Step-1: Creating a virtual machine:
sudo virt-install
--name ps-dev-tmpl
--os-variant ubuntu22.04
--vcpus 2
--memory 6144
--location /data1/os-iso/ubuntu-22.04.4-live-server-amd64.iso,kernel=casper/vmlinuz,initrd=casper/initrd
--network bridge=virbr20,model=virtio,mac=0a:01:e1:20:de:01
--network network=virho10,model=virtio,mac=0a:01:e1:10:de:01
--disk size=28,pool=adm-images --uuid=00000000-0000-0000-0000-0a01e120de01
--graphics none --extra-args='console=ttyS0' --debug
Step-2:: After creating this instance, connect to its serial console using virsh console command:
virsh console ps-dev-tmpl
Step-3: Now setup jenkins and run the installation+deployment script by setting up agent as described in the jenkins setup.
An example script would be:
def WEBAPP_DIR = '${WORKSPACE}/webappbuild'
def DEVOPS_DIR = '${WORKSPACE}'
def DEV1_DC = '172.21.2.7'
def DEV1_DC_LB = '172.21.2.7'
def SQL_UPDATE_SERVER = '172.21.2.7'
def CQL_UPDATE_SERVER = '172.21.2.7'
def remote = [:]
remote.user = 'root'
remote.password = '<remote-system-password>'
remote.port = 22
remote.allowAnyHosts = true
pipeline {
agent none
environment {
// JAVA_HOME="/opt/jdk-23"
M2_HOME="/opt/apache-maven-3.9.7"
MAVEN_HOME="/opt/apache-maven-3.9.7"
}
stages {
stage('Webapp Sourcecode Checkout') {
agent {
node {
label 'master'
}
}
steps {
script {
def res = sh(script: "test -d ${WEBAPP_DIR}/panchayatseva-webapp && echo '1' || echo '0'", returnStdout: true).trim()
if (res == '0') {
sh "cd ${WORKSPACE} && mkdir ${WEBAPP_DIR} || echo webappdir exists"
sh "cd ${WEBAPP_DIR} && git clone https://<git-auth-token-here>@github.com/PanchayatSeva-SB/panchayatseva-webapp.git"
}
// else {
// def changes = sh(script: "cd ${WEBAPP_DIR}/panchayatseva-webapp && git fetch && git rev-list HEAD..origin/main --count", returnStdout: true).trim()
// if (changes.toInteger() > 0) {
// echo 'Changes detected. Continue with the build.'
sh "cd ${WEBAPP_DIR}/panchayatseva-webapp && git pull "
// }
// else {
// currentBuild.result = 'ABORTED'
// error 'Pipeline aborted: No new changes in the repository.'
// }
// }
}
}
}
stage('Clone Repo') {
agent {
node {
label 'dev-tmpl'
customWorkspace '/root'
}
}
steps {
script {
def clone_exists = sh(script: "test -d DevOps-Automation-Scripts && echo '1' || echo '0'", returnStdout: true).trim()
if (clone_exists == '0') {
sh 'git clone https://<git-auth-token-here>@github.com/DevOps-Model/DevOps-Automation-Scripts.git'
}
else {
sh 'echo Repo exists... Executing git pull'
sh 'git -C DevOps-Automation-Scripts/ pull'
}
}
}
}
stage('Install App Stack') {
agent {
node {
label 'dev-tmpl'
customWorkspace '/root'
}
}
steps {
sh "mkdir -p /tmp/dev-tomcat-conf || echo 'dir exists'"
sh "mkdir -p /opt/pass/tomcat || echo 'dir exists'"
dir('DevOps-Automation-Scripts') {
sh 'python3 TechStackInstallation.py psDev haproxy tomcat redpanda scylla mysql'
}
}
}
stage('Artifact Install') {
agent {
node {
label 'master'
}
}
steps {
sh "cd ${WEBAPP_DIR}/panchayatseva-webapp && /opt/apache-maven-3.9.7/bin/mvn install package -DskipTests=true -Dtestng.dtd.http=true -Dsonar.skip=true -Dsnyk.skip=true"
}
}
stage('Config Update') {
agent {
node {
label 'master'
}
}
steps {
script {
remote.name = DEV1_DC
remote.host = DEV1_DC
sshCommand remote:remote, command:'rm -rf /tmp/dev-tomcat-conf'
sshCommand remote:remote, command:'mkdir /tmp/dev-tomcat-conf'
// sh "rm -rf ${DEVOPS_DIR}/panchayatseva-devops/target/webapp-config/"
sh "rm -rf ${DEVOPS_DIR}/panchayatseva-devops"
sh "git clone https://<git-auth-token-here>@github.com/PanchayatSeva-SB/panchayatseva-devops.git || echo repo exists"
sh "cd ${DEVOPS_DIR}/panchayatseva-devops && /opt/apache-maven-3.9.7/bin/mvn clean install || echo build dir fail"
sh "mkdir ${DEVOPS_DIR}/panchayatseva-devops/target/webapp-config || echo dir exists"
sh "mkdir ${DEVOPS_DIR}/panchayatseva-devops/target/webapp-config/sb || echo dir exists"
sh "mkdir ${DEVOPS_DIR}/panchayatseva-devops/target/webapp-config/sb/dev || echo dir exists"
//sh 'java -Denable.debug=true -cp ${WEBAPP_DIR}/panchayatseva-webapp/target/panchayatseva/WEB-INF/classes/:"${WEBAPP_DIR}/panchayatseva-webapp/target/panchayatseva/WEB-INF/lib/*":"${WEBAPP_DIR}/panchayatseva-webapp/target/dependency/*":${WORKSPACE}/target/classes/ com.sayukth.panchayatseva.webapp.conf.gen.sb.demo.WebAppConfigGeneratorDemo ${DEVOPS_DIR}/target/webapp-config/sb/demo/'
//sh "java -Denable.debug=true -cp ${WEBAPP_DIR}/panchayatseva-webapp/target/panchayatseva/WEB-INF/classes/:${WEBAPP_DIR}/panchayatseva-webapp/target/panchayatseva/WEB-INF/lib/*:${WEBAPP_DIR}/panchayatseva-webapp/target/dependency/*:${WORKSPACE}/target/classes/ com.sayukth.panchayatseva.webapp.conf.gen.sb.demo.WebAppConfigGeneratorDemo ${DEVOPS_DIR}/target/webapp-config/sb/demo/"
sh "java -Denable.debug=true -cp ${WEBAPP_DIR}/panchayatseva-webapp/target/panchayatseva/WEB-INF/classes/:\"${WEBAPP_DIR}/panchayatseva-webapp/target/panchayatseva/WEB-INF/lib/*\":\"${WEBAPP_DIR}/panchayatseva-webapp/target/dependency/*\":\"${WORKSPACE}/panchayatseva-devops/target/classes/\" com.sayukth.panchayatseva.webapp.conf.gen.dev_vm.WebAppConfigGeneratorDevVm \"${WORKSPACE}/panchayatseva-devops/target/webapp-config/sb/dev\" \"${WORKSPACE}/panchayatseva-devops/resources/tmpl/\""
sshPut remote:remote, from:"${WORKSPACE}/panchayatseva-devops/target/webapp-config/sb/dev/", into:'/tmp/dev-tomcat-conf/'
sshCommand remote:remote, command:'cp -rf /tmp/dev-tomcat-conf/dev/* /opt/ps/tomcat/conf/'
}
}
}
stage('Artifact Distribution') {
agent {
node {
label 'master'
}
}
steps {
script {
//sh "cp -f ${WEBAPP_DIR}/eev-webapp/target/generated-swagger/swagger-ui/swagger.json /opt/httpd-data/www/html/rest-api/"
remote.name = DEV1_DC
remote.host = DEV1_DC
sshCommand remote:remote, command:'rm -rf /tmp/dev-resources-conf'
sshCommand remote:remote, command:'mkdir /tmp/dev-resources-conf'
sshPut remote: remote, from: "${WORKSPACE}/webappbuild/panchayatseva-webapp/target/panchayatseva.war", into: '/tmp/dev-resources-conf'
sshPut remote: remote, from: "${WORKSPACE}/webappbuild/panchayatseva-webapp/src/main/resources", into: '/tmp/dev-resources-conf'
}
}
}
stage('Run Configuration') {
agent {
node {
label 'dev-tmpl'
customWorkspace '/root'
}
}
steps {
dir('DevOps-Automation-Scripts') {
sh 'apt install ufw -y'
sh 'python3 ConfigureScript.py psDev configuration.conf'
}
}
}
stage('Starting Servers') {
agent {
node {
label 'dev-tmpl'
customWorkspace '/root'
}
}
steps {
// dir('DevOps-Automation-Scripts') {
// sh 'python3 StartStopScript.py start'
// }
script{
sh "systemctl start redpanda.service"
sh "systemctl start mysql@bootstrap.service"
sh "systemctl start scylla-server.service"
sh "systemctl start tomcat-ps.service"
// sh "systemctl start haproxy"
}
}
}
stage('Run Database Schemas') {
agent {
node {
label 'dev-tmpl'
customWorkspace '/root'
}
}
steps {
dir('DevOps-Automation-Scripts') {
sh 'python3 ConfigureScript.py psDev databases.conf'
}
}
}
stage('Restart Servers') {
agent {
node {
label 'dev-tmpl'
customWorkspace '/root'
}
}
steps {
// dir('DevOps-Automation-Scripts') {
// sh 'python3 StartStopScript.py stop'
// sh 'python3 StartStopScript.py start'
// }
script{
sh "systemctl restart redpanda.service"
sh "systemctl restart mysql@bootstrap.service"
sh "systemctl restart scylla-sever.service"
sh "systemctl restart tomcat-ps.service"
// sh "systemctl start haproxy"
}
}
}
stage('Artifact Deploy') {
agent {
node {
label 'master'
}
}
steps {
script {
if (SQL_UPDATE_SERVER.isEmpty()) {
error('Did not find the sql_update_server')
}
else {
remote.name = SQL_UPDATE_SERVER
remote.host = SQL_UPDATE_SERVER
sshCommand remote:remote, command:"/usr/bin/mysql -uroot -p'<mysql-password-here>' psdb_dev < /tmp/dev-resources-conf/resources/db/update/ps_update.sql"
sshCommand remote:remote, command:"/usr/bin/mysql -uroot -p'<mysql-password-here>' psdb_dev < /opt/ps/tomcat/conf/user-passwd-update.sql"
}
if (CQL_UPDATE_SERVER.isEmpty()) {
error('Did not find the cql_update_server')
}
else {
remote.name = CQL_UPDATE_SERVER
remote.host = CQL_UPDATE_SERVER
sshCommand remote:remote, command:'/usr/bin/cqlsh -kps_blob_dev -uroot -p<cqlsh-password-here> 172.21.2.7 9042 -f /tmp/dev-resources-conf/resources/ds/update/ps_update.cql'
}
remote.name = DEV1_DC_LB
remote.host = DEV1_DC_LB
// sshCommand remote:remote, command:'cp -f /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy-copy.cfg'
// sshCommand remote:remote, command: "sed -i \" s| server $DEV1_DC|# server $DEV1_DC|g\" /etc/haproxy/haproxy.cfg "
// sshCommand remote:remote, command:'systemctl reload haproxy'
// sleep(time: 60, unit: 'SECONDS')
sshCommand remote:remote, command:'pgrep java || true'
sshCommand remote:remote, command:'systemctl stop tomcat-ps.service || echo tomcat doesnt exist'
sshCommand remote:remote, command:'rm -rf /opt/ps/tomcat/webapps/ROOT*'
sshCommand remote:remote, command:'rm -rf /opt/ps/tomcat/logs/*.log'
sshCommand remote:remote, command:'rm -rf /opt/ps/tomcat/logs/*.out'
sshCommand remote:remote, command:'cp /tmp/dev-resources-conf/panchayatseva.war /opt/ps/tomcat/webapps/ROOT.war'
sshCommand remote:remote, command:'chown tomcat:tomcat /opt/ps/tomcat/webapps/ROOT.war'
sshCommand remote:remote, command:'systemctl start tomcat-ps.service'
sshCommand remote:remote, command:'pgrep java'
sleep(time: 60, unit: 'SECONDS')
// remote.name=POD1_DC_LB
// remote.host=POD1_DC_LB
// sshCommand remote:remote, command:'cp -f /etc/haproxy/haproxy-copy.cfg /etc/haproxy/haproxy.cfg'
// sshCommand remote:remote, command:'systemctl reload haproxy'
}
}
}
}
}
**The automation script and devops code includes only the hostname wherever possible for the configurations to avoid hard coding private ip addresses which makes it difficult for us to make clones out of this template. But the /etc/hosts must be changed for each clone, to which we make use of virt-sysprep’s firstboot script option to edit the ip address in the host file for the newly creating virtual machine.
Step-3: We make a copy of our existing disk of our now created and configured virtual machine, from which we remove machine specifics and use the same as our source for the new clone.
cp /data2/adm-images/ps-dev-tmpl-1.qcow2 /tmp/dev-tmpl-test
Step-4: We now prepare this now copied disk and remove machine specific configurations using virt-sysprep:
virt-sysprep -a /tmp/dev-tmpl-test.qcow2
Step-5: Now, use virt-sysprep to setup hostname and insert firstboot-script which is executed when the guest powers up for the first time.
TO generate a specifice firstboot script we use a python script to automate this entire process from copying disk, preparing template, inserting firstboot script, changing hostname and defining xml for the vm. Refer to preceeding pages to view the python script.
A specific script for a VM would be(with the name of the guest):
#!/bin/bash
#chmod 777 /var/cache/debconf
#echo "ucf ucf/changeprompt_threeway select keep_current" | debconf-set-selections
#dpkg-reconfigure openssh-server
#chmod 600 /var/cache/debconf
apt purge openssh-server -y
apt install openssh-server -y
systemctl restart ssh
systemctl status ssh
cp /etc/machine-id /etc/machind-id.bkp
cp /var/lib/dbus/machine-id /var/lib/dbus/machine-id.bkp
echo "00000000-0000-0000-0a01e0200001" > /etc/machine-id
echo "00000000-0000-0000-0a01e0200001" > /var/lib/dbus/machine-id
cp /etc/hosts /etc/hosts.bkp
echo "HIn\n\n\nn\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
sed -iE 's/172.21.2.107/172.21.3.163/g' /etc/hosts
echo 172.21.3.163 ps-in-dev1a-node01 >> /etc/hosts
systemctl disable mysql
systemctl enable mysql@bootstrap
systemctl restart scylla-server mysql@bootstrap redpanda tomcat-ps haproxy ssh
import subprocess
def run_command(command):
print(f"Running command: {command}")
subprocess.run(command, shell=True, check=True)
# xml file format
# hostname
# Define the list of suffixes
empsuffix = ['32']
emplist = 'dev-ein00'
ip = '172.21.3.163'
# Loop through the suffix list
for emp_id in empsuffix:
# Nested loop from 1 to 3
for x in range(1, 2):
content=""
with open('/data1/guest-xml/dev-template.xml', 'r') as file:
# Read the entire file content
content = file.read()
brmac=f"0a:01:e0:20:00:0{x}"
homac=f"0a:01:e0:10:00:0{x}"
guestname=f"ps-devbox-{emplist}{emp_id}"
with open(f"{guestname}.xml", 'w') as file:
content=content.replace('dev-ein0034',guestname)
content=content.replace('0a:01:0d:20:de:35',brmac)
content=content.replace('0a:01:0d:10:de:35',homac)
#content=content.replace(f'00000000-0000-0000-0000-0a010d20de35',f"00000000-0000-0000-000{x}-0a010d20de{emp_id}")
content=content.replace('/data1/guest-images/dev-template.qcow2',f'/data1/guest-images/{guestname}.qcow2')
file.write(content)
run_command(f"cp /tmp/dev-tmpl-test/ps-dev-tmpl-1.qcow2 /data1/guest-images/{guestname}.qcow2")
#run_command(f"virt-sysprep -a dev-template.qcow2 --firstboot /root/scripts/startup-script.sh ")
# Replace machine id in start up shell script
script_content=""
with open("/root/scripts/startup-script.sh",'r') as script:
script_content=script.read()
script_content=script_content.replace("mac-id",f"00000000-0000-0000-0a01e020000{x}")
script_content=script_content.replace("iptobereplaced",ip)
print(script_content)
with open(f"/root/scripts/{guestname}.sh","w") as start_script:
start_script.write(script_content)
run_command(f"virt-sysprep -a /data1/guest-images/{guestname}.qcow2")
run_command(f"virt-sysprep --hostname {guestname} -a /data1/guest-images/{guestname}.qcow2 --firstboot /root/scripts/{guestname}.sh")
run_command(f"virsh define {guestname}.xml")
Step-5: Prepare the guest disk with custom hostname and firstboot script:
virt-sysprep --hostname {guestname} -a /data1/guest-images/{guestname}.qcow2 --firstboot /root/scripts/{guestname}.sh
Step-6: We make use of a template.xml file and replace specific configurations such as mac ids, uuids and guestnames and source disk files, the above python script will give more clarifying insights about the changes. After copying our own custom xml, we define the VM with our custom configurations made to xml file with source disk image prepared using virt-sysprep
virsh define <guestname>.xml
We fix MAC ids for virtual machines here and IPs on the microtik router.