容器云项目-持续集成实践(一)

一、测试说明

基于目前已有的情况分别对前端项目和后端项目作持续集成测试,前端为基于npm环境运行的,后端基于java环境运行,测试的前后端无关联,仅作为测试使用,均使用pipeline的方式来实现

二、前端项目持续集成测试

配置步骤
1、jenkins项目配置
2、Jenkinsfile编写
3、sonarqube配置
4、编写部署脚本
5、cicd部署应用

2.1、jenkins项目配置

这里只写pipeline项目配置,项目配置中涉及到的jenkins其他认证授权类配置请参考前面的CICD组件配置(一)

09abcd

09dev

09jhg

09hhn

2.2、Jenkinsfile编写

cat Jenkinsfile

pipeline {
    agent any
    tools {
        maven 'maven' 
    }
    environment {
        sendmail = 'yes'
        recipients = '2847602965@qq.com,1363832871@qq.com'
    }
    stages {
        stage('src checkout') {
          steps {
             sh 'rm -Rf *'
             script {
                def nodeHome = tool 'node'
                env.PATH = "${nodeHome}/bin:${env.PATH}"
                registry_url = "10.0.0.19:30002"
                checkout([$class: 'GitSCM', branches: [[name: '*/dev']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
                userRemoteConfigs: [[credentialsId: '55cd62f0-58c9-47ac-a9fc-0e77302e5556', url: "http://10.0.0.18:9002/gitlab/mayifan/xj-operation-maintenance.git"]]])
                       img_name = sh(script: "echo front", returnStdout: true).trim()
                       def dataObject = readJSON file: 'package.json'
                       img_tag = "${dataObject.version}"
                       img = "${img_name}:${img_tag}"
                        registry_img = "${registry_url}/xj/${img}"
             }               
          }
        }

        stage('sonar analysis') {
            steps {
               withSonarQubeEnv('Sonarqube') {
                  sh'sonar-scanner -Dsonar.projectKey=front -Dsonar.projectName=front -Dsonar.language=js -Dsonar.sources=. -Dsonar.sourceEncoding=UTF-8 -Dsonar.login=8d1beade677d3e63c03295dd5b8c002ca9928f36'
                }
            }
               }
        stage('src build') {
            steps {
                sh 'echo "" > /root/.npmrc'
                sh "npm config set registry http://nexus-svc:8081/nexus/repository/npm-group/"
                sh 'echo "//nexus-svc:8081/nexus/repository/npm-group/:_authToken=NpmToken.8e2db5d4-6716-301e-8d40-c8d6564380b3" >> /root/.npmrc'
                sh 'npm cache clean -f'
                sh "npm install --loglevel info --unsafe-perm"
                sh "npm run build"
            }
        }

        stage('image build') {
          steps {
                 script {
                     docker.withRegistry("http://${registry_url}",'ce3f3651-056b-4e36-b4ba-158aa8dfdefc') {
                     def customimage = docker.build("xj/${img}")
                      customimage.push()
                      }
                    }
                 sh "docker rmi ${registry_img}"
                  }
              }

       stage('app deploy') {
          steps {
                  sh './deploy.sh'
                 }
              }
    }
       post{             
        success {
            script {
                if (sendmail == 'yes') {
               emailext to: "${recipients}", body: '''
<html>
               <b style="font-size:12px">(<span style="color:red">本邮件是程序自动下发的,请勿回复</span>)<br></b><hr>

               <b style="font-size: 12px;">项目名称:$PROJECT_NAME<br></b><hr>

               <b style="font-size: 12px;">构建编号:$BUILD_NUMBER<br></b><hr>

               <b style="font-size: 12px;">GIT版本号:${GIT_REVISION}<br></b><hr>

               <b style="font-size: 12px;" >构建状态:<span style="color:red">$BUILD_STATUS</span><br></b><hr>

               <b style="font-size: 12px;">触发原因:${CAUSE}<br></b><hr>

              <b style="font-size: 12px;">构建日志地址:<a href="${BUILD_URL}console">${BUILD_URL}console</a><br></b><hr>

              <b style="font-size: 12px;">构建地址:<a href="$BUILD_URL">$BUILD_URL</a><br></b><hr>

              <b style="font-size: 12px;">变更集:${JELLY_SCRIPT,template="html"}<br></b><hr>
                     </html>
                  ''', subject: '${JOB_NAME}' 
                }
            }
        }
        failure { 
         emailext to: "${recipients}",body: '''
               <!DOCTYPE html>
               <html>
                   <b style="font-size:12px">(<span style="color:red">本邮件是程序自动下发的,请勿回复</span>)<br></b><hr>

               <b style="font-size: 12px;">项目名称:$PROJECT_NAME<br></b><hr>

               <b style="font-size: 12px;">构建编号:$BUILD_NUMBER<br></b><hr>

               <b style="font-size: 12px;">GIT版本号:${GIT_REVISION}<br></b><hr>

               <b style="font-size: 12px;" >构建状态:<span style="color:red">$BUILD_STATUS</span><br></b><hr>

               <b style="font-size: 12px;">触发原因:${CAUSE}<br></b><hr>

              <b style="font-size: 12px;">构建日志地址:<a href="${BUILD_URL}console">${BUILD_URL}console</a><br></b><hr>

              <b style="font-size: 12px;">构建地址:<a href="$BUILD_URL">$BUILD_URL</a><br></b><hr>

              <b style="font-size: 12px;">变更集:${JELLY_SCRIPT,template="html"}<br></b><hr>
               </html>
               ''',subject: '${JOB_NAME}'
        }
    }

}

ps:脚本中涉及到npm登录授权,这里采用的是token方式来实现授权,实现方式是先手动在服务器上登录,授权记录会持久化到文件/root/.npmrc中,然后配置该文件达到自动登录npm私有库的目的
该流水线文件包括源代码检出、代码质量分析、源代码构建、应用部署、发送邮件
上面的pipeline采用的是declare方式编写,也可以采用groovy脚本形式来编写

node {
    
    stage('src checkout') {
        sh "rm -Rf *" 
        def nodeHome = tool 'node'
        env.PATH = "${nodeHome}/bin:${env.PATH}"
        registry_url = "10.0.0.19:30002"
        checkout([$class: 'GitSCM', branches: [[name: '*/dev']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
        userRemoteConfigs: [[credentialsId: '55cd62f0-58c9-47ac-a9fc-0e77302e5556', url: "http://10.0.0.18:9002/gitlab/mayifan/xj-operation-maintenance.git"]]])
        img_name = sh(script: "echo front", returnStdout: true).trim()

        def dataObject = readJSON file: 'package.json'
        img_tag = "${dataObject.version}"
        img = "${img_name}:${img_tag}"
        registry_img = "${registry_url}/xj/${img}"
    }
    
    stage('src build') {
        sh 'echo "" > /root/.npmrc'
        sh "npm config set registry http://nexus-svc:8081/nexus/repository/npm-group/"
        sh 'echo "//nexus-svc:8081/nexus/repository/npm-group/:_authToken=NpmToken.8e2db5d4-6716-301e-8d40-c8d6564380b3" >> /root/.npmrc'
        sh 'npm cache clean -f'
        sh "npm install --loglevel info --unsafe-perm"
        sh "npm run build"
    }
  

    stage('image build') {
        docker.withRegistry("http://${registry_url}",'ce3f3651-056b-4e36-b4ba-158aa8dfdefc') {
        def customimage = docker.build("xj/${img}")
        customimage.push()
        sh "docker rmi ${registry_img}"
        }
                         }

    stage('app deploy') {
       sh './deploy.sh'
      }
  
}

2.3、jenkins配置sonarqube

jenkins系统配置sonarqube
09sonar

jenkins全局工具配置sonarqube
09qube

配置sonarqube服务端
新疆一个项目,并获取该项目的token,该token将用于jenkins连接sonaqube server

2.4、部署脚本

cat Dockerfile

FROM 10.0.0.19:30002/xj/nginx

ADD dist /usr/share/nginx/html/dist
RUN chmod -R 777 /usr/share/nginx/html/dist
RUN echo "server { " > /etc/nginx/conf.d/default.conf
RUN echo  "listen       8090;" >> /etc/nginx/conf.d/default.conf
RUN echo  "server_name  localhost;" >> /etc/nginx/conf.d/default.conf
RUN echo  "absolute_redirect off;" >> /etc/nginx/conf.d/default.conf
RUN echo  "location /xjOM {" >> /etc/nginx/conf.d/default.conf
RUN echo  "alias  /usr/share/nginx/html/dist;" >> /etc/nginx/conf.d/default.conf
RUN echo  "index index.html index.htm;" >> /etc/nginx/conf.d/default.conf
RUN echo  "      } " >> /etc/nginx/conf.d/default.conf
RUN echo  "    }"  >> /etc/nginx/conf.d/default.conf

cat deploy.sh

#!/bin/bash

a=$(grep -Eh '"version"' package.json | awk -F ':' '{print $2}' | awk -F '"' '{print $2}')

cat <<EOF> front.yaml
apiVersion: apps/v1
kind: Deployment 
metadata: 
  name: front
  namespace: devops
spec: 
  selector:
    matchLabels:
      name: front
  template: 
    metadata: 
      labels: 
        name: front
    spec: 
      containers: 
        - name: front
          image: 10.0.0.19:30002/xj/front:$a
          imagePullPolicy: IfNotPresent
          ports: 
            - containerPort: 8090
          volumeMounts:
            - name: tz-config
              mountPath: /etc/localtime
            - name: dockersock
              mountPath: "/var/run/docker.sock"
            - name: dockercli
              mountPath: "/usr/bin/docker"
            - name: dockerdaemon
              mountPath: /etc/docker/daemon.json
              subPath: daemon.json
      imagePullSecrets:
      - name: harbor-secret
      nodeSelector:
        app: devops
      volumes:
      - name: tz-config
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
      - name: dockerdaemon
        configMap:
          name: jenkins-configmap
          items:
          - key: daemon.json
            path: daemon.json
      - name: dockersock
        hostPath:
          path: /var/run/docker.sock
      - name: dockercli
        hostPath:
          path: /usr/bin/docker
---
kind: Service
apiVersion: v1
metadata:
  name: front-svc
  namespace: devops
  labels:
    name: front
  annotations:
    description: Exposes Front  Service
spec:
  type: NodePort
  selector:     
    name: front    
  ports:
    - name: front
      port: 8090
      targetPort: 8090
      nodePort: 30103
EOF

apk add sshpass
sshpass -p xxxxx\@xxxx ssh -o StrictHostKeyChecking=no  10.0.0.19 'if [ -f front.yaml ];then rm -f front.yaml;fi'
sshpass -p xxxx\@xxxx scp -o StrictHostKeyChecking=no  front.yaml 10.0.0.19:/root/deploy
sshpass -p xxxx\@xxxx ssh -o StrictHostKeyChecking=no  10.0.0.19 'if [ $(kubectl get deploy -n devops | grep front | wc -l) -eq 1 ];then  kubectl delete -n devops deploy front && kubectl delete -n devops svc front-svc &&  kubectl create -f /root/deploy/front.yaml;else kubectl create -f /root/deploy/front.yaml;fi'

2.5、cicd部署应用

这里没有啥好说的,放一张部署成功后blue ocean的图和代码分析图
09bnb

09mmn

三、后端项目持续集成测试

以java项目为例
配置步骤
1、jenkins项目配置
2、Jenkinsfile编写
3、sonarqube配置
4、编写部署脚本
5、cicd部署应用

3.1、jenkins项目配置

09htu1

09htu2

09hut3

09hut4

3.2、Jenkinsfile编写

cat Jenkinsfile

pipeline {
    agent any
    tools {
        maven 'maven' 
    }
    environment {
                CI = 'true'
                 GROUP = readMavenPom().getGroupId()
                ARTIFACT = readMavenPom().getArtifactId()
                VERSION = readMavenPom().getVersion()
    }
    stages {

        stage('Src Analysis') {
            steps {
                withSonarQubeEnv('Sonarqube') {
                         sh'sonar-scanner -Dsonar.projectKey=provider -Dsonar.projectName=provider -Dsonar.language=java -Dsonar.java.binaries=. -Dsonar.sources=. -Dsonar.sourceEncoding=UTF-8 -Dsonar.login=3d66f5f5fe195d72b1a738a3658fb898a758401a'
                         }   
                  }
           }        

        stage('Src Build') {
            steps {
                sh 'mvn clean install'
            }
        }
        stage('Image Build') {
            steps {
               script {
                    docker.withRegistry("http://10.0.0.19:30002",'ce3f3651-056b-4e36-b4ba-158aa8dfdefc') {
                    def customimage = docker.build("xj/${GROUP}-${ARTIFACT}:${VERSION}")
                    customimage.push()
                    }
                }
                sh "docker rmi xj/${GROUP}-${ARTIFACT}:${VERSION}"
                sh "docker rmi 10.0.0.19:30002/xj/${GROUP}-${ARTIFACT}:${VERSION}"
            }
        }

      stage('App Deploy') {
            steps {
               sh './deploy.sh'
               } 
       }
    }
}

3.3、Jenkins配置sonarqube

同前端测试中配置

3.4、部署脚本

cat Dockerfile

FROM 10.0.0.19:30002/xj/maven:3.6.3-jdk-8
ADD target/*jar  /tmp
WORKDIR /tmp
CMD java -jar $(ls *.jar)

cat deploy.sh

#!/bin/bash

jarfile=$(echo $(ls target/*jar) | awk -F '/' '{print $2}')
jarname=${jarfile%.*}
tag=${jarname#*-}


cat <<EOF> provider.yaml
apiVersion: apps/v1
kind: Deployment 
metadata: 
  name: provider
  namespace: devops
spec: 
  selector:
    matchLabels:
      name: provider
  template: 
    metadata: 
      labels: 
        name: provider
    spec: 
      containers: 
        - name: provider 
          image: 10.0.0.19:30002/xj/com.teng-provider:$tag
          imagePullPolicy: IfNotPresent
          ports: 
            - containerPort: 8083
              name: http-provider
              protocol: TCP
          volumeMounts:
            - name: tz-config
              mountPath: /etc/localtime
            - name: dockersock
              mountPath: "/var/run/docker.sock"
            - name: dockercli
              mountPath: "/usr/bin/docker"
            - name: dockerdaemon
              mountPath: /etc/docker/daemon.json
              subPath: daemon.json
      imagePullSecrets:
      - name: harbor-secret
      nodeSelector:
        app: devops
      volumes:
      - name: tz-config
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
      - name: dockerdaemon
        configMap:
          name: jenkins-configmap
          items:
          - key: daemon.json
            path: daemon.json
      - name: dockersock
        hostPath:
          path: /var/run/docker.sock
      - name: dockercli
        hostPath:
          path: /usr/bin/docker
---
kind: Service
apiVersion: v1
metadata:
  name: provider-svc
  namespace: devops
  labels:
    name: provider
  annotations:
    description: Exposes Provider  Service
spec:
  type: NodePort
  selector:     
    name: provider    
  ports:
    - name: provider
      port: 8083
      targetPort: 8083
      nodePort: 30101
EOF

apk add sshpass
sshpass -p xxxx\xxx ssh -o StrictHostKeyChecking=no  10.0.0.19 'if [ -f provider.yaml ];then rm -f provider.yaml;fi'
sshpass -p xxxx\xxx scp -o StrictHostKeyChecking=no  provider.yaml 10.0.0.19:/root/deploy
sshpass -p xxxx\xxx ssh -o StrictHostKeyChecking=no  10.0.0.19 'if [ $(kubectl get deploy -n devops | grep provider | wc -l) -eq 1 ];then  kubectl delete -n devops deploy provider && kubectl delete -n devops svc provider-svc &&  kubectl create -f /root/deploy/provider.yaml;else kubectl create -f /root/deploy/provider.yaml;fi'  

ps:注意这里部署的provider会在nacos上注册一个服务,所以这里还需要在同一命名空间内部署一个nacos
cat nacos-deploy.yaml

apiVersion: apps/v1
kind: Deployment 
metadata: 
  name: nacos
  namespace: devops
spec: 
  selector:
    matchLabels:
      name: nacos
  template: 
    metadata: 
      labels: 
        name: nacos
    spec: 
      containers: 
        - name: nacos 
          image: nacos/nacos-server:latest
          imagePullPolicy: IfNotPresent
          ports: 
            - containerPort: 8848
              name: http-nacos
              protocol: TCP
          env:
            - name: MODE
              value: standalone
          volumeMounts:
            - name: tz-config
              mountPath: /etc/localtime
      volumes:
      - name: tz-config
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
      nodeSelector:
        app: devops
---
kind: Service
apiVersion: v1
metadata:
  name: nacos-svc
  namespace: devops
  labels:
    name: nacos
  annotations:
    description: Exposes Nacos Service
spec:
  type: NodePort
  selector:     
    name: nacos    
  ports:
    - name: nacos
      port: 8848
      targetPort: 8848
      nodePort: 30102

09nbn

3.4、cicd部署应用

09ju1

09ju2