容器云项目-日志平台部署

一、工具介绍

工具平台 包含组件 部署节点 备注
日志监控 elasticsearch logstash kibana redis grafana node22 node23

二、日志工具部署

2.1、redis部署

redis用于暂时存储filebeat输入的数据
cat redis-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: redis
  namespace: monitor
  labels:
    k8s-app: redis
    addonmanager.kubernetes.io/mode: Reconcile
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: redis
  labels:
    k8s-app: redis
    addonmanager.kubernetes.io/mode: Reconcile
rules:
  - apiGroups:
      - ""
    resources:
      - "services"
      - "namespaces"
      - "endpoints"
    verbs:
      - "get"
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: monitor
  name: redis
  labels:
    k8s-app: redis
    addonmanager.kubernetes.io/mode: Reconcile
subjects:
  - kind: ServiceAccount
    name: redis
    namespace: monitor
    apiGroup: ""
roleRef:
  kind: ClusterRole
  name: redis
  apiGroup: ""

cat redis-configMap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-config
  namespace: monitor
data:
  redis.conf: |-
    bind 0.0.0.0
    port 6379
    pidfile /var/run/redis.pid 
    appendonly yes
    protected-mode no
    requirepass xxxxxxx

cat redis-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: redis
  namespace: monitor
  labels:
    name: redis
spec:
  ports:
  - name: redis
    port: 6379
    targetPort: 6379
  selector:
    name: redis

cat redis-sts.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  labels:
    name: redis
  namespace: monitor
spec:
  serviceName: redis
  selector:
    matchLabels:
      name: redis
  replicas: 1
  template:
    metadata:
      labels:
        name: redis
    spec:
      serviceAccountName: redis
      containers:
      - name: redis
        image: redis
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh","-c","redis-server /usr/local/etc/redis/redis.conf"]
        resources:
          limits:
            cpu: 1000m
            memory: 2Gi
          requests:
            cpu: 100m
            memory: 1Gi
        ports:
        - containerPort: 6379
        volumeMounts:
        - name: tz-config
          mountPath: /etc/localtime
        - name: config
          mountPath: /usr/local/etc/redis
        - name: redis-data
          mountPath: /data
      volumes:
      - name: tz-config
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai
      - name: config
        configMap:
          name: redis-config
      - name: redis-data
        emptyDir: {}
      nodeSelector:
        app: monitor

2.2、logstash部署

logstash主动去redis中抽取数据,并对数据进行过滤、解析、索引等处理,然后将处理过的数据输入elasticsearch
cat Dockerfile

FROM logstash:7.9.1

USER root
ADD GeoLite2-City_20210713.tar.gz /usr/share/logstash/ 

RUN mv /usr/share/logstash/GeoLite2-City_20210713 /usr/share/logstash/GeoLite2-City \
    && yum -y install geoipupdate \
    && echo "ProductIds GeoLite2-City" >> /etc/GeoIP.conf 

docker build -t logstash:7.9.1-TZ .

cat logstash-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: logstash
  namespace: monitor
  labels:
    k8s-app: logstash
    addonmanager.kubernetes.io/mode: Reconcile
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: logstash
  labels:
    k8s-app: logstash
    addonmanager.kubernetes.io/mode: Reconcile
rules:
  - apiGroups:
      - ""
    resources:
      - "services"
      - "namespaces"
      - "endpoints"
    verbs:
      - "get"
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: monitor
  name: logstash
  labels:
    k8s-app: logstash
    addonmanager.kubernetes.io/mode: Reconcile
subjects:
  - kind: ServiceAccount
    name: logstash
    namespace: monitor
    apiGroup: ""
roleRef:
  kind: ClusterRole
  name: logstash
  apiGroup: ""

cat logstash-configmap.yaml

kind: ConfigMap
apiVersion: v1
metadata:
  name: logstash-configmap
  namespace: monitor
data:
  logstash.conf: |-
    input {
      redis {
        data_type =>"list"
        key =>"nginx_logs"
        host =>"redis"
        port => 6379
        password => "xxxxxx"
        db => 0
      }
    }

    filter {
      geoip {
        target => "geoip"
        source => "client_ip"
        database => "/usr/share/logstash/GeoLite2-City/GeoLite2-City.mmdb"
        add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
        add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
        remove_field => ["[geoip][latitude]", "[geoip][longitude]", "[geoip][country_code]", "[geoip][country_code2]", "[geoip][country_code3]", "[geoip][timezone]", "[geoip][continent_code]", "[geoip][region_code]"]
        }
      mutate {
        convert => [ "size", "integer" ]
        convert => [ "status", "integer" ]
        convert => [ "responsetime", "float" ]
        convert => [ "upstreamtime", "float" ]
        convert => [ "[geoip][coordinates]", "float" ]
        remove_field => [ "ecs","agent","host","cloud","@version","input","logs_type" ]
      }
      useragent {
        source => "http_user_agent"
        target => "ua"
        remove_field => [ "[ua][minor]","[ua][major]","[ua][build]","[ua][patch]","[ua][os_minor]","[ua][os_major]" ]
      }
    }
    output {
      elasticsearch {
        hosts => "elasticsearch-service:9200"
        index => "logstash-nginx-%{+YYYY.MM.dd}"
      }
    }
  logstash.yml: |-
    http.host: "0.0.0.0"
    xpack.monitoring.enabled: false
    xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]

ps:该配置中的redis相关部分设计到的信息需要与redis、filebeat一致,filebeat在下一章节我们会用nginx访问日志的例子来进行一个简单的实践

cat logstash-deploy.yaml

apiVersion: apps/v1
kind: Deployment 
metadata: 
  name: logstash
  namespace: monitor
spec: 
  selector:
    matchLabels:
      name: logstash
  template: 
    metadata: 
      labels: 
        name: logstash
    spec: 
      serviceAccountName: logstash
      containers: 
        - name: logstash 
          image: logstash:7.9.1-TZ
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 1000m
              memory: 2Gi
            requests:
              cpu: 100m
              memory: 1Gi
          volumeMounts:
            - name: logstash-config
              mountPath: "/usr/share/logstash/pipeline/logstash.conf"
              subPath: logstash.conf
            - name: logstash-xpack
              mountPath: "/usr/share/logstash/config/logstash.yml"
              subPath: logstash.yml
            - name: tz-config
              mountPath: /etc/localtime
      volumes:
      - name: logstash-config
        configMap:
          name: logstash-configmap
          items:
            - key: logstash.conf
              path: logstash.conf
      - name: logstash-xpack
        configMap:
          name: logstash-configmap
          items:
            - key: logstash.yml
              path: logstash.yml
      - name: tz-config
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai         
      nodeSelector:
        app: monitor

2.3、elasticsearch部署

存储处理过的日志数据
cat Dockerfile

FROM elasticsearch:7.9.1


USER root
RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone

docker build -t elasticsearch:7.9.1-TZ .

cat elasticsearch-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: elasticsearch-log
  namespace: monitor
  labels:
    k8s-app: elasticsearch-logging
    addonmanager.kubernetes.io/mode: Reconcile
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: elasticsearch-log
  labels:
    k8s-app: elasticsearch-logging
    addonmanager.kubernetes.io/mode: Reconcile
rules:
  - apiGroups:
      - ""
    resources:
      - "services"
      - "namespaces"
      - "endpoints"
    verbs:
      - "get"
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: monitor
  name: elasticsearch-log
  labels:
    k8s-app: elasticsearch-logging
    addonmanager.kubernetes.io/mode: Reconcile
subjects:
  - kind: ServiceAccount
    name: elasticsearch-log
    namespace: monitor
    apiGroup: ""
roleRef:
  kind: ClusterRole
  name: elasticsearch-log
  apiGroup: ""

cat elasticsearch-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: elasticsearch
  namespace: monitor
data:
  elasticsearch.yml: |
    cluster.name: xj-elasticsearch
    node.name: node-1
    node.max_local_storage_nodes: 3
    network.host: 0.0.0.0
    http.port: 9200
    discovery.seed_hosts: ["127.0.0.1", "[::1]"]
    cluster.initial_master_nodes: ["node-1"]
    http.cors.enabled: true
    http.cors.allow-origin: /.*/

cat elasticsearch-pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: elasticsearch-pvc
  namespace: monitor
spec:
  storageClassName: es-nfs
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 15Gi

cat elasticsearch-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: elasticsearch-service
  namespace: monitor
  labels:
    name: elasticsearch
spec:
  type: NodePort
  ports:
  - name:  web-9200
    port: 9200
    targetPort: 9200
    protocol: TCP
    nodePort: 30001
  selector:
    name: elasticsearch

cat elasticsearch-sts.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: elasticsearch
  labels:
    name: elasticsearch
  namespace: monitor
spec:
  serviceName: elasticsearch-service
  selector:
    matchLabels:
      name: elasticsearch
  replicas: 1
  template:
    metadata:
      labels:
        name: elasticsearch
    spec:
      serviceAccountName: elasticsearch-log
      initContainers:
      - name: init-sysctl1
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh","-c","sysctl -w vm.max_map_count=262144"]
        securityContext:
          privileged: true
      containers:
      - name: elasticsearch
        image: elasticsearch:7.9.1-TZ
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            cpu: 1000m
            memory: 2Gi
          requests:
            cpu: 100m
            memory: 1Gi
        env:
        - name: ES_JAVA_OPTS
          value: -Xms512m -Xmx512m
        ports:
        - containerPort: 9200
          name: db
        - containerPort: 9300
          name: transport
        volumeMounts:
        - name: elasticsearch-data
          mountPath: /usr/share/elasticsearch/data/
        - name: es-config
          mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
          subPath: elasticsearch.yml
      volumes:
      - name: elasticsearch-data
        persistentVolumeClaim:
          claimName: elasticsearch-pvc
      - name: es-config
        configMap:
          name: elasticsearch
      nodeSelector:
        app: monitor

2.4、kibana部署

kibana可以进行日志搜索查询并展示具体的日志信息
cat kibana-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: kibana
  namespace: monitor
  labels:
    k8s-app: kibana
    addonmanager.kubernetes.io/mode: Reconcile
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: kibana
  labels:
    k8s-app: kibana
    addonmanager.kubernetes.io/mode: Reconcile
rules:
  - apiGroups:
      - ""
    resources:
      - "services"
      - "namespaces"
      - "endpoints"
    verbs:
      - "get"
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: monitor
  name: kibana
  labels:
    k8s-app: kibana
    addonmanager.kubernetes.io/mode: Reconcile
subjects:
  - kind: ServiceAccount
    name: kibana
    namespace: monitor
    apiGroup: ""
roleRef:
  kind: ClusterRole
  name: kibana
  apiGroup: ""

cat kibana-configmap.yaml

kind: ConfigMap
apiVersion: v1
metadata:
  name: kibana-configmap
  namespace: monitor
data:
  kibana.yml: |-
    server.name: kibana
    server.host: "0"
    elasticsearch.hosts: [ "http://elasticsearch-service:9200" ]

cat kibana-deploy.yaml

apiVersion: apps/v1
kind: Deployment 
metadata: 
  name: kibana
  namespace: monitor
spec: 
  selector:
    matchLabels:
      name: kibana
  template: 
    metadata: 
      labels: 
        name: kibana
    spec: 
      serviceAccountName: kibana
      containers: 
        - name: kibana 
          image: kibana:7.9.1
          imagePullPolicy: IfNotPresent
          ports: 
            - containerPort: 5601
          resources:
            limits:
              cpu: 1000m
              memory: 2Gi
            requests:
              cpu: 100m
              memory: 1Gi
          volumeMounts:
            - name: kibana-config
              mountPath: "/usr/share/kibana/config"
            - name: tz-config
              mountPath: /etc/localtime
      volumes:
      - name: kibana-config
        configMap:
          name: kibana-configmap
      - name: tz-config
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai         
      nodeSelector:
        app: monitor

cat kibana-service.yaml

kind: Service
apiVersion: v1
metadata:
  name: kibana-svc
  namespace: monitor
  labels:
    nfs: kibana-service
    deploy-kibana: support
  annotations:
    description: Exposes Nexus Service
spec:
  type: NodePort
  selector:     
    name: kibana    
  ports:
    - name: kibana
      port: 5601
      targetPort: 5601
      nodePort: 30005

2.5、grafana部署

elasticsearch同样也可以作为grafana的后端数据库
cat grafana-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: grafana
  namespace: monitor
  labels:
    k8s-app: grafana
    addonmanager.kubernetes.io/mode: Reconcile
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: grafana
  labels:
    k8s-app: grafana
    addonmanager.kubernetes.io/mode: Reconcile
rules:
  - apiGroups:
      - ""
    resources:
      - "services"
      - "namespaces"
      - "endpoints"
    verbs:
      - "get"
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: monitor
  name: grafana
  labels:
    k8s-app: grafana
    addonmanager.kubernetes.io/mode: Reconcile
subjects:
  - kind: ServiceAccount
    name: grafana
    namespace: monitor
    apiGroup: ""
roleRef:
  kind: ClusterRole
  name: grafana
  apiGroup: ""

cat grafana-pvc-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: grafana-pv
  namespace: monitor
  labels:
    app: grafana
spec:
  capacity:
    storage: 5Gi
  accessModes: 
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  nfs:
    server: 10.0.0.18
    path: /data/grafana
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: grafana-pvc
  namespace: monitor
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi
  selector:
    matchLabels:
      app: "grafana"

cat grafana-deploy.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: grafana
  name: grafana
  namespace: monitor
spec:
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      labels:
        app: grafana
    spec:
      securityContext:
        fsGroup: 472
        supplementalGroups:
          - 0
      serviceAccountName: grafana
      containers:
        - name: grafana
          image: grafana/grafana
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 3000
              name: http-grafana
              protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /robots.txt
              port: 3000
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 30
            successThreshold: 1
            timeoutSeconds: 2
          livenessProbe:
            failureThreshold: 3
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            tcpSocket:
              port: 3000
            timeoutSeconds: 1
          resources:
            requests:
              cpu: 250m
              memory: 750Mi
          volumeMounts:
            - mountPath: /var/lib/grafana
              name: grafana-pv
            - mountPath: /etc/localtime
              name: tz-config
      volumes:
        - name: grafana-pv
          persistentVolumeClaim:
            claimName: grafana-pvc
        - name: tz-config
          hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
      nodeSelector:
        app: monitor

cat grafana-service.yaml

kind: Service
apiVersion: v1
metadata:
  name: grafana-svc
  namespace: monitor
  labels:
    nfs: grafana-service
    deploy-grafana: support
  annotations:
    description: Exposes Grafana Service
spec:
  type: NodePort
  selector:     
    name: grafana    
  ports:
    - name: grafana
      port: 3000
      targetPort: 3000
      nodePort: 30009