k8s部署 elkElasticsearch,Kibana,Logstash,Redis,Filebea

目录

一、nfs存储

二、部署镜像，制作tag

三、 filebeat收集数据

 四、logstash过滤数据

五、elasticsearch存储数据 nfs做存储（自动注册pv详见前文）

六、kibana展示数据

七、验证安装

一、nfs存储

二、部署镜像，制作tag

1.  
   docker pull docker.elastic.co/elasticsearch/elasticsearch:7.17.2
2.  
   docker pull docker.elastic.co/kibana/kibana:7.17.2
3.  
   docker pull docker.elastic.co/logstash/logstash:7.17.2
4.  
   docker pull docker.elastic.co/beats/filebeat:7.17.2
5.  
   docker pull redis
6.  
    
7.  
   #注：提前下载，其他master将镜像copy过去load比较省时

三、 filebeat收集数据

1.  
   apiVersion: v1
2.  
   kind: ConfigMap
3.  
   metadata:
4.  
   name: filebeat-config
5.  
   namespace: logging
6.  
    
7.  
   data:
8.  
   filebeat.yml: |
9.  
   filebeat.inputs:
10.  
    - type: log
11.  
    paths:
12.  
    - /var/log/nginx
13.  
    document_type: k8s-nginx
14.  
     
15.  
    setup.template.name: "k8s-nginx"
16.  
    setup.template.pattern: "k8s-nginx-*"
17.  
    output.elasticsearch:
18.  
    hosts: ["elasticsearch:9200"]
19.  
    index: "k8s-nginx-%{ yyyy.MM.dd}"
20.  
    # output.logstash:
21.  
    # hosts: ['logging-logstash:5044']
22.  
    # enabled: true
23.  
    ---
24.  
     
25.  
    apiVersion: apps/v1
26.  
    kind: DaemonSet
27.  
    metadata:
28.  
    name: filebeat
29.  
    namespace: dev
30.  
    spec:
31.  
    selector:
32.  
    matchLabels:
33.  
    app: filebeat
34.  
    template:
35.  
    metadata:
36.  
    labels:
37.  
    k8s-app: filebeat
38.  
    app: filebeat
39.  
    spec:
40.  
    terminationGracePeriodSeconds: 30
41.  
    containers:
42.  
    - name: filebeat
43.  
    image: docker.elastic.co/beats/filebeat:7.17.2
44.  
    imagePullPolicy: IfNotPresent
45.  
    args: [
46.  
    "-c", "/etc/filebeat.yml",
47.  
    "-e",
48.  
    ]
49.  
    volumeMounts:
50.  
    - name: config
51.  
    mountPath: /etc/filebeat.yml
52.  
    readOnly: true
53.  
    subPath: filebeat.yml
54.  
    - name: log
55.  
    mountPath: /var/log/
56.  
    volumes:
57.  
    - name: config
58.  
    configMap:
59.  
    defaultMode: 0755
60.  
    name: filebeat-config
61.  
    - name: log
62.  
    hostPath:
63.  
    path: /var/log/
64.  
    type: Directory

 四、logstash过滤数据

1.  
   ---
2.  
   apiVersion: v1
3.  
   kind: ConfigMap
4.  
   metadata:
5.  
   name: logstash-configmap
6.  
   namespace: dev
7.  
   labels:
8.  
   k8s-app: logstash-configmap
9.  
   data:
10.  
    logstash.conf: |
11.  
    input {
12.  
    beats {
13.  
    port => "5044"
14.  
    codec => "json"
15.  
    }
16.  
    }
17.  
    filter{
18.  
    json{
19.  
    source => "message"
20.  
    remove_field => "message"
21.  
    }
22.  
    }
23.  
    output {
24.  
    elasticsearch {
25.  
    hosts => "elasticsearch:9200"
26.  
    index => "nginx-json-log-%{ YYYY.MM.dd}"
27.  
    }
28.  
    }
29.  
    ---
30.  
    apiVersion: apps/v1
31.  
    kind: Deployment
32.  
    metadata:
33.  
    name: logstash
34.  
    namespace: dev
35.  
    spec:
36.  
    replicas: 1
37.  
    selector:
38.  
    matchLabels:
39.  
    k8s-app: logstash
40.  
    template:
41.  
    metadata:
42.  
    labels:
43.  
    k8s-app: logstash
44.  
    spec:
45.  
    containers:
46.  
    - name: logstash
47.  
    image: docker.elastic.co/logstash/logstash:7.17.2
48.  
    imagePullPolicy: IfNotPresent
49.  
    ports:
50.  
    - containerPort: 5044
51.  
    volumeMounts:
52.  
    - name: config-volume
53.  
    mountPath: /usr/share/logstash/pipeline/
54.  
    volumes:
55.  
    - name: config-volume
56.  
    configMap:
57.  
    name: logstash-configmap
58.  
    items:
59.  
    - key: logstash.conf
60.  
    path: logstash.conf
61.  
    ---
62.  
    apiVersion: v1
63.  
    kind: Service
64.  
    metadata:
65.  
    name: logstash
66.  
    namespace: dev
67.  
    spec:
68.  
    ports:
69.  
    - port: 5044
70.  
    targetPort: 5044
71.  
    protocol: TCP
72.  
    selector:
73.  
    k8s-app: logstash
74.  
    type: ClusterIP

五、elasticsearch存储数据 nfs做存储（自动注册pv详见前文）

1.  
   apiVersion: v1
2.  
   kind: Service
3.  
   metadata:
4.  
   name: elasticsearch
5.  
   namespace: dev
6.  
   labels:
7.  
   app: elasticsearch
8.  
   spec:
9.  
   selector:
10.  
    k8s-app: elasticsearch
11.  
    clusterIP: None
12.  
    ports:
13.  
    - port: 9200
14.  
    name: db
15.  
    - port: 9300
16.  
    name: inter
17.  
     
18.  
    ---
19.  
    apiVersion: apps/v1
20.  
    kind: StatefulSet
21.  
    metadata:
22.  
    name: elasticsearch
23.  
    namespace: dev
24.  
    labels:
25.  
    k8s-app: elasticsearch
26.  
    spec:
27.  
    serviceName: elasticsearch
28.  
    selector:
29.  
    matchLabels:
30.  
    k8s-app: elasticsearch
31.  
    template:
32.  
    metadata:
33.  
    labels:
34.  
    k8s-app: elasticsearch
35.  
    spec:
36.  
    containers:
37.  
    - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.2
38.  
    name: elasticsearch
39.  
    resources:
40.  
    limits:
41.  
    cpu: 1
42.  
    memory: 2Gi
43.  
    requests:
44.  
    cpu: 0.5
45.  
    memory: 500Mi
46.  
    env:
47.  
    - name: "discovery.type"
48.  
    value: "single-node"
49.  
    - name: ES_JAVA_OPTS
50.  
    value: "-Xms512m -Xmx2g"
51.  
    ports:
52.  
    - containerPort: 9200
53.  
    name: db
54.  
    protocol: TCP
55.  
    - name: inter
56.  
    containerPort: 9300
57.  
    volumeMounts:
58.  
    - name: elasticsearch-data
59.  
    mountPath: /usr/share/elasticsearch/data
60.  
    volumeClaimTemplates:
61.  
    - metadata:
62.  
    name: elasticsearch-data
63.  
    spec:
64.  
    storageClassName: "nfs-storage"
65.  
    accessModes: [ "ReadWriteMany" ]
66.  
    resources:
67.  
    requests:
68.  
    storage: 1Gi

六、kibana展示数据

1.  
   apiVersion: apps/v1
2.  
   kind: Deployment
3.  
   metadata:
4.  
   name: kibana
5.  
   namespace: dev
6.  
   labels:
7.  
   k8s-app: kibana
8.  
   spec:
9.  
   replicas: 1
10.  
    selector:
11.  
    matchLabels:
12.  
    k8s-app: kibana
13.  
    template:
14.  
    metadata:
15.  
    labels:
16.  
    k8s-app: kibana
17.  
    spec:
18.  
    containers:
19.  
    - name: kibana
20.  
    image: docker.elastic.co/kibana/kibana:7.17.2
21.  
    resources:
22.  
    limits:
23.  
    cpu: 1
24.  
    memory: 1G
25.  
    requests:
26.  
    cpu: 0.5
27.  
    memory: 500Mi
28.  
    env:
29.  
    - name: ELASTICSEARCH_HOSTS
30.  
    value: http://elasticsearch:9200
31.  
    ports:
32.  
    - containerPort: 5601
33.  
    protocol: TCP
34.  
     
35.  
    ---
36.  
    apiVersion: v1
37.  
    kind: Service
38.  
    metadata:
39.  
    name: kibana
40.  
    namespace: dev
41.  
    spec:
42.  
    ports:
43.  
    - port: 5601
44.  
    protocol: TCP
45.  
    targetPort: 5601
46.  
    nodePort: 30000
47.  
    type: NodePort
48.  
    selector:
49.  
    k8s-app: kibana
50.  
     
51.  
    ---
52.  
    apiVersion: networking.k8s.io/v1
53.  
    kind: Ingress
54.  
    metadata:
55.  
    name: kibana
56.  
    namespace: dev
57.  
    spec:
58.  
    rules:
59.  
    - host: "dev。kibana.cn"
60.  
    http:
61.  
    paths:
62.  
    - path: /kibana
63.  
    pathType: Prefix
64.  
    backend:
65.  
    service:
66.  
    name: kibana
67.  
    port:
68.  
    number: 5601

七、验证安装

1.  
   [root@master-01 elk]# kubectl get pv,pvc -n dev |grep ela
2.  
   persistentvolume/pvc-9f405d08-2669-45de-81bb-e37f45ac53b7 1Gi RWX Delete Bound dev/elasticsearch-data-elasticsearch-0 nfs-storage 146m
3.  
   persistentvolumeclaim/elasticsearch-data-elasticsearch-0 Bound pvc-9f405d08-2669-45de-81bb-e37f45ac53b7 1Gi RWX nfs-storage 146m
4.  
   [root@master-01 elk]# kubectl get pod -n dev |grep filebeat
5.  
   filebeat-5255b 1/1 Running 0 151m
6.  
   filebeat-5fdgr 1/1 Running 0 151m
7.  
   filebeat-bv94f 1/1 Running 0 151m
8.  
   filebeat-fcljd 1/1 Running 0 151m
9.  
   filebeat-jbt87 1/1 Running 0 151m
10.  
    filebeat-p7jgx 1/1 Running 0 151m
11.  
    [root@master-01 elk]# kubectl get pod -n dev |grep elasticsearch
12.  
    elasticsearch-0 1/1 Running 0 147m
13.  
    [root@master-01 elk]# kubectl get pod -n dev |grep kibana
14.  
    kibana-5f5cb8b78d-g6fvz 1/1 Running 0 147m
15.  
    [root@master-01 elk]# kubectl get pod -n dev |grep logstash
16.  
    logstash-77bf676fb6-6rr9s 1/1 Running 0 150m

这篇好文章是转载于：学新通技术网