云计算运维一步步编译安装Kubernetes之master计算节点安装

部署kube-apiserver集群

主机名角色IP
k8s-node01.boysec.cnkube-apiserver10.1.1.100
k8s-node02.boysec.cnkube-apiserver10.1.1.110
k8s-master.boysec.cn4层负载均衡10.1.1.120
k8s-slave.boysec.cn4层负载均衡10.1.1.130

注意：这里10.1.1.120和10.1.1.130使用nginx做4层负载均衡器，用keepalived跑一个vip：10.1.1.50，代理两个kube-apiserver，实现高可用这里部署文档以k8s-node01.boysec.cn主机为例，另外一台运算节点安装部署方法类似

下载安装kube-apiserver

Kubernetes下载地址

1
2
3
4
5
cd /server/tools/ 
tar xf kubernetes-server-linux-amd64.tar.gz -C /opt  
mv /opt/kubernetes /opt/kubernetes-v1.17.9  
ln -s /opt/kubernetes-v1.17.9 /opt/kubernetes  
mkdir /opt/kubernetes/server/bin/{cert,conf} -p

签发证书

在k8s-dns.boysec.cn上

vim /opt/certs/client-csr.json

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
{
    "CN": "k8s-node",
    "hosts": [
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}

生成client证书和私钥

1
2
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json | cfssljson -bare client
ls client*

签发kube-apiserver证书

vim /opt/certs/apiserver-csr.json

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
{
    "CN": "apiserver",
    "hosts": [
        "127.0.0.1",
        "192.168.0.1",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.svc.cluster",
        "kubernetes.default.svc.cluster.local",
        "10.1.1.50",
        "10.1.1.100",
        "10.1.1.110",
        "10.1.1.120",
        "10.1.1.130"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}

生成apiserver证书和私钥

1
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server apiserver-csr.json |cfssljson -bare apiserver

拷贝证书至各运算节点，并创建配置

k8s-node01.boysec.cn上：

1
mkdir /opt/kubernetes/server/bin/{cert,conf} -p

拷贝证书、私钥，注意私钥文件属性600

1
[root@k8s-dns certs]# scp apiserver-key.pem apiserver.pem ca-key.pem ca.pem client-key.pem client.pem k8s-node01:/opt/kubernetes/server/bin/cert

检查

1
2
3
4
5
6
7
8
[root@k8s-node01 ]# ls -l /opt/kubernetes/server/bin/cert 
total 16  
-rw------- 1 root root 1679 Aug  4 22:09 apiserver-key.pem  
-rw-r--r-- 1 root root 1606 Aug  4 22:09 apiserver.pem  
-rw------- 1 root root 1679 Aug  4 22:11 ca-key.pem  
-rw-r--r-- 1 root root 1354 Aug  4 22:11 ca.pem  
-rw------- 1 root root 1675 Aug  4 22:09 client-key.pem  
-rw-r--r-- 1 root root 1371 Aug  4 22:09 client.pem

创建配置

vim /opt/kubernetes/server/bin/conf/audit.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
  - "RequestReceived"
rules:
  # Log pod changes at RequestResponse level
  - level: RequestResponse
    resources:
    - group: ""
      # Resource "pods" doesn't match requests to any subresource of pods,
      # which is consistent with the RBAC policy.
      resources: ["pods"]
  # Log "pods/log", "pods/status" at Metadata level
  - level: Metadata
    resources:
    - group: ""
      resources: ["pods/log", "pods/status"]

  # Don't log requests to a configmap called "controller-leader"
  - level: None
    resources:
    - group: ""
      resources: ["configmaps"]
      resourceNames: ["controller-leader"]

  # Don't log watch requests by the "system:kube-proxy" on endpoints or services
  - level: None
    users: ["system:kube-proxy"]
    verbs: ["watch"]
    resources:
    - group: "" # core API group
      resources: ["endpoints", "services"]

  # Don't log authenticated requests to certain non-resource URL paths.
  - level: None
    userGroups: ["system:authenticated"]
    nonResourceURLs:
    - "/api*" # Wildcard matching.
    - "/version"

  # Log the request body of configmap changes in kube-system.
  - level: Request
    resources:
    - group: "" # core API group
      resources: ["configmaps"]
    # This rule only applies to resources in the "kube-system" namespace.
    # The empty string "" can be used to select non-namespaced resources.
    namespaces: ["kube-system"]

  # Log configmap and secret changes in all other namespaces at the Metadata level.
  - level: Metadata
    resources:
    - group: "" # core API group
      resources: ["secrets", "configmaps"]

  # Log all other resources in core and extensions at the Request level.
  - level: Request
    resources:
    - group: "" # core API group
    - group: "extensions" # Version of group should NOT be included.

  # A catch-all rule to log all other requests at the Metadata level.
  - level: Metadata
    # Long-running requests like watches that fall under this rule will not
    # generate an audit event in RequestReceived.
    omitStages:
      - "RequestReceived"

创建启动脚本

k8s-node01.boysec.cn上：

vim /opt/kubernetes/server/bin/kube-apiserver.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#!/bin/bash
./kube-apiserver \
  --apiserver-count 2 \
  --audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log \
  --audit-policy-file ./conf/audit.yaml \
  --authorization-mode RBAC \
  --client-ca-file ./cert/ca.pem \
  --requestheader-client-ca-file ./cert/ca.pem \
  --enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota \
  --etcd-cafile ./cert/ca.pem \
  --etcd-certfile ./cert/client.pem \
  --etcd-keyfile ./cert/client-key.pem \
  --etcd-servers https://10.1.1.100:2379,https://10.1.1.110:2379,https://10.1.1.130:2379 \
  --service-account-key-file ./cert/ca-key.pem \
  --service-cluster-ip-range 192.168.0.0/16 \
  --service-node-port-range 3000-29999 \
  --target-ram-mb=1024 \
  --kubelet-client-certificate ./cert/client.pem \
  --kubelet-client-key ./cert/client-key.pem \
  --log-dir  /data/logs/kubernetes/kube-apiserver \
  --tls-cert-file ./cert/apiserver.pem \
  --tls-private-key-file ./cert/apiserver-key.pem \
  --v 2

调整权限和目录

1
2
chmod +x /opt/kubernetes/server/bin/kube-apiserver.sh
mkdir -p /data/logs/kubernetes/kube-apiserver

创建supervisor配置

vim /etc/supervisord.d/kube-apiserver.ini

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
[program:kube-apiserver-100]
command=/opt/kubernetes/server/bin/kube-apiserver.sh            ; the program (relative uses PATH, can take args)
numprocs=1                                                      ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin                            ; directory to cwd to before exec (def no cwd)
autostart=true                                                  ; start at supervisord start (default: true)
autorestart=true                                                ; retstart at unexpected quit (default: true)
startsecs=22                                                    ; number of secs prog must stay running (def. 1)
startretries=3                                                  ; max # of serial start failures (default 3)
exitcodes=0,2                                                   ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                 ; signal used to kill process (default TERM)
stopwaitsecs=10                                                 ; max num secs to wait b4 SIGKILL (default 10)
user=root                                                       ; setuid to this UNIX account to run the program
redirect_stderr=false                                           ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stdout.log        ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                    ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                        ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                     ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                     ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stderr.log        ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB                                    ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4                                        ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB                                     ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false                                     ; emit events on stderr writes (default false)

启动服务并检查

1
2
3
supervisorctl update
supervisorctl status
netstat -lnpt|grep 6443

配4层反向代理

k8s-master.boysec.cn,k8s-slave.boysec.cn上：

vim /etc/nginx/nginx.conf

1
2
3
4
5
6
7
8
9
10
11
12
stream {
    upstream kube-apiserver {
        server 10.1.1.100:6443     max_fails=3 fail_timeout=30s;
        server 10.1.1.110:6443     max_fails=3 fail_timeout=30s;
    }
    server {
        listen 7443;
        proxy_connect_timeout 2s;
        proxy_timeout 900s;
        proxy_pass kube-apiserver;
    }
}

2021-08-13报错

1
2
3
4
5
[root@k8s-master ~]# nginx -t
nginx: [emerg] unknown directive "stream" in /etc/nginx/nginx.conf:77
nginx: configuration file /etc/nginx/nginx.conf test failed
## 解决
yum -y install nginx-mod-stream

keepalived配置

vim /etc/keepalived/check_port.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
#!/bin/bash
nginxpid=$(ps -C nginx --no-header|wc -l)
#1.判断 Nginx 是否存活,如果不存活则尝试启动 Nginx
if [ $nginxpid -eq 0 ];then
    systemctl start nginx
    sleep 3
    #2.等待 3 秒后再次获取一次 Nginx 状态
    nginxpid=$(ps -C nginx --no-header|wc -l)
    #3.再次进行判断, 如 Nginx 还不存活则停止 Keepalived,让地址进行漂移,并退出脚本
    if [ $nginxpid -eq 0 ];then
        systemctl stop keepalived
    fi
fi

vim /etc/keepalived/keepalived.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
! Configuration File for keepalived

global_defs {
   router_id 10.1.1.120

}

vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh"
    interval 2
    weight -20
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 10.1.1.120
    nopreempt   # 非抢占式

    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        10.1.1.50
    }
}

vim /etc/keepalived/keepalived.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
! Configuration File for keepalived
global_defs {
	router_id 10.1.1.130
}
vrrp_script chk_nginx {
	script "/etc/keepalived/check_port.sh"
	interval 2
	weight -20
}
vrrp_instance VI_1 {
	state BACKUP
	interface eth0
	virtual_router_id 251
	mcast_src_ip 10.1.1.130
	nopreempt
	priority 90
	advert_int 1
	authentication {
		auth_type PASS
		auth_pass 11111111
	}
	track_script {
		chk_nginx
	}
	virtual_ipaddress {
		10.1.1.50
	}
}
nopreempt注解

VRRP will normally preempt a lower priority machine when a higher priority machine comes online. “nopreempt” allows the lower priority machine to maintain the master role, even when a higher priority machine comes back online.

NOTE: For this to work, the initial state of thisentry must be BACKUP.

当优先级较高的机器联机时，VRRP通常会抢占优先级较低的机器。“noprempt”允许低优先级机器保持主角色，即使高优先级机器重新联机。

注意：要使其工作，此条目的初始状态必须为BACKUP。

启动代理并检查

1
2
3
4
5
6
7
systemctl start keepalived
systemctl enable keepalived
systemctl start nginx
systemctl enable nginx

netstat -lnpt|grep 7443
ip addr|grep 10.1.1.50

部署controller-manager

集群规划

主机名角色ip
k8s-node01.boysec.cncontroller-manager10.1.1.100
k8s-node02.boysec.cncontroller-manager10.1.1.110

注意：这里部署文档以k8s-node01.boysec.cn主机为例，另外一台运算节点安装部署方法类似

创建启动脚本

vim /opt/kubernetes/server/bin/kube-controller-manager.sh

1
2
3
4
5
6
7
8
9
10
#!/bin/sh
./kube-controller-manager \
  --cluster-cidr 172.7.0.0/16 \
  --leader-elect true \
  --log-dir /data/logs/kubernetes/kube-controller-manager \
  --master http://127.0.0.1:8080 \
  --service-account-private-key-file ./cert/ca-key.pem \
  --service-cluster-ip-range 192.168.0.0/16 \
  --root-ca-file ./cert/ca.pem \
  --v 2

调整文件权限，创建目录

1
2
chmod +x /opt/kubernetes/server/bin/kube-controller-manager.sh
mkdir -p /data/logs/kubernetes/kube-controller-manager

创建supervisor配置

vim /etc/supervisord.d/kube-conntroller-manager.ini

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
[program:kube-controller-manager-100]
command=/opt/kubernetes/server/bin/kube-controller-manager.sh                     ; the program (relative uses PATH, can take args)
numprocs=1                                                                        ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin                                              ; directory to cwd to before exec (def no cwd)
autostart=true                                                                    ; start at supervisord start (default: true)
autorestart=true                                                                  ; retstart at unexpected quit (default: true)
startsecs=22                                                                      ; number of secs prog must stay running (def. 1)
startretries=3                                                                    ; max # of serial start failures (default 3)
exitcodes=0,2                                                                     ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                                   ; signal used to kill process (default TERM)
stopwaitsecs=10                                                                   ; max num secs to wait b4 SIGKILL (default 10)
user=root                                                                         ; setuid to this UNIX account to run the program
redirect_stderr=false                                                             ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-controller-manager/controll.stdout.log  ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                                      ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                                          ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                                       ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                                       ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-controller-manager/controll.stderr.log  ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB                                                      ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4                                                          ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB                                                       ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false                                                       ; emit events on stderr writes (default false)

启动服务并检查

1
2
supervisorctl update
supervisorctl status

部署kube-scheduler

集群规划

主机名角色ip
k8s-node01.boysec.cnkube-scheduler10.1.1.100
k8s-node02.boysec.cnkube-scheduler10.1.1.110

注意：这里部署文档以k8s-node01.boysec.cn主机为例，另外一台运算节点安装部署方法类似

创建启动脚本

vim /opt/kubernetes/server/bin/kube-scheduler.sh

1
2
3
4
5
6
#!/bin/sh
./kube-scheduler \
  --leader-elect  \
  --log-dir /data/logs/kubernetes/kube-scheduler \
  --master http://127.0.0.1:8080 \
  --v 2

调整文件权限，创建目录

1
2
chmod +x /opt/kubernetes/server/bin/kube-scheduler.sh
mkdir -p /data/logs/kubernetes/kube-scheduler

创建supervisor配置

vim /etc/supervisord.d/kube-scheduler.ini

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
[program:kube-scheduler-100]
command=/opt/kubernetes/server/bin/kube-scheduler.sh                     ; the program (relative uses PATH, can take args)
numprocs=1                                                               ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin                                     ; directory to cwd to before exec (def no cwd)
autostart=true                                                           ; start at supervisord start (default: true)
autorestart=true                                                         ; retstart at unexpected quit (default: true)
startsecs=22                                                             ; number of secs prog must stay running (def. 1)
startretries=3                                                           ; max # of serial start failures (default 3)
exitcodes=0,2                                                            ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                          ; signal used to kill process (default TERM)
stopwaitsecs=10                                                          ; max num secs to wait b4 SIGKILL (default 10)
user=root                                                                ; setuid to this UNIX account to run the program
redirect_stderr=false                                                    ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                             ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                                 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                              ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                              ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB                                             ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4                                                 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB                                              ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false                                              ; emit events on stderr writes (default false)

启动服务并检查

1
2
3
4
5
supervisorctl update
supervisorctl status

ln -s /opt/kubernetes/server/bin/kubectl /usr/bin/kubectl
kubectl get cs