[root@shkf6-245 ~]# docker pull nginx:1.7.9 1.7.9: Pulling from library/nginx Image docker.io/library/nginx:1.7.9 uses outdated schema1 manifest format. Please upgrade to a schema2 image for better future compatibility. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/ a3ed95caeb02: Pull complete 6f5424ebd796: Pull complete d15444df170a: Pull complete e83f073daa67: Pull complete a4d93e421023: Pull complete 084adbca2647: Pull complete c9cec474c523: Pull complete Digest: sha256:e3456c851a152494c3e4ff5fcc26f240206abac0c9d794affb40e0714846c451 Status: Downloaded newer image for nginx:1.7.9 docker.io/library/nginx:1.7.9
打tag
1 2 3
[root@shkf6-245 ~]# docker images|grep 1.7.9 nginx 1.7.9 84581e99d807 4 years ago 91.7MB [root@shkf6-245 ~]# docker tag 84581e99d807 harbor.od.com/public/nginx:v1.7.9
登录私有仓库，并推送镜像nginx
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
[root@shkf6-245 ~]# docker login harbor.od.com Username: admin Password: WARNING! Your password will be stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/engine/reference/commandline/login/#credentials-store
[root@shkf6-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json|cfssl-json -bare etcd-peer 2019/11/13 17:00:03 [INFO] generate received request 2019/11/13 17:00:03 [INFO] received CSR 2019/11/13 17:00:03 [INFO] generating key: rsa-2048 2019/11/13 17:00:04 [INFO] encoded CSR 2019/11/13 17:00:04 [INFO] signed certificate with serial number 69997016866371968425072677347883174107938471757 2019/11/13 17:00:04 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements").
检查生成的证书、私钥
1 2 3 4 5 6 7 8 9 10 11
[root@shkf6-245 certs]# ll total 36 -rw-r--r-- 1 root root 836 Nov 13 16:32 ca-config.json -rw-r--r-- 1 root root 993 Nov 12 16:31 ca.csr -rw-r--r-- 1 root root 328 Nov 12 16:06 ca-csr.json -rw------- 1 root root 1679 Nov 12 16:31 ca-key.pem -rw-r--r-- 1 root root 1346 Nov 12 16:31 ca.pem -rw-r--r-- 1 root root 1062 Nov 13 17:00 etcd-peer.csr -rw-r--r-- 1 root root 379 Nov 13 16:34 etcd-peer-csr.json -rw------- 1 root root 1679 Nov 13 17:00 etcd-peer-key.pem -rw-r--r-- 1 root root 1428 Nov 13 17:00 etcd-peer.pem
[root@shkf6-242 etcd]# vim /etc/supervisord.d/etcd-server.ini [root@shkf6-242 etcd]# cat /etc/supervisord.d/etcd-server.ini [program:etcd-server-6-242] command=/opt/etcd/etcd-server-startup.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/etcd ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=30 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=etcd ; setuid to this UNIX account to run the program redirect_stderr=true ; redirect proc stderr to stdout (default false) killasgroup=true ; kill all process in a group stopasgroup=true ; stop all process in a group stdout_logfile=/data/logs/etcd-server/etcd.stdout.log ; stdout log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false)
注意：etcd集群各主机启动配置略有不同，配置其他节点时注意修改。
启动etcd服务并检查
在shkf6-242机器上：
1 2 3 4 5 6 7 8 9
[root@shkf6-242 etcd]# supervisorctl update etcd-server-6-242: added process group
[root@shkf6-244 ~]# useradd -s /sbin/nologin -M etcd [root@shkf6-244 ~]# mkdir /opt/src [root@shkf6-244 ~]# cd /opt/src [root@shkf6-244 src]# wget https://github.com/etcd-io/etcd/releases/download/v3.1.20/etcd-v3.1.20-linux-amd64.tar.gz [root@shkf6-244 src]# useradd -s /sbin/nologin -M etcd useradd: user 'etcd' already exists [root@shkf6-244 src]# tar xf etcd-v3.1.20-linux-amd64.tar.gz -C /opt [root@shkf6-244 src]# cd /opt/ [root@shkf6-244 opt]# mv etcd-v3.1.20-linux-amd64/ etcd-v3.1.20 [root@shkf6-244 opt]# ln -s /opt/etcd-v3.1.20/ /opt/etcd [root@shkf6-244 opt]# mkdir -p /opt/etcd/certs /data/etcd /data/logs/etcd-server [root@shkf6-244 opt]# cd /opt/etcd/certs [root@shkf6-244 certs]# scp -P52113 shkf6-245:/opt/certs/ca.pem /opt/etcd/certs/ [root@shkf6-244 certs]# scp -P52113 shkf6-245:/opt/certs/etcd-peer.pem /opt/etcd/certs/ [root@shkf6-244 certs]# scp -P52113 shkf6-245:/opt/certs/etcd-peer-key.pem /opt/etcd/certs/ [root@shkf6-244 certs]# chown -R etcd.etcd /opt/etcd/certs /data/etcd /data/logs/etcd-server [root@shkf6-244 certs]# vim /opt/etcd/etcd-server-startup.sh [root@shkf6-244 etcd]# cat /opt/etcd/etcd-server-startup.sh #!/bin/sh ./etcd --name etcd-server-6-244 \ --data-dir /data/etcd/etcd-server \ --listen-peer-urls https://192.168.6.244:2380 \ --listen-client-urls https://192.168.6.244:2379,http://127.0.0.1:2379 \ --quota-backend-bytes 8000000000 \ --initial-advertise-peer-urls https://192.168.6.244:2380 \ --advertise-client-urls https://192.168.6.244:2379,http://127.0.0.1:2379 \ --initial-cluster etcd-server-6-242=https://192.168.6.242:2380,etcd-server-6-243=https://192.168.6.243:2380,etcd-server-6-244=https://192.168.6.244:2380 \ --ca-file ./certs/ca.pem \ --cert-file ./certs/etcd-peer.pem \ --key-file ./certs/etcd-peer-key.pem \ --client-cert-auth \ --trusted-ca-file ./certs/ca.pem \ --peer-ca-file ./certs/ca.pem \ --peer-cert-file ./certs/etcd-peer.pem \ --peer-key-file ./certs/etcd-peer-key.pem \ --peer-client-cert-auth \ --peer-trusted-ca-file ./certs/ca.pem \ --log-output stdout [root@shkf6-244 certs]# cd ../ [root@shkf6-244 etcd]# chmod +x etcd-server-startup.sh [root@shkf6-244 etcd]# yum install supervisor -y [root@shkf6-244 etcd]# systemctl start supervisord.service [root@shkf6-244 etcd]# systemctl enable supervisord.service [root@shkf6-244 etcd]# vim /etc/supervisord.d/etcd-server.ini [root@shkf6-244 etcd]# cat /etc/supervisord.d/etcd-server.ini [program:etcd-server-6-244] command=/opt/etcd/etcd-server-startup.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/etcd ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=30 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=etcd ; setuid to this UNIX account to run the program redirect_stderr=true ; redirect proc stderr to stdout (default false) killasgroup=true ; kill all process in a group stopasgroup=true ; stop all process in a group stdout_logfile=/data/logs/etcd-server/etcd.stdout.log ; stdout log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false) [root@shkf6-244 etcd]# supervisorctl update [root@shkf6-244 etcd]# supervisorctl status etcd-server-6-244 RUNNING pid 11748, uptime 0:00:33 [root@shkf6-244 etcd]# netstat -lntup|grep etcd tcp 0 0 192.168.6.244:2379 0.0.0.0:* LISTEN 11749/./etcd tcp 0 0 127.0.0.1:2379 0.0.0.0:* LISTEN 11749/./etcd tcp 0 0 192.168.6.244:2380 0.0.0.0:* LISTEN 11749/./etcd [root@shkf6-244 etcd]# ./etcdctl cluster-health member 4244d625c76d5482 is healthy: got healthy result from http://127.0.0.1:2379 member aa911af67b8285a2 is healthy: got healthy result from http://127.0.0.1:2379 member c751958d48e7e127 is healthy: got healthy result from http://127.0.0.1:2379 cluster is healthy
检查集群状态
三个etcd节点都起来后
在shkf6-242机器上：
1 2 3 4 5 6 7 8 9
[root@shkf6-242 etcd]# ./etcdctl cluster-health member 4244d625c76d5482 is healthy: got healthy result from http://127.0.0.1:2379 member aa911af67b8285a2 is healthy: got healthy result from http://127.0.0.1:2379 member c751958d48e7e127 is healthy: got healthy result from http://127.0.0.1:2379 cluster is healthy [root@shkf6-242 etcd]# ./etcdctl member list 4244d625c76d5482: name=etcd-server-6-242 peerURLs=https://192.168.6.242:2380 clientURLs=http://127.0.0.1:2379,https://192.168.6.242:2379 isLeader=true aa911af67b8285a2: name=etcd-server-6-243 peerURLs=https://192.168.6.243:2380 clientURLs=http://127.0.0.1:2379,https://192.168.6.243:2379 isLeader=false c751958d48e7e127: name=etcd-server-6-244 peerURLs=https://192.168.6.244:2380 clientURLs=http://127.0.0.1:2379,https://192.168.6.244:2379 isLeader=false
[root@shkf6-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json |cfssl-json -bare client 2019/11/14 13:59:24 [INFO] generate received request 2019/11/14 13:59:24 [INFO] received CSR 2019/11/14 13:59:24 [INFO] generating key: rsa-2048 2019/11/14 13:59:24 [INFO] encoded CSR 2019/11/14 13:59:24 [INFO] signed certificate with serial number 71787071397684874048844497862502145400133190813 2019/11/14 13:59:24 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements").
检查生成的证书和私钥
1 2 3 4 5
[root@shkf6-245 certs]# ll client* -rw-r--r-- 1 root root 993 Nov 14 13:59 client.csr -rw-r--r-- 1 root root 280 Nov 14 13:59 client-csr.json -rw------- 1 root root 1679 Nov 14 13:59 client-key.pem -rw-r--r-- 1 root root 1363 Nov 14 13:59 client.pem
[root@shkf6-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server apiserver-csr.json |cfssl-json -bare apiserver 2019/11/14 14:10:01 [INFO] generate received request 2019/11/14 14:10:01 [INFO] received CSR 2019/11/14 14:10:01 [INFO] generating key: rsa-2048 2019/11/14 14:10:02 [INFO] encoded CSR 2019/11/14 14:10:02 [INFO] signed certificate with serial number 531358145467350237994138515547646071524442824033 2019/11/14 14:10:02 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements").
检查生成的证书和私钥
1 2 3 4 5
[root@shkf6-245 certs]# ll apiserver* -rw-r--r-- 1 root root 1249 Nov 14 14:10 apiserver.csr -rw-r--r-- 1 root root 581 Nov 14 14:09 apiserver-csr.json -rw------- 1 root root 1679 Nov 14 14:10 apiserver-key.pem -rw-r--r-- 1 root root 1598 Nov 14 14:10 apiserver.pem
[root@shkf6-243 bin]# mkdir conf [root@shkf6-243 bin]# vi conf/audit.yaml [root@shkf6-243 bin]# cat conf/audit.yaml apiVersion: audit.k8s.io/v1beta1 # This is required. kind: Policy # Don't generate audit events for all requests in RequestReceived stage. omitStages: - "RequestReceived" rules: # Log pod changes at RequestResponse level - level: RequestResponse resources: - group: "" # Resource "pods" doesn't match requests to any subresource of pods, # which is consistent with the RBAC policy. resources: ["pods"] # Log "pods/log", "pods/status" at Metadata level - level: Metadata resources: - group: "" resources: ["pods/log", "pods/status"]
# Don't log requests to a configmap called "controller-leader" - level: None resources: - group: "" resources: ["configmaps"] resourceNames: ["controller-leader"]
# Don't log watch requests by the "system:kube-proxy" on endpoints or services - level: None users: ["system:kube-proxy"] verbs: ["watch"] resources: - group: "" # core API group resources: ["endpoints", "services"]
# Log the request body of configmap changes in kube-system. - level: Request resources: - group: "" # core API group resources: ["configmaps"] # This rule only applies to resources in the "kube-system" namespace. # The empty string "" can be used to select non-namespaced resources. namespaces: ["kube-system"]
# Log configmap and secret changes in all other namespaces at the Metadata level. - level: Metadata resources: - group: "" # core API group resources: ["secrets", "configmaps"]
# Log all other resources in core and extensions at the Request level. - level: Request resources: - group: "" # core API group - group: "extensions" # Version of group should NOT be included.
# A catch-all rule to log all other requests at the Metadata level. - level: Metadata # Long-running requests like watches that fall under this rule will not # generate an audit event in RequestReceived. omitStages: - "RequestReceived"
[root@shkf6-243 bin]# vi /etc/supervisord.d/kube-apiserver.ini [root@shkf6-243 bin]# cat /etc/supervisord.d/kube-apiserver.ini [program:kube-apiserver-6-243] command=/opt/kubernetes/server/bin/kube-apiserver.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=30 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=true ; redirect proc stderr to stdout (default false) killasgroup=true ; kill all process in a group stopasgroup=true ; stop all process in a group stdout_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stdout.log ; stderr log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false)
启动服务并检查
1 2 3 4 5 6 7 8 9
[root@shkf6-243 bin]# supervisorctl update kube-apiserver-6-243: added process group [root@shkf6-243 bin]# supervisorctl status etcd-server-6-243 RUNNING pid 12112, uptime 5:06:23 kube-apiserver-6-243 RUNNING pid 12824, uptime 0:00:46
[root@shkf6-244 src]# tar xf kubernetes-server-linux-amd64-v1.15.2.tar.gz -C /opt/ [root@shkf6-244 src]# cd /opt/ [root@shkf6-244 opt]# mv kubernetes kubernetes-v1.15.2 [root@shkf6-244 opt]# ln -s /opt/kubernetes-v1.15.2/ /opt/kubernetes [root@shkf6-244 opt]# cd kubernetes [root@shkf6-244 kubernetes]# rm -f kubernetes-src.tar.gz [root@shkf6-244 kubernetes]# cd server/bin [root@shkf6-244 bin]# rm -f *.tar [root@shkf6-244 bin]# rm -f *_tag [root@shkf6-244 bin]# mkdir cert [root@shkf6-244 bin]# scp -P52113 shkf6-245:/opt/certs/apiserver-key.pem /opt/kubernetes/server/bin/cert/ [root@shkf6-244 bin]# scp -P52113 shkf6-245:/opt/certs/apiserver.pem /opt/kubernetes/server/bin/cert/ [root@shkf6-244 bin]# scp -P52113 shkf6-245:/opt/certs/ca-key.pem /opt/kubernetes/server/bin/cert/ [root@shkf6-244 bin]# scp -P52113 shkf6-245:/opt/certs/ca.pem /opt/kubernetes/server/bin/cert/ [root@shkf6-244 bin]# scp -P52113 shkf6-245:/opt/certs/client-key.pem /opt/kubernetes/server/bin/cert/ [root@shkf6-244 bin]# scp -P52113 shkf6-245:/opt/certs/client.pem /opt/kubernetes/server/bin/cert/ [root@shkf6-244 bin]# mkdir conf [root@shkf6-244 bin]# vi conf/audit.yaml [root@shkf6-244 bin]# cat conf/audit.yaml apiVersion: audit.k8s.io/v1beta1 # This is required. kind: Policy # Don't generate audit events for all requests in RequestReceived stage. omitStages: - "RequestReceived" rules: # Log pod changes at RequestResponse level - level: RequestResponse resources: - group: "" # Resource "pods" doesn't match requests to any subresource of pods, # which is consistent with the RBAC policy. resources: ["pods"] # Log "pods/log", "pods/status" at Metadata level - level: Metadata resources: - group: "" resources: ["pods/log", "pods/status"]
# Don't log requests to a configmap called "controller-leader" - level: None resources: - group: "" resources: ["configmaps"] resourceNames: ["controller-leader"]
# Don't log watch requests by the "system:kube-proxy" on endpoints or services - level: None users: ["system:kube-proxy"] verbs: ["watch"] resources: - group: "" # core API group resources: ["endpoints", "services"]
# Log the request body of configmap changes in kube-system. - level: Request resources: - group: "" # core API group resources: ["configmaps"] # This rule only applies to resources in the "kube-system" namespace. # The empty string "" can be used to select non-namespaced resources. namespaces: ["kube-system"]
# Log configmap and secret changes in all other namespaces at the Metadata level. - level: Metadata resources: - group: "" # core API group resources: ["secrets", "configmaps"]
# Log all other resources in core and extensions at the Request level. - level: Request resources: - group: "" # core API group - group: "extensions" # Version of group should NOT be included.
# A catch-all rule to log all other requests at the Metadata level. - level: Metadata # Long-running requests like watches that fall under this rule will not # generate an audit event in RequestReceived. omitStages: - "RequestReceived" [root@shkf6-244 bin]# vi /opt/kubernetes/server/bin/kube-apiserver.sh [root@shkf6-244 bin]# cat /opt/kubernetes/server/bin/kube-apiserver.sh #!/bin/bash ./kube-apiserver \ --apiserver-count 2 \ --audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log \ --audit-policy-file ./conf/audit.yaml \ --authorization-mode RBAC \ --client-ca-file ./cert/ca.pem \ --requestheader-client-ca-file ./cert/ca.pem \ --enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota \ --etcd-cafile ./cert/ca.pem \ --etcd-certfile ./cert/client.pem \ --etcd-keyfile ./cert/client-key.pem \ --etcd-servers https://192.168.6.242:2379,https://192.168.6.243:2379,https://192.168.6.244:2379 \ --service-account-key-file ./cert/ca-key.pem \ --service-cluster-ip-range 10.96.0.0/22 \ --service-node-port-range 3000-29999 \ --target-ram-mb=1024 \ --kubelet-client-certificate ./cert/client.pem \ --kubelet-client-key ./cert/client-key.pem \ --log-dir /data/logs/kubernetes/kube-apiserver \ --tls-cert-file ./cert/apiserver.pem \ --tls-private-key-file ./cert/apiserver-key.pem \ --v 2 [root@shkf6-244 bin]# chmod +x /opt/kubernetes/server/bin/kube-apiserver.sh [root@shkf6-244 bin]# mkdir -p /data/logs/kubernetes/kube-apiserver [root@shkf6-244 bin]# vi /etc/supervisord.d/kube-apiserver.ini [root@shkf6-244 bin]# cat /etc/supervisord.d/kube-apiserver.ini [program:kube-apiserver-6-244] command=/opt/kubernetes/server/bin/kube-apiserver.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=30 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=true ; redirect proc stderr to stdout (default false) killasgroup=true ; kill all process in a group stopasgroup=true ; stop all process in a group stdout_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stdout.log ; stderr log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false) [root@shkf6-244 bin]# supervisorctl update kube-apiserver-6-244: added process group [root@shkf6-244 bin]# supervisorctl status etcd-server-6-244 RUNNING pid 11748, uptime 5:10:52 kube-apiserver-6-244 RUNNING pid 12408, uptime 0:00:43
配四层反向代理
部署nginx
在shkf6-241和shkf6-242上：
1
~]# yum install nginx -y
配置4层代理
在shkf6-241和shkf6-242上：
1 2 3 4 5 6 7 8 9 10 11 12 13 14
~]# vim /etc/nginx/nginx.conf
stream { upstream kube-apiserver { server 192.168.6.243:6443 max_fails=3 fail_timeout=30s; server 192.168.6.244:6443 max_fails=3 fail_timeout=30s; } server { listen 7443; proxy_connect_timeout 2s; proxy_timeout 900s; proxy_pass kube-apiserver; } }
启动nginx
在shkf6-241和shkf6-242上：
1 2 3 4 5
~]# nginx -t nginx: the configuration file /etc/nginx/nginx.conf syntax is ok nginx: configuration file /etc/nginx/nginx.conf test is successful ~]# systemctl start nginx ~]# systemctl enable nginx
[root@shkf6-241 ~]# ip a |grep eth0 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000 inet 192.168.6.241/24 brd 192.168.6.255 scope global eth0 inet 192.168.6.66/32 scope global eth0
[root@shkf6-243 bin]# vi /etc/supervisord.d/kube-conntroller-manager.ini [root@shkf6-243 bin]# cat /etc/supervisord.d/kube-conntroller-manager.ini [program:kube-controller-manager-6.243] command=/opt/kubernetes/server/bin/kube-controller-manager.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=30 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=true ; redirect proc stderr to stdout (default false) killasgroup=true ; kill all process in a group stopasgroup=true ; stop all process in a group stdout_logfile=/data/logs/kubernetes/kube-controller-manager/controller.stdout.log ; stderr log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false)
启动服务并检查
shkf6-243上和shkf6-244：
1 2 3 4 5 6
[root@shkf6-243 bin]# supervisorctl update kube-controller-manager-6.243: added process group [root@shkf6-243 bin]# supervisorctl status etcd-server-6-243 RUNNING pid 12112, uptime 23:36:23 kube-apiserver-6-243 RUNNING pid 12824, uptime 18:30:46 kube-controller-manager-6.243 RUNNING pid 14952, uptime 0:01:00
[root@shkf6-243 bin]# vi /etc/supervisord.d/kube-scheduler.ini [root@shkf6-243 bin]# cat /etc/supervisord.d/kube-scheduler.ini [program:kube-scheduler-6-243] command=/opt/kubernetes/server/bin/kube-scheduler.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=30 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=true ; redirect proc stderr to stdout (default false) killasgroup=true ; kill all process in a group stopasgroup=true ; stop all process in a group stdout_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stdout.log ; stderr log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false)
[root@shkf6-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kubelet-csr.json | cfssl-json -bare kubelet 2019/11/15 09:49:58 [INFO] generate received request 2019/11/15 09:49:58 [INFO] received CSR 2019/11/15 09:49:58 [INFO] generating key: rsa-2048 2019/11/15 09:49:59 [INFO] encoded CSR 2019/11/15 09:49:59 [INFO] signed certificate with serial number 609294877015122932833154151112494803106290808681 2019/11/15 09:49:59 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements").
检查生成证书的证书、私钥
1 2 3 4 5
[root@shkf6-245 certs]# ll kubelet* -rw-r--r-- 1 root root 1098 Nov 15 09:49 kubelet.csr -rw-r--r-- 1 root root 445 Nov 15 09:47 kubelet-csr.json -rw------- 1 root root 1675 Nov 15 09:49 kubelet-key.pem -rw-r--r-- 1 root root 1452 Nov 15 09:49 kubelet.pem
拷贝证书至各运算节点，并创建配置
shkf6-243上：
拷贝证书，私钥，注意私钥文件属性600
1 2 3 4 5 6 7 8 9 10 11 12
[root@shkf6-243 bin]# scp -P52113 shkf6-245:/opt/certs/kubelet-key.pem /opt/kubernetes/server/bin/cert/ [root@shkf6-243 bin]# scp -P52113 shkf6-245:/opt/certs/kubelet.pem /opt/kubernetes/server/bin/cert/ [root@shkf6-243 bin]# ll cert/ total 32 -rw------- 1 root root 1679 Nov 14 14:18 apiserver-key.pem -rw-r--r-- 1 root root 1598 Nov 14 14:18 apiserver.pem -rw------- 1 root root 1679 Nov 14 14:18 ca-key.pem -rw-r--r-- 1 root root 1346 Nov 14 14:19 ca.pem -rw------- 1 root root 1679 Nov 14 14:19 client-key.pem -rw-r--r-- 1 root root 1363 Nov 14 14:19 client.pem -rw------- 1 root root 1675 Nov 15 10:01 kubelet-key.pem -rw-r--r-- 1 root root 1452 Nov 15 10:02 kubelet.pem
[root@shkf6-243 conf]# vi /etc/supervisord.d/kube-kubelet.ini [root@shkf6-243 conf]# cat /etc/supervisord.d/kube-kubelet.ini
[program:kube-kubelet-6-243] command=/opt/kubernetes/server/bin/kubelet.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=30 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=true ; redirect proc stderr to stdout (default false) killasgroup=true ; kill all process in a group stopasgroup=true ; stop all process in a group stdout_logfile=/data/logs/kubernetes/kube-kubelet/kubelet.stdout.log ; stderr log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false)
[root@shkf6-245 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client kube-proxy-csr.json |cfssl-json -bare kube-proxy-client 2019/11/15 12:28:23 [INFO] generate received request 2019/11/15 12:28:23 [INFO] received CSR 2019/11/15 12:28:23 [INFO] generating key: rsa-2048 2019/11/15 12:28:24 [INFO] encoded CSR 2019/11/15 12:28:24 [INFO] signed certificate with serial number 499210659443234759487015805632579178164834077987 2019/11/15 12:28:24 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements").
[root@shkf6-243 conf]# vi /root/ipvs.sh [root@shkf6-243 conf]# cat /root/ipvs.sh #!/bin/bash ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs" for i in $(ls $ipvs_mods_dir|grep -o "^[^.]*") do /sbin/modinfo -F filename $i &>/dev/null if [ $? -eq 0 ];then /sbin/modprobe $i fi done
[root@shkf6-243 conf]# vi /etc/supervisord.d/kube-proxy.ini [root@shkf6-243 conf]# cat /etc/supervisord.d/kube-proxy.ini [program:kube-proxy-6-243] command=/opt/kubernetes/server/bin/kube-proxy.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=30 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=true ; redirect proc stderr to stdout (default false) killasgroup=true ; kill all process in a group stopasgroup=true ; stop all process in a group stdout_logfile=/data/logs/kubernetes/kube-proxy/proxy.stdout.log ; stderr log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false)
