初始化环境

| IP | Type | OS | | 192.168.1.111 | master、etcd、docker | Ubuntu 18.04.2、2C、4G | | 192.168.1.112 | master、etcd、docker | Ubuntu 18.04.2、2C、4G | | 192.168.1.113 | master、etcd、docker | Ubuntu 18.04.2、2C、4G | | 192.168.1.114 | node、docker | Ubuntu 18.04.2、2C、4G | | 192.168.1.115 | node、docker | Ubuntu 18.04.2、2C、4G | | 192.168.1.116 | node、docker | Ubuntu 18.04.2、2C、4G |

免密登陆SSH
1
2
3
ssh-keygen -t rsa  

for i in k8s-m2 k8s-m3 k8s-n1 k8s-n2 k8s-n3;do /usr/bin/ssh-copy-id -i .ssh/id-rsa.pub $i;done
关闭防火墙和SELINUX
1
ufw disable
关闭SWAP
1
2
3
4
5
swapoff -a && sysctl -w vm.swappiness=0  

vi /etc/fstab  
# 注释掉Swap行
# /swap.img	none	swap	sw	0	0
设置Docker所需参数
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
cat /etc/sysctl.d/k8s.conf  

net.ipv4.ip_forward = 1  
net.bridge.bridge-nf-call-ip6tables = 1  
net.bridge.bridge-nf-call-iptables = 1  

# 加载bridge模块
modprobe bridge 

sysctl -p /etc/sysctl.d/k8s.conf

编辑 /etc/hosts 文件,配置hostname 通信

1
2
3
4
5
6
192.168.1.111 k8s-m1  
192.168.1.112 k8s-m2  
192.168.1.113 k8s-m3  
192.168.1.114 k8s-n1  
192.168.1.115 k8s-n2  
192.168.1.116 k8s-n3
安装Docker

Docker安装教程官方文档

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 卸载老版本
sudo apt-get remove docker docker-engine docker.io containerd runc

# 安装库
sudo apt-get update
apt-get install \
apt-transport-https \
ca-certificates \
curl \
gnupg-agent \
software-properties-common

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
apt-key fingerprint 0EBFCD88

sudo add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"

# 安装docker
apt-get update
apt-cache madison docker-ce
apt-get install docker-ce=<VERSION_STRING>
安装CFSSL
1
2
3
4
5
6
7
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo

创建证书

创建CA证书
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
mkdir /etc/kubernetes/ssl -p  
cd /etc/kubernetes/ssl  
cat ca-config.json  

{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
cat ca-csr.json

{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "SC",
      "L": "CD",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

# 生成私钥
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
创建ETCD证书
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
cat etcd-csr.json

{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "192.168.1.111",
    "192.168.1.112",
    "192.168.1.113"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "SC",
      "L": "CD",
      "O": "k8s",
      "OU": "System"
    }
  ]
}


# 生成etcd私钥
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd  
创建kubernetes证书
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
cat kubernetes-csr.json 

{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "192.168.1.111",
    "192.168.1.112",
    "192.168.1.113",
    "192.168.1.114",
    "192.168.1.115",
    "192.168.1.116",
    "10.254.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "SC",
      "L": "CD",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

# 生成私钥
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
创建Admin证书
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
cat admin-csr.json

{         
  "CN": "admin",            
  "hosts": [],        
  "key": {                        
    "algo": "rsa",                            
    "size": 2048                                
  },                              
  "names": [                                          
    {                                                       
      "C": "CN",                                                                  
      "ST": "SC",
      "L": "CD",
      "O": "system:masters",                                                                                
      "OU": "System"                                                                                    
    }                                                                                     
  ]
}

# 生成私钥
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes admin-csr.json | cfssljson -bare admin
创建Kube-proxy证书
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
cat kube-proxy-csr.json

{
  "CN": "system:kube-proxy",        
  "hosts": [],        
  "key": {                        
    "algo": "rsa",                            
    "size": 2048                                
  },                              
  "names": [                                          
    {
      "C": "CN",                                                                  
      "ST": "SC",
      "L": "CD",
      "O": "k8s",                                                                                   
      "OU": "System"                                                                                    
    }                                                                                     
  ]
}

# 生成私钥
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

拷贝私钥到Master

1
for i in k8s-m2 k8s-m3;do scp /etc/kubernetes/ssl/*pem $i:/etc/kubernetes/ssl;done

拷贝kube-proxy私钥到Node

1
for i in k8s-n1 k8s-n2 k8s-n3;do scp /etc/kubernetes/ssl/kube-proxy*.pem $i:/etc/kubernetes/ssl;done

创建Kuberconfig文件

当集群开启了 TLS 认证后,每个节点的 kubelet 组件都要使用由 apiserver 使用的 CA 签发的有效证书才能与 apiserver 通讯;此时如果节点多起来,为每个节点单独签署证书将是一件非常繁琐的事情;TLS bootstrapping 功能就是让 kubelet 先使用一个预定的低权限用户连接到 apiserver,然后向 apiserver 申请证书,kubelet 的证书由 apiserver 动态签署

创建 TLS Bootstrapping Token
1
export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
1
2
3
4
5
6
7
cat > token.csv <<EOF 
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"  
EOF

# 确认${BOOTSTRAP_TOKEN}环境变量已经被真实值替换
cat token.csv
5021ae8be94c9b5f5d630b778e6ce01d,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
创建 kubelet bootstrapping kubeconfig 文件
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
# 安装工具
wget https://dl.k8s.io/v1.13.3/kubernetes-server-linux-amd64.tar.gz
tar xvf kubernetes-server-linux-amd64.tar.gz

cp kubernetes/server/bin/{kubeadm,kube-apiserver,kubectl,kube-controller-manager,kube-scheduler} /usr/local/bin/

for i in k8s-m2 k8s-m3;do scp kubernetes/server/bin/{kubeadm,kube-apiserver,kubectl,kube-controller-manager,kube-scheduler} $i:/usr/local/bin/;done  

#
for i in k8s-n1 k8s-n2 k8s-n3;do scp kubernetes/server/bin/{kubelet,kube-proxy} $i:/usr/local/bin
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
export KUBE_APISERVER="https://127.0.0.1:6443"

# 设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=bootstrap.kubeconfig

# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
  --token=${BOOTSTRAP_TOKEN} \
  --kubeconfig=bootstrap.kubeconfig

# 设置上下文参数
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kubelet-bootstrap \
  --kubeconfig=bootstrap.kubeconfig

# 设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
创建 kube-proxy kubeconfig 文件
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
export KUBE_APISERVER="https://127.0.0.1:6443"

# 设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=kube-proxy.kubeconfig

# 设置客户端认证参数
kubectl config set-credentials kube-proxy \
  --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
  --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-proxy.kubeconfig

# 设置上下文参数
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=kube-proxy.kubeconfig

# 设置默认上下文
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

拷贝配置文件到Node

1
2
3
4
5
#
for i in k8s-m2 k8s-m3 k8s-n1 k8s-n2 k8s-n3;do scp token.csv $i:/etc/kubernetes;done 

#
for i in k8s-n1 k8s-n2 k8s-n3;do scp bootstrap.kubeconfig kube-proxy.kubeconfig $i:/etc/kubernetes;done
安装ETCD工具
1
2
3
4
5
wget https://github.com/coreos/etcd/releases/download/v3.3.12/etcd-v3.3.12-linux-amd64.tar.gz  
tar xvf etcd-v3.3.12-linux-amd64.tar.gz  

cp etcd-v3.3.11-linux-amd64/etcd etcdctl /usr/local/bin
# 拷贝到k8s-m2,k8s-m3节点上
配置ETCD
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
# etcd-1

cat /etc/systemd/system/etcd.service

[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
# set GOMAXPROCS to number of processors
ExecStart=/usr/local/bin/etcd \
  --name=etcd1 \
  --cert-file=/etc/kubernetes/ssl/etcd.pem \
  --key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
  --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls=https://192.168.1.111:2380 \
  --listen-peer-urls=https://192.168.1.111:2380 \
  --listen-client-urls=https://192.168.1.111:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://192.168.1.111:2379 \
  --initial-cluster-token=k8s-etcd-cluster \
  --initial-cluster=etcd1=https://192.168.1.111:2380,etcd2=https://192.168.1.112:2380,etcd3=https://192.168.1.113:2380 \
  --initial-cluster-state=new \
  --data-dir=/var/lib/etcd/
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
# etcd-2

Wants=network-online.target

[Service]
Type=notify
# set GOMAXPROCS to number of processors
ExecStart=/usr/local/bin/etcd \
  --name=etcd2 \
  --cert-file=/etc/kubernetes/ssl/etcd.pem \
  --key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
  --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls=https://192.168.1.112:2380 \
  --listen-peer-urls=https://192.168.1.112:2380 \
  --listen-client-urls=https://192.168.1.112:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://192.168.1.112:2379 \
  --initial-cluster-token=k8s-etcd-cluster \
  --initial-cluster=etcd1=https://192.168.1.111:2380,etcd2=https://192.168.1.112:2380,etcd3=https://192.168.1.113:2380 \
  --initial-cluster-state=new \
  --data-dir=/var/lib/etcd/
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
# etcd-3

Wants=network-online.target

[Service]
Type=notify
# set GOMAXPROCS to number of processors
ExecStart=/usr/local/bin/etcd \
  --name=etcd3 \
  --cert-file=/etc/kubernetes/ssl/etcd.pem \
  --key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
  --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls=https://192.168.1.113:2380 \
  --listen-peer-urls=https://192.168.1.113:2380 \
  --listen-client-urls=https://192.168.1.113:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://192.168.1.113:2379 \
  --initial-cluster-token=k8s-etcd-cluster \
  --initial-cluster=etcd1=https://192.168.1.111:2380,etcd2=https://192.168.1.112:2380,etcd3=https://192.168.1.113:2380 \
  --initial-cluster-state=new \
  --data-dir=/var/lib/etcd/
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
启动etcd
1
2
systemctl daemon-reload && systemctl enable etcd && systemctl start etcd
systemctl status etcd
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
# 验证 etcd 集群状态
root@k8s-m1:~# etcdctl --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
   --cert-file=/etc/kubernetes/ssl/etcd.pem \
   --ca-file=/etc/kubernetes/ssl/ca.pem \
   --key-file=/etc/kubernetes/ssl/etcd-key.pem \
   cluster-health
member 95605c07b7eb732b is healthy: got healthy result from https://192.168.1.111:2379
member cc6dae876b8cf735 is healthy: got healthy result from https://192.168.1.112:2379
member f14198b921191b87 is healthy: got healthy result from https://192.168.1.113:2379
cluster is healthy
1
2
3
4
5
6
7
8
root@k8s-m1:~# etcdctl --endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379\
   --cert-file=/etc/kubernetes/ssl/etcd.pem \
   --ca-file=/etc/kubernetes/ssl/ca.pem \
   --key-file=/etc/kubernetes/ssl/etcd-key.pem \
   member list
95605c07b7eb732b: name=etcd1 peerURLs=https://192.168.1.111:2380 clientURLs=https://192.168.1.111:2379 isLeader=false
cc6dae876b8cf735: name=etcd2 peerURLs=https://192.168.1.112:2380 clientURLs=https://192.168.1.112:2379 isLeader=true
f14198b921191b87: name=etcd3 peerURLs=https://192.168.1.113:2380 clientURLs=https://192.168.1.113:2379 isLeader=false

部署Master节点

创建 Kubectl kubeconfig 文件
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
export KUBE_APISERVER="https://127.0.0.1:6443"

# 设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER}

# 设置客户端认证参数
kubectl config set-credentials admin \
  --client-certificate=/etc/kubernetes/ssl/admin.pem \
  --embed-certs=true \
  --client-key=/etc/kubernetes/ssl/admin-key.pem

# 设置上下文参数
kubectl config set-context kubernetes \
  --cluster=kubernetes \
  --user=admin

# 设置默认上下文
kubectl config use-context kubernetes

生成高级审核配置文件 官方说明

1
2
3
4
5
6
7
8
9
cat audit-policy.yaml 

# Log all requests at the Metadata level.
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
- level: Metadata

# 拷贝到k8s-m2,k8s-m3
创建 kube-apiserver.service 文件
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
cat /etc/systemd/system/kube-apiserver.service

[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
User=root
ExecStart=/usr/local/bin/kube-apiserver \
  --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction \
  --anonymous-auth=false \
  --token-auth-file=/etc/kubernetes/token.csv \
  --advertise-address=192.168.1.111 \
  --allow-privileged=true \
  --apiserver-count=3 \
  --audit-policy-file=/etc/kubernetes/audit-policy.yaml \
  --audit-log-maxage=30 \
  --audit-log-maxbackup=3 \
  --audit-log-maxsize=100 \
  --audit-log-path=/var/log/kubernetes/audit.log \
  --authorization-mode=Node,RBAC \
  --bind-address=0.0.0.0 \
  --secure-port=6443 \
  --client-ca-file=/etc/kubernetes/ssl/ca.pem \
  --enable-swagger-ui=true \
  --etcd-cafile=/etc/kubernetes/ssl/ca.pem \
  --etcd-certfile=/etc/kubernetes/ssl/etcd.pem \
  --etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem \
  --etcd-servers=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 \
  --event-ttl=1h \
  --kubelet-https=true \
  --insecure-bind-address=127.0.0.1 \
  --insecure-port=8080 \
  --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --service-cluster-ip-range=10.254.0.0/16 \
  --service-node-port-range=30000-32000 \
  --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \
  --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
  --enable-bootstrap-token-auth \
  --v=1
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
启动kube-apiserver
1
2
3
systemctl daemon-reload && systemctl enable kube-apiserver 
systemctl start kube-apiserver
systemctl status kube-apiserver
创建 Kube-controller-manager 文件
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
cat /etc/systemd/system/kube-controller-manager.service

[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
  --address=127.0.0.1 \
  --master=http://127.0.0.1:8080 \
  --service-cluster-ip-range=10.254.0.0/16 \
  --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
  --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --feature-gates=RotateKubeletServerCertificate=true \
  --experimental-cluster-signing-duration=86700h0m0s \
  --cluster-name=kubernetes \
  --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --root-ca-file=/etc/kubernetes/ssl/ca.pem \
  --leader-elect=true \
  --node-monitor-grace-period=40s \
  --node-monitor-period=5s \
  --pod-eviction-timeout=5m0s \
  --v=1
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
启动 Kube-controller-manager
1
2
3
4
systemctl daemon-reload 
systemctl enable kube-controller-manager.service
systemctl start kube-controller-manager.service
systemctl status kube-controller-manager.service
创建 Kube-scheduler 文件
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
cat /etc/systemd/system/kube-scheduler.service

[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
ExecStart=/usr/local/bin/kube-scheduler \
  --logtostderr=true \    
  --master=http://127.0.0.1:8080 \    
  --leader-elect=true \      
  --v=1 
Restart=on-failure      
RestartSec=5    

[Install]       
WantedBy=multi-user.target
启动 Kube-scheduler
1
2
3
systemctl daemon-reload && systemctl enable kube-scheduler.service
systemctl start kube-scheduler.service
systemctl status kube-scheduler.service

拷贝配置文件到k8-m2,k8-m3

验证 kube-scheduler 的 ha

kube-scheduler 通过配置 leader-elect=true 自动选择 leader

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
kubectl get endpoints kube-scheduler --namespace=kube-system  -o yaml

apiVersion: v1
kind: Endpoints
metadata:
  annotations:
    control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"k8s-m1_f4864882-37fa-11e9-8110-000c294c98c5","leaseDurationSeconds":15,"acquireTime":"2019-02-24T06:11:00Z","renewTime":"2019-02-24T06:32:00Z","leaderTransitions":0}'
  creationTimestamp: "2019-02-24T06:11:00Z"
  name: kube-scheduler
  namespace: kube-system
  resourceVersion: "1753"
  selfLink: /api/v1/namespaces/kube-system/endpoints/kube-scheduler
  uid: f5326aae-37fa-11e9-9ced-000c294c98c5
验证 Master 节点
1
2
3
4
5
6
7
kubectl get componentstatuses
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok                  
scheduler            Healthy   ok                  
etcd-0               Healthy   {"health":"true"}   
etcd-2               Healthy   {"health":"true"}   
etcd-1               Healthy   {"health":"true"}   

部署 Node 节点

安装 OS 依赖组件
1
apt-get -y install socat conntrack ipset
创建自动批准相关 CSR 请求的 CluseterRole
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
cat /etc/kubernetes/tls-instructs-csr.yaml

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeserver
rules:
- apiGroups: ["certificates.k8s.io"]
  resources: ["certificatesigningrequests/selfnodeserver"]
  verbs: ["create"]
导入 yaml文件
1
kubectl apply -f /etc/kubernetes/tls-instructs-csr.yaml
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
# 自动批准 system:bootstrappers 组用户 TLS bootstrapping 首次申请证书的 CSR 请求
kubectl create clusterrolebinding node-client-auto-approve-csr --clusterrole=system:certificates.k8s.io:certificatesigningrequests:nodeclient --group=system:bootstrappers


# 自动批准 system:nodes 组用户更新 kubelet 自身与 apiserver 通讯证书的 CSR 请求
kubectl create clusterrolebinding node-client-auto-renew-crt --clusterrole=system:certificates.k8s.io:certificatesigningrequests:selfnodeclient --group=system:nodes


# 自动批准 system:nodes 组用户更新 kubelet 10250 api 端口证书的 CSR 请求
kubectl create clusterrolebinding node-server-auto-renew-crt --clusterrole=system:certificates.k8s.io:certificatesigningrequests:selfnodeserver --group=system:nodes
配置 Kubelet

kubelet 启动时向 kube-apiserver 发送 TLS bootstrapping 请求,需要先将 bootstrap token 文件中的 kubelet-bootstrap 用户赋予 system:node-bootstrapper cluster role,使其能够发起 CSR 请求 然后 kubelet 才能有权限创建认证请求

1
2
3
4
# master 执行
kubectl create clusterrolebinding kubelet-bootstrap \
  --clusterrole=system:node-bootstrapper \
  --user=kubelet-bootstrap
创建 kubelet 配置文件

kubelet、kube-proxy、flannel都在node节点安装

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
cat /etc/systemd/system/kubelet.service

[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/bin/kubelet \
  --hostname-override=192.168.1.114 \
  --cgroup-driver=cgroupfs \
  --cluster-dns=10.254.0.2 \
  --fail-swap-on=false \
  --cluster-domain=cluster.local. \
  --serialize-image-pulls=false \
  --feature-gates=RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true \
  --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0 \
  --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \
  --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
  --cert-dir=/etc/kubernetes/ssl \
  --logtostderr=true \
  --v=1

[Install]
WantedBy=multi-user.target

分发到node2、node3

启动 kubelet
1
2
3
4
5
# 在启动kubelet之前,创建目录,否则会报错
mkdir /var/lib/kubelet

systemctl daemon-reload && systemctl enable kubelet 
systemctl start kubelet
1
2
3
4
5
# 查看CSR请求
kubectl get csr

# 手动签名请求
kubectl certificate approve csr-xx
配置 kube-proxy
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
vim /etc/systemd/system/kube-proxy.service

[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \
  --logtostderr=true \
  --masquerade-all \
  --proxy-mode=ipvs \
  --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \
  --cluster-cidr=10.254.0.0/16 \
  --hostname-override=192.168.1.114 \
  --v=4
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
启动 kube-proxy
1
2
systemctl daemon-reload && systemctl enable kube-proxy
systemctl start kube-proxy

分发到node2、node3

部署 Flannel

下载 Flannel
1
2
3
4
5
wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz

tar -xvf flannel-v0.11.0-linux-amd64.tar.gz

mv flanneld mk-docker-opts.sh /usr/local/bin/
etcd 注册网段
1
2
3
4
5
6
# Node 上执行
etcdctl --endpoints="https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379"\
--cert-file=/etc/kubernetes/ssl/etcd.pem \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--key-file=/etc/kubernetes/ssl/etcd-key.pem \ 
set /flannel/network/config '{"Network":"10.254.0.0/16","Backend":{"Type":"vxlan"}}'
配置 flanneld
1
2
3
vim /etc/kubernetes/flanneld 

FLANNEL_OPTIONS="--etcd-endpoints=https://192.168.1.111:2379,https://192.168.1.112:2379,https://192.168.1.113:2379 -etcd-cafile=/etc/kubernetes/ssl/ca.pem -etcd-certfile=/etc/kubernetes/ssl/etcd.pem -etcd-keyfile=/etc/kubernetes/ssl/etcd-key.pem -etcd-prefix=/flannel/network"
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
vim /etc/systemd/system/flanneld.service

[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service

[Service]
Type=notify
EnvironmentFile=/etc/kubernetes/flanneld
ExecStart=/usr/local/bin/flanneld --ip-masq $FLANNEL_OPTIONS
ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure

[Install]
WantedBy=multi-user.target
配置 docker

配置Docker启动指定子网 修改EnvironmentFile=/run/flannel/subnet.env,ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS即可

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
vim /lib/systemd/system/docker.service

[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
启动 flannel 服务
1
2
3
systemctl daemon-reload && systemctl start flanneld
systemctl enable flanneld && systemctl restart docker
systemctl restart kubelet && systemctl restart kube-proxy
验证服务
1
2
3
4
5
6
cat /run/flannel/subnet.env

DOCKER_OPT_BIP="--bip=10.254.50.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=false"
DOCKER_OPT_MTU="--mtu=1450"
DOCKER_NETWORK_OPTIONS=" --bip=10.254.50.1/24 --ip-masq=false --mtu=1450"
1
2
3
4
5
root@k8s-m1:~# kubectl get nodes
NAME            STATUS   ROLES    AGE   VERSION
192.168.1.114   Ready    <none>   22h   v1.13.3
192.168.1.115   Ready    <none>   22h   v1.13.3
192.168.1.116   Ready    <none>   22h   v1.13.3

创建一个nginx的service测试一下集群是否可用

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
root@k8s-m1:~# kubectl run nginx --replicas=2 --labels="run=load-balancer-example" --image=nginx  --port=80
kubectl run --generator=deployment/apps.v1 is DEPRECATED and will be removed in a future version. Use kubectl run --generator=run-pod/v1 or kubectl create instead.
deployment.apps/nginx created

root@k8s-m1:~# kubectl expose deployment nginx --type=NodePort --name=example-service
service/example-service exposed

root@k8s-m1:~# kubectl describe svc example-service
Name:                     example-service
Namespace:                default
Labels:                   run=load-balancer-example
Annotations:              <none>
Selector:                 run=load-balancer-example
Type:                     NodePort
IP:                       10.254.57.235
Port:                     <unset>  80/TCP
TargetPort:               80/TCP
NodePort:                 <unset>  30424/TCP
Endpoints:                10.254.66.2:80,10.254.69.2:80
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>

root@k8s-m1:~# kubectl get pod --all-namespaces -o wide
NAMESPACE   NAME                     READY   STATUS    RESTARTS   AGE     IP            NODE            NOMINATED NODE   READINESS GATES
default     nginx-58db6fdb58-gnzz4   1/1     Running   0          13m     10.254.69.2   192.168.1.116   <none>           <none>
default     nginx-58db6fdb58-z9prw   1/1     Running   0          3m46s   10.254.66.2   192.168.1.115   <none>           <none>

# 访问以下任何一个地址都可以看到nginx页面
192.168.1.114-116:30424

配置 CoreDNS

官方地址 https://coredns.io

下载 yaml 文件
1
2
3
wget https://github.com/coredns/deployment/blob/master/kubernetes/coredns.yaml.sed

mv coredns.yaml.sed coredns.yaml
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
vi coredns.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health
        kubernetes cluster.local 10.254.0.0/16 {
          pods insecure
          upstream
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        proxy . /etc/resolv.conf
        cache 30
        loop
        reload
        loadbalance
    }
...
  clusterIP: 10.254.0.2

# kubernetes cluster.local 为 创建 svc 的 IP 段
kubernetes cluster.local 10.254.0.0/16

#clusterIP  为 指定 DNS 的 IP
clusterIP: 10.254.0.2
导入 coredns yaml文件
1
2
3
4
5
6
7
root@k8s-m1:~# kubectl create -f coredns.yaml 
serviceaccount/coredns created
clusterrole.rbac.authorization.k8s.io/system:coredns created
clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
configmap/coredns created
deployment.apps/coredns created
service/kube-dns created
查看 coredns 服务
1
2
3
4
5
6
7
root@k8s-m1:~# kubectl get pod,svc -n kube-system
NAME                           READY   STATUS    RESTARTS   AGE
pod/coredns-5d668bd598-629rb   1/1     Running   0          14s
pod/coredns-5d668bd598-6k4wq   1/1     Running   0          14s

NAME               TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE
service/kube-dns   ClusterIP   10.254.0.2   <none>        53/UDP,53/TCP,9153/TCP   14s
查看日志
1
2
3
4
5
6
7
root@k8s-m1:~# kubectl logs -n kube-system pod/coredns-5d668bd598-629rb
.:53
2019-03-04T14:43:49.716Z [INFO] CoreDNS-1.3.1
2019-03-04T14:43:49.716Z [INFO] linux/amd64, go1.11.4, 6b56a9c
CoreDNS-1.3.1
linux/amd64, go1.11.4, 6b56a9c
2019-03-04T14:43:49.716Z [INFO] plugin/reload: Running configuration MD5 = 369e3e711c70345b5cbec6458ad5c091
验证 DNS 服务
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
vi test_coredns.yaml

apiVersion: v1
kind: Pod
metadata:
  name: alpine
spec:
  containers:
  - name: alpine
    image: alpine
    command:
    - sleep
    - "3600"

# 测试
root@k8s-m1:~# kubectl exec -it alpine nslookup kubernetes
nslookup: can't resolve '(null)': Name does not resolve

Name:      kubernetes
Address 1: 10.254.0.1 kubernetes.default.svc.cluster.local
部署 DNS 自动伸缩

按照 node 数量 自动伸缩 dns 数量 根据当前可调度节点和核心以及给定的缩放参数,计算所需的副本计数并将其应用于DNS后端

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
# 下载配置文件
curl -O https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml

# 替换image 
vi dns-horizontal-autoscaler.yaml

# Copyright 2016 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

kind: ServiceAccount
apiVersion: v1
metadata:
  name: kube-dns-autoscaler
  namespace: kube-system
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:kube-dns-autoscaler
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
rules:
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["list", "watch"]
  - apiGroups: [""]
    resources: ["replicationcontrollers/scale"]
    verbs: ["get", "update"]
  - apiGroups: ["extensions","apps"]
    resources: ["deployments/scale", "replicasets/scale"]
    verbs: ["get", "update"]
# Remove the configmaps rule once below issue is fixed:
# kubernetes-incubator/cluster-proportional-autoscaler#16
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["get", "create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:kube-dns-autoscaler
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
subjects:
  - kind: ServiceAccount
    name: kube-dns-autoscaler
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: system:kube-dns-autoscaler
  apiGroup: rbac.authorization.k8s.io

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kube-dns-autoscaler
  namespace: kube-system
  labels:
    k8s-app: kube-dns-autoscaler
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  selector:
    matchLabels:
      k8s-app: kube-dns-autoscaler
  template:
    metadata:
      labels:
        k8s-app: kube-dns-autoscaler
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
    spec:
      priorityClassName: system-cluster-critical
      securityContext:
        supplementalGroups: [ 65534 ]
        fsGroup: 65534
      containers:
      - name: autoscaler
        image: zhengwei919/cluster-proportional-autoscaler-amd64:1.4.0
        resources:
            requests:
                cpu: "20m"
                memory: "10Mi"
        command:
          - /cluster-proportional-autoscaler
          - --namespace=kube-system
          - --configmap=kube-dns-autoscaler
          # Should keep target in sync with cluster/addons/dns/kube-dns.yaml.base
          - --target=Deployment/kube-dns
          # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate.
          # If using small nodes, "nodesPerReplica" should dominate.
          - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}}
          - --logtostderr=true
          - --v=2
      tolerations:
      - key: "CriticalAddonsOnly"
        operator: "Exists"
      serviceAccountName: kube-dns-autoscaler
1
2
# 创建部署
kubectl create -f dns-horizontal-autoscaler.yaml

部署 Dashboard

官方地址

下载 dashboard.yaml 文件
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
curl -O https://github.com/kubernetes/dashboard/blob/master/aio/deploy/recommended/kubernetes-dashboard.yaml

# 替换所有images
sed -i 's/k8s\.gcr\.io/zhengwei919/g' kubernetes-dashboard.yaml

# 创建部署
kubectl create -f kubernetes-dashboard.yaml

# 因为 Service 是 ClusterIP 类型,为了方便使用,我们可通过下面命令修改成 NodePort 类型
kubectl patch svc kubernetes-dashboard -p '{"spec":{"type":"NodePort"}}' -n kube-system

# 查看pod和svc
kubectl get pods,svc -n kube-system
创建Admin账户

部署成功后可以直接访问 https://NODE_IP:30000 访问,但是想要登录进去查看的话需要使用 kubeconfig 或者 access token 的方式;实际上这个就是 RBAC 授权控制,以下提供一个创建 admin access token 的脚本

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# 首先创建一个 dashboard rbac 超级用户
vim dashboard-admin-rbac.yaml

---
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-admin
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kubernetes-dashboard-admin
  labels:
    k8s-app: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: kubernetes-dashboard-admin
  namespace: kube-system
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
# 导入文件
kubectl create -f dashboard-admin-rbac.yaml

serviceaccount/kubernetes-dashboard-admin created
clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard-admin created

# 查看创建的token
kubectl -n kube-system get secret | grep kubernetes-dashboard-admin

# 查看登陆令牌
kubectl describe secret kubernetes-dashboard-admin-token-rw2l8 -n kube-system

访问https://NODE_IP:port 令牌登陆

Dashboard-UI