Kubernetes双向TLS配置-Centos7

简介:

kubernetes 提供了多种安全认证机制,

其中对于集群通讯间可采用 TLS(https) 双向认证机制,也可采用基于 Token 或用户名密码的单向 tls 认证。

kubernetes 一般在内网部署,采用私有 IP 地址进行通讯,权威 CA 貌似只能签署域名证书,所以我们采用自建CA。

目标:

在kubernetes集群中,自建CA并配置各组件间的双向TLS认证

环境:

k8s-master:192.168.12.177

k8s-node01:192.168.12.178

软件版本:

k8s:v1.3.4

实施步骤:

1.master主机

1.1使用自建的CA签署证书

#新建CA
[root@master ~]# cd key/
[root@master key]# openssl genrsa -out ca-key.pem 2048
[root@master key]# openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca"
[root@master key]# ll
total 8
-rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
-rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem

#新建openssl.conf文件
[root@master key]# vi openssl.cnf 
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = kubernetes
DNS.2 = kubernetes.default
DNS.3 = kubernetes.default.svc
DNS.4 = kubernetes.default.svc.test01.com
IP.1 = 10.254.0.1
IP.2 = 192.168.12.177

#制作apiserver key
[root@master key]# openssl genrsa -out apiserver-key.pem 2048
[root@master key]# openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config openssl.cnf

#签署apiserver证书
[root@master key]# openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile openssl.cnf
[root@master key]# ll
total 28
-rw-r--r-- 1 root root 1115 Sep 21 11:11 apiserver.csr
-rw-r--r-- 1 root root 1679 Sep 21 11:11 apiserver-key.pem
-rw-r--r-- 1 root root 1184 Sep 21 11:12 apiserver.pem
-rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
-rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem
-rw-r--r-- 1 root root 17 Sep 21 11:12 ca.srl
-rw-r--r-- 1 root root 398 Sep 21 11:10 openssl.cnf

#新建node01的openssl.conf
[root@master key]# vi worker-openssl-001.cnf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[alt_names]
IP.1 = 192.168.12.178

#制作node01 key
[root@master key]# openssl genrsa -out k8s-node001-worker-key.pem 2048
[root@master key]# openssl req -new -key k8s-node001-worker-key.pem -out k8s-node001-worker.csr -subj "/CN=k8s-node001" -config worker-openssl-001.cnf

#签署node01证书
[root@master key]# openssl x509 -req -in k8s-node001-worker.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out k8s-node001-worker.pem -days 365 -extensions v3_req -extfile worker-openssl-001.cnf
[root@master key]# ll
total 44
-rw-r--r-- 1 root root 1115 Sep 21 11:11 apiserver.csr
-rw-r--r-- 1 root root 1679 Sep 21 11:11 apiserver-key.pem
-rw-r--r-- 1 root root 1184 Sep 21 11:12 apiserver.pem
-rw-r--r-- 1 root root 1675 Sep 21 11:05 ca-key.pem
-rw-r--r-- 1 root root 1090 Sep 21 11:06 ca.pem
-rw-r--r-- 1 root root 17 Sep 21 11:15 ca.srl
-rw-r--r-- 1 root root 972 Sep 21 11:14 k8s-node001-worker.csr
-rw-r--r-- 1 root root 1679 Sep 21 11:14 k8s-node001-worker-key.pem
-rw-r--r-- 1 root root 1046 Sep 21 11:15 k8s-node001-worker.pem
-rw-r--r-- 1 root root 398 Sep 21 11:10 openssl.cnf
-rw-r--r-- 1 root root 262 Sep 21 11:13 worker-openssl-001.cnf

1.2master主机修改相关配置

#复制apiserver的pem文件到ssl目录
[root@master key]# mkdir -p /etc/kubernetes/ssl
[root@master key]# cd /etc/kubernetes/ssl
[root@master ssl]# cp /root/key/ca.pem .
[root@master ssl]# cp /root/key/apiserver.pem .
[root@master ssl]# cp /root/key/apiserver-key.pem .

#新建kubeconfig文件
[root@master ~]# vi /etc/kubernetes/cm-kubeconfig.yaml 
apiVersion: v1
kind: Config
clusters:
- name: local
 cluster:
 certificate-authority: /etc/kubernetes/ssl/ca.pem
users:
- name: controllermanager
 user:
 client-certificate: /etc/kubernetes/ssl/apiserver.pem
 client-key: /etc/kubernetes/ssl/apiserver-key.pem
contexts:
- context:
 cluster: local
 user: controllermanager
 name: kubelet-context
current-context: kubelet-context

#按下面配置,修改各组件
#因controllmanager、scheduler与apiserver在同一台主机上,也可以使用非安全端口8080)
[root@master ~]# grep -v ^# /etc/kubernetes/config 
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=https://192.168.12.177:6443"
[root@master ~]# grep -v ^# /etc/kubernetes/apiserver 
KUBE_API_ADDRESS="--bind-address=0.0.0.0 --insecure-bind-address=127.0.0.1"
KUBE_API_PORT="--secure-port=6443 --insecure-port=8080"
KUBE_ETCD_SERVERS="--etcd-servers=http://192.168.12.177:2379"
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
KUBE_API_ARGS="--client-ca-file=/etc/kubernetes/ssl/ca.pem \
 --tls-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \
 --tls-cert-file=/etc/kubernetes/ssl/apiserver.pem \
 --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem"
[root@master ~]# grep -v ^# /etc/kubernetes/controller-manager 
KUBE_CONTROLLER_MANAGER_ARGS="--service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem \
 --root-ca-file=/etc/kubernetes/ssl/ca.pem \
 --master=http://127.0.0.1:8080 \
 --kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml"
[root@master ~]# grep -v ^# /etc/kubernetes/scheduler 
KUBE_SCHEDULER_ARGS="--master=http://127.0.0.1:8080 \
 --kubeconfig=/etc/kubernetes/cm-kubeconfig.yaml"

#重启各组件
[root@master kubernetes]# systemctl restart kube-controller-manager
[root@master kubernetes]# systemctl restart kube-scheduler
[root@master kubernetes]# systemctl status kube-apiserver

#查看状态
[root@master kubernetes]# systemctl status kube-controller-manager -l 
[root@master kubernetes]# systemctl status kube-scheduler -l 
[root@master kubernetes]# systemctl status kube-apiserver -l

#验证证书
[root@master kubernetes]# curl https://192.168.12.177:6443/api/v1/nodes --cert /etc/kubernetes/ssl/apiserver.pem --key /etc/kubernetes/ssl/apiserver-key.pem --cacert /etc/kubernetes/ssl/ca.pem

2.node01主机

#将master主机生成的node01证书复制到ssl目录
[root@node01 ~]# mkdir -p /etc/kubernetes/ssl/
[root@node01 ~]# cd /etc/kubernetes/ssl/
[root@node01 ssl]# ll
total 12
-rw-r--r-- 1 root root 1090 Sep 21 11:49 ca.pem
-rw-r--r-- 1 root root 1679 Sep 21 11:49 k8s-node001-worker-key.pem
-rw-r--r-- 1 root root 1046 Sep 21 11:50 k8s-node001-worker.pem

#做一个超链接
[root@node01 ssl]# ln -s k8s-node001-worker.pem worker.pem
[root@node01 ssl]# ln -s k8s-node001-worker-key.pem worker-key.pem

#新建kubeconfig
[root@node01 ssl]# vi /etc/kubernetes/worker-kubeconfig.yaml 
apiVersion: v1
kind: Config
clusters:
- name: local
 cluster:
 certificate-authority: /etc/kubernetes/ssl/ca.pem
users:
- name: kubelet
 user:
 client-certificate: /etc/kubernetes/ssl/worker.pem
 client-key: /etc/kubernetes/ssl/worker-key.pem
contexts:
- context:
 cluster: local
 user: kubelet
 name: kubelet-context
current-context: kubelet-context

#按照如下配置,修改各配置文件
[root@node01 ssl]# grep -v ^# /etc/kubernetes/config 
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=false"
KUBE_MASTER="--master=https://192.168.12.177:6443"
[root@node01 ssl]# grep -v ^# /etc/kubernetes/kubelet 
KUBELET_ADDRESS="--address=0.0.0.0"
KUBELET_PORT="--port=10250"
KUBELET_HOSTNAME="--hostname-override=node01.test01.com"
KUBELET_API_SERVER="--api-servers=https://192.168.12.177:6443"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"
KUBELET_ARGS="--cluster-dns=192.168.12.177 \
 --cluster-domain=test01.com \
 --tls-cert-file=/etc/kubernetes/ssl/worker.pem \
 --tls-private-key-file=/etc/kubernetes/ssl/worker-key.pem \
 --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"
[root@node01 ssl]# grep -v ^# /etc/kubernetes/proxy 
KUBE_PROXY_ARGS="--kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"

#重启服务
[root@node01 ssl]# systemctl restart docker kubelet kube-proxy

#查看
[root@node01 ssl]# systemctl status docker kubelet kube-proxy -l

#验证证书
[root@node01 ssl]# curl https://192.168.12.177:6443/api/v1/nodes --cert /etc/kubernetes/ssl/worker.pem --key /etc/kubernetes/ssl/worker-key.pem --cacert /etc/kubernetes/ssl/ca.pem

3.新建svc进行测试

#新建svc
[root@master template]# cd registry/
[root@master registry]# kubectl create -f registry-rc.yaml 
[root@master registry]# kubectl create -f registry-svc.yaml

#查看
[root@master registry]# kubectl get pods --namespace=kube-system
NAME READY STATUS RESTARTS AGE
kube-registry-v0-jypg1 1/1 Running 0 38s
[root@master registry]# kubectl get rc --namespace=kube-system
NAME DESIRED CURRENT AGE
kube-registry-v0 1 1 38s
[root@master registry]# kubectl get svc --namespace=kube-system
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kube-registry 10.254.80.217 <nodes> 5000/TCP 38s

#上传镜像至私有仓库
[root@master ssl]# docker pull alpine
[root@master ssl]# docker images
[root@master ssl]# docker tag docker.io/alpine 192.168.12.178:30099/alpine
[root@master ssl]# docker push 192.168.12.178:30099/alpine
The push refers to a repository [192.168.12.178:30099/alpine]
3fc666989c1d: Pushed 
latest: digest: sha256:02eb5cfe4b721495135728ab4aea87418fd4edbfbf83612130a81191f0b2aae3 size: 506

4.遗留问题-dashboard部署(已经解决)

4.1目前双向TLS配置后,dashboard报错如下(已经解决)

#错误提示
[root@master ui]# kubectl get pods --namespace=kube-system
NAME READY STATUS RESTARTS AGE
kube-registry-v0-jypg1 1/1 Running 2 2h
kubernetes-dashboard-v1.1.0-wd9g3 0/1 CrashLoopBackOff 5 4m
[root@master ui]# kubectl logs -f pods/kubernetes-dashboard-v1.1.0-wd9g3 --namespace=kube-system 
Starting HTTP server on port 9090
Creating API server client for https://192.168.12.177:6443
Error while initializing connection to Kubernetes apiserver. This most likely means that the cluster is misconfigured (e.g., it has invalid apiserver certificates or service accounts configuration) or the --apiserver-host param points to a server that does not exist. Reason: Get https://192.168.12.177:6443/version: x509: failed to load system roots and no roots provided

上述问题,是因为认证问题,我们需要修改一下yaml文件。

#完整的yaml文件如下:
[root@master ui]# vi dashboard-controller.yaml 
apiVersion: v1
kind: ReplicationController
metadata:
 name: kubernetes-dashboard-v1.1.0
 namespace: kube-system
 labels:
 k8s-app: kubernetes-dashboard
 version: v1.1.0
 kubernetes.io/cluster-service: "true"
spec:
 replicas: 1
 selector:
 k8s-app: kubernetes-dashboard
 template:
 metadata:
 labels:
 k8s-app: kubernetes-dashboard
 version: v1.1.0
 kubernetes.io/cluster-service: "true"
 spec:
 containers:
 - name: kubernetes-dashboard
 image: index.tenxcloud.com/google_containers/kubernetes-dashboard-amd64:v1.1.0
 resources:
 # keep request = limit to keep this container in guaranteed class
 limits:
 cpu: 100m
 memory: 50Mi
 requests:
 cpu: 100m
 memory: 50Mi
 ports:
 - containerPort: 9090
 livenessProbe:
 httpGet:
 path: /
 port: 9090
 initialDelaySeconds: 30
 timeoutSeconds: 30
 volumeMounts:
 - mountPath: /etc/kubernetes/ssl
 name: ssl-certs-kubernetes
 readOnly: true
 - mountPath: /etc/ssl/certs
 name: ssl-certs-host
 readOnly: true
 volumes:
 - hostPath:
 path: /etc/kubernetes/ssl
 name: ssl-certs-kubernetes
 - hostPath:
 path: /etc/pki/tls/certs
 name: ssl-certs-host

#创建
[root@master ui]# kubectl create -f dashboard-controller.yaml 
replicationcontroller "kubernetes-dashboard-v1.1.0" created

#查看
[root@master ui]# kubectl get pods --namespace=kube-system
NAME READY STATUS RESTARTS AGE
kubernetes-dashboard-v1.1.0-dt16g 1/1 Running 0 2s

可以看到,相比原来yaml文件,增加了ssl认证部分。

4.2自建CA,masterIP只能https访问了,那么windows如何访问dashboard(已经解决)

默认dashboard是通过master的ip和端口进行访问的,但是在windows下因无法加载自建ca证书到时无法访问dashboard。

解决办法是修改yaml,添加nodeport。

#完整的yaml文件,如下
[root@master ui]# vi dashboard-service.yaml 
apiVersion: v1
kind: Service
metadata:
 name: kubernetes-dashboard
 namespace: kube-system
 labels:
 k8s-app: kubernetes-dashboard
 kubernetes.io/cluster-service: "true"
spec:
 selector:
 k8s-app: kubernetes-dashboard
 type: NodePort
 ports:
 - port: 80
 nodePort: 30080
 targetPort: 9090

#创建
[root@master ui]# kubectl create -f dashboard-service.yaml
You have exposed your service on an external port on all nodes in your
cluster. If you want to expose this service to the external internet, you may
need to set up firewall rules for the service port(s) (tcp:30080) to serve traffic.

See http://releases.k8s.io/release-1.3/docs/user-guide/services-firewalls.md for more details.
service "kubernetes-dashboard" created

#查看
[root@master ui]# kubectl get svc --namespace=kube-system
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes-dashboard 10.254.0.253 <nodes> 80/TCP 1m

windows下浏览器打开

http://192.168.12.178:30080/

k8s-tls-001