1. 环境准备
- 二进制安装Kubernets1.15.2: https://www.yuque.com/duduniao/k8s/tr3hch
- dashboard版本: 2.0.1
- metrics-server: 0.3.3
| 主机 | 主机ip | 当前作用 |
|---|---|---|
| hdss7-21 | 10.4.7.11 10.4.7.10浮动IP | dns |
| hdss7-21 | 10.4.7.21 | master,node节点 |
| hdss7-22 | 10.4.7.22 | master,node节点 |
| hdss7-200 | 10.4.7.200 | 签发证书节点 |
注意:
- 由于 Kubernetes API 版本之间的重大更改，某些功能可能无法在dashbooard中正常使用
- 监控信息不需要通过 Heapster 来提供，而是通过 Metrics Server 来提供，Metrics Scraper服务来采集，不需要单独维护 Heapster(从kubernetes1.19.0+起,dashboard版本更改为2.0.0+和集成了Metrics Scraper)
2. 部署dashboard
2.1 准备资源配置清单
[root@hdss7-21 ~]# mkdir ~/dashboard
[root@hdss7-21 ~]# cd ~/dashboard
yaml文件下载: https://github.com/kubernetes/kubernetes/blob/v1.19.0/cluster/addons/dashboard/dashboard.yaml
dashboard.yaml
apiVersion: v1
kind: Namespace
metadata:
name: kubernetes-dashboard
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
ports:
- port: 443
targetPort: 8443
selector:
k8s-app: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: EnsureExists
name: kubernetes-dashboard-certs
namespace: kubernetes-dashboard
type: Opaque
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: EnsureExists
name: kubernetes-dashboard-csrf
namespace: kubernetes-dashboard
type: Opaque
data:
csrf: ""
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: EnsureExists
name: kubernetes-dashboard-key-holder
namespace: kubernetes-dashboard
type: Opaque
---
kind: ConfigMap
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: EnsureExists
name: kubernetes-dashboard-settings
namespace: kubernetes-dashboard
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
name: kubernetes-dashboard
namespace: kubernetes-dashboard
rules:
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
verbs: ["get", "update", "delete"]
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["kubernetes-dashboard-settings"]
verbs: ["get", "update"]
- apiGroups: [""]
resources: ["services"]
resourceNames: ["heapster", "dashboard-metrics-scraper"]
verbs: ["proxy"]
- apiGroups: [""]
resources: ["services/proxy"]
resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
verbs: ["get"]
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
name: kubernetes-dashboard
rules:
- apiGroups: ["metrics.k8s.io"]
resources: ["pods", "nodes"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
name: kubernetes-dashboard
namespace: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kubernetes-dashboard
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: kubernetesui/dashboard:v2.0.1
imagePullPolicy: Always
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
- --namespace=kubernetes-dashboard
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
- mountPath: /tmp
name: tmp-volume
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
nodeSelector:
"kubernetes.io/os": linux
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
- key: node-role.kubernetes.io/master
effect: NoSchedule
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kubernetes-dashboard
spec:
ports:
- port: 8000
targetPort: 8000
selector:
k8s-app: dashboard-metrics-scraper
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kubernetes-dashboard
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: dashboard-metrics-scraper
template:
metadata:
labels:
k8s-app: dashboard-metrics-scraper
annotations:
seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
spec:
containers:
- name: dashboard-metrics-scraper
image: kubernetesui/metrics-scraper:v1.0.4
ports:
- containerPort: 8000
protocol: TCP
livenessProbe:
httpGet:
scheme: HTTP
path: /
port: 8000
initialDelaySeconds: 30
timeoutSeconds: 30
volumeMounts:
- mountPath: /tmp
name: tmp-volume
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
serviceAccountName: kubernetes-dashboard
nodeSelector:
"kubernetes.io/os": linux
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
volumes:
- name: tmp-volume
emptyDir: {}
2.2 创建管理员用户
user.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: dashboard-admin-sa
namespace: kubernetes-dashboard
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: dashboard-admin-sa
namespace: kubernetes-dashboard
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: dashboard-admin-sa
namespace: kubernetes-dashboard
2.3. 创建ingress资源
如果当前没有使用ingress来提供服务, 可在dashboard的资源清单service资源指定NodePort提供服务
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: kubernetes-dashboard
namespace: kubernetes-dashboard
annotations:
kubernetes.io/ingress.class: traefik
spec:
rules:
- host: dashboard.odl.com
http:
paths:
- backend:
serviceName: kubernetes-dashboard
servicePort: 443
2.4. dashboard交付至K8s
[root@hdss7-21 dashboard ]# ll ~/dashboard
总用量 16
-rw-r--r-- 1 root root 6887 12月 7 10:42 dashboard.yaml
-rw-r--r-- 1 root root 328 12月 7 10:44 ingress.yaml
-rw-r--r-- 1 root root 605 12月 7 10:17 user.yaml
[root@hdss7-21 dashboard ]# kubectl apply -f .
namespace/kubernetes-dashboard created
serviceaccount/kubernetes-dashboard created
service/kubernetes-dashboard created
secret/kubernetes-dashboard-certs created
secret/kubernetes-dashboard-csrf created
secret/kubernetes-dashboard-key-holder created
configmap/kubernetes-dashboard-settings created
role.rbac.authorization.k8s.io/kubernetes-dashboard created
clusterrole.rbac.authorization.k8s.io/kubernetes-dashboard created
rolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
clusterrolebinding.rbac.authorization.k8s.io/kubernetes-dashboard created
deployment.apps/kubernetes-dashboard created
service/dashboard-metrics-scraper created
deployment.apps/dashboard-metrics-scraper created
ingress.extensions/kubernetes-dashboard created
serviceaccount/dashboard-admin-sa created
clusterrolebinding.rbac.authorization.k8s.io/dashboard-admin-sa created
2.5 修改dns节点
如果使用NodePort映射端口, 可忽略此步骤
[root@hdss7-11 ~]# vim /var/named/odl.com.zone
$ORIGIN odl.com.
$TTL 600 ; 10 minutes
@ IN SOA dns.odl.com. dnsadmin.odl.com. (
2020091712 ; serial
10800 ; refresh (3 hours)
900 ; retry (15 minutes)
604800 ; expire (1 week)
86400 ; minimum (1 day)
)
NS dns.odl.com.
$TTL 60 ; 1 minute
dns A 10.4.7.11
harbor A 10.4.7.200
k8s-yaml A 10.4.7.200
traefik A 10.4.7.10
dashboard A 10.4.7.10
2.6. 登录dashboard界面
2.6.1 查看secret资源
[root@hdss7-21 ~]# kubectl get secret -n kubernetes-dashboard
NAME TYPE DATA AGE
dashboard-admin-sa-token-qrkdl kubernetes.io/service-account-token 3 30m
default-token-h4p79 kubernetes.io/service-account-token 3 37m
kubernetes-dashboard-certs Opaque 0 37m
kubernetes-dashboard-csrf Opaque 1 37m
kubernetes-dashboard-key-holder Opaque 2 37m
kubernetes-dashboard-token-n8t4c kubernetes.io/service-account-token 3 37m
[root@hdss7-21 ~]# kubectl describe secret dashboard-admin-sa-token-qrkdl -n kubernetes-dashboard
Name: dashboard-admin-sa-token-qrkdl
Namespace: kubernetes-dashboard
Labels: <none>
Annotations: kubernetes.io/service-account.name: dashboard-admin-sa
kubernetes.io/service-account.uid: 661f4adb-b51b-46d5-b9f8-966c91161f20
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1346 bytes
namespace: 20 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9......(省略)
2.6.2. 使用token登录界面
image.png
image.png
image.png
3. 安装metrics-server
yaml文件: https://github.com/kubernetes/kubernetes/tree/v1.15.12/cluster/addons/metrics-server
metrics-server 0.3.3
addon-resizer:1.8.5
镜像的下载需要科学上网
3.1 准备资源配置清单
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: metrics-server:system:auth-delegator
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: metrics-server-auth-reader
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
name: v1beta1.metrics.k8s.io
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
service:
name: metrics-server
namespace: kube-system
group: metrics.k8s.io
version: v1beta1
insecureSkipTLSVerify: true
groupPriorityMinimum: 100
versionPriority: 100
metrics-server-deployment.yaml
参数 :
metrics-server
- 启动command新增参数 - --kubelet-insecure-tls
- 启动command注释参数 --kubelet-port=10250
- 启动command注释参数 - --deprecated-kubelet-completely-insecure=true
addon-resizer
- 修改 - --cpu={{ base_metrics_server_cpu }} ==> - --cpu=80m
- 修改 - --memory={{ base_metrics_server_memory }} ==> - --extra-memory=80Mi
- 修改 --extra-memory={{ metrics_server_memory_per_node }}Mi ==> --extra-memory=8Mi
- 注释 - --minClusterSize={{ metrics_server_min_cluster_size }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: metrics-server
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: v1
kind: ConfigMap
metadata:
name: metrics-server-config
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: EnsureExists
data:
NannyConfiguration: |-
apiVersion: nannyconfig/v1alpha1
kind: NannyConfiguration
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: metrics-server-v0.3.3
namespace: kube-system
labels:
k8s-app: metrics-server
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
version: v0.3.3
spec:
selector:
matchLabels:
k8s-app: metrics-server
version: v0.3.3
template:
metadata:
name: metrics-server
labels:
k8s-app: metrics-server
version: v0.3.3
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
spec:
priorityClassName: system-cluster-critical
serviceAccountName: metrics-server
containers:
- name: metrics-server
image: k8s.gcr.io/metrics-server-amd64:v0.3.3
command:
- /metrics-server
- --metric-resolution=30s
# These are needed for GKE, which doesn't support secure communication yet.
# Remove these lines for non-GKE clusters, and when GKE supports token-based auth.
#- --kubelet-port=10255
#- --deprecated-kubelet-completely-insecure=true
- --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP
- --kubelet-insecure-tls
ports:
- containerPort: 443
name: https
protocol: TCP
- name: metrics-server-nanny
image: k8s.gcr.io/addon-resizer:1.8.5
resources:
limits:
cpu: 100m
memory: 300Mi
requests:
cpu: 5m
memory: 50Mi
env:
- name: MY_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: MY_POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: metrics-server-config-volume
mountPath: /etc/config
command:
- /pod_nanny
- --config-dir=/etc/config
#- --cpu={{ base_metrics_server_cpu }}
- --cpu=80m
- --extra-cpu=0.5m
#- --memory={{ base_metrics_server_memory }}
- --memory=80Mi
#- --extra-memory={{ metrics_server_memory_per_node }}Mi
- --extra-memory=8Mi
- --threshold=5
- --deployment=metrics-server-v0.3.3
- --container=metrics-server
- --poll-period=300000
- --estimator=exponential
# Specifies the smallest cluster (defined in number of nodes)
# resources will be scaled to.
# 注释
# - --minClusterSize={{ metrics_server_min_cluster_size }}
volumes:
- name: metrics-server-config-volume
configMap:
name: metrics-server-config
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
apiVersion: v1
kind: Service
metadata:
name: metrics-server
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "Metrics-server"
spec:
selector:
k8s-app: metrics-server
ports:
- port: 443
protocol: TCP
targetPort: https
rules.resources 添加资源 - nodes/stats参数
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:metrics-server
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
rules:
- apiGroups:
- ""
resources:
- pods
- nodes
- namespaces
# 添加
- nodes/stats
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- deployments
verbs:
- get
- list
- update
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:metrics-server
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
3.2. 开启apiserver聚合层
- 在master节点要能访问metrics server pod ip（kubeadm部署默认已经满足该条件，二进制部署需注意要在master节点也部署node组件）
- 二进制安装需要开启聚合层（kubeadm默认已经启用，二进制部署需自己启用）
- 如果您未在 master 节点上运行 kube-proxy，则必须确保 kube-apiserver 启动参数中包含--enable-aggregator-routing=true
3.2.1. cfssl生成证书
[root@hdss7-200 certs]# vim metrics-server-csr.json
{
"CN": "aggregator",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "odl",
"OU": "System"
}
]
[root@hdss7-200 certs]# cfssl gencert \
-ca=/opt/certs/ca.pem \
-ca-key=/opt/certs/ca-key.pem \
-config=/opt/certs/ca-config.json \
-profile=clent metrics-server-csr.json | cfssl-json -bare metrics-server
报错: {"code":5100,"message":"Invalid policy: no key usage available"}
-profile=kubernetes metrics-server-csr.json 的kubernetes 在ca-config.json文件中不存在
ca-config.json添加
"kubernetes": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
3.2.2. 将证书复制至所有master节点
[root@hdss7-200 certs]# ll metrics-server*
-rw-r--r-- 1 root root 997 12月 7 16:38 metrics-server.csr
-rw-r--r-- 1 root root 220 12月 7 16:19 metrics-server-csr.json
-rw------- 1 root root 1675 12月 7 16:38 metrics-server-key.pem
-rw-r--r-- 1 root root 1371 12月 7 16:38 metrics-server.pem
[root@hdss7-200 certs]# scp metrics-server.pem metrics-server-key.pem hdss7-21:/opt/kubernetes/server/bin/certs
[root@hdss7-200 certs]# scp metrics-server.pem metrics-server-key.pem hdss7-22:/opt/kubernetes/server/bin/certs
3.2.3. 所有apiserver启动文件添加参数
[root@hdss7-21 ~]# vim /opt/kubernetes/server/bin/kube-apiserver-startup.sh
/opt/kubernetes/server/bin/kube-apiserver
.....
.....
--requestheader-client-ca-file=./certs/ca.pem \
--requestheader-allowed-names="aggregator" \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--proxy-client-cert-file=./certs/metrics-server.pem \
--proxy-client-key-file=./certs/metrics-server-key.pem
3.2.4. 所有kubelet添加authentication-token-webhook参数
[root@hdss7-21 ~]# vim /opt/kubernetes/server/bin/kubelet-startup.sh
/opt/kubernetes/server/bin/kubelet \
...
...
--authentication-token-webhook=true
3.2.5. 重启apiserver和kubelet
supervisor是用Python开发的一个client/server服务，是Linux/Unix系统下的一个进程管理工具
[root@hdss7-21 ~]# supervisorctl restart kube-apiserver-7-21 kube-kubelet-7-21
[root@hdss7-22 ~]# supervisorctl restart kube-apiserver-7-22 kube-kubelet-7-22
3.3. metrics-server交付至k8s
[root@hdss7-21 metrics-server]# ll
总用量 24
-rw-r--r-- 1 root root 398 11月 27 17:16 auth-delegator.yaml
-rw-r--r-- 1 root root 420 11月 27 17:16 auth-reader.yaml
-rw-r--r-- 1 root root 393 11月 27 17:18 metrics-apiservice.yaml
-rw-r--r-- 1 root root 3220 12月 7 10:02 metrics-server-deployment.yaml
-rw-r--r-- 1 root root 336 11月 27 17:19 metrics-server-service.yaml
-rw-r--r-- 1 root root 817 12月 1 17:26 resource-reader.yaml
[root@hdss7-21 metrics-server]# kubectl apply -f .
horization.k8s.io/metrics-server:system:auth-delegator created
rolebinding.rbac.authorization.k8s.io/metrics-server-auth-reader created
apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io created
clusterrole.rbac.authorization.k8s.io/system:metrics-server created
clusterrolebinding.rbac.authorization.k8s.io/system:metrics-server created
serviceaccount/metrics-server created
configmap/metrics-server-config created
deployment.apps/metrics-server-v0.3.3 created
service/metrics-server created
3.3. 查看kubectl top是否有信息
等待几分钟后查看
[root@hdss7-21 ~]# kubectl top nodes
NAME STATUS ROLES AGE VERSION
hdss7-21.host.com Ready master,node 5d21h v1.15.12
hdss7-22.host.com Ready master,node 22d v1.15.12
3.4. 登录dashboard界面查看
image.png
image.png
至此,Kubernetes-dashboard安装metrics-server实现完整的性能数据采集和监控功能
如有疑问,可留下评论.
