(5)kube-apiserver部署

创建相关目录

# 三台master主机
mkdir -p /k8s/kube-apiserver

生成证书

创建配置文件

cd /opt/k8s-playbook/ssl
vim kubernetes-csr.json
#################################
{
  "CN": "kubernetes-master",
  "hosts": [
    "127.0.0.1",
    "192.168.104.61",
    "192.168.104.62",
    "192.168.104.63",
    "10.254.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local.",
    "kubernetes.default.svc.zhfi.k8s."
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Shenzhen",
      "L": "Shenzhen",
      "O": "zhfi",
      "OU": "magina"
    }
  ]
}

生成

cfssl gencert -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes

分发

cp kubernetes*.pem /etc/k8s/cert/

scp kubernetes*.pem root@master-02:/etc/k8s/cert/
scp kubernetes*.pem root@master-03:/etc/k8s/cert/

创建加密文件

vim encryption-config.yaml
#################################
kind: EncryptionConfig
apiVersion: v1
resources:
  - resources:
      - secrets
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: pb2Sroltwv7BSYs4ssdC11Gu2JY3WKhJXP4X0PRPebk=
      - identity: {}
#################################

分发

cp encryption-config.yaml /etc/k8s/

scp encryption-config.yaml root@master-02:/etc/k8s/
scp encryption-config.yaml root@master-03:/etc/k8s/

创建审计文件

vim audit-policy.yaml
#################################
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
  # The following requests were manually identified as high-volume and low-risk, so drop them.
  - level: None
    resources:
      - group: ""
        resources:
          - endpoints
          - services
          - services/status
    users:
      - 'system:kube-proxy'
    verbs:
      - watch

  - level: None
    resources:
      - group: ""
        resources:
          - nodes
          - nodes/status
    userGroups:
      - 'system:nodes'
    verbs:
      - get

  - level: None
    namespaces:
      - kube-system
    resources:
      - group: ""
        resources:
          - endpoints
    users:
      - 'system:kube-controller-manager'
      - 'system:kube-scheduler'
      - 'system:serviceaccount:kube-system:endpoint-controller'
    verbs:
      - get
      - update

  - level: None
    resources:
      - group: ""
        resources:
          - namespaces
          - namespaces/status
          - namespaces/finalize
    users:
      - 'system:apiserver'
    verbs:
      - get

  # Don't log HPA fetching metrics.
  - level: None
    resources:
      - group: metrics.k8s.io
    users:
      - 'system:kube-controller-manager'
    verbs:
      - get
      - list

  # Don't log these read-only URLs.
  - level: None
    nonResourceURLs:
      - '/healthz*'
      - /version
      - '/swagger*'

  # Don't log events requests.
  - level: None
    resources:
      - group: ""
        resources:
          - events

  # node and pod status calls from nodes are high-volume and can be large, don't log responses
  # for expected updates from nodes
  - level: Request
    omitStages:
      - RequestReceived
    resources:
      - group: ""
        resources:
          - nodes/status
          - pods/status
    users:
      - kubelet
      - 'system:node-problem-detector'
      - 'system:serviceaccount:kube-system:node-problem-detector'
    verbs:
      - update
      - patch

  - level: Request
    omitStages:
      - RequestReceived
    resources:
      - group: ""
        resources:
          - nodes/status
          - pods/status
    userGroups:
      - 'system:nodes'
    verbs:
      - update
      - patch

  # deletecollection calls can be large, don't log responses for expected namespace deletions
  - level: Request
    omitStages:
      - RequestReceived
    users:
      - 'system:serviceaccount:kube-system:namespace-controller'
    verbs:
      - deletecollection

  # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
  # so only log at the Metadata level.
  - level: Metadata
    omitStages:
      - RequestReceived
    resources:
      - group: ""
        resources:
          - secrets
          - configmaps
      - group: authentication.k8s.io
        resources:
          - tokenreviews
  # Get repsonses can be large; skip them.
  - level: Request
    omitStages:
      - RequestReceived
    resources:
      - group: ""
      - group: admissionregistration.k8s.io
      - group: apiextensions.k8s.io
      - group: apiregistration.k8s.io
      - group: apps
      - group: authentication.k8s.io
      - group: authorization.k8s.io
      - group: autoscaling
      - group: batch
      - group: certificates.k8s.io
      - group: extensions
      - group: metrics.k8s.io
      - group: networking.k8s.io
      - group: policy
      - group: rbac.authorization.k8s.io
      - group: scheduling.k8s.io
      - group: settings.k8s.io
      - group: storage.k8s.io
    verbs:
      - get
      - list
      - watch

  # Default level for known APIs
  - level: RequestResponse
    omitStages:
      - RequestReceived
    resources:
      - group: ""
      - group: admissionregistration.k8s.io
      - group: apiextensions.k8s.io
      - group: apiregistration.k8s.io
      - group: apps
      - group: authentication.k8s.io
      - group: authorization.k8s.io
      - group: autoscaling
      - group: batch
      - group: certificates.k8s.io
      - group: extensions
      - group: metrics.k8s.io
      - group: networking.k8s.io
      - group: policy
      - group: rbac.authorization.k8s.io
      - group: scheduling.k8s.io
      - group: settings.k8s.io
      - group: storage.k8s.io

  # Default level for all other requests.
  - level: Metadata
    omitStages:
      - RequestReceived

分发

cp audit-policy.yaml /etc/k8s/

scp audit-policy.yaml root@master-02:/etc/k8s/
scp audit-policy.yaml root@master-03:/etc/k8s/

创建客户端证书

cd /opt/k8s-playbook/ssl
vim proxy-client-csr.json
#################################
{
  "CN": "aggregator",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Shenzhen",
      "L": "Shenzhen",
      "O": "zhfi",
      "OU": "magina"
    }
  ]
}

生成

cfssl gencert -ca=ca.pem \
  -ca-key=ca-key.pem  \
  -config=ca-config.json  \
  -profile=kubernetes proxy-client-csr.json | cfssljson -bare proxy-client

分发

cp proxy-client*.pem /etc/k8s/cert/

scp proxy-client*.pem root@master-02:/etc/k8s/cert/
scp proxy-client*.pem root@master-03:/etc/k8s/cert/

创建kube-apiserver.service

vim /etc/systemd/system/kube-apiserver.service 
#################################
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
WorkingDirectory=/k8s/kube-apiserver
ExecStart=/opt/k8s/bin/kube-apiserver \
  --advertise-address=192.168.104.61 \
  --bind-address=192.168.104.61 \
  --default-not-ready-toleration-seconds=360 \
  --default-unreachable-toleration-seconds=360 \
  --feature-gates=DynamicAuditing=true \
  --max-mutating-requests-inflight=2000 \
  --max-requests-inflight=4000 \
  --default-watch-cache-size=200 \
  --delete-collection-workers=2 \
  --encryption-provider-config=/etc/k8s/encryption-config.yaml \
  --etcd-cafile=/etc/k8s/cert/ca.pem \
  --etcd-certfile=/etc/k8s/cert/kubernetes.pem \
  --etcd-keyfile=/etc/k8s/cert/kubernetes-key.pem \
  --etcd-servers=https://192.168.104.61:2379,https://192.168.104.62:2379,https://192.168.104.63:2379 \
  --secure-port=6443 \
  --tls-cert-file=/etc/k8s/cert/kubernetes.pem \
  --tls-private-key-file=/etc/k8s/cert/kubernetes-key.pem \
  --insecure-port=0 \
  --audit-dynamic-configuration \
  --audit-log-maxage=15 \
  --audit-log-maxbackup=3 \
  --audit-log-maxsize=100 \
  --audit-log-truncate-enabled \
  --audit-log-path=/k8s/kube-apiserver/audit.log \
  --audit-policy-file=/etc/k8s/audit-policy.yaml \
  --profiling \
  --anonymous-auth=false \
  --client-ca-file=/etc/k8s/cert/ca.pem \
  --enable-bootstrap-token-auth \
  --requestheader-allowed-names="aggregator" \
  --requestheader-client-ca-file=/etc/k8s/cert/ca.pem \
  --requestheader-extra-headers-prefix="X-Remote-Extra-" \
  --requestheader-group-headers=X-Remote-Group \
  --requestheader-username-headers=X-Remote-User \
  --service-account-key-file=/etc/k8s/cert/ca.pem \
  --authorization-mode=Node,RBAC \
  --runtime-config=api/all=true \
  --enable-admission-plugins=NodeRestriction \
  --allow-privileged=true \
  --apiserver-count=3 \
  --event-ttl=168h \
  --kubelet-certificate-authority=/etc/k8s/cert/ca.pem \
  --kubelet-client-certificate=/etc/k8s/cert/kubernetes.pem \
  --kubelet-client-key=/etc/k8s/cert/kubernetes-key.pem \
  --kubelet-https=true \
  --kubelet-timeout=10s \
  --proxy-client-cert-file=/etc/k8s/cert/proxy-client.pem \
  --proxy-client-key-file=/etc/k8s/cert/proxy-client-key.pem \
  --service-cluster-ip-range=10.254.0.0/16 \
  --service-node-port-range=30000-32767 \
  --logtostderr=true \
  --v=2
Restart=on-failure
RestartSec=10
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
  • --advertise-address: 修改成当前主机IP
  • --bind-address: 修改成当前主机IP

复制二进制文件并启动

cd /opt/k8s-playbook/
cp kubernetes/server/bin/kube-apiserver /opt/k8s/bin/

前提是已将相关组件解压到/opt/k8s-playbook/

scp /opt/k8s/bin/kube-apiserver root@master-02:/opt/k8s/bin/
scp /opt/k8s/bin/kube-apiserver root@master-03:/opt/k8s/bin/

# 三台master分别执行
chmod +x /opt/k8s/bin/kube-apiserver

systemctl enable kube-apiserver
systemctl start kube-apiserver

查看状态

kubectl cluster-info

展示评论