[HARDWAY] 09.Bootstrapping the Kubernetes Worker Nodes

https://github.com/kelseyhightower/kubernetes-the-hard-way/blob/master/docs/09-bootstrapping-kubernetes-workers.md

### Provisioning a Kubernetes Worker Node

# Pre ENV

service firewalld stop

systemctl disable firewalld

iptables -L 

yum install -y wget

yum install -y bind-utils

sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux

sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

setenforce 0

# Install the OS dependencies:

yum -y install socat conntrack ipset

# Disable Swap

kubelet 은 swap 이 활설화 되어있는경우 동작하지 않음

swapon --show

swapoff -a

[root@worker01 ~]# cat /etc/fstab 

#

# /etc/fstab

# Created by anaconda on Fri Nov 13 16:49:56 2020

#

# Accessible filesystems, by reference, are maintained under '/dev/disk'

# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info

#

/dev/mapper/centos-root /                       xfs     defaults        0 0

UUID=1e96ed00-419d-481d-baaf-b43b7ef7c3bb /boot                   xfs     defaults        0 0

#/dev/mapper/centos-swap swap                    swap    defaults        0 0

# /etc/fstab 을 수정해야한다. sed 로 해도 되지만 fstab 은 위험도가 높은 파일이니 손으로 주석처리 해주자

#/dev/mapper/centos-swap swap                    swap    defaults        0 0

# Download and Install Worker Binaries

wget \

  https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.18.0/crictl-v1.18.0-linux-amd64.tar.gz \

  https://github.com/opencontainers/runc/releases/download/v1.0.0-rc91/runc.amd64 \

  https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz \

  https://github.com/containerd/containerd/releases/download/v1.3.6/containerd-1.3.6-linux-amd64.tar.gz \

  https://storage.googleapis.com/kubernetes-release/release/v1.18.6/bin/linux/amd64/kubectl \

  https://storage.googleapis.com/kubernetes-release/release/v1.18.6/bin/linux/amd64/kube-proxy \

  https://storage.googleapis.com/kubernetes-release/release/v1.18.6/bin/linux/amd64/kubelet

sudo mkdir -p \

  /etc/cni/net.d \

  /opt/cni/bin \

  /var/lib/kubelet \

  /var/lib/kube-proxy \

  /var/lib/kubernetes \

  /var/run/kubernetes

mkdir containerd

tar -xvf crictl-v1.18.0-linux-amd64.tar.gz

tar -xvf containerd-1.3.6-linux-amd64.tar.gz -C containerd

tar -xvf cni-plugins-linux-amd64-v0.8.6.tgz -C /opt/cni/bin/

mv runc.amd64 runc

chmod +x crictl kubectl kube-proxy kubelet runc 

mv crictl kubectl kube-proxy kubelet runc /usr/local/bin/

mv containerd/bin/* /bin/

### Configure CNI Networking

# Create the bridge network configuration file:

cat <<EOF | sudo tee /etc/cni/net.d/10-bridge.conf

{

    "cniVersion": "0.3.1",

    "name": "bridge",

    "type": "bridge",

    "bridge": "cnio0",

    "isGateway": true,

    "ipMasq": true,

    "ipam": {

        "type": "host-local",

        "ranges": [

          [{"subnet": "10.200.0.0/16"}]

        ],

        "routes": [{"dst": "0.0.0.0/0"}]

    }

}

EOF

# Create the loopback network configuration file:

cat <<EOF | sudo tee /etc/cni/net.d/99-loopback.conf

{

    "cniVersion": "0.3.1",

    "name": "lo",

    "type": "loopback"

}

EOF

### Configure containerd

mkdir -p /etc/containerd/

cat << EOF | sudo tee /etc/containerd/config.toml

[plugins]

  [plugins.cri.containerd]

    snapshotter = "overlayfs"

    [plugins.cri.containerd.default_runtime]

      runtime_type = "io.containerd.runtime.v1.linux"

      runtime_engine = "/usr/local/bin/runc"

      runtime_root = ""

EOF

# Create the containerd.service systemd unit file:

cat <<EOF | sudo tee /etc/systemd/system/containerd.service

[Unit]

Description=containerd container runtime

Documentation=https://containerd.io

After=network.target

[Service]

ExecStartPre=/sbin/modprobe overlay

ExecStart=/bin/containerd

Restart=always

RestartSec=5

Delegate=yes

KillMode=process

OOMScoreAdjust=-999

LimitNOFILE=1048576

LimitNPROC=infinity

LimitCORE=infinity

[Install]

WantedBy=multi-user.target

EOF

### Configure the Kubelet

mv ${HOSTNAME}-key.pem ${HOSTNAME}.pem /var/lib/kubelet/

mv ${HOSTNAME}.kubeconfig /var/lib/kubelet/kubeconfig

mv ca.pem /var/lib/kubernetes/

# Create the kubelet-config.yaml configuration file

cat <<EOF | sudo tee /var/lib/kubelet/kubelet-config.yaml

kind: KubeletConfiguration

apiVersion: kubelet.config.k8s.io/v1beta1

authentication:

  anonymous:

    enabled: false

  webhook:

    enabled: true

  x509:

    clientCAFile: "/var/lib/kubernetes/ca.pem"

authorization:

  mode: Webhook

clusterDomain: "cluster.local"

clusterDNS:

  - "10.32.0.10"

podCIDR: "10.200.0.0/16"

resolvConf: "/run/systemd/resolve/resolv.conf"

runtimeRequestTimeout: "15m"

tlsCertFile: "/var/lib/kubelet/${HOSTNAME}.pem"

tlsPrivateKeyFile: "/var/lib/kubelet/${HOSTNAME}-key.pem"

EOF

# Create the kubelet.service systemd unit file:

cat <<EOF | sudo tee /etc/systemd/system/kubelet.service

[Unit]

Description=Kubernetes Kubelet

Documentation=https://github.com/kubernetes/kubernetes

After=containerd.service

Requires=containerd.service

[Service]

ExecStart=/usr/local/bin/kubelet \\

  --config=/var/lib/kubelet/kubelet-config.yaml \\

  --container-runtime=remote \\

  --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock \\

  --image-pull-progress-deadline=2m \\

  --kubeconfig=/var/lib/kubelet/kubeconfig \\

  --network-plugin=cni \\

  --register-node=true \\

  --v=2

Restart=on-failure

RestartSec=5

[Install]

WantedBy=multi-user.target

EOF

### Configure the Kubernetes Proxy

mv kube-proxy.kubeconfig /var/lib/kube-proxy/kubeconfig

# Create the kube-proxy-config.yaml configuration file:

cat <<EOF | sudo tee /var/lib/kube-proxy/kube-proxy-config.yaml

kind: KubeProxyConfiguration

apiVersion: kubeproxy.config.k8s.io/v1alpha1

clientConnection:

  kubeconfig: "/var/lib/kube-proxy/kubeconfig"

mode: "iptables"

clusterCIDR: "10.200.0.0/16"

EOF

# Create the kube-proxy.service systemd unit file:

cat <<EOF | sudo tee /etc/systemd/system/kube-proxy.service

[Unit]

Description=Kubernetes Kube Proxy

Documentation=https://github.com/kubernetes/kubernetes

[Service]

ExecStart=/usr/local/bin/kube-proxy \\

  --config=/var/lib/kube-proxy/kube-proxy-config.yaml

Restart=on-failure

RestartSec=5

[Install]

WantedBy=multi-user.target

EOF

### Start the Worker Services

systemctl daemon-reload

systemctl enable containerd kubelet kube-proxy

systemctl restart containerd kubelet kube-proxy

##### Verification

Master Node 에서 Command 실행

"kubectl get nodes --kubeconfig admin.kubeconfig"

[root@master01 ~]# kubectl get node --kubeconfig admin.kubeconfig

NAME       STATUS   ROLES    AGE    VERSION

worker01   Ready    <none>   2d1h   v1.18.6

worker02   Ready    <none>   2d1h   v1.18.6

worker03   Ready    <none>   2d1h   v1.18.6

worker04   Ready    <none>   5m7s   v1.18.6