[HARDWAY] 07.Bootstrapping the etcd Cluster

https://github.com/kelseyhightower/kubernetes-the-hard-way/blob/master/docs/07-bootstrapping-etcd.md

ETCD 는 각 MASTER 에 설치된다. 

INTERNAL_IP 는 MASTER 의 IP

ETCD_NAME 은 master server 의 hostname 으로 세팅

### Download and Install the etcd Binaries

wget "https://github.com/etcd-io/etcd/releases/download/v3.4.10/etcd-v3.4.10-linux-amd64.tar.gz"

tar -xvf etcd-v3.4.10-linux-amd64.tar.gz

mv etcd-v3.4.10-linux-amd64/etcd* /usr/local/bin/

### Configure the etcd Server

mkdir -p /etc/etcd /var/lib/etcd

chmod 700 /var/lib/etcd

cp ca.pem kubernetes-key.pem kubernetes.pem /etc/etcd/

INTERNAL_IP=$(ip addr | grep global | awk '{print $2}' | cut -d/ -f1)

ETCD_NAME=$(hostname -s)

# MASTER IP 를 넣기위해 MASTER IP 변수 등록

MASTER_NODE=("192.168.1.21" "192.168.1.55" "192.168.1.56")

# ETCD Service 생성

cat <<EOF | sudo tee /etc/systemd/system/etcd.service

[Unit]

Description=etcd

Documentation=https://github.com/coreos

[Service]

Type=notify

ExecStart=/usr/local/bin/etcd \\

  --name ${ETCD_NAME} \\

  --cert-file=/etc/etcd/kubernetes.pem \\

  --key-file=/etc/etcd/kubernetes-key.pem \\

  --peer-cert-file=/etc/etcd/kubernetes.pem \\

  --peer-key-file=/etc/etcd/kubernetes-key.pem \\

  --trusted-ca-file=/etc/etcd/ca.pem \\

  --peer-trusted-ca-file=/etc/etcd/ca.pem \\

  --peer-client-cert-auth \\

  --client-cert-auth \\

  --initial-advertise-peer-urls https://${INTERNAL_IP}:2380 \\

  --listen-peer-urls https://${INTERNAL_IP}:2380 \\

  --listen-client-urls https://${INTERNAL_IP}:2379,https://127.0.0.1:2379 \\

  --advertise-client-urls https://${INTERNAL_IP}:2379 \\

  --initial-cluster-token etcd-cluster-0 \\

  --initial-cluster master01=https://${MASTER_NODE[0]}:2380,master02=https://${MASTER_NODE[1]}:2380,master03=https://${MASTER_NODE[2]}:2380 \\

  --initial-cluster-state new \\

  --data-dir=/var/lib/etcd

Restart=on-failure

RestartSec=5

[Install]

WantedBy=multi-user.target

EOF

### Start the etcd Server

systemctl daemon-reload

systemctl enable etcd

systemctl start etcd

### 확인

ETCDCTL_API=3 \

  etcdctl member list \

  --endpoints=https://127.0.0.1:2379 \

  --cacert=/etc/etcd/ca.pem \

  --cert=/etc/etcd/kubernetes.pem \

  --key=/etc/etcd/kubernetes-key.pem

# 결과물 예시

[root@master01 ~]# ETCDCTL_API=3 \

> etcdctl member list \

>   --endpoints=https://127.0.0.1:2379 \

>   --cacert=/etc/etcd/ca.pem \

>   --cert=/etc/etcd/kubernetes.pem \

>   --key=/etc/etcd/kubernetes-key.pem

27dce9766f396ef2, started, master03, https://192.168.1.56:2380, https://192.168.1.56:2379, false

4b07d8c024408053, started, master01, https://192.168.1.21:2380, https://192.168.1.21:2379, false

9e4bcb58402b8dba, started, master02, https://192.168.1.55:2380, https://192.168.1.55:2379, false

위의 내용을 정리해서 아래의 간단 설치 스크립트를 구성함

[root@master01 ~]# cat 031_etcd_install.sh 

#!/bin/bash

INTERNAL_IP=$(ip addr | grep global | awk '{print $2}' | cut -d/ -f1)

ETCD_NAME=$(hostname -s)

MASTER_NODE=("192.168.1.21" "192.168.1.55" "192.168.1.56")

### etcd config directroy create

mkdir -p /etc/etcd /var/lib/etcd

chmod 700 /var/lib/etcd

cp ca.pem kubernetes-key.pem kubernetes.pem /etc/etcd/

echo "ETCD Server INTERNAL_IP = ${INTERNAL_IP}"

echo "ETCD Server NAME = ${ETCD_NAME}"

### etcd birany download ###

echo ""

echo "ETCD Server Install Start"

echo ""

echo "ETCD Birnay Download"

wget "https://github.com/etcd-io/etcd/releases/download/v3.4.10/etcd-v3.4.10-linux-amd64.tar.gz"

sleep 2

### etcd birnay move

tar -xvf etcd-v3.4.10-linux-amd64.tar.gz

sleep 2

mv etcd-v3.4.10-linux-amd64/etcd* /usr/local/bin/

sleep 2

echo ""

echo "ETCD Service ADD"

### etcd Service Add

cat <<EOF | sudo tee /etc/systemd/system/etcd.service

[Unit]

Description=etcd

Documentation=https://github.com/coreos

[Service]

Type=notify

ExecStart=/usr/local/bin/etcd \\

  --name ${ETCD_NAME} \\

  --cert-file=/etc/etcd/kubernetes.pem \\

  --key-file=/etc/etcd/kubernetes-key.pem \\

  --peer-cert-file=/etc/etcd/kubernetes.pem \\

  --peer-key-file=/etc/etcd/kubernetes-key.pem \\

  --trusted-ca-file=/etc/etcd/ca.pem \\

  --peer-trusted-ca-file=/etc/etcd/ca.pem \\

  --peer-client-cert-auth \\

  --client-cert-auth \\

  --initial-advertise-peer-urls https://${INTERNAL_IP}:2380 \\

  --listen-peer-urls https://${INTERNAL_IP}:2380 \\

  --listen-client-urls https://${INTERNAL_IP}:2379,https://127.0.0.1:2379 \\

  --advertise-client-urls https://${INTERNAL_IP}:2379 \\

  --initial-cluster-token etcd-cluster-0 \\

  --initial-cluster master01=https://${MASTER_NODE[0]}:2380,master02=https://${MASTER_NODE[1]}:2380,master03=https://${MASTER_NODE[2]}:2380 \\

  --initial-cluster-state new \\

  --data-dir=/var/lib/etcd

Restart=on-failure

RestartSec=5

[Install]

WantedBy=multi-user.target

EOF

echo ""

echo "ETCD Service START"

systemctl daemon-reload

systemctl enable etcd

systemctl start etcd

echo ""

echo "ETCD Service Verification"

ETCDCTL_API=3 \

  etcdctl member list \

  --endpoints=https://127.0.0.1:2379 \

  --cacert=/etc/etcd/ca.pem \

  --cert=/etc/etcd/kubernetes.pem \

  --key=/etc/etcd/kubernetes-key.pem

echo ""

echo "ETCD Server Install END"

echo ""

# 스크립트 작성 후 저장

# etcd 를 master01 에서 원격으로 설치하는 스크립트

[root@master01 ~]# cat 032_remote_etcd_install_exec.sh 

#!/bin/bash

MASTER=("master01" "master02" "master03")

### etcd install scripts copy

for ((i=0; i<3; i++)); do

  echo "etcd ${MASTER[i]} install scripts copy START"

  scp 031_etcd_install.sh ${MASTER[i]}:~/

  echo "etcd ${MASTER[i]} install scripts copy END"

  echo ""

  sleep 2

done

### etcd install scripts execute

for ((i=0; i<3; i++)); do

  echo "etcd ${MASTER[i]} install scripts EXEC START"

  ssh ${MASTER[i]} sh ~/031_etcd_install.sh

  echo "etcd ${MASTER[i]} install scripts EXEC END"

  echo ""

  sleep 2

done

실행 후 etcd 상태값  확인

[root@master01 ~]# ETCDCTL_API=3 \

> etcdctl member list \

>   --endpoints=https://127.0.0.1:2379 \

>   --cacert=/etc/etcd/ca.pem \

>   --cert=/etc/etcd/kubernetes.pem \

>   --key=/etc/etcd/kubernetes-key.pem

27dce9766f396ef2, started, master03, https://192.168.1.56:2380, https://192.168.1.56:2379, false

4b07d8c024408053, started, master01, https://192.168.1.21:2380, https://192.168.1.21:2379, false

9e4bcb58402b8dba, started, master02, https://192.168.1.55:2380, https://192.168.1.55:2379, false