[kubernetes] HA 메모

https://kubernetes.io/docs/setup/independent/high-availability/

<master1>

# vi kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta1

kind: ClusterConfiguration

kubernetesVersion: stable

apiServer:

  certSANs:

  - "LOAD_BALANCER_DNS"

controlPlaneEndpoint: "LOAD_BALANCER_DNS:LOAD_BALANCER_PORT"

# kubeadm init --config=kubeadm-config.yaml

<아래 내용 복사>

 kubeadm join 192.168.0.200:6443 --token j04n3m.octy8zely83cy2ts --discovery-token-ca-cert-hash    sha256:84938d2a22203a8e56a787ec0c6ddad7bc7dbd52ebabc62fd5f4dbea72b14d1f

kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"

kubectl get pod -n kube-system -w

vi k8s1.sh

USER=centos # customizable

CONTROL_PLANE_IPS="10.0.0.7 10.0.0.8"

for host in ${CONTROL_PLANE_IPS}; do

    scp /etc/kubernetes/pki/ca.crt "${USER}"@$host:

    scp /etc/kubernetes/pki/ca.key "${USER}"@$host:

    scp /etc/kubernetes/pki/sa.key "${USER}"@$host:

    scp /etc/kubernetes/pki/sa.pub "${USER}"@$host:

    scp /etc/kubernetes/pki/front-proxy-ca.crt "${USER}"@$host:

    scp /etc/kubernetes/pki/front-proxy-ca.key "${USER}"@$host:

    scp /etc/kubernetes/pki/etcd/ca.crt "${USER}"@$host:etcd-ca.crt

    scp /etc/kubernetes/pki/etcd/ca.key "${USER}"@$host:etcd-ca.key

    scp /etc/kubernetes/admin.conf "${USER}"@$host:

done

<master2>

vi k8s2.sh

USER=centos

mkdir -p /etc/kubernetes/pki/etcd

mv /home/${USER}/ca.crt /etc/kubernetes/pki/

mv /home/${USER}/ca.key /etc/kubernetes/pki/

mv /home/${USER}/sa.pub /etc/kubernetes/pki/

mv /home/${USER}/sa.key /etc/kubernetes/pki/

mv /home/${USER}/front-proxy-ca.crt /etc/kubernetes/pki/

mv /home/${USER}/front-proxy-ca.key /etc/kubernetes/pki/

mv /home/${USER}/etcd-ca.crt /etc/kubernetes/pki/etcd/ca.crt

mv /home/${USER}/etcd-ca.key /etc/kubernetes/pki/etcd/ca.key

mv /home/${USER}/admin.conf /etc/kubernetes/admin.conf

kubeadm join 192.168.0.200:6443 --token j04n3m.octy8zely83cy2ts --discovery-token-ca-cert-hash sha256:84938d2a22203a8e56a787ec0c6ddad7bc7dbd52ebabc62fd5f4dbea72b14d1f --experimental-control-plane

kubectl get pod -n kube-system -w

---------------------------------------------

https://kubernetes.io/docs/setup/independent/setup-ha-etcd-with-kubeadm/

<master1>

cat << EOF > /etc/systemd/system/kubelet.service.d/20-etcd-service-manager.conf

[Service]

ExecStart=

ExecStart=/usr/bin/kubelet --address=127.0.0.1 --pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true

Restart=always

EOF

systemctl daemon-reload

systemctl restart kubelet

vi k8s3.sh

----

# Update HOST0, HOST1, and HOST2 with the IPs or resolvable names of your hosts

export HOST0=10.0.0.6

export HOST1=10.0.0.7

export HOST2=10.0.0.8

# Create temp directories to store files that will end up on other hosts.

mkdir -p /tmp/${HOST0}/ /tmp/${HOST1}/ /tmp/${HOST2}/

ETCDHOSTS=(${HOST0} ${HOST1} ${HOST2})

NAMES=("infra0" "infra1" "infra2")

for i in "${!ETCDHOSTS[@]}"; do

HOST=${ETCDHOSTS[$i]}

NAME=${NAMES[$i]}

cat << EOF > /tmp/${HOST}/kubeadmcfg.yaml

apiVersion: "kubeadm.k8s.io/v1beta1"

kind: ClusterConfiguration

etcd:

    local:

        serverCertSANs:

        - "${HOST}"

        peerCertSANs:

        - "${HOST}"

        extraArgs:

            initial-cluster: infra0=https://${ETCDHOSTS[0]}:2380,infra1=https://${ETCDHOSTS[1]}:2380,infra2=https://${ETCDHOSTS[2]}:2380

            initial-cluster-state: new

            name: ${NAME}

            listen-peer-urls: https://${HOST}:2380

            listen-client-urls: https://${HOST}:2379

            advertise-client-urls: https://${HOST}:2379

            initial-advertise-peer-urls: https://${HOST}:2380

EOF

done

----

kubeadm init phase certs etcd-ca

----

vi k8s4.sh

kubeadm init phase certs etcd-server --config=/tmp/${HOST2}/kubeadmcfg.yaml

kubeadm init phase certs etcd-peer --config=/tmp/${HOST2}/kubeadmcfg.yaml

kubeadm init phase certs etcd-healthcheck-client --config=/tmp/${HOST2}/kubeadmcfg.yaml

kubeadm init phase certs apiserver-etcd-client --config=/tmp/${HOST2}/kubeadmcfg.yaml

cp -R /etc/kubernetes/pki /tmp/${HOST2}/

# cleanup non-reusable certificates

find /etc/kubernetes/pki -not -name ca.crt -not -name ca.key -type f -delete

kubeadm init phase certs etcd-server --config=/tmp/${HOST1}/kubeadmcfg.yaml

kubeadm init phase certs etcd-peer --config=/tmp/${HOST1}/kubeadmcfg.yaml

kubeadm init phase certs etcd-healthcheck-client --config=/tmp/${HOST1}/kubeadmcfg.yaml

kubeadm init phase certs apiserver-etcd-client --config=/tmp/${HOST1}/kubeadmcfg.yaml

cp -R /etc/kubernetes/pki /tmp/${HOST1}/

find /etc/kubernetes/pki -not -name ca.crt -not -name ca.key -type f -delete

kubeadm init phase certs etcd-server --config=/tmp/${HOST0}/kubeadmcfg.yaml

kubeadm init phase certs etcd-peer --config=/tmp/${HOST0}/kubeadmcfg.yaml

kubeadm init phase certs etcd-healthcheck-client --config=/tmp/${HOST0}/kubeadmcfg.yaml

kubeadm init phase certs apiserver-etcd-client --config=/tmp/${HOST0}/kubeadmcfg.yaml

# No need to move the certs because they are for HOST0

# clean up certs that should not be copied off this host

find /tmp/${HOST2} -name ca.key -type f -delete

find /tmp/${HOST1} -name ca.key -type f -delete

----

 USER=ubuntu

 HOST=${HOST1}

 scp -r /tmp/${HOST}/* ${USER}@${HOST}:

 ssh ${USER}@${HOST}

 USER@HOST $ sudo -Es

 root@HOST $ chown -R root:root pki

 root@HOST $ mv pki /etc/kubernetes/

 ----

 

vi k8s5.sh

export CONTROL_PLANE="ubuntu@10.0.0.7"

+scp /etc/kubernetes/pki/etcd/ca.crt "${CONTROL_PLANE}":

+scp /etc/kubernetes/pki/apiserver-etcd-client.crt "${CONTROL_PLANE}":

+scp /etc/kubernetes/pki/apiserver-etcd-client.key "${CONTROL_PLANE}":

----

vi kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta1

kind: ClusterConfiguration

kubernetesVersion: stable

apiServer:

  certSANs:

  - "LOAD_BALANCER_DNS"

controlPlaneEndpoint: "LOAD_BALANCER_DNS:LOAD_BALANCER_PORT"

etcd:

    external:

        endpoints:

        - https://ETCD_0_IP:2379

        - https://ETCD_1_IP:2379

        - https://ETCD_2_IP:2379

        caFile: /etc/kubernetes/pki/etcd/ca.crt

        certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt

        keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key

----

kubeadm init --config kubeadm-config.yaml