[ Kans 3 Study - 4w ] 실습 환경 구성

Kubernetes

[ Kans 3 Study - 4w ] 실습 환경 구성

su''@ 2024. 9. 22. 18:52

CloudNetaStudy - Kubernets Networtk 3기 실습 스터디 게시글입니다.

wsl / kind 설치

# Ubuntu 배포 관련 Vagrantfile, init_cfg.sh 파일 다운로드
curl -O https://raw.githubusercontent.com/gasida/KANS/main/kind/Vagrantfile
curl -O https://raw.githubusercontent.com/gasida/KANS/main/kind/init_cfg.sh

# vagrant 로 ubuntu 가상머신 1대 프로비저닝
vagrant up
... # 설치 스크립트 init_cfg.sh 실행되어 다소 시간 소요

# vagrant 상태 확인
vagrant status

# Ubuntu SSH 접속
vagrant ssh
-----------
# 기본 정보 확인
whoami
hostnamectl
htop
ip -c a

# 툴 설치 확인
docker info
kind --version
kubectl version --client=true
helm version

# 외부 통신 확인
ping -c 3 8.8.8.8
curl ipinfo.io

# 실습 완료 후 전원 off
poweroff
-----------

# vagrant 상태 확인
vagrant status

# 추후 다시 실습 시 Ubuntu (재)시작 후 Ubuntu SSH 접속
vagrant reload
vagrant ssh

실습 환경 : K8S v1.31.0 , CNI(Kindnet, Direct Routing mode)

IPTABLES proxy mode

- 노드 네트워크 대역(실제로는 container) : 172.18.0.0/16

- 파드 네트워크 대역 : 10.10.0.0/16
(각각 10.10.1.0/24, 10.10.2.0/24, 10.10.3.0/24, 10.10.4.0/24)

- 서비스 사용 네트워크 대역 : 10.200.1.0/24

#
cat <<EOT> kind-svc-1w.yaml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
featureGates:
  "InPlacePodVerticalScaling": true
  "MultiCIDRServiceAllocator": true
nodes:
- role: control-plane
  labels:
    mynode: control-plane
    topology.kubernetes.io/zone: ap-northeast-2a
  extraPortMappings:
  - containerPort: 30000
    hostPort: 30000
  - containerPort: 30001
    hostPort: 30001
  - containerPort: 30002
    hostPort: 30002
  kubeadmConfigPatches:
  - |
    kind: ClusterConfiguration
    apiServer:
      extraArgs:
        runtime-config: api/all=true
    controllerManager:
      extraArgs:
        bind-address: 0.0.0.0
    etcd:
      local:
        extraArgs:
          listen-metrics-urls: http://0.0.0.0:2381
    scheduler:
      extraArgs:
        bind-address: 0.0.0.0
  - |
    kind: KubeProxyConfiguration
    metricsBindAddress: 0.0.0.0
- role: worker
  labels:
    mynode: worker1
    topology.kubernetes.io/zone: ap-northeast-2a
- role: worker
  labels:
    mynode: worker2
    topology.kubernetes.io/zone: ap-northeast-2b
- role: worker
  labels:
    mynode: worker3
    topology.kubernetes.io/zone: ap-northeast-2c
networking:
  podSubnet: 10.10.0.0/16
  serviceSubnet: 10.200.1.0/24
EOT

# k8s 클러스터 설치
kind create cluster --config kind-svc-1w.yaml --name myk8s --image kindest/node:v1.31.0
docker ps

# 노드에 기본 툴 설치
docker exec -it myk8s-control-plane sh -c 'apt update && apt install tree psmisc lsof wget bsdmainutils bridge-utils net-tools ipset ipvsadm nfacct tcpdump ngrep iputils-ping arping git vim arp-scan -y'
for i in worker worker2 worker3; do echo ">> node myk8s-$i <<"; docker exec -it myk8s-$i sh -c 'apt update && apt install tree psmisc lsof wget bsdmainutils bridge-utils net-tools ipset ipvsadm nfacct tcpdump ngrep iputils-ping arping -y'; echo; done


# k8s v1.31.0 버전 확인
kubectl get node

# 노드 labels 확인
kubectl get nodes -o jsonpath="{.items[*].metadata.labels}" | grep mynode
kubectl get nodes -o jsonpath="{.items[*].metadata.labels}" | jq | grep mynode
kubectl get nodes -o jsonpath="{.items[*].metadata.labels}" | jq | grep 'topology.kubernetes.io/zone'

# kind network 중 컨테이너(노드) IP(대역) 확인 : 172.18.0.2~ 부터 할당되며, control-plane 이 꼭 172.18.0.2가 안될 수 도 있음
docker ps -q | xargs docker inspect --format '{{.Name}} {{.NetworkSettings.Networks.kind.IPAddress}}'
/myk8s-control-plane 172.18.0.4
/myk8s-worker 172.18.0.3
/myk8s-worker2 172.18.0.5
/myk8s-worker3 172.18.0.2

# 파드CIDR 과 Service 대역 확인 : CNI는 kindnet 사용
kubectl get cm -n kube-system kubeadm-config -oyaml | grep -i subnet
      podSubnet: 10.10.0.0/16
      serviceSubnet: 10.200.1.0/24
kubectl cluster-info dump | grep -m 2 -E "cluster-cidr|service-cluster-ip-range"

# feature-gates 확인 : https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
kubectl describe pod -n kube-system | grep feature-gates
      --feature-gates=InPlacePodVerticalScaling=true
kubectl describe pod -n kube-system | grep runtime-config
      --runtime-config=api/all=true

# MultiCIDRServiceAllocator : https://kubernetes.io/docs/tasks/network/extend-service-ip-ranges/
kubectl get servicecidr
NAME         CIDRS           AGE
kubernetes   10.200.1.0/24   2m13s


# 노드마다 할당된 dedicated subnet (podCIDR) 확인
kubectl get nodes -o jsonpath="{.items[*].spec.podCIDR}"
10.10.0.0/24 10.10.4.0/24 10.10.3.0/24 10.10.1.0/24

# kube-proxy configmap 확인
kubectl describe cm -n kube-system kube-proxy
...
mode: iptables
iptables:
  localhostNodePorts: null
  masqueradeAll: false
  masqueradeBit: null
  minSyncPeriod: 1s
  syncPeriod: 0s
...


# 노드 별 네트워트 정보 확인 : CNI는 kindnet 사용
for i in control-plane worker worker2 worker3; do echo ">> node myk8s-$i <<"; docker exec -it myk8s-$i ls /opt/cni/bin/; echo; done
for i in control-plane worker worker2 worker3; do echo ">> node myk8s-$i <<"; docker exec -it myk8s-$i cat /etc/cni/net.d/10-kindnet.conflist; echo; done
for i in control-plane worker worker2 worker3; do echo ">> node myk8s-$i <<"; docker exec -it myk8s-$i ip -c route; echo; done
for i in control-plane worker worker2 worker3; do echo ">> node myk8s-$i <<"; docker exec -it myk8s-$i ip -c addr; echo; done
for i in control-plane worker worker2 worker3; do echo ">> node myk8s-$i <<"; docker exec -it myk8s-$i ip -c -4 addr show dev eth0; echo; done

# iptables 정보 확인
for i in filter nat mangle raw ; do echo ">> IPTables Type : $i <<"; docker exec -it myk8s-control-plane  iptables -t $i -S ; echo; done
for i in filter nat mangle raw ; do echo ">> IPTables Type : $i <<"; docker exec -it myk8s-worker  iptables -t $i -S ; echo; done
for i in filter nat mangle raw ; do echo ">> IPTables Type : $i <<"; docker exec -it myk8s-worker2 iptables -t $i -S ; echo; done
for i in filter nat mangle raw ; do echo ">> IPTables Type : $i <<"; docker exec -it myk8s-worker3 iptables -t $i -S ; echo; done

# 각 노드 bash 접속
docker exec -it myk8s-control-plane bash
docker exec -it myk8s-worker bash
docker exec -it myk8s-worker2 bash
docker exec -it myk8s-worker3 bash
----------------------------------------
exit
----------------------------------------

# kind 설치 시 kind 이름의 도커 브리지가 생성된다 : 172.18.0.0/16 대역
docker network ls
docker inspect kind

# arp scan 해두기
docker exec -it myk8s-control-plane arp-scan --interfac=eth0 --localnet

# mypc 컨테이너 기동 : kind 도커 브리지를 사용하고, 컨테이너 IP를 직접 지정
docker run -d --rm --name mypc --network kind --ip 172.18.0.100 nicolaka/netshoot sleep infinity
docker ps
## 만약 kind 네트워크 대역이 다를 경우 위 IP 지정이 실패할 수 있으니, 그냥 IP 지정 없이 mypc 컨테이너 기동 할 것
## docker run -d --rm --name mypc --network kind nicolaka/netshoot sleep infinity

# 통신 확인
docker exec -it mypc ping -c 1 172.18.0.1
for i in {1..5} ; do docker exec -it mypc ping -c 1 172.18.0.$i; done
docker exec -it mypc zsh
-------------
ifconfig
ping -c 1 172.18.0.2
exit
-------------

# kube-ops-view 설치
helm repo add geek-cookbook https://geek-cookbook.github.io/charts/
helm install kube-ops-view geek-cookbook/kube-ops-view --version 1.2.2 --set service.main.type=NodePort,service.main.ports.http.nodePort=30000 --set env.TZ="Asia/Seoul" --namespace kube-system

# myk8s-control-plane 배치
kubectl -n kube-system edit deploy kube-ops-view
---
spec:
  ...
  template:
    ...
    spec:
      nodeSelector:
        mynode: control-plane
      tolerations:
      - key: "node-role.kubernetes.io/control-plane"
        operator: "Equal"
        effect: "NoSchedule"
---

# 설치 확인
kubectl -n kube-system get pod -o wide -l app.kubernetes.io/instance=kube-ops-view

# kube-ops-view 접속 URL 확인 (1.5 , 2 배율) : macOS 사용자
echo -e "KUBE-OPS-VIEW URL = http://localhost:30000/#scale=1.5"
echo -e "KUBE-OPS-VIEW URL = http://localhost:30000/#scale=2"

# kube-ops-view 접속 URL 확인 (1.5 , 2 배율) : Windows 사용자
echo -e "KUBE-OPS-VIEW URL = http://192.168.50.10:30000/#scale=1.5"
echo -e "KUBE-OPS-VIEW URL = http://192.168.50.10:30000/#scale=2"

(옵션) 프로메테우스 스택 설치 : 그라파나만 외부 접속 가능하게 NodePort 30002 사용

#
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts

# 파라미터 파일 생성
cat <<EOT > monitor-values.yaml
prometheus:
  prometheusSpec:
    podMonitorSelectorNilUsesHelmValues: false
    serviceMonitorSelectorNilUsesHelmValues: false
    nodeSelector:
      mynode: control-plane
    tolerations:
    - key: "node-role.kubernetes.io/control-plane"
      operator: "Equal"
      effect: "NoSchedule"


grafana:
  defaultDashboardsTimezone: Asia/Seoul
  adminPassword: kans1234

  service:
    type: NodePort
    nodePort: 30002
  nodeSelector:
    mynode: control-plane
  tolerations:
  - key: "node-role.kubernetes.io/control-plane"
    operator: "Equal"
    effect: "NoSchedule"

defaultRules:
  create: false
alertmanager:
  enabled: false

EOT

# 배포
kubectl create ns monitoring
helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack --version 62.3.0 -f monitor-values.yaml --namespace monitoring

# 확인
helm list -n monitoring

# Grafana 접속 계정 : admin / kans1234 : macOS 사용자
echo -e "Grafana URL = http://localhost:30002"

# Grafana 접속 계정 : admin / kans1234 : Windows 사용자
echo -e "Grafana URL = http://192.168.50.10:30002"


# (참고) helm 삭제
helm uninstall -n monitoring kube-prometheus-stack

'Kubernetes' 카테고리의 다른 글

[ Kans 3 Study - 4w ] 2. Service - NodePort (1)	2024.09.29
[ Kans 3 Study - 4w ] 1. Service - ClusterIP (1)	2024.09.22
[ Kans 3 Study - 3w ] 4. Calico 네트워크 모드 (1)	2024.09.22
[ Kans 3 Study - 3w ] 3. Calico 기본 통신 이해 (1)	2024.09.21
[ Kans 3 Study - 3w ] 2. Calico Component (2)	2024.09.21

현재글[ Kans 3 Study - 4w ] 실습 환경 구성

su''@ diary

문제를 해결하고 배운 모든 것을 기록하는 공간

eksctl, Bastion, networt, Kubernetes, terraform, metallb, Security Group, kubeconfig, pod-identity-agent, kubernetes-dashboard, gateway api, kubectl, 클러스터 만들기, EKS, aws loadbalancer controller, istio, alb controller, service mesh, ipvs proxy 모드, network,

Today :
Yesterday :

su''@ diary