From 30312ee9a538cde23da499f554b355dced9997e0 Mon Sep 17 00:00:00 2001
From: lizw <1183409807@qq.com>
Date: Sat, 22 Oct 2022 22:55:35 +0800
Subject: [PATCH] 06k8s-install-centos.sh

---
 00base/06k8s-install-centos.sh | 175 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 158 insertions(+), 17 deletions(-)

diff --git a/00base/06k8s-install-centos.sh b/00base/06k8s-install-centos.sh
index 0370c67..a97b3d3 100644
--- a/00base/06k8s-install-centos.sh
+++ b/00base/06k8s-install-centos.sh
@@ -11,15 +11,14 @@ KUBE_VERSION="${KUBE_VERSION:-latest}"
 FLANNEL_VERSION="${FLANNEL_VERSION:-0.17.0}"
 CALICO_VERSION="${CALICO_VERSION:-3.22.1}"
 CILIUM_VERSION="${CILIUM_VERSION:-1.9.13}"
+HELM_VERSION="${HELM_VERSION:-3.10.1}"
+INGRESS_NGINX="${INGRESS_NGINX:-4.2.5}"
+RANCHER_VERSION="${RANCHER_VERSION:-2.6.8}"
 #METRICS_SERVER_VERSION="${METRICS_SERVER_VERSION:-0.6.1}"
-#INGRESS_NGINX="${INGRESS_NGINX:-1.1.2}"
-#TRAEFIK_VERSION="${TRAEFIK_VERSION:-2.6.1}"
 #KUBE_PROMETHEUS_VERSION="${KUBE_PROMETHEUS_VERSION:-0.10.0}"
 #ELASTICSEARCH_VERSION="${ELASTICSEARCH_VERSION:-8.1.0}"
 #ROOK_VERSION="${ROOK_VERSION:-1.8.7}"
 #LONGHORN_VERSION="${LONGHORN_VERSION:-1.2.4}"
-#KUBERNETES_DASHBOARD_VERSION="${KUBERNETES_DASHBOARD_VERSION:-2.5.1}"
-#KUBESPHERE_VERSION="${KUBESPHERE_VERSION:-3.2.1}"
 
 # 集群配置
 KUBE_DNSDOMAIN="${KUBE_DNSDOMAIN:-cluster.local}"
@@ -56,6 +55,9 @@ SUDO_USER="${SUDO_USER:-root}"
 # 节点设置
 HOSTNAME_PREFIX="${HOSTNAME_PREFIX:-k8s}"
 
+# nginx的端口配置
+NGINX_HTTP_PORT="${NGINX_HTTP_PORT:-80}"
+
 # 脚本设置
 TMP_DIR="$(rm -rf /tmp/kainstall* && mktemp -d -t kainstall.XXXXXXXXXX)"
 LOG_FILE="${TMP_DIR}/kainstall.log"
@@ -121,7 +123,7 @@ function utils::version_to_number() {
 }
 
 # 重试
-function utils::retry {
+function utils::retry() {
   local retries=$1
   shift
   local count=0
@@ -950,7 +952,7 @@ EOF
 # 安装 cri-o
 function script::install_cri-o() {
   local version="${1:-latest}"
-  version="${version##latest}"
+  version="${version#-latest}"
   os="CentOS_$(rpm --eval '%{centos_ver}')" && echo "${os}"
   # /etc/yum.repos.d/devel_kubic_libcontainers_stable.repo
   cat << EOF > /etc/yum.repos.d/devel_kubic_libcontainers_stable.repo
@@ -1024,7 +1026,6 @@ EOF
   systemctl enable crio
 }
 
-
 # 安装kube组件
 function script::install_kube() {
   local version="-${1:-latest}"
@@ -1110,6 +1111,25 @@ EOF
   systemctl restart haproxy
 }
 
+# 安装helm
+function script::install_helm() {
+  local version="${1:-3.10.1}"
+  version="${version#-3.10.1}"
+  local path="/tmp"
+  cd $path
+  # 下载软件(国内源)
+  wget https://mirrors.huaweicloud.com/helm/v$version/helm-v$version-linux-amd64.tar.gz
+  # 解压
+  tar -zxvf helm-v$version-linux-amd64.tar.gz
+  # 安装
+  sudo mv linux-amd64/helm /usr/local/bin/
+  # 清理
+  rm -rf helm-v$version-linux-amd64.tar.gz linux-amd64
+  # 验证
+  helm version
+  cd ~
+}
+
 # 检查用到的命令
 function check::command() {
   check::command_exists ssh openssh-clients
@@ -1200,6 +1220,7 @@ function check::preflight() {
 
 # 安装包
 function install::package() {
+  # 检查k8s最新稳定版本
   if [[ "${KUBE_CRI}" == "cri-o" && "${KUBE_CRI_VERSION}" == "latest" ]]; then
     KUBE_CRI_VERSION="${KUBE_VERSION}"
     if [[ "${KUBE_CRI_VERSION}" == "latest" ]]; then
@@ -1212,6 +1233,7 @@ function install::package() {
     fi
     KUBE_CRI_VERSION="${KUBE_CRI_VERSION%.*}"
   fi
+  # 安装 cri kube
   for host in $MASTER_NODES $WORKER_NODES
   do
     # install cri
@@ -1231,26 +1253,29 @@ function install::package() {
     "
     check::exit_code "$?" "install" "install kube on $host"
   done
+  # 配置 kube
   local apiservers=$MASTER_NODES
   if [[ "$apiservers" == "127.0.0.1" ]]; then
     command::exec "${MGMT_NODE}" "ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p'"
     get::command_output "apiservers" "$?"
   fi
+  # 输出 api-servers 信息 
   if [[ "${ADD_TAG:-}" == "1" ]]; then
     command::exec "${MGMT_NODE}" "
       kubectl get node --selector='node-role.kubernetes.io/master' -o jsonpath='{$.items[*].status.addresses[?(@.type==\"InternalIP\")].address}'
     "
     get::command_output "apiservers" "$?"
   fi
+  # 安装 haproxy
   for host in $WORKER_NODES
   do
     # install haproxy
     log::info "[install]" "install haproxy on $host"
-  command::exec "${host}" "
+    command::exec "${host}" "
       export OFFLINE_TAG=${OFFLINE_TAG:-0}
       $(declare -f script::install_haproxy)
       script::install_haproxy \"$apiservers\"
-  "
+    "
     check::exit_code "$?" "install" "install haproxy on $host"
   done
   # 10年证书
@@ -1282,10 +1307,21 @@ function install::package() {
           echo \"not found /tmp/kubeadm-linux-amd64\"
           exit 1
         fi
-    "
+      "
       check::exit_code "$?" "install" "$host: use kubeadm 10 years certs client"
     done
   fi
+  # 安装helm
+  for host in $MASTER_NODES $WORKER_NODES
+  do
+    log::info "[install]" "install helm on $host"
+    command::exec "${host}" "
+      export OFFLINE_TAG=${OFFLINE_TAG:-0}
+      $(declare -f script::install_helm)
+      script::install_helm $HELM_VERSION
+    "
+    check::exit_code "$?" "install" "install helm on $host"
+  done
 }
 
 # 升级节点内核
@@ -1917,6 +1953,7 @@ function get::ingress_conn() {
 ######################################################################################################
 # 主调用逻辑
 ######################################################################################################
+
 # 添加network组件
 function add::network() {
   if [[ "$KUBE_NETWORK" == "flannel" ]]; then
@@ -2011,8 +2048,74 @@ spec:
 
 # 添加ingress组件
 function add::ingress() {
-  # TODO add ingress
-  log::warning "[TODO]" "add ingress"
+  # 安装 ingress-nginx
+  log::info "[ingress]" "add ingress-nginx"
+  command::exec "${MGMT_NODE}" "
+    helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+    helm install ingress-nginx ingress-nginx/ingress-nginx \
+     --namespace ingress-nginx --create-namespace \
+     --version ${INGRESS_NGINX} \
+     --set controller.admissionWebhooks.patch.image.registry=registry.hub.docker.com \
+     --set controller.admissionWebhooks.patch.image.image=k8sgcrioingressnginx/kube-webhook-certgen \
+     --set controller.admissionWebhooks.patch.image.digest= \
+     --set controller.admissionWebhooks.enabled=true \
+     --set controller.kind=DaemonSet \
+     --set controller.replicaCount=1 \
+     --set controller.minAvailable=1 \
+     --set controller.image.registry=registry.hub.docker.com \
+     --set controller.image.image=k8sgcrioingressnginx/controller \
+     --set controller.image.digest= \
+     --set controller.ingressClassResource.name=nginx \
+     --set controller.ingressClassResource.enable=true \
+     --set controller.ingressClassResource.default=false \
+     --set controller.service.enabled=true \
+     --set controller.service.type=NodePort \
+     --set controller.service.enableHttp=true \
+     --set controller.service.enableHttps=true \
+     --set controller.service.nodePorts.http=30080 \
+     --set controller.service.nodePorts.https=30443 \
+     --set defaultBackend.image.registry=registry.hub.docker.com \
+     --set defaultBackend.image.image=gcmirrors/defaultbackend-amd64 \
+     --set defaultBackend.enabled=true \
+     --set defaultBackend.name=defaultbackend \
+     --set defaultBackend.replicaCount=1 \
+     --set defaultBackend.minAvailable=1 \
+     --set rbac.create=true \
+     --set serviceAccount.create=true \
+     --set podSecurityPolicy.enabled=true
+     kubectl get pod -n ingress-nginx -o wide
+     kubectl get svc -n ingress-nginx -o wide
+  "
+  # 安装 nginx
+  log::info "[nginx]" "add nginx"
+  command::exec "${MGMT_NODE}" "
+    sudo yum -y install nginx
+    nginx -v
+    sudo systemctl enable nginx
+    sudo service nginx start
+    cat << EOF > /etc/nginx/conf.d/k8s.ingress.conf
+upstream k8s-ingress {
+    $(for h in MASTER_NODES $WORKER_NODES;do echo "server $h:30080 max_fails=1 fail_timeout=15s;";done)
+    keepalive 128;
+}
+
+server {
+    listen       ${NGINX_HTTP_PORT};
+
+    location / {
+        proxy_http_version 1.1;
+        proxy_set_header Connection \"\";
+        proxy_next_upstream error;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        proxy_set_header Host \$http_host;
+        proxy_set_header X-Nginx-Proxy true;
+        proxy_pass http://k8s-ingress/;
+    }
+}
+EOF
+    sudo nginx -s reload
+  "
 }
 
 # 添加addon组件
@@ -2041,8 +2144,45 @@ function add::storage() {
 
 # 添加用户界面
 function add::ui() {
-  # TODO add ui
-  log::warning "[TODO]" "add ui"
+  local path="/tmp"
+  # 安装 rancher
+  log::info "[rancher]" "add rancher"
+  command::exec "${MGMT_NODE}" "
+    helm repo add rancher-stable http://rancher-mirror.oss-cn-beijing.aliyuncs.com/server-charts/stable
+    cd ${path}
+    helm pull rancher-stable/rancher --version ${RANCHER_VERSION} --untar
+    cat << EOF > rancher/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template \"rancher.fullname\" . }}
+  labels:
+{{ include \"rancher.labels\" . | indent 4 }}
+spec:
+  ports:
+  - port: 80
+    targetPort: 80
+    protocol: TCP
+    name: http
+    # 使用nodePort端口
+    nodePort: 31080
+  - port: 443
+    targetPort: 444
+    protocol: TCP
+    name: https-internal
+    # 使用nodePort端口
+    nodePort: 31443
+  selector:
+    app: {{ template \"rancher.fullname\" . }}
+EOF
+    helm install rancher ./rancher \
+     --namespace cattle-system --create-namespace \
+     --set replicas=1 \
+     --set ingress.tls.source=secret \
+     --set ingress.enabled=false
+  "
+  log::info "[rancher]" "获取初始密码 kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{.data.bootstrapPassword|base64decode}}{{ \"\n\" }}'"
+  log::info "[rancher]" "重置初始密码 kubectl -n cattle-system exec \$(kubectl -n cattle-system get pods -l app=rancher | grep '1/1' | head -1 | awk '{ print \$1 }') -- reset-password"
 }
 
 # 运维操作
@@ -2464,7 +2604,7 @@ function upgrade::cluster() {
 }
 
 # 脚本文件更新
-function update::self {
+function update::self() {
   log::info "[update]" "download kainstall script to $0"
   command::exec "127.0.0.1" "
     wget --timeout=10 --waitretry=3 --tries=5 --retry-connrefused https://cdn.jsdelivr.net/gh/lework/kainstall@master/kainstall-centos.sh -O /tmp/kainstall-centos.sh || exit 1
@@ -2475,7 +2615,7 @@ function update::self {
 }
 
 # 数据处理及限制
-function transform::data {
+function transform::data() {
   MASTER_NODES=$(echo "${MASTER_NODES}" | tr ',' ' ')
   WORKER_NODES=$(echo "${WORKER_NODES}" | tr ',' ' ')
   if ! utils::is_element_in_array "$KUBE_CRI" docker containerd cri-o ; then
@@ -2494,7 +2634,7 @@ function transform::data {
 }
 
 # 使用帮助
-function help::usage {
+function help::usage() {
   cat << EOF
 Install kubernetes cluster using kubeadm.
 
@@ -2582,6 +2722,7 @@ EOF
 ######################################################################################################
 # main
 ######################################################################################################
+
 [ "$#" == "0" ] && help::usage
 while [ "${1:-}" != "" ]; do
   case $1 in