离线安装高可用K3s--Server集群（离线安装libaio.so.1）

源码构建

将客户端的证书由1年到10年并生成需要镜像

--构建环境

root@op:~# apt install curl make make-guile docker.io

root@op:~# docker version

Client:
 Version:           19.03.6
 API version:       1.40
 Go version:        go1.12.17
 Git commit:        369ce74a3c
 Built:             Fri Feb 28 23:45:43 2020
 OS/Arch:           linux/amd64
 Experimental:      false
 
Server:
 Engine:
  Version:          19.03.6
  API version:      1.40 (minimum version 1.12)
  Go version:       go1.12.17
  Git commit:       369ce74a3c
  Built:            Wed Feb 19 01:06:16 2020
  OS/Arch:          linux/amd64
  Experimental:     false
 containerd:
  Version:          1.3.3-0ubuntu1~18.04.2
  GitCommit:
 runc:
  Version:          spec: 1.0.1-dev
  GitCommit:
 docker-init:
  Version:          0.18.0
  GitCommit:       
--源码

root@op:/opt# wget https://github.com/rancher/k3s/archive/v1.18.8+k3s1.tar.gz

root@op:/opt# tar zxvf v1.18.8+k3s1.tar.gz

---修改证书的时间

root@op:/opt/k3s-1.18.8-k3s1/vendor/github.com/rancher/dynamiclistener/cert# vim cert.go

# vim cert.go

  NotAfter:     time.Now().Add(duration365d).UTC(), 
  改为
  NotAfter:     time.Now().Add(duration365d * 10).UTC(), 

root@op:/opt/k3s-1.18.8-k3s1# git init

root@op:/opt/k3s-1.18.8-k3s1# git add .

root@op:/opt/k3s-1.18.8-k3s1# git config --global user.name root

root@op:/opt/k3s-1.18.8-k3s1# git config --global user.email root@yesnocom.com

root@op:/opt/k3s-1.18.8-k3s1# git commit -m "init"

---构建完整版本的二进制文件及需要的镜像

root@op:/opt/k3s-1.18.8-k3s1# SKIP_VALIDATE=true make

root@op:/opt/k3s-1.18.8-k3s1/dist/artifacts# ls -la

total 397292
drwxr-xr-x 2 root root      4096 Sep  7 04:42 .
drwxr-xr-x 3 root root      4096 Sep  7 04:41 ..
-rwxr-xr-x 1 root root  53448704 Sep  7 04:41 k3s
-rw------- 1 root root 352955392 Sep  7 04:42 k3s-airgap-images-amd64.tar
-rw-r--r-- 1 root root       272 Sep  7 04:42 k3s-images.txt

root@op:/opt/k3s-1.18.8-k3s1/dist/artifacts# ./k3s -v

k3s version v1.18.8+k3s-c8d17880 (c8d17880)

在3个server(u1/u2/u3)节点上：

mkdir -p /data/rancher/logs_k3s/pods

mkdir -p /data/rancher/logs_k3s/containers

mkdir -p /data/rancher/kubelet_k3s/kubelet

mkdir -p /data/rancher/data_k3s/rancher/k3s/agent/images

ln -s /data/rancher/data_k3s/rancher /var/lib/

ln -s /data/rancher/kubelet_k3s/kubelet /var/lib/

ln -s /data/rancher/logs_k3s/pods /var/log/

ln -s /data/rancher/logs_k3s/containers /var/log/

将构建好的二进制软件（k3s)分别分发到3个server节点上（u1/u2/u3）的/usr/local/bin目录中

root@u1:~# ls -l /usr/local/bin/k3s

-rwxr-xr-x 1 root root 53448704 Sep  7 08:41 /usr/local/bin/k3s

root@u2:~# ls -l /usr/local/bin/k3s

-rwxr-xr-x 1 root root 53448704 Sep  7 08:41 /usr/local/bin/k3s

root@u3:~# ls -l /usr/local/bin/k3s

-rwxr-xr-x 1 root root 53448704 Sep  7 08:41 /usr/local/bin/k3s

将构建好的需要的镜像包（k3s-airgap-images-amd64.tar）分别分发到3个server节点上（u1/u2/u3）的/var/lib/rancher/k3s/agent/images目录中

root@u1:~# ls -l /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

-rw------- 1 root root 352955392 Sep  7 08:42 /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

root@u2:~# ls -l /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

-rw------- 1 root root 352955392 Sep  7 08:42 /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

root@u3:~# ls -l /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

-rw------- 1 root root 352955392 Sep  7 08:42 /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

部署server节点上初始集群（u1/u2/u3）

root@u1:~# curl -sfL https://get.k3s.io | \
> INSTALL_K3S_SKIP_DOWNLOAD=true \
> INSTALL_K3S_EXEC=" \
> server \
> --write-kubeconfig-mode 644 \
> --datastore-endpoint 'https://g1.yesnocom.com:2379,https://g2.yesnocom.com,https://g3.yesnocom.com' \
> --datastore-cafile '/srv/etcd/pki/ca.crt' \
> --datastore-certfile '/srv/etcd/pki/client.crt' \
> --datastore-keyfile '/srv/etcd/pki/client.key' \
> -t agent-secret \
> --tls-san vip.yesnocom.com" \
> sh -
[INFO]  Skipping k3s download and verify
[INFO]  Creating /usr/local/bin/kubectl symlink to k3s
[INFO]  Creating /usr/local/bin/crictl symlink to k3s
[INFO]  Creating /usr/local/bin/ctr symlink to k3s
[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]  Creating uninstall script /usr/local/bin/k3s-uninstall.sh
[INFO]  env: Creating environment file /etc/systemd/system/k3s.service.env
[INFO]  systemd: Creating service file /etc/systemd/system/k3s.service
[INFO]  systemd: Enabling k3s unit
Created symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service.
[INFO]  systemd: Starting k3s

root@u2:~# curl -sfL https://get.k3s.io | \
> INSTALL_K3S_SKIP_DOWNLOAD=true \
> INSTALL_K3S_EXEC=" \
> server \
> --write-kubeconfig-mode 644 \
> --datastore-endpoint 'https://g1.yesnocom.com:2379,https://g2.yesnocom.com,https://g3.yesnocom.com' \
> --datastore-cafile '/srv/etcd/pki/ca.crt' \
> --datastore-certfile '/srv/etcd/pki/client.crt' \
> --datastore-keyfile '/srv/etcd/pki/client.key' \
> -t agent-secret \
> --tls-san vip.yesnocom.com" \
> sh -
[INFO]  Skipping k3s download and verify
[INFO]  Creating /usr/local/bin/kubectl symlink to k3s
[INFO]  Creating /usr/local/bin/crictl symlink to k3s
[INFO]  Creating /usr/local/bin/ctr symlink to k3s
[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]  Creating uninstall script /usr/local/bin/k3s-uninstall.sh
[INFO]  env: Creating environment file /etc/systemd/system/k3s.service.env
[INFO]  systemd: Creating service file /etc/systemd/system/k3s.service
[INFO]  systemd: Enabling k3s unit
Created symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service.
[INFO]  systemd: Starting k3s

root@u3:~# curl -sfL https://get.k3s.io | \
> INSTALL_K3S_SKIP_DOWNLOAD=true \
> INSTALL_K3S_EXEC=" \
> server \
> --write-kubeconfig-mode 644 \
> --datastore-endpoint 'https://g1.yesnocom.com:2379,https://g2.yesnocom.com,https://g3.yesnocom.com' \
> --datastore-cafile '/srv/etcd/pki/ca.crt' \
> --datastore-certfile '/srv/etcd/pki/client.crt' \
> --datastore-keyfile '/srv/etcd/pki/client.key' \
> -t agent-secret \
> --tls-san vip.yesnocom.com" \
> sh -
[INFO]  Skipping k3s download and verify
[INFO]  Creating /usr/local/bin/kubectl symlink to k3s
[INFO]  Creating /usr/local/bin/crictl symlink to k3s
[INFO]  Creating /usr/local/bin/ctr symlink to k3s
[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh
[INFO]  Creating uninstall script /usr/local/bin/k3s-uninstall.sh
[INFO]  env: Creating environment file /etc/systemd/system/k3s.service.env
[INFO]  systemd: Creating service file /etc/systemd/system/k3s.service
[INFO]  systemd: Enabling k3s unit
Created symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service.
[INFO]  systemd: Starting k3s

root@u1:~# kubectl get node -o wide

NAME   STATUS   ROLES    AGE     VERSION        INTERNAL-IP      EXTERNAL-IP   OS-IMAGE            KERNEL-VERSION       CONTAINER-RUNTIME
u1     Ready    master   2m31s   v1.18.8+k3s-   192.168.100.11   <none>        Ubuntu 18.04.5 LTS   4.15.0-115-generic   containerd://1.3.3-k3s2
u2     Ready    master   83s     v1.18.8+k3s-   192.168.100.12   <none>        Ubuntu 18.04.5 LTS   4.15.0-115-generic   containerd://1.3.3-k3s2
u3     Ready    master   52s     v1.18.8+k3s-   192.168.100.13   <none>        Ubuntu 18.04.5 LTS   4.15.0-115-generic   containerd://1.3.3-k3s2

root@u1:~# kubectl get pod -A

NAMESPACE     NAME                                     READY   STATUS      RESTARTS   AGE
kube-system   coredns-7944c66d8d-l7jtq                 1/1     Running     0          6m59s
kube-system   helm-install-traefik-hctws               0/1     Completed   0          6m59s
kube-system   local-path-provisioner-6d59f47c7-n9qjd   1/1     Running     0          6m59s
kube-system   metrics-server-7566d596c8-9wzsz          1/1     Running     0          6m59s
kube-system   svclb-traefik-47xws                      2/2     Running     0          5m43s
kube-system   svclb-traefik-sz8b8                      2/2     Running     0          6m40s
kube-system   svclb-traefik-wcn7s                      2/2     Running     0          5m13s
kube-system   traefik-758cd5fc85-f242c                 1/1     Running     0          6m40s

root@u1:~# kubectl get apiservices |grep 'metrics'

v1beta1.metrics.k8s.io                 kube-system/metrics-server   True        6m44s

root@u1:~# kubectl top node

NAME   CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%
u1     148m         7%     968Mi           49%
u2     94m          4%     726Mi           36%
u3     93m          4%     728Mi           36% 

HA部署（u1/u2/u3）

# apt install haproxy -y

# apt install keepalived -y

haproxy 配置（3个节点上【u1/u2/u3】配置文件相同）

# cat /etc/haproxy/haproxy.cfg

global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    stats socket /var/run/haproxy-admin.sock mode 660 level admin
    stats timeout 30s
    user  haproxy
    group haproxy
    daemon
    nbproc 1
 
defaults
    log     global
    timeout connect 5000
    timeout client  10m
    timeout server  10m
 
listen  admin_stats
    bind 0.0.0.0:10080
    mode http
    log 127.0.0.1 local0 err
    stats refresh 30s
    stats uri /status
    stats realm welcome login\ Haproxy
    stats auth admin:Jieshi11gR2.
    stats hide-version
    stats admin if TRUE
 
listen kube-master
    bind 0.0.0.0:8443
    mode tcp
    option tcplog
    balance source
    server 192.168.100.91 192.168.100.11:6443 check inter 2000 fall 2 rise 2 weight 1
    server 192.168.100.92 192.168.100.12:6443 check inter 2000 fall 2 rise 2 weight 1
    server 192.168.100.93 192.168.100.13:6443 check inter 2000 fall 2 rise 2 weight 1

# systemctl restart haproxy.service

# ss -tunpla|cat |grep 8443

tcp   LISTEN   0   128   0.0.0.0:8443    0.0.0.0:*  users:(("haproxy",pid=9399,fd=9))

keepalived 配置文件（采用一主多备）

u1节点上：

root@u1:~# cat /etc/keepalived/keepalived.conf

global_defs {
    router_id lb-master-105
}
 
vrrp_script check-haproxy {
    script "killall -0 haproxy"
    interval 5
    weight -30
}
 
vrrp_instance VI-kube-master {
    state MASTER
    priority 120
    dont_track_primary
    interface ens33
    virtual_router_id 68
    advert_int 3
    track_script {
        check-haproxy
    }
    virtual_ipaddress {
        192.168.100.88 dev ens33 label ens33:1
    }
}
 

root@u1:~# systemctl restart keepalived.service

root@u1:~# ifconfig |grep -A 3 ens33:1

ens33:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.100.88  netmask 255.255.255.255  broadcast 0.0.0.0
        ether 00:0c:29:61:7d:9a  txqueuelen 1000  (Ethernet)

备节点上（u2/u3 配置文件一样）

# cat /etc/keepalived/keepalived.conf

global_defs {
    router_id lb-backup-105
}
 
vrrp_script check-haproxy {
    script "killall -0 haproxy"
    interval 5
    weight -30
}
 
vrrp_instance VI-kube-master {
    state BACKUP
    priority 110
    dont_track_primary
    interface ens33
    virtual_router_id 68
    advert_int 3
    track_script {
        check-haproxy
    }
    virtual_ipaddress {
        192.168.100.88 dev ens33 label ens33:1
    }
} 
# systemctl restart keepalived.service

# ps -ef |grep keep

root       7883      1  0 09:48 ?        00:00:00 /usr/sbin/keepalived
root       7893   7883  0 09:48 ?        00:00:00 /usr/sbin/keepalived
root       7895   7883  0 09:48 ?        00:00:00 /usr/sbin/keepalived