阿里云ECS搭建高可用k8s集群及Terway、CCM、SLB配置

cat > /etc/sysctl.d/k8s.conf << EOF
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

# 使设置生效
sysctl --system
# 创建 modules 文件，保证在节点重启后能自动加载所需模块
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe br_netfilter
modprobe -- ip_vs
modprobe -- ip_vs_lc
modprobe -- ip_vs_wlc
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_lblc
modprobe -- ip_vs_lblcr
modprobe -- ip_vs_dh
modprobe -- ip_vs_sh
# 3.10 内核不支持
#modprobe -- ip_vs_fo
modprobe -- ip_vs_nq
modprobe -- ip_vs_sed
modprobe -- ip_vs_ftp
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
yum install ipset ipvsadm  -y
# 检查是否安装 成功 
lsmod | grep -e ip_vs -e nf_conntrack_ipv4

# haproxy的安装和配置都非常简单
yum install -y haproxy
# cat cat /etc/haproxy/haproxy.cfg
global
    log 127.0.0.1 local0
    log 127.0.0.1 local1 notice
 
    maxconn 4096
    chroot /var/lib/haproxy
    user haproxy
    group haproxy
    daemon
defaults
    log global
    mode tcp
    option tcplog
    option dontlognull
    retries 3
    option redispatch
    maxconn 2000
#    contimeout 5000
#    clitimeout 50000
#    srvtimeout 50000
    timeout connect 5000
    timeout client 50000
    timeout server 50000
listen zabbixServer
    bind 127.0.0.1:6443
    server k8s-01 192.168.1.27:5443 check inter 3000 fall 3 rise 3
    server k8s-02 192.168.1.28:5443 check inter 3000 fall 3 rise 3
    server k8s-03 192.168.1.29:5443 check inter 3000 fall 3 rise 3
  
  # 开机启动
  systemctl enable haproxy
  # 启动 haproxy
  systemctl start haproxy
  
  # 检查
  ss -anltp | grep haproxy

#!/bin/bash

. /etc/profile

function before_install() {
    getent group etcd >/dev/null || groupadd -r etcd
    getent passwd etcd >/dev/null || useradd -r -g etcd -d /var/lib/etcd -s /sbin/nologin -c "etcd user" etcd

    if [ ! -d "/var/lib/etcd" ]; then
        mkdir /var/lib/etcd
        chown -R etcd:etcd /var/lib/etcd
    fi
}

function get_cfssl_version(){
    local cfssl_version=$(curl -s https://api.github.com/repos/cloudflare/cfssl/releases/latest | grep tag_name | cut -d '"' -f 4| cut -b 2-)
    echo $cfssl_version
}

function install_cfssl(){
    local cfssl_version=$(get_cfssl_version)
    local cfssl_url="https://github.com/cloudflare/cfssl/releases/download/v${cfssl_version}/cfssl_${cfssl_version}_linux_amd64"
    local cfssl_bin="cfssl"
    local cfssl_bin_path="/usr/local/bin/${cfssl_bin}"

    local cfssljson_url="https://github.com/cloudflare/cfssl/releases/download/v${cfssl_version}/cfssljson_${cfssl_version}_linux_amd64"
    local cfssljson_bin="cfssljson"
    local cfssljson_bin_path="/usr/local/bin/${cfssljson_bin}"

    echo "cfssl_version: ${cfssl_version}"

    echo "Downloading cfssl..."
    rm -f ${cfssl_bin_path}
    wget -q -O ${cfssl_bin_path} ${cfssl_url}
    if [ $? -ne 0 ]; then
        echo "Failed to download cfssl."
        exit 1
    fi
    chmod +x ${cfssl_bin_path}
    echo "cfssl installed."
    
    echo "Downloading cfssljson..."
    rm -f ${cfssljson_bin_path}
    wget -q -O ${cfssljson_bin_path} ${cfssljson_url}
    if [ $? -ne 0 ]; then
        echo "Failed to download cfssljson."
        exit 1
    fi
    chmod +x ${cfssljson_bin_path}
    echo "cfssljson installed."
}

function get_etcd_version() {
    local etcd_version=$(curl -s https://api.github.com/repos/etcd-io/etcd/releases/latest | grep tag_name | cut -d '"' -f 4)
    echo $etcd_version
}

function install_etcd() {
    local etcd_version=$(get_etcd_version)
    local etcd_url="https://github.com/etcd-io/etcd/releases/download/${etcd_version}/etcd-${etcd_version}-linux-amd64.tar.gz"
    local etcd_file="etcd-${etcd_version}-linux-amd64.tar.gz"
    local etcd_dir="etcd-${etcd_version}-linux-amd64"

    echo "etcd_version: ${etcd_version}"

    cd /tmp
    echo "Downloading etcd..."
    wget -q -O ${etcd_file} ${etcd_url}
    if [ $? -ne 0 ]; then
        echo "download etcd failed"
        exit 1
    fi

    tar -xzf ${etcd_file}
    rm -f /usr/local/bin/etcd /usr/local/bin/etcdctl /usr/local/bin/etcdutl
    mv ${etcd_dir}/etcd /usr/local/bin/
    mv ${etcd_dir}/etcdctl /usr/local/bin/
    mv ${etcd_dir}/etcdutl /usr/local/bin/
    chmod +x /usr/local/bin/etcd /usr/local/bin/etcdctl /usr/local/bin/etcdutl
    echo "etcd installed successfully"

    echo "clean up..."
    rm -f ${etcd_file}
    rm -rf ${etcd_dir}

    cd -
}

function etcd_systemd() {
    cat > /etc/systemd/system/etcd.service << EOF
[Unit]
Description=etcd - highly-available key value store
Documentation=https://github.com/coreos/etcd
After=network.target
Wants=network-online.target

[Service]
#Type=notify
Type=simple
User=etcd
ExecStart=/usr/local/bin/etcd  --config-file /etc/etcd/etcd.yaml
Restart=on-failure
RestartSec=10s

LimitNOFILE=65536
#MemoryLow=200M
Nice=-10
IOSchedulingClass=best-effort
IOSchedulingPriority=2

[Install]
WantedBy=multi-user.target
EOF
}

function after_install() {
    mkdir -p /data/etcd/{data,wal}
    chown -R etcd:etcd /data/etcd

    mkdir -p /etc/etcd/{cfssl, ssl}
    chown -R etcd:etcd /etc/etcd

    echo "Installing systemd service..."
    etcd_systemd
    systemctl enable etcd
    
    grep 'ETCDCTL_API=3' /etc/profile > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        echo "export ETCDCTL_API=3" >> /etc/profile
        . /etc/profile
    fi
    
    grep '$PATH:/usr/local/bin' /etc/profile > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        echo 'export PATH=$PATH:/usr/local/bin' >> /etc/profile
    fi
    
    . /etc/profile
}

function main(){
    before_install

    install_etcd
    install_cfssl

    after_install
}

main

# hosts 中需要根据实际情况修改
# cat etcd-csr.json
{
    "key": {
        "algo": "rsa",
        "size": 2048 
    },
    "names": [
        {
            "O": "etcd",
            "OU": "etcd Security",
            "L": "Chengdu",
            "ST": "Sichuan",
            "C": "CN"
        }
    ],
    "CN": "etcd",
    "hosts": [
        "127.0.0.1",
        "localhost",
        "*.etcd.node",
        "*.kubernetes.node",
        "192.168.1.27",
        "192.168.1.28",
        "192.168.100.29"
    ]
}
# cat etcd-gencert.json
{
  "signing": {
    "default": {
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ],
        "expiry": "87600h"
    }
  }
}
# etcd-root-ca-csr.json
{
    "CN": "etcd-root-ca",
    "key": {
        "algo": "rsa",
        "size": 4096
    },
    "names": [
        {
            "O": "etcd",
            "OU": "etcd Security",
            "L": "Chengdu",
            "ST": "Sichuan",
            "C": "CN"
        }
    ],
    "ca": {
        "expiry": "87600h"
    }
}

# This is the configuration file for the etcd server.

# Human-readable name for this member.
# 不能重复
name: 'etcd-01'

# Path to the data directory.
data-dir: '/data/etcd/data'

# Path to the dedicated wal directory.
wal-dir: '/data/etcd/wal'

# Number of committed transactions to trigger a snapshot to disk.
#snapshot-count: 10000
snapshot-count: 1000

# Time (in milliseconds) of a heartbeat interval.
heartbeat-interval: 100

# Time (in milliseconds) for an election to timeout.
election-timeout: 1000

# Raise alarms when backend size exceeds the given quota. 0 means use the
# default quota.
#quota-backend-bytes: 0
# 5G
quota-backend-bytes: 5368709120

# List of comma separated URLs to listen on for peer traffic.
listen-peer-urls: https://192.168.1.27:2380

# List of comma separated URLs to listen on for client traffic.
listen-client-urls: https://192.168.1.27:2379,https://127.0.0.1:2379

# Maximum number of snapshot files to retain (0 is unlimited).
max-snapshots: 10

# Maximum number of wal files to retain (0 is unlimited).
max-wals: 10

# Comma-separated white list of origins for CORS (cross-origin resource sharing).
cors:

# List of this member's peer URLs to advertise to the rest of the cluster.
# The URLs needed to be a comma-separated list.
initial-advertise-peer-urls: https://192.168.1.27:2380

# List of this member's client URLs to advertise to the public.
# The URLs needed to be a comma-separated list.
advertise-client-urls: https://192.168.1.27:2379

# Discovery URL used to bootstrap the cluster.
discovery:

# Valid values include 'exit', 'proxy'
discovery-fallback: 'proxy'

# HTTP proxy to use for traffic to discovery service.
discovery-proxy:

# DNS domain used to bootstrap initial cluster.
discovery-srv:

# Initial cluster configuration for bootstrapping.
initial-cluster: etcd-01=https://192.168.1.27:2380,etcd-02=https://192.168.1.28:2380,etcd-03=https://192.168.1.29:2380

# Initial cluster token for the etcd cluster during bootstrap.
# 随机字符串，相同集群内，该配置需要一致
# 可使用shell命令生成： openssl rand -base64 15 
initial-cluster-token: 'krMWJC2ha5'

# Initial cluster state ('new' or 'existing').
initial-cluster-state: 'new'

# Reject reconfiguration requests that would cause quorum loss.
strict-reconfig-check: true

# Enable runtime profiling data via HTTP server
enable-pprof: true

# Valid values include 'on', 'readonly', 'off'
proxy: 'off'

# Time (in milliseconds) an endpoint will be held in a failed state.
proxy-failure-wait: 5000

# Time (in milliseconds) of the endpoints refresh interval.
proxy-refresh-interval: 30000

# Time (in milliseconds) for a dial to timeout.
proxy-dial-timeout: 1000

# Time (in milliseconds) for a write to timeout.
proxy-write-timeout: 5000

# Time (in milliseconds) for a read to timeout.
proxy-read-timeout: 0

client-transport-security:
  # Path to the client server TLS cert file.
  cert-file: /etc/etcd/ssl/etcd.pem

  # Path to the client server TLS key file.
  key-file: /etc/etcd/ssl/etcd-key.pem

  # Enable client cert authentication.
  client-cert-auth: true

  # Path to the client server TLS trusted CA cert file.
  trusted-ca-file: /etc/etcd/ssl/etcd-root-ca.pem

  # Client TLS using generated certificates
  auto-tls: true

peer-transport-security:
  # Path to the peer server TLS cert file.
  cert-file: /etc/etcd/ssl/etcd.pem

  # Path to the peer server TLS key file.
  key-file: /etc/etcd/ssl/etcd-key.pem

  # Enable peer client cert authentication.
  client-cert-auth: true

  # Path to the peer server TLS trusted CA cert file.
  trusted-ca-file: /etc/etcd/ssl/etcd-root-ca.pem

  # Peer TLS using generated certificates.
  auto-tls: true

# The validity period of the self-signed certificate, the unit is year.
self-signed-cert-validity: 1

# Enable debug-level logging for etcd.
log-level: debug

logger: zap

# Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.
log-outputs: [systemd/journal]

# Force to create a new one member cluster.
force-new-cluster: false

auto-compaction-mode: revision
auto-compaction-retention: "300"

# 阿里ECS 可以直接使用阿里云内部的源
# cat /etc/yum.repos.d/docker-ce.repo
[docker-ce-stable]
name=Docker CE Stable - $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/$basearch/stable
enabled=1
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-stable-debuginfo]
name=Docker CE Stable - Debuginfo $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/debug-$basearch/stable
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-stable-source]
name=Docker CE Stable - Sources
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/source/stable
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-test]
name=Docker CE Test - $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/$basearch/test
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-test-debuginfo]
name=Docker CE Test - Debuginfo $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/debug-$basearch/test
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-test-source]
name=Docker CE Test - Sources
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/source/test
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-nightly]
name=Docker CE Nightly - $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/$basearch/nightly
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-nightly-debuginfo]
name=Docker CE Nightly - Debuginfo $basearch
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/debug-$basearch/nightly
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

[docker-ce-nightly-source]
name=Docker CE Nightly - Sources
baseurl=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/$releasever/source/nightly
enabled=0
gpgcheck=1
gpgkey=http://mirrors.cloud.aliyuncs.com/docker-ce/linux/centos/gpg

# 更新yum 缓存
yum makecache fast
# 安装
yum -y install docker-ce-20.10.14-3.el7
# 配置
# registry-mirror 仅限公司内部使用
# cat /etc/docker/daemon.json 
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "registry-mirror": [
      "https://e88hruwq.mirror.aliyuncs.com" 
  ],
  "data-root": "/data/docker_data",
  "log-driver": "json-file",
  "log-level": "",
  "log-opts": {
      "max-size": "10m",
      "max-file": "3"
  }
}
# 开机启动
systemctl enable docker
systemctl restart docker

# cat /usr/lib/systemd/system/kubelet.service
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/
Wants=network-online.target
After=network-online.target

[Service]
#ExecStart=/usr/bin/kubelet
ExecStart=/usr/bin/kubelet --cloud-provider=external --hostname-override=xxx.xxx --provider-id=xxx.xxx
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target

apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
#nodeRegistration:
#    criSocket: /run/containerd/containerd.sock
#    name: containerd
localAPIEndpoint:
  # 第一个 master 节点 IP
  advertiseAddress: "192.168.1.27"
  # 6443 留给了 nginx，apiserver 换到 5443
  bindPort: 5443
# 这个 token 使用以下命令生成
# kubeadm certs certificate-key
certificateKey: e8734dd17eaab2d55cfb363037f0d27cdd0f0cf1016d163d07428ac3aa9e19f1
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
# 使用外部 etcd 配置
etcd:
  external:
    endpoints:
    - "https://192.168.1.27:2379"
    - "https://192.168.1.28:2379"
    - "https://192.168.1.29:2379"
    caFile: "/etc/etcd/ssl/etcd-root-ca.pem"
    certFile: "/etc/etcd/ssl/etcd.pem"
    keyFile: "/etc/etcd/ssl/etcd-key.pem"
# 网络配置
networking:
  serviceSubnet: "10.25.0.0/16"
  # flannel
  #podSubnet: "10.30.0.1/16"
  # terway
  # 需要在阿里控制面板创建专门的虚拟交换机给pod使用
  podSubnet: "192.168.32.0/19"
  dnsDomain: "cluster.local"
kubernetesVersion: "v1.21.13"
# 全局 apiserver LB 地址，由于采用了 nginx/haproxy 负载，所以直接指向本地既可
controlPlaneEndpoint: "127.0.0.1:6443"
apiServer:
  # apiserver 的自定义扩展参数
  extraArgs:
    v: "4"
    alsologtostderr: "true"
    # 审计日志相关配置
    audit-log-maxage: "20"
    audit-log-maxbackup: "10"
    audit-log-maxsize: "100"
    audit-log-path: "/var/log/kube-audit/audit.log"
    audit-policy-file: "/etc/kubernetes/audit-policy.yaml"
    authorization-mode: "Node,RBAC"
    event-ttl: "720h"
    runtime-config: "api/all=true"
    service-node-port-range: "30000-50000"
    service-cluster-ip-range: "10.25.0.0/16"
  # 由于自行定义了审计日志配置，所以需要将宿主机上的审计配置
  # 挂载到 kube-apiserver 的 pod 容器中
  extraVolumes:
  - name: "audit-config"
    hostPath: "/etc/kubernetes/audit-policy.yaml"
    mountPath: "/etc/kubernetes/audit-policy.yaml"
    readOnly: true
    pathType: "File"
  - name: "audit-log"
    hostPath: "/var/log/kube-audit"
    mountPath: "/var/log/kube-audit"
    pathType: "DirectoryOrCreate"
  # 这里是 apiserver 的证书地址配置
  # 为了防止以后出特殊情况，我增加了一个泛域名
  certSANs:
  - "*.k8s.node"
  - "192.168.1.27"
  - "192.168.1.28"
  - "192.168.1.30"
  - "127.0.0.1"
  timeoutForControlPlane: 5m
controllerManager:
  extraArgs:
    v: "4"
    # 宿主机 ip 掩码
    node-cidr-mask-size: "19"
    deployment-controller-sync-period: "10s"
    experimental-cluster-signing-duration: "87600h"
    node-monitor-grace-period: "20s"
    pod-eviction-timeout: "2m"
    terminated-pod-gc-threshold: "30"
scheduler:
  extraArgs:
    v: "4"
certificatesDir: "/etc/kubernetes/pki"
# gcr.io 被墙
imageRepository: "registry.aliyuncs.com/google_containers"
#imageRepository: "registry.cn-chengdu.aliyuncs.com/archky-k8s"
clusterName: "kuberentes"
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
# kubelet specific options here
# 此配置保证了 kubelet 能在 swap 开启的情况下启动
failSwapOn: false
nodeStatusUpdateFrequency: 5s
# 一些驱逐阀值，具体自行查文档修改
evictionSoft:
  "imagefs.available": "15%"
  "memory.available": "512Mi"
  "nodefs.available": "15%"
  "nodefs.inodesFree": "10%"
evictionSoftGracePeriod:
  "imagefs.available": "3m"
  "memory.available": "1m"
  "nodefs.available": "3m"
  "nodefs.inodesFree": "1m"
evictionHard:
  "imagefs.available": "10%"
  "memory.available": "256Mi"
  "nodefs.available": "10%"
  "nodefs.inodesFree": "5%"
evictionMaxPodGracePeriod: 30
imageGCLowThresholdPercent: 70
imageGCHighThresholdPercent: 80
kubeReserved:
  "cpu": "500m"
  "memory": "512Mi"
  "ephemeral-storage": "1Gi"
rotateCertificates: true
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
# kube-proxy specific options here
# 建议与 podSubnet 保持一致
clusterCIDR: "192.168.32.0/19"
# 启用 ipvs 模式
mode: "ipvs"
ipvs:
  minSyncPeriod: 5s
  syncPeriod: 5s
  # ipvs 负载策略
  scheduler: "wrr"

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a Pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  /docs/concepts/cluster-administration/addons/

You can now join any number of machines by running the following on each node
as root:

  kubeadm join 127.0.0.1:6443 --token <token> --discovery-token-ca-cert-hash sha256:<hash>
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 127.0.0.1:6443 --token <token> \
    --discovery-token-ca-cert-hash sha256:<hash>

{
  "Version": "1",
  "Statement": [
    {
      "Action": [
        "ecs:CreateNetworkInterface",
        "ecs:DescribeNetworkInterfaces",
        "ecs:AttachNetworkInterface",
        "ecs:DetachNetworkInterface",
        "ecs:DeleteNetworkInterface",
        "ecs:DescribeInstanceAttribute",
        "ecs:DescribeInstanceTypes",
        "ecs:AssignPrivateIpAddresses",
        "ecs:UnassignPrivateIpAddresses",
        "ecs:DescribeInstances",
        "ecs:ModifyNetworkInterfaceAttribute"
      ],
      "Resource": [
        "*"
      ],
      "Effect": "Allow"
    },
    {
      "Action": [
        "vpc:DescribeVSwitches"
      ],
      "Resource": [
        "*"
      ],
      "Effect": "Allow"
    }
  ]
}

apiVersion: v1
kind: ServiceAccount
metadata:
  name: terway
  namespace: kube-system

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: terway-pod-reader
  namespace: kube-system
rules:
  - apiGroups: [ "" ]
    resources: [ "pods", "nodes", "namespaces", "configmaps", "serviceaccounts" ]
    verbs: [ "get", "watch", "list", "update" ]
  - apiGroups: [ "" ]
    resources:
      - events
    verbs:
      - create
  - apiGroups: [ "networking.k8s.io" ]
    resources:
      - networkpolicies
    verbs:
      - get
      - list
      - watch
  - apiGroups: [ "extensions" ]
    resources:
      - networkpolicies
    verbs:
      - get
      - list
      - watch
  - apiGroups: [ "" ]
    resources:
      - pods/status
    verbs:
      - update
  - apiGroups: [ "crd.projectcalico.org" ]
    resources: [ "*" ]
    verbs: [ "*" ]

---

apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: terway-binding
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: terway-pod-reader
subjects:
  - kind: ServiceAccount
    name: terway
    namespace: kube-system

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: eni-config
  namespace: kube-system
data:
  eni_conf: |
    {
      "version": "1",
      "access_key": "ak",
      "access_secret": "ak",
      "security_group": "sg-xxx",
      "service_cidr": "10.25.0.0/16",
      "vswitches": {
        "cn-hongkong-b": ["vsw-xxx"]
      },
      "max_pool_size": 5,
      "min_pool_size": 0
    }
  10-terway.conf: |
    {
      "cniVersion": "0.3.1",
      "name": "terway",
      "type": "terway",
      "eniip_virtual_type": "Veth",
      "ip_stack": "ipv4"
    }
  # eniip_virtual_type: virtual type for eni multi ip "Veth" || "IPVlan"
  disable_network_policy: "false"

---

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: terway
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: terway
  template:
    metadata:
      labels:
        app: terway
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      hostPID: true
      nodeSelector:
        beta.kubernetes.io/arch: amd64
      tolerations:
        - operator: "Exists"
      terminationGracePeriodSeconds: 0
      serviceAccountName: terway
      hostNetwork: true
      initContainers:
        - name: terway-init
          image: registry.aliyuncs.com/acs/terway:v1.2.3
          imagePullPolicy: IfNotPresent
          securityContext:
            privileged: true
          command:
            - 'sh'
            - '-c'
            - 'cp /usr/bin/terway /opt/cni/bin/;
                  chmod +x /opt/cni/bin/terway;
                  cp /etc/eni/10-terway.conf /etc/cni/net.d/;
                  sysctl -w net.ipv4.conf.eth0.rp_filter=0;
                  modprobe sch_htb || true;
                  chroot /host sh -c "systemctl disable eni.service; rm -f /etc/udev/rules.d/75-persistent-net-generator.rules /lib/udev/rules.d/60-net.rules /lib/udev/rules.d/61-eni.rules /lib/udev/write_net_rules && udevadm control --reload-rules && udevadm trigger; true"'
          volumeMounts:
            - name: configvolume
              mountPath: /etc/eni
            - name: cni-bin
              mountPath: /opt/cni/bin/
            - name: cni
              mountPath: /etc/cni/net.d/
            - mountPath: /lib/modules
              name: lib-modules
            - mountPath: /host
              name: host-root
      containers:
        - name: terway
          image: registry.aliyuncs.com/acs/terway:v1.2.3
          imagePullPolicy: IfNotPresent
          command: [ '/usr/bin/terwayd', '-log-level', 'debug', '-daemon-mode', 'ENIMultiIP' ]
          securityContext:
            privileged: true
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          volumeMounts:
            - name: configvolume
              mountPath: /etc/eni
            - mountPath: /var/run/
              name: eni-run
            - mountPath: /opt/cni/bin/
              name: cni-bin
            - mountPath: /lib/modules
              name: lib-modules
            - mountPath: /var/lib/cni/networks
              name: cni-networks
            - mountPath: /var/lib/cni/terway
              name: cni-terway
            - mountPath: /var/lib/kubelet/device-plugins
              name: device-plugin-path
        - name: policy
          image: registry.aliyuncs.com/acs/terway:v1.2.3
          imagePullPolicy: IfNotPresent
          command: [ "/bin/policyinit.sh" ]
          env:
            - name: NODENAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: DISABLE_POLICY
              valueFrom:
                configMapKeyRef:
                  name: eni-config
                  key: disable_network_policy
                  optional: true
          securityContext:
            privileged: true
          resources:
            requests:
              cpu: 250m
          livenessProbe:
            tcpSocket:
              port: 9099
              host: localhost
            periodSeconds: 10
            initialDelaySeconds: 10
            failureThreshold: 6
          readinessProbe:
            tcpSocket:
              port: 9099
              host: localhost
            periodSeconds: 10
          volumeMounts:
            - mountPath: /lib/modules
              name: lib-modules
      volumes:
        - name: configvolume
          configMap:
            name: eni-config
            items:
              - key: eni_conf
                path: eni.json
              - key: 10-terway.conf
                path: 10-terway.conf
        - name: cni-bin
          hostPath:
            path: /opt/cni/bin
            type: "Directory"
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: eni-run
          hostPath:
            path: /var/run/
            type: "Directory"
        - name: lib-modules
          hostPath:
            path: /lib/modules
        - name: cni-networks
          hostPath:
            path: /var/lib/cni/networks
        - name: cni-terway
          hostPath:
            path: /var/lib/cni/terway
        - name: device-plugin-path
          hostPath:
            path: /var/lib/kubelet/device-plugins
            type: "Directory"
        - name: host-root
          hostPath:
            path: /
            type: "Directory"

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: felixconfigurations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: FelixConfiguration
    plural: felixconfigurations
    singular: felixconfiguration

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: bgpconfigurations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: BGPConfiguration
    plural: bgpconfigurations
    singular: bgpconfiguration

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ippools.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: IPPool
    plural: ippools
    singular: ippool

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: hostendpoints.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: HostEndpoint
    plural: hostendpoints
    singular: hostendpoint

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: clusterinformations.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: ClusterInformation
    plural: clusterinformations
    singular: clusterinformation

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: globalnetworkpolicies.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: GlobalNetworkPolicy
    plural: globalnetworkpolicies
    singular: globalnetworkpolicy

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: globalnetworksets.crd.projectcalico.org
spec:
  scope: Cluster
  group: crd.projectcalico.org
  version: v1
  names:
    kind: GlobalNetworkSet
    plural: globalnetworksets
    singular: globalnetworkset

---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: networkpolicies.crd.projectcalico.org
spec:
  scope: Namespaced
  group: crd.projectcalico.org
  version: v1
  names:
    kind: NetworkPolicy
    plural: networkpolicies
    singular: networkpolicy

# 配置说明
data:
  eni_conf: |
    {
      "version": "1",
      "access_key": "ak",
      "access_secret": "sk",
      "security_group": "ESC的使用的安全组",
      # 与 kubeadm.yaml 的 serviceSubnet 保持一致
      "service_cidr": "10.25.0.0/16",
      # 交换机信息配置，区域和ID
      "vswitches": {
        "cn-hangzhou-k": ["交换机ID"]
      }

---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: psp.flannel.unprivileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
  privileged: false
  volumes:
  - configMap
  - secret
  - emptyDir
  - hostPath
  allowedHostPaths:
  - pathPrefix: "/etc/cni/net.d"
  - pathPrefix: "/etc/kube-flannel"
  - pathPrefix: "/run/flannel"
  readOnlyRootFilesystem: false
  # Users and groups
  runAsUser:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  fsGroup:
    rule: RunAsAny
  # Privilege Escalation
  allowPrivilegeEscalation: false
  defaultAllowPrivilegeEscalation: false
  # Capabilities
  allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
  defaultAddCapabilities: []
  requiredDropCapabilities: []
  # Host namespaces
  hostPID: false
  hostIPC: false
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  # SELinux
  seLinux:
    # SELinux is unused in CaaSP
    rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
rules:
- apiGroups: ['extensions']
  resources: ['podsecuritypolicies']
  verbs: ['use']
  resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.30.0.0/16",
      "Backend": {
        "Type": "ali-vpc"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      # 增加DNS服务器223.6.6.6（别的公共服务器也行）
      dnsPolicy: "None"
      dnsConfig:
        nameservers:
        - 223.6.6.6
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay.io/coreos/flannel:v0.14.0
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay.io/coreos/flannel:v0.14.0
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        # ali-vpc
        - name: ACCESS_KEY_ID
          value: Ak
        - name: ACCESS_KEY_SECRET
          value: Sk
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg