ETCD集群部署

kubernetes
字数统计: 1.4k   |   阅读时长≈ 8 分钟

ETCD集群部署

环境信息

角色ip备注
etcd01.etcd.kubernetes10.10.10.31
etcd02.etcd.kubernetes10.10.10.32
etcd03.etcd.kubernetes10.10.10.33

配置hosts文件

1
2
3
4
5
cat << EOF >> /etc/hosts
10.10.10.31 etcd01.etcd.kubernetes
10.10.10.32 etcd02.etcd.kubernetes
10.10.10.33 etcd03.etcd.kubernetes
EOF

准备etcd二进制文件

1
2
3
mkdir -pv /opt/kubernetes/{cfg,data/etcd,bin,data,ssl,logs}
tar xf etcd-v3.3.10-linux-amd64.tar.gz
mv etcd-v3.3.10-linux-amd64/etcd* /opt/kubernetes/bin/

创建etcd所需证书文件

1、创建 etcd 证书的相关文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
cat << EOF > ca-csr.json 
{
    "CN":"kubernetes",
    "key":{
        "algo":"rsa",
        "size":2048
    },
    "ca":{
        "expiry":"262800h"
    },
    "names":[
        {

        }
    ]
}
EOF

cat << EOF > ca-config.json
{
    "signing":{
        "default":{
            "expiry":"8760h"
        },
        "profiles":{
            "kubernetes":{
                "expiry":"87600h",
                "usages":[
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}
EOF

cat << EOF > etcd-csr.json 
{
    "CN":"etcd.kubernetes",
    "hosts":[
        "127.0.0.1",
        "10.10.10.31",
        "10.10.10.32",
        "10.10.10.33",
        "localhost",
        "localhost.localdomain",
        "etcd-01.etcd.kubernetes",
        "etcd-02.etcd.kubernetes",
        "etcd-03.etcd.kubernetes"
    ],
    "key":{
        "algo":"rsa",
        "size":2048
    },
    "names":[
        {

        }
    ]
}
EOF

2、生成证书和私钥

1
2
3
4
5
6
cfssl gencert -initca ca-csr.json | cfssljson -bare ca

cfssl gencert -ca=ca.pem \
              -ca-key=ca-key.pem \
              -config=ca-config.json \
              -profile=kubernetes etcd-csr.json | cfssljson -bare etcd

3、分发证书

1
2
cp -a etcd*.pem /opt/kubernetes/ssl
ha scp ./etcd*.pem root@$I:/opt/kubernetes/ssl ;done

创建etcd配置文件

  • 节点一（10.10.10.31）
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
cat << 'EOF' > /opt/kubernetes/cfg/kube-etcd.conf
KUBE_ETCD_OPTS="--name=etcd1 \
--data-dir=/opt/kubernetes/data/etcd \
--listen-peer-urls=https://10.10.10.31:2380 \
--listen-client-urls=https://10.10.10.31:2379,http://127.0.0.1:2379 \
--initial-advertise-peer-urls=https://10.10.10.31:2380 \
--initial-cluster-token=etcd-cluster \
--advertise-client-urls=https://10.10.10.31:2379 \
--cert-file=/opt/kubernetes/ssl/etcd.pem \
--key-file=/opt/kubernetes/ssl/etcd-key.pem \
--peer-cert-file=/opt/kubernetes/ssl/etcd.pem \
--peer-key-file=/opt/kubernetes/ssl/etcd-key.pem \
--trusted-ca-file=/opt/kubernetes/ssl/ca.pem \
--peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem \
--client-cert-auth=True \
--initial-cluster etcd1=https://10.10.10.31:2380,etcd2=https://10.10.10.32:2380,etcd3=https://10.10.10.33:2380 \
--initial-cluster-state=new"
EOF
  • 节点二：(10.10.10.32)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
cat << 'EOF' > /opt/kubernetes/cfg/kube-etcd.conf
KUBE_ETCD_OPTS="--name=etcd2 \
--data-dir=/opt/kubernetes/data/etcd \
--listen-peer-urls=https://10.10.10.32:2380 \
--listen-client-urls=https://10.10.10.32:2379,http://127.0.0.1:2379 \
--initial-advertise-peer-urls=https://10.10.10.32:2380 \
--initial-cluster-token=etcd-cluster \
--advertise-client-urls=https://10.10.10.32:2379 \
--cert-file=/opt/kubernetes/ssl/etcd.pem \
--key-file=/opt/kubernetes/ssl/etcd-key.pem \
--peer-cert-file=/opt/kubernetes/ssl/etcd.pem \
--peer-key-file=/opt/kubernetes/ssl/etcd-key.pem \
--trusted-ca-file=/opt/kubernetes/ssl/ca.pem \
--peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem \
--client-cert-auth=True \
--initial-cluster etcd1=https://10.10.10.31:2380,etcd2=https://10.10.10.32:2380,etcd3=https://10.10.10.33:2380 \
--initial-cluster-state=new"
EOF
  • 节点三：(10.10.10.33)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
cat << 'EOF' > /opt/kubernetes/cfg/kube-etcd.conf
KUBE_ETCD_OPTS="--name=etcd3 \
--data-dir=/opt/kubernetes/data/etcd \
--listen-peer-urls=https://10.10.10.33:2380 \
--listen-client-urls=https://10.10.10.33:2379,http://127.0.0.1:2379 \
--initial-advertise-peer-urls=https://10.10.10.33:2380 \
--initial-cluster-token=etcd-cluster \
--advertise-client-urls=https://10.10.10.33:2379 \
--cert-file=/opt/kubernetes/ssl/etcd.pem \
--key-file=/opt/kubernetes/ssl/etcd-key.pem \
--peer-cert-file=/opt/kubernetes/ssl/etcd.pem \
--peer-key-file=/opt/kubernetes/ssl/etcd-key.pem \
--trusted-ca-file=/opt/kubernetes/ssl/ca.pem \
--peer-trusted-ca-file=/opt/kubernetes/ssl/ca.pem \
--client-cert-auth=True \
--initial-cluster etcd1=https://10.10.10.31:2380,etcd2=https://10.10.10.32:2380,etcd3=https://10.10.10.33:2380 \
--initial-cluster-state=new"
EOF

创建systemctl启动脚本

  • 三个etcd节点是一样的，三个节点分别创建
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
cat << 'EOF' > /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
Type=notify
WorkingDirectory=/opt/kubernetes/data/etcd
EnvironmentFile=/opt/kubernetes/cfg/kube-etcd.conf
ExecStart=/opt/kubernetes/bin/etcd $KUBE_ETCD_OPTS
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF

启动etcd集群

etcd节点要同时启动，才能启动成功

1
2
3
4
systemctl daemon-reload
systemctl restart etcd
systemctl status etcd
systemctl enable etcd

CTCD集群信息查看

由于是https，所以查看操作集群需要使用证书。

etcdctl 默认使用的是api v2版本

  • 方法一
1
2
3
cd /opt/kubernetes/ssl
etcdctl --ca-file=ca.pem --cert-file=etcd.pem --key-file=etcd-key.pem cluster-health
etcdctl --ca-file=ca.pem --cert-file=etcd.pem --key-file=etcd-key.pem member list
  • 方法二
1
2
3
4
5
6
alias etcdctl="etcdctl \
--endpoints='https://10.10.10.31:2379,https://10.10.10.32:2379,https://10.10.10.33:2379' \
--ca-file=/opt/kubernetes/ssl/ca.pem -cert-file=/opt/kubernetes/ssl/etcd.pem \
--key-file=/opt/kubernetes/ssl/etcd-key.pem"
etcdctl cluster-health
etcdctl member list

etcdctl 的api v3版本使用

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# 使用环境变量定义api版本
export ETCDCTL_API=3

# 申明etcd相关信息，etcdctl 默认连接的是http://127.0.0.1:2379，因无证书也能访问，建议关闭回环网卡监听。
export ETCDCTL_FILE=/opt/kubernetes/bin/etcdctl
export ETCD_ENDPOINTS=https://10.10.10.31:2379,https://10.10.10.32:2379,https://10.10.10.33:2379
export ETCD_CA_FILE=/opt/kubernetes/ssl/ca.pem
export ETCD_cert_FILE=/opt/kubernetes/ssl/etcd.pem
export ETCD_key_FILE=/opt/kubernetes/ssl/etcd-key.pem

# 配置etcdctl别名
alias etcdctl="$ETCDCTL_FILE --endpoints=$ETCD_ENDPOINTS --cacert=$ETCD_CA_FILE --cert=$ETCD_cert_FILE --key=$ETCD_key_FILE"

# 列出集群成员
etcdctl member list
etcdctl member list -w table

#----示例
[root@k8s-master ~]# etcdctl member list
a01381d0afc19e9, started, etcd1, https://10.10.10.31:2380, https://10.10.10.31:2379
53807b9bddd14168, started, etcd2, https://10.10.10.32:2380, https://10.10.10.32:2379
7b8d79f533deb5ed, started, etcd3, https://10.10.10.33:2380, https://10.10.10.33:2379
[root@k8s-master ~]# etcdctl member list -w table
+------------------+---------+-------+--------------------------+--------------------------+
|        ID        | STATUS  | NAME  |        PEER ADDRS        |       CLIENT ADDRS       |
+------------------+---------+-------+--------------------------+--------------------------+
|  a01381d0afc19e9 | started | etcd1 | https://10.10.10.31:2380 | https://10.10.10.31:2379 |
| 53807b9bddd14168 | started | etcd2 | https://10.10.10.32:2380 | https://10.10.10.32:2379 |
| 7b8d79f533deb5ed | started | etcd3 | https://10.10.10.33:2380 | https://10.10.10.33:2379 |
+------------------+---------+-------+--------------------------+--------------------------+

# 集群状态,集群状态主要是etcdctl endpoint status 和etcdctl endpoint health两条命令。
etcdctl endpoint health
etcdctl endpoint status --write-out=table 或etcdctl endpoint status -w table

#----示例
[root@k8s-master ~]# etcdctl endpoint status -w table
+--------------------------+------------------+---------+---------+--------
|         ENDPOINT         |        ID        | VERSION | DB SIZE |   .....
+--------------------------+------------------+---------+---------+--------
| https://10.10.10.31:2379 |  a01381d0afc19e9 |  3.3.10 |  3.8 MB |   .....
| https://10.10.10.32:2379 | 53807b9bddd14168 |  3.3.10 |  3.8 MB |   .....
| https://10.10.10.33:2379 | 7b8d79f533deb5ed |  3.3.10 |  3.8 MB |   .....
+--------------------------+------------------+---------+---------+--------
打赏
  • 版权声明： 本博客所有文章除特别声明外，著作权归作者所有。转载请注明出处！
  • Copyrights © 2018-2024 Outsrkem
  • 访问人数: | 浏览次数:

      请我喝杯咖啡吧~

      支付宝
      微信