Skip to content

Commit 83e0bda

Browse files
redscholarredscholar
authored
fix: scaling down etcd (#2775)
Signed-off-by: redscholar <blacktiledhouse@gmail.com>
1 parent 57e28b5 commit 83e0bda

46 files changed

Lines changed: 1896 additions & 341 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎builtin/core/defaults/inventory/localhost.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ spec:
1414
# port: 22
1515
# user: root
1616
# password: 123456
17+
# internal_ipv4: 1.1.1.1
1718
groups:
1819
# all kubernetes nodes.
1920
k8s_cluster:

‎builtin/core/playbooks/add_nodes.yaml‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,48 @@
3636
- etcd
3737
gather_facts: true
3838
roles:
39-
- role: etcd
39+
- role: etcd/prepare
4040
when: .etcd.deployment_type | eq "external"
41+
- role: etcd/backup
42+
when: .etcd.deployment_type | eq "external"
43+
- hosts:
44+
- etcd
45+
serial: 1
46+
roles:
47+
- role: etcd/scaling_up/learner
48+
when:
49+
- .etcd.deployment_type | eq "external"
50+
- .installed_etcd | empty | not
51+
- .need_installed_etcd | default list | has .inventory_hostname
52+
- role: etcd/install
53+
when:
54+
- .etcd.deployment_type | eq "external"
55+
- .installed_etcd | empty | not
56+
- .need_installed_etcd | default list | has .inventory_hostname
57+
- role: etcd/scaling_up/promote
58+
when:
59+
- .etcd.deployment_type | eq "external"
60+
- .installed_etcd | empty | not
61+
- .need_installed_etcd | default list | has .inventory_hostname
62+
- hosts:
63+
- etcd
64+
gather_facts: true
65+
roles:
66+
- role: etcd/postprocess
67+
when:
68+
- .etcd.deployment_type | eq "external"
69+
- .installed_etcd | empty | not
70+
- .need_installed_etcd | default list | has .inventory_hostname
71+
72+
- hosts:
73+
- kube_control_plane
74+
serial: 1
75+
roles:
76+
- role: kubernetes/sync-etcd-config
77+
when:
78+
- .need_installed_etcd | empty | not
79+
- .etcd.deployment_type | eq "external"
80+
- .kubernetes_install_ActiveState.stdout | eq "active"
4181

4282
- hosts:
4383
- k8s_cluster
@@ -47,7 +87,6 @@
4787
- role: kubernetes/pre-kubernetes
4888
when: or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname)
4989
- role: kubernetes/init-kubernetes
50-
when: or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname)
5190
- role: kubernetes/join-kubernetes
5291
when:
5392
- or (.add_nodes | default list | empty) (.add_nodes | default list | has .inventory_hostname)

‎builtin/core/playbooks/create_cluster.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,11 @@
3535
- hosts:
3636
- etcd
3737
roles:
38-
- role: etcd
38+
- role: etcd/prepare
3939
when: .etcd.deployment_type | eq "external"
40+
- role: etcd/install
41+
when: .etcd.deployment_type | eq "external"
42+
4043

4144
# Install the private image registry
4245
- hosts:

‎builtin/core/playbooks/delete_cluster.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@
3232
- hosts:
3333
- etcd
3434
roles:
35-
- role: uninstall/etcd
36-
when:
35+
- role: etcd/scaling_down
36+
when:
3737
- .delete.etcd
3838
- .etcd.deployment_type | eq "external"
3939

‎builtin/core/playbooks/delete_nodes.yaml‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,39 @@
1010
gather_facts: true
1111
roles:
1212
- defaults
13-
- precheck
1413

1514
- hosts:
16-
- kube_control_plane
15+
- etcd
16+
roles:
17+
- role: etcd/prepare
18+
when:
19+
- .delete.etcd
20+
- .etcd.deployment_type | eq "external"
21+
- hosts:
22+
- etcd
23+
serial: 1
24+
roles:
25+
- role: etcd/scaling_down
26+
when:
27+
- .delete.etcd
28+
- .etcd.deployment_type | eq "external"
29+
- .need_uninstall_etcd | has .inventory_hostname
30+
- hosts:
31+
- etcd
1732
gather_facts: true
18-
tasks:
33+
roles:
34+
- role: etcd/postprocess
35+
when:
36+
- .delete.etcd
37+
- .etcd.deployment_type | eq "external"
38+
- .need_uninstall_etcd | empty | not
39+
- .need_uninstall_etcd | has .inventory_hostname | not
40+
41+
- hosts:
42+
- kube_control_plane
43+
serial: 1
44+
pre_tasks:
1945
- name: DeleteNode | Ensure at least one control plane node remains in the cluster
20-
run_once: true
2146
command: |
2247
{{- $cpNodes := list -}}
2348
{{- range .groups.kube_control_plane -}}
@@ -29,6 +54,13 @@
2954
echo "At least one control plane node must be retained in the cluster." >&2
3055
exit 1
3156
{{- end }}
57+
roles:
58+
- role: kubernetes/sync-etcd-config
59+
when:
60+
- .need_installed_etcd | empty | not
61+
- .etcd.deployment_type | eq "external"
62+
- .kubernetes_install_ActiveState.stdout | eq "active"
63+
- .delete_nodes | default list | has .inventory_hostname | not
3264

3365
- hosts:
3466
- k8s_cluster
@@ -71,7 +103,7 @@
71103
- hosts:
72104
- etcd
73105
roles:
74-
- role: uninstall/etcd
106+
- role: etcd
75107
when:
76108
- .delete.etcd
77109
- .etcd.deployment_type | eq "external"

‎builtin/core/roles/certs/init/tasks/main.yaml‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242
{{ .binary_dir }}/pki/front-proxy.crt
4343
4444
- name: Cert | Generate the etcd certificate file
45+
loop: "{{ .groups.etcd | toJson }}"
46+
when: .item | empty | not
4547
gen_cert:
4648
root_key: >-
4749
{{ .binary_dir }}/pki/root.key
@@ -50,24 +52,43 @@
5052
cn: etcd
5153
sans: >-
5254
{{- $ips := list -}}
53-
{{- range .groups.etcd | default list -}}
54-
{{- $internalIPv4 := index $.hostvars . "internal_ipv4" | default "" -}}
55-
{{- $internalIPv6 := index $.hostvars . "internal_ipv6" | default "" -}}
56-
{{- if $internalIPv4 | empty | not -}}
57-
{{- $ips = append $ips $internalIPv4 -}}
58-
{{- end -}}
59-
{{- if $internalIPv6 | empty | not -}}
60-
{{- $ips = append $ips $internalIPv6 -}}
61-
{{- end -}}
55+
{{- $hostname := index .hostvars .item "hostname" | default "" -}}
56+
{{- if .native.set_hostname -}}
57+
{{- $hostname = .item -}}
58+
{{- end -}}
59+
{{- if $hostname | empty | not -}}
60+
{{- $ips = append $ips $hostname -}}
61+
{{- end -}}
62+
{{- $internalIPv4 := index .hostvars .item "internal_ipv4" | default "" -}}
63+
{{- if $internalIPv4 | empty | not -}}
64+
{{- $ips = append $ips $internalIPv4 -}}
65+
{{- end -}}
66+
{{- $internalIPv6 := index .hostvars .item "internal_ipv6" | default "" -}}
67+
{{- if $internalIPv6 | empty | not -}}
68+
{{- $ips = append $ips $internalIPv6 -}}
6269
{{- end -}}
6370
{{ $ips | toJson }}
6471
date: "{{ .certs.etcd.date }}"
6572
policy: "{{ .certs.etcd.gen_cert_policy }}"
6673
out_key: >-
67-
{{ .binary_dir }}/pki/etcd.key
74+
{{ .binary_dir }}/pki/etcd-{{ .item }}.key
6875
out_cert: >-
69-
{{ .binary_dir }}/pki/etcd.crt
76+
{{ .binary_dir }}/pki/etcd-{{ .item }}.crt
77+
78+
- name: Cert | Generate the etcd client certificate file
7079
when: .groups.etcd | default list | empty | not
80+
gen_cert:
81+
root_key: >-
82+
{{ .binary_dir }}/pki/root.key
83+
root_cert: >-
84+
{{ .binary_dir }}/pki/root.crt
85+
cn: etcd
86+
date: "{{ .certs.etcd.date }}"
87+
policy: "{{ .certs.etcd.gen_cert_policy }}"
88+
out_key: >-
89+
{{ .binary_dir }}/pki/etcd-client.key
90+
out_cert: >-
91+
{{ .binary_dir }}/pki/etcd-client.crt
7192
7293
- name: Cert | Generate the image registry certificate file
7394
tags: ["image_registry"]

‎builtin/core/roles/certs/renew/etcd/tasks/main.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,19 @@
22
- name: ETCD | Copy CA certificate to remote host
33
copy:
44
src: >-
5-
{{ ..etcd.ca_file }}
5+
{{ .etcd.ca_file }}
66
dest: /etc/ssl/etcd/ssl/ca.crt
77

88
- name: ETCD | Copy server certificate to remote host
99
copy:
1010
src: >-
11-
{{ .etcd.cert_file }}
11+
{{ tpl .etcd.server_cert_file .inventory_hostname }}
1212
dest: /etc/ssl/etcd/ssl/server.crt
1313

1414
- name: ETCD | Copy server private key to remote host
1515
copy:
1616
src: >-
17-
{{ .etcd.key_file }}
17+
{{ tpl .etcd.server_key_file .inventory_hostname }}
1818
dest: /etc/ssl/etcd/ssl/server.key
1919

2020
- name: ETCD | Restart etcd service to apply new certificates

‎builtin/core/roles/certs/renew/kubernetes/tasks/etcd.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,13 @@
99
- name: ETCD | Copy client certificate to remote host
1010
copy:
1111
src: >-
12-
{{ .etcd.cert_file }}
12+
{{ .etcd.client_cert_file }}
1313
dest: /etc/kubernetes/pki/etcd/client.crt
1414
mode: 0755
1515

1616
- name: ETCD | Copy client key to remote host
1717
copy:
1818
src: >-
19-
{{ .etcd.key_file }}
19+
{{ .etcd.client_key_file }}
2020
dest: /etc/kubernetes/pki/etcd/client.key
2121
mode: 0755

‎builtin/core/roles/defaults/defaults/main/02-certs.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
# CA (self-signed or provided)
99
# |- etcd.cert
1010
# |- etcd.key
11+
# |- etcd-client.cert
12+
# |- etcd-client.key
1113
# |
1214
# |- image_registry.cert
1315
# |- image_registry.key

‎builtin/core/roles/defaults/defaults/main/04-etcd.yaml‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ etcd:
99
{{ .image_registry.dockerio_registry }}
1010
repository: kubesphere/etcd
1111
tag: "{{ .etcd.etcd_version }}"
12-
# endpoints: ["https://127.1.1.1:2379"]
12+
port: 2379
13+
peer_port: 2380
1314
# Environment variables for etcd service
1415
env:
1516
election_timeout: 5000
@@ -37,7 +38,11 @@ etcd:
3738
traffic_priority: false
3839
ca_file: >-
3940
{{ .binary_dir }}/pki/root.crt
40-
cert_file: >-
41-
{{ .binary_dir }}/pki/etcd.crt
42-
key_file: >-
43-
{{ .binary_dir }}/pki/etcd.key
41+
server_cert_file: >-
42+
{{ .binary_dir }}/pki/etcd-{{ "{{ . }}" }}.crt
43+
server_key_file: >-
44+
{{ .binary_dir }}/pki/etcd-{{ "{{ . }}" }}.key
45+
client_cert_file: >-
46+
{{ .binary_dir }}/pki/etcd-client.crt
47+
client_key_file: >-
48+
{{ .binary_dir }}/pki/etcd-client.key

0 commit comments

Comments
 (0)