Repository navigation
Expand file tree
/
Copy pathetcdcluster_controller.go
More file actions
2262 lines (2104 loc) · 92 KB
/
Copy pathetcdcluster_controller.go
File metadata and controls
2262 lines (2104 loc) · 92 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
Copyright 2023 Timofey Larkin.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
"context"
"fmt"
"math"
"sort"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
policyv1 "k8s.io/api/policy/v1"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
clientv3 "go.etcd.io/etcd/client/v3"
lll "github.com/cozystack/etcd-operator/api/v1alpha2"
)
// DefaultProgressDeadlineSeconds is used when the user doesn't set one.
const DefaultProgressDeadlineSeconds = int32(600)
// EtcdClusterReconciler reconciles an EtcdCluster object.
type EtcdClusterReconciler struct {
client.Client
Scheme *runtime.Scheme
// EtcdClientFactory builds an etcd client. Tests inject a fake;
// production wiring uses DefaultEtcdClientFactory.
EtcdClientFactory EtcdClientFactory
// Recorder emits events on the EtcdCluster. Tests may leave it nil.
Recorder record.EventRecorder
// CertManagerAvailable reports whether the cert-manager.io/v1 API
// is registered on this cluster. Detected once at operator startup
// via the discovery API. When false, clusters using
// spec.tls.{client,peer}.certManager are surfaced as terminally
// Available=False/CertManagerNotInstalled rather than being
// reconciled into an informer hot-loop.
CertManagerAvailable bool
// ClusterDomain is the DNS suffix used for the cluster-domain half
// of Kubernetes service / pod DNS (e.g. "cluster.local",
// "cozy.local"). Threaded into the cert-manager-emitted Certificate
// SAN lists so the FQDN form (`*.<cluster>.<ns>.svc.<cluster-domain>`)
// matches what kube-dns returns for the peer reverse-DNS lookup
// etcd uses to authenticate incoming peer mTLS connections.
//
// Defaults to "cluster.local" when empty. Override at operator
// startup with the --cluster-domain flag.
ClusterDomain string
}
//+kubebuilder:rbac:groups=etcd-operator.cozystack.io,resources=etcdclusters,verbs=get;list;watch
//+kubebuilder:rbac:groups=etcd-operator.cozystack.io,resources=etcdclusters/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=etcd-operator.cozystack.io,resources=etcdclusters/finalizers,verbs=update
//+kubebuilder:rbac:groups=etcd-operator.cozystack.io,resources=etcdmembers,verbs=get;list;watch;create;delete
//+kubebuilder:rbac:groups=etcd-operator.cozystack.io,resources=etcdmembers/status,verbs=get;update;patch
//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch
//+kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch
// `delete` is intentionally omitted from the certificates verb list:
// each Certificate is owned by the EtcdCluster via SetControllerReference,
// and spec.tls is CEL-immutable post-create, so the only Certificate
// deletion path is the apiserver's owner-ref GC cascading from a cluster
// delete. If a future change relaxes the spec.tls immutability rule,
// add `delete` here.
//+kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch
func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := log.FromContext(ctx)
cluster := &lll.EtcdCluster{}
if err := r.Get(ctx, req.NamespacedName, cluster); err != nil {
if errors.IsNotFound(err) {
return ctrl.Result{}, nil
}
return ctrl.Result{}, err
}
// If the cluster is being deleted, don't keep reconciling it. Owned
// resources are cascaded out via owner refs; recreating a Service for a
// Terminating cluster races against the GC and pollutes logs.
if !cluster.DeletionTimestamp.IsZero() {
return ctrl.Result{}, nil
}
// Terminal-config gate. spec.tls.{client,peer}.certManager requires
// cert-manager.io/v1 to be registered on the apiserver (detected at
// operator startup). When it isn't, the cluster cannot form a single
// Pod (the member controller is gated on Secret existence; cert-
// manager would never materialise it). Surface Available=False/
// CertManagerNotInstalled and stop here so the rest of Reconcile
// (which would clobber this condition with ClusterUnreachable /
// WaitingForSeed once it tries to bootstrap a member) doesn't run.
if !r.CertManagerAvailable && hasCertManagerTLS(cluster) {
changed := setClusterCondition(cluster, lll.ClusterAvailable, metav1.ConditionFalse,
"CertManagerNotInstalled",
"spec.tls.{client,peer}.certManager is set but cert-manager.io/v1 is not registered on this cluster; install cert-manager and restart the operator, or switch to BYO Secrets")
if changed {
if err := r.Status().Update(ctx, cluster); err != nil {
if errors.IsConflict(err) {
return ctrl.Result{Requeue: true}, nil
}
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil
}
if err := r.ensureServices(ctx, cluster); err != nil {
log.Error(err, "failed to ensure services")
return ctrl.Result{}, err
}
if err := r.reconcileTLSCertificates(ctx, cluster); err != nil {
log.Error(err, "failed to reconcile cert-manager Certificates")
return ctrl.Result{}, err
}
memberList := &lll.EtcdMemberList{}
if err := r.List(ctx, memberList,
client.InNamespace(cluster.Namespace),
client.MatchingLabels{LabelCluster: cluster.Name},
); err != nil {
return ctrl.Result{}, err
}
active := filterActiveMembers(memberList.Items)
// running excludes dormant members. The cluster controller's replica
// accounting (`current`), readiness checks, and most scale decisions
// operate on this set: a dormant member has no Pod and contributes
// no etcd capacity, so it must not count as "we have N members" or
// "we need to wait for it to become Ready" — otherwise the resume
// flow could never decide to wake it.
running := filterRunningMembers(memberList.Items)
// ── First-reconcile init ──────────────────────────────────────────
// Combine the ClusterToken latch, initial Observed snapshot, and
// progress deadline into a single Status write so a brand-new cluster
// settles into a reconcile-ready shape in one pass.
current := int32(len(running))
now := metav1.Now()
if cluster.Status.ClusterToken == "" || cluster.Status.Observed == nil {
log.Info("initialising cluster status (token + observed snapshot)")
if cluster.Status.ClusterToken == "" {
cluster.Status.ClusterToken = deriveClusterToken(cluster)
}
if cluster.Status.Observed == nil {
snapshotSpecIntoObserved(cluster)
setProgressDeadline(cluster, now)
setClusterCondition(cluster, lll.ClusterProgressing, metav1.ConditionTrue, "InitialSnapshot",
"snapshotted initial spec into status.observed")
}
if err := r.Status().Update(ctx, cluster); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
}
// ── Subsequent spec-change handling ───────────────────────────────
// Triggers that update Observed from the latest spec:
// - the previous Observed target is complete and spec has changed
// - the deadline has elapsed (handled separately)
// reconciliationComplete uses the running set (non-dormant). A
// paused cluster has observed.Replicas=0 and len(running)=0 even
// while a dormant CR sits alongside; passing `active` here would
// count the dormant member against the target and never reach
// complete=true, so the spec-change-adoption path would never fire
// when the user scales the paused cluster back up.
complete := reconciliationComplete(cluster, running)
if !complete && deadlineExpired(cluster, now) {
return r.handleDeadlineExceeded(ctx, cluster, now)
}
if complete && !specEqualsObserved(cluster) {
log.Info("previous target reached; adopting new spec",
"observed", cluster.Status.Observed, "spec", cluster.Spec)
snapshotSpecIntoObserved(cluster)
setProgressDeadline(cluster, now)
setClusterCondition(cluster, lll.ClusterProgressing, metav1.ConditionTrue, "SpecChanged",
"adopting new spec after previous target reached")
if err := r.Status().Update(ctx, cluster); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
}
desired := cluster.Status.Observed.Replicas
// ── Bootstrap ──────────────────────────────────────────────────────
// Bootstrap creates a single-member etcd cluster (member -0 only) with
// --initial-cluster-state=new. Once ClusterID is latched, scale-up adds
// the remaining members via MemberAdd. This avoids the historical bug
// where multiple bootstrapping members had to agree on a single
// --initial-cluster value and any mid-flight replica change would
// corrupt that consensus.
if cluster.Status.ClusterID == "" {
// Pre-bootstrap fan-out:
// - observed.replicas == 0: user declared the cluster as paused
// from the start. Do NOT bootstrap a transient seed only to
// immediately tear it down — that would waste a pod + image
// pull + PVC write, surface a fleeting ClusterID for a cluster
// that was never used, and produce a misleading event sequence.
// Fall through to updateStatus, which has a Paused branch.
// The first non-zero spec the user submits will snapshot into
// observed and the bootstrap branch will fire then.
// - current == 0: fresh cluster, no seed yet → bootstrap() creates it.
// - hasPendingBootstrap: seed CR exists with empty InitialCluster
// (crash between Create and Patch) → bootstrap() completes it;
// skipping this and going to discovery would loop forever because
// the member controller won't start the seed's pod until
// InitialCluster is set.
// - Else: seed CR exists, fully wired, pod is coming up — proceed
// to discovery to latch ClusterID once etcd answers.
if desired == 0 {
log.Info("cluster declared paused from the start; not bootstrapping")
return r.updateStatus(ctx, cluster, active, nil)
}
if current == 0 || hasPendingBootstrap(running) {
log.Info("bootstrapping single-node cluster")
return r.bootstrap(ctx, cluster, running)
}
log.Info("waiting for bootstrap member to form cluster")
return r.tryDiscoverCluster(ctx, cluster, running)
}
// ── Scale ──────────────────────────────────────────────────────────
// Before deciding to scale either direction, wait for any in-flight
// EtcdMember deletion to finish. The EtcdMember finalizer calls
// MemberRemove against the etcd cluster; running that concurrently
// with a MemberAdd from scale-up, or with another MemberRemove from
// scale-down, races past quorum because each goroutine works from its
// own snapshot of the etcd member list. One mutation at a time.
for _, m := range memberList.Items {
if !m.DeletionTimestamp.IsZero() {
log.Info("waiting for in-flight member deletion before further scaling", "deleting", m.Name)
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
}
// If a half-baked scale-up CR is present (Spec.InitialCluster empty),
// route through scaleUp regardless of replica accounting. Two reasons:
//
// 1. The member controller gates pod creation on InitialCluster being
// set, so the pending CR has no pod and contributes a never-Ready
// member to `running`. The standard `current < desired` +
// allMembersReady gate would wait forever.
// 2. If the prior reconcile crashed AFTER MemberAddAsLearner but
// BEFORE the patch, etcd already has a learner whose peer URL
// points at the pending CR. promotePendingLearner cannot promote
// that learner (no pod = no sync), so any path that promotes
// first and patches second deadlocks. scaleUp patches first.
if hasPendingMember(running) {
log.Info("completing pending scale-up member before further action")
return r.scaleUp(ctx, cluster, active)
}
if current < desired {
// Resurrection from dormant: if a dormant member is parked, wake
// it before doing anything else. This is the inverse of the 1→0
// pause step — flip spec.Dormant back to false on the same
// member and let the member controller bring its pod back up.
// The dormant member is in `active` (it has no DeletionTimestamp)
// but not in `running` (filtered by Dormant), which is exactly
// the state that puts us here: current<desired with the dormant
// CR sitting alongside.
if findDormantMember(active) != nil {
log.Info("waking dormant member", "desired", desired)
return r.scaleUp(ctx, cluster, active)
}
// Don't add another member while existing ones aren't Ready yet.
// Even with learner-mode adds (which don't shift voting quorum until
// promotion), driving a sequence of MemberAddAsLearner without
// waiting for the previous pod to come up makes the cluster
// progressively more confused — the new pod has to fetch state from
// peers, and stacking joiners against a half-empty cluster is at
// best slow, at worst stalls indefinitely. Wait for everyone Ready
// before adding the next.
if !allMembersReady(running) {
log.Info("waiting for existing members to become Ready before next scale-up step")
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
log.Info("scaling up", "current", current, "desired", desired)
return r.scaleUp(ctx, cluster, active)
}
if current > desired {
log.Info("scaling down", "current", current, "desired", desired)
return r.scaleDown(ctx, cluster, running)
}
// ── current == desired ────────────────────────────────────────────
// One subtlety: the iteration that added the *last* learner returns
// before that learner gets promoted (current is now equal to desired
// so scaleUp won't run again on its own). We need a promote attempt
// here too. Cheap: list etcd once and try to promote any learner;
// no-op if none.
//
// A transient requeue from promote/auth is threaded through
// updateStatus via `pending`, not returned here: an early return skips
// updateStatus, freezing the cluster conditions at their last-healthy
// value while the members already read Ready=False.
var pending *ctrl.Result
if cluster.Status.ClusterID != "" && len(running) > 0 {
endpoints := memberEndpoints(clusterClientScheme(cluster), running, cluster.Namespace)
tlsCfg, tlsErr := buildOperatorTLSConfig(ctx, r.Client, cluster)
if tlsErr != nil {
// Don't bail out of the whole reconcile — updateStatus
// below still has useful work (PDB reconcile, cached-
// status fields). Just skip the promote attempt and
// surface the error so a missing/typo'd TLS Secret is
// debuggable without rg-ing through silent retries.
log.Error(tlsErr, "cannot build operator TLS config for promote-after-converged; skipping promotion this pass")
} else if user, pass, _, credErr := resolveEtcdCredentials(ctx, r.Client, cluster); credErr != nil {
// Same best-effort stance as the TLS-config error above: log and
// skip promotion, let updateStatus do its work, retry next pass.
log.Error(credErr, "cannot resolve etcd credentials for promote-after-converged; skipping promotion this pass")
} else {
etcdClient, err := r.EtcdClientFactory(ctx, endpoints, tlsCfg, user, pass)
if err != nil {
log.Error(err, "cannot dial etcd for promote-after-converged; skipping promotion this pass", "endpoints", endpoints)
} else {
defer etcdClient.Close()
res, perr := r.promotePendingLearner(ctx, cluster, running, etcdClient, endpoints)
if perr != nil {
return ctrl.Result{}, perr
}
pending = res
}
}
// Fall through to updateStatus — the next reconcile will retry
// promotion. This matches scaleUp's "log and continue" behaviour
// when the etcd client can't be built.
}
// ── Enable auth once converged ─────────────────────────────────────
// Provision the root user/role and turn on authentication, but only on
// a healthy, fully-converged cluster (all desired members ready, quorum
// formed). Gating on convergence keeps the auth flip from racing in-
// flight scale-up dials. No-op (and skipped) once status.authEnabled
// has latched.
//
// Skip when a promote requeue is already pending. A pending promote means
// the just-finished MemberList/MemberPromote either found etcd unreachable
// or a learner not yet caught up. reconcileAuth's clientv3 calls
// (AuthStatus/UserAdd/UserGrantRole/AuthEnable) run on the raw reconcile
// context with clientv3's default WaitForReady and no per-call deadline, so
// dialing again against a dead endpoint would block the worker rather than
// error out. Deferring auth to the pass after promote clears avoids that.
if pending == nil {
if res, err := r.reconcileAuth(ctx, cluster, running); err != nil {
return ctrl.Result{}, err
} else if res != nil {
pending = res
}
}
// ── Steady state ───────────────────────────────────────────────────
// Pass the full active set (including any dormant member), not
// `running`: updateStatus's Paused-message branch derives the
// dormant member from the slice via findDormantMember to name the
// preserved PVC. Stripping dormant here would silently fall back
// to the fresh-zero "no data has been written" message even when a
// dormant member with a real PVC exists. updateStatus re-derives
// `running` internally for its accounting, so passing `active`
// here is the correct shape.
return r.updateStatus(ctx, cluster, active, pending)
}
// ── Bootstrap ────────────────────────────────────────────────────────────
// bootstrap brings up the single seed member for a fresh cluster. Its
// --initial-cluster lists only itself, so the etcd protocol cannot get
// confused by partial agreement; subsequent members join via MemberAdd
// once ClusterID is latched.
//
// Members are named via GenerateName ("<cluster>-"), not ordinal-suffixed,
// so seed identity is detected by Spec.Bootstrap=true rather than by a
// predictable name. Idempotency therefore looks for an existing
// Bootstrap=true CR before creating a new one. Cross-incarnation safety
// (a leftover member with this cluster's label but a different
// controller) is enforced explicitly — see comment below.
//
// The Create-then-Update sequence is required: the seed's --initial-cluster
// flag references its own (apiserver-assigned) name, so we can only fill
// Spec.InitialCluster *after* Create returns the assigned name. The member
// controller refuses to start a pod with an empty InitialCluster (see
// EtcdMemberReconciler.Reconcile), so the half-baked state between Create
// and Update doesn't reach the data plane.
func (r *EtcdClusterReconciler) bootstrap(
ctx context.Context,
cluster *lll.EtcdCluster,
members []lll.EtcdMember,
) (ctrl.Result, error) {
var seed *lll.EtcdMember
for i := range members {
if members[i].Spec.Bootstrap {
seed = &members[i]
break
}
}
if seed == nil {
seedLabels, seedAnnotations := applyAdditionalMetadata(clusterLabels(cluster.Name), nil, observedAdditionalMetadata(cluster))
seed = &lll.EtcdMember{
ObjectMeta: metav1.ObjectMeta{
GenerateName: cluster.Name + "-",
Namespace: cluster.Namespace,
Labels: seedLabels,
Annotations: seedAnnotations,
},
Spec: lll.EtcdMemberSpec{
ClusterName: cluster.Name,
Version: cluster.Status.Observed.Version,
Storage: cluster.Status.Observed.Storage,
Resources: cluster.Status.Observed.Resources,
AdditionalMetadata: cluster.Status.Observed.AdditionalMetadata,
Affinity: cluster.Status.Observed.Affinity,
TopologySpreadConstraints: cluster.Status.Observed.TopologySpreadConstraints,
Options: cluster.Status.Observed.Options,
ImagePullSecrets: cluster.Status.Observed.ImagePullSecrets,
Bootstrap: true,
ClusterToken: cluster.Status.ClusterToken,
TLS: deriveMemberTLS(cluster),
// Restore is carried only by the seed: when the cluster
// requests a restore, the member controller runs a restore
// initContainer that populates the data dir from the snapshot
// before etcd starts. Inert once the data dir is initialized.
Restore: restoreForSeed(cluster),
// InitialCluster filled in below once apiserver assigns Name.
},
}
if err := controllerutil.SetControllerReference(cluster, seed, r.Scheme); err != nil {
return ctrl.Result{}, err
}
if err := r.Create(ctx, seed); err != nil {
return ctrl.Result{}, err
}
// Pre-stamp IsVoter=true. The seed is never a learner — it forms
// the cluster on its own with --initial-cluster-state=new. Setting
// this here means the member controller applies the role=voter
// Pod label on the first reconcile of the seed Pod, so the PDB
// selects it from creation rather than only after MemberList runs
// a few reconciles later.
//
// Written as a merge patch, not an Update: the member controller
// reconciles the seed the moment it is created and its own
// Status().Update races with this write — an optimistic-locked
// Update here loses that race with a conflict, and a cache-backed
// re-Get cannot recover (the informer may not even have the
// object yet). This stamp happens only in the create branch —
// losing it would leave the seed permanently IsVoter=false, and
// both learner memberID discovery and promotion filter their dial
// endpoints to voters, wedging every later scale-up.
stampBase := seed.DeepCopy()
seed.Status.IsVoter = true
if err := r.Status().Patch(ctx, seed, client.MergeFrom(stampBase)); err != nil {
return ctrl.Result{}, err
}
} else if !metav1.IsControlledBy(seed, cluster) {
// Defence-in-depth. The primary safety net against cross-
// incarnation adoption is that GenerateName produces a fresh
// suffix on every Create, so we cannot collide with a stale
// seed left over from a deleted prior cluster. This explicit
// IsControlledBy check catches the residual cases: a manually
// kubectl-created EtcdMember carrying our cluster label, or an
// orphan whose owner reference was reaped without GC catching
// the dependent. Refuse to adopt rather than risk treating
// someone else's etcd member as ours.
return ctrl.Result{}, fmt.Errorf(
"bootstrap-flagged EtcdMember %q in namespace %q is not controlled by this EtcdCluster (uid=%s); "+
"refusing to adopt", seed.Name, seed.Namespace, cluster.UID)
}
if seed.Spec.InitialCluster == "" {
original := seed.DeepCopy()
seed.Spec.InitialCluster = buildInitialCluster(clusterPeerScheme(cluster), []string{seed.Name}, memberServiceName(seed), cluster.Namespace)
// Now that apiserver-assigned name is known, populate the
// per-member component label so the seed CR's label set matches
// the Pod/PVC the member controller will create.
if seed.Labels == nil {
seed.Labels = map[string]string{}
}
for k, v := range memberLabels(cluster.Name, seed.Name) {
seed.Labels[k] = v
}
if err := r.Patch(ctx, seed, client.MergeFrom(original)); err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
// ── Cluster discovery ────────────────────────────────────────────────────
func (r *EtcdClusterReconciler) tryDiscoverCluster(
ctx context.Context,
cluster *lll.EtcdCluster,
members []lll.EtcdMember,
) (ctrl.Result, error) {
log := log.FromContext(ctx)
// Find the seed by Spec.Bootstrap. Member names are apiserver-assigned
// via GenerateName, so the seed cannot be located by a predictable
// name; Spec.Bootstrap is set on (and only on) the single CR that
// bootstrap() created. apiserver does not guarantee any List ordering,
// so trusting members[0] would silently anchor discovery to the wrong
// member when scale-up CRs land in front of the seed.
var seed *lll.EtcdMember
for i := range members {
if members[i].Spec.Bootstrap {
seed = &members[i]
break
}
}
if seed == nil {
log.Info("waiting for seed member to appear before discovery")
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
// If the seed's Pod hasn't been created yet, dialing its DNS name just
// times out the reconcile budget for no signal. Surface the wait as
// Progressing=True/WaitingForSeed — distinct from
// Available=False/ClusterUnreachable (which is reserved for "we
// expected to dial something and couldn't"). Skips the Status write if
// the condition didn't move (e.g. we already wrote WaitingForSeed last
// reconcile).
if seed.Status.PodName == "" {
if setClusterCondition(cluster, lll.ClusterProgressing, metav1.ConditionTrue, "WaitingForSeed",
fmt.Sprintf("seed member %q has no Pod yet", seed.Name)) {
if upErr := r.Status().Update(ctx, cluster); upErr != nil {
if errors.IsConflict(upErr) {
return ctrl.Result{Requeue: true}, nil
}
return ctrl.Result{}, upErr
}
}
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
endpoints := []string{clientURL(clusterClientScheme(cluster), seed.Name, memberServiceName(seed), cluster.Namespace)}
tlsCfg, err := buildOperatorTLSConfig(ctx, r.Client, cluster)
if err != nil {
return r.surfaceDiscoveryError(ctx, cluster, "operator TLS config", err)
}
// Bootstrap-window dial: discovery only runs while ClusterID is unset,
// strictly before reconcileAuth can turn auth on, so we always dial
// anonymously here (resolveEtcdCredentials would return empty creds too,
// since status.authEnabled cannot be true yet).
etcdClient, err := r.EtcdClientFactory(ctx, endpoints, tlsCfg, "", "")
if err != nil {
return r.surfaceDiscoveryError(ctx, cluster, "etcd client construction failed", err)
}
defer etcdClient.Close()
listCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
resp, err := etcdClient.MemberList(listCtx)
if err != nil {
return r.surfaceDiscoveryError(ctx, cluster, "MemberList failed", err)
}
// Only latch ClusterID when the response unambiguously describes the
// bootstrap state. With single-member bootstrap that means: exactly
// one member, and its name matches the seed (or its peer URL does, if
// etcd hasn't yet propagated the name).
if len(resp.Members) != 1 {
log.Info("waiting for definitive single-member response", "members", len(resp.Members))
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
expectedPeer := peerURL(clusterPeerScheme(cluster), seed.Name, memberServiceName(seed), cluster.Namespace)
matched := resp.Members[0].Name == seed.Name
if !matched {
for _, p := range resp.Members[0].PeerURLs {
if p == expectedPeer {
matched = true
break
}
}
}
if !matched {
log.Info("MemberList returned an unexpected member; retrying",
"got_name", resp.Members[0].Name, "got_urls", resp.Members[0].PeerURLs)
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
// Discovery success. Latch ClusterID and clear any stale failure
// condition from earlier retries so the cluster doesn't sit
// Available=False until the next reconcile cycle.
cluster.Status.ClusterID = fmt.Sprintf("%016x", resp.Header.ClusterId)
setClusterCondition(cluster, lll.ClusterAvailable, metav1.ConditionTrue, "ClusterDiscovered",
fmt.Sprintf("etcd cluster %s discovered", cluster.Status.ClusterID))
if err := r.Status().Update(ctx, cluster); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
// surfaceDiscoveryError reports that the seed pod exists but we can't talk
// to it. Gates the Status write on whether the condition actually moved so
// we don't bump resourceVersion every 10 s while waiting for the etcd
// process to come up. Errors with embedded timestamps would defeat the
// short-circuit, so we strip the variable portion of common timeouts.
func (r *EtcdClusterReconciler) surfaceDiscoveryError(
ctx context.Context,
cluster *lll.EtcdCluster,
what string,
err error,
) (ctrl.Result, error) {
log := log.FromContext(ctx)
log.Error(err, what)
// etcd is demanding authentication but the operator has no credentials to
// present (spec.auth unset) and auth hasn't latched. This is the
// restore-an-auth-enabled-snapshot-without-spec.auth case (or auth enabled
// out-of-band): anonymous dials are rejected, and because spec.auth is
// immutable post-create no spec edit can recover it — retrying forever would
// just hide the cause behind a generic ClusterUnreachable. Surface a
// specific, actionable Degraded condition and stop the retry loop; recovery
// is delete-and-recreate with spec.auth, which yields a fresh object.
if isAuthRequiredErr(err) && !authConfigured(cluster) && !cluster.Status.AuthEnabled {
const reason = "AuthRequiredNotConfigured"
msg := "etcd requires authentication but spec.auth is not configured, so the operator cannot manage this cluster. " +
"This usually means a snapshot from an auth-enabled cluster was restored without spec.auth. " +
"Auth is immutable post-create: delete and recreate the cluster with spec.auth.enabled and a rootCredentialsSecretRef " +
"holding the snapshot's original root password (see the restore runbook)."
changed := setClusterCondition(cluster, lll.ClusterAvailable, metav1.ConditionFalse, reason, msg)
if setClusterCondition(cluster, lll.ClusterDegraded, metav1.ConditionTrue, reason, msg) {
changed = true
}
if changed {
if upErr := r.Status().Update(ctx, cluster); upErr != nil {
if errors.IsConflict(upErr) {
return ctrl.Result{Requeue: true}, nil
}
return ctrl.Result{}, upErr
}
}
return ctrl.Result{}, nil
}
// etcd rejected the credentials we presented (wrong root password) — the
// most likely restore mistake: spec.auth.rootCredentialsSecretRef must hold
// the snapshot's *original* root password (etcd stores its bcrypt hash). This
// is recoverable by fixing the Secret's contents (the operator re-reads the
// password every dial), so keep requeueing — but surface a specific reason
// rather than an opaque ClusterUnreachable so the operator knows the fix.
if isAuthFailedErr(err) {
const reason = "AuthCredentialsRejected"
msg := "etcd rejected the operator's credentials (authentication failed). " +
"spec.auth.rootCredentialsSecretRef must hold the root password etcd actually has — " +
"for a cluster restored from an auth-enabled snapshot that is the snapshot's original password " +
"(etcd stores its bcrypt hash, so a fresh password will not authenticate). Correct the Secret's password (see the restore runbook)."
changed := setClusterCondition(cluster, lll.ClusterAvailable, metav1.ConditionFalse, reason, msg)
if setClusterCondition(cluster, lll.ClusterDegraded, metav1.ConditionTrue, reason, msg) {
changed = true
}
if changed {
if upErr := r.Status().Update(ctx, cluster); upErr != nil {
if errors.IsConflict(upErr) {
return ctrl.Result{Requeue: true}, nil
}
return ctrl.Result{}, upErr
}
}
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}
msg := fmt.Sprintf("%s: %s", what, stableErrorMessage(err))
if setClusterCondition(cluster, lll.ClusterAvailable, metav1.ConditionFalse, "ClusterUnreachable", msg) {
if upErr := r.Status().Update(ctx, cluster); upErr != nil {
if errors.IsConflict(upErr) {
return ctrl.Result{Requeue: true}, nil
}
return ctrl.Result{}, upErr
}
}
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
}
// authConfigured reports whether the operator has root credentials to present
// on its etcd dials (spec.auth enabled). Distinguishes "etcd demands auth but
// we have nothing to offer" from a transient dial failure.
func authConfigured(cluster *lll.EtcdCluster) bool {
return cluster.Spec.Auth != nil && cluster.Spec.Auth.Enabled
}
// stableErrorMessage strips per-call variable portions (timestamps, addresses
// embedded in DNS-lookup errors, etc.) from common etcd-client error strings.
// Keeps the kind-of-error stable across retries so the condition message
// doesn't move on every reconcile.
func stableErrorMessage(err error) string {
s := err.Error()
// "lookup foo on 127.0.0.53:53: server misbehaving" -> "lookup foo: server misbehaving"
if i, j := strings.Index(s, " on "), strings.Index(s, ": "); i > 0 && j > i {
s = s[:i] + s[j:]
}
return s
}
// ── Scale up ─────────────────────────────────────────────────────────────
// scaleUp adds one member to the etcd cluster — or, if a previous
// reconcile crashed mid-flight, completes that earlier attempt. The
// crash-recovery story shapes the function's ordering: there are two
// dangerous windows, and the steps that close them must run before any
// step that could short-circuit on an unrelated condition.
//
// Flow:
//
// 1. Fetch etcd's MemberList. We use the same snapshot for every
// decision below to avoid races against a moving cluster view.
// 2. Adopt any pending EtcdMember (Spec.InitialCluster=="") from a prior
// reconcile. There are two pending sub-states:
// a. peer URL ALREADY in etcd (post-AddAsLearner crash): patch
// Spec.InitialCluster immediately. This must happen BEFORE any
// promotion attempt — the orphan learner cannot sync until its
// pod is up, the pod cannot start until InitialCluster is set,
// and promotion would block forever on the un-synced learner.
// b. peer URL NOT yet in etcd (pre-AddAsLearner crash): retry
// MemberAddAsLearner, then patch.
// 3. With no pending CR, try to promote a learner (the iteration after
// a successful AddAsLearner + pod-up + sync).
// 4. If no learner was promotable, Create a fresh CR with GenerateName,
// MemberAddAsLearner, then patch.
//
// Until the patch in step 2a, 2b, or 4 completes, the member controller
// refuses to start a pod (see EtcdMemberReconciler.Reconcile) — keeping
// half-baked state out of the data plane. The finalizer is added by
// the member controller before that gate, so a mid-flight delete still
// triggers MemberRemove cleanup of the registered learner.
func (r *EtcdClusterReconciler) scaleUp(
ctx context.Context,
cluster *lll.EtcdCluster,
members []lll.EtcdMember,
) (ctrl.Result, error) {
log := log.FromContext(ctx)
// Wake from dormant: if a dormant member exists, flip its
// Spec.Dormant flag back to false. The member controller's
// reconcile loop then ensures the Pod is back up against the same
// PVC (which never lost its EtcdMember owner-ref). Same ClusterID,
// same member ID, same data — etcd resumes from the existing data
// dir. No name lookup, no Create-by-fixed-name, no foreign-CR
// adoption window to defend against.
if dormant := findDormantMember(members); dormant != nil {
log.Info("waking dormant member", "name", dormant.Name)
original := dormant.DeepCopy()
dormant.Spec.Dormant = false
if err := r.Patch(ctx, dormant, client.MergeFrom(original)); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
// `members` passed in here is the active set (including any dormant
// CR, which we just handled). From this point on we only care about
// running (non-dormant) members for the etcd-side flow.
running := filterRunningMembers(members)
endpoints := memberEndpoints(clusterClientScheme(cluster), running, cluster.Namespace)
tlsCfg, err := buildOperatorTLSConfig(ctx, r.Client, cluster)
if err != nil {
log.Error(err, "cannot build operator TLS config for scale-up")
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
}
user, pass, _, err := resolveEtcdCredentials(ctx, r.Client, cluster)
if err != nil {
log.Error(err, "cannot resolve etcd credentials for scale-up")
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
}
etcdClient, err := r.EtcdClientFactory(ctx, endpoints, tlsCfg, user, pass)
if err != nil {
log.Error(err, "cannot connect to etcd for scale-up", "endpoints", endpoints)
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
}
defer etcdClient.Close()
// Step 1: snapshot etcd's member list once.
listCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
listResp, err := etcdClient.MemberList(listCtx)
if err != nil {
log.Error(err, "MemberList failed", "endpoints", endpoints)
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
}
// Step 2: handle a pending CR if one exists. Defence-in-depth: see
// the comment in bootstrap() — GenerateName precludes name-collision
// adoption, but a manually-created CR or a label-matched orphan
// could in principle slip through; refuse rather than adopt.
pending := findPendingMember(running)
if pending != nil {
if !metav1.IsControlledBy(pending, cluster) {
return ctrl.Result{}, fmt.Errorf(
"pending EtcdMember %q in namespace %q is not controlled by this EtcdCluster (uid=%s); "+
"refusing to adopt", pending.Name, pending.Namespace, cluster.UID)
}
return r.completePendingMember(ctx, cluster, etcdClient, listResp, pending)
}
// Step 3: no pending CR. Try to promote any learner that's caught up.
if res, err := tryPromoteLearner(ctx, etcdClient, listResp); err != nil || res != nil {
return resultOrZero(res), err
}
// Step 4: no learner waiting. Create a fresh CR, AddAsLearner, patch.
mLabels, mAnnotations := applyAdditionalMetadata(clusterLabels(cluster.Name), nil, observedAdditionalMetadata(cluster))
newMember := &lll.EtcdMember{
ObjectMeta: metav1.ObjectMeta{
GenerateName: cluster.Name + "-",
Namespace: cluster.Namespace,
Labels: mLabels,
Annotations: mAnnotations,
},
Spec: lll.EtcdMemberSpec{
ClusterName: cluster.Name,
Version: cluster.Status.Observed.Version,
Storage: cluster.Status.Observed.Storage,
Resources: cluster.Status.Observed.Resources,
AdditionalMetadata: cluster.Status.Observed.AdditionalMetadata,
Affinity: cluster.Status.Observed.Affinity,
TopologySpreadConstraints: cluster.Status.Observed.TopologySpreadConstraints,
Options: cluster.Status.Observed.Options,
ImagePullSecrets: cluster.Status.Observed.ImagePullSecrets,
Bootstrap: false,
ClusterToken: cluster.Status.ClusterToken,
TLS: deriveMemberTLS(cluster),
// InitialCluster filled in by completePendingMember below.
},
}
if err := controllerutil.SetControllerReference(cluster, newMember, r.Scheme); err != nil {
return ctrl.Result{}, err
}
if err := r.Create(ctx, newMember); err != nil {
return ctrl.Result{}, err
}
return r.completePendingMember(ctx, cluster, etcdClient, listResp, newMember)
}
// completePendingMember finishes the AddAsLearner + Spec.InitialCluster
// patch for `pending`. It is idempotent in both directions of the crash
// window:
//
// - If `pending`'s peer URL is already in etcd's MemberList (passed in
// as `listResp`), MemberAddAsLearner is skipped and the patch uses
// the existing list.
// - Otherwise MemberAddAsLearner is called and the patch uses the
// resulting member list from the add response.
//
// Either way, after this call the CR has Spec.InitialCluster set and the
// member controller can start the pod.
func (r *EtcdClusterReconciler) completePendingMember(
ctx context.Context,
cluster *lll.EtcdCluster,
etcdClient EtcdClusterClient,
listResp *clientv3.MemberListResponse,
pending *lll.EtcdMember,
) (ctrl.Result, error) {
log := log.FromContext(ctx)
pendingPeerURL := peerURL(clusterPeerScheme(cluster), pending.Name, memberServiceName(pending), cluster.Namespace)
alreadyAdded := false
for _, m := range listResp.Members {
for _, p := range m.PeerURLs {
if p == pendingPeerURL {
alreadyAdded = true
break
}
}
if alreadyAdded {
break
}
}
etcdMembers := listResp.Members
if !alreadyAdded {
addCtx, addCancel := context.WithTimeout(ctx, 10*time.Second)
defer addCancel()
addResp, err := etcdClient.MemberAddAsLearner(addCtx, []string{pendingPeerURL})
if err != nil {
log.Error(err, "MemberAddAsLearner failed", "name", pending.Name)
return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
}
log.Info("added member as learner", "name", pending.Name)
etcdMembers = addResp.Members
}
// Build --initial-cluster from etcd's view, using each member's peer URL
// VERBATIM rather than reconstructing it from a single cluster Service
// name. During an in-place migration adopted members carry peer URLs
// under the legacy headless Service while the joining (native) member
// uses the cluster name, so a reconstruction keyed off one Service name
// would emit the wrong DNS for half the cluster. etcd already reports the
// correct persisted peer URL for every member (and the one we just
// registered via pendingPeerURL), so echo it back.
parts := make([]string, 0, len(etcdMembers))
for _, m := range etcdMembers {
if len(m.PeerURLs) == 0 {
continue
}
name := m.Name
if name == "" {
name = memberNameFromPeerURL(m.PeerURLs[0])
}
if name == "" {
continue
}
parts = append(parts, name+"="+m.PeerURLs[0])
}
sort.Strings(parts)
initialCluster := strings.Join(parts, ",")
original := pending.DeepCopy()
pending.Spec.InitialCluster = initialCluster
// Restore the per-member component label now that the apiserver-
// assigned name is known, so the CR's label set matches the Pod/PVC
// that the member controller will create.
if pending.Labels == nil {
pending.Labels = map[string]string{}
}
for k, v := range memberLabels(cluster.Name, pending.Name) {
pending.Labels[k] = v
}
if err := r.Patch(ctx, pending, client.MergeFrom(original)); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
// tryPromoteLearner promotes the first learner found in `listResp` (if
// any). Returns (&Result, nil) when a promotion happened or a learner
// is not yet caught up; (nil, nil) when there is no learner at all.
// Errors out only on apiserver-style MemberPromote failures.
func tryPromoteLearner(
ctx context.Context,
etcdClient EtcdClusterClient,
listResp *clientv3.MemberListResponse,
) (*ctrl.Result, error) {
log := log.FromContext(ctx)
for _, m := range listResp.Members {
if !m.IsLearner {
continue
}
promoteCtx, pCancel := context.WithTimeout(ctx, 10*time.Second)
_, perr := etcdClient.MemberPromote(promoteCtx, m.ID)
pCancel()
if perr != nil {
log.Info("learner not yet promotable; will retry",
"learner_id", fmt.Sprintf("%016x", m.ID), "err", perr.Error())
return &ctrl.Result{RequeueAfter: 5 * time.Second}, nil
}
log.Info("promoted learner", "learner_id", fmt.Sprintf("%016x", m.ID))
return &ctrl.Result{Requeue: true}, nil
}
return nil, nil
}
// findPendingMember returns the first non-bootstrap CR with an empty
// Spec.InitialCluster (the half-baked state between Create and the
// follow-up Patch). Returns nil if none.
func findPendingMember(members []lll.EtcdMember) *lll.EtcdMember {
for i := range members {
if !members[i].Spec.Bootstrap && members[i].Spec.InitialCluster == "" {
return &members[i]