Visitar URL original
fix: Improve error messaging · feast-dev/feast@c9be7aa · GitHub
Skip to content

Commit c9be7aa

Browse files
committed
fix: Improve error messaging
Signed-off-by: ntkathole <nikhilkathole2683@gmail.com>
1 parent 61c77a5 commit c9be7aa

3 files changed

Lines changed: 191 additions & 3 deletions

File tree

‎infra/feast-operator/internal/controller/featurestore_controller.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,11 +196,15 @@ func (r *FeatureStoreReconciler) deployFeast(ctx context.Context, cr *feastdevv1
196196
} else {
197197
isDeployAvailable := services.IsDeploymentAvailable(deployment.Status.Conditions)
198198
if !isDeployAvailable {
199+
msg := feastdevv1.DeploymentNotAvailableMessage
200+
if podMsg := feast.GetPodContainerFailureMessage(deployment); podMsg != "" {
201+
msg = msg + ": " + podMsg
202+
}
199203
condition = metav1.Condition{
200204
Type: feastdevv1.ReadyType,
201205
Status: metav1.ConditionUnknown,
202206
Reason: feastdevv1.DeploymentNotAvailableReason,
203-
Message: feastdevv1.DeploymentNotAvailableMessage,
207+
Message: msg,
204208
}
205209

206210
result = errResult

‎infra/feast-operator/internal/controller/services/services.go‎

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -733,8 +733,9 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64
733733
Command: []string{"feast", "apply"},
734734
WorkingDir: featureRepoDir,
735735
}
736-
// feast apply needs DB/store connectivity, so inherit env/envFrom
737-
// from all server container configs (registry, online, offline).
736+
// feast apply needs DB/store connectivity, so inherit env, envFrom
737+
// and volume mounts from all server container configs.
738+
seen := map[string]bool{}
738739
for _, feastType := range []FeastServiceType{RegistryFeastType, OnlineFeastType, OfflineFeastType} {
739740
if serverConfigs := feast.getServerConfigs(feastType); serverConfigs != nil {
740741
if serverConfigs.OptionalCtrConfigs.Env != nil {
@@ -743,6 +744,12 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64
743744
if serverConfigs.OptionalCtrConfigs.EnvFrom != nil {
744745
applyContainer.EnvFrom = append(applyContainer.EnvFrom, *serverConfigs.OptionalCtrConfigs.EnvFrom...)
745746
}
747+
for _, vm := range feast.getVolumeMounts(feastType) {
748+
if !seen[vm.MountPath] {
749+
applyContainer.VolumeMounts = append(applyContainer.VolumeMounts, vm)
750+
seen[vm.MountPath] = true
751+
}
752+
}
746753
}
747754
}
748755
podSpec.InitContainers = append(podSpec.InitContainers, applyContainer)
@@ -1427,6 +1434,67 @@ func IsDeploymentAvailable(conditions []appsv1.DeploymentCondition) bool {
14271434
return false
14281435
}
14291436

1437+
// GetPodContainerFailureMessage inspects pods belonging to the given deployment
1438+
// and returns a human-readable message describing the first init or regular
1439+
// container that is in a failing state. Returns empty string if no failure found.
1440+
func (feast *FeastServices) GetPodContainerFailureMessage(deploy appsv1.Deployment) string {
1441+
podList := corev1.PodList{}
1442+
labels := feast.getLabels()
1443+
if err := feast.Handler.Client.List(feast.Handler.Context, &podList,
1444+
client.InNamespace(deploy.Namespace),
1445+
client.MatchingLabels(labels),
1446+
); err != nil {
1447+
return ""
1448+
}
1449+
for i := range podList.Items {
1450+
pod := &podList.Items[i]
1451+
if msg := initContainerFailureMessage(pod); msg != "" {
1452+
return msg
1453+
}
1454+
if msg := containerFailureMessage(pod); msg != "" {
1455+
return msg
1456+
}
1457+
}
1458+
return ""
1459+
}
1460+
1461+
func initContainerFailureMessage(pod *corev1.Pod) string {
1462+
for _, cs := range pod.Status.InitContainerStatuses {
1463+
if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" && cs.State.Waiting.Reason != "PodInitializing" {
1464+
return "Init container '" + cs.Name + "' waiting: " + cs.State.Waiting.Reason +
1465+
messageIfPresent(cs.State.Waiting.Message)
1466+
}
1467+
if cs.State.Terminated != nil && cs.State.Terminated.ExitCode != 0 {
1468+
return "Init container '" + cs.Name + "' failed with exit code " +
1469+
strconv.Itoa(int(cs.State.Terminated.ExitCode)) +
1470+
messageIfPresent(cs.State.Terminated.Message)
1471+
}
1472+
}
1473+
return ""
1474+
}
1475+
1476+
func containerFailureMessage(pod *corev1.Pod) string {
1477+
for _, cs := range pod.Status.ContainerStatuses {
1478+
if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" && cs.State.Waiting.Reason != "ContainerCreating" {
1479+
return "Container '" + cs.Name + "' waiting: " + cs.State.Waiting.Reason +
1480+
messageIfPresent(cs.State.Waiting.Message)
1481+
}
1482+
if cs.State.Terminated != nil && cs.State.Terminated.ExitCode != 0 {
1483+
return "Container '" + cs.Name + "' failed with exit code " +
1484+
strconv.Itoa(int(cs.State.Terminated.ExitCode)) +
1485+
messageIfPresent(cs.State.Terminated.Message)
1486+
}
1487+
}
1488+
return ""
1489+
}
1490+
1491+
func messageIfPresent(msg string) string {
1492+
if msg != "" {
1493+
return " - " + msg
1494+
}
1495+
return ""
1496+
}
1497+
14301498
// GetFeastRestServiceName returns the feast REST service object name based on service type
14311499
func (feast *FeastServices) GetFeastRestServiceName(feastType FeastServiceType) string {
14321500
return feast.GetFeastServiceName(feastType) + "-rest"

‎infra/feast-operator/internal/controller/services/services_test.go‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -573,3 +573,119 @@ var _ = Describe("Registry Service", func() {
573573
})
574574
})
575575
})
576+
577+
var _ = Describe("Pod Container Failure Messages", func() {
578+
It("should detect init container in CrashLoopBackOff", func() {
579+
pod := &corev1.Pod{
580+
Status: corev1.PodStatus{
581+
InitContainerStatuses: []corev1.ContainerStatus{
582+
{
583+
Name: "feast-init",
584+
State: corev1.ContainerState{
585+
Terminated: &corev1.ContainerStateTerminated{ExitCode: 0},
586+
},
587+
},
588+
{
589+
Name: "feast-apply",
590+
State: corev1.ContainerState{
591+
Waiting: &corev1.ContainerStateWaiting{
592+
Reason: "CrashLoopBackOff",
593+
Message: "back-off 5m0s restarting failed container",
594+
},
595+
},
596+
},
597+
},
598+
},
599+
}
600+
msg := initContainerFailureMessage(pod)
601+
Expect(msg).To(ContainSubstring("feast-apply"))
602+
Expect(msg).To(ContainSubstring("CrashLoopBackOff"))
603+
Expect(msg).To(ContainSubstring("back-off 5m0s"))
604+
})
605+
606+
It("should detect init container terminated with non-zero exit code", func() {
607+
pod := &corev1.Pod{
608+
Status: corev1.PodStatus{
609+
InitContainerStatuses: []corev1.ContainerStatus{
610+
{
611+
Name: "feast-apply",
612+
State: corev1.ContainerState{
613+
Terminated: &corev1.ContainerStateTerminated{
614+
ExitCode: 1,
615+
Message: "feast apply failed",
616+
},
617+
},
618+
},
619+
},
620+
},
621+
}
622+
msg := initContainerFailureMessage(pod)
623+
Expect(msg).To(ContainSubstring("feast-apply"))
624+
Expect(msg).To(ContainSubstring("exit code 1"))
625+
Expect(msg).To(ContainSubstring("feast apply failed"))
626+
})
627+
628+
It("should return empty for init containers still initializing", func() {
629+
pod := &corev1.Pod{
630+
Status: corev1.PodStatus{
631+
InitContainerStatuses: []corev1.ContainerStatus{
632+
{
633+
Name: "feast-init",
634+
State: corev1.ContainerState{
635+
Waiting: &corev1.ContainerStateWaiting{
636+
Reason: "PodInitializing",
637+
},
638+
},
639+
},
640+
},
641+
},
642+
}
643+
Expect(initContainerFailureMessage(pod)).To(BeEmpty())
644+
})
645+
646+
It("should detect regular container failure", func() {
647+
pod := &corev1.Pod{
648+
Status: corev1.PodStatus{
649+
ContainerStatuses: []corev1.ContainerStatus{
650+
{
651+
Name: "registry",
652+
State: corev1.ContainerState{
653+
Waiting: &corev1.ContainerStateWaiting{
654+
Reason: "ImagePullBackOff",
655+
Message: "image not found",
656+
},
657+
},
658+
},
659+
},
660+
},
661+
}
662+
msg := containerFailureMessage(pod)
663+
Expect(msg).To(ContainSubstring("registry"))
664+
Expect(msg).To(ContainSubstring("ImagePullBackOff"))
665+
})
666+
667+
It("should return empty for healthy pods", func() {
668+
pod := &corev1.Pod{
669+
Status: corev1.PodStatus{
670+
InitContainerStatuses: []corev1.ContainerStatus{
671+
{
672+
Name: "feast-init",
673+
State: corev1.ContainerState{
674+
Terminated: &corev1.ContainerStateTerminated{ExitCode: 0},
675+
},
676+
},
677+
},
678+
ContainerStatuses: []corev1.ContainerStatus{
679+
{
680+
Name: "registry",
681+
State: corev1.ContainerState{
682+
Running: &corev1.ContainerStateRunning{},
683+
},
684+
},
685+
},
686+
},
687+
}
688+
Expect(initContainerFailureMessage(pod)).To(BeEmpty())
689+
Expect(containerFailureMessage(pod)).To(BeEmpty())
690+
})
691+
})

0 commit comments

Comments
 (0)