Recreate firewall on unhealthy condition

README.md

@@ -51,4 +51,4 @@ To play with the FCM, you can also run this controller inside the [mini-lab](htt
 1. Deploy the FCM into the mini-lab with `make deploy`
 1. Adapt the example [firewalldeployment.yaml](config/examples/firewalldeployment.yaml) and apply with `kubectl apply -f config/examples/firewalldeployment.yaml`
 1. Note that the firewall-controller will not be able to connect to the mini-lab due to network restrictions, so the firewall will not get ready.
-   - You can make the firewall become ready anyway by setting the annotation `kubectl annotate fw <fw-nsme> firewall.metal-stack.io/no-controller-connection=true`
+   - You can make the firewall become ready anyway by setting the annotation `kubectl annotate fw <fw-nsme> -n firewall firewall.metal-stack.io/no-controller-connection=true`

api/v2/config/controller.go

@@ -182,10 +182,10 @@ func (c *NewControllerConfig) validate() error {
 	if c.ProgressDeadline <= 0 {
 		return fmt.Errorf("progress deadline must be specified")
 	}
-	if c.FirewallHealthTimeout <= 0 {
+	if c.FirewallHealthTimeout < 0 {
 		return fmt.Errorf("firewall health timeout must be specified")
 	}
-	if c.CreateTimeout <= 0 {
+	if c.CreateTimeout < 0 {
 		return fmt.Errorf("create timeout must be specified")
 	}
 

api/v2/types_firewall.go

@@ -1,8 +1,10 @@
 package v2
 
 import (
+	"fmt"
 	"sort"
 	"strconv"
+	"time"
 
 	"github.com/metal-stack/metal-lib/pkg/pointer"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -183,6 +185,9 @@ const (
 	FirewallMonitorDeployed ConditionType = "MonitorDeployed"
 	// FirewallDistanceConfigured indicates that the firewall-controller has configured the given firewall distance.
 	FirewallDistanceConfigured ConditionType = "Distance"
+	// FirewallProvisioned indicates that all health conditions have been met at least once.
+	// Once set to true, it stays true and is used to detect condition degradation.
+	FirewallProvisioned ConditionType = "Provisioned"
 )
 
 // ShootAccess contains secret references to construct a shoot client in the firewall-controller to update its firewall monitor.
@@ -351,3 +356,169 @@ func SortFirewallsByImportance(fws []*Firewall) {
 		return !a.CreationTimestamp.Before(&b.CreationTimestamp)
 	})
 }
+
+type (
+	FirewallStatusResult string
+
+	FirewallStatusEvalResult struct {
+		Result    FirewallStatusResult
+		Reason    string
+		TimeoutIn *time.Duration
+	}
+)
+
+const (
+	FirewallStatusReady         FirewallStatusResult = "ready"
+	FirewallStatusProgressing   FirewallStatusResult = "progressing"
+	FirewallStatusUnhealthy     FirewallStatusResult = "unhealthy"
+	FirewallStatusHealthTimeout FirewallStatusResult = "health-timeout"
+	FirewallStatusCreateTimeout FirewallStatusResult = "create-timeout"
+)
+
+func EvaluateFirewallStatus(fw *Firewall, createTimeout, healthTimeout time.Duration) *FirewallStatusEvalResult {
+	var (
+		checkForTimeout = func(fw *Firewall, condition ConditionType, timeout time.Duration) (time.Duration, bool) {
+			if timeout == 0 {
+				return 0, false
+			}
+
+			var (
+				cond           = pointer.SafeDeref(fw.Status.Conditions.Get(condition))
+				transitionTime = cond.LastTransitionTime.Time
+				deadline       = time.Until(transitionTime.Add(timeout))
+			)
+
+			if deadline < 0 {
+				return 0, true
+			}
+
+			return deadline, false
+		}
+
+		collectUnhealthyConditions = func(cts ...ConditionType) []*Condition {
+			var res []*Condition
+
+			for _, ct := range cts {
+				cond := fw.Status.Conditions.Get(ct)
+				if cond == nil {
+					res = append(res, &Condition{Type: ct})
+				} else if cond.Status != ConditionTrue {
+					res = append(res, cond)
+				}
+			}
+
+			return res
+		}
+
+		unhealthyTypes []string
+		timeoutIn      *time.Duration
+	)
+
+	switch fw.Status.Phase {
+	case FirewallPhaseCreating, FirewallPhaseCrashing:
+		unhealthyConds := collectUnhealthyConditions(
+			FirewallCreated,
+			FirewallReady,
+			FirewallProvisioned,
+		)
+
+		if len(unhealthyConds) == 0 {
+			return &FirewallStatusEvalResult{
+				Result: FirewallStatusReady,
+				Reason: "",
+			}
+		}
+
+		if createTimeout > 0 {
+			if t, ok := checkForTimeout(fw, FirewallReady, createTimeout); ok {
+				return &FirewallStatusEvalResult{
+					Result: FirewallStatusCreateTimeout,
+					Reason: fmt.Sprintf("%s create timeout exceeded, firewall not provisioned in time", createTimeout.String()),
+				}
+			} else if createTimeout != 0 {
+				timeoutIn = &t
+			}
+		}
+
+		for _, c := range unhealthyConds {
+			unhealthyTypes = append(unhealthyTypes, string(c.Type))
+		}
+
+		return &FirewallStatusEvalResult{
+			Result:    FirewallStatusProgressing,
+			Reason:    fmt.Sprintf("not all health conditions are true: %v", unhealthyTypes),
+			TimeoutIn: timeoutIn,
+		}
+
+	case FirewallPhaseRunning:
+		fallthrough
+
+	default:
+		unhealthyConds := collectUnhealthyConditions(
+			FirewallCreated,
+			FirewallReady,
+			FirewallProvisioned,
+			FirewallControllerConnected,
+			FirewallControllerSeedConnected,
+			FirewallDistanceConfigured,
+		)
+
+		if len(unhealthyConds) == 0 {
+			return &FirewallStatusEvalResult{
+				Result: FirewallStatusReady,
+				Reason: "",
+			}
+		}
+
+		var (
+			ready         = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallReady)).Status == ConditionTrue
+			provisioned   = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallProvisioned)).Status == ConditionTrue
+			connected     = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallControllerConnected)).Status == ConditionTrue
+			seedConnected = pointer.SafeDeref(fw.Status.Conditions.Get(FirewallControllerSeedConnected)).Status == ConditionTrue
+		)
+
+		if provisioned {
+			switch {
+			case !seedConnected:
+				if t, ok := checkForTimeout(fw, FirewallControllerSeedConnected, healthTimeout); ok {
+					return &FirewallStatusEvalResult{
+						Result: FirewallStatusHealthTimeout,
+						Reason: fmt.Sprintf("%s health timeout exceeded, seed connection lost", healthTimeout.String()),
+					}
+				} else if healthTimeout != 0 {
+					timeoutIn = &t
+				}
+
+			case !connected:
+				if t, ok := checkForTimeout(fw, FirewallControllerConnected, healthTimeout); ok {
+					return &FirewallStatusEvalResult{
+						Result: FirewallStatusHealthTimeout,
+						Reason: fmt.Sprintf("%s health timeout exceeded, firewall monitor not reconciled anymore", healthTimeout.String()),
+					}
+				} else if healthTimeout != 0 {
+					timeoutIn = &t
+				}
+
+			case !ready:
+				if t, ok := checkForTimeout(fw, FirewallReady, healthTimeout); ok {
+					return &FirewallStatusEvalResult{
+						Result: FirewallStatusHealthTimeout,
+						Reason: fmt.Sprintf("%s health timeout exceeded, firewall is not ready from perspective of the metal-api", healthTimeout.String()),
+					}
+				} else if healthTimeout != 0 {
+					timeoutIn = &t
+				}
+			}
+		}
+
+		for _, c := range unhealthyConds {
+			unhealthyTypes = append(unhealthyTypes, string(c.Type))
+		}
+
+		return &FirewallStatusEvalResult{
+			Result:    FirewallStatusUnhealthy,
+			Reason:    fmt.Sprintf("not all health conditions are true: %v", unhealthyTypes),
+			TimeoutIn: timeoutIn,
+		}
+	}
+}