diff --git a/cmd/dataprotection/main.go b/cmd/dataprotection/main.go
index 4d7cc2c09ca..c28c7004d86 100644
--- a/cmd/dataprotection/main.go
+++ b/cmd/dataprotection/main.go
@@ -340,6 +340,14 @@ func main() {
os.Exit(1)
}
+ if err = (&dpcontrollers.ClusterRestoreReconciler{
+ Client: mgr.GetClient(),
+ Recorder: mgr.GetEventRecorderFor("cluster-restore-controller"),
+ }).SetupWithManager(mgr); err != nil {
+ setupLog.Error(err, "unable to create controller", "controller", "ClusterRestore")
+ os.Exit(1)
+ }
+
if err = (&dpcontrollers.BackupScheduleReconciler{
Client: dputils.NewCompatClient(mgr.GetClient()),
Scheme: mgr.GetScheme(),
diff --git a/controllers/dataprotection/cluster_restore_controller.go b/controllers/dataprotection/cluster_restore_controller.go
new file mode 100644
index 00000000000..304a7b4bc82
--- /dev/null
+++ b/controllers/dataprotection/cluster_restore_controller.go
@@ -0,0 +1,198 @@
+/*
+Copyright (C) 2022-2026 ApeCloud Co., Ltd
+
+This file is part of KubeBlocks project
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+*/
+
+package dataprotection
+
+import (
+ "context"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/meta"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/tools/record"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+ "sigs.k8s.io/controller-runtime/pkg/handler"
+ "sigs.k8s.io/controller-runtime/pkg/log"
+ "sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+ appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
+ dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1"
+ "github.com/apecloud/kubeblocks/pkg/constant"
+ intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
+ dprestore "github.com/apecloud/kubeblocks/pkg/dataprotection/restore"
+ dptypes "github.com/apecloud/kubeblocks/pkg/dataprotection/types"
+)
+
+// ClusterRestoreReconciler coordinates the Cluster-level restore lifecycle.
+type ClusterRestoreReconciler struct {
+ client.Client
+ Recorder record.EventRecorder
+}
+
+// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=clusters,verbs=get;list;watch;patch;update
+// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=clusters/finalizers,verbs=update;patch
+// +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=restores,verbs=get;list;watch
+// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch
+
+func (r *ClusterRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+ reqCtx := intctrlutil.RequestCtx{
+ Ctx: ctx, Req: req,
+ Log: log.FromContext(ctx).WithValues("cluster-restore", req.NamespacedName),
+ Recorder: r.Recorder,
+ }
+ cluster := &appsv1.Cluster{}
+ if err := r.Client.Get(ctx, req.NamespacedName, cluster); err != nil {
+ return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "")
+ }
+
+ restoring, err := r.isClusterRestoring(ctx, cluster)
+ if err != nil {
+ return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "failed to determine Cluster restore state")
+ }
+ if !restoring {
+ return r.releaseClusterRestoreProtection(reqCtx, cluster)
+ }
+ if !isClusterRestoreProtected(cluster) {
+ // If deletion began without our finalizer, this controller cannot acquire
+ // protection and must not delay Cluster deletion.
+ if !cluster.DeletionTimestamp.IsZero() {
+ return intctrlutil.Reconciled()
+ }
+ return r.protectClusterRestore(reqCtx, cluster)
+ }
+ if !cluster.DeletionTimestamp.IsZero() {
+ return intctrlutil.RequeueAfter(reconcileInterval, reqCtx.Log,
+ "waiting for restore owners to finish Cluster termination")
+ }
+ return intctrlutil.Reconciled()
+}
+
+func (r *ClusterRestoreReconciler) SetupWithManager(mgr ctrl.Manager) error {
+ return intctrlutil.NewControllerManagedBy(mgr).
+ For(&appsv1.Cluster{}).
+ Watches(&dpv1alpha1.Restore{}, handler.EnqueueRequestsFromMapFunc(r.mapObjectToCluster)).
+ Watches(&corev1.PersistentVolumeClaim{}, handler.EnqueueRequestsFromMapFunc(r.mapObjectToCluster)).
+ Complete(r)
+}
+
+func (r *ClusterRestoreReconciler) mapObjectToCluster(_ context.Context, obj client.Object) []reconcile.Request {
+ clusterName := obj.GetLabels()[constant.AppInstanceLabelKey]
+ if clusterName == "" {
+ return nil
+ }
+ return []reconcile.Request{{NamespacedName: client.ObjectKey{Namespace: obj.GetNamespace(), Name: clusterName}}}
+}
+
+func (r *ClusterRestoreReconciler) isClusterRestoring(ctx context.Context,
+ cluster *appsv1.Cluster) (bool, error) {
+ if cluster.DeletionTimestamp.IsZero() && clusterAllowsRestoreProgress(cluster) {
+ return true, nil
+ }
+
+ // Inspect PVCs before Restores. VP releases temporary target protection only
+ // after observing postReady Restore, so this order closes the handoff window.
+ pvcs := &corev1.PersistentVolumeClaimList{}
+ if err := r.Client.List(ctx, pvcs, client.InNamespace(cluster.Namespace), client.MatchingLabels{
+ constant.AppInstanceLabelKey: cluster.Name,
+ }); err != nil {
+ return false, err
+ }
+ for i := range pvcs.Items {
+ pvc := &pvcs.Items[i]
+ if pvc.Labels[dptypes.ClusterUIDLabelKey] != string(cluster.UID) {
+ continue
+ }
+ if isClusterRestoreHelperPVC(pvc) ||
+ (isClusterRestoreTargetPVC(pvc) && controllerutil.ContainsFinalizer(pvc, dptypes.DataProtectionFinalizerName)) {
+ return true, nil
+ }
+ }
+
+ restores := &dpv1alpha1.RestoreList{}
+ if err := r.Client.List(ctx, restores, client.InNamespace(cluster.Namespace), client.MatchingLabels{
+ constant.AppInstanceLabelKey: cluster.Name,
+ }); err != nil {
+ return false, err
+ }
+ for i := range restores.Items {
+ restore := &restores.Items[i]
+ if restore.Labels[dprestore.DataProtectionRestoreLabelKey] != restore.Name {
+ continue
+ }
+ owned := restore.Labels[dptypes.ClusterUIDLabelKey] == string(cluster.UID)
+ terminal := restore.Status.Phase == dpv1alpha1.RestorePhaseCompleted ||
+ restore.Status.Phase == dpv1alpha1.RestorePhaseFailed
+ if owned && (!cluster.DeletionTimestamp.IsZero() || !terminal || !restore.DeletionTimestamp.IsZero()) {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
+func clusterAllowsRestoreProgress(cluster *appsv1.Cluster) bool {
+ if cluster.Spec.Restore == nil {
+ return false
+ }
+ condition := meta.FindStatusCondition(cluster.Status.Conditions, appsv1.ConditionTypeRestore)
+ // Restore=False is terminal for status aggregation, but the Cluster still
+ // has initial-restore intent. Keep the lifecycle active until deletion so
+ // PVC restores cannot lose protection while converging on the failure.
+ return condition == nil || condition.Status != metav1.ConditionTrue
+}
+
+func isClusterRestoreProtected(cluster *appsv1.Cluster) bool {
+ return controllerutil.ContainsFinalizer(cluster, dptypes.RestoreProtectionFinalizerName)
+}
+
+func (r *ClusterRestoreReconciler) protectClusterRestore(reqCtx intctrlutil.RequestCtx,
+ cluster *appsv1.Cluster) (ctrl.Result, error) {
+ if isClusterRestoreProtected(cluster) {
+ return intctrlutil.Reconciled()
+ }
+ patch := client.MergeFromWithOptions(cluster.DeepCopy(), client.MergeFromWithOptimisticLock{})
+ controllerutil.AddFinalizer(cluster, dptypes.RestoreProtectionFinalizerName)
+ if err := r.Client.Patch(reqCtx.Ctx, cluster, patch); err != nil {
+ return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "failed to add Cluster restore-protection finalizer")
+ }
+ return intctrlutil.Reconciled()
+}
+
+func (r *ClusterRestoreReconciler) releaseClusterRestoreProtection(reqCtx intctrlutil.RequestCtx,
+ cluster *appsv1.Cluster) (ctrl.Result, error) {
+ if !isClusterRestoreProtected(cluster) {
+ return intctrlutil.Reconciled()
+ }
+ patch := client.MergeFromWithOptions(cluster.DeepCopy(), client.MergeFromWithOptimisticLock{})
+ controllerutil.RemoveFinalizer(cluster, dptypes.RestoreProtectionFinalizerName)
+ if err := r.Client.Patch(reqCtx.Ctx, cluster, patch); err != nil {
+ return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "failed to remove Cluster restore-protection finalizer")
+ }
+ return intctrlutil.Reconciled()
+}
+
+func isClusterRestoreHelperPVC(pvc *corev1.PersistentVolumeClaim) bool {
+ return pvc.Labels[dprestore.DataProtectionPopulatePVCLabelKey] != ""
+}
+
+func isClusterRestoreTargetPVC(pvc *corev1.PersistentVolumeClaim) bool {
+ return pvc.Spec.DataSourceRef != nil && pvc.Spec.DataSourceRef.APIGroup != nil &&
+ *pvc.Spec.DataSourceRef.APIGroup == dptypes.DataprotectionAPIGroup
+}
diff --git a/controllers/dataprotection/cluster_restore_controller_test.go b/controllers/dataprotection/cluster_restore_controller_test.go
new file mode 100644
index 00000000000..051960f4eea
--- /dev/null
+++ b/controllers/dataprotection/cluster_restore_controller_test.go
@@ -0,0 +1,149 @@
+/*
+Copyright (C) 2022-2026 ApeCloud Co., Ltd
+
+This file is part of KubeBlocks project
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+*/
+
+package dataprotection
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+ appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
+ dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1"
+ dptypes "github.com/apecloud/kubeblocks/pkg/dataprotection/types"
+)
+
+func TestClusterRestoreProtectionLifecycle(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ deleting bool
+ protected bool
+ status metav1.ConditionStatus
+ resource string
+ wantKeep bool
+ wantWait bool
+ }{
+ {"initial intent", false, false, metav1.ConditionUnknown, "", true, false},
+ {"deletion without protection", true, false, metav1.ConditionUnknown, "target", false, false},
+ {"failed restore", false, true, metav1.ConditionFalse, "", true, false},
+ {"successful restore", false, true, metav1.ConditionTrue, "completed restore", false, false},
+ {"target still protected", false, true, metav1.ConditionTrue, "target", true, false},
+ {"helper remains", false, true, metav1.ConditionTrue, "helper", true, false},
+ {"execution still running", false, true, metav1.ConditionTrue, "running restore", true, false},
+ {"deletion waits for target", true, true, metav1.ConditionTrue, "target", true, true},
+ {"deletion waits for helper", true, true, metav1.ConditionTrue, "helper", true, true},
+ {"deletion waits for failed restore", true, true, metav1.ConditionTrue, "failed restore", true, true},
+ {"deletion waits for completed restore", true, true, metav1.ConditionTrue, "completed restore", true, true},
+ {"deletion cleanup finished", true, true, metav1.ConditionTrue, "", false, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ ctx := context.Background()
+ scheme, cluster, _, _, target := parentRestoreObjects(t)
+ cluster.Finalizers = []string{"example.io/app-owner"}
+ if tc.protected {
+ cluster.Finalizers = append(cluster.Finalizers, dptypes.RestoreProtectionFinalizerName)
+ }
+ if tc.deleting {
+ now := metav1.Now()
+ cluster.DeletionTimestamp = &now
+ }
+ cluster.Status.Conditions = []metav1.Condition{{
+ Type: appsv1.ConditionTypeRestore, Status: tc.status,
+ }}
+ var resource client.Object
+ switch tc.resource {
+ case "target":
+ target.Finalizers = []string{dptypes.DataProtectionFinalizerName}
+ resource = target
+ case "helper":
+ resource = restoreHelperForTarget(target, cluster)
+ case "running restore", "failed restore", "completed restore":
+ restore := executionRestoreForTarget(target, cluster)
+ switch tc.resource {
+ case "failed restore":
+ restore.Status.Phase = dpv1alpha1.RestorePhaseFailed
+ case "completed restore":
+ restore.Status.Phase = dpv1alpha1.RestorePhaseCompleted
+ default:
+ restore.Status.Phase = dpv1alpha1.RestorePhaseRunning
+ }
+ restore.Finalizers = []string{dptypes.DataProtectionFinalizerName}
+ resource = restore
+ }
+ objects := []client.Object{cluster}
+ if resource != nil {
+ objects = append(objects, resource)
+ }
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build()
+ if resource != nil {
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(resource), resource))
+ }
+ reconciler := &ClusterRestoreReconciler{Client: cli}
+
+ result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(cluster)})
+ require.NoError(t, err)
+ require.Equal(t, tc.wantWait, result.RequeueAfter > 0)
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
+ expected := []string{"example.io/app-owner"}
+ if tc.wantKeep {
+ expected = append(expected, dptypes.RestoreProtectionFinalizerName)
+ }
+ require.ElementsMatch(t, expected, cluster.Finalizers)
+ if resource != nil {
+ current := resource.DeepCopyObject().(client.Object)
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(resource), current))
+ require.Equal(t, resource, current)
+ }
+ })
+ }
+}
+
+func TestClusterRestoreControllerIgnoresResourcesWithoutExactClusterUID(t *testing.T) {
+ for _, uid := range []string{"", "another-cluster-uid"} {
+ t.Run("uid="+uid, func(t *testing.T) {
+ ctx := context.Background()
+ scheme, cluster, _, _, target := parentRestoreObjects(t)
+ now := metav1.Now()
+ cluster.DeletionTimestamp = &now
+ cluster.Finalizers = append(cluster.Finalizers, "example.io/app-owner")
+ target.Finalizers = []string{dptypes.DataProtectionFinalizerName}
+ helper := restoreHelperForTarget(target, cluster)
+ restore := executionRestoreForTarget(target, cluster)
+ for _, obj := range []client.Object{target, helper, restore} {
+ obj.GetLabels()[dptypes.ClusterUIDLabelKey] = uid
+ }
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, target, helper, restore).Build()
+ reconciler := &ClusterRestoreReconciler{Client: cli}
+
+ _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(cluster)})
+ require.NoError(t, err)
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
+ require.Equal(t, []string{"example.io/app-owner"}, cluster.Finalizers)
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(target), &corev1.PersistentVolumeClaim{}))
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(helper), &corev1.PersistentVolumeClaim{}))
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(restore), &dpv1alpha1.Restore{}))
+ })
+ }
+}
diff --git a/controllers/dataprotection/suite_test.go b/controllers/dataprotection/suite_test.go
index 337d93d66f2..9da6f79fa2b 100644
--- a/controllers/dataprotection/suite_test.go
+++ b/controllers/dataprotection/suite_test.go
@@ -187,6 +187,12 @@ var _ = BeforeSuite(func() {
}).SetupWithManager(k8sManager)
Expect(err).ToNot(HaveOccurred())
+ err = (&ClusterRestoreReconciler{
+ Client: k8sClient,
+ Recorder: k8sManager.GetEventRecorderFor("cluster-restore-controller"),
+ }).SetupWithManager(k8sManager)
+ Expect(err).ToNot(HaveOccurred())
+
err = (&BackupScheduleReconciler{
Client: k8sClient,
Scheme: k8sManager.GetScheme(),
diff --git a/controllers/dataprotection/volumepopulator_controller.go b/controllers/dataprotection/volumepopulator_controller.go
index e4811694246..27731830954 100644
--- a/controllers/dataprotection/volumepopulator_controller.go
+++ b/controllers/dataprotection/volumepopulator_controller.go
@@ -60,7 +60,7 @@ import (
viper "github.com/apecloud/kubeblocks/pkg/viperx"
)
-// VolumePopulatorReconciler reconciles PVCs with Backup or Restore data sources.
+// VolumePopulatorReconciler coordinates data population and restore for PVCs.
type VolumePopulatorReconciler struct {
client.Client
Scheme *runtime.Scheme
@@ -88,7 +88,7 @@ type pvcRestoreDecision struct {
// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims/finalizers,verbs=update
-// +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=restores,verbs=get;list;watch
+// +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=restores,verbs=get;list;watch;delete
// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=clusters,verbs=get;list;watch
// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=components,verbs=get;list;watch
// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=componentdefinitions,verbs=get;list;watch
@@ -213,7 +213,7 @@ func (r *VolumePopulatorReconciler) mapClusterToPVCs(ctx context.Context, obj cl
}
return r.mapRestorePVCs(ctx, cluster.Namespace, client.MatchingLabels{
constant.AppInstanceLabelKey: cluster.Name,
- }, string(cluster.UID), false)
+ }, string(cluster.UID), !cluster.DeletionTimestamp.IsZero())
}
func (r *VolumePopulatorReconciler) mapRestorePVCs(ctx context.Context, namespace string,
@@ -343,7 +343,9 @@ func clusterDependencyPredicate() predicate.Predicate {
oldCluster, oldOK := e.ObjectOld.(*appsv1.Cluster)
newCluster, newOK := e.ObjectNew.(*appsv1.Cluster)
return oldOK && newOK && (oldCluster.Status.Phase != newCluster.Status.Phase ||
- !reflect.DeepEqual(oldCluster.DeletionTimestamp, newCluster.DeletionTimestamp))
+ !reflect.DeepEqual(oldCluster.DeletionTimestamp, newCluster.DeletionTimestamp) ||
+ controllerutil.ContainsFinalizer(oldCluster, dptypes.RestoreProtectionFinalizerName) !=
+ controllerutil.ContainsFinalizer(newCluster, dptypes.RestoreProtectionFinalizerName))
},
}
}
@@ -394,6 +396,17 @@ func (r *VolumePopulatorReconciler) syncPVC(reqCtx intctrlutil.RequestCtx, pvc *
if !matched {
return nil
}
+ terminated, err := r.handleRestoreClusterLifecycle(reqCtx, pvc)
+ if err != nil || terminated {
+ return err
+ }
+ // Parent deletion is checked first because it authorizes cleanup even after
+ // target protection has been handed off. Target deletion alone remains a
+ // no-op, and Kubernetes does not allow acquiring a new finalizer here.
+ if !pvc.DeletionTimestamp.IsZero() &&
+ !controllerutil.ContainsFinalizer(pvc, dptypes.DataProtectionFinalizerName) {
+ return nil
+ }
// A non-deleting bound PVC with a terminal Restore condition does not need
// its source Backup/Restore. Populating can finish while postReady is pending.
if pvc.Spec.VolumeName != "" && pvc.DeletionTimestamp.IsZero() && pvcRestoreTerminal(pvc) {
@@ -418,6 +431,351 @@ func (r *VolumePopulatorReconciler) syncPVC(reqCtx intctrlutil.RequestCtx, pvc *
return nil
}
+// handleRestoreClusterLifecycle validates the Cluster identity and protection
+// before restore work starts, and initiates owner-driven cleanup when the
+// Cluster is deleting. Target PVC deletion alone is not a termination signal.
+func (r *VolumePopulatorReconciler) handleRestoreClusterLifecycle(reqCtx intctrlutil.RequestCtx,
+ pvc *corev1.PersistentVolumeClaim) (bool, error) {
+ clusterName := pvc.Labels[constant.AppInstanceLabelKey]
+ componentName := pvc.Labels[constant.KBAppComponentLabelKey]
+ if clusterName == "" || componentName == "" {
+ return false, nil
+ }
+ hasClusterIdentity := pvc.Annotations[constant.KBAppClusterUIDKey] != "" ||
+ pvc.Labels[dptypes.ClusterUIDLabelKey] != ""
+ // App labels alone do not establish Cluster restore identity; standalone DP
+ // restores may use the same labels.
+ if !hasClusterIdentity {
+ return false, nil
+ }
+
+ cluster := &appsv1.Cluster{}
+ clusterKey := types.NamespacedName{Namespace: pvc.Namespace, Name: clusterName}
+ if err := r.Client.Get(reqCtx.Ctx, clusterKey, cluster); err != nil {
+ if apierrors.IsNotFound(err) {
+ if !controllerutil.ContainsFinalizer(pvc, dptypes.DataProtectionFinalizerName) {
+ return true, nil
+ }
+ // Registration can lose the race with Cluster deletion, but no work
+ // starts until a later reconcile has rechecked the parent.
+ if releaseErr := r.releaseUnusedTargetFinalizer(reqCtx, pvc); releaseErr != nil {
+ return true, restoreParentRequeue(releaseErr)
+ }
+ return true, nil
+ }
+ return false, restoreParentRequeue(err)
+ }
+ for _, clusterUID := range []string{
+ pvc.Annotations[constant.KBAppClusterUIDKey],
+ pvc.Labels[dptypes.ClusterUIDLabelKey],
+ } {
+ if clusterUID != "" && clusterUID != string(cluster.UID) {
+ return false, restoreParentRequeue(fmt.Errorf(
+ "PVC %s/%s identifies Cluster %s/%s UID %s, not current UID %s",
+ pvc.Namespace, pvc.Name, cluster.Namespace, cluster.Name, clusterUID, cluster.UID))
+ }
+ }
+ if !cluster.DeletionTimestamp.IsZero() {
+ return r.terminateClusterVolumePopulation(reqCtx, pvc, cluster)
+ }
+ if !pvc.DeletionTimestamp.IsZero() &&
+ !controllerutil.ContainsFinalizer(pvc, dptypes.DataProtectionFinalizerName) {
+ return true, nil
+ }
+
+ committed := volumePopulationIdentityCommitted(pvc, cluster)
+ if committed {
+ if _, err := r.committedVolumePopulationComponent(reqCtx.Ctx, pvc, cluster); err != nil {
+ return false, restoreParentRequeue(err)
+ }
+ }
+ // Aggregate restore status gates normal progression, not owner cleanup.
+ if !clusterAllowsRestoreProgress(cluster) && !pvcRestoreTerminal(pvc) {
+ return false, intctrlutil.NewRequeueError(reconcileInterval, "Cluster restore is no longer active")
+ }
+ if !committed {
+ if pvcRestoreTerminal(pvc) && !controllerutil.ContainsFinalizer(pvc, dptypes.DataProtectionFinalizerName) {
+ return false, nil
+ }
+ comp, err := r.validateClusterRestorePVCOwnership(reqCtx.Ctx, pvc, cluster)
+ if err != nil {
+ return false, restoreParentRequeue(err)
+ }
+ if err = r.registerVolumePopulation(reqCtx.Ctx, pvc, cluster, comp); err != nil {
+ return false, restoreParentRequeue(err)
+ }
+ return false, intctrlutil.NewRequeueError(reconcileInterval, "waiting for target PVC restore protection")
+ }
+ if !controllerutil.ContainsFinalizer(cluster, dptypes.RestoreProtectionFinalizerName) {
+ if pvcRestoreTerminal(pvc) && !controllerutil.ContainsFinalizer(pvc, dptypes.DataProtectionFinalizerName) {
+ return false, nil
+ }
+ return false, intctrlutil.NewRequeueError(reconcileInterval,
+ "waiting for Cluster restore-protection finalizer")
+ }
+ if !pvcPopulateReleased(pvc) && !pvcRestoreTerminal(pvc) {
+ if err := r.ensureTargetFinalizer(reqCtx, pvc); err != nil {
+ return false, err
+ }
+ }
+ return false, nil
+}
+
+// releaseUnusedTargetFinalizer rolls back an unused registration after its
+// Cluster disappears. Existing population or Restore resources retain it.
+func (r *VolumePopulatorReconciler) releaseUnusedTargetFinalizer(reqCtx intctrlutil.RequestCtx,
+ pvc *corev1.PersistentVolumeClaim) error {
+ clusterUID := clusterRestorePVCUID(pvc)
+ if clusterUID == "" || pvc.Labels[dptypes.ComponentUIDLabelKey] == "" {
+ return fmt.Errorf("restore PVC %s/%s has no committed parent identity", pvc.Namespace, pvc.Name)
+ }
+ if r.ContainPopulatingCondition(pvc) && !pvcPopulateReleased(pvc) {
+ return fmt.Errorf("cluster is missing after population started for PVC %s/%s", pvc.Namespace, pvc.Name)
+ }
+ key := types.NamespacedName{Namespace: pvc.Namespace, Name: getPopulatePVCName(pvc.UID)}
+ for _, obj := range []client.Object{&corev1.PersistentVolumeClaim{}, &dpv1alpha1.Restore{}} {
+ if err := r.Client.Get(reqCtx.Ctx, key, obj); !apierrors.IsNotFound(err) {
+ if err != nil {
+ return err
+ }
+ return fmt.Errorf("cluster is missing while restore resources for PVC %s/%s remain", pvc.Namespace, pvc.Name)
+ }
+ }
+
+ list := &dpv1alpha1.RestoreList{}
+ if err := r.Client.List(reqCtx.Ctx, list, client.InNamespace(pvc.Namespace), client.MatchingLabels{
+ constant.AppInstanceLabelKey: pvc.Labels[constant.AppInstanceLabelKey],
+ dptypes.ClusterUIDLabelKey: clusterUID,
+ }); err != nil {
+ return err
+ }
+ for i := range list.Items {
+ if internalPostReadyRestoreOwner(&list.Items[i]) != nil {
+ return fmt.Errorf("cluster is missing while postReady Restore %s/%s remains",
+ list.Items[i].Namespace, list.Items[i].Name)
+ }
+ }
+ return r.releaseTargetPVC(reqCtx, pvc)
+}
+
+func volumePopulationIdentityCommitted(pvc *corev1.PersistentVolumeClaim, cluster *appsv1.Cluster) bool {
+ return pvc.Labels[dptypes.ClusterUIDLabelKey] == string(cluster.UID) &&
+ pvc.Labels[dptypes.ComponentUIDLabelKey] != ""
+}
+
+// committedVolumePopulationComponent resolves the Component identity recorded
+// before VP creates restore resources. Retention may detach the workload owner
+// chain, but it does not change this identity.
+func (r *VolumePopulatorReconciler) committedVolumePopulationComponent(ctx context.Context,
+ pvc *corev1.PersistentVolumeClaim, cluster *appsv1.Cluster) (*appsv1.Component, error) {
+ componentName := pvc.Labels[constant.KBAppComponentLabelKey]
+ comp := &appsv1.Component{}
+ key := types.NamespacedName{
+ Namespace: pvc.Namespace,
+ Name: constant.GenerateClusterComponentName(cluster.Name, componentName),
+ }
+ if err := r.Client.Get(ctx, key, comp); err != nil {
+ return nil, err
+ }
+ if string(comp.UID) != pvc.Labels[dptypes.ComponentUIDLabelKey] {
+ return nil, fmt.Errorf("restore PVC %s/%s Component UID changed from %s to %s",
+ pvc.Namespace, pvc.Name, pvc.Labels[dptypes.ComponentUIDLabelKey], comp.UID)
+ }
+ return comp, nil
+}
+
+func restoreParentRequeue(err error) error {
+ return intctrlutil.NewRequeueError(reconcileInterval, err.Error())
+}
+
+func (r *VolumePopulatorReconciler) terminateClusterVolumePopulation(reqCtx intctrlutil.RequestCtx,
+ pvc *corev1.PersistentVolumeClaim, cluster *appsv1.Cluster) (bool, error) {
+ err := r.cleanupClusterVolumePopulation(reqCtx, pvc, cluster)
+ if err != nil && !intctrlutil.IsRequeueError(err) {
+ err = restoreParentRequeue(err)
+ }
+ return true, err
+}
+
+func (r *VolumePopulatorReconciler) cleanupClusterVolumePopulation(reqCtx intctrlutil.RequestCtx,
+ pvc *corev1.PersistentVolumeClaim, cluster *appsv1.Cluster) error {
+ pending, err := r.deleteExecutionRestoreAndWait(reqCtx.Ctx, pvc, cluster)
+ if err != nil {
+ return err
+ }
+ postReadyPending, err := r.deleteClusterPostReadyRestoresAndWait(reqCtx.Ctx, cluster)
+ if err != nil {
+ return err
+ }
+ if pending || postReadyPending {
+ return intctrlutil.NewRequeueError(reconcileInterval, "waiting for Restore owners to finish termination")
+ }
+ pending, err = r.deletePopulatePVCAndWait(reqCtx.Ctx, pvc, cluster)
+ if err != nil {
+ return err
+ }
+ if pending {
+ return intctrlutil.NewRequeueError(reconcileInterval, "waiting for helper PVC to disappear")
+ }
+ return r.releaseTargetPVC(reqCtx, pvc)
+}
+
+func (r *VolumePopulatorReconciler) deleteExecutionRestoreAndWait(ctx context.Context,
+ pvc *corev1.PersistentVolumeClaim, cluster *appsv1.Cluster) (bool, error) {
+ restore := &dpv1alpha1.Restore{}
+ key := types.NamespacedName{Namespace: pvc.Namespace, Name: getPopulatePVCName(pvc.UID)}
+ if err := r.Client.Get(ctx, key, restore); err != nil {
+ return false, client.IgnoreNotFound(err)
+ }
+ if restore.Labels[dptypes.ClusterUIDLabelKey] != string(cluster.UID) ||
+ restore.Labels[dprestore.DataProtectionRestoreLabelKey] != restore.Name ||
+ !hasExactOwnerReference(restore.OwnerReferences, corev1.SchemeGroupVersion.String(),
+ "PersistentVolumeClaim", pvc.Name, pvc.UID) {
+ return false, fmt.Errorf("refusing to delete execution Restore %s/%s without exact VP ownership",
+ restore.Namespace, restore.Name)
+ }
+ if restore.DeletionTimestamp.IsZero() {
+ if err := r.Client.Delete(ctx, restore); err != nil && !apierrors.IsNotFound(err) {
+ return false, err
+ }
+ }
+ return true, nil
+}
+
+func (r *VolumePopulatorReconciler) deleteClusterPostReadyRestoresAndWait(ctx context.Context,
+ cluster *appsv1.Cluster) (bool, error) {
+ list := &dpv1alpha1.RestoreList{}
+ if err := r.Client.List(ctx, list, client.InNamespace(cluster.Namespace), client.MatchingLabels{
+ constant.AppInstanceLabelKey: cluster.Name,
+ dptypes.ClusterUIDLabelKey: string(cluster.UID),
+ }); err != nil {
+ return false, err
+ }
+ pending := false
+ for i := range list.Items {
+ restore := &list.Items[i]
+ if internalPostReadyRestoreOwner(restore) == nil {
+ continue
+ }
+ pending = true
+ if !restore.DeletionTimestamp.IsZero() {
+ continue
+ }
+ if err := r.Client.Delete(ctx, restore); err != nil && !apierrors.IsNotFound(err) {
+ return false, err
+ }
+ }
+ return pending, nil
+}
+
+func (r *VolumePopulatorReconciler) deletePopulatePVCAndWait(ctx context.Context,
+ pvc *corev1.PersistentVolumeClaim, cluster *appsv1.Cluster) (bool, error) {
+ helper := &corev1.PersistentVolumeClaim{}
+ key := types.NamespacedName{Namespace: pvc.Namespace, Name: getPopulatePVCName(pvc.UID)}
+ if err := r.Client.Get(ctx, key, helper); err != nil {
+ return false, client.IgnoreNotFound(err)
+ }
+ if helper.Labels[dptypes.ClusterUIDLabelKey] != string(cluster.UID) ||
+ helper.Labels[dprestore.DataProtectionPopulatePVCLabelKey] != helper.Name {
+ return false, fmt.Errorf("refusing to delete helper PVC %s/%s without exact VP identity",
+ helper.Namespace, helper.Name)
+ }
+ if helper.DeletionTimestamp.IsZero() {
+ if err := r.Client.Delete(ctx, helper); err != nil && !apierrors.IsNotFound(err) {
+ return false, err
+ }
+ }
+ return true, nil
+}
+
+func hasExactOwnerReference(refs []metav1.OwnerReference, apiVersion, kind, name string, uid types.UID) bool {
+ for i := range refs {
+ ref := refs[i]
+ if ref.APIVersion == apiVersion && ref.Kind == kind && ref.Name == name && ref.UID == uid {
+ return true
+ }
+ }
+ return false
+}
+
+func (r *VolumePopulatorReconciler) validateClusterRestorePVCOwnership(ctx context.Context,
+ pvc *corev1.PersistentVolumeClaim, cluster *appsv1.Cluster) (*appsv1.Component, error) {
+ owner := metav1.GetControllerOf(pvc)
+ if owner == nil || owner.APIVersion != workloads.GroupVersion.String() {
+ return nil, fmt.Errorf("restore PVC %s/%s has no supported workload controller owner", pvc.Namespace, pvc.Name)
+ }
+ var itsOwner *metav1.OwnerReference
+ switch owner.Kind {
+ case workloads.InstanceSetKind:
+ itsOwner = owner
+ case "Instance":
+ instance := &workloads.Instance{}
+ if err := r.Client.Get(ctx, client.ObjectKey{Namespace: pvc.Namespace, Name: owner.Name}, instance); err != nil {
+ return nil, err
+ }
+ if instance.UID != owner.UID {
+ return nil, fmt.Errorf("restore PVC %s/%s Instance owner UID does not match", pvc.Namespace, pvc.Name)
+ }
+ itsOwner = metav1.GetControllerOf(instance)
+ if itsOwner == nil || itsOwner.APIVersion != workloads.GroupVersion.String() ||
+ itsOwner.Kind != workloads.InstanceSetKind {
+ return nil, fmt.Errorf("instance %s/%s has no InstanceSet controller owner", instance.Namespace, instance.Name)
+ }
+ default:
+ return nil, fmt.Errorf("restore PVC %s/%s has unsupported workload owner kind %s",
+ pvc.Namespace, pvc.Name, owner.Kind)
+ }
+
+ its := &workloads.InstanceSet{}
+ if err := r.Client.Get(ctx, client.ObjectKey{Namespace: pvc.Namespace, Name: itsOwner.Name}, its); err != nil {
+ return nil, err
+ }
+ if its.UID != itsOwner.UID {
+ return nil, fmt.Errorf("restore PVC %s/%s InstanceSet owner UID does not match", pvc.Namespace, pvc.Name)
+ }
+ if its.Labels[constant.AppInstanceLabelKey] != cluster.Name ||
+ its.Labels[constant.KBAppComponentLabelKey] != pvc.Labels[constant.KBAppComponentLabelKey] {
+ return nil, fmt.Errorf("InstanceSet %s/%s does not match restore PVC parent identity", its.Namespace, its.Name)
+ }
+ componentOwner := metav1.GetControllerOf(its)
+ if componentOwner == nil || componentOwner.APIVersion != appsv1.GroupVersion.String() ||
+ componentOwner.Kind != appsv1.ComponentKind {
+ return nil, fmt.Errorf("InstanceSet %s/%s has no Component controller owner", its.Namespace, its.Name)
+ }
+ comp := &appsv1.Component{}
+ if err := r.Client.Get(ctx, client.ObjectKey{Namespace: pvc.Namespace, Name: componentOwner.Name}, comp); err != nil {
+ return nil, err
+ }
+ if comp.UID != componentOwner.UID || comp.Labels[constant.AppInstanceLabelKey] != cluster.Name ||
+ comp.Labels[constant.KBAppComponentLabelKey] != pvc.Labels[constant.KBAppComponentLabelKey] {
+ return nil, fmt.Errorf("InstanceSet %s/%s is not owned by the PVC Component in Cluster %s/%s",
+ its.Namespace, its.Name, cluster.Namespace, cluster.Name)
+ }
+ clusterOwner := metav1.GetControllerOf(comp)
+ if clusterOwner == nil || clusterOwner.APIVersion != appsv1.GroupVersion.String() ||
+ clusterOwner.Kind != appsv1.ClusterKind || clusterOwner.Name != cluster.Name || clusterOwner.UID != cluster.UID {
+ return nil, fmt.Errorf("component %s/%s is not owned by current Cluster UID %s",
+ comp.Namespace, comp.Name, cluster.UID)
+ }
+ return comp, nil
+}
+
+// registerVolumePopulation records verified App ownership and target protection
+// together. The caller returns so the shared cache observes it before work starts.
+func (r *VolumePopulatorReconciler) registerVolumePopulation(ctx context.Context,
+ pvc *corev1.PersistentVolumeClaim, cluster *appsv1.Cluster, comp *appsv1.Component) error {
+ if uid := pvc.Labels[dptypes.ComponentUIDLabelKey]; uid != "" && uid != string(comp.UID) {
+ return fmt.Errorf("restore PVC %s/%s Component UID %s does not match %s",
+ pvc.Namespace, pvc.Name, uid, comp.UID)
+ }
+ originalPVC := pvc.DeepCopy()
+ pvc.Labels[dptypes.ClusterUIDLabelKey] = string(cluster.UID)
+ pvc.Labels[dptypes.ComponentUIDLabelKey] = string(comp.UID)
+ controllerutil.AddFinalizer(pvc, dptypes.DataProtectionFinalizerName)
+ return r.Client.Patch(ctx, pvc,
+ client.MergeFromWithOptions(originalPVC, client.MergeFromWithOptimisticLock{}))
+}
+
// dispatchUnboundPVC routes an unbound PVC to either Populate or ProvisionOnly.
// When mode is RestoreData but PrepareDataBackupSets is empty, it checks
// PostReadyBackupSets: if postReady actions exist, fall back to ProvisionOnly
@@ -939,6 +1297,7 @@ func internalRestoreLabels(pvc *corev1.PersistentVolumeClaim) map[string]string
constant.KBAppComponentLabelKey,
constant.KBAppShardingNameLabelKey,
constant.VolumeClaimTemplateNameLabelKey,
+ dptypes.ComponentUIDLabelKey,
} {
if value := pvc.Labels[key]; value != "" {
labels[key] = value
@@ -1132,13 +1491,8 @@ func (r *VolumePopulatorReconciler) Populate(reqCtx intctrlutil.RequestCtx, pvc
if err != nil || wait {
return err
}
- // Make sure the PVC finalizer is present
- if !slices.Contains(pvc.Finalizers, dptypes.DataProtectionFinalizerName) {
- pvcPatch := client.MergeFrom(pvc.DeepCopy())
- controllerutil.AddFinalizer(pvc, dptypes.DataProtectionFinalizerName)
- if err = r.Client.Patch(reqCtx.Ctx, pvc, pvcPatch); err != nil {
- return err
- }
+ if err = r.ensureTargetFinalizer(reqCtx, pvc); err != nil {
+ return err
}
if err = r.UpdatePVCConditions(reqCtx, pvc, ReasonPopulatingProcessing, "Populator started"); err != nil {
return err
@@ -1194,12 +1548,8 @@ func (r *VolumePopulatorReconciler) ProvisionOnly(reqCtx intctrlutil.RequestCtx,
if err != nil || wait {
return err
}
- if !slices.Contains(pvc.Finalizers, dptypes.DataProtectionFinalizerName) {
- pvcPatch := client.MergeFrom(pvc.DeepCopy())
- controllerutil.AddFinalizer(pvc, dptypes.DataProtectionFinalizerName)
- if err = r.Client.Patch(reqCtx.Ctx, pvc, pvcPatch); err != nil {
- return err
- }
+ if err = r.ensureTargetFinalizer(reqCtx, pvc); err != nil {
+ return err
}
if err = r.UpdatePVCConditions(reqCtx, pvc, ReasonPopulatingProcessing, "Provisioning PVC without data restore"); err != nil {
return err
@@ -1221,6 +1571,24 @@ func (r *VolumePopulatorReconciler) ProvisionOnly(reqCtx intctrlutil.RequestCtx,
return r.completeBoundPVCIfNeeded(reqCtx, pvc, restoreCtx)
}
+func (r *VolumePopulatorReconciler) ensureTargetFinalizer(
+ reqCtx intctrlutil.RequestCtx, pvc *corev1.PersistentVolumeClaim) error {
+ if slices.Contains(pvc.Finalizers, dptypes.DataProtectionFinalizerName) {
+ return nil
+ }
+ pvcPatch := client.MergeFromWithOptions(pvc.DeepCopy(), client.MergeFromWithOptimisticLock{})
+ controllerutil.AddFinalizer(pvc, dptypes.DataProtectionFinalizerName)
+ if err := r.Client.Patch(reqCtx.Ctx, pvc, pvcPatch); err != nil {
+ return err
+ }
+ if clusterRestorePVCUID(pvc) != "" {
+ // Reobserve the marker through the shared PVC cache before another
+ // informer can expose newly-created restore resources.
+ return intctrlutil.NewRequeueError(reconcileInterval, "waiting for target PVC restore protection")
+ }
+ return nil
+}
+
func (r *VolumePopulatorReconciler) completeBoundPVCIfNeeded(reqCtx intctrlutil.RequestCtx,
pvc *corev1.PersistentVolumeClaim,
restoreCtx *pvcRestoreContext) error {
@@ -1245,6 +1613,9 @@ func (r *VolumePopulatorReconciler) completeBoundPVCIfNeeded(reqCtx intctrlutil.
if !postReadyCompleted {
return intctrlutil.NewRequeueError(reconcileInterval, "waiting for postReady restore")
}
+ if err := r.releaseTargetPVC(reqCtx, pvc); err != nil {
+ return err
+ }
return r.UpdatePVCConditions(reqCtx, pvc, reason, message)
}
@@ -1477,6 +1848,11 @@ func (r *VolumePopulatorReconciler) ensurePostReadyRestoreCompleted(reqCtx intct
if !apierrors.IsNotFound(err) {
return false, err
}
+ if clusterRestorePVCUID(pvc) != "" {
+ if err = r.ensureTargetFinalizer(reqCtx, pvc); err != nil {
+ return false, err
+ }
+ }
if err = r.Client.Create(reqCtx.Ctx, postReadyRestore); err != nil && !apierrors.IsAlreadyExists(err) {
return false, err
}
@@ -1488,6 +1864,13 @@ func (r *VolumePopulatorReconciler) ensurePostReadyRestoreCompleted(reqCtx intct
if err = validatePostReadyRestore(existing, postReadyRestore, comp); err != nil {
return false, err
}
+ // The Restore is now visible to the Cluster lifecycle resource scan. The
+ // helper has already been released, so temporary target protection can go.
+ if pvcPopulateReleased(pvc) {
+ if err = r.releaseTargetPVC(reqCtx, pvc); err != nil {
+ return false, err
+ }
+ }
switch existing.Status.Phase {
case dpv1alpha1.RestorePhaseCompleted:
return true, nil
@@ -1920,7 +2303,7 @@ func (r *VolumePopulatorReconciler) deletePopulatePVC(reqCtx intctrlutil.Request
func (r *VolumePopulatorReconciler) releaseTargetPVC(reqCtx intctrlutil.RequestCtx, pvc *corev1.PersistentVolumeClaim) error {
if slices.Contains(pvc.Finalizers, dptypes.DataProtectionFinalizerName) {
- pvcPatch := client.MergeFrom(pvc.DeepCopy())
+ pvcPatch := client.MergeFromWithOptions(pvc.DeepCopy(), client.MergeFromWithOptimisticLock{})
controllerutil.RemoveFinalizer(pvc, dptypes.DataProtectionFinalizerName)
if err := r.Client.Patch(reqCtx.Ctx, pvc, pvcPatch); err != nil {
return client.IgnoreNotFound(err)
diff --git a/controllers/dataprotection/volumepopulator_controller_test.go b/controllers/dataprotection/volumepopulator_controller_test.go
index 8c5211fbb97..9ddc90e2f28 100644
--- a/controllers/dataprotection/volumepopulator_controller_test.go
+++ b/controllers/dataprotection/volumepopulator_controller_test.go
@@ -4221,6 +4221,415 @@ func dependencyRestorePVC(name, componentName string, uid types.UID) *corev1.Per
}
}
+func TestClusterDeletionTerminatesVolumePopulationInOrder(t *testing.T) {
+ for _, retained := range []bool{false, true} {
+ t.Run(fmt.Sprintf("retained=%t", retained), func(t *testing.T) {
+ ctx := context.Background()
+ scheme, cluster, component, its, target := parentRestoreObjects(t)
+ now := metav1.Now()
+ cluster.DeletionTimestamp = &now
+ cluster.Finalizers = append(cluster.Finalizers, "example.io/app-owner")
+ target.Finalizers = []string{dptypes.DataProtectionFinalizerName, "example.io/app-owner"}
+ target.Labels[dptypes.ComponentUIDLabelKey] = string(component.UID)
+ objects := []client.Object{cluster, target}
+ if retained {
+ target.OwnerReferences = nil
+ } else {
+ objects = append(objects, its)
+ }
+ helper := restoreHelperForTarget(target, cluster)
+ execution := executionRestoreForTarget(target, cluster)
+ execution.Finalizers = []string{"example.io/restore-owner"}
+ postReady := postReadyRestoreForComponent(target, cluster, component)
+ postReady.Finalizers = []string{"example.io/restore-owner"}
+ objects = append(objects, helper, execution, postReady)
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build()
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme}
+ reqCtx := intctrlutil.RequestCtx{Ctx: ctx}
+
+ require.ErrorContains(t, vp.syncPVC(reqCtx, target), "waiting for Restore owners")
+ for _, expected := range []*dpv1alpha1.Restore{execution, postReady} {
+ current := &dpv1alpha1.Restore{}
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(expected), current))
+ require.False(t, current.DeletionTimestamp.IsZero())
+ require.Equal(t, expected.Finalizers, current.Finalizers)
+ current.Finalizers = nil
+ require.NoError(t, cli.Update(ctx, current))
+ }
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(helper), &corev1.PersistentVolumeClaim{}))
+
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(target), target))
+ require.ErrorContains(t, vp.syncPVC(reqCtx, target), "waiting for helper PVC to disappear")
+ require.True(t, apierrors.IsNotFound(cli.Get(ctx, client.ObjectKeyFromObject(helper), helper)))
+
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(target), target))
+ require.NoError(t, vp.syncPVC(reqCtx, target))
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(target), target))
+ require.Equal(t, []string{"example.io/app-owner"}, target.Finalizers)
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
+ require.Contains(t, cluster.Finalizers, dptypes.RestoreProtectionFinalizerName)
+ })
+ }
+}
+
+func TestClusterLifecycleRegistersBeforeWaitingForProtection(t *testing.T) {
+ for _, existingFinalizer := range []bool{false, true} {
+ t.Run(fmt.Sprintf("existing-finalizer=%t", existingFinalizer), func(t *testing.T) {
+ ctx := context.Background()
+ scheme, cluster, component, its, target := parentRestoreObjects(t)
+ cluster.Finalizers = nil
+ delete(target.Labels, dptypes.ClusterUIDLabelKey)
+ if existingFinalizer {
+ target.Finalizers = []string{dptypes.DataProtectionFinalizerName}
+ }
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, component, its, target).Build()
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme}
+
+ require.ErrorContains(t, vp.syncPVC(intctrlutil.RequestCtx{Ctx: ctx}, target),
+ "waiting for target PVC restore protection")
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(target), target))
+ require.Equal(t, string(cluster.UID), target.Labels[dptypes.ClusterUIDLabelKey])
+ require.Equal(t, string(component.UID), target.Labels[dptypes.ComponentUIDLabelKey])
+ require.Contains(t, target.Finalizers, dptypes.DataProtectionFinalizerName)
+ require.Empty(t, target.Status.Conditions)
+
+ require.ErrorContains(t, vp.syncPVC(intctrlutil.RequestCtx{Ctx: ctx}, target),
+ "waiting for Cluster restore-protection finalizer")
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
+ require.NotContains(t, cluster.Finalizers, dptypes.RestoreProtectionFinalizerName,
+ "VolumePopulator must not add the Cluster finalizer")
+ pvcs := &corev1.PersistentVolumeClaimList{}
+ require.NoError(t, cli.List(ctx, pvcs))
+ require.Len(t, pvcs.Items, 1)
+ restores := &dpv1alpha1.RestoreList{}
+ require.NoError(t, cli.List(ctx, restores))
+ require.Empty(t, restores.Items)
+ })
+ }
+}
+
+func TestClusterLifecycleRegistrationReturnsBeforeRestoreValidation(t *testing.T) {
+ ctx := context.Background()
+ scheme, cluster, component, its, target := parentRestoreObjects(t)
+ delete(target.Labels, dptypes.ClusterUIDLabelKey)
+ target.Spec.DataSourceRef.Kind = dptypes.RestoreKind
+ target.Spec.DataSourceRef.Name = "source"
+ target.Annotations[constant.RestoreSourceKindAnnotationKey] = dptypes.RestoreKind
+ target.Annotations[constant.RestoreSourceNameAnnotationKey] = "source"
+ base := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, component, its, target).Build()
+ validated := false
+ cli := interceptor.NewClient(base, interceptor.Funcs{Get: func(ctx context.Context, inner client.WithWatch,
+ key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+ if _, ok := obj.(*dpv1alpha1.Restore); ok {
+ validated = true
+ }
+ return inner.Get(ctx, key, obj, opts...)
+ }})
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme}
+
+ require.ErrorContains(t, vp.syncPVC(intctrlutil.RequestCtx{Ctx: ctx}, target),
+ "waiting for target PVC restore protection")
+ require.False(t, validated)
+ require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(target), target))
+ require.Contains(t, target.Finalizers, dptypes.DataProtectionFinalizerName)
+}
+
+func TestClusterLifecyclePostReadyProtectionHandoff(t *testing.T) {
+ ctx := context.Background()
+ scheme, cluster, component, _, target := parentRestoreObjects(t)
+ target.Labels[dptypes.ComponentUIDLabelKey] = string(component.UID)
+ target.Finalizers = nil
+ target.Spec.VolumeName = "target-pv"
+ target.Status.Conditions = []corev1.PersistentVolumeClaimCondition{{
+ Type: PersistentVolumeClaimPopulating, Status: corev1.ConditionTrue, Reason: ReasonPopulatingProvisioned,
+ }}
+ component.Status.Phase = kbappsv1.RunningComponentPhase
+ cluster.Status.Conditions = []metav1.Condition{{
+ Type: kbappsv1.ConditionTypeRestore, Status: metav1.ConditionTrue,
+ }}
+ backup, actionSet := restoreBackupObjects()
+ actionSet.Spec.Restore.PostReady = []dpv1alpha1.ActionSpec{{Job: &dpv1alpha1.JobActionSpec{}}}
+ worker := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "worker", Namespace: target.Namespace}}
+ for key, value := range map[string]string{
+ dptypes.CfgKeyWorkerServiceAccountName: "worker",
+ dptypes.CfgKeyWorkerClusterRoleName: "worker-role",
+ } {
+ previous := viper.Get(key)
+ viper.Set(key, value)
+ t.Cleanup(func() { viper.Set(key, previous) })
+ }
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(target).
+ WithObjects(cluster, component, target, backup, actionSet, worker).Build()
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme, Recorder: record.NewFakeRecorder(10)}
+ mgr := dprestore.NewRestoreManager(&dpv1alpha1.Restore{}, nil, scheme, cli)
+ mgr.PostReadyBackupSets = []dprestore.BackupActionSet{{Backup: backup}}
+ restoreCtx := &pvcRestoreContext{restoreMgr: mgr, mode: pvcRestoreModeProvisionOnly}
+ reqCtx := intctrlutil.RequestCtx{Ctx: ctx}
+
+ require.ErrorContains(t, vp.completeBoundPVCIfNeeded(reqCtx, target, restoreCtx),
+ "waiting for target PVC restore protection")
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(target), target))
+ require.Contains(t, target.Finalizers, dptypes.DataProtectionFinalizerName)
+ restores := &dpv1alpha1.RestoreList{}
+ require.NoError(t, cli.List(ctx, restores))
+ require.Empty(t, restores.Items)
+
+ require.ErrorContains(t, vp.completeBoundPVCIfNeeded(reqCtx, target, restoreCtx), "waiting for postReady restore")
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(target), target))
+ require.Contains(t, target.Finalizers, dptypes.DataProtectionFinalizerName)
+ require.NoError(t, cli.List(ctx, restores))
+ require.Len(t, restores.Items, 1)
+
+ require.ErrorContains(t, vp.completeBoundPVCIfNeeded(reqCtx, target, restoreCtx), "waiting for postReady restore")
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(target), target))
+ require.NotContains(t, target.Finalizers, dptypes.DataProtectionFinalizerName)
+ coordinator := &ClusterRestoreReconciler{Client: cli}
+ _, err := coordinator.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(cluster)})
+ require.NoError(t, err)
+ require.NoError(t, cli.Get(ctx, client.ObjectKeyFromObject(cluster), cluster))
+ require.True(t, isClusterRestoreProtected(cluster))
+}
+
+func TestClusterLifecycleSafetyBoundaries(t *testing.T) {
+ t.Run("target deletion is not termination", func(t *testing.T) {
+ scheme, cluster, component, its, target := parentRestoreObjects(t)
+ now := metav1.Now()
+ target.DeletionTimestamp = &now
+ target.Finalizers = []string{dptypes.DataProtectionFinalizerName, "example.io/app-owner"}
+ target.Labels[dptypes.ComponentUIDLabelKey] = string(component.UID)
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, component, its, target).Build()
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme}
+
+ terminated, err := vp.handleRestoreClusterLifecycle(intctrlutil.RequestCtx{Ctx: context.Background()}, target)
+ require.NoError(t, err)
+ require.False(t, terminated)
+ })
+
+ t.Run("deleting target without VP finalizer is ignored", func(t *testing.T) {
+ scheme, cluster, _, _, target := parentRestoreObjects(t)
+ now := metav1.Now()
+ target.DeletionTimestamp = &now
+ target.Finalizers = []string{"kubernetes.io/pvc-protection"}
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, target).Build()
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme}
+ require.NoError(t, vp.syncPVC(intctrlutil.RequestCtx{Ctx: context.Background()}, target))
+ require.NoError(t, cli.Get(context.Background(), client.ObjectKeyFromObject(target), target))
+ require.Equal(t, []string{"kubernetes.io/pvc-protection"}, target.Finalizers)
+ restores := &dpv1alpha1.RestoreList{}
+ require.NoError(t, cli.List(context.Background(), restores))
+ require.Empty(t, restores.Items)
+ })
+
+ t.Run("Cluster deletion cleans postReady after target protection handoff", func(t *testing.T) {
+ scheme, cluster, component, _, target := parentRestoreObjects(t)
+ now := metav1.Now()
+ cluster.DeletionTimestamp = &now
+ cluster.Finalizers = append(cluster.Finalizers, "example.io/app-owner")
+ target.DeletionTimestamp = &now
+ target.Finalizers = []string{"example.io/app-owner"}
+ postReady := postReadyRestoreForComponent(target, cluster, component)
+ postReady.Finalizers = []string{"example.io/restore-owner"}
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, target, postReady).Build()
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme}
+
+ err := vp.syncPVC(intctrlutil.RequestCtx{Ctx: context.Background()}, target)
+
+ require.ErrorContains(t, err, "waiting for Restore owners")
+ require.NoError(t, cli.Get(context.Background(), client.ObjectKeyFromObject(postReady), postReady))
+ require.False(t, postReady.DeletionTimestamp.IsZero())
+ require.NoError(t, cli.Get(context.Background(), client.ObjectKeyFromObject(target), target))
+ require.NotContains(t, target.Finalizers, dptypes.DataProtectionFinalizerName)
+ })
+
+ t.Run("missing Cluster does not authorize active cleanup", func(t *testing.T) {
+ scheme, cluster, component, _, target := parentRestoreObjects(t)
+ target.Labels[dptypes.ComponentUIDLabelKey] = string(component.UID)
+ target.Finalizers = []string{dptypes.DataProtectionFinalizerName}
+ helper := restoreHelperForTarget(target, cluster)
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(target, helper).
+ WithInterceptorFuncs(interceptor.Funcs{
+ Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error {
+ t.Fatal("missing Cluster must not authorize deletion")
+ return nil
+ },
+ Patch: func(context.Context, client.WithWatch, client.Object, client.Patch, ...client.PatchOption) error {
+ t.Fatal("active protection must remain")
+ return nil
+ },
+ }).Build()
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme}
+ require.True(t, intctrlutil.IsRequeueError(vp.syncPVC(
+ intctrlutil.RequestCtx{Ctx: context.Background()}, target)))
+ })
+
+ t.Run("completed Cluster restore does not register", func(t *testing.T) {
+ scheme, cluster, component, its, target := parentRestoreObjects(t)
+ cluster.Status.Conditions = []metav1.Condition{{
+ Type: kbappsv1.ConditionTypeRestore, Status: metav1.ConditionTrue,
+ }}
+ delete(target.Labels, dptypes.ClusterUIDLabelKey)
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, component, its, target).Build()
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme}
+ err := vp.syncPVC(intctrlutil.RequestCtx{Ctx: context.Background()}, target)
+ require.ErrorContains(t, err, "Cluster restore is no longer active")
+ require.NoError(t, cli.Get(context.Background(), client.ObjectKeyFromObject(target), target))
+ require.NotContains(t, target.Finalizers, dptypes.DataProtectionFinalizerName)
+ })
+}
+
+func TestClusterLifecycleRefusesForeignExecutionRestore(t *testing.T) {
+ scheme, cluster, component, _, target := parentRestoreObjects(t)
+ now := metav1.Now()
+ cluster.DeletionTimestamp = &now
+ target.Finalizers = []string{dptypes.DataProtectionFinalizerName}
+ target.Labels[dptypes.ComponentUIDLabelKey] = string(component.UID)
+ helper := restoreHelperForTarget(target, cluster)
+ foreign := executionRestoreForTarget(target, cluster)
+ foreign.OwnerReferences[0].UID = "foreign-pvc"
+ cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cluster, target, helper, foreign).Build()
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme}
+
+ terminated, err := vp.handleRestoreClusterLifecycle(intctrlutil.RequestCtx{Ctx: context.Background()}, target)
+ require.True(t, terminated)
+ require.ErrorContains(t, err, "refusing to delete execution Restore")
+ require.NoError(t, cli.Get(context.Background(), client.ObjectKeyFromObject(foreign), &dpv1alpha1.Restore{}))
+ require.NoError(t, cli.Get(context.Background(), client.ObjectKeyFromObject(helper), &corev1.PersistentVolumeClaim{}))
+}
+
+func TestValidateClusterRestorePVCOwnershipThroughInstance(t *testing.T) {
+ scheme, cluster, component, its, target := parentRestoreObjects(t)
+ controller := true
+ instance := &workloadsv1.Instance{ObjectMeta: metav1.ObjectMeta{
+ Namespace: target.Namespace, Name: "cluster-mysql-0", UID: "instance-uid",
+ OwnerReferences: []metav1.OwnerReference{{
+ APIVersion: workloadsv1.GroupVersion.String(), Kind: workloadsv1.InstanceSetKind,
+ Name: its.Name, UID: its.UID, Controller: &controller,
+ }},
+ }}
+ target.OwnerReferences = []metav1.OwnerReference{{
+ APIVersion: workloadsv1.GroupVersion.String(), Kind: "Instance",
+ Name: instance.Name, UID: instance.UID, Controller: &controller,
+ }}
+ cli := fake.NewClientBuilder().WithScheme(scheme).
+ WithObjects(cluster, component, its, instance, target).Build()
+ vp := &VolumePopulatorReconciler{Client: cli, Scheme: scheme}
+
+ actual, err := vp.validateClusterRestorePVCOwnership(context.Background(), target, cluster)
+ require.NoError(t, err)
+ require.Equal(t, component.UID, actual.UID)
+}
+
+func restoreBackupObjects() (*dpv1alpha1.Backup, *dpv1alpha1.ActionSet) {
+ backup := newBackupForRestoreDecision([]string{"data"}, nil)
+ backup.Status.Phase = dpv1alpha1.BackupPhaseCompleted
+ backup.Status.BackupMethod.ActionSetName = "full"
+ backup.Status.Target.PodSelector = &dpv1alpha1.PodSelector{Strategy: dpv1alpha1.PodSelectionStrategyAny}
+ actionSet := &dpv1alpha1.ActionSet{ObjectMeta: metav1.ObjectMeta{Name: "full"}, Spec: dpv1alpha1.ActionSetSpec{
+ BackupType: dpv1alpha1.BackupTypeFull,
+ Restore: &dpv1alpha1.RestoreActionSpec{PrepareData: &dpv1alpha1.JobActionSpec{}},
+ }}
+ return backup, actionSet
+}
+
+func parentRestoreObjects(t *testing.T) (*runtime.Scheme, *kbappsv1.Cluster, *kbappsv1.Component,
+ *workloadsv1.InstanceSet, *corev1.PersistentVolumeClaim) {
+ t.Helper()
+ scheme := runtime.NewScheme()
+ require.NoError(t, corev1.AddToScheme(scheme))
+ require.NoError(t, kbappsv1.AddToScheme(scheme))
+ require.NoError(t, workloadsv1.AddToScheme(scheme))
+ require.NoError(t, dpv1alpha1.AddToScheme(scheme))
+ controller := true
+ cluster := &kbappsv1.Cluster{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: "default", Name: "cluster", UID: "cluster-uid",
+ Finalizers: []string{dptypes.RestoreProtectionFinalizerName},
+ },
+ Spec: kbappsv1.ClusterSpec{Restore: &kbappsv1.ClusterRestore{}},
+ }
+ component := &kbappsv1.Component{ObjectMeta: metav1.ObjectMeta{
+ Namespace: cluster.Namespace, Name: "cluster-mysql", UID: "component-uid",
+ Labels: map[string]string{
+ constant.AppInstanceLabelKey: cluster.Name, constant.KBAppComponentLabelKey: "mysql",
+ },
+ OwnerReferences: []metav1.OwnerReference{{
+ APIVersion: kbappsv1.GroupVersion.String(), Kind: kbappsv1.ClusterKind,
+ Name: cluster.Name, UID: cluster.UID, Controller: &controller,
+ }},
+ }}
+ its := &workloadsv1.InstanceSet{ObjectMeta: metav1.ObjectMeta{
+ Namespace: cluster.Namespace, Name: "cluster-mysql", UID: "its-uid",
+ Labels: map[string]string{
+ constant.AppInstanceLabelKey: cluster.Name, constant.KBAppComponentLabelKey: "mysql",
+ },
+ OwnerReferences: []metav1.OwnerReference{{
+ APIVersion: kbappsv1.GroupVersion.String(), Kind: kbappsv1.ComponentKind,
+ Name: component.Name, UID: component.UID, Controller: &controller,
+ }},
+ }}
+ target := dependencyRestorePVC("data-mysql-0", "mysql", "target-uid")
+ target.Annotations[constant.KBAppClusterUIDKey] = string(cluster.UID)
+ target.Labels[dptypes.ClusterUIDLabelKey] = string(cluster.UID)
+ target.OwnerReferences = []metav1.OwnerReference{{
+ APIVersion: workloadsv1.GroupVersion.String(), Kind: workloadsv1.InstanceSetKind,
+ Name: its.Name, UID: its.UID, Controller: &controller,
+ }}
+ return scheme, cluster, component, its, target
+}
+
+func restoreHelperForTarget(target *corev1.PersistentVolumeClaim,
+ cluster *kbappsv1.Cluster) *corev1.PersistentVolumeClaim {
+ name := getPopulatePVCName(target.UID)
+ return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{
+ Namespace: target.Namespace, Name: name,
+ Labels: map[string]string{
+ constant.AppInstanceLabelKey: cluster.Name,
+ constant.KBAppComponentLabelKey: target.Labels[constant.KBAppComponentLabelKey],
+ dptypes.ClusterUIDLabelKey: string(cluster.UID),
+ dprestore.DataProtectionRestoreLabelKey: name,
+ dprestore.DataProtectionRestoreNamespaceLabelKey: target.Namespace,
+ dprestore.DataProtectionPopulatePVCLabelKey: name,
+ },
+ }}
+}
+
+func executionRestoreForTarget(target *corev1.PersistentVolumeClaim,
+ cluster *kbappsv1.Cluster) *dpv1alpha1.Restore {
+ name := getPopulatePVCName(target.UID)
+ return &dpv1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{
+ Namespace: target.Namespace, Name: name,
+ Labels: map[string]string{
+ constant.AppInstanceLabelKey: cluster.Name,
+ constant.KBAppComponentLabelKey: target.Labels[constant.KBAppComponentLabelKey],
+ dptypes.ClusterUIDLabelKey: string(cluster.UID),
+ dprestore.DataProtectionRestoreLabelKey: name,
+ },
+ OwnerReferences: []metav1.OwnerReference{{
+ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "PersistentVolumeClaim",
+ Name: target.Name, UID: target.UID,
+ }},
+ }}
+}
+
+func postReadyRestoreForComponent(target *corev1.PersistentVolumeClaim, cluster *kbappsv1.Cluster,
+ component *kbappsv1.Component) *dpv1alpha1.Restore {
+ name := postReadyRestoreName(component.UID)
+ return &dpv1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{
+ Namespace: target.Namespace, Name: name,
+ Labels: map[string]string{
+ constant.AppInstanceLabelKey: cluster.Name,
+ constant.KBAppComponentLabelKey: target.Labels[constant.KBAppComponentLabelKey],
+ dptypes.ClusterUIDLabelKey: string(cluster.UID),
+ dptypes.ComponentUIDLabelKey: string(component.UID),
+ dprestore.DataProtectionRestoreLabelKey: name,
+ },
+ OwnerReferences: []metav1.OwnerReference{{
+ APIVersion: kbappsv1.GroupVersion.String(), Kind: kbappsv1.ComponentKind,
+ Name: component.Name, UID: component.UID,
+ }},
+ }}
+}
+
func TestEnsurePostReadyRestore_ShardingMissingTargetSkip_DoesNotRedirect(t *testing.T) {
scheme := runtime.NewScheme()
require.NoError(t, corev1.AddToScheme(scheme))
diff --git a/pkg/dataprotection/types/constant.go b/pkg/dataprotection/types/constant.go
index b653dc6e86a..810adca70ff 100644
--- a/pkg/dataprotection/types/constant.go
+++ b/pkg/dataprotection/types/constant.go
@@ -46,6 +46,9 @@ const (
const (
// DataProtectionFinalizerName is the name of our custom finalizer
DataProtectionFinalizerName = "dataprotection.kubeblocks.io/finalizer"
+ // RestoreProtectionFinalizerName prevents Cluster deletion from completing
+ // before its restore resources have been cleaned up.
+ RestoreProtectionFinalizerName = "dataprotection.kubeblocks.io/restore-protection-finalizer"
)
// annotation keys