Skip to content

fix(dataprotection): identify and route restore dependencies #10844

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
leon-ape merged 1 commit into from
Sep 4, 2026
Merged

fix(dataprotection): identify and route restore dependencies #10844

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions controllers/apps/cluster/restore_intent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func injectRestoreIntentToVCT(cluster *appsv1.Cluster, componentName string, vct
vct.Annotations[constant.RestoreSourceKindAnnotationKey] = restore.Source.Kind
vct.Annotations[constant.RestoreSourceNameAnnotationKey] = restore.Source.Name
vct.Annotations[constant.RestoreSourceNamespaceAnnotationKey] = sourceNamespace
vct.Annotations[constant.KBAppClusterUIDKey] = string(cluster.UID)
vct.Annotations[constant.RestoreComponentAnnotationKey] = componentName
vct.Annotations[constant.RestoreVolumeTemplateAnnotationKey] = vct.Name
delete(vct.Annotations, constant.RestorePITRAnnotationKey)
Expand Down Expand Up @@ -118,6 +119,7 @@ func cleanupRestoreIntentFromVCT(vct *appsv1.PersistentVolumeClaimTemplate) {
delete(vct.Annotations, constant.RestoreSourceKindAnnotationKey)
delete(vct.Annotations, constant.RestoreSourceNameAnnotationKey)
delete(vct.Annotations, constant.RestoreSourceNamespaceAnnotationKey)
delete(vct.Annotations, constant.KBAppClusterUIDKey)
delete(vct.Annotations, constant.RestorePITRAnnotationKey)
delete(vct.Annotations, constant.RestoreParametersAnnotationKey)
delete(vct.Annotations, constant.RestoreComponentAnnotationKey)
Expand All @@ -138,6 +140,7 @@ func hasRestoreIntent(vct *appsv1.PersistentVolumeClaimTemplate) bool {
constant.RestoreSourceKindAnnotationKey,
constant.RestoreSourceNameAnnotationKey,
constant.RestoreSourceNamespaceAnnotationKey,
constant.KBAppClusterUIDKey,
constant.RestoreComponentAnnotationKey,
constant.RestoreVolumeTemplateAnnotationKey,
} {
Expand Down
4 changes: 4 additions & 0 deletions controllers/apps/cluster/restore_intent_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ func TestInjectRestoreIntentRemovesStaleOptionalAnnotations(t *testing.T) {
cluster := &appsv1.Cluster{}
cluster.Name = "test-cluster"
cluster.Namespace = "test-ns"
cluster.UID = "cluster-uid"
cluster.Spec.Restore = &appsv1.ClusterRestore{
Source: appsv1.ClusterRestoreSource{
APIGroup: testRestoreSourceAPIGroup,
Expand All @@ -72,6 +73,7 @@ func TestInjectRestoreIntentRemovesStaleOptionalAnnotations(t *testing.T) {
require.Equal(t, "backup", vct.Spec.DataSourceRef.Name)
require.NotNil(t, vct.Spec.DataSourceRef.Namespace)
require.Equal(t, "backup-ns", *vct.Spec.DataSourceRef.Namespace)
require.Equal(t, string(cluster.UID), vct.Annotations[constant.KBAppClusterUIDKey])
}

func TestInjectRestoreIntentOmitsDataSourceRefNamespaceForSameNamespaceSource(t *testing.T) {
Expand Down Expand Up @@ -154,6 +156,7 @@ func TestApplyClusterRestoreIntentCleansTemplatesAfterRestoreCompleted(t *testin
Annotations: map[string]string{
constant.RestoreSourceKindAnnotationKey: testRestoreSourceKind,
constant.RestorePITRAnnotationKey: "stale-pitr",
constant.KBAppClusterUIDKey: "cluster-uid",
},
Spec: corev1.PersistentVolumeClaimSpec{
DataSourceRef: &corev1.TypedObjectReference{
Expand All @@ -172,6 +175,7 @@ func TestApplyClusterRestoreIntentCleansTemplatesAfterRestoreCompleted(t *testin
require.Nil(t, vct.Annotations)
require.NotContains(t, vct.Annotations, constant.RestoreSourceKindAnnotationKey)
require.NotContains(t, vct.Annotations, constant.RestorePITRAnnotationKey)
require.NotContains(t, vct.Annotations, constant.KBAppClusterUIDKey)
}

func TestApplyClusterRestoreIntentKeepsNonRestoreDataSourceAfterRestoreCompleted(t *testing.T) {
Expand Down
93 changes: 67 additions & 26 deletions controllers/dataprotection/volumepopulator_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,35 +156,32 @@ func (r *VolumePopulatorReconciler) mapRestoreToPVCs(ctx context.Context, obj cl
pvc := &corev1.PersistentVolumeClaim{}
key := types.NamespacedName{Namespace: restore.Namespace, Name: owner.Name}
if err := r.Client.Get(ctx, key, pvc); err != nil || pvc.UID != owner.UID ||
!isClusterRestorePVC(pvc) || restore.Name != getPopulatePVCName(pvc.UID) {
!isClusterRestorePVC(pvc) || restore.Name != getPopulatePVCName(pvc.UID) ||
restore.Labels[dptypes.ClusterUIDLabelKey] != clusterRestorePVCUID(pvc) {
return nil
}
return []reconcile.Request{{NamespacedName: key}}
}

owner := exactOwnerReference(restore.OwnerReferences, appsv1.GroupVersion.String(), "Component")
if owner == nil {
// A Component can disappear before a postReady Restore deletion event is
// observed. The Restore carries enough correlation identity to notify its
// PVC dependents without resolving the live Component.
if internalPostReadyRestoreOwner(restore) == nil {
return nil
}
comp := &appsv1.Component{}
key := types.NamespacedName{Namespace: restore.Namespace, Name: owner.Name}
if err := r.Client.Get(ctx, key, comp); err != nil || comp.UID != owner.UID ||
restore.Name != postReadyRestoreName(comp.UID) {
return nil
}
clusterName := comp.Labels[constant.AppInstanceLabelKey]
componentName := restore.Labels[constant.KBAppComponentLabelKey]
ownerComponentName := comp.Labels[constant.KBAppComponentLabelKey]
if clusterName == "" || componentName == "" || ownerComponentName == "" ||
restore.Labels[constant.AppInstanceLabelKey] != clusterName {
clusterName := restore.Labels[constant.AppInstanceLabelKey]
if clusterName == "" || restore.Labels[constant.KBAppComponentLabelKey] == "" {
return nil
}
// A postReady Restore is owned by its target Component, while its labels
// identify only the first source PVC that created it. Other Components in
// the Cluster can wait on the same Restore through postReady redirection.
includeTerminal := !restore.DeletionTimestamp.IsZero() ||
restore.Status.Phase == dpv1alpha1.RestorePhaseCompleted ||
restore.Status.Phase == dpv1alpha1.RestorePhaseFailed
// The component label identifies the first source PVC, while the owner
// reference identifies the target Component. Redirected postReady restores
// can therefore have dependents in other Components of the same Cluster.
return r.mapRestorePVCs(ctx, restore.Namespace, client.MatchingLabels{
constant.AppInstanceLabelKey: clusterName,
})
}, restore.Labels[dptypes.ClusterUIDLabelKey], includeTerminal)
}

func (r *VolumePopulatorReconciler) mapComponentToPVCs(ctx context.Context, obj client.Object) []reconcile.Request {
Expand All @@ -197,13 +194,16 @@ func (r *VolumePopulatorReconciler) mapComponentToPVCs(ctx context.Context, obj
if clusterName == "" || componentName == "" {
return nil
}
// A PVC can depend on another Component through a redirected postReady
// Restore. That relationship is derived from Backup status and is not
// represented on the Component, so a Component event must fan out to all
// active restore PVCs in its Cluster.
clusterOwner := exactOwnerReference(comp.OwnerReferences, appsv1.GroupVersion.String(), appsv1.ClusterKind)
if clusterOwner == nil || clusterOwner.Name != clusterName {
return nil
}
// A PVC can depend on another Component through redirected postReady. The
// dependency is not represented on the Component, so normal Component
// changes fan out to unfinished restore PVCs in the exact Cluster instance.
return r.mapRestorePVCs(ctx, comp.Namespace, client.MatchingLabels{
constant.AppInstanceLabelKey: clusterName,
})
}, string(clusterOwner.UID), false)
}

func (r *VolumePopulatorReconciler) mapClusterToPVCs(ctx context.Context, obj client.Object) []reconcile.Request {
Expand All @@ -213,19 +213,23 @@ func (r *VolumePopulatorReconciler) mapClusterToPVCs(ctx context.Context, obj cl
}
return r.mapRestorePVCs(ctx, cluster.Namespace, client.MatchingLabels{
constant.AppInstanceLabelKey: cluster.Name,
})
}, string(cluster.UID), false)
}

func (r *VolumePopulatorReconciler) mapRestorePVCs(ctx context.Context, namespace string,
labels client.MatchingLabels) []reconcile.Request {
labels client.MatchingLabels, clusterUID string, includeTerminal bool) []reconcile.Request {
if clusterUID == "" {
return nil
}
list := &corev1.PersistentVolumeClaimList{}
if err := r.Client.List(ctx, list, client.InNamespace(namespace), labels); err != nil {
return nil
}
requests := make([]reconcile.Request, 0, len(list.Items))
for i := range list.Items {
pvc := &list.Items[i]
if !isClusterRestorePVC(pvc) || pvcRestoreTerminal(pvc) {
if !isClusterRestorePVC(pvc) || clusterRestorePVCUID(pvc) != clusterUID ||
(!includeTerminal && pvcRestoreTerminal(pvc)) {
continue
}
requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(pvc)})
Expand All @@ -242,6 +246,9 @@ func isClusterRestorePVC(pvc *corev1.PersistentVolumeClaim) bool {
if pvc.Labels[constant.AppInstanceLabelKey] == "" || pvc.Labels[constant.KBAppComponentLabelKey] == "" {
return false
}
if clusterRestorePVCUID(pvc) == "" {
return false
}
for _, key := range []string{
constant.RestoreSourceAPIGroupAnnotationKey,
constant.RestoreSourceKindAnnotationKey,
Expand All @@ -259,6 +266,21 @@ func isClusterRestorePVC(pvc *corev1.PersistentVolumeClaim) bool {
restoreComponent == pvc.Labels[constant.KBAppShardTemplateLabelKey]
}

// clusterRestorePVCUID returns the Cluster correlation identity inherited
// from restore intent. A later verified label may repeat it, but conflicting
// identities are never accepted.
func clusterRestorePVCUID(pvc *corev1.PersistentVolumeClaim) string {
annotationUID := pvc.Annotations[constant.KBAppClusterUIDKey]
labelUID := pvc.Labels[dptypes.ClusterUIDLabelKey]
if annotationUID != "" && labelUID != "" && annotationUID != labelUID {
return ""
}
if labelUID != "" {
return labelUID
}
return annotationUID
}

func pvcRestoreTerminal(pvc *corev1.PersistentVolumeClaim) bool {
condition := findPVCConditionByType(pvc, appsv1.ConditionTypeRestore)
return condition != nil && (condition.Status == corev1.ConditionTrue || condition.Status == corev1.ConditionFalse)
Expand All @@ -273,6 +295,16 @@ func exactOwnerReference(refs []metav1.OwnerReference, apiVersion, kind string)
return nil
}

func internalPostReadyRestoreOwner(restore *dpv1alpha1.Restore) *metav1.OwnerReference {
owner := exactOwnerReference(restore.OwnerReferences, appsv1.GroupVersion.String(), appsv1.ComponentKind)
if owner == nil || restore.Name != postReadyRestoreName(owner.UID) ||
restore.Labels[dprestore.DataProtectionRestoreLabelKey] != restore.Name ||
restore.Labels[dptypes.ComponentUIDLabelKey] != string(owner.UID) {
return nil
}
return owner
}

func restoreDependencyPredicate() predicate.Predicate {
return predicate.Funcs{
CreateFunc: func(event.CreateEvent) bool { return true },
Expand Down Expand Up @@ -899,6 +931,9 @@ func internalRestoreLabels(pvc *corev1.PersistentVolumeClaim) map[string]string
dprestore.DataProtectionRestoreNamespaceLabelKey: pvc.Namespace,
dprestore.DataProtectionPopulatePVCLabelKey: getPopulatePVCName(pvc.UID),
}
if clusterUID := clusterRestorePVCUID(pvc); clusterUID != "" {
labels[dptypes.ClusterUIDLabelKey] = clusterUID
}
for _, key := range []string{
constant.AppInstanceLabelKey,
constant.KBAppComponentLabelKey,
Expand Down Expand Up @@ -1836,6 +1871,10 @@ func postReadyRestoreLabels(pvc *corev1.PersistentVolumeClaim, comp *appsv1.Comp
labels := map[string]string{
dprestore.DataProtectionRestoreLabelKey: restoreName,
dprestore.DataProtectionRestoreNamespaceLabelKey: pvc.Namespace,
dptypes.ComponentUIDLabelKey: string(comp.UID),
}
if clusterUID := clusterRestorePVCUID(pvc); clusterUID != "" {
labels[dptypes.ClusterUIDLabelKey] = clusterUID
}
for _, key := range []string{
constant.AppInstanceLabelKey,
Expand Down Expand Up @@ -1947,6 +1986,7 @@ func (r *VolumePopulatorReconciler) getPopulatePVC(reqCtx intctrlutil.RequestCtx
ObjectMeta: metav1.ObjectMeta{
Name: populatePVCName,
Namespace: pvc.Namespace,
Labels: internalRestoreLabels(pvc),
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: pvc.Spec.AccessModes,
Expand Down Expand Up @@ -2004,6 +2044,7 @@ func (r *VolumePopulatorReconciler) getProvisionOnlyPVC(reqCtx intctrlutil.Reque
ObjectMeta: metav1.ObjectMeta{
Name: populatePVCName,
Namespace: pvc.Namespace,
Labels: internalRestoreLabels(pvc),
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: pvc.Spec.AccessModes,
Expand Down
Loading
Loading