Merge pull request #661 from posit-dev/fix/publish-retry

fix(soci): retry transient registry errors on SOCI push

Author: ianpittwood

posit-bakery/posit_bakery/plugins/builtin/imagetools/soci.py

@@ -221,14 +221,23 @@ def run(self, dry_run: bool = False, runner: "CommandRunner | None" = None) -> S
             ).run(dry_run=dry_run, runner=runner)
 
             # 3. push converted layout -> registry, referenced by digest since
-            #    the converted layout carries no tag.
+            #    the converted layout carries no tag. Retry transient registry
+            #    errors: pushing by digest can race the same eventual-consistency
+            #    window as the pull above (e.g. cross-repo blob mounts of
+            #    just-pushed source layers).
             digest = "sha256:<dry-run>" if dry_run else self._read_converted_digest(out_layout)
-            OrasCopy(
+            push = OrasCopy(
                 oras_bin=self.oras_bin,
                 source=f"{out_layout}@{digest}",
                 destination=self.destination_ref,
                 from_oci_layout=True,
-            ).run(dry_run=dry_run, runner=runner)
+            )
+            retry_on_transient(
+                lambda: push.run(dry_run=dry_run, runner=runner),
+                policy=self.retry_policy,
+                description=f"soci push for '{self.image_target.uid}'",
+                sleep=runner.sleep if runner is not None else None,
+            )
         except BakeryToolRuntimeError as e:
             return SociConvertWorkflowResult(
                 success=False,

posit-bakery/test/plugins/builtin/imagetools/test_workflow.py

@@ -190,6 +190,41 @@ def fake_run(cmd, capture_output):
     sleep.assert_called_once()
 
 
+def test_push_retries_transient_then_succeeds(mock_target):
+    """A transient registry error on the SOCI push is retried, not fatal."""
+    from posit_bakery.retry import RetryPolicy
+
+    workflow = SociConvertWorkflow(
+        soci_bin="soci",
+        oras_bin="oras",
+        image_target=mock_target,
+        options=SociOptions(enabled=True),
+        source_ref="ghcr.io/posit-dev/test-image/tmp:merged",
+        retry_policy=RetryPolicy(max_attempts=5, initial_backoff=1.0),
+    )
+
+    attempts = {"push": 0}
+
+    def fake_run(cmd, capture_output):
+        # Second oras cp is the converted layout -> registry push; fail it transiently once.
+        if cmd[:2] == ["oras", "cp"] and "--from-oci-layout" in cmd:
+            attempts["push"] += 1
+            if attempts["push"] == 1:
+                return subprocess.CompletedProcess(args=cmd, returncode=1, stdout=b"", stderr=b"sha256:x: not found")
+        return _ok_proc(cmd)
+
+    with (
+        patch("subprocess.run", side_effect=fake_run),
+        patch("posit_bakery.retry.time.sleep") as sleep,
+        patch.object(SociConvertWorkflow, "_read_converted_digest", return_value="sha256:abc123"),
+    ):
+        result = workflow.run()
+
+    assert result.success is True
+    assert attempts["push"] == 2
+    sleep.assert_called_once()
+
+
 def test_pull_retry_uses_runner_sleep_when_runner_provided(mock_target):
     workflow = SociConvertWorkflow(
         soci_bin="soci",