Feature deterministic input outcomes and replay verification

Makefile

@@ -61,6 +61,12 @@ endif
 # Go artifacts
 GO_ARTIFACTS := $(addprefix cartesi-rollups-,node cli evm-reader advancer validator claimer jsonrpc-api prt machine-tool)
 
+# These artifacts embed the machine runtime and therefore require libcartesi.
+# Keep this list explicit: every other artifact is built with CGO_ENABLED=0, so
+# the normal build fails if a C dependency leaks into a pure service or tool.
+MACHINE_GO_ARTIFACTS := cartesi-rollups-node cartesi-rollups-advancer
+PURE_GO_ARTIFACTS := $(filter-out $(MACHINE_GO_ARTIFACTS),$(GO_ARTIFACTS))
+
 # fixme(vfusco): path on all oses
 CGO_CFLAGS:= -I$(PREFIX)/include
 CGO_LDFLAGS:= -L$(PREFIX)/lib
@@ -70,17 +76,25 @@ export CGO_LDFLAGS
 CARTESI_TEST_MACHINE_IMAGES_PATH:= $(PREFIX)/share/cartesi-machine/images/
 export CARTESI_TEST_MACHINE_IMAGES_PATH
 
-GO_BUILD_PARAMS := -ldflags "-s -w -X 'github.com/cartesi/rollups-node/internal/version.BuildVersion=$(ROLLUPS_NODE_VERSION)' -r $(PREFIX)/lib"
+GO_VERSION_LDFLAGS := -s -w -X 'github.com/cartesi/rollups-node/internal/version.BuildVersion=$(ROLLUPS_NODE_VERSION)'
+PURE_GO_BUILD_PARAMS := -ldflags "$(GO_VERSION_LDFLAGS)"
+MACHINE_GO_BUILD_PARAMS := -ldflags "$(GO_VERSION_LDFLAGS) -r $(PREFIX)/lib"
 ifeq ($(BUILD_TYPE),debug)
-	GO_BUILD_PARAMS += -gcflags "all=-N -l"
+	PURE_GO_BUILD_PARAMS += -gcflags "all=-N -l"
+	MACHINE_GO_BUILD_PARAMS += -gcflags "all=-N -l"
 endif
 
+# Tests and Go development tools cover machine packages, so retain the
+# machine-capable parameters for those existing recipes.
+GO_BUILD_PARAMS = $(MACHINE_GO_BUILD_PARAMS)
+
 GO_TEST_PACKAGES ?= ./...
 GO_TEST_FLAGS ?=
 
 VERBOSE ?=
 ifeq ($(VERBOSE),true)
-	GO_BUILD_PARAMS += -v
+	PURE_GO_BUILD_PARAMS += -v
+	MACHINE_GO_BUILD_PARAMS += -v
 	GO_TEST_FLAGS += -v
 endif
 
@@ -147,9 +161,13 @@ env:
 # =============================================================================
 # Artifacts
 # =============================================================================
-$(GO_ARTIFACTS):
+$(PURE_GO_ARTIFACTS):
+	@echo "Building pure Go artifact $@"
+	CGO_ENABLED=0 go build $(PURE_GO_BUILD_PARAMS) ./cmd/$@
+
+$(MACHINE_GO_ARTIFACTS):
 	@echo "Building Go artifact $@"
-	go build $(GO_BUILD_PARAMS) ./cmd/$@
+	CGO_ENABLED=1 go build $(MACHINE_GO_BUILD_PARAMS) ./cmd/$@
 
 tidy-go:
 	@go mod tidy

api/openapi/inspect.yaml

@@ -18,7 +18,7 @@ paths:
       description: |
         This POST method sends an inspect-state request to the DApp backend, using the body contents as a binary payload for the inspect method.
 
-        The response includes a status string and reports generated by the DApp backend. If an exception occurs, the `exception_payload` field will contain the exception details; otherwise, this field will be null.
+        The response includes the inspection completion status, reports generated by the DApp backend, and the number of previously processed inputs. The optional `error` field contains a sanitized description when the status is `Exception` or `Failed`. For `Exception`, `exception_data` contains the raw CMIO payload supplied by the guest.
 
         The inspect operation is executed on a temporary fork of the machine created upon request arrival, which is discarded afterward. Note that this method is synchronous and not recommended for resource-intensive operations.
 
@@ -88,10 +88,17 @@ components:
       properties:
         status:
           $ref: "#/components/schemas/CompletionStatus"
-        exception_payload:
-          $ref: "#/components/schemas/Payload"
+        error:
+          type: string
+          description: Sanitized error description, present only when status is Exception or Failed
+          example: "The node could not complete the inspection"
+        exception_data:
+          allOf:
+            - $ref: "#/components/schemas/Payload"
+          description: Raw guest-provided CMIO exception payload, present only when status is Exception
         reports:
           type: array
+          description: Reports emitted before completion; for Failed, this may be only a partial prefix
           items:
             $ref: "#/components/schemas/Report"
         processed_input_count:
@@ -100,22 +107,18 @@ components:
           example: 0
       required:
         - status
-        - exception_payload
         - reports
         - processed_input_count
 
     CompletionStatus:
       type: string
-      description: Whether inspection completed or not (and why not)
-      enum:
-        [
-          Accepted,
-          Rejected,
-          Exception,
-          MachineHalted,
-          CycleLimitExceeded,
-          TimeLimitExceeded,
-        ]
+      description: |
+        How inspection completed. MachineHalted means the temporary inspect
+        execution halted; the canonical application machine is unchanged.
+        Failed means the node could not complete inspection because of an
+        operational limit, cancellation, timeout, protocol error, or internal
+        failure. Reports returned with Failed may be partial.
+      enum: [Accepted, Rejected, Exception, MachineHalted, Failed]
       example: "Accepted"
 
     Payload:

cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters.go

@@ -449,17 +449,21 @@ func setParameterValue(params *model.ExecutionParameters, parameter, value strin
 }
 
 func printParameters(params *model.ExecutionParameters) {
-	fmt.Printf("snapshot_policy: %s\n", params.SnapshotPolicy)
-	fmt.Printf("advance_inc_cycles: %d\n", params.AdvanceIncCycles)
-	fmt.Printf("advance_max_cycles: %d\n", params.AdvanceMaxCycles)
-	fmt.Printf("inspect_inc_cycles: %d\n", params.InspectIncCycles)
-	fmt.Printf("inspect_max_cycles: %d\n", params.InspectMaxCycles)
-	fmt.Printf("advance_inc_deadline: %s\n", params.AdvanceIncDeadline)
-	fmt.Printf("advance_max_deadline: %s\n", params.AdvanceMaxDeadline)
-	fmt.Printf("inspect_inc_deadline: %s\n", params.InspectIncDeadline)
-	fmt.Printf("inspect_max_deadline: %s\n", params.InspectMaxDeadline)
-	fmt.Printf("load_deadline: %s\n", params.LoadDeadline)
-	fmt.Printf("store_deadline: %s\n", params.StoreDeadline)
-	fmt.Printf("fast_deadline: %s\n", params.FastDeadline)
-	fmt.Printf("max_concurrent_inspects: %d\n", params.MaxConcurrentInspects)
+	writeParameters(os.Stdout, params)
+}
+
+func writeParameters(w io.Writer, params *model.ExecutionParameters) {
+	fmt.Fprintf(w, "snapshot_policy: %s\n", params.SnapshotPolicy)
+	fmt.Fprintf(w, "advance_inc_cycles: %d\n", params.AdvanceIncCycles)
+	fmt.Fprintf(w, "advance_max_cycles: %d\n", params.AdvanceMaxCycles)
+	fmt.Fprintf(w, "inspect_inc_cycles: %d\n", params.InspectIncCycles)
+	fmt.Fprintf(w, "inspect_max_cycles: %d\n", params.InspectMaxCycles)
+	fmt.Fprintf(w, "advance_inc_deadline: %s\n", params.AdvanceIncDeadline)
+	fmt.Fprintf(w, "advance_max_deadline: %s\n", params.AdvanceMaxDeadline)
+	fmt.Fprintf(w, "inspect_inc_deadline: %s\n", params.InspectIncDeadline)
+	fmt.Fprintf(w, "inspect_max_deadline: %s\n", params.InspectMaxDeadline)
+	fmt.Fprintf(w, "load_deadline: %s\n", params.LoadDeadline)
+	fmt.Fprintf(w, "store_deadline: %s\n", params.StoreDeadline)
+	fmt.Fprintf(w, "fast_deadline: %s\n", params.FastDeadline)
+	fmt.Fprintf(w, "max_concurrent_inspects: %d\n", params.MaxConcurrentInspects)
 }

cmd/cartesi-rollups-cli/root/app/execution-parameters/execution_parameters_test.go

@@ -0,0 +1,38 @@
+// (c) Cartesi and individual authors (see AUTHORS)
+// SPDX-License-Identifier: Apache-2.0 (see LICENSE)
+
+package execution
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/cartesi/rollups-node/internal/model"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCycleMaximumParameters(t *testing.T) {
+	params := &model.ExecutionParameters{}
+	require.NoError(t, setParameterValue(params, "advance_max_cycles", "123"))
+	require.NoError(t, setParameterValue(params, "inspect_max_cycles", "456"))
+	value, err := getParameterValue(params, "advance_max_cycles")
+	require.NoError(t, err)
+	require.Equal(t, "123", value)
+	value, err = getParameterValue(params, "inspect_max_cycles")
+	require.NoError(t, err)
+	require.Equal(t, "456", value)
+}
+
+func TestWriteParametersIncludesCycleMaximums(t *testing.T) {
+	params := &model.ExecutionParameters{
+		AdvanceIncCycles: 11, AdvanceMaxCycles: 12,
+		InspectIncCycles: 21, InspectMaxCycles: 22,
+	}
+	var output bytes.Buffer
+	writeParameters(&output, params)
+
+	require.Contains(t, output.String(), "advance_inc_cycles: 11")
+	require.Contains(t, output.String(), "advance_max_cycles: 12")
+	require.Contains(t, output.String(), "inspect_inc_cycles: 21")
+	require.Contains(t, output.String(), "inspect_max_cycles: 22")
+}

internal/advancer/advancer.go

@@ -81,18 +81,18 @@ func (s *Service) Step(ctx context.Context) (bool, error) {
 	}
 
 	// Update the machine manager with any new or disabled applications
-	err := s.machineManager.UpdateMachines(ctx)
-	if err != nil {
-		return false, err
+	updateErr := s.machineManager.UpdateMachines(ctx)
+	if updateErr != nil && !manager.IsOnlyApplicationFailurePersistenceErrors(updateErr) {
+		return false, updateErr
 	}
 
 	// Get all applications with active machines (returned sorted by ID).
 	apps := s.machineManager.Applications()
 	if len(apps) == 0 {
-		return false, nil
+		return false, updateErr
 	}
 	anyWork := false
-	var errs []error
+	errs := []error{updateErr}
 	for _, app := range apps {
 		hadWork, err := s.stepApp(ctx, app)
 		if err != nil {
@@ -248,33 +248,25 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs []
 		result, err := machine.Advance(ctx, input.RawData, input.EpochIndex, input.Index, app.IsDaveConsensus())
 		input.RawData = nil // allow GC to collect payload while batch continues
 		if err != nil {
-			// Graceful shutdown: bail out quietly without marking FAILED.
-			if errors.Is(err, context.Canceled) {
-				s.Logger.Debug("Advance cancelled due to shutdown",
+			// Cancellation of this service context is the normal shutdown path,
+			// so it does not change the application's status. A returned error
+			// that happens to include context.Canceled is still a failure while
+			// the service context remains active.
+			if errors.Is(ctx.Err(), context.Canceled) {
+				s.Logger.Debug("Advance stopped because the service is shutting down",
 					"application", app.Name,
-					"index", input.Index)
+					"index", input.Index,
+					"error", err)
 				return err
 			}
 
-			// Anything else (including DeadlineExceeded) is a real failure.
+			// Anything else, including a deadline, is an execution failure.
 			s.Logger.Error("Error executing advance",
 				"application", app.Name,
 				"index", input.Index,
 				"error", err)
 
-			// DeadlineExceeded is a real failure but not a state-corruption
-			// signal — let the upper layer retry rather than marking FAILED.
-			if errors.Is(err, context.DeadlineExceeded) {
-				return err
-			}
-
-			if dbErr := appstatus.SetFailed(ctx, s.Logger, s.repository, app, err.Error()); dbErr != nil {
-				s.Logger.Error("Failed to persist FAILED status — machine will be closed "+
-					"but the app status remains unchanged in DB; it may be re-created "+
-					"from the last snapshot on the next tick. If the root cause "+
-					"persists, this may loop.",
-					"application", app.Name, "db_error", dbErr)
-			}
+			s.markApplicationFailed(ctx, app, err.Error())
 
 			// Eagerly close the machine to release the child process.
 			// The app has failed, so no further operations will succeed.
@@ -297,26 +289,41 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs []
 			"status", result.Status,
 			"outputs", len(result.Outputs),
 			"reports", len(result.Reports),
-			"hashes", len(result.Hashes),
-			"remaining_cycles", result.RemainingMetaCycles,
+			"periodic_state_hashes", len(result.PeriodicStateHashes),
+			"padding_repetitions", result.PaddingRepetitions,
 		)
 
 		// Store the result in the database
 		err = s.repository.StoreAdvanceResult(ctx, input.EpochApplicationID, result)
 		if err != nil {
-			// Machine state is now ahead of the database. This desync is
-			// unrecoverable without a restart — regardless of whether the
-			// failure was a DB error or a context timeout. Shut down the
-			// node so it can restart cleanly from the last snapshot.
+			// Advance has already changed the live machine, but the transaction
+			// did not confirm that its result was saved. The database may still
+			// show this input as pending. Reusing this machine could then execute
+			// the input again from the wrong state, so StoreAdvanceResult is not
+			// retried against this live machine.
 			s.Logger.Error(
-				"FATAL: failed to store advance result after machine state "+
-					"was already updated — shutting down to prevent permanent desync",
+				"Could not confirm that the advance result was saved; "+
+					"the live machine has already advanced, so services will stop; "+
+					"after the node is restarted, execution will use persisted state",
 				"application", app.Name,
 				"epoch", input.EpochIndex,
 				"index", input.Index,
 				"error", err)
+
+			// Try to close the machine now so the already-advanced runtime cannot
+			// be used again. Cancel services even if Close fails. After the node
+			// is restarted, the machine is rebuilt from persisted state, and the
+			// database decides whether this input is still pending and needs a
+			// safe retry.
+			closeErr := machine.Close()
 			s.Cancel() // triggers graceful shutdown of all services
-			return err
+			if closeErr != nil {
+				s.Logger.Error("Could not close the machine after its advance result "+
+					"was not confirmed saved; service shutdown is still required",
+					"application", app.Name,
+					"error", closeErr)
+			}
+			return errors.Join(err, closeErr)
 		}
 
 		// Create a snapshot if needed
@@ -341,6 +348,20 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs []
 	return nil
 }
 
+// markApplicationFailed persists FAILED or installs a local fence when the
+// status write cannot be confirmed. Keeping those operations together prevents
+// a failed application from being handed more work while durability is retried.
+func (s *Service) markApplicationFailed(ctx context.Context, app *Application, reason string) {
+	if err := appstatus.SetFailed(ctx, s.Logger, s.repository, app, reason); err != nil {
+		s.machineManager.FenceApplicationFailure(app, reason)
+		s.Logger.Error(
+			"Could not persist FAILED application status; the application remains fenced until the write is retried",
+			"application", app.Name,
+			"db_error", err,
+		)
+	}
+}
+
 func (s *Service) isEpochLastInput(ctx context.Context, app *Application, input *Input) (bool, error) {
 	if app == nil || input == nil {
 		return false, fmt.Errorf("application and input must not be nil")
@@ -408,10 +429,7 @@ func (s *Service) handleEpochAfterInputsProcessed(ctx context.Context, app *Appl
 			// If the runtime was destroyed (e.g., child process crashed),
 			// mark the app as failed to avoid an infinite retry loop.
 			if errors.Is(err, manager.ErrMachineClosed) {
-				if dbErr := appstatus.SetFailed(ctx, s.Logger, s.repository, app, err.Error()); dbErr != nil {
-					s.Logger.Error("Failed to persist FAILED status for crashed machine",
-						"application", app.Name, "db_error", dbErr)
-				}
+				s.markApplicationFailed(ctx, app, err.Error())
 			}
 			return fmt.Errorf("failed to get outputs proof from machine: %w", err)
 		}