sync_switch_configuration: Don't allow dueling Nexuses based on a stale blueprint

nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs

@@ -273,6 +273,7 @@ mod api_impl {
     use sled_agent_types_versions::v26;
     use sled_agent_types_versions::v30;
     use sled_agent_types_versions::v33;
+    use sled_agent_types_versions::v36;
     use sled_diagnostics::SledDiagnosticsQueryOutput;
     use std::collections::BTreeMap;
     use std::collections::BTreeSet;
@@ -773,6 +774,13 @@ mod api_impl {
             unimplemented!()
         }
 
+        async fn write_network_bootstore_config_v36(
+            _rqctx: RequestContext<Self::Context>,
+            _body: TypedBody<v36::system_networking::WriteNetworkConfigRequest>,
+        ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+            unimplemented!()
+        }
+
         async fn write_network_bootstore_config_v33(
             _rqctx: RequestContext<Self::Context>,
             _body: TypedBody<v33::system_networking::WriteNetworkConfigRequest>,

nexus/test-utils/src/starter.rs

@@ -923,7 +923,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> {
                     rack_subnet: "fd00:1122:3344:0100::/56".parse().unwrap(),
                 },
                 // TODO-correctness Can we fill this in for tests?
-                service_zone_nat_entries: None,
+                blueprint_external_networking_config: None,
             },
             generation: 1,
         };

openapi/sled-agent/sled-agent-35.0.0-93533c.json.gitstub

@@ -0,0 +1 @@
+5579a6d72e5f6be577d2b17ba940ccc0de10decd:openapi/sled-agent/sled-agent-35.0.0-93533c.json

openapi/sled-agent/sled-agent-35.0.0-93533c.json → openapi/sled-agent/sled-agent-36.0.0-187400.json

@@ -7,7 +7,7 @@
       "url": "https://oxide.computer",
       "email": "api@oxide.computer"
     },
-    "version": "35.0.0"
+    "version": "36.0.0"
   },
   "paths": {
     "/artifacts": {
@@ -3496,6 +3496,32 @@
         ],
         "additionalProperties": false
       },
+      "BlueprintExternalNetworkingConfig": {
+        "description": "External networking configuration controlled by Reconfigurator via blueprints.",
+        "type": "object",
+        "properties": {
+          "blueprint_external_networking_generation": {
+            "description": "The current generation number of the blueprint's external networking config.\n\nThis generation number is only bumped when a new blueprint is produced that changes the external networking configuration in some way.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Generation"
+              }
+            ]
+          },
+          "service_zone_nat_entries": {
+            "description": "Set of all Omicron service zone NAT entries.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/ServiceZoneNatEntries"
+              }
+            ]
+          }
+        },
+        "required": [
+          "blueprint_external_networking_generation",
+          "service_zone_nat_entries"
+        ]
+      },
       "Board": {
         "description": "A VM's mainboard.",
         "type": "object",
@@ -9779,17 +9805,17 @@
         "description": "All configuration needed to set up system-level networking.",
         "type": "object",
         "properties": {
-          "rack_network_config": {
-            "$ref": "#/components/schemas/RackNetworkConfig"
-          },
-          "service_zone_nat_entries": {
+          "blueprint_external_networking_config": {
             "nullable": true,
-            "description": "Set of all Omicron service zone NAT entries.",
+            "description": "External networking configuration specified by blueprints.",
             "allOf": [
               {
-                "$ref": "#/components/schemas/ServiceZoneNatEntries"
+                "$ref": "#/components/schemas/BlueprintExternalNetworkingConfig"
               }
             ]
+          },
+          "rack_network_config": {
+            "$ref": "#/components/schemas/RackNetworkConfig"
           }
         },
         "required": [

openapi/sled-agent/sled-agent-latest.json

@@ -1 +1 @@
-sled-agent-35.0.0-93533c.json
+sled-agent-36.0.0-187400.json

sled-agent/api/src/lib.rs

@@ -21,7 +21,7 @@ use omicron_common::api::internal::{
 };
 use sled_agent_types_versions::{
     latest, v1, v4, v6, v7, v9, v10, v11, v12, v14, v16, v17, v18, v20, v22,
-    v24, v25, v26, v28, v29, v30, v31, v33,
+    v24, v25, v26, v28, v29, v30, v31, v33, v36,
 };
 use sled_diagnostics::SledDiagnosticsQueryOutput;
 use slog_error_chain::InlineErrorChain;
@@ -38,6 +38,7 @@ api_versions!([
     // |  example for the next person.
     // v
     // (next_int, IDENT),
+    (36, BOOTSTORE_SERVICE_NAT_GENERATION),
     (35, INLINE_ROUTER_PEER_IP_ADDR),
     (34, MODIFY_SVCS_TYPES),
     (33, BOOTSTORE_SERVICE_NAT),
@@ -934,7 +935,20 @@ pub trait SledAgentApi {
     #[endpoint {
         method = PUT,
         path = "/network-bootstore-config",
-        versions = VERSION_BOOTSTORE_SERVICE_NAT..,
+        versions = VERSION_BOOTSTORE_SERVICE_NAT_GENERATION..,
+        operation_id = "write_network_bootstore_config",
+    }]
+    async fn write_network_bootstore_config_v36(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<v36::system_networking::WriteNetworkConfigRequest>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+
+    // As described above, this must not forward to newer versions; sled-agent
+    // must implement this by faithfully serializing the requested version.
+    #[endpoint {
+        method = PUT,
+        path = "/network-bootstore-config",
+        versions = VERSION_BOOTSTORE_SERVICE_NAT..VERSION_BOOTSTORE_SERVICE_NAT_GENERATION,
         operation_id = "write_network_bootstore_config",
     }]
     async fn write_network_bootstore_config_v33(

sled-agent/src/http_entrypoints.rs

@@ -79,7 +79,7 @@ use trust_quorum_types::messages::{
 use trust_quorum_types::status::{CommitStatus, CoordinatorStatus, NodeStatus};
 
 // Fixed identifiers for prior versions only
-use sled_agent_types_versions::{v1, v20, v25, v26, v30, v33};
+use sled_agent_types_versions::{v1, v20, v25, v26, v30, v33, v36};
 use sled_diagnostics::{
     SledDiagnosticsCommandHttpOutput, SledDiagnosticsQueryOutput,
 };
@@ -962,6 +962,7 @@ impl SledAgentApi for SledAgentImpl {
         use v20::early_networking::EarlyNetworkConfigBody as BodyV20;
         use v26::early_networking::EarlyNetworkConfigBody as BodyV26;
         use v30::early_networking::EarlyNetworkConfigBody as BodyV30;
+        use v33::system_networking::SystemNetworkingConfig as BodyV33;
         type LatestEnvelope = EarlyNetworkConfigEnvelope;
 
         let sa = rqctx.context();
@@ -989,7 +990,7 @@ impl SledAgentApi for SledAgentImpl {
                                     ))
                                 })?;
                         let body = BodyV20::from(BodyV26::from(BodyV30::from(
-                            latest_version_body,
+                            BodyV33::from(latest_version_body),
                         )));
                         v20::early_networking::EarlyNetworkConfig {
                             generation: config.generation,
@@ -1010,6 +1011,26 @@ impl SledAgentApi for SledAgentImpl {
             .await
     }
 
+    async fn write_network_bootstore_config_v36(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<v36::system_networking::WriteNetworkConfigRequest>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let sa = rqctx.context();
+        let bs = sa.bootstore();
+        let body = body.into_inner();
+        let config = EarlyNetworkConfigEnvelope::from(&body.body)
+            .serialize_to_bootstore_with_generation(body.generation);
+
+        bs.update_network_config(config).await.map_err(|e| {
+            HttpError::for_internal_error(format!(
+                "failed to write updated config to boot store: {}",
+                InlineErrorChain::new(&e),
+            ))
+        })?;
+
+        Ok(HttpResponseUpdatedNoContent())
+    }
+
     async fn write_network_bootstore_config_v33(
         rqctx: RequestContext<Self::Context>,
         body: TypedBody<v33::system_networking::WriteNetworkConfigRequest>,

sled-agent/src/rack_setup/service.rs

@@ -121,8 +121,10 @@ use sled_agent_types::inventory::{
     OmicronZoneConfig, OmicronZoneType, OmicronZonesConfig,
 };
 use sled_agent_types::rack_init::rack_init_bootstore_generation;
-use sled_agent_types::system_networking::ServiceZoneNatEntriesError;
 use sled_agent_types::system_networking::SystemNetworkingConfig;
+use sled_agent_types::system_networking::{
+    BlueprintExternalNetworkingConfig, ServiceZoneNatEntriesError,
+};
 use sled_hardware_types::BaseboardId;
 use sled_hardware_types::underlay::BootstrapInterface;
 use slog::Logger;
@@ -1318,7 +1320,7 @@ impl ServiceInner {
             // TODO-correctness could we wait to put this into the bootstore
             // until after the service plan is created, once we've finished
             // moving all system networking into scrimlet reconcilers?
-            service_zone_nat_entries: None,
+            blueprint_external_networking_config: None,
         };
         info!(self.log, "Writing initial network configuration to bootstore");
         rss_step.update(RssStep::InitialNetworkConfigUpdate);
@@ -1370,12 +1372,14 @@ impl ServiceInner {
             .map_err(SetupServiceError::ConvertPlanToBlueprint)?;
 
         // Now that we have a service plan (and therefore a blueprint), we can
-        // fill in the service_zone_nat_entries in the bootstore.
-        system_networking_config.service_zone_nat_entries = Some(
-            blueprint
-                .to_service_zone_nat_entries()
-                .map_err(SetupServiceError::InvalidServiceZoneNatEntries)?,
-        );
+        // fill in the `blueprint_external_networking_config` in the bootstore.
+        system_networking_config.blueprint_external_networking_config =
+            Some(BlueprintExternalNetworkingConfig {
+                blueprint_external_networking_generation: Generation::new(),
+                service_zone_nat_entries: blueprint
+                    .to_service_zone_nat_entries()
+                    .map_err(SetupServiceError::InvalidServiceZoneNatEntries)?,
+            });
         info!(
             self.log,
             "Writing final system networking configuration to bootstore",

sled-agent/src/sim/http_entrypoints.rs

@@ -89,6 +89,7 @@ use sled_agent_types_versions::v25;
 use sled_agent_types_versions::v26;
 use sled_agent_types_versions::v30;
 use sled_agent_types_versions::v33;
+use sled_agent_types_versions::v36;
 use sled_diagnostics::SledDiagnosticsQueryOutput;
 use slog_error_chain::InlineErrorChain;
 use std::collections::BTreeMap;
@@ -408,6 +409,7 @@ impl SledAgentApi for SledAgentSimImpl {
         use v20::early_networking::EarlyNetworkConfigBody as BodyV20;
         use v26::early_networking::EarlyNetworkConfigBody as BodyV26;
         use v30::early_networking::EarlyNetworkConfigBody as BodyV30;
+        use v33::system_networking::SystemNetworkingConfig as BodyV33;
 
         let config =
             rqctx.context().bootstore_network_config.lock().unwrap().clone();
@@ -430,8 +432,9 @@ impl SledAgentApi for SledAgentSimImpl {
 
         // Downconvert from the current version to the v20 version we have to
         // return from this endpoint.
-        let body =
-            BodyV20::from(BodyV26::from(BodyV30::from(latest_version_body)));
+        let body = BodyV20::from(BodyV26::from(BodyV30::from(BodyV33::from(
+            latest_version_body,
+        ))));
 
         Ok(HttpResponseOk(v20::early_networking::EarlyNetworkConfig {
             generation: config.generation,
@@ -440,6 +443,19 @@ impl SledAgentApi for SledAgentSimImpl {
         }))
     }
 
+    async fn write_network_bootstore_config_v36(
+        rqctx: RequestContext<Self::Context>,
+        body: TypedBody<v36::system_networking::WriteNetworkConfigRequest>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let mut config =
+            rqctx.context().bootstore_network_config.lock().unwrap();
+        let body = body.into_inner();
+
+        *config = EarlyNetworkConfigEnvelope::from(&body.body)
+            .serialize_to_bootstore_with_generation(body.generation);
+        Ok(HttpResponseUpdatedNoContent())
+    }
+
     async fn write_network_bootstore_config_v33(
         rqctx: RequestContext<Self::Context>,
         body: TypedBody<v33::system_networking::WriteNetworkConfigRequest>,

sled-agent/src/sim/sled_agent.rs

@@ -154,7 +154,7 @@ impl SledAgent {
                 },
                 // TODO-correctness Can we fill this in for the simulated
                 // sled-agent?
-                service_zone_nat_entries: None,
+                blueprint_external_networking_config: None,
             })
             .serialize_to_bootstore_with_generation(0),
         );