Add lockstep API

Author: smklein

dev-tools/omdb/src/bin/omdb/db.rs

@@ -1622,7 +1622,7 @@ impl DbArgs {
                         ).await
                     },
                     DbCommands::Ereport(args) => {
-                        cmd_db_ereport(&datastore, &fetch_opts, &args).await
+                        cmd_db_ereport(omdb, log, &datastore, &fetch_opts, &args).await
                     }
                     DbCommands::UserDataExport(args) => {
                         args.exec(&omdb, &opctx, &datastore).await

dev-tools/omdb/src/bin/omdb/db/ereport.rs

@@ -6,6 +6,8 @@
 
 use super::DbFetchOptions;
 use super::check_limit;
+use crate::Omdb;
+use crate::helpers::CONNECTION_OPTIONS_HEADING;
 use crate::helpers::const_max_len;
 use crate::helpers::datetime_opt_rfc3339_concise;
 use crate::helpers::datetime_rfc3339_concise;
@@ -23,6 +25,7 @@ use clap::Subcommand;
 use diesel::AggregateExpressionMethods;
 use diesel::dsl::{count, min};
 use diesel::prelude::*;
+use internal_dns_types::names::ServiceName;
 use nexus_db_lookup::DbConnection;
 use nexus_db_model::ereport as model;
 use nexus_db_model::ereport::DbEna;
@@ -57,9 +60,21 @@ enum Commands {
     Reporters(ReportersArgs),
 
     /// Summarize ereports by class, marking which classes a diagnosis engine
-    /// in Nexus consumes (per
-    /// `nexus_types::fm::ereport::known_ereport_classes`).
-    Classes,
+    /// in Nexus consumes (fetched from Nexus's lockstep API; falls back to
+    /// `?` if Nexus is unreachable).
+    Classes(ClassesArgs),
+}
+
+#[derive(Debug, Args, Clone)]
+struct ClassesArgs {
+    /// URL of the Nexus lockstep API. If not provided, looks up an instance
+    /// in internal DNS.
+    #[clap(
+        long,
+        env = "OMDB_NEXUS_URL",
+        help_heading = CONNECTION_OPTIONS_HEADING,
+    )]
+    nexus_internal_url: Option<String>,
 }
 
 #[derive(Debug, Args, Clone)]
@@ -105,6 +120,8 @@ struct ReportersArgs {
 }
 
 pub(super) async fn cmd_db_ereport(
+    omdb: &Omdb,
+    log: &slog::Logger,
     datastore: &DataStore,
     fetch_opts: &DbFetchOptions,
     args: &EreportArgs,
@@ -121,7 +138,9 @@ pub(super) async fn cmd_db_ereport(
             cmd_db_ereporters(datastore, args).await
         }
 
-        Commands::Classes => cmd_db_ereport_classes(datastore).await,
+        Commands::Classes(ref args) => {
+            cmd_db_ereport_classes(omdb, log, datastore, args).await
+        }
     }
 }
 
@@ -474,14 +493,30 @@ async fn cmd_db_ereporters(
     Ok(())
 }
 
-async fn cmd_db_ereport_classes(datastore: &DataStore) -> anyhow::Result<()> {
+async fn cmd_db_ereport_classes(
+    omdb: &Omdb,
+    log: &slog::Logger,
+    datastore: &DataStore,
+    args: &ClassesArgs,
+) -> anyhow::Result<()> {
     use std::collections::BTreeMap;
-
-    let known: std::collections::BTreeSet<&'static str> =
-        nexus_types::fm::ereport::known_ereport_classes()
-            .iter()
-            .copied()
-            .collect();
+    use std::collections::BTreeSet;
+
+    // Try to fetch the known list from Nexus. If anything fails, fall back
+    // to "?" for every row — DB totals are still useful even without Nexus.
+    let known_from_nexus =
+        fetch_known_classes_from_nexus(omdb, log, args).await;
+    let known: BTreeSet<String> = match &known_from_nexus {
+        Ok(list) => list.iter().cloned().collect(),
+        Err(err) => {
+            eprintln!(
+                "warning: could not fetch known ereport classes from Nexus: \
+                 {err:#}"
+            );
+            BTreeSet::new()
+        }
+    };
+    let nexus_reachable = known_from_nexus.is_ok();
 
     let conn = datastore.pool_connection_for_tests().await?;
 
@@ -513,32 +548,35 @@ async fn cmd_db_ereport_classes(datastore: &DataStore) -> anyhow::Result<()> {
         by_class.entry(class).or_default().unmarked = unmarked;
     }
 
-    // Whether *this* omdb's build has a diagnosis engine that consumes a
+    // Whether the deployed Nexus has a diagnosis engine that consumes a
     // given ereport class.
     #[derive(PartialEq, Eq)]
-    enum KnownToOmdb {
-        /// Class has rows in the DB AND is in `known_ereport_classes()`.
+    enum KnownToNexus {
+        /// Class has rows in the DB AND is in the list returned by Nexus.
         Yes,
-        /// Class has rows in the DB but is NOT in `known_ereport_classes()`.
+        /// Class has rows in the DB but is NOT in the list returned by Nexus.
         No,
         /// Class is NULL — strict-match policy means the loader never
         /// surfaces these to FM analysis.
         NullClass,
+        /// Could not reach Nexus — known/unknown is undetermined.
+        Unknown,
     }
-    impl std::fmt::Display for KnownToOmdb {
+    impl std::fmt::Display for KnownToNexus {
         fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
             f.write_str(match self {
                 Self::Yes => "yes",
                 Self::No => "no",
                 Self::NullClass => "-",
+                Self::Unknown => "?",
             })
         }
     }
 
     #[derive(Tabled)]
     #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
     struct ClassRow<'a> {
-        known: KnownToOmdb,
+        known: KnownToNexus,
         total: i64,
         unmarked: i64,
         /// Variable-length, so it goes last: wrapping on a narrow terminal
@@ -549,13 +587,16 @@ async fn cmd_db_ereport_classes(datastore: &DataStore) -> anyhow::Result<()> {
     let mut rows: Vec<ClassRow<'_>> = by_class
         .iter()
         .map(|(class, ClassCounts { total, unmarked })| {
-            let (known_marker, class_str): (KnownToOmdb, &str) = match class {
-                None => (KnownToOmdb::NullClass, "(NULL)"),
+            let (known_marker, class_str): (KnownToNexus, &str) = match class {
+                None => (KnownToNexus::NullClass, "(NULL)"),
+                Some(c) if !nexus_reachable => {
+                    (KnownToNexus::Unknown, c.as_str())
+                }
                 Some(c) => {
                     let k = if known.contains(c.as_str()) {
-                        KnownToOmdb::Yes
+                        KnownToNexus::Yes
                     } else {
-                        KnownToOmdb::No
+                        KnownToNexus::No
                     };
                     (k, c.as_str())
                 }
@@ -569,48 +610,79 @@ async fn cmd_db_ereport_classes(datastore: &DataStore) -> anyhow::Result<()> {
         })
         .collect();
 
-    // Sort: unknown-but-present first (highest unmarked), then known, then NULL.
+    // Sort: unknown-but-present first (highest unmarked), then known, then
+    // undetermined, then NULL.
     rows.sort_by(|a, b| {
         let priority = |row: &ClassRow<'_>| match row.known {
-            KnownToOmdb::No => 0,
-            KnownToOmdb::Yes => 1,
-            KnownToOmdb::NullClass => 2,
+            KnownToNexus::No => 0,
+            KnownToNexus::Yes => 1,
+            KnownToNexus::Unknown => 2,
+            KnownToNexus::NullClass => 3,
         };
         priority(a)
             .cmp(&priority(b))
             .then_with(|| b.unmarked.cmp(&a.unmarked))
             .then_with(|| a.class.cmp(b.class))
     });
 
-    println!(
-        "note: KNOWN reflects which classes have a diagnosis engine in Nexus \
-         as of\nthe control plane build that produced this omdb; the \
-         currently-deployed\nNexus may differ if it was built from a \
-         different commit.\n"
-    );
+    if nexus_reachable {
+        println!(
+            "note: KNOWN reflects which classes the currently-deployed Nexus \
+             knows how\nto consume.\n"
+        );
+    } else {
+        println!(
+            "note: could not reach Nexus to determine known ereport classes.\n"
+        );
+    }
 
     let mut table = tabled::Table::new(&rows);
     table
         .with(tabled::settings::Style::empty())
         .with(tabled::settings::Padding::new(0, 1, 0, 0));
     println!("{table}");
 
-    // Footer: classes this omdb knows about but has no DB rows for.
-    let seen_known: std::collections::BTreeSet<&str> = rows
-        .iter()
-        .filter(|r| r.known == KnownToOmdb::Yes)
-        .map(|r| r.class)
-        .collect();
-    let absent: Vec<&&'static str> =
-        known.iter().filter(|c| !seen_known.contains(*c)).collect();
-    if !absent.is_empty() {
-        println!(
-            "\nClasses known to this omdb but with no rows in the database:"
-        );
-        for c in absent {
-            println!("  {c}");
+    // Footer: classes Nexus knows about but with no rows in the database.
+    if nexus_reachable {
+        let seen_known: BTreeSet<&str> = rows
+            .iter()
+            .filter(|r| r.known == KnownToNexus::Yes)
+            .map(|r| r.class)
+            .collect();
+        let absent: Vec<&String> =
+            known.iter().filter(|c| !seen_known.contains(c.as_str())).collect();
+        if !absent.is_empty() {
+            println!(
+                "\nClasses Nexus knows about but with no rows in the database:"
+            );
+            for c in absent {
+                println!("  {c}");
+            }
         }
     }
 
     Ok(())
 }
+
+async fn fetch_known_classes_from_nexus(
+    omdb: &Omdb,
+    log: &slog::Logger,
+    args: &ClassesArgs,
+) -> anyhow::Result<Vec<String>> {
+    let nexus_url = match &args.nexus_internal_url {
+        Some(url) => url.clone(),
+        None => {
+            let addr = omdb
+                .dns_lookup_one(log.clone(), ServiceName::NexusLockstep)
+                .await
+                .context("resolving Nexus lockstep service via internal DNS")?;
+            format!("http://{addr}")
+        }
+    };
+    let client = nexus_lockstep_client::Client::new(&nexus_url, log.clone());
+    let resp = client
+        .fm_known_ereport_classes_list()
+        .await
+        .context("calling Nexus fm_known_ereport_classes_list")?;
+    Ok(resp.into_inner())
+}

dev-tools/omdb/tests/usage_errors.out

@@ -534,7 +534,7 @@ Commands:
   info       Show an ereport
   reporters  List ereport reporters
   classes    Summarize ereports by class, marking which classes a diagnosis engine in Nexus consumes
-             (per `nexus_types::fm::ereport::known_ereport_classes`)
+             (fetched from Nexus's lockstep API; falls back to `?` if Nexus is unreachable)
   help       Print this message or the help of the given subcommand(s)
 
 Options:

nexus/db-queries/src/db/datastore/ereport.rs

@@ -334,7 +334,7 @@ impl DataStore {
     /// Ereports with `class IS NULL` are intentionally never returned: the
     /// SQL filter is `class = ANY($1::text[])`, which never matches NULL.
     /// Callers (e.g. fm_analysis preparation) deliberately key off
-    /// [`nexus_types::fm::ereport::known_ereport_classes`] so that the loader
+    /// `nexus_fm::diagnosis::known_ereport_classes` so that the loader
     /// only surfaces ereports that FM analysis can consume; see that
     /// function's documentation for the policy and rationale.
     pub async fn ereports_list_unmarked(
@@ -396,8 +396,8 @@ impl DataStore {
         pagparams: &DataPageParams<'_, (Uuid, DbEna)>,
     ) -> impl RunnableQuery<Ereport> + use<> {
         // NULL-class ereports are intentionally excluded: `class = ANY(...)`
-        // never matches NULL. See `known_ereport_classes` in nexus-types for
-        // the policy.
+        // never matches NULL. See `nexus_fm::diagnosis::known_ereport_classes`
+        // for the policy.
         let classes: Vec<String> =
             classes.iter().map(|c| (*c).to_string()).collect();
         paginated_multicolumn(

nexus/fm/src/diagnosis.rs

@@ -11,3 +11,27 @@ pub fn analyze(
 ) -> anyhow::Result<()> {
     anyhow::bail!("FM analysis is not yet implemented")
 }
+
+/// Ereport classes that the diagnosis engine currently understands.
+/// Preparation only surfaces ereports whose class is in this set — there is
+/// no value in loading ereports FM analysis cannot consume.
+///
+/// Empty until [`analyze`] gains real handling. Grow this alongside FM
+/// analysis as new classes gain support.
+///
+/// **NULL-class ereports are intentionally excluded by the loader's SQL
+/// filter** (`class = ANY(...)` never matches NULL). If FM analysis ever
+/// needs to handle the "couldn't extract a class" or "reporter doesn't know
+/// its identity" cases, that's an explicit decision (e.g. a sentinel
+/// loader path), not a default of this list.
+///
+/// # Scaling
+///
+/// The loader filters ereports via `WHERE class = ANY($1::text[])` against
+/// the existing `lookup_ereports_by_class` index. This is comfortable up to
+/// a few hundred entries; past ~1000 entries, prefer either prefix matching
+/// (`class LIKE 'ereport.cpu.amd.%'`) or a `known_ereport_class` lookup
+/// table joined into the query. Revisit this if the list grows that large.
+pub fn known_ereport_classes() -> &'static [&'static str] {
+    &[]
+}

nexus/lockstep-api/src/lib.rs

@@ -587,6 +587,17 @@ pub trait NexusLockstepApi {
         rqctx: RequestContext<Self::Context>,
         path_params: Path<SledSelector>,
     ) -> Result<HttpResponseOk<Epoch>, HttpError>;
+
+    // Fault management
+
+    /// List ereport classes that this Nexus's diagnosis engines consume.
+    #[endpoint {
+        method = GET,
+        path = "/fm/known-ereport-classes",
+    }]
+    async fn fm_known_ereport_classes_list(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<Vec<String>>, HttpError>;
 }
 
 /// Path parameters for Rack requests.

nexus/src/app/background/tasks/fm_analysis.rs

@@ -81,11 +81,10 @@ impl FmAnalysis {
         // Snapshot the static known-classes set once, up front, so it's
         // reported in the activation status regardless of which outcome
         // variant fires.
-        let known_classes: Vec<String> =
-            nexus_types::fm::ereport::known_ereport_classes()
-                .iter()
-                .map(|s| (*s).to_string())
-                .collect();
+        let known_classes: Vec<String> = fm::diagnosis::known_ereport_classes()
+            .iter()
+            .map(|s| (*s).to_string())
+            .collect();
 
         let parent_sitrep = self.sitrep_rx.borrow_and_update().clone();
         let parent_sitrep_id = parent_sitrep.as_ref().map(|s| s.1.id());
@@ -209,7 +208,7 @@ impl FmAnalysis {
         errors: &mut Vec<String>,
     ) -> anyhow::Result<()> {
         // Only surface ereports a diagnosis engine will consume.
-        let classes = nexus_types::fm::ereport::known_ereport_classes();
+        let classes = fm::diagnosis::known_ereport_classes();
         let mut paginator = Paginator::new(
             nexus_db_queries::db::datastore::SQL_BATCH_SIZE,
             dropshot::PaginationOrder::Ascending,

nexus/src/lockstep_api/http_entrypoints.rs

@@ -1145,4 +1145,21 @@ impl NexusLockstepApi for NexusLockstepApiImpl {
             .instrument_dropshot_handler(&rqctx, handler)
             .await
     }
+
+    async fn fm_known_ereport_classes_list(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<Vec<String>>, HttpError> {
+        let apictx = &rqctx.context().context;
+        let handler = async {
+            let classes = nexus_fm::diagnosis::known_ereport_classes()
+                .iter()
+                .map(|s| (*s).to_string())
+                .collect();
+            Ok(HttpResponseOk(classes))
+        };
+        apictx
+            .internal_latencies
+            .instrument_dropshot_handler(&rqctx, handler)
+            .await
+    }
 }