Implement assessment Batch J: observability binding, LLM cost replay docs, and real-mode prereqs.

Ships Improvements #9, #18, and #20 — Grafana/Prometheus binding runbook, TB-023 live-rate semantics on ILlmCostEstimator, and verify_real_mode_prereqs.ps1 with BUILD/TB-007 documentation.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: joefrancisGA

.github/workflows/ci.yml

@@ -323,6 +323,9 @@ jobs:
       - name: Buyer-facing canonical claim drift guard unit tests
         run: python -m unittest discover -s scripts/ci/tests -p "test_check_buyer_claim_drift.py"
 
+      - name: Real-mode LLM prerequisite checklist unit test
+        run: python -m unittest discover -s scripts/ci/tests -p "test_verify_real_mode_prereqs.py"
+
       - name: Customer WIF templates — validate (warn-only when CLI missing)
         continue-on-error: true
         run: python scripts/ci/validate_customer_wif_templates.py

ArchLucid.AgentRuntime.Tests/LlmCostEstimatorTests.cs

@@ -12,6 +12,10 @@
 namespace ArchLucid.AgentRuntime.Tests;
 
 [Trait("Category", "Unit")]
+/// <summary>
+///     TB-023: <see cref="ILlmCostEstimator.EstimateUsd" /> uses live rates; run-level recomputation may diverge from
+///     persisted <c>AgentExecutionTrace.EstimatedCostUsd</c> after admin rate changes.
+/// </summary>
 public sealed class LlmCostEstimatorTests
 {
     [SkippableFact]

ArchLucid.AgentRuntime/LlmCostEstimator.cs

@@ -5,6 +5,10 @@
 namespace ArchLucid.AgentRuntime;
 
 /// <inheritdoc cref="ILlmCostEstimator" />
+/// <remarks>
+///     Resolves USD rates through <see cref="ILlmCostEstimationUsdRateOverride" /> on every call. FinOps replay of historical
+///     traces after admin rate tuning will change aggregates — see TB-023 on <see cref="ILlmCostEstimator" />.
+/// </remarks>
 public sealed class LlmCostEstimator(
     IOptions<LlmCostEstimationOptions> options,
     ILlmCostEstimationUsdRateOverride usdRateOverride) : ILlmCostEstimator

ArchLucid.Application/Agents/AgentExecutionTraceRunLlmCostAggregator.cs

@@ -13,7 +13,11 @@ public static class AgentExecutionTraceRunLlmCostAggregator
     /// <param name="traces">All execution traces for the run (typically from <c>GetByRunIdAsync</c>).</param>
     /// <param name="costEstimator">Host-configured estimator (respects <see cref="LlmCostEstimationOptions.Enabled" />).</param>
     /// <returns>Token totals, optional summed USD, and a display label for model/deployment names.</returns>
-    /// <remarks>Re-estimates using live rates.</remarks>
+    /// <remarks>
+    ///     Re-estimates each trace slice using <paramref name="costEstimator" /> (live rates). The returned
+    ///     <see cref="AgentExecutionTraceRunLlmCostSummary.EstimatedCostUsd" /> may differ from summing persisted
+    ///     <see cref="AgentExecutionTrace.EstimatedCostUsd" /> on each row when rates changed between trace recording and this call (TB-023).
+    /// </remarks>
     public static AgentExecutionTraceRunLlmCostSummary Compute(
         IReadOnlyList<AgentExecutionTrace> traces,
         ILlmCostEstimator costEstimator)

ArchLucid.Core/Configuration/ILlmCostEstimator.cs

@@ -1,10 +1,21 @@
 namespace ArchLucid.Core.Configuration;
 
 /// <summary>Estimates LLM call cost from token usage and <see cref="LlmCostEstimationOptions" />.</summary>
+/// <remarks>
+///     Estimates reflect the <strong>currently configured</strong> USD-per-million rates (including live admin overrides via
+///     <see cref="ILlmCostEstimationUsdRateOverride" />). Replaying historical token counts after a rate change produces a
+///     different USD total — that behavior is intentional and is <strong>not</strong> a stable audit-grade record.
+///     Per-trace <see cref="ArchLucid.Contracts.Agents.AgentExecutionTrace.EstimatedCostUsd" /> values captured at recording time may diverge from
+///     run-level recomputation via <c>AgentExecutionTraceRunLlmCostAggregator</c>.
+///     The OpenTelemetry counter <c>archlucid_llm_cost_usd_total</c> is pre-tax, monitoring-grade (decimal cast to double).
+/// </remarks>
 public interface ILlmCostEstimator
 {
     /// <summary>Returns pre-tax estimated cost, or <see langword="null" /> when estimation is disabled or counts are non-positive.</summary>
-    /// <remarks>Estimates reflect currently configured rates.</remarks>
+    /// <remarks>
+    ///     Uses live <see cref="LlmCostEstimationOptions" /> and override rates at call time — not rates frozen at trace persistence.
+    ///     See interface-level remarks for replay vs stored-per-trace divergence (TB-023).
+    /// </remarks>
     decimal? EstimateUsd(
         int inputTokens,
         int outputTokens,

docs/engineering/BUILD.md

@@ -88,6 +88,26 @@ python scripts/repo_digest/build_repo_digest.py
 
 **Merged .NET coverage gates:** After **`dotnet-full-regression`**, the workflow runs **`scripts/ci/assert_merged_line_coverage_min.py`** on merged Cobertura with **75%** merged line minimum; **63%** merged branch and **63%** per-product-package line remain merge-blocking (see **`docs/library/coverage-exclusions.md`**). **`--warn-below-package-line-pct 70`** still emits non-blocking warnings for packages in the **[63%, 70%)** band. **`assert_coverage_floor_ratchet.py`** and **`.coverage-floor`** are **not** run in CI until **V1.1** (**`docs/library/V1_DEFERRED.md`**).
 
+## Real-mode LLM CI and golden cohort (TB-007 partial)
+
+Optional jobs invoke live Azure OpenAI — they are **opt-in** and **never run on fork pull requests** (secrets unavailable).
+
+| Path | Gate | When it runs | Required configuration |
+|------|------|--------------|------------------------|
+| **Tier 2d — `dotnet-azure-openai-live-post-regression`** | `RealAzureOpenAIEndToEndTests` | After OpenAPI snapshot, merged coverage, and chaos; **`push` / `workflow_dispatch` / `merge_group` only** (not `pull_request`) | Repository variable **`ARCHLUCID_CI_REAL_AOAI_ENABLED=true`**; secrets **`ARCHLUCID_CI_REAL_AOAI_ENDPOINT`**, **`ARCHLUCID_CI_REAL_AOAI_KEY`**; optional var **`ARCHLUCID_CI_REAL_AOAI_DEPLOYMENT`** |
+| **`golden-cohort-nightly.yml`** — `cohort-real-llm-gate` | Structural gate + budget probe (no in-job AOAI call) | When **`ARCHLUCID_GOLDEN_COHORT_BASELINE_LOCKED=true`** and **`ARCHLUCID_GOLDEN_COHORT_REAL_LLM=true`** | See [`docs/runbooks/GOLDEN_COHORT_REAL_LLM_GATE.md`](../runbooks/GOLDEN_COHORT_REAL_LLM_GATE.md); cost cap in **`tests/golden-cohort/budget.config.json`** |
+| **`golden-cohort-nightly.yml`** — `cohort-real-llm-live` | Live drift CLI | Manual dispatch with **`run_live_invoke=true`**, or Sunday schedule when **`ARCHLUCID_GOLDEN_COHORT_LIVE_SCHEDULE_ENABLED=true`** | Secret **`ARCHLUCID_GOLDEN_COHORT_API_HOST`**; Azure OpenAI runner env documented in runbook |
+
+**Operator checklist (names only, no values):**
+
+```powershell
+.\scripts\ci\verify_real_mode_prereqs.ps1
+.\scripts\ci\verify_real_mode_prereqs.ps1 -Profile CiLiveAoai -Strict
+.\scripts\ci\verify_real_mode_prereqs.ps1 -UseGitHubCli
+```
+
+If a job is skipped, the script output plus [`docs/library/TECH_BACKLOG.md`](../library/TECH_BACKLOG.md) § **TB-007** explain which variable/secret is missing. Do **not** enable merge-blocking real-mode gates without owner approval ([`docs/assessments/LATEST.md`](../assessments/LATEST.md) Pending Questions — TB-007).
+
 **GA / demo workspace release gate (live Playwright `@release-gate`):** `archlucid-ui/e2e/demo-workspace-*.smoke.spec.ts` runs with **`playwright.config.ts`** in **`ci.yml`** `ui-e2e-live` against **DevelopmentBypass** + SQL seed (anchors in **`docs/go-to-market/DEMO_WORKSPACES.md`**). In **`ci.yml`**, `ui-e2e-live` uses **`if: github.event_name != 'pull_request'`** (runs on **`push`** / **`workflow_dispatch`** / **`merge_group`** per your triggers), so associate that check with the branch(es) where you certify GA. Failures block **general-availability readiness** alongside the broader live UI ladder (`scripts/release-smoke.ps1 -LivePlaywright`, **`-Profile LiveUiSql`**). Isolate locally: **`cd archlucid-ui`**, **`npm exec playwright test --grep "@release-gate"`** once ArchLucid.Api is reachable at **`LIVE_API_URL`** (with the demo workspace tenant/workspace/project triple in operator scope).
 
 **Secret scanning:** The **`gitleaks`** job scans the full Git history with **`gacts/gitleaks@v1.3.2`** (runs the upstream **`gitleaks`** binary; **`.gitleaks.toml`** extends default rules and allowlists only the two documented dev/CI SQL passwords that appear verbatim in-repo). To run locally: install [gitleaks](https://github.com/gitleaks/gitleaks) and run **`gitleaks detect --source . --verbose`** from the repo root.

docs/library/OPERATOR_ATLAS.md

@@ -89,4 +89,15 @@
 
 ---
 
+## Observability (Grafana + Prometheus)
+
+| Action | CLI / script | Primary signal | Operator UI | Runbook / doc |
+|--------|--------------|----------------|-------------|---------------|
+| Export readiness report | `python scripts/report_observability_export_readiness.py` | OTel export config | — | [OBSERVABILITY.md](OBSERVABILITY.md) |
+| Real-mode LLM CI prereqs | `.\scripts\ci\verify_real_mode_prereqs.ps1` | GitHub vars/secrets names | — | [BUILD.md](../engineering/BUILD.md), [GOLDEN_COHORT_REAL_LLM_GATE.md](../runbooks/GOLDEN_COHORT_REAL_LLM_GATE.md) |
+| Grafana dashboard import | — | Prometheus/Loki UIDs | — | [OBSERVABILITY_DASHBOARD_BINDING.md](../runbooks/OBSERVABILITY_DASHBOARD_BINDING.md) |
+| Prometheus alert rules | `promtool check rules infra/prometheus/archlucid-alerts.yml` | `archlucid-agent-output-quality` | — | [TECH_BACKLOG.md](TECH_BACKLOG.md) § TB-004 |
+
+---
+
 **Day-one role files** (`docs/onboarding/day-one-*.md`) stay for week-one checklists — use **this atlas** when you need the **canonical action map** (route × API × CLI) without narrative.

docs/library/PER_TENANT_COST_MODEL.md

@@ -13,7 +13,18 @@ Runtime cost estimates use `ILlmCostEstimator`, which applies USD-per-million ra
 
 When `AgentExecution:LlmCostEstimation:Enabled` is `false`, the estimator returns no USD value (previews show a null estimate).
 
-Note: Cost aggregations (like `AgentExecutionTraceRunLlmCostAggregator`) re-estimate costs using live rates rather than strictly summing historical point-in-time estimates.
+Note: Cost aggregations (like `AgentExecutionTraceRunLlmCostAggregator`) re-estimate costs using **live** rates rather than strictly summing historical point-in-time estimates.
+
+### Rate changes and replay (TB-023)
+
+| Surface | Behavior |
+|---------|----------|
+| **`ILlmCostEstimator.EstimateUsd`** | Always uses **current** `AgentExecution:LlmCostEstimation` rates and admin overrides. |
+| **`AgentExecutionTrace.EstimatedCostUsd`** | Snapshot at trace write time using rates then in effect. |
+| **Run detail / aggregator totals** | Recomputed from token counts × **live** rates — may **differ** from stored per-trace USD after operators tune rates. |
+| **`archlucid_llm_cost_usd_total`** | Pre-tax monitoring counter (IEEE 754 `double`); not invoice-reconciliation-grade. |
+
+This is intentional: operators tune forward-looking estimates without rewriting historical trace rows. For audit questions, cite persisted trace USD **and** the rate config effective at recording time (support bundle / config snapshot).
 
 ## Wizard preview (`GET /v1/agent-execution/cost-preview`)