Testing: watch buggify on by default in the repo's own harness backends (BG-2)

The library default stays clean (buggify-off is the emulator's shipped behavior, since it is
public and that decision belongs to each downstream consumer). One level up, where we own the
tests, the harness backends turn on seeded watch chaos by default so every watch-arming suite
exercises the weak watch contract (a watch may fire spuriously, and a net-reverted change may
never fire) instead of the emulator's deterministic-but-unrealistic clean watches. Each store
gets a per-test-name seed, so a run is reproducible and a failure replays.

- FakeDbStore.Buggify.EnableChaos(name, ...): the one-line, per-suite opt-in the design promised
  for SDK-level consumers. Derives a process-stable seed from the name (FNV-1a, not the
  per-run-randomized string.GetHashCode) and installs a chaos profile.
- Harness backends flipped chaos-on: FakeDbTestBackend (the dual-backend layer-suite backend) and
  the Docker-free conformance heads (Transaction, Smoke, RangeQuery, Retryable). Chaos is a no-op
  for any suite that arms no watches (nothing to fire, no checks to defer), so this is behaviorally
  inert everywhere except the transaction conformance watch facts.
- The scenario family (generator/trace/replay/dual-live/fuzz) is left buggify-OFF: it is a
  trace-comparison / exact-semantics engine, and the real-vs-FakeDb dual-live and FdbLite-vs-FakeDb
  emulator differentials must stay head-for-head faithful (BG-4). Injected glitches are designed
  divergence; a buggify-on differential would only re-prove the injection the injection facts
  already prove directly.
- The watch conformance facts that assert exact fire/pending timing opt out with one line -
  RequireCleanWatches() - which documents "this test needs clean watches" at the site. Two facts
  actually failed under their seed (spurious-fired a should-be-pending watch); the rest assert exact
  timing too and are annotated for robustness against a future seed/rate change.

Migration outcome: full FakeDb suite green (0 failed), the scenario dual-live/trace heads green with
buggify off.

Author: KrzysFR

FoundationDB.FakeDb.Tests/Conformance/RangeQueryConformanceFacts.cs

@@ -1668,14 +1668,14 @@ public class RangeQueryFakeDbFacts : RangeQueryConformanceFacts
 		protected override Task<IFdbDatabase> OpenTestDatabaseAsync(bool readOnly = false)
 		{
 			// mirror FdbTest.OpenTestDatabaseAsync: the real-cluster head seeds a 15s default timeout via FdbConnectionOptions.DefaultTimeout
-			var db = (this.Store ??= new FakeDbStore()).OpenDatabase(FdbPath.Root, readOnly);
+			var db = (this.Store ??= TestBuggify.ChaosStore()).OpenDatabase(FdbPath.Root, readOnly);
 			db.Options.WithDefaultTimeout(TimeSpan.FromSeconds(15));
 			return Task.FromResult<IFdbDatabase>(db);
 		}
 
 		protected override Task<IFdbDatabase> OpenTestPartitionAsync(string? testMethod = null)
 		{
-			var db = (this.Store ??= new FakeDbStore()).OpenDatabase(GetTestPartitionPath(testMethod), readOnly: false);
+			var db = (this.Store ??= TestBuggify.ChaosStore()).OpenDatabase(GetTestPartitionPath(testMethod), readOnly: false);
 			db.Options.WithDefaultTimeout(TimeSpan.FromSeconds(15));
 			return Task.FromResult<IFdbDatabase>(db);
 		}

FoundationDB.FakeDb.Tests/Conformance/RetryableConformanceFacts.cs

@@ -458,14 +458,14 @@ public class RetryableFakeDbFacts : RetryableConformanceFacts
 		protected override Task<IFdbDatabase> OpenTestDatabaseAsync(bool readOnly = false)
 		{
 			// mirror FdbTest.OpenTestDatabaseAsync: the real-cluster head seeds a 15s default timeout via FdbConnectionOptions.DefaultTimeout
-			var db = (this.Store ??= new FakeDbStore()).OpenDatabase(FdbPath.Root, readOnly);
+			var db = (this.Store ??= TestBuggify.ChaosStore()).OpenDatabase(FdbPath.Root, readOnly);
 			db.Options.WithDefaultTimeout(TimeSpan.FromSeconds(15));
 			return Task.FromResult<IFdbDatabase>(db);
 		}
 
 		protected override Task<IFdbDatabase> OpenTestPartitionAsync(string? testMethod = null)
 		{
-			var db = (this.Store ??= new FakeDbStore()).OpenDatabase(GetTestPartitionPath(testMethod), readOnly: false);
+			var db = (this.Store ??= TestBuggify.ChaosStore()).OpenDatabase(GetTestPartitionPath(testMethod), readOnly: false);
 			db.Options.WithDefaultTimeout(TimeSpan.FromSeconds(15));
 			return Task.FromResult<IFdbDatabase>(db);
 		}

FoundationDB.FakeDb.Tests/Conformance/SmokeConformanceFacts.cs

@@ -115,14 +115,14 @@ public class SmokeFakeDbFacts : SmokeConformanceFacts
 
 		protected override Task<IFdbDatabase> OpenTestDatabaseAsync(bool readOnly = false)
 		{
-			var db = (this.Store ??= new FakeDbStore()).OpenDatabase(FdbPath.Root, readOnly);
+			var db = (this.Store ??= TestBuggify.ChaosStore()).OpenDatabase(FdbPath.Root, readOnly);
 			db.Options.WithDefaultTimeout(TimeSpan.FromSeconds(15));
 			return Task.FromResult<IFdbDatabase>(db);
 		}
 
 		protected override Task<IFdbDatabase> OpenTestPartitionAsync(string? testMethod = null)
 		{
-			var db = (this.Store ??= new FakeDbStore()).OpenDatabase(GetTestPartitionPath(testMethod), readOnly: false);
+			var db = (this.Store ??= TestBuggify.ChaosStore()).OpenDatabase(GetTestPartitionPath(testMethod), readOnly: false);
 			db.Options.WithDefaultTimeout(TimeSpan.FromSeconds(15));
 			return Task.FromResult<IFdbDatabase>(db);
 		}

FoundationDB.FakeDb.Tests/Conformance/TransactionConformanceFacts.cs

@@ -45,6 +45,10 @@ namespace FoundationDB.Client.Tests
 	public abstract class TransactionConformanceFacts : FdbTest
 	{
 
+		/// <summary>Opts this test out of watch buggify so it sees clean, deterministic watch semantics (no-op on a real cluster; on the
+		/// FakeDb head it disables the seeded chaos the harness turns on by default). Call at the top of a test that asserts exact watch timing.</summary>
+		protected virtual void RequireCleanWatches() { }
+
 		[Test]
 		public async Task Test_Can_Create_And_Dispose_Transactions()
 		{
@@ -3447,6 +3451,8 @@ await AssertThrowsFdbErrorAsync(
 		[CoversCells("watches/no-fire-single-commit-aba")]
 		public async Task Test_Watch_Registration_Compares_Endpoint_Values()
 		{
+			RequireCleanWatches(); // clean watches: this test asserts exact fire/pending semantics
+
 			// the pre-arm window is judged by VALUES at the endpoints, not by version history: a key that changed
 			// and reverted across two commits between the watching transaction's read version and its arming
 			// commit registers as still-pending (oracle-pinned; contrast with the POST-arm two-commit case, where
@@ -3507,6 +3513,7 @@ public async Task Test_Same_Key_Watches_Fire_Together()
 		[Test]
 		public async Task Test_Can_Setup_And_Cancel_Watches()
 		{
+			RequireCleanWatches(); // clean watches: this test asserts exact fire/pending semantics
 			using var db = await OpenTestPartitionAsync();
 			await CleanLocation(db);
 
@@ -3597,6 +3604,7 @@ public async Task Test_Cannot_Use_Transaction_CancellationToken_With_Watch()
 		[Test]
 		public async Task Test_Setting_Key_To_Same_Value_Should_Not_Trigger_Watch()
 		{
+			RequireCleanWatches(); // clean watches: this test asserts exact fire/pending semantics
 			using var db = await OpenTestPartitionAsync();
 			await CleanLocation(db);
 
@@ -3658,6 +3666,8 @@ await db.WriteAsync(async tr =>
 		[Test]
 		public async Task Test_Watched_Key_Changed_By_Same_Transaction_Before_Commit_Should_Trigger_Watch()
 		{
+			RequireCleanWatches(); // clean watches: this test asserts exact fire/pending semantics
+
 			// Steps:
 			// - T1: set a watch on a key, but does not commit yet
 			// - T1: change the value of the watched key
@@ -3707,6 +3717,8 @@ await db.WriteAsync(async tr =>
 		[Test]
 		public async Task Test_Concurrent_Change_To_Watched_Key_Before_Commit_Should_Still_Trigger_Watch()
 		{
+			RequireCleanWatches(); // clean watches: this test asserts exact fire/pending semantics
+
 			// Steps:
 			// - T1: set a watch on a key, but do not commit yet
 			// - T2: update the watched key and commit before T1
@@ -3765,6 +3777,7 @@ public async Task Test_Can_Cancel_Awaited_Watch_With_CancellationToken()
 		{
 			// Test that calling watch.WaitAsync(CancellationToken) will throw if the token is triggered before the watch fires
 
+			RequireCleanWatches(); // clean watches: this test asserts the watch stays pending until cancelled
 			using var db = await OpenTestPartitionAsync();
 			await CleanLocation(db);
 
@@ -3816,6 +3829,7 @@ public async Task Test_Can_Cancel_Awaited_Watch_After_Timeout()
 		{
 			// Test that calling watch.WaitAsync(TimeSpan, CancellationToken) will throw if the timeout expires before the watch fires
 
+			RequireCleanWatches(); // clean watches: this test asserts the watch stays pending until the timeout
 			using var db = await OpenTestPartitionAsync();
 			await CleanLocation(db);
 

FoundationDB.FakeDb.Tests/Conformance/TransactionConformanceHeads.cs

@@ -33,25 +33,27 @@ namespace FoundationDB.Client.Tests
 	public class TransactionFakeDbFacts : TransactionConformanceFacts
 	{
 
-		/// <summary>Store shared by all databases opened during a single test, reset between tests.</summary>
+		/// <summary>Store shared by all databases opened during a single test, reset between tests. Watch buggify is on by default (see <see cref="TestBuggify"/>); a test needing clean watches calls <see cref="TransactionConformanceFacts.RequireCleanWatches"/>.</summary>
 		private FakeDbStore? Store { get; set; }
 
 		protected override bool UseRealServer => false;
 
 		[TearDown]
 		public void ResetFakeDbStore() => this.Store = null;
 
+		protected override void RequireCleanWatches() => (this.Store ??= TestBuggify.ChaosStore()).Buggify.Disable();
+
 		protected override Task<IFdbDatabase> OpenTestDatabaseAsync(bool readOnly = false)
 		{
 			// mirror FdbTest.OpenTestDatabaseAsync: the real-cluster head seeds a 15s default timeout via FdbConnectionOptions.DefaultTimeout
-			var db = (this.Store ??= new FakeDbStore()).OpenDatabase(FdbPath.Root, readOnly);
+			var db = (this.Store ??= TestBuggify.ChaosStore()).OpenDatabase(FdbPath.Root, readOnly);
 			db.Options.WithDefaultTimeout(TimeSpan.FromSeconds(15));
 			return Task.FromResult<IFdbDatabase>(db);
 		}
 
 		protected override Task<IFdbDatabase> OpenTestPartitionAsync(string? testMethod = null)
 		{
-			var db = (this.Store ??= new FakeDbStore()).OpenDatabase(GetTestPartitionPath(testMethod), readOnly: false);
+			var db = (this.Store ??= TestBuggify.ChaosStore()).OpenDatabase(GetTestPartitionPath(testMethod), readOnly: false);
 			db.Options.WithDefaultTimeout(TimeSpan.FromSeconds(15));
 			return Task.FromResult<IFdbDatabase>(db);
 		}

FoundationDB.FakeDb.Tests/TestBuggify.cs

@@ -0,0 +1,55 @@
+#region Copyright (c) 2023-2026 SnowBank SAS, (c) 2005-2023 Doxense SAS
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 	* Redistributions of source code must retain the above copyright
+// 	  notice, this list of conditions and the following disclaimer.
+// 	* Redistributions in binary form must reproduce the above copyright
+// 	  notice, this list of conditions and the following disclaimer in the
+// 	  documentation and/or other materials provided with the distribution.
+// 	* Neither the name of SnowBank nor the
+// 	  names of its contributors may be used to endorse or promote products
+// 	  derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL SNOWBANK SAS BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#endregion
+
+namespace FoundationDB.Client.Tests
+{
+	using FoundationDB.Testing;
+
+	/// <summary>Repo test-harness policy for FakeDb watch buggify: the repo's own harness backends are watch-realistic BY DEFAULT.</summary>
+	/// <remarks>
+	/// <para>The library default is buggify-off (that decision belongs to each downstream consumer, since the emulator is public), but
+	/// one level up - where we own the tests - the harness backends enable seeded chaos by default so every watch-arming suite exercises
+	/// the weak watch contract (a watch may fire spuriously, and a net-reverted change may never fire). Each test gets a stable,
+	/// per-test-name seed, so a run is reproducible and a failure replays.</para>
+	/// <para>A test that asserts exact, clean watch semantics disables it with one line: <c>store.Buggify.Disable()</c> (or, on a
+	/// conformance head, the fixture's <c>RequireCleanWatches()</c> hook), which documents "this test needs clean watches" at the site.</para>
+	/// <para>Chaos is a no-op for a suite that arms no watches (nothing to fire, no checks to defer), so enabling it on a non-watch
+	/// conformance head is harmless.</para>
+	/// </remarks>
+	internal static class TestBuggify
+	{
+
+		/// <summary>Creates a fresh FakeDb store with watch chaos enabled, seeded from the currently-running test's name.</summary>
+		public static FakeDbStore ChaosStore()
+		{
+			var store = new FakeDbStore();
+			store.Buggify.EnableChaos(NUnit.Framework.TestContext.CurrentContext.Test.FullName);
+			return store;
+		}
+
+	}
+
+}

FoundationDB.FakeDb/FakeDbDatabase.cs

@@ -4272,6 +4272,34 @@ internal FakeDbBuggify(FakeDbStore store)
 			/// is unaffected: it only fires when the test explicitly calls it, so there is nothing to disable.</remarks>
 			public void Disable() => this.Chaos = null;
 
+			/// <summary>Enables seeded chaos with a stable seed derived from <paramref name="name"/> - the one-line opt-in for a whole suite: buggify every watch-arming test with a profile that is distinct per name yet reproducible across runs.</summary>
+			/// <param name="name">A stable, distinct identifier (typically the test or suite name) that fixes the injection profile.</param>
+			/// <param name="spuriousFireRate">Per-commit probability of a fan-out spurious fire on one armed key.</param>
+			/// <param name="deferredCheckRate">Per-check probability that a watch check is deferred (skipped) this commit.</param>
+			/// <returns>The installed chaos profile (for further tuning, e.g. clearing one of the rates).</returns>
+			/// <remarks>Recommended for any suite whose code arms watches: it forces that code through the weak watch contract (spurious
+			/// and reverted-miss fires) without giving up reproducibility. Chaos never produces a contract-illegal outcome, so it is safe
+			/// to leave on for suites that do not assert exact watch timing.</remarks>
+			public FakeDbBuggifyChaos EnableChaos(string name, double spuriousFireRate = 0.25, double deferredCheckRate = 0.25)
+			{
+				Contract.NotNull(name);
+				var chaos = new FakeDbBuggifyChaos(StableSeed(name)) { SpuriousFireRate = spuriousFireRate, DeferredCheckRate = deferredCheckRate };
+				this.Chaos = chaos;
+				return chaos;
+
+				// FNV-1a over the name: a process-stable hash (unlike string.GetHashCode, which is randomized per run since .NET Core)
+				static int StableSeed(string s)
+				{
+					uint h = 2166136261u;
+					foreach (var c in s)
+					{
+						h = (h ^ (byte) c) * 16777619u;
+						h = (h ^ (byte) (c >> 8)) * 16777619u;
+					}
+					return unchecked((int) h);
+				}
+			}
+
 			/// <summary>Injects an immediate spurious fire of every watch registered on <paramref name="key"/>, then unregisters them (the FDBV-026 per-key fan-out shape).</summary>
 			/// <param name="key">The (fully-encoded) watched key, as registered by <c>tr.Watch(...)</c>.</param>
 			/// <returns>The number of watches fired (0 when no watch was armed on the key: a test can assert the injection landed).</returns>

FoundationDB.Layers.Tests/FakeDbTestBackend.cs

@@ -46,7 +46,7 @@ public Task<IFdbDatabase> OpenAsync(FdbPath path, bool readOnly = false)
 		{
 			try
 			{
-				var db = (this.Store ??= new FakeDbStore()).OpenDatabase(path, readOnly);
+				var db = (this.Store ??= NewStore()).OpenDatabase(path, readOnly);
 				db.Options.WithDefaultTimeout(TimeSpan.FromSeconds(15));
 				return Task.FromResult<IFdbDatabase>(db);
 			}
@@ -55,13 +55,23 @@ public Task<IFdbDatabase> OpenAsync(FdbPath path, bool readOnly = false)
 				// disposing a database opened from the store disposes the store with it: a test that opens
 				// several partitions in sequence (bench loops) gets a fresh, empty store per iteration,
 				// which matches the isolated-partition semantics the suites rely on
-				this.Store = new FakeDbStore();
+				this.Store = NewStore();
 				var db = this.Store.OpenDatabase(path, readOnly);
 				db.Options.WithDefaultTimeout(TimeSpan.FromSeconds(15));
 				return Task.FromResult<IFdbDatabase>(db);
 			}
 		}
 
+		/// <summary>Creates a fresh store with watch buggify enabled by default (seeded per running test): the repo's own harness is
+		/// watch-realistic so any layer suite that arms watches exercises the weak watch contract. A layer test needing clean, exact
+		/// watch semantics disables it with one line - <c>store.Buggify.Disable()</c>. Chaos is a no-op for a suite that arms no watches.</summary>
+		private static FakeDbStore NewStore()
+		{
+			var store = new FakeDbStore();
+			store.Buggify.EnableChaos(NUnit.Framework.TestContext.CurrentContext.Test.FullName);
+			return store;
+		}
+
 	}
 
 }