diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 54c5a03d68e..590238dd600 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -703,6 +703,8 @@ manifest: component_version: <3.22.0 tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v3.36.0 tests/ffe/test_dynamic_evaluation.py: v3.36.0 + tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation: bug (FFL-2184) + tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184) tests/ffe/test_exposures.py: v3.36.0 tests/ffe/test_flag_eval_metrics.py: missing_feature tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) diff --git a/manifests/golang.yml b/manifests/golang.yml index 628ce8f182d..45c78618694 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -864,8 +864,10 @@ manifest: tests/debugger/test_debugger_symdb.py::Test_Debugger_SymDb: v2.2.3 tests/debugger/test_debugger_telemetry.py::Test_Debugger_Telemetry: missing_feature tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v2.0.0 + tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation::test_valid_flag_unaffected: bug (FFL-2184) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v2.4.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: v2.4.0 + tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance::test_unknown_operator_errors: bug (FFL-2182) tests/ffe/test_exposures.py: v2.6.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0 tests/ffe/test_flag_eval_metrics.py: v2.9.0-dev tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Metric_Parse_Error_Invalid_Regex: irrelevant (Go validates regex at config load time) diff --git a/manifests/java.yml b/manifests/java.yml index f9244996399..99cdd8d760a 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3137,6 +3137,8 @@ manifest: - weblog_declaration: "*": irrelevant spring-boot: v1.56.0 + tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation::test_valid_flag_unaffected: bug (FFL-2184) + tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184) tests/ffe/test_exposures.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 9d5e38f6a33..e2642ea2985 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1614,6 +1614,7 @@ manifest: - weblog_declaration: "*": incomplete_test_app express4: *ref_5_77_0 + tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance::test_unknown_operator_errors: bug (FFL-2182) tests/ffe/test_exposures.py: - weblog_declaration: "*": incomplete_test_app diff --git a/tests/ffe/test_dynamic_evaluation.py b/tests/ffe/test_dynamic_evaluation.py index a518b6b638c..7eb4477da4e 100644 --- a/tests/ffe/test_dynamic_evaluation.py +++ b/tests/ffe/test_dynamic_evaluation.py @@ -1,6 +1,8 @@ """Test feature flags dynamic evaluation via Remote Config.""" +import copy import json +import uuid from http import HTTPStatus from utils import ( @@ -41,6 +43,318 @@ } +@scenarios.feature_flagging_and_experimentation +@features.feature_flags_dynamic_evaluation +class Test_FFE_Unknown_Operator_Tolerance: + """SDKs must ignore only the affected flag when a condition has an unknown operator. + + Two flags share the same UFC config: + - ``operator-grease-flag``: has a condition with a GREASE operator — the whole + flag must be ignored; the in-code default is returned. + - ``valid-flag``: fully valid, no unknown operators — must still evaluate + correctly, proving the bad flag did not poison config parsing. + """ + + @staticmethod + def _build_config() -> dict: + """UFC config with a GREASE operator in a condition. + + Two allocations: + 1. Rule with unknown operator — SDK must ignore the entire flag. + 2. Catch-all returning "on" — must NOT be reached. + + The catch-all distinguishes two wrong behaviours: + - Falls through to catch-all (returns "on") — SDK only skipped the condition. + - Raises / returns a variation anyway — SDK failed to handle unknown operator. + Correct: whole flag ignored, in-code default ("default") returned. + """ + grease_op = uuid.uuid4().hex # 32 hex chars — cannot collide with any real operator + return { + "createdAt": "2024-04-17T19:40:53.716Z", + "format": "SERVER", + "environment": {"name": "Test"}, + "flags": { + "operator-grease-flag": { + "key": "operator-grease-flag", + "enabled": True, + "variationType": "STRING", + "variations": { + "trap": {"key": "trap", "value": "trap"}, + "on": {"key": "on", "value": "on"}, + }, + "allocations": [ + { + "key": "grease-allocation", + "rules": [ + { + "conditions": [ + { + "attribute": "country", + "operator": grease_op, + "value": "anything", + } + ] + } + ], + "splits": [{"variationKey": "trap", "shards": []}], + "doLog": True, + }, + { + "key": "default-allocation", + "rules": [], + "splits": [{"variationKey": "on", "shards": []}], + "doLog": True, + }, + ], + }, + "valid-flag": { + "key": "valid-flag", + "enabled": True, + "variationType": "STRING", + "variations": {"expected": {"key": "expected", "value": "expected"}}, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [{"variationKey": "expected", "shards": []}], + "doLog": True, + } + ], + }, + }, + } + + def _setup(self): + rc.tracer_rc_state.reset().apply() + config = self._build_config() + rc.tracer_rc_state.set_config(f"{RC_PATH}/ffe-unknown-operator/config", config).apply() + + def setup_unknown_operator_errors(self): + self._setup() + self.r_grease = weblog.post( + "/ffe", + json={ + "flag": "operator-grease-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {"country": "US"}, + }, + ) + + def test_unknown_operator_errors(self): + assert self.r_grease.status_code == 200, f"Flag evaluation failed: {self.r_grease.text}" + result = json.loads(self.r_grease.text) + assert result["value"] == "default", ( + f"Unknown operator: expected in-code default 'default', got '{result['value']}' " + f"(SDK must ignore the whole flag, not fall through to catch-all allocations)" + ) + + def setup_valid_flag_unaffected(self): + self._setup() + self.r_valid = weblog.post( + "/ffe", + json={ + "flag": "valid-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {}, + }, + ) + + def test_valid_flag_unaffected(self): + assert self.r_valid.status_code == 200, f"Valid flag evaluation failed: {self.r_valid.text}" + result = json.loads(self.r_valid.text) + assert result["value"] == "expected", ( + f"Valid flag corrupted by bad neighbour: expected 'expected', got '{result['value']}' " + f"(unknown operator on one flag must not poison parsing of the whole config)" + ) + + +@scenarios.feature_flagging_and_experimentation +@features.feature_flags_dynamic_evaluation +class Test_FFE_Flag_Parse_Error_Isolation: + """A parse error in one flag must not prevent other flags from evaluating. + + Two flags share the same UFC config: + - ``malformed-flag``: has a structurally invalid field (``allocations`` is a + string instead of a list) — the SDK must skip this flag and return the + in-code default. + - ``valid-flag``: fully valid — must still evaluate correctly, proving the + malformed flag did not abort parsing of the whole config. + """ + + @staticmethod + def _build_config() -> dict: + return { + "createdAt": "2024-04-17T19:40:53.716Z", + "format": "SERVER", + "environment": {"name": "Test"}, + "flags": { + "malformed-flag": { + "key": "malformed-flag", + "enabled": True, + "variationType": "STRING", + "variations": {"on": {"key": "on", "value": "on"}}, + # allocations must be a list; a string is a deliberate parse error + "allocations": "this-is-not-a-list", + }, + "valid-flag": { + "key": "valid-flag", + "enabled": True, + "variationType": "STRING", + "variations": {"expected": {"key": "expected", "value": "expected"}}, + "allocations": [ + { + "key": "default-allocation", + "rules": [], + "splits": [{"variationKey": "expected", "shards": []}], + "doLog": True, + } + ], + }, + }, + } + + def _setup(self): + rc.tracer_rc_state.reset().apply() + config = self._build_config() + rc.tracer_rc_state.set_config(f"{RC_PATH}/ffe-flag-parse-error/config", config).apply() + + def setup_malformed_flag_returns_default(self): + self._setup() + self.r_malformed = weblog.post( + "/ffe", + json={ + "flag": "malformed-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {}, + }, + ) + + def test_malformed_flag_returns_default(self): + assert self.r_malformed.status_code == 200, f"Request failed: {self.r_malformed.text}" + result = json.loads(self.r_malformed.text) + assert result["value"] == "default", ( + f"Malformed flag: expected in-code default 'default', got '{result['value']}'" + ) + + def setup_valid_flag_unaffected(self): + self._setup() + self.r_valid = weblog.post( + "/ffe", + json={ + "flag": "valid-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {}, + }, + ) + + def test_valid_flag_unaffected(self): + assert self.r_valid.status_code == 200, f"Valid flag evaluation failed: {self.r_valid.text}" + result = json.loads(self.r_valid.text) + assert result["value"] == "expected", ( + f"Valid flag corrupted by malformed neighbour: expected 'expected', got '{result['value']}' " + f"(parse error on one flag must not abort processing of the whole config)" + ) + + +@scenarios.feature_flagging_and_experimentation +@features.feature_flags_dynamic_evaluation +class Test_FFE_Unknown_Fields_Tolerance: + """SDKs must silently ignore unknown fields in the UFC configuration. + + Injects nonsensical unknown fields at every level of a valid UFC config + (top-level, flag, variation, allocation, split) and verifies that flag + evaluation still returns the correct value. + """ + + @staticmethod + def _inject_unknown_fields(config: dict) -> dict: + """Inject random unknown fields at every *fixed-schema* level of a UFC config. + + Skips map-keyed levels (``flags``, ``variations``) because their keys carry + semantic meaning; injecting there would create phantom flags/variations. + Targets: top-level config, environment, flag objects, variation objects, + allocation objects, rule objects, condition objects, split objects, + shard objects, and range objects. + + Field names are raw UUID hex (RFC 8701 GREASE style) — no recognisable + prefix, so SDKs cannot pattern-match or special-case them. + """ + config = copy.deepcopy(config) + + def g() -> str: + return uuid.uuid4().hex + + # Top-level config object + config[g()] = "unknown-top-level-string" + config[g()] = 0 + config[g()] = None + + # environment object + env = config.get("environment") + if isinstance(env, dict): + env[g()] = "unknown-env-field" + + # flags is a map (flag key → flag object) — iterate values only + for flag_obj in config.get("flags", {}).values(): + flag_obj[g()] = "unknown-flag-field" + flag_obj[g()] = False + + # variations is a map (variation key → variation object) — iterate values only + for var_obj in flag_obj.get("variations", {}).values(): + var_obj[g()] = "unknown-variation-field" + + for alloc in flag_obj.get("allocations", []): + alloc[g()] = "unknown-allocation-field" + alloc[g()] = [] + + for rule in alloc.get("rules", []): + rule[g()] = "unknown-rule-field" + + for condition in rule.get("conditions", []): + condition[g()] = "unknown-condition-field" + + for split in alloc.get("splits", []): + split[g()] = "unknown-split-field" + + for shard in split.get("shards", []): + shard[g()] = "unknown-shard-field" + + for range_obj in shard.get("ranges", []): + range_obj[g()] = "unknown-range-field" + + return config + + def setup_unknown_fields_are_ignored(self): + rc.tracer_rc_state.reset().apply() + + poisoned_config = self._inject_unknown_fields(UFC_FIXTURE_DATA) + rc.tracer_rc_state.set_config(f"{RC_PATH}/ffe-unknown-fields/config", poisoned_config).apply() + + self.r = weblog.post( + "/ffe", + json={ + "flag": "test-flag", + "variationType": "STRING", + "defaultValue": "default", + "targetingKey": "user-1", + "attributes": {}, + }, + ) + + def test_unknown_fields_are_ignored(self): + assert self.r.status_code == 200, f"Flag evaluation failed: {self.r.text}" + result = json.loads(self.r.text) + assert result["value"] == "on", f"Unknown fields corrupted evaluation: expected 'on', got '{result['value']}'" + + @scenarios.feature_flagging_and_experimentation @features.feature_flags_dynamic_evaluation class Test_FFE_RC_Unavailable: