|
1 | 1 | from __future__ import annotations |
2 | 2 |
|
| 3 | +import ast |
3 | 4 | import json |
4 | 5 | import os |
5 | 6 | import re |
|
13 | 14 | MAX_COLLECTION_ITEMS = 6 |
14 | 15 | SECRET_KEY_PATTERN = r"(?:[A-Za-z0-9]+[_.-])*(?:authorization|credential|password|secret|token|api[_-]?key|access[_-]?key)(?:[_.-](?:hash|id|key|secret|value))?" |
15 | 16 | SENSITIVE_KEY_RE = re.compile(rf"^{SECRET_KEY_PATTERN}$", re.IGNORECASE) |
| 17 | +SECRET_ASSIGNMENT_PREFIX_RE = re.compile( |
| 18 | + rf"(?i)(?<![A-Za-z0-9_.{{,-])(['\"]?)({SECRET_KEY_PATTERN})\1(?![A-Za-z0-9_.-])\s*[:=]\s*" |
| 19 | +) |
16 | 20 | SECRET_QUOTED_ASSIGNMENT_RE = re.compile( |
17 | | - rf"(?i)(?<![A-Za-z0-9_.-])({SECRET_KEY_PATTERN})(?![A-Za-z0-9_.-])\s*[:=]\s*(['\"])(?:(?!\2).)*\2" |
| 21 | + rf"(?i)(?<![A-Za-z0-9_.{{,-])(['\"]?)({SECRET_KEY_PATTERN})\1(?![A-Za-z0-9_.-])\s*[:=]\s*(['\"])(?!<redacted>\3)(?:(?!\3).)*\3" |
18 | 22 | ) |
19 | 23 | SECRET_ASSIGNMENT_RE = re.compile( |
20 | | - rf"(?i)(?<![A-Za-z0-9_.-])({SECRET_KEY_PATTERN})(?![A-Za-z0-9_.-])\s*[:=]\s*(?:(?:bearer|basic|token)\s+)?[^\s,;]+" |
| 24 | + rf"(?i)(?<![A-Za-z0-9_.{{,-])(['\"]?)({SECRET_KEY_PATTERN})\1(?![A-Za-z0-9_.-])\s*[:=]\s*(?!['\"]?<redacted>['\"]?)(?:(?:bearer|basic|token)\s+)?[^\s,;}}]+" |
| 25 | +) |
| 26 | +COMMA_SECRET_ASSIGNMENT_RE = re.compile( |
| 27 | + rf"(?i)(?<=,)({SECRET_KEY_PATTERN})(?![A-Za-z0-9_.-])\s*[:=]\s*(?!['\"]?<redacted>['\"]?)(?:(?:bearer|basic|token)\s+)?[^\s,;}}]+" |
| 28 | +) |
| 29 | +COMMA_SECRET_QUOTED_ASSIGNMENT_RE = re.compile( |
| 30 | + rf"(?i)(?<=,)({SECRET_KEY_PATTERN})(?![A-Za-z0-9_.-])\s*[:=]\s*(['\"])(?!<redacted>\2)(?:(?!\2).)*\2" |
| 31 | +) |
| 32 | +COMMA_SECRET_COLLECTION_ASSIGNMENT_RE = re.compile( |
| 33 | + rf"(?i)(?<=,)({SECRET_KEY_PATTERN})(?![A-Za-z0-9_.-])\s*[:=]\s*[\[{{].*$" |
| 34 | +) |
| 35 | +JSON_LIKE_SECRET_FIELD_RE = re.compile( |
| 36 | + rf"(?i)([{{,]\s*)(['\"])({SECRET_KEY_PATTERN})\2\s*:\s*(['\"])(?:(?!\4).)*\4" |
| 37 | +) |
| 38 | +JSON_LIKE_SECRET_UNQUOTED_FIELD_RE = re.compile( |
| 39 | + rf"(?i)([{{,]\s*)(['\"])({SECRET_KEY_PATTERN})\2\s*:\s*(?!['\"]?<redacted>['\"]?)(?:\[[^\]}}]*(?:\]|$)|\{{[^\]}}]*(?:\}}|$)|[^,}}\s]+)" |
| 40 | +) |
| 41 | +JSON_LIKE_SECRET_BARE_FIELD_RE = re.compile( |
| 42 | + rf"(?i)([{{,]\s*)({SECRET_KEY_PATTERN})(?![A-Za-z0-9_.-])\s*:\s*(?!<redacted>)(?:\[[^\]}}]*(?:\]|$)|\{{[^\]}}]*(?:\}}|$)|[^,}}\s]+)" |
| 43 | +) |
| 44 | +ESCAPED_JSON_SECRET_FIELD_RE = re.compile( |
| 45 | + rf"(?i)((?:\\)?['\"])({SECRET_KEY_PATTERN})\1\s*:\s*((?:\\)?['\"])(?:(?!\3).)*(?:\3|(?=,|$))" |
21 | 46 | ) |
22 | 47 | SECRET_PATH_VALUE_ASSIGNMENT_RE = re.compile( |
23 | 48 | rf"(?i)(?<![A-Za-z0-9_.-])({SECRET_KEY_PATTERN})(?![A-Za-z0-9_.-])\s*[:=]\s*(?:(?:bearer|basic|token)\s+)?<path:[^>]+>" |
@@ -68,13 +93,13 @@ def serialize_issue(issue: DiagnosticIssue) -> dict[str, Any]: |
68 | 93 | return { |
69 | 94 | "type": issue.type, |
70 | 95 | "field": issue.field, |
71 | | - "expected": _json_safe(issue.expected), |
| 96 | + "expected": redact_actual(_json_safe(issue.expected)), |
72 | 97 | "actual": redact_actual(issue.actual), |
73 | 98 | "message": redact_actual(issue.message), |
74 | | - "recovery": issue.recovery, |
75 | | - "code": issue.code, |
| 99 | + "recovery": redact_actual(issue.recovery), |
| 100 | + "code": redact_actual(issue.code), |
76 | 101 | "severity": issue.severity, |
77 | | - "source": issue.source, |
| 102 | + "source": redact_actual(issue.source), |
78 | 103 | } |
79 | 104 |
|
80 | 105 |
|
@@ -173,18 +198,199 @@ def _json_safe(value: Any) -> Any: |
173 | 198 |
|
174 | 199 |
|
175 | 200 | def _redact_string(value: str) -> str: |
| 201 | + structured = _redact_json_string(value) |
| 202 | + if structured is not None: |
| 203 | + return structured |
| 204 | + value = JSON_LIKE_SECRET_FIELD_RE.sub(lambda match: f"{match.group(1)}{match.group(2)}{match.group(3)}{match.group(2)}:{match.group(4)}<redacted>{match.group(4)}", value) |
| 205 | + value = _redact_sensitive_json_assignments(value) |
| 206 | + value = _redact_quoted_json_strings(value) |
| 207 | + value = _redact_embedded_json(value) |
| 208 | + value = JSON_LIKE_SECRET_UNQUOTED_FIELD_RE.sub(lambda match: f"{match.group(1)}{match.group(2)}{match.group(3)}{match.group(2)}:<redacted>", value) |
| 209 | + value = JSON_LIKE_SECRET_BARE_FIELD_RE.sub(lambda match: f"{match.group(1)}{match.group(2)}:<redacted>", value) |
| 210 | + value = ESCAPED_JSON_SECRET_FIELD_RE.sub(lambda match: f"{match.group(1)}{match.group(2)}{match.group(1)}:{match.group(3)}<redacted>{match.group(3)}", value) |
176 | 211 | value = ABSOLUTE_PATH_WITH_EXT_RE.sub(_path_placeholder, value) |
177 | 212 | value = ABSOLUTE_PATH_BEFORE_SECRET_RE.sub(_path_before_secret_placeholder, value) |
178 | 213 | value = ABSOLUTE_PATH_RE.sub(_path_placeholder, value) |
179 | 214 | value = SECRET_PATH_VALUE_ASSIGNMENT_RE.sub(lambda match: f"{match.group(1)}=<redacted>", value) |
180 | 215 | value = SECRET_PATH_PLACEHOLDER_ASSIGNMENT_RE.sub(lambda match: f"{match.group(1)}=<redacted>", value) |
181 | | - value = SECRET_QUOTED_ASSIGNMENT_RE.sub(lambda match: f"{match.group(1)}=<redacted>", value) |
182 | | - value = SECRET_ASSIGNMENT_RE.sub(lambda match: f"{match.group(1)}=<redacted>", value) |
| 216 | + value = SECRET_QUOTED_ASSIGNMENT_RE.sub(lambda match: f"{match.group(2)}=<redacted>", value) |
| 217 | + value = SECRET_ASSIGNMENT_RE.sub(lambda match: f"{match.group(2)}=<redacted>", value) |
| 218 | + value = COMMA_SECRET_COLLECTION_ASSIGNMENT_RE.sub(lambda match: f"{match.group(1)}=<redacted>", value) |
| 219 | + value = COMMA_SECRET_QUOTED_ASSIGNMENT_RE.sub(lambda match: f"{match.group(1)}=<redacted>", value) |
| 220 | + value = COMMA_SECRET_ASSIGNMENT_RE.sub(lambda match: f"{match.group(1)}=<redacted>", value) |
183 | 221 | if len(value) > MAX_STRING_LENGTH: |
184 | 222 | return f"{value[:MAX_STRING_LENGTH]}...<truncated {len(value) - MAX_STRING_LENGTH} chars>" |
185 | 223 | return value |
186 | 224 |
|
187 | 225 |
|
| 226 | +def _redact_json_string(value: str) -> str | None: |
| 227 | + stripped = value.strip() |
| 228 | + if not (stripped.startswith("{") or stripped.startswith("[")): |
| 229 | + return None |
| 230 | + try: |
| 231 | + parsed = json.loads(stripped) |
| 232 | + except json.JSONDecodeError: |
| 233 | + try: |
| 234 | + parsed = ast.literal_eval(stripped) |
| 235 | + except (SyntaxError, ValueError): |
| 236 | + return None |
| 237 | + if isinstance(parsed, str) and parsed.strip().startswith(("{", "[")): |
| 238 | + redacted = redact_actual(parsed) |
| 239 | + encoded = json.dumps(redacted, separators=(",", ":")) |
| 240 | + return encoded if len(encoded) <= MAX_STRING_LENGTH else f"{encoded[:MAX_STRING_LENGTH]}...<truncated {len(encoded) - MAX_STRING_LENGTH} chars>" |
| 241 | + redacted = redact_actual(parsed) |
| 242 | + encoded = json.dumps(redacted, separators=(",", ":")) |
| 243 | + return encoded if len(encoded) <= MAX_STRING_LENGTH else f"{encoded[:MAX_STRING_LENGTH]}...<truncated {len(encoded) - MAX_STRING_LENGTH} chars>" |
| 244 | + |
| 245 | + |
| 246 | +def _redact_sensitive_json_assignments(value: str) -> str: |
| 247 | + output: list[str] = [] |
| 248 | + index = 0 |
| 249 | + changed = False |
| 250 | + while index < len(value): |
| 251 | + match = SECRET_ASSIGNMENT_PREFIX_RE.match(value, index) |
| 252 | + if not match: |
| 253 | + output.append(value[index]) |
| 254 | + index += 1 |
| 255 | + continue |
| 256 | + value_start = match.end() |
| 257 | + if value_start >= len(value) or value[value_start] not in "{[": |
| 258 | + output.append(value[index]) |
| 259 | + index += 1 |
| 260 | + continue |
| 261 | + decoded = _decode_collection_prefix(value[value_start:]) |
| 262 | + if decoded is None: |
| 263 | + output.append(f"{match.group(2)}=<redacted>") |
| 264 | + index = len(value) |
| 265 | + changed = True |
| 266 | + continue |
| 267 | + _parsed, end = decoded |
| 268 | + output.append(f"{match.group(2)}=<redacted>") |
| 269 | + index = value_start + end |
| 270 | + changed = True |
| 271 | + return "".join(output) if changed else value |
| 272 | + |
| 273 | + |
| 274 | +def _redact_quoted_json_strings(value: str) -> str: |
| 275 | + decoder = json.JSONDecoder() |
| 276 | + output: list[str] = [] |
| 277 | + index = 0 |
| 278 | + changed = False |
| 279 | + while index < len(value): |
| 280 | + if value[index] not in "\"'": |
| 281 | + output.append(value[index]) |
| 282 | + index += 1 |
| 283 | + continue |
| 284 | + try: |
| 285 | + parsed, end = decoder.raw_decode(value[index:]) |
| 286 | + except json.JSONDecodeError: |
| 287 | + literal = _decode_quoted_literal_prefix(value[index:]) |
| 288 | + if literal is None: |
| 289 | + output.append(value[index]) |
| 290 | + index += 1 |
| 291 | + continue |
| 292 | + parsed, end = literal |
| 293 | + if not (isinstance(parsed, str) and parsed.strip().startswith(("{", "["))): |
| 294 | + output.append(value[index]) |
| 295 | + index += 1 |
| 296 | + continue |
| 297 | + output.append(json.dumps(redact_actual(parsed), separators=(",", ":"))) |
| 298 | + index += end |
| 299 | + changed = True |
| 300 | + return "".join(output) if changed else value |
| 301 | + |
| 302 | + |
| 303 | +def _redact_embedded_json(value: str) -> str: |
| 304 | + output: list[str] = [] |
| 305 | + index = 0 |
| 306 | + changed = False |
| 307 | + while index < len(value): |
| 308 | + if value[index] not in "{[": |
| 309 | + output.append(value[index]) |
| 310 | + index += 1 |
| 311 | + continue |
| 312 | + decoded = _decode_collection_prefix(value[index:]) |
| 313 | + if decoded is None: |
| 314 | + output.append(value[index]) |
| 315 | + index += 1 |
| 316 | + continue |
| 317 | + parsed, end = decoded |
| 318 | + redacted = redact_actual(parsed) |
| 319 | + output.append(json.dumps(redacted, separators=(",", ":"))) |
| 320 | + index += end |
| 321 | + changed = True |
| 322 | + return "".join(output) if changed else value |
| 323 | + |
| 324 | + |
| 325 | +def _decode_collection_prefix(value: str) -> tuple[Any, int] | None: |
| 326 | + decoder = json.JSONDecoder() |
| 327 | + try: |
| 328 | + return decoder.raw_decode(value) |
| 329 | + except json.JSONDecodeError: |
| 330 | + pass |
| 331 | + end = _balanced_collection_end(value) |
| 332 | + if end <= 0: |
| 333 | + return None |
| 334 | + try: |
| 335 | + return ast.literal_eval(value[:end]), end |
| 336 | + except (SyntaxError, ValueError): |
| 337 | + return None |
| 338 | + |
| 339 | + |
| 340 | +def _decode_quoted_literal_prefix(value: str) -> tuple[Any, int] | None: |
| 341 | + end = _quoted_literal_end(value) |
| 342 | + if end <= 0: |
| 343 | + return None |
| 344 | + try: |
| 345 | + return ast.literal_eval(value[:end]), end |
| 346 | + except (SyntaxError, ValueError): |
| 347 | + return None |
| 348 | + |
| 349 | + |
| 350 | +def _balanced_collection_end(value: str) -> int: |
| 351 | + if not value or value[0] not in "{[": |
| 352 | + return -1 |
| 353 | + opening = {"{": "}", "[": "]"} |
| 354 | + stack = [opening[value[0]]] |
| 355 | + quote = "" |
| 356 | + escaped = False |
| 357 | + for index, char in enumerate(value[1:], start=1): |
| 358 | + if quote: |
| 359 | + if escaped: |
| 360 | + escaped = False |
| 361 | + elif char == "\\": |
| 362 | + escaped = True |
| 363 | + elif char == quote: |
| 364 | + quote = "" |
| 365 | + continue |
| 366 | + if char in {"'", '"'}: |
| 367 | + quote = char |
| 368 | + continue |
| 369 | + if char in opening: |
| 370 | + stack.append(opening[char]) |
| 371 | + continue |
| 372 | + if stack and char == stack[-1]: |
| 373 | + stack.pop() |
| 374 | + if not stack: |
| 375 | + return index + 1 |
| 376 | + return -1 |
| 377 | + |
| 378 | + |
| 379 | +def _quoted_literal_end(value: str) -> int: |
| 380 | + if not value or value[0] not in {"'", '"'}: |
| 381 | + return -1 |
| 382 | + quote = value[0] |
| 383 | + escaped = False |
| 384 | + for index, char in enumerate(value[1:], start=1): |
| 385 | + if escaped: |
| 386 | + escaped = False |
| 387 | + elif char == "\\": |
| 388 | + escaped = True |
| 389 | + elif char == quote: |
| 390 | + return index + 1 |
| 391 | + return -1 |
| 392 | + |
| 393 | + |
188 | 394 | def _path_placeholder(match: re.Match[str]) -> str: |
189 | 395 | path = match.group(0) |
190 | 396 | name = path.replace("\\", "/").rstrip("/").rsplit("/", 1)[-1] |
|
0 commit comments