Skip to content

Commit fc12bc2

Browse files
SamuelHassineSamuelHassine
committed
[ctm360] Address review feedback on CyberBlindSpot connector
Apply the outstanding reviewer feedback to make the connector merge-ready: - normalize_timestamp(): convert offset-aware ISO-8601 timestamps to UTC before formatting the trailing "Z" so values are no longer silently shifted (e.g. 18:00+02:00 now yields 16:00Z), and treat numeric 0 as a valid epoch (1970-01-01T00:00:00Z) instead of a missing value. - Tighten the timestamp tests to assert exact UTC output for positive and negative offsets and to cover the epoch-zero case. - Pass structured logging metadata via the meta= keyword across the connector, status tracker, API client, and converter, matching the pycti logger signature and the rest of the repo. - Seed the status tracker with the incident's normalised status instead of a hard-coded "unknown", so the first poll cycle does not re-apply an already-present status:<value> label.
1 parent ca0f48d commit fc12bc2

6 files changed

Lines changed: 65 additions & 40 deletions

File tree

‎external-import/ctm360-cyberblindspot-feed/src/connector/case_status_tracker.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ def start(self):
3838
self._thread.start()
3939
self.helper.connector_logger.info(
4040
"[STATUS-TRACKER] Background polling started",
41-
{"poll_interval": self.poll_interval},
41+
meta={"poll_interval": self.poll_interval},
4242
)
4343

4444
def stop(self):
@@ -72,7 +72,7 @@ def _poll_loop(self):
7272
self._check_all_cases()
7373
except Exception as e:
7474
self.helper.connector_logger.error(
75-
"[STATUS-TRACKER] Poll cycle error", {"error": str(e)}
75+
"[STATUS-TRACKER] Poll cycle error", meta={"error": str(e)}
7676
)
7777
self._stop_event.wait(self.poll_interval)
7878

@@ -86,7 +86,7 @@ def _check_all_cases(self):
8686

8787
self.helper.connector_logger.info(
8888
"[STATUS-TRACKER] Checking tracked cases",
89-
{"count": len(tracked)},
89+
meta={"count": len(tracked)},
9090
)
9191

9292
for ticket_id, info in tracked.items():
@@ -95,7 +95,7 @@ def _check_all_cases(self):
9595
except Exception as e:
9696
self.helper.connector_logger.error(
9797
"[STATUS-TRACKER] Failed to check case",
98-
{"ticket_id": ticket_id, "error": str(e)},
98+
meta={"ticket_id": ticket_id, "error": str(e)},
9999
)
100100

101101
def _check_single_case(self, ticket_id: str, info: dict):
@@ -111,7 +111,7 @@ def _check_single_case(self, ticket_id: str, info: dict):
111111

112112
self.helper.connector_logger.info(
113113
"[STATUS-TRACKER] Status change detected",
114-
{
114+
meta={
115115
"ticket_id": ticket_id,
116116
"old_status": last_status,
117117
"new_status": current_status,
@@ -154,10 +154,10 @@ def _update_case_label(
154154
)
155155
self.helper.connector_logger.info(
156156
"[STATUS-TRACKER] Updated case label",
157-
{"case_id": case_incident_id, "label": new_label},
157+
meta={"case_id": case_incident_id, "label": new_label},
158158
)
159159
except Exception as e:
160160
self.helper.connector_logger.error(
161161
"[STATUS-TRACKER] Failed to update case label",
162-
{"case_id": case_incident_id, "error": str(e)},
162+
meta={"case_id": case_incident_id, "error": str(e)},
163163
)

‎external-import/ctm360-cyberblindspot-feed/src/connector/connector.py‎

Lines changed: 28 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def _resolve_author_id(self) -> str:
4343
return result.get("id", "") if result else ""
4444
except Exception as e:
4545
self.helper.connector_logger.warning(
46-
"[CONNECTOR] Failed to resolve author identity", {"error": str(e)}
46+
"[CONNECTOR] Failed to resolve author identity", meta={"error": str(e)}
4747
)
4848
return ""
4949

@@ -53,15 +53,15 @@ def run(self):
5353
self.helper.connector_logger.info("[CONNECTOR] API connection verified")
5454
except Exception as exc:
5555
self.helper.connector_logger.error(
56-
"[CONNECTOR] API ping failed — stopping", {"error": str(exc)}
56+
"[CONNECTOR] API ping failed — stopping", meta={"error": str(exc)}
5757
)
5858
sys.exit(1)
5959

6060
self._author_opencti_id = self._resolve_author_id()
6161
if self._author_opencti_id:
6262
self.helper.connector_logger.info(
6363
"[CONNECTOR] Author identity resolved",
64-
{"id": self._author_opencti_id},
64+
meta={"id": self._author_opencti_id},
6565
)
6666

6767
if self._enable_tracking:
@@ -75,7 +75,7 @@ def run(self):
7575

7676
self.helper.connector_logger.info(
7777
"[CONNECTOR] Starting import loop",
78-
{"interval_seconds": self._interval},
78+
meta={"interval_seconds": self._interval},
7979
)
8080
while True:
8181
try:
@@ -85,7 +85,7 @@ def run(self):
8585
break
8686
except Exception as e:
8787
self.helper.connector_logger.error(
88-
"[CONNECTOR] Import cycle failed", {"error": str(e)}
88+
"[CONNECTOR] Import cycle failed", meta={"error": str(e)}
8989
)
9090
time.sleep(self._interval)
9191

@@ -114,12 +114,12 @@ def _import_data(self):
114114
all_objects.extend(objects)
115115
self.helper.connector_logger.info(
116116
"[CONNECTOR] Incidents processed",
117-
{"count": len(data), "stix_objects": len(objects)},
117+
meta={"count": len(data), "stix_objects": len(objects)},
118118
)
119119
except Exception as e:
120120
errors.append(f"incidents: {e}")
121121
self.helper.connector_logger.error(
122-
"[CONNECTOR] Incidents fetch failed", {"error": str(e)}
122+
"[CONNECTOR] Incidents fetch failed", meta={"error": str(e)}
123123
)
124124

125125
if self._import_malware_logs:
@@ -130,12 +130,12 @@ def _import_data(self):
130130
all_objects.extend(objects)
131131
self.helper.connector_logger.info(
132132
"[CONNECTOR] Malware logs processed",
133-
{"count": len(data), "stix_objects": len(objects)},
133+
meta={"count": len(data), "stix_objects": len(objects)},
134134
)
135135
except Exception as e:
136136
errors.append(f"malware_logs: {e}")
137137
self.helper.connector_logger.error(
138-
"[CONNECTOR] Malware logs fetch failed", {"error": str(e)}
138+
"[CONNECTOR] Malware logs fetch failed", meta={"error": str(e)}
139139
)
140140

141141
if self._import_breached_creds:
@@ -146,13 +146,13 @@ def _import_data(self):
146146
all_objects.extend(objects)
147147
self.helper.connector_logger.info(
148148
"[CONNECTOR] Breached credentials processed",
149-
{"count": len(data), "stix_objects": len(objects)},
149+
meta={"count": len(data), "stix_objects": len(objects)},
150150
)
151151
except Exception as e:
152152
errors.append(f"breached_credentials: {e}")
153153
self.helper.connector_logger.error(
154154
"[CONNECTOR] Breached credentials fetch failed",
155-
{"error": str(e)},
155+
meta={"error": str(e)},
156156
)
157157

158158
if self._import_card_leaks:
@@ -163,12 +163,12 @@ def _import_data(self):
163163
all_objects.extend(objects)
164164
self.helper.connector_logger.info(
165165
"[CONNECTOR] Card leaks processed",
166-
{"count": len(data), "stix_objects": len(objects)},
166+
meta={"count": len(data), "stix_objects": len(objects)},
167167
)
168168
except Exception as e:
169169
errors.append(f"card_leaks: {e}")
170170
self.helper.connector_logger.error(
171-
"[CONNECTOR] Card leaks fetch failed", {"error": str(e)}
171+
"[CONNECTOR] Card leaks fetch failed", meta={"error": str(e)}
172172
)
173173

174174
if self._import_domain_protection:
@@ -179,13 +179,13 @@ def _import_data(self):
179179
all_objects.extend(objects)
180180
self.helper.connector_logger.info(
181181
"[CONNECTOR] Domain protection processed",
182-
{"count": len(data), "stix_objects": len(objects)},
182+
meta={"count": len(data), "stix_objects": len(objects)},
183183
)
184184
except Exception as e:
185185
errors.append(f"domain_protection: {e}")
186186
self.helper.connector_logger.error(
187187
"[CONNECTOR] Domain protection fetch failed",
188-
{"error": str(e)},
188+
meta={"error": str(e)},
189189
)
190190

191191
if len(errors) == categories_attempted and categories_attempted > 0:
@@ -217,7 +217,9 @@ def _import_data(self):
217217
if errors:
218218
msg += f" (partial: {len(errors)} categories failed)"
219219

220-
self.helper.connector_logger.info("[CONNECTOR] Import done", {"msg": msg})
220+
self.helper.connector_logger.info(
221+
"[CONNECTOR] Import done", meta={"msg": msg}
222+
)
221223
self.helper.api.work.to_processed(work_id, msg)
222224
# Preserve other state keys (e.g. tracked_cases written by the status
223225
# tracker) and update under the shared lock to avoid races.
@@ -229,7 +231,7 @@ def _import_data(self):
229231
except Exception as e:
230232
if "All" not in str(e):
231233
self.helper.connector_logger.error(
232-
"[CONNECTOR] Import failed", {"error": str(e)}
234+
"[CONNECTOR] Import failed", meta={"error": str(e)}
233235
)
234236
self.helper.api.work.to_processed(work_id, str(e), in_error=True)
235237
raise
@@ -246,11 +248,11 @@ def _create_case_incidents(self, case_metadata: list) -> int:
246248
except Exception as e:
247249
self.helper.connector_logger.error(
248250
"[CONNECTOR] Failed to create CaseIncident",
249-
{"ticket_id": meta.get("ticket_id"), "error": str(e)},
251+
meta={"ticket_id": meta.get("ticket_id"), "error": str(e)},
250252
)
251253
self.helper.connector_logger.info(
252254
"[CONNECTOR] CaseIncidents created",
253-
{"created": created, "total": len(case_metadata)},
255+
meta={"created": created, "total": len(case_metadata)},
254256
)
255257
return created
256258

@@ -283,7 +285,7 @@ def _create_case_incident(self, meta: dict):
283285
if case_id:
284286
self.helper.connector_logger.info(
285287
"[CONNECTOR] CaseIncident created",
286-
{"case_id": case_id, "ticket_id": ticket_id, "name": meta["name"]},
288+
meta={"case_id": case_id, "ticket_id": ticket_id, "name": meta["name"]},
287289
)
288290
# Add labels individually
289291
for label in meta.get("labels", []):
@@ -294,15 +296,16 @@ def _create_case_incident(self, meta: dict):
294296
except Exception as e:
295297
self.helper.connector_logger.warning(
296298
"[CONNECTOR] Failed to add label",
297-
{"case_id": case_id, "label": label, "error": str(e)},
299+
meta={"case_id": case_id, "label": label, "error": str(e)},
298300
)
299-
# Register with status tracker
301+
# Register with status tracker, seeding the last-known status from
302+
# the incident metadata (matching the `status:<value>` label already
303+
# applied) so the first poll cycle does not re-add an existing label.
300304
if self._tracker:
301-
initial_status = "unknown"
302305
self._tracker.register_case(
303306
ticket_id=ticket_id,
304307
case_incident_id=case_id,
305-
initial_status=initial_status,
308+
initial_status=meta.get("status", "unknown"),
306309
)
307310
# Set response types
308311
resp_types = meta.get("response_types", [])
@@ -315,5 +318,5 @@ def _create_case_incident(self, meta: dict):
315318
except Exception as e:
316319
self.helper.connector_logger.warning(
317320
"[CONNECTOR] Failed to set response_types",
318-
{"case_id": case_id, "error": str(e)},
321+
meta={"case_id": case_id, "error": str(e)},
319322
)

‎external-import/ctm360-cyberblindspot-feed/src/connector/converter_to_stix.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ def incidents_to_stix(self, incidents: list) -> list:
7878
if not inc_id:
7979
self.helper.connector_logger.warning(
8080
"[CONVERTER] Skipping incident with no id",
81-
{"subject": subject},
81+
meta={"subject": subject},
8282
)
8383
continue
8484

@@ -129,6 +129,9 @@ def incidents_to_stix(self, incidents: list) -> list:
129129
"response_types": [inc_type] if inc_type else [],
130130
"labels": case_labels,
131131
"created": created,
132+
# Normalised current status, used to seed the status tracker
133+
# so the first poll cycle does not treat it as a change.
134+
"status": str(status).lower(),
132135
}
133136
)
134137

‎external-import/ctm360-cyberblindspot-feed/src/connector/utils.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ def normalize_timestamp(ts) -> str:
1212
- Epoch milliseconds (1772614383785)
1313
- Epoch seconds (1772614383)
1414
"""
15-
if not ts:
15+
# Use an explicit None/empty-string check so a numeric 0 (a valid epoch,
16+
# 1970-01-01T00:00:00Z) is handled by the epoch branch below.
17+
if ts is None or ts == "":
1618
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
1719

1820
# Handle epoch timestamps (int or float)
@@ -55,8 +57,13 @@ def normalize_timestamp(ts) -> str:
5557
try:
5658
ts = ts.replace("Z", "+00:00")
5759
dt = datetime.fromisoformat(ts)
60+
# Treat naive timestamps as UTC, and convert offset-aware timestamps
61+
# to UTC before formatting so the trailing "Z" is always accurate
62+
# (e.g. 18:00+02:00 -> 16:00Z, not 18:00Z).
5863
if dt.tzinfo is None:
5964
dt = dt.replace(tzinfo=timezone.utc)
65+
else:
66+
dt = dt.astimezone(timezone.utc)
6067
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
6168
except (ValueError, AttributeError):
6269
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")

‎external-import/ctm360-cyberblindspot-feed/src/ctm360_cbs_client/api_client.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ def _request(self, method: str, path: str, params: dict = None) -> dict:
2828
try:
2929
self.helper.connector_logger.debug(
3030
"[API] Request",
31-
{"method": method, "url": url, "params": params},
31+
meta={"method": method, "url": url, "params": params},
3232
)
3333
response = self.session.request(method, url, params=params, timeout=60)
3434
if response.status_code == 204:
@@ -41,7 +41,7 @@ def _request(self, method: str, path: str, params: dict = None) -> dict:
4141
)
4242
self.helper.connector_logger.warning(
4343
"[API] Rate limited, waiting",
44-
{"retry_after": retry_after},
44+
meta={"retry_after": retry_after},
4545
)
4646
time.sleep(retry_after)
4747
continue
@@ -100,7 +100,7 @@ def _paginated_request(
100100
total = data.get("count", 0)
101101
self.helper.connector_logger.debug(
102102
"[API] Page fetched",
103-
{"path": path, "page": page, "items": len(items), "total": total},
103+
meta={"path": path, "page": page, "items": len(items), "total": total},
104104
)
105105

106106
if len(items) < page_size or len(all_items) >= total:

‎external-import/ctm360-cyberblindspot-feed/tests/test_connector/test_utils.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,21 @@ def test_iso8601_without_timezone(self):
5454
assert normalize_timestamp("2026-03-04T18:00:00") == "2026-03-04T18:00:00Z"
5555

5656
def test_iso8601_with_offset(self):
57-
# +02:00 offset normalised back to UTC representation of the same wall clock
58-
result = normalize_timestamp("2026-03-04T18:00:00+02:00")
59-
assert ISO_Z_PATTERN.match(result)
57+
# +02:00 offset must be converted to UTC, not just stamped with "Z":
58+
# 18:00+02:00 is 16:00Z.
59+
assert (
60+
normalize_timestamp("2026-03-04T18:00:00+02:00") == "2026-03-04T16:00:00Z"
61+
)
62+
63+
def test_iso8601_with_negative_offset(self):
64+
# -05:00 offset -> UTC is five hours ahead of the local wall clock.
65+
assert (
66+
normalize_timestamp("2026-03-04T18:00:00-05:00") == "2026-03-04T23:00:00Z"
67+
)
68+
69+
def test_epoch_zero_is_unix_epoch(self):
70+
# 0 is a valid epoch (1970-01-01T00:00:00Z), not a missing value.
71+
assert normalize_timestamp(0) == "1970-01-01T00:00:00Z"
6072

6173
def test_invalid_string_returns_now(self):
6274
result = normalize_timestamp("definitely not a date")

0 commit comments

Comments
 (0)