[postgres] Prevent incidents by detecting idle sessions holding locks (DataDog#21182)

* Add idle in transaction with locks

* Fix metrix definition

* Add config

* Rename metric

* Rename changelog to match PR#

* Validate config

* Validate config

* Fix config with new ddev

* Restore test_deadlock.py

* Validate metadata

* Validate models

* Lint

* Include shared locks

* max_rows as parameter

* Fix max_rows

* Validate model

* Hide config

* Restore test_deadlock.py

* Select only exclusive locks

* Filter on state_change age instead of tx change to reduce noise

* State change not null

* Update postgres/changelog.d/21182.added

Co-authored-by: Eric Weaver <eweaver755@gmail.com>

---------

Co-authored-by: Eric Weaver <eweaver755@gmail.com>

Author: nenadnoveljic

postgres/assets/configuration/spec.yaml

@@ -282,6 +282,30 @@ files:
       value:
         type: boolean
         example: true
+    - name: locks_idle_in_transaction
+      description: Configure collection of idle in transaction lock age metrics
+      options:
+        - name: enabled
+          description: Enable collection of idle in transaction lock age metrics.
+          value:
+            type: boolean
+            example: true
+            display_default: true
+          hidden: true
+        - name: collection_interval
+          description: Set the collection interval (in seconds) for idle in transaction lock age metrics.
+          value:
+            type: number
+            example: 300
+            display_default: 300
+          hidden: true
+        - name: max_rows
+          description: Set the maximum number of rows to collect per check run.
+          value:
+            type: integer
+            example: 100
+            display_default: 100
+          hidden: true
     - name: collect_checksum_metrics
       description: Collect counts of database failed checksums. Only supported on versions >= 12.
       value:

postgres/changelog.d/21182.added

@@ -0,0 +1 @@
+Add gauge postgresql.locks.idle_in_transaction_age to measure age (s) of idle-in-transaction sessions holding exclusive relation locks; tags: pid, db, session_user, app, client_hostname, lock_mode, relation, relation_owner; limit 100 rows.

postgres/datadog_checks/postgres/config.py

@@ -91,6 +91,13 @@ def __init__(self, instance, init_config, check):
         self.collect_database_size_metrics = is_affirmative(instance.get('collect_database_size_metrics', True))
         self.collect_wal_metrics = self._should_collect_wal_metrics(instance.get('collect_wal_metrics'))
         self.collect_bloat_metrics = is_affirmative(instance.get('collect_bloat_metrics', False))
+        # Locks idle in transaction metrics config
+        locks_idle_cfg = instance.get('locks_idle_in_transaction', {}) or {}
+        self.locks_idle_in_transaction = {
+            'enabled': is_affirmative(locks_idle_cfg.get('enabled', True)),
+            'collection_interval': int(locks_idle_cfg.get('collection_interval', 300)),
+            'max_rows': int(locks_idle_cfg.get('max_rows', 100)),
+        }
         self.data_directory = instance.get('data_directory', None)
         self.ignore_databases = instance.get('ignore_databases', DEFAULT_IGNORE_DATABASES)
         if is_affirmative(instance.get('collect_default_database', True)):

postgres/datadog_checks/postgres/config_models/instance.py

@@ -136,6 +136,16 @@ class Gcp(BaseModel):
     project_id: Optional[str] = None
 
 
+class LocksIdleInTransaction(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    collection_interval: Optional[float] = None
+    enabled: Optional[bool] = None
+    max_rows: Optional[int] = None
+
+
 class ManagedIdentity(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
@@ -263,6 +273,7 @@ class InstanceConfig(BaseModel):
     idle_connection_timeout: Optional[int] = None
     ignore_databases: Optional[tuple[str, ...]] = None
     ignore_schemas_owned_by: Optional[tuple[str, ...]] = None
+    locks_idle_in_transaction: Optional[LocksIdleInTransaction] = None
     log_unobfuscated_plans: Optional[bool] = None
     log_unobfuscated_queries: Optional[bool] = None
     managed_identity: Optional[ManagedIdentity] = None

postgres/datadog_checks/postgres/data/conf.yaml.example

@@ -226,6 +226,10 @@ instances:
     #
     # collect_count_metrics: true
 
+    ## Configure collection of idle in transaction lock age metrics
+    #
+    # locks_idle_in_transaction:
+
     ## @param collect_checksum_metrics - boolean - optional - default: false
     ## Collect counts of database failed checksums. Only supported on versions >= 12.
     #

postgres/datadog_checks/postgres/postgres.py

@@ -47,6 +47,7 @@
     CONNECTION_METRICS,
     COUNT_METRICS,
     FUNCTION_METRICS,
+    IDLE_TX_LOCK_AGE_METRICS,
     INDEX_PROGRESS_METRICS,
     QUERY_PG_CONTROL_CHECKPOINT,
     QUERY_PG_CONTROL_CHECKPOINT_LT_10,
@@ -381,6 +382,13 @@ def dynamic_queries(self):
             if self._config.dbm_enabled:
                 queries.append(STAT_IO_METRICS)
 
+        if self._config.dbm_enabled and self._config.locks_idle_in_transaction['enabled']:
+            query_def = copy.deepcopy(IDLE_TX_LOCK_AGE_METRICS)
+            query_def['collection_interval'] = self._config.locks_idle_in_transaction['collection_interval']
+            max_rows = self._config.locks_idle_in_transaction.get('max_rows', 100)
+            query_def['query'] = query_def['query'].format(max_rows=max_rows)
+            per_database_queries.append(query_def)
+
         if not queries:
             self.log.debug("no dynamic queries defined")
             return None

postgres/datadog_checks/postgres/util.py

@@ -1085,3 +1085,45 @@ def trim_leading_set_stmts(sql):
         {'name': 'io.writes', 'type': 'monotonic_count'},
     ],
 }
+
+# Measures the age (in seconds) of idle-in-transaction sessions holding exclusive relation locks.
+# Limits result set to 10 rows to avoid tag explosion.
+IDLE_TX_LOCK_AGE_METRICS = {
+    'name': 'idle_tx_lock_age_metrics',
+    'query': (
+        """
+SELECT
+    l.pid,
+    a.datname,
+    a.usename             AS session_user,
+    a.application_name,
+    a.client_hostname,
+    l.mode,
+    c.oid::regclass       AS relation,
+    r.rolname             AS relation_owner,
+    EXTRACT(EPOCH FROM (now() - a.xact_start)) AS xact_age
+FROM pg_locks l
+JOIN pg_stat_activity a ON a.pid = l.pid
+JOIN pg_class c         ON c.oid = l.relation
+JOIN pg_roles r         ON r.oid = c.relowner
+WHERE l.locktype = 'relation'
+  AND l.granted = true
+  AND l.mode like '%Exclusive%'
+  AND a.state = 'idle in transaction'
+  AND a.state_change IS NOT NULL
+  AND now() - a.state_change > interval '60 seconds'
+ORDER BY xact_age DESC
+LIMIT {max_rows} ;        """
+    ).strip(),
+    'columns': [
+        {'name': 'pid', 'type': 'tag'},
+        {'name': 'db', 'type': 'tag'},
+        {'name': 'session_user', 'type': 'tag'},
+        {'name': 'app', 'type': 'tag'},
+        {'name': 'client_hostname', 'type': 'tag_not_null'},
+        {'name': 'lock_mode', 'type': 'tag'},
+        {'name': 'relation', 'type': 'tag'},
+        {'name': 'relation_owner', 'type': 'tag'},
+        {'name': 'locks.idle_in_transaction_age', 'type': 'gauge'},
+    ],
+}

postgres/metadata.csv

@@ -101,6 +101,7 @@ postgresql.last_autovacuum_age,gauge,,second,,"Last time at which this table was
 postgresql.last_vacuum_age,gauge,,second,,"Last time at which this table was manually vacuumed (not counting VACUUM FULL). This metric is tagged with db, schema, table.",0,postgres,age vacuum,,
 postgresql.live_rows,gauge,,row,,"Enabled with `relations`. The estimated number of live rows. This metric is tagged with db, schema, table.",0,postgres,live rows,,
 postgresql.locks,gauge,,lock,,"Enabled with `relations`. The number of locks active for this database. This metric is tagged with db, lock_mode, lock_type, schema, table, granted.",0,postgres,locks,,
+postgresql.locks.idle_in_transaction_age,gauge,,second,,Transaction age of idle in transaction sessions holding exclusive relation locks.,0,postgres,idle tx age with locks,,"pid,db,session_user,app,client_hostname,lock_mode,relation,relation_owner"
 postgresql.max_connections,gauge,,connection,, The maximum number of client connections allowed to this database.,0,postgres,max conns,,
 postgresql.percent_usage_connections,gauge,,fraction,,The number of connections to this database as a fraction of the maximum number of allowed connections.,0,postgres,pct usg conns,,
 postgresql.pg_stat_statements.dealloc,count,,,,The number of times pg_stat_statements had to evict least executed queries because pg_stat_statements.max was reached.,-1,postgres,pgss dealloc,,
@@ -235,4 +236,4 @@ postgresql.wal_receiver.last_msg_receipt_age,gauge,,second,,Time since the recep
 postgresql.wal_receiver.last_msg_send_age,gauge,,second,,The age of the latest message's send time received from the WAL sender. This metric is tagged with status.,0,postgres,wal receiver send age,,
 postgresql.wal_receiver.latest_end_age,gauge,,second,,Time since the reception of the last message from the WAL sender with an WAL location update. This metric is tagged with status.,0,postgres,wal receiver latest end,,
 postgresql.wal_receiver.received_timeline,gauge,,,,"Timeline number of last write-ahead log location received and flushed to disk, the initial value of this field being the timeline number of the first log location used when WAL receiver is started. This metric is tagged with status.",0,postgres,wal receiver tli,,
-postgresql.wal_size,gauge,,byte,,The sum of all WAL files on disk.,-1,postgres,wal size,,
+postgresql.wal_size,gauge,,byte,,The sum of all WAL files on disk.,-1,postgres,wal size,,