Use query executor to run index usage queries

mysql/assets/configuration/spec.yaml

@@ -337,6 +337,22 @@ files:
                   type: boolean
                   example: false
 
+              - name: index_metrics
+                description: |
+                  Set to `true` to collect index metrics.
+                  Metrics provided by the options:
+                    - mysql.index.size (per index)
+                    - mysql.index.reads (per index)
+                    - mysql.index.updates (per index)
+                    - mysql.index.deletes (per index)
+                  Note that some of these metrics require the `user` defined for this instance
+                  to have SELECT privileges. Take a look at the
+                  MySQL integration tile in the Datadog Web UI for further instructions.
+
+                value:
+                  type: boolean
+                  example: true
+
               - name: extra_performance_metrics
                 description: |
                   These metrics are reported if `performance_schema` is enabled in the MySQL instance

mysql/datadog_checks/mysql/const.py

@@ -307,9 +307,4 @@
     'Qcache_instant_utilization': ('mysql.performance.qcache.utilization.instant', GAUGE),
 }
 
-INDEX_VARS = {
-    'index_usage': ('mysql.index.usage', GAUGE),
-    'index_size': ('mysql.index.size', GAUGE),
-}
-
 BUILDS = ('log', 'standard', 'debug', 'valgrind', 'embedded')

mysql/datadog_checks/mysql/index_metrics.py

@@ -0,0 +1,74 @@
+# (C) Datadog, Inc. 2025-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+from datadog_checks.base import is_affirmative
+
+QUERY_INDEX_SIZE = {
+    'name': 'mysql.innodb_index_stats',
+    'query': """
+        SELECT 
+            database_name,
+            table_name,
+            index_name,
+            ROUND(stat_value * @@innodb_page_size/1024/1024, 2) AS index_size_mb
+        FROM
+            mysql.innodb_index_stats
+        WHERE
+            stat_name = 'size' AND
+            index_name != 'PRIMARY'
+    """.strip(),
+    'columns': [
+        {'name': 'db', 'type': 'tag'},
+        {'name': 'table', 'type': 'tag'},
+        {'name': 'index', 'type': 'tag'},
+        {'name': 'mysql.index.size', 'type': 'gauge'},
+    ],
+} 
+QUERY_INDEX_USAGE = {
+    'name': 'performance_schema.table_io_waits_summary_by_index_usage',
+    'query': """
+        SELECT
+            object_schema,
+            object_name,
+            index_name,
+            count_read,
+            count_update,
+            count_delete
+        FROM
+            performance_schema.table_io_waits_summary_by_index_usage
+        WHERE index_name IS NOT NULL AND
+        index_name != 'PRIMARY' AND
+        object_schema NOT IN ('mysql', 'performance_schema')
+    """.strip(),
+    'columns': [
+        {'name': 'db', 'type': 'tag'},
+        {'name': 'table', 'type': 'tag'},
+        {'name': 'index', 'type': 'tag'},
+        {'name': 'mysql.index.reads', 'type': 'gauge'},
+        {'name': 'mysql.index.updates', 'type': 'gauge'},
+        {'name': 'mysql.index.deletes', 'type': 'gauge'},
+    ],
+}
+
+class MySqlIndexMetrics():
+    def __init__(self, config):
+        self._config = config
+
+    @property
+    def include_index_metrics(self) -> bool:
+        return is_affirmative(self._config.options.get('index_metrics', True))
+    @property
+    def collection_interval(self) -> int:
+        # TODO ALLEN: change this to 300
+        return 60
+
+    @property
+    def queries(self):
+        # make a copy of the query to avoid modifying the original
+        # in case different instances have different collection intervals
+        usage_query = QUERY_INDEX_USAGE.copy()
+        size_query = QUERY_INDEX_SIZE.copy()
+        usage_query['collection_interval'] = self.collection_interval
+        size_query['collection_interval'] = self.collection_interval
+        return [size_query, usage_query]

mysql/datadog_checks/mysql/mysql.py

@@ -39,7 +39,6 @@
     GAUGE,
     GROUP_REPLICATION_VARS,
     GROUP_REPLICATION_VARS_8_0_2,
-    INDEX_VARS,
     INNODB_VARS,
     MONOTONIC,
     OPTIONAL_STATUS_VARS,
@@ -55,6 +54,7 @@
     TABLE_VARS,
     VARIABLES_VARS,
 )
+from .index_metrics import MySqlIndexMetrics
 from .innodb_metrics import InnoDBMetrics
 from .metadata import MySQLMetadata
 from .queries import (
@@ -69,8 +69,6 @@
     SQL_GROUP_REPLICATION_PLUGIN_STATUS,
     SQL_INNODB_ENGINES,
     SQL_PROCESS_LIST,
-    SQL_QUERY_INDEX_SIZE,
-    SQL_QUERY_INDEX_USAGE,
     SQL_QUERY_SCHEMA_SIZE,
     SQL_QUERY_SYSTEM_TABLE_SIZE,
     SQL_QUERY_TABLE_ROWS_STATS,
@@ -133,6 +131,7 @@ def __init__(self, name, init_config, instances):
         self._statement_samples = MySQLStatementSamples(self, self._config, self._get_connection_args())
         self._mysql_metadata = MySQLMetadata(self, self._config, self._get_connection_args())
         self._query_activity = MySQLActivity(self, self._config, self._get_connection_args())
+        self._index_metrics = MySqlIndexMetrics(self._config)
         # _database_instance_emitted: limit the collection and transmission of the database instance metadata
         self._database_instance_emitted = TTLCache(
             maxsize=1,
@@ -381,7 +380,8 @@ def _get_runtime_queries(self, db):
 
         if self.performance_schema_enabled:
             queries.extend([QUERY_USER_CONNECTIONS])
-
+        if self._index_metrics.include_index_metrics:
+            queries.extend(self._index_metrics.queries)
         self._runtime_queries_cached = self._new_query_executor(queries)
         self._runtime_queries_cached.compile_queries()
         self.log.debug("initialized runtime queries")
@@ -580,14 +580,6 @@ def _collect_metrics(self, db, tags):
                 results['information_table_data_size'] = table_data_size
             metrics.update(TABLE_VARS)
 
-        # TODO ALLEN: implement configuration option, and adjust collection interval
-        if is_affirmative(self._config.options.get('index_usage_metrics', True)):
-            with tracked_query(self, operation="index_size_metrics"):
-                results['index_size'] = self._query_index_size_per_index(db)
-            with tracked_query(self, operation="index_usage_metrics"):
-                results['index_usage'] = self._query_index_usage(db)
-            metrics.update(INDEX_VARS)
-
         if is_affirmative(self._config.options.get('replication', self._config.dbm_enabled)):
             if self.performance_schema_enabled and self._is_group_replication_active(db):
                 self.log.debug('Collecting group replication metrics.')
@@ -1198,67 +1190,67 @@ def _query_exec_time_per_schema(self, db):
 
         return {}
 
-    def _query_index_size_per_index(self, db):
-        try:
-            with closing(db.cursor(CommenterCursor)) as cursor:
-                cursor.execute(SQL_QUERY_INDEX_SIZE)
-                if cursor.rowcount < 1:
-                    # TODO ALLEN: link to documentation
-                    self.warning("Failed to fetch records from the mysql 'innodb_index_stats' table.")
-                    return None
-                index_sizes = {}
-                for row in cursor.fetchall():
-                    db_name = str(row[0])
-                    table_name = str(row[1])
-                    index_name = str(row[2])
-                    index_size = float(row[3])
-
-                    # set the tag as the dictionary key
-                    index_sizes["db:{},table:{},index:{}".format(db_name, table_name, index_name)] = index_size
-                return index_sizes
-        except (pymysql.err.InternalError, pymysql.err.OperationalError):
-            self.warning(
-                "Failed to fetch records from the performance schema " "'table_io_waits_summary_by_index_usage' table."
-            )
-
-            return None
-
-    def _query_index_usage(self, db):
-        try:
-            with closing(db.cursor(CommenterCursor)) as cursor:
-                cursor.execute(SQL_QUERY_INDEX_USAGE)
-                if cursor.rowcount < 1:
-                    self.warning(
-                        "Failed to fetch records from the performance schema "
-                        "'table_io_waits_summary_by_index_usage' table."
-                    )
-                    return None
-                index_usage = {}
-                for row in cursor.fetchall():
-                    db_name = str(row[0])
-                    table_name = str(row[1])
-                    index_name = str(row[2])
-                    count_read = int(row[3])
-                    count_update = int(row[4])
-                    count_delete = int(row[5])
-
-                    # set the tag as the dictionary key
-                    index_usage["db:{},table:{},index:{},operation:read".format(db_name, table_name, index_name)] = (
-                        count_read
-                    )
-                    index_usage["db:{},table:{},index:{},operation:update".format(db_name, table_name, index_name)] = (
-                        count_update
-                    )
-                    index_usage["db:{},table:{},index:{},operation:delete".format(db_name, table_name, index_name)] = (
-                        count_delete
-                    )
-                    self.warning("Allen! Found index usage: %s", index_usage)
-
-                return index_usage
-        except (pymysql.err.InternalError, pymysql.err.OperationalError) as e:
-            self.warning("Index usage metrics unavailable at this time: %s", e)
-
-            return None
+    # def _query_index_size_per_index(self, db):
+    #     try:
+    #         with closing(db.cursor(CommenterCursor)) as cursor:
+    #             cursor.execute(SQL_QUERY_INDEX_SIZE)
+    #             if cursor.rowcount < 1:
+    #                 # TODO ALLEN: link to documentation
+    #                 self.warning("Failed to fetch records from the mysql 'innodb_index_stats' table.")
+    #                 return None
+    #             index_sizes = {}
+    #             for row in cursor.fetchall():
+    #                 db_name = str(row[0])
+    #                 table_name = str(row[1])
+    #                 index_name = str(row[2])
+    #                 index_size = float(row[3])
+
+    #                 # set the tag as the dictionary key
+    #                 index_sizes["db:{},table:{},index:{}".format(db_name, table_name, index_name)] = index_size
+    #             return index_sizes
+    #     except (pymysql.err.InternalError, pymysql.err.OperationalError):
+    #         self.warning(
+    #             "Failed to fetch records from the performance schema " "'table_io_waits_summary_by_index_usage' table."
+    #         )
+
+    #         return None
+
+    # def _query_index_usage(self, db):
+    #     try:
+    #         with closing(db.cursor(CommenterCursor)) as cursor:
+    #             cursor.execute(SQL_QUERY_INDEX_USAGE)
+    #             if cursor.rowcount < 1:
+    #                 self.warning(
+    #                     "Failed to fetch records from the performance schema "
+    #                     "'table_io_waits_summary_by_index_usage' table."
+    #                 )
+    #                 return None
+    #             index_usage = {}
+    #             for row in cursor.fetchall():
+    #                 db_name = str(row[0])
+    #                 table_name = str(row[1])
+    #                 index_name = str(row[2])
+    #                 count_read = int(row[3])
+    #                 count_update = int(row[4])
+    #                 count_delete = int(row[5])
+
+    #                 # set the tag as the dictionary key
+    #                 index_usage["db:{},table:{},index:{},operation:read".format(db_name, table_name, index_name)] = (
+    #                     count_read
+    #                 )
+    #                 index_usage["db:{},table:{},index:{},operation:update".format(db_name, table_name, index_name)] = (
+    #                     count_update
+    #                 )
+    #                 index_usage["db:{},table:{},index:{},operation:delete".format(db_name, table_name, index_name)] = (
+    #                     count_delete
+    #                 )
+    #                 self.warning("Allen! Found index usage: %s", index_usage)
+
+    #             return index_usage
+    #     except (pymysql.err.InternalError, pymysql.err.OperationalError) as e:
+    #         self.warning("Index usage metrics unavailable at this time: %s", e)
+
+    #         return None
 
     def _query_size_per_table(self, db, system_tables=False):
         try:

mysql/datadog_checks/mysql/queries.py

@@ -35,17 +35,6 @@
 FROM information_schema.tables
 WHERE table_schema in ('mysql', 'performance_schema', 'information_schema')"""
 
-SQL_QUERY_INDEX_SIZE = """\
-SELECT database_name, table_name, index_name,
- ROUND(stat_value * @@innodb_page_size/1024/1024, 2) AS index_size_mb
-FROM mysql.innodb_index_stats
-WHERE stat_name = 'size' AND index_name != 'PRIMARY'"""
-
-SQL_QUERY_INDEX_USAGE = """\
-SELECT object_schema, object_name, index_name, count_read, count_update, count_delete
-FROM performance_schema.table_io_waits_summary_by_index_usage
-WHERE index_name IS NOT NULL AND index_name != 'PRIMARY' AND object_schema NOT IN ('mysql', 'performance_schema')"""
-
 SQL_AVG_QUERY_RUN_TIME = """\
 SELECT schema_name, ROUND((SUM(sum_timer_wait) / SUM(count_star)) / 1000000) AS avg_us
 FROM performance_schema.events_statements_summary_by_digest
@@ -223,7 +212,6 @@
     ],
 }
 
-
 def show_replica_status_query(version, is_mariadb, channel=''):
     if version.version_compatible((10, 5, 1)) or not is_mariadb and version.version_compatible((8, 0, 22)):
         base_query = "SHOW REPLICA STATUS"

mysql/metadata.csv

@@ -18,7 +18,9 @@ mysql.galera.wsrep_received,gauge,,,,Total number of write-sets received from ot
 mysql.galera.wsrep_received_bytes,gauge,,,,Total size (in bytes) of writesets received from other nodes.,0,mysql,mysql galera wsrep_received_bytes,
 mysql.galera.wsrep_replicated_bytes,gauge,,,,Total size (in bytes) of writesets sent to other nodes.,0,mysql,mysql galera wsrep_replicated_bytes,
 mysql.index.size,gauge,,mebibyte,,Size of indexes in MiB,0,mysql,mysql index size,
-mysql.index.usage,gauge,,operation,,The number of operations using an index.,0,mysql,mysql index usage,
+mysql.index.reads,gauge,,operation,,The number of read operation using an index.,0,mysql,mysql index read usage,
+mysql.index.deletes,gauge,,operation,,The number of delete operation using an index.,0,mysql,mysql index delete usage,
+mysql.index.updates,gauge,,operation,,The number of update operation using an index.,0,mysql,mysql index update usage,
 mysql.info.schema.size,gauge,,mebibyte,,Size of schemas in MiB,0,mysql,mysql schema size,
 mysql.info.table.data_size,gauge,,mebibyte,,Size of tables data in MiB,0,mysql,mysql data table size,memory
 mysql.info.table.index_size,gauge,,mebibyte,,Size of tables index in MiB,0,mysql,mysql index table size,

mysql/tests/conftest.py

@@ -74,7 +74,6 @@ def dd_environment(config_e2e):
             yield config_e2e, e2e_metadata
 
 
-# TODO ALLEN: adjust configs for index metric collection once the config is implemented
 @pytest.fixture(scope='session')
 def instance_basic():
     return {
@@ -103,6 +102,7 @@ def instance_complex():
             'table_size_metrics': True,
             'system_table_size_metrics': True,
             'table_row_stats_metrics': True,
+            'index_metrics': True,
         },
         'tags': tags.METRIC_TAGS,
         'queries': [