Merge pull request #2134 from mponomar/comdb2-index-usage

Add comdb2_index_usage tables

sqlite/CMakeLists.txt

@@ -47,6 +47,7 @@ add_library(sqlite
   ext/comdb2/connections.c 
   ext/comdb2/views.c
   ext/comdb2/sqlclientstats.c
+  ext/comdb2/indexuse.c
   ext/misc/completion.c
   ext/misc/regexp.c
   ext/misc/json1.c

sqlite/ext/comdb2/comdb2systblInt.h

@@ -52,6 +52,7 @@ int systblCronInit(sqlite3*db);
 int systblFingerprintsInit(sqlite3 *);
 int systblViewsInit(sqlite3 *);
 int systblSQLClientStats(sqlite3 *);
+int systblSQLIndexStatsInit(sqlite3 *);
 
 int comdb2_next_allowed_table(sqlite3_int64 *tabId);
 

sqlite/ext/comdb2/indexuse.c

@@ -0,0 +1,112 @@
+/*
+   Copyright 2020 Bloomberg Finance L.P.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+ */
+
+#if (!defined(SQLITE_CORE) || defined(SQLITE_BUILDING_FOR_COMDB2)) &&          \
+    !defined(SQLITE_OMIT_VIRTUALTABLE)
+
+#if defined(SQLITE_BUILDING_FOR_COMDB2) && !defined(SQLITE_CORE)
+#define SQLITE_CORE 1
+#endif
+
+#include "comdb2.h"
+#include "comdb2systblInt.h"
+#include "sql.h"
+#include "ezsystables.h"
+#include "cdb2api.h"
+#include "schema_lk.h"
+
+static sqlite3_module systblIndexUsageModule = {
+    .access_flag = CDB2_ALLOW_USER,
+};
+
+struct index_usage {
+    char *tablename;
+    int64_t ixnum;
+    char *cscname;
+    char *sqlname;
+    int64_t steps;
+    int64_t non_sql_steps;
+};
+typedef struct index_usage index_usage;
+
+static void free_index_usage(void *recsp, int nrecs) {
+    struct index_usage *ixs = (struct index_usage*) recsp;
+    for (int i = 0; i < nrecs; i++) {
+        struct index_usage *ix = &ixs[i];
+        free(ix->cscname);
+        free(ix->sqlname);
+        free(ix->tablename);
+    }
+    free(ixs);
+}
+
+static int get_index_usage(void **recsp, int *nrecs) {
+    struct dbtable *db;
+    struct index_usage *ixs = NULL;
+    int allocated = 0;
+    int nix = 0;
+
+    rdlock_schema_lk();
+
+    for (int dbn = 0; dbn < thedb->num_dbs; dbn++) {
+        db = thedb->dbs[dbn];
+        if (strncmp(db->tablename, "sqlite_stat", strlen("sqlite_stat")) == 0)
+            continue;
+        logmsg(LOGMSG_USER, "table '%s'\n", db->tablename);
+        for (int ixnum = 0; ixnum < db->nix; ixnum++) {
+            if (nix == allocated) {
+                allocated = allocated * 2 + 16;
+                struct index_usage *n = realloc(ixs, allocated * sizeof(struct index_usage));
+                if (n == NULL) {
+                    free_index_usage(ixs, nix);
+                    unlock_schema_lk();
+                    return -1;
+                }
+                ixs = n;
+            }
+            struct index_usage *ix = &ixs[nix];
+            ix->tablename = strdup(db->tablename);
+            ix->ixnum = ixnum;
+            ix->cscname = strdup(db->schema->ix[ixnum]->csctag);
+            ix->sqlname =  strdup(db->schema->ix[ixnum]->sqlitetag);
+            ix->steps = db->sqlixuse[ixnum];
+            ix->non_sql_steps = db->ixuse[ixnum];
+
+            nix++;
+        }
+    }
+    *nrecs = nix;
+    *recsp = ixs;
+    unlock_schema_lk();
+    return 0;
+}
+
+int systblSQLIndexStatsInit(sqlite3 *db) {
+    int rc = create_system_table(db, "comdb2_index_usage", &systblIndexUsageModule,
+            get_index_usage, free_index_usage, sizeof(index_usage),
+            CDB2_CSTRING, "table_name", -1, offsetof(index_usage, tablename),
+            CDB2_INTEGER, "ix_num", -1, offsetof(index_usage, ixnum),
+            CDB2_CSTRING, "csc_name", -1, offsetof(index_usage, cscname),
+            CDB2_CSTRING, "sql_name", -1, offsetof(index_usage, sqlname),
+            CDB2_INTEGER, "steps", -1, offsetof(index_usage, steps),
+            CDB2_INTEGER, "non_sql_steps", -1, offsetof(index_usage, non_sql_steps),
+            SYSTABLE_END_OF_FIELDS);
+    return rc;
+}
+
+
+#endif /* (!defined(SQLITE_CORE) || defined(SQLITE_BUILDING_FOR_COMDB2))       \
+          && !defined(SQLITE_OMIT_VIRTUALTABLE) */

sqlite/ext/comdb2/tables.c

@@ -283,7 +283,9 @@ int comdb2SystblInit(
   if (rc == SQLITE_OK)
     rc = systblViewsInit(db);
   if (rc == SQLITE_OK)
-    rc  = systblSQLClientStats(db);
+    rc = systblSQLClientStats(db);
+  if (rc == SQLITE_OK)
+    rc = systblSQLIndexStatsInit(db);
 #endif
   return rc;
 }

tests/auth.test/t09.expected

@@ -224,6 +224,7 @@
 (candidate='comdb2_cron_events')
 (candidate='comdb2_cron_schedulers')
 (candidate='comdb2_fingerprints')
+(candidate='comdb2_index_usage')
 (candidate='comdb2_keycomponents')
 (candidate='comdb2_keys')
 (candidate='comdb2_keywords')

tests/cdb2sql.test/t00.expected

@@ -29,6 +29,7 @@ unknown @ls sub-command foo
 (name='comdb2_cron_events')
 (name='comdb2_cron_schedulers')
 (name='comdb2_fingerprints')
+(name='comdb2_index_usage')
 (name='comdb2_keycomponents')
 (name='comdb2_keys')
 (name='comdb2_keywords')

tests/comdb2sys.test/comdb2sys.expected

@@ -356,6 +356,7 @@
 (name='comdb2_cron_events')
 (name='comdb2_cron_schedulers')
 (name='comdb2_fingerprints')
+(name='comdb2_index_usage')
 (name='comdb2_keycomponents')
 (name='comdb2_keys')
 (name='comdb2_keywords')

tests/comdb2sys.test/runit

@@ -111,4 +111,8 @@ ${TESTSROOTDIR}/tools/compare_results.sh -d $a_dbn -r req
 ./testsqlclients - $a_dbn
 rc=$?
 
+./testindexusage $a_dbn
+rc2=$?
+[[ $rc2 -ne 0 ]] && rc=$rc2
+
 exit $rc

tests/comdb2sys.test/testindexusage

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+a_dbn=$1
+default=default
+#default=local
+
+cdb2sql ${CDB2_OPTIONS} $a_dbn $default - <<EOF
+create table ixtest(a int, b int); \$\$
+create index ixtest_a on ixtest(a);
+create index ixtest_b on ixtest(b);
+EOF
+
+(
+echo "begin"
+for a in $(seq 1 99); do
+    for b in $(seq 1 99); do
+        echo "insert into ixtest values($a, $b);"
+    done
+done
+echo "commit"
+echo "analyze"
+echo 'select a from ixtest where a between 1 and 10'
+echo 'select a from ixtest where a between 1 and 10'
+echo 'select a from ixtest where a between 1 and 10'
+echo 'select a from ixtest where a between 1 and 10'
+echo 'select a from ixtest where a between 1 and 10'
+echo 'select b from ixtest where b between 1 and 10'
+) | cdb2sql ${CDB2_OPTIONS} $a_dbn $default - >/dev/null
+
+cdb2sql ${CDB2_OPTIONS} $a_dbn $default 'select * from comdb2_index_usage where table_name="ixtest"' > testindexusage.out
+diff testindexusage.out testindexusage.expected >/dev/null
+rc=$?
+if [[ $rc -ne 0  ]]; then
+    echo "Failed index usage test"
+    echo diff $(pwd)/testindexusage.out $(pwd)/testindexusage.expected
+fi
+
+exit $?

tests/comdb2sys.test/testindexusage.expected

@@ -0,0 +1,2 @@
+(table_name='ixtest', ix_num=0, csc_name='IXTEST_A', sql_name='$IXTEST_A_B7CCCE2', steps=4955, non_sql_steps=0)
+(table_name='ixtest', ix_num=1, csc_name='IXTEST_B', sql_name='$IXTEST_B_92759D58', steps=991, non_sql_steps=0)

tests/userschema.test/t00.expected

@@ -11,6 +11,7 @@
 (tablename='comdb2_cron_events', username='mohit', READ='Y', WRITE='Y', DDL='Y')
 (tablename='comdb2_cron_schedulers', username='mohit', READ='Y', WRITE='Y', DDL='Y')
 (tablename='comdb2_fingerprints', username='mohit', READ='Y', WRITE='Y', DDL='Y')
+(tablename='comdb2_index_usage', username='mohit', READ='Y', WRITE='Y', DDL='Y')
 (tablename='comdb2_keycomponents', username='mohit', READ='Y', WRITE='Y', DDL='Y')
 (tablename='comdb2_keys', username='mohit', READ='Y', WRITE='Y', DDL='Y')
 (tablename='comdb2_keywords', username='mohit', READ='Y', WRITE='Y', DDL='Y')

tests/userschema.test/t07.expected

@@ -35,6 +35,9 @@
 (tablename='comdb2_fingerprints', username='abcd', READ='N', WRITE='N', DDL='N')
 (tablename='comdb2_fingerprints', username='dcba', READ='N', WRITE='N', DDL='N')
 (tablename='comdb2_fingerprints', username='mohit', READ='Y', WRITE='Y', DDL='Y')
+(tablename='comdb2_index_usage', username='abcd', READ='N', WRITE='N', DDL='N')
+(tablename='comdb2_index_usage', username='dcba', READ='N', WRITE='N', DDL='N')
+(tablename='comdb2_index_usage', username='mohit', READ='Y', WRITE='Y', DDL='Y')
 (tablename='comdb2_keycomponents', username='abcd', READ='N', WRITE='N', DDL='N')
 (tablename='comdb2_keycomponents', username='dcba', READ='N', WRITE='N', DDL='N')
 (tablename='comdb2_keycomponents', username='mohit', READ='Y', WRITE='Y', DDL='Y')

tests/userschema.test/t09.expected

@@ -40,6 +40,9 @@
 (tablename='comdb2_fingerprints', username='abcd', READ='N', WRITE='N', DDL='N')
 (tablename='comdb2_fingerprints', username='dcba', READ='N', WRITE='N', DDL='N')
 (tablename='comdb2_fingerprints', username='mohit', READ='Y', WRITE='Y', DDL='Y')
+(tablename='comdb2_index_usage', username='abcd', READ='N', WRITE='N', DDL='N')
+(tablename='comdb2_index_usage', username='dcba', READ='N', WRITE='N', DDL='N')
+(tablename='comdb2_index_usage', username='mohit', READ='Y', WRITE='Y', DDL='Y')
 (tablename='comdb2_keycomponents', username='abcd', READ='N', WRITE='N', DDL='N')
 (tablename='comdb2_keycomponents', username='dcba', READ='N', WRITE='N', DDL='N')
 (tablename='comdb2_keycomponents', username='mohit', READ='Y', WRITE='Y', DDL='Y')