Skip to content

Commit

Permalink
Make schema generation work without a cluster (#57)
Browse files Browse the repository at this point in the history
  • Loading branch information
@0237h
0237h authored Aug 9, 2024
1 parent c99fce2 commit 7f774cc
Showing 1 changed file with 39 additions and 31 deletions.
70 changes: 39 additions & 31 deletions create_schema.sh
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,30 @@
#!/usr/bin/env bash

# Helper script for generating the `schema.sql` ClickHouse tables definition
# Specify a cluster name to add `ON CLUSTER` directives

show_usage() {
printf 'Usage: %s [(-o|--outfile) file (default: "schema.sql")] [(-c|--cluster) name (default: "antelope")] [(-h|--help)]\n' "$(basename "$0")"
printf 'Usage: %s [(-o|--outfile) file (default: "schema.sql")] [(-c|--cluster) name (default: none)] [(-h|--help)]\n' "$(basename "$0")"
exit 0
}

SCHEMA_FILE="./schema.sql"
CLUSTER_NAME="antelope"
CLUSTER_NAME=""
while [[ "$#" -gt 0 ]]; do
case $1 in
-o|--outfile) SCHEMA_FILE="$2"; shift ;;
-c|--cluster) CLUSTER_NAME="$2"; shift ;;
-h|--help) show_usage ;;
*) echo "Unknown parameter passed: $1"; exit 1 ;;
*) echo "Unknown parameter passed: $1"; show_usage; exit 1 ;;
esac
shift
done

ON_CLUSTER_DIRECTIVE=""
if [ -n "$CLUSTER_NAME" ]; then
ON_CLUSTER_DIRECTIVE="ON CLUSTER $CLUSTER_NAME"
fi

cat > $SCHEMA_FILE <<- EOM
--------------------------------------
-- AUTO-GENERATED FILE, DO NOT EDIT --
Expand All @@ -31,7 +37,7 @@ cat > $SCHEMA_FILE <<- EOM
-- Meta tables to store Substreams information --
-------------------------------------------------
CREATE TABLE IF NOT EXISTS cursors ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS cursors $ON_CLUSTER_DIRECTIVE
(
id String,
cursor String,
Expand All @@ -48,7 +54,7 @@ CREATE TABLE IF NOT EXISTS cursors ON CLUSTER "$CLUSTER_NAME"
-- The table to store all transfers. This uses the trx_id as first primary key so we can use this table to do
-- transfer lookups based on a transaction id.
CREATE TABLE IF NOT EXISTS transfer_events ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS transfer_events $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -74,7 +80,7 @@ CREATE TABLE IF NOT EXISTS transfer_events ON CLUSTER "$CLUSTER_NAME"
-- The table to store all account balance changes from the database operations. This uses the account and block_num as
-- first primary keys so we can use this table to lookup the account balance from a certain block number.
CREATE TABLE IF NOT EXISTS balance_change_events ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS balance_change_events $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -99,7 +105,7 @@ CREATE TABLE IF NOT EXISTS balance_change_events ON CLUSTER "$CLUSTER_NAME"
-- The table to store all token supply changes from the database operations. This uses the account and block_num as
-- first primary keys so we can use this table to lookup token supplies from a certain block number.
CREATE TABLE IF NOT EXISTS supply_change_events ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS supply_change_events $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -124,7 +130,7 @@ CREATE TABLE IF NOT EXISTS supply_change_events ON CLUSTER "$CLUSTER_NAME"
ORDER BY (contract, block_num, trx_id, action_index);
-- Table to contain all 'eosio.token:issue' transactions
CREATE TABLE IF NOT EXISTS issue_events ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS issue_events $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -149,7 +155,7 @@ CREATE TABLE IF NOT EXISTS issue_events ON CLUSTER "$CLUSTER_NAME"
ORDER BY (contract, symcode, to, amount, trx_id, action_index);
-- Table to contain all 'eosio.token:retire' transactions --
CREATE TABLE IF NOT EXISTS retire_events ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS retire_events $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -173,7 +179,7 @@ CREATE TABLE IF NOT EXISTS retire_events ON CLUSTER "$CLUSTER_NAME"
ORDER BY (contract, symcode, amount, trx_id, action_index);
-- Table to contain all 'eosio.token:create' transactions
CREATE TABLE IF NOT EXISTS create_events ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS create_events $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -200,7 +206,7 @@ CREATE TABLE IF NOT EXISTS create_events ON CLUSTER "$CLUSTER_NAME"
-----------------------------------------------
-- Table to store up to date balances per account and token
CREATE TABLE IF NOT EXISTS account_balances ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS account_balances $ON_CLUSTER_DIRECTIVE
(
account String,
Expand All @@ -219,7 +225,7 @@ CREATE TABLE IF NOT EXISTS account_balances ON CLUSTER "$CLUSTER_NAME"
PRIMARY KEY (account, contract, symcode)
ORDER BY (account, contract, symcode, value);
CREATE MATERIALIZED VIEW account_balances_mv ON CLUSTER "$CLUSTER_NAME"
CREATE MATERIALIZED VIEW account_balances_mv $ON_CLUSTER_DIRECTIVE
TO account_balances
AS
SELECT account,
Expand All @@ -234,7 +240,7 @@ SELECT account,
FROM balance_change_events;
-- Table to store historical balances per account and token
CREATE TABLE IF NOT EXISTS historical_account_balances ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS historical_account_balances $ON_CLUSTER_DIRECTIVE
(
account String,
Expand All @@ -253,7 +259,7 @@ CREATE TABLE IF NOT EXISTS historical_account_balances ON CLUSTER "$CLUSTER_NAME
PRIMARY KEY (block_num, account, contract, symcode)
ORDER BY (block_num, account, contract, symcode, value);
CREATE MATERIALIZED VIEW historical_account_balances_mv ON CLUSTER "$CLUSTER_NAME"
CREATE MATERIALIZED VIEW historical_account_balances_mv $ON_CLUSTER_DIRECTIVE
TO historical_account_balances
AS
SELECT account,
Expand All @@ -268,7 +274,7 @@ SELECT account,
FROM balance_change_events;
-- Table to store up to date positive balances per account and token for token holders
CREATE TABLE IF NOT EXISTS token_holders ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS token_holders $ON_CLUSTER_DIRECTIVE
(
account String,
Expand All @@ -288,7 +294,7 @@ CREATE TABLE IF NOT EXISTS token_holders ON CLUSTER "$CLUSTER_NAME"
PRIMARY KEY (has_positive_balance, contract, symcode)
ORDER BY (has_positive_balance, contract, symcode, value);
CREATE MATERIALIZED VIEW token_holders_mv ON CLUSTER "$CLUSTER_NAME"
CREATE MATERIALIZED VIEW token_holders_mv $ON_CLUSTER_DIRECTIVE
TO token_holders
AS
SELECT account,
Expand All @@ -304,7 +310,7 @@ SELECT account,
FROM balance_change_events;
-- Table to store up to date token supplies
CREATE TABLE IF NOT EXISTS token_supplies ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS token_supplies $ON_CLUSTER_DIRECTIVE
(
contract String,
symcode String,
Expand All @@ -324,7 +330,7 @@ CREATE TABLE IF NOT EXISTS token_supplies ON CLUSTER "$CLUSTER_NAME"
PRIMARY KEY (contract, symcode, issuer)
ORDER BY (contract, symcode, issuer);
CREATE MATERIALIZED VIEW token_supplies_mv ON CLUSTER "$CLUSTER_NAME"
CREATE MATERIALIZED VIEW token_supplies_mv $ON_CLUSTER_DIRECTIVE
TO token_supplies
AS
SELECT contract,
Expand All @@ -340,7 +346,7 @@ SELECT contract,
FROM supply_change_events;
-- Table to store historical token supplies per token
CREATE TABLE IF NOT EXISTS historical_token_supplies ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS historical_token_supplies $ON_CLUSTER_DIRECTIVE
(
contract String,
symcode String,
Expand All @@ -360,7 +366,7 @@ CREATE TABLE IF NOT EXISTS historical_token_supplies ON CLUSTER "$CLUSTER_NAME"
PRIMARY KEY (block_num, contract, symcode, issuer)
ORDER BY (block_num, contract, symcode, issuer);
CREATE MATERIALIZED VIEW historical_token_supplies_mv ON CLUSTER "$CLUSTER_NAME"
CREATE MATERIALIZED VIEW historical_token_supplies_mv $ON_CLUSTER_DIRECTIVE
TO historical_token_supplies
AS
SELECT contract,
Expand All @@ -376,7 +382,7 @@ SELECT contract,
FROM supply_change_events;
-- Table to store token transfers primarily indexed by the 'from' field --
CREATE TABLE IF NOT EXISTS transfers_from ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS transfers_from $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -400,7 +406,7 @@ CREATE TABLE IF NOT EXISTS transfers_from ON CLUSTER "$CLUSTER_NAME"
PRIMARY KEY (from, to, contract, symcode, trx_id, action_index)
ORDER BY (from, to, contract, symcode, trx_id, action_index);
CREATE MATERIALIZED VIEW transfers_from_mv ON CLUSTER "$CLUSTER_NAME"
CREATE MATERIALIZED VIEW transfers_from_mv $ON_CLUSTER_DIRECTIVE
TO transfers_from
AS
SELECT trx_id,
Expand All @@ -419,7 +425,7 @@ SELECT trx_id,
FROM transfer_events;
-- Table to store historical token transfers 'from' address --
CREATE TABLE IF NOT EXISTS historical_transfers_from ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS historical_transfers_from $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -443,7 +449,7 @@ CREATE TABLE IF NOT EXISTS historical_transfers_from ON CLUSTER "$CLUSTER_NAME"
PRIMARY KEY (block_num, from, to, contract, symcode, trx_id, action_index)
ORDER BY (block_num, from, to, contract, symcode, trx_id, action_index);
CREATE MATERIALIZED VIEW historical_transfers_from_mv ON CLUSTER "$CLUSTER_NAME"
CREATE MATERIALIZED VIEW historical_transfers_from_mv $ON_CLUSTER_DIRECTIVE
TO historical_transfers_from
AS
SELECT trx_id,
Expand All @@ -462,7 +468,7 @@ SELECT trx_id,
FROM transfer_events;
-- Table to store token transfers primarily indexed by the 'to' field --
CREATE TABLE IF NOT EXISTS transfers_to ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS transfers_to $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -486,7 +492,7 @@ CREATE TABLE IF NOT EXISTS transfers_to ON CLUSTER "$CLUSTER_NAME"
PRIMARY KEY (to, contract, symcode, trx_id, action_index)
ORDER BY (to, contract, symcode, trx_id, action_index);
CREATE MATERIALIZED VIEW transfers_to_mv ON CLUSTER "$CLUSTER_NAME"
CREATE MATERIALIZED VIEW transfers_to_mv $ON_CLUSTER_DIRECTIVE
TO transfers_to
AS
SELECT trx_id,
Expand All @@ -505,7 +511,7 @@ SELECT trx_id,
FROM transfer_events;
-- Table to store historical token transfers 'to' address --
CREATE TABLE IF NOT EXISTS historical_transfers_to ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS historical_transfers_to $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -529,7 +535,7 @@ CREATE TABLE IF NOT EXISTS historical_transfers_to ON CLUSTER "$CLUSTER_NAME"
PRIMARY KEY (block_num, to, contract, symcode, trx_id, action_index)
ORDER BY (block_num, to, contract, symcode, trx_id, action_index);
CREATE MATERIALIZED VIEW historical_transfers_to_mv ON CLUSTER "$CLUSTER_NAME"
CREATE MATERIALIZED VIEW historical_transfers_to_mv $ON_CLUSTER_DIRECTIVE
TO historical_transfers_to
AS
SELECT trx_id,
Expand All @@ -548,7 +554,7 @@ SELECT trx_id,
FROM transfer_events;
-- Table to store token transfers primarily indexed by the 'block_num' field
CREATE TABLE IF NOT EXISTS transfers_block_num ON CLUSTER "$CLUSTER_NAME"
CREATE TABLE IF NOT EXISTS transfers_block_num $ON_CLUSTER_DIRECTIVE
(
trx_id String,
action_index UInt32,
Expand All @@ -572,7 +578,7 @@ CREATE TABLE IF NOT EXISTS transfers_block_num ON CLUSTER "$CLUSTER_NAME"
PRIMARY KEY (block_num, contract, symcode, trx_id, action_index)
ORDER BY (block_num, contract, symcode, trx_id, action_index);
CREATE MATERIALIZED VIEW transfers_block_num_mv ON CLUSTER "$CLUSTER_NAME"
CREATE MATERIALIZED VIEW transfers_block_num_mv $ON_CLUSTER_DIRECTIVE
TO transfers_block_num
AS
SELECT trx_id,
Expand All @@ -591,4 +597,6 @@ SELECT trx_id,
FROM transfer_events;
EOM

echo "[+] Created '$SCHEMA_FILE' for cluster '$CLUSTER_NAME'"
echo "[+] Created '$SCHEMA_FILE'"
echo "[*] Run the following command to apply:"
echo "cat $SCHEMA_FILE | clickhouse client -h <host> --port 9000 -d <database> -u <user> --password <password>"

0 comments on commit 7f774cc

Please sign in to comment.