Tornado Cash: Fix sources, lowercase and improve materialization stra…

…tegy (#1460) * Fix sources, lowercase and improve materialization strat * ALTER TABLE, not VIEW * Revert "ALTER TABLE, not VIEW" This reverts commit f707c2e. * changed avalanche to avalanche_c in blockchain name for consistency across dune * Lowercase for blockchains and versiosn on schema.yml * avalanche to avalanche_c in schema and macro files * transactions_006 to transactions * Update alter_table_properties.sql * fix transactions_006 * Replace LEFT BY INNER JOINS on base transactions tables Co-authored-by: Hildebert Moulié <[email protected]>
duneanalytics · Sep 1, 2022 · b6adb5e · b6adb5e
1 parent 5c1734d
commit b6adb5e
Show file tree

Hide file tree

Showing 9 changed files with 77 additions and 117 deletions.
diff --git a/macros/alter_table_properties.sql b/macros/alter_table_properties.sql
@@ -579,16 +579,16 @@ ALTER TABLE airdrop_optimism.addresses SET TBLPROPERTIES ('dune.public'='true',
 
 {% set tornado_cash_deposits %}
 ALTER TABLE tornado_cash.deposits SET TBLPROPERTIES ('dune.public'='true',
-                                                'dune.data_explorer.blockchains'='["ethereum", "bnb", "avalanche", "gnosis", "optimism", "arbitrum"]',
+                                                'dune.data_explorer.blockchains'='["ethereum", "bnb", "avalanche_c", "gnosis", "optimism", "arbitrum"]',
                                                 'dune.data_explorer.category'='abstraction',
                                                 'dune.data_explorer.abstraction.type'='project',
                                                 'dune.data_explorer.abstraction.name'='tornado_cash',
                                                 'dune.data_explorer.contributors'='["hildobby", "dot2dotseurat"]');
 {% endset %}
 
 {% set tornado_cash_withdrawals %}
-ALTER TABLE tornado_cash.withdrawals SET TBLPROPERTIES ('dune.public'='true',
-                                                'dune.data_explorer.blockchains'='["ethereum", "bnb", "avalanche", "gnosis", "optimism", "arbitrum"]',
+ALTER VIEW tornado_cash.withdrawals SET TBLPROPERTIES ('dune.public'='true',
+                                                'dune.data_explorer.blockchains'='["ethereum", "bnb", "avalanche_c", "gnosis", "optimism", "arbitrum"]',
                                                 'dune.data_explorer.category'='abstraction',
                                                 'dune.data_explorer.abstraction.type'='project',
                                                 'dune.data_explorer.abstraction.name'='tornado_cash',

diff --git a/models/ens/ens_ethereum_sources.yml b/models/ens/ens_ethereum_sources.yml
@@ -24,9 +24,6 @@ sources:
           error_after: { count: 24, period: hour }
         loaded_at_field: evt_block_time
       - name: ENSRegistry_evt_NewOwner
-        freshness:
-          warn_after: { count: 12, period: hour }
-          error_after: { count: 24, period: hour }
         loaded_at_field: evt_block_time
       - name: ENSRegistryWithFallback_evt_NewOwner
         freshness:

diff --git a/models/sudoswap/ethereum/sudoswap_ethereum_sources.yml b/models/sudoswap/ethereum/sudoswap_ethereum_sources.yml
@@ -15,9 +15,6 @@ sources:
           error_after: { count: 24, period: hour }
         loaded_at_field: call_block_time
       - name: LSSVMPair_general_call_swapTokenForAnyNFTs
-        freshness:
-          warn_after: { count: 12, period: hour }
-          error_after: { count: 24, period: hour }
         loaded_at_field: call_block_time
       - name: LSSVMPair_general_call_swapTokenForSpecificNFTs
         freshness:

diff --git a/models/tornado_cash/tornado_cash_deposits.sql b/models/tornado_cash/tornado_cash_deposits.sql
@@ -15,8 +15,8 @@ FROM
         SELECT tc.evt_block_time AS block_time
         , '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' AS currency_contract
         , 'ETH' AS currency_symbol
-        , 'Ethereum' AS blockchain
-        , 'Classic' AS tornado_version
+        , 'ethereum' AS blockchain
+        , 'classic' AS tornado_version
         , et.from AS depositor
         , tc.contract_address AS contract_address
         , CASE WHEN tc.contract_address='0x12d66f87a04a9e220743712ce6d9bb1b5616b8fc' THEN 0.1
@@ -29,19 +29,19 @@ FROM
         , tc.evt_index
         , TRY_CAST(date_trunc('DAY', tc.evt_block_time) AS date) AS block_date
         FROM {{ source('tornado_cash_ethereum','eth_evt_Deposit') }} tc
-        LEFT JOIN {{ source('ethereum','transactions_0006') }} et
+        INNER JOIN {{ source('ethereum','transactions') }} et
                 ON et.hash=tc.evt_tx_hash
                 {% if not is_incremental() %}
                 AND et.block_time >= (select min(evt_block_time) from {{ source('tornado_cash_ethereum','eth_evt_Deposit') }})
                 {% endif %}
                 {% if is_incremental() %}
-                AND et.block_time >= (select max(block_time) from {{ this }})
+                AND et.block_time >= date_trunc("day", now() - interval '1 week')
                 {% endif %}
         {% if not is_incremental() %}
         WHERE tc.evt_block_time >= (select min(evt_block_time) from {{ source('tornado_cash_ethereum','eth_evt_Deposit') }})
         {% endif %}
         {% if is_incremental() %}
-        WHERE tc.evt_block_time >= (select max(block_time) from {{ this }})
+        WHERE tc.evt_block_time >= date_trunc("day", now() - interval '1 week')
         {% endif %}
 
         UNION
@@ -94,8 +94,8 @@ FROM
                 WHEN tc.contract_address='0x610b717796ad172b316836ac95a2ffad065ceab4' THEN 'WBTC'
                 WHEN tc.contract_address='0xbb93e510bbcd0b7beb5a853875f9ec60275cf498' THEN 'WBTC'
                 END AS currency_symbol
-        , 'Ethereum' AS blockchain
-        , 'Classic' AS tornado_version
+        , 'ethereum' AS blockchain
+        , 'classic' AS tornado_version
         , et.from AS depositor
         , tc.contract_address AS contract_address
         , CASE WHEN tc.contract_address='0xd4b88df4d29f5cedd6857912842cff3b20c8cfa3' THEN 100
@@ -126,19 +126,19 @@ FROM
         , tc.evt_index
         , TRY_CAST(date_trunc('DAY', tc.evt_block_time) AS date) AS block_date
         FROM {{ source('tornado_cash_ethereum','erc20_evt_Deposit') }} tc
-        LEFT JOIN {{ source('ethereum','transactions_0006') }} et
+        INNER JOIN {{ source('ethereum','transactions') }} et
                 ON et.hash=tc.evt_tx_hash
                 {% if not is_incremental() %}
                 AND et.block_time >= (select min(evt_block_time) from {{ source('tornado_cash_ethereum','erc20_evt_Deposit') }})
                 {% endif %}
                 {% if is_incremental() %}
-                AND et.block_time >= (select max(block_time) from {{ this }})
+                AND et.block_time >= date_trunc("day", now() - interval '1 week')
                 {% endif %}
         {% if not is_incremental() %}
         WHERE tc.evt_block_time >= (select min(evt_block_time) from {{ source('tornado_cash_ethereum','erc20_evt_Deposit') }})
         {% endif %}
         {% if is_incremental() %}
-        WHERE tc.evt_block_time >= (select max(block_time) from {{ this }})
+        WHERE tc.evt_block_time >= date_trunc("day", now() - interval '1 week')
         {% endif %}
 
         UNION
@@ -147,8 +147,8 @@ FROM
         SELECT tc.evt_block_time AS block_time
         , '0xb8c77482e45f1f44de1745f52c74426c631bdd52' AS currency_contract
         , 'BNB' AS currency_symbol
-        , 'BNB' AS blockchain
-        , 'Classic' AS tornado_version
+        , 'bnb' AS blockchain
+        , 'classic' AS tornado_version
         , bt.from AS depositor
         , tc.contract_address AS contract_address
         , CASE WHEN tc.contract_address='0x84443cfd09a48af6ef360c6976c5392ac5023a1f' THEN 0.1
@@ -161,19 +161,19 @@ FROM
         , tc.evt_index
         , TRY_CAST(date_trunc('DAY', tc.evt_block_time) AS date) AS block_date
         FROM {{ source('tornado_cash_bnb','TornadoCashBNB_evt_Deposit') }} tc
-        LEFT JOIN {{ source('bnb','transactions') }} bt
+        INNER JOIN {{ source('bnb','transactions') }} bt
                 ON bt.hash=tc.evt_tx_hash
                 {% if not is_incremental() %}
                 AND bt.block_time >= (select min(evt_block_time) from {{ source('tornado_cash_bnb','TornadoCashBNB_evt_Deposit') }})
                 {% endif %}
                 {% if is_incremental() %}
-                AND bt.block_time >= (select max(block_time) from {{ this }})
+                AND bt.block_time >= date_trunc("day", now() - interval '1 week')
                 {% endif %}
         {% if not is_incremental() %}
         WHERE tc.evt_block_time >= (select min(evt_block_time) from {{ source('tornado_cash_bnb','TornadoCashBNB_evt_Deposit') }})
         {% endif %}
         {% if is_incremental() %}
-        WHERE tc.evt_block_time >= (select max(block_time) from {{ this }})
+        WHERE tc.evt_block_time >= date_trunc("day", now() - interval '1 week')
         {% endif %}
 
         UNION
@@ -182,8 +182,8 @@ FROM
         SELECT tc.evt_block_time AS block_time
         , '0x6b175474e89094c44da98b954eedeac495271d0f' AS currency_contract
         , 'xDAI' AS currency_symbol
-        , 'Gnosis' AS blockchain
-        , 'Classic' AS tornado_version
+        , 'gnosis' AS blockchain
+        , 'classic' AS tornado_version
         , gt.from AS depositor
         , tc.contract_address AS contract_address
         , CASE WHEN tc.contract_address='0x1e34a77868e19a6647b1f2f47b51ed72dede95dd' THEN 100
@@ -196,19 +196,19 @@ FROM
         , tc.evt_index
         , TRY_CAST(date_trunc('DAY', tc.evt_block_time) AS date) AS block_date
         FROM {{ source('tornado_cash_gnosis','eth_evt_Deposit') }} tc
-        LEFT JOIN {{ source('gnosis','transactions') }} gt
+        INNER JOIN {{ source('gnosis','transactions') }} gt
                 ON gt.hash=tc.evt_tx_hash
                 {% if not is_incremental() %}
                 AND gt.block_time >= (select min(evt_block_time) from {{ source('tornado_cash_gnosis','eth_evt_Deposit') }})
                 {% endif %}
                 {% if is_incremental() %}
-                AND gt.block_time >= (select max(block_time) from {{ this }})
+                AND gt.block_time >= date_trunc("day", now() - interval '1 week')
                 {% endif %}
         {% if not is_incremental() %}
         WHERE tc.evt_block_time >= (select min(evt_block_time) from {{ source('tornado_cash_gnosis','eth_evt_Deposit') }})
         {% endif %}
         {% if is_incremental() %}
-        WHERE tc.evt_block_time >= (select max(block_time) from {{ this }})
+        WHERE tc.evt_block_time >= date_trunc("day", now() - interval '1 week')
         {% endif %}
 
         UNION
@@ -217,8 +217,8 @@ FROM
         SELECT tc.evt_block_time AS block_time
         , '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' AS currency_contract
         , 'ETH' AS currency_symbol
-        , 'Optimism' AS blockchain
-        , 'Classic' AS tornado_version
+        , 'optimism' AS blockchain
+        , 'classic' AS tornado_version
         , ot.from AS depositor
         , tc.contract_address AS contract_address
         , CASE WHEN tc.contract_address='0x84443cfd09a48af6ef360c6976c5392ac5023a1f' THEN 0.1
@@ -231,19 +231,19 @@ FROM
         , tc.evt_index
         , TRY_CAST(date_trunc('DAY', tc.evt_block_time) AS date) AS block_date
         FROM {{ source('tornado_cash_optimism','ETHTornado_evt_Deposit') }} tc
-        LEFT JOIN {{ source('optimism','transactions') }} ot
+        INNER JOIN {{ source('optimism','transactions') }} ot
                 ON ot.hash=tc.evt_tx_hash
                 {% if not is_incremental() %}
                 AND ot.block_time >= (select min(evt_block_time) from {{ source('tornado_cash_optimism','ETHTornado_evt_Deposit') }})
                 {% endif %}
                 {% if is_incremental() %}
-                AND ot.block_time >= (select max(block_time) from {{ this }})
+                AND ot.block_time >= date_trunc("day", now() - interval '1 week')
                 {% endif %}
         {% if not is_incremental() %}
         WHERE tc.evt_block_time >= (select min(evt_block_time) from {{ source('tornado_cash_optimism','ETHTornado_evt_Deposit') }})
         {% endif %}
         {% if is_incremental() %}
-        WHERE tc.evt_block_time >= (select max(block_time) from {{ this }})
+        WHERE tc.evt_block_time >= date_trunc("day", now() - interval '1 week')
         {% endif %}
 
         UNION
@@ -252,8 +252,8 @@ FROM
         SELECT tc.evt_block_time AS block_time
         , '0x85f138bfEE4ef8e540890CFb48F620571d67Eda3' AS currency_contract
         , 'AVAX' AS currency_symbol
-        , 'Avalanche' AS blockchain
-        , 'Classic' AS tornado_version
+        , 'avalanche_c' AS blockchain
+        , 'classic' AS tornado_version
         , at.from AS depositor
         , tc.contract_address AS contract_address
         , CASE WHEN tc.contract_address='0x330bdfade01ee9bf63c209ee33102dd334618e0a' THEN 10
@@ -265,19 +265,19 @@ FROM
         , tc.evt_index
         , TRY_CAST(date_trunc('DAY', tc.evt_block_time) AS date) AS block_date
         FROM {{ source('tornado_cash_avalanche_c','ETHTornado_evt_Deposit') }} tc
-        LEFT JOIN {{ source('avalanche_c','transactions') }} at
+        INNER JOIN {{ source('avalanche_c','transactions') }} at
                 ON at.hash=tc.evt_tx_hash
                 {% if not is_incremental() %}
                 AND at.block_time >= (select min(evt_block_time) from {{ source('tornado_cash_avalanche_c','ETHTornado_evt_Deposit') }})
                 {% endif %}
                 {% if is_incremental() %}
-                AND at.block_time >= (select max(block_time) from {{ this }})
+                AND at.block_time >= date_trunc("day", now() - interval '1 week')
                 {% endif %}
         {% if not is_incremental() %}
         WHERE tc.evt_block_time >= (select min(evt_block_time) from {{ source('tornado_cash_avalanche_c','ETHTornado_evt_Deposit') }})
         {% endif %}
         {% if is_incremental() %}
-        WHERE tc.evt_block_time >= (select max(block_time) from {{ this }})
+        WHERE tc.evt_block_time >= date_trunc("day", now() - interval '1 week')
         {% endif %}
 
         UNION
@@ -286,8 +286,8 @@ FROM
         SELECT tc.evt_block_time AS block_time
         , '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' AS currency_contract
         , 'ETH' AS currency_symbol
-        , 'Arbitrum' AS blockchain
-        , 'Classic' AS tornado_version
+        , 'arbitrum' AS blockchain
+        , 'classic' AS tornado_version
         , at.from AS depositor
         , tc.contract_address AS contract_address
         , CASE WHEN tc.contract_address='0x84443cfd09a48af6ef360c6976c5392ac5023a1f' THEN 0.1
@@ -300,18 +300,18 @@ FROM
         , tc.evt_index
         , TRY_CAST(date_trunc('DAY', tc.evt_block_time) AS date) AS block_date
         FROM {{ source('tornado_cash_arbitrum','ETHTornado_evt_Deposit') }} tc
-        LEFT JOIN {{ source('arbitrum','transactions') }} at
+        INNER JOIN {{ source('arbitrum','transactions') }} at
                 ON at.hash=tc.evt_tx_hash
                 {% if not is_incremental() %}
                 AND at.block_time >= (select min(evt_block_time) from {{ source('tornado_cash_arbitrum','ETHTornado_evt_Deposit') }})
                 {% endif %}
                 {% if is_incremental() %}
-                AND at.block_time >= (select max(block_time) from {{ this }})
+                AND at.block_time >= date_trunc("day", now() - interval '1 week')
                 {% endif %}
         {% if not is_incremental() %}
         WHERE tc.evt_block_time >= (select min(evt_block_time) from {{ source('tornado_cash_arbitrum','ETHTornado_evt_Deposit') }})
         {% endif %}
         {% if is_incremental() %}
-        WHERE tc.evt_block_time >= (select max(block_time) from {{ this }})
+        WHERE tc.evt_block_time >= date_trunc("day", now() - interval '1 week')
         {% endif %}
 )
diff --git a/models/tornado_cash/tornado_cash_schema.yml b/models/tornado_cash/tornado_cash_schema.yml
@@ -5,11 +5,11 @@ version: 2
 models:
   - name: tornado_cash_deposits
     meta:
-      blockchain: ethereum, bnb, avalanche, gnosis, optimism, arbitrum #, polygon
+      blockchain: ethereum, bnb, avalanche_c, gnosis, optimism, arbitrum #, polygon
       project: tornado_cash
       contibutors: [hildobby, dot2dotseurat]
     config:
-      tags: ['tornado_cash', 'deposits', 'ethereum', 'bnb', 'avalanche', 'gnosis', 'optimism', 'arbitrum']
+      tags: ['tornado_cash', 'deposits', 'ethereum', 'bnb', 'avalanche_c', 'gnosis', 'optimism', 'arbitrum']
     description: "Tornado Cash Deposits"
     columns:
       - &block_time
@@ -25,13 +25,13 @@ models:
         name: blockchain
         tests:
           - accepted_values:
-              values: ['Ethereum', 'BNB', 'Avalanche', 'Gnosis', 'Optimism', 'Arbitrum', 'Polygon']
+              values: ['ethereum', 'bnb', 'avalanche_c', 'gnosis', 'optimism', 'arbitrum', 'polygon']
         description: "Blockchain"
       - &tornado_version
         name: tornado_version
         tests:
           - accepted_values:
-              values: ['Classic', 'Nova']
+              values: ['classic', 'nova']
         description: "Version of Tornado Cash used for this transaction"
       - &depositor
         name: depositor

diff --git a/models/tornado_cash/tornado_cash_sources.yml b/models/tornado_cash/tornado_cash_sources.yml
@@ -4,7 +4,6 @@ sources:
   - name: tornado_cash_bnb
     freshness:
       warn_after: { count: 12, period: hour }
-      error_after: { count: 24, period: hour }
     tables:
       - name: TornadoCashBNB_evt_Deposit
         loaded_at_field: evt_block_time
@@ -16,18 +15,6 @@ sources:
           - name: commitment
           - name: leafIndex
           - name: timestamp
-      - name: ETHTornado_evt_Withdrawal
-        loaded_at_field: evt_block_time
-        columns:
-          - name: contract_address
-          - name: evt_tx_hash
-          - name: evt_index
-          - name: evt_block_time
-          - name: evt_block_number
-          - name: fee
-          - name: nullifierHash
-          - name: relayer
-          - name: to
       - name: TornadoCashBNB_evt_Withdrawal
         loaded_at_field: evt_block_time
         columns:
@@ -44,7 +31,6 @@ sources:
   - name: tornado_cash_arbitrum
     freshness:
       warn_after: { count: 12, period: hour }
-      error_after: { count: 24, period: hour }
     tables:
       - name: ETHTornado_evt_Deposit
         loaded_at_field: evt_block_time
@@ -73,7 +59,6 @@ sources:
   - name: tornado_cash_optimism
     freshness:
       warn_after: { count: 12, period: hour }
-      error_after: { count: 24, period: hour }
     tables:
       - name: ETHTornado_evt_Deposit
         loaded_at_field: evt_block_time
@@ -86,18 +71,6 @@ sources:
           - name: commitment
           - name: leafIndex
           - name: timestamp
-      - name: erc20_evt_Withdrawal
-        loaded_at_field: evt_block_time
-        columns:
-          - name: contract_address
-          - name: evt_tx_hash
-          - name: evt_index
-          - name: evt_block_time
-          - name: evt_block_number
-          - name: fee
-          - name: nullifierHash
-          - name: relayer
-          - name: to
       - name: ETHTornado_evt_Withdrawal
         loaded_at_field: evt_block_time
         columns:
@@ -114,7 +87,6 @@ sources:
   - name: tornado_cash_avalanche_c
     freshness:
       warn_after: { count: 12, period: hour }
-      error_after: { count: 24, period: hour }
     tables:
       - name: ETHTornado_evt_Deposit
         loaded_at_field: evt_block_time
@@ -143,7 +115,6 @@ sources:
   - name: tornado_cash_ethereum
     freshness:
       warn_after: { count: 12, period: hour }
-      error_after: { count: 24, period: hour }
     tables:
       - name: erc20_evt_Deposit
         loaded_at_field: evt_block_time
@@ -191,7 +162,6 @@ sources:
   - name: tornado_cash_gnosis
     freshness:
       warn_after: { count: 12, period: hour }
-      error_after: { count: 24, period: hour }
     tables:
       - name: eth_evt_Deposit
         loaded_at_field: evt_block_time