bridges: materialise chains + orbiter on all chains (#9034)

* bridges: orbiter on all chains

* fix

* bridges: materialise chains (test with ethereum)

* update all chains to materialize incrementally

* update evms raw to views

* fix spacing

* add partitions, update join logic for performance

* clean up block_month usage

* change unique_key

* switch to surrogate key for layerzero bridge_transfer_id

* fix

* fix

* switch to deposits/withdrawals macros per chaing

* join {{this}} on block_month

* make added chains use the deposits macros

* fix

* fix

* trigger raw models

* double timeout for CI tests

* revert CI timeout change

* add block_time incremental filter on this lookup

* remove unused hop macros

---------

Co-authored-by: jeff-dude <[email protected]>
Co-authored-by: jeff-dude <[email protected]>

Author: hildobby

dbt_subprojects/hourly_spellbook/macros/sector/bridges/bridges_deposits.sql

@@ -0,0 +1,70 @@
+{% macro bridges_deposits(blockchain, bridges_platforms) %}
+
+SELECT deposit_chain
+    , withdrawal_chain_id
+    , withdrawal_chain
+    , bridge_name
+    , bridge_version
+    , CAST(date_trunc('month', block_date) as date) as block_month
+    , block_date
+    , block_time
+    , block_number
+    , deposit_amount_raw
+    , sender
+    , recipient
+    , deposit_token_address
+    , deposit_token_standard
+    , tx_from
+    , tx_hash
+    , evt_index
+    , contract_address
+    , bridge_transfer_id
+FROM (
+    SELECT *
+        , ROW_NUMBER() OVER (PARTITION BY block_date, block_time, block_number, deposit_chain, tx_hash, evt_index, bridge_transfer_id ORDER BY block_number, block_time) AS rn
+    FROM (
+        {% for bridge_platform in bridges_platforms %}
+        SELECT d.deposit_chain
+        , d.withdrawal_chain_id
+        , d.withdrawal_chain
+        , d.bridge_name
+        , d.bridge_version
+        , d.block_date
+        , d.block_time
+        , d.block_number
+        , d.deposit_amount_raw
+        , d.sender
+        , d.recipient
+        , d.deposit_token_standard
+        , d.deposit_token_address
+        , d.tx_from
+        , d.tx_hash
+        , d.evt_index
+        , d.contract_address
+        , d.bridge_transfer_id
+        FROM {{ ref(bridge_platform) }} d
+        {% if is_incremental() %}
+        LEFT JOIN {{this}} t ON t.deposit_chain = '{{blockchain}}'
+            AND date_trunc('month', d.block_date) = t.block_month
+            AND d.block_time = t.block_time
+            AND d.block_number = t.block_number
+            AND d.bridge_name = t.bridge_name
+            AND d.bridge_version = t.bridge_version
+            AND d.withdrawal_chain_id = t.withdrawal_chain_id
+            AND d.tx_hash = t.tx_hash
+            AND d.evt_index = t.evt_index
+            AND d.bridge_transfer_id = t.bridge_transfer_id
+            AND {{ incremental_predicate('t.block_time') }}
+        WHERE {{ incremental_predicate('d.block_time') }}
+        AND t.block_time IS NULL
+        {% endif %}
+        {% if not loop.last %}
+        UNION ALL
+        {% endif %}
+        {% endfor %} 
+        )
+    )
+WHERE rn = 1
+
+{% endmacro %}
+

dbt_subprojects/hourly_spellbook/macros/sector/bridges/bridges_withdrawals.sql

@@ -0,0 +1,66 @@
+{% macro bridges_withdrawals(blockchain, bridges_platforms) %}
+
+SELECT deposit_chain_id
+    , deposit_chain
+    , withdrawal_chain
+    , bridge_name
+    , bridge_version
+    , cast(date_trunc('month', block_date) as date) as block_month
+    , block_date
+    , block_time
+    , block_number
+    , withdrawal_amount_raw
+    , sender
+    , recipient
+    , withdrawal_token_address
+    , withdrawal_token_standard
+    , tx_from
+    , tx_hash
+    , evt_index
+    , contract_address
+    , bridge_transfer_id
+FROM (
+        {% for bridge_platform in bridges_platforms %}
+        SELECT w.deposit_chain_id
+            , w.deposit_chain
+            , w.withdrawal_chain
+            , w.bridge_name
+            , w.bridge_version
+            , cast(date_trunc('month', w.block_date) as date) as block_month
+            , w.block_date
+            , w.block_time
+            , w.block_number
+            , w.withdrawal_amount_raw
+            , w.sender
+            , w.recipient
+            , w.withdrawal_token_address
+            , w.withdrawal_token_standard
+            , w.tx_from
+            , w.tx_hash
+            , w.evt_index
+            , w.contract_address
+            , w.bridge_transfer_id
+        FROM {{ ref(bridge_platform) }} w
+        {% if is_incremental() %}
+        LEFT JOIN {{this}} t ON t.withdrawal_chain = '{{blockchain}}'
+            AND date_trunc('month', w.block_date) = t.block_month
+            AND w.block_time = t.block_time
+            AND w.block_number = t.block_number
+            AND w.bridge_name = t.bridge_name
+            AND w.bridge_version = t.bridge_version
+            AND w.deposit_chain_id = t.deposit_chain_id
+            AND w.tx_hash = t.tx_hash
+            AND w.evt_index = t.evt_index
+            AND w.bridge_transfer_id = t.bridge_transfer_id
+            AND {{ incremental_predicate('t.block_time') }}
+        WHERE  {{ incremental_predicate('w.block_time') }}
+        AND t.block_time IS NULL
+        {% endif %}
+        {% if not loop.last %}
+        UNION ALL
+        {% endif %}
+        {% endfor %}
+        )
+
+{% endmacro %}
+

dbt_subprojects/hourly_spellbook/macros/sector/bridges/layerzero_v1_deposits.sql

@@ -73,7 +73,7 @@ SELECT distinct '{{blockchain}}' AS deposit_chain
 , sc.tx_hash
 , COALESCE(t.evt_index, -sc.call_send_index) AS evt_index
 , sc.contract_address
-, CAST(t.unique_key AS varchar) AS bridge_transfer_id
+, {{ dbt_utils.generate_surrogate_key(['sc.tx_hash', 't.evt_index', 'sc.call_send_index']) }} as bridge_transfer_id
 FROM send_calls sc
 LEFT JOIN transfers t ON t.block_number=sc.block_number
         AND t.tx_hash=sc.tx_hash

dbt_subprojects/hourly_spellbook/models/_sector/bridges/chain_indexes/bridges_hop_token_indexes.sql

@@ -0,0 +1,56 @@
+{{ config(
+    schema = 'bridges',
+    alias = 'hop_token_indexes',
+    materialized = 'view',
+    )
+}}
+
+-- source: https://github.com/DefiLlama/bridges-server/blob/dacbeefb5d076b438f58b15e57aa6d271ca04ae7/src/adapters/hop/index.ts
+
+SELECT blockchain, token_address, token_symbol, bridge_contract_address
+    FROM (VALUES
+    ('ethereum', '0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2', 'ETH', '0xb8901acB165ed027E32754E0FFe830802919727f')
+    , ('ethereum', '0x6B175474E89094C44Da98b954EedeAC495271d0F', 'DAI', '0x3d4Cc8A61c7528Fd86C55cfe061a78dCBA48EDd1')
+    , ('ethereum', '0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48', 'USDC', '0x3666f603Cc164936C1b87e207F36BEBa4AC5f18a')
+    , ('ethereum', '0xdAC17F958D2ee523a2206206994597C13D831ec7', 'USDT', '0x3E4a3a4796d16c0Cd582C382691998f7c06420B6')
+    , ('ethereum', '0x7D1AfA7B718fb893dB30A3aBc0Cfc608AaCfeBB0', 'MATIC', '0x22B1Cbb8D98a01a3B71D034BB899775A76Eb1cc2')
+    , ('ethereum', '0xc5102fE9359FD9a28f877a67E36B0F050d81a3CC', 'HOP', '0x914f986a44AcB623A277d6Bd17368171FCbe4273')
+    , ('ethereum', '0xB0c7a3Ba49C7a6EaBa6cD4a96C55a1391070Ac9A', 'MAGIC', '0xf074540eb83c86211F305E145eB31743E228E57d')
+    , ('ethereum', '0xae78736Cd615f374D3085123A210448E74Fc6393', 'rETH', '0x87269B23e73305117D0404557bAdc459CEd0dbEc')
+    , ('polygon', '0x7ceB23fD6bC0adD59E62ac25578270cFf1b9f619', 'ETH', '0xb98454270065A31D71Bf635F6F7Ee6A518dFb849')
+    , ('polygon', '0x8f3Cf7ad23Cd3CaDbD9735AFf958023239c6A063', 'DAI', '0xEcf268Be00308980B5b3fcd0975D47C4C8e1382a')
+    , ('polygon', '0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174', 'USDC', '0x25D8039bB044dC227f741a9e381CA4cEAE2E6aE8')
+    , ('polygon', '0xc2132D05D31c914a87C6611C10748AEb04B58e8F', 'USDT', '0x6c9a1ACF73bd85463A46B0AFc076FBdf602b690B')
+    , ('polygon', '0x0d500B1d8E8eF31E21C99d1Db9A6444d3ADf1270', 'MATIC', '0x553bC791D746767166fA3888432038193cEED5E2')
+    , ('polygon', '0xc5102fE9359FD9a28f877a67E36B0F050d81a3CC', 'HOP', '0x58c61AeE5eD3D748a1467085ED2650B697A66234')
+    , ('arbitrum', '0x82aF49447D8a07e3bd95BD0d56f35241523fBab1', 'ETH', '0x3749C4f034022c39ecafFaBA182555d4508caCCC')
+    , ('arbitrum', '0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1', 'DAI', '0x7aC115536FE3A185100B2c4DE4cb328bf3A58Ba6')
+    , ('arbitrum', '0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8', 'USDC', '0x0e0E3d2C5c292161999474247956EF542caBF8dd')
+    , ('arbitrum', '0xFd086bC7CD5C481DCC9C85ebE478A1C0b69FCbb9', 'USDT', '0x72209Fe68386b37A40d6bCA04f78356fd342491f')
+    , ('arbitrum', '0xc5102fE9359FD9a28f877a67E36B0F050d81a3CC', 'HOP', '0x25FB92E505F752F730cAD0Bd4fa17ecE4A384266')
+    , ('arbitrum', '0x539bdE0d7Dbd336b79148AA742883198BBF60342', 'MAGIC', '0xEa5abf2C909169823d939de377Ef2Bf897A6CE98')
+    , ('arbitrum', '0xEC70Dcb4A1EFa46b8F2D97C310C9c4790ba5ffA8', 'rETH', '0xc315239cFb05F1E130E7E28E603CEa4C014c57f0')
+    , ('optimism', '0x4200000000000000000000000000000000000006', 'ETH', '0x83f6244Bd87662118d96D9a6D44f09dffF14b30E')
+    , ('optimism', '0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1', 'DAI', '0x7191061D5d4C60f598214cC6913502184BAddf18')
+    , ('optimism', '0x7F5c764cBc14f9669B88837ca1490cCa17c31607', 'USDC', '0xa81D244A1814468C734E5b4101F7b9c0c577a8fC')
+    , ('optimism', '0x94b008aA00579c1307B0EF2c499aD98a8ce58e58', 'USDT', '0x46ae9BaB8CEA96610807a275EBD36f8e916b5C61')
+    , ('optimism', '0xc5102fE9359FD9a28f877a67E36B0F050d81a3CC', 'HOP', '0x03D7f750777eC48d39D080b020D83Eb2CB4e3547')
+    , ('optimism', '0x9Bcef72be871e61ED4fBbc7630889beE758eb81D', 'rETH', '0xA0075E8cE43dcB9970cB7709b9526c1232cc39c2')
+    , ('gnosis', '0x6A023CCd1ff6F2045C3309768eAd9E68F978f6e1', 'ETH', '0xD8926c12C0B2E5Cd40cFdA49eCaFf40252Af491B')
+    , ('gnosis', '0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d', 'DAI', '0x0460352b91D7CF42B0E1C1c30f06B602D9ef2238')
+    , ('gnosis', '0xDDAfbb505ad214D7b80b1f830fcCc89B60fb7A83', 'USDC', '0x25D8039bB044dC227f741a9e381CA4cEAE2E6aE8')
+    , ('gnosis', '0x4ECaBa5870353805a9F068101A40E0f32ed605C6', 'USDT', '0xFD5a186A7e8453Eb867A360526c5d987A00ACaC2')
+    , ('gnosis', '0x7122d7661c4564b7C6Cd4878B06766489a6028A2', 'MATIC', '0x7ac71c29fEdF94BAc5A5C9aB76E1Dd12Ea885CCC')
+    , ('gnosis', '0xc5102fE9359FD9a28f877a67E36B0F050d81a3CC', 'HOP', '0x6F03052743CD99ce1b29265E377e320CD24Eb632')
+    , ('arbitrum_nova', '0x722E8BdD2ce80A4422E880164f2079488e115365', 'ETH', '0x8796860ca1677Bf5d54cE5A348Fe4b779a8212f3')
+    , ('arbitrum_nova', '0xc5102fE9359FD9a28f877a67E36B0F050d81a3CC', 'HOP', '0x02D47f76523d2f059b617E4346de67482792eB83')
+    , ('arbitrum_nova', '0xe8936ac97A85d708d5312D52C30c18d4533b8A9c', 'MAGIC', '0xE638433e2C1dF5f7a3a21b0a6b5c4b37278e55DC')
+    , ('base', '0x4200000000000000000000000000000000000006', 'ETH', '0x3666f603Cc164936C1b87e207F36BEBa4AC5f18a')
+    , ('base', '0xd9aAEc86B65D86f6A7B5B1b0c42FFA531710b6CA', 'USDC', '0x46ae9BaB8CEA96610807a275EBD36f8e916b5C61')
+    , ('base', '0xc5102fE9359FD9a28f877a67E36B0F050d81a3CC', 'HOP', '0xe22D2beDb3Eca35E6397e0C6D62857094aA26F52')
+    , ('linea', '0xe5D7C2a44FfDDf6b295A15c148167daaAf5Cf34f', 'ETH', '0xCbb852A6274e03fA00fb4895dE0463f66dF27a11')
+    , ('linea', '0xc5102fE9359FD9a28f877a67E36B0F050d81a3CC', 'HOP', '0x0a6b1904369fE59E002ad0713ae89d4E3dF5A7Cf')
+    , ('polygon_zkevm', '0x4F9A0e7FD2Bf6067db6994CF12E4495Df938E6e9', 'ETH', '0x0ce6c85cF43553DE10FC56cecA0aef6Ff0DD444d')
+    , ('polygon_zkevm', '0xc5102fE9359FD9a28f877a67E36B0F050d81a3CC', 'HOP', '0x9ec9551d4A1a1593b0ee8124D98590CC71b3B09D')
+    ) AS x (blockchain, token_address, token_symbol, bridge_contract_address)
+

dbt_subprojects/hourly_spellbook/models/_sector/bridges/flows/bridges_crosschain_deposits.sql

@@ -2,7 +2,7 @@
     schema = 'bridges_crosschain'
     , alias = 'deposits'
     , materialized = 'view'
-    , post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c", "base", "blast", "bnb", "ethereum", "hyperevm", "ink", "lens", "linea", "optimism", "plasma", "polygon", "scroll", "unichain", "worldchain", "zksync", "zora", "fantom", "gnosis", "nova", "opbnb", "berachain", "corn", "flare", "sei", "boba"]\',
+    , post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c", "base", "blast", "bnb", "ethereum", "hyperevm", "ink", "lens", "linea", "optimism", "plasma", "polygon", "scroll", "unichain", "worldchain", "zksync", "zora", "fantom", "gnosis", "nova", "opbnb", "berachain", "corn", "flare", "sei", "boba", "abstract", "apechain", "bob", "celo", "kaia", "katana", "mantle", "plume", "ronin", "sonic", "sophon", "story", "taiko", "zkevm"]\',
                                 "sector",
                                 "bridges",
                                 \'["hildobby"]\') }}'

dbt_subprojects/hourly_spellbook/models/_sector/bridges/flows/bridges_crosschain_flows.sql

@@ -2,7 +2,7 @@
     schema = 'bridges_crosschain',
     alias = 'flows',
     materialized = 'view'
-    , post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c", "base", "blast", "bnb", "ethereum", "hyperevm", "ink", "lens", "linea", "optimism", "plasma", "polygon", "scroll", "unichain", "worldchain", "zksync", "zora", "fantom", "gnosis", "nova", "opbnb", "berachain", "corn", "flare", "sei", "boba", "mantle"]\',
+    , post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c", "base", "blast", "bnb", "ethereum", "hyperevm", "ink", "lens", "linea", "optimism", "plasma", "polygon", "scroll", "unichain", "worldchain", "zksync", "zora", "fantom", "gnosis", "nova", "opbnb", "berachain", "corn", "flare", "sei", "boba", "mantle", "abstract", "apechain", "bob", "celo", "kaia", "katana", "plume", "ronin", "sonic", "sophon", "story", "taiko", "zkevm"]\',
                                 "sector",
                                 "bridges",
                                 \'["hildobby"]\') }}'

dbt_subprojects/hourly_spellbook/models/_sector/bridges/flows/chains/abstract/bridges_abstract_deposits.sql

@@ -1,40 +1,20 @@
 {% set blockchain = 'abstract' %}
 
 {{ config(
-    schema = 'bridges_' + blockchain,
-    alias = 'deposits',
-    materialized = 'view'
+    schema = 'bridges_' + blockchain
+    , alias = 'deposits'
+    , materialized = 'incremental'
+    , file_format = 'delta'
+    , partition_by = ['block_month']
+    , incremental_strategy='merge'
+    , unique_key = ['tx_hash', 'evt_index', 'bridge_transfer_id']
+    , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
     )
 }}
 
 {% set bridges_platforms = [
     'bridges_' + blockchain + '_symbiosis_v1_deposits'
+    , 'bridges_' + blockchain + '_orbiter_v1_deposits'
 ] %}
 
-SELECT *
-FROM (
-    {% for bridge_platform in bridges_platforms %}
-    SELECT deposit_chain
-    , withdrawal_chain_id
-    , withdrawal_chain
-    , bridge_name
-    , bridge_version
-    , block_date
-    , block_time
-    , block_number
-    , deposit_amount_raw
-    , sender
-    , recipient
-    , deposit_token_standard
-    , deposit_token_address
-    , tx_from
-    , tx_hash
-    , evt_index
-    , contract_address
-    , bridge_transfer_id
-    FROM {{ ref(bridge_platform) }}
-    {% if not loop.last %}
-    UNION ALL
-    {% endif %}
-    {% endfor %}
-)
+{{ bridges_deposits(blockchain, bridges_platforms) }}

dbt_subprojects/hourly_spellbook/models/_sector/bridges/flows/chains/abstract/platforms/_schema.yml

@@ -9,4 +9,13 @@ models:
       contributors: [ 'hildobby']
     config:
       tags: [ 'abstract', 'bridges', 'flows', 'withdrawals' ]
-    description: "Symbiosis's v1 bridge deposits events on Abstract"
+    description: "Symbiosis's v1 bridge deposits events on Abstract"
+  - name: bridges_abstract_orbiter_v1_deposits
+    meta:
+      blockchain: abstract
+      sector: bridges
+      project: orbiter
+      contributors: [ 'hildobby']
+    config:
+      tags: [ 'abstract', 'bridges', 'flows', 'withdrawals' ]
+    description: "Orbiter's v1 bridge deposits events on Abstract"

dbt_subprojects/hourly_spellbook/models/_sector/bridges/flows/chains/abstract/platforms/bridges_abstract_orbiter_v1_deposits.sql

@@ -0,0 +1,13 @@
+{% set blockchain = 'abstract' %}
+
+{{ config(
+    schema = 'bridges_' + blockchain,
+    alias = 'orbiter_v1_deposits',
+    materialized = 'view',
+    )
+}}
+
+{{orbiter_v1_deposits(blockchain = blockchain
+    , first_block_number = 2066216
+    )}}
+

dbt_subprojects/hourly_spellbook/models/_sector/bridges/flows/chains/apechain/bridges_apechain_deposits.sql

@@ -0,0 +1,20 @@
+{% set blockchain = 'apechain' %}
+
+{{ config(
+    schema = 'bridges_' + blockchain
+    , alias = 'deposits'
+    , materialized = 'incremental'
+    , file_format = 'delta'
+    , partition_by = ['block_month']
+    , incremental_strategy='merge'
+    , unique_key = ['tx_hash', 'evt_index', 'bridge_transfer_id']
+    , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
+    )
+}}
+
+{% set bridges_platforms = [
+    'bridges_' + blockchain + '_orbiter_v1_deposits'
+] %}
+
+{{ bridges_deposits(blockchain, bridges_platforms) }}
+