1inch-U84: project_* lineage optimization & updates (#8898)

Author: max-morrow

dbt_subprojects/daily_spellbook/macros/project/oneinch/_meta/oneinch_mapped_contracts_macro.sql

@@ -70,7 +70,7 @@
         , ('0x8317c460c22a9958c27b4b6403b98d2ef4e2ad32', 'true', 'BabySwap'             , 'UNDEFINED'               , ['bnb'])
         , ('0xba12222222228d8ba445958a75a0704d566bf2c8', 'true', 'Balancer'             , 'Vault'                   , ['ethereum','bnb','polygon','arbitrum','avalanche_c','gnosis','optimism','base'])
         , ('0x6093aebac87d62b1a5a4ceec91204e35020e38be', 'true', 'Baoswap'              , 'UniswapV2Router02'       , ['gnosis'])
-        , ('0xbbbbbBB520d69a9775E85b458C58c648259FAD5F', 'true', 'Bebop'                , 'BebopSettlement'         , ['ethereum','arbitrum','base'])
+        , ('0xbbbbbBB520d69a9775E85b458C58c648259FAD5F', 'true', 'Bebop'                , 'BebopSettlement'         , ['ethereum','arbitrum','base','optimism'])
         , ('0xf4D3269fACF1FfD633195715e5d1357f0d4489d5', 'true', 'Bebop'                , 'BebopSettlement'         , ['base'])
         , ('0xbebebeb035351f58602e0c1c8b59ecbff5d5f47b', 'true', 'Bebop'                , 'JamSettlement'           , ['arbitrum'])
         , ('0xb300000b72DEAEb607a12d5f54773D1C19c7028d', 'true', 'Binance'              , 'DEXRouter'               , ['ethereum','bnb','polygon','arbitrum','avalanche_c','base','optimism','fantom','gnosis'])
@@ -482,7 +482,7 @@
         , ('0x8c42cf13fbea2ac15b0fe5a5f3cf35eec65d7d7d', 'false', 'Native'              , 'NativeRouterV1'          , ['polygon'])
         , ('0x41d7b3abcfecf1f1b4b1b962da8f086114b6cc5a', 'false', 'Native'              , 'NativeRouterV1'          , ['base'])
         , ('0xc6f7a7ba5388bfb5774bfaa87d350b7793fd9ef1', 'false', 'Native'              , 'NativeRouterV1'          , ['mantle'])
-        , ('0xd315a9c38ec871068fec378e4ce78af528c76293', 'false', 'Swaap'               , 'Vault'                   , ['ethereum','polygon','arbitrum','linea'])
+        , ('0xd315a9c38ec871068fec378e4ce78af528c76293', 'false', 'Swaap'               , 'Vault'                   , ['ethereum','polygon','arbitrum','linea','optimism','avalanche_c','sonic'])
         , ('0x03C01Acae3D0173a93d819efDc832C7C4F153B06', 'false', 'Swaap'               , 'Vault'                   , ['bnb','base'])
         , ('0xa57bd00134b2850b2a1c55860c9e9ea100fdd6cf', 'false', 'MEVBot'              , 'MEVBot'                  , ['ethereum'])
         , ('0xa69babef1ca67a37ffaf7a485dfff3382056e78c', 'false', 'MEVBot'              , 'MEVBot'                  , ['ethereum'])

dbt_subprojects/daily_spellbook/macros/project/oneinch/project/cfg/oneinch_project_orders_cfg_methods_macro.sql

@@ -741,7 +741,7 @@
     "taker_asset":      "substr(input, 4 + 32*4 + 12 + 1        , 20)",
     "maker_max_amount": "substr(input, 4 + 32*7 + 1             , 32)",
     "taker_max_amount": "substr(input, 4 + 32*6 + 1             , 32)",
-    "taking_amount":    "substr(input, 4 + 32*12 + 1            , 32)",
+    "taking_amount":    "if(bytearray_to_uint256(substr(input, 4 + 32*12 + 1, 32)) = 0, substr(input, 4 + 32*6 + 1, 32), substr(input, 4 + 32*12 + 1, 32))",
     "receiver":         "substr(input, 4 + 32*8 + 12 + 1        , 20)",
     "order_hash":       "substr(input, 4 + 32*10 + 1            , 16)",
 }] %}
@@ -765,7 +765,7 @@
     "taker_asset":      "substr(input, " ~ _beginning ~ " + " ~ _taker_tokens ~ " + 32*(" ~ _taker_tokens_number ~ "+2) + 12 + 1, 20)",
     "maker_max_amount": "substr(input, " ~ _signatures ~ " - 32*3 + 1, 32)",
     "taker_max_amount": "substr(input, " ~ _beginning ~ " + " ~ _taker_amounts ~ " + 32*(" ~ _taker_amounts_number ~ "+2) + 1, 32)",
-    "taking_amount":    "substr(input, 4 + 32*2 + 1, 32)",
+    "taking_amount":    "if(bytearray_to_uint256(substr(input, 4 + 32*2 + 1, 32)) = 0, substr(input, " ~ _beginning ~ " + " ~ _taker_amounts ~ " + 32*(" ~ _taker_amounts_number ~ "+2) + 1, 32), substr(input, 4 + 32*2 + 1, 32))",
     "deadline":         "substr(input, " ~ _beginning ~ " + 32*0 + 1, 32)",
     "order_hash":       "substr(input, " ~ _beginning ~ " + 32*10 + 1, 16)",
 }] %}

dbt_subprojects/daily_spellbook/macros/project/oneinch/project/oneinch_project_calls_macro.sql

@@ -1,11 +1,11 @@
-{% macro 
+{%- macro 
     oneinch_project_calls_macro(
-        blockchain
-        , date_from = '2019-01-01'
+        blockchain,
+        date_from = '2019-01-01'
     )
-%}
+-%}
 
-{% set native_addresses = '(0x0000000000000000000000000000000000000000, 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee)' %}
+{%- set native_addresses = '(0x0000000000000000000000000000000000000000, 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee)' -%}
 
 
 
@@ -26,13 +26,14 @@ static as (
         , any_value(tag) as tag
         , any_value(flags) as flags
     from {{ ref('oneinch_' + blockchain + '_mapped_contracts') }}
-    where
-        project not in ('MEVBot', 'Unknown')
+    where true
+        and project not in ('MEVBot', 'Unknown')
     group by 1, 2
 )
 
 , signatures as (
-    select *, reduce(cross_chain_suitables, false, (r, x) -> if(position(x in lower(replace(method, '_'))) > 0, true, r), r -> r) as cross_chain_method
+    select *
+        , reduce(cross_chain_suitables, false, (r, x) -> if(position(x in lower(replace(method, '_'))) > 0, true, r), r -> r) as cross_chain_method
     from (
         select
             id as selector
@@ -65,12 +66,9 @@ static as (
             , success as call_success
             , tx_success
         from {{ source(blockchain, 'traces') }}
-        where
-            {% if is_incremental() %}
-                {{ incremental_predicate('block_time') }}
-            {% else %}
-                block_time >= timestamp '{{date_from}}'
-            {% endif %}
+        where true
+            and block_time >= timestamp '{{ date_from }}'
+            {% if is_incremental() -%} and {{ incremental_predicate('block_time') }} {%- endif %}
     )
     join contracts using(call_to)
     join signatures using(selector)
@@ -83,12 +81,9 @@ static as (
             , "to" as tx_to
             , gas_used as tx_gas_used
         from {{ source(blockchain, 'transactions') }}
-        where
-            {% if is_incremental() %}
-                {{ incremental_predicate('block_time') }}
-            {% else %}
-                block_time >= timestamp '{{date_from}}'
-            {% endif %}
+        where true
+            and block_time >= timestamp '{{ date_from }}'
+            {% if is_incremental() -%} and {{ incremental_predicate('block_time') }} {%- endif %}
 
     ) using(block_number, block_time, tx_hash)
 )
@@ -120,4 +115,4 @@ select
     , date(date_trunc('month', block_time)) as block_month
 from calls
 
-{% endmacro %}
+{%- endmacro %}

dbt_subprojects/daily_spellbook/macros/project/oneinch/project/oneinch_project_orders_macro.sql

@@ -1,21 +1,23 @@
-{% macro oneinch_project_orders_macro(
-    blockchain
-    , date_from = '2019-01-01'
-)%}
+{%- macro
+    oneinch_project_orders_macro(
+        blockchain,
+        date_from = '2019-01-01'
+    )
+-%}
 
 
 
-{% set wrapping = 'array[0x0000000000000000000000000000000000000000, 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee, wrapped_native_token_address]' %}
+{%- set wrapping = 'array[0x0000000000000000000000000000000000000000, 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee, wrapped_native_token_address]' -%}
 
 with
 
 logs as (
 {% for event, event_data in oneinch_project_orders_cfg_events_macro().items() %}
     select
         '{{ blockchain }}' as blockchain
-        , '{{event_data["project"]}}' as project
+        , '{{ event_data["project"] }}' as project
+        , block_date
         , block_number
-        , block_time
         , tx_hash
         , tx_from
         , tx_to
@@ -51,25 +53,26 @@ logs as (
         , topic2
         , topic3
         , data
-        , row_number() over(partition by block_number, tx_hash order by index) as log_counter
+        , row_number() over(partition by block_number, tx_hash order by index) as trade_counter
     from {{ ref('oneinch_' + blockchain + '_project_orders_raw_logs') }}
-    where
-        topic0 = {{ event }}
-        and {% if is_incremental() %}{{ incremental_predicate('block_time') }}
-            {% else %}block_time >= timestamp '{{date_from}}'
-            {% endif %}
+    where true
+        and topic0 = {{ event }}
+        and block_time >= timestamp '{{ date_from }}'
+        {% if is_incremental() %}and {{ incremental_predicate('block_time') }}{% endif %}
     {% if not loop.last %}union all{% endif %}
 {% endfor %}
 )
 
 , calls as (
-    select *, row_number() over(partition by block_number, tx_hash, call_to, method order by call_trace_address, call_trade) as call_trade_counter -- trade counter in the tx: there may be multiple calls and multiple trades within a call in a single transaction
+    select *
+        , row_number() over(partition by block_number, tx_hash, call_to, method order by call_trace_address, call_trade) as trade_counter -- trade counter in the tx: there may be multiple calls and multiple trades within a call in a single transaction
     from (
         select
             blockchain
             , project
             , tag
             , flags
+            , block_date
             , block_number
             , block_time
             , tx_hash
@@ -120,6 +123,7 @@ logs as (
                 , project
                 , tag
                 , flags
+                , block_date
                 , block_number
                 , block_time
                 , tx_hash
@@ -137,7 +141,7 @@ logs as (
                 , {{ item.get("event", "null") }} as topic0
                 , coalesce({{ item.get("number", "1") }}, 1) as call_trades -- total trades in the call
                 , transform(sequence(1, coalesce({{ item.get("number", "1") }}, 1)), x -> map_from_entries(array[
-                      ('trade',             try(to_big_endian_64(x)))
+                    ('trade',               try(to_big_endian_64(x)))
                     , ('maker',             {{ item.get("maker", "null") }})
                     , ('taker',             {{ item.get("taker", "null") }})
                     , ('receiver',          {{ item.get("receiver", "null") }})
@@ -167,18 +171,19 @@ logs as (
                 , output
             from {{ ref('oneinch_' + blockchain + '_project_orders_raw_traces') }}
             join (
-                select *, address as "to"
+                select *
+                    , address as "to"
                 from {{ ref('oneinch_' + blockchain + '_mapped_contracts') }}
-                where
-                    blockchain = '{{ blockchain }}'
+                where true
+                    and blockchain = '{{ blockchain }}'
                     and project = '{{ item["project"] }}'
                     and coalesce({{ item.get("tag", "null") }} = tag, true)
             ) using("to")
-            where
-                {% if is_incremental() %}{{ incremental_predicate('block_time') }}
-                {% else %}block_time > greatest(first_created_at, timestamp '{{date_from}}'){% endif %}
+            where true
                 and substr(input, 1, 4) = {{ item["selector"] }}
                 and {{ item.get("condition", "true") }}
+                and block_time > greatest(first_created_at, timestamp '{{ date_from }}')
+                {% if is_incremental() %}and {{ incremental_predicate('block_time') }}{% endif %}
 
             {% if not loop.last %}union all{% endif %}
         {% endfor %}
@@ -187,8 +192,7 @@ logs as (
 )
 
 , joined as (
-    select
-        *
+    select *
         , coalesce(log_maker, call_maker, call_to) as maker
         , coalesce(log_taker, call_taker) as taker
         , coalesce(log_receiver, call_receiver) as receiver
@@ -212,15 +216,10 @@ logs as (
         , coalesce(log_fee_amount, call_fee_amount) as fee_amount
         , coalesce(log_fee_receiver, call_fee_receiver) as fee_receiver
         , coalesce(log_nonce, call_nonce) as order_nonce
-        , coalesce(log_order_hash, call_order_hash, concat(tx_hash, to_big_endian_32(cast(call_trade_counter as int))), concat(tx_hash, to_big_endian_32(cast(index as int)))) as order_hash
-        , count(*) over(partition by block_number, tx_hash, call_trace_address, call_trade) as call_trade_logs -- logs for each trade
-        , count(*) over(partition by block_number, tx_hash, index) as log_call_trades -- trades for each log
+        , coalesce(log_order_hash, call_order_hash, concat(tx_hash, to_utf8(array_join(call_trace_address, ',')), to_utf8('|'), substr(call_trade, 5))) as order_hash -- | required to eliminate duplicates: concat('0,1', '11') == concat('0,11', '1')
     from calls
-    full join logs using(blockchain, block_number, block_time, tx_hash, topic0, project)
-    join (
-        select * from {{ source('oneinch', 'blockchains') }}
-        where blockchain = '{{blockchain}}'
-    ) using(blockchain)
+    left join logs using(blockchain, block_date, block_number, tx_hash, topic0, project, trade_counter)
+    join (select * from {{ source('oneinch', 'blockchains') }} where blockchain = '{{blockchain}}') using(blockchain)
     where
             coalesce(call_maker = log_maker, true)
         and coalesce(call_taker = log_taker, true)
@@ -264,7 +263,7 @@ select
     , call_gas_used
     , call_type
     , call_error
-    , coalesce(bytearray_to_bigint(call_trade), 1) as call_trade
+    , from_big_endian_64(call_trade) as call_trade
     , call_trades
     , method
     , maker
@@ -303,9 +302,5 @@ select
     , to_unixtime(block_time) as block_unixtime
     , date(date_trunc('month', block_time)) as block_month
 from joined
-where
-    call_trade_logs = 1 -- if the log was not found or if found the only one log for a trade in the call
-    or log_call_trades = 1 -- if the call was not found or if found the only one call trade for the log
-    or call_trade_counter = log_counter -- if found several logs
 
-{% endmacro %}
+{%- endmacro -%}

dbt_subprojects/daily_spellbook/macros/project/oneinch/project/oneinch_project_orders_raw_logs_macro.sql

@@ -1,7 +1,9 @@
-{% macro oneinch_project_orders_raw_logs_macro(
-    blockchain
-    , date_from = '2019-01-01'
-)%}
+{%- macro
+    oneinch_project_orders_raw_logs_macro(
+        blockchain,
+        date_from = '2019-01-01'
+    )
+-%}
 
 select 
     block_number
@@ -19,15 +21,9 @@ select
     , row_number() over(partition by block_number, tx_hash order by index) as log_counter
     , date(block_time) as block_date
 from {{ source(blockchain, 'logs') }}
-where
-    {% if is_incremental() %}
-        {{ incremental_predicate('block_time') }}
-    {% else %}
-        block_time >= timestamp '{{date_from}}'
-    {% endif %}
-
-    and topic0 in (
-        {{ oneinch_project_orders_cfg_events_macro().keys() | join(', ') }}
-    )
+where true
+    and block_time >= timestamp '{{ date_from }}'
+    and topic0 in ({{ oneinch_project_orders_cfg_events_macro().keys() | join(', ') }})
+    {% if is_incremental() -%} and {{ incremental_predicate('block_time') }} {%- endif %}
 
-{% endmacro %}
+{%- endmacro -%}

dbt_subprojects/daily_spellbook/macros/project/oneinch/project/oneinch_project_orders_raw_traces_macro.sql

@@ -1,7 +1,9 @@
-{% macro oneinch_project_orders_raw_traces_macro(
-    blockchain
-    , date_from = '2019-01-01'
-)%}
+{%- macro
+    oneinch_project_orders_raw_traces_macro(
+        blockchain,
+        date_from = '2019-01-01'
+    )
+-%}
 
 select 
     block_number
@@ -20,19 +22,15 @@ select
     , output
     , block_date
 from {{ source(blockchain, 'traces') }}
-where
-    {% if is_incremental() %}
-        {{ incremental_predicate('block_time') }}
-    {% else %}
-        block_time >= timestamp '{{date_from}}'
-    {% endif %}
-
+where true
+    and block_time >= timestamp '{{ date_from }}'
     and substr(input, 1, 4) in (
         {% set selectors = [] %}
         {% for item in oneinch_project_orders_cfg_methods_macro() %}
             {% do selectors.append(item["selector"]) %}
         {% endfor %}
         {{ ','.join(selectors) }}
     )
+    {% if is_incremental() -%} and {{ incremental_predicate('block_time') }} {%- endif %}
 
-{% endmacro %}
+{%- endmacro -%}