Migrate NFT beta base_trades

macros/models/_sector/nft/enrich_trades.sql

@@ -37,7 +37,7 @@ WITH base_union as (
         row_number() over (partition by tx_hash, sub_tx_trade_id order by tx_hash) as duplicates_rank
     FROM {{ nft_model[2] }}
     {% if is_incremental() %}
-    WHERE block_time >= date_trunc("day", now() - interval '1 week')
+    WHERE block_time >= date_trunc('day', now() - interval '7' day)
     {% endif %}
     {% if not loop.last %}
     UNION ALL
@@ -58,8 +58,8 @@ SELECT
     base.project_contract_address,
     base.trade_category,
     base.trade_type,
-    case when base.buyer = coalesce(agg1.contract_address,agg2.contract_address) then tx.from else base.buyer end as buyer,
-    case when base.seller = coalesce(agg1.contract_address,agg2.contract_address) then tx.from else base.seller end as seller,
+    case when base.buyer = coalesce(agg1.contract_address,agg2.contract_address) then tx."from" else base.buyer end as buyer,
+    case when base.seller = coalesce(agg1.contract_address,agg2.contract_address) then tx."from" else base.seller end as seller,
     base.nft_contract_address,
     base.nft_token_id,
     base.nft_amount,
@@ -69,7 +69,7 @@ SELECT
     base.royalty_fee_amount_raw,
     base.platform_fee_address,
     base.royalty_fee_address,
-    tx.from as tx_from,
+    tx."from" as tx_from,
     tx.to as tx_to,
     nft.name as nft_collection,
     nft.standard as nft_standard,
@@ -80,8 +80,8 @@ SELECT
     base.price_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as price_usd,
     base.platform_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as platform_fee_amount_usd,
     base.royalty_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as royalty_fee_amount_usd,
-    coalesce(cast(100*base.platform_fee_amount_raw/base.price_raw as double),cast(0.0 as double)) as platform_fee_percentage,
-    coalesce(cast(100*base.royalty_fee_amount_raw/base.price_raw as double),cast(0.0 as double)) as royalty_fee_percentage,
+    case when base.price_raw > uint256 '0' then cast(100*base.platform_fee_amount_raw/base.price_raw as double) else double '0' end as platform_fee_percentage,
+    case when base.price_raw > uint256 '0' then cast(100*base.royalty_fee_amount_raw/base.price_raw as double) else double '0' end as royalty_fee_percentage,
     coalesce(agg1.contract_address,agg2.contract_address) as aggregator_address,
     {% if aggregator_markers != null %}
     coalesce(agg_mark.aggregator_name, agg1.name, agg2.name) as aggregator_name
@@ -93,7 +93,7 @@ INNER JOIN {{ transactions_model }} tx
 ON tx.block_number = base.block_number
     AND tx.hash = base.tx_hash
     {% if is_incremental() %}
-    AND tx.block_time >= date_trunc("day", now() - interval '1 week')
+    AND tx.block_time >= date_trunc('day', now() - interval '7' day)
     {% endif %}
 LEFT JOIN {{ tokens_nft_model }} nft
 ON nft.contract_address = base.nft_contract_address
@@ -104,7 +104,7 @@ ON p.blockchain = base.blockchain
     AND p.contract_address = base.currency_contract
     AND p.minute = date_trunc('minute',base.block_time)
     {% if is_incremental() %}
-    AND p.minute >= date_trunc("day", now() - interval '1 week')
+    AND p.minute >= date_trunc('day', now() - interval '7' day)
     {% endif %}
 LEFT JOIN {{ aggregators }} agg1
 ON (base.buyer = agg1.contract_address
@@ -114,9 +114,9 @@ ON agg1.contract_address is null    -- only match if agg1 produces no matches, t
     AND tx.to = agg2.contract_address
 {% if aggregator_markers != null %}
 LEFT JOIN {{ aggregator_markers }} agg_mark
-ON RIGHT(tx.data, agg_mark.hash_marker_size) = agg_mark.hash_marker
-{% endif %}
+ON bytearray_starts_with(bytearray_reverse(tx.data), bytearray_reverse(agg_mark.hash_marker)) -- eq to end_with()
 WHERE base.duplicates_rank = 1
+{% endif %}
 )
 
 

macros/models/_sector/nft/enrich_trades_legacy.sql

@@ -0,0 +1,124 @@
+{% macro enrich_trades_legacy(blockchain='', models=[], transactions_model=null, tokens_nft_model=null, tokens_erc20_model=null, prices_model=null, aggregators=null, aggregator_markers=null) %}
+-- Macro to apply the NFT trades enrichment(s) to base models
+-- 1. add transaction information
+-- 2. add NFT token information
+-- 3. add ERC20 token information + handle ERC20 decimals
+-- 4. handle USD columns
+-- 5. add aggregator columns
+-- 6. fix buyer or seller for aggregator txs
+-- 7. calculate platform and royalty rates
+-- 8. deduplicate based on sub_tx_trade_id
+
+WITH base_union as (
+    {% for nft_model in models %}
+    SELECT
+        '{{ blockchain }}' as blockchain,
+        '{{ nft_model[0] }}' as project,
+        '{{ nft_model[1] }}' as project_version,
+        block_date,
+        block_time,
+        block_number,
+        tx_hash,
+        project_contract_address,
+        trade_category,                 --buy/sell/swap
+        trade_type,                     --primary/secondary
+        buyer,
+        seller,
+        nft_contract_address,
+        nft_token_id,
+        nft_amount,                -- always 1 for erc721
+        price_raw,
+        currency_contract,
+        platform_fee_amount_raw,
+        royalty_fee_amount_raw,
+        platform_fee_address,   -- optional
+        royalty_fee_address,    -- optional
+        sub_tx_trade_id,
+        row_number() over (partition by tx_hash, sub_tx_trade_id order by tx_hash) as duplicates_rank
+    FROM {{ nft_model[2] }}
+    {% if is_incremental() %}
+    WHERE block_time >= date_trunc("day", now() - interval '1 week')
+    {% endif %}
+    {% if not loop.last %}
+    UNION ALL
+    {% endif %}
+    {% endfor %}
+),
+
+enrichments as (
+SELECT
+    base.blockchain,
+    base.project,
+    base.project_version,
+    base.block_date,
+    base.block_time,
+    base.block_number,
+    base.tx_hash,
+    base.sub_tx_trade_id,
+    base.project_contract_address,
+    base.trade_category,
+    base.trade_type,
+    case when base.buyer = coalesce(agg1.contract_address,agg2.contract_address) then tx.from else base.buyer end as buyer,
+    case when base.seller = coalesce(agg1.contract_address,agg2.contract_address) then tx.from else base.seller end as seller,
+    base.nft_contract_address,
+    base.nft_token_id,
+    base.nft_amount,
+    base.price_raw,
+    base.currency_contract,
+    base.platform_fee_amount_raw,
+    base.royalty_fee_amount_raw,
+    base.platform_fee_address,
+    base.royalty_fee_address,
+    tx.from as tx_from,
+    tx.to as tx_to,
+    nft.name as nft_collection,
+    nft.standard as nft_standard,
+    coalesce(erc20.symbol,p.symbol) as currency_symbol,
+    base.price_raw/pow(10,coalesce(erc20.decimals,p.decimals,18)) as price,
+    base.platform_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18)) as platform_fee_amount,
+    base.royalty_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18)) as royalty_fee_amount,
+    base.price_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as price_usd,
+    base.platform_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as platform_fee_amount_usd,
+    base.royalty_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as royalty_fee_amount_usd,
+    coalesce(cast(100*base.platform_fee_amount_raw/base.price_raw as double),cast(0.0 as double)) as platform_fee_percentage,
+    coalesce(cast(100*base.royalty_fee_amount_raw/base.price_raw as double),cast(0.0 as double)) as royalty_fee_percentage,
+    coalesce(agg1.contract_address,agg2.contract_address) as aggregator_address,
+    {% if aggregator_markers != null %}
+    coalesce(agg_mark.aggregator_name, agg1.name, agg2.name) as aggregator_name
+    {% else %}
+    coalesce(agg1.name,agg2.name) as aggregator_name
+    {% endif %}
+FROM base_union base
+INNER JOIN {{ transactions_model }} tx
+ON tx.block_number = base.block_number
+    AND tx.hash = base.tx_hash
+    {% if is_incremental() %}
+    AND tx.block_time >= date_trunc("day", now() - interval '1 week')
+    {% endif %}
+LEFT JOIN {{ tokens_nft_model }} nft
+ON nft.contract_address = base.nft_contract_address
+LEFT JOIN {{ tokens_erc20_model }} erc20
+ON erc20.contract_address = base.currency_contract
+LEFT JOIN {{ prices_model }} p
+ON p.blockchain = base.blockchain
+    AND p.contract_address = base.currency_contract
+    AND p.minute = date_trunc('minute',base.block_time)
+    {% if is_incremental() %}
+    AND p.minute >= date_trunc("day", now() - interval '1 week')
+    {% endif %}
+LEFT JOIN {{ aggregators }} agg1
+ON (base.buyer = agg1.contract_address
+    OR base.seller = agg1.contract_address)
+LEFT JOIN {{ aggregators }} agg2
+ON agg1.contract_address is null    -- only match if agg1 produces no matches, this prevents duplicates
+    AND tx.to = agg2.contract_address
+{% if aggregator_markers != null %}
+LEFT JOIN {{ aggregator_markers }} agg_mark
+ON RIGHT(tx.data, agg_mark.hash_marker_size) = agg_mark.hash_marker
+{% endif %}
+WHERE base.duplicates_rank = 1
+)
+
+
+select * from enrichments
+{% endmacro %}

macros/models/_sector/nft/platforms/element_v1_base_trades.sql

@@ -0,0 +1,123 @@
+-- Element NFT trades (re-usable macro for all chains)
+{% macro element_v1_base_trades(erc721_sell_order_filled, erc721_buy_order_filled, erc1155_sell_order_filled, erc1155_buy_order_filled) %}
+
+
+SELECT
+  cast(DATE_TRUNC('month', evt_block_time) as date) AS block_date,
+  evt_block_time AS block_time,
+  evt_block_number AS block_number,
+  'Buy' AS trade_category,
+  'secondary' AS trade_type,
+  erc721Token AS nft_contract_address,
+  cast(erc721TokenId as uint256) AS nft_token_id,
+  uint256 '1' AS nft_amount,
+  taker AS buyer,
+  maker AS seller,
+  cast(erc20TokenAmount AS UINT256) AS price_raw,
+  CASE
+    WHEN erc20Token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0x0000000000000000000000000000000000000000
+    ELSE erc20Token
+  END AS currency_contract,
+  cast(0 AS UINT256) AS platform_fee_amount_raw,
+  cast(0 AS UINT256) AS royalty_fee_amount_raw,
+  cast(NULL AS VARBINARY) AS platform_fee_address,
+  cast(NULL AS VARBINARY) AS royalty_fee_address,
+  contract_address AS project_contract_address,
+  evt_tx_hash AS tx_hash,
+  evt_index AS sub_tx_trade_id
+FROM {{ erc721_sell_order_filled }}
+{% if is_incremental() %}
+WHERE evt_block_time >= date_trunc('day', now() - interval '7' day)
+{% endif %}
+
+UNION ALL
+
+SELECT
+  cast(date_trunc('month', evt_block_time) as date) AS block_date,
+  evt_block_time AS block_time,
+  evt_block_number AS block_number,
+  'Sell' AS trade_category,
+  'secondary' AS trade_type,
+  erc721Token AS nft_contract_address,
+  cast(erc721TokenId as uint256) AS nft_token_id,
+  uint256 '1' AS nft_amount,
+  maker AS buyer,
+  taker AS seller,
+  cast(erc20TokenAmount AS UINT256) AS price_raw,
+  CASE
+    WHEN erc20Token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0x0000000000000000000000000000000000000000
+    ELSE erc20Token
+  END AS currency_contract,
+  cast(0 AS UINT256) AS platform_fee_amount_raw,
+  cast(0 AS UINT256) AS royalty_fee_amount_raw,
+  cast(NULL AS VARBINARY) AS platform_fee_address,
+  cast(NULL AS VARBINARY) AS royalty_fee_address,
+  contract_address AS project_contract_address,
+  evt_tx_hash AS tx_hash,
+  evt_index AS sub_tx_trade_id
+FROM {{ erc721_buy_order_filled }}
+{% if is_incremental() %}
+WHERE evt_block_time >= date_trunc('day', now() - interval '7' day)
+{% endif %}
+
+UNION ALL
+
+SELECT
+  cast(date_trunc('month', evt_block_time) as date) AS block_date,
+  evt_block_time AS block_time,
+  evt_block_number AS block_number,
+  'Buy' AS trade_category,
+  'secondary' AS trade_type,
+  erc1155Token AS nft_contract_address,
+  cast(erc1155TokenId as uint256) AS nft_token_id,
+  uint256 '1' AS nft_amount,
+  taker AS buyer,
+  maker AS seller,
+  cast(erc20FillAmount AS UINT256) AS price_raw,
+  CASE
+    WHEN erc20Token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0x0000000000000000000000000000000000000000
+    ELSE erc20Token
+  END AS currency_contract,
+  cast(0 AS UINT256) AS platform_fee_amount_raw,
+  cast(0 AS UINT256) AS royalty_fee_amount_raw,
+  cast(NULL AS VARBINARY) AS platform_fee_address,
+  cast(NULL AS VARBINARY) AS royalty_fee_address,
+  contract_address AS project_contract_address,
+  evt_tx_hash AS tx_hash,
+  evt_index AS sub_tx_trade_id
+FROM {{ erc1155_buy_order_filled }}
+{% if is_incremental() %}
+WHERE evt_block_time >= date_trunc('day', now() - interval '7' day)
+{% endif %}
+
+UNION ALL
+
+SELECT
+  cast(date_trunc('month', evt_block_time) as date) AS block_date,
+  evt_block_time AS block_time,
+  evt_block_number AS block_number,
+  'Buy' AS trade_category,
+  'secondary' AS trade_type,
+  erc1155Token AS nft_contract_address,
+  cast(erc1155TokenId as uint256) AS nft_token_id,
+  uint256 '1' AS nft_amount,
+  maker AS buyer,
+  taker AS seller,
+  cast(erc20FillAmount AS UINT256) AS price_raw,
+  CASE
+    WHEN erc20Token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0x0000000000000000000000000000000000000000
+    ELSE erc20Token
+  END AS currency_contract,
+  cast(0 AS UINT256) AS platform_fee_amount_raw,
+  cast(0 AS UINT256) AS royalty_fee_amount_raw,
+  cast(NULL AS VARBINARY) AS platform_fee_address,
+  cast(NULL AS VARBINARY) AS royalty_fee_address,
+  contract_address AS project_contract_address,
+  evt_tx_hash AS tx_hash,
+  evt_index AS sub_tx_trade_id
+FROM {{ erc1155_sell_order_filled }}
+{% if is_incremental() %}
+WHERE evt_block_time >= date_trunc('day', now() - interval '7' day)
+{% endif %}
+
+{% endmacro %}

macros/models/_sector/nft/element_v1_base_trades.sql → macros/models/_sector/nft/platforms/element_v1_base_trades_legacy.sql

@@ -1,5 +1,5 @@
 -- Element NFT trades (re-usable macro for all chains)
-{% macro element_v1_base_trades(erc721_sell_order_filled, erc721_buy_order_filled, erc1155_sell_order_filled, erc1155_buy_order_filled) %}
+{% macro element_v1_base_trades_legacy(erc721_sell_order_filled, erc721_buy_order_filled, erc1155_sell_order_filled, erc1155_buy_order_filled) %}
 
 
 SELECT

macros/models/_sector/nft/port_to_old_schema.sql

@@ -11,13 +11,13 @@ SELECT
     price_usd as amount_usd,
     nft_standard as token_standard,
     trade_type,
-    CAST(nft_amount as decimal(38)) as number_of_items,
+    CAST(nft_amount as uint256) as number_of_items,
     trade_category,
     'Trade' as evt_type,
     seller,
     buyer,
     CAST(price as double) as amount_original,
-    CAST(price_raw as decimal(38)) as amount_raw,
+    CAST(price_raw as uint256) as amount_raw,
     currency_symbol,
     currency_contract,
     nft_contract_address,
@@ -28,17 +28,17 @@ SELECT
     block_number,
     tx_from,
     tx_to,
-    CAST(platform_fee_amount_raw as decimal(38)) as platform_fee_amount_raw,
+    CAST(platform_fee_amount_raw as uint256) as platform_fee_amount_raw,
     CAST(platform_fee_amount as double) as platform_fee_amount,
     platform_fee_amount_usd,
     platform_fee_percentage,
     royalty_fee_address as royalty_fee_receive_address,
     currency_symbol as royalty_fee_currency_symbol,
-    CAST(royalty_fee_amount_raw as decimal(38)) as royalty_fee_amount_raw,
+    CAST(royalty_fee_amount_raw as uint256) as royalty_fee_amount_raw,
     CAST(royalty_fee_amount as double) as royalty_fee_amount,
     royalty_fee_amount_usd,
     royalty_fee_percentage,
-    concat(project,'-',cast(block_number as varchar(20)),'-',cast(tx_hash as varchar(66)),'-',cast(sub_tx_trade_id as varchar(20))) as unique_trade_id
+    concat(project,'-',cast(block_number as varchar),'-',cast(tx_hash as varchar),'-',cast(sub_tx_trade_id as varchar)) as unique_trade_id
 FROM {{ model }}
 
 {% endmacro %}