From 958fd09c23ac1d8b6c7f0085d88349403c5dc08f Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Fri, 7 Jul 2023 16:00:49 +0200 Subject: [PATCH 01/22] migrate all NFT base_trades beta models --- .../_sector/nft/element_v1_base_trades.sql | 115 ------------ .../nft/platforms/element_v1_base_trades.sql | 123 +++++++++++++ .../archipelago_ethereum_base_trades.sql | 35 ++-- .../platforms/blur_ethereum_base_trades.sql | 65 +++---- .../collectionswap_ethereum_base_trades.sql | 78 ++++---- .../cryptopunks_ethereum_base_trades.sql | 23 +-- .../element_ethereum_base_trades.sql | 3 +- .../looksrare_v1_ethereum_base_trades.sql | 31 ++-- .../looksrare_v2_ethereum_base_trades.sql | 51 +++--- .../sudoswap_ethereum_base_trades.sql | 69 +++---- .../superrare_ethereum_base_trades.sql | 172 +++++++++--------- .../platforms/x2y2_ethereum_base_trades.sql | 63 +++---- .../zora_v1_ethereum_base_trades.sql | 35 ++-- .../zora_v2_ethereum_base_trades.sql | 17 +- .../zora_v3_ethereum_base_trades.sql | 57 +++--- .../nft/trades/nft_ethereum_trades_beta.sql | 26 +-- .../sudoswap_ethereum_pool_trades_agg_day.sql | 2 +- 17 files changed, 498 insertions(+), 467 deletions(-) delete mode 100644 macros/models/_sector/nft/element_v1_base_trades.sql create mode 100644 macros/models/_sector/nft/platforms/element_v1_base_trades.sql diff --git a/macros/models/_sector/nft/element_v1_base_trades.sql b/macros/models/_sector/nft/element_v1_base_trades.sql deleted file mode 100644 index dfd960abd3c..00000000000 --- a/macros/models/_sector/nft/element_v1_base_trades.sql +++ /dev/null @@ -1,115 +0,0 @@ --- Element NFT trades (re-usable macro for all chains) -{% macro element_v1_base_trades(erc721_sell_order_filled, erc721_buy_order_filled, erc1155_sell_order_filled, erc1155_buy_order_filled) %} - - -SELECT - date_trunc('day',evt_block_time) as block_date -, evt_block_time AS block_time -, evt_block_number AS block_number -, 'Buy' AS trade_category -, 'secondary' AS trade_type -, erc721Token AS nft_contract_address -, erc721TokenId AS nft_token_id -, 1 AS nft_amount -, taker AS buyer -, maker AS seller -, cast(erc20TokenAmount as decimal(38)) AS price_raw -, CASE WHEN erc20Token='0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' THEN '{{ var("ETH_ERC20_ADDRESS") }}' - ELSE erc20Token END AS currency_contract -, cast(0 as decimal(38)) as platform_fee_amount_raw -, cast(0 as decimal(38)) as royalty_fee_amount_raw -, cast(null as varchar(1)) as platform_fee_address -, cast(null as varchar(1)) as royalty_fee_address -, contract_address AS project_contract_address -, evt_tx_hash AS tx_hash -, evt_index as sub_tx_trade_id -FROM {{ erc721_sell_order_filled }} -{% if is_incremental() %} -WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') -{% endif %} - -UNION ALL - -SELECT - date_trunc('day',evt_block_time) as block_date -, evt_block_time AS block_time -, evt_block_number AS block_number -, 'Sell' AS trade_category -, 'secondary' AS trade_type -, erc721Token AS nft_contract_address -, erc721TokenId AS nft_token_id -, 1 AS nft_amount -, maker AS buyer -, taker AS seller -, cast(erc20TokenAmount as decimal(38)) AS price_raw -, CASE WHEN erc20Token='0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' THEN '{{ var("ETH_ERC20_ADDRESS") }}' - ELSE erc20Token END AS currency_contract -, cast(0 as decimal(38)) as platform_fee_amount_raw -, cast(0 as decimal(38)) as royalty_fee_amount_raw -, cast(null as varchar(1)) as platform_fee_address -, cast(null as varchar(1)) as royalty_fee_address -, contract_address AS project_contract_address -, evt_tx_hash AS tx_hash -, evt_index as sub_tx_trade_id -FROM {{ erc721_buy_order_filled }} -{% if is_incremental() %} -WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') -{% endif %} - -UNION ALL - -SELECT - date_trunc('day',evt_block_time) as block_date -, evt_block_time AS block_time -, evt_block_number AS block_number -, 'Buy' AS trade_category -, 'secondary' AS trade_type -, erc1155Token AS nft_contract_address -, erc1155TokenId AS nft_token_id -, 1 AS nft_amount -, taker AS buyer -, maker AS seller -, cast(erc20FillAmount as decimal(38)) AS price_raw -, CASE WHEN erc20Token='0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' THEN '{{ var("ETH_ERC20_ADDRESS") }}' - ELSE erc20Token END AS currency_contract -, cast(0 as decimal(38)) as platform_fee_amount_raw -, cast(0 as decimal(38)) as royalty_fee_amount_raw -, cast(null as varchar(1)) as platform_fee_address -, cast(null as varchar(1)) as royalty_fee_address -, contract_address AS project_contract_address -, evt_tx_hash AS tx_hash -, evt_index as sub_tx_trade_id -FROM {{ erc1155_buy_order_filled }} -{% if is_incremental() %} -WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') -{% endif %} - -UNION ALL - -SELECT - date_trunc('day',evt_block_time) as block_date -, evt_block_time AS block_time -, evt_block_number AS block_number -, 'Buy' AS trade_category -, 'secondary' AS trade_type -, erc1155Token AS nft_contract_address -, erc1155TokenId AS nft_token_id -, 1 AS nft_amount -, maker AS buyer -, taker AS seller -, cast(erc20FillAmount as decimal(38)) AS price_raw -, CASE WHEN erc20Token='0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' THEN '{{ var("ETH_ERC20_ADDRESS") }}' - ELSE erc20Token END AS currency_contract -, cast(0 as decimal(38)) as platform_fee_amount_raw -, cast(0 as decimal(38)) as royalty_fee_amount_raw -, cast(null as varchar(1)) as platform_fee_address -, cast(null as varchar(1)) as royalty_fee_address -, contract_address AS project_contract_address -, evt_tx_hash AS tx_hash -, evt_index as sub_tx_trade_id -FROM {{ erc1155_sell_order_filled }} -{% if is_incremental() %} -WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') -{% endif %} - -{% endmacro %} diff --git a/macros/models/_sector/nft/platforms/element_v1_base_trades.sql b/macros/models/_sector/nft/platforms/element_v1_base_trades.sql new file mode 100644 index 00000000000..34eb0598b57 --- /dev/null +++ b/macros/models/_sector/nft/platforms/element_v1_base_trades.sql @@ -0,0 +1,123 @@ +-- Element NFT trades (re-usable macro for all chains) +{% macro element_v1_base_trades(erc721_sell_order_filled, erc721_buy_order_filled, erc1155_sell_order_filled, erc1155_buy_order_filled) %} + + +SELECT + DATE_TRUNC('day', evt_block_time) AS block_date, + evt_block_time AS block_time, + evt_block_number AS block_number, + 'Buy' AS trade_category, + 'secondary' AS trade_type, + erc721Token AS nft_contract_address, + cast(erc721TokenId as uint256) AS nft_token_id, + uint256 '1' AS nft_amount, + taker AS buyer, + maker AS seller, + cast(erc20TokenAmount AS UINT256) AS price_raw, + CASE + WHEN erc20Token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0x0000000000000000000000000000000000000000 + ELSE erc20Token + END AS currency_contract, + cast(0 AS UINT256) AS platform_fee_amount_raw, + cast(0 AS UINT256) AS royalty_fee_amount_raw, + cast(NULL AS VARBINARY) AS platform_fee_address, + cast(NULL AS VARBINARY) AS royalty_fee_address, + contract_address AS project_contract_address, + evt_tx_hash AS tx_hash, + evt_index AS sub_tx_trade_id +FROM {{ erc721_sell_order_filled }} +{% if is_incremental() %} +WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) +{% endif %} + +UNION ALL + +SELECT + DATE_TRUNC('day', evt_block_time) AS block_date, + evt_block_time AS block_time, + evt_block_number AS block_number, + 'Sell' AS trade_category, + 'secondary' AS trade_type, + erc721Token AS nft_contract_address, + cast(erc721TokenId as uint256) AS nft_token_id, + uint256 '1' AS nft_amount, + maker AS buyer, + taker AS seller, + cast(erc20TokenAmount AS UINT256) AS price_raw, + CASE + WHEN erc20Token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0x0000000000000000000000000000000000000000 + ELSE erc20Token + END AS currency_contract, + cast(0 AS UINT256) AS platform_fee_amount_raw, + cast(0 AS UINT256) AS royalty_fee_amount_raw, + cast(NULL AS VARBINARY) AS platform_fee_address, + cast(NULL AS VARBINARY) AS royalty_fee_address, + contract_address AS project_contract_address, + evt_tx_hash AS tx_hash, + evt_index AS sub_tx_trade_id +FROM {{ erc721_buy_order_filled }} +{% if is_incremental() %} +WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) +{% endif %} + +UNION ALL + +SELECT + DATE_TRUNC('day', evt_block_time) AS block_date, + evt_block_time AS block_time, + evt_block_number AS block_number, + 'Buy' AS trade_category, + 'secondary' AS trade_type, + erc1155Token AS nft_contract_address, + cast(erc1155TokenId as uint256) AS nft_token_id, + uint256 '1' AS nft_amount, + taker AS buyer, + maker AS seller, + cast(erc20FillAmount AS UINT256) AS price_raw, + CASE + WHEN erc20Token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0x0000000000000000000000000000000000000000 + ELSE erc20Token + END AS currency_contract, + cast(0 AS UINT256) AS platform_fee_amount_raw, + cast(0 AS UINT256) AS royalty_fee_amount_raw, + cast(NULL AS VARBINARY) AS platform_fee_address, + cast(NULL AS VARBINARY) AS royalty_fee_address, + contract_address AS project_contract_address, + evt_tx_hash AS tx_hash, + evt_index AS sub_tx_trade_id +FROM {{ erc1155_buy_order_filled }} +{% if is_incremental() %} +WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) +{% endif %} + +UNION ALL + +SELECT + DATE_TRUNC('day', evt_block_time) AS block_date, + evt_block_time AS block_time, + evt_block_number AS block_number, + 'Buy' AS trade_category, + 'secondary' AS trade_type, + erc1155Token AS nft_contract_address, + cast(erc1155TokenId as uint256) AS nft_token_id, + uint256 '1' AS nft_amount, + maker AS buyer, + taker AS seller, + cast(erc20FillAmount AS UINT256) AS price_raw, + CASE + WHEN erc20Token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0x0000000000000000000000000000000000000000 + ELSE erc20Token + END AS currency_contract, + cast(0 AS UINT256) AS platform_fee_amount_raw, + cast(0 AS UINT256) AS royalty_fee_amount_raw, + cast(NULL AS VARBINARY) AS platform_fee_address, + cast(NULL AS VARBINARY) AS royalty_fee_address, + contract_address AS project_contract_address, + evt_tx_hash AS tx_hash, + evt_index AS sub_tx_trade_id +FROM {{ erc1155_sell_order_filled }} +{% if is_incremental() %} +WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) +{% endif %} + +{% endmacro %} diff --git a/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql index e2f33024c78..627831b3851 100644 --- a/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'archipelago_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -8,23 +9,23 @@ unique_key = ['block_number','tx_hash','sub_tx_trade_id'], ) }} -{% set project_start_date='2022-6-20' %} +{% set project_start_date = "cast('2022-6-20' as timestamp)" %} WITH trade_events as ( SELECT * FROM {{ source('archipelago_ethereum','ArchipelagoMarket_evt_Trade') }} {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - WHERE evt_block_time >= '{{project_start_date}}' + WHERE evt_block_time >= {{project_start_date}} {% endif %} ), token_events as ( SELECT * FROM {{ source('archipelago_ethereum','ArchipelagoMarket_evt_TokenTrade') }} {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - WHERE evt_block_time >= '{{project_start_date}}' + WHERE evt_block_time >= {{project_start_date}} {% endif %} ), fee_events as ( @@ -32,16 +33,16 @@ fee_events as ( evt_block_number , tradeId , sum(amount) filter ( - where recipient not in ('0xa76456bb6abc50fb38e17c042026bc27a95c3314','0x1fc12c9f68a6b0633ba5897a40a8e61ed9274dc9') + where recipient not in (0xa76456bb6abc50fb38e17c042026bc27a95c3314,0x1fc12c9f68a6b0633ba5897a40a8e61ed9274dc9) ) as royalty_amount , sum(amount) filter ( - where recipient in ('0xa76456bb6abc50fb38e17c042026bc27a95c3314','0x1fc12c9f68a6b0633ba5897a40a8e61ed9274dc9') + where recipient in (0xa76456bb6abc50fb38e17c042026bc27a95c3314,0x1fc12c9f68a6b0633ba5897a40a8e61ed9274dc9) ) as platform_amount FROM {{ source('archipelago_ethereum','ArchipelagoMarket_evt_RoyaltyPayment') }} {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - WHERE evt_block_time >= '{{project_start_date}}' + WHERE evt_block_time >= {{project_start_date}} {% endif %} GROUP BY evt_block_number, tradeId ) @@ -52,19 +53,19 @@ SELECT ,trade.evt_block_number as block_number ,trade.evt_tx_hash as tx_hash ,trade.contract_address as project_contract_address - ,CAST(null as varchar(5)) as trade_category + ,CAST(null as varchar) as trade_category ,'secondary' as trade_type ,trade.buyer ,trade.seller ,tok.tokenAddress as nft_contract_address ,tok.tokenId as nft_token_id - ,1 as nft_amount + ,uint256 '1' as nft_amount ,trade.currency as currency_contract - ,cast(trade.cost as decimal(38)) as price_raw - ,cast(coalesce(fee.platform_amount,0) as decimal(38)) as platform_fee_amount_raw - ,cast(coalesce(fee.royalty_amount,0) as decimal(38)) as royalty_fee_amount_raw - ,CAST(null as varchar(5)) as platform_fee_address - ,CAST(null as varchar(5)) as royalty_fee_address + ,cast(trade.cost as uint256) as price_raw + ,cast(coalesce(fee.platform_amount,cast(0 as uint256)) as uint256) as platform_fee_amount_raw + ,cast(coalesce(fee.royalty_amount,cast(0 as uint256)) as uint256) as royalty_fee_amount_raw + ,CAST(null as varbinary) as platform_fee_address + ,CAST(null as varbinary) as royalty_fee_address ,trade.evt_index as sub_tx_trade_id FROM trade_events trade INNER JOIN token_events tok diff --git a/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql index 8f0f4252ed1..d478cb887bf 100644 --- a/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'blur_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -10,8 +11,8 @@ }} -{% set project_start_date = '2022-10-18' %} -{% set seaport_usage_start_date = '2023-01-25' %} +{% set project_start_date = "cast('2022-10-18' as timestamp)" %} +{% set seaport_usage_start_date = "cast('2023-01-25' as timestamp)" %} WITH @@ -20,30 +21,30 @@ regular_blur_sales as ( date_trunc('day', bm.evt_block_time) AS block_date , bm.evt_block_time AS block_time , bm.evt_block_number AS block_number - , get_json_object(bm.buy, '$.collection') AS nft_contract_address - , get_json_object(bm.sell, '$.tokenId') AS nft_token_id - , CAST(get_json_object(bm.buy, '$.amount') AS INT) AS nft_amount - , get_json_object(bm.sell, '$.trader') AS seller - , get_json_object(bm.buy, '$.trader') AS buyer - , CASE WHEN get_json_object(bm.buy, '$.matchingPolicy') IN ('0x00000000006411739da1c40b106f8511de5d1fac', '0x0000000000dab4a563819e8fd93dba3b25bc3495') THEN 'Buy' - WHEN get_json_object(bm.buy, '$.matchingPolicy') IN ('0x0000000000b92d5d043faf7cecf7e2ee6aaed232') THEN 'Sell' - ELSE CAST(null as VARCHAR(1)) + , from_hex(JSON_EXTRACT_SCALAR(bm.buy, '$.collection')) AS nft_contract_address + , cast(JSON_EXTRACT_SCALAR(bm.sell, '$.tokenId') as uint256) AS nft_token_id + , CAST(JSON_EXTRACT_SCALAR(bm.buy, '$.amount') AS uint256) AS nft_amount + , from_hex(JSON_EXTRACT_SCALAR(bm.sell, '$.trader')) AS seller + , from_hex(JSON_EXTRACT_SCALAR(bm.buy, '$.trader')) AS buyer + , CASE WHEN from_hex(JSON_EXTRACT_SCALAR(bm.buy, '$.matchingPolicy')) IN (0x00000000006411739da1c40b106f8511de5d1fac, 0x0000000000dab4a563819e8fd93dba3b25bc3495) THEN 'Buy' + WHEN from_hex(JSON_EXTRACT_SCALAR(bm.buy, '$.matchingPolicy')) IN (0x0000000000b92d5d043faf7cecf7e2ee6aaed232) THEN 'Sell' + ELSE CAST(null as VARCHAR) END AS trade_category , 'secondary' AS trade_type - , CAST(get_json_object(bm.buy, '$.price') AS DECIMAL(38,0)) AS price_raw - , get_json_object(bm.buy, '$.paymentToken') AS currency_contract + , CAST(JSON_EXTRACT_SCALAR(bm.buy, '$.price') AS uint256) AS price_raw + , from_hex(JSON_EXTRACT_SCALAR(bm.buy, '$.paymentToken')) AS currency_contract , bm.contract_address AS project_contract_address , bm.evt_tx_hash AS tx_hash - , 0 AS platform_fee_amount_raw -- Hardcoded 0% platform fee - , CAST(COALESCE(get_json_object(bm.buy, '$.price')*get_json_object(get_json_object(bm.sell, '$.fees[0]'), '$.rate')/10000, 0) AS DECIMAL(38,0)) AS royalty_fee_amount_raw - , get_json_object(bm.sell, '$.fees[0].recipient') AS royalty_fee_address - , cast(NULL as varchar(1)) as platform_fee_address + , cast(0 as uint256) AS platform_fee_amount_raw -- Hardcoded 0% platform fee + , CAST(COALESCE(cast(JSON_EXTRACT_SCALAR(bm.buy, '$.price') as uint256)*cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(bm.sell, '$.fees[1]'), '$.rate') as uint256)/10000, cast(0 as uint256)) AS uint256) AS royalty_fee_amount_raw + , from_hex(JSON_EXTRACT_SCALAR(bm.sell, '$.fees[1].recipient')) AS royalty_fee_address + , cast(NULL as varbinary) as platform_fee_address , bm.evt_index as sub_tx_trade_id FROM {{ source('blur_ethereum','BlurExchange_evt_OrdersMatched') }} bm {% if is_incremental() %} - WHERE bm.evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE bm.evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - WHERE bm.evt_block_time >= '{{ project_start_date }}' + WHERE bm.evt_block_time >= {{project_start_date}} {% endif %} ) @@ -52,30 +53,30 @@ SELECT date_trunc('day', s.evt_block_time) AS block_date , s.evt_block_time AS block_time , s.evt_block_number AS block_number - , get_json_object(s.offer[0], '$.token') AS nft_contract_address - , get_json_object(s.offer[0], '$.identifier') AS nft_token_id - , CAST(get_json_object(s.offer[0], '$.amount') AS DECIMAL(38,0)) AS nft_amount + , from_hex(JSON_EXTRACT_SCALAR(s.offer[1], '$.token')) AS nft_contract_address + , cast(JSON_EXTRACT_SCALAR(s.offer[1], '$.identifier') as uint256) AS nft_token_id + , CAST(JSON_EXTRACT_SCALAR(s.offer[1], '$.amount') AS uint256) AS nft_amount , s.offerer AS seller , s.recipient AS buyer , 'Buy' AS trade_category , 'secondary' AS trade_type - , CAST(get_json_object(s.consideration[0], '$.amount')+get_json_object(s.consideration[1], '$.amount') AS DECIMAL(38,0)) AS price_raw - , get_json_object(s.consideration[0], '$.token') AS currency_contract + , CAST(JSON_EXTRACT_SCALAR(element_at(s.consideration,1), '$.amount') as uint256)+cast(JSON_EXTRACT_SCALAR(element_at(s.consideration,2), '$.amount') AS uint256) AS price_raw + , from_hex(JSON_EXTRACT_SCALAR(element_at(s.consideration,1), '$.token')) AS currency_contract , s.contract_address AS project_contract_address , s.evt_tx_hash AS tx_hash - , CAST(0 AS DECIMAL(38,0)) AS platform_fee_amount_raw -- Hardcoded 0% platform fee - , LEAST(CAST(get_json_object(s.consideration[0], '$.amount') AS DECIMAL(38,0)), CAST(get_json_object(s.consideration[1], '$.amount') AS DECIMAL(38,0))) AS royalty_fee_amount_raw - , CASE WHEN get_json_object(s.consideration[0], '$.recipient')!=s.recipient THEN get_json_object(s.consideration[0], '$.recipient') - ELSE get_json_object(s.consideration[1], '$.recipient') + , CAST(0 AS uint256) AS platform_fee_amount_raw -- Hardcoded 0% platform fee + , LEAST(CAST(JSON_EXTRACT_SCALAR(element_at(s.consideration,1), '$.amount') AS uint256), CAST(JSON_EXTRACT_SCALAR(element_at(s.consideration,2), '$.amount') AS uint256)) AS royalty_fee_amount_raw + , CASE WHEN from_hex(JSON_EXTRACT_SCALAR(element_at(s.consideration,1), '$.recipient'))!=s.recipient THEN from_hex(JSON_EXTRACT_SCALAR(element_at(s.consideration,1), '$.recipient')) + ELSE from_hex(JSON_EXTRACT_SCALAR(element_at(s.consideration,2), '$.recipient')) END AS royalty_fee_address - , cast(NULL as varchar(1)) as platform_fee_address + , cast(NULL as varbinary) as platform_fee_address , s.evt_index as sub_tx_trade_id FROM {{ source('seaport_ethereum','Seaport_evt_OrderFulfilled') }} s -WHERE s.zone='0x0000000000d80cfcb8dfcd8b2c4fd9c813482938' +WHERE s.zone=0x0000000000d80cfcb8dfcd8b2c4fd9c813482938 {% if is_incremental() %} - AND s.evt_block_time >= date_trunc("day", now() - interval '1 week') + AND s.evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - AND s.evt_block_time >= '{{seaport_usage_start_date}}' + AND s.evt_block_time >= {{seaport_usage_start_date}} {% endif %} ) diff --git a/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades.sql index b3abd5986fe..efe49bca799 100644 --- a/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'collectionswap_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -9,7 +10,7 @@ ) }} -{%- set project_start_date = '2023-03-29' %} +{%- set project_start_date = "TIMESTAMP '2023-03-29'" %} WITH raw_trades as ( @@ -18,8 +19,7 @@ raw_trades as ( from( select block_number, block_time, evt_index, tx_hash, buyer, seller, - posexplode(nft_id_array) as (sub_order_id, nft_token_id), - cast(1 as DECIMAL(38,0)) as nft_amount, + cast(1 as uint256) as nft_amount, price_raw/number_of_items as price_raw, platform_fee_amount_raw/number_of_items as platform_fee_amount_raw, royalty_fee_amount_raw/number_of_items as royalty_fee_amount_raw, @@ -28,54 +28,62 @@ raw_trades as ( project_contract_address, number_of_items, 'secondary' as trade_type, - trade_category + trade_category, + nft_token_id, + sub_order_id from( select evt_block_number as block_number ,evt_block_time as block_time ,evt_index ,evt_tx_hash as tx_hash - ,null as buyer + ,cast(null as varbinary) as buyer ,contract_address as seller ,'Buy' as trade_category ,nftIds as nft_id_array - ,cardinality(nftIds) as number_of_items - ,cast(outputAmount as decimal(38)) as price_raw - ,cast(protocolFee as decimal(38)) as platform_fee_amount_raw - ,get_json_object(royaltyDue[0], '$.amount') as royalty_fee_amount_raw - ,get_json_object(royaltyDue[0], '$.recipient') as royalty_fee_address - ,cast(tradeFee as decimal(38)) as trade_fee_amount_raw + ,cast(cardinality(nftIds) as uint256) as number_of_items + ,cast(outputAmount as uint256) as price_raw + ,cast(protocolFee as uint256) as platform_fee_amount_raw + ,cast(JSON_EXTRACT_SCALAR(royaltyDue[1], '$.amount') as uint256) as royalty_fee_amount_raw + ,from_hex(JSON_EXTRACT_SCALAR(royaltyDue[1], '$.recipient')) as royalty_fee_address + ,cast(tradeFee as uint256) as trade_fee_amount_raw ,contract_address as project_contract_address from {{ source('collectionswap_ethereum','CollectionPool_evt_SwapNFTOutPool') }} e {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - WHERE evt_block_time >= '{{project_start_date}}' + WHERE evt_block_time >= {{project_start_date}} {% endif %} union all select - evt_block_number as block_number - ,evt_block_time as block_time - ,evt_index - ,evt_tx_hash as tx_hash - ,contract_address as buyer - ,null as seller - ,'Sell' as trade_category - ,nftIds as nft_id_array - ,cardinality(nftIds) as number_of_items - ,cast(inputAmount + protocolFee + cast(get_json_object(royaltyDue[0], '$.amount') as decimal(38)) as decimal(38)) as price_raw - ,cast(protocolFee as decimal(38)) as platform_fee_amount_raw - ,get_json_object(royaltyDue[0], '$.amount') as royalty_fee_amount_raw - ,get_json_object(royaltyDue[0], '$.recipient') as royalty_fee_address - ,cast(tradeFee as decimal(38)) as trade_fee_amount_raw - ,contract_address as project_contract_address + evt_block_number AS block_number, + evt_block_time AS block_time, + evt_index, + evt_tx_hash AS tx_hash, + contract_address AS buyer, + cast(null as varbinary) AS seller, + 'Sell' AS trade_category, + nftIds AS nft_id_array, + cast(cardinality(nftIds) as uint256) AS number_of_items, + TRY_CAST( + inputAmount + protocolFee + TRY_CAST( + JSON_EXTRACT_SCALAR(royaltyDue[1], '$.amount') AS UINT256 + ) AS UINT256 + ) AS price_raw, + TRY_CAST(protocolFee AS UINT256) AS platform_fee_amount_raw, + TRY_CAST(JSON_EXTRACT_SCALAR(royaltyDue[1], '$.amount') AS UINT256) AS royalty_fee_amount_raw, + from_hex(JSON_EXTRACT_SCALAR(royaltyDue[1], '$.recipient')) AS royalty_fee_address, + TRY_CAST(tradeFee AS UINT256) AS trade_fee_amount_raw, + contract_address AS project_contract_address from {{ source('collectionswap_ethereum','CollectionPool_evt_SwapNFTInPool') }} e {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - WHERE evt_block_time >= '{{project_start_date}}' + WHERE evt_block_time >= {{project_start_date}} {% endif %} ) + CROSS JOIN UNNEST(nft_id_array) + WITH ORDINALITY AS foo(nft_token_id, sub_order_id) ) ), @@ -104,10 +112,10 @@ SELECT , trade_type , trade_category , currency_contract -, cast(price_raw as decimal(38)) as price_raw -, cast(platform_fee_amount_raw as decimal(38)) as platform_fee_amount_raw -, cast(royalty_fee_amount_raw as decimal(38)) as royalty_fee_amount_raw -, cast(null as varchar(1)) as platform_fee_address +, cast(price_raw as uint256) as price_raw +, cast(platform_fee_amount_raw as uint256) as platform_fee_amount_raw +, cast(royalty_fee_amount_raw as uint256) as royalty_fee_amount_raw +, cast(null as varbinary) as platform_fee_address , royalty_fee_address , sub_tx_trade_id FROM base_trades diff --git a/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades.sql index fef1beb1847..15de7c34872 100644 --- a/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'cryptopunks_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -24,7 +25,7 @@ with accepted_bid_prices as ( and call.punkIndex = bid.punkIndex WHERE call_success {% if is_incremental() %} - AND call.call_block_time >= date_trunc("day", now() - interval '1 week') + AND call.call_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} group by 1,2,3 ) @@ -42,21 +43,21 @@ select date_trunc('day',evt.evt_block_time) as block_date , coalesce(call.latest_bidder,evt.toAddress) as buyer , evt.fromAddress as seller , evt.punkIndex as nft_token_id - , 1 as nft_amount + , uint256 '1' as nft_amount , evt.contract_address as nft_contract_address - , cast(coalesce(call.latest_bid, evt.value) as DECIMAL(38)) as price_raw - , '{{ var("ETH_ERC20_ADDRESS") }}' AS currency_contract -- all trades are in ETH - , cast(0 as DECIMAL(38)) as platform_fee_amount_raw - , cast(0 as DECIMAL(38)) as royalty_fee_amount_raw - , cast(null as VARCHAR(1)) as platform_fee_address - , cast(null as VARCHAR(1)) as royalty_fee_address + , cast(coalesce(call.latest_bid, evt.value) as UINT256) as price_raw + , {{ var("ETH_ERC20_ADDRESS") }} AS currency_contract -- all trades are in ETH + , cast(0 as UINT256) as platform_fee_amount_raw + , cast(0 as UINT256) as royalty_fee_amount_raw + , cast(null as varbinary) as platform_fee_address + , cast(null as varbinary) as royalty_fee_address from {{ source('cryptopunks_ethereum','CryptoPunksMarket_evt_PunkBought') }} evt left join accepted_bid_prices call on evt.evt_block_number = call.call_block_number and evt.evt_tx_hash = call.call_tx_hash and evt.punkIndex = call.punkIndex -where evt.evt_tx_hash not in ('0x92488a00dfa0746c300c66a716e6cc11ba9c0f9d40d8c58e792cc7fcebf432d0' -- flash loan https://twitter.com/cryptopunksnfts/status/1453903818308083720 +where evt.evt_tx_hash not in (0x92488a00dfa0746c300c66a716e6cc11ba9c0f9d40d8c58e792cc7fcebf432d0 -- flash loan https://twitter.com/cryptopunksnfts/status/1453903818308083720 ) {% if is_incremental() %} -and evt.evt_block_time >= date_trunc("day", now() - interval '1 week') +and evt.evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} diff --git a/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades.sql index 8e850f3e95b..a828c0aecb0 100644 --- a/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'element_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', diff --git a/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades.sql index cb2c50a31c0..375407dba12 100644 --- a/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'looksrare_v1_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -16,7 +17,7 @@ WITH looksrare_trades AS ( SELECT ta.evt_block_time AS block_time , ta.tokenId AS nft_token_id , ta.amount AS nft_amount - , CASE WHEN ta.strategy='0x58d83536d3efedb9f7f2a1ec3bdaad2b1a4dd98c' THEN 'Private Sale' ELSE 'Buy' END AS trade_category + , CASE WHEN ta.strategy=0x58d83536d3efedb9f7f2a1ec3bdaad2b1a4dd98c THEN 'Private Sale' ELSE 'Buy' END AS trade_category , ta.maker AS seller , ta.taker AS buyer , ta.price AS price_raw @@ -29,7 +30,7 @@ WITH looksrare_trades AS ( , ta.strategy FROM {{ source('looksrare_ethereum','LooksRareExchange_evt_TakerAsk') }} ta {% if is_incremental() %} - WHERE ta.evt_block_time >= date_trunc("day", NOW() - interval '1 week') + WHERE ta.evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} UNION ALL @@ -37,7 +38,7 @@ WITH looksrare_trades AS ( SELECT tb.evt_block_time AS block_time , tb.tokenId AS nft_token_id , tb.amount AS nft_amount - , CASE WHEN tb.strategy='0x58d83536d3efedb9f7f2a1ec3bdaad2b1a4dd98c' THEN 'Private Sale' ELSE 'Offer Accepted' END AS trade_category + , CASE WHEN tb.strategy=0x58d83536d3efedb9f7f2a1ec3bdaad2b1a4dd98c THEN 'Private Sale' ELSE 'Offer Accepted' END AS trade_category , tb.maker AS seller , tb.taker AS buyer , tb.price AS price_raw @@ -50,7 +51,7 @@ WITH looksrare_trades AS ( , tb.strategy FROM {{ source('looksrare_ethereum','LooksRareExchange_evt_TakerBid') }} tb {% if is_incremental() %} - WHERE tb.evt_block_time >= date_trunc("day", NOW() - interval '1 week') + WHERE tb.evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} ) ) @@ -66,29 +67,29 @@ WITH looksrare_trades AS ( , ROW_NUMBER() OVER (PARTITION BY evt_tx_hash, collection, tokenId ORDER BY evt_index ASC) AS id FROM {{ source('looksrare_ethereum','LooksRareExchange_evt_RoyaltyPayment') }} {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", NOW() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} ) , platform_fees AS ( SELECT distinct contract_address - , output_0/100 AS fee_percentage + , cast(output_0 as double)/100 AS fee_percentage FROM {{ source('looksrare_ethereum','StrategyStandardSaleForFixedPrice_call_viewProtocolFee') }} UNION ALL SELECT distinct contract_address - , output_0/100 AS fee_percentage + , cast(output_0 as double)/100 AS fee_percentage FROM {{ source('looksrare_ethereum','StrategyAnyItemFromCollectionForFixedPrice_call_viewProtocolFee') }} UNION ALL SELECT distinct contract_address - , output_0/100 AS fee_percentage + , cast(output_0 as double)/100 AS fee_percentage FROM {{ source('looksrare_ethereum','StrategyPrivateSale_call_viewProtocolFee') }} UNION ALL SELECT distinct contract_address - , output_0/100 AS fee_percentage + , cast(output_0 as double)/100 AS fee_percentage FROM {{ source('looksrare_ethereum','StrategyStandardSaleForFixedPriceV1B_call_viewProtocolFee') }} UNION ALL SELECT distinct contract_address - , output_0/100 AS fee_percentage + , cast(output_0 as double)/100 AS fee_percentage FROM {{ source('looksrare_ethereum','StrategyAnyItemFromCollectionForFixedPriceV1B_call_viewProtocolFee') }} ) @@ -107,10 +108,10 @@ SELECT , lr.buyer , lr.seller , lr.currency_contract -, CAST(lr.price_raw AS DECIMAL(38,0)) AS price_raw -, CAST(COALESCE((pf.fee_percentage/100)*lr.price_raw, 0) as DECIMAL(38,0)) AS platform_fee_amount_raw -, CAST(COALESCE(roy.amount, 0) AS DECIMAL(38,0)) AS royalty_fee_amount_raw -, cast(null as varchar(1)) as platform_fee_address +, lr.price_raw +, CAST(COALESCE((pf.fee_percentage/100)*CAST(lr.price_raw as uint256), DOUBLE '0') as UINT256) AS platform_fee_amount_raw +, COALESCE(roy.amount, cast(0 as uint256)) AS royalty_fee_amount_raw +, cast(null as varbinary) as platform_fee_address , roy.royaltyRecipient AS royalty_fee_address , lr.evt_index AS sub_tx_trade_id FROM looksrare_trades lr diff --git a/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades.sql index af028bb4ca2..be12c516e10 100644 --- a/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'looksrare_v2_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -9,31 +10,31 @@ ) }} -{% set looksrare_v2_start_date = '2023-04-01' %} +{% set looksrare_v2_start_date = "TIMESTAMP '2023-04-01'" %} WITH looksrare_v2_trades AS ( SELECT l.evt_block_time AS block_time , l.evt_block_number AS block_number , 'Offer Accepted' AS trade_category - , l.feeAmounts[0]+l.feeAmounts[1]+l.feeAmounts[2] AS price_raw + , l.feeAmounts[1]+l.feeAmounts[2]+l.feeAmounts[3] AS price_raw , l.askUser AS seller , l.bidUser AS buyer , l.collection AS nft_contract_address - , cast(element_at(l.amounts, 1) as DECIMAL(38,0)) AS nft_amount + , element_at(l.amounts, 2) AS nft_amount , l.currency - , l.itemIds[0] AS nft_token_id + , l.itemIds[1] AS nft_token_id , l.contract_address AS project_contract_address , l.evt_tx_hash AS tx_hash , l.evt_index - , l.feeAmounts[1] AS royalty_fee_amount_raw - , l.feeAmounts[2] AS platform_fee_amount_raw - , CASE WHEN l.feeRecipients[1]!='0x0000000000000000000000000000000000000000' THEN l.feeRecipients[1] END AS royalty_fee_address - , get_json_object(l.nonceInvalidationParameters, '$.orderHash') AS order_hash + , l.feeAmounts[2] AS royalty_fee_amount_raw + , l.feeAmounts[3] AS platform_fee_amount_raw + , CASE WHEN l.feeRecipients[2]!=0x0000000000000000000000000000000000000000 THEN l.feeRecipients[2] END AS royalty_fee_address +-- , JSON_EXTRACT_SCALAR(l.nonceInvalidationParameters, '$.orderHash') AS order_hash FROM {{ source('looksrare_v2_ethereum','LooksRareProtocol_evt_TakerAsk') }} l {% if is_incremental() %} - WHERE l.evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE l.evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - WHERE l.evt_block_time >= '{{looksrare_v2_start_date}}' + WHERE l.evt_block_time >= {{looksrare_v2_start_date}} {% endif %} UNION ALL @@ -41,25 +42,25 @@ WITH looksrare_v2_trades AS ( SELECT l.evt_block_time AS block_time , l.evt_block_number AS block_number , 'Buy' AS trade_category - , l.feeAmounts[0]+l.feeAmounts[1]+l.feeAmounts[2] AS price_raw - , l.feeRecipients[0] AS seller + , l.feeAmounts[1]+l.feeAmounts[2]+l.feeAmounts[3] AS price_raw + , l.feeRecipients[1] AS seller , l.bidUser AS buyer , l.collection AS nft_contract_address - , element_at(l.amounts, 1) AS nft_amount + , element_at(l.amounts, 2) AS nft_amount , l.currency - , l.itemIds[0] AS nft_token_id + , l.itemIds[1] AS nft_token_id , l.contract_address AS project_contract_address , l.evt_tx_hash AS tx_hash , l.evt_index - , l.feeAmounts[1] AS royalty_fee_amount_raw - , l.feeAmounts[2] AS platform_fee_amount_raw - , CASE WHEN l.feeRecipients[1]!='0x0000000000000000000000000000000000000000' THEN l.feeRecipients[1] END AS royalty_fee_address - , get_json_object(l.nonceInvalidationParameters, '$.orderHash') AS order_hash + , l.feeAmounts[2] AS royalty_fee_amount_raw + , l.feeAmounts[3] AS platform_fee_amount_raw + , CASE WHEN l.feeRecipients[2]!=0x0000000000000000000000000000000000000000 THEN l.feeRecipients[2] END AS royalty_fee_address +-- , JSON_EXTRACT_SCALAR(l.nonceInvalidationParameters, '$.orderHash') AS order_hash FROM {{ source('looksrare_v2_ethereum','LooksRareProtocol_evt_TakerBid') }} l {% if is_incremental() %} - WHERE l.evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE l.evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - WHERE l.evt_block_time >= '{{looksrare_v2_start_date}}' + WHERE l.evt_block_time >= {{looksrare_v2_start_date}} {% endif %} ) @@ -77,10 +78,10 @@ SELECT , buyer , seller , currency AS currency_contract -, CAST(price_raw as DECIMAL(38,0)) as price_raw -, CAST(platform_fee_amount_raw as DECIMAL(38,0)) as platform_fee_amount_raw -, CAST(royalty_fee_amount_raw as DECIMAL(38,0)) as royalty_fee_amount_raw +, price_raw +, platform_fee_amount_raw +, royalty_fee_amount_raw , royalty_fee_address -, CAST(null as VARCHAR(1)) as platform_fee_address +, CAST(null as varbinary) as platform_fee_address , evt_index as sub_tx_trade_id FROM looksrare_v2_trades diff --git a/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql index 3358e25e2a1..bab5c905084 100644 --- a/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'sudoswap_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -42,7 +43,7 @@ WITH WHERE call_success = true {% if is_incremental() %} -- this filter will only be applied on an incremental run. We only want to update with new swaps. - AND call_block_time >= date_trunc("day", now() - interval '1 week') + AND call_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} UNION ALL @@ -61,7 +62,7 @@ WITH WHERE call_success = true {% if is_incremental() %} -- this filter will only be applied on an incremental run. We only want to update with new swaps. - AND call_block_time >= date_trunc("day", now() - interval '1 week') + AND call_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} UNION ALL @@ -80,7 +81,7 @@ WITH WHERE call_success = true {% if is_incremental() %} -- this filter will only be applied on an incremental run. We only want to update with new swaps. - AND call_block_time >= date_trunc("day", now() - interval '1 week') + AND call_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} ) s ) @@ -88,16 +89,16 @@ WITH -- this join should be removed in the future when more call trace info is added to the _call_ tables, we need the call_from field to track down the eth traces. , swaps_with_calldata as ( select s.* - , tr.from as call_from - , CASE WHEN called_from_router = true THEN tr.from ELSE tr.to END as project_contract_address -- either the router or the pool if called directly + , tr."from" as call_from + , CASE WHEN called_from_router = true THEN tr."from" ELSE tr.to END as project_contract_address -- either the router or the pool if called directly from swaps s inner join {{ source('ethereum', 'traces') }} tr ON tr.success and s.call_block_number = tr.block_number and s.call_tx_hash = tr.tx_hash and s.call_trace_address = tr.trace_address {% if is_incremental() %} -- this filter will only be applied on an incremental run. We only want to update with new swaps. - AND tr.block_time >= date_trunc("day", now() - interval '1 week') + AND tr.block_time >= date_trunc('day', now() - interval '7' day) {% else %} - AND tr.block_time >= '2022-4-1' + AND tr.block_time >= TIMESTAMP '2022-4-1' {% endif %} ) @@ -171,22 +172,23 @@ WITH sb.call_block_time , sb.call_block_number , sb.trade_category - , SUM( + , cast(SUM( CASE WHEN sb.trade_category = 'Buy' -- caller buys, AMM sells THEN ( - CASE WHEN tr.from = sb.call_from THEN value -- amount of ETH payed - WHEN (tr.to = sb.call_from AND sb.call_from != sb.asset_recip) THEN -value --refunds unless the caller is also the asset recipient, no way to discriminate there. - ELSE 0 END) + CASE WHEN tr."from" = sb.call_from THEN cast(value as int256) -- amount of ETH payed + WHEN (tr.to = sb.call_from AND sb.call_from != sb.asset_recip) THEN -cast(value as int256) --refunds unless the caller is also the asset recipient, no way to discriminate there. + ELSE int256 '0' END) ELSE ( -- caller sells, AMM buys - CASE WHEN tr.from = sb.pair_address THEN value -- all ETH leaving the pool, nothing should be coming in on a sell. - ELSE 0 END) - END ) as trade_price -- what the buyer paid (incl all fees) + CASE WHEN tr."from" = sb.pair_address THEN cast(value as int256) -- all ETH leaving the pool, nothing should be coming in on a sell. + ELSE cast(0 as int256) END) + END ) as uint256) as trade_price -- what the buyer paid (incl all fees) , SUM( - CASE WHEN (tr.to = sb.protocolfee_recipient) THEN value - ELSE 0 END + CASE WHEN (tr.to = sb.protocolfee_recipient) THEN cast(value as uint256) + ELSE uint256 '0' END ) as protocol_fee_amount -- what the buyer paid - , ARRAY_AGG(distinct CASE WHEN substring(input,1,10)='0x42842e0e' THEN bytea2numeric_v3(substring(input,139,64)) END) - as nft_token_id + , filter(ARRAY_AGG(distinct CASE WHEN bytearray_substring(input,1,10)=0x42842e0e THEN bytearray_to_uint256(bytearray_substring(input,139,64)) END) + , x->x is not null + ) as nft_token_id , sb.call_tx_hash , sb.trade_recipient , sb.pair_address @@ -206,14 +208,16 @@ WITH AND tr.block_number = sb.call_block_number AND tr.tx_hash = sb.call_tx_hash AND ( - (cardinality(call_trace_address) != 0 AND call_trace_address = slice(tr.trace_address,1,cardinality(call_trace_address))) --either a normal tx where trace address helps us narrow down which subtraces to look at for ETH transfers or NFT transfers. + (cardinality(call_trace_address) != 0 + AND call_trace_address = slice(tr.trace_address,1,cardinality(call_trace_address)) + ) --either a normal tx where trace address helps us narrow down which subtraces to look at for ETH transfers or NFT transfers. OR cardinality(call_trace_address) = 0 -- In this case the swap function was called directly, all traces are thus subtraces of that call (like 0x34a52a94fce15c090cc16adbd6824948c731ecb19a39350633590a9cd163658b). ) {% if is_incremental() %} - AND tr.block_time >= date_trunc("day", now() - interval '1 week') + AND tr.block_time >= date_trunc('day', now() - interval '7' day) {% endif %} {% if not is_incremental() %} - AND tr.block_time >= '2022-4-1' + AND tr.block_time >= TIMESTAMP '2022-4-1' {% endif %} GROUP BY 1,2,3,7,8,9,10,11,12,13,14,15,16 ) @@ -234,7 +238,7 @@ WITH ELSE trade_recipient END as buyer , trade_price as price_raw - , '{{ var("ETH_ERC20_ADDRESS") }}' as currency_contract --ETH + , {{var("ETH_ERC20_ADDRESS")}} as currency_contract --ETH , nftcontractaddress as nft_contract_address , project_contract_address -- This is either the router or the pool address if called directly , call_tx_hash as tx_hash @@ -254,17 +258,18 @@ SELECT , buyer , seller , nft_contract_address - , explode(nft_token_id) as nft_token_id --nft.trades prefers each token id be its own row - , cast(1 as int) as nft_amount + , one_nft_token_id as nft_token_id --nft.trades prefers each token id be its own row + , cast(1 as uint256) as nft_amount , trade_type , trade_category , currency_contract - , cast(price_raw/number_of_items as DECIMAL(38,0)) as price_raw - , cast(platform_fee_amount_raw/number_of_items as decimal(38)) as platform_fee_amount_raw - , cast(0 as decimal(38)) as royalty_fee_amount_raw - , cast(pool_fee_amount_raw/number_of_items as decimal(38)) as pool_fee_amount_raw - , cast(protocolfee_recipient as VARCHAR(42)) as platform_fee_address - , cast(null as varchar(1)) as royalty_fee_address - , row_number() over (partition by tx_hash order by nft_token_id) as sub_tx_trade_id + , cast(price_raw/number_of_items as uint256) as price_raw + , cast(platform_fee_amount_raw/number_of_items as uint256) as platform_fee_amount_raw + , cast(0 as uint256) as royalty_fee_amount_raw + , cast(pool_fee_amount_raw/number_of_items as uint256) as pool_fee_amount_raw + , protocolfee_recipient as platform_fee_address + , cast(null as varbinary) as royalty_fee_address + , row_number() over (partition by tx_hash order by one_nft_token_id) as sub_tx_trade_id FROM swaps_cleaned +CROSS JOIN UNNEST(nft_token_id) as foo(one_nft_token_id) diff --git a/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades.sql index f75763096b8..09557c9b5a5 100644 --- a/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'superrare_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -8,23 +9,23 @@ unique_key = ['block_number','tx_hash','sub_tx_trade_id'], ) }} -{% set project_start_date='2018-04-05' %} +{% set project_start_date="TIMESTAMP '2018-04-05'" %} -- raw data table with all sales on superrare platform -- both primary and secondary with all_superrare_sales as ( select evt_block_time as block_time , evt_block_number as block_number - , `_originContract` as contract_address - , `_tokenId` as nft_token_id - , `_seller` as seller - , `_buyer` as buyer - , `_amount` as price_raw + , "_originContract" as contract_address + , "_tokenId" as nft_token_id + , "_seller" as seller + , "_buyer" as buyer + , "_amount" as price_raw , evt_tx_hash - , '{{ var("ETH_ERC20_ADDRESS") }}' as currency_contract + , {{var("ETH_ERC20_ADDRESS")}} as currency_contract , evt_index as sub_tx_trade_id from {{ source('superrare_ethereum','SuperRareMarketAuction_evt_Sold') }} {% if is_incremental() %} - where evt_block_time >= date_trunc("day", now() - interval '1 week') + where evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} union all @@ -32,33 +33,33 @@ with all_superrare_sales as ( select evt_block_time , evt_block_number , contract_address - , `_tokenId` - , `_seller` - , `_buyer` - , `_amount` + , "_tokenId" + , "_seller" + , "_buyer" + , "_amount" , evt_tx_hash - , '{{ var("ETH_ERC20_ADDRESS") }}' + , {{var("ETH_ERC20_ADDRESS")}} , evt_index from {{ source('superrare_ethereum','SuperRare_evt_Sold') }} {% if is_incremental() %} - where evt_block_time >= date_trunc("day", now() - interval '1 week') + where evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} union all select evt_block_time , evt_block_number - , `_originContract` as contract_address - , `_tokenId` - , `_seller` - , `_bidder` - , `_amount` + , "_originContract" as contract_address + , "_tokenId" + , "_seller" + , "_bidder" + , "_amount" , evt_tx_hash - , '{{ var("ETH_ERC20_ADDRESS") }}' + , {{var("ETH_ERC20_ADDRESS")}} , evt_index from {{ source('superrare_ethereum','SuperRareMarketAuction_evt_AcceptBid') }} {% if is_incremental() %} - where evt_block_time >= date_trunc("day", now() - interval '1 week') + where evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} union all @@ -66,67 +67,67 @@ with all_superrare_sales as ( select evt_block_time , evt_block_number , contract_address - , `_tokenId` - , `_seller` - , `_bidder` - , `_amount` + , "_tokenId" + , "_seller" + , "_bidder" + , "_amount" , evt_tx_hash - , '{{ var("ETH_ERC20_ADDRESS") }}' + , {{var("ETH_ERC20_ADDRESS")}} , evt_index from {{ source('superrare_ethereum','SuperRare_evt_AcceptBid') }} {% if is_incremental() %} - where evt_block_time >= date_trunc("day", now() - interval '1 week') + where evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} union all select evt_block_time , evt_block_number - , `_originContract` - , `_tokenId` - , `_seller` - , `_bidder` - , `_amount` + , "_originContract" + , "_tokenId" + , "_seller" + , "_bidder" + , "_amount" , evt_tx_hash - , `_currencyAddress` + , "_currencyAddress" , evt_index from {{ source('superrare_ethereum','SuperRareBazaar_evt_AcceptOffer') }} {% if is_incremental() %} - where evt_block_time >= date_trunc("day", now() - interval '1 week') + where evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} union all select evt_block_time , evt_block_number - , `_contractAddress` - , `_tokenId` - , `_seller` - , `_bidder` - , `_amount` + , "_contractAddress" + , "_tokenId" + , "_seller" + , "_bidder" + , "_amount" , evt_tx_hash - , `_currencyAddress` + , "_currencyAddress" , evt_index from {{ source('superrare_ethereum','SuperRareBazaar_evt_AuctionSettled') }} {% if is_incremental() %} - where evt_block_time >= date_trunc("day", now() - interval '1 week') + where evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} union all select evt_block_time , evt_block_number - , `_originContract` - , `_tokenId` - , `_seller` - , `_buyer` - , `_amount` + , "_originContract" + , "_tokenId" + , "_seller" + , "_buyer" + , "_amount" , evt_tx_hash - , `_currencyAddress` + , "_currencyAddress" , evt_index from {{ source('superrare_ethereum','SuperRareBazaar_evt_Sold') }} {% if is_incremental() %} - where evt_block_time >= date_trunc("day", now() - interval '1 week') + where evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} union all @@ -134,21 +135,21 @@ with all_superrare_sales as ( -- Superrare AuctionHouse (not decoded) select block_time , block_number - , concat('0x',substring(topic2 from 27 for 40)) as contract_address - , bytea2numeric_v3(substring(topic4 from 3)) as nft_token_id - , concat('0x',substring(data from 27 for 40)) as seller - , concat('0x',substring(topic3 from 27 for 40)) as buyer - , bytea2numeric_v3(substring(data from 67 for 64)) as price_raw + , bytearray_substring(topic1, 13, 20) AS contract_address + , bytearray_to_uint256(topic3) AS nft_token_id + , bytearray_substring(data, 13, 20) AS seller + , bytearray_substring(topic2, 13, 20) AS buyer + , bytearray_to_uint256(bytearray_substring(data, 33, 32)) AS price_raw , tx_hash - , '{{ var("ETH_ERC20_ADDRESS") }}' + , {{var("ETH_ERC20_ADDRESS")}} , index from {{ source('ethereum','logs') }} - where contract_address = lower('0x8c9f364bf7a56ed058fc63ef81c6cf09c833e656') - and topic1 = lower('0xea6d16c6bfcad11577aef5cc6728231c9f069ac78393828f8ca96847405902a9') + where contract_address = 0x8c9f364bf7a56ed058fc63ef81c6cf09c833e656 + and topic0 = 0xea6d16c6bfcad11577aef5cc6728231c9f069ac78393828f8ca96847405902a9 {% if is_incremental() %} - and block_time >= date_trunc("day", now() - interval '1 week') + and block_time >= date_trunc('day', now() - interval '7' day) {% else %} - and block_time >= '{{ project_start_date }}' + and block_time >= {{ project_start_date }} {% endif %} union all @@ -156,23 +157,22 @@ with all_superrare_sales as ( -- Superrare Marketplace (not decoded) select block_time , block_number - , concat('0x',substring(topic2 from 27 for 40)) as contract_address - , bytea2numeric_v3(substring(data from 67 for 64)) as nft_token_id - , concat('0x',substring(topic4 from 27 for 40)) as seller - , concat('0x',substring(topic3 from 27 for 40)) as buyer - , bytea2numeric_v3(substring(data from 3 for 64)) as price_raw + , bytearray_substring(topic1, 13, 20) AS contract_address + , bytearray_to_uint256(bytearray_substring(data, 33, 32)) AS nft_token_id + , bytearray_substring(topic3, 13, 20) AS seller + , bytearray_substring(topic2, 13, 20) AS buyer + , bytearray_to_uint256(bytearray_substring(data, 1, 32)) as price_raw , tx_hash - , '{{ var("ETH_ERC20_ADDRESS") }}' + , {{var("ETH_ERC20_ADDRESS")}} , index from {{ source('ethereum','logs') }} - where contract_address = lower('0x65b49f7aee40347f5a90b714be4ef086f3fe5e2c') - and topic1 in (lower('0x2a9d06eec42acd217a17785dbec90b8b4f01a93ecd8c127edd36bfccf239f8b6') - , lower('0x5764dbcef91eb6f946584f4ea671217c686fa7e858ce4f9f42d08422b86556a9') - ) + where contract_address = 0x65b49f7aee40347f5a90b714be4ef086f3fe5e2c + and topic0 in (0x2a9d06eec42acd217a17785dbec90b8b4f01a93ecd8c127edd36bfccf239f8b6 + ,0x5764dbcef91eb6f946584f4ea671217c686fa7e858ce4f9f42d08422b86556a9) {% if is_incremental() %} - and block_time >= date_trunc("day", now() - interval '1 week') + and block_time >= date_trunc('day', now() - interval '7' day) {% else %} - and block_time >= '{{ project_start_date }}' + and block_time >= {{ project_start_date }} {% endif %} ) @@ -181,14 +181,14 @@ SELECT a.block_time, a.block_number, a.nft_token_id, - 1 as nft_amount, + uint256 '1' as nft_amount, 'Buy' as trade_category, a.seller, a.buyer, - CAST(a.price_raw AS DECIMAL(38,0)) as price_raw, + a.price_raw as price_raw, a.currency_contract, a.contract_address as nft_contract_address, - cast(NULL as varchar(1)) as project_contract_address, + cast(NULL as varbinary) as project_contract_address, a.evt_tx_hash as tx_hash, case when a.seller = coalesce(minter.to, minter_superrare.to) @@ -197,33 +197,33 @@ SELECT end as trade_type, case when a.seller = coalesce(minter.to, minter_superrare.to) - then cast(ROUND((0.03+0.15) * (a.price_raw)) as decimal(38)) -- superrare takes fixed 3% fee + 15% commission on primary sales - else cast(ROUND(0.03 * (a.price_raw)) as decimal(38)) -- fixed 3% fee + then cast(ROUND((0.03+0.15) * (a.price_raw)) as uint256) -- superrare takes fixed 3% fee + 15% commission on primary sales + else cast(ROUND(0.03 * (a.price_raw)) as uint256) -- fixed 3% fee end as platform_fee_amount_raw, case when a.seller = coalesce(minter.to, minter_superrare.to) - then cast(0 as decimal(38)) - else cast(ROUND(0.10 * (a.price_raw)) as decimal(38)) -- fixed 10% royalty fee on secondary sales + then cast(0 as uint256) + else cast(ROUND(0.10 * (a.price_raw)) as uint256) -- fixed 10% royalty fee on secondary sales end as royalty_fee_amount_raw, - cast(NULL as varchar(1)) as royalty_fee_address, - cast(NULL as varchar(1)) as platform_fee_address, + cast(NULL as varbinary) as royalty_fee_address, + cast(NULL as varbinary) as platform_fee_address, sub_tx_trade_id from all_superrare_sales a left join {{ source('erc721_ethereum','evt_transfer') }} minter on minter.contract_address = a.contract_address and minter.tokenId = a.nft_token_id - and minter.from = '0x0000000000000000000000000000000000000000' + and minter."from" = 0x0000000000000000000000000000000000000000 {% if is_incremental() %} - and minter.evt_block_time >= date_trunc("day", now() - interval '1 week') + and minter.evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - and minter.evt_block_time >= '{{ project_start_date }}' + and minter.evt_block_time >= {{ project_start_date }} {% endif %} left join {{ source('erc20_ethereum','evt_transfer') }} minter_superrare on minter_superrare.contract_address = a.contract_address and minter_superrare.value = a.nft_token_id - and minter_superrare.from = '0x0000000000000000000000000000000000000000' + and minter_superrare."from" = 0x0000000000000000000000000000000000000000 {% if is_incremental() %} - and minter_superrare.evt_block_time >= date_trunc("day", now() - interval '1 week') + and minter_superrare.evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - and minter_superrare.evt_block_time >= '{{ project_start_date }}' + and minter_superrare.evt_block_time >= {{ project_start_date }} {% endif %} diff --git a/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql index 414e09dd74f..8e7046cbba8 100644 --- a/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'x2y2_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -9,8 +10,8 @@ ) }} -{%- set project_start_date = '2022-02-04' %} -{%- set fee_management_addr = '0xd823c605807cc5e6bd6fc0d7e4eea50d3e2d66cd' %} +{%- set project_start_date = "TIMESTAMP '2022-02-04'" %} +{%- set fee_management_addr = "0xd823c605807cc5e6bd6fc0d7e4eea50d3e2d66cd" %} WITH src_evt_inventory as ( @@ -20,42 +21,42 @@ WITH src_evt_inventory as ( ,evt_block_number as block_number ,evt_tx_hash as tx_hash ,contract_address as project_contract_address - ,case when intent = 1 then taker else maker end as buyer - ,case when intent = 1 then maker else taker end as seller - ,'0x' || substring(get_json_object(inv.item, '$.data'), 155, 40) as nft_contract_address - ,bytea2numeric_v3(substring(get_json_object(inv.item, '$.data'), 195,64)) as nft_token_id - ,CAST(1 AS DECIMAL(38,0)) AS nft_amount - ,case when intent = 1 then 'Buy' else 'Offer Accepted' end as trade_category + ,case when intent = cast(1 as uint256) then taker else maker end as buyer + ,case when intent = cast(1 as uint256) then maker else taker end as seller + ,from_hex('0x' || substring(JSON_EXTRACT_SCALAR(inv.item, '$.data'), 155, 40)) as nft_contract_address + ,bytearray_to_uint256(from_hex(substring(JSON_EXTRACT_SCALAR(inv.item, '$.data'), 195,64))) as nft_token_id + ,CAST(1 AS UINT256) AS nft_amount + ,case when intent = cast(1 as uint256) then 'Buy' else 'Offer Accepted' end as trade_category ,'secondary' as trade_type ,currency as currency_contract - ,get_json_object(inv.item, '$.price') as price_raw - ,get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.to') as fees_0_to - ,get_json_object(get_json_object(inv.detail, '$.fees[1]'), '$.to') as fees_1_to - ,CASE WHEN get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.to') ='{{fee_management_addr}}' - THEN get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.percentage') + ,cast(JSON_EXTRACT_SCALAR(inv.item, '$.price') as UINT256) as price_raw + ,from_hex(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[0]'), '$.to')) as fees_0_to + ,from_hex(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[1]'), '$.to')) as fees_1_to + ,CASE WHEN JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[0]'), '$.to') = '{{fee_management_addr}}' + THEN cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[0]'), '$.percentage') as DOUBLE) ELSE 0 END as platform_fee_percentage - ,CASE WHEN get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.to') ='{{fee_management_addr}}' - THEN (COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[1]'), '$.percentage'), 0) - +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[2]'), '$.percentage'), 0) - +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[3]'), '$.percentage'), 0) - +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[4]'), '$.percentage'), 0)) - ELSE ( COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.percentage'), 0) - +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[1]'), '$.percentage'), 0) - +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[2]'), '$.percentage'), 0) - +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[3]'), '$.percentage'), 0)) + ,CASE WHEN JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[0]'), '$.to') = '{{fee_management_addr}}' + THEN (COALESCE(cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[1]'), '$.percentage') as double), 0) + +COALESCE(cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[2]'), '$.percentage')as double), 0) + +COALESCE(cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[3]'), '$.percentage')as double), 0) + +COALESCE(cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[4]'), '$.percentage')as double), 0)) + ELSE ( COALESCE(cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[0]'), '$.percentage')as double), 0) + +COALESCE(cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[1]'), '$.percentage')as double), 0) + +COALESCE(cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[2]'), '$.percentage')as double), 0) + +COALESCE(cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[3]'), '$.percentage')as double), 0)) END as royalty_fee_percentage - ,'{{fee_management_addr}}' as platform_fee_address - ,CASE WHEN get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.to')='{{fee_management_addr}}' - THEN get_json_object(get_json_object(inv.detail, '$.fees[1]'), '$.to') - ELSE get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.to') + ,{{fee_management_addr}} as platform_fee_address + ,CASE WHEN JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[0]'), '$.to')='{{fee_management_addr}}' + THEN from_hex(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[1]'), '$.to')) + ELSE from_hex(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(inv.detail, '$.fees[0]'), '$.to')) END AS royalty_fee_address ,evt_index as sub_tx_trade_id FROM {{ source('x2y2_ethereum','X2Y2_r1_evt_EvInventory') }} inv {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% else %} - WHERE evt_block_time >= '{{project_start_date}}' + WHERE evt_block_time >= {{project_start_date}} {% endif %} ) @@ -75,8 +76,8 @@ SELECT , trade_category , currency_contract , price_raw -, CAST(COALESCE(price_raw*platform_fee_percentage/1e6, 0) as decimal(38)) AS platform_fee_amount_raw -, CAST(COALESCE(price_raw*royalty_fee_percentage/1e6, 0) as decimal(38)) AS royalty_fee_amount_raw +, CAST(COALESCE(price_raw*platform_fee_percentage/1e6, 0) as uint256) AS platform_fee_amount_raw +, CAST(COALESCE(price_raw*royalty_fee_percentage/1e6, 0) as uint256) AS royalty_fee_amount_raw , platform_fee_address , royalty_fee_address , sub_tx_trade_id diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades.sql index 793fb73749c..28d6b60c202 100644 --- a/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'zora_v1_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -15,34 +16,34 @@ SELECT , bf.evt_block_number AS block_number , bf.contract_address AS project_contract_address , bf.evt_tx_hash AS tx_hash - , '0xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7' AS nft_contract_address -- hardcoded ZORA address + , 0xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7 AS nft_contract_address -- hardcoded ZORA address , bf.tokenId AS nft_token_id - , CAST(1 as INT) as nft_amount + , CAST(1 as UINT256) as nft_amount , 'Offer accepted' AS trade_category - , CASE WHEN mt.from = mint.to + , CASE WHEN mt."from" = mint.to THEN 'primary' ELSE 'secondary' END AS trade_type - , get_json_object(bf.bid, '$.bidder') AS buyer - , mt.from AS seller - , CAST(get_json_object(bf.bid, '$.amount') as DECIMAL(38)) AS price_raw - , get_json_object(bf.bid, '$.currency') AS currency_contract - , CAST(0 as DECIMAL(38)) AS platform_fee_amount_raw - , CAST(0 as DECIMAL(38)) AS royalty_fee_amount_raw - , CAST(NULL as VARCHAR(1)) AS platform_fee_address - , CAST(NULL as VARCHAR(1)) AS royalty_fee_address + , from_hex(JSON_EXTRACT_SCALAR(bf.bid, '$.bidder')) AS buyer + , mt."from" AS seller + , CAST(JSON_EXTRACT_SCALAR(bf.bid, '$.amount') as UINT256) AS price_raw + , from_hex(JSON_EXTRACT_SCALAR(bf.bid, '$.currency')) AS currency_contract + , CAST(0 as UINT256) AS platform_fee_amount_raw + , CAST(0 as UINT256) AS royalty_fee_amount_raw + , CAST(NULL as varbinary) AS platform_fee_address + , CAST(NULL as varbinary) AS royalty_fee_address , bf.evt_index as sub_tx_trade_id FROM {{ source('zora_ethereum','Market_evt_BidFinalized') }} bf LEFT JOIN {{ source('zora_ethereum','Media_evt_Transfer') }} mint - ON mint.from = '0x0000000000000000000000000000000000000000' + ON mint."from" = 0x0000000000000000000000000000000000000000 AND mint.tokenId = bf.tokenId LEFT JOIN {{ source('zora_ethereum','Media_evt_Transfer') }} mt ON bf.evt_block_number = mt.evt_block_number AND bf.evt_tx_hash = mt.evt_tx_hash AND bf.tokenId = mt.tokenId - AND get_json_object(bf.bid, '$.bidder') = mt.to -WHERE get_json_object(bf.bid, '$.bidder') != '0xe468ce99444174bd3bbbed09209577d25d1ad673' -- these are sells through the auction house which are included in V2 + AND from_hex(JSON_EXTRACT_SCALAR(bf.bid, '$.bidder')) = mt.to +WHERE from_hex(JSON_EXTRACT_SCALAR(bf.bid, '$.bidder')) != 0xe468ce99444174bd3bbbed09209577d25d1ad673 -- these are sells through the auction house which are included in V2 {% if is_incremental() %} -AND bf.evt_block_time >= date_trunc("day", now() - interval '1 week') -AND mt.evt_block_time >= date_trunc("day", now() - interval '1 week') +AND bf.evt_block_time >= date_trunc('day', now() - interval '7' day) +AND mt.evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades.sql index aaa331bc5de..d1e290047d3 100644 --- a/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'zora_v2_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -17,19 +18,19 @@ SELECT , evt_tx_hash AS tx_hash , tokenContract AS nft_contract_address , tokenId AS nft_token_id - , CAST(1 as INT) as nft_amount + , uint256 '1' as nft_amount , 'Auction' AS trade_category , 'secondary'AS trade_type , winner AS buyer , tokenOwner AS seller - , CAST(amount+curatorFee as DECIMAL(38)) AS price_raw + , CAST(amount+curatorFee as uint256) AS price_raw , auctionCurrency AS currency_contract - , CAST(0 as DECIMAL(38)) AS platform_fee_amount_raw - , CAST(0 as DECIMAL(38)) AS royalty_fee_amount_raw - , CAST(NULL as VARCHAR(1)) AS platform_fee_address - , CAST(NULL as VARCHAR(1)) AS royalty_fee_address + , CAST(0 as uint256) AS platform_fee_amount_raw + , CAST(0 as uint256) AS royalty_fee_amount_raw + , CAST(NULL as varbinary) AS platform_fee_address + , CAST(NULL as varbinary) AS royalty_fee_address , evt_index as sub_tx_trade_id FROM {{ source('zora_ethereum','AuctionHouse_evt_AuctionEnded') }} {% if is_incremental() %} -WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') +WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades.sql index bb08d9a0e9c..ca4c0f9438d 100644 --- a/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'zora_v3_ethereum', - alias ='base_trades', + tags = ['dunesql'], + alias = alias('base_trades'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -13,13 +14,13 @@ WITH v3_trades as ( SELECT evt_block_time AS block_time , evt_block_number AS block_number - , get_json_object(a, '$.tokenId') AS nft_token_id + , CAST(JSON_EXTRACT_SCALAR(a, '$.tokenId') as uint256) AS nft_token_id , 'Offer Accepted' AS trade_category , userA AS seller , userB AS buyer - , get_json_object(b, '$.amount') AS price_raw - , get_json_object(b, '$.tokenContract') AS currency_contract - , get_json_object(a, '$.tokenContract') AS nft_contract_address + , CAST(JSON_EXTRACT_SCALAR(b, '$.amount') as uint256) AS price_raw + , from_hex(JSON_EXTRACT_SCALAR(b, '$.tokenContract')) AS currency_contract + , from_hex(JSON_EXTRACT_SCALAR(a, '$.tokenContract')) AS nft_contract_address , contract_address AS project_contract_address , evt_tx_hash AS tx_hash , evt_index AS sub_tx_trade_id @@ -32,7 +33,7 @@ WITH v3_trades as ( FROM {{ source('zora_v3_ethereum','AsksV1_1_evt_ExchangeExecuted') }} ) {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} UNION ALL SELECT @@ -40,10 +41,10 @@ WITH v3_trades as ( , evt_block_number AS block_number , tokenId AS nft_token_id , 'Buy' AS trade_category - , get_json_object(auction, '$.seller') AS seller - , get_json_object(auction, '$.highestBidder') AS buyer - , get_json_object(auction, '$.highestBid') AS price_raw - , coalesce(get_json_object(auction, '$.currency'),'0x0000000000000000000000000000000000000000') AS currency_contract + , from_hex(JSON_EXTRACT_SCALAR(auction, '$.seller')) AS seller + , from_hex(JSON_EXTRACT_SCALAR(auction, '$.highestBidder')) AS buyer + , CAST(JSON_EXTRACT_SCALAR(auction, '$.highestBid') as uint256) AS price_raw + , coalesce(from_hex(JSON_EXTRACT_SCALAR(auction, '$.currency')),0x0000000000000000000000000000000000000000) AS currency_contract , tokenContract AS nft_contract_address , contract_address AS project_contract_address , evt_tx_hash AS tx_hash @@ -63,7 +64,7 @@ WITH v3_trades as ( FROM {{ source('zora_v3_ethereum','ReserveAuctionListingErc20_evt_AuctionEnded') }} ) {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} UNION ALL SELECT @@ -71,17 +72,17 @@ WITH v3_trades as ( , evt_block_number AS block_number , tokenId AS nft_token_id , 'Private Sale' AS trade_category - , get_json_object(ask, '$.seller') AS seller - , get_json_object(ask, '$.buyer') AS buyer - , get_json_object(ask, '$.price') AS price_raw - , '0x0000000000000000000000000000000000000000' AS currency_contract + , from_hex(JSON_EXTRACT_SCALAR(ask, '$.seller')) AS seller + , from_hex(JSON_EXTRACT_SCALAR(ask, '$.buyer')) AS buyer + , CAST(JSON_EXTRACT_SCALAR(ask, '$.price') as uint256) AS price_raw + , 0x0000000000000000000000000000000000000000 AS currency_contract , tokenContract AS nft_contract_address , contract_address AS project_contract_address , evt_tx_hash AS tx_hash , evt_index AS sub_tx_trade_id FROM {{ source('zora_v3_ethereum','AsksPrivateEth_evt_AskFilled') }} {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} UNION ALL SELECT @@ -92,14 +93,14 @@ WITH v3_trades as ( , seller AS seller , buyer AS buyer , price AS price_raw - , '0x0000000000000000000000000000000000000000' AS currency_contract + , 0x0000000000000000000000000000000000000000 AS currency_contract , tokenContract AS nft_contract_address , contract_address AS project_contract_address , evt_tx_hash AS tx_hash , evt_index AS sub_tx_trade_id FROM {{ source('zora_v3_ethereum','AsksCoreEth_evt_AskFilled') }} {% if is_incremental() %} - WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} ) @@ -109,10 +110,10 @@ WITH v3_trades as ( ,evt_tx_hash ,tokenContract ,tokenId - ,sum(cast(amount as decimal(38))) as royalty_fee_amount_raw + ,sum(cast(amount as uint256)) as royalty_fee_amount_raw ,case when count(distinct recipient) = 1 then min(recipient) - else cast(null as varchar(1)) + else cast(null as varbinary) end as royalty_fee_address FROM ( SELECT * FROM ( @@ -139,9 +140,9 @@ WITH v3_trades as ( UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient FROM {{ source('zora_v3_ethereum','ReserveAuctionListingErc20_evt_RoyaltyPayout') }} ) - WHERE cast(amount as decimal(38)) > 0 + WHERE cast(amount as uint256) > cast(0 as uint256) {% if is_incremental() %} - AND evt_block_time >= date_trunc("day", now() - interval '1 week') + AND evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} ) GROUP BY 1,2,3,4 @@ -155,17 +156,17 @@ SELECT , tx_hash , nft_contract_address , nft_token_id - , CAST(1 as INT) as nft_amount + , CAST(1 as uint256) as nft_amount , trade_category , 'secondary' as trade_type , buyer , seller - , CAST(price_raw as DECIMAL(38)) as price_raw + , price_raw as price_raw , currency_contract - , CAST(0 as DECIMAL(38)) AS platform_fee_amount_raw - , CAST(coalesce(roy.royalty_fee_amount_raw,0) as DECIMAL(38)) as royalty_fee_amount_raw - , CAST(NULL as VARCHAR(1)) AS platform_fee_address - , CAST(roy.royalty_fee_address as VARCHAR(42)) as royalty_fee_address + , CAST(0 as uint256) AS platform_fee_amount_raw + , coalesce(roy.royalty_fee_amount_raw,cast(0 as uint256)) as royalty_fee_amount_raw + , CAST(NULL as varbinary) AS platform_fee_address + , roy.royalty_fee_address as royalty_fee_address , sub_tx_trade_id FROM v3_trades LEFT JOIN royalty_payouts roy diff --git a/models/_sector/nft/trades/nft_ethereum_trades_beta.sql b/models/_sector/nft/trades/nft_ethereum_trades_beta.sql index 0f6984521eb..7942d173f31 100644 --- a/models/_sector/nft/trades/nft_ethereum_trades_beta.sql +++ b/models/_sector/nft/trades/nft_ethereum_trades_beta.sql @@ -10,20 +10,20 @@ }} -- (project, project_version, model) {% set base_models = [ - ('archipelago', 'v1', ref('archipelago_ethereum_base_trades')) - ,('superrare', 'v1', ref('superrare_ethereum_base_trades')) + ('archipelago', 'v1', ref('archipelago_ethereum_base_trades_legacy')) + ,('superrare', 'v1', ref('superrare_ethereum_base_trades_legacy')) ,('foundation', 'v1', ref('foundation_ethereum_base_trades_legacy')) - ,('blur', 'v1', ref('blur_ethereum_base_trades')) - ,('element', 'v1', ref('element_ethereum_base_trades')) - ,('x2y2', 'v1', ref('x2y2_ethereum_base_trades')) - ,('zora', 'v1', ref('zora_v1_ethereum_base_trades')) - ,('zora', 'v2', ref('zora_v2_ethereum_base_trades')) - ,('zora', 'v3', ref('zora_v3_ethereum_base_trades')) - ,('cryptopunks', 'v1', ref('cryptopunks_ethereum_base_trades')) - ,('sudoswap', 'v1', ref('sudoswap_ethereum_base_trades')) - ,('collectionswap', 'v1', ref('collectionswap_ethereum_base_trades')) - ,('looksrare', 'v1', ref('looksrare_v1_ethereum_base_trades')) - ,('looksrare', 'v2', ref('looksrare_v2_ethereum_base_trades')) + ,('blur', 'v1', ref('blur_ethereum_base_trades_legacy')) + ,('element', 'v1', ref('element_ethereum_base_trades_legacy')) + ,('x2y2', 'v1', ref('x2y2_ethereum_base_trades_legacy')) + ,('zora', 'v1', ref('zora_v1_ethereum_base_trades_legacy')) + ,('zora', 'v2', ref('zora_v2_ethereum_base_trades_legacy')) + ,('zora', 'v3', ref('zora_v3_ethereum_base_trades_legacy')) + ,('cryptopunks', 'v1', ref('cryptopunks_ethereum_base_trades_legacy')) + ,('sudoswap', 'v1', ref('sudoswap_ethereum_base_trades_legacy')) + ,('collectionswap', 'v1', ref('collectionswap_ethereum_base_trades_legacy')) + ,('looksrare', 'v1', ref('looksrare_v1_ethereum_base_trades_legacy')) + ,('looksrare', 'v2', ref('looksrare_v2_ethereum_base_trades_legacy')) ] %} -- We should remove this CTE and include ETH into the general prices table once everything is migrated diff --git a/models/sudoswap/ethereum/sudoswap_ethereum_pool_trades_agg_day.sql b/models/sudoswap/ethereum/sudoswap_ethereum_pool_trades_agg_day.sql index 75eb8a475a5..b6d918d8532 100644 --- a/models/sudoswap/ethereum/sudoswap_ethereum_pool_trades_agg_day.sql +++ b/models/sudoswap/ethereum/sudoswap_ethereum_pool_trades_agg_day.sql @@ -39,7 +39,7 @@ SELECT ELSE -1 * nft_amount END ) AS nft_change_trading -FROM {{ ref('sudoswap_ethereum_base_trades') }} t +FROM {{ ref('sudoswap_ethereum_base_trades_legacy') }} t LEFT JOIN {{ ref('prices_usd_forward_fill') }} usd ON usd.blockchain = null and usd.symbol = 'ETH' AND usd.minute = date_trunc('minute',t.block_time) From 3cccbef06849d32a0ad8a195f75cb848e5711423 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Fri, 7 Jul 2023 16:00:59 +0200 Subject: [PATCH 02/22] migrate all NFT base_trades beta models --- ...rchipelago_ethereum_base_trades_legacy.sql | 76 +++++ .../blur_ethereum_base_trades_legacy.sql | 87 ++++++ ...ectionswap_ethereum_base_trades_legacy.sql | 115 ++++++++ ...ryptopunks_ethereum_base_trades_legacy.sql | 62 ++++ .../element_ethereum_base_trades_legacy.sql | 22 ++ ...oksrare_v1_ethereum_base_trades_legacy.sql | 122 ++++++++ ...oksrare_v2_ethereum_base_trades_legacy.sql | 86 ++++++ .../sudoswap_ethereum_base_trades_legacy.sql | 270 ++++++++++++++++++ .../superrare_ethereum_base_trades_legacy.sql | 229 +++++++++++++++ .../x2y2_ethereum_base_trades_legacy.sql | 84 ++++++ .../zora_v1_ethereum_base_trades_legacy.sql | 48 ++++ .../zora_v2_ethereum_base_trades_legacy.sql | 35 +++ .../zora_v3_ethereum_base_trades_legacy.sql | 177 ++++++++++++ 13 files changed, 1413 insertions(+) create mode 100644 models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades_legacy.sql create mode 100644 models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..c23be5f893a --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades_legacy.sql @@ -0,0 +1,76 @@ +{{ config( + schema = 'archipelago_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} +{% set project_start_date='2022-6-20' %} + +WITH +trade_events as ( + SELECT * FROM {{ source('archipelago_ethereum','ArchipelagoMarket_evt_Trade') }} + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + WHERE evt_block_time >= '{{project_start_date}}' + {% endif %} +), +token_events as ( + SELECT * FROM {{ source('archipelago_ethereum','ArchipelagoMarket_evt_TokenTrade') }} + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + WHERE evt_block_time >= '{{project_start_date}}' + {% endif %} +), +fee_events as ( + SELECT + evt_block_number + , tradeId + , sum(amount) filter ( + where recipient not in ('0xa76456bb6abc50fb38e17c042026bc27a95c3314','0x1fc12c9f68a6b0633ba5897a40a8e61ed9274dc9') + ) as royalty_amount + , sum(amount) filter ( + where recipient in ('0xa76456bb6abc50fb38e17c042026bc27a95c3314','0x1fc12c9f68a6b0633ba5897a40a8e61ed9274dc9') + ) as platform_amount + FROM {{ source('archipelago_ethereum','ArchipelagoMarket_evt_RoyaltyPayment') }} + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + WHERE evt_block_time >= '{{project_start_date}}' + {% endif %} + GROUP BY evt_block_number, tradeId +) + +SELECT + date_trunc('day',trade.evt_block_time) as block_date + ,trade.evt_block_time as block_time + ,trade.evt_block_number as block_number + ,trade.evt_tx_hash as tx_hash + ,trade.contract_address as project_contract_address + ,CAST(null as varchar(5)) as trade_category + ,'secondary' as trade_type + ,trade.buyer + ,trade.seller + ,tok.tokenAddress as nft_contract_address + ,tok.tokenId as nft_token_id + ,1 as nft_amount + ,trade.currency as currency_contract + ,cast(trade.cost as decimal(38)) as price_raw + ,cast(coalesce(fee.platform_amount,0) as decimal(38)) as platform_fee_amount_raw + ,cast(coalesce(fee.royalty_amount,0) as decimal(38)) as royalty_fee_amount_raw + ,CAST(null as varchar(5)) as platform_fee_address + ,CAST(null as varchar(5)) as royalty_fee_address + ,trade.evt_index as sub_tx_trade_id +FROM trade_events trade +INNER JOIN token_events tok +ON trade.evt_block_number = tok.evt_block_number + AND trade.tradeId = tok.tradeId +LEFT JOIN fee_events fee +ON trade.evt_block_number = fee.evt_block_number + AND trade.tradeId = fee.tradeId + diff --git a/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..63498c0d29a --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades_legacy.sql @@ -0,0 +1,87 @@ +{{ config( + schema = 'blur_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} + + +{% set project_start_date = '2022-10-18' %} +{% set seaport_usage_start_date = '2023-01-25' %} + + +WITH +regular_blur_sales as ( + SELECT + date_trunc('day', bm.evt_block_time) AS block_date + , bm.evt_block_time AS block_time + , bm.evt_block_number AS block_number + , get_json_object(bm.buy, '$.collection') AS nft_contract_address + , get_json_object(bm.sell, '$.tokenId') AS nft_token_id + , CAST(get_json_object(bm.buy, '$.amount') AS INT) AS nft_amount + , get_json_object(bm.sell, '$.trader') AS seller + , get_json_object(bm.buy, '$.trader') AS buyer + , CASE WHEN get_json_object(bm.buy, '$.matchingPolicy') IN ('0x00000000006411739da1c40b106f8511de5d1fac', '0x0000000000dab4a563819e8fd93dba3b25bc3495') THEN 'Buy' + WHEN get_json_object(bm.buy, '$.matchingPolicy') IN ('0x0000000000b92d5d043faf7cecf7e2ee6aaed232') THEN 'Sell' + ELSE CAST(null as VARCHAR(1)) + END AS trade_category + , 'secondary' AS trade_type + , CAST(get_json_object(bm.buy, '$.price') AS DECIMAL(38,0)) AS price_raw + , get_json_object(bm.buy, '$.paymentToken') AS currency_contract + , bm.contract_address AS project_contract_address + , bm.evt_tx_hash AS tx_hash + , 0 AS platform_fee_amount_raw -- Hardcoded 0% platform fee + , CAST(COALESCE(get_json_object(bm.buy, '$.price')*get_json_object(get_json_object(bm.sell, '$.fees[0]'), '$.rate')/10000, 0) AS DECIMAL(38,0)) AS royalty_fee_amount_raw + , get_json_object(bm.sell, '$.fees[0].recipient') AS royalty_fee_address + , cast(NULL as varchar(1)) as platform_fee_address + , bm.evt_index as sub_tx_trade_id + FROM {{ source('blur_ethereum','BlurExchange_evt_OrdersMatched') }} bm + {% if is_incremental() %} + WHERE bm.evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + WHERE bm.evt_block_time >= '{{ project_start_date }}' + {% endif %} +) + +, seaport_blur_sales as ( +SELECT + date_trunc('day', s.evt_block_time) AS block_date + , s.evt_block_time AS block_time + , s.evt_block_number AS block_number + , get_json_object(s.offer[0], '$.token') AS nft_contract_address + , get_json_object(s.offer[0], '$.identifier') AS nft_token_id + , CAST(get_json_object(s.offer[0], '$.amount') AS DECIMAL(38,0)) AS nft_amount + , s.offerer AS seller + , s.recipient AS buyer + , 'Buy' AS trade_category + , 'secondary' AS trade_type + , CAST(get_json_object(s.consideration[0], '$.amount')+get_json_object(s.consideration[1], '$.amount') AS DECIMAL(38,0)) AS price_raw + , get_json_object(s.consideration[0], '$.token') AS currency_contract + , s.contract_address AS project_contract_address + , s.evt_tx_hash AS tx_hash + , CAST(0 AS DECIMAL(38,0)) AS platform_fee_amount_raw -- Hardcoded 0% platform fee + , LEAST(CAST(get_json_object(s.consideration[0], '$.amount') AS DECIMAL(38,0)), CAST(get_json_object(s.consideration[1], '$.amount') AS DECIMAL(38,0))) AS royalty_fee_amount_raw + , CASE WHEN get_json_object(s.consideration[0], '$.recipient')!=s.recipient THEN get_json_object(s.consideration[0], '$.recipient') + ELSE get_json_object(s.consideration[1], '$.recipient') + END AS royalty_fee_address + , cast(NULL as varchar(1)) as platform_fee_address + , s.evt_index as sub_tx_trade_id +FROM {{ source('seaport_ethereum','Seaport_evt_OrderFulfilled') }} s +WHERE s.zone='0x0000000000d80cfcb8dfcd8b2c4fd9c813482938' + {% if is_incremental() %} + AND s.evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + AND s.evt_block_time >= '{{seaport_usage_start_date}}' + {% endif %} +) + +select * +from regular_blur_sales +union all +select * +from seaport_blur_sales + diff --git a/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..4d21287a72c --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades_legacy.sql @@ -0,0 +1,115 @@ +{{ config( + schema = 'collectionswap_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} + +{%- set project_start_date = '2023-03-29' %} + +WITH +raw_trades as ( + select * + , row_number() over (partition by tx_hash order by evt_index asc, sub_order_id asc) as sub_tx_trade_id + from( + select + block_number, block_time, evt_index, tx_hash, buyer, seller, + posexplode(nft_id_array) as (sub_order_id, nft_token_id), + cast(1 as DECIMAL(38,0)) as nft_amount, + price_raw/number_of_items as price_raw, + platform_fee_amount_raw/number_of_items as platform_fee_amount_raw, + royalty_fee_amount_raw/number_of_items as royalty_fee_amount_raw, + trade_fee_amount_raw/number_of_items as trade_fee_amount_raw, + royalty_fee_address, + project_contract_address, + number_of_items, + 'secondary' as trade_type, + trade_category + from( + select + evt_block_number as block_number + ,evt_block_time as block_time + ,evt_index + ,evt_tx_hash as tx_hash + ,null as buyer + ,contract_address as seller + ,'Buy' as trade_category + ,nftIds as nft_id_array + ,cardinality(nftIds) as number_of_items + ,cast(outputAmount as decimal(38)) as price_raw + ,cast(protocolFee as decimal(38)) as platform_fee_amount_raw + ,get_json_object(royaltyDue[0], '$.amount') as royalty_fee_amount_raw + ,get_json_object(royaltyDue[0], '$.recipient') as royalty_fee_address + ,cast(tradeFee as decimal(38)) as trade_fee_amount_raw + ,contract_address as project_contract_address + from {{ source('collectionswap_ethereum','CollectionPool_evt_SwapNFTOutPool') }} e + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + WHERE evt_block_time >= '{{project_start_date}}' + {% endif %} + union all + select + evt_block_number as block_number + ,evt_block_time as block_time + ,evt_index + ,evt_tx_hash as tx_hash + ,contract_address as buyer + ,null as seller + ,'Sell' as trade_category + ,nftIds as nft_id_array + ,cardinality(nftIds) as number_of_items + ,cast(inputAmount + protocolFee + cast(get_json_object(royaltyDue[0], '$.amount') as decimal(38)) as decimal(38)) as price_raw + ,cast(protocolFee as decimal(38)) as platform_fee_amount_raw + ,get_json_object(royaltyDue[0], '$.amount') as royalty_fee_amount_raw + ,get_json_object(royaltyDue[0], '$.recipient') as royalty_fee_address + ,cast(tradeFee as decimal(38)) as trade_fee_amount_raw + ,contract_address as project_contract_address + from {{ source('collectionswap_ethereum','CollectionPool_evt_SwapNFTInPool') }} e + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + WHERE evt_block_time >= '{{project_start_date}}' + {% endif %} + ) + ) +), + +base_trades as ( + select + t.*, + p.nft_contract_address, + p.token_address as currency_contract + from raw_trades t + left join {{ ref('collectionswap_ethereum_pools') }} p + on t.project_contract_address = p.pool_address +) + +-- results +SELECT + date_trunc('day',block_time ) as block_date +, block_time +, block_number +, tx_hash +, project_contract_address +, buyer +, seller +, nft_contract_address +, nft_token_id +, nft_amount +, trade_type +, trade_category +, currency_contract +, cast(price_raw as decimal(38)) as price_raw +, cast(platform_fee_amount_raw as decimal(38)) as platform_fee_amount_raw +, cast(royalty_fee_amount_raw as decimal(38)) as royalty_fee_amount_raw +, cast(null as varchar(1)) as platform_fee_address +, royalty_fee_address +, sub_tx_trade_id +FROM base_trades + + diff --git a/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..44c6a59f7aa --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades_legacy.sql @@ -0,0 +1,62 @@ +{{ config( + schema = 'cryptopunks_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} + + +with accepted_bid_prices as ( + select + call_block_number + ,call_tx_hash + ,call.punkIndex + ,max_by(bid.value, evt_block_number) as latest_bid + ,max_by(bid.fromAddress, evt_block_number) as latest_bidder + from {{ source('cryptopunks_ethereum','CryptoPunksMarket_call_acceptBidForPunk')}} call + left join {{ source('cryptopunks_ethereum','CryptoPunksMarket_evt_PunkBidEntered') }} bid + on call_success + and call.call_block_number >= bid.evt_block_number + and call.punkIndex = bid.punkIndex + WHERE call_success + {% if is_incremental() %} + AND call.call_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + group by 1,2,3 +) + +select date_trunc('day',evt.evt_block_time) as block_date + , evt.evt_block_time as block_time + , evt.evt_block_number as block_number + , evt.evt_tx_hash as tx_hash + , evt.contract_address as project_contract_address + , evt.evt_index as sub_tx_trade_id + , case when call.latest_bid is null + then 'Buy' + else 'Bid accepted' end as trade_category + , 'secondary' as trade_type + , coalesce(call.latest_bidder,evt.toAddress) as buyer + , evt.fromAddress as seller + , evt.punkIndex as nft_token_id + , 1 as nft_amount + , evt.contract_address as nft_contract_address + , cast(coalesce(call.latest_bid, evt.value) as DECIMAL(38)) as price_raw + , '{{ var("ETH_ERC20_ADDRESS") }}' AS currency_contract -- all trades are in ETH + , cast(0 as DECIMAL(38)) as platform_fee_amount_raw + , cast(0 as DECIMAL(38)) as royalty_fee_amount_raw + , cast(null as VARCHAR(1)) as platform_fee_address + , cast(null as VARCHAR(1)) as royalty_fee_address +from {{ source('cryptopunks_ethereum','CryptoPunksMarket_evt_PunkBought') }} evt +left join accepted_bid_prices call +on evt.evt_block_number = call.call_block_number + and evt.evt_tx_hash = call.call_tx_hash + and evt.punkIndex = call.punkIndex +where evt.evt_tx_hash not in ('0x92488a00dfa0746c300c66a716e6cc11ba9c0f9d40d8c58e792cc7fcebf432d0' -- flash loan https://twitter.com/cryptopunksnfts/status/1453903818308083720 + ) +{% if is_incremental() %} +and evt.evt_block_time >= date_trunc("day", now() - interval '1 week') +{% endif %} diff --git a/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..7ea4b787ff0 --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades_legacy.sql @@ -0,0 +1,22 @@ +{{ config( + schema = 'element_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} + +WITH base_trades as ( + {{ element_v1_base_trades( + erc721_sell_order_filled = source('element_ex_ethereum','OrdersFeature_evt_ERC721SellOrderFilled') + , erc721_buy_order_filled = source('element_ex_ethereum','OrdersFeature_evt_ERC721BuyOrderFilled') + , erc1155_sell_order_filled = source('element_ex_ethereum','OrdersFeature_evt_ERC1155SellOrderFilled') + , erc1155_buy_order_filled = source('element_ex_ethereum','OrdersFeature_evt_ERC1155BuyOrderFilled') + ) + }} +) + +SELECT * FROM base_trades diff --git a/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..cc690625e34 --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades_legacy.sql @@ -0,0 +1,122 @@ +{{ config( + schema = 'looksrare_v1_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} + +WITH looksrare_trades AS ( + SELECT * + , ROW_NUMBER() OVER (PARTITION BY tx_hash, nft_contract_address, nft_token_id ORDER BY evt_index ASC) AS id + FROM ( + SELECT ta.evt_block_time AS block_time + , ta.tokenId AS nft_token_id + , ta.amount AS nft_amount + , CASE WHEN ta.strategy='0x58d83536d3efedb9f7f2a1ec3bdaad2b1a4dd98c' THEN 'Private Sale' ELSE 'Buy' END AS trade_category + , ta.maker AS seller + , ta.taker AS buyer + , ta.price AS price_raw + , ta.currency AS currency_contract + , ta.collection AS nft_contract_address + , ta.contract_address AS project_contract_address + , ta.evt_tx_hash AS tx_hash + , ta.evt_block_number AS block_number + , ta.evt_index + , ta.strategy + FROM {{ source('looksrare_ethereum','LooksRareExchange_evt_TakerAsk') }} ta + {% if is_incremental() %} + WHERE ta.evt_block_time >= date_trunc("day", NOW() - interval '1 week') + {% endif %} + + UNION ALL + + SELECT tb.evt_block_time AS block_time + , tb.tokenId AS nft_token_id + , tb.amount AS nft_amount + , CASE WHEN tb.strategy='0x58d83536d3efedb9f7f2a1ec3bdaad2b1a4dd98c' THEN 'Private Sale' ELSE 'Offer Accepted' END AS trade_category + , tb.maker AS seller + , tb.taker AS buyer + , tb.price AS price_raw + , tb.currency AS currency_contract + , tb.collection AS nft_contract_address + , tb.contract_address AS project_contract_address + , tb.evt_tx_hash AS tx_hash + , tb.evt_block_number AS block_number + , tb.evt_index + , tb.strategy + FROM {{ source('looksrare_ethereum','LooksRareExchange_evt_TakerBid') }} tb + {% if is_incremental() %} + WHERE tb.evt_block_time >= date_trunc("day", NOW() - interval '1 week') + {% endif %} + ) + ) + +, royalties AS ( + SELECT evt_block_time AS block_time + , evt_tx_hash AS tx_hash + , evt_index + , collection AS nft_contract_address + , tokenId AS nft_token_id + , amount + , royaltyRecipient + , ROW_NUMBER() OVER (PARTITION BY evt_tx_hash, collection, tokenId ORDER BY evt_index ASC) AS id + FROM {{ source('looksrare_ethereum','LooksRareExchange_evt_RoyaltyPayment') }} + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", NOW() - interval '1 week') + {% endif %} + ) + +, platform_fees AS ( + SELECT distinct contract_address + , output_0/100 AS fee_percentage + FROM {{ source('looksrare_ethereum','StrategyStandardSaleForFixedPrice_call_viewProtocolFee') }} + UNION ALL + SELECT distinct contract_address + , output_0/100 AS fee_percentage + FROM {{ source('looksrare_ethereum','StrategyAnyItemFromCollectionForFixedPrice_call_viewProtocolFee') }} + UNION ALL + SELECT distinct contract_address + , output_0/100 AS fee_percentage + FROM {{ source('looksrare_ethereum','StrategyPrivateSale_call_viewProtocolFee') }} + UNION ALL + SELECT distinct contract_address + , output_0/100 AS fee_percentage + FROM {{ source('looksrare_ethereum','StrategyStandardSaleForFixedPriceV1B_call_viewProtocolFee') }} + UNION ALL + SELECT distinct contract_address + , output_0/100 AS fee_percentage + FROM {{ source('looksrare_ethereum','StrategyAnyItemFromCollectionForFixedPriceV1B_call_viewProtocolFee') }} + ) + + +SELECT + date_trunc('day', lr.block_time) AS block_date +, lr.block_time +, lr.block_number +, lr.tx_hash +, lr.project_contract_address +, lr.trade_category +, 'secondary' as trade_type +, lr.nft_contract_address +, lr.nft_token_id +, lr.nft_amount +, lr.buyer +, lr.seller +, lr.currency_contract +, CAST(lr.price_raw AS DECIMAL(38,0)) AS price_raw +, CAST(COALESCE((pf.fee_percentage/100)*lr.price_raw, 0) as DECIMAL(38,0)) AS platform_fee_amount_raw +, CAST(COALESCE(roy.amount, 0) AS DECIMAL(38,0)) AS royalty_fee_amount_raw +, cast(null as varchar(1)) as platform_fee_address +, roy.royaltyRecipient AS royalty_fee_address +, lr.evt_index AS sub_tx_trade_id +FROM looksrare_trades lr +LEFT JOIN royalties roy ON roy.block_time=lr.block_time + AND roy.tx_hash=lr.tx_hash + AND roy.nft_contract_address=lr.nft_contract_address + AND roy.nft_token_id=lr.nft_token_id + AND roy.id = lr.id +LEFT JOIN platform_fees pf ON pf.contract_address=lr.strategy diff --git a/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..88adb281ee4 --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades_legacy.sql @@ -0,0 +1,86 @@ +{{ config( + schema = 'looksrare_v2_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} + +{% set looksrare_v2_start_date = '2023-04-01' %} + +WITH looksrare_v2_trades AS ( + SELECT l.evt_block_time AS block_time + , l.evt_block_number AS block_number + , 'Offer Accepted' AS trade_category + , l.feeAmounts[0]+l.feeAmounts[1]+l.feeAmounts[2] AS price_raw + , l.askUser AS seller + , l.bidUser AS buyer + , l.collection AS nft_contract_address + , cast(element_at(l.amounts, 1) as DECIMAL(38,0)) AS nft_amount + , l.currency + , l.itemIds[0] AS nft_token_id + , l.contract_address AS project_contract_address + , l.evt_tx_hash AS tx_hash + , l.evt_index + , l.feeAmounts[1] AS royalty_fee_amount_raw + , l.feeAmounts[2] AS platform_fee_amount_raw + , CASE WHEN l.feeRecipients[1]!='0x0000000000000000000000000000000000000000' THEN l.feeRecipients[1] END AS royalty_fee_address + , get_json_object(l.nonceInvalidationParameters, '$.orderHash') AS order_hash + FROM {{ source('looksrare_v2_ethereum','LooksRareProtocol_evt_TakerAsk') }} l + {% if is_incremental() %} + WHERE l.evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + WHERE l.evt_block_time >= '{{looksrare_v2_start_date}}' + {% endif %} + + UNION ALL + + SELECT l.evt_block_time AS block_time + , l.evt_block_number AS block_number + , 'Buy' AS trade_category + , l.feeAmounts[0]+l.feeAmounts[1]+l.feeAmounts[2] AS price_raw + , l.feeRecipients[0] AS seller + , l.bidUser AS buyer + , l.collection AS nft_contract_address + , element_at(l.amounts, 1) AS nft_amount + , l.currency + , l.itemIds[0] AS nft_token_id + , l.contract_address AS project_contract_address + , l.evt_tx_hash AS tx_hash + , l.evt_index + , l.feeAmounts[1] AS royalty_fee_amount_raw + , l.feeAmounts[2] AS platform_fee_amount_raw + , CASE WHEN l.feeRecipients[1]!='0x0000000000000000000000000000000000000000' THEN l.feeRecipients[1] END AS royalty_fee_address + , get_json_object(l.nonceInvalidationParameters, '$.orderHash') AS order_hash + FROM {{ source('looksrare_v2_ethereum','LooksRareProtocol_evt_TakerBid') }} l + {% if is_incremental() %} + WHERE l.evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + WHERE l.evt_block_time >= '{{looksrare_v2_start_date}}' + {% endif %} + ) + +SELECT + date_trunc('day', block_time) AS block_date +, block_time +, block_number +, tx_hash +, project_contract_address +, nft_contract_address +, nft_token_id +, nft_amount +, trade_category +, 'secondary' AS trade_type +, buyer +, seller +, currency AS currency_contract +, CAST(price_raw as DECIMAL(38,0)) as price_raw +, CAST(platform_fee_amount_raw as DECIMAL(38,0)) as platform_fee_amount_raw +, CAST(royalty_fee_amount_raw as DECIMAL(38,0)) as royalty_fee_amount_raw +, royalty_fee_address +, CAST(null as VARCHAR(1)) as platform_fee_address +, evt_index as sub_tx_trade_id +FROM looksrare_v2_trades diff --git a/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..b962065d04c --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades_legacy.sql @@ -0,0 +1,270 @@ +{{ config( + schema = 'sudoswap_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} +--base table CTEs +WITH + pairs_created as ( + SELECT + _nft as nftcontractaddress + , _initialNFTIDs as nft_ids + , _fee as initialfee + , _assetRecipient as asset_recip + , output_pair as pair_address + , call_block_time as block_time + , contract_address as protocolfee_recipient -- the factory used to create the pair is the protocol fee recipient + FROM {{ source('sudo_amm_ethereum','LSSVMPairFactory_call_createPairETH') }} + WHERE call_success + ) + + , swaps as ( + SELECT + * + FROM ( + SELECT + contract_address + , call_tx_hash + , call_trace_address + , call_block_time + , call_block_number + , call_success + , tokenRecipient as trade_recipient + , 'Sell' as trade_category + , isRouter as called_from_router + , routerCaller as router_caller + FROM {{ source('sudo_amm_ethereum','LSSVMPair_general_call_swapNFTsForToken') }} + WHERE call_success = true + {% if is_incremental() %} + -- this filter will only be applied on an incremental run. We only want to update with new swaps. + AND call_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + + UNION ALL + SELECT + contract_address + , call_tx_hash + , call_trace_address + , call_block_time + , call_block_number + , call_success + , nftRecipient as trade_recipient + , 'Buy' as trade_category + , isRouter as called_from_router + , routerCaller as router_caller + FROM {{ source('sudo_amm_ethereum','LSSVMPair_general_call_swapTokenForAnyNFTs') }} + WHERE call_success = true + {% if is_incremental() %} + -- this filter will only be applied on an incremental run. We only want to update with new swaps. + AND call_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + + UNION ALL + SELECT + contract_address + , call_tx_hash + , call_trace_address + , call_block_time + , call_block_number + , call_success + , nftRecipient as trade_recipient + , 'Buy' as trade_category + , isRouter as called_from_router + , routerCaller as router_caller + FROM {{ source('sudo_amm_ethereum','LSSVMPair_general_call_swapTokenForSpecificNFTs') }} + WHERE call_success = true + {% if is_incremental() %} + -- this filter will only be applied on an incremental run. We only want to update with new swaps. + AND call_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + ) s + ) + + -- this join should be removed in the future when more call trace info is added to the _call_ tables, we need the call_from field to track down the eth traces. + , swaps_with_calldata as ( + select s.* + , tr.from as call_from + , CASE WHEN called_from_router = true THEN tr.from ELSE tr.to END as project_contract_address -- either the router or the pool if called directly + from swaps s + inner join {{ source('ethereum', 'traces') }} tr + ON tr.success and s.call_block_number = tr.block_number and s.call_tx_hash = tr.tx_hash and s.call_trace_address = tr.trace_address + {% if is_incremental() %} + -- this filter will only be applied on an incremental run. We only want to update with new swaps. + AND tr.block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + AND tr.block_time >= '2022-4-1' + {% endif %} + ) + + + ,pool_fee_update as ( + SELECT + * + FROM {{ source('sudo_amm_ethereum','LSSVMPair_general_evt_FeeUpdate') }} + ) + + ,protocol_fee_update as ( + SELECT + * + FROM {{ source('sudo_amm_ethereum','LSSVMPairFactory_evt_ProtocolFeeMultiplierUpdate') }} + ) + + ,asset_recipient_update as ( + SELECT + * + FROM {{ source('sudo_amm_ethereum','LSSVMPair_general_evt_AssetRecipientChange') }} + ) + +--logic CTEs + ,swaps_w_fees as ( + SELECT + * + FROM ( + SELECT + call_tx_hash + , call_block_time + , call_block_number + , contract_address as pair_address + , call_trace_address + , call_from + , router_caller + , pool_fee + , protocolfee + , protocolfee_recipient + , trade_category + , nftcontractaddress + , asset_recip + , trade_recipient + , project_contract_address + , row_number() OVER (partition by call_tx_hash, contract_address, call_trace_address order by fee_update_time desc, protocolfee_update_time desc, asset_recip_update_time desc) as ordering + FROM ( + SELECT + swaps.* + , COALESCE(fu.newfee, pc.initialfee)/1e18 as pool_fee --most recent pool_fee, depends on bonding curve to implement it correctly. See explanation in fee table schema. + , COALESCE(fu.evt_block_time, pc.block_time) as fee_update_time + , pfu.newMultiplier/1e18 as protocolfee --most recent protocolfee, depends on bonding curve to implement it correctly. See explanation in fee table schema. + , pfu.evt_block_time as protocolfee_update_time + , pc.protocolfee_recipient + , pc.nftcontractaddress + , coalesce(aru.a, pc.asset_recip) as asset_recip + , coalesce(aru.evt_block_time, pc.block_time) as asset_recip_update_time + FROM swaps_with_calldata swaps + JOIN pairs_created pc ON pc.pair_address = contract_address --remember swaps from other NFT addresses won't appear! + -- we might need to do these joins separately since we're exploding into a lot of rows.. + -- should not matter a lot since # of changes per pool should be small + LEFT JOIN pool_fee_update fu ON swaps.call_block_time >= fu.evt_block_time AND swaps.contract_address = fu.contract_address + LEFT JOIN protocol_fee_update pfu ON swaps.call_block_time >= pfu.evt_block_time + LEFT JOIN asset_recipient_update aru on swaps.call_block_time >= aru.evt_block_time AND swaps.contract_address = aru.contract_address + ) a + ) b + WHERE ordering = 1 --we want to keep the most recent pool_fee and protocol fee for each individual call (trade) + ) + + ,swaps_w_traces as ( + -- we traces to get NFT and ETH transfer data because sudoswap doesn't emit any data in events for swaps, so we have to piece it together manually based on trace_address. + SELECT + sb.call_block_time + , sb.call_block_number + , sb.trade_category + , SUM( + CASE WHEN sb.trade_category = 'Buy' -- caller buys, AMM sells + THEN ( + CASE WHEN tr.from = sb.call_from THEN value -- amount of ETH payed + WHEN (tr.to = sb.call_from AND sb.call_from != sb.asset_recip) THEN -value --refunds unless the caller is also the asset recipient, no way to discriminate there. + ELSE 0 END) + ELSE ( -- caller sells, AMM buys + CASE WHEN tr.from = sb.pair_address THEN value -- all ETH leaving the pool, nothing should be coming in on a sell. + ELSE 0 END) + END ) as trade_price -- what the buyer paid (incl all fees) + , SUM( + CASE WHEN (tr.to = sb.protocolfee_recipient) THEN value + ELSE 0 END + ) as protocol_fee_amount -- what the buyer paid + , ARRAY_AGG(distinct CASE WHEN substring(input,1,10)='0x42842e0e' THEN bytea2numeric_v3(substring(input,139,64)) END) + as nft_token_id + , sb.call_tx_hash + , sb.trade_recipient + , sb.pair_address + , sb.nftcontractaddress + , sb.pool_fee + , sb.protocolfee + , sb.protocolfee_recipient + , project_contract_address + -- these 2 are used for matching the aggregator address, dropped later + , router_caller + , call_from + FROM swaps_w_fees sb + INNER JOIN {{ source('ethereum', 'traces') }} tr + ON tr.type = 'call' + AND tr.call_type = 'call' + AND success + AND tr.block_number = sb.call_block_number + AND tr.tx_hash = sb.call_tx_hash + AND ( + (cardinality(call_trace_address) != 0 AND call_trace_address = slice(tr.trace_address,1,cardinality(call_trace_address))) --either a normal tx where trace address helps us narrow down which subtraces to look at for ETH transfers or NFT transfers. + OR cardinality(call_trace_address) = 0 -- In this case the swap function was called directly, all traces are thus subtraces of that call (like 0x34a52a94fce15c090cc16adbd6824948c731ecb19a39350633590a9cd163658b). + ) + {% if is_incremental() %} + AND tr.block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + {% if not is_incremental() %} + AND tr.block_time >= '2022-4-1' + {% endif %} + GROUP BY 1,2,3,7,8,9,10,11,12,13,14,15,16 + ) + + ,swaps_cleaned as ( + SELECT + date_trunc('DAY', call_block_time) AS block_date + , call_block_time as block_time + , call_block_number as block_number + , nft_token_id + , cardinality(nft_token_id) as number_of_items + , 'secondary' as trade_type + , trade_category + , CASE WHEN trade_category = 'Buy' THEN pair_address --AMM is selling if an NFT is being bought + ELSE trade_recipient + END as seller + , CASE WHEN trade_category = 'Sell' THEN pair_address --AMM is buying if an NFT is being sold + ELSE trade_recipient + END as buyer + , trade_price as price_raw + , '{{ var("ETH_ERC20_ADDRESS") }}' as currency_contract --ETH + , nftcontractaddress as nft_contract_address + , project_contract_address -- This is either the router or the pool address if called directly + , call_tx_hash as tx_hash + , protocol_fee_amount as platform_fee_amount_raw + , protocolfee_recipient + -- trade_price = baseprice + (baseprice*pool_fee) + (baseprice*protocolfee) + , (trade_price-protocol_fee_amount)/(1+pool_fee)*pool_fee as pool_fee_amount_raw + FROM swaps_w_traces + ) + +SELECT + block_date + , block_time + , block_number + , tx_hash + , project_contract_address + , buyer + , seller + , nft_contract_address + , explode(nft_token_id) as nft_token_id --nft.trades prefers each token id be its own row + , cast(1 as int) as nft_amount + , trade_type + , trade_category + , currency_contract + , cast(price_raw/number_of_items as DECIMAL(38,0)) as price_raw + , cast(platform_fee_amount_raw/number_of_items as decimal(38)) as platform_fee_amount_raw + , cast(0 as decimal(38)) as royalty_fee_amount_raw + , cast(pool_fee_amount_raw/number_of_items as decimal(38)) as pool_fee_amount_raw + , cast(protocolfee_recipient as VARCHAR(42)) as platform_fee_address + , cast(null as varchar(1)) as royalty_fee_address + , row_number() over (partition by tx_hash order by nft_token_id) as sub_tx_trade_id +FROM swaps_cleaned + diff --git a/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..5f726a3a80a --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades_legacy.sql @@ -0,0 +1,229 @@ +{{ config( + schema = 'superrare_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} +{% set project_start_date='2018-04-05' %} + +-- raw data table with all sales on superrare platform -- both primary and secondary +with all_superrare_sales as ( + select evt_block_time as block_time + , evt_block_number as block_number + , `_originContract` as contract_address + , `_tokenId` as nft_token_id + , `_seller` as seller + , `_buyer` as buyer + , `_amount` as price_raw + , evt_tx_hash + , '{{ var("ETH_ERC20_ADDRESS") }}' as currency_contract + , evt_index as sub_tx_trade_id + from {{ source('superrare_ethereum','SuperRareMarketAuction_evt_Sold') }} + {% if is_incremental() %} + where evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + + union all + + select evt_block_time + , evt_block_number + , contract_address + , `_tokenId` + , `_seller` + , `_buyer` + , `_amount` + , evt_tx_hash + , '{{ var("ETH_ERC20_ADDRESS") }}' + , evt_index + from {{ source('superrare_ethereum','SuperRare_evt_Sold') }} + {% if is_incremental() %} + where evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + + union all + + select evt_block_time + , evt_block_number + , `_originContract` as contract_address + , `_tokenId` + , `_seller` + , `_bidder` + , `_amount` + , evt_tx_hash + , '{{ var("ETH_ERC20_ADDRESS") }}' + , evt_index + from {{ source('superrare_ethereum','SuperRareMarketAuction_evt_AcceptBid') }} + {% if is_incremental() %} + where evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + + union all + + select evt_block_time + , evt_block_number + , contract_address + , `_tokenId` + , `_seller` + , `_bidder` + , `_amount` + , evt_tx_hash + , '{{ var("ETH_ERC20_ADDRESS") }}' + , evt_index + from {{ source('superrare_ethereum','SuperRare_evt_AcceptBid') }} + {% if is_incremental() %} + where evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + + union all + + select evt_block_time + , evt_block_number + , `_originContract` + , `_tokenId` + , `_seller` + , `_bidder` + , `_amount` + , evt_tx_hash + , `_currencyAddress` + , evt_index + from {{ source('superrare_ethereum','SuperRareBazaar_evt_AcceptOffer') }} + {% if is_incremental() %} + where evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + + union all + + select evt_block_time + , evt_block_number + , `_contractAddress` + , `_tokenId` + , `_seller` + , `_bidder` + , `_amount` + , evt_tx_hash + , `_currencyAddress` + , evt_index + from {{ source('superrare_ethereum','SuperRareBazaar_evt_AuctionSettled') }} + {% if is_incremental() %} + where evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + + union all + + select evt_block_time + , evt_block_number + , `_originContract` + , `_tokenId` + , `_seller` + , `_buyer` + , `_amount` + , evt_tx_hash + , `_currencyAddress` + , evt_index + from {{ source('superrare_ethereum','SuperRareBazaar_evt_Sold') }} + {% if is_incremental() %} + where evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + + union all + + -- Superrare AuctionHouse (not decoded) + select block_time + , block_number + , concat('0x',substring(topic2 from 27 for 40)) as contract_address + , bytea2numeric_v3(substring(topic4 from 3)) as nft_token_id + , concat('0x',substring(data from 27 for 40)) as seller + , concat('0x',substring(topic3 from 27 for 40)) as buyer + , bytea2numeric_v3(substring(data from 67 for 64)) as price_raw + , tx_hash + , '{{ var("ETH_ERC20_ADDRESS") }}' + , index + from {{ source('ethereum','logs') }} + where contract_address = lower('0x8c9f364bf7a56ed058fc63ef81c6cf09c833e656') + and topic1 = lower('0xea6d16c6bfcad11577aef5cc6728231c9f069ac78393828f8ca96847405902a9') + {% if is_incremental() %} + and block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + and block_time >= '{{ project_start_date }}' + {% endif %} + + union all + + -- Superrare Marketplace (not decoded) + select block_time + , block_number + , concat('0x',substring(topic2 from 27 for 40)) as contract_address + , bytea2numeric_v3(substring(data from 67 for 64)) as nft_token_id + , concat('0x',substring(topic4 from 27 for 40)) as seller + , concat('0x',substring(topic3 from 27 for 40)) as buyer + , bytea2numeric_v3(substring(data from 3 for 64)) as price_raw + , tx_hash + , '{{ var("ETH_ERC20_ADDRESS") }}' + , index + from {{ source('ethereum','logs') }} + where contract_address = lower('0x65b49f7aee40347f5a90b714be4ef086f3fe5e2c') + and topic1 in (lower('0x2a9d06eec42acd217a17785dbec90b8b4f01a93ecd8c127edd36bfccf239f8b6') + , lower('0x5764dbcef91eb6f946584f4ea671217c686fa7e858ce4f9f42d08422b86556a9') + ) + {% if is_incremental() %} + and block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + and block_time >= '{{ project_start_date }}' + {% endif %} +) + +SELECT + cast(date_trunc('day', a.block_time) AS date) AS block_date, + a.block_time, + a.block_number, + a.nft_token_id, + 1 as nft_amount, + 'Buy' as trade_category, + a.seller, + a.buyer, + CAST(a.price_raw AS DECIMAL(38,0)) as price_raw, + a.currency_contract, + a.contract_address as nft_contract_address, + cast(NULL as varchar(1)) as project_contract_address, + a.evt_tx_hash as tx_hash, + case + when a.seller = coalesce(minter.to, minter_superrare.to) + then 'primary' + else 'secondary' + end as trade_type, + case + when a.seller = coalesce(minter.to, minter_superrare.to) + then cast(ROUND((0.03+0.15) * (a.price_raw)) as decimal(38)) -- superrare takes fixed 3% fee + 15% commission on primary sales + else cast(ROUND(0.03 * (a.price_raw)) as decimal(38)) -- fixed 3% fee + end as platform_fee_amount_raw, + case + when a.seller = coalesce(minter.to, minter_superrare.to) + then cast(0 as decimal(38)) + else cast(ROUND(0.10 * (a.price_raw)) as decimal(38)) -- fixed 10% royalty fee on secondary sales + end as royalty_fee_amount_raw, + cast(NULL as varchar(1)) as royalty_fee_address, + cast(NULL as varchar(1)) as platform_fee_address, + sub_tx_trade_id +from all_superrare_sales a +left join {{ source('erc721_ethereum','evt_transfer') }} minter on minter.contract_address = a.contract_address + and minter.tokenId = a.nft_token_id + and minter.from = '0x0000000000000000000000000000000000000000' + {% if is_incremental() %} + and minter.evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + and minter.evt_block_time >= '{{ project_start_date }}' + {% endif %} + +left join {{ source('erc20_ethereum','evt_transfer') }} minter_superrare on minter_superrare.contract_address = a.contract_address + and minter_superrare.value = a.nft_token_id + and minter_superrare.from = '0x0000000000000000000000000000000000000000' + {% if is_incremental() %} + and minter_superrare.evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + and minter_superrare.evt_block_time >= '{{ project_start_date }}' + {% endif %} + diff --git a/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..5922e60c026 --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades_legacy.sql @@ -0,0 +1,84 @@ +{{ config( + schema = 'x2y2_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} + +{%- set project_start_date = '2022-02-04' %} +{%- set fee_management_addr = '0xd823c605807cc5e6bd6fc0d7e4eea50d3e2d66cd' %} + + +WITH src_evt_inventory as ( + SELECT + date_trunc('day',evt_block_time) as block_date + ,evt_block_time as block_time + ,evt_block_number as block_number + ,evt_tx_hash as tx_hash + ,contract_address as project_contract_address + ,case when intent = 1 then taker else maker end as buyer + ,case when intent = 1 then maker else taker end as seller + ,'0x' || substring(get_json_object(inv.item, '$.data'), 155, 40) as nft_contract_address + ,bytea2numeric_v3(substring(get_json_object(inv.item, '$.data'), 195,64)) as nft_token_id + ,CAST(1 AS DECIMAL(38,0)) AS nft_amount + ,case when intent = 1 then 'Buy' else 'Offer Accepted' end as trade_category + ,'secondary' as trade_type + ,currency as currency_contract + ,get_json_object(inv.item, '$.price') as price_raw + ,get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.to') as fees_0_to + ,get_json_object(get_json_object(inv.detail, '$.fees[1]'), '$.to') as fees_1_to + ,CASE WHEN get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.to') ='{{fee_management_addr}}' + THEN get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.percentage') + ELSE 0 + END as platform_fee_percentage + ,CASE WHEN get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.to') ='{{fee_management_addr}}' + THEN (COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[1]'), '$.percentage'), 0) + +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[2]'), '$.percentage'), 0) + +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[3]'), '$.percentage'), 0) + +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[4]'), '$.percentage'), 0)) + ELSE ( COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.percentage'), 0) + +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[1]'), '$.percentage'), 0) + +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[2]'), '$.percentage'), 0) + +COALESCE(get_json_object(get_json_object(inv.detail, '$.fees[3]'), '$.percentage'), 0)) + END as royalty_fee_percentage + ,'{{fee_management_addr}}' as platform_fee_address + ,CASE WHEN get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.to')='{{fee_management_addr}}' + THEN get_json_object(get_json_object(inv.detail, '$.fees[1]'), '$.to') + ELSE get_json_object(get_json_object(inv.detail, '$.fees[0]'), '$.to') + END AS royalty_fee_address + ,evt_index as sub_tx_trade_id + FROM {{ source('x2y2_ethereum','X2Y2_r1_evt_EvInventory') }} inv + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + {% else %} + WHERE evt_block_time >= '{{project_start_date}}' + {% endif %} +) + +-- results +SELECT + block_date +, block_time +, block_number +, tx_hash +, project_contract_address +, buyer +, seller +, nft_contract_address +, nft_token_id +, nft_amount +, trade_type +, trade_category +, currency_contract +, price_raw +, CAST(COALESCE(price_raw*platform_fee_percentage/1e6, 0) as decimal(38)) AS platform_fee_amount_raw +, CAST(COALESCE(price_raw*royalty_fee_percentage/1e6, 0) as decimal(38)) AS royalty_fee_amount_raw +, platform_fee_address +, royalty_fee_address +, sub_tx_trade_id +FROM src_evt_inventory + diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..669677c09fe --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades_legacy.sql @@ -0,0 +1,48 @@ +{{ config( + schema = 'zora_v1_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} + +SELECT + date_trunc('day',bf.evt_block_time) AS block_date + , bf.evt_block_time AS block_time + , bf.evt_block_number AS block_number + , bf.contract_address AS project_contract_address + , bf.evt_tx_hash AS tx_hash + , '0xabefbc9fd2f806065b4f3c237d4b59d9a97bcac7' AS nft_contract_address -- hardcoded ZORA address + , bf.tokenId AS nft_token_id + , CAST(1 as INT) as nft_amount + , 'Offer accepted' AS trade_category + , CASE WHEN mt.from = mint.to + THEN 'primary' + ELSE 'secondary' + END AS trade_type + , get_json_object(bf.bid, '$.bidder') AS buyer + , mt.from AS seller + , CAST(get_json_object(bf.bid, '$.amount') as DECIMAL(38)) AS price_raw + , get_json_object(bf.bid, '$.currency') AS currency_contract + , CAST(0 as DECIMAL(38)) AS platform_fee_amount_raw + , CAST(0 as DECIMAL(38)) AS royalty_fee_amount_raw + , CAST(NULL as VARCHAR(1)) AS platform_fee_address + , CAST(NULL as VARCHAR(1)) AS royalty_fee_address + , bf.evt_index as sub_tx_trade_id +FROM {{ source('zora_ethereum','Market_evt_BidFinalized') }} bf +LEFT JOIN {{ source('zora_ethereum','Media_evt_Transfer') }} mint + ON mint.from = '0x0000000000000000000000000000000000000000' + AND mint.tokenId = bf.tokenId +LEFT JOIN {{ source('zora_ethereum','Media_evt_Transfer') }} mt + ON bf.evt_block_number = mt.evt_block_number + AND bf.evt_tx_hash = mt.evt_tx_hash + AND bf.tokenId = mt.tokenId + AND get_json_object(bf.bid, '$.bidder') = mt.to +WHERE get_json_object(bf.bid, '$.bidder') != '0xe468ce99444174bd3bbbed09209577d25d1ad673' -- these are sells through the auction house which are included in V2 +{% if is_incremental() %} +AND bf.evt_block_time >= date_trunc("day", now() - interval '1 week') +AND mt.evt_block_time >= date_trunc("day", now() - interval '1 week') +{% endif %} diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..c955741f96a --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades_legacy.sql @@ -0,0 +1,35 @@ +{{ config( + schema = 'zora_v2_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} + +SELECT + date_trunc('day',evt_block_time) AS block_date + , evt_block_time AS block_time + , evt_block_number AS block_number + , contract_address AS project_contract_address + , evt_tx_hash AS tx_hash + , tokenContract AS nft_contract_address + , tokenId AS nft_token_id + , CAST(1 as INT) as nft_amount + , 'Auction' AS trade_category + , 'secondary'AS trade_type + , winner AS buyer + , tokenOwner AS seller + , CAST(amount+curatorFee as DECIMAL(38)) AS price_raw + , auctionCurrency AS currency_contract + , CAST(0 as DECIMAL(38)) AS platform_fee_amount_raw + , CAST(0 as DECIMAL(38)) AS royalty_fee_amount_raw + , CAST(NULL as VARCHAR(1)) AS platform_fee_address + , CAST(NULL as VARCHAR(1)) AS royalty_fee_address + , evt_index as sub_tx_trade_id +FROM {{ source('zora_ethereum','AuctionHouse_evt_AuctionEnded') }} +{% if is_incremental() %} +WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') +{% endif %} diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades_legacy.sql new file mode 100644 index 00000000000..183c924b7a7 --- /dev/null +++ b/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades_legacy.sql @@ -0,0 +1,177 @@ +{{ config( + schema = 'zora_v3_ethereum', + alias = alias('base_trades', legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_number','tx_hash','sub_tx_trade_id'], + ) +}} + +WITH v3_trades as ( + SELECT + evt_block_time AS block_time + , evt_block_number AS block_number + , get_json_object(a, '$.tokenId') AS nft_token_id + , 'Offer Accepted' AS trade_category + , userA AS seller + , userB AS buyer + , get_json_object(b, '$.amount') AS price_raw + , get_json_object(b, '$.tokenContract') AS currency_contract + , get_json_object(a, '$.tokenContract') AS nft_contract_address + , contract_address AS project_contract_address + , evt_tx_hash AS tx_hash + , evt_index AS sub_tx_trade_id + FROM ( + SELECT evt_block_time, evt_block_number, evt_tx_hash, evt_index, contract_address, userA, a, userB, b + FROM {{ source('zora_v3_ethereum','OffersV1_evt_ExchangeExecuted') }} + UNION ALL SELECT evt_block_time, evt_block_number, evt_tx_hash, evt_index, contract_address, userA, a, userB, b + FROM {{ source('zora_v3_ethereum','AsksV1_0_evt_ExchangeExecuted') }} + UNION ALL SELECT evt_block_time, evt_block_number, evt_tx_hash, evt_index, contract_address, userA, a, userB, b + FROM {{ source('zora_v3_ethereum','AsksV1_1_evt_ExchangeExecuted') }} + ) + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + UNION ALL + SELECT + evt_block_time AS block_time + , evt_block_number AS block_number + , tokenId AS nft_token_id + , 'Buy' AS trade_category + , get_json_object(auction, '$.seller') AS seller + , get_json_object(auction, '$.highestBidder') AS buyer + , get_json_object(auction, '$.highestBid') AS price_raw + , coalesce(get_json_object(auction, '$.currency'),'0x0000000000000000000000000000000000000000') AS currency_contract + , tokenContract AS nft_contract_address + , contract_address AS project_contract_address + , evt_tx_hash AS tx_hash + , evt_index AS sub_tx_trade_id + FROM ( + SELECT evt_block_time, evt_block_number, evt_tx_hash, evt_index, tokenId, auction, tokenContract, contract_address + FROM {{ source('zora_v3_ethereum','ReserveAuctionFindersEth_evt_AuctionEnded') }} + UNION ALL SELECT evt_block_time, evt_block_number, evt_tx_hash, evt_index, tokenId, auction, tokenContract, contract_address + FROM {{ source('zora_v3_ethereum','ReserveAuctionFindersErc20_evt_AuctionEnded') }} + UNION ALL SELECT evt_block_time, evt_block_number, evt_tx_hash, evt_index, tokenId, auction, tokenContract, contract_address + FROM {{ source('zora_v3_ethereum','ReserveAuctionCoreEth_evt_AuctionEnded') }} + UNION ALL SELECT evt_block_time, evt_block_number, evt_tx_hash, evt_index, tokenId, auction, tokenContract, contract_address + FROM {{ source('zora_v3_ethereum','ReserveAuctionCoreErc20_evt_AuctionEnded') }} + UNION ALL SELECT evt_block_time, evt_block_number, evt_tx_hash, evt_index, tokenId, auction, tokenContract, contract_address + FROM {{ source('zora_v3_ethereum','ReserveAuctionListingEth_evt_AuctionEnded') }} + UNION ALL SELECT evt_block_time, evt_block_number, evt_tx_hash, evt_index, tokenId, auction, tokenContract, contract_address + FROM {{ source('zora_v3_ethereum','ReserveAuctionListingErc20_evt_AuctionEnded') }} + ) + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + UNION ALL + SELECT + evt_block_time AS block_time + , evt_block_number AS block_number + , tokenId AS nft_token_id + , 'Private Sale' AS trade_category + , get_json_object(ask, '$.seller') AS seller + , get_json_object(ask, '$.buyer') AS buyer + , get_json_object(ask, '$.price') AS price_raw + , '0x0000000000000000000000000000000000000000' AS currency_contract + , tokenContract AS nft_contract_address + , contract_address AS project_contract_address + , evt_tx_hash AS tx_hash + , evt_index AS sub_tx_trade_id + FROM {{ source('zora_v3_ethereum','AsksPrivateEth_evt_AskFilled') }} + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + UNION ALL + SELECT + evt_block_time AS block_time + , evt_block_number AS block_number + , tokenId AS nft_token_id + , 'Buy' AS trade_category + , seller AS seller + , buyer AS buyer + , price AS price_raw + , '0x0000000000000000000000000000000000000000' AS currency_contract + , tokenContract AS nft_contract_address + , contract_address AS project_contract_address + , evt_tx_hash AS tx_hash + , evt_index AS sub_tx_trade_id + FROM {{ source('zora_v3_ethereum','AsksCoreEth_evt_AskFilled') }} + {% if is_incremental() %} + WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + ) + +, royalty_payouts as ( + SELECT + evt_block_time + ,evt_tx_hash + ,tokenContract + ,tokenId + ,sum(cast(amount as decimal(38))) as royalty_fee_amount_raw + ,case when count(distinct recipient) = 1 + then min(recipient) + else cast(null as varchar(1)) + end as royalty_fee_address + FROM ( + SELECT * FROM ( + SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','OffersV1_evt_RoyaltyPayout') }} + UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','AsksV1_0_evt_RoyaltyPayout') }} + UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','AsksV1_1_evt_RoyaltyPayout') }} + UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','AsksPrivateEth_evt_RoyaltyPayout') }} + UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','AsksCoreEth_evt_RoyaltyPayout') }} + UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','ReserveAuctionCoreEth_evt_RoyaltyPayout') }} + UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','ReserveAuctionCoreErc20_evt_RoyaltyPayout') }} + UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','ReserveAuctionFindersEth_evt_RoyaltyPayout') }} + UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','ReserveAuctionFindersErc20_evt_RoyaltyPayout') }} + UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','ReserveAuctionListingEth_evt_RoyaltyPayout') }} + UNION ALL SELECT evt_block_time, evt_tx_hash, tokenContract, tokenId, amount, recipient + FROM {{ source('zora_v3_ethereum','ReserveAuctionListingErc20_evt_RoyaltyPayout') }} + ) + WHERE cast(amount as decimal(38)) > 0 + {% if is_incremental() %} + AND evt_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + ) + GROUP BY 1,2,3,4 +) + +SELECT + date_trunc('day',block_time) as block_date + , block_time + , block_number + , project_contract_address + , tx_hash + , nft_contract_address + , nft_token_id + , CAST(1 as INT) as nft_amount + , trade_category + , 'secondary' as trade_type + , buyer + , seller + , CAST(price_raw as DECIMAL(38)) as price_raw + , currency_contract + , CAST(0 as DECIMAL(38)) AS platform_fee_amount_raw + , CAST(coalesce(roy.royalty_fee_amount_raw,0) as DECIMAL(38)) as royalty_fee_amount_raw + , CAST(NULL as VARCHAR(1)) AS platform_fee_address + , CAST(roy.royalty_fee_address as VARCHAR(42)) as royalty_fee_address + , sub_tx_trade_id +FROM v3_trades +LEFT JOIN royalty_payouts roy + ON block_time = roy.evt_block_time + AND tx_hash = roy.evt_tx_hash + AND nft_contract_address = roy.tokenContract + AND nft_token_id = roy.tokenId + + From a00284617193ac9a1e39aa9e9ed09e166e52f8d8 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Fri, 7 Jul 2023 16:04:46 +0200 Subject: [PATCH 03/22] migrate seeds --- seeds/_sector/nft/_schema.yml | 160 ++++++++++++++++++++++++++-------- 1 file changed, 125 insertions(+), 35 deletions(-) diff --git a/seeds/_sector/nft/_schema.yml b/seeds/_sector/nft/_schema.yml index cfae0341cb2..3599d555b6f 100644 --- a/seeds/_sector/nft/_schema.yml +++ b/seeds/_sector/nft/_schema.yml @@ -3,10 +3,16 @@ version: 2 seeds: - name: archipelago_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 - name: foundation_ethereum_base_trades_seed config: @@ -23,73 +29,157 @@ seeds: - name: superrare_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 - name: blur_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 - name: element_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 - name: x2y2_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 - name: zora_v1_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 - name: zora_v2_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 - name: zora_v3_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 - name: cryptopunks_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 - name: looksrare_v1_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) - nft_token_id: string + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 - name: looksrare_v2_ethereum_base_trades_seed config: + tags: dunesql column_types: - price_raw: decimal(38) - platform_fee_amount_raw: decimal(38) - royalty_fee_amount_raw: decimal(38) - nft_token_id: string + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 + + - name: collectionswap_ethereum_base_trades_seed + config: + tags: dunesql + column_types: + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 + + - name: sudoswap_ethereum_base_trades_seed + config: + tags: dunesql + column_types: + tx_hash: varbinary + buyer: varbinary + seller: varbinary + nft_contract_address: varbinary + price_raw: uint256 + platform_fee_amount_raw: uint256 + royalty_fee_amount_raw: uint256 + nft_token_id: uint256 From 8116d008ecffa84d139157fe293718f4e9df4585 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Fri, 7 Jul 2023 16:07:13 +0200 Subject: [PATCH 04/22] add collectionswap_ethereum_pools --- ...ectionswap_ethereum_base_trades_legacy.sql | 2 +- .../collectionswap_ethereum_pools.sql | 3 +- .../collectionswap_ethereum_pools_legacy.sql | 58 +++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 models/collectionswap/ethereum/collectionswap_ethereum_pools_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades_legacy.sql index 4d21287a72c..bd0f849416f 100644 --- a/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades_legacy.sql +++ b/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades_legacy.sql @@ -85,7 +85,7 @@ base_trades as ( p.nft_contract_address, p.token_address as currency_contract from raw_trades t - left join {{ ref('collectionswap_ethereum_pools') }} p + left join {{ ref('collectionswap_ethereum_pools_legacy') }} p on t.project_contract_address = p.pool_address ) diff --git a/models/collectionswap/ethereum/collectionswap_ethereum_pools.sql b/models/collectionswap/ethereum/collectionswap_ethereum_pools.sql index 423c4905507..bf9222e78df 100644 --- a/models/collectionswap/ethereum/collectionswap_ethereum_pools.sql +++ b/models/collectionswap/ethereum/collectionswap_ethereum_pools.sql @@ -1,6 +1,7 @@ {{ config( schema='collectionswap_ethereum', - alias = 'pools', + tags = ['dunesql'], + alias = alias('pools'), materialized = 'incremental', file_format = 'delta', incremental_strategy = 'merge', diff --git a/models/collectionswap/ethereum/collectionswap_ethereum_pools_legacy.sql b/models/collectionswap/ethereum/collectionswap_ethereum_pools_legacy.sql new file mode 100644 index 00000000000..bc0e241b6fb --- /dev/null +++ b/models/collectionswap/ethereum/collectionswap_ethereum_pools_legacy.sql @@ -0,0 +1,58 @@ +{{ config( + schema='collectionswap_ethereum', + alias = alias('pools', legacy_model=True), + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['pool_address'] + ) +}} + +select + poolAddress as pool_address + ,collection as nft_contract_address + ,token_address + ,evt_tx_hash as create_tx_hash + ,evt_block_time as create_block_time +from {{ source('collectionswap_ethereum','CollectionPoolFactory_evt_NewPool') }} e +inner join ( + select + output_pool + ,get_json_object(params, '$.token') AS token_address + from {{ source('collectionswap_ethereum','CollectionPoolFactory_call_createPoolERC20') }} + where call_success + {% if is_incremental() %} + and call_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + union all + select + output_pool + ,'0x0000000000000000000000000000000000000000' AS token_address + from {{ source('collectionswap_ethereum','CollectionPoolFactory_call_createPoolETH') }} + where call_success + {% if is_incremental() %} + and call_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + union all + select + output_pool + ,get_json_object(params, '$.token') AS token_address + from {{ source('collectionswap_ethereum','CollectionPoolFactory_call_createPoolERC20Filtered') }} + where call_success + {% if is_incremental() %} + and call_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + union all + select + output_pool + ,'0x0000000000000000000000000000000000000000' AS token_address + from {{ source('collectionswap_ethereum','CollectionPoolFactory_call_createPoolETHFiltered') }} + where call_success + {% if is_incremental() %} + and call_block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} +) c +on e.poolAddress = c.output_pool +{% if is_incremental() %} +WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') +{% endif %} From fc0a9191a9fb59bbeb66eabd4c078298868cbb8b Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Fri, 7 Jul 2023 16:11:55 +0200 Subject: [PATCH 05/22] actually migrate the pools --- .../ethereum/collectionswap_ethereum_pools.sql | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/models/collectionswap/ethereum/collectionswap_ethereum_pools.sql b/models/collectionswap/ethereum/collectionswap_ethereum_pools.sql index bf9222e78df..ec850b1453b 100644 --- a/models/collectionswap/ethereum/collectionswap_ethereum_pools.sql +++ b/models/collectionswap/ethereum/collectionswap_ethereum_pools.sql @@ -19,41 +19,41 @@ from {{ source('collectionswap_ethereum','CollectionPoolFactory_evt_NewPool') }} inner join ( select output_pool - ,get_json_object(params, '$.token') AS token_address + ,from_hex(JSON_EXTRACT_SCALAR(params, '$.token')) AS token_address from {{ source('collectionswap_ethereum','CollectionPoolFactory_call_createPoolERC20') }} where call_success {% if is_incremental() %} - and call_block_time >= date_trunc("day", now() - interval '1 week') + and call_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} union all select output_pool - ,'0x0000000000000000000000000000000000000000' AS token_address + ,0x0000000000000000000000000000000000000000 AS token_address from {{ source('collectionswap_ethereum','CollectionPoolFactory_call_createPoolETH') }} where call_success {% if is_incremental() %} - and call_block_time >= date_trunc("day", now() - interval '1 week') + and call_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} union all select output_pool - ,get_json_object(params, '$.token') AS token_address + ,from_hex(JSON_EXTRACT_SCALAR(params, '$.token')) AS token_address from {{ source('collectionswap_ethereum','CollectionPoolFactory_call_createPoolERC20Filtered') }} where call_success {% if is_incremental() %} - and call_block_time >= date_trunc("day", now() - interval '1 week') + and call_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} union all select output_pool - ,'0x0000000000000000000000000000000000000000' AS token_address + ,0x0000000000000000000000000000000000000000 AS token_address from {{ source('collectionswap_ethereum','CollectionPoolFactory_call_createPoolETHFiltered') }} where call_success {% if is_incremental() %} - and call_block_time >= date_trunc("day", now() - interval '1 week') + and call_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} ) c on e.poolAddress = c.output_pool {% if is_incremental() %} -WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') +WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) {% endif %} From f2cc70b095b7f0b01e000df7477d951f695374fa Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Fri, 7 Jul 2023 16:21:36 +0200 Subject: [PATCH 06/22] fix tests --- .../nft/trades/ethereum/platforms/_schema.yml | 26 +++++++++---------- tests/_sector/nft/check_nft_rowcount_blur.sql | 2 +- .../nft/check_nft_rowcount_superrare.sql | 10 +++---- .../nft/check_nft_rowcount_zora_v1.sql | 2 +- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/models/_sector/nft/trades/ethereum/platforms/_schema.yml b/models/_sector/nft/trades/ethereum/platforms/_schema.yml index 4a4da72a21c..7f74e93ff93 100644 --- a/models/_sector/nft/trades/ethereum/platforms/_schema.yml +++ b/models/_sector/nft/trades/ethereum/platforms/_schema.yml @@ -16,7 +16,7 @@ models: - equal_rowcount_with_sources: evt_sources: - source('archipelago_ethereum','ArchipelagoMarket_evt_Trade') - - check_seed_legacy: + - check_seed: seed_file: ref('archipelago_ethereum_base_trades_seed') match_columns: - block_number @@ -44,7 +44,7 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ] - - check_seed_legacy: + - check_seed: seed_file: ref('superrare_ethereum_base_trades_seed') match_columns: - block_number @@ -109,7 +109,7 @@ models: error_if: ">1" evt_sources: - source('cryptopunks_ethereum','CryptoPunksMarket_evt_PunkBought') - - check_seed_legacy: + - check_seed: seed_file: ref('cryptopunks_ethereum_base_trades_seed') match_columns: - block_number @@ -136,7 +136,7 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ] - - check_seed_legacy: + - check_seed: seed_file: ref('blur_ethereum_base_trades_seed') match_columns: - block_number @@ -169,7 +169,7 @@ models: - source('element_ex_ethereum','OrdersFeature_evt_ERC721BuyOrderFilled') - source('element_ex_ethereum','OrdersFeature_evt_ERC1155SellOrderFilled') - source('element_ex_ethereum','OrdersFeature_evt_ERC1155BuyOrderFilled') - - check_seed_legacy: + - check_seed: seed_file: ref('element_ethereum_base_trades_seed') match_columns: - block_number @@ -199,7 +199,7 @@ models: - equal_rowcount_with_sources: evt_sources: - source('x2y2_ethereum','X2Y2_r1_evt_EvInventory') - - check_seed_legacy: + - check_seed: seed_file: ref('x2y2_ethereum_base_trades_seed') match_columns: - block_number @@ -230,7 +230,7 @@ models: evt_sources: - source('looksrare_ethereum','LooksRareExchange_evt_TakerAsk') - source('looksrare_ethereum','LooksRareExchange_evt_TakerBid') - - check_seed_legacy: + - check_seed: seed_file: ref('looksrare_v1_ethereum_base_trades_seed') match_columns: - block_number @@ -261,7 +261,7 @@ models: evt_sources: - source('looksrare_v2_ethereum','LooksRareProtocol_evt_TakerAsk') - source('looksrare_v2_ethereum','LooksRareProtocol_evt_TakerBid') - - check_seed_legacy: + - check_seed: seed_file: ref('looksrare_v2_ethereum_base_trades_seed') match_columns: - block_number @@ -287,7 +287,7 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ] - - check_seed_legacy: + - check_seed: seed_file: ref('zora_v1_ethereum_base_trades_seed') match_columns: - block_number @@ -317,7 +317,7 @@ models: - equal_rowcount_with_sources: evt_sources: - source('zora_ethereum','AuctionHouse_evt_AuctionEnded') - - check_seed_legacy: + - check_seed: seed_file: ref('zora_v2_ethereum_base_trades_seed') match_columns: - block_number @@ -357,7 +357,7 @@ models: - source('zora_v3_ethereum','ReserveAuctionListingErc20_evt_AuctionEnded') - source('zora_v3_ethereum','AsksPrivateEth_evt_AskFilled') - source('zora_v3_ethereum','AsksCoreEth_evt_AskFilled') - - check_seed_legacy: + - check_seed: seed_file: ref('zora_v3_ethereum_base_trades_seed') match_columns: - block_number @@ -384,7 +384,7 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ] - - check_seed_legacy: + - check_seed: seed_file: ref('sudoswap_ethereum_base_trades_seed') match_columns: - block_number @@ -411,7 +411,7 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ] - - check_seed_legacy: + - check_seed: seed_file: ref('collectionswap_ethereum_base_trades_seed') match_columns: - block_number diff --git a/tests/_sector/nft/check_nft_rowcount_blur.sql b/tests/_sector/nft/check_nft_rowcount_blur.sql index e8f0b301061..480542b4105 100644 --- a/tests/_sector/nft/check_nft_rowcount_blur.sql +++ b/tests/_sector/nft/check_nft_rowcount_blur.sql @@ -13,7 +13,7 @@ full outer join union all select count(*) as count_b from {{ source('seaport_ethereum','Seaport_evt_OrderFulfilled') }} where evt_block_time <= (select max(block_time) from {{ ref('blur_ethereum_base_trades')}}) - and zone='0x0000000000d80cfcb8dfcd8b2c4fd9c813482938' + and zone=0x0000000000d80cfcb8dfcd8b2c4fd9c813482938 ) ) on 1=1 diff --git a/tests/_sector/nft/check_nft_rowcount_superrare.sql b/tests/_sector/nft/check_nft_rowcount_superrare.sql index 44d34f425a2..e84c3c7eb49 100644 --- a/tests/_sector/nft/check_nft_rowcount_superrare.sql +++ b/tests/_sector/nft/check_nft_rowcount_superrare.sql @@ -30,14 +30,14 @@ full outer join where evt_block_time <= (select max(block_time) from {{ ref('superrare_ethereum_base_trades')}}) union all select count(*) as count_b from {{ source('ethereum','logs') }} - where contract_address = '0x8c9f364bf7a56ed058fc63ef81c6cf09c833e656' - and topic1 = '0xea6d16c6bfcad11577aef5cc6728231c9f069ac78393828f8ca96847405902a9' + where contract_address = 0x8c9f364bf7a56ed058fc63ef81c6cf09c833e656 + and topic0 = 0xea6d16c6bfcad11577aef5cc6728231c9f069ac78393828f8ca96847405902a9 and block_time <= (select max(block_time) from {{ ref('superrare_ethereum_base_trades')}}) union all select count(*) as count_b from {{ source('ethereum','logs') }} - where contract_address = '0x65b49f7aee40347f5a90b714be4ef086f3fe5e2c' - and topic1 in ('0x2a9d06eec42acd217a17785dbec90b8b4f01a93ecd8c127edd36bfccf239f8b6' - ,'0x5764dbcef91eb6f946584f4ea671217c686fa7e858ce4f9f42d08422b86556a9') + where contract_address = 0x65b49f7aee40347f5a90b714be4ef086f3fe5e2c + and topic0 in (0x2a9d06eec42acd217a17785dbec90b8b4f01a93ecd8c127edd36bfccf239f8b6 + ,0x5764dbcef91eb6f946584f4ea671217c686fa7e858ce4f9f42d08422b86556a9) and block_time <= (select max(block_time) from {{ ref('superrare_ethereum_base_trades')}}) ) ) diff --git a/tests/_sector/nft/check_nft_rowcount_zora_v1.sql b/tests/_sector/nft/check_nft_rowcount_zora_v1.sql index 0c22d5dff7c..c36c31dee22 100644 --- a/tests/_sector/nft/check_nft_rowcount_zora_v1.sql +++ b/tests/_sector/nft/check_nft_rowcount_zora_v1.sql @@ -8,7 +8,7 @@ from full outer join (select count(*) as count_b from {{ source('zora_ethereum','Market_evt_BidFinalized') }} - where get_json_object(bid, '$.bidder') != '0xe468ce99444174bd3bbbed09209577d25d1ad673' + where from_hex(get_json_object(bid, '$.bidder')) != 0xe468ce99444174bd3bbbed09209577d25d1ad673 and evt_block_time <= (select max(block_time) from {{ ref('zora_v1_ethereum_base_trades')}}) ) on 1=1 From a2eac846558d208a5ece3e85add4e6316c838bc3 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Fri, 7 Jul 2023 16:30:38 +0200 Subject: [PATCH 07/22] fix test --- tests/_sector/nft/check_nft_rowcount_zora_v1.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/_sector/nft/check_nft_rowcount_zora_v1.sql b/tests/_sector/nft/check_nft_rowcount_zora_v1.sql index c36c31dee22..013969fba27 100644 --- a/tests/_sector/nft/check_nft_rowcount_zora_v1.sql +++ b/tests/_sector/nft/check_nft_rowcount_zora_v1.sql @@ -8,7 +8,7 @@ from full outer join (select count(*) as count_b from {{ source('zora_ethereum','Market_evt_BidFinalized') }} - where from_hex(get_json_object(bid, '$.bidder')) != 0xe468ce99444174bd3bbbed09209577d25d1ad673 + where from_hex(json_extract_scalar(bid, '$.bidder')) != 0xe468ce99444174bd3bbbed09209577d25d1ad673 and evt_block_time <= (select max(block_time) from {{ ref('zora_v1_ethereum_base_trades')}}) ) on 1=1 From 5fbb9995c16d904a7d7b3fe24b55cf705087ac42 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Fri, 7 Jul 2023 17:09:21 +0200 Subject: [PATCH 08/22] fix blur --- .../trades/ethereum/platforms/blur_ethereum_base_trades.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql index d478cb887bf..bffad0f7099 100644 --- a/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql @@ -36,8 +36,8 @@ regular_blur_sales as ( , bm.contract_address AS project_contract_address , bm.evt_tx_hash AS tx_hash , cast(0 as uint256) AS platform_fee_amount_raw -- Hardcoded 0% platform fee - , CAST(COALESCE(cast(JSON_EXTRACT_SCALAR(bm.buy, '$.price') as uint256)*cast(JSON_EXTRACT_SCALAR(JSON_EXTRACT_SCALAR(bm.sell, '$.fees[1]'), '$.rate') as uint256)/10000, cast(0 as uint256)) AS uint256) AS royalty_fee_amount_raw - , from_hex(JSON_EXTRACT_SCALAR(bm.sell, '$.fees[1].recipient')) AS royalty_fee_address + , CAST(COALESCE(cast(JSON_EXTRACT_SCALAR(bm.buy, '$.price') as uint256)*cast(JSON_EXTRACT_SCALAR(cast(JSON_EXTRACT(bm.sell, '$.fees[0]') as varchar), '$.rate') as uint256)/10000, cast(0 as uint256)) AS uint256) AS royalty_fee_amount_raw + , from_hex(JSON_EXTRACT_SCALAR(cast(json_extract(bm.sell, '$.fees[0]') as varchar), '$.recipient')) AS royalty_fee_address , cast(NULL as varbinary) as platform_fee_address , bm.evt_index as sub_tx_trade_id FROM {{ source('blur_ethereum','BlurExchange_evt_OrdersMatched') }} bm From 7ecdcffbaf0a544b181540fcb95cec73f60bf9ce Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Fri, 7 Jul 2023 17:18:33 +0200 Subject: [PATCH 09/22] fix sudoswap --- .../trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql index bab5c905084..a64802bae6c 100644 --- a/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql @@ -186,7 +186,7 @@ WITH CASE WHEN (tr.to = sb.protocolfee_recipient) THEN cast(value as uint256) ELSE uint256 '0' END ) as protocol_fee_amount -- what the buyer paid - , filter(ARRAY_AGG(distinct CASE WHEN bytearray_substring(input,1,10)=0x42842e0e THEN bytearray_to_uint256(bytearray_substring(input,139,64)) END) + , filter(ARRAY_AGG(distinct CASE WHEN bytearray_substring(input,1,4)=0x42842e0e THEN bytearray_to_uint256(bytearray_substring(input,69,32)) END) , x->x is not null ) as nft_token_id , sb.call_tx_hash From aebe51a88f41534d3fe58e85e728f96aef5da13c Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Fri, 7 Jul 2023 18:04:26 +0200 Subject: [PATCH 10/22] fix sudoswap seed --- seeds/_sector/nft/sudoswap_ethereum_base_trades_seed.csv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/seeds/_sector/nft/sudoswap_ethereum_base_trades_seed.csv b/seeds/_sector/nft/sudoswap_ethereum_base_trades_seed.csv index 7fb35d6ea82..989f47f6331 100644 --- a/seeds/_sector/nft/sudoswap_ethereum_base_trades_seed.csv +++ b/seeds/_sector/nft/sudoswap_ethereum_base_trades_seed.csv @@ -1,7 +1,7 @@ block_number,tx_hash,sub_tx_trade_id,buyer,seller,nft_contract_address,nft_token_id,price_raw,platform_fee_amount_raw,royalty_fee_amount_raw,description 15354581,0x03b4168904dc5d25d3f35be9653c5f459d5a0c829469cf806bf44377991b2df7,1,0x4bfee8eb04d1df1ef8509a92b211436000e5950c,0xf7bcafade91139b428acd9494767a972ebab2e69,0xf10a71e53398bf4e3cf5e2a0cad3c529ad19b834,199,67600000000000000,338000000000000,0,swapNFTsForToken 15349124,0x044e30a995a3e05ec113450321942e71d6bbd017969bc4a4768c0b69ac29ddb5,1,0x1a4fbd328ed410720f7a53cd26aef307ccf5e0e4,0xbede55bdded9c808c1d0e791abe269e61f20cd84,0x881c822cba3325a8692c37c470d2f9654454df84,3935,5519309296786866,29048996298878,0,swapNFTsForToken -15349608,0x7f34d65b0d65198f3c59796c2691bcd73857b5ec6feeb75369c66982a8454895,1,0xc22630f6e3124eccc8156648973461c30b2f84d1,0x8fb854f52e1773edbd07041195e37c1688996c0e,0x8a3590348d1dae26b38676bc07e26f0d024af368,1670,12859658176722586,63978398889167,0,swapTokenForAnyNFTs -15349608,0x7f34d65b0d65198f3c59796c2691bcd73857b5ec6feeb75369c66982a8454895,2,0xc22630f6e3124eccc8156648973461c30b2f84d1,0x8fb854f52e1773edbd07041195e37c1688996c0e,0x8a3590348d1dae26b38676bc07e26f0d024af368,1669,12859658176722586,63978398889167,0,swapTokenForAnyNFTs -15351515,0x01c9a24ac9e47f00d905bc30aa5c4b912be2839e9005553e9171d69d2b0db4c6,1,0xc6257136f3d07716a2d2027b386b5957e79463ee,0x328a54df862cfb6233277be8666c32f2a2d10235,0xb2df5bde8432c83a8a1919cc610fa9aef5e53348,4611,12607499999999956,59190140845070,0,swapTokenForSpecificNFTs -15354961,0x0dbd29b3695e099047dd83e6f2fbc4775c0df35c13747be1e210c3898924cc75,1,0x34cae1d9e2d014b7b9e6295c66c554d7e79713d3,0xed837564e8b67f55a05c0b91cce1509cf09bc592,0xb5455326c6d00aa75d426f8be044d4efaa201e1a,68,9673800000000000,48128358208955,0,swapTokenForSpecificNFTs +15349608,0x7f34d65b0d65198f3c59796c2691bcd73857b5ec6feeb75369c66982a8454895,1,0xc22630f6e3124eccc8156648973461c30b2f84d1,0x8fb854f52e1773edbd07041195e37c1688996c0e,0x8a3590348d1dae26b38676bc07e26f0d024af368,1651,12859658176722585,63978398889167,0,swapTokenForAnyNFTs +15349608,0x7f34d65b0d65198f3c59796c2691bcd73857b5ec6feeb75369c66982a8454895,2,0xc22630f6e3124eccc8156648973461c30b2f84d1,0x8fb854f52e1773edbd07041195e37c1688996c0e,0x8a3590348d1dae26b38676bc07e26f0d024af368,1652,12859658176722585,63978398889167,0,swapTokenForAnyNFTs +15351515,0x01c9a24ac9e47f00d905bc30aa5c4b912be2839e9005553e9171d69d2b0db4c6,1,0xc6257136f3d07716a2d2027b386b5957e79463ee,0x328a54df862cfb6233277be8666c32f2a2d10235,0xb2df5bde8432c83a8a1919cc610fa9aef5e53348,4611,12607499999999957,59190140845070,0,swapTokenForSpecificNFTs +15354961,0x0dbd29b3695e099047dd83e6f2fbc4775c0df35c13747be1e210c3898924cc75,1,0x34cae1d9e2d014b7b9e6295c66c554d7e79713d3,0xed837564e8b67f55a05c0b91cce1509cf09bc592,0xb5455326c6d00aa75d426f8be044d4efaa201e1a,13,9673800000000000,48128358208955,0,swapTokenForSpecificNFTs From 5c9d7c3a05c22164c9a503dd921156b7a22559bb Mon Sep 17 00:00:00 2001 From: 0xRob <83790096+0xRobin@users.noreply.github.com> Date: Mon, 10 Jul 2023 09:28:30 +0200 Subject: [PATCH 11/22] Update models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql Co-authored-by: Jonas Irgens Kylling --- .../ethereum/platforms/archipelago_ethereum_base_trades.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql index 627831b3851..e57c27aa766 100644 --- a/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql @@ -9,7 +9,7 @@ unique_key = ['block_number','tx_hash','sub_tx_trade_id'], ) }} -{% set project_start_date = "cast('2022-6-20' as timestamp)" %} +{% set project_start_date = "timestamp '2022-6-20'" %} WITH trade_events as ( From 8be72b79e6381c28f6985ed4fcd671b450ac8e09 Mon Sep 17 00:00:00 2001 From: 0xRob <83790096+0xRobin@users.noreply.github.com> Date: Mon, 10 Jul 2023 09:28:39 +0200 Subject: [PATCH 12/22] Update models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql Co-authored-by: Jonas Irgens Kylling --- .../nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql index 8e7046cbba8..ed2d40b533f 100644 --- a/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql @@ -23,7 +23,7 @@ WITH src_evt_inventory as ( ,contract_address as project_contract_address ,case when intent = cast(1 as uint256) then taker else maker end as buyer ,case when intent = cast(1 as uint256) then maker else taker end as seller - ,from_hex('0x' || substring(JSON_EXTRACT_SCALAR(inv.item, '$.data'), 155, 40)) as nft_contract_address + ,from_hex(substring(JSON_EXTRACT_SCALAR(inv.item, '$.data'), 155, 40)) as nft_contract_address ,bytearray_to_uint256(from_hex(substring(JSON_EXTRACT_SCALAR(inv.item, '$.data'), 195,64))) as nft_token_id ,CAST(1 AS UINT256) AS nft_amount ,case when intent = cast(1 as uint256) then 'Buy' else 'Offer Accepted' end as trade_category From fd6840274c2b72ad065b34d26cae3935b5d757cc Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Mon, 10 Jul 2023 11:20:36 +0200 Subject: [PATCH 13/22] fix element base trades legacy --- .../element_v1_base_trades_legacy.sql | 115 ++++++++++++++++++ .../element_ethereum_base_trades_legacy.sql | 2 +- 2 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 macros/models/_sector/nft/platforms/element_v1_base_trades_legacy.sql diff --git a/macros/models/_sector/nft/platforms/element_v1_base_trades_legacy.sql b/macros/models/_sector/nft/platforms/element_v1_base_trades_legacy.sql new file mode 100644 index 00000000000..35a1cc371a5 --- /dev/null +++ b/macros/models/_sector/nft/platforms/element_v1_base_trades_legacy.sql @@ -0,0 +1,115 @@ +-- Element NFT trades (re-usable macro for all chains) +{% macro element_v1_base_trades_legacy(erc721_sell_order_filled, erc721_buy_order_filled, erc1155_sell_order_filled, erc1155_buy_order_filled) %} + + +SELECT + date_trunc('day',evt_block_time) as block_date +, evt_block_time AS block_time +, evt_block_number AS block_number +, 'Buy' AS trade_category +, 'secondary' AS trade_type +, erc721Token AS nft_contract_address +, erc721TokenId AS nft_token_id +, 1 AS nft_amount +, taker AS buyer +, maker AS seller +, cast(erc20TokenAmount as decimal(38)) AS price_raw +, CASE WHEN erc20Token='0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' THEN '{{ var("ETH_ERC20_ADDRESS") }}' + ELSE erc20Token END AS currency_contract +, cast(0 as decimal(38)) as platform_fee_amount_raw +, cast(0 as decimal(38)) as royalty_fee_amount_raw +, cast(null as varchar(1)) as platform_fee_address +, cast(null as varchar(1)) as royalty_fee_address +, contract_address AS project_contract_address +, evt_tx_hash AS tx_hash +, evt_index as sub_tx_trade_id +FROM {{ erc721_sell_order_filled }} +{% if is_incremental() %} +WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') +{% endif %} + +UNION ALL + +SELECT + date_trunc('day',evt_block_time) as block_date +, evt_block_time AS block_time +, evt_block_number AS block_number +, 'Sell' AS trade_category +, 'secondary' AS trade_type +, erc721Token AS nft_contract_address +, erc721TokenId AS nft_token_id +, 1 AS nft_amount +, maker AS buyer +, taker AS seller +, cast(erc20TokenAmount as decimal(38)) AS price_raw +, CASE WHEN erc20Token='0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' THEN '{{ var("ETH_ERC20_ADDRESS") }}' + ELSE erc20Token END AS currency_contract +, cast(0 as decimal(38)) as platform_fee_amount_raw +, cast(0 as decimal(38)) as royalty_fee_amount_raw +, cast(null as varchar(1)) as platform_fee_address +, cast(null as varchar(1)) as royalty_fee_address +, contract_address AS project_contract_address +, evt_tx_hash AS tx_hash +, evt_index as sub_tx_trade_id +FROM {{ erc721_buy_order_filled }} +{% if is_incremental() %} +WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') +{% endif %} + +UNION ALL + +SELECT + date_trunc('day',evt_block_time) as block_date +, evt_block_time AS block_time +, evt_block_number AS block_number +, 'Buy' AS trade_category +, 'secondary' AS trade_type +, erc1155Token AS nft_contract_address +, erc1155TokenId AS nft_token_id +, 1 AS nft_amount +, taker AS buyer +, maker AS seller +, cast(erc20FillAmount as decimal(38)) AS price_raw +, CASE WHEN erc20Token='0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' THEN '{{ var("ETH_ERC20_ADDRESS") }}' + ELSE erc20Token END AS currency_contract +, cast(0 as decimal(38)) as platform_fee_amount_raw +, cast(0 as decimal(38)) as royalty_fee_amount_raw +, cast(null as varchar(1)) as platform_fee_address +, cast(null as varchar(1)) as royalty_fee_address +, contract_address AS project_contract_address +, evt_tx_hash AS tx_hash +, evt_index as sub_tx_trade_id +FROM {{ erc1155_buy_order_filled }} +{% if is_incremental() %} +WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') +{% endif %} + +UNION ALL + +SELECT + date_trunc('day',evt_block_time) as block_date +, evt_block_time AS block_time +, evt_block_number AS block_number +, 'Buy' AS trade_category +, 'secondary' AS trade_type +, erc1155Token AS nft_contract_address +, erc1155TokenId AS nft_token_id +, 1 AS nft_amount +, maker AS buyer +, taker AS seller +, cast(erc20FillAmount as decimal(38)) AS price_raw +, CASE WHEN erc20Token='0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' THEN '{{ var("ETH_ERC20_ADDRESS") }}' + ELSE erc20Token END AS currency_contract +, cast(0 as decimal(38)) as platform_fee_amount_raw +, cast(0 as decimal(38)) as royalty_fee_amount_raw +, cast(null as varchar(1)) as platform_fee_address +, cast(null as varchar(1)) as royalty_fee_address +, contract_address AS project_contract_address +, evt_tx_hash AS tx_hash +, evt_index as sub_tx_trade_id +FROM {{ erc1155_sell_order_filled }} +{% if is_incremental() %} +WHERE evt_block_time >= date_trunc("day", now() - interval '1 week') +{% endif %} + +{% endmacro %} diff --git a/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades_legacy.sql index 7ea4b787ff0..006ff931488 100644 --- a/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades_legacy.sql +++ b/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades_legacy.sql @@ -10,7 +10,7 @@ }} WITH base_trades as ( - {{ element_v1_base_trades( + {{ element_v1_base_trades_legacy( erc721_sell_order_filled = source('element_ex_ethereum','OrdersFeature_evt_ERC721SellOrderFilled') , erc721_buy_order_filled = source('element_ex_ethereum','OrdersFeature_evt_ERC721BuyOrderFilled') , erc1155_sell_order_filled = source('element_ex_ethereum','OrdersFeature_evt_ERC1155SellOrderFilled') From 4072f292d98035295a14fb00bdb2a37089c3fec8 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Tue, 11 Jul 2023 12:24:32 +0200 Subject: [PATCH 14/22] migrate nft trades beta setup --- macros/models/_sector/nft/enrich_trades.sql | 23 ++-- .../_sector/nft/enrich_trades_legacy.sql | 124 ++++++++++++++++++ .../models/_sector/nft/port_to_old_schema.sql | 10 +- .../_sector/nft/port_to_old_schema_legacy.sql | 44 +++++++ .../nft/trades/nft_ethereum_trades_beta.sql | 25 ++-- .../nft_ethereum_trades_beta_legacy.sql | 73 +++++++++++ .../nft_ethereum_trades_beta_ported.sql | 3 +- ...nft_ethereum_trades_beta_ported_legacy.sql | 8 ++ models/_sector/nft/trades/nft_events.sql | 4 +- 9 files changed, 281 insertions(+), 33 deletions(-) create mode 100644 macros/models/_sector/nft/enrich_trades_legacy.sql create mode 100644 macros/models/_sector/nft/port_to_old_schema_legacy.sql create mode 100644 models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql create mode 100644 models/_sector/nft/trades/nft_ethereum_trades_beta_ported_legacy.sql diff --git a/macros/models/_sector/nft/enrich_trades.sql b/macros/models/_sector/nft/enrich_trades.sql index a537550f76d..bb2c506b18d 100644 --- a/macros/models/_sector/nft/enrich_trades.sql +++ b/macros/models/_sector/nft/enrich_trades.sql @@ -7,7 +7,6 @@ -- 5. add aggregator columns -- 6. fix buyer or seller for aggregator txs -- 7. calculate platform and royalty rates --- 8. deduplicate based on sub_tx_trade_id WITH base_union as ( {% for nft_model in models %} @@ -33,11 +32,10 @@ WITH base_union as ( royalty_fee_amount_raw, platform_fee_address, -- optional royalty_fee_address, -- optional - sub_tx_trade_id, - row_number() over (partition by tx_hash, sub_tx_trade_id order by tx_hash) as duplicates_rank + sub_tx_trade_id FROM {{ nft_model[2] }} {% if is_incremental() %} - WHERE block_time >= date_trunc("day", now() - interval '1 week') + WHERE block_time >= date_trunc('day', now() - interval '7' day) {% endif %} {% if not loop.last %} UNION ALL @@ -58,8 +56,8 @@ SELECT base.project_contract_address, base.trade_category, base.trade_type, - case when base.buyer = coalesce(agg1.contract_address,agg2.contract_address) then tx.from else base.buyer end as buyer, - case when base.seller = coalesce(agg1.contract_address,agg2.contract_address) then tx.from else base.seller end as seller, + case when base.buyer = coalesce(agg1.contract_address,agg2.contract_address) then tx."from" else base.buyer end as buyer, + case when base.seller = coalesce(agg1.contract_address,agg2.contract_address) then tx."from" else base.seller end as seller, base.nft_contract_address, base.nft_token_id, base.nft_amount, @@ -69,7 +67,7 @@ SELECT base.royalty_fee_amount_raw, base.platform_fee_address, base.royalty_fee_address, - tx.from as tx_from, + tx."from" as tx_from, tx.to as tx_to, nft.name as nft_collection, nft.standard as nft_standard, @@ -80,8 +78,8 @@ SELECT base.price_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as price_usd, base.platform_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as platform_fee_amount_usd, base.royalty_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as royalty_fee_amount_usd, - coalesce(cast(100*base.platform_fee_amount_raw/base.price_raw as double),cast(0.0 as double)) as platform_fee_percentage, - coalesce(cast(100*base.royalty_fee_amount_raw/base.price_raw as double),cast(0.0 as double)) as royalty_fee_percentage, + case when base.price_raw > uint256 '0' then cast(100*base.platform_fee_amount_raw/base.price_raw as double) else double '0' end as platform_fee_percentage, + case when base.price_raw > uint256 '0' then cast(100*base.royalty_fee_amount_raw/base.price_raw as double) else double '0' end as royalty_fee_percentage, coalesce(agg1.contract_address,agg2.contract_address) as aggregator_address, {% if aggregator_markers != null %} coalesce(agg_mark.aggregator_name, agg1.name, agg2.name) as aggregator_name @@ -93,7 +91,7 @@ INNER JOIN {{ transactions_model }} tx ON tx.block_number = base.block_number AND tx.hash = base.tx_hash {% if is_incremental() %} - AND tx.block_time >= date_trunc("day", now() - interval '1 week') + AND tx.block_time >= date_trunc('day', now() - interval '7' day) {% endif %} LEFT JOIN {{ tokens_nft_model }} nft ON nft.contract_address = base.nft_contract_address @@ -104,7 +102,7 @@ ON p.blockchain = base.blockchain AND p.contract_address = base.currency_contract AND p.minute = date_trunc('minute',base.block_time) {% if is_incremental() %} - AND p.minute >= date_trunc("day", now() - interval '1 week') + AND p.minute >= date_trunc('day', now() - interval '7' day) {% endif %} LEFT JOIN {{ aggregators }} agg1 ON (base.buyer = agg1.contract_address @@ -114,9 +112,8 @@ ON agg1.contract_address is null -- only match if agg1 produces no matches, t AND tx.to = agg2.contract_address {% if aggregator_markers != null %} LEFT JOIN {{ aggregator_markers }} agg_mark -ON RIGHT(tx.data, agg_mark.hash_marker_size) = agg_mark.hash_marker +ON bytearray_starts_with(bytearray_reverse(tx.data), bytearray_reverse(agg_mark.hash_marker)) -- eq to end_with() {% endif %} -WHERE base.duplicates_rank = 1 ) diff --git a/macros/models/_sector/nft/enrich_trades_legacy.sql b/macros/models/_sector/nft/enrich_trades_legacy.sql new file mode 100644 index 00000000000..a537550f76d --- /dev/null +++ b/macros/models/_sector/nft/enrich_trades_legacy.sql @@ -0,0 +1,124 @@ +{% macro enrich_trades(blockchain='', models=[], transactions_model=null, tokens_nft_model=null, tokens_erc20_model=null, prices_model=null, aggregators=null, aggregator_markers=null) %} +-- Macro to apply the NFT trades enrichment(s) to base models +-- 1. add transaction information +-- 2. add NFT token information +-- 3. add ERC20 token information + handle ERC20 decimals +-- 4. handle USD columns +-- 5. add aggregator columns +-- 6. fix buyer or seller for aggregator txs +-- 7. calculate platform and royalty rates +-- 8. deduplicate based on sub_tx_trade_id + +WITH base_union as ( + {% for nft_model in models %} + SELECT + '{{ blockchain }}' as blockchain, + '{{ nft_model[0] }}' as project, + '{{ nft_model[1] }}' as project_version, + block_date, + block_time, + block_number, + tx_hash, + project_contract_address, + trade_category, --buy/sell/swap + trade_type, --primary/secondary + buyer, + seller, + nft_contract_address, + nft_token_id, + nft_amount, -- always 1 for erc721 + price_raw, + currency_contract, + platform_fee_amount_raw, + royalty_fee_amount_raw, + platform_fee_address, -- optional + royalty_fee_address, -- optional + sub_tx_trade_id, + row_number() over (partition by tx_hash, sub_tx_trade_id order by tx_hash) as duplicates_rank + FROM {{ nft_model[2] }} + {% if is_incremental() %} + WHERE block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} +), + +enrichments as ( +SELECT + base.blockchain, + base.project, + base.project_version, + base.block_date, + base.block_time, + base.block_number, + base.tx_hash, + base.sub_tx_trade_id, + base.project_contract_address, + base.trade_category, + base.trade_type, + case when base.buyer = coalesce(agg1.contract_address,agg2.contract_address) then tx.from else base.buyer end as buyer, + case when base.seller = coalesce(agg1.contract_address,agg2.contract_address) then tx.from else base.seller end as seller, + base.nft_contract_address, + base.nft_token_id, + base.nft_amount, + base.price_raw, + base.currency_contract, + base.platform_fee_amount_raw, + base.royalty_fee_amount_raw, + base.platform_fee_address, + base.royalty_fee_address, + tx.from as tx_from, + tx.to as tx_to, + nft.name as nft_collection, + nft.standard as nft_standard, + coalesce(erc20.symbol,p.symbol) as currency_symbol, + base.price_raw/pow(10,coalesce(erc20.decimals,p.decimals,18)) as price, + base.platform_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18)) as platform_fee_amount, + base.royalty_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18)) as royalty_fee_amount, + base.price_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as price_usd, + base.platform_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as platform_fee_amount_usd, + base.royalty_fee_amount_raw/pow(10,coalesce(erc20.decimals,p.decimals,18))*p.price as royalty_fee_amount_usd, + coalesce(cast(100*base.platform_fee_amount_raw/base.price_raw as double),cast(0.0 as double)) as platform_fee_percentage, + coalesce(cast(100*base.royalty_fee_amount_raw/base.price_raw as double),cast(0.0 as double)) as royalty_fee_percentage, + coalesce(agg1.contract_address,agg2.contract_address) as aggregator_address, + {% if aggregator_markers != null %} + coalesce(agg_mark.aggregator_name, agg1.name, agg2.name) as aggregator_name + {% else %} + coalesce(agg1.name,agg2.name) as aggregator_name + {% endif %} +FROM base_union base +INNER JOIN {{ transactions_model }} tx +ON tx.block_number = base.block_number + AND tx.hash = base.tx_hash + {% if is_incremental() %} + AND tx.block_time >= date_trunc("day", now() - interval '1 week') + {% endif %} +LEFT JOIN {{ tokens_nft_model }} nft +ON nft.contract_address = base.nft_contract_address +LEFT JOIN {{ tokens_erc20_model }} erc20 +ON erc20.contract_address = base.currency_contract +LEFT JOIN {{ prices_model }} p +ON p.blockchain = base.blockchain + AND p.contract_address = base.currency_contract + AND p.minute = date_trunc('minute',base.block_time) + {% if is_incremental() %} + AND p.minute >= date_trunc("day", now() - interval '1 week') + {% endif %} +LEFT JOIN {{ aggregators }} agg1 +ON (base.buyer = agg1.contract_address + OR base.seller = agg1.contract_address) +LEFT JOIN {{ aggregators }} agg2 +ON agg1.contract_address is null -- only match if agg1 produces no matches, this prevents duplicates + AND tx.to = agg2.contract_address +{% if aggregator_markers != null %} +LEFT JOIN {{ aggregator_markers }} agg_mark +ON RIGHT(tx.data, agg_mark.hash_marker_size) = agg_mark.hash_marker +{% endif %} +WHERE base.duplicates_rank = 1 +) + + +select * from enrichments +{% endmacro %} diff --git a/macros/models/_sector/nft/port_to_old_schema.sql b/macros/models/_sector/nft/port_to_old_schema.sql index eccdf8ddd0f..c64a2bd19f5 100644 --- a/macros/models/_sector/nft/port_to_old_schema.sql +++ b/macros/models/_sector/nft/port_to_old_schema.sql @@ -11,13 +11,13 @@ SELECT price_usd as amount_usd, nft_standard as token_standard, trade_type, - CAST(nft_amount as decimal(38)) as number_of_items, + CAST(nft_amount as uint256) as number_of_items, trade_category, 'Trade' as evt_type, seller, buyer, CAST(price as double) as amount_original, - CAST(price_raw as decimal(38)) as amount_raw, + CAST(price_raw as uint256) as amount_raw, currency_symbol, currency_contract, nft_contract_address, @@ -28,17 +28,17 @@ SELECT block_number, tx_from, tx_to, - CAST(platform_fee_amount_raw as decimal(38)) as platform_fee_amount_raw, + CAST(platform_fee_amount_raw as uint256) as platform_fee_amount_raw, CAST(platform_fee_amount as double) as platform_fee_amount, platform_fee_amount_usd, platform_fee_percentage, royalty_fee_address as royalty_fee_receive_address, currency_symbol as royalty_fee_currency_symbol, - CAST(royalty_fee_amount_raw as decimal(38)) as royalty_fee_amount_raw, + CAST(royalty_fee_amount_raw as uint256) as royalty_fee_amount_raw, CAST(royalty_fee_amount as double) as royalty_fee_amount, royalty_fee_amount_usd, royalty_fee_percentage, - concat(project,'-',cast(block_number as varchar(20)),'-',cast(tx_hash as varchar(66)),'-',cast(sub_tx_trade_id as varchar(20))) as unique_trade_id + concat(project,'-',cast(block_number as varchar),'-',cast(tx_hash as varchar),'-',cast(sub_tx_trade_id as varchar)) as unique_trade_id FROM {{ model }} {% endmacro %} diff --git a/macros/models/_sector/nft/port_to_old_schema_legacy.sql b/macros/models/_sector/nft/port_to_old_schema_legacy.sql new file mode 100644 index 00000000000..eccdf8ddd0f --- /dev/null +++ b/macros/models/_sector/nft/port_to_old_schema_legacy.sql @@ -0,0 +1,44 @@ +{% macro port_to_old_schema(model) %} + +SELECT + blockchain, + project, + project_version as version, + block_date, + block_time, + nft_token_id as token_id, + nft_collection as collection, + price_usd as amount_usd, + nft_standard as token_standard, + trade_type, + CAST(nft_amount as decimal(38)) as number_of_items, + trade_category, + 'Trade' as evt_type, + seller, + buyer, + CAST(price as double) as amount_original, + CAST(price_raw as decimal(38)) as amount_raw, + currency_symbol, + currency_contract, + nft_contract_address, + project_contract_address, + aggregator_name, + aggregator_address, + tx_hash, + block_number, + tx_from, + tx_to, + CAST(platform_fee_amount_raw as decimal(38)) as platform_fee_amount_raw, + CAST(platform_fee_amount as double) as platform_fee_amount, + platform_fee_amount_usd, + platform_fee_percentage, + royalty_fee_address as royalty_fee_receive_address, + currency_symbol as royalty_fee_currency_symbol, + CAST(royalty_fee_amount_raw as decimal(38)) as royalty_fee_amount_raw, + CAST(royalty_fee_amount as double) as royalty_fee_amount, + royalty_fee_amount_usd, + royalty_fee_percentage, + concat(project,'-',cast(block_number as varchar(20)),'-',cast(tx_hash as varchar(66)),'-',cast(sub_tx_trade_id as varchar(20))) as unique_trade_id +FROM {{ model }} + +{% endmacro %} diff --git a/models/_sector/nft/trades/nft_ethereum_trades_beta.sql b/models/_sector/nft/trades/nft_ethereum_trades_beta.sql index cb762d64886..4f9c6e5ba9c 100644 --- a/models/_sector/nft/trades/nft_ethereum_trades_beta.sql +++ b/models/_sector/nft/trades/nft_ethereum_trades_beta.sql @@ -1,6 +1,7 @@ {{ config( schema = 'nft_ethereum', - alias ='trades_beta', + tags = ['dunesql'], + alias = alias('trades_beta'), partition_by = ['block_date'], materialized = 'incremental', file_format = 'delta', @@ -12,7 +13,7 @@ {% set base_models = [ ('archipelago', 'v1', ref('archipelago_ethereum_base_trades')) ,('superrare', 'v1', ref('superrare_ethereum_base_trades')) - ,('foundation', 'v1', ref('foundation_ethereum_base_trades_legacy')) + ,('foundation', 'v1', ref('foundation_ethereum_base_trades')) ,('blur', 'v1', ref('blur_ethereum_base_trades')) ,('element', 'v1', ref('element_ethereum_base_trades')) ,('x2y2', 'v1', ref('x2y2_ethereum_base_trades')) @@ -26,7 +27,7 @@ ,('looksrare', 'v2', ref('looksrare_v2_ethereum_base_trades')) ] %} --- We should remove this CTE and include ETH into the general prices table once everything is migrated +-- TODO: We should remove this CTE and include ETH into the general prices table once everything is migrated WITH cte_prices_patch as ( SELECT contract_address @@ -35,23 +36,23 @@ WITH cte_prices_patch as ( ,minute ,price ,symbol - FROM {{ ref('prices_usd_forward_fill_legacy') }} + FROM {{ ref('prices_usd_forward_fill') }} WHERE blockchain = 'ethereum' {% if is_incremental() %} - AND minute >= date_trunc("day", now() - interval '1 week') + AND minute >= date_trunc('day', now() - interval '7' day) {% endif %} UNION ALL SELECT - '{{ var("ETH_ERC20_ADDRESS") }}' as contract_address + {{ var("ETH_ERC20_ADDRESS") }} as contract_address ,'ethereum' as blockchain ,18 as decimals ,minute ,price ,'ETH' as symbol - FROM {{ ref('prices_usd_forward_fill_legacy') }} + FROM {{ ref('prices_usd_forward_fill') }} WHERE blockchain is null AND symbol = 'ETH' {% if is_incremental() %} - AND minute >= date_trunc("day", now() - interval '1 week') + AND minute >= date_trunc('day', now() - interval '7' day) {% endif %} ), enriched_trades as ( @@ -61,11 +62,11 @@ enriched_trades as ( blockchain='ethereum', models=base_models, transactions_model=source('ethereum','transactions'), - tokens_nft_model=ref('tokens_ethereum_nft_legacy'), - tokens_erc20_model=ref('tokens_ethereum_erc20_legacy'), + tokens_nft_model=ref('tokens_ethereum_nft'), + tokens_erc20_model=ref('tokens_ethereum_erc20'), prices_model='cte_prices_patch', - aggregators=ref('nft_ethereum_aggregators_legacy'), - aggregator_markers=ref('nft_ethereum_aggregators_markers_legacy') + aggregators=ref('nft_ethereum_aggregators'), + aggregator_markers=ref('nft_ethereum_aggregators_markers') ) }} ) diff --git a/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql b/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql new file mode 100644 index 00000000000..57724f4f600 --- /dev/null +++ b/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql @@ -0,0 +1,73 @@ +{{ config( + schema = 'nft_ethereum', + alias = alias('trades_beta',legacy_model=True), + partition_by = ['block_date'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['project','project_version','tx_hash','sub_tx_trade_id'] + ) +}} +-- (project, project_version, model) +{% set base_models = [ + ('archipelago', 'v1', ref('archipelago_ethereum_base_trades')) + ,('superrare', 'v1', ref('superrare_ethereum_base_trades')) + ,('foundation', 'v1', ref('foundation_ethereum_base_trades_legacy')) + ,('blur', 'v1', ref('blur_ethereum_base_trades')) + ,('element', 'v1', ref('element_ethereum_base_trades')) + ,('x2y2', 'v1', ref('x2y2_ethereum_base_trades')) + ,('zora', 'v1', ref('zora_v1_ethereum_base_trades')) + ,('zora', 'v2', ref('zora_v2_ethereum_base_trades')) + ,('zora', 'v3', ref('zora_v3_ethereum_base_trades')) + ,('cryptopunks', 'v1', ref('cryptopunks_ethereum_base_trades')) + ,('sudoswap', 'v1', ref('sudoswap_ethereum_base_trades')) + ,('collectionswap', 'v1', ref('collectionswap_ethereum_base_trades')) + ,('looksrare', 'v1', ref('looksrare_v1_ethereum_base_trades')) + ,('looksrare', 'v2', ref('looksrare_v2_ethereum_base_trades')) +] %} + +-- We should remove this CTE and include ETH into the general prices table once everything is migrated +WITH cte_prices_patch as ( + SELECT + contract_address + ,blockchain + ,decimals + ,minute + ,price + ,symbol + FROM {{ ref('prices_usd_forward_fill_legacy') }} + WHERE blockchain = 'ethereum' + {% if is_incremental() %} + AND minute >= date_trunc("day", now() - interval '1 week') + {% endif %} + UNION ALL + SELECT + '{{ var("ETH_ERC20_ADDRESS") }}' as contract_address + ,'ethereum' as blockchain + ,18 as decimals + ,minute + ,price + ,'ETH' as symbol + FROM {{ ref('prices_usd_forward_fill_legacy') }} + WHERE blockchain is null AND symbol = 'ETH' + {% if is_incremental() %} + AND minute >= date_trunc("day", now() - interval '1 week') + {% endif %} +), +enriched_trades as ( +-- macros/models/sector/nft +{{ + enrich_trades_legacy( + blockchain='ethereum', + models=base_models, + transactions_model=source('ethereum','transactions'), + tokens_nft_model=ref('tokens_ethereum_nft_legacy'), + tokens_erc20_model=ref('tokens_ethereum_erc20_legacy'), + prices_model='cte_prices_patch', + aggregators=ref('nft_ethereum_aggregators_legacy'), + aggregator_markers=ref('nft_ethereum_aggregators_markers_legacy') + ) +}} +) + +select * from enriched_trades diff --git a/models/_sector/nft/trades/nft_ethereum_trades_beta_ported.sql b/models/_sector/nft/trades/nft_ethereum_trades_beta_ported.sql index ab12091c558..8a2ff4d5123 100644 --- a/models/_sector/nft/trades/nft_ethereum_trades_beta_ported.sql +++ b/models/_sector/nft/trades/nft_ethereum_trades_beta_ported.sql @@ -1,6 +1,7 @@ {{ config( schema = 'nft_ethereum', - alias ='trades_beta_ported', + tags = ['dunesql'], + alias = alias('trades_beta_ported'), materialized = 'view' ) }} diff --git a/models/_sector/nft/trades/nft_ethereum_trades_beta_ported_legacy.sql b/models/_sector/nft/trades/nft_ethereum_trades_beta_ported_legacy.sql new file mode 100644 index 00000000000..d750aa3ed09 --- /dev/null +++ b/models/_sector/nft/trades/nft_ethereum_trades_beta_ported_legacy.sql @@ -0,0 +1,8 @@ +{{ config( + schema = 'nft_ethereum', + alias = alias('trades_beta_ported', legacy_model=True), + materialized = 'view' + ) +}} + +{{ port_to_old_schema_legacy(ref('nft_ethereum_trades_beta_legacy')) }} diff --git a/models/_sector/nft/trades/nft_events.sql b/models/_sector/nft/trades/nft_events.sql index 0656e22c1af..04631e6ab48 100644 --- a/models/_sector/nft/trades/nft_events.sql +++ b/models/_sector/nft/trades/nft_events.sql @@ -51,7 +51,7 @@ SELECT royalty_fee_amount_usd, royalty_fee_percentage, unique_trade_id -FROM {{ ref('nft_ethereum_trades_beta_ported')}} +FROM {{ ref('nft_ethereum_trades_beta_ported_legacy')}} {% if is_incremental() %} WHERE block_time >= date_trunc("day", now() - interval '1 week') {% endif %} @@ -95,7 +95,7 @@ SELECT blockchain, royalty_fee_percentage, unique_trade_id FROM {{ref('nft_events_old')}} -WHERE (project, version) not in (SELECT distinct project, version FROM {{ref('nft_ethereum_trades_beta_ported')}}) +WHERE (project, version) not in (SELECT distinct project, version FROM {{ref('nft_ethereum_trades_beta_ported_legacy')}}) {% if is_incremental() %} AND block_time >= date_trunc("day", now() - interval '1 week') {% endif %} From ac127ae24f1ee176cc4fe1b873d492647fc7a4f2 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Tue, 11 Jul 2023 12:29:43 +0200 Subject: [PATCH 15/22] re-apply duplicate fix --- macros/models/_sector/nft/enrich_trades.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/macros/models/_sector/nft/enrich_trades.sql b/macros/models/_sector/nft/enrich_trades.sql index bb2c506b18d..558b77fb6ad 100644 --- a/macros/models/_sector/nft/enrich_trades.sql +++ b/macros/models/_sector/nft/enrich_trades.sql @@ -32,7 +32,8 @@ WITH base_union as ( royalty_fee_amount_raw, platform_fee_address, -- optional royalty_fee_address, -- optional - sub_tx_trade_id + sub_tx_trade_id, + row_number() over (partition by tx_hash, sub_tx_trade_id order by tx_hash) as duplicates_rank FROM {{ nft_model[2] }} {% if is_incremental() %} WHERE block_time >= date_trunc('day', now() - interval '7' day) @@ -113,6 +114,7 @@ ON agg1.contract_address is null -- only match if agg1 produces no matches, t {% if aggregator_markers != null %} LEFT JOIN {{ aggregator_markers }} agg_mark ON bytearray_starts_with(bytearray_reverse(tx.data), bytearray_reverse(agg_mark.hash_marker)) -- eq to end_with() +WHERE base.duplicates_rank = 1 {% endif %} ) From bc2052daa833ba2f5f6fa72020faf36595cf6a1c Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Tue, 11 Jul 2023 12:30:03 +0200 Subject: [PATCH 16/22] re-apply duplicate fix --- macros/models/_sector/nft/enrich_trades.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/macros/models/_sector/nft/enrich_trades.sql b/macros/models/_sector/nft/enrich_trades.sql index 558b77fb6ad..73e8f04a7f9 100644 --- a/macros/models/_sector/nft/enrich_trades.sql +++ b/macros/models/_sector/nft/enrich_trades.sql @@ -7,6 +7,7 @@ -- 5. add aggregator columns -- 6. fix buyer or seller for aggregator txs -- 7. calculate platform and royalty rates +-- 8. deduplicate based on sub_tx_trade_id WITH base_union as ( {% for nft_model in models %} From 187431b5fa4ba24b38ffc79d8fae36f41417811c Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Tue, 11 Jul 2023 12:34:47 +0200 Subject: [PATCH 17/22] fixes to trades_beta --- .../nft/trades/nft_ethereum_trades_beta.sql | 26 ++++++++--------- .../nft_ethereum_trades_beta_legacy.sql | 28 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/models/_sector/nft/trades/nft_ethereum_trades_beta.sql b/models/_sector/nft/trades/nft_ethereum_trades_beta.sql index 9b30c9d6401..4f9c6e5ba9c 100644 --- a/models/_sector/nft/trades/nft_ethereum_trades_beta.sql +++ b/models/_sector/nft/trades/nft_ethereum_trades_beta.sql @@ -11,20 +11,20 @@ }} -- (project, project_version, model) {% set base_models = [ - ('archipelago', 'v1', ref('archipelago_ethereum_base_trades_legacy')) - ,('superrare', 'v1', ref('superrare_ethereum_base_trades_legacy')) + ('archipelago', 'v1', ref('archipelago_ethereum_base_trades')) + ,('superrare', 'v1', ref('superrare_ethereum_base_trades')) ,('foundation', 'v1', ref('foundation_ethereum_base_trades')) - ,('blur', 'v1', ref('blur_ethereum_base_trades_legacy')) - ,('element', 'v1', ref('element_ethereum_base_trades_legacy')) - ,('x2y2', 'v1', ref('x2y2_ethereum_base_trades_legacy')) - ,('zora', 'v1', ref('zora_v1_ethereum_base_trades_legacy')) - ,('zora', 'v2', ref('zora_v2_ethereum_base_trades_legacy')) - ,('zora', 'v3', ref('zora_v3_ethereum_base_trades_legacy')) - ,('cryptopunks', 'v1', ref('cryptopunks_ethereum_base_trades_legacy')) - ,('sudoswap', 'v1', ref('sudoswap_ethereum_base_trades_legacy')) - ,('collectionswap', 'v1', ref('collectionswap_ethereum_base_trades_legacy')) - ,('looksrare', 'v1', ref('looksrare_v1_ethereum_base_trades_legacy')) - ,('looksrare', 'v2', ref('looksrare_v2_ethereum_base_trades_legacy')) + ,('blur', 'v1', ref('blur_ethereum_base_trades')) + ,('element', 'v1', ref('element_ethereum_base_trades')) + ,('x2y2', 'v1', ref('x2y2_ethereum_base_trades')) + ,('zora', 'v1', ref('zora_v1_ethereum_base_trades')) + ,('zora', 'v2', ref('zora_v2_ethereum_base_trades')) + ,('zora', 'v3', ref('zora_v3_ethereum_base_trades')) + ,('cryptopunks', 'v1', ref('cryptopunks_ethereum_base_trades')) + ,('sudoswap', 'v1', ref('sudoswap_ethereum_base_trades')) + ,('collectionswap', 'v1', ref('collectionswap_ethereum_base_trades')) + ,('looksrare', 'v1', ref('looksrare_v1_ethereum_base_trades')) + ,('looksrare', 'v2', ref('looksrare_v2_ethereum_base_trades')) ] %} -- TODO: We should remove this CTE and include ETH into the general prices table once everything is migrated diff --git a/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql b/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql index 57724f4f600..a782b789c6e 100644 --- a/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql +++ b/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql @@ -10,20 +10,20 @@ }} -- (project, project_version, model) {% set base_models = [ - ('archipelago', 'v1', ref('archipelago_ethereum_base_trades')) - ,('superrare', 'v1', ref('superrare_ethereum_base_trades')) - ,('foundation', 'v1', ref('foundation_ethereum_base_trades_legacy')) - ,('blur', 'v1', ref('blur_ethereum_base_trades')) - ,('element', 'v1', ref('element_ethereum_base_trades')) - ,('x2y2', 'v1', ref('x2y2_ethereum_base_trades')) - ,('zora', 'v1', ref('zora_v1_ethereum_base_trades')) - ,('zora', 'v2', ref('zora_v2_ethereum_base_trades')) - ,('zora', 'v3', ref('zora_v3_ethereum_base_trades')) - ,('cryptopunks', 'v1', ref('cryptopunks_ethereum_base_trades')) - ,('sudoswap', 'v1', ref('sudoswap_ethereum_base_trades')) - ,('collectionswap', 'v1', ref('collectionswap_ethereum_base_trades')) - ,('looksrare', 'v1', ref('looksrare_v1_ethereum_base_trades')) - ,('looksrare', 'v2', ref('looksrare_v2_ethereum_base_trades')) + ('archipelago', 'v1', ref('archipelago_ethereum_base_trades_legacy')) + ,('superrare', 'v1', ref('superrare_ethereum_base_trades_legacy')) + ,('foundation', 'v1', ref('foundation_ethereum_base_trades')) + ,('blur', 'v1', ref('blur_ethereum_base_trades_legacy')) + ,('element', 'v1', ref('element_ethereum_base_trades_legacy')) + ,('x2y2', 'v1', ref('x2y2_ethereum_base_trades_legacy')) + ,('zora', 'v1', ref('zora_v1_ethereum_base_trades_legacy')) + ,('zora', 'v2', ref('zora_v2_ethereum_base_trades_legacy')) + ,('zora', 'v3', ref('zora_v3_ethereum_base_trades_legacy')) + ,('cryptopunks', 'v1', ref('cryptopunks_ethereum_base_trades_legacy')) + ,('sudoswap', 'v1', ref('sudoswap_ethereum_base_trades_legacy')) + ,('collectionswap', 'v1', ref('collectionswap_ethereum_base_trades_legacy')) + ,('looksrare', 'v1', ref('looksrare_v1_ethereum_base_trades_legacy')) + ,('looksrare', 'v2', ref('looksrare_v2_ethereum_base_trades_legacy')) ] %} -- We should remove this CTE and include ETH into the general prices table once everything is migrated From 1282088995e0dd02e0a7b4e3b0bb41c5d920da9f Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Tue, 11 Jul 2023 12:36:18 +0200 Subject: [PATCH 18/22] fix macro refs --- macros/models/_sector/nft/enrich_trades_legacy.sql | 2 +- macros/models/_sector/nft/port_to_old_schema_legacy.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/models/_sector/nft/enrich_trades_legacy.sql b/macros/models/_sector/nft/enrich_trades_legacy.sql index a537550f76d..4b58d5466a3 100644 --- a/macros/models/_sector/nft/enrich_trades_legacy.sql +++ b/macros/models/_sector/nft/enrich_trades_legacy.sql @@ -1,4 +1,4 @@ -{% macro enrich_trades(blockchain='', models=[], transactions_model=null, tokens_nft_model=null, tokens_erc20_model=null, prices_model=null, aggregators=null, aggregator_markers=null) %} +{% macro enrich_trades_legacy(blockchain='', models=[], transactions_model=null, tokens_nft_model=null, tokens_erc20_model=null, prices_model=null, aggregators=null, aggregator_markers=null) %} -- Macro to apply the NFT trades enrichment(s) to base models -- 1. add transaction information -- 2. add NFT token information diff --git a/macros/models/_sector/nft/port_to_old_schema_legacy.sql b/macros/models/_sector/nft/port_to_old_schema_legacy.sql index eccdf8ddd0f..91a19456055 100644 --- a/macros/models/_sector/nft/port_to_old_schema_legacy.sql +++ b/macros/models/_sector/nft/port_to_old_schema_legacy.sql @@ -1,4 +1,4 @@ -{% macro port_to_old_schema(model) %} +{% macro port_to_old_schema_legacy(model) %} SELECT blockchain, From 219b3a93a3f3b4ba8ef063fbf417050e5d21d69b Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Tue, 11 Jul 2023 12:51:26 +0200 Subject: [PATCH 19/22] update downstream refs --- models/cryptopunks/ethereum/cryptopunks_ethereum_all_events.sql | 2 +- .../cryptopunks/ethereum/cryptopunks_ethereum_current_bids.sql | 2 +- .../ethereum/cryptopunks_ethereum_current_listings.sql | 2 +- .../ethereum/cryptopunks_ethereum_floor_price_over_time.sql | 2 +- .../ethereum/cryptopunks_ethereum_listings_over_time.sql | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/cryptopunks/ethereum/cryptopunks_ethereum_all_events.sql b/models/cryptopunks/ethereum/cryptopunks_ethereum_all_events.sql index 76210553319..2b8b43d0163 100644 --- a/models/cryptopunks/ethereum/cryptopunks_ethereum_all_events.sql +++ b/models/cryptopunks/ethereum/cryptopunks_ethereum_all_events.sql @@ -57,7 +57,7 @@ from , price_usd as amount_usd , block_number , tx_hash - from {{ ref('nft_ethereum_trades_beta') }} + from {{ ref('nft_ethereum_trades_beta_legacy') }} where project = 'cryptopunks' union all diff --git a/models/cryptopunks/ethereum/cryptopunks_ethereum_current_bids.sql b/models/cryptopunks/ethereum/cryptopunks_ethereum_current_bids.sql index ff539b1a696..fad444cbfa5 100644 --- a/models/cryptopunks/ethereum/cryptopunks_ethereum_current_bids.sql +++ b/models/cryptopunks/ethereum/cryptopunks_ethereum_current_bids.sql @@ -50,7 +50,7 @@ with combined_events_table as ( , block_number , sub_tx_trade_id as evt_index , tx_hash - from {{ ref('nft_ethereum_trades_beta') }} + from {{ ref('nft_ethereum_trades_beta_legacy') }} WHERE project = 'cryptopunks' ) a ) diff --git a/models/cryptopunks/ethereum/cryptopunks_ethereum_current_listings.sql b/models/cryptopunks/ethereum/cryptopunks_ethereum_current_listings.sql index d34b42fe62c..5b7eebc1976 100644 --- a/models/cryptopunks/ethereum/cryptopunks_ethereum_current_listings.sql +++ b/models/cryptopunks/ethereum/cryptopunks_ethereum_current_listings.sql @@ -32,7 +32,7 @@ with all_listing_events as ( , sub_tx_trade_id as evt_index , block_time as evt_block_time , tx_hash as evt_tx_hash - from {{ ref('nft_ethereum_trades_beta') }} + from {{ ref('nft_ethereum_trades_beta_legacy') }} where project = 'cryptopunks' ) , all_transfers as ( diff --git a/models/cryptopunks/ethereum/cryptopunks_ethereum_floor_price_over_time.sql b/models/cryptopunks/ethereum/cryptopunks_ethereum_floor_price_over_time.sql index c52847eecfc..8d0627b68a7 100644 --- a/models/cryptopunks/ethereum/cryptopunks_ethereum_floor_price_over_time.sql +++ b/models/cryptopunks/ethereum/cryptopunks_ethereum_floor_price_over_time.sql @@ -32,7 +32,7 @@ with all_listing_events as ( , sub_tx_trade_id as evt_index , block_time as evt_block_time , tx_hash as evt_tx_hash - from {{ ref('nft_ethereum_trades_beta') }} + from {{ ref('nft_ethereum_trades_beta_legacy') }} where project = 'cryptopunks' ) , all_transfers as ( diff --git a/models/cryptopunks/ethereum/cryptopunks_ethereum_listings_over_time.sql b/models/cryptopunks/ethereum/cryptopunks_ethereum_listings_over_time.sql index c5e10c15cc5..54caf42dcc3 100644 --- a/models/cryptopunks/ethereum/cryptopunks_ethereum_listings_over_time.sql +++ b/models/cryptopunks/ethereum/cryptopunks_ethereum_listings_over_time.sql @@ -32,7 +32,7 @@ with all_listing_events as ( , sub_tx_trade_id as evt_index , block_time as evt_block_time , tx_hash as evt_tx_hash - from {{ ref('nft_ethereum_trades_beta') }} + from {{ ref('nft_ethereum_trades_beta_legacy') }} where project = 'cryptopunks' ) , all_transfers as ( From f6a9c0bb1fc027cb85c04744e7bd19b2f6f7d347 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Tue, 11 Jul 2023 13:19:27 +0200 Subject: [PATCH 20/22] fix legacy parent --- models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql b/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql index a782b789c6e..7f066433abd 100644 --- a/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql +++ b/models/_sector/nft/trades/nft_ethereum_trades_beta_legacy.sql @@ -12,7 +12,7 @@ {% set base_models = [ ('archipelago', 'v1', ref('archipelago_ethereum_base_trades_legacy')) ,('superrare', 'v1', ref('superrare_ethereum_base_trades_legacy')) - ,('foundation', 'v1', ref('foundation_ethereum_base_trades')) + ,('foundation', 'v1', ref('foundation_ethereum_base_trades_legacy')) ,('blur', 'v1', ref('blur_ethereum_base_trades_legacy')) ,('element', 'v1', ref('element_ethereum_base_trades_legacy')) ,('x2y2', 'v1', ref('x2y2_ethereum_base_trades_legacy')) From 23caf1e4866c492e9a3fb5aa3bb4121043605c63 Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Tue, 11 Jul 2023 13:49:43 +0200 Subject: [PATCH 21/22] partition by month --- .../_sector/nft/platforms/element_v1_base_trades.sql | 8 ++++---- .../platforms/archipelago_ethereum_base_trades.sql | 2 +- .../ethereum/platforms/blur_ethereum_base_trades.sql | 4 ++-- .../platforms/collectionswap_ethereum_base_trades.sql | 2 +- .../platforms/cryptopunks_ethereum_base_trades.sql | 2 +- .../platforms/foundation_ethereum_base_trades.sql | 2 +- .../archipelago_ethereum_base_trades_legacy.sql | 0 .../{ => legacy}/blur_ethereum_base_trades_legacy.sql | 0 .../collectionswap_ethereum_base_trades_legacy.sql | 0 .../cryptopunks_ethereum_base_trades_legacy.sql | 0 .../{ => legacy}/element_ethereum_base_trades_legacy.sql | 0 .../foundation_ethereum_base_trades_legacy.sql | 0 .../looksrare_v1_ethereum_base_trades_legacy.sql | 0 .../looksrare_v2_ethereum_base_trades_legacy.sql | 0 .../{ => legacy}/sudoswap_ethereum_base_trades_legacy.sql | 0 .../superrare_ethereum_base_trades_legacy.sql | 0 .../{ => legacy}/x2y2_ethereum_base_trades_legacy.sql | 0 .../{ => legacy}/zora_v1_ethereum_base_trades_legacy.sql | 0 .../{ => legacy}/zora_v2_ethereum_base_trades_legacy.sql | 0 .../{ => legacy}/zora_v3_ethereum_base_trades_legacy.sql | 0 .../platforms/looksrare_v1_ethereum_base_trades.sql | 2 +- .../platforms/looksrare_v2_ethereum_base_trades.sql | 2 +- .../ethereum/platforms/sudoswap_ethereum_base_trades.sql | 2 +- .../ethereum/platforms/superrare_ethereum_base_trades.sql | 2 +- .../ethereum/platforms/x2y2_ethereum_base_trades.sql | 2 +- .../ethereum/platforms/zora_v1_ethereum_base_trades.sql | 2 +- .../ethereum/platforms/zora_v2_ethereum_base_trades.sql | 2 +- .../ethereum/platforms/zora_v3_ethereum_base_trades.sql | 2 +- 28 files changed, 18 insertions(+), 18 deletions(-) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/archipelago_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/blur_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/collectionswap_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/cryptopunks_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/element_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/foundation_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/looksrare_v1_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/looksrare_v2_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/sudoswap_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/superrare_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/x2y2_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/zora_v1_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/zora_v2_ethereum_base_trades_legacy.sql (100%) rename models/_sector/nft/trades/ethereum/platforms/{ => legacy}/zora_v3_ethereum_base_trades_legacy.sql (100%) diff --git a/macros/models/_sector/nft/platforms/element_v1_base_trades.sql b/macros/models/_sector/nft/platforms/element_v1_base_trades.sql index 34eb0598b57..362c7b15e1d 100644 --- a/macros/models/_sector/nft/platforms/element_v1_base_trades.sql +++ b/macros/models/_sector/nft/platforms/element_v1_base_trades.sql @@ -3,7 +3,7 @@ SELECT - DATE_TRUNC('day', evt_block_time) AS block_date, + cast(DATE_TRUNC('month', evt_block_time) as date) AS block_date, evt_block_time AS block_time, evt_block_number AS block_number, 'Buy' AS trade_category, @@ -33,7 +33,7 @@ WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) UNION ALL SELECT - DATE_TRUNC('day', evt_block_time) AS block_date, + cast(date_trunc('month', evt_block_time) as date) AS block_date, evt_block_time AS block_time, evt_block_number AS block_number, 'Sell' AS trade_category, @@ -63,7 +63,7 @@ WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) UNION ALL SELECT - DATE_TRUNC('day', evt_block_time) AS block_date, + cast(date_trunc('month', evt_block_time) as date) AS block_date, evt_block_time AS block_time, evt_block_number AS block_number, 'Buy' AS trade_category, @@ -93,7 +93,7 @@ WHERE evt_block_time >= date_trunc('day', now() - interval '7' day) UNION ALL SELECT - DATE_TRUNC('day', evt_block_time) AS block_date, + cast(date_trunc('month', evt_block_time) as date) AS block_date, evt_block_time AS block_time, evt_block_number AS block_number, 'Buy' AS trade_category, diff --git a/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql index e57c27aa766..07e8525befe 100644 --- a/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades.sql @@ -48,7 +48,7 @@ fee_events as ( ) SELECT - date_trunc('day',trade.evt_block_time) as block_date + cast(date_trunc('month',trade.evt_block_time) as date) as block_date ,trade.evt_block_time as block_time ,trade.evt_block_number as block_number ,trade.evt_tx_hash as tx_hash diff --git a/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql index bffad0f7099..135d9a1051a 100644 --- a/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades.sql @@ -18,7 +18,7 @@ WITH regular_blur_sales as ( SELECT - date_trunc('day', bm.evt_block_time) AS block_date + cast(date_trunc('month', bm.evt_block_time) as date) AS block_date , bm.evt_block_time AS block_time , bm.evt_block_number AS block_number , from_hex(JSON_EXTRACT_SCALAR(bm.buy, '$.collection')) AS nft_contract_address @@ -50,7 +50,7 @@ regular_blur_sales as ( , seaport_blur_sales as ( SELECT - date_trunc('day', s.evt_block_time) AS block_date + cast(date_trunc('month', s.evt_block_time) as date) AS block_date , s.evt_block_time AS block_time , s.evt_block_number AS block_number , from_hex(JSON_EXTRACT_SCALAR(s.offer[1], '$.token')) AS nft_contract_address diff --git a/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades.sql index efe49bca799..50922c7fb2b 100644 --- a/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades.sql @@ -99,7 +99,7 @@ base_trades as ( -- results SELECT - date_trunc('day',block_time ) as block_date + cast(date_trunc('month',block_time ) as date) as block_date , block_time , block_number , tx_hash diff --git a/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades.sql index 15de7c34872..49ce6616ca1 100644 --- a/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades.sql @@ -30,7 +30,7 @@ with accepted_bid_prices as ( group by 1,2,3 ) -select date_trunc('day',evt.evt_block_time) as block_date +select cast(date_trunc('month',evt.evt_block_time) as date) as block_date , evt.evt_block_time as block_time , evt.evt_block_number as block_number , evt.evt_tx_hash as tx_hash diff --git a/models/_sector/nft/trades/ethereum/platforms/foundation_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/foundation_ethereum_base_trades.sql index 424dcb8ee48..231cbfe650d 100644 --- a/models/_sector/nft/trades/ethereum/platforms/foundation_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/foundation_ethereum_base_trades.sql @@ -103,7 +103,7 @@ WITH all_foundation_trades AS ( ) SELECT - date_trunc('day', t.block_time) AS block_date + cast(date_trunc('month', t.block_time) as date) AS block_date , t.block_time , t.block_number , t.nft_token_id diff --git a/models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/archipelago_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/archipelago_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/archipelago_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/blur_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/blur_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/blur_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/collectionswap_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/collectionswap_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/collectionswap_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/cryptopunks_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/cryptopunks_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/cryptopunks_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/element_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/element_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/element_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/foundation_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/foundation_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/foundation_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/foundation_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/looksrare_v1_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/looksrare_v1_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/looksrare_v2_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/looksrare_v2_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/sudoswap_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/sudoswap_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/superrare_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/superrare_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/x2y2_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/x2y2_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/zora_v1_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/zora_v1_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/zora_v2_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/zora_v2_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades_legacy.sql b/models/_sector/nft/trades/ethereum/platforms/legacy/zora_v3_ethereum_base_trades_legacy.sql similarity index 100% rename from models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades_legacy.sql rename to models/_sector/nft/trades/ethereum/platforms/legacy/zora_v3_ethereum_base_trades_legacy.sql diff --git a/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades.sql index 375407dba12..91ef968f590 100644 --- a/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/looksrare_v1_ethereum_base_trades.sql @@ -95,7 +95,7 @@ WITH looksrare_trades AS ( SELECT - date_trunc('day', lr.block_time) AS block_date + cast(date_trunc('month', lr.block_time) as date) AS block_date , lr.block_time , lr.block_number , lr.tx_hash diff --git a/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades.sql index be12c516e10..3866f338aaa 100644 --- a/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/looksrare_v2_ethereum_base_trades.sql @@ -65,7 +65,7 @@ WITH looksrare_v2_trades AS ( ) SELECT - date_trunc('day', block_time) AS block_date + cast(date_trunc('month', block_time) as date) AS block_date , block_time , block_number , tx_hash diff --git a/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql index a64802bae6c..eecc544084e 100644 --- a/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/sudoswap_ethereum_base_trades.sql @@ -224,7 +224,7 @@ WITH ,swaps_cleaned as ( SELECT - date_trunc('DAY', call_block_time) AS block_date + cast(date_trunc('month', call_block_time) as date) AS block_date , call_block_time as block_time , call_block_number as block_number , nft_token_id diff --git a/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades.sql index 09557c9b5a5..ab255c9e12b 100644 --- a/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/superrare_ethereum_base_trades.sql @@ -177,7 +177,7 @@ with all_superrare_sales as ( ) SELECT - cast(date_trunc('day', a.block_time) AS date) AS block_date, + cast(date_trunc('month', a.block_time) AS date) AS block_date, a.block_time, a.block_number, a.nft_token_id, diff --git a/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql index ed2d40b533f..077f1a26c24 100644 --- a/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/x2y2_ethereum_base_trades.sql @@ -16,7 +16,7 @@ WITH src_evt_inventory as ( SELECT - date_trunc('day',evt_block_time) as block_date + cast(date_trunc('month',evt_block_time) as date) as block_date ,evt_block_time as block_time ,evt_block_number as block_number ,evt_tx_hash as tx_hash diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades.sql index 28d6b60c202..2a98b7746d1 100644 --- a/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/zora_v1_ethereum_base_trades.sql @@ -11,7 +11,7 @@ }} SELECT - date_trunc('day',bf.evt_block_time) AS block_date + cast(date_trunc('month',bf.evt_block_time) as date) AS block_date , bf.evt_block_time AS block_time , bf.evt_block_number AS block_number , bf.contract_address AS project_contract_address diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades.sql index d1e290047d3..9738895d19c 100644 --- a/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/zora_v2_ethereum_base_trades.sql @@ -11,7 +11,7 @@ }} SELECT - date_trunc('day',evt_block_time) AS block_date + cast(date_trunc('month',evt_block_time) as date) AS block_date , evt_block_time AS block_time , evt_block_number AS block_number , contract_address AS project_contract_address diff --git a/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades.sql b/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades.sql index ca4c0f9438d..d3da86ab7c6 100644 --- a/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades.sql +++ b/models/_sector/nft/trades/ethereum/platforms/zora_v3_ethereum_base_trades.sql @@ -149,7 +149,7 @@ WITH v3_trades as ( ) SELECT - date_trunc('day',block_time) as block_date + cast(date_trunc('month',block_time) as date) as block_date , block_time , block_number , project_contract_address From c642271f305dd1d7e7678f8be9bd5077894af54a Mon Sep 17 00:00:00 2001 From: 0xRob <0xRobin5@gmail.com> Date: Wed, 12 Jul 2023 11:55:28 +0200 Subject: [PATCH 22/22] fix tests --- .../foundation_ethereum_base_trades_seed.csv | 2 +- .../sudoswap_ethereum_example_trades.csv | 24 --------------- .../x2y2_ethereum_trades_etherscan.csv | 16 ---------- tests/_sector/nft/check_backport_columns.sql | 2 +- .../nft/check_nft_migrations_ethereum.sql | 2 +- .../sudoswap_ethereum_assert_samples.sql | 30 ------------------- .../ethereum/x2y2_ethereum_assert_samples.sql | 11 ------- 7 files changed, 3 insertions(+), 84 deletions(-) delete mode 100644 seeds/sudoswap/sudoswap_ethereum_example_trades.csv delete mode 100644 seeds/x2y2/ethereum/x2y2_ethereum_trades_etherscan.csv delete mode 100644 tests/sudoswap/ethereum/sudoswap_ethereum_assert_samples.sql delete mode 100644 tests/x2y2/ethereum/x2y2_ethereum_assert_samples.sql diff --git a/seeds/_sector/nft/foundation_ethereum_base_trades_seed.csv b/seeds/_sector/nft/foundation_ethereum_base_trades_seed.csv index fc83bc057f9..d451c578d97 100644 --- a/seeds/_sector/nft/foundation_ethereum_base_trades_seed.csv +++ b/seeds/_sector/nft/foundation_ethereum_base_trades_seed.csv @@ -5,5 +5,5 @@ block_number,tx_hash,sub_tx_trade_id,buyer,seller,nft_contract_address,nft_token 15140017,0x0971db8bdf8d32d50ec6e29882adf4138b1ee3dd95543ce9bc903832772be607,118,0xc9aefca21844fdb3fe8f58d883f908991d757692,0xb050d94e2663811c499714459cc4b046c572bc0f,0x13e79523fb5dbc6fb25c8269ec4bcf0712bd6453,36,150000000000000000,7500000000000000,0,BuyPriceAccepted 15138308,0xaed38a403c194458f75256b381abfc9f5b27a0c39708a7d2b81a9134175585f3,451,0x7191124e9208cbc0a62dd3f78ec87d3f40c1bba2,0x636e2ce8ededd4a0cadd638d3d14664332fa56a0,0x53ae47906995831c63b052665eb71690db2575ed,13,69000000000000000,3450000000000000,0,OfferAccepted 15138341,0x85f8ffd1736f79018e889f27e6f46236b09cf34b6d979f91f22bc0f4a739c036,416,0x00ec445a9bf5039930a9be1739cab75ac75404cb,0x810657d67c68cf7299f7ecbc321b293ee064c5df,0x705690d86496f1be06c996a91894d7240b9dadce,35,650000000000000000,32500000000000000,65000000000000000,OfferAccepted -13148421,0x7078278e7dfffd37679c55376a41d89c62486c6b62a162bc63083786c4434d39,149,0x830cc132dd66f6491ceaa20206f398247143d9cf,0x101419e2d84470d900c41b86974b5912c92f01fe,0x3b3ee1931dc30c1957379fac9aba94d1c48a5405,80134,2000000000000000000,300000000000000000,0,PrivateSaleFinalized 13148396,0x3f9490cf8ff2df399adb60862825b4109494fb8a9dbdd34cff51714761f3fd86,310,0x5a39e2d5a81fe9e630ee574df446b78f742f5d9e,0x05d1a84cfe7acef6fac12d901f1b0e74393c6f63,0x3b3ee1931dc30c1957379fac9aba94d1c48a5405,80127,100000000000000000,15000000000000000,0,PrivateSaleFinalized +13148421,0x7078278e7dfffd37679c55376a41d89c62486c6b62a162bc63083786c4434d39,149,0x830cc132dd66f6491ceaa20206f398247143d9cf,0x101419e2d84470d900c41b86974b5912c92f01fe,0x3b3ee1931dc30c1957379fac9aba94d1c48a5405,80134,2000000000000000000,300000000000000000,0,PrivateSaleFinalized diff --git a/seeds/sudoswap/sudoswap_ethereum_example_trades.csv b/seeds/sudoswap/sudoswap_ethereum_example_trades.csv deleted file mode 100644 index dea64a58581..00000000000 --- a/seeds/sudoswap/sudoswap_ethereum_example_trades.csv +++ /dev/null @@ -1,24 +0,0 @@ -block_number,tx_hash,trade_category,nft_contract_address,token_id,amount_original,platform_fee_amount,pool_fee_amount -15432497,0x1eaa197f025cfe070767a85824b16576fafe1876d1d091f72bb2593c54490bd1,Buy,0x34f08f2a3f4a86531e9c4139fde571a62689afec,18542,0.14,0.000696517412935323,0 -15432495,0xa0458f553bd8862dd3b8818399048ef6aa3de2ee91eb8ccce661169a0fe68dd2,Sell,0xec0a7a26456b8451aefc4b00393ce1beff5eb3e9,5286,0.024267627963430793,0.000126657765988678,0.0009730525415475708 -15432495,0xa0458f553bd8862dd3b8818399048ef6aa3de2ee91eb8ccce661169a0fe68dd2,Sell,0xec0a7a26456b8451aefc4b00393ce1beff5eb3e9,6353,0.024773253381963498,0.00012704232503571,0.0006011270989494582 -15432495,0xc6235a303b0f8b3f2ace5866334727fe05d4f854c91ed37f1aa08841b38eba9d,Sell,0xca21d4228cdcc68d4e23807e5e370c07577dd152,52321,0.02332108374384237,0.000117783251231527,0.0002297356484416915 -15432490,0x85c0abd68e0656b4d97ef0e0fde6641f9f56f54055defe25af1ee1678ea9f649,Sell,0x3a112d86899578e63b373b6d09322e18aed924cb,1562,0.014898710357536758,0.000082770613097426,0.0013469036131308482 -15432487,0xda52c4775c93201e1ff4c53912807ea55f302bbab3c734c3e661531f1f6d0bec,Buy,0xd93206bd0062cc054e397ecccdb8436c3fa5700e,37959,0.005000000000000001,0.000024875621890547,0 -15432487,0xeb898b0cd98a0e8a499d78f2827919b7fe6f1a66bf38c0b0ad954e0d34eff52f,Buy,0xda4c9ffb9a96ef44865114be4af25004f0ee385d,2889,0.058714793272315983,0.000275656306442798,0.0033078756773135764 -15432485,0x23a50cc42e0656c3ccfea231e7e277ab827f0bfec68e9c5b9cd16d783eb2403c,Sell,0x209e639a0ec166ac7a1a4ba41968fa967db30221,3448,0.09404553759276155,0.000522475208848675,0.008502096580355716 -15432485,0xc9f8ea24925f0a16f02093897c4d4b83f5b5e9e833495d6f631db54f6f47221e,Sell,0x6661c87764adf7fffa3c7922fa6edfa2fd62ccfc,752,0.07035781345505236,0.000390876741416957,0.006360630610330491 -15432484,0x84d6424dc7be3fa61bb874472b773e08e860acd3ccc311d6ef3aad23d540ed93,Buy,0x9690b63eb85467be5267a3603f770589ab12dc95,12133,0.14,0.000696517412935323,0 -15432483,0x4e389c6e1af375d783c993fa5812ca2b035de9047d96fb2ab361d34a2aeb11d2,Sell,0x65784d6f23de30a17122e96c0f0986c378ed6947,2401,0.049060256626310146,0.000247779073870253,0.00048329185695485046 -15432482,0x4245161dd68ceeff680018abf7dff76dc48c6b343551e45423930f6e933b08c4,Sell,0x65784d6f23de30a17122e96c0f0986c378ed6947,4013,0.05028676304196789,0.000253973550717009,0.0004953741533787216 -15432481,0xa666ff76bd2304bf4ef3fff4a02fda2a875721b57e3ec43f4fe6c43959e39d17,Buy,0x65784d6f23de30a17122e96c0f0986c378ed6947,1248,0.050927891395607175,0.000250876312293631,0.0005017526245872628 -15432481,0xa666ff76bd2304bf4ef3fff4a02fda2a875721b57e3ec43f4fe6c43959e39d17,Buy,0x65784d6f23de30a17122e96c0f0986c378ed6947,3877,0.050927891395607175,0.000250876312293631,0.0005017526245872628 -15432480,0x568eb6b515323d98c8da96c3fe86993e525e9dcbbdafa02a9503ba947897eab1,Buy,0xca21d4228cdcc68d4e23807e5e370c07577dd152,52321,0.02398,0.000119303482587064,0 -15432479,0x1779ae7730bd3eb6888602de9168cb0f6c27e376674c70e2370d27d07d87f471,Buy,0x6c5a06ae6b773457480c12f12c2fb22627507a3a,4338,0.012,0.000059701492537313,0 -15432479,0x6d52e85aa2c431bde274a9e55a9a9f3224eb093c6c97cf94d4f8e202084a29fa,Sell,0x65784d6f23de30a17122e96c0f0986c378ed6947,7982,0.049060256626310146,0.000247779073870253,0.00048329185695485046 -15432477,0x3ef9a47879b8678d76f45ff825f8764fe5847a5dcab465299d415d9033f06eec,Buy,0x65784d6f23de30a17122e96c0f0986c378ed6947,3088,0.049827285463986515,0.000225462830153785,0.004509256603075703 -15432474,0x8fbb16b280543c807192faea8c5084a7372803da5d00d6b1e551342cf64ebbfc,Sell,0x00b784c0e9dd20fc865f89d05d0ce4417efb77a9,2251,0.05194903846842098,0.000259745192342105,0 -15432474,0x8fbb16b280543c807192faea8c5084a7372803da5d00d6b1e551342cf64ebbfc,Sell,0x00b784c0e9dd20fc865f89d05d0ce4417efb77a9,2249,0.05194903846842098,0.000259745192342105,0 -15432474,0x4d4353fbb604b15298ba8133941274e39624482cd8e999589d8ad4ff7c5af937,Buy,0xf10a71e53398bf4e3cf5e2a0cad3c529ad19b834,161,0.012444574248598796,0.000060705240237067,0.00024282096094826916 -15432471,0x04e2f0da231969e8ef950b99ad6beb13ec0c95f07a46e3ffd6273563a21e5b23,Sell,0x6661c87764adf7fffa3c7922fa6edfa2fd62ccfc,396,0.07387570412780498,0.000410420578487805,0.006678662140847015 -15432470,0x20f5126d305c93d8fa5aade194e67607e571bfc212fbd1a1147d8f5d2b32c460,Buy,0x740c178e10662bbb050bde257bfa318defe3cabc,3002,0.08829526864409548,0.000418460988834575,0.00418460988834575 diff --git a/seeds/x2y2/ethereum/x2y2_ethereum_trades_etherscan.csv b/seeds/x2y2/ethereum/x2y2_ethereum_trades_etherscan.csv deleted file mode 100644 index ac900f71cae..00000000000 --- a/seeds/x2y2/ethereum/x2y2_ethereum_trades_etherscan.csv +++ /dev/null @@ -1,16 +0,0 @@ -tx_hash,block_time,token_id,amount_original -0x9cf7d16af0b2192ef1f47a98c7f2b9aa93c3d881c1b5102dd467b24f1e94d1ee,2022-07-20T13:40:17+00:00,9556,0.75 -0x711062341ab9f350683d54e4c22aae50e0bc66977f65815bc4c04eb23bea0165,2022-07-20T13:39:23+00:00,153058,2 -0xf3760cf46e7a1e2815a71d481c8fc3b84743f65871540893aaebc60a5fa30ef4,2022-07-20T13:35:57+00:00,136,0.022 -0xc3bab3213bdaedff616aa2a27a4b6148e5b6489692b0e3e0d9e81830d9244c45,2022-07-20T13:57:17+00:00,501,2.1 -0xdcabaa83f18fc115816b59a71ed2ad42cef3c042f7e07e789192bf48d0574726,2022-07-20T13:57:17+00:00,2880,0.15 -0xda84b080888e9de4afcd990f0c1cecb26c3bd8fa29cabd446e011e7bf7a3bda8,2022-07-20T13:56:23+00:00,9439,0.37 -0xfee16424961ad8141114c561c34725018b4840c58d198ae517ffafc8977cbef3,2022-07-20T13:54:30+00:00,3712,0.199 -0x4d5d4b7d301e9db7af229ebe632634d40f8ec1fe6dceb43194b17da633653204,2022-07-20T13:53:37+00:00,9024,11.9 -0x26999d51ac98ee97dd43084eb708e6bb622c5bdee1b507f231c377a4672d8c36,2022-07-20T13:52:38+00:00,8214,0.375 -0x030d06148286e167a5fdc4f70bb4a25e46e0d0a0626918edd74f434f7b3162f8,2022-07-20T14:19:30+00:00,49380,0.3 -0x6131f902817a0b6f66117ccde31de96813ce794c3b810932b341ada6169d195f,2022-07-20T14:17:37+00:00,10992,9.4001 -0x92e9843c7e9041325ea1cd47ff0cf09950dffd777c5ad8327dcf348a3c55519b,2022-07-20T14:17:37+00:00,21710,16.05 -0x058f24ef8f39b6b0982c1ae84efa76a43fb5f2e4599e7f439049669d55ac73fe,2022-07-20T14:08:50+00:00,102,0.1666 -0x20cef85503a169cdba68566c017faf3a57d59f11354d1d41ef74dabad11fc883,2022-06-09T22:33:20+00:00,2605,0.055 -0x9dc0851957041d6e71506f973b8d6c022818044eed75d91be4ce968cf1ec223c,2022-06-09T22:33:38+00:00,6799,0.06 diff --git a/tests/_sector/nft/check_backport_columns.sql b/tests/_sector/nft/check_backport_columns.sql index 8110303a21e..6f2eea5f17e 100644 --- a/tests/_sector/nft/check_backport_columns.sql +++ b/tests/_sector/nft/check_backport_columns.sql @@ -83,7 +83,7 @@ SELECT royalty_fee_amount_usd, royalty_fee_percentage, unique_trade_id -from {{ ref('nft_ethereum_trades_beta_ported') }} +from {{ ref('nft_ethereum_trades_beta_ported_legacy') }} limit 1 ) ) diff --git a/tests/_sector/nft/check_nft_migrations_ethereum.sql b/tests/_sector/nft/check_nft_migrations_ethereum.sql index 09e602decb1..0d06fcbcb3c 100644 --- a/tests/_sector/nft/check_nft_migrations_ethereum.sql +++ b/tests/_sector/nft/check_nft_migrations_ethereum.sql @@ -6,7 +6,7 @@ WITH migrated as ( ,sum(platform_fee_amount_raw/pow(10,18)) as mig_total_platform_amount ,sum(royalty_fee_amount_raw/pow(10,18)) as mig_total_royalty_amount - from {{ ref('nft_ethereum_trades_beta')}} + from {{ ref('nft_ethereum_trades_beta_legacy')}} where (project, project_version) in (select distinct project, version from {{ ref('nft_events_old') }}) group by 1,2 ) diff --git a/tests/sudoswap/ethereum/sudoswap_ethereum_assert_samples.sql b/tests/sudoswap/ethereum/sudoswap_ethereum_assert_samples.sql deleted file mode 100644 index 4c43895cd98..00000000000 --- a/tests/sudoswap/ethereum/sudoswap_ethereum_assert_samples.sql +++ /dev/null @@ -1,30 +0,0 @@ --- trades between block 15432470 and 15432500 -WITH trades as ( - select block_number,tx_hash,trade_category,nft_contract_address,token_id,amount_original,platform_fee_amount - from {{ ref('nft_ethereum_trades_beta_ported') }} - where project = 'sudoswap' and block_number >= 15432470 and block_number < 15432500 -) -, examples as ( - select * from {{ ref('sudoswap_ethereum_example_trades') }} -) - -, matched as ( - select - coalesce(t.block_number, ex.block_number) as block_number - , coalesce(t.tx_hash, ex.tx_hash) as tx_hash - , coalesce(t.nft_contract_address, ex.nft_contract_address) as nft_contract_Address - , coalesce(t.token_id, ex.token_id) as token_id - , case when (t.amount_original = ex.amount_original) then true else false end as correct_amt_orig - , case when (t.platform_fee_amount = ex.platform_fee_amount) then true else false end as correct_platform_fee - , t.amount_original as reported_amount_original - , ex.amount_original as seed_amount_original - , t.platform_fee_amount as reported_platform_fee_amount - , ex.platform_fee_amount as seed_platform_fee_amount - from trades t - full outer join examples ex - on ex.block_number = t.block_number and ex.tx_hash=t.tx_hash - and ex.nft_contract_address=t.nft_contract_address and ex.token_id=t.token_id -) - -select * from matched -where not (correct_amt_orig and correct_platform_fee) diff --git a/tests/x2y2/ethereum/x2y2_ethereum_assert_samples.sql b/tests/x2y2/ethereum/x2y2_ethereum_assert_samples.sql deleted file mode 100644 index bb4aaf90ea8..00000000000 --- a/tests/x2y2/ethereum/x2y2_ethereum_assert_samples.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Bootstrapped correctness test against legacy Postgres values. --- Also manually check etherscan info for the first 5 rows -WITH unit_tests as -(SELECT case when test_data.amount_original = x2y2_trades.amount_original then True else False end as price_test -FROM {{ ref('nft_ethereum_trades_beta_ported') }} x2y2_trades -JOIN {{ ref('x2y2_ethereum_trades_etherscan') }} test_data ON test_data.tx_hash = x2y2_trades.tx_hash -WHERE project = 'x2y2' and x2y2_trades.block_time = '2022-07-20' or x2y2_trades.block_time = '2022-06-09' -) -select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows -from unit_tests -having count(case when price_test = false then 1 else null end) > count(*)*0.05