Fixed missing operations when handler pattern contains item without entrypoint (#976)

Author: droserasprout

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog], and this project adheres to [Semantic Versioning].
 
+## [Unreleased]
+
+### Fixed
+
+- tezos.tzkt.operations: Fixed missing operations when handler pattern contains item without entrypoint.
+
 ## [7.5.2] - 2024-03-20
 
 ### Fixed

docs/15.thanks.md

@@ -33,6 +33,7 @@ We are grateful to all the people who helped us with the project.
 - [Igor Sereda](https://github.com/igorsereda)
 - [Javier Graciá Carpio](https://github.com/jagracar)
 - [Karan Dua](https://github.com/Karantezsure)
+- [Kornii](https://github.com/lourenc)
 - [Nick Kalomoiris](https://github.com/nikos-kalomoiris)
 - [magicCity](https://github.com/tezosmiami)
 - [pravind](https://github.com/pravind)

src/dipdup/codegen/tezos_tzkt.py

@@ -38,6 +38,7 @@
 from dipdup.datasources.tezos_tzkt import late_tzkt_initialization
 from dipdup.exceptions import ConfigurationError
 from dipdup.exceptions import FrameworkException
+from dipdup.models.tezos_tzkt import DEFAULT_ENTRYPOINT
 from dipdup.package import DipDupPackage
 from dipdup.utils import json_dumps
 from dipdup.utils import pascal_to_snake
@@ -46,9 +47,8 @@
 
 
 def match_entrypoint_schema(entrypoint_name: str, entrypoint_schemas: list[dict[str, Any]]) -> dict[str, Any]:
-    if entrypoint_name == 'default' and len(entrypoint_schemas) == 1:
+    if entrypoint_name == DEFAULT_ENTRYPOINT and len(entrypoint_schemas) == 1:
         return entrypoint_schemas[0]['parameterSchema']  # type: ignore[no-any-return]
-
     return next(ep['parameterSchema'] for ep in entrypoint_schemas if ep['name'] == entrypoint_name)
 
 

src/dipdup/dipdup.py

@@ -96,7 +96,7 @@ def __init__(self, ctx: DipDupContext) -> None:
         self._ctx = ctx
         self._indexes: dict[str, Index[Any, Any, Any]] = {}
         # FIXME: Tezos-specific
-        self._entrypoint_filter: set[str | None] = set()
+        self._entrypoint_filter: set[str] = set()
         self._address_filter: set[str] = set()
         self._code_hash_filter: set[int] = set()
         # NOTE: Monitoring purposes

src/dipdup/indexes/tezos_tzkt_operations/index.py

@@ -29,6 +29,7 @@
 from dipdup.indexes.tezos_tzkt_operations.matcher import match_operation_subgroup
 from dipdup.indexes.tezos_tzkt_operations.matcher import match_operation_unfiltered_subgroup
 from dipdup.models import RollbackMessage
+from dipdup.models.tezos_tzkt import DEFAULT_ENTRYPOINT
 from dipdup.models.tezos_tzkt import TzktMessageType
 from dipdup.models.tezos_tzkt import TzktOperationData
 from dipdup.prometheus import Metrics
@@ -38,14 +39,14 @@
 QueueItem = tuple[OperationSubgroup, ...] | RollbackMessage
 
 
-def entrypoint_filter(handlers: tuple[TzktOperationsHandlerConfig, ...]) -> set[str | None]:
+def entrypoint_filter(handlers: tuple[TzktOperationsHandlerConfig, ...]) -> set[str]:
     """Set of entrypoints to filter operations with before an actual matching"""
     entrypoints = set()
     for handler_config in handlers:
         for pattern_config in handler_config.pattern:
             if not isinstance(pattern_config, TransactionPatternConfig):
                 continue
-            entrypoints.add(pattern_config.entrypoint)
+            entrypoints.add(pattern_config.entrypoint or DEFAULT_ENTRYPOINT)
 
     return entrypoints
 
@@ -101,7 +102,7 @@ def code_hash_filter(handlers: tuple[TzktOperationsHandlerConfig, ...]) -> set[i
 def extract_operation_subgroups(
     operations: Iterable[TzktOperationData],
     addresses: set[str],
-    entrypoints: set[str | None],
+    entrypoints: set[str],
     code_hashes: set[int],
 ) -> Iterator[OperationSubgroup]:
     filtered: int = 0
@@ -112,7 +113,8 @@ def extract_operation_subgroups(
     for _operation_index, op in enumerate(operations):
         # NOTE: Filtering out operations that are not part of any index
         if op.type == 'transaction':
-            if entrypoints and op.entrypoint not in entrypoints:
+            entrypoint = op.entrypoint or DEFAULT_ENTRYPOINT
+            if entrypoints and entrypoint not in entrypoints:
                 filtered += 1
                 continue
 
@@ -140,12 +142,10 @@ def extract_operation_subgroups(
 
     for key, operations in operation_subgroups.items():
         hash_, counter = key
-        entrypoints = {op.entrypoint for op in operations}
         yield OperationSubgroup(
             hash=hash_,
             counter=counter,
             operations=tuple(operations),
-            entrypoints=entrypoints,
         )
 
 
@@ -160,11 +160,11 @@ def __init__(
         datasource: TzktDatasource,
     ) -> None:
         super().__init__(ctx, config, datasource)
-        self._entrypoint_filter: set[str | None] = set()
+        self._entrypoint_filter: set[str] = set()
         self._address_filter: set[str] = set()
         self._code_hash_filter: set[int] = set()
 
-    async def get_filters(self) -> tuple[set[str | None], set[str], set[int]]:
+    async def get_filters(self) -> tuple[set[str], set[str], set[int]]:
         if isinstance(self._config, TzktOperationsUnfilteredIndexConfig):
             return set(), set(), set()
 

src/dipdup/indexes/tezos_tzkt_operations/matcher.py

@@ -35,7 +35,6 @@ class OperationSubgroup:
     hash: str
     counter: int
     operations: tuple[TzktOperationData, ...]
-    entrypoints: set[str | None]
 
 
 OperationsHandlerArgumentU = (

src/dipdup/models/tezos_tzkt.py

@@ -17,6 +17,8 @@
 from dipdup.models import MessageType
 from dipdup.subscriptions import Subscription
 
+DEFAULT_ENTRYPOINT = 'default'
+
 ParameterType = TypeVar('ParameterType', bound=BaseModel)
 StorageType = TypeVar('StorageType', bound=BaseModel)
 KeyType = TypeVar('KeyType', bound=BaseModel)
@@ -241,7 +243,7 @@ def from_json(
         if target_json.get('address', '').startswith('KT1'):
             # NOTE: TzKT returns None for `default` entrypoint
             if entrypoint is None:
-                entrypoint = 'default'
+                entrypoint = DEFAULT_ENTRYPOINT
 
                 # NOTE: Empty parameter in this case means `{"prim": "Unit"}`
                 if parameter is None:

tests/configs/demo_dex.yml

@@ -161,8 +161,8 @@ indexes:
       token_contract: kusd_token_mainnet
       symbol: kUSD
       decimals: 18
-    first_level: 1434927
-    last_level: 1435026
+    first_level: 3032136
+    last_level: 3032136
 
   hdao_mainnet:
     template: quipuswap_fa2

tests/test_demos.py

@@ -100,8 +100,8 @@ async def assert_run_dex() -> None:
     async with in_transaction() as conn:
         symbols = (await conn.execute_query('select count(distinct(symbol)) from trade group by symbol;'))[0]
     assert symbols == 2
-    assert trades == 56
-    assert positions == 133
+    assert trades == 55
+    assert positions == 125
 
 
 async def assert_run_domains() -> None: