Implement high-precision 128-bit value types (#2072)

Co-authored-by: Chris Sellers <[email protected]>
RFC #2084

Author: twitu

.github/workflows/build.yml

@@ -81,10 +81,10 @@ jobs:
       - name: Run pre-commit
         continue-on-error: true
         run: |
-          pre-commit run --all-files
+          pre-commit run --all-files || echo "PRECOMMIT_FAILED=true" > $GITHUB_ENV
 
       - name: Add PR comment on failure
-        if: failure() && github.event_name == 'pull_request'
+        if: env.PRECOMMIT_FAILED == 'true' && github.event_name == 'pull_request'
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -95,11 +95,185 @@ jobs:
             })
 
       - name: Fail job if pre-commit failed
-        if: failure()
+        if: env.PRECOMMIT_FAILED == 'true'
         run: |
           echo "Pre-commit checks failed, exiting"
           exit 1
 
+  build-linux-low-precision:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+    defaults:
+      run:
+        shell: bash
+    name: build low-precision (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    needs: [pre-commit]
+    env:
+      BUILD_MODE: debug
+      HIGH_PRECISION: false
+      RUST_BACKTRACE: 1
+      # > --------------------------------------------------
+      # > sccache
+      # https://github.com/Mozilla-Actions/sccache-action
+      SCCACHE_IDLE_TIMEOUT: 0
+      SCCACHE_DIRECT: "true"
+      SCCACHE_CACHE_MULTIARCH: 1
+      SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache
+      RUSTC_WRAPPER: "sccache"
+      CC: "sccache clang"
+      CXX: "sccache clang++"
+      # Incrementally compiled crates cannot be cached by sccache
+      # https://github.com/mozilla/sccache#rust
+      CARGO_INCREMENTAL: 0
+      # > --------------------------------------------------
+
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DB: nautilus
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+      # - name: Free disk space  # Continue to monitor
+      #   uses: jlumbroso/free-disk-space@main
+      #   with:
+      #     tool-cache: true
+      #     android: false
+      #     dotnet: false
+      #     haskell: false
+      #     large-packages: true
+      #     docker-images: true
+      #     swap-storage: true
+
+      - name: Install runner dependencies
+        run: sudo apt-get install -y curl clang git libssl-dev make pkg-config
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Rust toolchain
+        run: |
+          rustup toolchain add --profile minimal stable --component clippy,rustfmt
+
+      - name: Set up Python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Get Python version
+        run: |
+          version=$(bash scripts/python-version.sh)
+          echo "PYTHON_VERSION=$version" >> $GITHUB_ENV
+
+      - name: Get Poetry version from poetry-version
+        run: |
+          version=$(cat poetry-version)
+          echo "POETRY_VERSION=$version" >> $GITHUB_ENV
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+
+      - name: Install build dependencies
+        run: python -m pip install --upgrade pip setuptools wheel poetry-plugin-export pre-commit msgspec
+
+      - name: Cached sccache
+        id: cached-sccache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.SCCACHE_DIR }}
+          key: sccache-${{ runner.os }}-${{ github.workflow }}-${{ github.job }}-${{ hashFiles('**/Cargo.lock', '**/poetry.lock') }}
+          restore-keys: |
+            sccache-${{ runner.os }}-${{ github.workflow }}-${{ github.job }}-
+            sccache-${{ runner.os }}-${{ github.workflow }}-
+            sccache-${{ runner.os }}-
+
+      - name: Run sccache
+        uses: mozilla-actions/[email protected]
+
+      - name: Cached cargo
+        id: cached-cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
+      - name: Cache Python site-packages
+        id: cached-site-packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.local/lib/python${{ matrix.python-version }}/site-packages
+          key: ${{ runner.os }}-${{ matrix.python-version }}-site-packages
+          restore-keys: |
+            ${{ runner.os }}-site-packages-
+
+      - name: Cached test data
+        id: cached-testdata-large
+        uses: actions/cache@v4
+        with:
+          path: tests/test_data/large
+          key: ${{ runner.os }}-large-files-${{ hashFiles('tests/test_data/large/checksums.json') }}
+          restore-keys: ${{ runner.os }}-large-files-
+
+      - name: Install Nautilus CLI and run init postgres
+        run: |
+          make install-cli
+          nautilus database init --schema ${{ github.workspace }}/schema
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          POSTGRES_USERNAME: postgres
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DATABASE: nautilus
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: nextest
+
+      - name: Run nautilus_core tests
+        run: |
+          make cargo-test-low-precision
+
+      - name: Build Python wheel
+        run: |
+          poetry build --format wheel
+          ls -lh dist/
+
+      - name: Install Python wheel
+        run: |
+          poetry export --with test --all-extras --format requirements.txt --output requirements-test.txt
+          python -m pip install -r requirements-test.txt
+          pip install "$(ls dist/*.whl)"
+
+      - name: Run tests
+        run: pytest --ignore=tests/performance_tests
+
   build-linux:
     strategy:
       fail-fast: false
@@ -525,6 +699,8 @@ jobs:
     needs: [pre-commit]
     env:
       BUILD_MODE: debug # Not building wheels, so debug is fine
+      HIGH_PRECISION: false
+      PARALLEL_BUILD: false
       RUST_BACKTRACE: 1
       # > --------------------------------------------------
       # > sccache
@@ -627,8 +803,6 @@ jobs:
         run: |
           poetry install --with test --all-extras
           poetry run pytest --ignore=tests/performance_tests
-        env:
-          PARALLEL_BUILD: false
 
   publish-wheels:
     name: publish-packages

Makefile

@@ -99,7 +99,17 @@ cargo-test:
 		echo "cargo-nextest is not installed. You can install it using 'cargo install cargo-nextest'"; \
 		exit 1; \
 	fi
-	RUST_BACKTRACE=1 && (cd nautilus_core && cargo nextest run --workspace)
+	RUST_BACKTRACE=1 && (cd nautilus_core && cargo nextest run --workspace --features "python,ffi,high-precision")
+
+
+.PHONY: cargo-test-low-precision
+cargo-test-low-precision:
+	@if ! cargo nextest --version >/dev/null 2>&1; then \
+    echo "cargo-nextest is not installed. You can install it using 'cargo install cargo-nextest'"; \
+    exit 1; \
+	fi
+	RUST_BACKTRACE=1 && (cd nautilus_core && cargo nextest run --workspace --features "python,ffi")
+
 
 .PHONY: cargo-test-coverage
 cargo-test-coverage:

build.py

@@ -29,11 +29,13 @@
 # If ANNOTATION mode is enabled, generate an annotated HTML version of the input source files
 ANNOTATION_MODE = bool(os.getenv("ANNOTATION_MODE", ""))
 # If PARALLEL build is enabled, uses all CPUs for compile stage of build
-PARALLEL_BUILD = os.getenv("PARALLEL_BUILD", "true") == "true"
+PARALLEL_BUILD = os.getenv("PARALLEL_BUILD", "true").lower() == "true"
 # If COPY_TO_SOURCE is enabled, copy built *.so files back into the source tree
-COPY_TO_SOURCE = os.getenv("COPY_TO_SOURCE", "true") == "true"
+COPY_TO_SOURCE = os.getenv("COPY_TO_SOURCE", "true").lower() == "true"
 # If PyO3 only then don't build C extensions to reduce compilation time
-PYO3_ONLY = os.getenv("PYO3_ONLY", "") != ""
+PYO3_ONLY = os.getenv("PYO3_ONLY", "").lower() != ""
+# If high-precision mode is enabled (value types use 128-bit integers, rather than 64-bit)
+HIGH_PRECISION = os.getenv("HIGH_PRECISION", "true").lower() == "true"
 
 if PROFILE_MODE:
     # For subsequent debugging, the C source needs to be in the same tree as
@@ -99,11 +101,17 @@ def _build_rust_libs() -> None:
 
         build_options = " --release" if BUILD_MODE == "release" else ""
 
+        if HIGH_PRECISION:
+            features = ["--all-features"]
+        else:
+            # Enable features needed for main build, but not high_precision
+            features = ["--features", "ffi,python,extension-module"]
+
         cmd_args = [
             "cargo",
             "build",
             *build_options.split(),
-            "--all-features",
+            *features,
         ]
 
         if RUST_TOOLCHAIN == "nightly":
@@ -132,8 +140,6 @@ def _build_rust_libs() -> None:
 Options.annotate = ANNOTATION_MODE  # Create annotated HTML files for each .pyx
 if ANNOTATION_MODE:
     Options.annotate_coverage_xml = "coverage.xml"
-Options.warning_errors = True  # Treat compiler warnings as errors
-Options.extra_warnings = True
 
 CYTHON_COMPILER_DIRECTIVES = {
     "language_level": "3",
@@ -145,6 +151,12 @@ def _build_rust_libs() -> None:
     "warn.maybe_uninitialized": True,
 }
 
+# TODO: Temporarily separate Cython configuration while we require v3.0.11 for coverage
+if cython_compiler_version == "3.1.0a1":
+    Options.warning_errors = True  # Treat compiler warnings as errors
+    Options.extra_warnings = True
+    CYTHON_COMPILER_DIRECTIVES["warn.deprecated.IF"] = False
+
 
 def _build_extensions() -> list[Extension]:
     # Regarding the compiler warning: #warning "Using deprecated NumPy API,
@@ -371,6 +383,7 @@ def build() -> None:
     print(f"PARALLEL_BUILD={PARALLEL_BUILD}")
     print(f"COPY_TO_SOURCE={COPY_TO_SOURCE}")
     print(f"PYO3_ONLY={PYO3_ONLY}")
+    print(f"HIGH_PRECISION={HIGH_PRECISION}")
     print(f"CC={os.environ['CC']}") if "CC" in os.environ else None
     print(f"CXX={os.environ['CXX']}") if "CXX" in os.environ else None
     print(f"LDSHARED={os.environ['LDSHARED']}") if "LDSHARED" in os.environ else None

nautilus_core/Cargo.lock

@@ -42,7 +42,7 @@ rstest = { workspace = true }
 tracing-test = { workspace = true }
 
 [features]
-default = ["python"]
+default = ["python", "nautilus-core/ffi"]
 extension-module = [
   "pyo3/extension-module",
   "nautilus-core/extension-module",