atomic data preparation #1421
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Pipeline Run Check | |
| on: | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| # Explicitly include paths that should trigger the workflow | |
| - 'pipelines/**' | |
| - 'pyrenew_hew/**' | |
| - '.github/workflows/pipeline-run-check.yaml' | |
| - '.github/actions/**' | |
| - 'uv.lock' | |
| - 'pyproject.toml' | |
| - '.python-version' | |
| # Explicitly exclude batch script changes from triggering this workflow | |
| - '!pipelines/batch/**' | |
| - '!pipelines/azure_command_center.py' | |
| push: | |
| branches: [main] | |
| paths: | |
| # Explicitly include paths that should trigger the workflow | |
| - 'pipelines/**' | |
| - 'pyrenew_hew/**' | |
| - '.github/workflows/pipeline-run-check.yaml' | |
| - '.github/actions/**' | |
| - 'uv.lock' | |
| - 'pyproject.toml' | |
| - '.python-version' | |
| # Explicitly exclude batch script changes from triggering this workflow | |
| - '!pipelines/batch/**' | |
| - '!pipelines/azure_command_center.py' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| BASE_DIR: "pipelines/tests/end_to_end_test_output" | |
| DATA_DIR: "pipelines/tests/end_to_end_test_output/private_data" | |
| FORECAST_DIR: "pipelines/tests/end_to_end_test_output/2024-12-21_forecasts" | |
| jobs: | |
| generate-test-data: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Pyrenew-HEW via UV | |
| uses: ./.github/actions/setup-pyrenew-hew | |
| - name: Generate test data | |
| run: uv run python pipelines/generate_test_data.py ${{ env.BASE_DIR }} | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-data | |
| path: ${{ env.DATA_DIR }} | |
| retention-days: 1 | |
| model-fit-data: | |
| needs: generate-test-data | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| disease: [COVID-19, Influenza] | |
| location: [US, CA, MT, DC] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Pyrenew-HEW via UV | |
| uses: ./.github/actions/setup-pyrenew-hew | |
| - name: Set up R with hewr package | |
| uses: ./.github/actions/setup-hewr | |
| - name: Download test data | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: test-data | |
| path: ${{ env.DATA_DIR }} | |
| - name: Prepare model fit data | |
| run: | | |
| uv run bash pipelines/tests/test_prep_data.sh ${{ env.BASE_DIR }} \ | |
| ${{ matrix.disease }} ${{ matrix.location }} ${{ matrix.model }} | |
| - name: Upload model fitting data | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: | | |
| model-fit-data-${{ matrix.disease }}-${{ matrix.location }}-${{ matrix.model }} | |
| path: ${{ env.BASE_DIR }} | |
| fit-timeseries-models: | |
| needs: model-fit-data | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| model: [e] | |
| disease: [COVID-19, Influenza] | |
| location: [US, CA, MT, DC] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Pyrenew-HEW via UV | |
| uses: ./.github/actions/setup-pyrenew-hew | |
| - name: Set up R with hewr package | |
| uses: ./.github/actions/setup-hewr | |
| - name: Download test data | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: model-fit-data-* | |
| path: ${{ env.BASE_DIR }} | |
| merge-multiple: true | |
| - name: Fit timeseries model | |
| run: | | |
| uv run bash pipelines/tests/test_ts_fit.sh ${{ env.BASE_DIR }} \ | |
| ${{ matrix.disease }} ${{ matrix.location }} ${{ matrix.model }} | |
| - name: Upload model fitting output | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: | | |
| test-fit-ts-${{ matrix.disease }}-${{ matrix.location }}-${{ matrix.model }} | |
| path: ${{ env.BASE_DIR }} | |
| fit-pyrenew-models: | |
| needs: fit-timeseries-models | |
| runs-on: ubuntu-latest | |
| env: | |
| NHSN_API_KEY_ID: ${{ secrets.NHSN_API_KEY_ID }} | |
| NHSN_API_KEY_SECRET: ${{ secrets.NHSN_API_KEY_SECRET }} | |
| XLA_FLAGS: --xla_force_host_platform_device_count=2 | |
| strategy: | |
| matrix: | |
| model: [h, e, he, hw, ew, hew] | |
| disease: [COVID-19, Influenza] | |
| location: [US, CA, MT, DC] | |
| exclude: | |
| - model: hw | |
| disease: Influenza | |
| - model: ew | |
| disease: Influenza | |
| - model: hew | |
| disease: Influenza | |
| - model: hw | |
| location: US | |
| - model: ew | |
| location: US | |
| - model: hew | |
| location: US | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Pyrenew-HEW via UV | |
| uses: ./.github/actions/setup-pyrenew-hew | |
| - name: Set up R with hewr package | |
| uses: ./.github/actions/setup-hewr | |
| - name: Download test data | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: test-data | |
| path: ${{ env.DATA_DIR }} | |
| - name: Download timeseries output | |
| if: ${{ matrix.model != 'hw' && matrix.model != 'h' }} | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: test-fit-ts-* | |
| path: ${{ env.BASE_DIR }} | |
| merge-multiple: true | |
| - name: Fit pyrenew model | |
| run: | | |
| uv run bash pipelines/tests/test_pyrenew_fit.sh ${{ env.BASE_DIR }} \ | |
| ${{ matrix.disease }} ${{ matrix.location }} ${{ matrix.model }} | |
| - name: Upload model fitting output | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: | | |
| test-fit-${{ matrix.disease }}-${{ matrix.location }}-${{ matrix.model }} | |
| path: ${{ env.BASE_DIR }} | |
| postprocess-models: | |
| needs: fit-pyrenew-models | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Pyrenew-HEW via UV | |
| uses: ./.github/actions/setup-pyrenew-hew | |
| - name: Set up R with hewr package | |
| uses: ./.github/actions/setup-hewr | |
| - name: Download model fitting output | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: test-fit-* | |
| path: ${{ env.BASE_DIR }} | |
| merge-multiple: true | |
| - name: Run postprocessing | |
| run: | | |
| uv run python pipelines/postprocess_forecast_batches.py \ | |
| ${{ env.FORECAST_DIR }} |