From 6c35a892e5bcef01ba1c09e5c2780911a0e99cc4 Mon Sep 17 00:00:00 2001 From: Nicholas Ipsen Date: Sun, 29 Oct 2023 04:55:11 +0100 Subject: [PATCH 1/8] initial draft of no-gil CI/CD workflow --- .github/workflows/build.yml | 69 +++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3373a0616b66c4..a6a4c61598c0fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -332,6 +332,75 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: xvfb-run make test + build_ubuntu: + name: 'Ubuntu' + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + env: + OPENSSL_VER: 3.0.11 + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - uses: actions/checkout@v4 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: Create directories for read-only out-of-tree builds + run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + - name: Bind mount sources read-only + run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-pydebug \ + --with-openssl=$OPENSSL_DIR \ + --disable-gil + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j4 + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + - name: Tests + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: xvfb-run make test + + build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' runs-on: ubuntu-20.04 From d7b420b52501f9f42ec0ace8afda6f4e1901ee96 Mon Sep 17 00:00:00 2001 From: Nicholas Ipsen Date: Sun, 29 Oct 2023 04:58:16 +0100 Subject: [PATCH 2/8] Fixed workflow name --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a6a4c61598c0fd..ff1b82021b3913 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -332,7 +332,7 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: xvfb-run make test - build_ubuntu: + build_ubuntu_nogil: name: 'Ubuntu' runs-on: ubuntu-latest timeout-minutes: 60 From 591817225437e377f740e5294fc5667d8cb22286 Mon Sep 17 00:00:00 2001 From: Nicholas Ipsen Date: Sun, 29 Oct 2023 05:06:52 +0100 Subject: [PATCH 3/8] Disabled build-steps to speed up debugging --- .github/workflows/build.yml | 552 ++++++++++++++++++------------------ 1 file changed, 276 insertions(+), 276 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ff1b82021b3913..000a14cb2bdf41 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -146,37 +146,37 @@ jobs: grep "aclocal 1.16.4" aclocal.m4 grep -q "runstatedir" configure grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 - - name: Configure CPython - run: | - # Build Python with the libpython dynamic library - ./configure --config-cache --with-pydebug --enable-shared - - name: Regenerate autoconf files with container image - run: make regen-configure - - name: Build CPython - run: | - make -j4 regen-all - make regen-stdlib-module-names - - name: Check for changes - run: | - git add -u - changes=$(git status --porcelain) - # Check for changes in regenerated files - if test -n "$changes"; then - echo "Generated files not up to date." - echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)" - echo "configure files must be regenerated with a specific version of autoconf." - echo "$changes" - echo "" - git diff --staged || true - exit 1 - fi - - name: Check exported libpython symbols - run: make smelly - - name: Check limited ABI symbols - run: make check-limited-abi - - name: Check for unsupported C global variables - if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME - run: make check-c-globals + # - name: Configure CPython + # run: | + # # Build Python with the libpython dynamic library + # ./configure --config-cache --with-pydebug --enable-shared + # - name: Regenerate autoconf files with container image + # run: make regen-configure + # - name: Build CPython + # run: | + # make -j4 regen-all + # make regen-stdlib-module-names + # - name: Check for changes + # run: | + # git add -u + # changes=$(git status --porcelain) + # # Check for changes in regenerated files + # if test -n "$changes"; then + # echo "Generated files not up to date." + # echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)" + # echo "configure files must be regenerated with a specific version of autoconf." + # echo "$changes" + # echo "" + # git diff --staged || true + # exit 1 + # fi + # - name: Check exported libpython symbols + # run: make smelly + # - name: Check limited ABI symbols + # run: make check-limited-abi + # - name: Check for unsupported C global variables + # if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME + # run: make check-c-globals build_win32: name: 'Windows (x86)' @@ -207,12 +207,12 @@ jobs: - uses: actions/checkout@v4 - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p x64 - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci + # - name: Build CPython + # run: .\PCbuild\build.bat -e -d -p x64 + # - name: Display build info + # run: .\python.bat -m test.pythoninfo + # - name: Tests + # run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci build_win_arm64: name: 'Windows (arm64)' @@ -226,8 +226,8 @@ jobs: - uses: actions/checkout@v4 - name: Register MSVC problem matcher run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p arm64 + # - name: Build CPython + # run: .\PCbuild\build.bat -e -d -p arm64 build_macos: name: 'macOS' @@ -242,28 +242,28 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk - - name: Configure CPython - run: | - GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ - GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ - ./configure \ - --config-cache \ - --with-pydebug \ - --prefix=/opt/python-dev \ - --with-openssl="$(brew --prefix openssl@3.0)" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: make test + # - name: Restore config.cache + # uses: actions/cache@v3 + # with: + # path: config.cache + # key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + # - name: Install Homebrew dependencies + # run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + # - name: Configure CPython + # run: | + # GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ + # GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ + # ./configure \ + # --config-cache \ + # --with-pydebug \ + # --prefix=/opt/python-dev \ + # --with-openssl="$(brew --prefix openssl@3.0)" + # - name: Build CPython + # run: make -j4 + # - name: Display build info + # run: make pythoninfo + # - name: Tests + # run: make test build_ubuntu: name: 'Ubuntu' @@ -280,57 +280,57 @@ jobs: run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v3 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV - - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --with-openssl=$OPENSSL_DIR - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - - name: Tests - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make test + # - name: Configure OpenSSL env vars + # run: | + # echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + # echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + # echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + # - name: 'Restore OpenSSL build' + # id: cache-openssl + # uses: actions/cache@v3 + # with: + # path: ./multissl/openssl/${{ env.OPENSSL_VER }} + # key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + # - name: Install OpenSSL + # if: steps.cache-openssl.outputs.cache-hit != 'true' + # run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + # - name: Add ccache to PATH + # run: | + # echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + # - name: Configure ccache action + # uses: hendrikmuhs/ccache-action@v1.2 + # - name: Setup directory envs for out-of-tree builds + # run: | + # echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV + # echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + # - name: Create directories for read-only out-of-tree builds + # run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + # - name: Bind mount sources read-only + # run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + # - name: Restore config.cache + # uses: actions/cache@v3 + # with: + # path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + # key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + # - name: Configure CPython out-of-tree + # working-directory: ${{ env.CPYTHON_BUILDDIR }} + # run: | + # ../cpython-ro-srcdir/configure \ + # --config-cache \ + # --with-pydebug \ + # --with-openssl=$OPENSSL_DIR + # - name: Build CPython out-of-tree + # working-directory: ${{ env.CPYTHON_BUILDDIR }} + # run: make -j4 + # - name: Display build info + # working-directory: ${{ env.CPYTHON_BUILDDIR }} + # run: make pythoninfo + # - name: Remount sources writable for tests + # # some tests write to srcdir, lack of pyc files slows down testing + # run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + # - name: Tests + # working-directory: ${{ env.CPYTHON_BUILDDIR }} + # run: xvfb-run make test build_ubuntu_nogil: name: 'Ubuntu' @@ -419,41 +419,41 @@ jobs: steps: - uses: actions/checkout@v4 - name: Restore config.cache - uses: actions/cache@v3 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v3 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Configure CPython - run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: SSL tests - run: ./python Lib/test/ssltests.py + # uses: actions/cache@v3 + # with: + # path: config.cache + # key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + # - name: Register gcc problem matcher + # run: echo "::add-matcher::.github/problem-matchers/gcc.json" + # - name: Install Dependencies + # run: sudo ./.github/workflows/posix-deps-apt.sh + # - name: Configure OpenSSL env vars + # run: | + # echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + # echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + # echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + # - name: 'Restore OpenSSL build' + # id: cache-openssl + # uses: actions/cache@v3 + # with: + # path: ./multissl/openssl/${{ env.OPENSSL_VER }} + # key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + # - name: Install OpenSSL + # if: steps.cache-openssl.outputs.cache-hit != 'true' + # run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + # - name: Add ccache to PATH + # run: | + # echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + # - name: Configure ccache action + # uses: hendrikmuhs/ccache-action@v1.2 + # - name: Configure CPython + # run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR + # - name: Build CPython + # run: make -j4 + # - name: Display build info + # run: make pythoninfo + # - name: SSL tests + # run: ./python Lib/test/ssltests.py test_hypothesis: name: "Hypothesis tests on Ubuntu" @@ -468,100 +468,100 @@ jobs: - uses: actions/checkout@v4 - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v3 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV - - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --with-openssl=$OPENSSL_DIR - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV - - name: "Create hypothesis venv" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - VENV_LOC=$(realpath -m .)/hypovenv - VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV - echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV - ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt - - name: 'Restore Hypothesis database' - id: cache-hypothesis-database - uses: actions/cache@v3 - with: - path: ./hypothesis - key: hypothesis-database-${{ github.head_ref || github.run_id }} - restore-keys: | - - hypothesis-database- - - name: "Run tests" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - # Most of the excluded tests are slow test suites with no property tests - # - # (GH-104097) test_sysconfig is skipped because it has tests that are - # failing when executed from inside a virtual environment. - ${{ env.VENV_PYTHON }} -m test \ - -W \ - -o \ - -j4 \ - -x test_asyncio \ - -x test_multiprocessing_fork \ - -x test_multiprocessing_forkserver \ - -x test_multiprocessing_spawn \ - -x test_concurrent_futures \ - -x test_socket \ - -x test_subprocess \ - -x test_signal \ - -x test_sysconfig - - uses: actions/upload-artifact@v3 - if: always() - with: - name: hypothesis-example-db - path: .hypothesis/examples/ + # - name: Install Dependencies + # run: sudo ./.github/workflows/posix-deps-apt.sh + # - name: Configure OpenSSL env vars + # run: | + # echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + # echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + # echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + # - name: 'Restore OpenSSL build' + # id: cache-openssl + # uses: actions/cache@v3 + # with: + # path: ./multissl/openssl/${{ env.OPENSSL_VER }} + # key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + # - name: Install OpenSSL + # if: steps.cache-openssl.outputs.cache-hit != 'true' + # run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + # - name: Add ccache to PATH + # run: | + # echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + # - name: Configure ccache action + # uses: hendrikmuhs/ccache-action@v1.2 + # - name: Setup directory envs for out-of-tree builds + # run: | + # echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV + # echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + # - name: Create directories for read-only out-of-tree builds + # run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + # - name: Bind mount sources read-only + # run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + # - name: Restore config.cache + # uses: actions/cache@v3 + # with: + # path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + # key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + # - name: Configure CPython out-of-tree + # working-directory: ${{ env.CPYTHON_BUILDDIR }} + # run: | + # ../cpython-ro-srcdir/configure \ + # --config-cache \ + # --with-pydebug \ + # --with-openssl=$OPENSSL_DIR + # - name: Build CPython out-of-tree + # working-directory: ${{ env.CPYTHON_BUILDDIR }} + # run: make -j4 + # - name: Display build info + # working-directory: ${{ env.CPYTHON_BUILDDIR }} + # run: make pythoninfo + # - name: Remount sources writable for tests + # # some tests write to srcdir, lack of pyc files slows down testing + # run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + # - name: Setup directory envs for out-of-tree builds + # run: | + # echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + # - name: "Create hypothesis venv" + # working-directory: ${{ env.CPYTHON_BUILDDIR }} + # run: | + # VENV_LOC=$(realpath -m .)/hypovenv + # VENV_PYTHON=$VENV_LOC/bin/python + # echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV + # echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV + # ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt + # - name: 'Restore Hypothesis database' + # id: cache-hypothesis-database + # uses: actions/cache@v3 + # with: + # path: ./hypothesis + # key: hypothesis-database-${{ github.head_ref || github.run_id }} + # restore-keys: | + # - hypothesis-database- + # - name: "Run tests" + # working-directory: ${{ env.CPYTHON_BUILDDIR }} + # run: | + # # Most of the excluded tests are slow test suites with no property tests + # # + # # (GH-104097) test_sysconfig is skipped because it has tests that are + # # failing when executed from inside a virtual environment. + # ${{ env.VENV_PYTHON }} -m test \ + # -W \ + # -o \ + # -j4 \ + # -x test_asyncio \ + # -x test_multiprocessing_fork \ + # -x test_multiprocessing_forkserver \ + # -x test_multiprocessing_spawn \ + # -x test_concurrent_futures \ + # -x test_socket \ + # -x test_subprocess \ + # -x test_signal \ + # -x test_sysconfig + # - uses: actions/upload-artifact@v3 + # if: always() + # with: + # name: hypothesis-example-db + # path: .hypothesis/examples/ build_asan: @@ -581,41 +581,41 @@ jobs: with: path: config.cache key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 10 - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v3 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: xvfb-run make test + # - name: Register gcc problem matcher + # run: echo "::add-matcher::.github/problem-matchers/gcc.json" + # - name: Install Dependencies + # run: sudo ./.github/workflows/posix-deps-apt.sh + # - name: Set up GCC-10 for ASAN + # uses: egor-tensin/setup-gcc@v1 + # with: + # version: 10 + # - name: Configure OpenSSL env vars + # run: | + # echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + # echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + # echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + # - name: 'Restore OpenSSL build' + # id: cache-openssl + # uses: actions/cache@v3 + # with: + # path: ./multissl/openssl/${{ env.OPENSSL_VER }} + # key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + # - name: Install OpenSSL + # if: steps.cache-openssl.outputs.cache-hit != 'true' + # run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + # - name: Add ccache to PATH + # run: | + # echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + # - name: Configure ccache action + # uses: hendrikmuhs/ccache-action@v1.2 + # - name: Configure CPython + # run: ./configure --config-cache --with-address-sanitizer --without-pymalloc + # - name: Build CPython + # run: make -j4 + # - name: Display build info + # run: make pythoninfo + # - name: Tests + # run: xvfb-run make test # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: From a9d5aea2f017dcc000471c713f26d8396ca4a813 Mon Sep 17 00:00:00 2001 From: Nicholas Ipsen Date: Sun, 29 Oct 2023 05:10:12 +0100 Subject: [PATCH 4/8] Disabled tests to speed up debugging --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 000a14cb2bdf41..e1eefb26e6a02e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -393,12 +393,12 @@ jobs: - name: Display build info working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - - name: Tests - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make test + # - name: Remount sources writable for tests + # # some tests write to srcdir, lack of pyc files slows down testing + # run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + # - name: Tests + # working-directory: ${{ env.CPYTHON_BUILDDIR }} + # run: xvfb-run make test build_ubuntu_ssltests: From 0e3eafc3263efbcc2ebabc52d6fbc09f23dfb210 Mon Sep 17 00:00:00 2001 From: Nicholas Ipsen Date: Sun, 29 Oct 2023 05:21:19 +0100 Subject: [PATCH 5/8] Re-enabled neccesary step. --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e1eefb26e6a02e..be85e99633fd04 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -419,10 +419,10 @@ jobs: steps: - uses: actions/checkout@v4 - name: Restore config.cache - # uses: actions/cache@v3 - # with: - # path: config.cache - # key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} # - name: Register gcc problem matcher # run: echo "::add-matcher::.github/problem-matchers/gcc.json" # - name: Install Dependencies From 21f62c4ec4877464142b09520a3e3598c7a1397f Mon Sep 17 00:00:00 2001 From: Nicholas Ipsen Date: Sun, 29 Oct 2023 05:30:01 +0100 Subject: [PATCH 6/8] Disable missed windows tests for development --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index be85e99633fd04..feafcc8d271ea0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -192,8 +192,8 @@ jobs: run: .\PCbuild\build.bat -e -d -p Win32 - name: Display build info run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci + # - name: Tests + # run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci build_win_amd64: name: 'Windows (x64)' From 1a56987ef3c03b971a7df8a2cf9e9a0ee4a0dcae Mon Sep 17 00:00:00 2001 From: Nicholas Ipsen Date: Sun, 29 Oct 2023 05:39:18 +0100 Subject: [PATCH 7/8] Enable build_ubuntu_nogil as non-required --- .github/workflows/build.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index feafcc8d271ea0..a75b31cdb2c9c5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -188,10 +188,10 @@ jobs: IncludeUwp: 'true' steps: - uses: actions/checkout@v4 - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p Win32 - - name: Display build info - run: .\python.bat -m test.pythoninfo + # - name: Build CPython + # run: .\PCbuild\build.bat -e -d -p Win32 + # - name: Display build info + # run: .\python.bat -m test.pythoninfo # - name: Tests # run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci @@ -670,6 +670,7 @@ jobs: - build_win_arm64 - build_macos - build_ubuntu + - build_ubuntu_nogil # Leave disabled until considered required - build_ubuntu_ssltests - test_hypothesis - build_asan @@ -683,6 +684,7 @@ jobs: with: allowed-failures: >- build_macos, + build_ubuntu_nogil, build_ubuntu_ssltests, build_win32, build_win_arm64, @@ -705,6 +707,7 @@ jobs: build_win_arm64, build_macos, build_ubuntu, + build_ubuntu_nogil, build_ubuntu_ssltests, build_asan, ' From ee4635af51f7a5259ecaa3ec58ae8e2f7d5b231e Mon Sep 17 00:00:00 2001 From: Nicholas Ipsen Date: Sun, 29 Oct 2023 05:45:05 +0100 Subject: [PATCH 8/8] Changed new workflow name for clarity --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a75b31cdb2c9c5..472e3412a82f2d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -333,7 +333,7 @@ jobs: # run: xvfb-run make test build_ubuntu_nogil: - name: 'Ubuntu' + name: 'Ubuntu nogil-build' runs-on: ubuntu-latest timeout-minutes: 60 needs: check_source