Test 3.11, run test with -u all, improve WASI runner script (#80)

tiran · web-flow · commit 7561a70c7c69 · 2022-06-10T09:28:32.000+02:00
* Run test with -u all, improve WASI runner script

* Build on Python 3.11, too

* Create sentinel for getpath.py

* Set date in env
diff --git a/.github/actions/prepare/action.yml b/.github/actions/prepare/action.yml
@@ -11,13 +11,13 @@ runs:
       id: get-date
       shell: bash
       run: |
-        echo "::set-output name=date::$(/bin/date -u "+%Y%m%d")"
+        echo "TODAY=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_ENV
     - name: Get container cache
       id: container-cache
       uses: actions/cache@v1
       with:
         path: /tmp/container-cache
-        key: image-cache-${{ steps.get-date.outputs.date }}
+        key: image-cache-${{ env.TODAY }}
     - name: "Pull build image"
       if: ${{ steps.container-cache.outputs.cache-hit != 'true' }}
       shell: bash
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -17,9 +17,22 @@ on:
         required: false
 
 jobs:
+  pull-buildcontainer:
+    name: "Pull & cache build container"
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: "checkout python-wasm"
+        uses: "actions/checkout@v2"
+      - name: "Common prepare step"
+        uses: ./.github/actions/prepare
   build-python:
-    name: "Build build Python"
+    name: "Build build Python ${{ matrix.pythonbranch }}"
     runs-on: "ubuntu-latest"
+    needs: pull-buildcontainer
+    strategy:
+      fail-fast: false
+      matrix:
+        pythonbranch: [main, 3.11]
     steps:
       - name: "checkout python-wasm"
         uses: "actions/checkout@v2"
@@ -28,7 +41,7 @@ jobs:
         with:
           repository: python/cpython
           path: cpython
-          ref: main
+          ref: ${{ matrix.pythonbranch }}
       - name: "Verify checkout"
         shell: bash
         run: |
@@ -42,25 +55,29 @@ jobs:
         uses: actions/cache@v2
         with:
           path: cpython
-          key: cpython-${{ runner.os }}-${{ github.sha }}
+          key: cpython-${{ matrix.pythonbranch }}-${{ runner.os }}-${{ env.TODAY }}-${{ github.sha }}
   emscripte-node:
-    name: "Build Emscripten node"
+    name: "Build Emscripten node ${{ matrix.pythonbranch }}"
     runs-on: "ubuntu-latest"
     needs: build-python
+    strategy:
+      fail-fast: false
+      matrix:
+        pythonbranch: [main, 3.11]
     steps:
       - name: "checkout python-wasm"
         uses: "actions/checkout@v2"
+      - name: "Common prepare step"
+        uses: ./.github/actions/prepare
       - name: "Fetch cached build Python"
         uses: actions/cache@v2
         with:
           path: cpython
-          key: cpython-${{ runner.os }}-${{ github.sha }}
+          key: cpython-${{ matrix.pythonbranch }}-${{ runner.os }}-${{ env.TODAY }}-${{ github.sha }}
       - name: "Check build Python"
         run: |
           test -e cpython/builddir/build/python || exit 1
           test -e cpython/configure || exit 2
-      - name: "Common prepare step"
-        uses: ./.github/actions/prepare
       - name: "Build emscripten Python for node"
         run: docker run --rm -v $(pwd):/build -w /build quay.io/tiran/cpythonbuild:emsdk3 ./build-python-emscripten-node.sh
       - name: "Check artifacts"
@@ -70,11 +87,11 @@ jobs:
       - name: "Print test.pythoninfo"
         run: docker run --rm -v $(pwd):/build -w /build quay.io/tiran/cpythonbuild:emsdk3 ./run-python-node.sh -m test.pythoninfo
       - name: "Run tests"
-        run: docker run --rm -v $(pwd):/build -w /build quay.io/tiran/cpythonbuild:emsdk3 ./test-emscripten-node.sh
+        run: docker run --rm -v $(pwd):/build -w /build quay.io/tiran/cpythonbuild:emsdk3 ./test-emscripten-node.sh -u all -W --slowest --fail-env-changed
       - name: "Upload node build artifacts"
         uses: actions/upload-artifact@v2
         with:
-          name: emscripten-node
+          name: emscripten-node-${{ matrix.pythonbranch }}
           path: |
             cpython/builddir/emscripten-node/python.wasm
             cpython/builddir/emscripten-node/python.worker.js
@@ -85,7 +102,7 @@ jobs:
       - name: "Upload build artifacts"
         uses: actions/upload-artifact@v2
         with:
-          name: build-node
+          name: build-node-${{ matrix.pythonbranch }}
           path: |
             cpython/builddir/emscripten-node/config.log
             cpython/builddir/emscripten-node/config.cache
@@ -100,23 +117,27 @@ jobs:
             cpython/builddir/emscripten-node/Programs/python.o
           if-no-files-found: error
   emscripte-browser:
-    name: "Build Emscripten browser"
+    name: "Build Emscripten browser ${{ matrix.pythonbranch }}"
     runs-on: "ubuntu-latest"
     needs: build-python
+    strategy:
+      fail-fast: false
+      matrix:
+        pythonbranch: [main, 3.11]
     steps:
       - name: "checkout python-wasm"
         uses: "actions/checkout@v2"
+      - name: "Common prepare step"
+        uses: ./.github/actions/prepare
       - name: "Fetch cached build Python"
         uses: actions/cache@v2
         with:
           path: cpython
-          key: cpython-${{ runner.os }}-${{ github.sha }}
+          key: cpython-${{ matrix.pythonbranch }}-${{ runner.os }}-${{ env.TODAY }}-${{ github.sha }}
       - name: "Check build Python"
         run: |
           test -e cpython/builddir/build/python || exit 1
           test -e cpython/configure || exit 2
-      - name: "Common prepare step"
-        uses: ./.github/actions/prepare
       - name: "Build emscripten Python for browser"
         run: docker run --rm -v $(pwd):/build -w /build quay.io/tiran/cpythonbuild:emsdk3 ./build-python-emscripten-browser.sh
       - name: "Check artifacts"
@@ -127,7 +148,7 @@ jobs:
       - name: "Upload browser build artifacts"
         uses: actions/upload-artifact@v2
         with:
-          name: emscripten-browser
+          name: emscripten-browser-${{ matrix.pythonbranch }}
           path: |
             cpython/builddir/emscripten-browser/python.wasm
             cpython/builddir/emscripten-browser/python.html
@@ -138,7 +159,7 @@ jobs:
       - name: "Upload build artifacts"
         uses: actions/upload-artifact@v2
         with:
-          name: build-browser
+          name: build-browser-${{ matrix.pythonbranch }}
           path: |
             cpython/builddir/emscripten-browser/config.log
             cpython/builddir/emscripten-browser/config.cache
@@ -154,23 +175,27 @@ jobs:
             cpython/builddir/emscripten-browser/Programs/python.o
           if-no-files-found: error
   wasi:
-    name: "Build WASI"
+    name: "Build WASI ${{ matrix.pythonbranch }}"
     runs-on: "ubuntu-latest"
     needs: build-python
+    strategy:
+      fail-fast: false
+      matrix:
+        pythonbranch: [main, 3.11]
     steps:
       - name: "checkout python-wasm"
         uses: "actions/checkout@v2"
+      - name: "Common prepare step"
+        uses: ./.github/actions/prepare
       - name: "Fetch cached build Python"
         uses: actions/cache@v2
         with:
           path: cpython
-          key: cpython-${{ runner.os }}-${{ github.sha }}
+          key: cpython-${{ matrix.pythonbranch }}-${{ runner.os }}-${{ env.TODAY }}-${{ github.sha }}
       - name: "Check build Python"
         run: |
           test -e cpython/builddir/build/python || exit 1
           test -e cpython/configure || exit 2
-      - name: "Common prepare step"
-        uses: ./.github/actions/prepare
       - name: "Build WASI Python"
         run: docker run --rm -v $(pwd):/build -w /build quay.io/tiran/cpythonbuild:emsdk3 ./build-python-wasi.sh
       - name: "Check artifacts"
@@ -180,13 +205,13 @@ jobs:
       - name: "Print test.pythoninfo"
         run: docker run --rm -v $(pwd):/build -w /build quay.io/tiran/cpythonbuild:emsdk3 ./run-python-wasi.sh -m test.pythoninfo
       - name: "Run tests"
-        run: docker run --rm -v $(pwd):/build -w /build quay.io/tiran/cpythonbuild:emsdk3 ./test-wasi.sh
+        run: docker run --rm -v $(pwd):/build -w /build quay.io/tiran/cpythonbuild:emsdk3 ./test-wasi.sh -u all -W --slowest --fail-env-changed
         # some WASI tests are failing
         continue-on-error: true
       - name: "Upload WASI artifacts"
         uses: actions/upload-artifact@v2
         with:
-          name: wasi
+          name: wasi-${{ matrix.pythonbranch }}
           path: |
             cpython/builddir/wasi/python.wasm
             cpython/builddir/wasi/pybuilddir.txt
@@ -195,7 +220,7 @@ jobs:
       - name: "Upload build artifacts"
         uses: actions/upload-artifact@v2
         with:
-          name: build-wasi
+          name: build-wasi-${{ matrix.pythonbranch }}
           path: |
             cpython/builddir/wasi/config.log
             cpython/builddir/wasi/config.cache
@@ -219,7 +244,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/download-artifact@v2
         with:
-          name: emscripten-browser
+          name: emscripten-browser-main
           path: wasm
       - name: "Prepare artifacts for Github Pages"
         run: |
diff --git a/build-python-wasi.sh b/build-python-wasi.sh
@@ -25,12 +25,7 @@ CONFIG_SITE=../../Tools/wasm/config.site-wasm32-wasi \
     --disable-ipv6
 
 make -j$(nproc)
-
-# XXX hack
-# Symlink pybuilddir.txt and build lib directory with sysconfig data into
-# VPATH root. "--mapdir .::../../" uses VPATH root as CWD.
-mkdir -p ../../build
-ln -srf -t ../../ pybuilddir.txt
-ln -srf -t ../../build/ build/lib.wasi-wasm32-3.*
-
 popd
+
+# sentinel for getpath.py
+touch cpython/Modules/Setup.local
diff --git a/run-python-wasi.sh b/run-python-wasi.sh
@@ -3,5 +3,8 @@ set -e
 
 export PATH="$PATH:/root/.wasmtime/bin"
 
-cd cpython/builddir/wasi
-exec wasmtime run --mapdir .::../../ -- python.wasm "$@"
+# PYTHONPATH is relative to mapped cpython/ directory.
+exec wasmtime run \
+    --env PYTHONPATH=/builddir/wasi/$(cat cpython/builddir/wasi/pybuilddir.txt) \
+    --mapdir /::cpython/ -- \
+    cpython/builddir/wasi/python.wasm "$@"
