feat: Update github python releaase workflow to release wheels for linux, macos and windows

2020-06-11 11:54:35 +01:00 · 2020-06-11 11:54:35 +01:00 · c95a8d5903
parent 059ac3238b
commit c95a8d5903
9 changed files with 250 additions and 124 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -21,11 +21,11 @@ jobs:
    name: Generic pre-commit checks
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
        with:
          fetch-depth: 1

-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v2
        with:
          python-version: 3.7

@ -54,12 +54,16 @@ jobs:
      - uses: actions/checkout@v2
        with:
          submodules: true
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
      - name: Start background server  # User for integration tests
        run: |
-          # This assumes that python3 is installed on the system
-          /usr/bin/env python3 -m pip install flask
+          python -m pip install flask
          # Starts the server in background
-          /usr/bin/env python3 ./tests/suite/bin/jsonschema_suite serve &
+          python ./tests/suite/bin/jsonschema_suite serve &
      - uses: actions-rs/toolchain@v1
        with:
          profile: minimal
@ -77,15 +81,18 @@ jobs:
      - uses: actions/checkout@v2
        with:
          submodules: true
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
      - name: Start background server  # User for integration tests
        run: |
-          # This assumes that python3 is installed on the system
-          /usr/bin/env python3 -m pip install flask
+          python -m pip install flask
          # Starts the server in background
-          /usr/bin/env python3 ./tests/suite/bin/jsonschema_suite serve &
+          python ./tests/suite/bin/jsonschema_suite serve &
      - uses: actions-rs/toolchain@v1
        with:
-          toolchain: nightly
+          # TODO: Using nightly makes `cargo build` fail (at least it happens with 2020-06-12 nightly compiler)
+          toolchain: nightly-2020-06-11
          override: true
      - uses: actions-rs/cargo@v1
        with:
@ -109,37 +116,43 @@ jobs:

  test-python:
    strategy:
+      fail-fast: false
      matrix:
-        os: [ubuntu-latest]
-        python: [3.5, 3.6, 3.7, 3.8]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.5', '3.6', '3.7', '3.8']

-    name: Python ${{ matrix.python }} on ${{ matrix.os }}
+    name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v2
        with:
-          python-version: ${{ matrix.python }}
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
      - name: Start background server  # User for integration tests
        run: |
-          # This assumes that python3 is installed on the system
-          /usr/bin/env python3 -m pip install flask
+          python -m pip install flask
          # Starts the server in background
-          /usr/bin/env python3 ./tests/suite/bin/jsonschema_suite serve &
+          python ./tests/suite/bin/jsonschema_suite serve &

-      - run: /usr/bin/env python3 -m pip install setuptools_rust tox
+      - run: python -m pip install tox
        working-directory: ./python

-      - run: ./build-sdist.sh
-        working-directory: ./python
+      - uses: actions-rs/toolchain@v1
+        with:
+          # TODO: Using nightly makes `cargo build` fail (at least it happens with 2020-06-12 nightly compiler)
+          toolchain: nightly-2020-06-11
+          override: true
+
+      # Use the unreleased version of jsonschema rust library
+      # The UNRELEASED_JSONSCHEMA_PATH variable is used by `python/setup.py`
+      - run: echo "::set-env name=UNRELEASED_JSONSCHEMA_PATH::$(pwd)"

      - name: Run ${{ matrix.python }} tox job
-        run: tox -e py${TOX_JOB//.} # Strip dot from python version to match tox job
+        run: tox -e py
        working-directory: ./python
-        env:
-          TOX_JOB: ${{ matrix.python }}

  fmt:
    name: Rustfmt
@ -172,6 +185,7 @@ jobs:
        with:
          command: clippy
          args: -- -D warnings
+
  benchmark:
    name: Benchmark
    runs-on: ubuntu-latest
@ -187,6 +201,7 @@ jobs:
      with:
        command: bench
        args: jsonschema-rs  # Run the benchmarks only for the `jsonschema` package
+
  benchmark-all:
    name: Benchmark All
    runs-on: ubuntu-latest
--- a/.github/workflows/python-release.yml
+++ b/.github/workflows/python-release.yml
@ -6,46 +6,132 @@ on:
      - python-v*

 jobs:
+  create_and_upload_macos_and_windows_wheels:
+    name: Wheels for Python ${{ matrix.python-version }} / ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest, windows-latest]
+        python-version: ['3.5', '3.6', '3.7', '3.8']
+        architecture: [x86, x64]
+        exclude:
+        - os: macos-latest
+          architecture: x86
+        - os: windows-latest
+          # TODO: Re-enable windows 32bits
+          architecture: x86
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: ${{ matrix.architecture }}
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+
+      - name: Install Tox
+        run: pip install tox
+
+      - name: Build wheel
+        working-directory: ./python
+        run: tox -e build-wheel
+
+      - name: Publish distribution package to PyPI
+        uses: pypa/gh-action-pypi-publish@v1.2.2
+        with:
+          user: ${{ secrets.PYPI_USERNAME }}
+          password: ${{ secrets.PYPI_PASSWORD }}
+          packages_dir: python/dist/

  create_wheels_manylinux:
    runs-on: ubuntu-latest
-    name: Create wheels for manylinux
-    container: quay.io/pypa/manylinux2010_x86_64
+    name: Wheels for Python ${{ matrix.PYTHON_IMPLEMENTATION_ABI }} / Linux
+    container: quay.io/pypa/manylinux2014_x86_64  # Builds wheels on CentOS 7 (supported until 2024)
+    strategy:
+      fail-fast: false
+      matrix:
+        # List of the language-implementation API pairs to publish wheels for
+        # The list of supported is obtainable by running `docker run quay.io/pypa/manylinux2014_x86_64 ls /opt/python`
+        PYTHON_IMPLEMENTATION_ABI: [cp35-cp35m, cp36-cp36m, cp37-cp37m, cp38-cp38]
+    env:
+      # Variable needed for PyO3 to properly identify the python interpreter
+      PYTHON_SYS_EXECUTABLE: /opt/python/${{ matrix.PYTHON_IMPLEMENTATION_ABI }}/bin/python
    steps:
-      - uses: actions/checkout@v1

-      - name: Build and audit wheels
+      - uses: actions/checkout@v2
+
+      - name: Install/Update OpenSSL
+        run: |
+          retryCount=0
+          # yum install seems to be flakey (due to network timeouts)
+          # retry up to 5 times with a 10s sleep in case of failure
+          until yum install openssl-devel --assumeyes --noplugins; do
+            # For some reason the install has failed
+            if [ ${retryCount} -eq 5 ]; then
+                false
+            else
+              retryCount=$((${retryCount}+1))
+            fi
+            sleep 10
+          done
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+
+      - name: Install Tox
+        run: ${{ env.PYTHON_SYS_EXECUTABLE }} -m pip install tox
+
+      - name: Build wheel
        working-directory: ./python
-        run: sh build-wheels.sh
+        run: |
+          ${{ env.PYTHON_SYS_EXECUTABLE }} -m tox -e build-wheel
+          # Ensure that the wheel is tagged as manylinux2014 platform
+          auditwheel repair \
+            --wheel-dir=./dist \
+            --plat manylinux2014_x86_64 \
+            ./dist/jsonschema_rs-*-${{ matrix.PYTHON_IMPLEMENTATION_ABI }}-linux_x86_64.whl

-      - run: /opt/python/cp37-cp37m/bin/pip install twine
+      - name: Publish distribution package to PyPI
+        uses: pypa/gh-action-pypi-publish@v1.2.2
+        with:
+          user: ${{ secrets.PYPI_USERNAME }}
+          password: ${{ secrets.PYPI_PASSWORD }}
+          packages_dir: python/dist/

-      - name: Upload package
-        working-directory: ./python
-        run: /opt/python/cp37-cp37m/bin/twine upload ./dist/* --username=${PYPI_USERNAME} --password=${PYPI_PASSWORD}
-        env:
-          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-
-  create_source_dist:
+  create_and_upload_source_dist:
    name: Create sdist package
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2

-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v2
        with:
          python-version: 3.7

-      - run: pip install setuptools-rust twine
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+
+      - name: Install Tox
+        run: pip install tox

      - name: Build sdist
        working-directory: ./python
-        run: sh build-sdist.sh
+        run: tox -e build-sdist

-      - name: Upload package
-        working-directory: ./python
-        run: twine upload ./dist/* --username=${PYPI_USERNAME} --password=${PYPI_PASSWORD}
-        env:
-          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      - name: Publish distribution package to PyPI
+        uses: pypa/gh-action-pypi-publish@v1.2.2
+        with:
+          user: ${{ secrets.PYPI_USERNAME }}
+          password: ${{ secrets.PYPI_PASSWORD }}
+          packages_dir: python/dist/
--- a/python/MANIFEST.in
+++ b/python/MANIFEST.in
@ -2,5 +2,5 @@ include Cargo.toml
 include pyproject.toml
 include rust-toolchain
 recursive-include src *
-recursive-include jsonschema/src/ *
+recursive-include jsonschema/src *
 include jsonschema/Cargo.toml
--- a/python/build-sdist.sh
+++ b/python/build-sdist.sh
@ -1,15 +0,0 @@
-#!/bin/bash
-# `setuptools_rust` and `maturin` don't support some local dependencies as `jsonschema` is (it is in the parent directory)
-# As a workaround we create a modified distribution of this library that has `jsonschema` crate as a dependency in
-# the same directory, then the sources are copied as declared in MANIFEST.in and the resulting package can be
-# installed properly
-set -ex
-
-ln -sf ../ jsonschema
-# Modify cargo.toml to include this symlink
-sed -i 's/\.\.\//jsonschema/' Cargo.toml
-# Build the source distribution
-python setup.py sdist
-# Rollback local changes after a source distribution is ready
-rm jsonschema
-sed -i 's/"jsonschema"/"\.\.\/"/' Cargo.toml
--- a/python/build-wheels.sh
+++ b/python/build-wheels.sh
@ -1,20 +0,0 @@
-#!/bin/bash
-set -ex
-
-yum install openssl-devel -y
-
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly -y
-export PATH="$HOME/.cargo/bin:$PATH"
-
-for PYBIN in /opt/python/{cp35-cp35m,cp36-cp36m,cp37-cp37m,cp38-cp38}/bin; do
-    export PYTHON_SYS_EXECUTABLE="$PYBIN/python"
-
-    "${PYBIN}/pip" install -U setuptools wheel setuptools-rust
-    "${PYBIN}/python" setup.py bdist_wheel
-    rm build/lib/jsonschema_rs/*.so
-done
-
-for whl in dist/*.whl; do
-    auditwheel repair "$whl" -w dist/
-    rm "$whl"
-done
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools", "wheel", "setuptools-rust"]
+requires = ["setuptools", "setuptools-rust", "toml", "wheel"]

 [tool.black]
 line-length = 120
--- a/python/rust-toolchain
+++ b/python/rust-toolchain
@ -1 +1,2 @@
-nightly
+nightly-2020-06-10
+# TODO: Using nightly makes `cargo build` fail (at least it happens with 2020-06-12 nightly compiler)
--- a/python/setup.py
+++ b/python/setup.py
@ -1,10 +1,12 @@
-from textwrap import dedent
+import os

 from setuptools import find_packages, setup

 try:
    from setuptools_rust import Binding, RustExtension
 except ImportError:
+    from textwrap import dedent
+
    raise ImportError(
        dedent(
            """
@ -17,35 +19,87 @@ except ImportError:
    )


-setup(
-    name="jsonschema_rs",
-    version="0.3.0",
-    description="Fast JSON Schema validation for Python implemented in Rust",
-    long_description=open("README.rst", encoding="utf-8").read(),
-    long_description_content_type="text/x-rst",
-    keywords="jsonschema validation rust",
-    author="Dmitry Dygalo",
-    author_email="dadygalo@gmail.com",
-    maintainer="Dmitry Dygalo",
-    maintainer_email="dadygalo@gmail.com",
-    python_requires=">=3.5",
-    url="https://github.com/Stranger6667/jsonschema-rs/python",
-    license="MIT",
-    rust_extensions=[RustExtension("jsonschema_rs.jsonschema_rs", binding=Binding.PyO3)],
-    classifiers=[
-        "Development Status :: 3 - Alpha",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: MIT License",
-        "Operating System :: OS Independent",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: Implementation :: CPython",
-        "Programming Language :: Rust",
-    ],
-    packages=find_packages(where="pysrc"),
-    package_dir={"": "pysrc"},
-    zip_safe=False,
-)
+def call_setup():
+    setup(
+        name="jsonschema_rs",
+        version="0.3.0",
+        description="Fast JSON Schema validation for Python implemented in Rust",
+        long_description=open("README.rst", encoding="utf-8").read(),
+        long_description_content_type="text/x-rst",
+        keywords="jsonschema validation rust",
+        author="Dmitry Dygalo",
+        author_email="dadygalo@gmail.com",
+        maintainer="Dmitry Dygalo",
+        maintainer_email="dadygalo@gmail.com",
+        python_requires=">=3.5",
+        url="https://github.com/Stranger6667/jsonschema-rs/python",
+        license="MIT",
+        rust_extensions=[RustExtension("jsonschema_rs.jsonschema_rs", binding=Binding.PyO3)],
+        classifiers=[
+            "Development Status :: 3 - Alpha",
+            "Intended Audience :: Developers",
+            "License :: OSI Approved :: MIT License",
+            "Operating System :: OS Independent",
+            "Programming Language :: Python :: 3",
+            "Programming Language :: Python :: 3.5",
+            "Programming Language :: Python :: 3.6",
+            "Programming Language :: Python :: 3.7",
+            "Programming Language :: Python :: 3.8",
+            "Programming Language :: Python :: Implementation :: CPython",
+            "Programming Language :: Rust",
+        ],
+        packages=find_packages(where="pysrc"),
+        package_dir={"": "pysrc"},
+        zip_safe=False,
+    )
+
+
+if "UNRELEASED_JSONSCHEMA_PATH" in os.environ:
+    import sys
+
+    from contextlib import contextmanager
+    from tempfile import NamedTemporaryFile
+    import toml
+
+    # Modify Cargo.toml to apply crates.io patch to link jsonschema to the version available
+    # on the repository.
+    # This is done by using a contextmanager in order to ensure that the Cargo.toml modification
+    # is discarded once setup.py ends (regardless of success or failure)
+    @contextmanager
+    def cargo_toml_context():
+        tmp_file = NamedTemporaryFile(buffering=False)
+        with open("Cargo.toml", "rb") as f:
+            tmp_file.writelines(f.readlines())
+
+        cargo_file = toml.load("Cargo.toml")
+
+        cargo_file.setdefault("patch", {}).setdefault("crates-io", {})["jsonschema"] = {
+            "path": os.environ["UNRELEASED_JSONSCHEMA_PATH"],
+        }
+
+        with open("Cargo.toml", "w") as f:
+            toml.dump(cargo_file, f)
+
+        try:
+            print(
+                "Modified Cargo.toml file by patching jsonschema dependency to {}".format(
+                    os.environ["UNRELEASED_JSONSCHEMA_PATH"]
+                ),
+                file=sys.stderr,
+            )
+            yield
+        except:
+            print("Cargo.toml used during the build", file=sys.stderr)
+            with open("Cargo.toml", "r") as f:
+                print(f.read(), file=sys.stderr)
+
+            raise
+        finally:
+            with open("Cargo.toml", "wb") as f:
+                tmp_file.seek(0)
+                f.writelines(tmp_file.readlines())
+
+    with cargo_toml_context():
+        call_setup()
+else:
+    call_setup()
--- a/python/tox.ini
+++ b/python/tox.ini
@ -1,19 +1,21 @@
 [tox]
-# Skip Source distribution build to allow each task to
-# install it via pip (workaround the fact that setup.py
-# does not honor pyproject.toml)
+# Skip Source distribution build to allow each task to install it via pip
+# (workaround the fact that setup.py does not honor pyproject.toml)
 skipsdist = True

-envlist =
-  py{35,36,37,38}
+envlist = py{35,36,37,38}

 [testenv]
+passenv =
+  # Env variable that would allow to build the jsonschema-rs library pointing to
+  # the jsonschema rust-library in the project (unreleased). Check setup.py for more details.
+  UNRELEASED_JSONSCHEMA_PATH
 deps =
  pytest
  pytest-benchmark
  hypothesis
-  .
 commands =
+  pip install -e .  # Installing it within commands allow faster evn build (NOTE: uses debug rust build)
  python -m pytest tests-py {posargs:}

 [testenv:build-sdist]
@ -23,6 +25,9 @@ commands =
  python setup.py sdist

 [testenv:build-wheel]
+passenv =
+  {[testenv]passenv}
+  PYTHON_SYS_EXECUTABLE
 deps =
  setuptools-rust
  wheel