fix: Build platform specific python packages with ci-build-wheel (#2555)

* ci build wheel

Signed-off-by: pyalex <[email protected]>

* compile go only with env var

Signed-off-by: pyalex <[email protected]>

* filter go extensions

Signed-off-by: pyalex <[email protected]>

* copy full directory on inplace build_ext

Signed-off-by: pyalex <[email protected]>

* copytree from distutils

Signed-off-by: pyalex <[email protected]>

Author: pyalex

.github/workflows/publish.yml

@@ -126,95 +126,121 @@ jobs:
         run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX
 
   publish-python-sdk:
+    runs-on: ubuntu-latest
+    needs: [build-python-sdk, build-python-sdk-no-telemetry, build-python-sdk-macos-py310]
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: wheels
+          path: dist
+      - uses: pypa/[email protected]
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_PASSWORD }}
+
+
+  build-python-sdk:
+    name: Build wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
       matrix:
-        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
-        os: [ ubuntu-latest, macOS-latest ]
-        compile-go: [ True ]
-        include:
-          - python-version: "3.7"
-            os: ubuntu-latest
-            compile-go: False
-    env:
-      TWINE_USERNAME: __token__
-      TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      COMPILE_GO: ${{ matrix.compile-go }}
+        os: [ ubuntu-latest, macos-10.15 ]
+
     steps:
       - uses: actions/checkout@v2
-      - name: Setup Python
-        id: setup-python
-        uses: actions/setup-python@v2
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.4.0
         with:
-          python-version: ${{ matrix.python-version }}
-          architecture: x64
-      - name: Setup Go
-        id: setup-go
-        uses: actions/setup-go@v2
+          package-dir: sdk/python
+        env:
+          CIBW_BUILD: "cp3*_x86_64"
+          CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
+          CIBW_ARCHS: "native"
+          CIBW_ENVIRONMENT: >
+            COMPILE_GO=True
+          CIBW_BEFORE_ALL_LINUX: |
+            yum install -y golang
+          CIBW_BEFORE_ALL_MACOS: |
+            curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg
+            sudo installer -pkg python.pkg -target /
+          CIBW_BEFORE_BUILD: |
+            make install-protoc-dependencies
+            make install-go-proto-dependencies
+            make install-go-ci-dependencies
+
+      - uses: actions/upload-artifact@v2
         with:
-          go-version: 1.17.7
-      - name: Upgrade pip version
-        run: |
-          pip install --upgrade "pip>=21.3.1"
-      - name: Install pip-tools
-        run: pip install pip-tools
-      - name: Install dependencies
-        run: make install-python-ci-dependencies PYTHON=${{ matrix.python-version }}
-      - name: Publish Python Package
-        run: |
-          cd sdk/python
-          python3 -m pip install --user --upgrade setuptools wheel twine
-          python3 setup.py sdist bdist_wheel
-          python3 -m twine upload --verbose dist/*.whl
+          name: wheels
+          path: ./wheelhouse/*.whl
+
 
-  publish-python-sdk-no-telemetry:
+  build-python-sdk-no-telemetry:
+    name: Build no telemetry wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
       matrix:
-        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
-        os: [ ubuntu-latest, macOS-latest ]
-        compile-go: [ True ]
-        include:
-          - python-version: "3.7"
-            os: ubuntu-latest
-            compile-go: False
+        os: [ ubuntu-latest, macos-10.15 ]
     needs: get-version
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          cd sdk/python
+          sed -i.bak 's/DEFAULT_FEAST_USAGE_VALUE = "True"/DEFAULT_FEAST_USAGE_VALUE = "False"/g' feast/constants.py
+          sed -i.bak 's/NAME = "feast"/NAME = "feast-no-telemetry"/g' setup.py
+      - name: Build wheels
+        uses: pypa/[email protected]
+        with:
+          package-dir: sdk/python
+        env:
+          CIBW_BUILD: "cp3*_x86_64"
+          CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
+          CIBW_ARCHS: "native"
+          CIBW_ENVIRONMENT: >
+            COMPILE_GO=True SETUPTOOLS_SCM_PRETEND_VERSION="${{ needs.get-version.outputs.version_without_prefix }}"
+          CIBW_BEFORE_ALL_LINUX: |
+            yum install -y golang
+          CIBW_BEFORE_ALL_MACOS: |
+            curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg
+            sudo installer -pkg python.pkg -target /
+          CIBW_BEFORE_BUILD: |
+            make install-protoc-dependencies
+            make install-go-proto-dependencies
+            make install-go-ci-dependencies
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: wheels
+          path: ./wheelhouse/*.whl
+
+  build-python-sdk-macos-py310:
+    runs-on: macos-10.15
     env:
-      TWINE_USERNAME: __token__
-      TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      COMPILE_GO: ${{ matrix.compile-go }}
+      COMPILE_GO: True
     steps:
       - uses: actions/checkout@v2
       - name: Setup Python
         id: setup-python
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.10"
           architecture: x64
-      - name: Setup Go
-        id: setup-go
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.17.7
-      - name: Upgrade pip version
-        run: |
-          pip install --upgrade "pip>=21.3.1"
-      - name: Install pip-tools
-        run: pip install pip-tools
       - name: Install dependencies
-        run: make install-python-ci-dependencies PYTHON=${{ matrix.python-version }}
-      - name: Publish Python Package
-        env:
-          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ needs.get-version.outputs.version_without_prefix }}
+        run: |
+          pip install -U pip setuptools wheel twine
+          make install-protoc-dependencies
+          make install-go-proto-dependencies
+          make install-go-ci-dependencies
+      - name: Build
         run: |
           cd sdk/python
-          sed -i 's/DEFAULT_FEAST_USAGE_VALUE = "True"/DEFAULT_FEAST_USAGE_VALUE = "False"/g' feast/constants.py
-          sed -i 's/NAME = "feast"/NAME = "feast-no-telemetry"/g' setup.py
-          python3 -m pip install --user --upgrade setuptools wheel twine
           python3 setup.py sdist bdist_wheel
-          python3 -m twine upload --verbose dist/*.whl
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: wheels
+          path: sdk/python/dist/*
+
 
   publish-java-sdk:
     container: maven:3.6-jdk-11

Makefile

@@ -145,15 +145,15 @@ install-go-ci-dependencies:
 	go get github.com/go-python/gopy
 	go install golang.org/x/tools/cmd/goimports
 	go install github.com/go-python/gopy
+	python -m pip install pybindgen==0.22.0
 
 install-protoc-dependencies:
-	pip install grpcio-tools==1.34.0
+	pip install grpcio-tools==1.44.0 mypy-protobuf==3.1.0
 
 compile-protos-go: install-go-proto-dependencies install-protoc-dependencies
 	cd sdk/python && python setup.py build_go_protos
 
 compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies
-	python -m pip install pybindgen==0.22.0
 	cd sdk/python && python setup.py build_go_lib
 
 # Needs feast package to setup the feature store

sdk/python/setup.py

@@ -11,29 +11,33 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import copy
 import glob
+import json
 import os
 import pathlib
 import re
 import shutil
 import subprocess
 import sys
 from distutils.cmd import Command
+from distutils.dir_util import copy_tree
 from pathlib import Path
 from subprocess import CalledProcessError
 
-from setuptools import find_packages
+from setuptools import find_packages, Extension
 
 try:
     from setuptools import setup
     from setuptools.command.build_py import build_py
+    from setuptools.command.build_ext import build_ext as _build_ext
     from setuptools.command.develop import develop
     from setuptools.command.install import install
-    from setuptools.dist import Distribution
+
 except ImportError:
     from distutils.command.build_py import build_py
+    from distutils.command.build_ext import build_ext as _build_ext
     from distutils.core import setup
-    from distutils.dist import Distribution
 
 NAME = "feast"
 DESCRIPTION = "Python SDK for Feast"
@@ -189,7 +193,7 @@ class BuildPythonProtosCommand(Command):
 
     def initialize_options(self):
         self.python_protoc = [
-            "python",
+            sys.executable,
             "-m",
             "grpc_tools.protoc",
         ]  # find_executable("protoc")
@@ -292,7 +296,7 @@ class BuildGoProtosCommand(Command):
 
     def initialize_options(self):
         self.go_protoc = [
-            "python",
+            sys.executable,
             "-m",
             "grpc_tools.protoc",
         ]  # find_executable("protoc")
@@ -331,45 +335,6 @@ def run(self):
             self._generate_go_protos(f"feast/{sub_folder}/*.proto")
 
 
-class BuildGoEmbeddedCommand(Command):
-    description = "Builds Go embedded library"
-    user_options = []
-
-    def initialize_options(self) -> None:
-        self.path_val = _generate_path_with_gopath()
-
-        self.go_env = {}
-        for var in ("GOCACHE", "GOPATH"):
-            self.go_env[var] = subprocess \
-                .check_output(["go", "env", var]) \
-                .decode("utf-8") \
-                .strip()
-
-    def finalize_options(self) -> None:
-        pass
-
-    def _compile_embedded_lib(self):
-        print("Compile embedded go")
-        subprocess.check_call([
-            "gopy",
-            "build",
-            "-output",
-            "feast/embedded_go/lib",
-            "-vm",
-            # Path of current python executable
-            sys.executable,
-            "-no-make",
-            "github.com/feast-dev/feast/go/embedded"
-        ], env={
-            "PATH": self.path_val,
-            "CGO_LDFLAGS_ALLOW": ".*",
-            **self.go_env,
-        })
-
-    def run(self):
-        self._compile_embedded_lib()
-
-
 class BuildCommand(build_py):
     """Custom build command."""
 
@@ -378,7 +343,7 @@ def run(self):
         if os.getenv("COMPILE_GO", "false").lower() == "true":
             _ensure_go_and_proto_toolchain()
             self.run_command("build_go_protos")
-            self.run_command("build_go_lib")
+
         build_py.run(self)
 
 
@@ -390,15 +355,61 @@ def run(self):
         if os.getenv("COMPILE_GO", "false").lower() == "true":
             _ensure_go_and_proto_toolchain()
             self.run_command("build_go_protos")
-            self.run_command("build_go_lib")
+
         develop.run(self)
 
 
-class BinaryDistribution(Distribution):
-    """Distribution which forces a binary package with platform name
-     when go compilation is enabled"""
-    def has_ext_modules(self):
-        return os.getenv("COMPILE_GO", "false").lower() == "true"
+class build_ext(_build_ext):
+    def finalize_options(self) -> None:
+        super().finalize_options()
+        if os.getenv("COMPILE_GO", "false").lower() == "false":
+            self.extensions = [e for e in self.extensions if not self._is_go_ext(e)]
+
+    def _is_go_ext(self, ext: Extension):
+        return any(source.endswith('.go') or source.startswith('github') for source in ext.sources)
+
+    def build_extension(self, ext: Extension):
+        if not self._is_go_ext(ext):
+            # the base class may mutate `self.compiler`
+            compiler = copy.deepcopy(self.compiler)
+            self.compiler, compiler = compiler, self.compiler
+            try:
+                return _build_ext.build_extension(self, ext)
+            finally:
+                self.compiler, compiler = compiler, self.compiler
+
+        bin_path = _generate_path_with_gopath()
+        go_env = json.loads(
+            subprocess.check_output(["go", "env", "-json"]).decode("utf-8").strip()
+        )
+
+        destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name)))
+        subprocess.check_call([
+            "gopy",
+            "build",
+            "-output",
+            destination,
+            "-vm",
+            sys.executable,
+            "-no-make",
+            *ext.sources
+        ], env={
+            "PATH": bin_path,
+            "CGO_LDFLAGS_ALLOW": ".*",
+            **go_env,
+        })
+
+    def copy_extensions_to_source(self):
+        build_py = self.get_finalized_command('build_py')
+        for ext in self.extensions:
+            fullname = self.get_ext_fullname(ext.name)
+            modpath = fullname.split('.')
+            package = '.'.join(modpath[:-1])
+            package_dir = build_py.get_package_dir(package)
+            src = os.path.join(self.build_lib, package_dir)
+
+            # copy whole directory
+            copy_tree(src, package_dir)
 
 
 setup(
@@ -453,9 +464,10 @@ def has_ext_modules(self):
     cmdclass={
         "build_python_protos": BuildPythonProtosCommand,
         "build_go_protos": BuildGoProtosCommand,
-        "build_go_lib": BuildGoEmbeddedCommand,
         "build_py": BuildCommand,
         "develop": DevelopCommand,
+        "build_ext": build_ext,
     },
-    distclass=BinaryDistribution,  # generate wheel with platform-specific name
+    ext_modules=[Extension('feast.embedded_go.lib._embedded',
+                           ["github.com/feast-dev/feast/go/embedded"])],
 )