Skip to content

Support specifying variants for wheel builds #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 23 commits into from
Closed
Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
da7bf41
Support specifying variants for wheel builds
mgorny Mar 20, 2025
2cd858a
Fix breakage without variants
mgorny Mar 20, 2025
3bd701e
Include a variantlib dependency
mgorny Mar 20, 2025
b87c479
Try via URL instead
mgorny Mar 20, 2025
0717750
Support `-Dvariant` that gets passed through to meson
mgorny Mar 21, 2025
baff058
Fix handling variant-name without variant
mgorny Mar 26, 2025
8c752d5
Support appending plugin labels to wheel names
mgorny Mar 26, 2025
88645e2
variantlib changes were merged to main
mgorny Mar 26, 2025
5b8429c
Fix other variantlib reference
mgorny Mar 26, 2025
c2df897
Actually, we need variant-labels branch
mgorny Mar 26, 2025
d6ddb8c
On my fork
mgorny Mar 26, 2025
c93f2be
Install demo_plugins as part of meson-python
mgorny Mar 27, 2025
9ba73d3
Update to follow variantlib API changes
mgorny Apr 7, 2025
b2e9b80
Update the demo plugin API
mgorny Apr 7, 2025
7a71cc5
Add variant validation
mgorny Apr 8, 2025
5886b42
Use newer variant validation API
mgorny Apr 9, 2025
c100caa
Set CFLAGS/CXXFLAGS via get_build_setup()
mgorny Apr 10, 2025
61d95a0
Remove demo-plugins (they are being moved to a separate package)
mgorny Apr 10, 2025
4c73d71
Restore blas demo plugin
mgorny Apr 14, 2025
3035504
Use numpy-demo branch of variantlib
mgorny Apr 14, 2025
dc3d905
Support other meson environment variables via get_build_setup
mgorny Apr 16, 2025
d0fb02e
Use variantlib from main branch
mgorny Apr 17, 2025
6809ddb
Update for new metadata API, support Variant-Provider metadata
mgorny Apr 17, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file added demo-plugins/blas_plugin.py
Empty file.
29 changes: 29 additions & 0 deletions demo-plugins/demo_plugins.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
from variantlib.base import PluginBase
from variantlib.config import ProviderConfig
from variantlib.meta import VariantDescription


class BlasPlugin(PluginBase):
namespace = "blas"

def get_supported_configs(self) -> ProviderConfig:
return None

def get_variant_labels(self, variant_desc: VariantDescription) -> list[str]:
for meta in variant_desc:
if meta.namespace == "blas" and meta.key == "variant":
return [meta.value]
return []


class X8664Plugin(PluginBase):
namespace = "x86_64"

def get_supported_configs(self) -> ProviderConfig:
return None

def get_variant_labels(self, variant_desc: VariantDescription) -> list[str]:
for meta in variant_desc:
if meta.namespace == "x86_64" and meta.key == "baseline":
return [f"x86_64_{meta.value}"]
return []
13 changes: 13 additions & 0 deletions demo-plugins/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[build-system]
requires = ["flit_core>=3.2,<4"]
build-backend = "flit_core.buildapi"

[project]
name = "demo-plugins"
version = "0"
description = "Demo plugins for meson-python/numpy variants"
dependencies = ["variantlib"]

[project.entry-points."variantlib.plugins"]
blas = "demo_plugins:BlasPlugin"
x86_64 = "demo_plugins:X8664Plugin"
54 changes: 49 additions & 5 deletions mesonpy/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@
import packaging.version
import pyproject_metadata

from variantlib.meta import VariantMeta, VariantDescription
from variantlib.plugins import PluginLoader

import mesonpy._compat
import mesonpy._rpath
import mesonpy._tags
Expand Down Expand Up @@ -310,11 +313,13 @@ def __init__(
manifest: Dict[str, List[Tuple[pathlib.Path, str]]],
limited_api: bool,
allow_windows_shared_libs: bool,
variant: Optional[VariantDescriptor],
) -> None:
self._metadata = metadata
self._manifest = manifest
self._limited_api = limited_api
self._allow_windows_shared_libs = allow_windows_shared_libs
self._variant = variant

@property
def _has_internal_libs(self) -> bool:
Expand Down Expand Up @@ -351,7 +356,17 @@ def tag(self) -> mesonpy._tags.Tag:
@property
def name(self) -> str:
"""Wheel name, this includes the basename and tag."""
return f'{self._metadata.distribution_name}-{self._metadata.version}-{self.tag}'
name = f'{self._metadata.distribution_name}-{self._metadata.version}-{self.tag}'
if self._variant is not None:
name += f'-{self._variant.hexdigest}'
labels = PluginLoader().get_variant_labels(self._variant)
for numlabels in range(len(labels), 0, -1):
long_name = "+".join((name, *labels[:numlabels]))
# if labels would give us filename that's longer than 128
# characters (124 + .whl), strip them

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this take into account that we're running this code before auditwheel is going to make it quite a bit longer?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, it obviously can't predict what will happen once meson-python is done with it. I guess the only way to handle that is to adjust the limit — and I've put a totally arbitrary number here. I suppose we could even make it configurable via config_settings, so people can adjust when their workflows alter the tags afterwards.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense. We can always tweak the number later. My gut feel is that 128 is on the generous side, but it doesn't matter too much for now.

if len(long_name) < 124:
return long_name
return name

@property
def _distinfo_dir(self) -> str:
Expand Down Expand Up @@ -448,7 +463,20 @@ def _install_path(self, wheel_file: mesonpy._wheelfile.WheelFile, origin: Path,

def _wheel_write_metadata(self, whl: mesonpy._wheelfile.WheelFile) -> None:
# add metadata
whl.writestr(f'{self._distinfo_dir}/METADATA', bytes(self._metadata.as_rfc822()))
metadata = self._metadata.as_rfc822()

# inject variant metadata
if self._variant is not None:
# hack to avoid forking pyproject-metadata
import pyproject_metadata.constants as c
c.KNOWN_METADATA_FIELDS.add('variant')
c.KNOWN_METADATA_FIELDS.add('variant-hash')

metadata['Variant-Hash'] = self._variant.hexdigest
for meta in self._variant:
metadata['Variant'] = meta.to_str()

whl.writestr(f'{self._distinfo_dir}/METADATA', bytes(metadata))
whl.writestr(f'{self._distinfo_dir}/WHEEL', self.wheel)
if self.entrypoints_txt:
whl.writestr(f'{self._distinfo_dir}/entry_points.txt', self.entrypoints_txt)
Expand Down Expand Up @@ -599,6 +627,11 @@ def _bool(value: Any, name: str) -> bool:
def _string_or_strings(value: Any, name: str) -> List[str]:
return list([value,] if isinstance(value, str) else value)

def _variant_names(value: Any, name: str) -> list[VariantMeta]:
if isinstance(value, str):
value = [value]
return [VariantMeta.from_str(x) for x in value]

options = {
'builddir': _string,
'build-dir': _string,
Expand All @@ -607,6 +640,8 @@ def _string_or_strings(value: Any, name: str) -> List[str]:
'setup-args': _string_or_strings,
'compile-args': _string_or_strings,
'install-args': _string_or_strings,
'variant': _variant_names,
'variant-name': _variant_names,
}
assert all(f'{name}-args' in options for name in _MESON_ARGS_KEYS)

Expand Down Expand Up @@ -644,10 +679,12 @@ def __init__(
build_dir: Path,
meson_args: Optional[MesonArgs] = None,
editable_verbose: bool = False,
variant: Optional[VariantDescription] = None,
) -> None:
self._source_dir = pathlib.Path(source_dir).absolute()
self._build_dir = pathlib.Path(build_dir).absolute()
self._editable_verbose = editable_verbose
self._variant = variant
self._meson_native_file = self._build_dir / 'meson-python-native-file.ini'
self._meson_cross_file = self._build_dir / 'meson-python-cross-file.ini'
self._meson_args: MesonArgs = collections.defaultdict(list)
Expand Down Expand Up @@ -1001,13 +1038,13 @@ def sdist(self, directory: Path) -> pathlib.Path:
def wheel(self, directory: Path) -> pathlib.Path:
"""Generates a wheel in the specified directory."""
self.build()
builder = _WheelBuilder(self._metadata, self._manifest, self._limited_api, self._allow_windows_shared_libs)
builder = _WheelBuilder(self._metadata, self._manifest, self._limited_api, self._allow_windows_shared_libs, self._variant)
return builder.build(directory)

def editable(self, directory: Path) -> pathlib.Path:
"""Generates an editable wheel in the specified directory."""
self.build()
builder = _EditableWheelBuilder(self._metadata, self._manifest, self._limited_api, self._allow_windows_shared_libs)
builder = _EditableWheelBuilder(self._metadata, self._manifest, self._limited_api, self._allow_windows_shared_libs, self._variant)
return builder.build(directory, self._source_dir, self._build_dir, self._build_command, self._editable_verbose)


Expand All @@ -1020,11 +1057,18 @@ def _project(config_settings: Optional[Dict[Any, Any]] = None) -> Iterator[Proje
source_dir = os.path.curdir
build_dir = settings.get('build-dir')
editable_verbose = bool(settings.get('editable-verbose'))
variants = settings.get('variant', [])
variant_names = settings.get('variant-name', []) + variants

variant_desc = VariantDescription(variant_names) if variant_names else None
if variants:
meson_args.setdefault('setup', [])
meson_args['setup'].append(f'-Dvariant={[x.to_str() for x in variants]!r}')

with contextlib.ExitStack() as ctx:
if build_dir is None:
build_dir = ctx.enter_context(tempfile.TemporaryDirectory(prefix='.mesonpy-', dir=source_dir))
yield Project(source_dir, build_dir, meson_args, editable_verbose)
yield Project(source_dir, build_dir, meson_args, editable_verbose, variant_desc)


def _parse_version_string(string: str) -> Tuple[int, ...]:
Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ requires = [
'packaging >= 19.0',
'pyproject-metadata >= 0.9.0',
'tomli >= 1.0.0; python_version < "3.11"',
'variantlib @ https://github.com/wheelnext/variantlib/archive/main.tar.gz',
]

[project]
Expand Down Expand Up @@ -39,6 +40,7 @@ dependencies = [
'packaging >= 19.0',
'pyproject-metadata >= 0.9.0',
'tomli >= 1.0.0; python_version < "3.11"',
'variantlib @ https://github.com/wheelnext/variantlib/archive/variant-json.tar.gz',
]

[project.optional-dependencies]
Expand Down