# SPDX-License-Identifier: Apache-2. # Copyright 2022 The Meson development team from __future__ import annotations import functools, json, operator, os, textwrap from pathlib import Path import typing as T from .. import mesonlib, mlog from .base import process_method_kw, DependencyException, DependencyMethods, ExternalDependency, SystemDependency from .configtool import ConfigToolDependency from .detect import packages from .factory import DependencyFactory from .framework import ExtraFrameworkDependency from .pkgconfig import PkgConfigDependency from ..envconfig import detect_cpu_family from ..programs import ExternalProgram from ..options import OptionKey if T.TYPE_CHECKING: from typing_extensions import Final, TypedDict from .factory import DependencyGenerator from ..environment import Environment from ..mesonlib import MachineChoice from .base import DependencyObjectKWs class PythonIntrospectionDict(TypedDict): install_paths: T.Dict[str, str] is_pypy: bool is_venv: bool is_freethreaded: bool link_libpython: bool sysconfig_paths: T.Dict[str, str] paths: T.Dict[str, str] platform: str suffix: str limited_api_suffix: str variables: T.Dict[str, str] version: str _Base = ExternalDependency else: _Base = object class Pybind11ConfigToolDependency(ConfigToolDependency): tools = ['pybind11-config'] # any version of the tool is valid, since this is header-only allow_default_for_cross = True # pybind11 in 2.10.4 added --version, sanity-check another flag unique to it # in the meantime skip_version = '--pkgconfigdir' def __init__(self, name: str, environment: Environment, kwargs: DependencyObjectKWs): super().__init__(name, environment, kwargs) if not self.is_found: return self.compile_args = self.get_config_value(['--includes'], 'compile_args') class NumPyConfigToolDependency(ConfigToolDependency): tools = ['numpy-config'] def __init__(self, name: str, environment: Environment, kwargs: DependencyObjectKWs): super().__init__(name, environment, kwargs) if not self.is_found: return self.compile_args = self.get_config_value(['--cflags'], 'compile_args') class PythonBuildConfig: """PEP 739 build-details.json config file.""" IMPLEMENTED_VERSION: Final[str] = '1.0' """Schema version currently implemented.""" _PATH_KEYS = ( 'base_interpreter', 'libpython.dynamic', 'libpython.dynamic_stableabi', 'libpython.static', 'c_api.headers', 'c_api.pkgconfig_path', ) """Path keys — may be relative, need to be expanded.""" def __init__(self, path: str) -> None: self._path = Path(path) try: self._data = json.loads(self._path.read_text(encoding='utf8')) except OSError as e: raise DependencyException(f'Failed to read python.build_config: {e}') from e self._validate_data() self._expand_paths() def __getitem__(self, key: str) -> T.Any: return functools.reduce(operator.getitem, key.split('.'), self._data) def __contains__(self, key: str) -> bool: try: self[key] except KeyError: return False else: return True def get(self, key: str, default: T.Any = None) -> T.Any: try: return self[key] except KeyError: return default def _validate_data(self) -> None: schema_version = self._data['schema_version'] if mesonlib.version_compare(schema_version, '< 1.0'): raise DependencyException(f'Invalid schema_version in python.build_config: {schema_version}') if mesonlib.version_compare(schema_version, '>= 2.0'): raise DependencyException( f'Unsupported schema_version {schema_version!r} in python.build_config, ' f'but we only implement support for {self.IMPLEMENTED_VERSION!r}' ) # Schema version that we currently understand if mesonlib.version_compare(schema_version, f'> {self.IMPLEMENTED_VERSION}'): mlog.log( f'python.build_config has schema_version {schema_version!r}, ' f'but we only implement support for {self.IMPLEMENTED_VERSION!r}, ' 'new functionality might be missing' ) def _expand_paths(self) -> None: """Expand relative path (they're relative to base_prefix).""" for key in self._PATH_KEYS: if key not in self: continue parent, _, child = key.rpartition('.') container = self[parent] if parent else self._data path = Path(container[child]) if not path.is_absolute(): container[child] = os.fspath(self.base_prefix / path) @property def config_path(self) -> Path: return self._path @mesonlib.lazy_property def base_prefix(self) -> Path: path = Path(self._data['base_prefix']) if path.is_absolute(): return path # Non-absolute paths are relative to the build config directory return self.config_path.parent / path class BasicPythonExternalProgram(ExternalProgram): def __init__(self, name: str, command: T.Optional[T.List[str]] = None, ext_prog: T.Optional[ExternalProgram] = None, build_config_path: T.Optional[str] = None): if ext_prog is None: super().__init__(name, command=command, silent=True) else: self.name = name self.command = ext_prog.command self.path = ext_prog.path self.cached_version = None self.version_arg = '--version' self.build_config = PythonBuildConfig(build_config_path) if build_config_path else None # We want strong key values, so we always populate this with bogus data. # Otherwise to make the type checkers happy we'd have to do .get() for # everycall, even though we know that the introspection data will be # complete self.info: 'PythonIntrospectionDict' = { 'install_paths': {}, 'is_pypy': False, 'is_venv': False, 'is_freethreaded': False, 'link_libpython': False, 'sysconfig_paths': {}, 'paths': {}, 'platform': 'sentinel', 'suffix': 'sentinel', 'limited_api_suffix': 'sentinel', 'variables': {}, 'version': '0.0', } self.pure: bool = True @property def version(self) -> str: if self.build_config: value = self.build_config['language']['version'] else: value = self.info['variables'].get('LDVERSION') or self.info['version'] assert isinstance(value, str) return value def _check_version(self, version: str) -> bool: if self.name == 'python2': return mesonlib.version_compare(version, '< 3.0') elif self.name == 'python3': return mesonlib.version_compare(version, '>= 3.0') return True def sanity(self) -> bool: # Sanity check, we expect to have something that at least quacks in tune if self.build_config: if not self.build_config['libpython']: mlog.debug('This Python installation does not provide a libpython') return False if not self.build_config['c_api']: mlog.debug('This Python installation does support the C API') return False import importlib.resources with importlib.resources.path('mesonbuild.scripts', 'python_info.py') as f: cmd = self.get_command() + [str(f)] env = os.environ.copy() env['SETUPTOOLS_USE_DISTUTILS'] = 'stdlib' p, stdout, stderr = mesonlib.Popen_safe(cmd, env=env) try: info = json.loads(stdout) except json.JSONDecodeError: info = None mlog.debug('Could not introspect Python (%s): exit code %d' % (str(p.args), p.returncode)) mlog.debug('Program stdout:\n') mlog.debug(stdout) mlog.debug('Program stderr:\n') mlog.debug(stderr) if info is not None and self._check_version(info['version']): self.info = T.cast('PythonIntrospectionDict', info) return True else: return False class _PythonDependencyBase(_Base): def __init__(self, python_holder: 'BasicPythonExternalProgram', embed: bool, for_machine: 'MachineChoice'): self.for_machine = for_machine self.embed = embed self.build_config = python_holder.build_config if self.build_config: self.version = self.build_config['language']['version'] self.platform = self.build_config['platform'] self.is_freethreaded = 't' in self.build_config['abi']['flags'] self.link_libpython = self.build_config['libpython']['link_extensions'] # TODO: figure out how to deal with frameworks # see the logic at the bottom of PythonPkgConfigDependency.__init__() if self.env.machines.host.is_darwin(): raise DependencyException('--python.build-config is not supported on Darwin') else: self.version = python_holder.info['version'] self.platform = python_holder.info['platform'] self.is_freethreaded = python_holder.info['is_freethreaded'] self.link_libpython = python_holder.info['link_libpython'] # This data shouldn't be needed when build_config is set self.is_pypy = python_holder.info['is_pypy'] self.variables = python_holder.info['variables'] self.paths = python_holder.info['paths'] # The "-embed" version of python.pc / python-config was introduced in 3.8, # and distutils extension linking was changed to be considered a non embed # usage. Before then, this dependency always uses the embed=True handling # because that is the only one that exists. # # On macOS and some Linux distros (Debian) distutils doesn't link extensions # against libpython, even on 3.7 and below. We call into distutils and # mirror its behavior. See https://github.com/mesonbuild/meson/issues/4117 if not self.link_libpython: self.link_libpython = embed self.info: T.Optional[T.Dict[str, str]] = None if mesonlib.version_compare(self.version, '>= 3.0'): self.major_version = 3 else: self.major_version = 2 # pyconfig.h is shared between regular and free-threaded builds in the # Windows installer from python.org, and hence does not define # Py_GIL_DISABLED correctly. So do it here: if mesonlib.is_windows() and self.is_freethreaded: self.compile_args += ['-DPy_GIL_DISABLED'] def find_libpy(self, environment: 'Environment') -> None: if self.build_config: path = self.build_config['libpython'].get('dynamic') if not path: raise DependencyException('Python does not provide a dynamic libpython library') sysroot = environment.properties[self.for_machine].get_sys_root() or '' path = sysroot + path if not os.path.isfile(path): raise DependencyException('Python dynamic library does not exist or is not a file') self.link_args = [path] self.is_found = True return if self.is_pypy: if self.major_version == 3: libname = 'pypy3-c' else: libname = 'pypy-c' libdir = os.path.join(self.variables.get('base'), 'bin') libdirs = [libdir] else: libname = f'python{self.version}' if 'DEBUG_EXT' in self.variables: libname += self.variables['DEBUG_EXT'] if 'ABIFLAGS' in self.variables: libname += self.variables['ABIFLAGS'] libdirs = [] largs = self.clib_compiler.find_library(libname, libdirs) if largs is not None: self.link_args = largs self.is_found = True def get_windows_python_arch(self) -> str: if self.platform.startswith('mingw'): if 'x86_64' in self.platform: return 'x86_64' elif 'i686' in self.platform: return 'x86' elif 'aarch64' in self.platform: return 'aarch64' else: raise DependencyException(f'MinGW Python built with unknown platform {self.platform!r}, please file a bug') elif self.platform == 'win32': return 'x86' elif self.platform in {'win64', 'win-amd64'}: return 'x86_64' elif self.platform in {'win-arm64'}: return 'aarch64' raise DependencyException('Unknown Windows Python platform {self.platform!r}') def get_windows_link_args(self, limited_api: bool, environment: 'Environment') -> T.Optional[T.List[str]]: if self.build_config: if self.static: key = 'static' elif limited_api: key = 'dynamic-stableabi' else: key = 'dynamic' sysroot = environment.properties[self.for_machine].get_sys_root() or '' return [sysroot + self.build_config['libpython'][key]] if self.platform.startswith('win'): vernum = self.variables.get('py_version_nodot') verdot = self.variables.get('py_version_short') imp_lower = self.variables.get('implementation_lower', 'python') if self.static: libpath = Path('libs') / f'libpython{vernum}.a' else: if limited_api: vernum = vernum[0] comp = self.get_compiler() if comp.id == "gcc": if imp_lower == 'pypy' and verdot == '3.8': # The naming changed between 3.8 and 3.9 libpath = Path('libpypy3-c.dll') elif imp_lower == 'pypy': libpath = Path(f'libpypy{verdot}-c.dll') else: if self.is_freethreaded: libpath = Path(f'python{vernum}t.dll') else: libpath = Path(f'python{vernum}.dll') else: if self.is_freethreaded: libpath = Path('libs') / f'python{vernum}t.lib' else: libpath = Path('libs') / f'python{vernum}.lib' # For a debug build, pyconfig.h may force linking with # pythonX_d.lib (see meson#10776). This cannot be avoided # and won't work unless we also have a debug build of # Python itself (except with pybind11, which has an ugly # hack to work around this) - so emit a warning to explain # the cause of the expected link error. buildtype = self.env.coredata.optstore.get_value_for(OptionKey('buildtype')) assert isinstance(buildtype, str) debug = self.env.coredata.optstore.get_value_for(OptionKey('debug')) # `debugoptimized` buildtype may not set debug=True currently, see gh-11645 is_debug_build = debug or buildtype == 'debug' vscrt_debug = False if OptionKey('b_vscrt') in self.env.coredata.optstore: vscrt = self.env.coredata.optstore.get_value_for('b_vscrt') if vscrt in {'mdd', 'mtd', 'from_buildtype', 'static_from_buildtype'}: vscrt_debug = True if is_debug_build and vscrt_debug and not self.variables.get('Py_DEBUG'): mlog.warning(textwrap.dedent('''\ Using a debug build type with MSVC or an MSVC-compatible compiler when the Python interpreter is not also a debug build will almost certainly result in a failed build. Prefer using a release build type or a debug Python interpreter. ''')) # base_prefix to allow for virtualenvs. lib = Path(self.variables.get('base_prefix')) / libpath elif self.platform.startswith('mingw'): if self.static: if limited_api: libname = self.variables.get('ABI3DLLLIBRARY') else: libname = self.variables.get('LIBRARY') else: if limited_api: libname = self.variables.get('ABI3LDLIBRARY') else: libname = self.variables.get('LDLIBRARY') lib = Path(self.variables.get('LIBDIR')) / libname else: raise mesonlib.MesonBugException( 'On a Windows path, but the OS doesn\'t appear to be Windows or MinGW.') if not lib.exists(): mlog.log('Could not find Python3 library {!r}'.format(str(lib))) return None return [str(lib)] def find_libpy_windows(self, env: 'Environment', limited_api: bool = False) -> None: ''' Find python3 libraries on Windows and also verify that the arch matches what we are building for. ''' try: pyarch = self.get_windows_python_arch() except DependencyException as e: mlog.log(str(e)) self.is_found = False return arch = detect_cpu_family(env.coredata.compilers.host) if arch != pyarch: mlog.log('Need', mlog.bold(self.name), f'for {arch}, but found {pyarch}') self.is_found = False return # This can fail if the library is not found largs = self.get_windows_link_args(limited_api, env) if largs is None: self.is_found = False return self.link_args = largs self.is_found = True class PythonPkgConfigDependency(PkgConfigDependency, _PythonDependencyBase): def __init__(self, environment: 'Environment', kwargs: DependencyObjectKWs, installation: 'BasicPythonExternalProgram', embed: bool, for_machine: 'MachineChoice'): pkg_embed = '-embed' if embed and mesonlib.version_compare(installation.info['version'], '>=3.8') else '' pkg_name = f'python-{installation.version}{pkg_embed}' if installation.build_config: pkg_libdir = installation.build_config.get('c_api.pkgconfig_path') pkg_libdir_origin = 'c_api.pkgconfig_path from the Python build config' else: pkg_libdir = installation.info['variables'].get('LIBPC') pkg_libdir_origin = 'LIBPC' if pkg_libdir is None: # we do not fall back to system directories, since this could lead # to using pkg-config of another Python installation, for example # we could end up using CPython .pc file for PyPy mlog.debug(f'Skipping pkgconfig lookup, {pkg_libdir_origin} is unset') self.is_found = False return sysroot = environment.properties[for_machine].get_sys_root() or '' pkg_libdir = sysroot + pkg_libdir mlog.debug(f'Searching for {pkg_libdir!r} via pkgconfig lookup in {pkg_libdir_origin}') pkgconfig_paths = [pkg_libdir] if pkg_libdir else [] PkgConfigDependency.__init__(self, pkg_name, environment, kwargs, extra_paths=pkgconfig_paths) _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False), for_machine) if pkg_libdir and not self.is_found: mlog.debug(f'{pkg_name!r} could not be found in {pkg_libdir_origin}, ' 'this is likely due to a relocated python installation') return # pkg-config files are usually accurate starting with python 3.8 if not self.link_libpython and mesonlib.version_compare(self.version, '< 3.8'): self.link_args = [] # But not Apple, because it's a framework if self.env.machines.host.is_darwin() and 'PYTHONFRAMEWORKPREFIX' in self.variables: framework_prefix = self.variables['PYTHONFRAMEWORKPREFIX'] # Add rpath, will be de-duplicated if necessary if framework_prefix.startswith('/Applications/Xcode.app/'): self.link_args += ['-Wl,-rpath,' + framework_prefix] if self.raw_link_args is not None: # When None, self.link_args is used self.raw_link_args += ['-Wl,-rpath,' + framework_prefix] class PythonFrameworkDependency(ExtraFrameworkDependency, _PythonDependencyBase): def __init__(self, name: str, environment: 'Environment', kwargs: DependencyObjectKWs, installation: 'BasicPythonExternalProgram', for_machine: 'MachineChoice'): ExtraFrameworkDependency.__init__(self, name, environment, kwargs) _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False), for_machine) class PythonSystemDependency(SystemDependency, _PythonDependencyBase): def __init__(self, name: str, environment: 'Environment', kwargs: DependencyObjectKWs, installation: 'BasicPythonExternalProgram', for_machine: 'MachineChoice'): SystemDependency.__init__(self, name, environment, kwargs) _PythonDependencyBase.__init__(self, installation, kwargs.get('embed', False), for_machine) # For most platforms, match pkg-config behavior. iOS is a special case; # check for that first, so that check takes priority over # `link_libpython` (which *shouldn't* be set, but just in case) if self.platform.startswith('ios-'): # iOS doesn't use link_libpython - it links with the *framework*. self.link_args = ['-framework', 'Python', '-F', self.variables.get('base_prefix')] self.is_found = True elif self.link_libpython: # link args if mesonlib.is_windows(): self.find_libpy_windows(environment, limited_api=False) else: self.find_libpy(environment) else: self.is_found = True # compile args if self.build_config: sysroot = environment.properties[for_machine].get_sys_root() or '' inc_paths = mesonlib.OrderedSet([sysroot + self.build_config['c_api']['headers']]) else: inc_paths = mesonlib.OrderedSet([ self.variables.get('INCLUDEPY'), self.paths.get('include'), self.paths.get('platinclude')]) self.compile_args += ['-I' + path for path in inc_paths if path] # https://sourceforge.net/p/mingw-w64/mailman/message/30504611/ # https://github.com/python/cpython/pull/100137 if mesonlib.is_windows() and self.get_windows_python_arch().endswith('64') and mesonlib.version_compare(self.version, '<3.12'): self.compile_args += ['-DMS_WIN64='] if not self.clib_compiler.has_header('Python.h', '', extra_args=self.compile_args)[0]: self.is_found = False @staticmethod def log_tried() -> str: return 'sysconfig' def python_factory(env: 'Environment', for_machine: 'MachineChoice', kwargs: DependencyObjectKWs, installation: T.Optional['BasicPythonExternalProgram'] = None) -> T.List['DependencyGenerator']: # We can't use the factory_methods decorator here, as we need to pass the # extra installation argument methods = process_method_kw({DependencyMethods.PKGCONFIG, DependencyMethods.SYSTEM}, kwargs) embed = kwargs.get('embed', False) candidates: T.List['DependencyGenerator'] = [] from_installation = installation is not None # When not invoked through the python module, default installation. if installation is None: installation = BasicPythonExternalProgram('python3', mesonlib.python_command) installation.sanity() if DependencyMethods.PKGCONFIG in methods: if from_installation: candidates.append(functools.partial(PythonPkgConfigDependency, env, kwargs, installation, embed, for_machine)) else: candidates.append(functools.partial(PkgConfigDependency, 'python3', env, kwargs)) if DependencyMethods.SYSTEM in methods: candidates.append(functools.partial(PythonSystemDependency, 'python', env, kwargs, installation, for_machine)) if DependencyMethods.EXTRAFRAMEWORK in methods: nkwargs = kwargs.copy() if mesonlib.version_compare(installation.version, '>= 3'): # There is a python in /System/Library/Frameworks, but that's python 2.x, # Python 3 will always be in /Library nkwargs['paths'] = ['/Library/Frameworks'] candidates.append(functools.partial(PythonFrameworkDependency, 'Python', env, nkwargs, installation, for_machine)) return candidates packages['python3'] = python_factory packages['pybind11'] = pybind11_factory = DependencyFactory( 'pybind11', [DependencyMethods.PKGCONFIG, DependencyMethods.CONFIG_TOOL, DependencyMethods.CMAKE], configtool_class=Pybind11ConfigToolDependency, ) packages['numpy'] = numpy_factory = DependencyFactory( 'numpy', [DependencyMethods.PKGCONFIG, DependencyMethods.CONFIG_TOOL], configtool_class=NumPyConfigToolDependency, )