# SPDX-License-Identifier: Apache-2.0 # Copyright 2018 The Meson development team from __future__ import annotations import copy, json, os, shutil, re import typing as T from . import ExtensionModule, ModuleInfo from .. import mesonlib from .. import mlog from ..options import UserFeatureOption from ..build import known_shmod_kwargs, CustomTarget, CustomTargetIndex, BuildTarget, GeneratedList, StructuredSources, ExtractedObjects, SharedModule from ..dependencies import NotFoundDependency from ..dependencies.detect import get_dep_identifier, find_external_dependency from ..dependencies.python import BasicPythonExternalProgram, python_factory, _PythonDependencyBase from ..interpreter import extract_required_kwarg, primitives as P_OBJ from ..interpreter.interpreterobjects import _ExternalProgramHolder from ..interpreter.type_checking import NoneType, DEPENDENCY_KWS, PRESERVE_PATH_KW, SHARED_MOD_KWS from ..interpreterbase import ( noPosargs, noKwargs, permittedKwargs, ContainerTypeInfo, InvalidArguments, typed_pos_args, typed_kwargs, KwargInfo, FeatureNew, disablerIfNotFound, InterpreterObject ) from ..mesonlib import MachineChoice from ..options import OptionKey from ..programs import ExternalProgram, NonExistingExternalProgram if T.TYPE_CHECKING: from typing_extensions import TypedDict, NotRequired from . import ModuleState from ..build import Build, Data from ..dependencies.base import Dependency, DependencyObjectKWs from ..interpreter import Interpreter from ..interpreter.interpreter import BuildTargetSource from ..interpreter.kwargs import ExtractRequired, SharedModule as SharedModuleKw, FuncDependency from ..interpreterbase.baseobjects import TYPE_var, TYPE_kwargs class PyInstallKw(TypedDict): pure: T.Optional[bool] subdir: str install_tag: T.Optional[str] class FindInstallationKw(ExtractRequired): disabler: bool modules: T.List[str] pure: T.Optional[bool] class ExtensionModuleKw(SharedModuleKw): subdir: NotRequired[T.Optional[str]] MaybePythonProg = T.Union[NonExistingExternalProgram, 'PythonExternalProgram'] mod_kwargs = {'subdir', 'limited_api'} mod_kwargs.update(known_shmod_kwargs) mod_kwargs -= {'name_prefix', 'name_suffix'} _MOD_KWARGS = [k for k in SHARED_MOD_KWS if k.name not in {'name_prefix', 'name_suffix'}] class PythonExternalProgram(BasicPythonExternalProgram): # This is a ClassVar instead of an instance bool, because although an # installation is cached, we actually copy it, modify attributes such as pure, # and return a temporary one rather than the cached object. run_bytecompile: T.ClassVar[T.Dict[str, bool]] = {} def sanity(self, state: T.Optional['ModuleState'] = None) -> bool: ret = super().sanity() if ret: self.platlib = self._get_path(state, 'platlib') self.purelib = self._get_path(state, 'purelib') self.run_bytecompile.setdefault(self.info['version'], False) return ret def _get_path(self, state: T.Optional['ModuleState'], key: str) -> str: rel_path = self.info['install_paths'][key][1:] if not state: # This happens only from run_project_tests.py return rel_path value = T.cast('str', state.get_option(f'python.{key}dir')) if value: if state.is_user_defined_option('python.install_env'): raise mesonlib.MesonException(f'python.{key}dir and python.install_env are mutually exclusive') return value install_env = state.get_option('python.install_env') if install_env == 'auto': install_env = 'venv' if self.info['is_venv'] else 'system' if install_env == 'system': rel_path = os.path.join(self.info['variables']['prefix'], rel_path) elif install_env == 'venv': if not self.info['is_venv']: raise mesonlib.MesonException('python.install_env cannot be set to "venv" unless you are in a venv!') # inside a venv, deb_system is *never* active hence info['paths'] may be wrong rel_path = self.info['sysconfig_paths'][key] return rel_path _PURE_KW = KwargInfo('pure', (bool, NoneType)) _SUBDIR_KW = KwargInfo('subdir', str, default='') _LIMITED_API_KW = KwargInfo('limited_api', str, default='', since='1.3.0') _DEFAULTABLE_SUBDIR_KW = KwargInfo('subdir', (str, NoneType)) class PythonInstallation(_ExternalProgramHolder['PythonExternalProgram']): def __init__(self, python: 'PythonExternalProgram', interpreter: 'Interpreter'): _ExternalProgramHolder.__init__(self, python, interpreter) info = python.info prefix = self.interpreter.environment.coredata.optstore.get_value_for(OptionKey('prefix')) assert isinstance(prefix, str), 'for mypy' if python.build_config: self.version = python.build_config['language']['version'] self.platform = python.build_config['platform'] self.suffix = python.build_config['abi']['extension_suffix'] self.limited_api_suffix = python.build_config['abi']['stable_abi_suffix'] self.link_libpython = python.build_config['libpython']['link_extensions'] self.is_pypy = python.build_config['implementation']['name'] == 'pypy' else: self.version = info['version'] self.platform = info['platform'] self.suffix = info['suffix'] self.limited_api_suffix = info['limited_api_suffix'] self.link_libpython = info['link_libpython'] self.is_pypy = info['is_pypy'] self.variables = info['variables'] self.paths = info['paths'] self.pure = python.pure self.platlib_install_path = os.path.join(prefix, python.platlib) self.purelib_install_path = os.path.join(prefix, python.purelib) @permittedKwargs(mod_kwargs) @typed_pos_args('python.extension_module', str, varargs=(str, mesonlib.File, CustomTarget, CustomTargetIndex, GeneratedList, StructuredSources, ExtractedObjects, BuildTarget)) @typed_kwargs('python.extension_module', *_MOD_KWARGS, _DEFAULTABLE_SUBDIR_KW, _LIMITED_API_KW, allow_unknown=True) @InterpreterObject.method('extension_module') def extension_module_method(self, args: T.Tuple[str, T.List[BuildTargetSource]], kwargs: ExtensionModuleKw) -> 'SharedModule': if 'install_dir' in kwargs: if kwargs['subdir'] is not None: raise InvalidArguments('"subdir" and "install_dir" are mutually exclusive') else: # We want to remove 'subdir', but it may be None and we want to replace it with '' # It must be done this way since we don't allow both `install_dir` # and `subdir` to be set at the same time subdir = kwargs.pop('subdir') or '' kwargs['install_dir'] = self._get_install_dir_impl(False, subdir) target_suffix = self.suffix new_deps = mesonlib.extract_as_list(kwargs, 'dependencies') pydep = next((dep for dep in new_deps if isinstance(dep, _PythonDependencyBase)), None) if pydep is None: pydep = self._dependency_method_impl({}) if not pydep.found(): raise mesonlib.MesonException('Python dependency not found') new_deps.append(pydep) FeatureNew.single_use('python_installation.extension_module with implicit dependency on python', '0.63.0', self.subproject, 'use python_installation.dependency()', self.current_node) limited_api_version = kwargs.pop('limited_api') allow_limited_api = self.interpreter.environment.coredata.optstore.get_value_for(OptionKey('python.allow_limited_api')) if limited_api_version != '' and allow_limited_api: target_suffix = self.limited_api_suffix limited_api_version_hex = self._convert_api_version_to_py_version_hex(limited_api_version, pydep.version) limited_api_definition = f'-DPy_LIMITED_API={limited_api_version_hex}' new_c_args = mesonlib.extract_as_list(kwargs, 'c_args') new_c_args.append(limited_api_definition) kwargs['c_args'] = new_c_args new_cpp_args = mesonlib.extract_as_list(kwargs, 'cpp_args') new_cpp_args.append(limited_api_definition) kwargs['cpp_args'] = new_cpp_args # On Windows, the limited API DLL is python3.dll, not python3X.dll. for_machine = kwargs['native'] if self.interpreter.environment.machines[for_machine].is_windows(): pydep_copy = copy.copy(pydep) pydep_copy.find_libpy_windows(self.env, limited_api=True) if not pydep_copy.found(): raise mesonlib.MesonException('Python dependency supporting limited API not found') new_deps.remove(pydep) new_deps.append(pydep_copy) # When compiled under MSVC, Python's PC/pyconfig.h forcibly inserts pythonMAJOR.MINOR.lib # into the linker path when not running in debug mode via a series #pragma comment(lib, "") # directives. We manually override these here as this interferes with the intended # use of the 'limited_api' kwarg compilers = self.interpreter.environment.coredata.compilers[for_machine] if any(compiler.get_id() == 'msvc' for compiler in compilers.values()): pyver = pydep.version.replace('.', '') python_windows_debug_link_exception = f'/NODEFAULTLIB:python{pyver}_d.lib' python_windows_release_link_exception = f'/NODEFAULTLIB:python{pyver}.lib' new_link_args = mesonlib.extract_as_list(kwargs, 'link_args') is_debug = self.interpreter.environment.coredata.optstore.get_value_for('debug') if is_debug: new_link_args.append(python_windows_debug_link_exception) else: new_link_args.append(python_windows_release_link_exception) kwargs['link_args'] = new_link_args kwargs['dependencies'] = new_deps # msys2's python3 has "-cpython-36m.dll", we have to be clever # FIXME: explain what the specific cleverness is here split, target_suffix = target_suffix.rsplit('.', 1) args = (args[0] + split, args[1]) kwargs['name_prefix'] = '' kwargs['name_suffix'] = target_suffix if kwargs['gnu_symbol_visibility'] == '' and \ (self.is_pypy or mesonlib.version_compare(self.version, '>=3.9')): kwargs['gnu_symbol_visibility'] = 'inlineshidden' return self.interpreter.build_target(self.current_node, args, kwargs, SharedModule) def _convert_api_version_to_py_version_hex(self, api_version: str, detected_version: str) -> str: python_api_version_format = re.compile(r'[0-9]\.[0-9]{1,2}') decimal_match = python_api_version_format.fullmatch(api_version) if not decimal_match: raise InvalidArguments(f'Python API version invalid: "{api_version}".') if mesonlib.version_compare(api_version, '<3.2'): raise InvalidArguments(f'Python Limited API version invalid: {api_version} (must be greater than 3.2)') if mesonlib.version_compare(api_version, '>' + detected_version): raise InvalidArguments(f'Python Limited API version too high: {api_version} (detected {detected_version})') version_components = api_version.split('.') major = int(version_components[0]) minor = int(version_components[1]) return '0x{:02x}{:02x}0000'.format(major, minor) def _dependency_method_impl(self, kwargs: DependencyObjectKWs) -> Dependency: for_machine = kwargs.get('native', MachineChoice.HOST) identifier = get_dep_identifier(self._full_path(), kwargs) dep = self.interpreter.coredata.deps[for_machine].get(identifier) if dep is not None: return dep build_config = self.interpreter.environment.coredata.optstore.get_value_for(OptionKey('python.build_config')) new_kwargs = kwargs.copy() new_kwargs['required'] = False if build_config: new_kwargs['build_config'] = build_config candidates = python_factory(self.interpreter.environment, for_machine, new_kwargs, self.held_object) dep = find_external_dependency('python', self.interpreter.environment, new_kwargs, candidates) self.interpreter.coredata.deps[for_machine].put(identifier, dep) return dep @noPosargs @typed_kwargs( 'python_installation.dependency', *DEPENDENCY_KWS, KwargInfo('embed', bool, default=False, since='0.53.0'), ) @disablerIfNotFound @InterpreterObject.method('dependency') def dependency_method(self, args: T.List['TYPE_var'], kwargs: FuncDependency) -> 'Dependency': disabled, required, feature = extract_required_kwarg(kwargs, self.subproject) nkwargs = T.cast('DependencyObjectKWs', kwargs.copy()) nkwargs['required'] = required if disabled: mlog.log('Dependency', mlog.bold('python'), 'skipped: feature', mlog.bold(feature), 'disabled') return NotFoundDependency('python', self.interpreter.environment) else: dep = self._dependency_method_impl(nkwargs) if required and not dep.found(): raise mesonlib.MesonException('Python dependency not found') return dep @typed_pos_args('install_data', varargs=(str, mesonlib.File)) @typed_kwargs( 'python_installation.install_sources', _PURE_KW, _SUBDIR_KW, PRESERVE_PATH_KW, KwargInfo('install_tag', (str, NoneType), since='0.60.0') ) @InterpreterObject.method('install_sources') def install_sources_method(self, args: T.Tuple[T.List[T.Union[str, mesonlib.File]]], kwargs: 'PyInstallKw') -> 'Data': self.held_object.run_bytecompile[self.version] = True tag = kwargs['install_tag'] or 'python-runtime' pure = kwargs['pure'] if kwargs['pure'] is not None else self.pure install_dir = self._get_install_dir_impl(pure, kwargs['subdir']) return self.interpreter.install_data_impl( self.interpreter.source_strings_to_files(args[0]), install_dir, mesonlib.FileMode(), rename=None, tag=tag, install_data_type='python', preserve_path=kwargs['preserve_path']) @noPosargs @typed_kwargs('python_installation.install_dir', _PURE_KW, _SUBDIR_KW) @InterpreterObject.method('get_install_dir') def get_install_dir_method(self, args: T.List['TYPE_var'], kwargs: 'PyInstallKw') -> str: self.held_object.run_bytecompile[self.version] = True pure = kwargs['pure'] if kwargs['pure'] is not None else self.pure return self._get_install_dir_impl(pure, kwargs['subdir']) def _get_install_dir_impl(self, pure: bool, subdir: str) -> P_OBJ.OptionString: if pure: base = self.purelib_install_path name = '{py_purelib}' else: base = self.platlib_install_path name = '{py_platlib}' return P_OBJ.OptionString(os.path.join(base, subdir), os.path.join(name, subdir)) @noPosargs @noKwargs @InterpreterObject.method('language_version') def language_version_method(self, args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') -> str: return self.version @typed_pos_args('python_installation.has_path', str) @noKwargs @InterpreterObject.method('has_path') def has_path_method(self, args: T.Tuple[str], kwargs: 'TYPE_kwargs') -> bool: return args[0] in self.paths @typed_pos_args('python_installation.get_path', str, optargs=[object]) @noKwargs @InterpreterObject.method('get_path') def get_path_method(self, args: T.Tuple[str, T.Optional['TYPE_var']], kwargs: 'TYPE_kwargs') -> 'TYPE_var': path_name, fallback = args try: return self.paths[path_name] except KeyError: if fallback is not None: return fallback raise InvalidArguments(f'{path_name} is not a valid path name') @typed_pos_args('python_installation.has_variable', str) @noKwargs @InterpreterObject.method('has_variable') def has_variable_method(self, args: T.Tuple[str], kwargs: 'TYPE_kwargs') -> bool: return args[0] in self.variables @typed_pos_args('python_installation.get_variable', str, optargs=[object]) @noKwargs @InterpreterObject.method('get_variable') def get_variable_method(self, args: T.Tuple[str, T.Optional['TYPE_var']], kwargs: 'TYPE_kwargs') -> 'TYPE_var': var_name, fallback = args try: return self.variables[var_name] except KeyError: if fallback is not None: return fallback raise InvalidArguments(f'{var_name} is not a valid variable name') @noPosargs @noKwargs @FeatureNew('Python module path method', '0.50.0') @InterpreterObject.method('path') def path_method(self, args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') -> str: return super().path_method(args, kwargs) class PythonModule(ExtensionModule): INFO = ModuleInfo('python', '0.46.0') def __init__(self, interpreter: 'Interpreter') -> None: super().__init__(interpreter) self.installations: T.Dict[str, MaybePythonProg] = {} self.methods.update({ 'find_installation': self.find_installation, }) def _get_install_scripts(self) -> T.List[mesonlib.ExecutableSerialisation]: backend = self.interpreter.backend ret = [] optlevel = self.interpreter.environment.coredata.optstore.get_value_for(OptionKey('python.bytecompile')) if optlevel == -1: return ret if not any(PythonExternalProgram.run_bytecompile.values()): return ret installdata = backend.create_install_data() py_files = [] def should_append(f, isdir: bool = False): # This uses the install_plan decorated names to see if the original source was propagated via # install_sources() or get_install_dir(). return f.startswith(('{py_platlib}', '{py_purelib}')) and (f.endswith('.py') or isdir) for t in installdata.targets: if should_append(t.out_name): py_files.append((t.out_name, os.path.join(installdata.prefix, t.outdir, os.path.basename(t.fname)))) for d in installdata.data: if should_append(d.install_path_name): py_files.append((d.install_path_name, os.path.join(installdata.prefix, d.install_path))) for d in installdata.install_subdirs: if should_append(d.install_path_name, True): py_files.append((d.install_path_name, os.path.join(installdata.prefix, d.install_path))) import importlib.resources pycompile = os.path.join(self.interpreter.environment.get_scratch_dir(), 'pycompile.py') with open(pycompile, 'wb') as f: f.write(importlib.resources.read_binary('mesonbuild.scripts', 'pycompile.py')) for i in self.installations.values(): if isinstance(i, PythonExternalProgram) and i.run_bytecompile[i.info['version']]: i = T.cast('PythonExternalProgram', i) manifest = f'python-{i.info["version"]}-installed.json' manifest_json = [] for name, f in py_files: if f.startswith((os.path.join(installdata.prefix, i.platlib), os.path.join(installdata.prefix, i.purelib))): manifest_json.append(name) with open(os.path.join(self.interpreter.environment.get_scratch_dir(), manifest), 'w', encoding='utf-8') as f: json.dump(manifest_json, f) cmd = i.command + [pycompile, manifest, str(optlevel)] script = backend.get_executable_serialisation(cmd, verbose=True, tag='python-runtime', installdir_map={'py_purelib': i.purelib, 'py_platlib': i.platlib}) ret.append(script) return ret def postconf_hook(self, b: Build) -> None: b.install_scripts.extend(self._get_install_scripts()) # https://www.python.org/dev/peps/pep-0397/ @staticmethod def _get_win_pythonpath(name_or_path: str) -> T.Optional[str]: if not name_or_path.startswith(('python2', 'python3')): return None if not shutil.which('py'): # program not installed, return without an exception return None ver = f'-{name_or_path[6:]}' cmd = ['py', ver, '-c', "import sysconfig; print(sysconfig.get_config_var('BINDIR'))"] _, stdout, _ = mesonlib.Popen_safe(cmd) directory = stdout.strip() if os.path.exists(directory): return os.path.join(directory, 'python') else: return None def _find_installation_impl(self, state: 'ModuleState', display_name: str, name_or_path: str, required: bool) -> MaybePythonProg: build_config = self.interpreter.environment.coredata.optstore.get_value_for(OptionKey('python.build_config')) if not name_or_path: python = PythonExternalProgram('python3', mesonlib.python_command, build_config_path=build_config) else: tmp_python = ExternalProgram.from_entry(display_name, name_or_path) python = PythonExternalProgram(display_name, ext_prog=tmp_python, build_config_path=build_config) if not python.found() and mesonlib.is_windows(): pythonpath = self._get_win_pythonpath(name_or_path) if pythonpath is not None: name_or_path = pythonpath python = PythonExternalProgram(name_or_path) # Last ditch effort, python2 or python3 can be named python # on various platforms, let's not give up just yet, if an executable # named python is available and has a compatible version, let's use # it if not python.found() and name_or_path in {'python2', 'python3'}: tmp_python = ExternalProgram.from_entry(display_name, 'python') python = PythonExternalProgram(name_or_path, ext_prog=tmp_python, build_config_path=build_config) if python.found(): if python.sanity(state): return python else: sanitymsg = f'{python} is not a valid python or it is missing distutils' if required: raise mesonlib.MesonException(sanitymsg) else: mlog.warning(sanitymsg, location=state.current_node) return NonExistingExternalProgram(python.name) @disablerIfNotFound @typed_pos_args('python.find_installation', optargs=[str]) @typed_kwargs( 'python.find_installation', KwargInfo('required', (bool, UserFeatureOption), default=True), KwargInfo('disabler', bool, default=False, since='0.49.0'), KwargInfo('modules', ContainerTypeInfo(list, str), listify=True, default=[], since='0.51.0'), _PURE_KW.evolve(default=True, since='0.64.0'), ) def find_installation(self, state: 'ModuleState', args: T.Tuple[T.Optional[str]], kwargs: 'FindInstallationKw') -> MaybePythonProg: feature_check = FeatureNew('Passing "feature" option to find_installation', '0.48.0') disabled, required, feature = extract_required_kwarg(kwargs, state.subproject, feature_check) # FIXME: this code is *full* of sharp corners. It assumes that it's # going to get a string value (or now a list of length 1), of `python2` # or `python3` which is completely nonsense. On windows the value could # easily be `['py', '-3']`, or `['py', '-3.7']` to get a very specific # version of python. On Linux we might want a python that's not in # $PATH, or that uses a wrapper of some kind. np: T.List[str] = state.environment.lookup_binary_entry(MachineChoice.HOST, 'python') or [] fallback = args[0] display_name = fallback or 'python' if not np and fallback is not None: np = [fallback] name_or_path = np[0] if np else None if disabled: mlog.log('Program', name_or_path or 'python', 'found:', mlog.red('NO'), '(disabled by:', mlog.bold(feature), ')') return NonExistingExternalProgram() python = self.installations.get(name_or_path) if not python: python = self._find_installation_impl(state, display_name, name_or_path, required) self.installations[name_or_path] = python want_modules = kwargs['modules'] found_modules: T.List[str] = [] missing_modules: T.List[str] = [] if python.found() and want_modules: for mod in want_modules: p, *_ = mesonlib.Popen_safe( python.command + ['-c', f'import {mod}']) if p.returncode != 0: missing_modules.append(mod) else: found_modules.append(mod) msg: T.List['mlog.TV_Loggable'] = ['Program', python.name] if want_modules: msg.append('({})'.format(', '.join(want_modules))) msg.append('found:') if python.found() and not missing_modules: msg.extend([mlog.green('YES'), '({})'.format(' '.join(python.command))]) else: msg.append(mlog.red('NO')) if found_modules: msg.append('modules:') msg.append(', '.join(found_modules)) mlog.log(*msg) if not python.found(): if required: raise mesonlib.MesonException('{} not found'.format(name_or_path or 'python')) return NonExistingExternalProgram(python.name) elif missing_modules: if required: raise mesonlib.MesonException('{} is missing modules: {}'.format(name_or_path or 'python', ', '.join(missing_modules))) return NonExistingExternalProgram(python.name) else: assert isinstance(python, PythonExternalProgram), 'for mypy' python = copy.copy(python) python.pure = kwargs['pure'] return python raise mesonlib.MesonBugException('Unreachable code was reached (PythonModule.find_installation).') def initialize(interpreter: 'Interpreter') -> PythonModule: mod = PythonModule(interpreter) mod.interpreter.append_holder_map(PythonExternalProgram, PythonInstallation) return mod