diff --git a/README.md b/README.md index e997c68..ea95c83 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,72 @@ cmake_env_args = {} # env-specific cmake args to pass include_flags = {} # include flags to pass -D ``` +### Linking an extension against a library + +A `generic` library can ship next to the extension that links it, in the layout pyarrow uses. List the library first and name it in the extension's `depends`: + +```toml +[[tool.hatch.build.hooks.hatch-cpp.libraries]] +name = "project/lib/libdemo" +sources = ["cpp/demo.c"] +language = "c" +binding = "generic" +include-dirs = ["cpp/include"] +define-macros = ["DEMO_BUILD"] + +[[tool.hatch.build.hooks.hatch-cpp.libraries]] +name = "project/extension" +sources = ["cpp/extension.c"] +language = "c" +include-dirs = ["cpp/include"] +depends = ["project/lib/libdemo"] +``` + +The library builds first. On Linux it gets `-soname libdemo.so` and on macOS the install name `@rpath/libdemo.dylib`, unless `extra-link-args` already set one. The extension links the built file and gets an rpath to its directory (`$ORIGIN/lib`, `@loader_path/lib`), so the wheel works wherever it is installed. On Windows the extension links the import library `project/lib/libdemo.lib`, which is packaged next to the DLL. Windows has no rpath, so the package `__init__.py` must call `os.add_dll_directory` on `lib/` before importing the extension, and the library's header must mark its functions `__declspec(dllexport)` when building it and `__declspec(dllimport)` otherwise. `depends` is not supported for Emscripten builds. + +`library-source` (or `HATCH_CPP_LIBRARY_SOURCE`, which wins) picks how libraries named in `depends` are provided: + +- `bundled` (default): built and shipped in the wheel as above. +- `external`: not built. Each one is found under `library-root` (or `HATCH_CPP_LIBRARY_ROOT`) with `lib/` and `include/`, through `pkg-config = "demo"`, or under `$PREFIX`/`$CONDA_PREFIX` (`Library/` on Windows), by its name without the `lib` prefix. The extension links it with `-L`/`-l` and gets no rpath, so the packager supplies the runtime path; this is the mode a conda recipe uses. Validation fields on the entry check the external library. +- `direct`: not built. Its sources, include directories, and macros are compiled into each entry that depends on it, and no library ships. Use this for Pyodide. + +`static` and `python-package` are not supported yet. + +Wheel repair tools need to leave the bundled library in place: `auditwheel repair --exclude 'libdemo.so'`, `delocate-wheel --exclude libdemo`, `delvewheel repair --exclude libdemo.dll`. + +### C ABI validation + +A library that other code links against can be checked after it is built. Validation reads the export table from the binary, so it also works for cross builds and Pyodide. The field names match [hatch-rs](https://github.com/python-project-templates/hatch-rs) and come from [hatch-native](https://github.com/python-project-templates/hatch-native). + +Set the fields on a `libraries` entry: + +```toml +[[tool.hatch.build.hooks.hatch-cpp.libraries]] +name = "project/lib/libdemo" +sources = ["cpp/demo.c"] +language = "c" +binding = "generic" +include-dirs = ["cpp/include"] +expected-headers = ["cpp/include/demo/demo.h"] +header-symbol-prefix = "demo_" # every demo_* function in the headers must be exported +expected-abi-strings = ["DEMO_ABI_VERSION 1"] +expected-symbols = [] # extra symbols to require +validate = false # also load the library with ctypes +``` + +Or list `artifacts` to validate files that `libraries` does not name, such as the output of a CMake install. Each entry takes the same fields plus `name` and an optional `path` relative to the project root. Without `path`, `name` matches a `libraries` entry, or else the one packaged file called `lib.so`, `lib.dylib`, or `.dll`: + +```toml +[[tool.hatch.build.hooks.hatch-cpp.artifacts]] +name = "demo" +expected-headers = ["project/include/demo/demo.h"] +header-symbol-prefix = "demo_" +``` + +`validation-commands` run after the checks, with `{name}`, `{path}`, and `{header}` substituted and `HATCH_CPP_ARTIFACT_NAME` and `HATCH_CPP_ARTIFACT_PATH` in the environment. + +To record what was packaged, set `artifact-manifest = true` and `artifact-manifest-destination = "project/lib/hatch-cpp-artifacts.json"`. The JSON lists each packaged binary with its role (`python-extension` or `library`), platform, and profile, and is added to the wheel. + ### CLI `hatch-cpp` is integrated with [`hatch-build`](https://github.com/python-project-templates/hatch-build) to allow easy configuration of options via command line: diff --git a/hatch_cpp/config.py b/hatch_cpp/config.py index 124b06a..3bd7478 100644 --- a/hatch_cpp/config.py +++ b/hatch_cpp/config.py @@ -1,12 +1,33 @@ from __future__ import annotations +from importlib.machinery import EXTENSION_SUFFIXES from os import environ, system as system_call from pathlib import Path +from hatch_native import ( + ExternalLibrary, + LibraryValidationConfig, + check_exports, + check_headers, + prefixed_header_symbols, + resolve_external_library, + resolve_library_source, + run_validation_command, + shared_library_name, + write_artifact_manifest, +) from pkn import getSimpleLogger -from pydantic import BaseModel, Field, model_validator +from pydantic import AliasChoices, BaseModel, Field, PrivateAttr, model_validator -from .toolchains import BuildType, HatchCppCmakeConfiguration, HatchCppLibrary, HatchCppPlatform, HatchCppVcpkgConfiguration, Toolchain +from .toolchains import ( + BuildType, + HatchCppArtifact, + HatchCppCmakeConfiguration, + HatchCppLibrary, + HatchCppPlatform, + HatchCppVcpkgConfiguration, + Toolchain, +) __all__ = ( "HatchCppBuildConfig", @@ -27,6 +48,18 @@ class HatchCppBuildConfig(BaseModel): cmake: HatchCppCmakeConfiguration | None = Field(default=None) platform: HatchCppPlatform = Field(default_factory=HatchCppPlatform.default) vcpkg: HatchCppVcpkgConfiguration | None = Field(default_factory=HatchCppVcpkgConfiguration) + library_source: str = Field( + default="bundled", + validation_alias=AliasChoices("library-source", "library_source"), + description="How libraries named in depends are provided: bundled, external, or direct. Overridden by HATCH_CPP_LIBRARY_SOURCE.", + ) + artifacts: list[HatchCppArtifact] = Field(default_factory=list) + artifact_manifest: bool = Field(default=False, validation_alias=AliasChoices("artifact-manifest", "artifact_manifest")) + artifact_manifest_destination: str | None = Field( + default=None, + validation_alias=AliasChoices("artifact-manifest-destination", "artifact_manifest_destination"), + description="Wheel-relative path of the artifact manifest JSON, such as project/lib/hatch-cpp-artifacts.json.", + ) @model_validator(mode="wrap") @classmethod @@ -50,6 +83,16 @@ def validate_model(cls, data, handler): model = handler(data) if model.cmake and model.libraries: raise ValueError("Must not provide libraries when using cmake toolchain.") + declared: dict[str, HatchCppLibrary] = {} + for library in model.libraries: + if library.python_package is not None: + raise ValueError(f"Library '{library.name}' sets python-package, which hatch-cpp does not support yet.") + for name in library.depends: + if name not in declared or declared[name].binding != "generic": + raise ValueError(f"Library '{library.name}' depends on '{name}', which must be a generic library declared before it.") + declared[library.name] = library + if model.artifact_manifest and not model.artifact_manifest_destination: + raise ValueError("artifact-manifest requires artifact-manifest-destination.") return model @@ -58,6 +101,48 @@ class HatchCppBuildPlan(HatchCppBuildConfig): commands: list[str] = Field(default_factory=list) _active_toolchains: list[Toolchain] = [] + _external_libraries: dict[str, ExternalLibrary] = PrivateAttr(default_factory=dict) + _skipped_libraries: set[str] = PrivateAttr(default_factory=set) + + def built_libraries(self) -> list[HatchCppLibrary]: + """Libraries this build compiles; external and direct mode skip the ones other entries depend on.""" + return [library for library in self.libraries if library.name not in self._skipped_libraries] + + def _resolve_external(self, library: HatchCppLibrary) -> ExternalLibrary: + root = environ.get("HATCH_CPP_LIBRARY_ROOT") or (str(library.library_root) if library.library_root is not None else None) + name = Path(library.name).name + if name.startswith("lib") and len(name) > 3: + name = name[3:] + return resolve_external_library( + name, + label=library.name, + platform=self.platform.platform, + root_env_var="HATCH_CPP_LIBRARY_ROOT", + root=Path(root).resolve() if root else None, + pkg_config=library.pkg_config, + ) + + def _consumer(self, library: HatchCppLibrary, source: str) -> tuple[HatchCppLibrary, list[HatchCppLibrary]]: + """The library to compile and the libraries it links, after applying the library-source mode to its depends.""" + dependencies = [next(other for other in self.libraries if other.name == name) for name in library.depends] + if not dependencies or source == "bundled": + return library, dependencies + if source == "external": + externals = [self._external_libraries[dependency.name] for dependency in dependencies] + return library.model_copy( + update={ + "include_dirs": [*library.include_dirs, *(str(external.include_dir) for external in externals if external.include_dir)], + "library_dirs": [*library.library_dirs, *(str(external.library_dir) for external in externals)], + "libraries": [*library.libraries, *(external.link_name for external in externals)], + } + ), [] + return library.model_copy( + update={ + "sources": [*library.sources, *(path for dependency in dependencies for path in dependency.sources)], + "include_dirs": [*library.include_dirs, *(path for dependency in dependencies for path in dependency.include_dirs)], + "define_macros": [*library.define_macros, *(macro for dependency in dependencies for macro in dependency.define_macros)], + } + ), [] def generate(self): self.commands = [] @@ -94,9 +179,20 @@ def generate(self): if "vcpkg" in self._active_toolchains: log.warning("vcpkg toolchain is active; ensure that your compiler is configured to use vcpkg includes and libs.") + source = resolve_library_source(self.library_source, env_var="HATCH_CPP_LIBRARY_SOURCE") + if source == "static": + raise ValueError("library-source 'static' is not supported by hatch-cpp; use bundled, external, or direct.") + linked = {name for library in self.libraries for name in library.depends} + self._external_libraries = {} + self._skipped_libraries = set() if source == "bundled" else linked for library_index, library in enumerate(self.libraries): + if library.name in self._skipped_libraries: + if source == "external": + self._external_libraries[library.name] = self._resolve_external(library) + continue + library, dependencies = self._consumer(library, source) compile_flags = self.platform.get_compile_flags(library, self.build_type) - link_flags = self.platform.get_link_flags(library, self.build_type) + link_flags = self.platform.get_link_flags(library, self.build_type, dependencies) compiler = self.platform.cc if library.language == "c" else self.platform.cxx if self.platform.platform == "emscripten": objects = [] @@ -117,6 +213,9 @@ def generate(self): def execute(self): if self.platform.platform == "emscripten" and "vanilla" in self._active_toolchains: Path("build/hatch-cpp").mkdir(parents=True, exist_ok=True) + if "vanilla" in self._active_toolchains: + for library in self.built_libraries(): + Path(self._library_output(library)).parent.mkdir(parents=True, exist_ok=True) for command in self.commands: ret = system_call(command) if ret != 0: @@ -127,3 +226,91 @@ def cleanup(self): if self.platform.platform == "win32": for temp_obj in Path(".").glob("*.obj"): temp_obj.unlink() + + def _library_output(self, library: HatchCppLibrary) -> str: + return library.get_qualified_name(self.platform.platform) + + def _artifact_path(self, artifact: HatchCppArtifact, packaged: list[str]) -> str: + if artifact.path: + return artifact.path + library = next((library for library in self.libraries if library.name == artifact.name), None) + if library is not None: + return self._library_output(library) + filename = shared_library_name(Path(artifact.name).name, platform=self.platform.platform) + matches = [path for path in packaged if Path(path).name == filename] + if len(matches) != 1: + found = ", ".join(matches) if matches else "none" + raise FileNotFoundError(f"Artifact '{artifact.name}' needs a path: expected one packaged '{filename}', found {found}.") + return matches[0] + + def validation_targets(self, packaged: list[str]) -> list[tuple[str, LibraryValidationConfig, str]]: + """Libraries with validation fields set, plus every `artifacts` entry, as (label, config, project-relative path).""" + targets: list[tuple[str, LibraryValidationConfig, str]] = [] + for library in self.libraries: + if not _has_validation(library): + continue + if library.name in self._external_libraries: + targets.append((library.name, library, str(self._external_libraries[library.name].library))) + elif library.name not in self._skipped_libraries: + targets.append((library.name, library, self._library_output(library))) + targets.extend((artifact.name, artifact, self._artifact_path(artifact, packaged)) for artifact in self.artifacts) + return targets + + def validate_artifacts(self, root: Path, packaged: list[str]) -> None: + """Check headers, ABI strings, and exports of each validation target, then run its validation commands.""" + for label, config, relative in self.validation_targets(packaged): + library = root / relative + if not library.is_file(): + raise FileNotFoundError(f"Artifact '{label}' library was not built: {relative}") + headers = check_headers(config.expected_headers, config.expected_abi_strings, base=root, label=label) + expected = list(dict.fromkeys([*config.expected_symbols, *prefixed_header_symbols(headers, config.header_symbol_prefix, label=label)])) + check_exports(library, expected=expected, load=config.validate_artifact, label=label, display_path=relative) + tokens = {"name": label, "path": str(library), "header": str(headers[0]) if headers else ""} + for command in config.validation_commands: + env = {**environ, **{key: value.format(**tokens) for key, value in command.env.items()}} + env.update({"HATCH_CPP_ARTIFACT_NAME": label, "HATCH_CPP_ARTIFACT_PATH": str(library)}) + cwd = root / command.working_directory if command.working_directory else root + run_validation_command([value.format(**tokens) for value in command.command], cwd=cwd, env=env, label=label) + + def write_manifest(self, root: Path, packaged: list[str]) -> str | None: + """Write the manifest of packaged binaries under the project root and return its relative path.""" + if not self.artifact_manifest: + return None + assert self.artifact_manifest_destination is not None + records = [ + { + "name": Path(path).name, + "role": _artifact_role(path, self.platform.platform), + "platform": self.platform.platform, + "profile": self.build_type, + "destination": Path(path).as_posix(), + } + for path in sorted(packaged) + ] + write_artifact_manifest(root / self.artifact_manifest_destination, module=self.name or "", records=records) + return self.artifact_manifest_destination + + +def _has_validation(config: LibraryValidationConfig) -> bool: + return bool( + config.expected_symbols + or config.expected_headers + or config.expected_abi_strings + or config.header_symbol_prefix + or config.validate_artifact + or config.validation_commands + ) + + +def _artifact_role(path: str, platform: str) -> str: + """Classify a packaged binary: a Python extension module or a plain shared library.""" + filename = Path(path).name + if filename.endswith(".pyd"): + return "python-extension" + if platform != "win32" and filename.startswith("lib"): + return "library" + if any(filename.endswith(suffix) for suffix in EXTENSION_SUFFIXES if suffix != ".so") or ".abi3." in filename: + return "python-extension" + if filename.endswith(".so") and platform in ("linux", "darwin", "emscripten"): + return "python-extension" + return "library" diff --git a/hatch_cpp/plugin.py b/hatch_cpp/plugin.py index 4d8a4f2..162979a 100644 --- a/hatch_cpp/plugin.py +++ b/hatch_cpp/plugin.py @@ -88,9 +88,12 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: if build_plan.libraries: # force include libraries - for library in build_plan.libraries: + for library in build_plan.built_libraries(): name = library.get_qualified_name(build_plan.platform.platform) build_data["force_include"][name] = name + import_library = Path(name).with_suffix(".lib") + if build_plan.platform.platform == "win32" and library.binding == "generic" and import_library.is_file(): + build_data["force_include"][import_library.as_posix()] = import_library.as_posix() build_data["pure_python"] = False machine = platform_machine() @@ -101,7 +104,7 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: machine, version_major, version_minor, - all(lib.py_limited_api for lib in build_plan.libraries), + all(lib.py_limited_api for lib in build_plan.built_libraries()), ) else: assert build_plan.cmake is not None @@ -120,5 +123,11 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: if path.suffix in (".pyd", ".dll", ".so", ".dylib"): build_data["force_include"][str(path)] = str(path) + packaged = list(build_data["force_include"]) + build_plan.validate_artifacts(Path(self.root), packaged) + manifest = build_plan.write_manifest(Path(self.root), packaged) + if manifest is not None: + build_data["force_include"][manifest] = manifest + for path in build_data["force_include"]: self._logger.info(f"Force include: {path}") diff --git a/hatch_cpp/tests/test_project_c_abi_library/cpp/demo.c b/hatch_cpp/tests/test_project_c_abi_library/cpp/demo.c new file mode 100644 index 0000000..92c9a61 --- /dev/null +++ b/hatch_cpp/tests/test_project_c_abi_library/cpp/demo.c @@ -0,0 +1,5 @@ +#include "demo/demo.h" + +int demo_abi_version(void) { return DEMO_ABI_VERSION; } + +int demo_answer(void) { return 42; } diff --git a/hatch_cpp/tests/test_project_c_abi_library/cpp/extension.c b/hatch_cpp/tests/test_project_c_abi_library/cpp/extension.c new file mode 100644 index 0000000..d58f769 --- /dev/null +++ b/hatch_cpp/tests/test_project_c_abi_library/cpp/extension.c @@ -0,0 +1,11 @@ +#include + +#include "demo/demo.h" + +static PyObject *answer(PyObject *self, PyObject *args) { return PyLong_FromLong(demo_answer()); } + +static PyMethodDef methods[] = {{"answer", answer, METH_NOARGS, NULL}, {NULL, NULL, 0, NULL}}; + +static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, "extension", NULL, -1, methods}; + +PyMODINIT_FUNC PyInit_extension(void) { return PyModule_Create(&module); } diff --git a/hatch_cpp/tests/test_project_c_abi_library/cpp/include/demo/demo.h b/hatch_cpp/tests/test_project_c_abi_library/cpp/include/demo/demo.h new file mode 100644 index 0000000..e5cbb47 --- /dev/null +++ b/hatch_cpp/tests/test_project_c_abi_library/cpp/include/demo/demo.h @@ -0,0 +1,22 @@ +#pragma once + +#define DEMO_ABI_VERSION 1 + +#if defined(_WIN32) && defined(DEMO_BUILD) +#define DEMO_API __declspec(dllexport) +#elif defined(_WIN32) +#define DEMO_API __declspec(dllimport) +#else +#define DEMO_API __attribute__((visibility("default"))) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +DEMO_API int demo_abi_version(void); +DEMO_API int demo_answer(void); + +#ifdef __cplusplus +} +#endif diff --git a/hatch_cpp/tests/test_project_c_abi_library/project/__init__.py b/hatch_cpp/tests/test_project_c_abi_library/project/__init__.py new file mode 100644 index 0000000..af8e3ca --- /dev/null +++ b/hatch_cpp/tests/test_project_c_abi_library/project/__init__.py @@ -0,0 +1,5 @@ +import os +import sys + +if sys.platform == "win32": + os.add_dll_directory(os.path.join(os.path.dirname(__file__), "lib")) diff --git a/hatch_cpp/tests/test_project_c_abi_library/pyproject.toml b/hatch_cpp/tests/test_project_c_abi_library/pyproject.toml new file mode 100644 index 0000000..2c7bef1 --- /dev/null +++ b/hatch_cpp/tests/test_project_c_abi_library/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["hatchling>=1.20"] +build-backend = "hatchling.build" + +[project] +name = "hatch-cpp-test-project-c-abi-library" +description = "C ABI library validated by hatch-cpp" +version = "0.1.0" +requires-python = ">=3.9" +dependencies = [ + "hatchling>=1.20", + "hatch-cpp", +] + +[tool.hatch.build] +artifacts = [ + "project/*.pyd", + "project/*.so", + "project/lib/*.lib", + "project/lib/*.dll", + "project/lib/*.dylib", + "project/lib/*.so", + "project/lib/*.json", +] + +[tool.hatch.build.sources] +src = "/" + +[tool.hatch.build.targets.sdist] +packages = ["project"] + +[tool.hatch.build.targets.wheel] +packages = ["project"] + +[tool.hatch.build.hooks.hatch-cpp] +verbose = true +artifact-manifest = true +artifact-manifest-destination = "project/lib/hatch-cpp-artifacts.json" + +[[tool.hatch.build.hooks.hatch-cpp.libraries]] +name = "project/lib/libdemo" +sources = ["cpp/demo.c"] +language = "c" +binding = "generic" +include-dirs = ["cpp/include"] +define-macros = ["DEMO_BUILD"] +expected-headers = ["cpp/include/demo/demo.h"] +header-symbol-prefix = "demo_" +expected-abi-strings = ["DEMO_ABI_VERSION 1"] +validate = true + +[[tool.hatch.build.hooks.hatch-cpp.libraries]] +name = "project/extension" +sources = ["cpp/extension.c"] +language = "c" +include-dirs = ["cpp/include"] +depends = ["project/lib/libdemo"] diff --git a/hatch_cpp/tests/test_project_cmake_c_abi/CMakeLists.txt b/hatch_cpp/tests/test_project_cmake_c_abi/CMakeLists.txt new file mode 100644 index 0000000..0a037fd --- /dev/null +++ b/hatch_cpp/tests/test_project_cmake_c_abi/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 3.20.0) +project(hatch-cpp-test-project-cmake-c-abi VERSION "0.1.0" LANGUAGES C) + +set(CMAKE_INSTALL_NAME_DIR "@rpath") + +add_library(demo SHARED cpp/demo.c) +target_include_directories(demo PUBLIC "${CMAKE_SOURCE_DIR}/cpp/include") +target_compile_definitions(demo PRIVATE DEMO_BUILD) +set_target_properties(demo PROPERTIES PUBLIC_HEADER cpp/include/demo/demo.h) +install(TARGETS demo + LIBRARY DESTINATION project/lib + RUNTIME DESTINATION project/lib + ARCHIVE DESTINATION project/lib + PUBLIC_HEADER DESTINATION project/include/demo + ) + +find_package(Python ${HATCH_CPP_TEST_PROJECT_CMAKE_C_ABI_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development.Module) +python_add_library(extension MODULE WITH_SOABI cpp/extension.c) +target_link_libraries(extension PRIVATE demo) +if(APPLE) + set_target_properties(extension PROPERTIES INSTALL_RPATH "@loader_path/lib") +elseif(UNIX) + set_target_properties(extension PROPERTIES INSTALL_RPATH "\$ORIGIN/lib") +endif() +install(TARGETS extension + LIBRARY DESTINATION project + RUNTIME DESTINATION project + ) diff --git a/hatch_cpp/tests/test_project_cmake_c_abi/cpp/demo.c b/hatch_cpp/tests/test_project_cmake_c_abi/cpp/demo.c new file mode 100644 index 0000000..92c9a61 --- /dev/null +++ b/hatch_cpp/tests/test_project_cmake_c_abi/cpp/demo.c @@ -0,0 +1,5 @@ +#include "demo/demo.h" + +int demo_abi_version(void) { return DEMO_ABI_VERSION; } + +int demo_answer(void) { return 42; } diff --git a/hatch_cpp/tests/test_project_cmake_c_abi/cpp/extension.c b/hatch_cpp/tests/test_project_cmake_c_abi/cpp/extension.c new file mode 100644 index 0000000..d58f769 --- /dev/null +++ b/hatch_cpp/tests/test_project_cmake_c_abi/cpp/extension.c @@ -0,0 +1,11 @@ +#include + +#include "demo/demo.h" + +static PyObject *answer(PyObject *self, PyObject *args) { return PyLong_FromLong(demo_answer()); } + +static PyMethodDef methods[] = {{"answer", answer, METH_NOARGS, NULL}, {NULL, NULL, 0, NULL}}; + +static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, "extension", NULL, -1, methods}; + +PyMODINIT_FUNC PyInit_extension(void) { return PyModule_Create(&module); } diff --git a/hatch_cpp/tests/test_project_cmake_c_abi/cpp/include/demo/demo.h b/hatch_cpp/tests/test_project_cmake_c_abi/cpp/include/demo/demo.h new file mode 100644 index 0000000..e5cbb47 --- /dev/null +++ b/hatch_cpp/tests/test_project_cmake_c_abi/cpp/include/demo/demo.h @@ -0,0 +1,22 @@ +#pragma once + +#define DEMO_ABI_VERSION 1 + +#if defined(_WIN32) && defined(DEMO_BUILD) +#define DEMO_API __declspec(dllexport) +#elif defined(_WIN32) +#define DEMO_API __declspec(dllimport) +#else +#define DEMO_API __attribute__((visibility("default"))) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +DEMO_API int demo_abi_version(void); +DEMO_API int demo_answer(void); + +#ifdef __cplusplus +} +#endif diff --git a/hatch_cpp/tests/test_project_cmake_c_abi/project/__init__.py b/hatch_cpp/tests/test_project_cmake_c_abi/project/__init__.py new file mode 100644 index 0000000..af8e3ca --- /dev/null +++ b/hatch_cpp/tests/test_project_cmake_c_abi/project/__init__.py @@ -0,0 +1,5 @@ +import os +import sys + +if sys.platform == "win32": + os.add_dll_directory(os.path.join(os.path.dirname(__file__), "lib")) diff --git a/hatch_cpp/tests/test_project_cmake_c_abi/pyproject.toml b/hatch_cpp/tests/test_project_cmake_c_abi/pyproject.toml new file mode 100644 index 0000000..a1d1d8c --- /dev/null +++ b/hatch_cpp/tests/test_project_cmake_c_abi/pyproject.toml @@ -0,0 +1,50 @@ +[build-system] +requires = ["hatchling>=1.20"] +build-backend = "hatchling.build" + +[project] +name = "hatch-cpp-test-project-cmake-c-abi" +description = "CMake-built C ABI library validated by hatch-cpp" +version = "0.1.0" +requires-python = ">=3.9" +dependencies = [ + "hatchling>=1.20", + "hatch-cpp", +] + +[tool.hatch.build] +artifacts = [ + "project/*.pyd", + "project/*.so", + "project/include/**/*.h", + "project/lib/*.dll", + "project/lib/*.dylib", + "project/lib/*.json", + "project/lib/*.lib", + "project/lib/*.so", +] + +[tool.hatch.build.sources] +src = "/" + +[tool.hatch.build.targets.sdist] +packages = ["project"] + +[tool.hatch.build.targets.wheel] +packages = ["project"] + +[tool.hatch.build.hooks.hatch-cpp] +verbose = true +artifact-manifest = true +artifact-manifest-destination = "project/lib/hatch-cpp-artifacts.json" + +[tool.hatch.build.hooks.hatch-cpp.cmake] +root = "CMakeLists.txt" +include_flags = {"python_version" = true} + +# No path: hatch-cpp finds the one packaged libdemo.so, libdemo.dylib, or demo.dll. +[[tool.hatch.build.hooks.hatch-cpp.artifacts]] +name = "demo" +expected-headers = ["project/include/demo/demo.h"] +header-symbol-prefix = "demo_" +expected-abi-strings = ["DEMO_ABI_VERSION 1"] diff --git a/hatch_cpp/tests/test_validation.py b/hatch_cpp/tests/test_validation.py new file mode 100644 index 0000000..bc667c3 --- /dev/null +++ b/hatch_cpp/tests/test_validation.py @@ -0,0 +1,216 @@ +from json import loads +from os import environ +from pathlib import Path +from shutil import copy2, copytree +from subprocess import check_call, run +from sys import executable, platform + +import pytest +from pydantic import ValidationError + +from hatch_cpp import HatchCppBuildPlan +from hatch_cpp.config import _artifact_role + +FIXTURE = Path(__file__).parent / "test_project_c_abi_library" +LIBRARY = { + "name": "project/lib/libdemo", + "sources": ["cpp/demo.c"], + "language": "c", + "binding": "generic", + "include-dirs": ["cpp/include"], +} + + +def _plan(**config): + return HatchCppBuildPlan(name="demo", **config) + + +def test_c_abi_library_is_validated_linked_and_recorded(tmp_path): + project = copytree(FIXTURE, tmp_path / "build_root") + check_call(["hatch-build", "--hooks-only"], cwd=project) + + suffix = _library_suffix() + manifest = loads((project / "project/lib/hatch-cpp-artifacts.json").read_text()) + roles = {record["destination"]: record["role"] for record in manifest["artifacts"]} + assert roles[f"project/lib/libdemo.{suffix}"] == "library" + assert [role for role in roles.values() if role == "python-extension"] == ["python-extension"] + + # The extension finds libdemo through a package-relative rpath, so the package still imports after it moves. + installed = copytree(project / "project", tmp_path / "site" / "project") + check_call([executable, "-c", "import project.extension as e; assert e.answer() == 42"], cwd=installed.parent) + + +def _library_suffix(): + return {"win32": "dll", "darwin": "dylib"}.get(platform, "so") + + +@pytest.mark.skipif(platform == "win32", reason="the external prefix is found through the loader path, which Windows does not have") +def test_external_mode_links_a_prebuilt_library(tmp_path): + bundled = copytree(FIXTURE, tmp_path / "bundled") + check_call(["hatch-build", "--hooks-only"], cwd=bundled) + prefix = tmp_path / "prefix" + (prefix / "lib").mkdir(parents=True) + copy2(bundled / f"project/lib/libdemo.{_library_suffix()}", prefix / "lib") + copytree(bundled / "cpp/include", prefix / "include") + + external = copytree(FIXTURE, tmp_path / "external") + env = {**environ, "HATCH_CPP_LIBRARY_SOURCE": "external", "HATCH_CPP_LIBRARY_ROOT": str(prefix)} + check_call(["hatch-build", "--hooks-only"], cwd=external, env=env) + + assert not (external / f"project/lib/libdemo.{_library_suffix()}").exists() + manifest = loads((external / "project/lib/hatch-cpp-artifacts.json").read_text()) + assert [record["role"] for record in manifest["artifacts"]] == ["python-extension"] + loader_path = "DYLD_LIBRARY_PATH" if platform == "darwin" else "LD_LIBRARY_PATH" + check_call( + [executable, "-c", "import project.extension as e; assert e.answer() == 42"], + cwd=external, + env={**environ, loader_path: str(prefix / "lib")}, + ) + + +def test_direct_mode_compiles_the_library_into_the_extension(tmp_path): + project = copytree(FIXTURE, tmp_path / "direct") + check_call(["hatch-build", "--hooks-only"], cwd=project, env={**environ, "HATCH_CPP_LIBRARY_SOURCE": "direct"}) + + assert not (project / f"project/lib/libdemo.{_library_suffix()}").exists() + check_call([executable, "-c", "import project.extension as e; assert e.answer() == 42"], cwd=project) + + +def test_static_mode_is_rejected(monkeypatch): + monkeypatch.setenv("HATCH_CPP_LIBRARY_SOURCE", "static") + extension = {"name": "project/extension", "sources": ["ext.c"], "depends": ["project/lib/libdemo"]} + with pytest.raises(ValueError, match="'static' is not supported"): + _plan(libraries=[LIBRARY, extension]).generate() + + +def test_python_package_is_rejected(): + with pytest.raises(ValidationError, match="python-package, which hatch-cpp does not support yet"): + _plan(libraries=[{**LIBRARY, "python-package": "other"}]) + + +CMAKE_FIXTURE = Path(__file__).parent / "test_project_cmake_c_abi" + + +def test_cmake_library_is_validated_by_name(tmp_path): + project = copytree(CMAKE_FIXTURE, tmp_path / "cmake") + check_call(["hatch-build", "--hooks-only"], cwd=project) + + manifest = loads((project / "project/lib/hatch-cpp-artifacts.json").read_text()) + roles = sorted(record["role"] for record in manifest["artifacts"]) + assert roles == ["library", "python-extension"] + installed = copytree(project / "project", tmp_path / "site" / "project") + check_call([executable, "-c", "import project.extension as e; assert e.answer() == 42"], cwd=installed.parent) + + +def test_cmake_library_validation_failure_stops_the_build(tmp_path): + project = copytree(CMAKE_FIXTURE, tmp_path / "cmake") + pyproject = project / "pyproject.toml" + pyproject.write_text(pyproject.read_text().replace("DEMO_ABI_VERSION 1", "DEMO_ABI_VERSION 2")) + completed = run(["hatch-build", "--hooks-only"], check=False, cwd=project, capture_output=True, text=True) + assert completed.returncode != 0 + assert "Artifact 'demo' expected ABI string was not found in headers: DEMO_ABI_VERSION 2" in completed.stderr + completed.stdout + + +def test_validation_fails_on_unexported_header_function(tmp_path, monkeypatch): + monkeypatch.setattr("hatch_native.validation.exported_symbols", lambda path: {"demo_answer"}) + library = {**LIBRARY, "expected-headers": ["include/demo/demo.h"], "header-symbol-prefix": "demo_"} + (tmp_path / "include/demo").mkdir(parents=True) + (tmp_path / "include/demo/demo.h").write_text("int demo_answer(void);\nint demo_abi_version(void);\n") + plan = _plan(libraries=[library]) + output = tmp_path / plan.libraries[0].get_qualified_name(plan.platform.platform) + output.parent.mkdir(parents=True) + output.write_bytes(b"") + + with pytest.raises(RuntimeError, match="expected symbol was not exported: demo_abi_version"): + plan.validate_artifacts(tmp_path, []) + + +def test_validation_fails_on_missing_abi_string(tmp_path): + library = {**LIBRARY, "expected-headers": ["demo.h"], "expected-abi-strings": ["DEMO_ABI_VERSION 2"]} + (tmp_path / "demo.h").write_text("#define DEMO_ABI_VERSION 1\n") + plan = _plan(libraries=[library]) + output = tmp_path / plan.libraries[0].get_qualified_name(plan.platform.platform) + output.parent.mkdir(parents=True) + output.write_bytes(b"") + + with pytest.raises(RuntimeError, match="ABI string was not found"): + plan.validate_artifacts(tmp_path, []) + + +def test_libraries_without_validation_fields_are_not_targets(): + assert _plan(libraries=[LIBRARY]).validation_targets([]) == [] + + +def test_artifact_targets_resolve_by_path_library_and_packaged_name(): + plan = _plan( + libraries=[LIBRARY], + artifacts=[ + {"name": "explicit", "path": "somewhere/libexplicit.so"}, + {"name": "project/lib/libdemo", "expected-symbols": ["demo_answer"]}, + {"name": "installed", "expected-symbols": ["installed_answer"]}, + ], + ) + installed = {"win32": "installed.dll", "darwin": "libinstalled.dylib"}.get(plan.platform.platform, "libinstalled.so") + paths = [path for _, _, path in plan.validation_targets([f"project/lib/{installed}", "project/extension.so"])] + assert paths == ["somewhere/libexplicit.so", plan.libraries[0].get_qualified_name(plan.platform.platform), f"project/lib/{installed}"] + + +def test_artifact_without_unique_packaged_match_needs_a_path(): + plan = _plan(artifacts=[{"name": "missing"}]) + with pytest.raises(FileNotFoundError, match="Artifact 'missing' needs a path"): + plan.validation_targets([]) + + +def test_depends_must_name_an_earlier_generic_library(): + with pytest.raises(ValidationError, match="must be a generic library declared before it"): + _plan(libraries=[{"name": "project/extension", "sources": ["ext.c"], "depends": ["project/lib/libdemo"]}, LIBRARY]) + + +def test_generic_library_gets_layout_neutral_identity(): + plan = _plan(libraries=[LIBRARY]) + library = plan.libraries[0] + plan.platform.toolchain = "clang" + plan.platform.platform = "darwin" + assert "-Wl,-install_name,@rpath/libdemo.dylib" in plan.platform.get_link_flags(library) + plan.platform.platform = "linux" + assert "-Wl,-soname,libdemo.so" in plan.platform.get_link_flags(library) + library.extra_link_args = ["-Wl,-soname,libdemo.so.1"] + assert "-Wl,-soname,libdemo.so " not in plan.platform.get_link_flags(library) + + +def test_depends_links_output_and_sets_rpath(): + extension = {"name": "project/extension", "sources": ["ext.c"], "depends": ["project/lib/libdemo"]} + plan = _plan(libraries=[LIBRARY, extension]) + producer, consumer = plan.libraries + plan.platform.toolchain = "gcc" + plan.platform.platform = "linux" + flags = plan.platform.get_link_flags(consumer, dependencies=[producer]) + assert "project/lib/libdemo.so -Wl,-rpath,\\$ORIGIN/lib" in flags + plan.platform.toolchain = "msvc" + plan.platform.platform = "win32" + assert "project/lib/libdemo.lib" in plan.platform.get_link_flags(consumer, dependencies=[producer]).replace("\\", "/") + plan.platform.toolchain = "clang" + plan.platform.platform = "emscripten" + with pytest.raises(ValueError, match="not supported for Emscripten"): + plan.platform.get_link_flags(consumer, dependencies=[producer]) + + +def test_artifact_manifest_requires_destination(): + with pytest.raises(ValidationError, match="artifact-manifest requires artifact-manifest-destination"): + _plan(**{"artifact-manifest": True}) + + +@pytest.mark.parametrize( + "path,target_platform,role", + [ + ("project/extension.cpython-312-x86_64-linux-gnu.so", "linux", "python-extension"), + ("project/extension.abi3.so", "darwin", "python-extension"), + ("project/extension.so", "linux", "python-extension"), + ("project/extension.pyd", "win32", "python-extension"), + ("project/lib/libdemo.so", "linux", "library"), + ("project/lib/libdemo.dylib", "darwin", "library"), + ("project/lib/demo.dll", "win32", "library"), + ], +) +def test_artifact_role(path, target_platform, role): + assert _artifact_role(path, target_platform) == role diff --git a/hatch_cpp/toolchains/common.py b/hatch_cpp/toolchains/common.py index c92c88a..40ebd8a 100644 --- a/hatch_cpp/toolchains/common.py +++ b/hatch_cpp/toolchains/common.py @@ -1,6 +1,7 @@ from __future__ import annotations from os import environ +from os.path import relpath from pathlib import Path from re import match from shutil import which @@ -8,12 +9,14 @@ from sysconfig import get_config_var, get_path from typing import Any, Literal +from hatch_native import ExternalLibraryConfig, LibraryValidationConfig from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator __all__ = ( "Binding", "BuildType", "CompilerToolchain", + "HatchCppArtifact", "HatchCppLibrary", "HatchCppPlatform", "Language", @@ -38,7 +41,7 @@ } -class HatchCppLibrary(BaseModel, validate_assignment=True): +class HatchCppLibrary(LibraryValidationConfig, ExternalLibraryConfig, validate_assignment=True): """A C++ library.""" name: str @@ -221,6 +224,13 @@ def get_effective_undef_macros(self, platform: Platform) -> list[str]: return macros +class HatchCppArtifact(LibraryValidationConfig): + """A built or installed library to validate, named by a `libraries` entry, a path, or a CMake-installed library name.""" + + name: str + path: str | None = Field(default=None, description="Library path relative to the project root. Inferred from `name` when unset.") + + def _normalize_rpath(value: str, platform: Platform) -> str: r"""Translate and escape rpath values for the target platform. @@ -369,9 +379,10 @@ def get_compile_flags(self, library: HatchCppLibrary, build_type: BuildType = "r flags = flags.replace(" ", " ") return flags - def get_link_flags(self, library: HatchCppLibrary, build_type: BuildType = "release") -> str: + def get_link_flags(self, library: HatchCppLibrary, build_type: BuildType = "release", dependencies: list[HatchCppLibrary] | None = None) -> str: flags = "" effective_link_args = library.get_effective_link_args(self.platform) + effective_link_args.extend(self._library_identity_args(library, effective_link_args)) effective_extra_objects = library.get_effective_extra_objects(self.platform) effective_libraries = library.get_effective_libraries(self.platform) effective_library_dirs = library.get_effective_library_dirs(self.platform) @@ -383,6 +394,7 @@ def get_link_flags(self, library: HatchCppLibrary, build_type: BuildType = "rele flags += " -shared" flags += " " + " ".join(effective_link_args) flags += " " + " ".join(effective_extra_objects) + flags += " " + " ".join(self._dependency_link_args(library, dependencies or [])) flags += " " + " ".join(f"-l{lib}" for lib in effective_libraries) flags += " " + " ".join(f"-L{lib}" for lib in effective_library_dirs) flags += f" -o {library.get_qualified_name(self.platform)}" @@ -413,8 +425,41 @@ def get_link_flags(self, library: HatchCppLibrary, build_type: BuildType = "rele flags += f" /LIBPATH:{libs_path!s}" break flags += " " + " ".join(f"{lib}.lib" for lib in effective_libraries) + flags += " " + " ".join(self._dependency_link_args(library, dependencies or [])) flags += " " + " ".join(f"/LIBPATH:{lib}" for lib in effective_library_dirs) # clean while flags.count(" "): flags = flags.replace(" ", " ") return flags + + def _library_identity_args(self, library: HatchCppLibrary, link_args: list[str]) -> list[str]: + """Give a generic library a layout-neutral SONAME or @rpath install name unless the project sets one.""" + if library.binding != "generic" or self.toolchain == "msvc": + return [] + filename = Path(library.get_qualified_name(self.platform)).name + existing = " ".join(link_args) + if self.platform == "linux" and "soname" not in existing: + return [f"-Wl,-soname,{filename}"] + if self.platform == "darwin" and "install_name" not in existing: + return [f"-Wl,-install_name,@rpath/{filename}"] + return [] + + def _dependency_link_args(self, library: HatchCppLibrary, dependencies: list[HatchCppLibrary]) -> list[str]: + """Link `library` against the outputs of its `depends` entries and add an rpath to each output directory.""" + if dependencies and self.platform == "emscripten": + raise ValueError(f"Library '{library.name}' sets depends, which is not supported for Emscripten builds.") + output_dir = Path(library.get_qualified_name(self.platform)).parent + args: list[str] = [] + rpaths: list[str] = [] + for dependency in dependencies: + dependency_path = Path(dependency.get_qualified_name(self.platform)) + if self.platform == "win32": + args.append(str(dependency_path.with_suffix(".lib"))) + continue + args.append(dependency_path.as_posix()) + relative = Path(relpath(dependency_path.parent, output_dir)).as_posix() + origin = "@loader_path" if self.platform == "darwin" else "$ORIGIN" + rpath = _normalize_rpath(origin if relative == "." else f"{origin}/{relative}", self.platform) + if rpath not in rpaths: + rpaths.append(rpath) + return args + [f"-Wl,-rpath,{rpath}" for rpath in rpaths] diff --git a/pyproject.toml b/pyproject.toml index 26da470..2272262 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ classifiers = [ ] dependencies = [ + "hatch-native>=0.1.0,<0.2", "hatchling", "hatch-build>=0.5,<0.6", "pkn",