Validate C ABI libraries and link extensions against them

README.md

@@ -83,6 +83,72 @@ cmake_env_args = {} # env-specific cmake args to pass
 include_flags = {} # include flags to pass -D
 ```
 
+### Linking an extension against a library
+
+A `generic` library can ship next to the extension that links it, in the layout pyarrow uses. List the library first and name it in the extension's `depends`:
+
+```toml
+[[tool.hatch.build.hooks.hatch-cpp.libraries]]
+name = "project/lib/libdemo"
+sources = ["cpp/demo.c"]
+language = "c"
+binding = "generic"
+include-dirs = ["cpp/include"]
+define-macros = ["DEMO_BUILD"]
+
+[[tool.hatch.build.hooks.hatch-cpp.libraries]]
+name = "project/extension"
+sources = ["cpp/extension.c"]
+language = "c"
+include-dirs = ["cpp/include"]
+depends = ["project/lib/libdemo"]
+```
+
+The library builds first. On Linux it gets `-soname libdemo.so` and on macOS the install name `@rpath/libdemo.dylib`, unless `extra-link-args` already set one. The extension links the built file and gets an rpath to its directory (`$ORIGIN/lib`, `@loader_path/lib`), so the wheel works wherever it is installed. On Windows the extension links the import library `project/lib/libdemo.lib`, which is packaged next to the DLL. Windows has no rpath, so the package `__init__.py` must call `os.add_dll_directory` on `lib/` before importing the extension, and the library's header must mark its functions `__declspec(dllexport)` when building it and `__declspec(dllimport)` otherwise. `depends` is not supported for Emscripten builds.
+
+`library-source` (or `HATCH_CPP_LIBRARY_SOURCE`, which wins) picks how libraries named in `depends` are provided:
+
+- `bundled` (default): built and shipped in the wheel as above.
+- `external`: not built. Each one is found under `library-root` (or `HATCH_CPP_LIBRARY_ROOT`) with `lib/` and `include/`, through `pkg-config = "demo"`, or under `$PREFIX`/`$CONDA_PREFIX` (`Library/` on Windows), by its name without the `lib` prefix. The extension links it with `-L`/`-l` and gets no rpath, so the packager supplies the runtime path; this is the mode a conda recipe uses. Validation fields on the entry check the external library.
+- `direct`: not built. Its sources, include directories, and macros are compiled into each entry that depends on it, and no library ships. Use this for Pyodide.
+
+`static` and `python-package` are not supported yet.
+
+Wheel repair tools need to leave the bundled library in place: `auditwheel repair --exclude 'libdemo.so'`, `delocate-wheel --exclude libdemo`, `delvewheel repair --exclude libdemo.dll`.
+
+### C ABI validation
+
+A library that other code links against can be checked after it is built. Validation reads the export table from the binary, so it also works for cross builds and Pyodide. The field names match [hatch-rs](https://github.com/python-project-templates/hatch-rs) and come from [hatch-native](https://github.com/python-project-templates/hatch-native).
+
+Set the fields on a `libraries` entry:
+
+```toml
+[[tool.hatch.build.hooks.hatch-cpp.libraries]]
+name = "project/lib/libdemo"
+sources = ["cpp/demo.c"]
+language = "c"
+binding = "generic"
+include-dirs = ["cpp/include"]
+expected-headers = ["cpp/include/demo/demo.h"]
+header-symbol-prefix = "demo_"           # every demo_* function in the headers must be exported
+expected-abi-strings = ["DEMO_ABI_VERSION 1"]
+expected-symbols = []                    # extra symbols to require
+validate = false                         # also load the library with ctypes
+```
+
+Or list `artifacts` to validate files that `libraries` does not name, such as the output of a CMake install. Each entry takes the same fields plus `name` and an optional `path` relative to the project root. Without `path`, `name` matches a `libraries` entry, or else the one packaged file called `lib<name>.so`, `lib<name>.dylib`, or `<name>.dll`:
+
+```toml
+[[tool.hatch.build.hooks.hatch-cpp.artifacts]]
+name = "demo"
+expected-headers = ["project/include/demo/demo.h"]
+header-symbol-prefix = "demo_"
+```
+
+`validation-commands` run after the checks, with `{name}`, `{path}`, and `{header}` substituted and `HATCH_CPP_ARTIFACT_NAME` and `HATCH_CPP_ARTIFACT_PATH` in the environment.
+
+To record what was packaged, set `artifact-manifest = true` and `artifact-manifest-destination = "project/lib/hatch-cpp-artifacts.json"`. The JSON lists each packaged binary with its role (`python-extension` or `library`), platform, and profile, and is added to the wheel.
+
 ### CLI
 
 `hatch-cpp` is integrated with [`hatch-build`](https://github.com/python-project-templates/hatch-build) to allow easy configuration of options via command line:

hatch_cpp/config.py

@@ -1,12 +1,33 @@
 from __future__ import annotations
 
+from importlib.machinery import EXTENSION_SUFFIXES
 from os import environ, system as system_call
 from pathlib import Path
 
+from hatch_native import (
+    ExternalLibrary,
+    LibraryValidationConfig,
+    check_exports,
+    check_headers,
+    prefixed_header_symbols,
+    resolve_external_library,
+    resolve_library_source,
+    run_validation_command,
+    shared_library_name,
+    write_artifact_manifest,
+)
 from pkn import getSimpleLogger
-from pydantic import BaseModel, Field, model_validator
+from pydantic import AliasChoices, BaseModel, Field, PrivateAttr, model_validator
 
-from .toolchains import BuildType, HatchCppCmakeConfiguration, HatchCppLibrary, HatchCppPlatform, HatchCppVcpkgConfiguration, Toolchain
+from .toolchains import (
+    BuildType,
+    HatchCppArtifact,
+    HatchCppCmakeConfiguration,
+    HatchCppLibrary,
+    HatchCppPlatform,
+    HatchCppVcpkgConfiguration,
+    Toolchain,
+)
 
 __all__ = (
     "HatchCppBuildConfig",
@@ -27,6 +48,18 @@ class HatchCppBuildConfig(BaseModel):
     cmake: HatchCppCmakeConfiguration | None = Field(default=None)
     platform: HatchCppPlatform = Field(default_factory=HatchCppPlatform.default)
     vcpkg: HatchCppVcpkgConfiguration | None = Field(default_factory=HatchCppVcpkgConfiguration)
+    library_source: str = Field(
+        default="bundled",
+        validation_alias=AliasChoices("library-source", "library_source"),
+        description="How libraries named in depends are provided: bundled, external, or direct. Overridden by HATCH_CPP_LIBRARY_SOURCE.",
+    )
+    artifacts: list[HatchCppArtifact] = Field(default_factory=list)
+    artifact_manifest: bool = Field(default=False, validation_alias=AliasChoices("artifact-manifest", "artifact_manifest"))
+    artifact_manifest_destination: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("artifact-manifest-destination", "artifact_manifest_destination"),
+        description="Wheel-relative path of the artifact manifest JSON, such as project/lib/hatch-cpp-artifacts.json.",
+    )
 
     @model_validator(mode="wrap")
     @classmethod
@@ -50,6 +83,16 @@ def validate_model(cls, data, handler):
         model = handler(data)
         if model.cmake and model.libraries:
             raise ValueError("Must not provide libraries when using cmake toolchain.")
+        declared: dict[str, HatchCppLibrary] = {}
+        for library in model.libraries:
+            if library.python_package is not None:
+                raise ValueError(f"Library '{library.name}' sets python-package, which hatch-cpp does not support yet.")
+            for name in library.depends:
+                if name not in declared or declared[name].binding != "generic":
+                    raise ValueError(f"Library '{library.name}' depends on '{name}', which must be a generic library declared before it.")
+            declared[library.name] = library
+        if model.artifact_manifest and not model.artifact_manifest_destination:
+            raise ValueError("artifact-manifest requires artifact-manifest-destination.")
         return model
 
 
@@ -58,6 +101,48 @@ class HatchCppBuildPlan(HatchCppBuildConfig):
     commands: list[str] = Field(default_factory=list)
 
     _active_toolchains: list[Toolchain] = []
+    _external_libraries: dict[str, ExternalLibrary] = PrivateAttr(default_factory=dict)
+    _skipped_libraries: set[str] = PrivateAttr(default_factory=set)
+
+    def built_libraries(self) -> list[HatchCppLibrary]:
+        """Libraries this build compiles; external and direct mode skip the ones other entries depend on."""
+        return [library for library in self.libraries if library.name not in self._skipped_libraries]
+
+    def _resolve_external(self, library: HatchCppLibrary) -> ExternalLibrary:
+        root = environ.get("HATCH_CPP_LIBRARY_ROOT") or (str(library.library_root) if library.library_root is not None else None)
+        name = Path(library.name).name
+        if name.startswith("lib") and len(name) > 3:
+            name = name[3:]
+        return resolve_external_library(
+            name,
+            label=library.name,
+            platform=self.platform.platform,
+            root_env_var="HATCH_CPP_LIBRARY_ROOT",
+            root=Path(root).resolve() if root else None,
+            pkg_config=library.pkg_config,
+        )
+
+    def _consumer(self, library: HatchCppLibrary, source: str) -> tuple[HatchCppLibrary, list[HatchCppLibrary]]:
+        """The library to compile and the libraries it links, after applying the library-source mode to its depends."""
+        dependencies = [next(other for other in self.libraries if other.name == name) for name in library.depends]
+        if not dependencies or source == "bundled":
+            return library, dependencies
+        if source == "external":
+            externals = [self._external_libraries[dependency.name] for dependency in dependencies]
+            return library.model_copy(
+                update={
+                    "include_dirs": [*library.include_dirs, *(str(external.include_dir) for external in externals if external.include_dir)],
+                    "library_dirs": [*library.library_dirs, *(str(external.library_dir) for external in externals)],
+                    "libraries": [*library.libraries, *(external.link_name for external in externals)],
+                }
+            ), []
+        return library.model_copy(
+            update={
+                "sources": [*library.sources, *(path for dependency in dependencies for path in dependency.sources)],
+                "include_dirs": [*library.include_dirs, *(path for dependency in dependencies for path in dependency.include_dirs)],
+                "define_macros": [*library.define_macros, *(macro for dependency in dependencies for macro in dependency.define_macros)],
+            }
+        ), []
 
     def generate(self):
         self.commands = []
@@ -94,9 +179,20 @@ def generate(self):
             if "vcpkg" in self._active_toolchains:
                 log.warning("vcpkg toolchain is active; ensure that your compiler is configured to use vcpkg includes and libs.")
 
+            source = resolve_library_source(self.library_source, env_var="HATCH_CPP_LIBRARY_SOURCE")
+            if source == "static":
+                raise ValueError("library-source 'static' is not supported by hatch-cpp; use bundled, external, or direct.")
+            linked = {name for library in self.libraries for name in library.depends}
+            self._external_libraries = {}
+            self._skipped_libraries = set() if source == "bundled" else linked
             for library_index, library in enumerate(self.libraries):
+                if library.name in self._skipped_libraries:
+                    if source == "external":
+                        self._external_libraries[library.name] = self._resolve_external(library)
+                    continue
+                library, dependencies = self._consumer(library, source)
                 compile_flags = self.platform.get_compile_flags(library, self.build_type)
-                link_flags = self.platform.get_link_flags(library, self.build_type)
+                link_flags = self.platform.get_link_flags(library, self.build_type, dependencies)
                 compiler = self.platform.cc if library.language == "c" else self.platform.cxx
                 if self.platform.platform == "emscripten":
                     objects = []
@@ -117,6 +213,9 @@ def generate(self):
     def execute(self):
         if self.platform.platform == "emscripten" and "vanilla" in self._active_toolchains:
             Path("build/hatch-cpp").mkdir(parents=True, exist_ok=True)
+        if "vanilla" in self._active_toolchains:
+            for library in self.built_libraries():
+                Path(self._library_output(library)).parent.mkdir(parents=True, exist_ok=True)
         for command in self.commands:
             ret = system_call(command)
             if ret != 0:
@@ -127,3 +226,91 @@ def cleanup(self):
         if self.platform.platform == "win32":
             for temp_obj in Path(".").glob("*.obj"):
                 temp_obj.unlink()
+
+    def _library_output(self, library: HatchCppLibrary) -> str:
+        return library.get_qualified_name(self.platform.platform)
+
+    def _artifact_path(self, artifact: HatchCppArtifact, packaged: list[str]) -> str:
+        if artifact.path:
+            return artifact.path
+        library = next((library for library in self.libraries if library.name == artifact.name), None)
+        if library is not None:
+            return self._library_output(library)
+        filename = shared_library_name(Path(artifact.name).name, platform=self.platform.platform)
+        matches = [path for path in packaged if Path(path).name == filename]
+        if len(matches) != 1:
+            found = ", ".join(matches) if matches else "none"
+            raise FileNotFoundError(f"Artifact '{artifact.name}' needs a path: expected one packaged '{filename}', found {found}.")
+        return matches[0]
+
+    def validation_targets(self, packaged: list[str]) -> list[tuple[str, LibraryValidationConfig, str]]:
+        """Libraries with validation fields set, plus every `artifacts` entry, as (label, config, project-relative path)."""
+        targets: list[tuple[str, LibraryValidationConfig, str]] = []
+        for library in self.libraries:
+            if not _has_validation(library):
+                continue
+            if library.name in self._external_libraries:
+                targets.append((library.name, library, str(self._external_libraries[library.name].library)))
+            elif library.name not in self._skipped_libraries:
+                targets.append((library.name, library, self._library_output(library)))
+        targets.extend((artifact.name, artifact, self._artifact_path(artifact, packaged)) for artifact in self.artifacts)
+        return targets
+
+    def validate_artifacts(self, root: Path, packaged: list[str]) -> None:
+        """Check headers, ABI strings, and exports of each validation target, then run its validation commands."""
+        for label, config, relative in self.validation_targets(packaged):
+            library = root / relative
+            if not library.is_file():
+                raise FileNotFoundError(f"Artifact '{label}' library was not built: {relative}")
+            headers = check_headers(config.expected_headers, config.expected_abi_strings, base=root, label=label)
+            expected = list(dict.fromkeys([*config.expected_symbols, *prefixed_header_symbols(headers, config.header_symbol_prefix, label=label)]))
+            check_exports(library, expected=expected, load=config.validate_artifact, label=label, display_path=relative)
+            tokens = {"name": label, "path": str(library), "header": str(headers[0]) if headers else ""}
+            for command in config.validation_commands:
+                env = {**environ, **{key: value.format(**tokens) for key, value in command.env.items()}}
+                env.update({"HATCH_CPP_ARTIFACT_NAME": label, "HATCH_CPP_ARTIFACT_PATH": str(library)})
+                cwd = root / command.working_directory if command.working_directory else root
+                run_validation_command([value.format(**tokens) for value in command.command], cwd=cwd, env=env, label=label)
+
+    def write_manifest(self, root: Path, packaged: list[str]) -> str | None:
+        """Write the manifest of packaged binaries under the project root and return its relative path."""
+        if not self.artifact_manifest:
+            return None
+        assert self.artifact_manifest_destination is not None
+        records = [
+            {
+                "name": Path(path).name,
+                "role": _artifact_role(path, self.platform.platform),
+                "platform": self.platform.platform,
+                "profile": self.build_type,
+                "destination": Path(path).as_posix(),
+            }
+            for path in sorted(packaged)
+        ]
+        write_artifact_manifest(root / self.artifact_manifest_destination, module=self.name or "", records=records)
+        return self.artifact_manifest_destination
+
+
+def _has_validation(config: LibraryValidationConfig) -> bool:
+    return bool(
+        config.expected_symbols
+        or config.expected_headers
+        or config.expected_abi_strings
+        or config.header_symbol_prefix
+        or config.validate_artifact
+        or config.validation_commands
+    )
+
+
+def _artifact_role(path: str, platform: str) -> str:
+    """Classify a packaged binary: a Python extension module or a plain shared library."""
+    filename = Path(path).name
+    if filename.endswith(".pyd"):
+        return "python-extension"
+    if platform != "win32" and filename.startswith("lib"):
+        return "library"
+    if any(filename.endswith(suffix) for suffix in EXTENSION_SUFFIXES if suffix != ".so") or ".abi3." in filename:
+        return "python-extension"
+    if filename.endswith(".so") and platform in ("linux", "darwin", "emscripten"):
+        return "python-extension"
+    return "library"

hatch_cpp/plugin.py

@@ -88,9 +88,12 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None:
 
         if build_plan.libraries:
             # force include libraries
-            for library in build_plan.libraries:
+            for library in build_plan.built_libraries():
                 name = library.get_qualified_name(build_plan.platform.platform)
                 build_data["force_include"][name] = name
+                import_library = Path(name).with_suffix(".lib")
+                if build_plan.platform.platform == "win32" and library.binding == "generic" and import_library.is_file():
+                    build_data["force_include"][import_library.as_posix()] = import_library.as_posix()
 
             build_data["pure_python"] = False
             machine = platform_machine()
@@ -101,7 +104,7 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None:
                 machine,
                 version_major,
                 version_minor,
-                all(lib.py_limited_api for lib in build_plan.libraries),
+                all(lib.py_limited_api for lib in build_plan.built_libraries()),
             )
         else:
             assert build_plan.cmake is not None
@@ -120,5 +123,11 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None:
                 if path.suffix in (".pyd", ".dll", ".so", ".dylib"):
                     build_data["force_include"][str(path)] = str(path)
 
+        packaged = list(build_data["force_include"])
+        build_plan.validate_artifacts(Path(self.root), packaged)
+        manifest = build_plan.write_manifest(Path(self.root), packaged)
+        if manifest is not None:
+            build_data["force_include"][manifest] = manifest
+
         for path in build_data["force_include"]:
             self._logger.info(f"Force include: {path}")

hatch_cpp/tests/test_project_c_abi_library/cpp/demo.c

@@ -0,0 +1,5 @@
+#include "demo/demo.h"
+
+int demo_abi_version(void) { return DEMO_ABI_VERSION; }
+
+int demo_answer(void) { return 42; }

hatch_cpp/tests/test_project_c_abi_library/cpp/extension.c

@@ -0,0 +1,11 @@
+#include <Python.h>
+
+#include "demo/demo.h"
+
+static PyObject *answer(PyObject *self, PyObject *args) { return PyLong_FromLong(demo_answer()); }
+
+static PyMethodDef methods[] = {{"answer", answer, METH_NOARGS, NULL}, {NULL, NULL, 0, NULL}};
+
+static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, "extension", NULL, -1, methods};
+
+PyMODINIT_FUNC PyInit_extension(void) { return PyModule_Create(&module); }