From 9e14a1d1ab53b51f36b7019105dbaaa499eecd74 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Thu, 13 Nov 2025 11:10:45 +0100 Subject: [PATCH 01/15] feat: add exportLibs feature to python-gardenlinux-lib --- pyproject.toml | 1 + src/gardenlinux/export_libs/__main__.py | 58 +++++++++++ src/gardenlinux/export_libs/exporter.py | 132 ++++++++++++++++++++++++ 3 files changed, 191 insertions(+) create mode 100644 src/gardenlinux/export_libs/__main__.py create mode 100644 src/gardenlinux/export_libs/exporter.py diff --git a/pyproject.toml b/pyproject.toml index d05accef..3a74d892 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ gl-flavors-parse = "gardenlinux.flavors.__main__:main" gl-oci = "gardenlinux.oci.__main__:main" gl-s3 = "gardenlinux.s3.__main__:main" gl-gh = "gardenlinux.github.__main__:main" +gl-python-exportlibs = "gardenlinux.export_libs.__main__:main" [tool.pytest.ini_options] pythonpath = ["src"] diff --git a/src/gardenlinux/export_libs/__main__.py b/src/gardenlinux/export_libs/__main__.py new file mode 100644 index 00000000..58e8a0da --- /dev/null +++ b/src/gardenlinux/export_libs/__main__.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +gl-python-exportlibs main entrypoint +""" + +from argparse import ArgumentParser + +from .exporter import _get_default_package_dir, export + +_ARGS_TYPE_ALLOWED = [ + "copy", +] + + +def parse_args(): + """ + Parses arguments used for main() + + :return: (object) Parsed argparse.ArgumentParser namespace + :since: TODO + """ + + parser = ArgumentParser( + description="Export shared libraries required by installed pip packages to a portable directory" + ) + + parser.add_argument("type", choices=_ARGS_TYPE_ALLOWED, default=None) + parser.add_argument( + "--output-dir", + default="/required_libs", + help="Directory containing the shared libraries.", + ) + parser.add_argument( + "--package-dir", + default=_get_default_package_dir(), + help="Path of the generated output", + ) + + return parser.parse_args() + + +def main(): + """ + gl-python-exportlibs main() + + :since: TODO + """ + + args = parse_args() + + export(output_dir=args.output_dir, package_dir=args.package_dir) + + +if __name__ == "__main__": + # Create a null logger as default + main() diff --git a/src/gardenlinux/export_libs/exporter.py b/src/gardenlinux/export_libs/exporter.py new file mode 100644 index 00000000..56fe8b52 --- /dev/null +++ b/src/gardenlinux/export_libs/exporter.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- + +import os +import re +import shutil +import subprocess +from os import PathLike + +from ..logger import LoggerSetup + +# Parses dependencies from ld output +parse_output = re.compile("(?:.*=>)?\\s*(/\\S*).*\n") +# Remove leading / +remove_root = re.compile("^/") + + +# Check for ELF header +def _isElf(path: str) -> bool: + """ + Checks if a file is an ELF by looking for the ELF header. + + :param path: Path to file + + :return: (bool) If the file found at path is an ELF + :since: TODO + """ + + with open(path, "rb") as f: + return f.read(4) == b"\x7f\x45\x4c\x46" + + +def _getInterpreter(path: str, logger) -> str: + """ + Returns the interpreter of an ELF. Supported architectures: x86_64, aarch64, i686. + + :param path: Path to file + :param logger: Logger to log errors + + :return: (str) Path of the interpreter + :since: TODO + """ + + with open(path, "rb") as f: + head = f.read(19) + + if head[5] == 1: + arch = head[17:] + elif head[5] == 2: + arch = head[17:][::-1] + else: + logger.error( + f"Error: Unknown endianess value for {path}: expected 1 or 2, but was {head[5]}" + ) + exit(1) + + if arch == b"\x00\xb7": # 00b7: aarch64 + return "/lib/ld-linux-aarch64.so.1" + elif arch == b"\x00\x3e": # 003e: x86_64 + return "/lib64/ld-linux-x86-64.so.2" + elif arch == b"\x00\x03": # 0003: i686 + return "/lib/ld-linux.so.2" + else: + logger.error( + f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}" + ) + exit(1) + + +def _get_default_package_dir() -> str: + """ + Finds the default site-packages or dist-packages directory of the default python3 environment + + :return: (str) Path to directory + :since: TODO + """ + + # Needs to escape the virtual environment python-gardenlinx-lib is running in + out = subprocess.run( + ["/bin/sh", "-c", 'python3 -c "import site; print(site.getsitepackages()[0])"'], + stdout=subprocess.PIPE, + ) + return out.stdout.decode().strip() + + +def export( + output_dir: str | PathLike[str] = "/required_libs", + package_dir: str | PathLike[str] = None, + logger=None, +) -> None: + """ + Identifies shared library dependencies of `package_dir` and copies them to `output_dir`. + + :param output_dir: Path to output_dir + :param package_dir: Path to package_dir + :param logger: Logger to log errors + + :since: TODO + """ + + if not package_dir: + package_dir = _get_default_package_dir() + if logger is None or not logger.hasHandlers(): + logger = LoggerSetup.get_logger("gardenlinux.export_libs") + # Collect ld dependencies for installed pip packages + dependencies = set() + for root, dirs, files in os.walk(package_dir): + for file in files: + path = f"{root}/{file}" + if not os.path.islink(path) and _isElf(path): + out = subprocess.run( + [_getInterpreter(path, logger), "--inhibit-cache", "--list", path], + stdout=subprocess.PIPE, + ) + for dependency in parse_output.findall(out.stdout.decode()): + dependencies.add(os.path.realpath(dependency)) + + # Copy dependencies into output_dir folder + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + + for dependency in dependencies: + path = os.path.join(output_dir, remove_root.sub("", dependency)) + os.makedirs(os.path.dirname(path), exist_ok=True) + shutil.copy2(dependency, path) + + # Reset timestamps of the parent directories + if len(dependencies) > 0: + mtime = int(os.stat(dependencies.pop()).st_mtime) + os.utime(output_dir, (mtime, mtime)) + for root, dirs, _ in os.walk(output_dir): + for dir in dirs: + os.utime(f"{root}/{dir}", (mtime, mtime)) From 60c71a422a81e587bb157822071c080358a55629 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Tue, 18 Nov 2025 09:30:20 +0100 Subject: [PATCH 02/15] move export-libs to gardenlinux.build and use pyelftools to retrieve interpreter --- .gitignore | 1 + poetry.lock | 14 +++++- pyproject.toml | 3 +- .../{export_libs => build}/__main__.py | 12 +++-- .../{export_libs => build}/exporter.py | 50 +++++++++---------- src/gardenlinux/s3/__main__.py | 4 +- 6 files changed, 51 insertions(+), 33 deletions(-) rename src/gardenlinux/{export_libs => build}/__main__.py (79%) rename src/gardenlinux/{export_libs => build}/exporter.py (77%) diff --git a/.gitignore b/.gitignore index 5569953a..2553cf14 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ __pycache__/ # Distribution / packaging .Python build/ +!/src/gardenlinux/build develop-eggs/ dist/ downloads/ diff --git a/poetry.lock b/poetry.lock index fdd36388..012f9feb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1094,6 +1094,18 @@ files = [ ] markers = {main = "implementation_name != \"PyPy\"", dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\""} +[[package]] +name = "pyelftools" +version = "0.32" +description = "Library for analyzing ELF files and DWARF debugging information" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "pyelftools-0.32-py3-none-any.whl", hash = "sha256:013df952a006db5e138b1edf6d8a68ecc50630adbd0d83a2d41e7f846163d738"}, + {file = "pyelftools-0.32.tar.gz", hash = "sha256:6de90ee7b8263e740c8715a925382d4099b354f29ac48ea40d840cf7aa14ace5"}, +] + [[package]] name = "pygit2" version = "1.19.0" @@ -1901,4 +1913,4 @@ test = ["pytest", "pytest-cov"] [metadata] lock-version = "2.1" python-versions = ">=3.13" -content-hash = "ea1c3fd1bb706d5202222d76cea0a99b8f1091ea3d3b25532c53f0e48276d8ed" +content-hash = "9e9d65ab3a2ba41e4a109299cfb2d2ea04b94c647471f020922148f2e6a906c8" diff --git a/pyproject.toml b/pyproject.toml index 3a74d892..26238a96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ cryptography = "^46.0.1" jsonschema = "^4.25.1" networkx = "^3.5" oras = "^0.2.38" +pyelftools = "^0.32" pygit2 = "^1.19.0" pygments = "^2.19.2" PyYAML = "^6.0.2" @@ -42,7 +43,7 @@ gl-flavors-parse = "gardenlinux.flavors.__main__:main" gl-oci = "gardenlinux.oci.__main__:main" gl-s3 = "gardenlinux.s3.__main__:main" gl-gh = "gardenlinux.github.__main__:main" -gl-python-exportlibs = "gardenlinux.export_libs.__main__:main" +gl-build = "gardenlinux.build.__main__:main" [tool.pytest.ini_options] pythonpath = ["src"] diff --git a/src/gardenlinux/export_libs/__main__.py b/src/gardenlinux/build/__main__.py similarity index 79% rename from src/gardenlinux/export_libs/__main__.py rename to src/gardenlinux/build/__main__.py index 58e8a0da..fc7c7d2b 100644 --- a/src/gardenlinux/export_libs/__main__.py +++ b/src/gardenlinux/build/__main__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ -gl-python-exportlibs main entrypoint +gl-build main entrypoint """ from argparse import ArgumentParser @@ -10,7 +10,7 @@ from .exporter import _get_default_package_dir, export _ARGS_TYPE_ALLOWED = [ - "copy", + "export-python-libs", ] @@ -43,14 +43,18 @@ def parse_args(): def main(): """ - gl-python-exportlibs main() + gl-gl-build main() :since: TODO """ args = parse_args() - export(output_dir=args.output_dir, package_dir=args.package_dir) + match args.type: + case "export-python-libs": + export(output_dir=args.output_dir, package_dir=args.package_dir) + case _: + raise NotImplementedError if __name__ == "__main__": diff --git a/src/gardenlinux/export_libs/exporter.py b/src/gardenlinux/build/exporter.py similarity index 77% rename from src/gardenlinux/export_libs/exporter.py rename to src/gardenlinux/build/exporter.py index 56fe8b52..ecf840a4 100644 --- a/src/gardenlinux/export_libs/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -1,11 +1,14 @@ # -*- coding: utf-8 -*- import os +import pathlib import re import shutil import subprocess from os import PathLike +from elftools.elf.elffile import ELFFile + from ..logger import LoggerSetup # Parses dependencies from ld output @@ -41,32 +44,27 @@ def _getInterpreter(path: str, logger) -> str: """ with open(path, "rb") as f: - head = f.read(19) + elf = ELFFile(f) + interp = elf.get_section_by_name(".interp") - if head[5] == 1: - arch = head[17:] - elif head[5] == 2: - arch = head[17:][::-1] - else: - logger.error( - f"Error: Unknown endianess value for {path}: expected 1 or 2, but was {head[5]}" - ) - exit(1) - - if arch == b"\x00\xb7": # 00b7: aarch64 - return "/lib/ld-linux-aarch64.so.1" - elif arch == b"\x00\x3e": # 003e: x86_64 - return "/lib64/ld-linux-x86-64.so.2" - elif arch == b"\x00\x03": # 0003: i686 - return "/lib/ld-linux.so.2" + if interp: + return interp.data().split(b"\x00")[0].decode() else: - logger.error( - f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}" - ) - exit(1) - - -def _get_default_package_dir() -> str: + match elf.header["e_machine"]: + case "EM_AARCH64": + return "/lib/ld-linux-aarch64.so.1" + case "EM_386": + return "/lib/ld-linux.so.2" + case "EM_X86_64": + return "/lib64/ld-linux-x86-64.so.2" + case arch: + logger.error( + f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}" + ) + exit(1) + + +def _get_default_package_dir() -> PathLike[str]: """ Finds the default site-packages or dist-packages directory of the default python3 environment @@ -79,12 +77,12 @@ def _get_default_package_dir() -> str: ["/bin/sh", "-c", 'python3 -c "import site; print(site.getsitepackages()[0])"'], stdout=subprocess.PIPE, ) - return out.stdout.decode().strip() + return pathlib.Path(out.stdout.decode().strip()) def export( output_dir: str | PathLike[str] = "/required_libs", - package_dir: str | PathLike[str] = None, + package_dir: str | PathLike[str] | None = None, logger=None, ) -> None: """ diff --git a/src/gardenlinux/s3/__main__.py b/src/gardenlinux/s3/__main__.py index eed51403..3d82b972 100644 --- a/src/gardenlinux/s3/__main__.py +++ b/src/gardenlinux/s3/__main__.py @@ -36,4 +36,6 @@ def main() -> None: if args.action == "download-artifacts-from-bucket": S3Artifacts(args.bucket).download_to_directory(args.cname, args.path) elif args.action == "upload-artifacts-to-bucket": - S3Artifacts(args.bucket).upload_from_directory(args.cname, args.path, dry_run=args.dry_run) + S3Artifacts(args.bucket).upload_from_directory( + args.cname, args.path, dry_run=args.dry_run + ) From fd226377b69828446c9d7ab3bf5104754ced63f1 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Tue, 18 Nov 2025 09:58:17 +0100 Subject: [PATCH 03/15] revert formatting change of unrelated feature --- src/gardenlinux/s3/__main__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/gardenlinux/s3/__main__.py b/src/gardenlinux/s3/__main__.py index 3d82b972..eed51403 100644 --- a/src/gardenlinux/s3/__main__.py +++ b/src/gardenlinux/s3/__main__.py @@ -36,6 +36,4 @@ def main() -> None: if args.action == "download-artifacts-from-bucket": S3Artifacts(args.bucket).download_to_directory(args.cname, args.path) elif args.action == "upload-artifacts-to-bucket": - S3Artifacts(args.bucket).upload_from_directory( - args.cname, args.path, dry_run=args.dry_run - ) + S3Artifacts(args.bucket).upload_from_directory(args.cname, args.path, dry_run=args.dry_run) From 2e2f289ac79096437faa8bff7db2c22cd74414c4 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Wed, 19 Nov 2025 11:30:55 +0100 Subject: [PATCH 04/15] Identify ELF files using pyelftools --- src/gardenlinux/build/exporter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/gardenlinux/build/exporter.py b/src/gardenlinux/build/exporter.py index ecf840a4..dcb956d4 100644 --- a/src/gardenlinux/build/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -7,6 +7,7 @@ import subprocess from os import PathLike +from elftools.common.exceptions import ELFError from elftools.elf.elffile import ELFFile from ..logger import LoggerSetup @@ -29,7 +30,11 @@ def _isElf(path: str) -> bool: """ with open(path, "rb") as f: - return f.read(4) == b"\x7f\x45\x4c\x46" + try: + ELFFile(f) + return True + except ELFError: + return False def _getInterpreter(path: str, logger) -> str: From e5cab984aa1fac9e8284d219f126773413df7137 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Wed, 19 Nov 2025 13:09:58 +0100 Subject: [PATCH 05/15] use pathlib.Path and retrieve primary python interpreter directly from script --- src/gardenlinux/build/__main__.py | 3 +- src/gardenlinux/build/exporter.py | 46 +++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/gardenlinux/build/__main__.py b/src/gardenlinux/build/__main__.py index fc7c7d2b..f86dcd90 100644 --- a/src/gardenlinux/build/__main__.py +++ b/src/gardenlinux/build/__main__.py @@ -5,6 +5,7 @@ gl-build main entrypoint """ +import pathlib from argparse import ArgumentParser from .exporter import _get_default_package_dir, export @@ -52,7 +53,7 @@ def main(): match args.type: case "export-python-libs": - export(output_dir=args.output_dir, package_dir=args.package_dir) + export(output_dir=pathlib.Path(args.output_dir), package_dir=pathlib.Path(args.package_dir)) case _: raise NotImplementedError diff --git a/src/gardenlinux/build/exporter.py b/src/gardenlinux/build/exporter.py index dcb956d4..d60b6aee 100644 --- a/src/gardenlinux/build/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -19,7 +19,7 @@ # Check for ELF header -def _isElf(path: str) -> bool: +def _isElf(path: str | PathLike[str]) -> bool: """ Checks if a file is an ELF by looking for the ELF header. @@ -37,7 +37,7 @@ def _isElf(path: str) -> bool: return False -def _getInterpreter(path: str, logger) -> str: +def _getInterpreter(path: str | PathLike[str], logger) -> PathLike[str]: """ Returns the interpreter of an ELF. Supported architectures: x86_64, aarch64, i686. @@ -53,36 +53,48 @@ def _getInterpreter(path: str, logger) -> str: interp = elf.get_section_by_name(".interp") if interp: - return interp.data().split(b"\x00")[0].decode() + return pathlib.Path(interp.data().split(b"\x00")[0].decode()) else: match elf.header["e_machine"]: case "EM_AARCH64": - return "/lib/ld-linux-aarch64.so.1" + return pathlib.Path("/lib/ld-linux-aarch64.so.1") case "EM_386": - return "/lib/ld-linux.so.2" + return pathlib.Path("/lib/ld-linux.so.2") case "EM_X86_64": - return "/lib64/ld-linux-x86-64.so.2" + return pathlib.Path("/lib64/ld-linux-x86-64.so.2") case arch: logger.error( f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}" ) exit(1) +def _get_python_from_path() -> str | None: + interpreter = None + for dir in os.environ["PATH"].split(":"): + binary = os.path.join(dir, "python3") + if os.path.isfile(binary): + interpreter = binary + break + return interpreter -def _get_default_package_dir() -> PathLike[str]: +def _get_default_package_dir() -> PathLike[str] | None: """ Finds the default site-packages or dist-packages directory of the default python3 environment :return: (str) Path to directory :since: TODO """ - + # Needs to escape the virtual environment python-gardenlinx-lib is running in - out = subprocess.run( - ["/bin/sh", "-c", 'python3 -c "import site; print(site.getsitepackages()[0])"'], - stdout=subprocess.PIPE, - ) - return pathlib.Path(out.stdout.decode().strip()) + interpreter = _get_python_from_path() + if interpreter: + out = subprocess.run( + [interpreter, "-c", "import site; print(site.getsitepackages()[0])"], + stdout=subprocess.PIPE, + ) + return pathlib.Path(out.stdout.decode().strip()) + else: + return None def export( @@ -102,13 +114,19 @@ def export( if not package_dir: package_dir = _get_default_package_dir() + if not package_dir: + logger.error( + f"Error: Couldn't identify a default python package directory. Please specifiy one using the --package-dir option. Use -h for more information." + ) + exit(1) + if logger is None or not logger.hasHandlers(): logger = LoggerSetup.get_logger("gardenlinux.export_libs") # Collect ld dependencies for installed pip packages dependencies = set() for root, dirs, files in os.walk(package_dir): for file in files: - path = f"{root}/{file}" + path = os.path.join(root, file) if not os.path.islink(path) and _isElf(path): out = subprocess.run( [_getInterpreter(path, logger), "--inhibit-cache", "--list", path], From ba84e472352b2ef6dc4e1c3dc60f2964b8f7d7a2 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Wed, 19 Nov 2025 13:39:06 +0100 Subject: [PATCH 06/15] replace os.path functions with pathlib functions --- src/gardenlinux/build/exporter.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/gardenlinux/build/exporter.py b/src/gardenlinux/build/exporter.py index d60b6aee..d29ca225 100644 --- a/src/gardenlinux/build/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -37,7 +37,7 @@ def _isElf(path: str | PathLike[str]) -> bool: return False -def _getInterpreter(path: str | PathLike[str], logger) -> PathLike[str]: +def _getInterpreter(path: str | PathLike[str], logger) -> pathlib.Path: """ Returns the interpreter of an ELF. Supported architectures: x86_64, aarch64, i686. @@ -68,16 +68,16 @@ def _getInterpreter(path: str | PathLike[str], logger) -> PathLike[str]: ) exit(1) -def _get_python_from_path() -> str | None: +def _get_python_from_path() -> pathlib.Path | None: interpreter = None for dir in os.environ["PATH"].split(":"): - binary = os.path.join(dir, "python3") - if os.path.isfile(binary): + binary = pathlib.Path(dir).joinpath("python3") + if binary.is_file(): interpreter = binary break return interpreter -def _get_default_package_dir() -> PathLike[str] | None: +def _get_default_package_dir() -> pathlib.Path | None: """ Finds the default site-packages or dist-packages directory of the default python3 environment @@ -119,15 +119,18 @@ def export( f"Error: Couldn't identify a default python package directory. Please specifiy one using the --package-dir option. Use -h for more information." ) exit(1) + else: + package_dir = pathlib.Path(package_dir) + output_dir = pathlib.Path(output_dir) if logger is None or not logger.hasHandlers(): logger = LoggerSetup.get_logger("gardenlinux.export_libs") # Collect ld dependencies for installed pip packages dependencies = set() - for root, dirs, files in os.walk(package_dir): + for root, dirs, files in package_dir.walk(): for file in files: - path = os.path.join(root, file) - if not os.path.islink(path) and _isElf(path): + path = root.joinpath(file) + if not path.is_symlink() and _isElf(path): out = subprocess.run( [_getInterpreter(path, logger), "--inhibit-cache", "--list", path], stdout=subprocess.PIPE, @@ -136,18 +139,18 @@ def export( dependencies.add(os.path.realpath(dependency)) # Copy dependencies into output_dir folder - if not os.path.isdir(output_dir): - os.mkdir(output_dir) + if not output_dir.is_dir(): + output_dir.mkdir() for dependency in dependencies: - path = os.path.join(output_dir, remove_root.sub("", dependency)) - os.makedirs(os.path.dirname(path), exist_ok=True) + path = output_dir.joinpath(remove_root.sub("", dependency)) + path.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(dependency, path) # Reset timestamps of the parent directories if len(dependencies) > 0: mtime = int(os.stat(dependencies.pop()).st_mtime) os.utime(output_dir, (mtime, mtime)) - for root, dirs, _ in os.walk(output_dir): + for root, dirs, _ in output_dir.walk(): for dir in dirs: - os.utime(f"{root}/{dir}", (mtime, mtime)) + os.utime(root.joinpath(dir), (mtime, mtime)) From c1e63771c5d0d8e175d123ab9c9a9a3bdeb8bf44 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Wed, 19 Nov 2025 13:42:41 +0100 Subject: [PATCH 07/15] updpate .gitignore to only exclude build folder at root level --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 2553cf14..5894f4f8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,7 @@ __pycache__/ # Distribution / packaging .Python -build/ -!/src/gardenlinux/build +/build/ develop-eggs/ dist/ downloads/ From bd780c6bf12497fe1c3900ef5f2dcceccbabd864 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Thu, 20 Nov 2025 10:05:44 +0100 Subject: [PATCH 08/15] replace logger infos with exceptions --- src/gardenlinux/build/exporter.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/src/gardenlinux/build/exporter.py b/src/gardenlinux/build/exporter.py index d29ca225..c1ea5bfb 100644 --- a/src/gardenlinux/build/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -37,12 +37,11 @@ def _isElf(path: str | PathLike[str]) -> bool: return False -def _getInterpreter(path: str | PathLike[str], logger) -> pathlib.Path: +def _getInterpreter(path: str | PathLike[str]) -> pathlib.Path: """ Returns the interpreter of an ELF. Supported architectures: x86_64, aarch64, i686. :param path: Path to file - :param logger: Logger to log errors :return: (str) Path of the interpreter :since: TODO @@ -63,15 +62,12 @@ def _getInterpreter(path: str | PathLike[str], logger) -> pathlib.Path: case "EM_X86_64": return pathlib.Path("/lib64/ld-linux-x86-64.so.2") case arch: - logger.error( - f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}" - ) - exit(1) + raise RuntimeError(f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}") def _get_python_from_path() -> pathlib.Path | None: interpreter = None for dir in os.environ["PATH"].split(":"): - binary = pathlib.Path(dir).joinpath("python3") + binary = pathlib.Path(dir, "python3").joinpath("python3") if binary.is_file(): interpreter = binary break @@ -99,15 +95,13 @@ def _get_default_package_dir() -> pathlib.Path | None: def export( output_dir: str | PathLike[str] = "/required_libs", - package_dir: str | PathLike[str] | None = None, - logger=None, + package_dir: str | PathLike[str] | None = None ) -> None: """ Identifies shared library dependencies of `package_dir` and copies them to `output_dir`. :param output_dir: Path to output_dir :param package_dir: Path to package_dir - :param logger: Logger to log errors :since: TODO """ @@ -115,16 +109,11 @@ def export( if not package_dir: package_dir = _get_default_package_dir() if not package_dir: - logger.error( - f"Error: Couldn't identify a default python package directory. Please specifiy one using the --package-dir option. Use -h for more information." - ) - exit(1) + raise RuntimeError(f"Error: Couldn't identify a default python package directory. Please specifiy one using the --package-dir option. Use -h for more information.") else: package_dir = pathlib.Path(package_dir) output_dir = pathlib.Path(output_dir) - if logger is None or not logger.hasHandlers(): - logger = LoggerSetup.get_logger("gardenlinux.export_libs") # Collect ld dependencies for installed pip packages dependencies = set() for root, dirs, files in package_dir.walk(): @@ -132,7 +121,7 @@ def export( path = root.joinpath(file) if not path.is_symlink() and _isElf(path): out = subprocess.run( - [_getInterpreter(path, logger), "--inhibit-cache", "--list", path], + [_getInterpreter(path), "--inhibit-cache", "--list", path], stdout=subprocess.PIPE, ) for dependency in parse_output.findall(out.stdout.decode()): From cdc6983f971a8ef662959b46b98f22a835790306 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Thu, 20 Nov 2025 10:11:10 +0100 Subject: [PATCH 09/15] directly raise RuntimeError instead of returning None --- src/gardenlinux/build/exporter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/gardenlinux/build/exporter.py b/src/gardenlinux/build/exporter.py index c1ea5bfb..edaea261 100644 --- a/src/gardenlinux/build/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -90,7 +90,7 @@ def _get_default_package_dir() -> pathlib.Path | None: ) return pathlib.Path(out.stdout.decode().strip()) else: - return None + raise RuntimeError(f"Error: Couldn't identify a default python package directory. Please specifiy one using the --package-dir option. Use -h for more information.") def export( @@ -108,8 +108,6 @@ def export( if not package_dir: package_dir = _get_default_package_dir() - if not package_dir: - raise RuntimeError(f"Error: Couldn't identify a default python package directory. Please specifiy one using the --package-dir option. Use -h for more information.") else: package_dir = pathlib.Path(package_dir) output_dir = pathlib.Path(output_dir) From 38fa08c44d7713fee4b9d550ac634c5fe0aef3bf Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Thu, 20 Nov 2025 10:34:14 +0100 Subject: [PATCH 10/15] fix: remove duplicated python3 in path --- src/gardenlinux/build/exporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gardenlinux/build/exporter.py b/src/gardenlinux/build/exporter.py index edaea261..405fd025 100644 --- a/src/gardenlinux/build/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -67,7 +67,7 @@ def _getInterpreter(path: str | PathLike[str]) -> pathlib.Path: def _get_python_from_path() -> pathlib.Path | None: interpreter = None for dir in os.environ["PATH"].split(":"): - binary = pathlib.Path(dir, "python3").joinpath("python3") + binary = pathlib.Path(dir, "python3") if binary.is_file(): interpreter = binary break From 389bf5c2ec54a3cd94d14cb7d8645c1382b8587b Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Thu, 20 Nov 2025 10:36:00 +0100 Subject: [PATCH 11/15] insert 'since' value 1.0.0 --- src/gardenlinux/build/__main__.py | 4 ++-- src/gardenlinux/build/exporter.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/gardenlinux/build/__main__.py b/src/gardenlinux/build/__main__.py index f86dcd90..319bbe93 100644 --- a/src/gardenlinux/build/__main__.py +++ b/src/gardenlinux/build/__main__.py @@ -20,7 +20,7 @@ def parse_args(): Parses arguments used for main() :return: (object) Parsed argparse.ArgumentParser namespace - :since: TODO + :since: 1.0.0 """ parser = ArgumentParser( @@ -46,7 +46,7 @@ def main(): """ gl-gl-build main() - :since: TODO + :since: 1.0.0 """ args = parse_args() diff --git a/src/gardenlinux/build/exporter.py b/src/gardenlinux/build/exporter.py index 405fd025..8c44e7e4 100644 --- a/src/gardenlinux/build/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -26,7 +26,7 @@ def _isElf(path: str | PathLike[str]) -> bool: :param path: Path to file :return: (bool) If the file found at path is an ELF - :since: TODO + :since: 1.0.0 """ with open(path, "rb") as f: @@ -44,7 +44,7 @@ def _getInterpreter(path: str | PathLike[str]) -> pathlib.Path: :param path: Path to file :return: (str) Path of the interpreter - :since: TODO + :since: 1.0.0 """ with open(path, "rb") as f: @@ -78,7 +78,7 @@ def _get_default_package_dir() -> pathlib.Path | None: Finds the default site-packages or dist-packages directory of the default python3 environment :return: (str) Path to directory - :since: TODO + :since: 1.0.0 """ # Needs to escape the virtual environment python-gardenlinx-lib is running in @@ -103,7 +103,7 @@ def export( :param output_dir: Path to output_dir :param package_dir: Path to package_dir - :since: TODO + :since: 1.0.0 """ if not package_dir: From b3b1e8086e68b5cb87b100841084d5eac06ca253 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Mon, 24 Nov 2025 09:16:07 +0100 Subject: [PATCH 12/15] replace PATH readout with shutil.which() --- src/gardenlinux/build/exporter.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/gardenlinux/build/exporter.py b/src/gardenlinux/build/exporter.py index 8c44e7e4..148571af 100644 --- a/src/gardenlinux/build/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -63,16 +63,7 @@ def _getInterpreter(path: str | PathLike[str]) -> pathlib.Path: return pathlib.Path("/lib64/ld-linux-x86-64.so.2") case arch: raise RuntimeError(f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}") - -def _get_python_from_path() -> pathlib.Path | None: - interpreter = None - for dir in os.environ["PATH"].split(":"): - binary = pathlib.Path(dir, "python3") - if binary.is_file(): - interpreter = binary - break - return interpreter - + def _get_default_package_dir() -> pathlib.Path | None: """ Finds the default site-packages or dist-packages directory of the default python3 environment @@ -82,7 +73,7 @@ def _get_default_package_dir() -> pathlib.Path | None: """ # Needs to escape the virtual environment python-gardenlinx-lib is running in - interpreter = _get_python_from_path() + interpreter = shutil.which("python3") if interpreter: out = subprocess.run( [interpreter, "-c", "import site; print(site.getsitepackages()[0])"], From 7c11b65a7aaf42c6c1b7c2d48f6be0b94d12a937 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Tue, 25 Nov 2025 09:10:45 +0100 Subject: [PATCH 13/15] add test for exporter --- tests/build/__init__.py | 0 tests/build/test_exporter.py | 48 ++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 tests/build/__init__.py create mode 100644 tests/build/test_exporter.py diff --git a/tests/build/__init__.py b/tests/build/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/build/test_exporter.py b/tests/build/test_exporter.py new file mode 100644 index 00000000..069a4e11 --- /dev/null +++ b/tests/build/test_exporter.py @@ -0,0 +1,48 @@ +import pathlib +import shutil +import subprocess +import tempfile + +from gardenlinux.build.exporter import export + +arm64 = {'usr', 'usr/lib', 'usr/lib/aarch64-linux-gnu', 'usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1', 'usr/lib/aarch64-linux-gnu/libc.so.6', 'usr/lib/aarch64-linux-gnu/libpthread.so.0'} +amd64 = {'usr', 'usr/lib', 'usr/lib/x86_64-linux-gnu', 'usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2', 'usr/lib/x86_64-linux-gnu/libc.so.6', 'usr/lib/x86_64-linux-gnu/libpthread.so.0'} + +def test_requests_export(): + dpkg = shutil.which("dpkg") + assert dpkg + + out = subprocess.run([dpkg, "--print-architecture"], stdout=subprocess.PIPE) + assert out.returncode == 0 + arch = out.stdout.decode().strip() + + with tempfile.TemporaryDirectory() as target_dir: + target_dir = pathlib.Path(target_dir) + site_packages = target_dir.joinpath("site-packages") + required_libs = target_dir.joinpath("required_libs") + + site_packages.mkdir() + required_libs.mkdir() + + pip3 = shutil.which("pip3") + assert pip3 + + out = subprocess.run([pip3, "install", "--target", site_packages, "requests"]) + + assert out.returncode == 0 + + export(required_libs, site_packages) + + exported = set() + + for path in required_libs.rglob("*"): + exported.add(str(path.relative_to(required_libs))) + + if arch == "arm64": + expected = arm64 + elif arch == "amd64": + expected = amd64 + else: + raise NotImplementedError(f"Architecture {arch} not supported") + + assert exported == expected \ No newline at end of file From a219facdd89b8c5b070d99d0c16981778aa835a2 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Wed, 26 Nov 2025 10:45:22 +0100 Subject: [PATCH 14/15] enhance control flow --- src/gardenlinux/build/__main__.py | 5 ++++- src/gardenlinux/build/exporter.py | 30 ++++++++++++++++++------------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/gardenlinux/build/__main__.py b/src/gardenlinux/build/__main__.py index 319bbe93..dcb44b73 100644 --- a/src/gardenlinux/build/__main__.py +++ b/src/gardenlinux/build/__main__.py @@ -53,7 +53,10 @@ def main(): match args.type: case "export-python-libs": - export(output_dir=pathlib.Path(args.output_dir), package_dir=pathlib.Path(args.package_dir)) + export( + output_dir=pathlib.Path(args.output_dir), + package_dir=pathlib.Path(args.package_dir), + ) case _: raise NotImplementedError diff --git a/src/gardenlinux/build/exporter.py b/src/gardenlinux/build/exporter.py index 148571af..8f1beb9d 100644 --- a/src/gardenlinux/build/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -62,8 +62,11 @@ def _getInterpreter(path: str | PathLike[str]) -> pathlib.Path: case "EM_X86_64": return pathlib.Path("/lib64/ld-linux-x86-64.so.2") case arch: - raise RuntimeError(f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}") - + raise RuntimeError( + f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}" + ) + + def _get_default_package_dir() -> pathlib.Path | None: """ Finds the default site-packages or dist-packages directory of the default python3 environment @@ -71,22 +74,25 @@ def _get_default_package_dir() -> pathlib.Path | None: :return: (str) Path to directory :since: 1.0.0 """ - + # Needs to escape the virtual environment python-gardenlinx-lib is running in interpreter = shutil.which("python3") - if interpreter: - out = subprocess.run( - [interpreter, "-c", "import site; print(site.getsitepackages()[0])"], - stdout=subprocess.PIPE, + + if not interpreter: + raise RuntimeError( + f"Error: Couldn't identify a default python package directory. Please specifiy one using the --package-dir option. Use -h for more information." ) - return pathlib.Path(out.stdout.decode().strip()) - else: - raise RuntimeError(f"Error: Couldn't identify a default python package directory. Please specifiy one using the --package-dir option. Use -h for more information.") + + out = subprocess.run( + [interpreter, "-c", "import site; print(site.getsitepackages()[0])"], + stdout=subprocess.PIPE, + ) + return pathlib.Path(out.stdout.decode().strip()) def export( output_dir: str | PathLike[str] = "/required_libs", - package_dir: str | PathLike[str] | None = None + package_dir: str | PathLike[str] | None = None, ) -> None: """ Identifies shared library dependencies of `package_dir` and copies them to `output_dir`. @@ -102,7 +108,7 @@ def export( else: package_dir = pathlib.Path(package_dir) output_dir = pathlib.Path(output_dir) - + # Collect ld dependencies for installed pip packages dependencies = set() for root, dirs, files in package_dir.walk(): From 2418ffb6d316344bd957517474a1d44877963b59 Mon Sep 17 00:00:00 2001 From: Leon Kniffki Date: Thu, 27 Nov 2025 08:19:41 +0100 Subject: [PATCH 15/15] minor cleanups --- src/gardenlinux/build/exporter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/gardenlinux/build/exporter.py b/src/gardenlinux/build/exporter.py index 8f1beb9d..80c0ae6c 100644 --- a/src/gardenlinux/build/exporter.py +++ b/src/gardenlinux/build/exporter.py @@ -10,8 +10,6 @@ from elftools.common.exceptions import ELFError from elftools.elf.elffile import ELFFile -from ..logger import LoggerSetup - # Parses dependencies from ld output parse_output = re.compile("(?:.*=>)?\\s*(/\\S*).*\n") # Remove leading / @@ -67,7 +65,7 @@ def _getInterpreter(path: str | PathLike[str]) -> pathlib.Path: ) -def _get_default_package_dir() -> pathlib.Path | None: +def _get_default_package_dir() -> pathlib.Path: """ Finds the default site-packages or dist-packages directory of the default python3 environment