diff --git a/features/pythonDev/exec.config b/features/pythonDev/exec.config new file mode 100755 index 000000000..3f8a34f3c --- /dev/null +++ b/features/pythonDev/exec.config @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# Set up a virtual environment for exportLibs.py to keep the system-wide packages clean +# Otherwise, pyelftools would be copied to the bare-python container in the multistage build +cd root +mkdir venv +python3 -m venv venv + +# shellcheck source=/dev/null +source venv/bin/activate +pip install pyelftools +deactivate + +cd .. diff --git a/features/pythonDev/file.include/usr/bin/exportLibs.py b/features/pythonDev/file.include/usr/bin/exportLibs.py index c0cf78605..aaffbc242 100755 --- a/features/pythonDev/file.include/usr/bin/exportLibs.py +++ b/features/pythonDev/file.include/usr/bin/exportLibs.py @@ -1,70 +1,166 @@ -#!/bin/python3 +#!/root/venv/bin/python3 -import site import os -import subprocess -import sys import re import shutil +import subprocess +from argparse import ArgumentParser +from os import PathLike +from pathlib import Path + +from elftools.common.exceptions import ELFError +from elftools.elf.elffile import ELFFile # Parses dependencies from ld output parse_output = re.compile("(?:.*=>)?\\s*(/\\S*).*\n") +# Remove leading / +remove_root = re.compile("^/") + + +def parse_args(): + """ + Parses arguments used for main() + :return: (object) Parsed argparse.ArgumentParser namespace + """ + + parser = ArgumentParser( + description="Export shared libraries required by installed pip packages to a portable directory" + ) + + parser.add_argument( + "--output-dir", + default="/required_libs", + help="Directory containing the shared libraries.", + ) + parser.add_argument( + "--package-dir", + default=_get_default_package_dir(), + help="Path of the generated output", + ) + + return parser.parse_args() # Check for ELF header -def isElf(path: str) -> bool: +def _isElf(path: str | PathLike[str]) -> bool: + """ + Checks if a file is an ELF by looking for the ELF header. + + :param path: Path to file + + :return: (bool) If the file found at path is an ELF + """ + with open(path, "rb") as f: - return f.read(4) == b"\x7f\x45\x4c\x46" - + try: + ELFFile(f) + return True + except ELFError: + return False + + +def _getInterpreter(path: str | PathLike[str]) -> Path: + """ + Returns the interpreter of an ELF. Supported architectures: x86_64, aarch64, i686. + + :param path: Path to file + + :return: (str) Path of the interpreter + """ -def getInterpreter(path: str) -> str: with open(path, "rb") as f: - head = f.read(19) + elf = ELFFile(f) + interp = elf.get_section_by_name(".interp") - if head[5] == 1: - arch = head[17:] - elif head[5] == 2: - arch = head[17:][::-1] - else: - print(f"Error: Unknown endianess value for {path}: expected 1 or 2, but was {head[5]}", file=sys.stderr) - exit(1) - - if arch == b"\x00\xb7": # 00b7: aarch64 - return "/lib/ld-linux-aarch64.so.1" - elif arch == b"\x00\x3e": # 003e: x86_64 - return "/lib64/ld-linux-x86-64.so.2" - elif arch == b"\x00\x03": # 0003: i686 - return "/lib/ld-linux.so.2" + if interp: + return Path(interp.data().split(b"\x00")[0].decode()) else: - print(f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}", file=sys.stderr) - exit(1) - - -package_dir = site.getsitepackages()[0] - -# Collect ld dependencies for installed pip packages -dependencies = set() -for root, dirs, files in os.walk(package_dir): - for file in files: - path = f"{root}/{file}" - if not os.path.islink(path) and isElf(path): - out = subprocess.run([getInterpreter(path),"--inhibit-cache", "--list", path], stdout=subprocess.PIPE) - for dependency in parse_output.findall(out.stdout.decode()): - dependencies.add(os.path.realpath(dependency)) - - -# Copy dependencies into required_libs folder -if not os.path.isdir("/required_libs"): - os.mkdir("/required_libs") - -for dependency in dependencies: - os.makedirs(f"/required_libs{os.path.dirname(dependency)}", exist_ok=True) - shutil.copy2(dependency, f"/required_libs{dependency}") - -# Reset timestamps of the parent directories -if len(dependencies) > 0: - mtime = int(os.stat(dependencies.pop()).st_mtime) - os.utime("/required_libs", (mtime, mtime)) - for root, dirs, files in os.walk("/required_libs"): - for dir in dirs: - os.utime(f"{root}/{dir}", (mtime, mtime)) + match elf.header["e_machine"]: + case "EM_AARCH64": + return Path("/lib/ld-linux-aarch64.so.1") + case "EM_386": + return Path("/lib/ld-linux.so.2") + case "EM_X86_64": + return Path("/lib64/ld-linux-x86-64.so.2") + case arch: + raise RuntimeError( + f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}" + ) + + +def _get_default_package_dir() -> Path: + """ + Finds the default site-packages or dist-packages directory of the default python3 environment + + :return: (str) Path to directory + """ + + # Needs to escape the virtual environment python-gardenlinx-lib is running in + interpreter = shutil.which("python3") + + if not interpreter: + raise RuntimeError( + f"Error: Couldn't identify a default python package directory. Please specifiy one using the --package-dir option. Use -h for more information." + ) + + out = subprocess.run( + [interpreter, "-c", "import site; print(site.getsitepackages()[0])"], + stdout=subprocess.PIPE, + ) + return Path(out.stdout.decode().strip()) + + +def export( + output_dir: str | PathLike[str] = "/required_libs", + package_dir: str | PathLike[str] | None = None, +) -> None: + """ + Identifies shared library dependencies of `package_dir` and copies them to `output_dir`. + + :param output_dir: Path to output_dir + :param package_dir: Path to package_dir + """ + + if not package_dir: + package_dir = _get_default_package_dir() + else: + package_dir = Path(package_dir) + output_dir = Path(output_dir) + + # Collect ld dependencies for installed pip packages + dependencies = set() + for root, dirs, files in package_dir.walk(): + for file in files: + path = root.joinpath(file) + if not path.is_symlink() and _isElf(path): + out = subprocess.run( + [_getInterpreter(path), "--inhibit-cache", "--list", path], + stdout=subprocess.PIPE, + ) + for dependency in parse_output.findall(out.stdout.decode()): + dependencies.add(os.path.realpath(dependency)) + + # Copy dependencies into output_dir folder + if not output_dir.is_dir(): + output_dir.mkdir() + + for dependency in dependencies: + path = output_dir.joinpath(remove_root.sub("", dependency)) + path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(dependency, path) + + # Reset timestamps of the parent directories + if len(dependencies) > 0: + mtime = int(os.stat(dependencies.pop()).st_mtime) + os.utime(output_dir, (mtime, mtime)) + for root, dirs, _ in output_dir.walk(): + for dir in dirs: + os.utime(root.joinpath(dir), (mtime, mtime)) + + +if __name__ == "__main__": + args = parse_args() + export( + output_dir=Path(args.output_dir), + package_dir=Path(args.package_dir), + ) diff --git a/tests-ng/handlers/pip.py b/tests-ng/handlers/pip.py new file mode 100644 index 000000000..5166bc71f --- /dev/null +++ b/tests-ng/handlers/pip.py @@ -0,0 +1,34 @@ +import os +import shutil + +import pytest +from plugins.shell import ShellRunner + + +@pytest.fixture +def pip_requests(shell: ShellRunner): + out = shell( + '/bin/python3 -c "import site; print(site.getsitepackages()[0])"', + capture_output=True, + ) + package_dir = out.stdout.strip("\n") + + restore_backup = False + if os.path.isdir(package_dir): + restore_backup = True + shutil.move(package_dir, package_dir + ".backup") + + delete_normalizer = not os.path.isfile("/usr/local/bin/normalizer") + + os.mkdir(package_dir) + + shell("/bin/pip3 install requests --break-system-packages --no-cache-dir") + + yield package_dir + + shutil.rmtree(package_dir) + if restore_backup: + shutil.move(package_dir + ".backup", package_dir) + + if delete_normalizer: + os.remove("/usr/local/bin/normalizer") diff --git a/tests-ng/plugins/utils.py b/tests-ng/plugins/utils.py index 1a181a124..2f2d84f21 100644 --- a/tests-ng/plugins/utils.py +++ b/tests-ng/plugins/utils.py @@ -1,4 +1,5 @@ import logging +import os from pathlib import Path from typing import Dict, List, Optional, TypeVar @@ -21,6 +22,18 @@ def is_set(obj) -> bool: return isinstance(obj, set) +def tree(path: str) -> set[str]: + """Returns all subpaths of `path`, like the find command""" + tree = {path} + for root, dirs, files in os.walk(path): + for file in files: + tree.add(f"{root}/{file}") + for dir in dirs: + tree.add(f"{root}/{dir}") + + return tree + + T = TypeVar("T") diff --git a/tests-ng/test_pythonDev.py b/tests-ng/test_pythonDev.py new file mode 100644 index 000000000..f626379e1 --- /dev/null +++ b/tests-ng/test_pythonDev.py @@ -0,0 +1,66 @@ +import os +import shutil + +import pytest +from handlers.pip import pip_requests +from plugins.dpkg import Dpkg +from plugins.shell import ShellRunner +from plugins.utils import tree + + +@pytest.mark.feature("pythonDev") +def test_python_environment_is_installed(shell: ShellRunner): + dpkg = Dpkg(shell) + + assert dpkg.package_is_installed("python3"), "python3 package is not installed" + assert dpkg.package_is_installed( + "python3-pip" + ), "python3-pip package is not installed" + assert dpkg.package_is_installed( + "python3.13-venv" + ), "python3.13-venv package is not installed" + + +dependencies = { + "arm64": { + "/required_libs_test", + "/required_libs_test/usr", + "/required_libs_test/usr/lib", + "/required_libs_test/usr/lib/aarch64-linux-gnu", + "/required_libs_test/usr/lib/aarch64-linux-gnu/libpthread.so.0", + "/required_libs_test/usr/lib/aarch64-linux-gnu/libc.so.6", + "/required_libs_test/usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1", + }, + "amd64": { + "/required_libs_test", + "/required_libs_test/usr", + "/required_libs_test/usr/lib", + "/required_libs_test/usr/lib/x86_64-linux-gnu", + "/required_libs_test/usr/lib/x86_64-linux-gnu/libpthread.so.0", + "/required_libs_test/usr/lib/x86_64-linux-gnu/libc.so.6", + "/required_libs_test/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + }, +} + + +@pytest.mark.feature("pythonDev") +@pytest.mark.root(reason="Installs pip packages") +@pytest.mark.modify(reason="Installs pip packages") +def test_python_export_libs(shell: ShellRunner, pip_requests): + dpkg = Dpkg(shell) + arch = dpkg.architecture_native() + + # Check if requests is installed + shell("/bin/python3 -c 'import requests'") + + shell( + f"exportLibs.py --package-dir {pip_requests} --output-dir /required_libs_test" + ) + + assert os.path.isdir("/required_libs_test"), "/required_libs_test was not created" + + assert ( + tree("/required_libs_test") == dependencies[arch] + ), "required_libs content differs" + + shutil.rmtree("/required_libs_test")