Skip to content

Enhance and test exportLibs.py #3914

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Leon-hk wants to merge 9 commits into
base: main
Choose a base branch
from
+279 −55
Open

Enhance and test exportLibs.py #3914

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
9b6a87a
Install python-gardenlinux-lib for pythonDev image, added feature tests
Leon-hk Nov 20, 2025
7407838
Fix shellcheck
Leon-hk Nov 20, 2025
3801858
Fix shellcheck again
Leon-hk Nov 20, 2025
6c6cef8
Merge branch 'main' into feat/port-exportlibs-to-python-lib
Leon-hk Dec 1, 2025
49c6953
enhanced exportLibs.py script using pyelftools
Leon-hk Dec 1, 2025
de27dfd
Merge branch 'main' into feat/port-exportlibs-to-python-lib
Leon-hk Jan 26, 2026
5afce6a
Merge branch 'main' into feat/port-exportlibs-to-python-lib
Leon-hk Jan 26, 2026
b221131
imrpove test structure
Leon-hk Jan 28, 2026
6128f3a
Add comment to exec.config
Leon-hk Jan 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions features/pythonDev/exec.config
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -Eeuo pipefail

# Set up a virtual environment for exportLibs.py to keep the system-wide packages clean
# Otherwise, pyelftools would be copied to the bare-python container in the multistage build
cd root
mkdir venv
python3 -m venv venv

# shellcheck source=/dev/null
source venv/bin/activate
pip install pyelftools
Copy link
Member

@fwilhe fwilhe Jan 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what's the rationale for this installation method? did you evaluate packaging this as a debian package? not sure if this is what we want, please get feedback from @nkraetzschmar

Copy link
Contributor Author

@Leon-hk Leon-hk Jan 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The exportLibs.py script runs in a virtual environment, as it requires pyelftools. This is necessary to have a completely fresh python environment of the system, as the dist-packages are later copied to the bare-python image. If we would install pyelftools system wide, it would be always contained in the bare-python image, even if it is not used.

Copy link
Member

@fwilhe fwilhe Jan 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that seems reasonable, thanks for explaining, but I think this is obscure enough that it should be documented (a comment in the script is fine imo)

Copy link
Contributor Author

@Leon-hk Leon-hk Jan 29, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, makes sense 👍 I've added a comment.

deactivate

cd ..
206 changes: 151 additions & 55 deletions features/pythonDev/file.include/usr/bin/exportLibs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,70 +1,166 @@
#!/bin/python3
#!/root/venv/bin/python3

import site
import os
import subprocess
import sys
import re
import shutil
import subprocess
from argparse import ArgumentParser
from os import PathLike
from pathlib import Path

from elftools.common.exceptions import ELFError
from elftools.elf.elffile import ELFFile

# Parses dependencies from ld output
parse_output = re.compile("(?:.*=>)?\\s*(/\\S*).*\n")
# Remove leading /
remove_root = re.compile("^/")


def parse_args():
"""
Parses arguments used for main()
:return: (object) Parsed argparse.ArgumentParser namespace
"""

parser = ArgumentParser(
description="Export shared libraries required by installed pip packages to a portable directory"
)

parser.add_argument(
"--output-dir",
default="/required_libs",
help="Directory containing the shared libraries.",
)
parser.add_argument(
"--package-dir",
default=_get_default_package_dir(),
help="Path of the generated output",
)

return parser.parse_args()


# Check for ELF header
def isElf(path: str) -> bool:
def _isElf(path: str | PathLike[str]) -> bool:
"""
Checks if a file is an ELF by looking for the ELF header.

:param path: Path to file

:return: (bool) If the file found at path is an ELF
"""

with open(path, "rb") as f:
return f.read(4) == b"\x7f\x45\x4c\x46"

try:
ELFFile(f)
return True
except ELFError:
return False


def _getInterpreter(path: str | PathLike[str]) -> Path:
"""
Returns the interpreter of an ELF. Supported architectures: x86_64, aarch64, i686.

:param path: Path to file

:return: (str) Path of the interpreter
"""

def getInterpreter(path: str) -> str:
with open(path, "rb") as f:
head = f.read(19)
elf = ELFFile(f)
interp = elf.get_section_by_name(".interp")

if head[5] == 1:
arch = head[17:]
elif head[5] == 2:
arch = head[17:][::-1]
else:
print(f"Error: Unknown endianess value for {path}: expected 1 or 2, but was {head[5]}", file=sys.stderr)
exit(1)

if arch == b"\x00\xb7": # 00b7: aarch64
return "/lib/ld-linux-aarch64.so.1"
elif arch == b"\x00\x3e": # 003e: x86_64
return "/lib64/ld-linux-x86-64.so.2"
elif arch == b"\x00\x03": # 0003: i686
return "/lib/ld-linux.so.2"
if interp:
return Path(interp.data().split(b"\x00")[0].decode())
else:
print(f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}", file=sys.stderr)
exit(1)


package_dir = site.getsitepackages()[0]

# Collect ld dependencies for installed pip packages
dependencies = set()
for root, dirs, files in os.walk(package_dir):
for file in files:
path = f"{root}/{file}"
if not os.path.islink(path) and isElf(path):
out = subprocess.run([getInterpreter(path),"--inhibit-cache", "--list", path], stdout=subprocess.PIPE)
for dependency in parse_output.findall(out.stdout.decode()):
dependencies.add(os.path.realpath(dependency))


# Copy dependencies into required_libs folder
if not os.path.isdir("/required_libs"):
os.mkdir("/required_libs")

for dependency in dependencies:
os.makedirs(f"/required_libs{os.path.dirname(dependency)}", exist_ok=True)
shutil.copy2(dependency, f"/required_libs{dependency}")

# Reset timestamps of the parent directories
if len(dependencies) > 0:
mtime = int(os.stat(dependencies.pop()).st_mtime)
os.utime("/required_libs", (mtime, mtime))
for root, dirs, files in os.walk("/required_libs"):
for dir in dirs:
os.utime(f"{root}/{dir}", (mtime, mtime))
match elf.header["e_machine"]:
case "EM_AARCH64":
return Path("/lib/ld-linux-aarch64.so.1")
case "EM_386":
return Path("/lib/ld-linux.so.2")
case "EM_X86_64":
return Path("/lib64/ld-linux-x86-64.so.2")
case arch:
raise RuntimeError(
f"Error: Unsupported architecture for {path}: only support x86_64 (003e), aarch64 (00b7) and i686 (0003), but was {arch}"
)


def _get_default_package_dir() -> Path:
"""
Finds the default site-packages or dist-packages directory of the default python3 environment

:return: (str) Path to directory
"""

# Needs to escape the virtual environment python-gardenlinx-lib is running in
interpreter = shutil.which("python3")

if not interpreter:
raise RuntimeError(
f"Error: Couldn't identify a default python package directory. Please specifiy one using the --package-dir option. Use -h for more information."
)

out = subprocess.run(
[interpreter, "-c", "import site; print(site.getsitepackages()[0])"],
stdout=subprocess.PIPE,
)
return Path(out.stdout.decode().strip())


def export(
output_dir: str | PathLike[str] = "/required_libs",
package_dir: str | PathLike[str] | None = None,
) -> None:
"""
Identifies shared library dependencies of `package_dir` and copies them to `output_dir`.

:param output_dir: Path to output_dir
:param package_dir: Path to package_dir
"""

if not package_dir:
package_dir = _get_default_package_dir()
else:
package_dir = Path(package_dir)
output_dir = Path(output_dir)

# Collect ld dependencies for installed pip packages
dependencies = set()
for root, dirs, files in package_dir.walk():
for file in files:
path = root.joinpath(file)
if not path.is_symlink() and _isElf(path):
out = subprocess.run(
[_getInterpreter(path), "--inhibit-cache", "--list", path],
stdout=subprocess.PIPE,
)
for dependency in parse_output.findall(out.stdout.decode()):
dependencies.add(os.path.realpath(dependency))

# Copy dependencies into output_dir folder
if not output_dir.is_dir():
output_dir.mkdir()

for dependency in dependencies:
path = output_dir.joinpath(remove_root.sub("", dependency))
path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(dependency, path)

# Reset timestamps of the parent directories
if len(dependencies) > 0:
mtime = int(os.stat(dependencies.pop()).st_mtime)
os.utime(output_dir, (mtime, mtime))
for root, dirs, _ in output_dir.walk():
for dir in dirs:
os.utime(root.joinpath(dir), (mtime, mtime))


if __name__ == "__main__":
args = parse_args()
export(
output_dir=Path(args.output_dir),
package_dir=Path(args.package_dir),
)
34 changes: 34 additions & 0 deletions tests-ng/handlers/pip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
import os
import shutil

import pytest
from plugins.shell import ShellRunner


@pytest.fixture
def pip_requests(shell: ShellRunner):
out = shell(
'/bin/python3 -c "import site; print(site.getsitepackages()[0])"',
capture_output=True,
)
package_dir = out.stdout.strip("\n")

restore_backup = False
if os.path.isdir(package_dir):
restore_backup = True
shutil.move(package_dir, package_dir + ".backup")

delete_normalizer = not os.path.isfile("/usr/local/bin/normalizer")

os.mkdir(package_dir)

shell("/bin/pip3 install requests --break-system-packages --no-cache-dir")

yield package_dir

shutil.rmtree(package_dir)
if restore_backup:
shutil.move(package_dir + ".backup", package_dir)

if delete_normalizer:
os.remove("/usr/local/bin/normalizer")
13 changes: 13 additions & 0 deletions tests-ng/plugins/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import logging
import os
from pathlib import Path
from typing import Dict, List, Optional, TypeVar

Expand All @@ -21,6 +22,18 @@ def is_set(obj) -> bool:
return isinstance(obj, set)


def tree(path: str) -> set[str]:
"""Returns all subpaths of `path`, like the find command"""
tree = {path}
for root, dirs, files in os.walk(path):
for file in files:
tree.add(f"{root}/{file}")
for dir in dirs:
tree.add(f"{root}/{dir}")

return tree


T = TypeVar("T")


Expand Down
66 changes: 66 additions & 0 deletions tests-ng/test_pythonDev.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import os
import shutil

import pytest
from handlers.pip import pip_requests
from plugins.dpkg import Dpkg
from plugins.shell import ShellRunner
from plugins.utils import tree


@pytest.mark.feature("pythonDev")
def test_python_environment_is_installed(shell: ShellRunner):
dpkg = Dpkg(shell)

assert dpkg.package_is_installed("python3"), "python3 package is not installed"
assert dpkg.package_is_installed(
"python3-pip"
), "python3-pip package is not installed"
assert dpkg.package_is_installed(
"python3.13-venv"
), "python3.13-venv package is not installed"


dependencies = {
"arm64": {
"/required_libs_test",
"/required_libs_test/usr",
"/required_libs_test/usr/lib",
"/required_libs_test/usr/lib/aarch64-linux-gnu",
"/required_libs_test/usr/lib/aarch64-linux-gnu/libpthread.so.0",
"/required_libs_test/usr/lib/aarch64-linux-gnu/libc.so.6",
"/required_libs_test/usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1",
},
"amd64": {
"/required_libs_test",
"/required_libs_test/usr",
"/required_libs_test/usr/lib",
"/required_libs_test/usr/lib/x86_64-linux-gnu",
"/required_libs_test/usr/lib/x86_64-linux-gnu/libpthread.so.0",
"/required_libs_test/usr/lib/x86_64-linux-gnu/libc.so.6",
"/required_libs_test/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2",
},
}


@pytest.mark.feature("pythonDev")
@pytest.mark.root(reason="Installs pip packages")
@pytest.mark.modify(reason="Installs pip packages")
def test_python_export_libs(shell: ShellRunner, pip_requests):
dpkg = Dpkg(shell)
arch = dpkg.architecture_native()

# Check if requests is installed
shell("/bin/python3 -c 'import requests'")

shell(
f"exportLibs.py --package-dir {pip_requests} --output-dir /required_libs_test"
)

assert os.path.isdir("/required_libs_test"), "/required_libs_test was not created"

assert (
tree("/required_libs_test") == dependencies[arch]
), "required_libs content differs"

shutil.rmtree("/required_libs_test")
Loading