diff --git a/poetry.lock b/poetry.lock index 4429bf3b..52655eab 100644 --- a/poetry.lock +++ b/poetry.lock @@ -77,18 +77,18 @@ yaml = ["PyYAML"] [[package]] name = "boto3" -version = "1.42.10" +version = "1.42.11" description = "The AWS SDK for Python" optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "boto3-1.42.10-py3-none-any.whl", hash = "sha256:70720926eab4306a724414286480ec4efa301f3e67e5a53ad4b62f6eb6dbd5b4"}, - {file = "boto3-1.42.10.tar.gz", hash = "sha256:8b7a1eb83ab7f0c89bb449ccac400eeca6f4ba6e33ba312e2281c6d864602bc3"}, + {file = "boto3-1.42.11-py3-none-any.whl", hash = "sha256:54939f7fc1b2777771c2a66ecc77025b2af86e567b5cf68d30dc3838205f0a4a"}, + {file = "boto3-1.42.11.tar.gz", hash = "sha256:2537d9462b70f4432385202709d1c8aa2291f802cfd8588d33334112116c554a"}, ] [package.dependencies] -botocore = ">=1.42.10,<1.43.0" +botocore = ">=1.42.11,<1.43.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.16.0,<0.17.0" @@ -97,14 +97,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.42.10" +version = "1.42.11" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "botocore-1.42.10-py3-none-any.whl", hash = "sha256:41eaa73694c0f9e5e281d81f18325f1181d332dce21ea47f58426250b31889fe"}, - {file = "botocore-1.42.10.tar.gz", hash = "sha256:84312c37ddc34cd0cce25436f26370af1edb9e1b1944359ee15350239537cdaa"}, + {file = "botocore-1.42.11-py3-none-any.whl", hash = "sha256:73b0796870f16ccd44729c767ade20e8ed62b31b3aa2be07b35377338dcf6d7c"}, + {file = "botocore-1.42.11.tar.gz", hash = "sha256:4c5278b9e0f6217f428aade811d409e321782bd14f0a202ff95a298d841be1f7"}, ] [package.dependencies] @@ -1060,16 +1060,37 @@ files = [ dev = ["pre-commit", "tox"] testing = ["coverage", "pytest", "pytest-benchmark"] +[[package]] +name = "podman" +version = "5.6.0" +description = "Bindings for Podman RESTful API" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "podman-5.6.0-py3-none-any.whl", hash = "sha256:967ff8ad8c6b851bc5da1a9410973882d80e235a9410b7d1e931ce0c3324fbe3"}, + {file = "podman-5.6.0.tar.gz", hash = "sha256:cc5f7aa9562e30f992fc170a48da970a7132be60d8a2e2941e6c17bd0a0b35c9"}, +] + +[package.dependencies] +requests = ">=2.24" +urllib3 = "*" + +[package.extras] +docs = ["sphinx"] +progress-bar = ["rich (>=12.5.1)"] +test = ["coverage", "fixtures", "pytest", "requests-mock", "tox"] + [[package]] name = "pre-commit" -version = "4.5.0" +version = "4.5.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "pre_commit-4.5.0-py2.py3-none-any.whl", hash = "sha256:25e2ce09595174d9c97860a95609f9f852c0614ba602de3561e267547f2335e1"}, - {file = "pre_commit-4.5.0.tar.gz", hash = "sha256:dc5a065e932b19fc1d4c653c6939068fe54325af8e741e74e88db4d28a4dd66b"}, + {file = "pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77"}, + {file = "pre_commit-4.5.1.tar.gz", hash = "sha256:eb545fcff725875197837263e977ea257a402056661f09dae08e4b149b030a61"}, ] [package.dependencies] @@ -1883,4 +1904,4 @@ test = ["pytest", "pytest-cov"] [metadata] lock-version = "2.1" python-versions = ">=3.13, <3.14" -content-hash = "72c619d1320245804ef38e5d9d45f0e026329d19e2ccbff02d65ec69dc8939f4" +content-hash = "34dfded5d5bcaf052b4d2208ec39b1f1d323af27ada6dfa5fa9be4c0fcad1411" diff --git a/pyproject.toml b/pyproject.toml index 7fd177bd..b25e13f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,12 +10,13 @@ packages = [{ include = "gardenlinux", from = "src" }] [tool.poetry.dependencies] python = ">=3.13, <3.14" apt-repo = "^0.5" -boto3 = "^1.42.10" +boto3 = "^1.42.11" click = "^8.3.1" cryptography = "^46.0.3" jsonschema = "^4.25.1" networkx = "^3.6" oras = "^0.2.38" +podman = "^5.6.0" pygit2 = "^1.19.0" pygments = "^2.19.2" PyYAML = "^6.0.2" @@ -23,12 +24,12 @@ gitpython = "^3.1.45" [tool.poetry.group.dev.dependencies] bandit = "^1.9.2" +isort = "^7.0.0" moto = "^5.1.16" -pre-commit = "^4.5.0" +pre-commit = "^4.5.1" python-dotenv = "^1.2.1" pytest = "^9.0.2" pytest-cov = "^7.0.0" -isort = "^7.0.0" requests-mock = "^1.12.1" [tool.poetry.group.docs.dependencies] diff --git a/src/gardenlinux/constants.py b/src/gardenlinux/constants.py index 293086d8..135cf0a3 100644 --- a/src/gardenlinux/constants.py +++ b/src/gardenlinux/constants.py @@ -165,6 +165,8 @@ GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME = "gardenlinux-github-releases" GLVD_BASE_URL = "https://security.gardenlinux.org/v1" +PODMAN_CONNECTION_MAX_IDLE_SECONDS = 3 + # https://github.com/gardenlinux/gardenlinux/issues/3044 # Empty string is the 'legacy' variant with traditional root fs and still needed/supported IMAGE_VARIANTS = ["", "_usi", "_tpm2_trustedboot"] diff --git a/src/gardenlinux/github/release/__init__.py b/src/gardenlinux/github/release/__init__.py index 18dd9cb8..cae6493b 100644 --- a/src/gardenlinux/github/release/__init__.py +++ b/src/gardenlinux/github/release/__init__.py @@ -1,4 +1,5 @@ import json +import logging import os import sys @@ -7,7 +8,7 @@ from gardenlinux.constants import RELEASE_ID_FILE, REQUESTS_TIMEOUTS from gardenlinux.logger import LoggerSetup -LOGGER = LoggerSetup.get_logger("gardenlinux.github.release", "INFO") +LOGGER = LoggerSetup.get_logger("gardenlinux.github.release", logging.INFO) def create_github_release(owner, repo, tag, commitish, latest, body): diff --git a/src/gardenlinux/github/release/__main__.py b/src/gardenlinux/github/release/__main__.py index 61c37338..29c6d2fd 100644 --- a/src/gardenlinux/github/release/__main__.py +++ b/src/gardenlinux/github/release/__main__.py @@ -1,4 +1,5 @@ import argparse +import logging from gardenlinux.constants import GARDENLINUX_GITHUB_RELEASE_BUCKET_NAME from gardenlinux.logger import LoggerSetup @@ -10,10 +11,10 @@ write_to_release_id_file, ) -LOGGER = LoggerSetup.get_logger("gardenlinux.github", "INFO") +LOGGER = LoggerSetup.get_logger("gardenlinux.github", logging.INFO) -def main(): +def main() -> None: parser = argparse.ArgumentParser(description="GitHub Release Script") subparsers = parser.add_subparsers(dest="command") diff --git a/src/gardenlinux/oci/__init__.py b/src/gardenlinux/oci/__init__.py index 96814084..d152bb37 100644 --- a/src/gardenlinux/oci/__init__.py +++ b/src/gardenlinux/oci/__init__.py @@ -9,5 +9,15 @@ from .index import Index from .layer import Layer from .manifest import Manifest +from .podman import Podman +from .podman_context import PodmanContext -__all__ = ["Container", "ImageManifest", "Index", "Layer", "Manifest"] +__all__ = [ + "Container", + "ImageManifest", + "Index", + "Layer", + "Manifest", + "Podman", + "PodmanContext", +] diff --git a/src/gardenlinux/oci/__main__.py b/src/gardenlinux/oci/__main__.py index 1d6371ee..28fc57c3 100755 --- a/src/gardenlinux/oci/__main__.py +++ b/src/gardenlinux/oci/__main__.py @@ -4,18 +4,23 @@ gl-oci main entrypoint """ +import json from typing import List import click from .container import Container from .image_manifest import ImageManifest +from .podman import Podman +from .podman_context import PodmanContext @click.group() def cli() -> None: """ - gl-oci click argument entrypoint + gl-oci provides functionality to handle OCI containers. It can pull and push + images from remote repositories as well as handle GardenLinux artifacts, OCI + image indices and manifests. :since: 0.7.0 """ @@ -23,38 +28,320 @@ def cli() -> None: pass +@cli.command() +@click.option( + "--index", + required=True, + help="OCI image index name", +) +@click.option( + "--container", + required=True, + help="Container Name", +) +@click.option( + "--tag", + required=True, + help="OCI tag of image", +) +@click.option( + "--insecure", + type=bool, + default=False, + help="Use HTTP to communicate with the registry", +) +@click.option( + "--additional_tag", + required=False, + multiple=True, + help="Additional tag to push the index with", +) +def add_container_to_index( + index: str, container: str, tag: str, insecure: bool, additional_tag: List[str] +) -> None: + """ + Adds an image container to an OCI image index. + + :since: 1.0.0 + """ + + manifest_container = Container(f"{container}:{tag}", insecure=insecure) + + manifest = manifest_container.read_manifest() + + index_container = Container(index, insecure=insecure) + + image_index = index_container.read_or_generate_index() + image_index.append_manifest(manifest) + + index_container.push_index(image_index) + index_container.push_index_for_tags(image_index, additional_tag) + + @cli.command() @click.option( "--container", required=True, + help="Container Name", +) +@click.option( + "--tag", + required=True, + help="OCI tag of image", +) +@click.option( + "--dir", + "directory", + required=True, + type=click.Path(), + help="Path to the build Containerfile", +) +@click.option( + "--platform", + required=False, + help="OCI platform as os/arch/variant", +) +@click.option( + "--additional_tag", + required=False, + multiple=True, + help="Additional tag to push the manifest with", +) +@click.option( + "--build_arg", + required=False, + default=[], + multiple=True, + help="Additional build args for Containerfile", +) +@click.option( + "--oci_archive", + required=False, + help="Write build result to the OCI archive path and file name", +) +def build_container( + container: str, + tag: str, + directory: str, + platform: str, + additional_tag: List[str], + build_arg: List[str], + oci_archive: str, +) -> None: + """ + Build an OCI container based on the defined `Containerfile`. + + :since: 1.0.0 + """ + + podman = Podman() + + with PodmanContext() as podman_context: + if oci_archive is None: + image_id = podman.build( + directory, + podman=podman_context, + platform=platform, + oci_tag=f"{container}:{tag}", + build_args=Podman.parse_build_args_list(build_arg), + ) + else: + build_result_data = podman.build_and_save_oci_archive( + directory, + oci_archive, + podman=podman_context, + platform=platform, + oci_tag=f"{container}:{tag}", + build_args=Podman.parse_build_args_list(build_arg), + ) + + _, image_id = build_result_data.popitem() + + if additional_tag is not None: + podman.tag_list( + image_id, Podman.get_container_tag_list(container, additional_tag) + ) + + print(image_id) + + +@cli.command() +@click.option( + "--oci_archive", + required=False, + help="Write build result to the OCI archive path and file name", +) +@click.option( + "--additional_tag", + required=False, + multiple=True, + help="Additional tag to push the manifest with", +) +def load_container(oci_archive: str, additional_tag: List[str]) -> None: + """ + Load an OCI archive. + + :since: 1.0.0 + """ + + podman = Podman() + + with PodmanContext() as podman_context: + image_id = podman.load_oci_archive(oci_archive, podman=podman_context) + + if additional_tag is not None: + podman.tag_list(image_id, additional_tag, podman=podman_context) + + print(image_id) + + +@cli.command() +@click.option( + "--dir", + "directory", + required=True, type=click.Path(), + help="path to the build artifacts", +) +def load_containers_from_directory(directory: str) -> None: + """ + Load multiple OCI archives. + + :since: 1.0.0 + """ + + result = Podman().load_oci_archives_from_directory(directory) + print(json.dumps(result)) + + +@cli.command() +@click.option( + "--index", + required=True, + help="OCI image index name", +) +@click.option( + "--insecure", + type=bool, + default=False, + help="Use HTTP to communicate with the registry", +) +@click.option( + "--additional_tag", + required=False, + multiple=True, + help="Additional tag to push the index with", +) +def new_index(index: str, insecure: bool, additional_tag: List[str]) -> None: + """ + Create a new OCI image index. + + :since: 1.0.0 + """ + + index_container = Container(index, insecure=insecure) + image_index = index_container.generate_index() + index_container.push_index(image_index) + index_container.push_index_for_tags(image_index, additional_tag) + + +@cli.command() +@click.option( + "--container", + required=True, + help="Container Name", +) +@click.option( + "--tag", + required=False, + help="OCI tag of image", +) +@click.option( + "--platform", + required=False, + help="OCI platform as os/arch/variant", +) +@click.option( + "--insecure", + type=bool, + default=False, + help="Use HTTP to communicate with the registry", +) +def pull_container(container: str, tag: str, platform: str, insecure: bool) -> None: + """ + Pull an OCI image container from a remote OCI registry. + + :since: 1.0.0 + """ + + image_id = Podman(insecure=insecure).pull(container, oci_tag=tag, platform=platform) + print(image_id) + + +@cli.command() +@click.option( + "--container", + required=True, help="Container Name", ) @click.option( - "--cname", required=True, type=click.Path(), help="Canonical Name of Image" + "--tag", + required=False, + help="OCI tag of image", ) +@click.option( + "--destination", + required=False, + help="OCI container destination", +) +@click.option( + "--insecure", + type=bool, + default=False, + help="Use HTTP to communicate with the registry", +) +def push_container(container: str, tag: str, destination: str, insecure: bool) -> None: + """ + Push an OCI image container to a remote OCI registry. + + :since: 1.0.0 + """ + + Podman(insecure=insecure).push(container, oci_tag=tag, destination=destination) + + +@cli.command() +@click.option( + "--container", + required=True, + help="Container Name", +) +@click.option("--cname", required=True, help="Canonical Name of Image") @click.option( "--arch", required=False, - type=click.Path(), default=None, help="Target Image CPU Architecture", ) @click.option( "--version", required=False, - type=click.Path(), default=None, help="Version of image", ) @click.option( "--commit", required=False, - type=click.Path(), default=None, help="Commit of image", ) -@click.option("--dir", "directory", required=True, help="path to the build artifacts") +@click.option( + "--dir", + "directory", + required=True, + type=click.Path(), + help="path to the build artifacts", +) @click.option( "--cosign_file", required=False, @@ -62,11 +349,13 @@ def cli() -> None: ) @click.option( "--manifest_file", + type=click.Path(), default="manifests/manifest.json", help="A file where the index entry for the pushed manifest is written to.", ) @click.option( "--insecure", + type=bool, default=False, help="Use HTTP to communicate with the registry", ) @@ -89,7 +378,8 @@ def push_manifest( additional_tag: List[str], ) -> None: """ - Push artifacts and the manifest from a directory to a registry. + Push to an OCI image container given GardenLinux canonical named artifacts + in a specified directory. :since: 0.7.0 """ @@ -116,39 +406,35 @@ def push_manifest( @click.option( "--container", required=True, - type=click.Path(), help="Container Name", ) @click.option( "--cname", required=False, - type=click.Path(), default=None, help="Canonical Name of Image", ) @click.option( "--arch", required=False, - type=click.Path(), default=None, help="Target Image CPU Architecture", ) @click.option( "--version", required=False, - type=click.Path(), default=None, help="Version of image", ) @click.option( "--commit", required=False, - type=click.Path(), default=None, help="Commit of image", ) @click.option( "--insecure", + type=bool, default=False, help="Use HTTP to communicate with the registry", ) @@ -168,7 +454,7 @@ def push_manifest_tags( tag: List[str], ) -> None: """ - Push artifacts and the manifest from a directory to a registry. + Push tags to an OCI image container for a given GardenLinux canonical named image. :since: 0.10.0 """ @@ -182,28 +468,94 @@ def push_manifest_tags( container.push_manifest_for_tags(manifest, tag) +@cli.command() +@click.option( + "--container", + required=True, + help="Container Name", +) +@click.option( + "--tag", + required=False, + help="OCI tag of image", +) +@click.option( + "--oci_archive", + required=False, + help="Write build result to the OCI archive path and file name", +) +def save_container(container: str, tag: str, oci_archive: str) -> None: + """ + Saves a given OCI image container as an OCI archive. + + :since: 1.0.0 + """ + + podman = Podman() + + image_id = podman.get_image_id(container, oci_tag=tag) + podman.save_oci_archive(image_id, oci_archive, oci_tag=tag) + + +@cli.command() +@click.option( + "--container", + required=True, + help="Container Name", +) +@click.option( + "--tag", + required=False, + help="OCI tag of image", +) +@click.option( + "--additional_tag", + required=False, + multiple=True, + help="Additional tag to push the manifest with", +) +def tag_container(container: str, tag: str, additional_tag: List[str]) -> None: + """ + Adds additional tags to a given OCI image container. + + :since: 1.0.0 + """ + + podman = Podman() + + with PodmanContext() as podman_context: + image_id = podman.get_image_id(container, podman=podman_context, oci_tag=tag) + + if additional_tag is not None: + podman.tag_list( + image_id, + Podman.get_container_tag_list(container, additional_tag), + podman=podman_context, + ) + + @cli.command() @click.option( "--container", "container", required=True, - type=click.Path(), help="Container Name", ) @click.option( "--version", "version", required=True, - type=click.Path(), help="Version of image", ) @click.option( "--manifest_folder", + type=click.Path(), default="manifests", help="A folder where the index entries are read from.", ) @click.option( "--insecure", + type=bool, default=False, help="Use HTTP to communicate with the registry", ) @@ -221,7 +573,7 @@ def update_index( additional_tag: List[str], ) -> None: """ - Push a list of files from the `manifest_folder` to an index. + Pushes manifests stored in a directory to a given OCI image index. :since: 0.7.0 """ diff --git a/src/gardenlinux/oci/container.py b/src/gardenlinux/oci/container.py index 54c3eb52..bccf9acb 100644 --- a/src/gardenlinux/oci/container.py +++ b/src/gardenlinux/oci/container.py @@ -19,7 +19,7 @@ from oras.container import Container as OrasContainer from oras.provider import Registry from oras.utils import extract_targz, make_targz -from requests import Response +from requests import HTTPError, Response from ..constants import OCI_IMAGE_INDEX_MEDIA_TYPE from ..features.cname import CName @@ -62,24 +62,22 @@ def __init__( :since: 0.7.0 """ - if "://" in container_url: - container_data = container_url.rsplit(":", 2) + container_data = container_url.rsplit(":", 1) - if len(container_data) < 3: - raise RuntimeError("Container name given is invalid") + if len(container_data) < 2: + raise RuntimeError("Container name given is invalid") - self._container_url = f"{container_data[0]}:{container_data[1]}" - self._container_version = container_data[2] - else: - container_data = container_url.rsplit(":", 1) + self._container_version = container_data[1] - if len(container_data) < 2: - raise RuntimeError("Container name given is invalid") + if "://" in container_data[0]: + scheme = container_data[0].split(":", 1)[0].lower() + insecure = scheme != "https" + self._container_url = container_data[0] + else: scheme = "http" if insecure else "https" self._container_url = f"{scheme}://{container_data[0]}" - self._container_version = container_data[1] container_url_data = urlsplit(self._container_url) self._token = None @@ -249,6 +247,23 @@ def _get_manifest_without_response_parsing(self, reference: str) -> Response: headers={"Accept": "application/vnd.oci.image.manifest.v1+json"}, ) + def push_index(self, index: Index, tag: Optional[str] = None) -> None: + """ + Replaces an old manifest entries with new ones + + :param manifests_dir: Directory where the manifest entries are read from + :param additional_tags: Additional tags to push the index with + + :since: 1.0.0 + """ + + index_kwargs = {} + + if tag is not None: + index_kwargs["reference"] = tag + + self._check_200_response(self._upload_index(index, **index_kwargs)) + def push_index_from_directory( self, manifests_dir: PathLike[str] | str, @@ -298,7 +313,7 @@ def push_index_from_directory( new_entries += 1 - self._check_200_response(self._upload_index(index)) + self.push_index(index) self._logger.info(f"Index pushed with {new_entries} new entries") if isinstance(additional_tags, Sequence) and len(additional_tags) > 0: @@ -321,7 +336,7 @@ def push_index_for_tags(self, index: Index, tags: List[str]) -> None: # For each additional tag, push the manifest using Registry.upload_manifest for tag in tags: - self._check_200_response(self._upload_index(index, tag)) + self.push_index(index, tag) def push_manifest( self, @@ -536,44 +551,38 @@ def push_manifest_for_tags(self, manifest: Manifest, tags: List[str]) -> None: self._check_200_response(self.upload_manifest(manifest, manifest_container)) - def read_or_generate_index(self) -> Index: + def read_index(self) -> Index: """ - Reads from registry or generates the OCI image index. + Reads the OCI image index from registry. - :return: OCI image index - :since: 0.7.0 + :return: OCI image manifest + :since: 1.0.0 """ response = self._get_index_without_response_parsing() if response.ok: index = Index(**response.json()) - elif response.status_code == 404: - index = self.generate_index() else: response.raise_for_status() return index - def read_or_generate_manifest( + def read_manifest( self, cname: Optional[str] = None, architecture: Optional[str] = None, version: Optional[str] = None, - commit: Optional[str] = None, - feature_set: Optional[str] = None, - ) -> Manifest | ImageManifest: + ) -> Manifest: """ - Reads from registry or generates the OCI manifest. + Reads the OCI manifest from registry. :param cname: Canonical name of the manifest :param architecture: Target architecture of the manifest :param version: Artifacts version of the manifest - :param commit: The Git commit ID of the manifest - :param feature_set: The expanded list of the included features of this manifest :return: OCI image manifest - :since: 0.7.0 + :since: 1.0.0 """ if cname is None: @@ -594,15 +603,62 @@ def read_or_generate_manifest( if response.ok: manifest = manifest_type(**response.json()) - elif response.status_code == 404: + else: + response.raise_for_status() + + return manifest + + def read_or_generate_index(self) -> Index: + """ + Reads from registry or generates the OCI image index. + + :return: OCI image index + :since: 0.7.0 + """ + + try: + index = self.read_index() + except HTTPError as exc: + if exc.response.status_code != 404: + raise + + index = self.generate_index() + + return index + + def read_or_generate_manifest( + self, + cname: Optional[str] = None, + architecture: Optional[str] = None, + version: Optional[str] = None, + commit: Optional[str] = None, + feature_set: Optional[str] = None, + ) -> Manifest | ImageManifest: + """ + Reads from registry or generates the OCI manifest. + + :param cname: Canonical name of the manifest + :param architecture: Target architecture of the manifest + :param version: Artifacts version of the manifest + :param commit: The Git commit ID of the manifest + :param feature_set: The expanded list of the included features of this manifest + + :return: OCI image manifest + :since: 0.7.0 + """ + + try: + manifest = self.read_manifest(cname, architecture, version) + except HTTPError as exc: + if exc.response.status_code != 404: + raise + if cname is None: manifest = self.generate_manifest(version, commit) else: manifest = self.generate_image_manifest( cname, architecture, version, commit, feature_set ) - else: - response.raise_for_status() return manifest diff --git a/src/gardenlinux/oci/podman.py b/src/gardenlinux/oci/podman.py new file mode 100644 index 00000000..d58000d3 --- /dev/null +++ b/src/gardenlinux/oci/podman.py @@ -0,0 +1,342 @@ +# -*- coding: utf-8 -*- + +""" +OCI podman +""" + +import json +import logging +from collections.abc import Mapping, Sequence +from os import PathLike +from pathlib import Path +from typing import Any, Dict, List, Optional + +from ..logger import LoggerSetup +from .podman_context import PodmanContext + + +class Podman(object): + """ + OCI podman provides access to an local podman installation. + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: oci + :since: 1.0.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + + def __init__( + self, insecure: Optional[bool] = False, logger: Optional[logging.Logger] = None + ): + """ + Constructor __init__(Podman) + + :since: 1.0.0 + """ + + if logger is None or not logger.hasHandlers(): + logger = LoggerSetup.get_logger("gardenlinux.oci") + + self._insecure = insecure + self._logger = logger + + @PodmanContext.wrap + def build( + self, + build_path: str, + podman: PodmanContext, + platform: Optional[str] = None, + build_args: Optional[Dict[str, str]] = None, + oci_tag: Optional[str] = None, + log_build_output: bool = False, + **kwargs: Any, + ) -> str: + """ + Build a container. + + :since: 1.0.0 + """ + + if not Path(build_path, "Containerfile").exists(): + raise RuntimeError(f"No Containerfile found at: {build_path}") + + if isinstance(build_args, Mapping): + kwargs["buildargs"] = build_args + + if platform is not None: + kwargs["platform"] = platform + + if oci_tag is not None: + kwargs["tag"] = oci_tag + + image, log = podman.images.build( + path=build_path, dockerfile="Containerfile", **kwargs + ) + + if log_build_output: + for line in log: + self._logger.info(json.loads(line)["stream"].strip()) + + return image.id # type: ignore[no-any-return] + + @PodmanContext.wrap + def build_and_save_oci_archive( + self, + build_path: str, + oci_archive_file_name: str | PathLike[str], + podman: PodmanContext, + platform: Optional[str] = None, + build_args: Optional[Dict[str, str]] = None, + oci_tag: Optional[str] = None, + log_build_output: bool = False, + **kwargs: Any, + ) -> Dict[str, str]: + """ + Build a container and save the result as an OCI archive with the given path and file name. + + :since: 1.0.0 + """ + + oci_archive_file_name = Path(oci_archive_file_name) + + image_id = self.build( + build_path, + podman=podman, + platform=platform, + build_args=build_args, + oci_tag=oci_tag, + log_build_output=log_build_output, + ) + + self.save_oci_archive(image_id, oci_archive_file_name, podman=podman) + + return {oci_archive_file_name.name: image_id} + + @PodmanContext.wrap + def get_image_id( + self, + container: str, + podman: PodmanContext, + oci_tag: Optional[str] = None, + ) -> str: + """ + Returns the Podman image ID for a given OCI container tag. + + :since: 1.0.0 + """ + + container_tag = container + + if oci_tag is not None: + if ":" in oci_tag: + container_tag = oci_tag + else: + container_tag += f":{oci_tag}" + + image = podman.images.get(container_tag) + return image.id # type: ignore[no-any-return] + + @PodmanContext.wrap + def load_oci_archive( + self, oci_archive_file_name: str | PathLike[str], /, podman: PodmanContext + ) -> str: + """ + Load OCI archives from the given directory. + + :since: 1.0.0 + """ + + oci_archive_file_name = Path(oci_archive_file_name) + + image = next(podman.images.load(file_path=oci_archive_file_name)) + return image.id # type: ignore[no-any-return] + + @PodmanContext.wrap + def load_oci_archives_from_directory( + self, oci_dir: str | PathLike[str], /, podman: PodmanContext + ) -> Dict[str, str]: + """ + Load OCI archives from the given directory. + + :since: 1.0.0 + """ + + oci_archives = {} + oci_dir = Path(oci_dir) + + for oci_archive in oci_dir.iterdir(): + if not oci_archive.match("*.oci"): + continue + + image = next(podman.images.load(file_path=oci_archive)) + oci_archives[oci_archive.name] = image.id + + return oci_archives + + @PodmanContext.wrap + def pull( + self, + container: str, + podman: PodmanContext, + platform: Optional[str] = None, + oci_tag: Optional[str] = None, + ) -> None: + """ + Pulls a given OCI container. + + :since: 1.0.0 + """ + + kwargs: Dict[str, Any] = {} + + if self._insecure: + kwargs["tlsVerify"] = False + + if platform is not None: + kwargs["platform"] = platform + + if oci_tag is not None: + kwargs["tag"] = oci_tag + + image = podman.images.pull(container, **kwargs) + return image.id + + @PodmanContext.wrap + def push( + self, + container: str, + podman: PodmanContext, + destination: Optional[str] = None, + oci_tag: Optional[str] = None, + ) -> None: + """ + Pushs a given OCI container. + + :since: 1.0.0 + """ + + kwargs: Dict[str, Any] = {} + + if self._insecure: + kwargs["tlsVerify"] = False + + if destination is not None: + kwargs["destination"] = destination + + if oci_tag is not None: + kwargs["tag"] = oci_tag + + podman.images.push(container, **kwargs) + + @PodmanContext.wrap + def save_oci_archive( + self, + image_id: str, + oci_archive_file_name: str | PathLike[str], + podman: PodmanContext, + oci_tag: Optional[str] = None, + ) -> None: + """ + Save the given Podman image ID as an OCI archive with the given path and + file name. + + :since: 1.0.0 + """ + + oci_archive_file_name = Path(oci_archive_file_name) + + if oci_archive_file_name.exists(): + raise RuntimeError("OCI archive file does already exist") + + image = podman.images.get(image_id) + + with oci_archive_file_name.open("wb") as fp: + named: bool | str = True + + if oci_tag is not None: + if oci_tag not in image.tags: + self.tag(image.id, oci_tag, podman=podman) + + named = oci_tag + + for chunk in image.save(named=named): + fp.write(chunk) + + @PodmanContext.wrap + def tag( + self, + image_id: str, + oci_container_tag: str, + podman: PodmanContext, + ) -> None: + """ + Tags a given Podman image ID with the container tag given. + + :since: 1.0.0 + """ + + oci_data = oci_container_tag.rsplit(":", 1) + + if len(oci_data) < 2: + raise RuntimeError("No tag given") + + image = podman.images.get(image_id) + image.tag(oci_data[0], oci_data[1]) + + @PodmanContext.wrap + def tag_list( + self, + image_id: str, + oci_container_tags_list: List[str], + podman: PodmanContext, + ) -> None: + """ + Tags a given Podman image ID with the list of container tags given. + + :since: 1.0.0 + """ + + for container_tag in oci_container_tags_list: + self.tag(image_id, container_tag, podman=podman) + + @staticmethod + def get_container_tag_list(container: str, tag_list: Sequence[str]) -> List[str]: + """ + Returns a list of "container:tag" values. + + :since: 1.0.0 + """ + + container_tag_list = [] + + if isinstance(tag_list, Sequence): + for tag in tag_list: + if ":" in tag: + container_tag_list.append(tag) + else: + container_tag_list.append(f"{container}:{tag}") + + return container_tag_list + + @staticmethod + def parse_build_args_list(args_list: List[str]) -> Dict[str, str]: + """ + Returns a mapping of build arguments based on the given list. + + :since: 1.0.0 + """ + + args = {} + + for arg_line in args_list: + arg_data = arg_line.split("=", 1) + + if len(arg_data) < 2: + raise RuntimeError(f"Failed to parse build argument: {arg_line}") + + args[arg_data[0]] = arg_data[1] + + return args diff --git a/src/gardenlinux/oci/podman_context.py b/src/gardenlinux/oci/podman_context.py new file mode 100644 index 00000000..e6076a24 --- /dev/null +++ b/src/gardenlinux/oci/podman_context.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- + +""" +OCI podman context +""" + +import logging +from contextlib import ExitStack +from functools import wraps +from os import rmdir +from pathlib import Path +from subprocess import PIPE, STDOUT, Popen +from tempfile import mkdtemp +from time import sleep +from typing import Any, Optional + +from podman.client import PodmanClient + +from ..constants import PODMAN_CONNECTION_MAX_IDLE_SECONDS +from ..logger import LoggerSetup + + +class PodmanContext(ExitStack): + """ + OCI podman context provides a context manager to be used to interact with + the podman API from Python. + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: oci + :since: 1.0.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + + def __init__(self, logger: Optional[logging.Logger] = None): + """ + Constructor __init__(PodmanContext) + + :since: 1.0.0 + """ + + ExitStack.__init__(self) + + if logger is None or not logger.hasHandlers(): + logger = LoggerSetup.get_logger("gardenlinux.oci") + + self._logger = logger + self._podman = None + self._podman_daemon: Optional[Popen[bytes]] = None + self._tmpdir: Optional[str] = None + + def __enter__(self) -> Any: + """ + python.org: Enter the runtime context related to this object. + + :return: (object) Podman context instance + :since: 1.0.0 + """ + + self._tmpdir = mkdtemp() + + podman_sock = str(Path(self._tmpdir, "podman.sock").absolute()) + + self._podman = PodmanClient(base_url=f"unix://{podman_sock}") + self._podman_daemon = Popen( + args=[ + "podman", + "system", + "service", + f"--time={PODMAN_CONNECTION_MAX_IDLE_SECONDS}", + f"unix://{podman_sock}", + ], + executable="podman", + stdout=PIPE, + stderr=STDOUT, + ) + + self.enter_context(self._podman_daemon) + self._wait_for_socket(podman_sock) + self.enter_context(self._podman) # type: ignore[arg-type] + + return self + + def __exit__( # type: ignore[exit-return] + self, exc_type: Any = None, exc_value: Any = None, traceback: Any = None + ) -> bool: + """ + python.org: Exit the runtime context related to this object. + + :return: (bool) True to suppress exceptions + :since: 1.0.0 + """ + + try: + if self._podman_daemon is not None: + self._podman_daemon.terminate() + self._podman_daemon.wait(PODMAN_CONNECTION_MAX_IDLE_SECONDS) + + if exc_type is not None and self._podman_daemon.stdout is not None: + stdout = self._podman_daemon.stdout.read() + + self._logger.error( + f"Podman context encountered an error. Process output: {stdout!r}" + ) + finally: + self._podman_daemon = None + + if self._tmpdir is not None: + rmdir(self._tmpdir) + self._tmpdir = None + + return False + + def __getattr__(self, name: str) -> Any: + """ + python.org: Called when an attribute lookup has not found the attribute in + the usual places (i.e. it is not an instance attribute nor is it found in the + class tree for self). + + :param name: Attribute name + + :return: (mixed) Attribute + :since: 1.0.0 + """ + + if self._podman_daemon is None: + raise RuntimeError("Podman context not ready") + + return getattr(self._podman, name) + + def _wait_for_socket(self, sock: str) -> None: + """ + Waits for the socket file to be created. + + :since: 1.0.0 + """ + + sock_path = Path(sock) + + for _ in range(0, 5 * PODMAN_CONNECTION_MAX_IDLE_SECONDS): + if sock_path.exists(): + break + + sleep(0.2) + + if not sock_path.exists(): + raise TimeoutError() + + @staticmethod + def wrap(f: Any) -> Any: + """ + Wraps the given function to provide access to a podman client. + + :since: 1.0.0 + """ + + @wraps(f) + def decorator(*args: Any, **kwargs: Any) -> Any: + """ + Decorator for wrapping a function or method with a call context. + """ + + if "podman" in kwargs: + if not isinstance(kwargs["podman"], PodmanContext): + raise ValueError( + "Podman context wrapped functions can not be called with `kwargs['podman']`" + ) + + return f(*args, **kwargs) + + with PodmanContext() as podman: + kwargs["podman"] = podman + return f(*args, **kwargs) + + return decorator diff --git a/test-data/oci/build/Containerfile b/test-data/oci/build/Containerfile new file mode 100644 index 00000000..c35f1b5f --- /dev/null +++ b/test-data/oci/build/Containerfile @@ -0,0 +1 @@ +FROM scratch diff --git a/tests/oci/test_main.py b/tests/oci/test_main.py new file mode 100644 index 00000000..6c67f3a2 --- /dev/null +++ b/tests/oci/test_main.py @@ -0,0 +1,275 @@ +import json +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest + +import gardenlinux.oci.__main__ as oci_main +from gardenlinux.oci import Container, Podman +from gardenlinux.oci.podman_context import PodmanContext + +from ..constants import REGISTRY, REPO_NAME, TEST_DATA_DIR + + +def test_main_build_container( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + monkeypatch.setattr( + sys, + "argv", + [ + "__main__.py", + "build-container", + "--dir", + f"{TEST_DATA_DIR}/oci/build", + "--container", + "container-test", + "--tag", + "latest", + ], + ) + + with pytest.raises(SystemExit, match="0"): + oci_main.main() + + captured = capsys.readouterr() + image_id = captured.out.strip() + + with PodmanContext() as podman: + image = podman.images.get(image_id) + + try: + assert podman.images.exists("localhost/container-test:latest") + assert "localhost/container-test:latest" in image.tags + finally: + podman.images.remove(image) + + +def test_main_build_container_and_save_as_oci_archive( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + with TemporaryDirectory() as tmpdir: + monkeypatch.setattr( + sys, + "argv", + [ + "__main__.py", + "build-container", + "--dir", + f"{TEST_DATA_DIR}/oci/build", + "--container", + "container-test", + "--tag", + "latest", + "--oci_archive", + f"{tmpdir}/archive.oci", + ], + ) + + image_id = None + + try: + with pytest.raises(SystemExit, match="0"): + oci_main.main() + + captured = capsys.readouterr() + image_id = captured.out.strip() + + assert Path(tmpdir, "archive.oci").exists() + finally: + if image_id is not None: + with PodmanContext() as podman: + image = podman.images.get(image_id) + podman.images.remove(image) + + +def test_main_load_container( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + with PodmanContext() as podman_context, TemporaryDirectory() as tmpdir: + podman = Podman() + + image_id = podman.build(f"{TEST_DATA_DIR}/oci/build", podman=podman_context) + + try: + podman.save_oci_archive( + image_id, f"{tmpdir}/archive.oci", podman=podman_context + ) + + monkeypatch.setattr( + sys, + "argv", + [ + "__main__.py", + "load-container", + "--oci_archive", + f"{tmpdir}/archive.oci", + ], + ) + + with pytest.raises(SystemExit, match="0"): + oci_main.main() + + captured = capsys.readouterr() + image_id_exported = captured.out.strip() + + assert Path(tmpdir, "archive.oci").exists() + assert image_id_exported == image_id + finally: + image = podman_context.images.get(image_id) + podman_context.images.remove(image) + + +def test_main_load_containers_from_directory( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + with PodmanContext() as podman_context, TemporaryDirectory() as tmpdir: + podman = Podman() + + image_id = podman.build(f"{TEST_DATA_DIR}/oci/build", podman=podman_context) + + try: + podman.save_oci_archive(image_id, f"{tmpdir}/1.oci", podman=podman_context) + podman.save_oci_archive(image_id, f"{tmpdir}/2.oci", podman=podman_context) + podman.save_oci_archive(image_id, f"{tmpdir}/3.oci", podman=podman_context) + + monkeypatch.setattr( + sys, + "argv", + [ + "__main__.py", + "load-containers-from-directory", + "--dir", + f"{tmpdir}", + ], + ) + + with pytest.raises(SystemExit, match="0"): + oci_main.main() + + captured = capsys.readouterr() + result = json.loads(captured.out) + + assert len(result) == 3 + assert "1.oci" in result + assert "2.oci" in result + assert "3.oci" in result + finally: + image = podman_context.images.get(image_id) + podman_context.images.remove(image) + + +@pytest.mark.usefixtures("zot_session") # type: ignore[misc] +def test_main_push_container( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + with PodmanContext() as podman_context, TemporaryDirectory(): + podman = Podman(insecure=True) + + image_built = None + image_id = podman.build(f"{TEST_DATA_DIR}/oci/build", podman=podman_context) + + try: + image_built = podman_context.images.get(image_id) + + monkeypatch.setattr( + sys, + "argv", + [ + "__main__.py", + "push-container", + "--container", + image_id, + "--destination", + f"docker://{REGISTRY}/{REPO_NAME}/kidden:latest", + "--insecure", + "true", + ], + ) + + with pytest.raises(SystemExit, match="0"): + oci_main.main() + + container = Container( + f"http://{REGISTRY}/{REPO_NAME}/kidden:latest", insecure=True + ) + # Assert - the following read would fail if push has not been successful + _ = container.read_manifest() + finally: + if image_built is not None: + podman_context.images.remove(image_built) + + +def test_main_tag_container( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + with PodmanContext() as podman_context, TemporaryDirectory(): + podman = Podman() + + image = None + image_id = podman.build(f"{TEST_DATA_DIR}/oci/build", podman=podman_context) + + try: + image = podman_context.images.get(image_id) + + monkeypatch.setattr( + sys, + "argv", + [ + "__main__.py", + "tag-container", + "--container", + image_id, + "--additional_tag", + "latest", + "--additional_tag", + "test:latest", + "--additional_tag", + "localhost/kidden:latest", + ], + ) + + with pytest.raises(SystemExit, match="0"): + oci_main.main() + + image = podman_context.images.get(image_id) + + assert len(image.tags) == 3 + assert f"localhost/{image_id}:latest" in image.tags + assert "localhost/test:latest" in image.tags + assert "localhost/kidden:latest" in image.tags + finally: + if image is not None: + podman_context.images.remove(image, force=True) + + +def test_main_save_container(monkeypatch: pytest.MonkeyPatch) -> None: + with PodmanContext() as podman, TemporaryDirectory() as tmpdir: + image_id = Podman().build( + f"{TEST_DATA_DIR}/oci/build", oci_tag="container-test:latest", podman=podman + ) + + monkeypatch.setattr( + sys, + "argv", + [ + "__main__.py", + "save-container", + "--container", + "container-test:latest", + "--tag", + "localhost/container-test:latest", + "--oci_archive", + f"{tmpdir}/archive.oci", + ], + ) + + try: + with pytest.raises(SystemExit, match="0"): + oci_main.main() + + assert Path(tmpdir, "archive.oci").exists() + finally: + image = podman.images.get(image_id) + podman.images.remove(image) diff --git a/tests/oci/test_podman.py b/tests/oci/test_podman.py new file mode 100644 index 00000000..c0688adc --- /dev/null +++ b/tests/oci/test_podman.py @@ -0,0 +1,55 @@ +from contextlib import contextmanager +from tempfile import TemporaryDirectory +from typing import Any + +import pytest + +from gardenlinux.oci import Podman +from gardenlinux.oci.podman_context import PodmanContext + +from ..constants import TEST_DATA_DIR + + +def test_podman_tag_list( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + with PodmanContext() as podman_context, TemporaryDirectory(): + podman = Podman() + + image_id = podman.build(f"{TEST_DATA_DIR}/oci/build", podman=podman_context) + + try: + podman.tag_list( + image_id, + Podman.get_container_tag_list("container-test", ["a", "b", "c"]), + ) + + image = podman_context.images.get(image_id) + + assert len(image.tags) == 3 + assert "localhost/container-test:a" in image.tags + assert "localhost/container-test:b" in image.tags + assert "localhost/container-test:c" in image.tags + finally: + image = podman_context.images.get(image_id) + podman_context.images.remove(image, force=True) + + +def test_podmancontext_socket_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + @contextmanager + def Popen(*args: Any, **kwargs: Any) -> Any: + yield + + monkeypatch.setattr("gardenlinux.oci.podman_context.Popen", Popen) + + with pytest.raises(TimeoutError): + with PodmanContext(): + pass + + +def test_podmancontext_podman_argument() -> None: + with PodmanContext(): + with pytest.raises( + ValueError, match="Podman context wrapped functions can not be called with" + ): + Podman().get_image_id("container-test", podman=object())