diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index faec1c7bc3..439d78c822 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -78,7 +78,7 @@ jobs: - name: Write tags file 🏷 run: > - python3 -m tagging.write_tags_file + python3 -m tagging.apps.write_tags_file --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --short-image-name ${{ inputs.image }} @@ -94,7 +94,7 @@ jobs: - name: Write manifest and build history file 🏷 run: > - python3 -m tagging.write_manifest + python3 -m tagging.apps.write_manifest --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --short-image-name ${{ inputs.image }} diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml index 125194731e..0787e48f29 100644 --- a/.github/workflows/docker-merge-tags.yml +++ b/.github/workflows/docker-merge-tags.yml @@ -68,7 +68,7 @@ jobs: - name: Merge tags for the images 🔀 if: env.PUSH_TO_REGISTRY == 'true' run: > - python3 -m tagging.merge_tags + python3 -m tagging.apps.merge_tags --short-image-name ${{ inputs.image }} --variant ${{ inputs.variant }} --tags-dir /tmp/jupyter/tags/ diff --git a/.github/workflows/docker-tag-push.yml b/.github/workflows/docker-tag-push.yml index a53ccf1933..87e9855d9c 100644 --- a/.github/workflows/docker-tag-push.yml +++ b/.github/workflows/docker-tag-push.yml @@ -62,7 +62,7 @@ jobs: path: /tmp/jupyter/tags/ - name: Apply tags to the loaded image 🏷 run: > - python3 -m tagging.apply_tags + python3 -m tagging.apps.apply_tags --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --short-image-name ${{ inputs.image }} diff --git a/.github/workflows/docker-wiki-update.yml b/.github/workflows/docker-wiki-update.yml index 82cf82711f..a381fca20d 100644 --- a/.github/workflows/docker-wiki-update.yml +++ b/.github/workflows/docker-wiki-update.yml @@ -37,12 +37,12 @@ jobs: uses: actions/checkout@v4 with: repository: ${{ github.repository }}.wiki - path: wiki/ + path: wiki_src/ - name: Update wiki 🏷 run: > - python3 -m tagging.update_wiki - --wiki-dir wiki/ + python3 -m wiki.update_wiki + --wiki-dir wiki_src/ --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --repository ${{ github.repository }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 98693807fd..9e1491bbc3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,6 +30,7 @@ on: - "!tagging/README.md" - "tests/**" - "!tests/README.md" + - "wiki/**" - "requirements-dev.txt" push: branches: @@ -50,6 +51,7 @@ on: - "!tagging/README.md" - "tests/**" - "!tests/README.md" + - "wiki/**" - "requirements-dev.txt" workflow_dispatch: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a1d239e32..038e8a82ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ This changelog only contains breaking and/or significant changes manually introduced to this repository (using Pull Requests). All image manifests can be found in [the wiki](https://github.com/jupyter/docker-stacks/wiki). +## 2025-02-21 + +Affected: all images. + +- **Non-breaking:**: Better tagging directory structure ([#2228](https://github.com/jupyter/docker-stacks/pull/2228)). + ## 2025-02-18 Affected: all images. diff --git a/Makefile b/Makefile index 1b609d80d9..9c9a5b64d9 100644 --- a/Makefile +++ b/Makefile @@ -78,20 +78,20 @@ linkcheck-docs: ## check broken links hook/%: VARIANT?=default hook/%: ## run post-build hooks for an image - python3 -m tagging.write_tags_file \ + python3 -m tagging.apps.write_tags_file \ --registry "$(REGISTRY)" \ --owner "$(OWNER)" \ --short-image-name "$(notdir $@)" \ --variant "$(VARIANT)" \ --tags-dir /tmp/jupyter/tags/ - python3 -m tagging.write_manifest \ + python3 -m tagging.apps.write_manifest \ --registry "$(REGISTRY)" \ --owner "$(OWNER)" \ --short-image-name "$(notdir $@)" \ --variant "$(VARIANT)" \ --hist-lines-dir /tmp/jupyter/hist_lines/ \ --manifests-dir /tmp/jupyter/manifests/ - python3 -m tagging.apply_tags \ + python3 -m tagging.apps.apply_tags \ --registry "$(REGISTRY)" \ --owner "$(OWNER)" \ --short-image-name "$(notdir $@)" \ diff --git a/docs/index.rst b/docs/index.rst index e8cd032138..df09573f2d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,6 +35,7 @@ Table of Contents :caption: Maintainer Guide maintaining/new-images-and-packages-policy + maintaining/tagging maintaining/tasks .. toctree:: diff --git a/docs/maintaining/tagging.md b/docs/maintaining/tagging.md new file mode 100644 index 0000000000..377304f517 --- /dev/null +++ b/docs/maintaining/tagging.md @@ -0,0 +1,94 @@ +# Tagging and manifest creation + +The main purpose of the source code in [the `tagging` folder](https://github.com/jupyter/docker-stacks/tree/main/tagging) is to properly write tag files and manifests for single-platform images, +apply these tags, and merge single-platform images into one multi-arch image. + +## What is a tag and a manifest + +A tag is a label attached to a Docker image identifying specific attributes or versions. +For example, an image `jupyter/base-notebook` with Python 3.10.5 will have a full image name `quay.io/jupyter/base-notebook:python-3.10.5`. +These tags are pushed to our [Quay.io registry](https://quay.io/organization/jupyter). + +A manifest is a description of important image attributes written in Markdown format. +For example, we dump all `conda` packages with their versions into the manifest. + +## Main principles + +- All images are organized in a hierarchical tree. + More info on [image relationships](../using/selecting.md#image-relationships). +- Classes inherit from `TaggerInterface` and `ManifestInterface` to generate tags and manifest pieces by running commands in Docker containers. +- Tags and manifests are reevaluated for each image in the hierarchy since values may change between parent and child images. +- To tag an image and create its manifest, run `make hook/` (e.g., `make hook/base-notebook`). + +## Utils + +### DockerRunner + +`DockerRunner` is a helper class to easily run a docker container and execute commands inside this container: + +```{literalinclude} tagging_examples/docker_runner.py +:language: py +``` + +### GitHelper + +`GitHelper` methods are run in the current `git` repo and give the information about the last commit hash and commit message: + +```{literalinclude} tagging_examples/git_helper.py +:language: py +``` + +The prefix of commit hash (namely, 12 letters) is used as an image tag to make it easy to inherit from a fixed version of a docker image. + +## Taggers and Manifests + +### Tagger + +`Tagger` is a class that can be run inside a docker container to calculate some tag for an image. + +All the taggers are inherited from `TaggerInterface`: + +```{literalinclude} ../../tagging/taggers/tagger_interface.py +:language: py +``` + +So, the `tag_value(container)` method gets a docker container as an input and returns a tag. + +`SHATagger` example: + +```{literalinclude} ../../tagging/taggers/sha.py +:language: py +``` + +- `taggers/` subdirectory contains all the taggers. +- `apps/write_tags_file.py`, `apps/apply_tags.py`, and `apps/merge_tags.py` are Python executable used to write tags for an image, apply tags from a file, and create multi-arch images. + +### Manifest + +`ManifestHeader` is a build manifest header. +It contains the following sections: `Build timestamp`, `Docker image size`, and `Git commit` info. + +All the other manifest classes are inherited from `ManifestInterface`: + +```{literalinclude} ../../tagging/manifests/manifest_interface.py +:language: py +``` + +- The `markdown_piece(container)` method returns a piece of markdown file to be used as a part of the build manifest. + +`AptPackagesManifest` example: + +```{literalinclude} ../../tagging/manifests/apt_packages.py +:language: py +``` + +- `quoted_output(container, cmd)` simply runs the command inside a container using `DockerRunner.run_simple_command` and wraps it to triple quotes to create a valid markdown piece. + It also adds the command which was run to the markdown piece. +- `manifests/` subdirectory contains all the manifests. +- `apps/write_manifest.py` is a Python executable to create the build manifest and history line for an image. + +### Images Hierarchy + +All images' dependencies on each other and what taggers and manifest are applicable to them are defined in `hierarchy/images_hierarchy.py`. + +`hierarchy/get_taggers_and_manifests.py` defines a function to get the taggers and manifests for a specific image. diff --git a/docs/maintaining/tagging_examples/docker_runner.py b/docs/maintaining/tagging_examples/docker_runner.py new file mode 100644 index 0000000000..528bc36a6e --- /dev/null +++ b/docs/maintaining/tagging_examples/docker_runner.py @@ -0,0 +1,4 @@ +from tagging.utils.docker_runner import DockerRunner + +with DockerRunner("ubuntu") as container: + DockerRunner.run_simple_command(container, cmd="env", print_result=True) diff --git a/docs/maintaining/tagging_examples/git_helper.py b/docs/maintaining/tagging_examples/git_helper.py new file mode 100644 index 0000000000..de90923267 --- /dev/null +++ b/docs/maintaining/tagging_examples/git_helper.py @@ -0,0 +1,4 @@ +from tagging.utils.git_helper import GitHelper + +print("Git hash:", GitHelper.commit_hash()) +print("Git message:", GitHelper.commit_message()) diff --git a/tagging/README.md b/tagging/README.md index 5d4ff4a5ab..8d6fe9e57d 100644 --- a/tagging/README.md +++ b/tagging/README.md @@ -1,126 +1,3 @@ # Docker stacks tagging and manifest creation -The main purpose of the source code in this folder is to properly tag all the images and to update [build manifests](https://github.com/jupyter/docker-stacks/wiki). -These two processes are closely related, so the source code is widely reused. - -A basic example of a tag is a `Python` version tag. -For example, an image `jupyter/base-notebook` with `python 3.10.5` will have a full image name `quay.io/jupyter/base-notebook:python-3.10.5`. -This tag (and all the other tags) are pushed to Quay.io. - -Manifest is a description of some important part of the image in a `markdown`. -For example, we dump all the `conda` packages, including their versions. - -## Main principles - -- All the images are located in a hierarchical tree. - More info on [image relationships](../docs/using/selecting.md#image-relationships). -- We have `tagger` and `manifest` classes, which can be run inside docker containers to obtain tags and build manifest pieces. -- These classes are inherited from the parent image to all the child images. -- Because manifests and tags might change from parent to child, `taggers` and `manifests` are reevaluated on each image. - So, the values are not inherited. -- To tag an image and create a manifest, run `make hook/base-notebook` (or another image of your choice). - -## Source code description - -In this section, we will briefly describe the source code in this folder and give examples of how to use it. - -### DockerRunner - -`DockerRunner` is a helper class to easily run a docker container and execute commands inside this container: - -```python -from tagging.docker_runner import DockerRunner - -with DockerRunner("ubuntu:22.04") as container: - DockerRunner.run_simple_command(container, cmd="env", print_result=True) -``` - -### GitHelper - -`GitHelper` methods are run in the current `git` repo and give the information about the last commit hash and commit message: - -```python -from tagging.git_helper import GitHelper - -print("Git hash:", GitHelper.commit_hash()) -print("Git message:", GitHelper.commit_message()) -``` - -The prefix of commit hash (namely, 12 letters) is used as an image tag to make it easy to inherit from a fixed version of a docker image. - -### Tagger - -`Tagger` is a class that can be run inside a docker container to calculate some tag for an image. - -All the taggers are inherited from `TaggerInterface`: - -```python -class TaggerInterface: - """Common interface for all taggers""" - - @staticmethod - def tag_value(container) -> str: - raise NotImplementedError -``` - -So, the `tag_value(container)` method gets a docker container as an input and returns a tag. - -`SHATagger` example: - -```python -from tagging.git_helper import GitHelper -from tagging.taggers import TaggerInterface - - -class SHATagger(TaggerInterface): - @staticmethod - def tag_value(container): - return GitHelper.commit_hash_tag() -``` - -- `taggers.py` contains all the taggers. -- `tag_image.py` is a Python executable that is used to tag the image. - -### Manifest - -`ManifestHeader` is a build manifest header. -It contains the following sections: `Build timestamp`, `Docker image size`, and `Git commit` info. - -All the other manifest classes are inherited from `ManifestInterface`: - -```python -class ManifestInterface: - """Common interface for all manifests""" - - @staticmethod - def markdown_piece(container) -> str: - raise NotImplementedError -``` - -- The `markdown_piece(container)` method returns a piece of markdown file to be used as a part of the build manifest. - -`AptPackagesManifest` example: - -```python -from tagging.manifests import ManifestInterface, quoted_output - - -class AptPackagesManifest(ManifestInterface): - @staticmethod - def markdown_piece(container) -> str: - return f"""\ -## Apt Packages - -{quoted_output(container, "apt list --installed")}""" -``` - -- `quoted_output` simply runs the command inside a container using `DockerRunner.run_simple_command` and wraps it to triple quotes to create a valid markdown piece. - It also adds the command which was run to the markdown piece. -- `manifests.py` contains all the manifests. -- `write_manifest.py` is a Python executable that is used to create the build manifest and history line for an image. - -### Images Hierarchy - -All images' dependencies on each other and what taggers and manifest they make use of are defined in `images_hierarchy.py`. - -`get_taggers_and_manifests.py` defines a helper function to get the taggers and manifests for a specific image. +Please, refer to the [tagging section of documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/maintaing/tagging.html) to see how tags and manifests are created. diff --git a/tagging/apps/__init__.py b/tagging/apps/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tagging/apply_tags.py b/tagging/apps/apply_tags.py similarity index 88% rename from tagging/apply_tags.py rename to tagging/apps/apply_tags.py index 613b7826d6..34166029be 100755 --- a/tagging/apply_tags.py +++ b/tagging/apps/apply_tags.py @@ -6,9 +6,9 @@ import plumbum -from tagging.common_arguments import common_arguments_parser -from tagging.get_platform import unify_aarch64 -from tagging.get_prefix import get_file_prefix_for_platform +from tagging.apps.common_cli_arguments import common_arguments_parser +from tagging.utils.get_platform import unify_aarch64 +from tagging.utils.get_prefix import get_file_prefix_for_platform docker = plumbum.local["docker"] diff --git a/tagging/common_arguments.py b/tagging/apps/common_cli_arguments.py similarity index 100% rename from tagging/common_arguments.py rename to tagging/apps/common_cli_arguments.py diff --git a/tagging/merge_tags.py b/tagging/apps/merge_tags.py similarity index 91% rename from tagging/merge_tags.py rename to tagging/apps/merge_tags.py index 5eaa9c7cbd..d9f0432bff 100755 --- a/tagging/merge_tags.py +++ b/tagging/apps/merge_tags.py @@ -6,9 +6,9 @@ import plumbum -from tagging.common_arguments import common_arguments_parser -from tagging.get_platform import ALL_PLATFORMS -from tagging.get_prefix import get_file_prefix_for_platform +from tagging.apps.common_cli_arguments import common_arguments_parser +from tagging.utils.get_platform import ALL_PLATFORMS +from tagging.utils.get_prefix import get_file_prefix_for_platform docker = plumbum.local["docker"] diff --git a/tagging/write_manifest.py b/tagging/apps/write_manifest.py similarity index 89% rename from tagging/write_manifest.py rename to tagging/apps/write_manifest.py index ca7b0e038b..30d0764852 100755 --- a/tagging/write_manifest.py +++ b/tagging/apps/write_manifest.py @@ -7,12 +7,15 @@ from docker.models.containers import Container -from tagging.common_arguments import common_arguments_parser -from tagging.docker_runner import DockerRunner -from tagging.get_prefix import get_file_prefix, get_tag_prefix -from tagging.get_taggers_and_manifests import get_taggers_and_manifests -from tagging.git_helper import GitHelper -from tagging.manifests import ManifestHeader, ManifestInterface +from tagging.apps.common_cli_arguments import common_arguments_parser +from tagging.hierarchy.get_taggers_and_manifests import ( + get_taggers_and_manifests, +) +from tagging.manifests.header import ManifestHeader +from tagging.manifests.manifest_interface import ManifestInterface +from tagging.utils.docker_runner import DockerRunner +from tagging.utils.get_prefix import get_file_prefix, get_tag_prefix +from tagging.utils.git_helper import GitHelper LOGGER = logging.getLogger(__name__) diff --git a/tagging/write_tags_file.py b/tagging/apps/write_tags_file.py similarity index 85% rename from tagging/write_tags_file.py rename to tagging/apps/write_tags_file.py index 4c77dc8a96..f32ad925be 100755 --- a/tagging/write_tags_file.py +++ b/tagging/apps/write_tags_file.py @@ -4,10 +4,12 @@ import logging from pathlib import Path -from tagging.common_arguments import common_arguments_parser -from tagging.docker_runner import DockerRunner -from tagging.get_prefix import get_file_prefix, get_tag_prefix -from tagging.get_taggers_and_manifests import get_taggers_and_manifests +from tagging.apps.common_cli_arguments import common_arguments_parser +from tagging.hierarchy.get_taggers_and_manifests import ( + get_taggers_and_manifests, +) +from tagging.utils.docker_runner import DockerRunner +from tagging.utils.get_prefix import get_file_prefix, get_tag_prefix LOGGER = logging.getLogger(__name__) diff --git a/tagging/hierarchy/__init__.py b/tagging/hierarchy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tagging/get_taggers_and_manifests.py b/tagging/hierarchy/get_taggers_and_manifests.py similarity index 75% rename from tagging/get_taggers_and_manifests.py rename to tagging/hierarchy/get_taggers_and_manifests.py index 47ee262bb1..e817445d8b 100644 --- a/tagging/get_taggers_and_manifests.py +++ b/tagging/hierarchy/get_taggers_and_manifests.py @@ -1,9 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -from tagging.images_hierarchy import ALL_IMAGES -from tagging.manifests import ManifestInterface -from tagging.taggers import TaggerInterface +from tagging.hierarchy.images_hierarchy import ALL_IMAGES +from tagging.manifests.manifest_interface import ManifestInterface +from tagging.taggers.tagger_interface import TaggerInterface def get_taggers_and_manifests( diff --git a/tagging/images_hierarchy.py b/tagging/hierarchy/images_hierarchy.py similarity index 80% rename from tagging/images_hierarchy.py rename to tagging/hierarchy/images_hierarchy.py index 3a15faa0fe..2a49790bd8 100644 --- a/tagging/images_hierarchy.py +++ b/tagging/hierarchy/images_hierarchy.py @@ -2,16 +2,17 @@ # Distributed under the terms of the Modified BSD License. from dataclasses import dataclass, field -from tagging.manifests import ( - AptPackagesManifest, - CondaEnvironmentManifest, - JuliaPackagesManifest, - ManifestInterface, - RPackagesManifest, - SparkInfoManifest, -) -from tagging.taggers import ( - DateTagger, +from tagging.manifests.apt_packages import AptPackagesManifest +from tagging.manifests.conda_environment import CondaEnvironmentManifest +from tagging.manifests.julia_packages import JuliaPackagesManifest +from tagging.manifests.manifest_interface import ManifestInterface +from tagging.manifests.r_packages import RPackagesManifest +from tagging.manifests.spark_info import SparkInfoManifest +from tagging.taggers.date import DateTagger +from tagging.taggers.sha import SHATagger +from tagging.taggers.tagger_interface import TaggerInterface +from tagging.taggers.ubuntu_version import UbuntuVersionTagger +from tagging.taggers.versions import ( JavaVersionTagger, JuliaVersionTagger, JupyterHubVersionTagger, @@ -21,11 +22,8 @@ PythonVersionTagger, PytorchVersionTagger, RVersionTagger, - SHATagger, SparkVersionTagger, - TaggerInterface, TensorflowVersionTagger, - UbuntuVersionTagger, ) diff --git a/tagging/manifests.py b/tagging/manifests.py deleted file mode 100644 index b09db50b52..0000000000 --- a/tagging/manifests.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import plumbum -from docker.models.containers import Container - -from tagging.docker_runner import DockerRunner -from tagging.git_helper import GitHelper - -docker = plumbum.local["docker"] - - -def quoted_output(container: Container, cmd: str) -> str: - cmd_output = DockerRunner.run_simple_command(container, cmd, print_result=False) - # For example, `mamba info` adds redundant empty lines - cmd_output = cmd_output.strip("\n") - # For example, R packages list contains trailing backspaces - cmd_output = "\n".join(line.rstrip() for line in cmd_output.split("\n")) - - assert cmd_output, f"Command `{cmd}` returned empty output" - - return f"""\ -`{cmd}`: - -```text -{cmd_output} -```""" - - -class ManifestHeader: - """ManifestHeader doesn't fall under common interface, and we run it separately""" - - @staticmethod - def create_header( - short_image_name: str, registry: str, owner: str, build_timestamp: str - ) -> str: - commit_hash = GitHelper.commit_hash() - commit_hash_tag = GitHelper.commit_hash_tag() - commit_message = GitHelper.commit_message() - - # Unfortunately, `docker images` doesn't work when specifying `docker.io` as registry - fixed_registry = registry + "/" if registry != "docker.io" else "" - - image_size = docker[ - "images", - f"{fixed_registry}{owner}/{short_image_name}:latest", - "--format", - "{{.Size}}", - ]().rstrip() - - return f"""\ -# Build manifest for image: {short_image_name}:{commit_hash_tag} - -## Build Info - -- Build timestamp: {build_timestamp} -- Docker image: `{registry}/{owner}/{short_image_name}:{commit_hash_tag}` -- Docker image size: {image_size} -- Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash}) -- Git commit message: - -```text -{commit_message} -```""" - - -class ManifestInterface: - """Common interface for all manifests""" - - @staticmethod - def markdown_piece(container: Container) -> str: - raise NotImplementedError - - -class CondaEnvironmentManifest(ManifestInterface): - @staticmethod - def markdown_piece(container: Container) -> str: - return f"""\ -## Python Packages - -{DockerRunner.run_simple_command(container, "python --version")} - -{quoted_output(container, "conda info")} - -{quoted_output(container, "mamba info")} - -{quoted_output(container, "mamba list")}""" - - -class AptPackagesManifest(ManifestInterface): - @staticmethod - def markdown_piece(container: Container) -> str: - return f"""\ -## Apt Packages - -{quoted_output(container, "apt list --installed")}""" - - -class RPackagesManifest(ManifestInterface): - @staticmethod - def markdown_piece(container: Container) -> str: - return f"""\ -## R Packages - -{quoted_output(container, "R --version")} - -{quoted_output(container, "R --silent -e 'installed.packages(.Library)[, c(1,3)]'")}""" - - -class JuliaPackagesManifest(ManifestInterface): - @staticmethod - def markdown_piece(container: Container) -> str: - return f"""\ -## Julia Packages - -{quoted_output(container, "julia -E 'using InteractiveUtils; versioninfo()'")} - -{quoted_output(container, "julia -E 'import Pkg; Pkg.status()'")}""" - - -class SparkInfoManifest(ManifestInterface): - @staticmethod - def markdown_piece(container: Container) -> str: - return f"""\ -## Apache Spark - -{quoted_output(container, "/usr/local/spark/bin/spark-submit --version")}""" diff --git a/tagging/manifests/__init__.py b/tagging/manifests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tagging/manifests/apt_packages.py b/tagging/manifests/apt_packages.py new file mode 100644 index 0000000000..939e1ac0d4 --- /dev/null +++ b/tagging/manifests/apt_packages.py @@ -0,0 +1,15 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from docker.models.containers import Container + +from tagging.manifests.manifest_interface import ManifestInterface +from tagging.utils.quoted_output import quoted_output + + +class AptPackagesManifest(ManifestInterface): + @staticmethod + def markdown_piece(container: Container) -> str: + return f"""\ +## Apt Packages + +{quoted_output(container, "apt list --installed")}""" diff --git a/tagging/manifests/conda_environment.py b/tagging/manifests/conda_environment.py new file mode 100644 index 0000000000..a2f8ceefff --- /dev/null +++ b/tagging/manifests/conda_environment.py @@ -0,0 +1,22 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from docker.models.containers import Container + +from tagging.manifests.manifest_interface import ManifestInterface +from tagging.utils.docker_runner import DockerRunner +from tagging.utils.quoted_output import quoted_output + + +class CondaEnvironmentManifest(ManifestInterface): + @staticmethod + def markdown_piece(container: Container) -> str: + return f"""\ +## Python Packages + +{DockerRunner.run_simple_command(container, "python --version")} + +{quoted_output(container, "conda info")} + +{quoted_output(container, "mamba info")} + +{quoted_output(container, "mamba list")}""" diff --git a/tagging/manifests/header.py b/tagging/manifests/header.py new file mode 100644 index 0000000000..f54c3c705f --- /dev/null +++ b/tagging/manifests/header.py @@ -0,0 +1,44 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import plumbum + +from tagging.utils.git_helper import GitHelper + +docker = plumbum.local["docker"] + + +class ManifestHeader: + """ManifestHeader doesn't fall under common interface, and we run it separately""" + + @staticmethod + def create_header( + short_image_name: str, registry: str, owner: str, build_timestamp: str + ) -> str: + commit_hash = GitHelper.commit_hash() + commit_hash_tag = GitHelper.commit_hash_tag() + commit_message = GitHelper.commit_message() + + # Unfortunately, `docker images` doesn't work when specifying `docker.io` as registry + fixed_registry = registry + "/" if registry != "docker.io" else "" + + image_size = docker[ + "images", + f"{fixed_registry}{owner}/{short_image_name}:latest", + "--format", + "{{.Size}}", + ]().rstrip() + + return f"""\ +# Build manifest for image: {short_image_name}:{commit_hash_tag} + +## Build Info + +- Build timestamp: {build_timestamp} +- Docker image: `{registry}/{owner}/{short_image_name}:{commit_hash_tag}` +- Docker image size: {image_size} +- Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash}) +- Git commit message: + +```text +{commit_message} +```""" diff --git a/tagging/manifests/julia_packages.py b/tagging/manifests/julia_packages.py new file mode 100644 index 0000000000..23f4036ba8 --- /dev/null +++ b/tagging/manifests/julia_packages.py @@ -0,0 +1,17 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from docker.models.containers import Container + +from tagging.manifests.manifest_interface import ManifestInterface +from tagging.utils.quoted_output import quoted_output + + +class JuliaPackagesManifest(ManifestInterface): + @staticmethod + def markdown_piece(container: Container) -> str: + return f"""\ +## Julia Packages + +{quoted_output(container, "julia -E 'using InteractiveUtils; versioninfo()'")} + +{quoted_output(container, "julia -E 'import Pkg; Pkg.status()'")}""" diff --git a/tagging/manifests/manifest_interface.py b/tagging/manifests/manifest_interface.py new file mode 100644 index 0000000000..2677810b4a --- /dev/null +++ b/tagging/manifests/manifest_interface.py @@ -0,0 +1,9 @@ +from docker.models.containers import Container + + +class ManifestInterface: + """Common interface for all manifests""" + + @staticmethod + def markdown_piece(container: Container) -> str: + raise NotImplementedError diff --git a/tagging/manifests/r_packages.py b/tagging/manifests/r_packages.py new file mode 100644 index 0000000000..961f43c131 --- /dev/null +++ b/tagging/manifests/r_packages.py @@ -0,0 +1,17 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from docker.models.containers import Container + +from tagging.manifests.manifest_interface import ManifestInterface +from tagging.utils.quoted_output import quoted_output + + +class RPackagesManifest(ManifestInterface): + @staticmethod + def markdown_piece(container: Container) -> str: + return f"""\ +## R Packages + +{quoted_output(container, "R --version")} + +{quoted_output(container, "R --silent -e 'installed.packages(.Library)[, c(1,3)]'")}""" diff --git a/tagging/manifests/spark_info.py b/tagging/manifests/spark_info.py new file mode 100644 index 0000000000..20bbc7e878 --- /dev/null +++ b/tagging/manifests/spark_info.py @@ -0,0 +1,15 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from docker.models.containers import Container + +from tagging.manifests.manifest_interface import ManifestInterface +from tagging.utils.quoted_output import quoted_output + + +class SparkInfoManifest(ManifestInterface): + @staticmethod + def markdown_piece(container: Container) -> str: + return f"""\ +## Apache Spark + +{quoted_output(container, "/usr/local/spark/bin/spark-submit --version")}""" diff --git a/tagging/taggers/__init__.py b/tagging/taggers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tagging/taggers/date.py b/tagging/taggers/date.py new file mode 100644 index 0000000000..d03b4f4b9d --- /dev/null +++ b/tagging/taggers/date.py @@ -0,0 +1,13 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import datetime + +from docker.models.containers import Container + +from tagging.taggers.tagger_interface import TaggerInterface + + +class DateTagger(TaggerInterface): + @staticmethod + def tag_value(container: Container) -> str: + return datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d") diff --git a/tagging/taggers/sha.py b/tagging/taggers/sha.py new file mode 100644 index 0000000000..675cb47d52 --- /dev/null +++ b/tagging/taggers/sha.py @@ -0,0 +1,12 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from docker.models.containers import Container + +from tagging.taggers.tagger_interface import TaggerInterface +from tagging.utils.git_helper import GitHelper + + +class SHATagger(TaggerInterface): + @staticmethod + def tag_value(container: Container) -> str: + return GitHelper.commit_hash_tag() diff --git a/tagging/taggers/tagger_interface.py b/tagging/taggers/tagger_interface.py new file mode 100644 index 0000000000..2540815158 --- /dev/null +++ b/tagging/taggers/tagger_interface.py @@ -0,0 +1,9 @@ +from docker.models.containers import Container + + +class TaggerInterface: + """Common interface for all taggers""" + + @staticmethod + def tag_value(container: Container) -> str: + raise NotImplementedError diff --git a/tagging/taggers/ubuntu_version.py b/tagging/taggers/ubuntu_version.py new file mode 100644 index 0000000000..e3ea133e5e --- /dev/null +++ b/tagging/taggers/ubuntu_version.py @@ -0,0 +1,19 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from docker.models.containers import Container + +from tagging.taggers.tagger_interface import TaggerInterface +from tagging.utils.docker_runner import DockerRunner + + +class UbuntuVersionTagger(TaggerInterface): + @staticmethod + def tag_value(container: Container) -> str: + os_release = DockerRunner.run_simple_command( + container, + "cat /etc/os-release", + ).split("\n") + for line in os_release: + if line.startswith("VERSION_ID"): + return "ubuntu-" + line.split("=")[1].strip('"') + raise RuntimeError(f"did not find ubuntu version in: {os_release}") diff --git a/tagging/taggers.py b/tagging/taggers/versions.py similarity index 75% rename from tagging/taggers.py rename to tagging/taggers/versions.py index 5e0400d9a8..f7571b7656 100644 --- a/tagging/taggers.py +++ b/tagging/taggers/versions.py @@ -1,11 +1,9 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -import datetime - from docker.models.containers import Container -from tagging.docker_runner import DockerRunner -from tagging.git_helper import GitHelper +from tagging.taggers.tagger_interface import TaggerInterface +from tagging.utils.docker_runner import DockerRunner def _get_program_version(container: Container, program: str) -> str: @@ -25,39 +23,6 @@ def _get_pip_package_version(container: Container, package: str) -> str: return version_line[len(PIP_VERSION_PREFIX) :] -class TaggerInterface: - """Common interface for all taggers""" - - @staticmethod - def tag_value(container: Container) -> str: - raise NotImplementedError - - -class SHATagger(TaggerInterface): - @staticmethod - def tag_value(container: Container) -> str: - return GitHelper.commit_hash_tag() - - -class DateTagger(TaggerInterface): - @staticmethod - def tag_value(container: Container) -> str: - return datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d") - - -class UbuntuVersionTagger(TaggerInterface): - @staticmethod - def tag_value(container: Container) -> str: - os_release = DockerRunner.run_simple_command( - container, - "cat /etc/os-release", - ).split("\n") - for line in os_release: - if line.startswith("VERSION_ID"): - return "ubuntu-" + line.split("=")[1].strip('"') - raise RuntimeError(f"did not find ubuntu version in: {os_release}") - - class PythonVersionTagger(TaggerInterface): @staticmethod def tag_value(container: Container) -> str: diff --git a/tagging/utils/__init__.py b/tagging/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tagging/docker_runner.py b/tagging/utils/docker_runner.py similarity index 100% rename from tagging/docker_runner.py rename to tagging/utils/docker_runner.py diff --git a/tagging/get_platform.py b/tagging/utils/get_platform.py similarity index 100% rename from tagging/get_platform.py rename to tagging/utils/get_platform.py diff --git a/tagging/get_prefix.py b/tagging/utils/get_prefix.py similarity index 92% rename from tagging/get_prefix.py rename to tagging/utils/get_prefix.py index 6e7c718576..8fedbf4864 100644 --- a/tagging/get_prefix.py +++ b/tagging/utils/get_prefix.py @@ -1,6 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -from tagging.get_platform import get_platform +from tagging.utils.get_platform import get_platform DEFAULT_VARIANT = "default" diff --git a/tagging/git_helper.py b/tagging/utils/git_helper.py similarity index 100% rename from tagging/git_helper.py rename to tagging/utils/git_helper.py diff --git a/tagging/utils/quoted_output.py b/tagging/utils/quoted_output.py new file mode 100644 index 0000000000..4df8b34ea6 --- /dev/null +++ b/tagging/utils/quoted_output.py @@ -0,0 +1,22 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +from docker.models.containers import Container + +from tagging.utils.docker_runner import DockerRunner + + +def quoted_output(container: Container, cmd: str) -> str: + cmd_output = DockerRunner.run_simple_command(container, cmd, print_result=False) + # For example, `mamba info` adds redundant empty lines + cmd_output = cmd_output.strip("\n") + # For example, R packages list contains trailing backspaces + cmd_output = "\n".join(line.rstrip() for line in cmd_output.split("\n")) + + assert cmd_output, f"Command `{cmd}` returned empty output" + + return f"""\ +`{cmd}`: + +```text +{cmd_output} +```""" diff --git a/tagging/Home.md b/wiki/Home.md similarity index 100% rename from tagging/Home.md rename to wiki/Home.md diff --git a/wiki/__init__.py b/wiki/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tagging/update_wiki.py b/wiki/update_wiki.py similarity index 100% rename from tagging/update_wiki.py rename to wiki/update_wiki.py