Better tagging directory structure (#2228)

Author: mathbunnyru

.github/workflows/docker-build-test-upload.yml

@@ -78,7 +78,7 @@ jobs:
 
       - name: Write tags file 🏷
         run: >
-          python3 -m tagging.write_tags_file
+          python3 -m tagging.apps.write_tags_file
           --registry ${{ env.REGISTRY }}
           --owner ${{ env.OWNER }}
           --short-image-name ${{ inputs.image }}
@@ -94,7 +94,7 @@ jobs:
 
       - name: Write manifest and build history file 🏷
         run: >
-          python3 -m tagging.write_manifest
+          python3 -m tagging.apps.write_manifest
           --registry ${{ env.REGISTRY }}
           --owner ${{ env.OWNER }}
           --short-image-name ${{ inputs.image }}

.github/workflows/docker-merge-tags.yml

@@ -68,7 +68,7 @@ jobs:
       - name: Merge tags for the images 🔀
         if: env.PUSH_TO_REGISTRY == 'true'
         run: >
-          python3 -m tagging.merge_tags
+          python3 -m tagging.apps.merge_tags
           --short-image-name ${{ inputs.image }}
           --variant ${{ inputs.variant }}
           --tags-dir /tmp/jupyter/tags/

.github/workflows/docker-tag-push.yml

@@ -62,7 +62,7 @@ jobs:
           path: /tmp/jupyter/tags/
       - name: Apply tags to the loaded image 🏷
         run: >
-          python3 -m tagging.apply_tags
+          python3 -m tagging.apps.apply_tags
           --registry ${{ env.REGISTRY }}
           --owner ${{ env.OWNER }}
           --short-image-name ${{ inputs.image }}

.github/workflows/docker-wiki-update.yml

@@ -37,12 +37,12 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}.wiki
-          path: wiki/
+          path: wiki_src/
 
       - name: Update wiki 🏷
         run: >
-          python3 -m tagging.update_wiki
-          --wiki-dir wiki/
+          python3 -m wiki.update_wiki
+          --wiki-dir wiki_src/
           --hist-lines-dir /tmp/jupyter/hist_lines/
           --manifests-dir /tmp/jupyter/manifests/
           --repository ${{ github.repository }}

.github/workflows/docker.yml

@@ -30,6 +30,7 @@ on:
       - "!tagging/README.md"
       - "tests/**"
       - "!tests/README.md"
+      - "wiki/**"
       - "requirements-dev.txt"
   push:
     branches:
@@ -50,6 +51,7 @@ on:
       - "!tagging/README.md"
       - "tests/**"
       - "!tests/README.md"
+      - "wiki/**"
       - "requirements-dev.txt"
   workflow_dispatch:
 

CHANGELOG.md

@@ -3,6 +3,12 @@
 This changelog only contains breaking and/or significant changes manually introduced to this repository (using Pull Requests).
 All image manifests can be found in [the wiki](https://github.com/jupyter/docker-stacks/wiki).
 
+## 2025-02-21
+
+Affected: all images.
+
+- **Non-breaking:**: Better tagging directory structure ([#2228](https://github.com/jupyter/docker-stacks/pull/2228)).
+
 ## 2025-02-18
 
 Affected: all images.

Makefile

@@ -78,20 +78,20 @@ linkcheck-docs: ## check broken links
 
 hook/%: VARIANT?=default
 hook/%: ## run post-build hooks for an image
-	python3 -m tagging.write_tags_file \
+	python3 -m tagging.apps.write_tags_file \
 	  --registry "$(REGISTRY)" \
 	  --owner "$(OWNER)" \
 	  --short-image-name "$(notdir $@)" \
 	  --variant "$(VARIANT)" \
 	  --tags-dir /tmp/jupyter/tags/
-	python3 -m tagging.write_manifest \
+	python3 -m tagging.apps.write_manifest \
 	  --registry "$(REGISTRY)" \
 	  --owner "$(OWNER)" \
 	  --short-image-name "$(notdir $@)" \
 	  --variant "$(VARIANT)" \
 	  --hist-lines-dir /tmp/jupyter/hist_lines/ \
 	  --manifests-dir /tmp/jupyter/manifests/
-	python3 -m tagging.apply_tags \
+	python3 -m tagging.apps.apply_tags \
 	  --registry "$(REGISTRY)" \
 	  --owner "$(OWNER)" \
 	  --short-image-name "$(notdir $@)" \

docs/index.rst

@@ -35,6 +35,7 @@ Table of Contents
    :caption: Maintainer Guide
 
    maintaining/new-images-and-packages-policy
+   maintaining/tagging
    maintaining/tasks
 
 .. toctree::

docs/maintaining/tagging.md

@@ -0,0 +1,94 @@
+# Tagging and manifest creation
+
+The main purpose of the source code in [the `tagging` folder](https://github.com/jupyter/docker-stacks/tree/main/tagging) is to properly write tag files and manifests for single-platform images,
+apply these tags, and merge single-platform images into one multi-arch image.
+
+## What is a tag and a manifest
+
+A tag is a label attached to a Docker image identifying specific attributes or versions.
+For example, an image `jupyter/base-notebook` with Python 3.10.5 will have a full image name `quay.io/jupyter/base-notebook:python-3.10.5`.
+These tags are pushed to our [Quay.io registry](https://quay.io/organization/jupyter).
+
+A manifest is a description of important image attributes written in Markdown format.
+For example, we dump all `conda` packages with their versions into the manifest.
+
+## Main principles
+
+- All images are organized in a hierarchical tree.
+  More info on [image relationships](../using/selecting.md#image-relationships).
+- Classes inherit from `TaggerInterface` and `ManifestInterface` to generate tags and manifest pieces by running commands in Docker containers.
+- Tags and manifests are reevaluated for each image in the hierarchy since values may change between parent and child images.
+- To tag an image and create its manifest, run `make hook/<somestack>` (e.g., `make hook/base-notebook`).
+
+## Utils
+
+### DockerRunner
+
+`DockerRunner` is a helper class to easily run a docker container and execute commands inside this container:
+
+```{literalinclude} tagging_examples/docker_runner.py
+:language: py
+```
+
+### GitHelper
+
+`GitHelper` methods are run in the current `git` repo and give the information about the last commit hash and commit message:
+
+```{literalinclude} tagging_examples/git_helper.py
+:language: py
+```
+
+The prefix of commit hash (namely, 12 letters) is used as an image tag to make it easy to inherit from a fixed version of a docker image.
+
+## Taggers and Manifests
+
+### Tagger
+
+`Tagger` is a class that can be run inside a docker container to calculate some tag for an image.
+
+All the taggers are inherited from `TaggerInterface`:
+
+```{literalinclude} ../../tagging/taggers/tagger_interface.py
+:language: py
+```
+
+So, the `tag_value(container)` method gets a docker container as an input and returns a tag.
+
+`SHATagger` example:
+
+```{literalinclude} ../../tagging/taggers/sha.py
+:language: py
+```
+
+- `taggers/` subdirectory contains all the taggers.
+- `apps/write_tags_file.py`, `apps/apply_tags.py`, and `apps/merge_tags.py` are Python executable used to write tags for an image, apply tags from a file, and create multi-arch images.
+
+### Manifest
+
+`ManifestHeader` is a build manifest header.
+It contains the following sections: `Build timestamp`, `Docker image size`, and `Git commit` info.
+
+All the other manifest classes are inherited from `ManifestInterface`:
+
+```{literalinclude} ../../tagging/manifests/manifest_interface.py
+:language: py
+```
+
+- The `markdown_piece(container)` method returns a piece of markdown file to be used as a part of the build manifest.
+
+`AptPackagesManifest` example:
+
+```{literalinclude} ../../tagging/manifests/apt_packages.py
+:language: py
+```
+
+- `quoted_output(container, cmd)` simply runs the command inside a container using `DockerRunner.run_simple_command` and wraps it to triple quotes to create a valid markdown piece.
+  It also adds the command which was run to the markdown piece.
+- `manifests/` subdirectory contains all the manifests.
+- `apps/write_manifest.py` is a Python executable to create the build manifest and history line for an image.
+
+### Images Hierarchy
+
+All images' dependencies on each other and what taggers and manifest are applicable to them are defined in `hierarchy/images_hierarchy.py`.
+
+`hierarchy/get_taggers_and_manifests.py` defines a function to get the taggers and manifests for a specific image.

docs/maintaining/tagging_examples/docker_runner.py

@@ -0,0 +1,4 @@
+from tagging.utils.docker_runner import DockerRunner
+
+with DockerRunner("ubuntu") as container:
+    DockerRunner.run_simple_command(container, cmd="env", print_result=True)

docs/maintaining/tagging_examples/git_helper.py

@@ -0,0 +1,4 @@
+from tagging.utils.git_helper import GitHelper
+
+print("Git hash:", GitHelper.commit_hash())
+print("Git message:", GitHelper.commit_message())

tagging/README.md

@@ -1,126 +1,3 @@
 # Docker stacks tagging and manifest creation
 
-The main purpose of the source code in this folder is to properly tag all the images and to update [build manifests](https://github.com/jupyter/docker-stacks/wiki).
-These two processes are closely related, so the source code is widely reused.
-
-A basic example of a tag is a `Python` version tag.
-For example, an image `jupyter/base-notebook` with `python 3.10.5` will have a full image name `quay.io/jupyter/base-notebook:python-3.10.5`.
-This tag (and all the other tags) are pushed to Quay.io.
-
-Manifest is a description of some important part of the image in a `markdown`.
-For example, we dump all the `conda` packages, including their versions.
-
-## Main principles
-
-- All the images are located in a hierarchical tree.
-  More info on [image relationships](../docs/using/selecting.md#image-relationships).
-- We have `tagger` and `manifest` classes, which can be run inside docker containers to obtain tags and build manifest pieces.
-- These classes are inherited from the parent image to all the child images.
-- Because manifests and tags might change from parent to child, `taggers` and `manifests` are reevaluated on each image.
-  So, the values are not inherited.
-- To tag an image and create a manifest, run `make hook/base-notebook` (or another image of your choice).
-
-## Source code description
-
-In this section, we will briefly describe the source code in this folder and give examples of how to use it.
-
-### DockerRunner
-
-`DockerRunner` is a helper class to easily run a docker container and execute commands inside this container:
-
-```python
-from tagging.docker_runner import DockerRunner
-
-with DockerRunner("ubuntu:22.04") as container:
-    DockerRunner.run_simple_command(container, cmd="env", print_result=True)
-```
-
-### GitHelper
-
-`GitHelper` methods are run in the current `git` repo and give the information about the last commit hash and commit message:
-
-```python
-from tagging.git_helper import GitHelper
-
-print("Git hash:", GitHelper.commit_hash())
-print("Git message:", GitHelper.commit_message())
-```
-
-The prefix of commit hash (namely, 12 letters) is used as an image tag to make it easy to inherit from a fixed version of a docker image.
-
-### Tagger
-
-`Tagger` is a class that can be run inside a docker container to calculate some tag for an image.
-
-All the taggers are inherited from `TaggerInterface`:
-
-```python
-class TaggerInterface:
-    """Common interface for all taggers"""
-
-    @staticmethod
-    def tag_value(container) -> str:
-        raise NotImplementedError
-```
-
-So, the `tag_value(container)` method gets a docker container as an input and returns a tag.
-
-`SHATagger` example:
-
-```python
-from tagging.git_helper import GitHelper
-from tagging.taggers import TaggerInterface
-
-
-class SHATagger(TaggerInterface):
-    @staticmethod
-    def tag_value(container):
-        return GitHelper.commit_hash_tag()
-```
-
-- `taggers.py` contains all the taggers.
-- `tag_image.py` is a Python executable that is used to tag the image.
-
-### Manifest
-
-`ManifestHeader` is a build manifest header.
-It contains the following sections: `Build timestamp`, `Docker image size`, and `Git commit` info.
-
-All the other manifest classes are inherited from `ManifestInterface`:
-
-```python
-class ManifestInterface:
-    """Common interface for all manifests"""
-
-    @staticmethod
-    def markdown_piece(container) -> str:
-        raise NotImplementedError
-```
-
-- The `markdown_piece(container)` method returns a piece of markdown file to be used as a part of the build manifest.
-
-`AptPackagesManifest` example:
-
-```python
-from tagging.manifests import ManifestInterface, quoted_output
-
-
-class AptPackagesManifest(ManifestInterface):
-    @staticmethod
-    def markdown_piece(container) -> str:
-        return f"""\
-## Apt Packages
-
-{quoted_output(container, "apt list --installed")}"""
-```
-
-- `quoted_output` simply runs the command inside a container using `DockerRunner.run_simple_command` and wraps it to triple quotes to create a valid markdown piece.
-  It also adds the command which was run to the markdown piece.
-- `manifests.py` contains all the manifests.
-- `write_manifest.py` is a Python executable that is used to create the build manifest and history line for an image.
-
-### Images Hierarchy
-
-All images' dependencies on each other and what taggers and manifest they make use of are defined in `images_hierarchy.py`.
-
-`get_taggers_and_manifests.py` defines a helper function to get the taggers and manifests for a specific image.
+Please, refer to the [tagging section of documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/maintaing/tagging.html) to see how tags and manifests are created.

tagging/apps/__init__.py

@@ -6,9 +6,9 @@
 
 import plumbum
 
-from tagging.common_arguments import common_arguments_parser
-from tagging.get_platform import unify_aarch64
-from tagging.get_prefix import get_file_prefix_for_platform
+from tagging.apps.common_cli_arguments import common_arguments_parser
+from tagging.utils.get_platform import unify_aarch64
+from tagging.utils.get_prefix import get_file_prefix_for_platform
 
 docker = plumbum.local["docker"]
 

tagging/apply_tags.py renamed to tagging/apps/apply_tags.py

@@ -6,9 +6,9 @@
 
 import plumbum
 
-from tagging.common_arguments import common_arguments_parser
-from tagging.get_platform import ALL_PLATFORMS
-from tagging.get_prefix import get_file_prefix_for_platform
+from tagging.apps.common_cli_arguments import common_arguments_parser
+from tagging.utils.get_platform import ALL_PLATFORMS
+from tagging.utils.get_prefix import get_file_prefix_for_platform
 
 docker = plumbum.local["docker"]
 

tagging/common_arguments.py renamed to tagging/apps/common_cli_arguments.py

@@ -7,12 +7,15 @@
 
 from docker.models.containers import Container
 
-from tagging.common_arguments import common_arguments_parser
-from tagging.docker_runner import DockerRunner
-from tagging.get_prefix import get_file_prefix, get_tag_prefix
-from tagging.get_taggers_and_manifests import get_taggers_and_manifests
-from tagging.git_helper import GitHelper
-from tagging.manifests import ManifestHeader, ManifestInterface
+from tagging.apps.common_cli_arguments import common_arguments_parser
+from tagging.hierarchy.get_taggers_and_manifests import (
+    get_taggers_and_manifests,
+)
+from tagging.manifests.header import ManifestHeader
+from tagging.manifests.manifest_interface import ManifestInterface
+from tagging.utils.docker_runner import DockerRunner
+from tagging.utils.get_prefix import get_file_prefix, get_tag_prefix
+from tagging.utils.git_helper import GitHelper
 
 LOGGER = logging.getLogger(__name__)