feat: reusable containers

adresses testcontainers#109

Author: matthiasschaub

core/testcontainers/core/config.py

@@ -39,7 +39,10 @@ def read_tc_properties() -> dict[str, str]:
     return settings
 
 
-_WARNINGS = {"DOCKER_AUTH_CONFIG": "DOCKER_AUTH_CONFIG is experimental, see testcontainers/testcontainers-python#566"}
+_WARNINGS = {
+    "DOCKER_AUTH_CONFIG": "DOCKER_AUTH_CONFIG is experimental, see testcontainers/testcontainers-python#566",
+    "tc_properties_get_tc_host": "this method has moved to property 'tc_properties_tc_host'",
+}
 
 
 @dataclass
@@ -73,8 +76,19 @@ def docker_auth_config(self, value: str):
         self._docker_auth_config = value
 
     def tc_properties_get_tc_host(self) -> Union[str, None]:
+        if "tc_properties_get_tc_host" in _WARNINGS:
+            warning(_WARNINGS.pop("tc_properties_get_tc_host"))
         return self.tc_properties.get("tc.host")
 
+    @property
+    def tc_properties_tc_host(self) -> Union[str, None]:
+        return self.tc_properties.get("tc.host")
+
+    @property
+    def tc_properties_testcontainers_reuse_enable(self) -> bool:
+        enabled = self.tc_properties.get("testcontainers.reuse.enable")
+        return enabled == "true"
+
     @property
     def timeout(self):
         return self.max_tries * self.sleep_time

core/testcontainers/core/container.py

@@ -1,4 +1,6 @@
 import contextlib
+import hashlib
+import logging
 from platform import system
 from socket import socket
 from typing import TYPE_CHECKING, Optional
@@ -49,6 +51,7 @@ def __init__(
         self._name = None
         self._network: Optional[Network] = None
         self._network_aliases: Optional[list[str]] = None
+        self._reuse: bool = False
         self._kwargs = kwargs
 
     def with_env(self, key: str, value: str) -> Self:
@@ -76,6 +79,10 @@ def with_kwargs(self, **kwargs) -> Self:
         self._kwargs = kwargs
         return self
 
+    def with_reuse(self, reuse=True) -> Self:
+        self._reuse = reuse
+        return self
+
     def maybe_emulate_amd64(self) -> Self:
         if is_arm():
             return self.with_kwargs(platform="linux/amd64")
@@ -86,8 +93,49 @@ def start(self) -> Self:
             logger.debug("Creating Ryuk container")
             Reaper.get_instance()
         logger.info("Pulling image %s", self.image)
-        docker_client = self.get_docker_client()
         self._configure()
+
+        # container hash consisting of run arguments
+        args = (
+            self.image,
+            self._command,
+            self.env,
+            self.ports,
+            self._name,
+            self.volumes,
+            str(tuple(sorted(self._kwargs.items()))),
+        )
+        hash_ = hashlib.sha256(bytes(str(args), encoding="utf-8")).hexdigest()
+
+        # TODO: check also if ryuk is disabled
+        if self._reuse and not c.tc_properties_testcontainers_reuse_enable:
+            logging.warning(
+                "Reuse was requested (`with_reuse`) but the environment does not "
+                + "support the reuse of containers. To enable container reuse, add "
+                + "the property 'testcontainers.reuse.enable=true' to a file at "
+                + "~/.testcontainers.properties (you may need to create it)."
+            )
+
+        if self._reuse and c.tc_properties_testcontainers_reuse_enable:
+            docker_client = self.get_docker_client()
+            container = docker_client.find_container_by_hash(hash_)
+            if container:
+                if container.status != "running":
+                    container.start()
+                    logger.info("Existing container started: %s", container.id)
+                logger.info("Container is already running: %s", container.id)
+                self._container = container
+            else:
+                self._start(hash_)
+        else:
+            self._start(hash_)
+
+        if self._network:
+            self._network.connect(self._container.id, self._network_aliases)
+        return self
+
+    def _start(self, hash_):
+        docker_client = self.get_docker_client()
         self._container = docker_client.run(
             self.image,
             command=self._command,
@@ -96,16 +144,17 @@ def start(self) -> Self:
             ports=self.ports,
             name=self._name,
             volumes=self.volumes,
+            labels={"hash": hash_},
             **self._kwargs,
         )
         logger.info("Container started: %s", self._container.short_id)
-        if self._network:
-            self._network.connect(self._container.id, self._network_aliases)
-        return self
 
     def stop(self, force=True, delete_volume=True) -> None:
         if self._container:
-            self._container.remove(force=force, v=delete_volume)
+            if self._reuse and c.tc_properties_testcontainers_reuse_enable:
+                self._container.stop()
+            else:
+                self._container.remove(force=force, v=delete_volume)
         self.get_docker_client().client.close()
 
     def __enter__(self) -> Self:

core/testcontainers/core/docker_client.py

@@ -215,9 +215,15 @@ def client_networks_create(self, name: str, param: dict):
         labels = create_labels("", param.get("labels"))
         return self.client.networks.create(name, **{**param, "labels": labels})
 
+    def find_container_by_hash(self, hash_: str) -> Container | None:
+        for container in self.client.containers.list(all=True):
+            if container.labels.get("hash", None) == hash_:
+                return container
+        return None
+
 
 def get_docker_host() -> Optional[str]:
-    return c.tc_properties_get_tc_host() or os.getenv("DOCKER_HOST")
+    return c.tc_properties_tc_host or os.getenv("DOCKER_HOST")
 
 
 def get_docker_auth_config() -> Optional[str]:

core/tests/test_reusable_containers.py

@@ -0,0 +1,83 @@
+from time import sleep
+
+from docker.models.containers import Container
+
+from testcontainers.core.config import testcontainers_config
+from testcontainers.core.container import DockerContainer
+from testcontainers.core.docker_client import DockerClient
+from testcontainers.core.waiting_utils import wait_for_logs
+from testcontainers.core.container import Reaper
+
+
+def test_docker_container_reuse_default():
+    with DockerContainer("hello-world") as container:
+        assert container._reuse == False
+        id = container._container.id
+        wait_for_logs(container, "Hello from Docker!")
+    containers = DockerClient().client.containers.list(all=True)
+    assert id not in [container.id for container in containers]
+
+
+def test_docker_container_with_reuse_reuse_disabled():
+    with DockerContainer("hello-world").with_reuse() as container:
+        assert container._reuse == True
+        id = container._container.id
+        wait_for_logs(container, "Hello from Docker!")
+    containers = DockerClient().client.containers.list(all=True)
+    assert id not in [container.id for container in containers]
+
+
+def test_docker_container_with_reuse_reuse_enabled_ryuk_enabled(monkeypatch):
+    # Make sure Ryuk cleanup is not active from previous test runs
+    Reaper.delete_instance()
+    tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"}
+    monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock)
+    monkeypatch.setattr(testcontainers_config, "ryuk_reconnection_timeout", "0.1s")
+
+    with DockerContainer("hello-world").with_reuse() as container:
+        id = container._container.id
+        wait_for_logs(container, "Hello from Docker!")
+
+    Reaper._socket.close()
+    # Sleep until Ryuk reaps all dangling containers
+    sleep(0.6)
+
+    containers = DockerClient().client.containers.list(all=True)
+    assert id not in [container.id for container in containers]
+
+    # Cleanup Ryuk class fields after manual Ryuk shutdown
+    Reaper.delete_instance()
+
+
+def test_docker_container_with_reuse_reuse_enabled_ryuk_disabled(monkeypatch):
+    # Make sure Ryuk cleanup is not active from previous test runs
+    Reaper.delete_instance()
+    tc_properties_mock = testcontainers_config.tc_properties | {"testcontainers.reuse.enable": "true"}
+    monkeypatch.setattr(testcontainers_config, "tc_properties", tc_properties_mock)
+    monkeypatch.setattr(testcontainers_config, "ryuk_disabled", True)
+    with DockerContainer("hello-world").with_reuse() as container:
+        assert container._reuse == True
+        id = container._container.id
+        wait_for_logs(container, "Hello from Docker!")
+    containers = DockerClient().client.containers.list(all=True)
+    assert id in [container.id for container in containers]
+    # Cleanup after keeping container alive (with_reuse)
+    container._container.remove(force=True)
+
+
+def test_docker_container_labels_hash():
+    expected_hash = "91fde3c09244e1d3ec6f18a225b9261396b9a1cb0f6365b39b9795782817c128"
+    with DockerContainer("hello-world").with_reuse() as container:
+        assert container._container.labels["hash"] == expected_hash
+
+
+def test_docker_client_find_container_by_hash_not_existing():
+    with DockerContainer("hello-world"):
+        assert DockerClient().find_container_by_hash("foo") == None
+
+
+def test_docker_client_find_container_by_hash_existing():
+    with DockerContainer("hello-world").with_reuse() as container:
+        hash_ = container._container.labels["hash"]
+        found_container = DockerClient().find_container_by_hash(hash_)
+        assert isinstance(found_container, Container)