Support inode resolution mechanism for Origin Detection (#813)

* [dogstatsd] Send in-<inode> when container_id cannot be retrieved

Signed-off-by: Wassim DHIF <[email protected]>

* Update datadog/dogstatsd/container.py

Co-authored-by: Vickenty Fesunov <[email protected]>

* Update datadog/dogstatsd/container.py

Co-authored-by: Vickenty Fesunov <[email protected]>

* Update datadog/dogstatsd/container.py

Co-authored-by: Vickenty Fesunov <[email protected]>

* Update datadog/dogstatsd/container.py

Co-authored-by: Vickenty Fesunov <[email protected]>

* Add inode > 2 case

Signed-off-by: Wassim DHIF <[email protected]>

* Rename _is_cgroup_namespace and add inode comment

Signed-off-by: Wassim DHIF <[email protected]>

* Add try/except for _is_host_cgroup_namespace()

Signed-off-by: Wassim DHIF <[email protected]>

* Add controller priority unittest

Signed-off-by: Wassim DHIF <[email protected]>

---------

Signed-off-by: Wassim DHIF <[email protected]>
Co-authored-by: Vickenty Fesunov <[email protected]>

Author: wdhif

datadog/dogstatsd/base.py

@@ -32,7 +32,7 @@
     DistributedContextManagerDecorator,
 )
 from datadog.dogstatsd.route import get_default_route
-from datadog.dogstatsd.container import ContainerID
+from datadog.dogstatsd.container import Cgroup
 from datadog.util.compat import is_p3k, text
 from datadog.util.format import normalize_tags
 from datadog.version import __version__
@@ -1288,7 +1288,7 @@ def _set_container_id(self, container_id, origin_detection_enabled):
             return
         if origin_detection_enabled:
             try:
-                reader = ContainerID()
+                reader = Cgroup()
                 self._container_id = reader.container_id
             except Exception as e:
                 log.debug("Couldn't get container ID: %s", str(e))

datadog/dogstatsd/container.py

@@ -4,6 +4,7 @@
 # Copyright 2015-Present Datadog, Inc
 
 import errno
+import os
 import re
 
 
@@ -13,31 +14,54 @@ class UnresolvableContainerID(Exception):
     """
 
 
-class ContainerID(object):
+class Cgroup(object):
     """
-    A reader class that retrieves the current container ID parsed from a the cgroup file.
+    A reader class that retrieves either:
+    - The current container ID parsed from the cgroup file
+    - The cgroup controller inode.
 
     Returns:
-    object: ContainerID
+    object: Cgroup
 
     Raises:
         `NotImplementedError`: No proc filesystem is found (non-Linux systems)
         `UnresolvableContainerID`: Unable to read the container ID
     """
 
     CGROUP_PATH = "/proc/self/cgroup"
+    CGROUP_MOUNT_PATH = "/sys/fs/cgroup"  # cgroup mount path.
+    CGROUP_NS_PATH = "/proc/self/ns/cgroup"  # path to the cgroup namespace file.
+    CGROUPV1_BASE_CONTROLLER = "memory"  # controller used to identify the container-id in cgroup v1 (memory).
+    CGROUPV2_BASE_CONTROLLER = ""  # controller used to identify the container-id in cgroup v2.
+    HOST_CGROUP_NAMESPACE_INODE = 0xEFFFFFFB  # inode of the host cgroup namespace.
+
     UUID_SOURCE = r"[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}"
     CONTAINER_SOURCE = r"[0-9a-f]{64}"
     TASK_SOURCE = r"[0-9a-f]{32}-\d+"
     LINE_RE = re.compile(r"^(\d+):([^:]*):(.+)$")
     CONTAINER_RE = re.compile(r"(?:.+)?({0}|{1}|{2})(?:\.scope)?$".format(UUID_SOURCE, CONTAINER_SOURCE, TASK_SOURCE))
 
     def __init__(self):
-        self.container_id = self._read_container_id(self.CGROUP_PATH)
+        if self._is_host_cgroup_namespace():
+            self.container_id = self._read_cgroup_path()
+            return
+        self.container_id = self._get_cgroup_from_inode()
+
+    def _is_host_cgroup_namespace(self):
+        """Check if the current process is in a host cgroup namespace."""
+        try:
+            return (
+                os.stat(self.CGROUP_NS_PATH).st_ino == self.HOST_CGROUP_NAMESPACE_INODE
+                if os.path.exists(self.CGROUP_NS_PATH)
+                else False
+            )
+        except Exception:
+            return False
 
-    def _read_container_id(self, fpath):
+    def _read_cgroup_path(self):
+        """Read the container ID from the cgroup file."""
         try:
-            with open(fpath, mode="r") as fp:
+            with open(self.CGROUP_PATH, mode="r") as fp:
                 for line in fp:
                     line = line.strip()
                     match = self.LINE_RE.match(line)
@@ -55,3 +79,33 @@ def _read_container_id(self, fpath):
         except Exception as e:
             raise UnresolvableContainerID("Unable to read the container ID: " + str(e))
         return None
+
+    def _get_cgroup_from_inode(self):
+        """Read the container ID from the cgroup inode."""
+        # Parse /proc/self/cgroup and get a map of controller to its associated cgroup node path.
+        cgroup_controllers_paths = {}
+        with open(self.CGROUP_PATH, mode="r") as fp:
+            for line in fp:
+                tokens = line.strip().split(":")
+                if len(tokens) != 3:
+                    continue
+                if tokens[1] == self.CGROUPV1_BASE_CONTROLLER or tokens[1] == self.CGROUPV2_BASE_CONTROLLER:
+                    cgroup_controllers_paths[tokens[1]] = tokens[2]
+
+        # Retrieve the cgroup inode from "/sys/fs/cgroup + controller + cgroupNodePath"
+        for controller in [
+            self.CGROUPV1_BASE_CONTROLLER,
+            self.CGROUPV2_BASE_CONTROLLER,
+        ]:
+            if controller in cgroup_controllers_paths:
+                inode_path = os.path.join(
+                    self.CGROUP_MOUNT_PATH,
+                    controller,
+                    cgroup_controllers_paths[controller] if cgroup_controllers_paths[controller] != "/" else "",
+                )
+                inode = os.stat(inode_path).st_ino
+                # 0 is not a valid inode. 1 is a bad block inode and 2 is the root of a filesystem.
+                if inode > 2:
+                    return "in-{0}".format(inode)
+
+        return None

tests/unit/dogstatsd/test_container.py

@@ -10,7 +10,7 @@
 import mock
 import pytest
 
-from datadog.dogstatsd.container import ContainerID
+from datadog.dogstatsd.container import Cgroup
 
 
 def get_mock_open(read_data=None):
@@ -125,12 +125,59 @@ def get_mock_open(read_data=None):
         ),
     ),
 )
-def test_container_id(file_contents, expected_container_id):
+def test_container_id_from_cgroup(file_contents, expected_container_id):
     with get_mock_open(read_data=file_contents) as mock_open:
         if file_contents is None:
             mock_open.side_effect = IOError
 
-        reader = ContainerID()
+        with mock.patch("os.stat", mock.MagicMock(return_value=mock.Mock(st_ino=0xEFFFFFFB))):
+            reader = Cgroup()
         assert expected_container_id == reader.container_id
 
         mock_open.assert_called_once_with("/proc/self/cgroup", mode="r")
+
+
+def test_container_id_inode():
+    """Test that the inode is returned when the container ID cannot be found."""
+    with mock.patch("datadog.dogstatsd.container.open", mock.mock_open(read_data="0::/")) as mock_open:
+        with mock.patch("os.stat", mock.MagicMock(return_value=mock.Mock(st_ino=1234))):
+            reader = Cgroup()
+        assert reader.container_id == "in-1234"
+        mock_open.assert_called_once_with("/proc/self/cgroup", mode="r")
+
+    cgroupv1_priority = """
+12:cpu,cpuacct:/
+11:hugetlb:/
+10:devices:/
+9:rdma:/
+8:net_cls,net_prio:/
+7:memory:/
+6:cpuset:/
+5:pids:/
+4:freezer:/
+3:perf_event:/
+2:blkio:/
+1:name=systemd:/
+0::/
+"""
+
+    paths_checked = []
+
+    def inode_stat_mock(path):
+        paths_checked.append(path)
+
+        # The cgroupv1 controller is mounted on inode 0. This will cause a fallback to the cgroupv2 controller.
+        if path == "/sys/fs/cgroup/memory/":
+            return mock.Mock(st_ino=0)
+        elif path == "/sys/fs/cgroup/":
+            return mock.Mock(st_ino=1234)
+
+    with mock.patch("datadog.dogstatsd.container.open", mock.mock_open(read_data=cgroupv1_priority)) as mock_open:
+        with mock.patch("os.stat", mock.MagicMock(side_effect=inode_stat_mock)):
+            reader = Cgroup()
+        assert reader.container_id == "in-1234"
+        assert paths_checked[-2:] == [
+            "/sys/fs/cgroup/memory/",
+            "/sys/fs/cgroup/"
+        ]
+        mock_open.assert_called_once_with("/proc/self/cgroup", mode="r")