Fix silent drop of ztunnel counter metrics in Istio ambient mode

istio/README.md

@@ -90,6 +90,25 @@ This annotation specifies the container `discovery` to match the default contain
 
 The method for applying these annotations varies depending on the [Istio deployment strategy (Istioctl, Helm, Operator)][22] used. Consult the Istio documentation for the proper method to apply these pod annotations. See the [sample istio.d/conf.yaml][8] for all available configuration options.
 
+##### Ambient mode configuration
+
+Istio ambient mode, generally available in Istio v1.24, replaces sidecar injection with two shared components: the `ztunnel` DaemonSet (L4 zero-trust tunneling) and optional `waypoint` proxies (L7 HTTP/gRPC processing). Set `istio_mode: ambient` and configure one or more of `ztunnel_endpoint`, `waypoint_endpoint`, and `istiod_endpoint` on the same instance. The check scrapes each endpoint that is set. Adjust the URLs in the example below to match your cluster's hostnames and ports.
+
+Example static configuration in `istio.d/conf.yaml` covering all three components:
+
+```yaml
+init_config:
+
+instances:
+  - istio_mode: ambient
+    use_openmetrics: true
+    ztunnel_endpoint: http://ztunnel.istio-system.svc:15020/stats/prometheus
+    waypoint_endpoint: http://waypoint.<NAMESPACE>.svc:15020/stats/prometheus
+    istiod_endpoint: http://istiod.istio-system.svc:15014/metrics
+```
+
+Replace `<NAMESPACE>` with the namespace where you ran `istioctl waypoint apply`. Omit `waypoint_endpoint` if you have not deployed a waypoint proxy. The same options can be set via the Autodiscovery annotation syntax shown in the [Control plane configuration](#control-plane-configuration) section above.
+
 #### Disable sidecar injection for Datadog Agent pods
 
 If you are installing the [Datadog Agent in a container][10], Datadog recommends that you first disable Istio's sidecar injection.

istio/assets/configuration/spec.yaml

@@ -24,6 +24,7 @@ files:
             Specify the Istio data plane mode to monitor.
             - `sidecar`: Monitor Istio sidecar proxies (traditional mode)
             - `ambient`: Monitor Istio ambient mode components (ztunnel, waypoint proxies)
+          fleet_configurable: true
           value:
             example: sidecar
             display_default: sidecar
@@ -35,6 +36,7 @@ files:
             Ztunnel is the L4 proxy that provides zero-trust tunneling and mTLS for ambient mesh.
             Only used when `istio_mode` is set to `ambient`.
             Ztunnel metrics are exposed on port 15020.
+          fleet_configurable: true
           value:
             display_default: null
             example: http://ztunnel.istio-system:15020/stats/prometheus
@@ -45,6 +47,7 @@ files:
             Waypoint proxies provide optional L7 processing (HTTP/gRPC traffic management) in ambient mesh.
             Only used when `istio_mode` is set to `ambient`.
             Waypoint metrics are exposed on port 15020.
+          fleet_configurable: true
           value:
             display_default: null
             example: http://waypoint.istio-system:15020/stats/prometheus

istio/changelog.d/23707.fixed

@@ -0,0 +1 @@
+Restore Istio ambient mode metric collection broken in 9.4.0: ztunnel counters are no longer silently dropped, proxy management metrics use the `workload_manager_*` names ztunnel actually emits, and the missing xDS message counters are now registered.

istio/datadog_checks/istio/check.py

@@ -3,7 +3,7 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 from collections import ChainMap
 
-from datadog_checks.base import ConfigurationError, OpenMetricsBaseCheckV2
+from datadog_checks.base import ConfigurationError, OpenMetricsBaseCheckV2, is_affirmative
 from datadog_checks.base.checks.openmetrics.v2.scraper import OpenMetricsCompatibilityScraper
 
 from .constants import ISTIOD_NAMESPACE
@@ -72,10 +72,24 @@ def _parse_ambient_config(self, istiod_endpoint, istiod_namespace):
                 "`ztunnel_endpoint`, `waypoint_endpoint`, or `istiod_endpoint`."
             )
 
-        # Ztunnel provides L4 TCP metrics for ambient mesh
+        # Ztunnel uses the modern OpenMetrics counter convention; force the v2 parser so counters are not dropped.
         ztunnel_namespace = istiod_namespace + ".ztunnel"
         if ztunnel_endpoint:
-            self.scraper_configs.append(self._generate_config(ztunnel_endpoint, ZTUNNEL_METRICS, ztunnel_namespace))
+            if not is_affirmative(self.instance.get("use_latest_spec", True)):
+                self.log.warning(
+                    "`use_latest_spec: false` is set with `ztunnel_endpoint` configured. "
+                    "ztunnel emits the modern OpenMetrics counter convention which the "
+                    "legacy parser silently drops, so every ztunnel counter metric will be "
+                    "missed. Remove `use_latest_spec: false` to restore ztunnel metrics."
+                )
+            self.scraper_configs.append(
+                self._generate_config(
+                    ztunnel_endpoint,
+                    ZTUNNEL_METRICS,
+                    ztunnel_namespace,
+                    scraper_defaults={'use_latest_spec': True},
+                )
+            )
 
         # Waypoint provides L7 HTTP/gRPC metrics (optional in ambient mode)
         waypoint_namespace = istiod_namespace + ".waypoint"
@@ -86,16 +100,12 @@ def _parse_ambient_config(self, istiod_endpoint, istiod_namespace):
         if istiod_endpoint:
             self.scraper_configs.append(self._generate_config(istiod_endpoint, ISTIOD_METRICS, istiod_namespace))
 
-    def _generate_config(self, endpoint, metrics, namespace):
+    def _generate_config(self, endpoint, metrics, namespace, *, scraper_defaults=None):
         metrics = construct_metrics_config(metrics)
         metrics.append(ISTIOD_VERSION)
-        config = {
-            'openmetrics_endpoint': endpoint,
-            'metrics': metrics,
-            'namespace': namespace,
-        }
+        config = {**(scraper_defaults or {}), 'openmetrics_endpoint': endpoint, 'metrics': metrics}
+        # Instance keys override scraper_defaults; per-scraper namespace is restored on the next line.
         config.update(self.instance)
-        # Restore per-scraper namespace so custom ztunnel/waypoint/mesh namespaces are not overwritten by instance
         config['namespace'] = namespace
         return config
 

istio/datadog_checks/istio/metrics.py

@@ -276,17 +276,16 @@
     'istio_dns_upstream_failures_total': 'dns.upstream_failures.total',
     'istio_dns_upstream_request_duration_seconds': 'dns.upstream_request_duration_seconds',
     'istio_on_demand_dns_total': 'on_demand_dns.total',
-    # In-pod proxy management metrics (unstable)
-    'istio_active_proxy_count_total': 'active_proxy_count.total',
-    'istio_pending_proxy_count_total': 'pending_proxy_count.total',
-    'istio_proxies_started_total': 'proxies_started.total',
-    'istio_proxies_stopped_total': 'proxies_stopped.total',
+    # In-pod proxy management metrics (unstable). Ztunnel exposes these under the
+    # workload_manager_* family, not istio_*.
+    'workload_manager_active_proxy_count': 'active_proxy_count',
+    'workload_manager_pending_proxy_count': 'pending_proxy_count',
+    'workload_manager_proxies_started_total': 'proxies_started.total',
+    'workload_manager_proxies_stopped_total': 'proxies_stopped.total',
     # XDS metrics (unstable)
     'istio_xds_connection_terminations_total': 'xds.connection_terminations.total',
-    # Connection metrics (unstable)
-    'istio_connection_opens_total': 'connection.opens.total',
-    'istio_connection_closes_total': 'connection.closes.total',
-    'istio_connection_termination_total': 'connection.termination.total',
+    'istio_xds_message_total': 'xds.message.total',
+    'istio_xds_message_bytes_total': 'xds.message_bytes.total',
 }
 
 

istio/hatch.toml

@@ -5,14 +5,34 @@ dependencies = [
   "requests-mock==1.4.0",
 ]
 
+# Istio supports two data plane modes: traditional sidecar injection, and ambient
+# (sidecar-less). Ambient mode graduated to GA in Istio 1.24. The matrix below is
+# split into version × mode blocks so each block declares only the (version, mode)
+# combinations that are actually supported by the version and have a working setup
+# function in conftest.py. To add a new Istio version, extend the relevant block's
+# `version` list; to add a new mode on an existing version, add the entry to the
+# right block (and extend conftest.setup_istio* accordingly).
+
+# Sidecar-mode envs. 1.13 stays for the legacy Go-runtime safety net; 1.24 is where
+# ambient GA'd; 1.29 is the current supported release.
+[[envs.default.matrix]]
+python = ["3.13"]
+version = ["1.13", "1.24", "1.29"]
+mode = ["sidecar"]
+
+# Ambient-mode envs. Requires Istio >= 1.24.
 [[envs.default.matrix]]
 python = ["3.13"]
-version = ["1.13"]
+version = ["1.24", "1.29"]
+mode = ["ambient"]
 
 [envs.default.overrides]
 matrix.version.env-vars = [
   { key = "ISTIO_VERSION", value = "1.13.3", if = ["1.13"] },
+  { key = "ISTIO_VERSION", value = "1.24.3", if = ["1.24"] },
+  { key = "ISTIO_VERSION", value = "1.29.2", if = ["1.29"] },
 ]
+matrix.mode.env-vars = "ISTIO_MODE"
 
 [envs.default.env-vars]
 DDEV_SKIP_GENERIC_TAGS_CHECK = "true"

istio/metadata.csv

@@ -447,24 +447,23 @@ istio.galley.istio.networking.virtualservices,gauge,,,,,0,istio,,
 istio.galley.istio.networking.destinationrules,gauge,,,,,0,istio,,
 istio.galley.istio.networking.gateways,gauge,,,,,0,istio,,
 istio.galley.istio.authentication.meshpolicies,gauge,,,,,0,istio,,
-istio.ztunnel.tcp.connections_opened.total,count,,connection,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total TCP connections opened through ztunnel",0,istio,ztunnel connections opened,
-istio.ztunnel.tcp.connections_closed.total,count,,connection,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total TCP connections closed through ztunnel",0,istio,ztunnel connections closed,
-istio.ztunnel.tcp.send_bytes.total,count,,byte,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total bytes sent through ztunnel TCP connections",0,istio,ztunnel bytes sent,
-istio.ztunnel.tcp.received_bytes.total,count,,byte,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total bytes received through ztunnel TCP connections",0,istio,ztunnel bytes received,
-istio.ztunnel.dns.requests.total,count,,request,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total DNS requests handled by ztunnel",0,istio,ztunnel dns requests,
-istio.ztunnel.dns.upstream_requests.total,count,,request,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total DNS requests forwarded to upstream by ztunnel",0,istio,ztunnel dns upstream requests,
-istio.ztunnel.dns.upstream_failures.total,count,,request,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total DNS upstream request failures in ztunnel",0,istio,ztunnel dns failures,
+istio.ztunnel.tcp.connections_opened.count,count,,connection,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total TCP connections opened through ztunnel",0,istio,ztunnel connections opened,
+istio.ztunnel.tcp.connections_closed.count,count,,connection,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total TCP connections closed through ztunnel",0,istio,ztunnel connections closed,
+istio.ztunnel.tcp.send_bytes.count,count,,byte,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total bytes sent through ztunnel TCP connections",0,istio,ztunnel bytes sent,
+istio.ztunnel.tcp.received_bytes.count,count,,byte,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total bytes received through ztunnel TCP connections",0,istio,ztunnel bytes received,
+istio.ztunnel.dns.requests.count,count,,request,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total DNS requests handled by ztunnel",0,istio,ztunnel dns requests,
+istio.ztunnel.dns.upstream_requests.count,count,,request,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total DNS requests forwarded to upstream by ztunnel",0,istio,ztunnel dns upstream requests,
+istio.ztunnel.dns.upstream_failures.count,count,,request,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total DNS upstream request failures in ztunnel",0,istio,ztunnel dns failures,
 istio.ztunnel.dns.upstream_request_duration_seconds.count,count,,second,,"[OpenMetrics V1 and V2 and Istio v1.24+] Count of DNS upstream request durations in ztunnel",0,istio,ztunnel dns duration count,
 istio.ztunnel.dns.upstream_request_duration_seconds.sum,count,,second,,"[OpenMetrics V1 and V2 and Istio v1.24+] Sum of DNS upstream request durations in ztunnel",0,istio,ztunnel dns duration sum,
-istio.ztunnel.on_demand_dns.total,count,,request,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total on-demand DNS requests in ztunnel",0,istio,ztunnel on-demand dns,
-istio.ztunnel.active_proxy_count.total,gauge,,,,"[OpenMetrics V1 and V2 and Istio v1.24+] Number of active in-pod proxies managed by ztunnel",0,istio,ztunnel active proxies,
-istio.ztunnel.pending_proxy_count.total,gauge,,,,"[OpenMetrics V1 and V2 and Istio v1.24+] Number of pending in-pod proxies in ztunnel",0,istio,ztunnel pending proxies,
-istio.ztunnel.proxies_started.total,count,,,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total number of in-pod proxies started by ztunnel",0,istio,ztunnel proxies started,
-istio.ztunnel.proxies_stopped.total,count,,,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total number of in-pod proxies stopped by ztunnel",0,istio,ztunnel proxies stopped,
-istio.ztunnel.xds.connection_terminations.total,count,,connection,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total XDS connection terminations in ztunnel",0,istio,ztunnel xds terminations,
-istio.ztunnel.connection.opens.total,count,,connection,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total connections opened in ztunnel",0,istio,ztunnel connection opens,
-istio.ztunnel.connection.closes.total,count,,connection,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total connections closed in ztunnel",0,istio,ztunnel connection closes,
-istio.ztunnel.connection.termination.total,count,,connection,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total connection terminations in ztunnel",0,istio,ztunnel connection terminations,
+istio.ztunnel.on_demand_dns.count,count,,request,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total on-demand DNS requests in ztunnel",0,istio,ztunnel on-demand dns,
+istio.ztunnel.active_proxy_count,gauge,,,,"[OpenMetrics V1 and V2 and Istio v1.24+] Number of active in-pod proxies managed by ztunnel",0,istio,ztunnel active proxies,
+istio.ztunnel.pending_proxy_count,gauge,,,,"[OpenMetrics V1 and V2 and Istio v1.24+] Number of pending in-pod proxies in ztunnel",0,istio,ztunnel pending proxies,
+istio.ztunnel.proxies_started.count,count,,,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total number of in-pod proxies started by ztunnel",0,istio,ztunnel proxies started,
+istio.ztunnel.proxies_stopped.count,count,,,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total number of in-pod proxies stopped by ztunnel",0,istio,ztunnel proxies stopped,
+istio.ztunnel.xds.connection_terminations.count,count,,connection,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total XDS connection terminations in ztunnel",0,istio,ztunnel xds terminations,
+istio.ztunnel.xds.message.count,count,,message,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total XDS messages exchanged between ztunnel and istiod",0,istio,ztunnel xds messages,
+istio.ztunnel.xds.message_bytes.count,count,,byte,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total bytes of XDS messages exchanged between ztunnel and istiod",0,istio,ztunnel xds message bytes,
 istio.waypoint.request.count,count,,request,,"[OpenMetrics V1 and V2 and Istio v1.24+] Total HTTP requests through waypoint proxy",0,istio,waypoint requests,
 istio.waypoint.request.duration.milliseconds.count,count,,request,,"[OpenMetrics V1 and V2 and Istio v1.24+] Count of HTTP request durations through waypoint proxy",0,istio,waypoint request duration count,
 istio.waypoint.request.duration.milliseconds.sum,count,,millisecond,,"[OpenMetrics V1 and V2 and Istio v1.24+] Sum of HTTP request durations through waypoint proxy",0,istio,waypoint request duration sum,

istio/tests/common.py

@@ -409,19 +409,26 @@
     'istio.galley.istio.authentication.meshpolicies',
 ]
 
-# Ambient mode (ztunnel) - default namespace istio.ztunnel (OpenMetrics submits counters as .count)
-V2_ZTUNNEL_METRICS = [
+# Ambient mode (ztunnel) - default namespace istio.ztunnel.
+# Ztunnel counters use `# TYPE foo counter` + `foo_total{} N`, which the legacy parser drops; require the v2 parser.
+V2_ZTUNNEL_COUNTER_METRICS = [
     'istio.ztunnel.tcp.connections_opened.count',
     'istio.ztunnel.tcp.connections_closed.count',
     'istio.ztunnel.tcp.send_bytes.count',
     'istio.ztunnel.tcp.received_bytes.count',
-    'istio.ztunnel.dns.requests.count',
-    'istio.ztunnel.dns.upstream_requests.count',
-    'istio.ztunnel.dns.upstream_failures.count',
-    'istio.ztunnel.connection.opens.count',
-    'istio.ztunnel.connection.closes.count',
+    'istio.ztunnel.xds.message.count',
+    'istio.ztunnel.xds.message_bytes.count',
+    'istio.ztunnel.proxies_started.count',
 ]
 
+# Gauges, unaffected by the legacy-parser counter bug; split out so the regression test pins only counters.
+V2_ZTUNNEL_GAUGE_METRICS = [
+    'istio.ztunnel.active_proxy_count',
+    'istio.ztunnel.pending_proxy_count',
+]
+
+V2_ZTUNNEL_METRICS = V2_ZTUNNEL_COUNTER_METRICS + V2_ZTUNNEL_GAUGE_METRICS
+
 # Ambient mode (waypoint) - default namespace istio.waypoint
 V2_WAYPOINT_METRICS = [
     'istio.waypoint.request.count',