feat(llmobs): add prompt tracking telemetry tags (#15637)

## Description

Adds prompt tracking telemetry tags for OpenAI integration:
- Adds `prompt_tracking_instrumentation_method:auto` tag for
auto-instrumented prompts (OpenAI, LangChain)
- Adds `prompt_tracking_instrumentation_method:annotated` tag for
annotated prompts (manual)
- Adds `prompt_multimodal:true` tag for OpenAI prompts containing
image/file inputs

### Motivation

Enable dd-go to emit statsd metrics tagged by prompt tracking source and
multimodal content for internal dashboard analytics.

## Testing

- Updated unit tests for OpenAI and LangChain integrations
- All existing tests pass with new tag assertions
- System tests

## Risks

<!-- Note any risks associated with this change, or "None" if no risks
-->
🤷 

## Related PRs

- dd-trace-js: DataDog/dd-trace-js#7106
- dd-source: DataDog/dd-source#323345
- system-tests: DataDog/system-tests#5876

Author: PROFeNoM

ddtrace/llmobs/_constants.py

@@ -94,6 +94,12 @@
 # Prompt constants
 DEFAULT_PROMPT_NAME = "unnamed-prompt"
 
+# Prompt tracking tags
+PROMPT_TRACKING_INSTRUMENTATION_METHOD = "prompt_tracking_instrumentation_method"
+PROMPT_MULTIMODAL = "prompt_multimodal"
+INSTRUMENTATION_METHOD_AUTO = "auto"
+INSTRUMENTATION_METHOD_ANNOTATED = "annotated"
+
 DECORATOR = "_ml_obs.decorator"
 INTEGRATION = "_ml_obs.integration"
 

ddtrace/llmobs/_integrations/langchain.py

@@ -38,6 +38,7 @@
 from ddtrace.llmobs._integrations.utils import LANGCHAIN_ROLE_MAPPING
 from ddtrace.llmobs._integrations.utils import extract_instance_metadata_from_stack
 from ddtrace.llmobs._integrations.utils import format_langchain_io
+from ddtrace.llmobs._integrations.utils import set_prompt_tracking_tags
 from ddtrace.llmobs._integrations.utils import update_proxy_workflow_input_output_value
 from ddtrace.llmobs._utils import _get_attr
 from ddtrace.llmobs._utils import _get_nearest_llmobs_ancestor
@@ -957,5 +958,6 @@ def llmobs_set_prompt_tag(self, instance, span: Span):
             try:
                 prompt = _validate_prompt(prompt, strict_validation=True)
                 span._set_ctx_item(INPUT_PROMPT, prompt)
+                set_prompt_tracking_tags(span)
             except Exception as e:
                 log.debug("Failed to validate langchain prompt", e)

ddtrace/llmobs/_integrations/utils.py

@@ -24,11 +24,15 @@
 from ddtrace.llmobs._constants import INPUT_TYPE_IMAGE
 from ddtrace.llmobs._constants import INPUT_TYPE_TEXT
 from ddtrace.llmobs._constants import INPUT_VALUE
+from ddtrace.llmobs._constants import INSTRUMENTATION_METHOD_AUTO
 from ddtrace.llmobs._constants import METADATA
 from ddtrace.llmobs._constants import OAI_HANDOFF_TOOL_ARG
 from ddtrace.llmobs._constants import OUTPUT_MESSAGES
 from ddtrace.llmobs._constants import OUTPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import OUTPUT_VALUE
+from ddtrace.llmobs._constants import PROMPT_MULTIMODAL
+from ddtrace.llmobs._constants import PROMPT_TRACKING_INSTRUMENTATION_METHOD
+from ddtrace.llmobs._constants import TAGS
 from ddtrace.llmobs._constants import TOOL_DEFINITIONS
 from ddtrace.llmobs._constants import TOTAL_TOKENS_METRIC_KEY
 from ddtrace.llmobs._utils import _get_attr
@@ -880,6 +884,35 @@ def _extract_chat_template_from_instructions(
     return chat_template
 
 
+def _has_multimodal_inputs(variables: Dict[str, Any]) -> bool:
+    """Check if prompt variables contain multimodal inputs (image/file)."""
+    if not variables or not isinstance(variables, dict):
+        return False
+    for value in variables.values():
+        item_type = _get_attr(value, "type", None)
+        if item_type in (INPUT_TYPE_IMAGE, INPUT_TYPE_FILE):
+            return True
+    return False
+
+
+def set_prompt_tracking_tags(span: Span, *, is_multimodal: bool = False) -> None:
+    """Set prompt tracking telemetry tags on a span.
+
+    Args:
+        span: The span to tag
+        is_multimodal: Whether the prompt contains image/file inputs
+    """
+    new_tags = {PROMPT_TRACKING_INSTRUMENTATION_METHOD: INSTRUMENTATION_METHOD_AUTO}
+    if is_multimodal:
+        new_tags[PROMPT_MULTIMODAL] = "true"
+
+    existing_tags = span._get_ctx_item(TAGS)
+    if existing_tags:
+        existing_tags.update(new_tags)
+    else:
+        span._set_ctx_item(TAGS, new_tags)
+
+
 def openai_set_meta_tags_from_response(
     span: Span, kwargs: Dict[str, Any], response: Optional[Any], integration: Any = None
 ) -> None:
@@ -908,12 +941,13 @@ def openai_set_meta_tags_from_response(
     if prompt_data:
         try:
             prompt_data = dict(prompt_data)  # Make a copy to avoid modifying the original
+            variables = prompt_data.get("variables", {})
+            has_multimodal = _has_multimodal_inputs(variables)
 
             # Extract chat_template from response instructions if not already provided
             if response and not prompt_data.get("chat_template") and not prompt_data.get("template"):
                 instructions = _get_attr(response, "instructions", None)
                 if instructions:
-                    variables = prompt_data.get("variables", {})
                     normalized_variables = _normalize_prompt_variables(variables)
                     chat_template = _extract_chat_template_from_instructions(instructions, normalized_variables)
                     if chat_template:
@@ -922,6 +956,8 @@ def openai_set_meta_tags_from_response(
 
             validated_prompt = _validate_prompt(prompt_data, strict_validation=False)
             span._set_ctx_item(INPUT_PROMPT, validated_prompt)
+
+            set_prompt_tracking_tags(span, is_multimodal=has_multimodal)
         except (TypeError, ValueError, AttributeError) as e:
             logger.debug("Failed to validate prompt for OpenAI response: %s", e)
 

ddtrace/llmobs/_llmobs.py

@@ -70,6 +70,7 @@
 from ddtrace.llmobs._constants import INPUT_MESSAGES
 from ddtrace.llmobs._constants import INPUT_PROMPT
 from ddtrace.llmobs._constants import INPUT_VALUE
+from ddtrace.llmobs._constants import INSTRUMENTATION_METHOD_ANNOTATED
 from ddtrace.llmobs._constants import INTEGRATION
 from ddtrace.llmobs._constants import LLMOBS_TRACE_ID
 from ddtrace.llmobs._constants import METADATA
@@ -81,6 +82,7 @@
 from ddtrace.llmobs._constants import OUTPUT_MESSAGES
 from ddtrace.llmobs._constants import OUTPUT_VALUE
 from ddtrace.llmobs._constants import PARENT_ID_KEY
+from ddtrace.llmobs._constants import PROMPT_TRACKING_INSTRUMENTATION_METHOD
 from ddtrace.llmobs._constants import PROPAGATED_LLMOBS_TRACE_ID_KEY
 from ddtrace.llmobs._constants import PROPAGATED_ML_APP_KEY
 from ddtrace.llmobs._constants import PROPAGATED_PARENT_ID_KEY
@@ -1655,6 +1657,9 @@ def annotate(
                 try:
                     validated_prompt = _validate_prompt(prompt, strict_validation=False)
                     cls._set_dict_attribute(span, INPUT_PROMPT, validated_prompt)
+                    cls._set_dict_attribute(
+                        span, TAGS, {PROMPT_TRACKING_INSTRUMENTATION_METHOD: INSTRUMENTATION_METHOD_ANNOTATED}
+                    )
                 except (ValueError, TypeError) as e:
                     error = "invalid_prompt"
                     raise LLMObsAnnotateSpanError("Failed to validate prompt with error:", str(e))

tests/contrib/langchain/test_langchain_llmobs.py

@@ -93,6 +93,7 @@ def _expected_langchain_llmobs_llm_span(
         tags={"ml_app": "langchain_test", "service": "tests.contrib.langchain"},
         span_links=span_links,
         prompt=prompt,
+        prompt_tracking_instrumentation_method="auto" if prompt else None,
     )
 
 
@@ -212,9 +213,9 @@ def test_llmobs_string_prompt_template_invoke(langchain_core, langchain_openai,
     assert actual_prompt["id"] == "test_langchain_llmobs.prompt_template"
     assert actual_prompt["template"] == template_string
     assert actual_prompt["variables"] == variable_dict
-    # Check that metadata from the prompt template is preserved
     assert "tags" in actual_prompt
     assert actual_prompt["tags"] == {"test_type": "basic_invoke", "author": "test_suite"}
+    assert "prompt_tracking_instrumentation_method:auto" in llmobs_events[1]["tags"]
 
 
 def test_llmobs_string_prompt_template_direct_invoke(
@@ -237,14 +238,13 @@ def test_llmobs_string_prompt_template_direct_invoke(
     llmobs_events.sort(key=lambda span: span["start_ns"])
     assert len(llmobs_events) == 1  # Only LLM span, prompt template invoke doesn't create LLMObs event by itself
 
-    # The prompt should be attached to the LLM span
     actual_prompt = llmobs_events[0]["meta"]["input"]["prompt"]
     assert actual_prompt["id"] == "test_langchain_llmobs.greeting_template"
     assert actual_prompt["template"] == template_string
     assert actual_prompt["variables"] == variable_dict
-    # Check that metadata from the prompt template is preserved
     assert "tags" in actual_prompt
     assert actual_prompt["tags"] == {"test_type": "direct_invoke", "interaction": "greeting"}
+    assert "prompt_tracking_instrumentation_method:auto" in llmobs_events[0]["tags"]
 
 
 def test_llmobs_string_prompt_template_invoke_chat_model(

tests/contrib/openai/test_openai_llmobs.py

@@ -2407,6 +2407,7 @@ def test_response_with_mixed_input_prompt_tracking_url_stripped(self, openai, mo
                     ),
                 }
             ],
+            prompt_multimodal=True,
         )
 
     @pytest.mark.skipif(
@@ -2485,6 +2486,7 @@ def test_response_with_mixed_input_prompt_tracking_url_preserved(self, openai, m
                     ),
                 }
             ],
+            prompt_multimodal=True,
         )
 
     @pytest.mark.skipif(

tests/contrib/openai/utils.py

@@ -267,7 +267,17 @@ def get_openai_vcr(subdirectory_name=""):
     )
 
 
-def assert_prompt_tracking(span_event, prompt_id, prompt_version, variables, expected_chat_template, expected_messages):
+def assert_prompt_tracking(
+    span_event,
+    prompt_id,
+    prompt_version,
+    variables,
+    expected_chat_template,
+    expected_messages,
+    *,
+    prompt_tracking_instrumentation_method="auto",
+    prompt_multimodal=False,
+):
     """Helper to assert prompt tracking metadata and template extraction."""
     assert "prompt" in span_event["meta"]["input"]
     actual_prompt = span_event["meta"]["input"]["prompt"]
@@ -277,3 +287,6 @@ def assert_prompt_tracking(span_event, prompt_id, prompt_version, variables, exp
     assert "chat_template" in actual_prompt
     assert actual_prompt["chat_template"] == expected_chat_template
     assert span_event["meta"]["input"]["messages"] == expected_messages
+    assert f"prompt_tracking_instrumentation_method:{prompt_tracking_instrumentation_method}" in span_event["tags"]
+    if prompt_multimodal:
+        assert "prompt_multimodal:true" in span_event["tags"]

tests/llmobs/_utils.py

@@ -75,6 +75,8 @@ def _expected_llmobs_llm_span_event(
     span,
     span_kind="llm",
     prompt=None,
+    prompt_tracking_instrumentation_method=None,
+    prompt_multimodal=None,
     input_messages=None,
     input_documents=None,
     output_messages=None,
@@ -94,6 +96,9 @@ def _expected_llmobs_llm_span_event(
     """
     Helper function to create an expected LLM span event.
     span_kind: either "llm" or "agent" or "embedding"
+    prompt: prompt metadata dict (id, version, variables, template)
+    prompt_tracking_instrumentation_method: prompt tracking source tag ('auto' for auto-instrumented)
+    prompt_multimodal: whether prompt contains multimodal inputs (True if present)
     input_messages: list of input messages in format {"content": "...", "optional_role", "..."}
     output_messages: list of output messages in format {"content": "...", "optional_role", "..."}
     metadata: dict of metadata key value pairs
@@ -109,7 +114,16 @@ def _expected_llmobs_llm_span_event(
     tool_definitions: list of tool definitions that were available to the LLM
     """
     span_event = _llmobs_base_span_event(
-        span, span_kind, tags, session_id, error, error_message, error_stack, span_links
+        span,
+        span_kind,
+        tags,
+        session_id,
+        error,
+        error_message,
+        error_stack,
+        span_links,
+        prompt_tracking_instrumentation_method,
+        prompt_multimodal,
     )
     meta_dict = {"input": {}, "output": {}}
     if span_kind == "llm":
@@ -171,6 +185,8 @@ def _expected_llmobs_non_llm_span_event(
     error_message=None,
     error_stack=None,
     span_links=False,
+    prompt_tracking_instrumentation_method=None,
+    prompt_multimodal=None,
 ):
     """
     Helper function to create an expected span event of type (workflow, task, tool, retrieval).
@@ -185,9 +201,20 @@ def _expected_llmobs_non_llm_span_event(
     error_message: error message
     error_stack: error stack
     span_links: whether there are span links present on this span.
+    prompt_tracking_instrumentation_method: prompt tracking source tag ('auto' for auto-instrumented)
+    prompt_multimodal: whether prompt contains multimodal inputs (True if present)
     """
     span_event = _llmobs_base_span_event(
-        span, span_kind, tags, session_id, error, error_message, error_stack, span_links
+        span,
+        span_kind,
+        tags,
+        session_id,
+        error,
+        error_message,
+        error_stack,
+        span_links,
+        prompt_tracking_instrumentation_method,
+        prompt_multimodal,
     )
     meta_dict = {"input": {}, "output": {}}
     if span_kind == "retrieval":
@@ -221,7 +248,14 @@ def _llmobs_base_span_event(
     error_message=None,
     error_stack=None,
     span_links=False,
+    prompt_tracking_instrumentation_method=None,
+    prompt_multimodal=None,
 ):
+    expected_tags = _expected_llmobs_tags(span, tags=tags, error=error, session_id=session_id)
+    if prompt_tracking_instrumentation_method:
+        expected_tags.append(f"prompt_tracking_instrumentation_method:{prompt_tracking_instrumentation_method}")
+    if prompt_multimodal:
+        expected_tags.append(f"prompt_multimodal:{prompt_multimodal}")
     span_event = {
         "trace_id": mock.ANY,
         "span_id": str(span.span_id),
@@ -232,7 +266,7 @@ def _llmobs_base_span_event(
         "status": "error" if error else "ok",
         "meta": _Meta(span=_SpanField(kind=span_kind)),
         "metrics": {},
-        "tags": _expected_llmobs_tags(span, tags=tags, error=error, session_id=session_id),
+        "tags": expected_tags,
         "_dd": {
             "span_id": str(span.span_id),
             "trace_id": format_trace_id(span.trace_id),

tests/llmobs/test_llmobs_service.py

@@ -25,6 +25,7 @@
 from ddtrace.llmobs._constants import OUTPUT_DOCUMENTS
 from ddtrace.llmobs._constants import OUTPUT_MESSAGES
 from ddtrace.llmobs._constants import OUTPUT_VALUE
+from ddtrace.llmobs._constants import PROMPT_TRACKING_INSTRUMENTATION_METHOD
 from ddtrace.llmobs._constants import PROPAGATED_ML_APP_KEY
 from ddtrace.llmobs._constants import PROPAGATED_PARENT_ID_KEY
 from ddtrace.llmobs._constants import SESSION_ID
@@ -791,6 +792,7 @@ def test_annotate_prompt_dict(llmobs):
             "_dd_context_variable_keys": ["context"],
             "_dd_query_variable_keys": ["question"],
         }
+        assert span._get_ctx_item(TAGS) == {PROMPT_TRACKING_INSTRUMENTATION_METHOD: "annotated"}
 
 
 def test_annotate_prompt_dict_with_context_var_keys(llmobs):
@@ -814,6 +816,7 @@ def test_annotate_prompt_dict_with_context_var_keys(llmobs):
             "_dd_context_variable_keys": ["var1", "var2"],
             "_dd_query_variable_keys": ["user_input"],
         }
+        assert span._get_ctx_item(TAGS) == {PROMPT_TRACKING_INSTRUMENTATION_METHOD: "annotated"}
 
 
 def test_annotate_prompt_typed_dict(llmobs):
@@ -837,6 +840,7 @@ def test_annotate_prompt_typed_dict(llmobs):
             "_dd_context_variable_keys": ["var1", "var2"],
             "_dd_query_variable_keys": ["user_input"],
         }
+        assert span._get_ctx_item(TAGS) == {PROMPT_TRACKING_INSTRUMENTATION_METHOD: "annotated"}
 
 
 def test_annotate_prompt_wrong_type(llmobs):
@@ -1322,6 +1326,7 @@ def test_annotation_context_modifies_prompt(llmobs):
                 "_dd_context_variable_keys": ["context"],
                 "_dd_query_variable_keys": ["question"],
             }
+            assert span._get_ctx_item(TAGS) == {PROMPT_TRACKING_INSTRUMENTATION_METHOD: "annotated"}
 
 
 def test_annotation_context_modifies_name(llmobs):
@@ -1507,6 +1512,7 @@ async def test_annotation_context_async_modifies_prompt(llmobs):
                 "_dd_context_variable_keys": ["context"],
                 "_dd_query_variable_keys": ["question"],
             }
+            assert span._get_ctx_item(TAGS) == {PROMPT_TRACKING_INSTRUMENTATION_METHOD: "annotated"}
 
 
 async def test_annotation_context_async_modifies_name(llmobs):