Skip to content

Commit 9b8bf9e

Browse files
authored
lint(seer): Prevent new generic LLM callers (#128292)
Adding linter to stop usage of llm_generate method which will soon be deprecated. Once users are ported over to seer I will remove the linter. ## Summary - add S030 to reject new imports and calls of the legacy generic Seer LLM API - pin the four existing Sentry callers to exact functions and occurrence counts in a shrink-only allowlist - reject aliases, module-qualified calls, duplicate calls in allowed functions, and direct `/v1/llm/generate` usage - report stale allowlist entries as callers migrate to named Seer one-shots ## Validation - `.venv/bin/pytest -q tests/tools/test_flake8_plugin.py` - `.venv/bin/prek run -q flake8-sentry --files tools/flake8_plugin.py tests/tools/test_flake8_plugin.py src/sentry/integrations/utils/external_issues.py src/sentry/seer/endpoints/issue_view_title_generate.py src/sentry/tasks/seer/autofix_issue_data.py src/sentry/uptime/seer_assertions.py` - repository commit hooks - pre-push mypy checks
1 parent 73dc979 commit 9b8bf9e

2 files changed

Lines changed: 196 additions & 0 deletions

File tree

‎tests/tools/test_flake8_plugin.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -604,6 +604,105 @@ def test_S029_ignores_non_seer_pool() -> None:
604604
assert _run(src, filename="src/sentry/example.py") == []
605605

606606

607+
def test_S030_allows_exact_legacy_callsite_baseline() -> None:
608+
src = """\
609+
from sentry.seer.signed_seer_api import LlmGenerateRequest, make_llm_generate_request
610+
611+
612+
def generate_title_from_query():
613+
body = LlmGenerateRequest(prompt="hello")
614+
return make_llm_generate_request(body)
615+
"""
616+
617+
assert _run(src, filename="src/sentry/seer/endpoints/issue_view_title_generate.py") == []
618+
619+
620+
def test_S030_rejects_new_legacy_caller() -> None:
621+
src = """\
622+
from sentry.seer.signed_seer_api import LlmGenerateRequest, make_llm_generate_request
623+
624+
625+
def new_caller():
626+
body = LlmGenerateRequest(prompt="hello")
627+
return make_llm_generate_request(body)
628+
"""
629+
630+
errors = _run(src, filename="src/sentry/seer/new_caller.py")
631+
assert len(errors) == 4
632+
assert all("S030 Do not add new callers" in error for error in errors)
633+
634+
635+
def test_S030_rejects_second_call_in_allowlisted_function() -> None:
636+
src = """\
637+
from sentry.seer.signed_seer_api import LlmGenerateRequest, make_llm_generate_request
638+
639+
640+
def generate_title_from_query():
641+
first = LlmGenerateRequest(prompt="first")
642+
second = LlmGenerateRequest(prompt="second")
643+
make_llm_generate_request(first)
644+
return make_llm_generate_request(second)
645+
"""
646+
647+
errors = _run(src, filename="src/sentry/seer/endpoints/issue_view_title_generate.py")
648+
assert errors == [
649+
"t.py:6:13: S030 Do not add new callers of the legacy Seer LLM generate API; "
650+
"add a named Seer one-shot so prompts and model selection stay in Seer",
651+
"t.py:8:11: S030 Do not add new callers of the legacy Seer LLM generate API; "
652+
"add a named Seer one-shot so prompts and model selection stay in Seer",
653+
]
654+
655+
656+
def test_S030_rejects_aliased_and_module_qualified_calls() -> None:
657+
aliased = """\
658+
from sentry.seer.signed_seer_api import LlmGenerateRequest as LegacyRequest
659+
660+
LegacyRequest(prompt="hello")
661+
"""
662+
module_qualified = """\
663+
import sentry.seer.signed_seer_api as seer_api
664+
665+
seer_api.make_llm_generate_request({})
666+
"""
667+
668+
assert len(_run(aliased, filename="src/sentry/example.py")) == 2
669+
assert len(_run(module_qualified, filename="src/sentry/example.py")) == 2
670+
671+
672+
def test_S030_rejects_direct_route_bypass() -> None:
673+
src = 'make_signed_seer_api_request(pool, "/v1/llm/generate", b"{}")\n'
674+
675+
assert _run(src, filename="src/sentry/example.py") == [
676+
"t.py:1:35: S030 Do not add new callers of the legacy Seer LLM generate API; "
677+
"add a named Seer one-shot so prompts and model selection stay in Seer"
678+
]
679+
680+
681+
def test_S030_reports_stale_allowlist_entry() -> None:
682+
src = """\
683+
from sentry.seer.signed_seer_api import LlmGenerateRequest, make_llm_generate_request
684+
685+
686+
def generate_title_from_query():
687+
return None
688+
"""
689+
690+
errors = _run(src, filename="src/sentry/seer/endpoints/issue_view_title_generate.py")
691+
assert len(errors) == 2
692+
assert all("S030 stale legacy Seer LLM generate allowlist entry" in error for error in errors)
693+
694+
695+
def test_S030_allows_legacy_api_definition() -> None:
696+
src = """\
697+
class LlmGenerateRequest: ...
698+
699+
def make_llm_generate_request(body):
700+
return make_signed_seer_api_request(pool, "/v1/llm/generate", body)
701+
"""
702+
703+
assert _run(src, filename="src/sentry/seer/signed_seer_api.py") == []
704+
705+
607706
def _codes(src: str) -> list[str]:
608707
"""`line:CODE` for each diagnostic, for terse assertions."""
609708
tree = ast.parse(src)

‎tools/flake8_plugin.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import ast
44
import os
5+
from collections import Counter
56
from collections.abc import Generator
67
from datetime import datetime, timezone
78
from typing import Any
@@ -164,6 +165,48 @@
164165
"raw urlopen bypasses ViewerContext propagation"
165166
)
166167

168+
S030_msg = (
169+
"S030 Do not add new callers of the legacy Seer LLM generate API; "
170+
"add a named Seer one-shot so prompts and model selection stay in Seer"
171+
)
172+
S030_stale_msg = (
173+
"S030 stale legacy Seer LLM generate allowlist entry for {} in {}; "
174+
"remove it from S030_allowlist"
175+
)
176+
_S030_LEGACY_MODULE = "sentry.seer.signed_seer_api"
177+
_S030_LEGACY_NAMES = frozenset(("LlmGenerateRequest", "make_llm_generate_request"))
178+
_S030_LEGACY_ROUTE = "/v1/llm/generate"
179+
_S030_MODULE_SCOPE = "<module>"
180+
181+
# Existing callers are pinned to their exact function and occurrence count. This is a
182+
# shrink-only migration baseline: remove entries as callers move to named Seer one-shots.
183+
S030_allowlist: dict[str, dict[tuple[str, str], int]] = {
184+
"src/sentry/integrations/utils/external_issues.py": {
185+
(_S030_MODULE_SCOPE, "import LlmGenerateRequest"): 1,
186+
(_S030_MODULE_SCOPE, "import make_llm_generate_request"): 1,
187+
("_make_generate_external_issue_details_request", "call LlmGenerateRequest"): 1,
188+
("_make_generate_external_issue_details_request", "call make_llm_generate_request"): 1,
189+
},
190+
"src/sentry/seer/endpoints/issue_view_title_generate.py": {
191+
(_S030_MODULE_SCOPE, "import LlmGenerateRequest"): 1,
192+
(_S030_MODULE_SCOPE, "import make_llm_generate_request"): 1,
193+
("generate_title_from_query", "call LlmGenerateRequest"): 1,
194+
("generate_title_from_query", "call make_llm_generate_request"): 1,
195+
},
196+
"src/sentry/tasks/seer/autofix_issue_data.py": {
197+
(_S030_MODULE_SCOPE, "import LlmGenerateRequest"): 1,
198+
(_S030_MODULE_SCOPE, "import make_llm_generate_request"): 1,
199+
("_judge_issue", "call LlmGenerateRequest"): 1,
200+
("_judge_issue", "call make_llm_generate_request"): 1,
201+
},
202+
"src/sentry/uptime/seer_assertions.py": {
203+
(_S030_MODULE_SCOPE, "import LlmGenerateRequest"): 1,
204+
(_S030_MODULE_SCOPE, "import make_llm_generate_request"): 1,
205+
("generate_assertion_suggestions", "call LlmGenerateRequest"): 1,
206+
("generate_assertion_suggestions", "call make_llm_generate_request"): 1,
207+
},
208+
}
209+
167210

168211
# Rules whose diagnostics are fatal. Empty ships every rule off; adding one
169212
# gates every endpoint of that shape at once, with no baseline to maintain.
@@ -715,6 +758,10 @@ def __init__(
715758
# S029: pools created from SEER_*_URL must not make raw requests.
716759
self._s029_seer_pool_names: set[str] = set()
717760
self._s029_urlopen_calls: list[tuple[int, int, str]] = []
761+
# S030: legacy generic Seer LLM API imports, calls, and route bypasses.
762+
self._s030_legacy_names: dict[str, str] = {}
763+
self._s030_module_aliases: set[str] = set()
764+
self._s030_uses: list[tuple[int, int, str, str]] = []
718765
# Module-level classes, indexed as encountered. A base is always defined
719766
# before its subclass at module level, so in-order indexing suffices.
720767
self._module_classes: dict[str, ast.ClassDef] = {}
@@ -727,6 +774,13 @@ def __init__(
727774
self._function_stack: list[str] = []
728775

729776
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
777+
if node.module == _S030_LEGACY_MODULE and not node.level:
778+
for alias in node.names:
779+
if alias.name in _S030_LEGACY_NAMES:
780+
local_name = alias.asname or alias.name
781+
self._s030_legacy_names[local_name] = alias.name
782+
self._s030_record(node, f"import {alias.name}")
783+
730784
if node.module and not node.level:
731785
if node.module.split(".")[0] in S003_modules:
732786
self.errors.append((node.lineno, node.col_offset, S003_msg))
@@ -776,6 +830,10 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
776830

777831
def visit_Import(self, node: ast.Import) -> None:
778832
for alias in node.names:
833+
if alias.name == _S030_LEGACY_MODULE:
834+
if alias.asname:
835+
self._s030_module_aliases.add(alias.asname)
836+
self._s030_record(node, f"import {alias.name}")
779837
if alias.name.split(".")[0] in S003_modules:
780838
self.errors.append((node.lineno, node.col_offset, S003_msg))
781839
elif (
@@ -818,6 +876,8 @@ def visit_Name(self, node: ast.Name) -> None:
818876
def visit_Constant(self, node: ast.Constant) -> None:
819877
if isinstance(node.value, str) and node.value == S018_fqn:
820878
self.errors.append((node.lineno, node.col_offset, S018_msg))
879+
elif isinstance(node.value, str) and node.value == _S030_LEGACY_ROUTE:
880+
self._s030_record(node, f"use {_S030_LEGACY_ROUTE}")
821881

822882
self.generic_visit(node)
823883

@@ -1032,6 +1092,22 @@ def _s024_visit_call(self, node: ast.Call) -> None:
10321092
if self._s024_parses is None:
10331093
self._s024_parses = (node.lineno, node.col_offset)
10341094

1095+
def _s030_record(self, node: ast.stmt | ast.expr, kind: str) -> None:
1096+
function = self._function_stack[-1] if self._function_stack else _S030_MODULE_SCOPE
1097+
self._s030_uses.append((node.lineno, node.col_offset, function, kind))
1098+
1099+
def _s030_call_name(self, node: ast.expr) -> str | None:
1100+
if isinstance(node, ast.Name):
1101+
return self._s030_legacy_names.get(node.id)
1102+
1103+
qualified = _name_of(node)
1104+
for legacy_name in _S030_LEGACY_NAMES:
1105+
if qualified == f"{_S030_LEGACY_MODULE}.{legacy_name}":
1106+
return legacy_name
1107+
if any(qualified == f"{alias}.{legacy_name}" for alias in self._s030_module_aliases):
1108+
return legacy_name
1109+
return None
1110+
10351111
def visit_Subscript(self, node: ast.Subscript) -> None:
10361112
# Load only: `request.data["title"] = ...` writes a value, it does not
10371113
# read a parameter the client sent.
@@ -1120,6 +1196,8 @@ def _exit_input(self) -> None:
11201196

11211197
def visit_Call(self, node: ast.Call) -> None:
11221198
self._s024_visit_call(node)
1199+
if legacy_name := self._s030_call_name(node.func):
1200+
self._s030_record(node, f"call {legacy_name}")
11231201
if isinstance(node.func, ast.Attribute) and node.func.attr == "urlopen":
11241202
self._s029_urlopen_calls.append(
11251203
(node.lineno, node.col_offset, _name_of(node.func.value))
@@ -1201,6 +1279,7 @@ def run(self) -> Generator[tuple[int, int, str, type[Any]]]:
12011279

12021280
yield from self._s024(visitor)
12031281
yield from self._s029(visitor)
1282+
yield from self._s030(visitor)
12041283

12051284
for e in visitor.errors:
12061285
yield (*e, type(self))
@@ -1221,3 +1300,21 @@ def _s029(self, visitor: SentryVisitor) -> Generator[tuple[int, int, str, type[A
12211300
for line, col, receiver in visitor._s029_urlopen_calls:
12221301
if receiver in visitor._s029_seer_pool_names:
12231302
yield (line, col, S029_msg, type(self))
1303+
1304+
def _s030(self, visitor: SentryVisitor) -> Generator[tuple[int, int, str, type[Any]]]:
1305+
rel = _repo_relative(self.filename)
1306+
if not rel.startswith("src/sentry/") or rel == "src/sentry/seer/signed_seer_api.py":
1307+
return
1308+
1309+
allowed = S030_allowlist.get(rel, {})
1310+
seen: Counter[tuple[str, str]] = Counter()
1311+
for line, col, function, kind in visitor._s030_uses:
1312+
key = (function, kind)
1313+
seen[key] += 1
1314+
if seen[key] > allowed.get(key, 0):
1315+
yield (line, col, S030_msg, type(self))
1316+
1317+
for key, expected_count in allowed.items():
1318+
if seen[key] < expected_count:
1319+
function, kind = key
1320+
yield (1, 0, S030_stale_msg.format(kind, function), type(self))

0 commit comments

Comments
 (0)