Skip to content

Commit 58f4e6b

Browse files
joshuarlijoshuarli
committed
fix(tests): Fix flaky test_get_feedback_categories_basic with explicit time ranges
The test was flaky because feedbacks from different test methods share the same project/org and the endpoint queries a 7-day window by default. Feedbacks created by one test method were visible in other test methods' queries, causing numFeedbacksContext to return 5 instead of 4. Fix by using explicit timestamps for each feedback and passing narrow start/end time ranges to the API request, isolating each test's data. Fixes #108654
1 parent 87c8531 commit 58f4e6b

1 file changed

Lines changed: 90 additions & 24 deletions

File tree

‎tests/sentry/feedback/endpoints/test_organization_feedback_categories.py‎

Lines changed: 90 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from datetime import datetime
1+
from datetime import UTC, datetime, timedelta
22
from unittest.mock import patch
33

44
from django.urls import reverse
@@ -85,10 +85,23 @@ def test_get_feedback_categories_without_seer_access(self) -> None:
8585
assert response.status_code == 403
8686

8787
def test_get_feedback_categories_basic(self) -> None:
88-
self._create_feedback("a", ["User Interface", "Speed"], self.project1)
89-
self._create_feedback("b", ["Performance", "Usability", "Loading"], self.project1)
90-
self._create_feedback("c", ["Security", "Performance"], self.project2)
91-
self._create_feedback("d", ["Performance", "User Interface", "Speed"], self.project2)
88+
base_time = datetime.now(UTC) - timedelta(hours=1)
89+
self._create_feedback("a", ["User Interface", "Speed"], self.project1, dt=base_time)
90+
self._create_feedback(
91+
"b",
92+
["Performance", "Usability", "Loading"],
93+
self.project1,
94+
dt=base_time + timedelta(seconds=1),
95+
)
96+
self._create_feedback(
97+
"c", ["Security", "Performance"], self.project2, dt=base_time + timedelta(seconds=2)
98+
)
99+
self._create_feedback(
100+
"d",
101+
["Performance", "User Interface", "Speed"],
102+
self.project2,
103+
dt=base_time + timedelta(seconds=3),
104+
)
92105

93106
self.mock_make_signed_seer_api_request.return_value = MockSeerResponse(
94107
200,
@@ -106,7 +119,11 @@ def test_get_feedback_categories_basic(self) -> None:
106119
)
107120

108121
with self.feature(self.features):
109-
response = self.get_success_response(self.org.slug)
122+
response = self.get_success_response(
123+
self.org.slug,
124+
start=base_time.isoformat(),
125+
end=(base_time + timedelta(minutes=1)).isoformat(),
126+
)
110127

111128
assert response.data["success"] is True
112129
assert response.data["numFeedbacksContext"] == 4
@@ -130,10 +147,23 @@ def test_get_feedback_categories_basic(self) -> None:
130147
assert category["feedbackCount"] == 0
131148

132149
def test_get_feedback_categories_with_project_filter(self) -> None:
133-
self._create_feedback("a", ["User Interface", "Performance"], self.project1)
134-
self._create_feedback("b", ["Performance", "Loading"], self.project1)
135-
self._create_feedback("c", ["Security", "Performance"], self.project2)
136-
self._create_feedback("d", ["Performance", "User Interface", "Speed"], self.project2)
150+
base_time = datetime.now(UTC) - timedelta(hours=2)
151+
self._create_feedback("a", ["User Interface", "Performance"], self.project1, dt=base_time)
152+
self._create_feedback(
153+
"b", ["Performance", "Loading"], self.project1, dt=base_time + timedelta(seconds=1)
154+
)
155+
self._create_feedback(
156+
"c",
157+
["Security", "Performance"],
158+
self.project2,
159+
dt=base_time + timedelta(seconds=2),
160+
)
161+
self._create_feedback(
162+
"d",
163+
["Performance", "User Interface", "Speed"],
164+
self.project2,
165+
dt=base_time + timedelta(seconds=3),
166+
)
137167

138168
self.mock_make_signed_seer_api_request.return_value = MockSeerResponse(
139169
200,
@@ -149,7 +179,12 @@ def test_get_feedback_categories_with_project_filter(self) -> None:
149179
)
150180

151181
with self.feature(self.features):
152-
response = self.get_success_response(self.org.slug, project=[self.project1.id])
182+
response = self.get_success_response(
183+
self.org.slug,
184+
project=[self.project1.id],
185+
start=base_time.isoformat(),
186+
end=(base_time + timedelta(minutes=1)).isoformat(),
187+
)
153188

154189
assert response.data["success"] is True
155190
assert response.data["numFeedbacksContext"] == 2
@@ -172,9 +207,14 @@ def test_get_feedback_categories_with_project_filter(self) -> None:
172207
)
173208
def test_max_group_labels_limit(self) -> None:
174209
"""Test that MAX_GROUP_LABELS constant is respected when processing label groups."""
175-
self._create_feedback("a", ["User Interface"], self.project1)
176-
self._create_feedback("b", ["User Interface", "Usability"], self.project1)
177-
self._create_feedback("c", ["Accessibility"], self.project1)
210+
base_time = datetime.now(UTC) - timedelta(hours=3)
211+
self._create_feedback("a", ["User Interface"], self.project1, dt=base_time)
212+
self._create_feedback(
213+
"b", ["User Interface", "Usability"], self.project1, dt=base_time + timedelta(seconds=1)
214+
)
215+
self._create_feedback(
216+
"c", ["Accessibility"], self.project1, dt=base_time + timedelta(seconds=2)
217+
)
178218

179219
# Mock Seer to return a label group with more than MAX_GROUP_LABELS labels
180220
self.mock_make_signed_seer_api_request.return_value = MockSeerResponse(
@@ -190,7 +230,11 @@ def test_max_group_labels_limit(self) -> None:
190230
)
191231

192232
with self.feature(self.features):
193-
response = self.get_success_response(self.org.slug)
233+
response = self.get_success_response(
234+
self.org.slug,
235+
start=base_time.isoformat(),
236+
end=(base_time + timedelta(minutes=1)).isoformat(),
237+
)
194238

195239
assert response.data["success"] is True
196240
categories = response.data["categories"]
@@ -204,8 +248,11 @@ def test_filter_invalid_associated_labels_by_count_ratio(self) -> None:
204248
"""Test that associated labels with too many feedbacks (relative to primary label) are filtered out."""
205249
# Create feedbacks where associated label feedbacks are >= primary label feedbacks.
206250
# This should cause them to be filtered out from the label group.
207-
self._create_feedback("a", ["User Interface", "Issues UI"], self.project1)
208-
self._create_feedback("b", ["Usability", "Issues UI"], self.project1)
251+
base_time = datetime.now(UTC) - timedelta(hours=4)
252+
self._create_feedback("a", ["User Interface", "Issues UI"], self.project1, dt=base_time)
253+
self._create_feedback(
254+
"b", ["Usability", "Issues UI"], self.project1, dt=base_time + timedelta(seconds=1)
255+
)
209256

210257
# XXX: the endpoint checks for assoc >= 3/4 * primary, but this test is more lenient in case the ratio changes.
211258

@@ -222,7 +269,11 @@ def test_filter_invalid_associated_labels_by_count_ratio(self) -> None:
222269
)
223270

224271
with self.feature(self.features):
225-
response = self.get_success_response(self.org.slug)
272+
response = self.get_success_response(
273+
self.org.slug,
274+
start=base_time.isoformat(),
275+
end=(base_time + timedelta(minutes=1)).isoformat(),
276+
)
226277

227278
assert response.data["success"] is True
228279
categories = response.data["categories"]
@@ -232,39 +283,54 @@ def test_filter_invalid_associated_labels_by_count_ratio(self) -> None:
232283
assert categories[0]["feedbackCount"] == 1
233284

234285
def test_seer_request_error(self) -> None:
235-
self._create_feedback("a", ["User Interface", "Issues UI"], self.project1)
286+
base_time = datetime.now(UTC) - timedelta(hours=5)
287+
self._create_feedback("a", ["User Interface", "Issues UI"], self.project1, dt=base_time)
236288
self.mock_make_signed_seer_api_request.side_effect = Exception("seer failed")
237289

238290
with self.feature(self.features):
239-
response = self.get_error_response(self.org.slug)
291+
response = self.get_error_response(
292+
self.org.slug,
293+
start=base_time.isoformat(),
294+
end=(base_time + timedelta(minutes=1)).isoformat(),
295+
)
240296

241297
assert response.status_code == 500
242298
assert response.data["detail"] == "Failed to generate user feedback label groups"
243299

244300
def test_seer_http_errors(self) -> None:
245-
self._create_feedback("a", ["User Interface", "Issues UI"], self.project1)
301+
base_time = datetime.now(UTC) - timedelta(hours=6)
302+
self._create_feedback("a", ["User Interface", "Issues UI"], self.project1, dt=base_time)
246303
for status in [400, 401, 403, 404, 429, 500, 502, 503, 504]:
247304
self.mock_make_signed_seer_api_request.return_value = MockSeerResponse(
248305
status=status, json_data={"detail": "seer failed"}
249306
)
250307

251308
with self.feature(self.features):
252-
response = self.get_error_response(self.org.slug)
309+
response = self.get_error_response(
310+
self.org.slug,
311+
start=base_time.isoformat(),
312+
end=(base_time + timedelta(minutes=1)).isoformat(),
313+
)
253314

254315
assert response.status_code == 500
255316
assert response.data["detail"] == "Failed to generate user feedback label groups"
256317

257318
def test_fallback_to_primary_labels_when_below_threshold(self) -> None:
258319
"""Test that when feedback count is below THRESHOLD_TO_GET_ASSOCIATED_LABELS, we fall back to primary labels only (no Seer request)."""
320+
base_time = datetime.now(UTC) - timedelta(hours=7)
259321

260322
with patch(
261323
"sentry.feedback.endpoints.organization_feedback_categories.THRESHOLD_TO_GET_ASSOCIATED_LABELS",
262324
2,
263325
):
264-
self._create_feedback("a", ["User Interface", "Usability"], self.project1)
326+
self._create_feedback("a", ["User Interface", "Usability"], self.project1, dt=base_time)
265327

266328
with self.feature(self.features):
267-
response = self.get_success_response(self.org.slug)
329+
response = self.get_success_response(
330+
self.org.slug,
331+
start=base_time.isoformat(),
332+
end=(base_time + timedelta(minutes=1)).isoformat(),
333+
)
268334

269335
assert self.mock_make_signed_seer_api_request.call_count == 0
270336

0 commit comments

Comments
 (0)