Skip to content

Commit 7214301

Browse files
marcusquinnmarcusquinn
authored
fix: verify OpenAI OAuth fallback health (#24695)
1 parent 810aad7 commit 7214301

2 files changed

Lines changed: 188 additions & 35 deletions

File tree

‎.agents/scripts/model-availability-probe-lib.sh‎

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -474,33 +474,34 @@ _probe_allows_oauth_fallback_after_rejected_key() {
474474

475475
_probe_openai_oauth_fallback_verified() {
476476
local auth_file="$1"
477+
local quiet="${2:-false}"
477478

478-
[[ -f "$auth_file" ]] || return 1
479-
command -v jq >/dev/null 2>&1 || return 1
480-
481-
# OpenAI ChatGPT OAuth is usable by the runtime when auth.json contains a
482-
# refresh token, or a still-live access token. Do not treat openai.type ==
483-
# "oauth" alone as provider health after a rejected static key (GH#24636).
484-
local has_refresh
485-
has_refresh=$(jq -r '.openai.refresh // empty' "$auth_file" 2>/dev/null) || has_refresh=""
486-
if [[ -n "$has_refresh" ]]; then
487-
return 0
488-
fi
479+
[[ -f "$auth_file" ]] || return 3
480+
command -v jq >/dev/null 2>&1 || return 3
489481

482+
# OpenAI ChatGPT OAuth fallback is only provider-health evidence after a
483+
# successful OpenAI API probe with a currently-live access token. Refresh-token
484+
# presence alone proved too weak: quota-exhausted accounts still looked
485+
# healthy and were selected for workers (GH#24694).
490486
local access_token expires_ms now_s now_ms
491487
access_token=$(jq -r '.openai.access // empty' "$auth_file" 2>/dev/null) || access_token=""
492488
expires_ms=$(jq -r '.openai.expires // empty' "$auth_file" 2>/dev/null) || expires_ms=""
493-
if [[ -n "$access_token" && "$expires_ms" =~ ^[0-9]+$ ]]; then
494-
now_s=$(date +%s 2>/dev/null) || now_s=""
495-
if [[ "$now_s" =~ ^[0-9]+$ ]]; then
496-
now_ms=$((now_s * 1000))
497-
if [[ "$expires_ms" -gt "$now_ms" ]]; then
498-
return 0
499-
fi
500-
fi
489+
if [[ -z "$access_token" || ! "$expires_ms" =~ ^[0-9]+$ ]]; then
490+
return 3
501491
fi
502492

503-
return 1
493+
now_s=$(date +%s 2>/dev/null) || now_s=""
494+
if [[ ! "$now_s" =~ ^[0-9]+$ ]]; then
495+
return 3
496+
fi
497+
498+
now_ms=$((now_s * 1000))
499+
if [[ "$expires_ms" -le "$now_ms" ]]; then
500+
return 3
501+
fi
502+
503+
_probe_execute_http openai "$access_token" "$quiet"
504+
return $?
504505
}
505506

506507
# _probe_check_oauth_fallback: called when an HTTP probe rejected the resolved
@@ -529,9 +530,13 @@ _probe_check_oauth_fallback() {
529530
local auth_type
530531
auth_type=$(jq -r --arg p "$provider" '.[$p].type // empty' "$auth_file" 2>/dev/null) || auth_type=""
531532
if [[ "$auth_type" == "oauth" ]]; then
532-
if [[ "$provider" == openai ]] && ! _probe_openai_oauth_fallback_verified "$auth_file"; then
533-
[[ "$quiet" != "true" ]] && print_warning "$provider: env key rejected (HTTP 401/403); OpenAI OAuth in auth.json is not refreshable or currently valid"
534-
return 3
533+
if [[ "$provider" == openai ]]; then
534+
local oauth_exit=0
535+
_probe_openai_oauth_fallback_verified "$auth_file" "$quiet" || oauth_exit=$?
536+
if [[ "$oauth_exit" -ne 0 ]]; then
537+
[[ "$quiet" != "true" ]] && print_warning "$provider: env key rejected (HTTP 401/403); OpenAI OAuth API verification failed"
538+
return "$oauth_exit"
539+
fi
535540
fi
536541
[[ "$quiet" != "true" ]] && print_success "$provider: env key rejected (HTTP 401/403) but OAuth found in auth.json — recording healthy (t3229)"
537542
_record_health "$provider" "healthy" 0 0 "OAuth available (env key rejected, t3229)" 0

‎.agents/scripts/tests/test-model-availability-oauth-probe.sh‎

Lines changed: 160 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,32 @@ init_db >/dev/null 2>&1 || {
157157
# Keep going so the summary still reports useful state.
158158
}
159159

160+
OPENAI_OAUTH_STUB_DIR="${TEST_ROOT}/openai-oauth-curl-stub"
161+
OPENAI_OAUTH_CURL_LOG="${TEST_ROOT}/openai-oauth-curl.log"
162+
mkdir -p "$OPENAI_OAUTH_STUB_DIR"
163+
cat >"${OPENAI_OAUTH_STUB_DIR}/curl" <<'SH'
164+
#!/usr/bin/env bash
165+
printf '%s\n' "$*" >>"${OPENAI_OAUTH_CURL_LOG:?}"
166+
case "${OPENAI_OAUTH_STUB_MODE:-success}" in
167+
success)
168+
printf 'HTTP/2 200\n\n{"data":[{"id":"gpt-5.5"}]}\n0.001\n200\n'
169+
;;
170+
quota)
171+
printf 'HTTP/2 403\n\n{"error":{"message":"You exceeded your current quota, please check your plan and billing details.","type":"insufficient_quota","code":"insufficient_quota"}}\n0.001\n403\n'
172+
;;
173+
auth)
174+
printf 'HTTP/2 401\n\n{"error":{"message":"Invalid API key","type":"invalid_request_error","code":"invalid_api_key"}}\n0.001\n401\n'
175+
;;
176+
network)
177+
exit 7
178+
;;
179+
*)
180+
exit 2
181+
;;
182+
esac
183+
SH
184+
chmod +x "${OPENAI_OAUTH_STUB_DIR}/curl"
185+
160186
# Call the validator with quiet=true to suppress the print_success line.
161187
# Discard stdout (it echoes the api_key on success, which we don't want
162188
# printed even masked). Capture only the exit code.
@@ -211,24 +237,47 @@ else
211237
"(got rc=$rc — expected 3; built-in openai would reuse the rejected env key)"
212238
fi
213239

214-
# Assertion 4b — OAuth still wins when the rejected helper key is not from env.
240+
# Assertion 4b — OAuth still wins when the rejected helper key is not from env,
241+
# but only after a successful OpenAI API verification with the OAuth access
242+
# token (GH#24694).
215243
unset OPENAI_API_KEY
216-
_probe_check_oauth_fallback openai true "gopass:OPENAI_API_KEY"
217-
rc=$?
244+
future_ms=$(($(date +%s) * 1000 + 3600000))
245+
cat >"$AUTH_FILE" <<JSON
246+
{
247+
"openai": {
248+
"type": "oauth",
249+
"access": "fake-openai-live-access-token",
250+
"refresh": "fake-openai-refresh-token",
251+
"expires": ${future_ms}
252+
}
253+
}
254+
JSON
255+
rc=$(
256+
OPENAI_OAUTH_STUB_MODE=success \
257+
OPENAI_OAUTH_CURL_LOG="$OPENAI_OAUTH_CURL_LOG" \
258+
PATH="${OPENAI_OAUTH_STUB_DIR}:${PATH}" \
259+
_probe_check_oauth_fallback openai true "gopass:OPENAI_API_KEY" >/dev/null
260+
printf '%s\n' "$?"
261+
)
218262
if [[ "$rc" -eq 0 ]]; then
219-
print_result "rejected-non-env-key+openai-oauth: _probe_check_oauth_fallback returns 0 (t3229 preserved)" 0
263+
print_result "rejected-non-env-key+openai-oauth: verified API fallback returns 0" 0
220264
else
221-
print_result "rejected-non-env-key+openai-oauth: _probe_check_oauth_fallback returns 0 (t3229 preserved)" 1 \
222-
"(got rc=$rc — expected 0; non-env stale key should not mask OAuth in auth.json)"
265+
print_result "rejected-non-env-key+openai-oauth: verified API fallback returns 0" 1 \
266+
"(got rc=$rc — expected 0; successful OAuth API verification should record healthy)"
223267
fi
224268

225269
# Assertion 4b.1 — Sourced credentials.sh-style variables are not process env.
226270
# resolve_api_key sources credentials.sh into the helper process, leaving an
227271
# OPENAI_API_KEY shell variable behind even though key_source is credentials:*.
228272
# That variable must not be mistaken for an exported runtime env key.
229273
OPENAI_API_KEY="[redacted-credential]"
230-
_probe_check_oauth_fallback openai true "credentials:OPENAI_API_KEY"
231-
rc=$?
274+
rc=$(
275+
OPENAI_OAUTH_STUB_MODE=success \
276+
OPENAI_OAUTH_CURL_LOG="$OPENAI_OAUTH_CURL_LOG" \
277+
PATH="${OPENAI_OAUTH_STUB_DIR}:${PATH}" \
278+
_probe_check_oauth_fallback openai true "credentials:OPENAI_API_KEY" >/dev/null
279+
printf '%s\n' "$?"
280+
)
232281
if [[ "$rc" -eq 0 ]]; then
233282
print_result "rejected-credentials-key+openai-oauth: sourced variable does not block OAuth fallback" 0
234283
else
@@ -257,7 +306,7 @@ else
257306
fi
258307

259308
# Assertion 4b.3 — A currently-live OpenAI OAuth access token is usable even
260-
# without refresh, preserving runtime-compatible OAuth fallback when verified.
309+
# without refresh, but only when the OpenAI API verification succeeds.
261310
future_ms=$(($(date +%s) * 1000 + 3600000))
262311
cat >"$AUTH_FILE" <<JSON
263312
{
@@ -268,13 +317,112 @@ cat >"$AUTH_FILE" <<JSON
268317
}
269318
}
270319
JSON
271-
_probe_check_oauth_fallback openai true "gopass:OPENAI_API_KEY"
272-
rc=$?
320+
rc=$(
321+
OPENAI_OAUTH_STUB_MODE=success \
322+
OPENAI_OAUTH_CURL_LOG="$OPENAI_OAUTH_CURL_LOG" \
323+
PATH="${OPENAI_OAUTH_STUB_DIR}:${PATH}" \
324+
_probe_check_oauth_fallback openai true "gopass:OPENAI_API_KEY" >/dev/null
325+
printf '%s\n' "$?"
326+
)
273327
if [[ "$rc" -eq 0 ]]; then
274328
print_result "openai-oauth-live-access: verified fallback remains healthy" 0
275329
else
276330
print_result "openai-oauth-live-access: verified fallback remains healthy" 1 \
277-
"(got rc=$rc — expected 0; live OAuth access should remain eligible)"
331+
"(got rc=$rc — expected 0; live OAuth access with successful API verification should remain eligible)"
332+
fi
333+
334+
# Assertion 4b.3a — Refresh-token existence alone no longer proves health.
335+
cat >"$AUTH_FILE" <<JSON
336+
{
337+
"openai": {
338+
"type": "oauth",
339+
"refresh": "fake-openai-refresh-token"
340+
}
341+
}
342+
JSON
343+
rm -f "$OPENAI_OAUTH_CURL_LOG"
344+
rc=$(
345+
OPENAI_OAUTH_STUB_MODE=success \
346+
OPENAI_OAUTH_CURL_LOG="$OPENAI_OAUTH_CURL_LOG" \
347+
PATH="${OPENAI_OAUTH_STUB_DIR}:${PATH}" \
348+
_probe_check_oauth_fallback openai true "gopass:OPENAI_API_KEY" >/dev/null
349+
printf '%s\n' "$?"
350+
)
351+
if [[ "$rc" -eq 3 && ! -f "$OPENAI_OAUTH_CURL_LOG" ]]; then
352+
print_result "openai-oauth-refresh-only: fallback fails closed without API verification" 0
353+
else
354+
print_result "openai-oauth-refresh-only: fallback fails closed without API verification" 1 \
355+
"(got rc=$rc, curl_log_exists=$([[ -f "$OPENAI_OAUTH_CURL_LOG" ]] && printf yes || printf no) — expected rc=3 and no curl call)"
356+
fi
357+
358+
# Assertion 4b.3b — Quota-exhausted OAuth verification is unhealthy, not healthy.
359+
cat >"$AUTH_FILE" <<JSON
360+
{
361+
"openai": {
362+
"type": "oauth",
363+
"access": "fake-openai-quota-access-token",
364+
"refresh": "fake-openai-refresh-token",
365+
"expires": ${future_ms}
366+
}
367+
}
368+
JSON
369+
rc=$(
370+
OPENAI_OAUTH_STUB_MODE=quota \
371+
OPENAI_OAUTH_CURL_LOG="$OPENAI_OAUTH_CURL_LOG" \
372+
PATH="${OPENAI_OAUTH_STUB_DIR}:${PATH}" \
373+
_probe_check_oauth_fallback openai true "gopass:OPENAI_API_KEY" >/dev/null
374+
printf '%s\n' "$?"
375+
)
376+
if [[ "$rc" -eq 1 ]]; then
377+
print_result "openai-oauth-quota: verification returns unhealthy instead of healthy" 0
378+
else
379+
print_result "openai-oauth-quota: verification returns unhealthy instead of healthy" 1 \
380+
"(got rc=$rc — expected 1; insufficient quota must not record healthy)"
381+
fi
382+
383+
# Assertion 4b.3c — Generic OAuth auth failure remains key-invalid.
384+
rc=$(
385+
OPENAI_OAUTH_STUB_MODE=auth \
386+
OPENAI_OAUTH_CURL_LOG="$OPENAI_OAUTH_CURL_LOG" \
387+
PATH="${OPENAI_OAUTH_STUB_DIR}:${PATH}" \
388+
_probe_check_oauth_fallback openai true "gopass:OPENAI_API_KEY" >/dev/null
389+
printf '%s\n' "$?"
390+
)
391+
if [[ "$rc" -eq 3 ]]; then
392+
print_result "openai-oauth-auth-failure: verification returns key-invalid" 0
393+
else
394+
print_result "openai-oauth-auth-failure: verification returns key-invalid" 1 \
395+
"(got rc=$rc — expected 3; invalid OAuth token must not record healthy)"
396+
fi
397+
398+
# Assertion 4b.3d — Network failure while verifying OAuth fails closed.
399+
rc=$(
400+
OPENAI_OAUTH_STUB_MODE=network \
401+
OPENAI_OAUTH_CURL_LOG="$OPENAI_OAUTH_CURL_LOG" \
402+
PATH="${OPENAI_OAUTH_STUB_DIR}:${PATH}" \
403+
_probe_check_oauth_fallback openai true "gopass:OPENAI_API_KEY" >/dev/null
404+
printf '%s\n' "$?"
405+
)
406+
if [[ "$rc" -eq 1 ]]; then
407+
print_result "openai-oauth-network-failure: verification fails closed" 0
408+
else
409+
print_result "openai-oauth-network-failure: verification fails closed" 1 \
410+
"(got rc=$rc — expected 1; network failure must not record healthy)"
411+
fi
412+
413+
# Assertion 4b.3e — Probe output must not expose OAuth access tokens.
414+
oauth_output=$(
415+
OPENAI_OAUTH_STUB_MODE=quota \
416+
OPENAI_OAUTH_CURL_LOG="$OPENAI_OAUTH_CURL_LOG" \
417+
PATH="${OPENAI_OAUTH_STUB_DIR}:${PATH}" \
418+
_probe_check_oauth_fallback openai false "gopass:OPENAI_API_KEY" 2>&1
419+
)
420+
rc=$?
421+
if [[ "$rc" -eq 1 && "$oauth_output" != *"fake-openai-quota-access-token"* ]]; then
422+
print_result "openai-oauth-token-logging: output does not expose access token" 0
423+
else
424+
print_result "openai-oauth-token-logging: output does not expose access token" 1 \
425+
"(got rc=$rc; output must not contain OAuth access token)"
278426
fi
279427

280428
# Assertion 4b.4 — date failures fail closed instead of producing arithmetic

0 commit comments

Comments
 (0)