Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/ray-core/api/exceptions.rst
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,4 @@ Exceptions
ray.exceptions.RaySystemError
ray.exceptions.NodeDiedError
ray.exceptions.UnserializableException
ray.exceptions.RayAuthenticationError
Comment thread
edoakes marked this conversation as resolved.
Outdated
24 changes: 4 additions & 20 deletions python/ray/_private/authentication/authentication_constants.py
Original file line number Diff line number Diff line change
@@ -1,27 +1,11 @@
# Token setup instructions (used in multiple contexts)
TOKEN_SETUP_INSTRUCTIONS = """Please provide an authentication token using one of these methods:
1. Set the RAY_AUTH_TOKEN environment variable
2. Set the RAY_AUTH_TOKEN_PATH environment variable (pointing to a token file)
3. Create a token file at the default location: ~/.ray/auth_token"""

# When token auth is enabled but no token is found anywhere
# Authentication error messages
TOKEN_AUTH_ENABLED_BUT_NO_TOKEN_FOUND_ERROR_MESSAGE = (
"Token authentication is enabled but no authentication token was found. "
+ TOKEN_SETUP_INSTRUCTIONS
)

# When HTTP request fails with 401 (Unauthorized - missing token)
HTTP_REQUEST_MISSING_TOKEN_ERROR_MESSAGE = (
"The Ray cluster requires authentication, but no token was provided.\n\n"
+ TOKEN_SETUP_INSTRUCTIONS
"Token authentication is enabled but no authentication token was found."
)

# When HTTP request fails with 403 (Forbidden - invalid token)
HTTP_REQUEST_INVALID_TOKEN_ERROR_MESSAGE = (
"The authentication token you provided is invalid or incorrect.\n\n"
+ TOKEN_SETUP_INSTRUCTIONS
)
TOKEN_INVALID_ERROR_MESSAGE = "Token authentication is enabled but the authentication token is invalid or incorrect." # noqa: E501

# HTTP header and cookie constants
AUTHORIZATION_HEADER_NAME = "authorization"
AUTHORIZATION_BEARER_PREFIX = "Bearer "

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
AuthenticationTokenLoader,
get_authentication_mode,
)
from ray.exceptions import RayAuthenticationError

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -97,4 +98,6 @@ def ensure_token_if_auth_enabled(
# Reload the cache so subsequent calls to token_loader read the new token.
token_loader.reset_cache()
else:
raise RuntimeError(TOKEN_AUTH_ENABLED_BUT_NO_TOKEN_FOUND_ERROR_MESSAGE)
raise RayAuthenticationError(
TOKEN_AUTH_ENABLED_BUT_NO_TOKEN_FOUND_ERROR_MESSAGE
)
27 changes: 15 additions & 12 deletions python/ray/_private/authentication/http_token_authentication.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,18 +104,21 @@ def get_auth_headers_if_auth_enabled(user_headers: Dict[str, str]) -> Dict[str,


def format_authentication_http_error(status: int, body: str) -> Optional[str]:
"""Return a user-friendly authentication error message, if applicable."""
"""Return a user-friendly authentication error message, if applicable.

if status == 401:
return "Authentication required: {body}\n\n{details}".format(
body=body,
details=authentication_constants.HTTP_REQUEST_MISSING_TOKEN_ERROR_MESSAGE,
)
Args:
status: HTTP status code
body: Response body text

if status == 403:
return "Authentication failed: {body}\n\n{details}".format(
body=body,
details=authentication_constants.HTTP_REQUEST_INVALID_TOKEN_ERROR_MESSAGE,
)
Returns:
Formatted error message if auth-related (401/403), None otherwise
"""
Comment thread
sampan-s-nayak marked this conversation as resolved.
Outdated
if status == 401: # Unauthorized - missing token
return (
authentication_constants.TOKEN_AUTH_ENABLED_BUT_NO_TOKEN_FOUND_ERROR_MESSAGE
) # noqa: E501

if status == 403: # Forbidden - invalid token
return authentication_constants.TOKEN_INVALID_ERROR_MESSAGE

return None
return None # Not an auth error, let caller handle it
3 changes: 2 additions & 1 deletion python/ray/dashboard/modules/dashboard_sdk.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from ray._private.utils import split_address
from ray.autoscaler._private.cli_logger import cli_logger
from ray.dashboard.modules.job.common import uri_to_http_components
from ray.exceptions import RayAuthenticationError
from ray.util.annotations import DeveloperAPI, PublicAPI

try:
Expand Down Expand Up @@ -323,7 +324,7 @@ def _do_request(
response.status_code, response.text
)
if formatted_error:
raise RuntimeError(formatted_error)
raise RayAuthenticationError(formatted_error)

return response

Expand Down
22 changes: 22 additions & 0 deletions python/ray/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,27 @@ def __str__(self):
return error_msg


@PublicAPI
class RayAuthenticationError(RayError):
"""Indicates that an authentication error occurred. only applicable when RAY_AUTH_MODE is not disabled."""
Comment thread
sampan-s-nayak marked this conversation as resolved.
Outdated

def __init__(self, message: str):
"""Initialize RayAuthenticationError.

Args:
message: Error message describing the authentication failure
"""
self.message = message
# Always hide traceback for cleaner output
self.__suppress_context__ = True
super().__init__(message)

def __str__(self) -> str:
error_msg = f"{self.message}\n\n"
error_msg += "For more information, see: https://docs.ray.io/en/latest/ray-security/auth.html"
return error_msg


@DeveloperAPI
class UserCodeException(RayError):
"""Indicates that an exception occurred while executing user code.
Expand Down Expand Up @@ -963,4 +984,5 @@ def __str__(self):
OufOfBandObjectRefSerializationException,
RayCgraphCapacityExceeded,
UnserializableException,
RayAuthenticationError,
]
1 change: 1 addition & 0 deletions python/ray/includes/common.pxd
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ cdef extern from "ray/common/status.h" namespace "ray" nogil:
c_bool IsUnexpectedSystemExit()
c_bool IsChannelError()
c_bool IsChannelTimeoutError()
c_bool IsUnauthenticated()

c_string ToString()
c_string CodeAsString()
Expand Down
3 changes: 3 additions & 0 deletions python/ray/includes/common.pxi
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ from ray.exceptions import (
ActorDiedError,
RayError,
RaySystemError,
RayAuthenticationError,
RayTaskError,
ObjectStoreFullError,
OutOfDiskError,
Expand Down Expand Up @@ -105,6 +106,8 @@ cdef int check_status(const CRayStatus& status) except -1 nogil:
raise ValueError(message)
elif status.IsIOError():
raise IOError(message)
elif status.IsUnauthenticated():
raise RayAuthenticationError(message)
elif status.IsRpcError():
raise RpcError(message, rpc_code=status.rpc_code())
elif status.IsIntentionalSystemExit():
Expand Down
10 changes: 4 additions & 6 deletions python/ray/scripts/scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -2735,14 +2735,10 @@ def get_auth_token(generate):
# Check if token auth mode is enabled and provide guidance if not
if get_authentication_mode() != AuthenticationMode.TOKEN:
click.echo(
"Note: Token authentication is not currently enabled.",
"Token authentication is not currently enabled. To enable token authentication, set: export RAY_AUTH_MODE=token\n For more instructions, see: https://docs.ray.io/en/latest/ray-security/auth.html",
err=True,
)
click.echo(
"To enable token authentication, set: export RAY_AUTH_MODE=token",
err=True,
)
click.echo("", err=True)
sys.exit(1)

# Try to load existing token
loader = AuthenticationTokenLoader.instance()
Expand All @@ -2768,6 +2764,8 @@ def get_auth_token(generate):

# Print token to stdout (for piping) without newline
click.echo(token, nl=False)
# Print newline to stderr for clean terminal display (doesn't affect piping)
click.echo("", err=True)
Comment thread
cursor[bot] marked this conversation as resolved.


def add_command_alias(command, name, hidden):
Expand Down
29 changes: 29 additions & 0 deletions python/ray/tests/test_exceptions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
"""Tests for Ray exceptions."""
import pytest

from ray.exceptions import RayAuthenticationError, RayError


class TestRayAuthenticationError:
"""Tests for RayAuthenticationError exception."""

auth_doc_url = "https://docs.ray.io/en/latest/ray-security/auth.html"

def test_basic_creation(self):
"""Test basic RayAuthenticationError creation."""
error = RayAuthenticationError("Token is missing")
error_str = str(error)

assert "Token is missing" in error_str
assert self.auth_doc_url in error_str

def test_is_ray_error_subclass(self):
"""Test that RayAuthenticationError is a RayError subclass."""
error = RayAuthenticationError("Test")
assert isinstance(error, RayError)


if __name__ == "__main__":
import sys

sys.exit(pytest.main(["-v", __file__]))
8 changes: 6 additions & 2 deletions python/ray/tests/test_runtime_env_agent_auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@

import ray
from ray._common.test_utils import wait_for_condition
from ray._private.authentication.authentication_constants import (
TOKEN_AUTH_ENABLED_BUT_NO_TOKEN_FOUND_ERROR_MESSAGE,
TOKEN_INVALID_ERROR_MESSAGE,
)
from ray._private.authentication.http_token_authentication import (
format_authentication_http_error,
get_auth_headers_if_auth_enabled,
Expand Down Expand Up @@ -66,7 +70,7 @@ def test_runtime_env_agent_requires_auth_missing_token(setup_cluster_with_token_
body = exc_info.value.read().decode("utf-8", "ignore")
assert "Missing authentication token" in body
formatted = format_authentication_http_error(401, body)
Comment thread
sampan-s-nayak marked this conversation as resolved.
assert formatted.startswith("Authentication required")
assert formatted == TOKEN_AUTH_ENABLED_BUT_NO_TOKEN_FOUND_ERROR_MESSAGE


def test_runtime_env_agent_rejects_invalid_token(setup_cluster_with_token_auth):
Expand All @@ -92,7 +96,7 @@ def test_runtime_env_agent_rejects_invalid_token(setup_cluster_with_token_auth):
body = exc_info.value.read().decode("utf-8", "ignore")
assert "Invalid authentication token" in body
formatted = format_authentication_http_error(403, body)
Comment thread
sampan-s-nayak marked this conversation as resolved.
assert formatted.startswith("Authentication failed")
assert formatted == TOKEN_INVALID_ERROR_MESSAGE


def test_runtime_env_agent_accepts_valid_token(setup_cluster_with_token_auth):
Expand Down
47 changes: 19 additions & 28 deletions python/ray/tests/test_submission_client_auth.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import pytest

from ray._private.authentication.authentication_constants import (
HTTP_REQUEST_INVALID_TOKEN_ERROR_MESSAGE,
HTTP_REQUEST_MISSING_TOKEN_ERROR_MESSAGE,
TOKEN_AUTH_ENABLED_BUT_NO_TOKEN_FOUND_ERROR_MESSAGE,
TOKEN_INVALID_ERROR_MESSAGE,
)
from ray._private.authentication_test_utils import (
clear_auth_token_sources,
Expand All @@ -12,6 +12,7 @@
)
from ray.dashboard.modules.dashboard_sdk import SubmissionClient
from ray.dashboard.modules.job.sdk import JobSubmissionClient
from ray.exceptions import RayAuthenticationError
from ray.util.state import StateApiClient


Expand All @@ -37,15 +38,13 @@ def test_submission_client_without_token_shows_helpful_error(

client = SubmissionClient(address=setup_cluster_with_token_auth["dashboard_url"])

# Make a request - should fail with helpful message
with pytest.raises(RuntimeError) as exc_info:
# Make a request - should fail with RayAuthenticationError
with pytest.raises(RayAuthenticationError) as exc_info:
client.get_version()

expected_message = (
"Authentication required: Unauthorized: Missing authentication token\n\n"
f"{HTTP_REQUEST_MISSING_TOKEN_ERROR_MESSAGE}"
)
assert str(exc_info.value) == expected_message
error_str = str(exc_info.value)
# Check that the error contains the simple message and auto-added docs link
assert TOKEN_AUTH_ENABLED_BUT_NO_TOKEN_FOUND_ERROR_MESSAGE in error_str


def test_submission_client_with_invalid_token_shows_helpful_error(
Expand All @@ -60,15 +59,13 @@ def test_submission_client_with_invalid_token_shows_helpful_error(

client = SubmissionClient(address=setup_cluster_with_token_auth["dashboard_url"])

# Make a request - should fail with helpful message
with pytest.raises(RuntimeError) as exc_info:
# Make a request - should fail with RayAuthenticationError
with pytest.raises(RayAuthenticationError) as exc_info:
client.get_version()

expected_message = (
"Authentication failed: Forbidden: Invalid authentication token\n\n"
f"{HTTP_REQUEST_INVALID_TOKEN_ERROR_MESSAGE}"
)
assert str(exc_info.value) == expected_message
error_str = str(exc_info.value)
# Check that the error contains the simple message and auto-added docs link
assert TOKEN_INVALID_ERROR_MESSAGE in error_str


def test_submission_client_with_valid_token_succeeds(setup_cluster_with_token_auth):
Expand Down Expand Up @@ -150,14 +147,11 @@ def test_error_messages_contain_instructions(setup_cluster_with_token_auth):

client = SubmissionClient(address=setup_cluster_with_token_auth["dashboard_url"])

with pytest.raises(RuntimeError) as exc_info:
with pytest.raises(RayAuthenticationError) as exc_info:
client.get_version()

expected_missing = (
"Authentication required: Unauthorized: Missing authentication token\n\n"
f"{HTTP_REQUEST_MISSING_TOKEN_ERROR_MESSAGE}"
)
assert str(exc_info.value) == expected_missing
error_str = str(exc_info.value)
assert TOKEN_AUTH_ENABLED_BUT_NO_TOKEN_FOUND_ERROR_MESSAGE in error_str

# Test 403 error (invalid token)
set_env_auth_token("wrong_token_00000000000000000000000000000000")
Expand All @@ -166,14 +160,11 @@ def test_error_messages_contain_instructions(setup_cluster_with_token_auth):

client2 = SubmissionClient(address=setup_cluster_with_token_auth["dashboard_url"])

with pytest.raises(RuntimeError) as exc_info:
with pytest.raises(RayAuthenticationError) as exc_info:
client2.get_version()

expected_invalid = (
"Authentication failed: Forbidden: Invalid authentication token\n\n"
f"{HTTP_REQUEST_INVALID_TOKEN_ERROR_MESSAGE}"
)
assert str(exc_info.value) == expected_invalid
error_str = str(exc_info.value)
assert TOKEN_INVALID_ERROR_MESSAGE in error_str


def test_no_token_added_when_auth_disabled(setup_cluster_without_token_auth):
Expand Down
3 changes: 2 additions & 1 deletion python/ray/util/client/server/proxier.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from ray._private.utils import detect_fate_sharing_support
from ray._raylet import GcsClient
from ray.cloudpickle.compat import pickle
from ray.exceptions import RayAuthenticationError
from ray.job_config import JobConfig
from ray.util.client.common import (
CLIENT_SERVER_MAX_THREADS,
Expand Down Expand Up @@ -287,7 +288,7 @@ def _create_runtime_env(

formatted_error = format_authentication_http_error(e.code, body or "")
Comment thread
sampan-s-nayak marked this conversation as resolved.
if formatted_error:
raise RuntimeError(formatted_error) from e
raise RayAuthenticationError(formatted_error) from e

# Treat non-auth HTTP errors like URLError (retry with backoff)
last_exception = e
Expand Down