[historyserver] Migrate S3 credentials to standard AWS environment variables (#4665)

* [historyserver] Migrate S3 credentials to standard AWS environment variables

Replace non-standard env vars with the standard AWS SDK ones:
- AWS_S3ID -> AWS_ACCESS_KEY_ID
- AWS_S3SECRET -> AWS_SECRET_ACCESS_KEY
- AWS_S3TOKEN -> AWS_SESSION_TOKEN

Fixes #4664

Signed-off-by: Ashutosh Agarwal <ashutosh18011@gmail.com>

* Update historyserver/pkg/storage/s3/config.go

Signed-off-by: Han-Ju Chen (Future-Outlier) <eric901201@gmail.com>

---------

Signed-off-by: Ashutosh Agarwal <ashutosh18011@gmail.com>
Signed-off-by: Han-Ju Chen (Future-Outlier) <eric901201@gmail.com>
Co-authored-by: Han-Ju Chen (Future-Outlier) <eric901201@gmail.com>

Author: ashutosh1807

historyserver/config/historyserver.yaml

@@ -36,11 +36,11 @@ spec:
         env:
           - name: S3DISABLE_SSL
             value: "true"
-          - name: AWS_S3ID
+          - name: AWS_ACCESS_KEY_ID
             value: minioadmin
-          - name: AWS_S3SECRET
+          - name: AWS_SECRET_ACCESS_KEY
             value: minioadmin
-          - name: AWS_S3TOKEN
+          - name: AWS_SESSION_TOKEN
             value: ""
           - name: S3_BUCKET
             value: "ray-historyserver"

historyserver/config/raycluster.yaml

@@ -100,11 +100,11 @@ spec:
             value: "30s"
           - name: S3DISABLE_SSL
             value: "true"
-          - name: AWS_S3ID
+          - name: AWS_ACCESS_KEY_ID
             value: minioadmin
-          - name: AWS_S3SECRET
+          - name: AWS_SECRET_ACCESS_KEY
             value: minioadmin
-          - name: AWS_S3TOKEN
+          - name: AWS_SESSION_TOKEN
             value: ""
           - name: S3_BUCKET
             value: "ray-historyserver"
@@ -203,11 +203,11 @@ spec:
           image: collector:v0.1.0
           imagePullPolicy: IfNotPresent
           env:
-          - name: AWS_S3ID
+          - name: AWS_ACCESS_KEY_ID
             value: minioadmin
-          - name: AWS_S3SECRET
+          - name: AWS_SECRET_ACCESS_KEY
             value: minioadmin
-          - name: AWS_S3TOKEN
+          - name: AWS_SESSION_TOKEN
             value: ""
           - name: S3_BUCKET
             value: "ray-historyserver"

historyserver/docs/set_up_historyserver.md

@@ -107,9 +107,9 @@ debugging in your own IDE. For example, you can set up `.vscode/launch.json` as
                 // Use localhost rather than the Kubernetes service name.
                 "S3_ENDPOINT": "localhost:9000",
                 "S3_BUCKET": "ray-historyserver",
-                "AWS_S3ID": "minioadmin",
-                "AWS_S3SECRET": "minioadmin",
-                "AWS_S3TOKEN": "",
+                "AWS_ACCESS_KEY_ID": "minioadmin",
+                "AWS_SECRET_ACCESS_KEY": "minioadmin",
+                "AWS_SESSION_TOKEN": "",
                 "S3FORCE_PATH_STYLE": "true",
                 "S3DISABLE_SSL": "true"
             }
@@ -132,9 +132,9 @@ make buildhistoryserver
 export S3_REGION=test
 export S3_ENDPOINT=localhost:9000
 export S3_BUCKET=ray-historyserver
-export AWS_S3ID=minioadmin
-export AWS_S3SECRET=minioadmin
-export AWS_S3TOKEN=
+export AWS_ACCESS_KEY_ID=minioadmin
+export AWS_SECRET_ACCESS_KEY=minioadmin
+export AWS_SESSION_TOKEN=
 export S3FORCE_PATH_STYLE=true
 export S3DISABLE_SSL=true
 

historyserver/pkg/storage/s3/config.go

@@ -16,9 +16,9 @@ type config struct {
 	S3Endpoint       string
 	S3Bucket         string
 	S3Region         string
-	S3ID             string
-	S3Secret         string
-	S3Token          string
+	AccessKeyID      string
+	SecretAccessKey  string
+	SessionToken     string
 	types.RayCollectorConfig
 }
 
@@ -32,9 +32,9 @@ func getS3BucketWithDefault() string {
 
 func (c *config) complete(rcc *types.RayCollectorConfig, jd map[string]interface{}) {
 	c.RayCollectorConfig = *rcc
-	c.S3ID = os.Getenv("AWS_S3ID")
-	c.S3Secret = os.Getenv("AWS_S3SECRET")
-	c.S3Token = os.Getenv("AWS_S3TOKEN")
+	c.AccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID")
+	c.SecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
+	c.SessionToken = os.Getenv("AWS_SESSION_TOKEN")
 	c.S3Bucket = getS3BucketWithDefault()
 	if len(jd) == 0 {
 		c.S3Endpoint = os.Getenv("S3_ENDPOINT")
@@ -68,10 +68,10 @@ func (c *config) completeHSConfig(rcc *types.RayHistoryServerConfig, jd map[stri
 	c.RayCollectorConfig = types.RayCollectorConfig{
 		RootDir: rcc.RootDir,
 	}
-	c.S3ID = os.Getenv("AWS_S3ID")
-	c.S3Secret = os.Getenv("AWS_S3SECRET")
-	c.S3Token = os.Getenv("AWS_S3TOKEN")
-	c.S3Bucket = getS3BucketWithDefault() // Use default if S3_BUCKET not set
+	c.AccessKeyID = os.Getenv("AWS_ACCESS_KEY_ID")
+	c.SecretAccessKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
+	c.SessionToken = os.Getenv("AWS_SESSION_TOKEN")
+	c.S3Bucket = getS3BucketWithDefault()
 	if len(jd) == 0 {
 		c.S3Endpoint = os.Getenv("S3_ENDPOINT")
 		c.S3Region = os.Getenv("S3_REGION")

historyserver/pkg/storage/s3/s3.go

@@ -342,8 +342,8 @@ func New(c *config) (*RayLogsHandler, error) {
 	// Only use static credentials when explicitly provided; otherwise let the
 	// SDK fall back to the default credential chain (IRSA, instance role, etc.).
 	var creds *credentials.Credentials
-	if c.S3ID != "" {
-		creds = credentials.NewStaticCredentials(c.S3ID, c.S3Secret, c.S3Token)
+	if c.AccessKeyID != "" {
+		creds = credentials.NewStaticCredentials(c.AccessKeyID, c.SecretAccessKey, c.SessionToken)
 	}
 
 	// Create AWS session