fix: compatibility for S3

Put bucket into host instead of path.
Add option "s3_bucket_in_path" to use deprecated APIs.

I wonder why the offical SDK do not update api definition. They instead do it with monkey patch code?!

Author: StarlightIbuki

src/resty/aws/init.lua

@@ -273,6 +273,23 @@ local unsigned = {
 }
 
 
+local function s3_patch(request, bucket)
+  if not bucket then
+    return
+  end
+
+  request.headers.Host = bucket .. "." .. request.headers.Host
+
+  local path = request.path
+  if bucket and path then
+    path = path:sub(#bucket + 2)
+    if path == "/" then
+      path = ""
+    end
+
+    request.path = path
+  end
+end
 
 -- Generate a function for each operation in the service api "operations" table
 local function generate_service_methods(service)
@@ -317,6 +334,10 @@ local function generate_service_methods(service)
         self.config.signatureVersion = "none"
       end
 
+      if not self.config.s3_bucket_in_path then
+        s3_patch(request, params.Bucket)
+      end
+
       -- sign the request according to the signature version required
       local signed_request, err = sign_request(self.config, request)
       if old_sig then

src/resty/aws/request/build.lua

@@ -72,6 +72,39 @@ local function parse_query(q)
   return query_tbl
 end
 
+local function get_host_port(config)
+  local host = config.endpoint
+  do
+    local s, e = host:find("://")
+    if s then
+      -- the "globalSSL" one from the region_config_data file
+      local scheme = host:sub(1, s-1):lower()
+      host = host:sub(e+1, -1)
+      if config.tls == nil then
+        config.tls = scheme == "https"
+      end
+    end
+  end
+
+  local tls = config.tls
+  local port = config.port or (tls and 443 or 80)
+
+  local host_header do -- If the "standard" port is not in use, the port should be added to the Host header
+    local with_port
+    if tls then
+      with_port = port ~= 443
+    else
+      with_port = port ~= 80
+    end
+    if with_port then
+      host_header = string.format("%s:%d", host, port)
+    else
+      host_header = host
+    end
+  end
+
+  return host_header, port
+end
 
 -- implement AWS api protocols.
 -- returns a request table;
@@ -95,12 +128,15 @@ local function build_request(operation, config, params)
   local http = operation.http or {}
   local uri = http.requestUri or ""
 
+  local host, port = get_host_port(config)
 
   local request = {
     path =  uri,
     method = http.method,
     query = {},
     headers = {},
+    host = host,
+    port = port,
   }
 
   local body_tbl = {}

src/resty/aws/request/signatures/none.lua

@@ -22,44 +22,14 @@
 --     hostname template: see https://github.com/aws/aws-sdk-js/blob/ae07e498e77000e55da70b20996dc8fd2f8b3051/lib/region_config_data.json
 local function prepare_request(config, request_data)
   local tls = config.tls
-  local host = config.endpoint
-  do
-    local s, e = host:find("://")
-    if s then
-      -- the "globalSSL" one from the region_config_data file
-      local scheme = host:sub(1, s-1):lower()
-      host = host:sub(e+1, -1)
-      if config.tls == nil then
-        config.tls = scheme == "https"
-      end
-    end
-  end
-
-  if tls == nil then
-    tls = true
-  end
-
-  local port = config.port or (tls and 443 or 80)
+  local host = request_data.host
+  local port = request_data.port
   local timestamp = ngx.time()
   local req_date = os.date("!%Y%m%dT%H%M%SZ", timestamp)
 
-  local host_header do -- If the "standard" port is not in use, the port should be added to the Host header
-    local with_port
-    if tls then
-      with_port = port ~= 443
-    else
-      with_port = port ~= 80
-    end
-    if with_port then
-      host_header = string.format("%s:%d", host, port)
-    else
-      host_header = host
-    end
-  end
-
   local headers = {
     ["X-Amz-Date"] = req_date,
-    ["Host"] = host_header,
+    ["Host"] = host,
   }
   for k, v in pairs(request_data.headers or {}) do
     headers[k] = v

src/resty/aws/request/signatures/v4.lua

@@ -141,7 +141,7 @@ local function prepare_awsv4_request(config, request_data)
     canonical_querystring = canonicalise_query_string(query)
   end
 
-  local req_headers = request_data.headers or {}
+  local req_headers = request_data.headers
   local req_payload = request_data.body
 
   -- get credentials
@@ -157,44 +157,16 @@ local function prepare_awsv4_request(config, request_data)
   end
 
   local tls = config.tls
-  local host = config.endpoint
-  do
-    local s, e = host:find("://")
-    if s then
-      -- the "globalSSL" one from the region_config_data file
-      local scheme = host:sub(1, s-1):lower()
-      host = host:sub(e+1, -1)
-      if config.tls == nil then
-        config.tls = scheme == "https"
-      end
-    end
-  end
 
-  if tls == nil then
-    tls = true
-  end
-  local port = config.port or (tls and 443 or 80)
+  local host = request_data.host
+  local port = request_data.port
   local timestamp = ngx.time()
   local req_date = os.date("!%Y%m%dT%H%M%SZ", timestamp)
   local date = os.date("!%Y%m%d", timestamp)
 
-  local host_header do -- If the "standard" port is not in use, the port should be added to the Host header
-    local with_port
-    if tls then
-      with_port = port ~= 443
-    else
-      with_port = port ~= 80
-    end
-    if with_port then
-      host_header = string.format("%s:%d", host, port)
-    else
-      host_header = host
-    end
-  end
-
   local headers = {
     ["X-Amz-Date"] = req_date,
-    ["Host"] = host_header,
+    ["Host"] = host,
     ["X-Amz-Security-Token"] = session_token,
   }
 

src/resty/aws/s3.lua

@@ -0,0 +1,96 @@
+local AWS = require("resty.aws")
+local normalize = require("kong.tools.uri").normalize
+
+-- this happens at init phase because the kong.clustering.new() is called by kong.init at init phase
+-- please make sure this is required at init phase if that changes
+local AWS_config = require("resty.aws.config")
+local third_party_env = os.getenv("AWS_THIRD_PARTY_ENDPOINT")
+
+local xml = require("pl.xml")
+local xml_parse = xml.parse
+local s3_instance
+local re_match = ngx.re.match
+
+local _M = {}
+
+
+local empty = {}
+local function endpoint_settings(config)
+    local endpoint = config.third_party_endpoint or third_party_env
+    if endpoint then
+        local m = re_match(endpoint, [[^(?:(https?):\/\/)?([^:]+)(?::(\d+))?$]], "jo")
+        assert(m and m[2], "invalid endpoint")
+        local scheme = m[1] or "https"
+        local tls = scheme == "https"
+        return {
+            scheme = scheme,
+            tls = tls,
+            endpoint = m[2],
+            port = tonumber(m[3]) or (tls and 443 or 80),
+        }
+    else
+        return empty
+    end
+end
+
+function _M.init_worker()
+    -- TODO: avoid http request to get the region
+    local global_config = AWS_config.global
+    global_config.s3_update = true
+    local aws_instance = assert(AWS(global_config))
+    s3_instance = assert(aws_instance:S3(endpoint_settings(global_config)))
+end
+
+function _M.new(gateway_version, url)
+    local self = {
+        url = url,
+        gateway_version = gateway_version,
+    }
+    local m = assert(re_match(url, [[^s3:\/\/([^\/]+)\/(.+)$]], "jo"), "invalid S3 URL")
+    self.bucket = m[1]
+    self.key = normalize(m[2] .. "/" .. gateway_version .. ".json", true)
+    
+    return setmetatable(self, { __index = _M })
+end
+
+function _M:backup_config(config, config_hash)
+    local res, err = s3_instance:putObject{
+        Bucket = self.bucket,
+        Key = self.key,
+        Body = config,
+        -- optional, to help DP recalculate the hash
+        Metadata = config_hash and {
+            ["config-hash"] = config_hash,
+        },
+        ContentType = "application/json",
+    }
+
+    if not res then
+        return nil, err
+    end
+
+    if not (res.status == 200) then
+        return nil, xml_parse(res.body, nil, true):child_with_name("Message")[1]
+    end
+
+    return true
+end
+
+function _M:fetch_config()
+    local res, err = s3_instance:getObject{
+        Bucket = self.bucket,
+        Key = self.key,
+    }
+
+    if not res then
+        return nil, err
+    end
+
+    if not (res.status == 200) then
+        return nil, xml_parse(res.body, nil, true):child_with_name("Message")[1]
+    end
+
+    return res.body, res.headers["x-amz-meta-config-hash"]
+end
+
+return _M