diff --git a/Makefile b/Makefile index 888c5b4..809fe75 100644 --- a/Makefile +++ b/Makefile @@ -15,9 +15,11 @@ all: ; install: all $(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/api-gateway/tracking/ $(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/api-gateway/tracking/log/ + $(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/api-gateway/tracking/util/ $(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/api-gateway/tracking/validator/ $(INSTALL) src/lua/api-gateway/tracking/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/api-gateway/tracking/ $(INSTALL) src/lua/api-gateway/tracking/log/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/api-gateway/tracking/log/ + $(INSTALL) src/lua/api-gateway/tracking/util/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/api-gateway/tracking/util/ $(INSTALL) src/lua/api-gateway/tracking/validator/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/api-gateway/tracking/validator/ test: redis diff --git a/src/lua/api-gateway/tracking/RequestTrackingManager.lua b/src/lua/api-gateway/tracking/RequestTrackingManager.lua index fdfa30b..4760e98 100644 --- a/src/lua/api-gateway/tracking/RequestTrackingManager.lua +++ b/src/lua/api-gateway/tracking/RequestTrackingManager.lua @@ -14,7 +14,7 @@ -- --- Exposes utility functions to add/remove Tracking rules ( BLOCK, TRACK, DEBUG, DELAY, RETRY-AFTER ) +-- Exposes utility functions to add/remove Tracking rules ( BLOCK, TRACK, TRACE, DEBUG, DELAY, RETRY-AFTER ) -- -- You should map this to a REST API Endpoint: -- @@ -34,6 +34,7 @@ local KNWON_RULES = { BLOCK = "blocking_rules_dict", TRACK = "tracking_rules_dict", + TRACE = "tracing_rules_dict", DEBUG = "debuging_rules_dict", DELAY = "delaying_rules_dict", REWRITE = "rewriting_rules_dict", @@ -43,6 +44,7 @@ local KNWON_RULES = local last_modified_date = { BLOCK = -1, TRACK = -1, + TRACE = -1, DEBUG = -1, DELAY = -1, REWRITE = -1, @@ -52,6 +54,7 @@ local last_modified_date = { local cached_rules = { BLOCK = {}, --- holds a per worker cache of the BLOCK rules TRACK = {}, --- holds a per worker cache of the TRACK rules + TRACE = {}, --- holds a per worker cache of the TRACE rules DEBUG = {}, --- holds a per worker cache of the DEBUG rules DELAY = {}, --- holds a per worker cache of the DELAY rules REWRITE = {}, --- holds a per worker cache of the REWRITE rules @@ -140,7 +143,7 @@ end --- Returns an object with the current active rules for the given rule_type -- --- @param rule_type BLOCK, TRACK, DEBUG, DELAY or RETRY-AFTER +-- @param rule_type BLOCK, TRACK, TRACE, DEBUG, DELAY or RETRY-AFTER -- function _M:getRulesForType(rule_type) local rule_type = string.upper(rule_type) @@ -268,7 +271,7 @@ end --- -- Returns an object with only the rules matching the current request variables. It's up to the caller to decide what to do with the result --- @param rule_type BLOCK, TRACK, DEBUG, DELAY or RETRY-AFTER +-- @param rule_type BLOCK, TRACK, TRACE, DEBUG, DELAY or RETRY-AFTER -- @param separator For instance ";". It's the character separating the values and variables -- @param exit_on_first_match Default:true. When true the method exits on the first match. -- This is useful for BLOCK, DELAY or RETRY-AFTER behaviours when the first match would alter the request status. diff --git a/src/lua/api-gateway/tracking/RequestVariableManager.lua b/src/lua/api-gateway/tracking/RequestVariableManager.lua index 550e70b..68de4c6 100644 --- a/src/lua/api-gateway/tracking/RequestVariableManager.lua +++ b/src/lua/api-gateway/tracking/RequestVariableManager.lua @@ -31,12 +31,16 @@ function _M:getRequestVariable(request_var, cache) local ctx_var = ngx.ctx[request_var] if ctx_var ~= nil then - cache[request_var] = ctx_var + if cache ~= nil then + cache[request_var] = ctx_var + end return ctx_var end local ngx_var = ngx.var[request_var] - cache[request_var] = ngx_var + if cache ~= nil then + cache[request_var] = ngx_var + end return ngx_var end diff --git a/src/lua/api-gateway/tracking/factory.lua b/src/lua/api-gateway/tracking/factory.lua index 5b126ba..dc252de 100644 --- a/src/lua/api-gateway/tracking/factory.lua +++ b/src/lua/api-gateway/tracking/factory.lua @@ -30,6 +30,7 @@ local RequestTrackingManager = require "api-gateway.tracking.RequestTrackingMana local RequestVariableManager = require "api-gateway.tracking.RequestVariableManager" local BlockingRulesValidator = require "api-gateway.tracking.validator.blockingRulesValidator" local DelayingRulesValidator = require "api-gateway.tracking.validator.delayingRulesValidator" +local TracingRulesLogger = require "api-gateway.tracking.log.tracingRulesLogger" local TrackingRulesLogger = require "api-gateway.tracking.log.trackingRulesLogger" local cjson = require "cjson" @@ -79,8 +80,11 @@ end -- This method should be called from the log phase ( log_by_lua ) -- local function _trackRequest() + local tracingRulesLogger = TracingRulesLogger:new() + tracingRulesLogger:log_trace_rules() + local trackingRulesLogger = TrackingRulesLogger:new() - return trackingRulesLogger:log() + return trackingRulesLogger:log() end return { diff --git a/src/lua/api-gateway/tracking/log/tracingRulesLogger.lua b/src/lua/api-gateway/tracking/log/tracingRulesLogger.lua new file mode 100644 index 0000000..c6cedf4 --- /dev/null +++ b/src/lua/api-gateway/tracking/log/tracingRulesLogger.lua @@ -0,0 +1,88 @@ +--[[ + Copyright 2016 Adobe Systems Incorporated. All rights reserved. + + This file is licensed to you under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR RESPRESENTATIONS OF ANY KIND, + either express or implied. See the License for the specific language governing permissions and + limitations under the License. + ]] + +local cjson = require "cjson" +local su = require "api-gateway.tracking.util.stringutil" + +local _M = {} + +function _M:new(o) + local o = o or {} + setmetatable(o, self) + self.__index = self + return o +end + +function _M:buildTraceMessage(rule) + local data = {} + + data["id"] = tostring(rule.id) + data["domain"] = tostring(rule.domain) + data["request_body"] = tostring(ngx.var.request_body) + data["status"] = tostring(ngx.var.status) + + for key,value in pairs(ngx.header) do + data[tostring(key)] = tostring(value) + end + + local meta = rule.meta:split(";") + local meta_length = table.getn(meta) + + local var_value, index + for index = 1, meta_length do + local variableManager = ngx.apiGateway.tracking.variableManager + local var_name = string.sub(su.trim(meta[index]), 2); + var_value = variableManager:getRequestVariable(var_name, nil) + data[var_name] = tostring(var_value) + end + + return data +end + +--- +-- @param config_obj configuration object +-- +function _M:log_trace_rules(config_obj) + local trackingManager = ngx.apiGateway.tracking.manager + if ( trackingManager == nil ) then + ngx.log(ngx.WARN, "Please initialize RequestTrackingManager before calling this method") + end + + local tracingLogger = ngx.apiGateway.getAsyncLogger("api-gateway-debugging") + if ( tracingLogger == nil ) then + ngx.log(ngx.WARN, "Could not track request. Tracing logger should not be nil") + return + end + + -- 1. read the keys in the shared dict and compare them with the current request + local stop_at_first_block_match = false + local tracing_rules = trackingManager:getMatchingRulesForRequest("trace", ";", stop_at_first_block_match) + if (tracing_rules == nil) then + return + end + -- 2. for each tracing rule matching the request publish a tracing message asyncronously + for i, rule in pairs(tracing_rules) do + if ( rule ~= nil ) then + local message = self:buildTraceMessage(rule) + if ( message ~= nil ) then + local partition_key = ngx.utctime() .."-".. math.random(ngx.now() * 1000) + ngx.log(ngx.DEBUG, "Logging tracing info: " .. cjson.encode(message)) + tracingLogger:logMetrics(partition_key, cjson.encode(message)); + end + end + end +end + +return _M + diff --git a/src/lua/api-gateway/tracking/util/stringutil.lua b/src/lua/api-gateway/tracking/util/stringutil.lua new file mode 100644 index 0000000..a8afc79 --- /dev/null +++ b/src/lua/api-gateway/tracking/util/stringutil.lua @@ -0,0 +1,26 @@ +local stringutil = {} + +-- http://coronalabs.com/blog/2013/04/16/lua-string-magic/ +function string:split( pattern, results ) + if not results then + results = { } + end + local start = 1 + local split_start, split_end = string.find( self, pattern, start ) + while split_start do + table.insert( results, string.sub( self, start, split_start - 1 ) ) + start = split_end + 1 + split_start, split_end = string.find( self, pattern, start ) + end + table.insert( results, string.sub( self, start ) ) + return results +end + +-- http://coronalabs.com/blog/2013/04/16/lua-string-magic/ +local function trim( s ) + return string.match( s,"^()%s*$") and "" or string.match(s,"^%s*(.*%S)" ) +end + +stringutil.trim = trim + +return stringutil \ No newline at end of file diff --git a/src/lua/api-gateway/tracking/validator/delayingRulesValidator.lua b/src/lua/api-gateway/tracking/validator/delayingRulesValidator.lua index b46b9e9..4e68bde 100644 --- a/src/lua/api-gateway/tracking/validator/delayingRulesValidator.lua +++ b/src/lua/api-gateway/tracking/validator/delayingRulesValidator.lua @@ -39,6 +39,9 @@ local function getActualDelay( delaying_rule ) return math.random( actualDelay / 2, actualDelay ) end +--- +-- @param config_obj configuration object +-- function _M:validate_delaying_rules(config_obj) local trackingManager = ngx.apiGateway.tracking.manager if ( trackingManager == nil ) then diff --git a/test/perl/api-gateway/tracking/log/tracingRulesLogger.t b/test/perl/api-gateway/tracking/log/tracingRulesLogger.t new file mode 100644 index 0000000..3cc7199 --- /dev/null +++ b/test/perl/api-gateway/tracking/log/tracingRulesLogger.t @@ -0,0 +1,118 @@ +# /* +# * Copyright 2016 Adobe Systems Incorporated. All rights reserved. +# * +# * This file is licensed to you under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software distributed under the License +# * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR RESPRESENTATIONS OF ANY KIND, +# * either express or implied. See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# vim:set ft= ts=4 sw=4 et fdm=marker: +use lib 'lib'; +use strict; +use warnings; +use Test::Nginx::Socket::Lua; +use Cwd qw(cwd); + +#worker_connections(1014); +#master_process_enabled(1); +#log_level('warn'); + +repeat_each(1); + +plan tests => repeat_each() * (blocks() * 5) + 1 ; + +my $pwd = cwd(); + +our $HttpConfig = <<_EOC_; + # lua_package_path "$pwd/scripts/?.lua;;"; + lua_package_path "src/lua/?.lua;/usr/local/lib/lua/?.lua;;"; + init_by_lua ' + local v = require "jit.v" + v.on("$Test::Nginx::Util::ErrLogFile") + require "resty.core" + '; + init_worker_by_lua ' + ngx.apiGateway = ngx.apiGateway or {} + ngx.apiGateway.validation = require "api-gateway.validation.factory" + ngx.apiGateway.tracking = require "api-gateway.tracking.factory" + + local function get_logger(name) + return { + logMetrics = function (self, key, value) + ngx.log(ngx.INFO, "Received " .. tostring(value)) + end + } + end + ngx.apiGateway.getAsyncLogger = get_logger + '; + include "$pwd/conf.d/http.d/*.conf"; + upstream cache_rw_backend { + server 127.0.0.1:6379; + } + upstream cache_read_only_backend { # Default config for redis health check test + server 127.0.0.1:6379; + } + lua_shared_dict blocking_rules_dict 5m; + lua_shared_dict tracking_rules_dict 5m; + lua_shared_dict tracing_rules_dict 5m; + lua_shared_dict debugging_rules_dict 5m; + lua_shared_dict delaying_rules_dict 5m; + lua_shared_dict retrying_rules_dict 5m; + + client_body_temp_path /tmp/; + proxy_temp_path /tmp/; + fastcgi_temp_path /tmp/; +_EOC_ + +#no_diff(); +no_long_string(); +run_tests(); + +__DATA__ + + +=== TEST 1: test that we can trace the request +--- http_config eval: $::HttpConfig +--- config + include ../../api-gateway/default_validators.conf; + include ../../api-gateway/tracking_service.conf; + set $publisher_org_name 'pub1'; + + error_log ../test-logs/tracingRequestValidator_test1_error.log debug; + + location /trace { + + log_by_lua ' + ngx.apiGateway.tracking.track() + '; + content_by_lua 'ngx.say("OK")'; + + } +--- timeout: 10 +--- pipelined_requests eval +['POST /tracking/ +[{ + "id": 222, + "domain" : "pub1", + "format": "$publisher_org_name", + "expire_at_utc": 1583910454, + "action" : "TRACE", + "meta" : "$request_uri; $request_method;$publisher_org_name" +}] +', +"GET /trace" +] +--- response_body_like eval +[ +'\{"result":"success"\}.*', +'OK' +] +--- error_code_like eval + [200, 200] +--- no_error_log +[error] diff --git a/test/perl/api-gateway/tracking/log/trackingRulesLogger.t b/test/perl/api-gateway/tracking/log/trackingRulesLogger.t index 5ac6c65..4ee93ed 100644 --- a/test/perl/api-gateway/tracking/log/trackingRulesLogger.t +++ b/test/perl/api-gateway/tracking/log/trackingRulesLogger.t @@ -41,6 +41,11 @@ our $HttpConfig = <<_EOC_; ngx.apiGateway.validation = require "api-gateway.validation.factory" ngx.apiGateway.tracking = require "api-gateway.tracking.factory" + local function get_logger(name) + return {} + end + ngx.apiGateway.getAsyncLogger = get_logger + local function loadrequire(module) local function requiref(module) require(module)