From a5b43bbc665fbe5251bfaccad3216b09961b7847 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 28 Nov 2022 09:35:37 -0800 Subject: [PATCH 1/5] log query params in query frontend when error Signed-off-by: Ben Ye --- pkg/frontend/transport/handler.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index 5eb406571f7..78e840d2e6f 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -136,6 +136,19 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { resp, err := f.roundTripper.RoundTrip(r) queryResponseTime := time.Since(startTime) + // Check whether we should parse the query string. + shouldReportSlowQuery := f.cfg.LogQueriesLongerThan != 0 && queryResponseTime > f.cfg.LogQueriesLongerThan + if shouldReportSlowQuery || f.cfg.QueryStatsEnabled { + queryString = f.parseRequestQueryString(r, buf) + } + + if shouldReportSlowQuery { + f.reportSlowQuery(r, queryString, queryResponseTime) + } + if f.cfg.QueryStatsEnabled { + f.reportQueryStats(r, queryString, queryResponseTime, stats) + } + if err != nil { writeError(w, err) return @@ -156,19 +169,6 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err != nil && !errors.Is(err, syscall.EPIPE) { level.Error(util_log.WithContext(r.Context(), f.log)).Log("msg", "write response body error", "bytesCopied", bytesCopied, "err", err) } - - // Check whether we should parse the query string. - shouldReportSlowQuery := f.cfg.LogQueriesLongerThan != 0 && queryResponseTime > f.cfg.LogQueriesLongerThan - if shouldReportSlowQuery || f.cfg.QueryStatsEnabled { - queryString = f.parseRequestQueryString(r, buf) - } - - if shouldReportSlowQuery { - f.reportSlowQuery(r, queryString, queryResponseTime) - } - if f.cfg.QueryStatsEnabled { - f.reportQueryStats(r, queryString, queryResponseTime, stats) - } } // reportSlowQuery reports slow queries. From 2268ded694bdff10b5c842894bebbc3fd28791d9 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 28 Nov 2022 09:59:58 -0800 Subject: [PATCH 2/5] log errors Signed-off-by: Ben Ye --- pkg/frontend/transport/handler.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index 78e840d2e6f..be9ececbe15 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -6,6 +6,7 @@ import ( "errors" "flag" "fmt" + "google.golang.org/grpc/status" "io" "net/http" "net/url" @@ -146,7 +147,7 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.reportSlowQuery(r, queryString, queryResponseTime) } if f.cfg.QueryStatsEnabled { - f.reportQueryStats(r, queryString, queryResponseTime, stats) + f.reportQueryStats(r, queryString, queryResponseTime, stats, err) } if err != nil { @@ -184,7 +185,7 @@ func (f *Handler) reportSlowQuery(r *http.Request, queryString url.Values, query level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) } -func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, queryResponseTime time.Duration, stats *querier_stats.Stats) { +func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, queryResponseTime time.Duration, stats *querier_stats.Stats, error error) { tenantIDs, err := tenant.TenantIDs(r.Context()) if err != nil { return @@ -215,6 +216,15 @@ func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, quer "fetched_data_bytes", numDataBytes, }, formatQueryString(queryString)...) + if error != nil { + s, ok := status.FromError(error) + if !ok { + logMessage = append(logMessage, "error", error) + } else { + logMessage = append(logMessage, "error", s.Message()) + } + } + level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) } From 815e457491861a779245f098ede59e425f7ac86c Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 28 Nov 2022 11:46:33 -0800 Subject: [PATCH 3/5] change log level when error Signed-off-by: Ben Ye --- pkg/frontend/transport/handler.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index be9ececbe15..5b5c688197e 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -223,9 +223,10 @@ func (f *Handler) reportQueryStats(r *http.Request, queryString url.Values, quer } else { logMessage = append(logMessage, "error", s.Message()) } + level.Error(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) + } else { + level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) } - - level.Info(util_log.WithContext(r.Context(), f.log)).Log(logMessage...) } func (f *Handler) parseRequestQueryString(r *http.Request, bodyBuf bytes.Buffer) url.Values { From 90126bff4ae8663663d4373ca6d091052e4677ae Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 28 Nov 2022 13:19:42 -0800 Subject: [PATCH 4/5] fix lint Signed-off-by: Ben Ye --- pkg/frontend/transport/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index 5b5c688197e..99f2116e5b1 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -6,7 +6,6 @@ import ( "errors" "flag" "fmt" - "google.golang.org/grpc/status" "io" "net/http" "net/url" @@ -21,6 +20,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/httpgrpc/server" + "google.golang.org/grpc/status" querier_stats "github.com/cortexproject/cortex/pkg/querier/stats" "github.com/cortexproject/cortex/pkg/tenant" From d57b85c1dd64fdfed53706719946ab1070913a5b Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 28 Nov 2022 18:36:12 -0800 Subject: [PATCH 5/5] update changelog Signed-off-by: Ben Ye --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae223659546..2639a810d56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,10 @@ ## master / unreleased * [ENHANCEMENT] Update Go version to 1.19.3. #4988 +* [ENHANCEMENT] Querier: limit series query to only ingesters if `start` param is not specified. #4976 * [FEATURE] Querier/Query Frontend: support Prometheus /api/v1/status/buildinfo API. #4978 -* [ENHANCEMENT] Querier: limit series query to only ingesters if `start` param is not been specified. #4976 - * [FEATURE] Ingester: Add active series to all_user_stats page. #4972 +* [FEATURE] Query Frontend: Log query params in query frontend even if error happens. #5005 ## 1.14.0 in progress