From 90390b2441785c1e6c659c59ea56342bd76968c1 Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Sun, 15 Nov 2020 16:53:58 -0500 Subject: [PATCH 1/4] [aggregator] Reduce delays in integration tests --- src/aggregator/integration/integration.go | 2 +- src/aggregator/integration/setup.go | 64 +++++++++++++++++------ 2 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/aggregator/integration/integration.go b/src/aggregator/integration/integration.go index 94c56d3d67..fcf23f84fe 100644 --- a/src/aggregator/integration/integration.go +++ b/src/aggregator/integration/integration.go @@ -32,7 +32,7 @@ func waitUntil(fn conditionFn, timeout time.Duration) bool { if fn() { return true } - time.Sleep(time.Second) + time.Sleep(10 * time.Millisecond) } return false } diff --git a/src/aggregator/integration/setup.go b/src/aggregator/integration/setup.go index 1df66c7fbd..db2f2373b4 100644 --- a/src/aggregator/integration/setup.go +++ b/src/aggregator/integration/setup.go @@ -21,13 +21,14 @@ package integration import ( + "encoding/json" "errors" "fmt" + "io/ioutil" "net/http" "sort" "sync" "testing" - "time" "github.com/m3db/m3/src/aggregator/aggregator" "github.com/m3db/m3/src/aggregator/aggregator/handler" @@ -246,14 +247,41 @@ func (ts *testServerSetup) newClient() *client { return newClient(ts.rawTCPAddr, ts.opts.ClientBatchSize(), connectTimeout) } +func (ts *testServerSetup) getStatusResponse(path string, response interface{}) error { + resp, err := http.Get("http://" + ts.httpAddr + path) //nolint + if err != nil { + return err + } + + defer resp.Body.Close() //nolint:errcheck + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return errors.New("aggregator returned a non-400") + } + return json.Unmarshal(b, response) +} + func (ts *testServerSetup) waitUntilServerIsUp() error { - c := ts.newClient() - defer c.close() + if waitUntil(func() bool { + var resp httpserver.Response + if err := ts.getStatusResponse(httpserver.HealthPath, &resp); err != nil { + return false + } + + if resp.State == "OK" { + return true + } - serverIsUp := func() bool { return c.testConnection() } - if waitUntil(serverIsUp, ts.opts.ServerStateChangeTimeout()) { + return false + }, + ts.opts.ElectionStateChangeTimeout(), + ) { return nil } + return errServerStartTimedOut } @@ -301,21 +329,23 @@ func (ts *testServerSetup) startServer() error { } func (ts *testServerSetup) waitUntilLeader() error { - isLeader := func() bool { - leader, err := ts.leaderService.Leader(ts.electionKey) - if err != nil { + if waitUntil(func() bool { + var resp httpserver.StatusResponse + if err := ts.getStatusResponse(httpserver.StatusPath, &resp); err != nil { return false } - return leader == ts.leaderValue - } - if !waitUntil(isLeader, ts.opts.ElectionStateChangeTimeout()) { - return errLeaderElectionTimeout + + if resp.Status.FlushStatus.ElectionState == aggregator.LeaderState { + return true + } + return false + }, + ts.opts.ElectionStateChangeTimeout(), + ) { + return nil } - // TODO(xichen): replace the sleep here by using HTTP client to explicit - // curl the server for election status. - // Give the server some time to transition into leader state if needed. - time.Sleep(time.Second) - return nil + + return errLeaderElectionTimeout } func (ts *testServerSetup) sortedResults() []aggregated.MetricWithStoragePolicy { From f09d5648b1c84cd43b1302c728be2fb687fa5532 Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Sun, 15 Nov 2020 17:41:00 -0500 Subject: [PATCH 2/4] fix typo --- src/aggregator/integration/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aggregator/integration/setup.go b/src/aggregator/integration/setup.go index db2f2373b4..ff9c9421a7 100644 --- a/src/aggregator/integration/setup.go +++ b/src/aggregator/integration/setup.go @@ -277,7 +277,7 @@ func (ts *testServerSetup) waitUntilServerIsUp() error { return false }, - ts.opts.ElectionStateChangeTimeout(), + ts.opts.ServerStateChangeTimeout(), ) { return nil } From 2298e481c85e8c2de78b6f7124ce9a21ab1446b1 Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Mon, 16 Nov 2020 09:32:13 -0500 Subject: [PATCH 3/4] fix typo --- src/aggregator/integration/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aggregator/integration/setup.go b/src/aggregator/integration/setup.go index ff9c9421a7..026f061063 100644 --- a/src/aggregator/integration/setup.go +++ b/src/aggregator/integration/setup.go @@ -259,7 +259,7 @@ func (ts *testServerSetup) getStatusResponse(path string, response interface{}) return err } if resp.StatusCode != http.StatusOK { - return errors.New("aggregator returned a non-400") + return fmt.Errorf("got a non-200 status code: %v", resp.StatusCode) } return json.Unmarshal(b, response) } From 73f037c78ef5c0616c9bd49bedf23d257c31b22d Mon Sep 17 00:00:00 2001 From: Vytenis Darulis Date: Mon, 16 Nov 2020 09:38:33 -0500 Subject: [PATCH 4/4] split --- src/aggregator/integration/setup.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/aggregator/integration/setup.go b/src/aggregator/integration/setup.go index 026f061063..1f5d254a01 100644 --- a/src/aggregator/integration/setup.go +++ b/src/aggregator/integration/setup.go @@ -265,7 +265,7 @@ func (ts *testServerSetup) getStatusResponse(path string, response interface{}) } func (ts *testServerSetup) waitUntilServerIsUp() error { - if waitUntil(func() bool { + isUp := func() bool { var resp httpserver.Response if err := ts.getStatusResponse(httpserver.HealthPath, &resp); err != nil { return false @@ -276,9 +276,9 @@ func (ts *testServerSetup) waitUntilServerIsUp() error { } return false - }, - ts.opts.ServerStateChangeTimeout(), - ) { + } + + if waitUntil(isUp, ts.opts.ServerStateChangeTimeout()) { return nil } @@ -329,7 +329,7 @@ func (ts *testServerSetup) startServer() error { } func (ts *testServerSetup) waitUntilLeader() error { - if waitUntil(func() bool { + isLeader := func() bool { var resp httpserver.StatusResponse if err := ts.getStatusResponse(httpserver.StatusPath, &resp); err != nil { return false @@ -339,9 +339,9 @@ func (ts *testServerSetup) waitUntilLeader() error { return true } return false - }, - ts.opts.ElectionStateChangeTimeout(), - ) { + } + + if waitUntil(isLeader, ts.opts.ElectionStateChangeTimeout()) { return nil }