Include per second latency histogram on json output

benchmark_runner/benchmark_runner.go

@@ -18,7 +18,7 @@ import (
 	"time"
 
 	"code.cloudfoundry.org/bytefmt"
-	"github.com/filipecosta90/hdrhistogram"
+	hdrhistogram "github.com/HdrHistogram/hdrhistogram-go"
 )
 
 const (
@@ -58,6 +58,9 @@ type BenchmarkRunner struct {
 	inst_setupWriteHistogram   *hdrhistogram.Histogram
 	setupWriteTs               []DataPoint
 
+	perSecondHistograms map[uint64]*hdrhistogram.Histogram
+	perSecondHistogramsMutex sync.RWMutex
+
 	writeHistogram      *hdrhistogram.Histogram
 	inst_writeHistogram *hdrhistogram.Histogram
 
@@ -233,7 +236,15 @@ func (b *BenchmarkRunner) GetTimeSeriesMap() map[string]interface{} {
 	configs["deleteTs"] = b.deleteTs
 
 	return configs
+}
 
+func (b *BenchmarkRunner) GetPerSecondEncodedHistogramsMap() map[uint64]string {
+	configs := map[uint64]string{}
+	for k := range b.perSecondHistograms {
+		encodedV, _ := b.perSecondHistograms[k].Encode(hdrhistogram.V2CompressedEncodingCookieBase)
+		configs[k]= string(encodedV)
+	}
+	return configs
 }
 
 var loader = &BenchmarkRunner{
@@ -259,6 +270,7 @@ var loader = &BenchmarkRunner{
 	inst_totalHistogram:      hdrhistogram.New(1, 1000000, 3),
 	totalTs:                  make([]DataPoint, 0, 10),
 	detailedMapHistograms:    make(map[string]*hdrhistogram.Histogram),
+	perSecondHistograms:    make(map[uint64]*hdrhistogram.Histogram),
 }
 
 // GetBenchmarkRunner returns the singleton BenchmarkRunner for use in a benchmark program
@@ -327,6 +339,7 @@ func (l *BenchmarkRunner) RunBenchmark(b Benchmark, workQueues uint) {
 	l.testResult.OverallRates = l.GetOverallRatesMap()
 	l.testResult.TimeSeries = l.GetTimeSeriesMap()
 	l.testResult.OverallQuantiles = l.GetOverallQuantiles()
+	l.testResult.PerSecondEncodedHistograms = l.GetPerSecondEncodedHistogramsMap()
 	l.testResult.Limit = l.limit
 	l.testResult.Workers = l.workers
 	l.testResult.MaxRps = l.maxRPS
@@ -420,6 +433,14 @@ func (l *BenchmarkRunner) work(b Benchmark, wg *sync.WaitGroup, c *duplexChannel
 			l.detailedMapHistograms[groupAndQuery].RecordValue(int64(cmdStat.Latency()))
 			l.detailedMapHistogramsMutex.Unlock()
 
+			ts := cmdStat.StartTs()
+			l.perSecondHistogramsMutex.Lock()
+			if _, exist := l.perSecondHistograms[ts]; !exist {
+				l.perSecondHistograms[ts] = hdrhistogram.New(1, 1000000, 3)
+			}
+			l.perSecondHistograms[ts].RecordValue(int64(cmdStat.Latency()))
+			l.perSecondHistogramsMutex.Unlock()
+
 			switch labelStr {
 			case "SETUP_WRITE":
 				_ = l.setupWriteHistogram.RecordValue(int64(cmdStat.Latency()))

benchmark_runner/stat.go

@@ -18,13 +18,22 @@ func (s *Stat) SetCmdStats(cmdStats []CmdStat) {
 type CmdStat struct {
 	cmdQueryGroup []byte // READ, WRITE, etc...
 	cmdQueryId    []byte // R1, R2, etc...
+	startTs       uint64 // start timestamp in seconds since epoch
 	latency       uint64 // microseconds latency
 	error         bool
 	timedOut      bool
 	rx            uint64 // bytes received
 	tx            uint64 // bytes received
 }
 
+func (c *CmdStat) StartTs() uint64 {
+	return c.startTs
+}
+
+func (c *CmdStat) SetStartTs(startTs uint64) {
+	c.startTs = startTs
+}
+
 func (c *CmdStat) Tx() uint64 {
 	return c.tx
 }
@@ -72,9 +81,9 @@ func NewStat() *Stat {
 	}
 }
 
-func (s *Stat) AddEntry(cmdGroup []byte, cmdQueryId []byte, latencyUs uint64, error bool, timedOut bool, rx, tx uint64) *Stat {
+func (s *Stat) AddEntry(cmdGroup []byte, cmdQueryId []byte, startTs, latencyUs uint64, error bool, timedOut bool, rx, tx uint64) *Stat {
 	s.totalCmds++
-	entry := CmdStat{cmdGroup, cmdQueryId, latencyUs, error, timedOut, rx, tx}
+	entry := CmdStat{cmdGroup, cmdQueryId, startTs,latencyUs, error, timedOut, rx, tx}
 	s.cmdStats = append(s.cmdStats, entry)
 	return s
 }

benchmark_runner/indexing_result.go → benchmark_runner/test_result.go

@@ -50,4 +50,6 @@ type TestResult struct {
 
 	// Time-Series
 	TimeSeries map[string]interface{} `json:"TimeSeries"`
+
+	PerSecondEncodedHistograms map[uint64]string `json:"PerSecondEncodedHistograms"`
 }

cmd/ftsb_redisearch/cmd_processor.go

@@ -143,7 +143,7 @@ func sendIfRequired(p *processor, client radix.Client, cmdType string, cmdQueryI
 			duration := endT.Sub(t)
 			took := uint64(duration.Microseconds())
 			rxBytesCount += getRxLen(rcv)
-			stat := benchmark_runner.NewStat().AddEntry([]byte(cmdType), []byte(cmdQueryId), took, false, false, txBytesCount, rxBytesCount)
+			stat := benchmark_runner.NewStat().AddEntry([]byte(cmdType), []byte(cmdQueryId), uint64(t.Unix()), took, false, false, txBytesCount, rxBytesCount)
 			p.cmdChan <- *stat
 		}
 		cmds = nil

go.mod

@@ -4,9 +4,7 @@ go 1.13
 
 require (
 	code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48
-	github.com/filipecosta90/hdrhistogram v0.1.1
+	github.com/HdrHistogram/hdrhistogram-go v0.9.1-0.20201006155429-aada4ab574ea
 	github.com/mediocregopher/radix/v3 v3.5.2
-	github.com/mitchellh/gox v1.0.1 // indirect
-	github.com/tcnksm/ghr v0.13.0 // indirect
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 )