Skip to content

Commit 68b2319

Browse files
waynexiawaynexia
authored
feat: improve Prometheus compliance (#1022)
* initial impl Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * minor (useless) refactor Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * retrieve metric name Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * add time index column to group by columns filter out NaN in normalize remove NULL in instant manipulator accept form data as HTTP params correct API URL accept second literal as step param * happy clippy Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * update test result Signed-off-by: Ruihang Xia <waynestxia@gmail.com> --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
1 parent 6e9964a commit 68b2319

File tree

9 files changed

+332
-195
lines changed

9 files changed

+332
-195
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/datanode/src/tests/promql_test.rs‎

Lines changed: 56 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -82,16 +82,16 @@ async fn sql_insert_promql_query_ceil() {
8282
UNIX_EPOCH.checked_add(Duration::from_secs(100)).unwrap(),
8383
Duration::from_secs(5),
8484
Duration::from_secs(1),
85-
"+---------------------+-------------------------------+----------------------------------+\
86-
\n| ts | ceil(http_requests_total.cpu) | ceil(http_requests_total.memory) |\
87-
\n+---------------------+-------------------------------+----------------------------------+\
88-
\n| 1970-01-01T00:00:00 | 67 | 1024 |\
89-
\n| 1970-01-01T00:00:05 | 67 | 4096 |\
90-
\n| 1970-01-01T00:00:10 | 100 | 20480 |\
91-
\n| 1970-01-01T00:00:50 | 12424 | 1334 |\
92-
\n| 1970-01-01T00:01:20 | 0 | 2334 |\
93-
\n| 1970-01-01T00:01:40 | 49 | 3334 |\
94-
\n+---------------------+-------------------------------+----------------------------------+",
85+
"+---------------------+-------------------------------+----------------------------------+-------+\
86+
\n| ts | ceil(http_requests_total.cpu) | ceil(http_requests_total.memory) | host |\
87+
\n+---------------------+-------------------------------+----------------------------------+-------+\
88+
\n| 1970-01-01T00:00:00 | 67 | 1024 | host1 |\
89+
\n| 1970-01-01T00:00:05 | 67 | 4096 | host1 |\
90+
\n| 1970-01-01T00:00:10 | 100 | 20480 | host1 |\
91+
\n| 1970-01-01T00:01:20 | 0 | 2334 | host1 |\
92+
\n| 1970-01-01T00:01:40 | 49 | 3334 | host1 |\
93+
\n| 1970-01-01T00:00:50 | 12424 | 1334 | host1 |\
94+
\n+---------------------+-------------------------------+----------------------------------+-------+"
9595
)
9696
.await;
9797
}
@@ -142,13 +142,12 @@ async fn aggregators_simple_sum() {
142142
unix_epoch_plus_100s(),
143143
Duration::from_secs(60),
144144
Duration::from_secs(0),
145-
"+------------+--------------------------+\
146-
\n| group | SUM(http_requests.value) |\
147-
\n+------------+--------------------------+\
148-
\n| | |\
149-
\n| canary | 700 |\
150-
\n| production | 300 |\
151-
\n+------------+--------------------------+",
145+
"+------------+---------------------+--------------------------+\
146+
\n| group | ts | SUM(http_requests.value) |\
147+
\n+------------+---------------------+--------------------------+\
148+
\n| production | 1970-01-01T00:00:00 | 300 |\
149+
\n| canary | 1970-01-01T00:00:00 | 700 |\
150+
\n+------------+---------------------+--------------------------+",
152151
)
153152
.await;
154153
}
@@ -167,13 +166,12 @@ async fn aggregators_simple_avg() {
167166
unix_epoch_plus_100s(),
168167
Duration::from_secs(60),
169168
Duration::from_secs(0),
170-
"+------------+--------------------------+\
171-
\n| group | AVG(http_requests.value) |\
172-
\n+------------+--------------------------+\
173-
\n| | 0 |\
174-
\n| production | 150 |\
175-
\n| canary | 350 |\
176-
\n+------------+--------------------------+",
169+
"+------------+---------------------+--------------------------+\
170+
\n| group | ts | AVG(http_requests.value) |\
171+
\n+------------+---------------------+--------------------------+\
172+
\n| production | 1970-01-01T00:00:00 | 150 |\
173+
\n| canary | 1970-01-01T00:00:00 | 350 |\
174+
\n+------------+---------------------+--------------------------+",
177175
)
178176
.await;
179177
}
@@ -192,13 +190,12 @@ async fn aggregators_simple_count() {
192190
unix_epoch_plus_100s(),
193191
Duration::from_secs(60),
194192
Duration::from_secs(0),
195-
"+------------+----------------------------+\
196-
\n| group | COUNT(http_requests.value) |\
197-
\n+------------+----------------------------+\
198-
\n| | 0 |\
199-
\n| canary | 2 |\
200-
\n| production | 2 |\
201-
\n+------------+----------------------------+",
193+
"+------------+---------------------+----------------------------+\
194+
\n| group | ts | COUNT(http_requests.value) |\
195+
\n+------------+---------------------+----------------------------+\
196+
\n| canary | 1970-01-01T00:00:00 | 2 |\
197+
\n| production | 1970-01-01T00:00:00 | 2 |\
198+
\n+------------+---------------------+----------------------------+",
202199
)
203200
.await;
204201
}
@@ -217,13 +214,12 @@ async fn aggregators_simple_without() {
217214
unix_epoch_plus_100s(),
218215
Duration::from_secs(60),
219216
Duration::from_secs(0),
220-
"+------------+------------+--------------------------+\
221-
\n| group | job | SUM(http_requests.value) |\
222-
\n+------------+------------+--------------------------+\
223-
\n| | | |\
224-
\n| canary | api-server | 700 |\
225-
\n| production | api-server | 300 |\
226-
\n+------------+------------+--------------------------+",
217+
"+------------+------------+---------------------+--------------------------+\
218+
\n| group | job | ts | SUM(http_requests.value) |\
219+
\n+------------+------------+---------------------+--------------------------+\
220+
\n| production | api-server | 1970-01-01T00:00:00 | 300 |\
221+
\n| canary | api-server | 1970-01-01T00:00:00 | 700 |\
222+
\n+------------+------------+---------------------+--------------------------+",
227223
)
228224
.await;
229225
}
@@ -241,11 +237,11 @@ async fn aggregators_empty_by() {
241237
unix_epoch_plus_100s(),
242238
Duration::from_secs(60),
243239
Duration::from_secs(0),
244-
"+--------------------------+\
245-
\n| SUM(http_requests.value) |\
246-
\n+--------------------------+\
247-
\n| 1000 |\
248-
\n+--------------------------+",
240+
"+---------------------+--------------------------+\
241+
\n| ts | SUM(http_requests.value) |\
242+
\n+---------------------+--------------------------+\
243+
\n| 1970-01-01T00:00:00 | 1000 |\
244+
\n+---------------------+--------------------------+",
249245
)
250246
.await;
251247
}
@@ -263,11 +259,11 @@ async fn aggregators_no_by_without() {
263259
unix_epoch_plus_100s(),
264260
Duration::from_secs(60),
265261
Duration::from_secs(0),
266-
"+--------------------------+\
267-
\n| SUM(http_requests.value) |\
268-
\n+--------------------------+\
269-
\n| 1000 |\
270-
\n+--------------------------+",
262+
"+---------------------+--------------------------+\
263+
\n| ts | SUM(http_requests.value) |\
264+
\n+---------------------+--------------------------+\
265+
\n| 1970-01-01T00:00:00 | 1000 |\
266+
\n+---------------------+--------------------------+",
271267
)
272268
.await;
273269
}
@@ -286,13 +282,12 @@ async fn aggregators_empty_without() {
286282
unix_epoch_plus_100s(),
287283
Duration::from_secs(60),
288284
Duration::from_secs(0),
289-
"+------------+----------+------------+--------------------------+\
290-
\n| group | instance | job | SUM(http_requests.value) |\
291-
\n+------------+----------+------------+--------------------------+\
292-
\n| | | | |\
293-
\n| production | 0 | api-server | 100 |\
294-
\n| production | 1 | api-server | 200 |\
295-
\n+------------+----------+------------+--------------------------+",
285+
"+------------+----------+------------+---------------------+--------------------------+\
286+
\n| group | instance | job | ts | SUM(http_requests.value) |\
287+
\n+------------+----------+------------+---------------------+--------------------------+\
288+
\n| production | 0 | api-server | 1970-01-01T00:00:00 | 100 |\
289+
\n| production | 1 | api-server | 1970-01-01T00:00:00 | 200 |\
290+
\n+------------+----------+------------+---------------------+--------------------------+",
296291
)
297292
.await;
298293
}
@@ -356,11 +351,12 @@ async fn stddev_by_label() {
356351
unix_epoch_plus_100s(),
357352
Duration::from_secs(60),
358353
Duration::from_secs(0),
359-
"+----------+-----------------------------+\
360-
\n| instance | STDDEV(http_requests.value) |\
361-
\n+----------+-----------------------------+\
362-
\n| 0 | 258.19888974716116 |\
363-
\n+----------+-----------------------------+",
354+
"+----------+---------------------+-----------------------------+\
355+
\n| instance | ts | STDDEV(http_requests.value) |\
356+
\n+----------+---------------------+-----------------------------+\
357+
\n| 0 | 1970-01-01T00:00:00 | 258.19888974716116 |\
358+
\n| 1 | 1970-01-01T00:00:00 | 258.19888974716116 |\
359+
\n+----------+---------------------+-----------------------------+",
364360
)
365361
.await;
366362
}

‎src/promql/src/extension_plan/instant_manipulate.rs‎

Lines changed: 14 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,7 @@ use crate::extension_plan::Millisecond;
4141
///
4242
/// This plan will try to align the input time series, for every timestamp between
4343
/// `start` and `end` with step `interval`. Find in the `lookback` range if data
44-
/// is missing at the given timestamp. If data is absent in some timestamp, all columns
45-
/// except the time index will left blank.
44+
/// is missing at the given timestamp.
4645
#[derive(Debug)]
4746
pub struct InstantManipulate {
4847
start: Millisecond,
@@ -341,6 +340,17 @@ impl InstantManipulateStream {
341340
take_indices: Vec<Option<u64>>,
342341
aligned_ts: Vec<Millisecond>,
343342
) -> ArrowResult<RecordBatch> {
343+
let aligned_ts = aligned_ts
344+
.into_iter()
345+
.zip(take_indices.iter())
346+
.filter_map(|(ts, i)| i.map(|_| ts))
347+
.collect::<Vec<_>>();
348+
let take_indices = take_indices
349+
.iter()
350+
.filter(|i| i.is_some())
351+
.copied()
352+
.collect::<Vec<_>>();
353+
344354
let indices_array = UInt64Array::from(take_indices);
345355
let mut arrays = record_batch
346356
.columns()
@@ -349,7 +359,8 @@ impl InstantManipulateStream {
349359
.collect::<ArrowResult<Vec<_>>>()?;
350360
arrays[self.time_index] = Arc::new(TimestampMillisecondArray::from(aligned_ts));
351361

352-
RecordBatch::try_new(record_batch.schema(), arrays)
362+
let result = RecordBatch::try_new(record_batch.schema(), arrays)?;
363+
Ok(result)
353364
}
354365
}
355366

@@ -430,11 +441,8 @@ mod test {
430441
\n| 1970-01-01T00:01:00 | 1 | foo |\
431442
\n| 1970-01-01T00:01:30 | 1 | foo |\
432443
\n| 1970-01-01T00:02:00 | 1 | foo |\
433-
\n| 1970-01-01T00:02:30 | | |\
434444
\n| 1970-01-01T00:03:00 | 1 | foo |\
435-
\n| 1970-01-01T00:03:30 | | |\
436445
\n| 1970-01-01T00:04:00 | 1 | foo |\
437-
\n| 1970-01-01T00:04:30 | | |\
438446
\n| 1970-01-01T00:05:00 | 1 | foo |\
439447
\n+---------------------+-------+------+",
440448
);
@@ -449,34 +457,19 @@ mod test {
449457
\n+---------------------+-------+------+\
450458
\n| 1970-01-01T00:00:00 | 1 | foo |\
451459
\n| 1970-01-01T00:00:10 | 1 | foo |\
452-
\n| 1970-01-01T00:00:20 | | |\
453460
\n| 1970-01-01T00:00:30 | 1 | foo |\
454461
\n| 1970-01-01T00:00:40 | 1 | foo |\
455-
\n| 1970-01-01T00:00:50 | | |\
456462
\n| 1970-01-01T00:01:00 | 1 | foo |\
457463
\n| 1970-01-01T00:01:10 | 1 | foo |\
458-
\n| 1970-01-01T00:01:20 | | |\
459464
\n| 1970-01-01T00:01:30 | 1 | foo |\
460465
\n| 1970-01-01T00:01:40 | 1 | foo |\
461-
\n| 1970-01-01T00:01:50 | | |\
462466
\n| 1970-01-01T00:02:00 | 1 | foo |\
463467
\n| 1970-01-01T00:02:10 | 1 | foo |\
464-
\n| 1970-01-01T00:02:20 | | |\
465-
\n| 1970-01-01T00:02:30 | | |\
466-
\n| 1970-01-01T00:02:40 | | |\
467-
\n| 1970-01-01T00:02:50 | | |\
468468
\n| 1970-01-01T00:03:00 | 1 | foo |\
469469
\n| 1970-01-01T00:03:10 | 1 | foo |\
470-
\n| 1970-01-01T00:03:20 | | |\
471-
\n| 1970-01-01T00:03:30 | | |\
472-
\n| 1970-01-01T00:03:40 | | |\
473-
\n| 1970-01-01T00:03:50 | | |\
474470
\n| 1970-01-01T00:04:00 | 1 | foo |\
475471
\n| 1970-01-01T00:04:10 | 1 | foo |\
476-
\n| 1970-01-01T00:04:20 | | |\
477-
\n| 1970-01-01T00:04:30 | | |\
478472
\n| 1970-01-01T00:04:40 | 1 | foo |\
479-
\n| 1970-01-01T00:04:50 | | |\
480473
\n| 1970-01-01T00:05:00 | 1 | foo |\
481474
\n+---------------------+-------+------+",
482475
);
@@ -527,14 +520,10 @@ mod test {
527520
\n| 1970-01-01T00:02:10 | 1 | foo |\
528521
\n| 1970-01-01T00:02:20 | 1 | foo |\
529522
\n| 1970-01-01T00:02:30 | 1 | foo |\
530-
\n| 1970-01-01T00:02:40 | | |\
531-
\n| 1970-01-01T00:02:50 | | |\
532523
\n| 1970-01-01T00:03:00 | 1 | foo |\
533524
\n| 1970-01-01T00:03:10 | 1 | foo |\
534525
\n| 1970-01-01T00:03:20 | 1 | foo |\
535526
\n| 1970-01-01T00:03:30 | 1 | foo |\
536-
\n| 1970-01-01T00:03:40 | | |\
537-
\n| 1970-01-01T00:03:50 | | |\
538527
\n| 1970-01-01T00:04:00 | 1 | foo |\
539528
\n| 1970-01-01T00:04:10 | 1 | foo |\
540529
\n| 1970-01-01T00:04:20 | 1 | foo |\
@@ -617,22 +606,8 @@ mod test {
617606
"+---------------------+-------+------+\
618607
\n| timestamp | value | path |\
619608
\n+---------------------+-------+------+\
620-
\n| 1970-01-01T00:03:50 | | |\
621-
\n| 1970-01-01T00:03:51 | | |\
622-
\n| 1970-01-01T00:03:52 | | |\
623-
\n| 1970-01-01T00:03:53 | | |\
624-
\n| 1970-01-01T00:03:54 | | |\
625-
\n| 1970-01-01T00:03:55 | | |\
626-
\n| 1970-01-01T00:03:56 | | |\
627-
\n| 1970-01-01T00:03:57 | | |\
628-
\n| 1970-01-01T00:03:58 | | |\
629-
\n| 1970-01-01T00:03:59 | | |\
630609
\n| 1970-01-01T00:04:00 | 1 | foo |\
631610
\n| 1970-01-01T00:04:01 | 1 | foo |\
632-
\n| 1970-01-01T00:04:02 | | |\
633-
\n| 1970-01-01T00:04:03 | | |\
634-
\n| 1970-01-01T00:04:04 | | |\
635-
\n| 1970-01-01T00:04:05 | | |\
636611
\n+---------------------+-------+------+",
637612
);
638613
do_normalize_test(230_000, 245_000, 0, 1_000, expected).await;
@@ -646,7 +621,6 @@ mod test {
646621
\n+---------------------+-------+------+\
647622
\n| 1970-01-01T00:00:00 | 1 | foo |\
648623
\n| 1970-01-01T00:00:10 | 1 | foo |\
649-
\n| 1970-01-01T00:00:20 | | |\
650624
\n| 1970-01-01T00:00:30 | 1 | foo |\
651625
\n+---------------------+-------+------+",
652626
);
@@ -659,37 +633,12 @@ mod test {
659633
"+---------------------+-------+------+\
660634
\n| timestamp | value | path |\
661635
\n+---------------------+-------+------+\
662-
\n| 1969-12-31T23:45:00 | | |\
663-
\n| 1969-12-31T23:46:00 | | |\
664-
\n| 1969-12-31T23:47:00 | | |\
665-
\n| 1969-12-31T23:48:00 | | |\
666-
\n| 1969-12-31T23:49:00 | | |\
667-
\n| 1969-12-31T23:50:00 | | |\
668-
\n| 1969-12-31T23:51:00 | | |\
669-
\n| 1969-12-31T23:52:00 | | |\
670-
\n| 1969-12-31T23:53:00 | | |\
671-
\n| 1969-12-31T23:54:00 | | |\
672-
\n| 1969-12-31T23:55:00 | | |\
673-
\n| 1969-12-31T23:56:00 | | |\
674-
\n| 1969-12-31T23:57:00 | | |\
675-
\n| 1969-12-31T23:58:00 | | |\
676-
\n| 1969-12-31T23:59:00 | | |\
677636
\n| 1970-01-01T00:00:00 | 1 | foo |\
678637
\n| 1970-01-01T00:01:00 | 1 | foo |\
679638
\n| 1970-01-01T00:02:00 | 1 | foo |\
680639
\n| 1970-01-01T00:03:00 | 1 | foo |\
681640
\n| 1970-01-01T00:04:00 | 1 | foo |\
682641
\n| 1970-01-01T00:05:00 | 1 | foo |\
683-
\n| 1970-01-01T00:06:00 | | |\
684-
\n| 1970-01-01T00:07:00 | | |\
685-
\n| 1970-01-01T00:08:00 | | |\
686-
\n| 1970-01-01T00:09:00 | | |\
687-
\n| 1970-01-01T00:10:00 | | |\
688-
\n| 1970-01-01T00:11:00 | | |\
689-
\n| 1970-01-01T00:12:00 | | |\
690-
\n| 1970-01-01T00:13:00 | | |\
691-
\n| 1970-01-01T00:14:00 | | |\
692-
\n| 1970-01-01T00:15:00 | | |\
693642
\n+---------------------+-------+------+",
694643
);
695644
do_normalize_test(-900_000, 900_000, 30_000, 60_000, expected).await;
@@ -704,8 +653,6 @@ mod test {
704653
\n| 1970-01-01T00:03:10 | 1 | foo |\
705654
\n| 1970-01-01T00:03:20 | 1 | foo |\
706655
\n| 1970-01-01T00:03:30 | 1 | foo |\
707-
\n| 1970-01-01T00:03:40 | | |\
708-
\n| 1970-01-01T00:03:50 | | |\
709656
\n| 1970-01-01T00:04:00 | 1 | foo |\
710657
\n| 1970-01-01T00:04:10 | 1 | foo |\
711658
\n| 1970-01-01T00:04:20 | 1 | foo |\

0 commit comments

Comments
 (0)