Add support to generate a Promethus exposition format output

seshat:text_format/3 produces a binary that contains the text
representation of the metrics in the Prometheus exposition format,
so that it can be returned as-is by a Prometheus endpoint

Author: mkuratczyk

rebar3.crashdump

@@ -0,0 +1,23 @@
+Error: terminated
+[{io,format,
+     ["rebar ~ts on Erlang/OTP ~ts Erts ~ts~n",
+      ["3.24.0+build.5437.ref5495da14","27","15.2.3"]],
+     [{file,"io.erl"},
+      {line,198},
+      {error_info,#{cause => {io,terminated},module => erl_stdlib_errors}}]},
+ {rebar_prv_version,do,1,
+                    [{file,"/Users/mkuratczyk/workspace/rebar3/apps/rebar/src/rebar_prv_version.erl"},
+                     {line,36}]},
+ {rebar_core,do,2,
+             [{file,"/Users/mkuratczyk/workspace/rebar3/apps/rebar/src/rebar_core.erl"},
+              {line,155}]},
+ {rebar3,run_aux,2,
+         [{file,"/Users/mkuratczyk/workspace/rebar3/apps/rebar/src/rebar3.erl"},
+          {line,212}]},
+ {rebar3,main,1,
+         [{file,"/Users/mkuratczyk/workspace/rebar3/apps/rebar/src/rebar3.erl"},
+          {line,66}]},
+ {escript,run,2,[{file,"escript.erl"},{line,904}]},
+ {escript,start,1,[{file,"escript.erl"},{line,418}]},
+ {init,start_it,1,[]}]
+

src/seshat.erl

@@ -19,6 +19,7 @@
          delete/2,
          format/1,
          format/2,
+         text_format/3,
          format_one/2
         ]).
 
@@ -310,3 +311,77 @@ format_fields(Fields, CRef, Labels, Acc) ->
                 MetricAcc1 = MetricAcc#{values => ValuesAcc1},
                 Acc0#{Name => MetricAcc1}
         end, Acc, Fields).
+
+%% @doc Return a Prometheus-formated text (as a binary),
+%% which can be directly returned by a Prometheus endpoint.
+%% The returned binary has the following structure:
+%% prefix_name_unit{label_key="value_value",...} Value
+%%
+%% Units are automatically appended based on the metric type.
+%%
+%% @param Group the name of an existing group
+%% @param Names the list of metrics to return
+%%
+-spec text_format(group(), string(), [atom()]) -> binary().
+text_format(Group, Prefix, Names) when is_list(Names) ->
+    PrefixBin = list_to_binary(Prefix ++ "_"),
+    Metrics = ets:foldl(fun
+                    ({_Name, _CRef, _FieldSpec, Labels}, Acc) when map_size(Labels) == 0 ->
+                        %% skip metrics with no labels; at some point we might want to export them
+                        Acc;
+                    ({_Name, CRef, FieldSpec, Labels}, Acc) ->
+                        Fields0 = resolve_fields_spec(FieldSpec),
+                        Fields = lists:filter(fun (F) -> lists:member(element(1, F), Names) end, Fields0),
+                        text_format_fields(Fields, CRef, Labels, PrefixBin, Acc)
+        end, #{}, seshat_counters_server:get_table(Group)),
+    MetricsBin = maps:fold(fun(_Name, Lines, Acc) ->
+        <<Acc/binary, Lines/binary, <<"\n">>/binary>>
+    end, <<>>, Metrics),
+    MetricsBin.
+
+text_format_fields(Fields, CRef, Labels, PrefixBin, Acc) ->
+    LabelsList = maps:to_list(Labels),
+    % transform the map of labels into "label1=value1,label2=value2"
+    LabelsBin0 = lists:foldl(
+        fun ({Name, Value}, LabelsAcc) ->
+            LabelKey = atom_to_binary(Name, utf8),
+            LabelValue = list_to_binary(Value),
+            <<LabelsAcc/binary, LabelKey/binary, "=\"", LabelValue/binary, "\",">>
+        end, <<>>, LabelsList),
+    % remove the final comma
+    LabelsBin = case LabelsBin0 of
+        <<>> -> <<>>;
+        _ -> binary:part(LabelsBin0, 0, byte_size(LabelsBin0) - 1)
+        end,
+    % produce the lines for each metric; we are itearting by object, but Prometheus
+    % output should be sorted by metric name with one HELP and one TYPE line for
+    % a given metric, so we accumulate in a map, with the metric name as key
+    lists:foldl(
+        fun ({Name, Index, Type, Help}, Acc0) ->
+                ComputedType = case Type of
+                                    ratio -> gauge;
+                                    Other -> Other
+                                end,
+                ComputedUnit = case Type of
+                                    ratio -> <<"_ratio">>;
+                                    _ -> <<"">>
+                                end,
+                ComputedValue = case Type of
+                                    ratio -> Ratio = counters:get(CRef, Index) / 100,
+                                    Ratio1= float_to_list(Ratio, [{decimals, 2}, compact]),
+                                    list_to_binary(Ratio1)
+                                    ;
+                                    _ -> integer_to_binary(counters:get(CRef, Index))
+                                end,
+                                NameBin = <<PrefixBin/binary, (atom_to_binary(Name, utf8))/binary, ComputedUnit/binary>>,
+                Line = <<NameBin/binary, "{", LabelsBin/binary, "} ", ComputedValue/binary>>,
+                case maps:get(Name, Acc0, <<>>) of
+                    <<>> ->
+                        HelpLine = <<"# HELP ", NameBin/binary, <<" ">>/binary, (list_to_binary(Help))/binary>>,
+                        TypeBin = atom_to_binary(ComputedType, utf8),
+                        TypeLine = <<"# TYPE ", NameBin/binary, <<" ">>/binary, TypeBin/binary>>,
+                        Acc0#{Name => <<HelpLine/binary, <<"\n">>/binary, TypeLine/binary, <<"\n">>/binary, Line/binary>>};
+                    Lines ->
+                        Acc0#{Name => <<Lines/binary, <<"\n">>/binary, Line/binary>>}
+                end
+        end, Acc, Fields).

test/seshat_test.erl

@@ -20,11 +20,12 @@ test_suite_test_() ->
      fun cleanup/1,
      [ fun overview/0,
        fun counters_with_persistent_term_field_spec/0,
-       fun prometheus_format_group/0,
-       fun prometheus_format_one/0,
-       fun prometheus_format_with_many_labels/0,
-       fun prometheus_format_ratio/0,
-       fun prometheus_format_selected_metrics/0,
+       fun format_group/0,
+       fun format_one/0,
+       fun format_with_many_labels/0,
+       fun format_ratio/0,
+       fun format_selected_metrics/0,
+       fun text_format_selected_metrics/0,
        fun invalid_fields/0 ]}.
 
 overview() ->
@@ -90,7 +91,7 @@ counters_with_persistent_term_field_spec() ->
 
     ok.
 
-prometheus_format_group() ->
+format_group() ->
     Group = widgets,
     Counters = [{reads, 1, counter, "Total reads"}],
     seshat:new_group(Group),
@@ -105,7 +106,7 @@ prometheus_format_group() ->
     ?assertEqual(ExpectedPrometheusFormat, PrometheusFormat),
     ok.
 
-prometheus_format_one() ->
+format_one() ->
     Group = widgets,
     Counters = [{reads, 1, counter, "Total reads"}],
     seshat:new_group(Group),
@@ -118,7 +119,7 @@ prometheus_format_one() ->
     ?assertEqual(ExpectedPrometheusFormat, PrometheusFormat),
     ok.
 
-prometheus_format_with_many_labels() ->
+format_with_many_labels() ->
     Group = widgets,
     Counters = [{reads, 1, counter, "Total reads"}],
     seshat:new_group(Group),
@@ -134,7 +135,7 @@ prometheus_format_with_many_labels() ->
     ?assertEqual(ExpectedPrometheusFormat, PrometheusFormat),
     ok.
 
-prometheus_format_selected_metrics() ->
+format_selected_metrics() ->
     Group = widgets,
     Counters = [
                 {reads, 1, counter, "Total reads"},
@@ -166,7 +167,7 @@ invalid_fields() ->
 
     ok.
 
-prometheus_format_ratio() ->
+format_ratio() ->
     Group = widgets,
     Counters = [{pings, 1, ratio, "Some ratio that happens to be 0%"},
                 {pongs, 2, ratio, "Some ratio that happens to be 17%"},
@@ -196,6 +197,46 @@ prometheus_format_ratio() ->
     ?assertEqual(ExpectedPrometheusFormat, PrometheusFormat),
     ok.
 
+text_format_selected_metrics() ->
+    Group = widgets,
+    Counters = [
+                {reads, 1, counter, "Total reads"},
+                {writes, 2, counter, "Total writes"},
+                {cached, 3, ratio, "Ratio of things served from cache"}
+                ],
+    seshat:new_group(Group),
+    seshat:new(Group, thing1, Counters, #{component => "thing1", version => "1.2.3"}),
+    seshat:new(Group, thing2, Counters, #{component => "thing2"}),
+    seshat:new(Group, thing3, Counters, #{component => "thing3"}),
+    set_value(Group, thing1, reads, 1),
+    set_value(Group, thing1, writes, 2),
+    set_value(Group, thing1, cached, 10),
+    set_value(Group, thing2, reads, 3),
+    set_value(Group, thing2, writes, 4),
+    set_value(Group, thing2, cached, 100),
+    set_value(Group, thing3, reads, 1234),
+    set_value(Group, thing3, writes, 4321),
+    set_value(Group, thing3, cached, 17),
+    PrometheusFormat = binary_to_list(seshat:text_format(Group, "acme", [reads, writes, cached])),
+    ExpectedPrometheusFormat = "# HELP acme_reads Total reads\n"
+    "# TYPE acme_reads counter\n"
+    "acme_reads{version=\"1.2.3\",component=\"thing1\"} 1\n"
+    "acme_reads{component=\"thing2\"} 3\n"
+    "acme_reads{component=\"thing3\"} 1234\n"
+    "# HELP acme_writes Total writes\n"
+    "# TYPE acme_writes counter\n"
+    "acme_writes{version=\"1.2.3\",component=\"thing1\"} 2\n"
+    "acme_writes{component=\"thing2\"} 4\n"
+    "acme_writes{component=\"thing3\"} 4321\n"
+    "# HELP acme_cached_ratio Ratio of things served from cache\n"
+    "# TYPE acme_cached_ratio gauge\n"
+    "acme_cached_ratio{version=\"1.2.3\",component=\"thing1\"} 0.1\n"
+    "acme_cached_ratio{component=\"thing2\"} 1.0\n"
+    "acme_cached_ratio{component=\"thing3\"} 0.17\n",
+
+    ?assertEqual(ExpectedPrometheusFormat, PrometheusFormat),
+    ok.
+
 %% test helpers
 
 set_value(Group, Id, Name, Value) ->