Detailed storage stats

include/rts.hrl

@@ -32,10 +32,28 @@
         ['Count' , 'UserErrorCount' , 'SystemErrorCount',
          'BytesIn' , 'UserErrorBytesIn' , 'SystemErrorBytesIn',
          'BytesOut' , 'UserErrorBytesOut' , 'SystemErrorBytesOut',
-         'BytesOutIncomplete', 'Objects', 'Bytes']).
+         'BytesOutIncomplete',
+         'wr_old-ct', 'wr_old-by', 'wr_old-bl',
+         'wr_new-ct', 'wr_new-by', 'wr_new-bl',
+         'wr_mp-ct', 'wr_mp-by', 'wr_mp-bl',
+         'Objects', 'Bytes', 'user-bl',
+         'sd_old-ct', 'sd_old-by', 'sd_old-bl',
+         'sd_new-ct', 'sd_new-by', 'sd_new-bl',
+         'pd_old-ct','pd_old-by', 'pd_old-bl',
+         'pd_new-ct','pd_new-by', 'pd_new-bl',
+         'ac_de-ct','ac_de-by', 'ac_de-bl']).
 
 -define(SUPPORTED_USAGE_FIELD_BIN,
         [<<"Count">> , <<"UserErrorCount">> , <<"SystemErrorCount">>,
          <<"BytesIn">> , <<"UserErrorBytesIn">> , <<"SystemErrorBytesIn">>,
          <<"BytesOut">> , <<"UserErrorBytesOut">> , <<"SystemErrorBytesOut">>,
-         <<"BytesOutIncomplete">>, <<"Objects">>, <<"Bytes">>]).
+         <<"BytesOutIncomplete">>,
+         <<"wr_old-ct">>, <<"wr_old-by">>, <<"wr_old-bl">>,
+         <<"wr_new-ct">>, <<"wr_new-by">>, <<"wr_new-bl">>,
+         <<"wr_mp-ct">>, <<"wr_mp-by">>, <<"wr_mp-bl">>,
+         <<"Objects">>, <<"Bytes">>, <<"user-bl">>,
+         <<"sd_old-ct">>, <<"sd_old-by">>, <<"sd_old-bl">>,
+         <<"sd_new-ct">>, <<"sd_new-by">>, <<"sd_new-bl">>,
+         <<"pd_old-ct">>,<<"pd_old-by">>, <<"pd_old-bl">>,
+         <<"pd_new-ct">>,<<"pd_new-by">>, <<"pd_new-bl">>,
+         <<"ac_de-ct">>,<<"ac_de-by">>, <<"ac_de-bl">>]).

priv/tools/internal/riak_cs_inspector.erl

@@ -120,7 +120,9 @@
                   {show, [{key, undefined, undefined, string, "access stats key"}]}]}).
 -define(storage_cmd_spec,
         {storage, [{list, ?sort_opt_spec},
-                   {show, [{key, undefined, undefined, string, "storage stats key"}]}]}).
+                   {show, [{key, undefined, undefined, string, "storage stats key"},
+                           {print_zeros, undefined, "print-zeros", {boolean, false},
+                            "print zeros for detaild stat items"}]}]}).
 -define(user_cmd_spec,
         {user, [{list, ?sort_opt_spec},
                 {show, [{key, undefined, "key", string, "User ID"},
@@ -192,7 +194,8 @@ process_command(gc, count, _Opts, RiakcPid) ->
 process_command(storage, list, _Opts, RiakcPid) ->
     list_storage(RiakcPid);
 process_command(storage, show, Opts, RiakcPid) ->
-    show_storage(RiakcPid, proplists:get_value(key, Opts));
+    show_storage(RiakcPid, proplists:get_value(key, Opts),
+                 proplists:get_value(print_zeros, Opts));
 process_command(block, list, Opts, RiakcPid) ->
     list_blocks(RiakcPid,
                proplists:get_value(bucket, Opts),
@@ -362,11 +365,11 @@ list_storage(RiakcPid) ->
                    end]]
      || Key <- lists:sort(Keys)].
 
-show_storage(_RiakcPid, undefined)->
+show_storage(_RiakcPid, undefined, _)->
     usage(storage, show),
     halt(1);
-show_storage(RiakcPid, Key)->
-    [print_storage_stats(Key, SiblingNo, StatsBin)
+show_storage(RiakcPid, Key, PrintZeros)->
+    [print_storage_stats(Key, SiblingNo, StatsBin, PrintZeros)
      || {SiblingNo, {_RiakMD, StatsBin}}
             <- get_riak_object(RiakcPid, "moss.storage", Key)].
 
@@ -454,7 +457,10 @@ stats_sample_from_binary([{<<"MossNode">>, Node} | Rest],
     stats_sample_from_binary(Rest, {Start, End, Node, Ops});
 stats_sample_from_binary([{OpName, {struct, Stats}} | Rest],
                          {Start, End, Node, Ops}) ->
-    stats_sample_from_binary(Rest, {Start, End, Node, [{OpName, Stats} | Ops]}).
+    stats_sample_from_binary(Rest, {Start, End, Node, [{OpName, Stats} | Ops]});
+stats_sample_from_binary([{OpName, Other} | Rest],
+                         {Start, End, Node, Ops}) ->
+    stats_sample_from_binary(Rest, {Start, End, Node, [{OpName, Other} | Ops]}).
 
 count_riak_bucket(RiakcPid, Bucket, BucketToDisplay, Timeout) ->
     case riakc_pb_socket:stream_list_keys(RiakcPid, Bucket) of
@@ -513,7 +519,7 @@ print_access_stats(Key, SiblingNo, StatsBin) ->
         {Op, OpStats} <- Ops,
         {StatsKey, StatsValue} <- OpStats].
 
-print_storage_stats(Key, SiblingNo, StatsBin) ->
+print_storage_stats(Key, SiblingNo, StatsBin, PrintZeros) ->
     {Start, End, _Node, Buckets} = stats_sample_from_binary(StatsBin),
     io:nl(),
     io:format("Key       : ~s~n", [Key]),
@@ -522,11 +528,21 @@ print_storage_stats(Key, SiblingNo, StatsBin) ->
     io:format("EndTime   : ~s~n", [End]),
     io:format("~-36..=s: ~-32..=s ~-32..=s~n",
               ["Bucket ", "Objects ", "Bytes "]),
-    %% TODO: Error handling, e.g. StatItems = "{error,{timeout,[]}}"
-    [io:format("~-36s: ~32B ~32B~n", [Bucket, Objects, Bytes]) ||
-        {Bucket, StatItems} <- Buckets,
-        {ObjectsKey, Objects} <- StatItems, ObjectsKey =:= <<"Objects">>,
-        {BytesKey, Bytes}     <- StatItems, BytesKey   =:= <<"Bytes">>].
+    [case StatItems of
+         ErrorMessage when is_binary(ErrorMessage) ->
+             io:format("~-36s: ~s~n", [Bucket, ErrorMessage]);
+         _ ->
+             {<<"Objects">>, Objects} = lists:keyfind(<<"Objects">>, 1, StatItems),
+             {<<"Bytes">>, Bytes}     = lists:keyfind(<<"Bytes">>, 1, StatItems),
+             io:format("~-36s: ~32B ~32B~n", [Bucket, Objects, Bytes]),
+             case length(StatItems) =< 2 of
+                 true -> ok;
+                 false ->
+                     [io:format("  +--- ~-29s= ~32B~n", [K, V]) ||
+                         {K, V} <- StatItems,
+                         (PrintZeros orelse V =/= 0)]
+             end
+     end || {Bucket, StatItems} <- Buckets].
 
 print_users(RiakcPid, Bucket, Options) ->
     {ok, Keys} = riakc_pb_socket:list_keys(RiakcPid, Bucket),

riak_test/tests/storage_stats_detailed_test.erl

@@ -0,0 +1,108 @@
+%% ---------------------------------------------------------------------
+%%
+%% Copyright (c) 2007-2014 Basho Technologies, Inc.  All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+%% ---------------------------------------------------------------------
+
+-module(storage_stats_detailed_test).
+%% @doc Integration test for storage statistics.
+
+-compile(export_all).
+-export([confirm/0]).
+
+-include_lib("erlcloud/include/erlcloud_aws.hrl").
+-include_lib("xmerl/include/xmerl.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-include("riak_cs.hrl").
+
+-define(BUCKET, "storage-stats-detailed").
+
+-define(KEY1, "1").
+-define(KEY2, "2").
+-define(KEY3, "3").
+
+confirm() ->
+    Config = [{riak, rtcs:riak_config()},
+              {stanchion, rtcs:stanchion_config()},
+              {cs, rtcs:cs_config([{fold_objects_for_list_keys, true},
+                                   {detailed_storage_calc, true}])}],
+    SetupRes = rtcs:setup(1, Config),
+    {UserConfig, {_RiakNodes, CSNodes, _Stanchion}} = SetupRes,
+    setup_objects(UserConfig, ?BUCKET),
+    %% Set up to grep logs to verify messages
+    rt:setup_log_capture(hd(CSNodes)),
+
+    {Begin, End} = storage_stats_test:calc_storage_stats(hd(CSNodes)),
+    {JsonStat, XmlStat} = storage_stats_test:storage_stats_request(
+                            UserConfig, Begin, End),
+
+    ?assert(rtcs:json_get([<<"StartTime">>], JsonStat) =/= notfound),
+    ?assert(rtcs:json_get([<<"EndTime">>],   JsonStat) =/= notfound),
+    ?assert(proplists:get_value('StartTime', XmlStat)  =/= notfound),
+    ?assert(proplists:get_value('EndTime',   XmlStat)  =/= notfound),
+    lists:foreach(fun({K, V}) ->
+                          assert_storage_json_stats(K, V, JsonStat),
+                          assert_storage_xml_stats(K, V, XmlStat)
+                  end,
+                  [{"Objects",   1 + 2},
+                   {"Bytes",     300 + 2 * 2*1024*1024},
+                   {"user-bl",   1 + 4},
+                   {"wr_mp-ct",  2},
+                   {"wr_mp-by",  2 * 2*1024*1024},
+                   {"wr_mp-bl",  2 * 2},
+                   {"sd_new-ct", 2},
+                   {"sd_new-by", 100 + 200},
+                   {"sd_new-bl", 2}]),
+
+    storage_stats_test:confirm_2(SetupRes),
+    rtcs:pass().
+
+setup_objects(UserConfig, Bucket) ->
+    ?assertEqual(ok, erlcloud_s3:create_bucket(Bucket, UserConfig)),
+    Block1 = crypto:rand_bytes(100),
+    ?assertEqual([{version_id, "null"}],
+                 erlcloud_s3:put_object(Bucket, ?KEY1, Block1, UserConfig)),
+    Block1Overwrite = crypto:rand_bytes(300),
+    ?assertEqual([{version_id, "null"}],
+                 erlcloud_s3:put_object(Bucket, ?KEY1, Block1Overwrite, UserConfig)),
+    Block2 = crypto:rand_bytes(200),
+    ?assertEqual([{version_id, "null"}],
+                 erlcloud_s3:put_object(Bucket, ?KEY2, Block2, UserConfig)),
+    ?assertEqual([{delete_marker, false}, {version_id, "null"}],
+                 erlcloud_s3:delete_object(Bucket, ?KEY2, UserConfig)),
+
+    InitRes = erlcloud_s3_multipart:initiate_upload(
+                Bucket, ?KEY3, "text/plain", [], UserConfig),
+    UploadId = erlcloud_xml:get_text(
+                 "/InitiateMultipartUploadResult/UploadId", InitRes),
+    MPBlocks = crypto:rand_bytes(2*1024*1024),
+    {_RespHeaders1, _UploadRes} = erlcloud_s3_multipart:upload_part(
+                                    Bucket, ?KEY3, UploadId, 1, MPBlocks, UserConfig),
+    {_RespHeaders2, _UploadRes} = erlcloud_s3_multipart:upload_part(
+                                    Bucket, ?KEY3, UploadId, 2, MPBlocks, UserConfig),
+    ok.
+
+assert_storage_json_stats(K, V, Sample) ->
+    lager:debug("assert json: ~p", [{K, V}]),
+    ?assertEqual(V, rtcs:json_get([list_to_binary(?BUCKET), list_to_binary(K)],
+                                  Sample)).
+
+assert_storage_xml_stats(K, V, Sample) ->
+    lager:debug("assert xml: ~p", [{K, V}]),
+    ?assertEqual(V, proplists:get_value(list_to_atom(K),
+                                        proplists:get_value(?BUCKET, Sample))).

riak_test/tests/storage_stats_test.erl

@@ -54,7 +54,10 @@ confirm_1(Use2iForStorageCalc) when is_boolean(Use2iForStorageCalc) ->
               {stanchion, rtcs:stanchion_config()},
               {cs, rtcs:cs_config([{fold_objects_for_list_keys, true},
                                    {use_2i_for_storage_calc, Use2iForStorageCalc}])}],
-    {UserConfig, {RiakNodes, CSNodes, _Stanchion}} = rtcs:setup(1, Config),
+    SetupRes = rtcs:setup(1, Config),
+    confirm_2(SetupRes).
+
+confirm_2({UserConfig, {RiakNodes, CSNodes, _Stanchion}}) ->
     {AccessKey2, SecretKey2} = rtcs:create_user(hd(RiakNodes), 1),
     UserConfig2 = rtcs:config(AccessKey2, SecretKey2, rtcs:cs_port(hd(RiakNodes))),
 

src/riak_cs_config.erl

@@ -60,7 +60,8 @@
          read_before_last_manifest_write/0,
          region/0,
          stanchion/0,
-         use_2i_for_storage_calc/0
+         use_2i_for_storage_calc/0,
+         detailed_storage_calc/0
         ]).
 
 %% Timeouts hitting Riak
@@ -431,6 +432,12 @@ region() ->
 use_2i_for_storage_calc() ->
     get_env(riak_cs, use_2i_for_storage_calc, false).
 
+%% @doc Cauculate detailed summary information of bucket
+%% usage or not.
+-spec detailed_storage_calc() -> boolean().
+detailed_storage_calc() ->
+    get_env(riak_cs, detailed_storage_calc, false).
+
 %% ===================================================================
 %% ALL Timeouts hitting Riak
 %% ===================================================================