Merge pull request #7 from ChunweiYan/feature/link_sdk_with_server

Author: Superjomn

server/visualdl/visual_dl.py

@@ -13,6 +13,7 @@
 from visualdl.log import logger
 import visualdl.mock.data as mock_data
 import visualdl.mock.tags as mock_tags
+import storage
 
 app = Flask(__name__, static_url_path="")
 
@@ -31,7 +32,14 @@ def option_parser():
         default=8040,
         action="store",
         dest="port",
-        help="rest api service port")
+        help="api service port")
+    parser.add_option(
+        "-t",
+        "--host",
+        type=str,
+        default="0.0.0.0",
+        action="store",
+        help="api service ip")
     parser.add_option(
         "--logdir", action="store", dest="logdir", help="log file directory")
     return parser.parse_args()
@@ -42,6 +50,8 @@ def option_parser():
 static_file_path = "./frontend/dist/"
 mock_data_path = "./mock_data/"
 
+storage = storage.StorageReader(options.logdir)
+
 
 # return data
 # status, msg, data
@@ -85,7 +95,22 @@ def runs():
 @app.route("/data/plugin/scalars/tags")
 def tags():
     is_debug = bool(request.args.get('debug'))
-    result = gen_result(0, "", mock_tags.data())
+    tag = request.args.get('tag')
+    if is_debug:
+        result = mock_tags.data()
+    else:
+        result = {}
+        print 'modes', storage.modes()
+        for mode in storage.modes():
+            result[mode] = {}
+            reader = storage.as_mode(mode)
+            for tag in reader.tags("scalar"):
+                result[mode][tag] = {
+                    'displayName': reader.scalar(tag).caption(),
+                    'description': ""
+                }
+    print 'tags', result
+    result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')
 
 
@@ -94,10 +119,22 @@ def scalars():
     run = request.args.get('run')
     tag = request.args.get('tag')
     is_debug = bool(request.args.get('debug'))
-    result = gen_result(0, "", mock_data.sequence_data())
+    if is_debug:
+        result = gen_result(0, "", mock_data.sequence_data())
+    else:
+        reader = storage.as_mode(run)
+        scalar = reader.scalar(tag)
+
+        records = scalar.records()
+        ids = scalar.ids()
+        timestamps = scalar.timestamps()
+
+        result = zip(timestamps, ids, records)
+        result = gen_result(0, "", result)
+
     return Response(json.dumps(result), mimetype='application/json')
 
 
 if __name__ == '__main__':
     logger.info(" port=" + str(options.port))
-    app.run(debug=False, host="0.0.0.0", port=options.port)
+    app.run(debug=False, host=options.host, port=options.port)

visualdl/logic/pybind.cc

@@ -38,11 +38,13 @@ PYBIND11_PLUGIN(core) {
     return vs::components::ScalarReader<T>(std::move(tablet));          \
   })
   py::class_<vs::Reader>(m, "Reader")
-      .def(
-          "__init__",
-          [](vs::Reader& instance,
-             const std::string& mode,
-             const std::string& dir) { new (&instance) vs::Reader(mode, dir); })
+      .def("__init__",
+           [](vs::Reader& instance, const std::string& dir) {
+             new (&instance) vs::Reader(dir);
+           })
+      .def("as_mode", &vs::Reader::AsMode)
+      .def("modes", [](vs::Reader& self) { return self.storage().modes(); })
+      .def("tags", &vs::Reader::tags)
       // clang-format off
     ADD_SCALAR(float)
     ADD_SCALAR(double)
@@ -59,8 +61,7 @@ PYBIND11_PLUGIN(core) {
   py::class_<vs::Writer>(m, "Writer")
       .def("__init__",
            [](vs::Writer& instance, const std::string& dir, int sync_cycle) {
-             new (&instance) vs::Writer(dir);
-             instance.storage().meta.cycle = sync_cycle;
+             new (&instance) vs::Writer(dir, sync_cycle);
            })
       .def("as_mode", &vs::Writer::AsMode)
       // clang-format off

visualdl/logic/sdk.h

@@ -6,16 +6,24 @@
 #include "visualdl/utils/string.h"
 namespace visualdl {
 
+const static std::string kDefaultMode{"default"};
+
 class Writer {
 public:
-  Writer(const std::string& dir) {
+  Writer(const std::string& dir, int sync_cycle) {
     storage_.SetDir(dir);
+    storage_.meta.cycle = sync_cycle;
+  }
+  Writer(const Writer& other) {
+    storage_ = other.storage_;
+    mode_ = other.mode_;
   }
 
-  Writer& AsMode(const std::string& mode) {
-    mode_ = mode;
+  Writer AsMode(const std::string& mode) {
+    Writer writer = *this;
     storage_.AddMode(mode);
-    return *this;
+    writer.mode_ = mode;
+    return writer;
   }
 
   Tablet AddTablet(const std::string& tag) {
@@ -31,23 +39,64 @@ class Writer {
 
 private:
   Storage storage_;
-  std::string mode_;
+  std::string mode_{kDefaultMode};
 };
 
 class Reader {
 public:
-  Reader(const std::string& mode, const std::string& dir)
-      : mode_(mode), reader_(dir) {}
+  Reader(const std::string& dir) : reader_(dir) {}
+
+  Reader AsMode(const std::string& mode) {
+    auto tmp = *this;
+    tmp.mode_ = mode;
+    return tmp;
+  }
 
   TabletReader tablet(const std::string& tag) {
     auto tmp = mode_ + "/" + tag;
     string::TagEncode(tmp);
     return reader_.tablet(tmp);
   }
 
+  std::vector<std::string> all_tags() {
+    auto tags = reader_.all_tags();
+    auto it =
+        std::remove_if(tags.begin(), tags.end(), [&](const std::string& tag) {
+          return !TagMatchMode(tag);
+        });
+    tags.erase(it + 1);
+    return tags;
+  }
+
+  std::vector<std::string> tags(const std::string& component) {
+    auto type = Tablet::type(component);
+    auto tags = reader_.tags(type);
+    CHECK(!tags.empty());
+    std::vector<std::string> res;
+    for (const auto& tag : tags) {
+      if (TagMatchMode(tag)) {
+        res.push_back(GenReadableTag(tag));
+      }
+    }
+    return res;
+  }
+
+  StorageReader& storage() { return reader_; }
+
+protected:
+  bool TagMatchMode(const std::string& tag) {
+    if (tag.size() <= mode_.size()) return false;
+    return tag.substr(0, mode_.size()) == mode_;
+  }
+  std::string GenReadableTag(const std::string& tag) {
+    auto tmp = tag;
+    string::TagDecode(tmp);
+    return tmp.substr(mode_.size() + 1);  // including `/`
+  }
+
 private:
   StorageReader reader_;
-  std::string mode_{"default"};
+  std::string mode_{kDefaultMode};
 };
 
 namespace components {
@@ -84,14 +133,13 @@ struct ScalarReader {
   std::vector<T> ids() const;
   std::vector<T> timestamps() const;
   std::string caption() const;
-  size_t total_records() {return reader_.total_records();}
+  size_t total_records() { return reader_.total_records(); }
   size_t size() const;
 
 private:
   TabletReader reader_;
 };
 
-
 }  // namespace components
 }  // namespace visualdl
 

visualdl/logic/sdk_test.cc

@@ -6,18 +6,19 @@ namespace visualdl {
 
 TEST(Scalar, write) {
   const auto dir = "./tmp/sdk_test";
-  Storage storage;
+  Writer writer__(dir, 1);
+  auto writer = writer__.AsMode("train");
   // write disk every time
-  storage.meta.cycle = 1;
-  storage.SetDir(dir);
-  auto tablet = storage.AddTablet("scalar0");
+  auto tablet = writer.AddTablet("scalar0");
   components::Scalar<int> scalar(tablet);
-  scalar.SetCaption("train");
   scalar.AddRecord(0, 12);
-  storage.PersistToDisk();
+  auto tablet1 = writer.AddTablet("model/layer/min");
+  components::Scalar<float> scalar1(tablet1);
+  scalar1.SetCaption("customized caption");
 
   // read from disk
-  StorageReader reader(dir);
+  Reader reader_(dir);
+  auto reader = reader_.AsMode("train");
   auto tablet_reader = reader.tablet("scalar0");
   auto scalar_reader = components::ScalarReader<int>(std::move(tablet_reader));
   auto captioin = scalar_reader.caption();
@@ -27,6 +28,16 @@ TEST(Scalar, write) {
   ASSERT_EQ(record.size(), 1);
   // check the first entry of first record
   ASSERT_EQ(record.front(), 12);
+
+  // check tags
+  ASSERT_EQ(reader.all_tags().size(), 1);
+  auto tags = reader.tags("scalar");
+  ASSERT_EQ(tags.size(), 2);
+  ASSERT_EQ(tags.front(), "scalar0");
+  ASSERT_EQ(tags[1], "model/layer/min");
+  components::ScalarReader<float> scalar_reader1(
+      reader.tablet("model/layer/min"));
+  ASSERT_EQ(scalar_reader1.caption(), "customized caption");
 }
 
 }  // namespace visualdl

visualdl/python/storage.py

@@ -9,8 +9,19 @@
 
 class StorageReader(object):
 
-    def __init__(self, mode, dir):
-        self.reader = core.Reader(mode, dir)
+    def __init__(self, dir, reader=None):
+        self.dir = dir
+        self.reader = reader if reader else core.Reader(dir)
+
+    def as_mode(self, mode):
+        tmp = StorageReader(dir, self.reader.as_mode(mode))
+        return tmp
+
+    def modes(self):
+        return self.reader.modes()
+
+    def tags(self, kind):
+        return self.reader.tags(kind)
 
     def scalar(self, tag, type='float'):
         type2scalar = {
@@ -23,12 +34,14 @@ def scalar(self, tag, type='float'):
 
 class StorageWriter(object):
 
-    def __init__(self, dir, sync_cycle):
-        self.writer = core.Writer(dir, sync_cycle)
+    def __init__(self, dir, sync_cycle, writer=None):
+        self.dir = dir
+        self.sync_cycle = sync_cycle
+        self.writer = writer if writer else core.Writer(dir, sync_cycle)
 
     def as_mode(self, mode):
-        self.writer = self.writer.as_mode(mode)
-        return self
+        tmp = StorageWriter(self.dir, self.sync_cycle, self.writer.as_mode(mode))
+        return tmp
 
     def scalar(self, tag, type='float'):
         type2scalar = {

visualdl/python/test_read_service.py

@@ -4,20 +4,22 @@
 import random
 import time
 
+
 class StorageTest(unittest.TestCase):
     def setUp(self):
         self.dir = "./tmp/storage_test"
 
     def test_read(self):
         print 'test write'
-        self.writer = storage.StorageWriter(self.dir, sync_cycle=1).as_mode("train")
+        self.writer = storage.StorageWriter(
+            self.dir, sync_cycle=1).as_mode("train")
         scalar = self.writer.scalar("model/scalar/min")
         # scalar.set_caption("model/scalar/min")
         for i in range(10):
             scalar.add_record(i, float(i))
 
         print 'test read'
-        self.reader = storage.StorageReader("train", self.dir)
+        self.reader = storage.StorageReader(self.dir).as_mode("train")
         scalar = self.reader.scalar("model/scalar/min")
         self.assertEqual(scalar.caption(), "train")
         records = scalar.records()