format all the code

__ocr_recognition/continuous_evaluation.py

@@ -6,7 +6,6 @@
 train_avg_loss_kpi = CostKpi('train_avg_loss', 0.2, 0)
 train_seq_err_kpi = CostKpi('train_seq_err', 0.2, 0)
 
-
 tracking_kpis = [
     train_avg_loss_kpi,
     train_seq_err_kpi,

__ocr_recognition/ctc_reader.py

@@ -167,7 +167,8 @@ def train(batch_size, train_images_dir=None, train_list_file=None):
         train_images_dir = path.join(data_dir, TRAIN_DATA_DIR_NAME)
     if train_list_file is None:
         train_list_file = path.join(data_dir, TRAIN_LIST_FILE_NAME)
-    return generator.train_reader(train_images_dir, train_list_file, batch_size)
+    return generator.train_reader(train_images_dir, train_list_file,
+                                  batch_size)
 
 
 def test(batch_size=1, test_images_dir=None, test_list_file=None):

__ocr_recognition/ctc_train.py

@@ -104,8 +104,12 @@ def train(args, data_reader=ctc_reader):
             if batch_id == args.iterations - 1:
                 avg_seq_err = batch_seq_error[0] / args.batch_size
                 avg_loss = batch_loss[0] / args.batch_size
-                train_avg_loss_kpi.add_record(np.array(avg_loss, dtype='float32'))
-                train_seq_err_kpi.add_record(np.array(avg_seq_err, dtype='float32'))
+                train_avg_loss_kpi.add_record(
+                    np.array(
+                        avg_loss, dtype='float32'))
+                train_seq_err_kpi.add_record(
+                    np.array(
+                        avg_seq_err, dtype='float32'))
                 break
             # evaluate
             if batch_id % args.eval_period == 0:
@@ -121,7 +125,7 @@ def train(args, data_reader=ctc_reader):
 
             batch_id += 1
     train_avg_loss_kpi.persist()
-    train_seq_err_kpi.persist()	
+    train_seq_err_kpi.persist()
 
 
 def main():

__ocr_recognition/eval.py

@@ -54,7 +54,8 @@ def evaluate(args, eval=ctc_eval, data_reader=ctc_reader):
     count = 0
     for data in test_reader():
         count += 1
-        exe.run(fluid.default_main_program(), feed=get_feeder_data(data, place))
+        exe.run(fluid.default_main_program(),
+                feed=get_feeder_data(data, place))
     avg_distance, avg_seq_error = evaluator.eval(exe)
     print "Read %d samples; avg_distance: %s; avg_seq_error: %s" % (
         count, avg_distance, avg_seq_error)

image_classification/continuous_evaluation.py

@@ -7,9 +7,4 @@
 train_acc_top5_kpi = AccKpi('train_acc_top5_kpi', 0.2, 0)
 train_cost_kpi = CostKpi('train_cost_kpi', 0.2, 0)
 
-
-tracking_kpis = [
-	train_acc_top1_kpi,
-	train_acc_top5_kpi,
-	train_cost_kpi
-]
+tracking_kpis = [train_acc_top1_kpi, train_acc_top5_kpi, train_cost_kpi]

image_classification/latest_kpis/train_acc_top1_kpi_factor.txt

@@ -1 +1 @@
-[0.33659327030181885]
+[0.33659327030181885]

image_classification/latest_kpis/train_acc_top5_kpi_factor.txt

@@ -1 +1 @@
-[0.5419688820838928]
+[0.5419688820838928]

image_classification/latest_kpis/train_cost_kpi_factor.txt

@@ -1 +1 @@
-[2.975904941558838]
+[2.975904941558838]

image_classification/se_resnext.py

@@ -2,7 +2,11 @@
 import paddle.fluid as fluid
 
 
-def conv_bn_layer(input, num_filters, filter_size, stride=1, groups=1,
+def conv_bn_layer(input,
+                  num_filters,
+                  filter_size,
+                  stride=1,
+                  groups=1,
                   act=None):
     conv = fluid.layers.conv2d(
         input=input,

image_classification/train.py

@@ -12,7 +12,8 @@
 import functools
 from utility import add_arguments, print_arguments
 
-from continuous_evaluation import (train_acc_top1_kpi, train_acc_top5_kpi, train_cost_kpi)
+from continuous_evaluation import (train_acc_top1_kpi, train_acc_top5_kpi,
+                                   train_cost_kpi)
 parser = argparse.ArgumentParser(description=__doc__)
 add_arg = functools.partial(add_arguments, argparser=parser)
 # yapf: disable
@@ -244,14 +245,13 @@ def train_parallel_exe(args,
         train_loss = np.array(train_info[0]).mean()
         train_acc1 = np.array(train_info[1]).mean()
         train_acc5 = np.array(train_info[2]).mean()
-	if pass_id == num_passes - 1:
-		train_acc_top1_kpi.add_record(train_acc1)
-		train_acc_top5_kpi.add_record(train_acc5)
-		train_cost_kpi.add_record(train_loss)
+        if pass_id == num_passes - 1:
+            train_acc_top1_kpi.add_record(train_acc1)
+            train_acc_top5_kpi.add_record(train_acc5)
+            train_cost_kpi.add_record(train_loss)
         for data in test_reader():
             t1 = time.time()
-            loss, acc1, acc5 = test_exe.run(fetch_list,
-                                            feed=feeder.feed(data))
+            loss, acc1, acc5 = test_exe.run(fetch_list, feed=feeder.feed(data))
             t2 = time.time()
             period = t2 - t1
             loss = np.mean(np.array(loss))

language_model/continuous_evaluation.py

@@ -8,8 +8,8 @@
 from kpi import DurationKpi
 
 imikolov_20_avg_ppl_kpi = CostKpi('imikolov_20_avg_ppl', 0.2, 0)
-imikolov_20_pass_duration_kpi = DurationKpi('imikolov_20_pass_duration', 0.2, 0)
-
+imikolov_20_pass_duration_kpi = DurationKpi('imikolov_20_pass_duration', 0.2,
+                                            0)
 
 tracking_kpis = [
     imikolov_20_avg_ppl_kpi,

language_model/latest_kpis/imikolov_20_avg_ppl_factor.txt

@@ -1 +1 @@
-[32.465272032979705]
+[32.465272032979705]

language_model/latest_kpis/imikolov_20_pass_duration_factor.txt

@@ -1 +1 @@
-[17.121339857578278]
+[17.121339857578278]

language_model/train.py

@@ -5,12 +5,13 @@
 import math
 
 import paddle.fluid as fluid
-import  paddle
+import paddle
 
 import utils
 
 from continuous_evaluation import imikolov_20_avg_ppl_kpi, imikolov_20_pass_duration_kpi
 
+
 def network(src, dst, vocab_size, hid_size, init_low_bound, init_high_bound):
     """ network definition """
     emb_lr_x = 10.0
@@ -129,16 +130,17 @@ def train(train_reader,
 
         t1 = time.time()
         total_time += t1 - t0
-        print "epoch:%d num_steps:%d time_cost(s):%f" % (epoch_idx, i,
-                                                         total_time / epoch_idx)
+        print "epoch:%d num_steps:%d time_cost(s):%f" % (
+            epoch_idx, i, total_time / epoch_idx)
 
         if pass_idx == pass_num - 1:
             imikolov_20_pass_duration_kpi.add_record(total_time / epoch_idx)
-	    imikolov_20_avg_ppl_kpi.add_record(newest_ppl)
+            imikolov_20_avg_ppl_kpi.add_record(newest_ppl)
         save_dir = "%s/epoch_%d" % (model_dir, epoch_idx)
         feed_var_names = ["src_wordseq", "dst_wordseq"]
         fetch_vars = [avg_cost]
-        fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, exe)
+        fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars,
+                                      exe)
         print("model saved in %s" % save_dir)
     imikolov_20_pass_duration_kpi.persist()
     imikolov_20_avg_ppl_kpi.persist()

language_model/utils.py

@@ -35,6 +35,7 @@ def prepare_data(batch_size, buffer_size=1000, word_freq_threshold=0):
         batch_size)
     test_reader = paddle.batch(
         paddle.dataset.imikolov.test(
-            vocab, buffer_size, data_type=paddle.dataset.imikolov.DataType.SEQ),
+            vocab, buffer_size,
+            data_type=paddle.dataset.imikolov.DataType.SEQ),
         batch_size)
     return vocab, train_reader, test_reader

lstm/continuous_evaluation.py

@@ -11,7 +11,6 @@
 imdb_32_train_speed_kpi = AccKpi('imdb_32_train_speed', 0.2, 0)
 imdb_32_gpu_memory_kpi = DurationKpi('imdb_32_gpu_memory', 0.2, 0)
 
-
 tracking_kpis = [
     imdb_32_train_speed_kpi,
     imdb_32_gpu_memory_kpi,

lstm/get_gpu_data.py

@@ -5,7 +5,6 @@
 # Copyright (c) 2018 Baidu.com, Inc. All Rights Reserved
 # 
 ########################################################################
-
 """
 File: get_gpu_data.py
 Author: paddle([email protected])
@@ -14,7 +13,6 @@
 import argparse
 from continuous_evaluation import tracking_kpis
 
-
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument(
     '--batch_size', type=int, default=128, help="Batch size for training.")
@@ -25,20 +23,21 @@
     help='Optional dataset for benchmark.')
 args = parser.parse_args()
 
+
 def save_gpu_data():
     mem_list = []
     with open('memory.txt', 'r') as f:
-        for i , data in enumerate(f.readlines()):
+        for i, data in enumerate(f.readlines()):
             if i == 0:
                 continue
             mem_list.append(int(data.split("\n")[0].split(" ")[0]))
     gpu_memory_factor = None
-    for kpi in tracking_kpis: 
+    for kpi in tracking_kpis:
         if kpi.name == '%s_%s_gpu_memory' % (args.data_set, args.batch_size):
             gpu_memory_kpi = kpi
     gpu_memory_kpi.add_record(max(mem_list))
     gpu_memory_kpi.persist()
 
+
 if __name__ == "__main__":
     save_gpu_data()
-

lstm/latest_kpis/imdb_32_gpu_memory_factor.txt

@@ -1 +1 @@
-[1560]
+[1560]

lstm/latest_kpis/imdb_32_train_speed_factor.txt

@@ -1 +1 @@
-[883.2451171875]
+[883.2451171875]

lstm/model.py

@@ -24,6 +24,7 @@
 
 from continuous_evaluation import tracking_kpis
 
+
 def parse_args():
     parser = argparse.ArgumentParser("Understand Sentiment by Dynamic RNN.")
     parser.add_argument(
@@ -38,7 +39,10 @@ def parse_args():
         help='The first num of minibatch num to skip, for better performance test'
     )
     parser.add_argument(
-        '--iterations', type=int, default=80, help='The number of minibatches.')
+        '--iterations',
+        type=int,
+        default=80,
+        help='The number of minibatches.')
     parser.add_argument(
         '--emb_dim',
         type=int,
@@ -173,7 +177,7 @@ def gate_common(
             crop_sentence(imdb.train(word_dict), args.crop_size),
             buf_size=25000),
         batch_size=args.batch_size)
-    
+
     train_acc_kpi = None
     for kpi in tracking_kpis:
         if kpi.name == 'imdb_%s_train_acc' % (args.batch_size):
@@ -247,19 +251,19 @@ def collect_gpu_memory_data(alive):
     global is_alive
     status, output = commands.getstatusoutput('rm -rf memory.txt')
     if status == 0:
-    	print('del memory.txt')
+        print('del memory.txt')
     command = "nvidia-smi --id=%s --query-compute-apps=used_memory --format=csv -lms 1 > memory.txt" % args.gpu_id
     p = subprocess.Popen(command, shell=True)
     if p.pid < 0:
-    	print('Get GPU memory data error')
-    while(is_alive):
+        print('Get GPU memory data error')
+    while (is_alive):
         time.sleep(1)
     p.kill()
 
 
 def save_gpu_data(mem_list):
     gpu_memory_kpi = None
-    for kpi in tracking_kpis: 
+    for kpi in tracking_kpis:
         if kpi.name == 'imdb_%s_gpu_memory' % (args.batch_size):
             gpu_memory_kpi = kpi
     gpu_memory_kpi.add_record(max(mem_list))
@@ -272,7 +276,7 @@ def save_gpu_data(mem_list):
     global is_alive
     is_alive = True
     collect_memory_thread = threading.Thread(
-            target=collect_gpu_memory_data, args=(is_alive,))
+        target=collect_gpu_memory_data, args=(is_alive, ))
     collect_memory_thread.setDaemon(True)
     collect_memory_thread.start()
     main()

mnist/latest_kpis/test_acc_factor.txt

@@ -2,4 +2,4 @@
 [0.9839000105857849]
 [0.9868000149726868]
 [0.9866999983787537]
-[0.9879000186920166]
+[0.9879000186920166]

mnist/latest_kpis/train_acc_factor.txt

@@ -2,4 +2,4 @@
 [0.9831333160400391]
 [0.9886166453361511]
 [0.9915000200271606]
-[0.9929666519165039]
+[0.9929666519165039]

mnist/latest_kpis/train_duration_factor.txt

@@ -2,4 +2,4 @@
 [36.04332995414734]
 [36.20732808113098]
 [36.188393115997314]
-[35.95417380332947]
+[35.95417380332947]

mnist/model.py

@@ -49,8 +49,8 @@ def parse_args():
 
 
 def print_arguments(args):
-    vars(args)['use_nvprof'] = (vars(args)['use_nvprof']
-                                and vars(args)['device'] == 'GPU')
+    vars(args)['use_nvprof'] = (vars(args)['use_nvprof'] and
+                                vars(args)['device'] == 'GPU')
     print('-----------  Configuration Arguments -----------')
     for arg, value in sorted(vars(args).iteritems()):
         print('%s: %s' % (arg, value))
@@ -99,13 +99,10 @@ def eval_test(exe, batch_acc, batch_size_tensor, inference_program):
         y_data = np.array(map(lambda x: x[1], data)).astype("int64")
         y_data = y_data.reshape([len(y_data), 1])
 
-        acc, weight = exe.run(
-            inference_program,
-            feed={
-                "pixel": img_data,
-                "label": y_data
-            },
-            fetch_list=[batch_acc, batch_size_tensor])
+        acc, weight = exe.run(inference_program,
+                              feed={"pixel": img_data,
+                                    "label": y_data},
+                              fetch_list=[batch_acc, batch_size_tensor])
         test_pass_acc.add(value=acc, weight=weight)
         pass_acc = test_pass_acc.eval()
     return pass_acc
@@ -167,10 +164,8 @@ def run_benchmark(model, args):
             start = time.time()
             outs = exe.run(
                 fluid.default_main_program(),
-                feed={
-                    "pixel": img_data,
-                    "label": y_data
-                },
+                feed={"pixel": img_data,
+                      "label": y_data},
                 fetch_list=[avg_cost, batch_acc, batch_size_tensor]
             )  # The accuracy is the accumulation of batches, but not the current batch.
             accuracy.add(value=outs[1], weight=outs[2])

object_detection/continuous_evaluation.py

@@ -6,8 +6,4 @@
 train_cost_kpi = CostKpi('train_cost', 0.2, 0)
 train_speed_kpi = AccKpi('train_speed', 0.2, 0)
 
-
-tracking_kpis = [
-	train_cost_kpi,
-	train_speed_kpi
-]
+tracking_kpis = [train_cost_kpi, train_speed_kpi]

object_detection/labels/create_list.py

@@ -36,8 +36,10 @@ def walk_dir(devkit_dir, year):
                 added.add(name_prefix)
                 ann_path = osp.join(annotation_dir, name_prefix + '.xml')
                 img_path = osp.join(img_dir, name_prefix + '.jpg')
-                assert os.path.isfile(ann_path), 'file %s not found.' % ann_path
-                assert os.path.isfile(img_path), 'file %s not found.' % img_path
+                assert os.path.isfile(
+                    ann_path), 'file %s not found.' % ann_path
+                assert os.path.isfile(
+                    img_path), 'file %s not found.' % img_path
                 img_ann_list.append((img_path, ann_path))
 
     return trainval_list, test_list

object_detection/latest_kpis/train_cost_factor.txt

@@ -1 +1 @@
-[8.533137321472168]
+[8.533137321472168]

object_detection/latest_kpis/train_speed_factor.txt

@@ -1 +1 @@
-[73.42477785941152]
+[73.42477785941152]