From cf253873d127acf8e443da81a5763c1db79e430d Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Tue, 16 Mar 2021 17:49:01 +0630 Subject: [PATCH 1/9] refactor: explicit template for image classification --- .github/workflows/ci.yml | 2 +- app/streamlit_app.py | 2 +- .../base/{base_config.py => base_sidebar.py} | 7 +- templates/base/utils.py.jinja | 5 - templates/image_classification/fn.py.jinja | 1 - .../image_classification/generate_metadata.py | 30 --- .../image_classification_config.py | 19 -- .../image_classification_sidebar.py | 36 +++ templates/image_classification/main.py.jinja | 137 ++++++++++- templates/image_classification/metadata.json | 51 ----- templates/image_classification/utils.py.jinja | 212 +++++++++++++++++- 11 files changed, 376 insertions(+), 126 deletions(-) rename templates/base/{base_config.py => base_sidebar.py} (96%) delete mode 100644 templates/image_classification/fn.py.jinja delete mode 100644 templates/image_classification/generate_metadata.py delete mode 100644 templates/image_classification/image_classification_config.py create mode 100644 templates/image_classification/image_classification_sidebar.py delete mode 100644 templates/image_classification/metadata.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40f6baf5..d03d5e56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: restore-keys: | ${{ steps.get-date.outputs.date }}-${{ runner.os }}-${{ matrix.python-version }}- - - run: pip install --pre -r requirements-dev.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html --progress-bar off + - run: pip install -r requirements-dev.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html --progress-bar off - run: python -m torch.utils.collect_env - run: bash .github/run_test.sh generate - run: bash .github/run_test.sh unittest diff --git a/app/streamlit_app.py b/app/streamlit_app.py index 6e322c12..7309f031 100644 --- a/app/streamlit_app.py +++ b/app/streamlit_app.py @@ -47,7 +47,7 @@ def render_code(self, fname="", code="", fold=False): def add_sidebar(self): def config(template_name): - return import_from_file("template_config", f"./templates/{template_name}/{template_name}_config.py") + return import_from_file("template_config", f"./templates/{template_name}/{template_name}_sidebar.py") self.sidebar(self.codegen.template_list, config) diff --git a/templates/base/base_config.py b/templates/base/base_sidebar.py similarity index 96% rename from templates/base/base_config.py rename to templates/base/base_sidebar.py index 0d6ba3b4..8fa3f604 100644 --- a/templates/base/base_config.py +++ b/templates/base/base_sidebar.py @@ -7,10 +7,6 @@ "app": ["None", "amp", "apex"], "test": ["None", "amp", "apex"], }, - "device": { - "app": ["cpu", "cuda", "xla"], - "test": ["cpu", "cuda"], - }, "data_path": { "app": {"value": "./"}, "test": {"prefix": "tmp", "suffix": ""}, @@ -96,8 +92,7 @@ def get_configs() -> dict: st.info("Common base training configurations. Those in the parenthesis are used in the code.") # group by streamlit function type - config["amp_mode"] = st.selectbox("AMP mode (amp_mode)", params.amp_mode.app) - config["device"] = st.selectbox("Device to use (device)", params.device.app) + # config["amp_mode"] = st.selectbox("AMP mode (amp_mode)", params.amp_mode.app) config["data_path"] = st.text_input("Dataset path (data_path)", **params.data_path.app) config["filepath"] = st.text_input("Logging file path (filepath)", **params.filepath.app) diff --git a/templates/base/utils.py.jinja b/templates/base/utils.py.jinja index a8c8c702..d36608aa 100644 --- a/templates/base/utils.py.jinja +++ b/templates/base/utils.py.jinja @@ -23,11 +23,6 @@ DEFAULTS = { "type": str, "help": "datasets path ({{ data_path }})", }, - "device": { - "default": "{{ device }}", - "type": torch.device, - "help": "device to use for training / evaluation / testing ({{ device }})", - }, "filepath": { "default": "{{ filepath }}", "type": str, diff --git a/templates/image_classification/fn.py.jinja b/templates/image_classification/fn.py.jinja deleted file mode 100644 index 108701ae..00000000 --- a/templates/image_classification/fn.py.jinja +++ /dev/null @@ -1 +0,0 @@ -{% extends "base/fn.py.jinja" %} diff --git a/templates/image_classification/generate_metadata.py b/templates/image_classification/generate_metadata.py deleted file mode 100644 index 2c13ac1a..00000000 --- a/templates/image_classification/generate_metadata.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -Generates the metadata used in the sidebar of template configurations. -The goal is to make Code-Generator free of pytorch and its related libraries free -when running the app on the streamlit. -""" - -import json - -from torchvision import models - -METADATA_FILEPATH = "./templates/image_classification/metadata.json" - - -def generate_classi_model_names(): - """Generate image classification models from torchvision.""" - names = [] - for k in models.__dict__.keys(): - if ( - not k.startswith("_") - and not k[0].istitle() - and k not in ["utils", "segmentation", "detection", "video", "quantization"] - ): - names.append(k) - obj = {"model_name": names} - with open(METADATA_FILEPATH, "w") as fp: - json.dump(obj, fp) - - -if __name__ == "__main__": - generate_classi_model_names() diff --git a/templates/image_classification/image_classification_config.py b/templates/image_classification/image_classification_config.py deleted file mode 100644 index 959869b7..00000000 --- a/templates/image_classification/image_classification_config.py +++ /dev/null @@ -1,19 +0,0 @@ -import json -import sys - -import streamlit as st - -sys.path.append("./templates/base") - -from base_config import get_configs as base_configs - - -def get_configs() -> dict: - with open("./templates/image_classification/metadata.json", "r") as fp: - model_names = json.load(fp)["model_name"] - - config = base_configs() - with st.beta_expander("Template Configurations"): - config["model_name"] = st.selectbox("Model name (model_name)", model_names, 2) - - return config diff --git a/templates/image_classification/image_classification_sidebar.py b/templates/image_classification/image_classification_sidebar.py new file mode 100644 index 00000000..e226bddb --- /dev/null +++ b/templates/image_classification/image_classification_sidebar.py @@ -0,0 +1,36 @@ +import sys + +import streamlit as st + +sys.path.append("./templates/base") + +from base_sidebar import get_configs as base_configs + + +params = { + "model_name": [ + "alexnet", + "resnet", + "vgg", + "squeezenet", + "inception_v3", + "densenet", + "googlenet", + "mobilenetv2", + "mobilenetv3", + "mnasnet", + "shufflenetv2", + ], + "exp_logging": [ + "wandb", + ], +} + + +def get_configs() -> dict: + config = base_configs() + with st.beta_expander("Template Configurations"): + config["model_name"] = st.selectbox("Model name (model_name)", params["model_name"], 1) + config["exp_logging"] = st.selectbox("Experiment Tracking (exp_logging)", params["exp_logging"]) + + return config diff --git a/templates/image_classification/main.py.jinja b/templates/image_classification/main.py.jinja index e3b3f150..628a1936 100644 --- a/templates/image_classification/main.py.jinja +++ b/templates/image_classification/main.py.jinja @@ -1,6 +1,30 @@ -{% extends "base/main.py.jinja" %} -{% block datasets_and_dataloaders %} - train_dataset, eval_dataset = get_datasets(root=config.data_path) +{% block imports %} +from argparse import ArgumentParser +from datetime import datetime +from pathlib import Path +from typing import Any + +import logging + +import ignite.distributed as idist +from ignite.engine import create_supervised_evaluator, create_supervised_trainer +from ignite.engine.events import Events +from ignite.utils import setup_logger, manual_seed +from ignite.metrics import Accuracy, Loss + +from datasets import get_datasets, get_data_loaders +from utils import log_metrics, get_default_parser, initialize, setup_common_handlers, setup_exp_logging +{% endblock %} + + +{% block run %} +def run(local_rank: int, config: Any, *args: Any, **kwags: Any): + + # ----------------------------- + # datasets and dataloaders + # ----------------------------- + {% block datasets_and_dataloaders %} + train_dataset, eval_dataset = get_datasets(config.data_path) train_dataloader, eval_dataloader = get_data_loaders( train_dataset=train_dataset, eval_dataset=eval_dataset, @@ -8,16 +32,101 @@ eval_batch_size=config.eval_batch_size, num_workers=config.num_workers, ) -{% endblock %} + {% endblock %} -{% block model_optimizer_loss %} - model = idist.auto_model(get_model(config.model_name)) - optimizer = idist.auto_optim(optim.Adam(model.parameters(), lr=config.lr)) - loss_fn = nn.CrossEntropyLoss() -{% endblock %} + # ------------------------------------------ + # model, optimizer, loss function, device + # ------------------------------------------ + {% block model_optimizer_loss %} + device, model, optimizer, loss_fn = initialize(config) + {% endblock %} + + # ---------------------- + # train / eval engine + # ---------------------- + {% block engines %} + train_engine = create_supervised_trainer( + model=model, + optimizer=optimizer, + loss_fn=loss_fn, + device=device, + output_transform=lambda x, y, y_pred, loss: {'train_loss': loss.item()}, + ) + metrics = { + 'eval_accuracy': Accuracy(device=device), + 'eval_loss': Loss(loss_fn=loss_fn, device=device) + } + eval_engine = create_supervised_evaluator( + model=model, + metrics=metrics, + device=device, + ) + {% endblock %} -{% block metrics %} - Accuracy(device=config.device).attach(eval_engine, "eval_accuracy") + # --------------- + # setup logging + # --------------- + {% block loggers %} + name = f"bs{config.train_batch_size}-lr{config.lr}-{optimizer.__class__.__name__}" + now = datetime.now().strftime("%Y%m%d-%X") + train_engine.logger = setup_logger("trainer", level=config.verbose, filepath=config.filepath / f"{name}-{now}.log") + eval_engine.logger = setup_logger("evaluator", level=config.verbose, filepath=config.filepath / f"{name}-{now}.log") + {% endblock %} + + # ----------------------------------------- + # checkpoint and common training handlers + # ----------------------------------------- + {% block eval_ckpt_common_training %} + eval_ckpt_handler = setup_common_handlers( + config=config, + eval_engine=eval_engine, + train_engine=train_engine, + model=model, + optimizer=optimizer + ) + {% endblock %} + + # -------------------------------- + # setup common experiment loggers + # -------------------------------- + {% block exp_logging %} + exp_logger = setup_exp_logging( + config=config, + eval_engine=eval_engine, + train_engine=train_engine, + optimizer=optimizer, + name=name + ) + {% endblock %} + + # ---------------------- + # engines log and run + # ---------------------- + {% block engines_run_and_log %} + {% block log_training_results %} + @train_engine.on(Events.ITERATION_COMPLETED(every=config.log_train)) + def log_training_results(engine): + train_engine.state.metrics = train_engine.state.output + log_metrics(train_engine, "Train", device) + {% endblock %} + + {% block run_eval_engine_and_log %} + @train_engine.on(Events.EPOCH_COMPLETED(every=config.log_eval)) + def run_eval_engine_and_log(engine): + eval_engine.run( + eval_dataloader, + max_epochs=config.eval_max_epochs, + epoch_length=config.eval_epoch_length + ) + log_metrics(eval_engine, "Eval", device) + {% endblock %} + + train_engine.run( + train_dataloader, + max_epochs=config.train_max_epochs, + epoch_length=config.train_epoch_length + ) + {% endblock %} {% endblock %} {% block main_fn %} @@ -46,3 +155,9 @@ def main(): ) as parallel: parallel.run(run, config=config) {% endblock %} + + +{% block entrypoint %} +if __name__ == "__main__": + main() +{% endblock %} diff --git a/templates/image_classification/metadata.json b/templates/image_classification/metadata.json deleted file mode 100644 index fa5a1fd9..00000000 --- a/templates/image_classification/metadata.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "model_name": [ - "alexnet", - "resnet", - "resnet18", - "resnet34", - "resnet50", - "resnet101", - "resnet152", - "resnext50_32x4d", - "resnext101_32x8d", - "wide_resnet50_2", - "wide_resnet101_2", - "vgg", - "vgg11", - "vgg11_bn", - "vgg13", - "vgg13_bn", - "vgg16", - "vgg16_bn", - "vgg19_bn", - "vgg19", - "squeezenet", - "squeezenet1_0", - "squeezenet1_1", - "inception", - "inception_v3", - "densenet", - "densenet121", - "densenet169", - "densenet201", - "densenet161", - "googlenet", - "mobilenetv2", - "mobilenetv3", - "mobilenet", - "mobilenet_v2", - "mobilenet_v3_large", - "mobilenet_v3_small", - "mnasnet", - "mnasnet0_5", - "mnasnet0_75", - "mnasnet1_0", - "mnasnet1_3", - "shufflenetv2", - "shufflenet_v2_x0_5", - "shufflenet_v2_x1_0", - "shufflenet_v2_x1_5", - "shufflenet_v2_x2_0" - ] -} diff --git a/templates/image_classification/utils.py.jinja b/templates/image_classification/utils.py.jinja index 4b6479ee..14da90cb 100644 --- a/templates/image_classification/utils.py.jinja +++ b/templates/image_classification/utils.py.jinja @@ -1 +1,211 @@ -{% extends "base/utils.py.jinja" %} +{% block imports %} +from argparse import ArgumentParser +from typing import Any + +from ignite.contrib.engines import common +from ignite.engine.engine import Engine +import ignite.distributed as idist + +import torch +from torch import optim, nn + +from models import get_model + +{% endblock %} + +{% block get_default_parser %} +DEFAULTS = { + "amp_mode": { + "default": "{{ amp_mode }}", + "type": str, + "help": "automatic mixed precision mode to use: `amp` or `apex` ({{ amp_mode }})", + }, + "train_batch_size": { + "default": {{ train_batch_size }}, + "type": int, + "help": "will be equally divided by number of GPUs if in distributed ({{ train_batch_size }})", + }, + "data_path": { + "default": "{{ data_path }}", + "type": str, + "help": "datasets path ({{ data_path }})", + }, + "filepath": { + "default": "{{ filepath }}", + "type": str, + "help": "logging file path ({{ filepath }})", + }, + "num_workers": { + "default": {{ num_workers }}, + "type": int, + "help": "num_workers for DataLoader ({{ num_workers }})", + }, + "train_max_epochs": { + "default": {{ train_max_epochs }}, + "type": int, + "help": "max_epochs of ignite.Engine.run() for training ({{ train_max_epochs }})", + }, + "eval_max_epochs": { + "default": {{ eval_max_epochs }}, + "type": int, + "help": "max_epochs of ignite.Engine.run() for evaluation ({{ eval_max_epochs }})", + }, + "train_epoch_length": { + "default": {{ train_epoch_length }}, + "type": int, + "help": "epoch_length of ignite.Engine.run() for training ({{ train_epoch_length }})", + }, + "eval_epoch_length": { + "default": {{ eval_epoch_length }}, + "type": int, + "help": "epoch_length of ignite.Engine.run() for evaluation ({{ eval_epoch_length }})", + }, + "lr": { + "default": {{ lr }}, + "type": float, + "help": "learning rate used by torch.optim.* ({{ lr }})", + }, + "log_train": { + "default": {{ log_train }}, + "type": int, + "help": "logging interval of training iteration ({{ log_train }})", + }, + "log_eval": { + "default": {{ log_eval }}, + "type": int, + "help": "logging interval of evaluation epoch ({{ log_eval }})", + }, + "seed": { + "default": {{ seed }}, + "type": int, + "help": "used in ignite.utils.manual_seed() ({{ seed }})", + }, + "eval_batch_size": { + "default": {{ eval_batch_size }}, + "type": int, + "help": "will be equally divided by number of GPUs if in distributed ({{ eval_batch_size }})", + }, + "verbose": { + "action": "store_true", + "help": "use logging.INFO in ignite.utils.setup_logger", + }, + "nproc_per_node": { + "default": {{ nproc_per_node }}, + "type": int, + "help": """number of processes to launch on each node, for GPU training + this is recommended to be set to the number of GPUs in your system + so that each process can be bound to a single GPU ({{ nproc_per_node }})""", + }, + "nnodes": { + "default": {{ nnodes }}, + "type": int, + "help": "number of nodes to use for distributed training ({{ nnodes }})", + }, + "node_rank": { + "default": {{ node_rank }}, + "type": int, + "help": "rank of the node for multi-node distributed training ({{ node_rank }})", + }, + "master_addr": { + "default": {{ master_addr }}, + "type": str, + "help": "master node TCP/IP address for torch native backends ({{ master_addr }})", + }, + "master_port": { + "default": {{ master_port }}, + "type": int, + "help": "master node port for torch native backends {{ master_port }}" + } +} + + +def get_default_parser(): + """Get the default configs for training.""" + parser = ArgumentParser(add_help=False) + + for key, value in DEFAULTS.items(): + parser.add_argument(f"--{key}", **value) + + return parser +{% endblock %} + + +{% block log_metrics %} +def log_metrics(engine: Engine, tag: str, device: torch.device) -> None: + """Log ``engine.state.metrics`` with given ``engine`` + and memory info with given ``device``. + + Args: + engine (Engine): instance of ``Engine`` which metrics to log. + tag (str): a string to add at the start of output. + device (torch.device): current torch.device to log memory info. + """ + max_epochs = len(str(engine.state.max_epochs)) + max_iters = len(str(engine.state.max_iters)) + metrics_format = "{tag} [{epoch:>{max_epochs}}/{iteration:0{max_iters}d}]: {metrics}".format( + tag=tag, + epoch=engine.state.epoch, + max_epochs=max_epochs, + iteration=engine.state.iteration, + max_iters=max_iters, + metrics=engine.state.metrics, + ) + if "cuda" in device.type: + metrics_format += " Memory - {:.2f} MB".format( + torch.cuda.max_memory_allocated(device) / (1024.0 * 1024.0) + ) + engine.logger.info(metrics_format) +{% endblock %} + + +{% block initialize %} +def initialize(config: Any): + device = idist.device() + model = idist.auto_model(get_model(config.model_name)) + optimizer = idist.auto_optim(optim.Adam(model.parameters(), lr=config.lr)) + loss_fn = nn.CrossEntropyLoss().to(device) + + return device, model, optimizer, loss_fn +{% endblock %} + + +{% block eval_ckpt_common_training %} +def setup_common_handlers(config, eval_engine, train_engine, model, optimizer): + eval_ckpt_handler = common.save_best_model_by_val_score( + output_path=config.filepath, + evaluator=eval_engine, + model=model, + metric_name='eval_accuracy', + n_saved=config.n_saved, + trainer=train_engine, + tag='eval', + ) + to_save = { + 'model': model, + 'optimizer': optimizer, + 'train_enginer': train_engine + } + common.setup_common_training_handlers( + trainer=train_engine, + to_save=to_save, + output_path=config.filepath, + with_pbars=False, + clear_cuda_cache=False + ) + return eval_ckpt_handler +{% endblock %} + +{% block exp_logging %} +def setup_exp_logging(config, eval_engine, train_engine, optimizer, name): + {% if exp_logging == "wandb" %} + exp_logger = common.setup_wandb_logging( + trainer=train_engine, + optimizers=optimizer, + evaluators=eval_engine, + config=config, + project=config.project_name, + name=name, + ) + {% endif %} + return exp_logger +{% endblock %} From 79f7a13b808abd1306106f8d317f92ab00a3c020 Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Tue, 16 Mar 2021 18:08:37 +0630 Subject: [PATCH 2/9] rm amp_mode, add exp project_name --- templates/base/base_sidebar.py | 10 ++-------- templates/base/utils.py.jinja | 5 ----- .../image_classification_sidebar.py | 6 +++++- templates/image_classification/main.py.jinja | 6 ------ templates/image_classification/utils.py.jinja | 15 ++++++++++----- 5 files changed, 17 insertions(+), 25 deletions(-) diff --git a/templates/base/base_sidebar.py b/templates/base/base_sidebar.py index 8fa3f604..d03fe0e6 100644 --- a/templates/base/base_sidebar.py +++ b/templates/base/base_sidebar.py @@ -3,10 +3,6 @@ import streamlit as st params = { - "amp_mode": { - "app": ["None", "amp", "apex"], - "test": ["None", "amp", "apex"], - }, "data_path": { "app": {"value": "./"}, "test": {"prefix": "tmp", "suffix": ""}, @@ -16,11 +12,11 @@ "test": {"prefix": "tmp", "suffix": ""}, }, "train_batch_size": { - "app": {"min_value": 1, "value": 1}, + "app": {"min_value": 1, "value": 4}, "test": {"min_value": 1, "max_value": 2}, }, "eval_batch_size": { - "app": {"min_value": 1, "value": 1}, + "app": {"min_value": 1, "value": 4}, "test": {"min_value": 1, "max_value": 2}, }, "num_workers": { @@ -92,8 +88,6 @@ def get_configs() -> dict: st.info("Common base training configurations. Those in the parenthesis are used in the code.") # group by streamlit function type - # config["amp_mode"] = st.selectbox("AMP mode (amp_mode)", params.amp_mode.app) - config["data_path"] = st.text_input("Dataset path (data_path)", **params.data_path.app) config["filepath"] = st.text_input("Logging file path (filepath)", **params.filepath.app) diff --git a/templates/base/utils.py.jinja b/templates/base/utils.py.jinja index d36608aa..ea081f60 100644 --- a/templates/base/utils.py.jinja +++ b/templates/base/utils.py.jinja @@ -8,11 +8,6 @@ import torch {% block get_default_parser %} DEFAULTS = { - "amp_mode": { - "default": "{{ amp_mode }}", - "type": str, - "help": "automatic mixed precision mode to use: `amp` or `apex` ({{ amp_mode }})", - }, "train_batch_size": { "default": {{ train_batch_size }}, "type": int, diff --git a/templates/image_classification/image_classification_sidebar.py b/templates/image_classification/image_classification_sidebar.py index e226bddb..0230baa6 100644 --- a/templates/image_classification/image_classification_sidebar.py +++ b/templates/image_classification/image_classification_sidebar.py @@ -22,6 +22,7 @@ "shufflenetv2", ], "exp_logging": [ + None, "wandb", ], } @@ -29,8 +30,11 @@ def get_configs() -> dict: config = base_configs() + config["project_name"] = None with st.beta_expander("Template Configurations"): config["model_name"] = st.selectbox("Model name (model_name)", params["model_name"], 1) - config["exp_logging"] = st.selectbox("Experiment Tracking (exp_logging)", params["exp_logging"]) + config["exp_logging"] = st.selectbox("Experiment tracking (exp_logging)", params["exp_logging"]) + if config["exp_logging"] is not None: + config["project_name"] = st.text_input("Project name of experiment tracking system (project_name)") return config diff --git a/templates/image_classification/main.py.jinja b/templates/image_classification/main.py.jinja index 628a1936..44c66a7c 100644 --- a/templates/image_classification/main.py.jinja +++ b/templates/image_classification/main.py.jinja @@ -132,12 +132,6 @@ def run(local_rank: int, config: Any, *args: Any, **kwags: Any): {% block main_fn %} def main(): parser = ArgumentParser(parents=[get_default_parser()]) - parser.add_argument( - "--model_name", - default="{{ model_name }}", - type=str, - help="Image classification model name ({{ model_name}})" - ) config = parser.parse_args() manual_seed(config.seed) config.verbose = logging.INFO if config.verbose else logging.WARNING diff --git a/templates/image_classification/utils.py.jinja b/templates/image_classification/utils.py.jinja index 14da90cb..28119acd 100644 --- a/templates/image_classification/utils.py.jinja +++ b/templates/image_classification/utils.py.jinja @@ -15,11 +15,6 @@ from models import get_model {% block get_default_parser %} DEFAULTS = { - "amp_mode": { - "default": "{{ amp_mode }}", - "type": str, - "help": "automatic mixed precision mode to use: `amp` or `apex` ({{ amp_mode }})", - }, "train_batch_size": { "default": {{ train_batch_size }}, "type": int, @@ -115,6 +110,16 @@ DEFAULTS = { "default": {{ master_port }}, "type": int, "help": "master node port for torch native backends {{ master_port }}" + }, + "model_name": { + "default": "{{ model_name }}", + "type": str, + "help": "image classification model name ({{ model_name }})", + }, + "project_name": { + "default": "{{ project_name }}", + "type": str, + "help": "project name of experiment tracking system ({{ project_name }})" } } From cc8900f533c747df5cd3e2466f4f5148faba4804 Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Tue, 16 Mar 2021 18:09:43 +0630 Subject: [PATCH 3/9] epochs and epoch_length 2 --- tests/integration/test_image_classification.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_image_classification.sh b/tests/integration/test_image_classification.sh index f2a25797..7b4b4642 100755 --- a/tests/integration/test_image_classification.sh +++ b/tests/integration/test_image_classification.sh @@ -1,10 +1,10 @@ cd ./tests/dist/image_classification python main.py \ --verbose \ - --train_max_epochs 1 \ - --train_epoch_length 1 \ - --eval_max_epochs 1 \ - --eval_epoch_length 1 \ + --train_max_epochs 2 \ + --train_epoch_length 2 \ + --eval_max_epochs 2 \ + --eval_epoch_length 2 \ --log_train 1 \ --log_eval 1 \ --train_batch_size 4 \ From 4751757ed0bea8ea3d075479b1088ccd10d93dc2 Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Tue, 16 Mar 2021 18:39:54 +0630 Subject: [PATCH 4/9] {template_name}_sidebar -> sidebar, add missing config --- app/streamlit_app.py | 2 +- .../base/{base_sidebar.py => sidebar.py} | 0 templates/image_classification/main.py.jinja | 2 +- ...e_classification_sidebar.py => sidebar.py} | 24 +++++++++++-------- templates/image_classification/utils.py.jinja | 9 ++++++- tests/generate.py | 2 +- tests/unittest/test_image_classification.py | 10 +++++--- 7 files changed, 32 insertions(+), 17 deletions(-) rename templates/base/{base_sidebar.py => sidebar.py} (100%) rename templates/image_classification/{image_classification_sidebar.py => sidebar.py} (62%) diff --git a/app/streamlit_app.py b/app/streamlit_app.py index 7309f031..52b98cd1 100644 --- a/app/streamlit_app.py +++ b/app/streamlit_app.py @@ -47,7 +47,7 @@ def render_code(self, fname="", code="", fold=False): def add_sidebar(self): def config(template_name): - return import_from_file("template_config", f"./templates/{template_name}/{template_name}_sidebar.py") + return import_from_file("template_config", f"./templates/{template_name}/sidebar.py") self.sidebar(self.codegen.template_list, config) diff --git a/templates/base/base_sidebar.py b/templates/base/sidebar.py similarity index 100% rename from templates/base/base_sidebar.py rename to templates/base/sidebar.py diff --git a/templates/image_classification/main.py.jinja b/templates/image_classification/main.py.jinja index 44c66a7c..894ea86a 100644 --- a/templates/image_classification/main.py.jinja +++ b/templates/image_classification/main.py.jinja @@ -89,7 +89,7 @@ def run(local_rank: int, config: Any, *args: Any, **kwags: Any): # -------------------------------- # setup common experiment loggers # -------------------------------- - {% block exp_logging %} + {% block exp_loggers %} exp_logger = setup_exp_logging( config=config, eval_engine=eval_engine, diff --git a/templates/image_classification/image_classification_sidebar.py b/templates/image_classification/sidebar.py similarity index 62% rename from templates/image_classification/image_classification_sidebar.py rename to templates/image_classification/sidebar.py index 0230baa6..0b0e8761 100644 --- a/templates/image_classification/image_classification_sidebar.py +++ b/templates/image_classification/sidebar.py @@ -2,24 +2,24 @@ import streamlit as st -sys.path.append("./templates/base") +sys.path.append("./templates/") -from base_sidebar import get_configs as base_configs +from base.sidebar import get_configs as base_configs params = { "model_name": [ "alexnet", - "resnet", - "vgg", - "squeezenet", + "resnet18", + "vgg16", + "squeezenet1_0", "inception_v3", - "densenet", + "densenet161", "googlenet", - "mobilenetv2", - "mobilenetv3", - "mnasnet", - "shufflenetv2", + "mobilenet_v2", + "mobilenet_v3_small", + "resnext50_32x4d", + "shufflenet_v2_x1_0", ], "exp_logging": [ None, @@ -32,9 +32,13 @@ def get_configs() -> dict: config = base_configs() config["project_name"] = None with st.beta_expander("Template Configurations"): + + # group by streamlit function type config["model_name"] = st.selectbox("Model name (model_name)", params["model_name"], 1) config["exp_logging"] = st.selectbox("Experiment tracking (exp_logging)", params["exp_logging"]) if config["exp_logging"] is not None: config["project_name"] = st.text_input("Project name of experiment tracking system (project_name)") + config["n_saved"] = st.number_input("Number of best models to store", min_value=1, value=2) + return config diff --git a/templates/image_classification/utils.py.jinja b/templates/image_classification/utils.py.jinja index 28119acd..21c218c4 100644 --- a/templates/image_classification/utils.py.jinja +++ b/templates/image_classification/utils.py.jinja @@ -120,6 +120,11 @@ DEFAULTS = { "default": "{{ project_name }}", "type": str, "help": "project name of experiment tracking system ({{ project_name }})" + }, + "n_saved": { + "default": {{ n_saved }}, + "type": int, + "help": "number of best models to store ({{ n_saved }})", } } @@ -211,6 +216,8 @@ def setup_exp_logging(config, eval_engine, train_engine, optimizer, name): project=config.project_name, name=name, ) - {% endif %} return exp_logger + {% else %} + return None + {% endif %} {% endblock %} diff --git a/tests/generate.py b/tests/generate.py index b792e0ea..adf538c9 100644 --- a/tests/generate.py +++ b/tests/generate.py @@ -13,7 +13,7 @@ def generate(): if p.is_dir(): sys.path.append(f"./templates/{p.stem}") target_dir = "./tests/dist" - configs = import_from_file("template_config", f"./templates/{p.stem}/{p.stem}_config.py").get_configs() + configs = import_from_file("template_config", f"./templates/{p.stem}/sidebar.py").get_configs() code_gen = CodeGenerator(target_dir=target_dir) [*code_gen.render_templates(p.stem, configs)] code_gen.create_target_template_dir(p.stem) diff --git a/tests/unittest/test_image_classification.py b/tests/unittest/test_image_classification.py index 4e4d369c..3d22657e 100644 --- a/tests/unittest/test_image_classification.py +++ b/tests/unittest/test_image_classification.py @@ -22,7 +22,7 @@ class ImageClassiTester(unittest.TestCase): # test datasets.py @settings(deadline=None, derandomize=True) - @given(st.integers(min_value=1, max_value=128), st.integers(min_value=0, max_value=2)) + @given(st.integers(min_value=1, max_value=4), st.integers(min_value=0, max_value=2)) def test_datasets(self, train_batch_size, num_workers): train_dataset, eval_dataset = get_datasets("/tmp/cifar10") @@ -75,11 +75,15 @@ def test_get_default_parser(self): # test log_metrics of utils.py def test_log_metrics(self): + device = idist.device() engine = Engine(lambda engine, batch: None) engine.run(list(range(100)), max_epochs=2) with self.assertLogs() as log: - log_metrics(engine, "train", "cpu") - self.assertEqual(log.output[0], "INFO:ignite.engine.engine.Engine:train [2/0200]: {}") + log_metrics(engine, "train", device) + if device.type == "cpu": + self.assertEqual(log.output[0], "INFO:ignite.engine.engine.Engine:train [2/0200]: {}") + else: + self.assertEqual(log.output[0], "INFO:ignite.engine.engine.Engine:train [2/0200]: {} Memory - 0.00 MB") if __name__ == "__main__": From f9ab2cc9ed084d151a0685b16213be72a79f5e48 Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Tue, 16 Mar 2021 18:53:05 +0630 Subject: [PATCH 5/9] integration data_path, num_workers to 1 --- tests/integration/test_image_classification.sh | 1 + tests/unittest/test_image_classification.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_image_classification.sh b/tests/integration/test_image_classification.sh index 7b4b4642..3700c975 100755 --- a/tests/integration/test_image_classification.sh +++ b/tests/integration/test_image_classification.sh @@ -9,3 +9,4 @@ python main.py \ --log_eval 1 \ --train_batch_size 4 \ --eval_batch_size 4 \ + --data_path /tmp/cifar10 diff --git a/tests/unittest/test_image_classification.py b/tests/unittest/test_image_classification.py index 3d22657e..4a1fb18f 100644 --- a/tests/unittest/test_image_classification.py +++ b/tests/unittest/test_image_classification.py @@ -22,7 +22,7 @@ class ImageClassiTester(unittest.TestCase): # test datasets.py @settings(deadline=None, derandomize=True) - @given(st.integers(min_value=1, max_value=4), st.integers(min_value=0, max_value=2)) + @given(st.integers(min_value=1, max_value=4), st.integers(min_value=0, max_value=1)) def test_datasets(self, train_batch_size, num_workers): train_dataset, eval_dataset = get_datasets("/tmp/cifar10") From 264b1924a6a45d27461ad5d43901e9693bd850fc Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Tue, 16 Mar 2021 18:53:52 +0630 Subject: [PATCH 6/9] install streamlit in ci --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d03d5e56..d203c0d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,7 @@ jobs: restore-keys: | ${{ steps.get-date.outputs.date }}-${{ runner.os }}-${{ matrix.python-version }}- + - run: pip install -r requirements.txt --progress-bar off - run: pip install -r requirements-dev.txt -f https://download.pytorch.org/whl/cpu/torch_stable.html --progress-bar off - run: python -m torch.utils.collect_env - run: bash .github/run_test.sh generate From aaa7918154b324a3643cb21bcfcc2a60dc37ca93 Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Tue, 16 Mar 2021 18:55:01 +0630 Subject: [PATCH 7/9] add (n_saved) in sidebar --- templates/image_classification/sidebar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/image_classification/sidebar.py b/templates/image_classification/sidebar.py index 0b0e8761..296c6c42 100644 --- a/templates/image_classification/sidebar.py +++ b/templates/image_classification/sidebar.py @@ -39,6 +39,6 @@ def get_configs() -> dict: if config["exp_logging"] is not None: config["project_name"] = st.text_input("Project name of experiment tracking system (project_name)") - config["n_saved"] = st.number_input("Number of best models to store", min_value=1, value=2) + config["n_saved"] = st.number_input("Number of best models to store (n_saved)", min_value=1, value=2) return config From fb145317bccec88e4b76771540e95f58c7470447 Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Tue, 16 Mar 2021 22:02:00 +0630 Subject: [PATCH 8/9] add test for functions in utils.py --- app/streamlit_app.py | 2 +- templates/image_classification/sidebar.py | 11 ++-- templates/image_classification/utils.py.jinja | 25 +++++++- tests/unittest/common_utils.py | 16 ++++++ tests/unittest/test_image_classification.py | 57 +++++++++++++++++-- 5 files changed, 97 insertions(+), 14 deletions(-) create mode 100644 tests/unittest/common_utils.py diff --git a/app/streamlit_app.py b/app/streamlit_app.py index aa69aa5b..43512802 100644 --- a/app/streamlit_app.py +++ b/app/streamlit_app.py @@ -1,6 +1,6 @@ import shutil -from pathlib import Path from datetime import datetime +from pathlib import Path import streamlit as st from codegen import CodeGenerator diff --git a/templates/image_classification/sidebar.py b/templates/image_classification/sidebar.py index 296c6c42..273068f7 100644 --- a/templates/image_classification/sidebar.py +++ b/templates/image_classification/sidebar.py @@ -6,7 +6,6 @@ from base.sidebar import get_configs as base_configs - params = { "model_name": [ "alexnet", @@ -22,7 +21,6 @@ "shufflenet_v2_x1_0", ], "exp_logging": [ - None, "wandb", ], } @@ -36,9 +34,12 @@ def get_configs() -> dict: # group by streamlit function type config["model_name"] = st.selectbox("Model name (model_name)", params["model_name"], 1) config["exp_logging"] = st.selectbox("Experiment tracking (exp_logging)", params["exp_logging"]) - if config["exp_logging"] is not None: - config["project_name"] = st.text_input("Project name of experiment tracking system (project_name)") - + config["project_name"] = st.text_input( + "Project name of experiment tracking system (project_name)", "code-generator" + ) config["n_saved"] = st.number_input("Number of best models to store (n_saved)", min_value=1, value=2) + config["save_every_iters"] = st.number_input( + "Model saving interval (save_every_iters)", min_value=10, value=1000 + ) return config diff --git a/templates/image_classification/utils.py.jinja b/templates/image_classification/utils.py.jinja index 21c218c4..c43f3d0a 100644 --- a/templates/image_classification/utils.py.jinja +++ b/templates/image_classification/utils.py.jinja @@ -1,6 +1,6 @@ {% block imports %} from argparse import ArgumentParser -from typing import Any +from typing import Any, Optional from ignite.contrib.engines import common from ignite.engine.engine import Engine @@ -8,6 +8,7 @@ import ignite.distributed as idist import torch from torch import optim, nn +from torch.optim.optimizer import Optimizer from models import get_model @@ -125,6 +126,11 @@ DEFAULTS = { "default": {{ n_saved }}, "type": int, "help": "number of best models to store ({{ n_saved }})", + }, + "save_every_iters": { + "default": {{ save_every_iters }}, + "type": int, + "help": "model saving interval ({{ save_every_iters }})", } } @@ -180,7 +186,13 @@ def initialize(config: Any): {% block eval_ckpt_common_training %} -def setup_common_handlers(config, eval_engine, train_engine, model, optimizer): +def setup_common_handlers( + config: Any, + eval_engine: Engine, + train_engine: Engine, + model: nn.Module, + optimizer: Optimizer +): eval_ckpt_handler = common.save_best_model_by_val_score( output_path=config.filepath, evaluator=eval_engine, @@ -198,6 +210,7 @@ def setup_common_handlers(config, eval_engine, train_engine, model, optimizer): common.setup_common_training_handlers( trainer=train_engine, to_save=to_save, + save_every_iters=config.save_every_iters, output_path=config.filepath, with_pbars=False, clear_cuda_cache=False @@ -206,7 +219,13 @@ def setup_common_handlers(config, eval_engine, train_engine, model, optimizer): {% endblock %} {% block exp_logging %} -def setup_exp_logging(config, eval_engine, train_engine, optimizer, name): +def setup_exp_logging( + train_engine: Engine, + config: Any, + eval_engine: Optional[Engine] = None, + optimizer: Optional[Optimizer] = None, + name: Optional[str] = None +): {% if exp_logging == "wandb" %} exp_logger = common.setup_wandb_logging( trainer=train_engine, diff --git a/tests/unittest/common_utils.py b/tests/unittest/common_utils.py new file mode 100644 index 00000000..94e19844 --- /dev/null +++ b/tests/unittest/common_utils.py @@ -0,0 +1,16 @@ +import contextlib +import os +import shutil +import tempfile + + +@contextlib.contextmanager +def get_tmp_dir(src=None, **kwargs): + tmp_dir = tempfile.mkdtemp(**kwargs) + if src is not None: + os.rmdir(tmp_dir) + shutil.copytree(src, tmp_dir) + try: + yield tmp_dir + finally: + shutil.rmtree(tmp_dir) diff --git a/tests/unittest/test_image_classification.py b/tests/unittest/test_image_classification.py index 4a1fb18f..bba194bd 100644 --- a/tests/unittest/test_image_classification.py +++ b/tests/unittest/test_image_classification.py @@ -1,12 +1,16 @@ +import os import sys import unittest -from argparse import ArgumentParser +from argparse import ArgumentParser, Namespace import ignite.distributed as idist +import torch +from common_utils import get_tmp_dir from hypothesis import given, settings from hypothesis import strategies as st from ignite.engine import Engine -from torch.nn import Module +from ignite.handlers import Checkpoint +from torch import nn, optim from torch.utils.data import DataLoader, Dataset from torch.utils.data._utils.collate import default_collate from torch.utils.data.sampler import RandomSampler, SequentialSampler @@ -15,11 +19,16 @@ from datasets import get_data_loaders, get_datasets from models import get_model -from utils import get_default_parser, log_metrics +from utils import ( + get_default_parser, + initialize, + log_metrics, + setup_common_handlers, + setup_exp_logging, +) class ImageClassiTester(unittest.TestCase): - # test datasets.py @settings(deadline=None, derandomize=True) @given(st.integers(min_value=1, max_value=4), st.integers(min_value=0, max_value=1)) @@ -64,7 +73,7 @@ def test_datasets(self, train_batch_size, num_workers): @given(st.sampled_from(["squeezenet1_0", "squeezenet1_1"])) def test_models(self, name): model = get_model(name) - self.assertIsInstance(model, Module) + self.assertIsInstance(model, nn.Module) self.assertEqual(model.num_classes, 10) # test get_default_parser of utils.py @@ -85,6 +94,44 @@ def test_log_metrics(self): else: self.assertEqual(log.output[0], "INFO:ignite.engine.engine.Engine:train [2/0200]: {} Memory - 0.00 MB") + # test initialize of utils.py + def test_initialize(self): + config = Namespace(model_name="alexnet", lr=0.01) + device, model, optimizer, loss_fn = initialize(config) + self.assertIsInstance(device, torch.device) + self.assertIsInstance(model, nn.Module) + self.assertIsInstance(optimizer, optim.Optimizer) + self.assertIsInstance(loss_fn, nn.Module) + + # test setup_common_handlers of utils.py + def test_setup_common_handlers(self): + data = [1, 2, 3] + max_epochs = 3 + model = nn.Linear(1, 1) + optimizer = optim.Adam(model.parameters()) + engine = Engine(lambda e, b: b) + engine.state.metrics = {"eval_accuracy": 1} + with get_tmp_dir() as tmpdir: + config = Namespace(filepath=tmpdir, n_saved=2, save_every_iters=1) + handler = setup_common_handlers( + config=config, + eval_engine=engine, + train_engine=engine, + model=model, + optimizer=optimizer, + ) + engine.run(data, max_epochs=max_epochs) + self.assertIsInstance(handler, Checkpoint) + self.assertTrue(os.path.isfile(f"{tmpdir}/training_checkpoint_{len(data) * max_epochs}.pt")) + self.assertTrue(handler.last_checkpoint, f"{tmpdir}/best_model_3_eval_eval_accuracy=1.0000.pt") + + # test setup_exp_logging of utils.py + def test_setup_exp_logging(self): + engine = Engine(lambda e, b: b) + config = Namespace(project_name="abc") + with self.assertRaisesRegex(RuntimeError, r"This contrib module requires wandb to be installed."): + setup_exp_logging(train_engine=engine, config=config) + if __name__ == "__main__": unittest.main() From 52463e4a6df07f1107d5c14ac3c52430f3c100a5 Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Tue, 16 Mar 2021 22:15:54 +0630 Subject: [PATCH 9/9] exp logging default to None --- templates/image_classification/sidebar.py | 8 +++++--- tests/unittest/test_image_classification.py | 12 +++--------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/templates/image_classification/sidebar.py b/templates/image_classification/sidebar.py index 273068f7..a7841883 100644 --- a/templates/image_classification/sidebar.py +++ b/templates/image_classification/sidebar.py @@ -21,6 +21,7 @@ "shufflenet_v2_x1_0", ], "exp_logging": [ + None, "wandb", ], } @@ -34,9 +35,10 @@ def get_configs() -> dict: # group by streamlit function type config["model_name"] = st.selectbox("Model name (model_name)", params["model_name"], 1) config["exp_logging"] = st.selectbox("Experiment tracking (exp_logging)", params["exp_logging"]) - config["project_name"] = st.text_input( - "Project name of experiment tracking system (project_name)", "code-generator" - ) + if config["exp_logging"] is not None: + config["project_name"] = st.text_input( + "Project name of experiment tracking system (project_name)", "code-generator" + ) config["n_saved"] = st.number_input("Number of best models to store (n_saved)", min_value=1, value=2) config["save_every_iters"] = st.number_input( "Model saving interval (save_every_iters)", min_value=10, value=1000 diff --git a/tests/unittest/test_image_classification.py b/tests/unittest/test_image_classification.py index bba194bd..19cba75d 100644 --- a/tests/unittest/test_image_classification.py +++ b/tests/unittest/test_image_classification.py @@ -88,11 +88,8 @@ def test_log_metrics(self): engine = Engine(lambda engine, batch: None) engine.run(list(range(100)), max_epochs=2) with self.assertLogs() as log: - log_metrics(engine, "train", device) - if device.type == "cpu": - self.assertEqual(log.output[0], "INFO:ignite.engine.engine.Engine:train [2/0200]: {}") - else: - self.assertEqual(log.output[0], "INFO:ignite.engine.engine.Engine:train [2/0200]: {} Memory - 0.00 MB") + log_metrics(engine, "train", torch.device("cpu")) + self.assertEqual(log.output[0], "INFO:ignite.engine.engine.Engine:train [2/0200]: {}") # test initialize of utils.py def test_initialize(self): @@ -127,10 +124,7 @@ def test_setup_common_handlers(self): # test setup_exp_logging of utils.py def test_setup_exp_logging(self): - engine = Engine(lambda e, b: b) - config = Namespace(project_name="abc") - with self.assertRaisesRegex(RuntimeError, r"This contrib module requires wandb to be installed."): - setup_exp_logging(train_engine=engine, config=config) + self.assertIsNone(setup_exp_logging(train_engine=None, config=None)) if __name__ == "__main__":