From 0f12064307cf3254b4d45640cce8574e45c686dd Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Wed, 10 Mar 2021 22:42:10 +0630 Subject: [PATCH 1/5] feat(download): add an option to archive and download --- app/codegen.py | 18 ++++++++++++------ app/streamlit_app.py | 26 +++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/app/codegen.py b/app/codegen.py index 0c09e4da..bc984b80 100644 --- a/app/codegen.py +++ b/app/codegen.py @@ -3,11 +3,15 @@ from pathlib import Path from jinja2 import Environment, FileSystemLoader +import shutil + class CodeGenerator: def __init__(self, templates_dir=None): templates_dir = templates_dir or "./templates" - self.template_list = [p.stem for p in Path(templates_dir).iterdir() if p.is_dir()] + self.template_list = [ + p.stem for p in Path(templates_dir).iterdir() if p.is_dir() + ] self.env = Environment( loader=FileSystemLoader(templates_dir), trim_blocks=True, lstrip_blocks=True ) @@ -33,9 +37,11 @@ def render_templates(self, template_name: str, config: dict): def generate(self, template_name: str, fname: str, code: str) -> None: """Generates `fname` with content `code` in `path`. """ - path = Path(f"dist/{template_name}") - path.mkdir(parents=True, exist_ok=True) - (path / fname).write_text(code) + self.path = Path(f"./dist/{template_name}") + self.path.mkdir(parents=True, exist_ok=True) + (self.path / fname).write_text(code) - def make_archive(self): - raise NotImplementedError + def make_archive(self, format): + return shutil.make_archive( + base_name=str(self.path), format=format, base_dir=self.path + ) diff --git a/app/streamlit_app.py b/app/streamlit_app.py index 06a08b88..109fe6c1 100644 --- a/app/streamlit_app.py +++ b/app/streamlit_app.py @@ -1,3 +1,6 @@ +import shutil +from pathlib import Path + import streamlit as st from codegen import CodeGenerator @@ -45,7 +48,9 @@ def render_code(self, fname="", code="", fold=False): st.code(code) def add_sidebar(self): - config = lambda template_name: import_from_file("template_config", f"./templates/{template_name}/config.py") + config = lambda template_name: import_from_file( + "template_config", f"./templates/{template_name}/config.py" + ) self.sidebar(self.codegen.template_list, config) def add_content(self): @@ -60,9 +65,28 @@ def add_content(self): for fname, code in content: self.render_code(fname, code, fold) + def add_download(self): + st.markdown("") + format = st.selectbox( + "Archive format", [name for name, _ in shutil.get_archive_formats()] + ) + # temporary hack until streamlit has official download option + # https://github.com/streamlit/streamlit/issues/400 + # https://github.com/streamlit/streamlit/issues/400#issuecomment-648580840 + if st.button("Generate an archive"): + archive_fname = self.codegen.make_archive(format) + # this is where streamlit serves static files + # ~/site-packages/streamlit/static/static/ + dist_path = Path(st.__path__[0]) / "static/static/dist" + if not dist_path.is_dir(): + dist_path.mkdir() + shutil.copy(archive_fname, dist_path) + st.markdown(f"Download link : [{archive_fname}](./static/{archive_fname})") + def run(self): self.add_sidebar() self.add_content() + self.add_download() def main(): From f28ab7e2c3c2f5070ab3e571ff0802ed997b1d5c Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Wed, 10 Mar 2021 22:45:34 +0630 Subject: [PATCH 2/5] fix utils.py.jinja, use replace instead of strip --- app/codegen.py | 2 +- templates/base/utils.py.jinja | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/codegen.py b/app/codegen.py index bc984b80..bde4db95 100644 --- a/app/codegen.py +++ b/app/codegen.py @@ -30,7 +30,7 @@ def render_templates(self, template_name: str, config: dict): # Render template code = template.render(**config) # Write python file - fname = fname.strip(f"{template_name}/").strip(".jinja") + fname = fname.replace(f"{template_name}/", "").replace(".jinja", "") self.generate(template_name, fname, code) yield fname, code diff --git a/templates/base/utils.py.jinja b/templates/base/utils.py.jinja index df705434..0929b854 100644 --- a/templates/base/utils.py.jinja +++ b/templates/base/utils.py.jinja @@ -90,7 +90,7 @@ DEFAULTS = { "help": "rank of the node for multi-node distributed training ({{ node_rank }})", }, "master_addr": { - "default": {{ master_addr|safe }}, + "default": {{ master_addr }}, "type": str, "help": "master node TCP/IP address for torch native backends ({{ master_addr }})", }, From e5db6f30429b49737dab5b683ce3e4ef75fa2795 Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Wed, 10 Mar 2021 22:56:32 +0630 Subject: [PATCH 3/5] use radio instead of selectbox for mobile --- app/streamlit_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/streamlit_app.py b/app/streamlit_app.py index 109fe6c1..8faf279d 100644 --- a/app/streamlit_app.py +++ b/app/streamlit_app.py @@ -67,7 +67,7 @@ def add_content(self): def add_download(self): st.markdown("") - format = st.selectbox( + format = st.radio( "Archive format", [name for name, _ in shutil.get_archive_formats()] ) # temporary hack until streamlit has official download option From 633411874f9d1acd22848674e9643b31b4235e94 Mon Sep 17 00:00:00 2001 From: Jeff Yang Date: Thu, 11 Mar 2021 12:04:23 +0630 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Taras Savchyn <30748114+trsvchn@users.noreply.github.com> --- app/streamlit_app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/streamlit_app.py b/app/streamlit_app.py index 8faf279d..2f92b1ea 100644 --- a/app/streamlit_app.py +++ b/app/streamlit_app.py @@ -68,7 +68,8 @@ def add_content(self): def add_download(self): st.markdown("") format = st.radio( - "Archive format", [name for name, _ in shutil.get_archive_formats()] + "Archive format", [name for name, _ in sorted(shutil.get_archive_formats(), key=lambda x: x[0], reverse=True)] + ) # temporary hack until streamlit has official download option # https://github.com/streamlit/streamlit/issues/400 @@ -81,7 +82,7 @@ def add_download(self): if not dist_path.is_dir(): dist_path.mkdir() shutil.copy(archive_fname, dist_path) - st.markdown(f"Download link : [{archive_fname}](./static/{archive_fname})") + st.success(f"Download link : [{archive_fname}](./static/{archive_fname})") def run(self): self.add_sidebar() From 63a2b42d42903c5d25501309c575f49b1f7a65ed Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Thu, 11 Mar 2021 12:09:25 +0630 Subject: [PATCH 5/5] format to format_, black 120, isorted --- app/codegen.py | 22 +++++--------- app/streamlit_app.py | 13 ++++----- templates/base/config.py | 62 +++++++++++----------------------------- 3 files changed, 28 insertions(+), 69 deletions(-) diff --git a/app/codegen.py b/app/codegen.py index bde4db95..2dd70f9e 100644 --- a/app/codegen.py +++ b/app/codegen.py @@ -1,28 +1,22 @@ """Code Generator base module. """ +import shutil from pathlib import Path -from jinja2 import Environment, FileSystemLoader -import shutil +from jinja2 import Environment, FileSystemLoader class CodeGenerator: def __init__(self, templates_dir=None): templates_dir = templates_dir or "./templates" - self.template_list = [ - p.stem for p in Path(templates_dir).iterdir() if p.is_dir() - ] - self.env = Environment( - loader=FileSystemLoader(templates_dir), trim_blocks=True, lstrip_blocks=True - ) + self.template_list = [p.stem for p in Path(templates_dir).iterdir() if p.is_dir()] + self.env = Environment(loader=FileSystemLoader(templates_dir), trim_blocks=True, lstrip_blocks=True) def render_templates(self, template_name: str, config: dict): """Renders all the templates from template folder for the given config. """ file_template_list = ( - template - for template in self.env.list_templates(".jinja") - if template.startswith(template_name) + template for template in self.env.list_templates(".jinja") if template.startswith(template_name) ) for fname in file_template_list: # Get template @@ -41,7 +35,5 @@ def generate(self, template_name: str, fname: str, code: str) -> None: self.path.mkdir(parents=True, exist_ok=True) (self.path / fname).write_text(code) - def make_archive(self, format): - return shutil.make_archive( - base_name=str(self.path), format=format, base_dir=self.path - ) + def make_archive(self, format_): + return shutil.make_archive(base_name=str(self.path), format=format_, base_dir=self.path) diff --git a/app/streamlit_app.py b/app/streamlit_app.py index 2f92b1ea..09c27dc2 100644 --- a/app/streamlit_app.py +++ b/app/streamlit_app.py @@ -2,7 +2,6 @@ from pathlib import Path import streamlit as st - from codegen import CodeGenerator from utils import import_from_file @@ -48,9 +47,7 @@ def render_code(self, fname="", code="", fold=False): st.code(code) def add_sidebar(self): - config = lambda template_name: import_from_file( - "template_config", f"./templates/{template_name}/config.py" - ) + config = lambda template_name: import_from_file("template_config", f"./templates/{template_name}/config.py") self.sidebar(self.codegen.template_list, config) def add_content(self): @@ -67,15 +64,15 @@ def add_content(self): def add_download(self): st.markdown("") - format = st.radio( - "Archive format", [name for name, _ in sorted(shutil.get_archive_formats(), key=lambda x: x[0], reverse=True)] - + format_ = st.radio( + "Archive format", + [name for name, _ in sorted(shutil.get_archive_formats(), key=lambda x: x[0], reverse=True)], ) # temporary hack until streamlit has official download option # https://github.com/streamlit/streamlit/issues/400 # https://github.com/streamlit/streamlit/issues/400#issuecomment-648580840 if st.button("Generate an archive"): - archive_fname = self.codegen.make_archive(format) + archive_fname = self.codegen.make_archive(format_) # this is where streamlit serves static files # ~/site-packages/streamlit/static/static/ dist_path = Path(st.__path__[0]) / "static/static/dist" diff --git a/templates/base/config.py b/templates/base/config.py index add0f930..e0938f77 100644 --- a/templates/base/config.py +++ b/templates/base/config.py @@ -4,74 +4,44 @@ def get_configs() -> dict: config = {} with st.beta_expander("Training Configurations"): - st.info( - "Common base training configurations. Those in the parenthesis are used in the code." - ) + st.info("Common base training configurations. Those in the parenthesis are used in the code.") # group by streamlit function type - config["amp_mode"] = st.selectbox( - "AMP mode (amp_mode)", ("None", "amp", "apex") - ) - config["device"] = st.selectbox( - "Device to use (device)", ("cpu", "cuda", "xla") - ) + config["amp_mode"] = st.selectbox("AMP mode (amp_mode)", ("None", "amp", "apex")) + config["device"] = st.selectbox("Device to use (device)", ("cpu", "cuda", "xla")) config["data_path"] = st.text_input("Dataset path (data_path)", value="./") - config["filepath"] = st.text_input( - "Logging file path (filepath)", value="./logs" - ) + config["filepath"] = st.text_input("Logging file path (filepath)", value="./logs") - config["train_batch_size"] = st.number_input( - "Train batch size (train_batch_size)", min_value=1, value=1 - ) - config["eval_batch_size"] = st.number_input( - "Eval batch size (eval_batch_size)", min_value=1, value=1 - ) - config["num_workers"] = st.number_input( - "Number of workers (num_workers)", min_value=0, value=2 - ) - config["max_epochs"] = st.number_input( - "Maximum epochs to train (max_epochs)", min_value=1, value=2 - ) + config["train_batch_size"] = st.number_input("Train batch size (train_batch_size)", min_value=1, value=1) + config["eval_batch_size"] = st.number_input("Eval batch size (eval_batch_size)", min_value=1, value=1) + config["num_workers"] = st.number_input("Number of workers (num_workers)", min_value=0, value=2) + config["max_epochs"] = st.number_input("Maximum epochs to train (max_epochs)", min_value=1, value=2) config["lr"] = st.number_input( - "Learning rate used by torch.optim.* (lr)", - min_value=0.0, - value=1e-3, - format="%e", + "Learning rate used by torch.optim.* (lr)", min_value=0.0, value=1e-3, format="%e", ) config["log_train"] = st.number_input( "Logging interval of training iterations (log_train)", min_value=0, value=50 ) - config["log_eval"] = st.number_input( - "Logging interval of evaluation epoch (log_eval)", min_value=0, value=1 - ) - config["seed"] = st.number_input( - "Seed used in ignite.utils.manual_seed() (seed)", min_value=0, value=666 - ) + config["log_eval"] = st.number_input("Logging interval of evaluation epoch (log_eval)", min_value=0, value=1) + config["seed"] = st.number_input("Seed used in ignite.utils.manual_seed() (seed)", min_value=0, value=666) if st.checkbox("Use distributed training"): config["nproc_per_node"] = st.number_input( - "Number of processes to launch on each node (nproc_per_node)", - min_value=1, - ) - config["nnodes"] = st.number_input( - "Number of nodes to use for distributed training (nnodes)", min_value=1, + "Number of processes to launch on each node (nproc_per_node)", min_value=1, ) + config["nnodes"] = st.number_input("Number of nodes to use for distributed training (nnodes)", min_value=1,) if config["nnodes"] > 1: st.info( "The following options are only supported by torch.distributed, namely 'gloo' and 'nccl' backends." " For other backends, please specify spawn_kwargs in main.py" ) config["node_rank"] = st.number_input( - "Rank of the node for multi-node distributed training (node_rank)", - min_value=0, + "Rank of the node for multi-node distributed training (node_rank)", min_value=0, ) if config["node_rank"] > (config["nnodes"] - 1): - st.error( - f"node_rank should be between 0 and {config['nnodes'] - 1}" - ) + st.error(f"node_rank should be between 0 and {config['nnodes'] - 1}") config["master_addr"] = st.text_input( - "Master node TCP/IP address for torch native backends (master_addr)", - "'127.0.0.1'", + "Master node TCP/IP address for torch native backends (master_addr)", "'127.0.0.1'", ) st.warning("Please include single quote in master_addr.") config["master_port"] = st.text_input(