From 945427767d8163bdf8900c57ebbd7146052e9bd7 Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Mon, 14 Aug 2023 11:59:27 +0530 Subject: [PATCH 01/12] Added Manual dataset download --- functions/colab.js | 17 +++++++++++------ .../template-vision-segmentation/README.md | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index bb35246c..e368a6a1 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -42,6 +42,16 @@ exports.handler = async function (event, _) { .map((v) => v[0].toUpperCase() + v.slice(1)) .join(' ') // notebook cell structure + let commands = [ + `!wget ${zipRes}\n`, + `!unzip ${template}.zip\n`, + '!pip install -r requirements.txt\n', + '!python main.py config.yaml\n' + ] + + if(title === 'Template Vision Segmentation'){ + commands.splice(3, 0, '!python -c "from data import download_datasets; download_datasets(\'./\')"') + } const nb = { nbformat: 4, nbformat_minor: 0, @@ -68,12 +78,7 @@ exports.handler = async function (event, _) { metadata: {}, execution_count: null, outputs: [], - source: [ - `!wget ${zipRes}\n`, - `!unzip ${template}.zip\n`, - '!pip install -r requirements.txt\n', - '!python main.py config.yaml\n' - ] + source: commands, } ] } diff --git a/src/templates/template-vision-segmentation/README.md b/src/templates/template-vision-segmentation/README.md index 2ded1c80..1b209cf9 100644 --- a/src/templates/template-vision-segmentation/README.md +++ b/src/templates/template-vision-segmentation/README.md @@ -4,4 +4,19 @@ This is the segmentation template by Code-Generator using `deeplabv3_resnet101` and `cifar10` dataset from TorchVision and training is powered by PyTorch and PyTorch-Ignite. +**Note:** +The dataset used in this is quite substantial, with a size of several GBs and automatically downloading it can be seen as an unexpected behaviour. To prevent unexpected behavior or excessive bandwidth usage, the automatic downloading of the dataset has been disabled by default. + +To download the dataset: +```python +python -c "from data import download_datasets; download_datasets('/path/to/data')" +``` + +or + +```py +from data import download_datasets +download_datasets('/path/to/data') +``` + #::= from_template_common ::# From a8295c45fb53856cd851f3bc5f9badeb10b8aa1f Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Mon, 14 Aug 2023 12:13:59 +0530 Subject: [PATCH 02/12] Modified formatting --- functions/colab.js | 10 +++++++--- src/templates/template-vision-segmentation/README.md | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index e368a6a1..fe9ef6f3 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -49,8 +49,12 @@ exports.handler = async function (event, _) { '!python main.py config.yaml\n' ] - if(title === 'Template Vision Segmentation'){ - commands.splice(3, 0, '!python -c "from data import download_datasets; download_datasets(\'./\')"') + if (title === 'Template Vision Segmentation') { + commands.splice( + 3, + 0, + '!python -c "from data import download_datasets; download_datasets(\'./\')"\n' + ) } const nb = { nbformat: 4, @@ -78,7 +82,7 @@ exports.handler = async function (event, _) { metadata: {}, execution_count: null, outputs: [], - source: commands, + source: commands } ] } diff --git a/src/templates/template-vision-segmentation/README.md b/src/templates/template-vision-segmentation/README.md index 1b209cf9..896c2a01 100644 --- a/src/templates/template-vision-segmentation/README.md +++ b/src/templates/template-vision-segmentation/README.md @@ -8,6 +8,7 @@ This is the segmentation template by Code-Generator using `deeplabv3_resnet101` The dataset used in this is quite substantial, with a size of several GBs and automatically downloading it can be seen as an unexpected behaviour. To prevent unexpected behavior or excessive bandwidth usage, the automatic downloading of the dataset has been disabled by default. To download the dataset: + ```python python -c "from data import download_datasets; download_datasets('/path/to/data')" ``` From 793ecec112c1f1a667f63e234a6f6ac610d0b01f Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Tue, 15 Aug 2023 16:17:54 +0530 Subject: [PATCH 03/12] Added Specific Commands in different Cells --- functions/colab.js | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index fe9ef6f3..341188ee 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -42,20 +42,22 @@ exports.handler = async function (event, _) { .map((v) => v[0].toUpperCase() + v.slice(1)) .join(' ') // notebook cell structure - let commands = [ + let common_commands = [ `!wget ${zipRes}\n`, `!unzip ${template}.zip\n`, - '!pip install -r requirements.txt\n', - '!python main.py config.yaml\n' + '!pip install -r requirements.txt' ] + let specific_commands = [] + if (title === 'Template Vision Segmentation') { - commands.splice( - 3, - 0, - '!python -c "from data import download_datasets; download_datasets(\'./\')"\n' + specific_commands.push( + '!python -c "from data import download_datasets; download_datasets(\'./\')"' ) } + + let execution_command = ['!python main.py config.yaml'] + const nb = { nbformat: 4, nbformat_minor: 0, @@ -82,7 +84,21 @@ exports.handler = async function (event, _) { metadata: {}, execution_count: null, outputs: [], - source: commands + source: common_commands + }, + { + cell_type: 'code', + metadata: {}, + execution_count: null, + outputs: [], + source: specific_commands + }, + { + cell_type: 'code', + metadata: {}, + execution_count: null, + outputs: [], + source: execution_command } ] } From 9e04e68f57c1ac9b578a2a3926ae781d0096434d Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Tue, 15 Aug 2023 16:53:24 +0530 Subject: [PATCH 04/12] Added a different list with all the notebook commands --- functions/colab.js | 86 +++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index 341188ee..f2e87954 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -42,11 +42,17 @@ exports.handler = async function (event, _) { .map((v) => v[0].toUpperCase() + v.slice(1)) .join(' ') // notebook cell structure - let common_commands = [ - `!wget ${zipRes}\n`, - `!unzip ${template}.zip\n`, - '!pip install -r requirements.txt' - ] + let common_nb_commands = { + cell_type: 'code', + metadata: {}, + execution_count: null, + outputs: [], + source: [ + `!wget ${zipRes}\n`, + `!unzip ${template}.zip\n`, + '!pip install -r requirements.txt' + ] + } let specific_commands = [] @@ -56,7 +62,41 @@ exports.handler = async function (event, _) { ) } - let execution_command = ['!python main.py config.yaml'] + let specific_nb_commands = { + cell_type: 'code', + metadata: {}, + execution_count: null, + outputs: [], + source: specific_commands + } + + let execution_nb_commands = { + cell_type: 'code', + metadata: {}, + execution_count: null, + outputs: [], + source: ['!python main.py config.yaml'] + } + + var nb_commands = [ + { + cell_type: 'markdown', + metadata: {}, + execution_count: null, + outputs: [], + source: [ + `# ${title} by PyTorch-Ignite Code-Generator\n\n`, + 'Please, run the cell below to execute your code.' + ] + }, + common_nb_commands + ] + + if (specific_commands.length > 0) { + nb_commands.push(specific_commands) + } + + nb_commands.push(execution_nb_commands) const nb = { nbformat: 4, @@ -68,39 +108,7 @@ exports.handler = async function (event, _) { }, accelerator: 'GPU' }, - cells: [ - { - cell_type: 'markdown', - metadata: {}, - execution_count: null, - outputs: [], - source: [ - `# ${title} by PyTorch-Ignite Code-Generator\n\n`, - 'Please, run the cell below to execute your code.' - ] - }, - { - cell_type: 'code', - metadata: {}, - execution_count: null, - outputs: [], - source: common_commands - }, - { - cell_type: 'code', - metadata: {}, - execution_count: null, - outputs: [], - source: specific_commands - }, - { - cell_type: 'code', - metadata: {}, - execution_count: null, - outputs: [], - source: execution_command - } - ] + cells: nb_commands } // Create the notebook on GitHub await pushToGitHub( From 1c621750ca9d0399b591f57b64c658eee1ab5242 Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Tue, 15 Aug 2023 16:57:23 +0530 Subject: [PATCH 05/12] Correct typo --- functions/colab.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/colab.js b/functions/colab.js index f2e87954..d7b807cc 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -93,7 +93,7 @@ exports.handler = async function (event, _) { ] if (specific_commands.length > 0) { - nb_commands.push(specific_commands) + nb_commands.push(specific_nb_commands) } nb_commands.push(execution_nb_commands) From fec48650b8bba10299a99519b8c5fff74ec9ceb8 Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Tue, 15 Aug 2023 17:03:48 +0530 Subject: [PATCH 06/12] Refactored Code --- functions/colab.js | 23 +++++++++---------- .../template-vision-segmentation/README.md | 2 +- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index d7b807cc..dcfd3d44 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -42,17 +42,6 @@ exports.handler = async function (event, _) { .map((v) => v[0].toUpperCase() + v.slice(1)) .join(' ') // notebook cell structure - let common_nb_commands = { - cell_type: 'code', - metadata: {}, - execution_count: null, - outputs: [], - source: [ - `!wget ${zipRes}\n`, - `!unzip ${template}.zip\n`, - '!pip install -r requirements.txt' - ] - } let specific_commands = [] @@ -89,7 +78,17 @@ exports.handler = async function (event, _) { 'Please, run the cell below to execute your code.' ] }, - common_nb_commands + { + cell_type: 'code', + metadata: {}, + execution_count: null, + outputs: [], + source: [ + `!wget ${zipRes}\n`, + `!unzip ${template}.zip\n`, + '!pip install -r requirements.txt' + ] + } ] if (specific_commands.length > 0) { diff --git a/src/templates/template-vision-segmentation/README.md b/src/templates/template-vision-segmentation/README.md index 896c2a01..5d84c807 100644 --- a/src/templates/template-vision-segmentation/README.md +++ b/src/templates/template-vision-segmentation/README.md @@ -5,7 +5,7 @@ This is the segmentation template by Code-Generator using `deeplabv3_resnet101` and `cifar10` dataset from TorchVision and training is powered by PyTorch and PyTorch-Ignite. **Note:** -The dataset used in this is quite substantial, with a size of several GBs and automatically downloading it can be seen as an unexpected behaviour. To prevent unexpected behavior or excessive bandwidth usage, the automatic downloading of the dataset has been disabled by default. +The dataset used in this template is quite substantial, with a size of several GBs and automatically downloading it can be seen as an unexpected behaviour. To prevent unexpected behavior or excessive bandwidth usage, the automatic downloading of the dataset has been disabled by default. To download the dataset: From e0392dde199d3265add6012b8bb7a81506f34efd Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Tue, 15 Aug 2023 20:30:27 +0530 Subject: [PATCH 07/12] Refactored Code --- functions/colab.js | 72 ++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index dcfd3d44..e5b03c05 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -43,6 +43,18 @@ exports.handler = async function (event, _) { .join(' ') // notebook cell structure + function create_nb_cell(source_array, cell_type){ + if(source_array.length > 0){ + return { + cell_type: cell_type, + metadata: {}, + execution_count: null, + outputs: [], + source: source_array, + } + } + } + let specific_commands = [] if (title === 'Template Vision Segmentation') { @@ -51,51 +63,29 @@ exports.handler = async function (event, _) { ) } - let specific_nb_commands = { - cell_type: 'code', - metadata: {}, - execution_count: null, - outputs: [], - source: specific_commands - } - - let execution_nb_commands = { - cell_type: 'code', - metadata: {}, - execution_count: null, - outputs: [], - source: ['!python main.py config.yaml'] - } + const md_cell = [ + `# ${title} by PyTorch-Ignite Code-Generator\n\n`, + 'Please, run the cell below to execute your code.' + ] - var nb_commands = [ - { - cell_type: 'markdown', - metadata: {}, - execution_count: null, - outputs: [], - source: [ - `# ${title} by PyTorch-Ignite Code-Generator\n\n`, - 'Please, run the cell below to execute your code.' - ] - }, - { - cell_type: 'code', - metadata: {}, - execution_count: null, - outputs: [], - source: [ - `!wget ${zipRes}\n`, - `!unzip ${template}.zip\n`, - '!pip install -r requirements.txt' - ] - } + const common_nb_commands = [ + `!wget ${zipRes}\n`, + `!unzip ${template}.zip\n`, + '!pip install -r requirements.txt' ] - if (specific_commands.length > 0) { - nb_commands.push(specific_nb_commands) - } + const specific_nb_commands = specific_commands.length > 0 ? specific_commands : null - nb_commands.push(execution_nb_commands) + const execution_nb_commands = [ + '!python main.py config.yaml' + ] + + const nb_commands = [ + create_nb_cell(md_cell, 'markdown'), + create_nb_cell(common_nb_commands, 'code'), + create_nb_cell(specific_nb_commands, 'code'), + create_nb_cell(execution_nb_commands, 'code') + ] const nb = { nbformat: 4, From 85f11d1a754e481404fdc335b05feedb1da53464 Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Tue, 15 Aug 2023 20:42:01 +0530 Subject: [PATCH 08/12] Fixed failing null error --- functions/colab.js | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index e5b03c05..fa16a70d 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -43,14 +43,14 @@ exports.handler = async function (event, _) { .join(' ') // notebook cell structure - function create_nb_cell(source_array, cell_type){ - if(source_array.length > 0){ + function create_nb_cell(source_array, cell_type) { + if (source_array.length > 0) { return { cell_type: cell_type, metadata: {}, execution_count: null, outputs: [], - source: source_array, + source: source_array } } } @@ -74,18 +74,16 @@ exports.handler = async function (event, _) { '!pip install -r requirements.txt' ] - const specific_nb_commands = specific_commands.length > 0 ? specific_commands : null + const execution_nb_commands = ['!python main.py config.yaml'] - const execution_nb_commands = [ - '!python main.py config.yaml' - ] - - const nb_commands = [ + let nb_commands = [ create_nb_cell(md_cell, 'markdown'), - create_nb_cell(common_nb_commands, 'code'), - create_nb_cell(specific_nb_commands, 'code'), - create_nb_cell(execution_nb_commands, 'code') + create_nb_cell(common_nb_commands, 'code') ] + if(specific_commands.length > 0){ + nb_commands.push(create_nb_cell(specific_commands, 'code')) + } + nb_commands.push(execution_nb_commands, 'code') const nb = { nbformat: 4, From 26748b32e292bac88f7340f95accc16a2294cc67 Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Tue, 15 Aug 2023 20:46:16 +0530 Subject: [PATCH 09/12] Fix Typo for function --- functions/colab.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index fa16a70d..536aaa72 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -80,10 +80,10 @@ exports.handler = async function (event, _) { create_nb_cell(md_cell, 'markdown'), create_nb_cell(common_nb_commands, 'code') ] - if(specific_commands.length > 0){ + if (specific_commands.length > 0) { nb_commands.push(create_nb_cell(specific_commands, 'code')) } - nb_commands.push(execution_nb_commands, 'code') + nb_commands.push(create_nb_cell(execution_nb_commands, 'code')) const nb = { nbformat: 4, From c4c0d5e95e4f4da92dca1073ef802306ac4a3da5 Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Thu, 17 Aug 2023 12:46:30 +0530 Subject: [PATCH 10/12] Removed redundant if condition --- functions/colab.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index 536aaa72..98114081 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -44,7 +44,6 @@ exports.handler = async function (event, _) { // notebook cell structure function create_nb_cell(source_array, cell_type) { - if (source_array.length > 0) { return { cell_type: cell_type, metadata: {}, @@ -52,7 +51,6 @@ exports.handler = async function (event, _) { outputs: [], source: source_array } - } } let specific_commands = [] From 8070046719e774d1e6ddebbd57ccc9b5c0e9e541 Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Thu, 17 Aug 2023 12:54:04 +0530 Subject: [PATCH 11/12] Fix lint --- functions/colab.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index 98114081..ee8aafc6 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -44,13 +44,13 @@ exports.handler = async function (event, _) { // notebook cell structure function create_nb_cell(source_array, cell_type) { - return { - cell_type: cell_type, - metadata: {}, - execution_count: null, - outputs: [], - source: source_array - } + return { + cell_type: cell_type, + metadata: {}, + execution_count: null, + outputs: [], + source: source_array + } } let specific_commands = [] From e99ef9b9c67f2aa99294cc6e160f4e8607b36e9c Mon Sep 17 00:00:00 2001 From: Jyotirmay Khavasi Date: Thu, 17 Aug 2023 13:19:09 +0530 Subject: [PATCH 12/12] Renamed variable nb_commands to nb_cells --- functions/colab.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/colab.js b/functions/colab.js index ee8aafc6..07f1024d 100644 --- a/functions/colab.js +++ b/functions/colab.js @@ -74,14 +74,14 @@ exports.handler = async function (event, _) { const execution_nb_commands = ['!python main.py config.yaml'] - let nb_commands = [ + let nb_cells = [ create_nb_cell(md_cell, 'markdown'), create_nb_cell(common_nb_commands, 'code') ] if (specific_commands.length > 0) { - nb_commands.push(create_nb_cell(specific_commands, 'code')) + nb_cells.push(create_nb_cell(specific_commands, 'code')) } - nb_commands.push(create_nb_cell(execution_nb_commands, 'code')) + nb_cells.push(create_nb_cell(execution_nb_commands, 'code')) const nb = { nbformat: 4, @@ -93,7 +93,7 @@ exports.handler = async function (event, _) { }, accelerator: 'GPU' }, - cells: nb_commands + cells: nb_cells } // Create the notebook on GitHub await pushToGitHub(