Merge remote-tracking branch 'origin/main' into tpi-tweaks

Author: alex000kim

.github/workflows/1-experiment.yaml

@@ -67,8 +67,6 @@ jobs:
           pipenv run dvc exp run
           pipenv run dvc push
 
-          cml pr dvc.lock metrics.json training_metrics.json training_metrics_dvc_plots/**
-
           # Create CML report
           echo "## Metrics" >> report.md
           pipenv run dvc metrics show --md >> report.md

.github/workflows/2-develop.yaml

@@ -66,8 +66,6 @@ jobs:
           pipenv run dvc exp run
           pipenv run dvc push
 
-          cml pr dvc.lock metrics.json training_metrics.json training_metrics_dvc_plots/**
-
           # Create CML report
           echo "## Metrics" >> report.md
           pipenv run dvc metrics show --md >> report.md

Pipfile

@@ -9,20 +9,12 @@ torchvision = "==0.12.0"
 fastai = "==2.5.6"
 fastcore = "==1.4.1"
 tqdm = "==4.64.0"
-pillow = "==9.1.0"
+pillow = "==9.1.1"
 dvc = {version = "==2.10.2", extras = ["s3"]}
 fastapi = {extras = ["all"], version = "==0.75.1"}
 python-box = "==6.0.2"
-dvclive = "==0.7.1"
+dvclive = "==0.8.2"
 uvicorn = "==0.15.0"
 
-[dev-packages]
-ipykernel = "*"
-autopep8 = "*"
-pylint = "*"
-bandit = "*"
-ipywidgets = "*"
-notebook = "*"
-
 [requires]
 python_version = "3"

dvc.lock

@@ -4,14 +4,14 @@ stages:
     cmd: pipenv run pip freeze > requirements.txt
     outs:
     - path: requirements.txt
-      md5: 7edda101df092201b03b26a2b476e7c4
-      size: 3445
+      md5: 0fbfbeb39bc8c13a8c663523a4c2f7db
+      size: 2711
   data_load:
     cmd: python src/stages/data_load.py --config=params.yaml
     deps:
     - path: requirements.txt
-      md5: 7edda101df092201b03b26a2b476e7c4
-      size: 3445
+      md5: 0fbfbeb39bc8c13a8c663523a4c2f7db
+      size: 2711
     - path: src/stages/data_load.py
       md5: 481ef35d6faa72224f0f0842194becbc
       size: 797
@@ -26,9 +26,6 @@ stages:
           data_dir: data
           orig_dirname: Magnetic-tile-defect-datasets.-master
           new_dirname: MAGNETIC_TILE_SURFACE_DEFECTS
-          code_names:
-          - 0
-          - 1
     outs:
     - path: data/MAGNETIC_TILE_SURFACE_DEFECTS/images
       md5: b7178caeac712acf11359c9f183f2cdc.dir
@@ -50,8 +47,8 @@ stages:
       size: 199417
       nfiles: 392
     - path: requirements.txt
-      md5: 7edda101df092201b03b26a2b476e7c4
-      size: 3445
+      md5: 0fbfbeb39bc8c13a8c663523a4c2f7db
+      size: 2711
     - path: src/stages/data_split.py
       md5: 34f52be01f05d45585264c1c998765d4
       size: 1295
@@ -66,9 +63,6 @@ stages:
           data_dir: data
           orig_dirname: Magnetic-tile-defect-datasets.-master
           new_dirname: MAGNETIC_TILE_SURFACE_DEFECTS
-          code_names:
-          - 0
-          - 1
         data_split:
           test_pct: 0.2
           train_img_dir_path: data/MAGNETIC_TILE_SURFACE_DEFECTS/train_images
@@ -77,35 +71,35 @@ stages:
           test_mask_dir_path: data/MAGNETIC_TILE_SURFACE_DEFECTS/test_masks
     outs:
     - path: data/MAGNETIC_TILE_SURFACE_DEFECTS/test_images
-      md5: 945201efae1f89c50a4c3c325853fc49.dir
-      size: 2804818
+      md5: aec21b0f9a77d82b3aa1e88a45d8b5b7.dir
+      size: 2955133
       nfiles: 78
     - path: data/MAGNETIC_TILE_SURFACE_DEFECTS/test_masks
-      md5: e074f6b72133fb84ea4a4099431ad50d.dir
-      size: 34264
+      md5: 92a47a0bfebeaa9813818548aea03e3e.dir
+      size: 39362
       nfiles: 78
     - path: data/MAGNETIC_TILE_SURFACE_DEFECTS/train_images
-      md5: 6928706298a6d7ef07013289bd41948f.dir
-      size: 11861699
+      md5: db4f51d18ab93e49665a49ea0682e8da.dir
+      size: 11711384
       nfiles: 314
     - path: data/MAGNETIC_TILE_SURFACE_DEFECTS/train_masks
-      md5: 2607e5f281c5037af71d5b28b9f653bd.dir
-      size: 165153
+      md5: 3ad55c909b7e950bd5faca37648d95d8.dir
+      size: 160055
       nfiles: 314
   train:
     cmd: python src/stages/train.py --config=params.yaml
     deps:
     - path: data/MAGNETIC_TILE_SURFACE_DEFECTS/train_images
-      md5: 6928706298a6d7ef07013289bd41948f.dir
-      size: 11861699
+      md5: db4f51d18ab93e49665a49ea0682e8da.dir
+      size: 11711384
       nfiles: 314
     - path: data/MAGNETIC_TILE_SURFACE_DEFECTS/train_masks
-      md5: 2607e5f281c5037af71d5b28b9f653bd.dir
-      size: 165153
+      md5: 3ad55c909b7e950bd5faca37648d95d8.dir
+      size: 160055
       nfiles: 314
     - path: requirements.txt
-      md5: 7edda101df092201b03b26a2b476e7c4
-      size: 3445
+      md5: 0fbfbeb39bc8c13a8c663523a4c2f7db
+      size: 2711
     - path: src/stages/train.py
       md5: a6ec2486d381039eac01b2463dfae760
       size: 1597
@@ -132,31 +126,31 @@ stages:
           model_pickle_fname: model_pickle_fastai.pkl
     outs:
     - path: models
-      md5: e68c53794a1fbcf967a067da7b4103ba.dir
+      md5: b1646d6a7376fb92bbd6a2c73e7f73e0.dir
       size: 190185141
       nfiles: 1
     - path: training_metrics
-      md5: ccc588b1adce5fef7427d647fe5e7ddd.dir
-      size: 2270
+      md5: fdda54abf3ade1a584cd8a841a9c3a5d.dir
+      size: 2374
       nfiles: 6
   evaluate:
     cmd: python src/stages/eval.py --config=params.yaml
     deps:
     - path: data/MAGNETIC_TILE_SURFACE_DEFECTS/test_images
-      md5: 945201efae1f89c50a4c3c325853fc49.dir
-      size: 2804818
+      md5: aec21b0f9a77d82b3aa1e88a45d8b5b7.dir
+      size: 2955133
       nfiles: 78
     - path: data/MAGNETIC_TILE_SURFACE_DEFECTS/test_masks
-      md5: e074f6b72133fb84ea4a4099431ad50d.dir
-      size: 34264
+      md5: 92a47a0bfebeaa9813818548aea03e3e.dir
+      size: 39362
       nfiles: 78
     - path: models
-      md5: e68c53794a1fbcf967a067da7b4103ba.dir
+      md5: b1646d6a7376fb92bbd6a2c73e7f73e0.dir
       size: 190185141
       nfiles: 1
     - path: requirements.txt
-      md5: 7edda101df092201b03b26a2b476e7c4
-      size: 3445
+      md5: 0fbfbeb39bc8c13a8c663523a4c2f7db
+      size: 2711
     - path: src/stages/eval.py
       md5: 4d1ae58faaf33683ce0fbe6563f93878
       size: 1472
@@ -186,5 +180,5 @@ stages:
           model_pickle_fname: model_pickle_fastai.pkl
     outs:
     - path: metrics.json
-      md5: d62c799d50f66150fb5684fc1995527d
-      size: 64
+      md5: 8f9a36cdec542ba6ba31076d240c7ead
+      size: 65

metrics.json

@@ -1 +1 @@
-{"dice_mean": 0.707909783307654, "acc_mean": 0.9804131410256411}
+{"dice_mean": 0.7390305060702425, "acc_mean": 0.9721134615384617}

notebooks/1_ProcessData.ipynb

@@ -27,12 +27,13 @@
     }
    ],
    "source": [
+    "import os\n",
+    "import shutil\n",
     "import numpy as np\n",
     "\n",
     "from fastai.vision.all import *\n",
     "from pathlib import Path\n",
-    "from tqdm.auto import tqdm\n",
-    "from shutil import copyfile, rmtree, move"
+    "from tqdm.auto import tqdm"
    ]
   },
   {
@@ -96,9 +97,9 @@
    "source": [
     "dataset_path = DATA_PATH/'MAGNETIC_TILE_SURFACE_DEFECTS'\n",
     "if dataset_path.is_dir():\n",
-    "    rmtree(dataset_path)\n",
+    "    shutil.rmtree(dataset_path)\n",
     "if extract_dir.is_dir():\n",
-    "    move(extract_dir, dataset_path)"
+    "    shutil.move(extract_dir, dataset_path)"
    ]
   },
   {
@@ -130,7 +131,7 @@
    "outputs": [],
    "source": [
     "if (dataset_path/'MT_Free').exists():\n",
-    "    rmtree(dataset_path/'MT_Free')"
+    "    shutil.rmtree(dataset_path/'MT_Free')"
    ]
   },
   {
@@ -672,7 +673,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 392/392 [00:01<00:00, 308.99it/s]\n"
+      "100%|██████████| 392/392 [00:00<00:00, 560.50it/s]\n"
      ]
     }
    ],
@@ -685,7 +686,7 @@
     "\n",
     "    new_img_path = img_dir_path/img_path.name\n",
     "    new_mask_path = mask_dir_path/msk_path.name\n",
-    "    copyfile(img_path, new_img_path)\n",
+    "    shutil.copyfile(img_path, new_img_path)\n",
     "    Image.fromarray(msk).save(new_mask_path)"
    ]
   },
@@ -737,10 +738,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 19,
    "id": "33489364",
    "metadata": {},
    "outputs": [],
+   "source": [
+    "dirs_to_keep = [img_dir_path, mask_dir_path, train_img_dir_path, train_mask_dir_path, test_img_dir_path, test_mask_dir_path]\n",
+    "\n",
+    "for ii in (dataset_path).ls():\n",
+    "    if ii not in dirs_to_keep:\n",
+    "        try: # if file\n",
+    "            os.remove(ii)\n",
+    "        except: # if directory\n",
+    "            shutil.rmtree(ii)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "021bf550",
+   "metadata": {},
+   "outputs": [],
    "source": []
   }
  ],

params.yaml

@@ -8,7 +8,6 @@ data_load:
   data_dir: data
   orig_dirname: Magnetic-tile-defect-datasets.-master
   new_dirname: MAGNETIC_TILE_SURFACE_DEFECTS
-  code_names: [0, 1]
 
 data_split:
   test_pct: 0.20

src/stages/train.py

@@ -14,7 +14,6 @@ def train_and_save_model(params_path):
     random_state = params.base.random_state
     train_img_dir_path = Path(params.data_split.train_img_dir_path)
     train_mask_dir_path = Path(params.data_split.train_mask_dir_path)
-    code_names = params.data_load.code_names
     img_size = params.train.img_size
     valid_pct = params.train.valid_pct
     learning_rate = params.train.learning_rate
@@ -27,7 +26,6 @@ def train_and_save_model(params_path):
     model_pickle_path = (model_pickle_dir_path/model_pickle_fname).absolute()
     train_model(train_img_dir_path=train_img_dir_path,
                 train_mask_dir_path=train_mask_dir_path,
-                code_names=code_names,
                 n_epochs=n_epochs,
                 lr=learning_rate,
                 use_cpu=use_cpu,

src/train_utils.py

@@ -16,7 +16,6 @@ def get_mask_path(img_path, train_mask_dir_path):
 
 def train_model(train_img_dir_path,
                 train_mask_dir_path,
-                code_names,
                 n_epochs,
                 use_cpu,
                 lr,
@@ -35,7 +34,7 @@ def train_model(train_img_dir_path,
         fnames=fpaths,
         label_func=partial(
             get_mask_path, train_mask_dir_path=train_mask_dir_path),
-        codes=code_names,
+        codes=[0, 1],
         bs=batch_size,
         splitter=RandomSplitter(valid_pct=valid_pct, seed=seed),
         item_tfms=Resize(item_tfms_sz),