CI cleaning (#4941)

* set

* cut

* env

* oonce

* env

* env

* env

Author: Borda

.drone.yml

@@ -25,13 +25,16 @@ steps:
   environment:
     CODECOV_TOKEN:
       from_secret: codecov_token
+    AUTH_TOKEN:
+      from_secret: gh_auth_token
     MKL_THREADING_LAYER: GNU
 
   commands:
     - python --version
     - pip --version
     - nvidia-smi
     - pip install -r ./requirements/devel.txt --upgrade-strategy only-if-needed -v --no-cache-dir
+    - pip install git+https://${AUTH_TOKEN}@github.com/PyTorchLightning/[email protected] -v --no-cache-dir
     # when Image has defined CUDa version we can switch to this package spec "nvidia-dali-cuda${CUDA_VERSION%%.*}0"
     # todo: temprarl fix till https://github.com/PyTorchLightning/pytorch-lightning/pull/4922 is resolved
     - pip install --extra-index-url https://developer.download.nvidia.com/compute/redist "nvidia-dali-cuda100<0.27" --upgrade-strategy only-if-needed

benchmarks/test_sharded_parity.py

@@ -10,7 +10,7 @@
 from pytorch_lightning.plugins.ddp_plugin import DDPPlugin
 from pytorch_lightning.plugins.sharded_plugin import DDPShardedPlugin
 from pytorch_lightning.utilities import FAIRSCALE_AVAILABLE, NATIVE_AMP_AVAILABLE
-from tests.backends.launcher import DDPLauncher
+from tests.backends import DDPLauncher
 from tests.base.boring_model import BoringModel, RandomDataset
 
 

tests/__init__.py

@@ -3,8 +3,11 @@
 import numpy as np
 
 TEST_ROOT = os.path.dirname(__file__)
-PACKAGE_ROOT = os.path.dirname(TEST_ROOT)
-TEMP_PATH = os.path.join(PACKAGE_ROOT, 'test_temp')
+PROJECT_ROOT = os.path.dirname(TEST_ROOT)
+TEMP_PATH = os.path.join(PROJECT_ROOT, 'test_temp')
+
+if PROJECT_ROOT not in os.getenv('PYTHONPATH', ""):
+    os.environ['PYTHONPATH'] = f'{PROJECT_ROOT}:{os.environ.get("PYTHONPATH", "")}'
 
 # generate a list of random seeds for each test
 RANDOM_PORTS = list(np.random.randint(12000, 19000, 1000))

tests/backends/__init__.py

@@ -0,0 +1,10 @@
+from functools import wraps
+
+try:
+    from dtrun.launcher import DDPLauncher
+except ImportError:
+    class DDPLauncher:
+        def run(cmd_line, **kwargs):
+            def inner(func):
+                pass
+            return inner

tests/backends/launcher.py

@@ -12,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
-from time import sleep
 
 import pytest
 import torch
 
 from tests.backends import ddp_model
-from tests.backends.launcher import DDPLauncher
+from tests.backends import DDPLauncher
 from tests.utilities.distributed import call_training_script
 
 

tests/backends/test_ddp.py

@@ -22,10 +22,10 @@
 from torch import Tensor
 from torch.utils.data import Dataset
 
-from tests import PACKAGE_ROOT
+from tests import PROJECT_ROOT
 
 #: local path to test datasets
-PATH_DATASETS = os.path.join(PACKAGE_ROOT, 'Datasets')
+PATH_DATASETS = os.path.join(PROJECT_ROOT, 'Datasets')
 
 
 class MNIST(Dataset):

tests/base/datasets.py

@@ -17,7 +17,7 @@
 import torch
 
 from pytorch_lightning import Trainer
-from tests.backends.launcher import DDPLauncher
+from tests.backends import DDPLauncher
 from tests.base.boring_model import BoringModel