Skip to content

Commit ad6ead4

Browse files
authored
Merge pull request #245 from acwooding/extend-environment
Extend environment management to handle arbitrary conda channels
2 parents b7aff58 + 2e4563b commit ad6ead4

File tree

10 files changed

+132
-63
lines changed

10 files changed

+132
-63
lines changed

.circleci/config.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,15 @@ jobs:
4040
pwd
4141
which make
4242
cookiecutter --config-file .cookiecutter-easydata-test-circleci.yml . -f --no-input
43-
43+
4444
- run:
4545
name: Create test-env environment and contrive to always use it
4646
command: |
4747
conda activate cookiecutter
4848
cd test-env
4949
export CONDA_EXE=/opt/conda/bin/conda
5050
make create_environment
51+
python scripts/tests/add-extra-channel-dependency.py
5152
conda activate test-env
5253
conda install -c anaconda make
5354
touch environment.yml

{{ cookiecutter.repo_name }}/Makefile

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -75,17 +75,12 @@ test: update_environment
7575
$(if $(CI_RUNNING),--ignore=$(TESTS_NO_CI)) \
7676
$(MODULE_NAME)
7777

78-
## Run all Unit Tests with coverage
78+
## Run all Unit and code coverage tests
7979
test_with_coverage: update_environment
8080
$(SET) LOGLEVEL=DEBUG; coverage run -m pytest --pyargs --doctest-modules --doctest-continue-on-failure --verbose \
8181
$(if $(CI_RUNNING),--ignore=$(TESTS_NO_CI)) \
8282
$(MODULE_NAME)
8383

84-
.PHONY: lint
85-
## Lint using flake8
86-
lint:
87-
flake8 $(MODULE_NAME)
88-
8984
.phony: help_update_easydata
9085
help_update_easydata:
9186
@$(PYTHON_INTERPRETER) scripts/help-update.py
@@ -105,7 +100,7 @@ debug:
105100
# Self Documenting Commands #
106101
#################################################################################
107102

108-
HELP_VARS := PROJECT_NAME DEBUG_FILE ARCH PLATFORM
103+
HELP_VARS := PROJECT_NAME DEBUG_FILE ARCH PLATFORM SHELL
109104

110105
.DEFAULT_GOAL := show-help
111106
.PHONY: show-help

{{ cookiecutter.repo_name }}/Makefile.envs

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,28 +4,20 @@
44

55
include Makefile.include
66

7-
$(LOCKFILE): check_installation .make.bootstrap .make.pip-requirements.txt .make.environment-default.yml .make.conda-forge-requirements.txt
7+
$(LOCKFILE): check_installation .make.bootstrap split_environment_files
88
ifeq (conda, $(VIRTUALENV))
9-
$(CONDA_EXE) env update -n $(PROJECT_NAME) -f .make.environment-default.yml --prune
10-
$(CONDA_EXE) install -n $(PROJECT_NAME) --file .make.conda-forge-requirements.txt --channel defaults --channel conda-forge --strict-channel-priority --yes
9+
$(foreach channel, $(shell $(CAT) .make.channel-order.include),\
10+
$(CONDA_EXE) install -n $(PROJECT_NAME) --file .make.$(channel)-environment.txt --channel defaults --channel $(channel) --strict-channel-priority --yes $(CMDSEP))
1111
$(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture pip install -r .make.pip-requirements.txt
1212
$(CONDA_EXE) env export -n $(PROJECT_NAME) -f $(LOCKFILE)
1313
else
1414
$(error Unsupported Environment `$(VIRTUALENV)`. Use conda)
1515
endif
1616

17-
# extract multi-phase dependencies from environment.yml
18-
.make.environment-pip.yml: environment.yml .make.bootstrap
19-
$(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py pip-yaml $(PROJECT_DIR)environment.yml > $@
20-
21-
.make.pip-requirements.txt: environment.yml .make.bootstrap
22-
$(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py pip $(PROJECT_DIR)environment.yml > $@
23-
24-
.make.conda-forge-requirements.txt: environment.yml .make.bootstrap
25-
$(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py conda-forge $(PROJECT_DIR)environment.yml > $@
26-
27-
.make.environment-default.yml: environment.yml .make.bootstrap
28-
$(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py default $(PROJECT_DIR)environment.yml > $@
17+
.PHONY: split_environment_files
18+
# extract multi-phase dependencies from environment.yml and create ordering file
19+
split_environment_files: environment.yml .make.bootstrap
20+
$(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py $(PROJECT_DIR)environment.yml
2921

3022
.make.bootstrap: scripts/bootstrap.yml
3123
$(CONDA_EXE) env update -n $(PROJECT_NAME) -f scripts/bootstrap.yml
@@ -69,6 +61,7 @@ endif
6961
# Checks that the conda environment is active
7062
environment_enabled:
7163
ifeq (conda,$(VIRTUALENV))
64+
$(CONDA_EXE) config --env --set channel_priority strict
7265
ifneq ($(notdir ${CONDA_DEFAULT_ENV}), $(PROJECT_NAME))
7366
$(error Run "$(VIRTUALENV) activate $(PROJECT_NAME)" before proceeding...)
7467
endif

{{ cookiecutter.repo_name }}/Makefile.include

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,4 @@ CAT ?= cat
1919
SET ?= export
2020
WHICH ?= which
2121
DEVNULL ?= /dev/null
22-
23-
$(warning From here on, using SHELL = $(SHELL))
22+
CMDSEP ?= ;

{{ cookiecutter.repo_name }}/Makefile.win32

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ CAT = type
55
SET = set
66
WHICH = where
77
DEVNULL = nul
8+
CMDSEP = &
89

910
# Some UNIXish packages force the installation of a Bourne-compatible shell, and Make
1011
# prefers using this when it sees it. We thus force the usage of the good ole Batch

{{ cookiecutter.repo_name }}/environment.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{% macro pyver() -%}
22
{% if cookiecutter.python_version == 'latest' -%}
3-
- python=3
3+
- python
44
{% else -%}
55
- python={{ cookiecutter.python_version }}
66
{% endif -%}

{{ cookiecutter.repo_name }}/reference/easydata/conda-environments.md

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ When adding packages to your python environment, **do not `pip install` or `cond
8181
Your `environment.yml` file will look something like this:
8282
```
8383
name: {{ cookiecutter.repo_name }}
84+
dependencies:
8485
- pip
8586
- pip:
8687
- -e . # conda >= 4.4 only
@@ -106,7 +107,7 @@ name: {{ cookiecutter.repo_name }}
106107
```
107108
To add any package available from conda, add it to the end of the list. If you have a PYPI dependency that's not avaible via conda, add it to the list of pip installable dependencies under ` - pip:`.
108109

109-
You can include any {{ cookiecutter.upstream_location }} python-based project in the `pip` section via `git+https://{{ cookiecutter.upstream_location }}/<my_git_handle>/<package>`.
110+
You can include any `{{ cookiecutter.upstream_location }}` python-based project in the `pip` section via `git+https://{{ cookiecutter.upstream_location }}/<my_git_handle>/<package>`.
110111

111112
In particular, if you're working off of a fork or a work in progress branch of a repo in {{ cookiecutter.upstream_location }} (say, your personal version of <package>), you can change `git+https://{{ cookiecutter.upstream_location }}/<my_git_handle>/<package>` to
112113

@@ -117,6 +118,43 @@ Once you're done your edits, run `make update_environment` and voila, you're upd
117118

118119
To share your updated environment, check in your `environment.yml` file. (More on this in [Sharing your Work](sharing-your-work.md))
119120

121+
#### Adding packages from other conda channels
122+
Say we want to add a package only available from the `conda-forge` conda channel and not the default conda channel. (The conda channel is what follows `-c` when using `conda install -c my-channel my-package`. Suppose we want to use `make` on windows. Then we need to use `conda-forge` since the default conda channel only has linux and macOS installations of `make`. To normally conda install this, we would use `conda install -c conda-forge make`. **We won't do that here**.
123+
124+
Instead, we add a `channel-order` section that starts with `defaults` and lists the other channels we want to use in the order we want to install from them (note that this is a custom EasyData section to the `environment.yml`). Then we add our package in the dependency list in the form `channel-name::package-name`, for example, `conda-forge::make`.
125+
126+
In this case an updated `environment.yml` file looks like this:
127+
```
128+
name: {{ cookiecutter.repo_name }}
129+
channel-order:
130+
- defaults
131+
- conda-forge
132+
dependencies:
133+
- pip
134+
- pip:
135+
- -e . # conda >= 4.4 only
136+
- python-dotenv>=0.5.1
137+
- nbval
138+
- nbdime
139+
- umap-learn
140+
- gdown
141+
- setuptools
142+
- wheel
143+
- git>=2.5 # for git worktree template updating
144+
- sphinx
145+
- bokeh
146+
- click
147+
- colorcet
148+
- coverage
149+
- coveralls
150+
- datashader
151+
- holoviews
152+
- matplotlib
153+
- jupyter
154+
- conda-forge::make
155+
...
156+
```
157+
120158

121159
#### Lock files
122160
Now, we'll admit that this workflow isn't perfectly reproducible in the sense that conda still has to resolve versions from the `environment.yml`. To make it more reproducible, running either `make create_environment` or `make update_environment` will generate an `environment.{$ARCH}.lock.yml` (e.g. `environment.i386.lock.yml`). This file keeps a record of the exact environment that is currently installed in your conda environment `{{ cookiecutter.repo_name }}`. If you ever need to reproduce an environment exactly, you can install from the `.lock.yml` file. (Note: These are architecture dependent).
Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
1+
{% macro pyver() -%}
2+
{% if cookiecutter.python_version == 'latest' -%}
3+
- python
4+
{% else -%}
5+
- python={{ cookiecutter.python_version }}
6+
{% endif -%}
7+
{% endmacro -%}
8+
name: {{ cookiecutter.repo_name }}
19
channels:
2-
- defaults
10+
- defaults
311
dependencies:
4-
- python=3.7
5-
- pyyaml
12+
- pyyaml
13+
{{ pyver()|indent(3, true) }}

{{ cookiecutter.repo_name }}/scripts/split_pip.py

Lines changed: 53 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,19 @@
22
import json
33
import sys
44
import yaml
5+
from collections import defaultdict
56

6-
ACCEPTABLE_FORMATS = ["default", "pip", "pip-yaml", "conda-forge"]
77

8-
def env_split(conda_env, kind="default"):
9-
"""Given a conda_environment dict, split into pip/nonpip versions
8+
def env_split(conda_env, channel_order):
9+
"""Given a conda_environment dict, and a channel order, split into versions for each channel.
10+
11+
Returns:
12+
13+
conda_env: (list)
14+
remaining setup bits of the environment.yml file
15+
channel_dict: (dict)
16+
dict containing the list of dependencies by channel name
1017
11-
conda_env: dict
1218
Python object corresponding to environment.yml"""
1319
# Cheater way to make deep Copies
1420
json_copy = json.dumps(conda_env)
@@ -17,49 +23,63 @@ def env_split(conda_env, kind="default"):
1723

1824
pipdeps = None
1925
deplist = conda_env.pop('dependencies')
20-
conda_forge_list = []
26+
channel_dict = defaultdict(list)
2127

2228
for k, dep in enumerate(deplist[:]): # Note: copy list, as we mutate it
2329
if isinstance(dep, dict): # nested yaml
2430
if dep.get('pip', None):
25-
pipdeps = ["pip", deplist.pop(k)]
31+
channel_dict['pip'] = deplist.pop(k)
2632
else:
27-
prefix = 'conda-forge::'
28-
if dep.startswith(prefix):
29-
conda_forge_list.append(dep[len(prefix):])
33+
prefix_check = dep.split('::')
34+
if len(prefix_check) > 1:
35+
channel = prefix_check[0]
36+
if not channel in channel_order:
37+
raise Exception(f'the channel {channel} required for {dep} is not specified in a channel-order section of the environment file')
38+
channel_dict[f'{channel}'].append(prefix_check[1])
3039
deplist.remove(dep)
3140

32-
conda_env['dependencies'] = deplist
33-
pip_env['dependencies'] = pipdeps
34-
return conda_env, pip_env, conda_forge_list
41+
channel_dict['defaults'] = deplist
42+
conda_env.pop('channel-order', None)
43+
return conda_env, channel_dict
44+
45+
def get_channel_order(conda_env):
46+
"""
47+
Given a conda_environment dict, get the channels from the channel order.
48+
"""
49+
channel_order = conda_env.get('channel-order')
50+
51+
if channel_order is None:
52+
channel_order = ['defaults']
53+
if not 'defaults' in channel_order:
54+
channel_order.insert(0, 'defaults')
55+
channel_order.append('pip')
56+
return channel_order
3557

3658
def usage():
3759
print(f"""
38-
Usage: split_pip.py [{"|".join(ACCEPTABLE_FORMATS)}] path/to/environment.yml
60+
Usage: split_pip.py path/to/environment.yml
3961
""")
4062
if __name__ == '__main__':
41-
if len(sys.argv) != 3:
42-
usage()
43-
exit(1)
44-
45-
kind = sys.argv[1]
46-
if kind not in ACCEPTABLE_FORMATS:
63+
if len(sys.argv) != 2:
4764
usage()
4865
exit(1)
4966

50-
with open(sys.argv[2], 'r') as yamlfile:
67+
with open(sys.argv[1], 'r') as yamlfile:
5168
conda_env = yaml.safe_load(yamlfile)
5269

53-
cenv, penv, forgelist = env_split(conda_env)
54-
if kind == "pip-yaml":
55-
_ = yaml.dump(penv, sys.stdout, allow_unicode=True, default_flow_style=False)
56-
elif kind == "pip":
57-
print("\n".join(penv["dependencies"].pop(-1)["pip"]))
58-
elif kind == "pip-yaml":
59-
_ = yaml.dump(penv, sys.stdout, allow_unicode=True, default_flow_style=False)
60-
elif kind == "default":
61-
_ = yaml.dump(cenv, sys.stdout, allow_unicode=True, default_flow_style=False)
62-
elif kind == "conda-forge":
63-
print("\n".join(forgelist))
64-
else:
65-
raise Exception(f"Invalid Kind: {kind}")
70+
#check for acceptable formats
71+
channel_order = get_channel_order(conda_env)
72+
with open('.make.channel-order.include', 'w') as f:
73+
f. write(' '.join(channel_order[:-1])) #exclude pip as a channel here
74+
75+
cenv, channel_dict = env_split(conda_env, channel_order)
76+
77+
for kind in channel_order:
78+
if kind == "pip":
79+
filename = '.make.pip-requirements.txt'
80+
with open(filename, 'w') as f:
81+
f.write("\n".join(channel_dict['pip']['pip']))
82+
else:
83+
filename = f'.make.{kind}-environment.txt'
84+
with open(filename, 'w') as f:
85+
f.write("\n".join(channel_dict[kind]))
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import sys
2+
import yaml
3+
4+
5+
if __name__ == "__main__":
6+
channel_order = ['defaults', 'pytorch']
7+
dependency_new = "pytorch::cpuonly"
8+
9+
with open("environment.yml", "rt", encoding="utf-8") as file_env:
10+
env = yaml.safe_load(file_env)
11+
env["dependencies"].append(dependency_new)
12+
env["channel-order"] = channel_order
13+
with open("environment.yml", "wt", encoding="utf-8") as file_env:
14+
yaml.safe_dump(env, file_env)

0 commit comments

Comments
 (0)