Skip to content

Create environment using venv or conda #19848

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 31 commits into from
Sep 23, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
b5cbdc8
Create environment provider.
karthiknadig Sep 9, 2022
2e5ae43
Add classes supporting environment creation.
karthiknadig Sep 9, 2022
4e4f60e
Add general quickpick API.
karthiknadig Sep 11, 2022
48e31f8
Implement venv support
karthiknadig Sep 13, 2022
d11fff5
Fix build.
karthiknadig Sep 13, 2022
d323df9
Conda env creation support.
karthiknadig Sep 14, 2022
703dc9e
Fix conda env creation.
karthiknadig Sep 14, 2022
9f44ab6
Add status.
karthiknadig Sep 14, 2022
cd09e93
Add detailed status.
karthiknadig Sep 14, 2022
b4c8d00
Remove decoder.
karthiknadig Sep 14, 2022
9fca478
Improve localization
karthiknadig Sep 14, 2022
71ab854
Add tests for createEnvironment function.
karthiknadig Sep 14, 2022
e9e7ca4
Add tests for create environment provider quick pick.
karthiknadig Sep 15, 2022
7995c2c
Clean-up localization on workspace selection.
karthiknadig Sep 15, 2022
1eb96cc
More fixes and clean up.
karthiknadig Sep 15, 2022
952454c
Fix localization quick pick.
karthiknadig Sep 15, 2022
afc9c4f
Fix venv creation localization.
karthiknadig Sep 15, 2022
37fb908
Fix conda provider localization.
karthiknadig Sep 15, 2022
6b4cceb
Add workspace selection tests.
karthiknadig Sep 16, 2022
d7729ca
Fix linting.
karthiknadig Sep 16, 2022
69729f4
Fix typing.
karthiknadig Sep 16, 2022
29ba248
Use the new picker API.
karthiknadig Sep 16, 2022
bc83068
Add conda creation tests.
karthiknadig Sep 16, 2022
7f6e503
Add venv tests.
karthiknadig Sep 21, 2022
7463805
Add conda tests
karthiknadig Sep 21, 2022
e7b3516
Fix build and localization.
karthiknadig Sep 21, 2022
7e38079
Rebase with main.
karthiknadig Sep 21, 2022
861cd98
Remove unused libs.
karthiknadig Sep 21, 2022
c55d4c6
Address comments.
karthiknadig Sep 23, 2022
cbb337a
Add link to conda issue.
karthiknadig Sep 23, 2022
503c2da
Add doc link for venv binary location.
karthiknadig Sep 23, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,11 @@
"command": "python.createTerminal",
"title": "%python.command.python.createTerminal.title%"
},
{
"category": "Python",
"command": "python.createEnvironment",
"title": "%python.command.python.createEnvironment.title%"
},
{
"category": "Python",
"command": "python.enableLinting",
Expand Down Expand Up @@ -1540,6 +1545,12 @@
"title": "%python.command.python.configureTests.title%",
"when": "!virtualWorkspace && shellExecutionSupported"
},
{
"category": "Python",
"command": "python.createEnvironment",
"title": "%python.command.python.createEnvironment.title%",
"when": "!virtualWorkspace && shellExecutionSupported"
},
{
"category": "Python",
"command": "python.createTerminal",
Expand Down
5 changes: 3 additions & 2 deletions package.nls.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
{
"python.command.python.sortImports.title": "Sort Imports",
"python.command.python.startREPL.title": "Start REPL",
"python.command.python.createEnvironment.title": "Create Environment",
"python.command.python.createTerminal.title": "Create Terminal",
"python.command.python.execInTerminal.title": "Run Python File in Terminal",
"python.command.python.debugInTerminal.title": "Debug Python File",
Expand All @@ -24,15 +25,15 @@
"python.command.python.launchTensorBoard.title": "Launch TensorBoard",
"python.command.python.refreshTensorBoard.title": "Refresh TensorBoard",
"python.menu.createNewFile.title": "Python File",
"python.autoComplete.extraPaths.description":"List of paths to libraries and the like that need to be imported by auto complete engine. E.g. when using Google App SDK, the paths are not in system path, hence need to be added into this list.",
"python.autoComplete.extraPaths.description": "List of paths to libraries and the like that need to be imported by auto complete engine. E.g. when using Google App SDK, the paths are not in system path, hence need to be added into this list.",
"python.condaPath.description": "Path to the conda executable to use for activation (version 4.4+).",
"python.defaultInterpreterPath.description": "Path to default Python to use when extension loads up for the first time, no longer used once an interpreter is selected for the workspace. See https://aka.ms/AAfekmf to understand when this is used",
"python.diagnostics.sourceMapsEnabled.description": "Enable source map support for meaningful stack traces in error logs.",
"python.envFile.description": "Absolute path to a file containing environment variable definitions.",
"python.experiments.enabled.description": "Enables A/B tests experiments in the Python extension. If enabled, you may get included in proposed enhancements and/or features.",
"python.experiments.optInto.description": "List of experiment to opt into. If empty, user is assigned the default experiment groups. See https://github.com/microsoft/vscode-python/wiki/Experiments for more details.",
"python.experiments.optOutFrom.description": "List of experiment to opt out of. If empty, user is assigned the default experiment groups. See https://github.com/microsoft/vscode-python/wiki/Experiments for more details.",
"python.formatting.autopep8Args.description":"Arguments passed in. Each argument is a separate item in the array.",
"python.formatting.autopep8Args.description": "Arguments passed in. Each argument is a separate item in the array.",
"python.formatting.autopep8Path.description": "Path to autopep8, you can use a custom version of autopep8 by modifying this setting to include the full path.",
"python.formatting.blackArgs.description": "Arguments passed in. Each argument is a separate item in the array.",
"python.formatting.blackPath.description": "Path to Black, you can use a custom version of Black by modifying this setting to include the full path.",
Expand Down
128 changes: 128 additions & 0 deletions pythonFiles/create_conda.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import argparse
import os
import pathlib
import subprocess
import sys
from typing import Optional, Sequence, Union

CONDA_ENV_NAME = ".conda"
CWD = pathlib.PurePath(os.getcwd())


class VenvError(Exception):
pass


def parse_args(argv: Sequence[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument(
"--python",
action="store",
help="Python version to install in the virtual environment.",
default=f"{sys.version_info.major}.{sys.version_info.minor}",
)
parser.add_argument(
"--install",
action="store_true",
default=False,
help="Install packages into the virtual environment.",
)
parser.add_argument(
"--git-ignore",
action="store_true",
default=False,
help="Add .gitignore to the newly created virtual environment.",
)
parser.add_argument(
"--name",
default=CONDA_ENV_NAME,
type=str,
help="Name of the virtual environment.",
metavar="NAME",
action="store",
)
return parser.parse_args(argv)


def file_exists(path: Union[str, pathlib.PurePath]) -> bool:
return os.path.exists(path)


def conda_env_exists(name: Union[str, pathlib.PurePath]) -> bool:
return os.path.exists(CWD / name)


def run_process(args: Sequence[str], error_message: str) -> None:
try:
print("Running: " + " ".join(args))
subprocess.run(args, cwd=os.getcwd(), check=True)
except subprocess.CalledProcessError:
raise VenvError(error_message)


def get_conda_env_path(name: str) -> str:
return os.fspath(CWD / name)


def install_packages(env_path: str) -> None:
yml = os.fspath(CWD / "environment.yml")
if file_exists(yml):
print(f"CONDA_INSTALLING_YML: {yml}")
run_process(
[
sys.executable,
"-m",
"conda",
"env",
"update",
"--prefix",
env_path,
"--file",
yml,
],
"CREATE_CONDA.FAILED_INSTALL_YML",
)


def add_gitignore(name: str) -> None:
git_ignore = os.fspath(CWD / name / ".gitignore")
if not file_exists(git_ignore):
print(f"Creating: {git_ignore}")
with open(git_ignore, "w") as f:
f.write("*")


def main(argv: Optional[Sequence[str]] = None) -> None:
if argv is None:
argv = []
args = parse_args(argv)

if not conda_env_exists(args.name):
run_process(
[
sys.executable,
"-m",
"conda",
"create",
"--yes",
"--prefix",
args.name,
f"python={args.python}",
],
"CREATE_CONDA.ENV_FAILED_CREATION",
)
if args.git_ignore:
add_gitignore(args.name)

env_path = get_conda_env_path(args.name)
print(f"CREATED_CONDA_ENV:{env_path}")

if args.install:
install_packages(env_path)


if __name__ == "__main__":
main(sys.argv[1:])
130 changes: 130 additions & 0 deletions pythonFiles/create_venv.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import argparse
import importlib.util as import_util
import os
import pathlib
import subprocess
import sys
from typing import Optional, Sequence, Union

VENV_NAME = ".venv"
CWD = pathlib.PurePath(os.getcwd())


class VenvError(Exception):
pass


def parse_args(argv: Sequence[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument(
"--install",
action="store_true",
default=False,
help="Install packages into the virtual environment.",
)
parser.add_argument(
"--git-ignore",
action="store_true",
default=False,
help="Add .gitignore to the newly created virtual environment.",
)
parser.add_argument(
"--name",
default=VENV_NAME,
type=str,
help="Name of the virtual environment.",
metavar="NAME",
action="store",
)
return parser.parse_args(argv)


def is_installed(module: str) -> bool:
return import_util.find_spec(module) is not None


def file_exists(path: Union[str, pathlib.PurePath]) -> bool:
return os.path.exists(path)


def venv_exists(name: str) -> bool:
return os.path.exists(CWD / name)


def run_process(args: Sequence[str], error_message: str) -> None:
try:
print("Running: " + " ".join(args))
subprocess.run(args, cwd=os.getcwd(), check=True)
except subprocess.CalledProcessError:
raise VenvError(error_message)


def get_venv_path(name: str) -> str:
# See `venv` doc here for more details on binary location:
# https://docs.python.org/3/library/venv.html#creating-virtual-environments
if sys.platform == "win32":
return os.fspath(CWD / name / "Scripts" / "python.exe")

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any documentation for this assumption?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the same assumption we have in the extension. The docs here describe the "Scripts" or "bin" : https://docs.python.org/3/library/venv.html#creating-virtual-environments

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gotcha.

Maybe still add a one line comment linking the doc? I'm asking because it looks like we actually don't use that assumption in discovery and identification:

function getPyvenvConfigPathsFrom(interpreterPath: string): string[] {
const pyvenvConfigFile = 'pyvenv.cfg';
// Check if the pyvenv.cfg file is in the parent directory relative to the interpreter.
// env
// |__ pyvenv.cfg <--- check if this file exists
// |__ bin or Scripts
// |__ python <--- interpreterPath
const venvPath1 = path.join(path.dirname(path.dirname(interpreterPath)), pyvenvConfigFile);
// Check if the pyvenv.cfg file is in the directory as the interpreter.
// env
// |__ pyvenv.cfg <--- check if this file exists
// |__ python <--- interpreterPath
const venvPath2 = path.join(path.dirname(interpreterPath), pyvenvConfigFile);
// The paths are ordered in the most common to least common
return [venvPath1, venvPath2];
}

and this is the first time.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The purpose of the code here and in the discovery is different. In discovery we are tyring to find the .cfg file. in the create env, we created the env, so we know what type it is and where the python.exe will be. I can add the docs nonetheless.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think the comment is misleading:

// envFolder
// |__ pyvenv.cfg  <--- check if this file exists
// |__ python  <--- interpreterPath

It seems to indicate python binary can be directly under the env folder.

else:
return os.fspath(CWD / name / "bin" / "python")


def install_packages(venv_path: str) -> None:
if not is_installed("pip"):
raise VenvError("CREATE_VENV.PIP_NOT_FOUND")

requirements = os.fspath(CWD / "requirements.txt")
pyproject = os.fspath(CWD / "pyproject.toml")

run_process(
[venv_path, "-m", "pip", "install", "--upgrade", "pip"],
"CREATE_VENV.PIP_UPGRADE_FAILED",
)

if file_exists(requirements):
print(f"VENV_INSTALLING_REQUIREMENTS: {requirements}")
run_process(
[venv_path, "-m", "pip", "install", "-r", requirements],
"CREATE_VENV.PIP_FAILED_INSTALL_REQUIREMENTS",
)
elif file_exists(pyproject):
print(f"VENV_INSTALLING_PYPROJECT: {pyproject}")
run_process(
[venv_path, "-m", "pip", "install", "-e", ".[extras]"],
"CREATE_VENV.PIP_FAILED_INSTALL_PYPROJECT",
)


def add_gitignore(name: str) -> None:
git_ignore = CWD / name / ".gitignore"
if not file_exists(git_ignore):
print("Creating: " + os.fspath(git_ignore))
with open(git_ignore, "w") as f:
f.write("*")


def main(argv: Optional[Sequence[str]] = None) -> None:
if argv is None:
argv = []
args = parse_args(argv)

if is_installed("venv"):
if not venv_exists(args.name):
run_process(
[sys.executable, "-m", "venv", args.name],
"CREATE_VENV.VENV_FAILED_CREATION",
)
if args.git_ignore:
add_gitignore(args.name)
venv_path = get_venv_path(args.name)
print(f"CREATED_VENV:{venv_path}")
if args.install:
install_packages(venv_path)
else:
raise VenvError("CREATE_VENV.VENV_NOT_FOUND")


if __name__ == "__main__":
main(sys.argv[1:])
72 changes: 72 additions & 0 deletions pythonFiles/tests/test_create_conda.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import importlib
import sys

import create_conda
import pytest


@pytest.mark.parametrize("env_exists", [True, False])
@pytest.mark.parametrize("git_ignore", [True, False])
@pytest.mark.parametrize("install", [True, False])
@pytest.mark.parametrize("python", [True, False])
def test_create_env(env_exists, git_ignore, install, python):
importlib.reload(create_conda)
create_conda.conda_env_exists = lambda _n: env_exists

install_packages_called = False

def install_packages(_name):
nonlocal install_packages_called
install_packages_called = True

create_conda.install_packages = install_packages

run_process_called = False

def run_process(args, error_message):
nonlocal run_process_called
run_process_called = True
version = (
"12345" if python else f"{sys.version_info.major}.{sys.version_info.minor}"
)
if not env_exists:
assert args == [
sys.executable,
"-m",
"conda",
"create",
"--yes",
"--prefix",
create_conda.CONDA_ENV_NAME,
f"python={version}",
]
assert error_message == "CREATE_CONDA.ENV_FAILED_CREATION"

create_conda.run_process = run_process

add_gitignore_called = False

def add_gitignore(_name):
nonlocal add_gitignore_called
add_gitignore_called = True

create_conda.add_gitignore = add_gitignore

args = []
if git_ignore:
args.append("--git-ignore")
if install:
args.append("--install")
if python:
args.extend(["--python", "12345"])
create_conda.main(args)
assert install_packages_called == install

# run_process is called when the venv does not exist
assert run_process_called != env_exists

# add_gitignore is called when new venv is created and git_ignore is True
assert add_gitignore_called == (not env_exists and git_ignore)
Loading