|
29 | 29 | import sys
|
30 | 30 | import tarfile
|
31 | 31 | import tempfile
|
| 32 | +from functools import cache |
32 | 33 |
|
33 | 34 | from distutils.spawn import find_executable
|
34 | 35 | from threading import Thread
|
|
58 | 59 |
|
59 | 60 | logger = logging.getLogger(__name__)
|
60 | 61 |
|
| 62 | +@cache |
| 63 | +def get_docker_compose_cmd(*args): |
| 64 | + cmd = [] |
| 65 | + # check docker compose v2 first |
| 66 | + try: |
| 67 | + output = subprocess.check_output( |
| 68 | + ['docker', 'compose', 'version'], |
| 69 | + universal_newlines=True, |
| 70 | + ) |
| 71 | + if 'v2' in output.strip(): |
| 72 | + cmd.extend(['docker', 'compose']) |
| 73 | + |
| 74 | + except subprocess.CalledProcessError: |
| 75 | + # check if docker-compose is installed |
| 76 | + if find_executable("docker-compose") is None: |
| 77 | + raise ImportError( |
| 78 | + "'docker-compose' is not installed. " |
| 79 | + "Local Mode features will not work without docker-compose. " |
| 80 | + "For more information on how to install 'docker-compose', please, see " |
| 81 | + "https://docs.docker.com/compose/install/" |
| 82 | + ) |
| 83 | + cmd.append('docker-compose') |
| 84 | + |
| 85 | + cmd.extend(args) |
| 86 | + return cmd |
| 87 | + |
61 | 88 |
|
62 | 89 | class _SageMakerContainer(object):
|
63 | 90 | """Handle the lifecycle and configuration of a local container execution.
|
@@ -93,15 +120,6 @@ def __init__(
|
93 | 120 | """
|
94 | 121 | from sagemaker.local.local_session import LocalSession
|
95 | 122 |
|
96 |
| - # check if docker-compose is installed |
97 |
| - if find_executable("docker-compose") is None: |
98 |
| - raise ImportError( |
99 |
| - "'docker-compose' is not installed. " |
100 |
| - "Local Mode features will not work without docker-compose. " |
101 |
| - "For more information on how to install 'docker-compose', please, see " |
102 |
| - "https://docs.docker.com/compose/install/" |
103 |
| - ) |
104 |
| - |
105 | 123 | self.sagemaker_session = sagemaker_session or LocalSession()
|
106 | 124 | self.instance_type = instance_type
|
107 | 125 | self.instance_count = instance_count
|
@@ -715,17 +733,15 @@ def _compose(self, detached=False):
|
715 | 733 | Args:
|
716 | 734 | detached:
|
717 | 735 | """
|
718 |
| - compose_cmd = "docker-compose" |
719 |
| - |
720 |
| - command = [ |
| 736 | + compose_args = [ |
721 | 737 | compose_cmd,
|
722 | 738 | "-f",
|
723 | 739 | os.path.join(self.container_root, DOCKER_COMPOSE_FILENAME),
|
724 | 740 | "up",
|
725 | 741 | "--build",
|
726 | 742 | "--abort-on-container-exit" if not detached else "--detach", # mutually exclusive
|
727 | 743 | ]
|
728 |
| - |
| 744 | + command = get_docker_compose_cmd(*compose_args) |
729 | 745 | logger.info("docker command: %s", " ".join(command))
|
730 | 746 | return command
|
731 | 747 |
|
@@ -768,7 +784,7 @@ def _create_docker_host(self, host, environment, optml_subdirs, command, volumes
|
768 | 784 | # to setting --runtime=nvidia in the docker commandline.
|
769 | 785 | if self.instance_type == "local_gpu":
|
770 | 786 | host_config["deploy"] = {
|
771 |
| - "resources": {"reservations": {"devices": [{"capabilities": ["gpu"]}]}} |
| 787 | + "resources": {"reservations": {"devices": [{"driver": "nvidia", "count": "all", "capabilities": ["gpu"]}]}} |
772 | 788 | }
|
773 | 789 |
|
774 | 790 | if command == "serve":
|
|
0 commit comments