diff --git a/.ci/scripts/setup-linux.sh b/.ci/scripts/setup-linux.sh index 776bf6f7953..a090571ab49 100755 --- a/.ci/scripts/setup-linux.sh +++ b/.ci/scripts/setup-linux.sh @@ -10,19 +10,17 @@ set -exu # shellcheck source=/dev/null source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" -BUILD_TOOL=$1 -if [[ -z "${BUILD_TOOL:-}" ]]; then - echo "Missing build tool (require buck2 or cmake), exiting..." - exit 1 -else - echo "Setup Linux for ${BUILD_TOOL} ..." -fi +read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@") # As Linux job is running inside a Docker container, all of its dependencies # have already been installed, so we use PyTorch build from source here instead # of nightly. This allows CI to test against latest commits from PyTorch -install_executorch "use-pt-pinned-commit" -build_executorch_runner "${BUILD_TOOL}" "${2:-Release}" +if [[ "${EDITABLE:-false}" == "true" ]]; then + install_executorch --use-pt-pinned-commit --editable +else + install_executorch --use-pt-pinned-commit +fi +build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}" if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then do_not_use_nightly_on_ci diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index bb8e45f23f1..4b43a730710 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -10,13 +10,7 @@ set -exu # shellcheck source=/dev/null source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" -BUILD_TOOL=$1 -if [[ -z "${BUILD_TOOL:-}" ]]; then - echo "Missing build tool (require buck2 or cmake), exiting..." - exit 1 -else - echo "Setup MacOS for ${BUILD_TOOL} ..." -fi +read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@") install_buck() { if ! command -v zstd &> /dev/null; then @@ -135,8 +129,12 @@ print_cmake_info install_pytorch_and_domains # We build PyTorch from source here instead of using nightly. This allows CI to test against # the pinned commit from PyTorch -install_executorch "use-pt-pinned-commit" -build_executorch_runner "${BUILD_TOOL}" "${2:-Release}" +if [[ "$EDITABLE" == "true" ]]; then + install_executorch --use-pt-pinned-commit --editable +else + install_executorch --use-pt-pinned-commit +fi +build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}" if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then do_not_use_nightly_on_ci diff --git a/.ci/scripts/unittest-linux.sh b/.ci/scripts/unittest-linux.sh index 27da8d4e4f9..f8ff9df773e 100755 --- a/.ci/scripts/unittest-linux.sh +++ b/.ci/scripts/unittest-linux.sh @@ -6,21 +6,10 @@ # LICENSE file in the root directory of this source tree. set -eux -BUILD_TOOL=$1 -if [[ $BUILD_TOOL =~ ^(cmake|buck2)$ ]]; then - echo "Running unittests for ${BUILD_TOOL} ..." -else - echo "Missing build tool (require buck2 or cmake), exiting..." - exit 1 -fi +# shellcheck source=/dev/null +source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" -BUILD_MODE=$2 -if [[ "${BUILD_MODE:-}" =~ ^(Debug|Release)$ ]]; then - echo "Running tests in build mode ${BUILD_MODE} ..." -else - echo "Unsupported build mode ${BUILD_MODE}, options are Debug or Release." - exit 1 -fi +read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@") # The generic Linux job chooses to use base env, not the one setup by the image eval "$(conda shell.bash hook)" @@ -34,7 +23,7 @@ if [[ "$BUILD_TOOL" == "cmake" ]]; then PYTHON_EXECUTABLE=python \ EXECUTORCH_BUILD_PYBIND=ON \ CMAKE_ARGS="-DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \ - .ci/scripts/setup-linux.sh "$BUILD_TOOL" "$BUILD_MODE" + .ci/scripts/setup-linux.sh "$@" # Install llama3_2_vision dependencies. PYTHON_EXECUTABLE=python ./examples/models/llama3_2_vision/install_requirements.sh @@ -45,7 +34,7 @@ elif [[ "$BUILD_TOOL" == "buck2" ]]; then # because TMPDIR gets messed up? Please feel free to fix this and # speed up this CI job! PYTHON_EXECUTABLE=python \ - .ci/scripts/setup-linux.sh "$BUILD_TOOL" "$BUILD_MODE" + .ci/scripts/setup-linux.sh "$@" .ci/scripts/unittest-buck2.sh else diff --git a/.ci/scripts/unittest-macos.sh b/.ci/scripts/unittest-macos.sh index 9f7fafa35ce..960f69389ce 100755 --- a/.ci/scripts/unittest-macos.sh +++ b/.ci/scripts/unittest-macos.sh @@ -6,21 +6,10 @@ # LICENSE file in the root directory of this source tree. set -eux -BUILD_TOOL=$1 -if [[ $BUILD_TOOL =~ ^(cmake|buck2)$ ]]; then - echo "Running unittests for ${BUILD_TOOL} ..." -else - echo "Missing build tool (require buck2 or cmake), exiting..." - exit 1 -fi +# shellcheck source=/dev/null +source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" -BUILD_MODE=$2 -if [[ $BUILD_MODE =~ ^(Debug|Release)$ ]]; then - echo "Running tests in build mode ${BUILD_MODE} ..." -else - echo "Unsupported build mode ${BUILD_MODE}, options are Debug or Release." - exit 1 -fi +read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@") bash .ci/scripts/setup-conda.sh eval "$(conda shell.bash hook)" @@ -36,7 +25,7 @@ if [[ "$BUILD_TOOL" == "cmake" ]]; then EXECUTORCH_BUILD_PYBIND=ON \ CMAKE_ARGS="-DEXECUTORCH_BUILD_COREML=ON -DEXECUTORCH_BUILD_MPS=ON -DEXECUTORCH_BUILD_XNNPACK=ON -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON" \ ${CONDA_RUN} --no-capture-output \ - .ci/scripts/setup-macos.sh "${BUILD_TOOL}" "${BUILD_MODE}" + .ci/scripts/setup-macos.sh "$@" # Install llama3_2_vision dependencies. PYTHON_EXECUTABLE=python \ diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index e0bc935e861..8d99a0022cf 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -24,11 +24,7 @@ install_executorch() { which pip # Install executorch, this assumes that Executorch is checked out in the # current directory. - if [[ "${1:-}" == "use-pt-pinned-commit" ]]; then - ./install_executorch.sh --pybind xnnpack --use-pt-pinned-commit - else - ./install_executorch.sh --pybind xnnpack - fi + ./install_executorch.sh --pybind xnnpack "$@" # Just print out the list of packages for debugging pip list } @@ -166,3 +162,52 @@ do_not_use_nightly_on_ci() { exit 1 fi } + + +parse_args() { + local args=("$@") + local i + local BUILD_TOOL="" + local BUILD_MODE="" + local EDITABLE="" + for ((i=0; i<${#args[@]}; i++)); do + case "${args[$i]}" in + --build-tool) + BUILD_TOOL="${args[$((i+1))]}" + i=$((i+1)) + ;; + --build-mode) + BUILD_MODE="${args[$((i+1))]}" + i=$((i+1)) + ;; + --editable) + EDITABLE="${args[$((i+1))]}" + i=$((i+1)) + ;; + *) + echo "Invalid argument: ${args[$i]}" + exit 1 + ;; + esac + done + + if [ -z "$BUILD_TOOL" ]; then + echo "Missing build tool (require buck2 or cmake), exiting..." + exit 1 + elif ! [[ $BUILD_TOOL =~ ^(cmake|buck2)$ ]]; then + echo "Require buck2 or cmake for --build-tool, got ${BUILD_TOOL}, exiting..." + exit 1 + fi + BUILD_MODE="${BUILD_MODE:-Release}" + if ! [[ "$BUILD_MODE" =~ ^(Debug|Release)$ ]]; then + echo "Unsupported build mode ${BUILD_MODE}, options are Debug or Release." + exit 1 + fi + EDITABLE="${EDITABLE:-false}" + if ! [[ $EDITABLE =~ ^(true|false)$ ]]; then + echo "Require true or false for --editable, got ${EDITABLE}, exiting..." + exit 1 + fi + + echo "$BUILD_TOOL $BUILD_MODE $EDITABLE" +} diff --git a/.github/workflows/_android.yml b/.github/workflows/_android.yml index fa7c331311f..82e49d6672e 100644 --- a/.github/workflows/_android.yml +++ b/.github/workflows/_android.yml @@ -25,7 +25,7 @@ jobs: # The generic Linux job chooses to use base env, not the one setup by the image CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh buck2 + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool buck2 export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android diff --git a/.github/workflows/_unittest.yml b/.github/workflows/_unittest.yml index f2eb2cfdb51..6b08b6d1259 100644 --- a/.github/workflows/_unittest.yml +++ b/.github/workflows/_unittest.yml @@ -15,6 +15,10 @@ on: required: true type: string description: Build tool to use, cmake or buck2. + editable: + required: false + type: string + description: Install ExecuTorch in editable mode or not. python-version: required: false type: string @@ -34,7 +38,7 @@ jobs: timeout: 90 script: | set -eux - .ci/scripts/unittest-linux.sh "${{ inputs.build-tool }}" "${{ inputs.build-mode }}" + .ci/scripts/unittest-linux.sh --build-tool "${{ inputs.build-tool }}" --build-mode "${{ inputs.build-mode }}" --editable "${{ inputs.editable }}" macos: uses: pytorch/test-infra/.github/workflows/macos_job.yml@main @@ -45,4 +49,4 @@ jobs: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | set -eux - .ci/scripts/unittest-macos.sh "${{ inputs.build-tool }}" "${{ inputs.build-mode }}" + .ci/scripts/unittest-macos.sh --build-tool "${{ inputs.build-tool }}" --build-mode "${{ inputs.build-mode }}" --editable "${{ inputs.editable }}" diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index 201fb3b7a8f..00ba907845a 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -181,7 +181,7 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh fi - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake" # Install requirements for export_llama PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh @@ -356,7 +356,7 @@ jobs: # The generic Linux job chooses to use base env, not the one setup by the image CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh cmake + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml index 8d2c1d354cc..26423e59233 100644 --- a/.github/workflows/android-release-artifacts.yml +++ b/.github/workflows/android-release-artifacts.yml @@ -49,7 +49,7 @@ jobs: # The generic Linux job chooses to use base env, not the one setup by the image CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh buck2 + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool buck2 export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml index ea88be441cb..ccdd24e8ed8 100644 --- a/.github/workflows/apple-perf.yml +++ b/.github/workflows/apple-perf.yml @@ -181,7 +181,7 @@ jobs: BUILD_TOOL=cmake # Setup MacOS dependencies as there is no Docker support on MacOS atm GITHUB_RUNNER=1 PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \ - .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" if [[ ${{ matrix.config }} == *"coreml"* ]]; then PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \ @@ -367,7 +367,7 @@ jobs: BUILD_TOOL=cmake # Setup MacOS dependencies as there is no Docker support on MacOS atm GITHUB_RUNNER=1 PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \ - .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Setup Apple certificate for iOS development diff --git a/.github/workflows/apple.yml b/.github/workflows/apple.yml index 6929e12fa6d..19460a35379 100644 --- a/.github/workflows/apple.yml +++ b/.github/workflows/apple.yml @@ -69,7 +69,7 @@ jobs: # Setup MacOS dependencies as there is no Docker support on MacOS atm GITHUB_RUNNER=1 PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \ - .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded @@ -160,7 +160,7 @@ jobs: # Setup MacOS dependencies as there is no Docker support on MacOS atm GITHUB_RUNNER=1 PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \ - .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" # Install CoreML Backend Requirements PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \ @@ -293,7 +293,7 @@ jobs: BUILD_TOOL=cmake # Setup MacOS dependencies as there is no Docker support on MacOS atm GITHUB_RUNNER=1 PYTHON_EXECUTABLE=python ${CONDA_RUN} --no-capture-output \ - .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Setup Apple certificate for iOS development diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 8d9081615be..aa6f164d7b6 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -38,7 +38,7 @@ jobs: BUILD_TOOL=${{ matrix.build-tool }} # Setup dependencies as there is no Docker support - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then export CHANNEL=test diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 53d2bd7910b..69fcc11a347 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -35,7 +35,7 @@ jobs: # For mypy linting, we need to first install executorch first so that # it builds the python package information. BUILD_TOOL="cmake" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" CACHE_DIRECTORY="/tmp/.lintbin" # Try to recover the cached binaries diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml index 6b4644bb522..fa3fa6e1cd2 100644 --- a/.github/workflows/periodic.yml +++ b/.github/workflows/periodic.yml @@ -63,6 +63,6 @@ jobs: BACKEND=${{ matrix.backend }} DEMO_BACKEND_DELEGATION=${{ matrix.demo_backend_delegation }} - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Build and test ExecuTorch PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" "${DEMO_BACKEND_DELEGATION}" diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index d9a7aa6bd5b..2a2109bcdc1 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -52,34 +52,10 @@ jobs: BUILD_TOOL="cmake" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Build and test ExecuTorch with the add model on portable backend. PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "add" "${BUILD_TOOL}" "portable" - test-pip-install-editable-mode-linux: - name: test-pip-install-editable-mode-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - with: - runner: linux.2xlarge - docker-image: executorch-ubuntu-22.04-clang12 - submodules: 'true' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 90 - script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - # Debug - which pip - PYTHON_EXECUTABLE=python bash ./install_executorch.sh --editable --pybind xnnpack --use-pt-pinned-commit - # Try to import extension library - python -c "from executorch.extension.llm.custom_ops import custom_ops" - test-models-linux: name: test-models-linux uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main @@ -106,7 +82,7 @@ jobs: BACKEND=${{ matrix.backend }} DEMO_BACKEND_DELEGATION=${{ matrix.demo_backend_delegation }} - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Build and test ExecuTorch PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" "${DEMO_BACKEND_DELEGATION}" @@ -146,7 +122,7 @@ jobs: ARTIFACTS_DIR_NAME="${ARTIFACTS_DIR_NAME/+/-}" # Setup executorch - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Install requirements for export_llama PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh # Test llama2 @@ -172,7 +148,7 @@ jobs: conda activate "${CONDA_ENV}" source .ci/scripts/utils.sh - install_executorch "use-pt-pinned-commit" + install_executorch "--use-pt-pinned-commit" BUILD_TOOL="cmake" PYTHON_EXECUTABLE=python \ bash .ci/scripts/build_llama_android.sh "${BUILD_TOOL}" @@ -197,7 +173,7 @@ jobs: conda activate "${CONDA_ENV}" BUILD_TOOL="cmake" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Test custom ops PYTHON_EXECUTABLE=python bash examples/portable/custom_ops/test_custom_ops.sh "${BUILD_TOOL}" @@ -221,7 +197,7 @@ jobs: conda activate "${CONDA_ENV}" BUILD_TOOL="cmake" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Test selective build PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}" @@ -244,7 +220,7 @@ jobs: CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake" # install Llava requirements bash examples/models/llama/install_requirements.sh @@ -276,7 +252,7 @@ jobs: conda activate "${CONDA_ENV}" BUILD_TOOL="cmake" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" PYTHON_EXECUTABLE=python bash examples/xnnpack/quantization/test_quantize.sh "${BUILD_TOOL}" mv2 test-pybind-build-linux: @@ -303,7 +279,7 @@ jobs: PYTHON_EXECUTABLE=python \ EXECUTORCH_BUILD_XNNPACK=ON \ EXECUTORCH_BUILD_PYBIND=ON \ - bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # see if we can import the module successfully python -c "from executorch.extension.pybindings import portable_lib; print('success!')" @@ -429,7 +405,7 @@ jobs: PYTHON_EXECUTABLE=python \ EXECUTORCH_BUILD_PYBIND=ON \ EXECUTORCH_BUILD_ARM_BAREMETAL=ON \ - .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Install Arm dependencies .ci/scripts/setup-arm-baremetal-tools.sh @@ -469,7 +445,7 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh # Setup executorch - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Install requirements for export_llama PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh # Test llama2 @@ -500,7 +476,7 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh # Setup executorch - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Setup install_requirements for llama PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh @@ -549,7 +525,7 @@ jobs: CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake" # install phi-3-mini requirements bash examples/models/phi-3-mini/install_requirements.sh @@ -576,7 +552,7 @@ jobs: CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake" # install llama requirements bash examples/models/llama/install_requirements.sh @@ -603,7 +579,7 @@ jobs: CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake" # install llama requirements bash examples/models/llama/install_requirements.sh @@ -630,7 +606,7 @@ jobs: CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "cmake" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake" # install llama requirements bash examples/models/llama/install_requirements.sh diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index d8ec745b75c..410e95d9a84 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -36,31 +36,6 @@ jobs: PYTHONPATH="${PWD}" python .ci/scripts/gather_test_models.py --target-os macos --event "${GITHUB_EVENT_NAME}" - test-pip-install-editable-mode-macos: - name: test-pip-install-editable-mode-macos - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - permissions: - id-token: write - contents: read - strategy: - fail-fast: false - with: - runner: macos-m1-stable - python-version: '3.11' - submodules: 'true' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 90 - script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - # Debug - which pip - bash .ci/scripts/setup-conda.sh - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash ./install_executorch.sh --editable --pybind xnnpack - # Try to import extension library - python -c "from executorch.extension.llm.custom_ops import custom_ops" - test-models-macos: name: test-models-macos uses: pytorch/test-infra/.github/workflows/macos_job.yml@main @@ -82,7 +57,7 @@ jobs: bash .ci/scripts/setup-conda.sh # Setup MacOS dependencies as there is no Docker support on MacOS atm - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" # Build and test executorch PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" "${DEMO_BACKEND_DELEGATION}" @@ -104,7 +79,7 @@ jobs: bash .ci/scripts/setup-conda.sh # Setup MacOS dependencies as there is no Docker support on MacOS atm - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" # Build and test custom ops PYTHON_EXECUTABLE=python ${CONDA_RUN} bash examples/portable/custom_ops/test_custom_ops.sh "${BUILD_TOOL}" @@ -126,7 +101,7 @@ jobs: bash .ci/scripts/setup-conda.sh # Setup MacOS dependencies as there is no Docker support on MacOS atm - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" # Build and test selective build PYTHON_EXECUTABLE=python ${CONDA_RUN} bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}" @@ -153,7 +128,7 @@ jobs: conda activate "${CONDA_ENV}" BUILD_TOOL=${{ matrix.build-tool }} - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Test selective build PYTHON_EXECUTABLE=python bash examples/portable/scripts/test_demo_backend_delegation.sh "${BUILD_TOOL}" @@ -175,7 +150,7 @@ jobs: conda activate "${CONDA_ENV}" source .ci/scripts/utils.sh - install_executorch "use-pt-pinned-commit" + install_executorch "--use-pt-pinned-commit" .ci/scripts/setup-arm-baremetal-tools.sh @@ -205,7 +180,7 @@ jobs: conda activate "${CONDA_ENV}" source .ci/scripts/utils.sh - install_executorch "use-pt-pinned-commit" + install_executorch "--use-pt-pinned-commit" .ci/scripts/setup-arm-baremetal-tools.sh @@ -226,7 +201,7 @@ jobs: bash .ci/scripts/setup-conda.sh # Setup MacOS dependencies as there is no Docker support on MacOS atm - GITHUB_RUNNER=1 PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + GITHUB_RUNNER=1 PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" # Build and test coreml delegate PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/build_all.sh @@ -249,7 +224,7 @@ jobs: # build module for executorch.extension.pybindings.portable_lib BUILD_TOOL=${{ matrix.build-tool }} - EXECUTORCH_BUILD_PYBIND=ON PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + EXECUTORCH_BUILD_PYBIND=ON PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" # see if we can import the module successfully ${CONDA_RUN} python -c "from executorch.extension.pybindings import portable_lib; print('success!')" @@ -281,7 +256,7 @@ jobs: bash .ci/scripts/setup-conda.sh # Setup executorch - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh cmake + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool cmake if [[ "${MODE}" == "mps" ]]; then # Install mps delegate @@ -315,7 +290,7 @@ jobs: # bash .ci/scripts/setup-conda.sh # # Setup MacOS dependencies as there is no Docker support on MacOS atm - # GITHUB_RUNNER=1 PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + # GITHUB_RUNNER=1 PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" # # install Llava requirements # ${CONDA_RUN} bash examples/models/llama/install_requirements.sh @@ -348,7 +323,7 @@ jobs: # The generic Linux job chooses to use base env, not the one setup by the image CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh cmake + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh ${{ matrix.model }} "cmake" "qnn" @@ -370,7 +345,7 @@ jobs: bash .ci/scripts/setup-conda.sh # Setup MacOS dependencies as there is no Docker support on MacOS atm - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/install_requirements.sh echo "Finishing installing coreml." PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/mps/install_requirements.sh @@ -419,7 +394,7 @@ jobs: # The generic Linux job chooses to use base env, not the one setup by the image CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") conda activate "${CONDA_ENV}" - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh cmake + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake echo "::endgroup::" echo "::group::Set up Hugging Face" @@ -488,7 +463,7 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh # Setup executorch - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" # Install requirements for export_llama PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh # Test llama2