From 04755bd33e371c6772057f27002e4c237dca9191 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Thu, 18 Jul 2024 13:21:24 -0700 Subject: [PATCH 1/6] xnnpack test --- .gitmodules | 2 +- backends/xnnpack/third-party/XNNPACK | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9164ee5700a..32e67ccb8a6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -27,7 +27,7 @@ url = https://github.com/Maratyszcza/FXdiv.git [submodule "backends/xnnpack/third-party/XNNPACK"] path = backends/xnnpack/third-party/XNNPACK - url = https://github.com/digantdesai/XNNPACK.git + url = https://github.com/kirklandsign/XNNPACK.git [submodule "backends/arm/third-party/serialization_lib"] path = backends/arm/third-party/serialization_lib url = https://review.mlplatform.org/tosa/serialization_lib diff --git a/backends/xnnpack/third-party/XNNPACK b/backends/xnnpack/third-party/XNNPACK index 20c0d886fb7..6dd7e0b25f2 160000 --- a/backends/xnnpack/third-party/XNNPACK +++ b/backends/xnnpack/third-party/XNNPACK @@ -1 +1 @@ -Subproject commit 20c0d886fb78d6497362e8303b999bf5d67aaa02 +Subproject commit 6dd7e0b25f29143bd9e329875d723b904206de1d From 9de7c846b0c4e3b816e99e2c4d74214cd8433ba8 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Thu, 18 Jul 2024 14:41:50 -0700 Subject: [PATCH 2/6] Update android device test --- .github/workflows/android.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 1853afcddd5..cfa43760a63 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -76,3 +76,35 @@ jobs: retention-days: 14 if-no-files-found: ignore path: ${{ runner.temp }}/artifacts/ + + # Let's see how expensive this job is, we might want to tone it down by running it periodically + test-llama-app: + needs: upload-artifacts + permissions: + id-token: write + contents: read + uses: pytorch/test-infra/.github/workflows/mobile_job.yml@main + strategy: + matrix: + # https://github.com/pytorch/executorch/blob/main/examples/demo-apps/android/LlamaDemo/README.md#alternative-2-build-from-local-machine + # mentions that tiktoken is only for Llama3. So, we can export it later in another archive + # like https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b-0717.zip when this is + # updated to run Llama3 + tokenizer: [bpe] + with: + device-type: android + runner: linux.2xlarge + test-infra-ref: '' + # This is the ARN of ExecuTorch project on AWS + project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:02a2cf0f-6d9b-45ee-ba1a-a086587469e6 + # This is the custom Android device pool that only includes Samsung Galaxy S2x + device-pool-arn: arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa + # Uploaded to S3 from the previous job, the name of the app comes from the project itself + android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug.apk + android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug-androidTest.apk + # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml + test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/abd86868-fa63-467e-a5c7-218194665a77 + # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. + # Among the input, this is the biggest file, so it is cached on AWS to make the test faster. Note that the file is deleted by AWS after 30 + # days and the job will automatically re-upload the file when that happens. + extra-data: https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b-0717.zip From 003ceee71ce21b12a8fe129aceb0a3561c5a9e04 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Fri, 19 Jul 2024 14:08:14 -0700 Subject: [PATCH 3/6] trigger again --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index cfa43760a63..878b46c03e2 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -91,7 +91,7 @@ jobs: # like https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b-0717.zip when this is # updated to run Llama3 tokenizer: [bpe] - with: + with: device-type: android runner: linux.2xlarge test-infra-ref: '' From f8cd915ec2cb421787984eb96259e173f600fa7c Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 23 Jul 2024 07:58:39 -0700 Subject: [PATCH 4/6] update xnnpack --- .gitmodules | 2 +- backends/xnnpack/third-party/XNNPACK | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 32e67ccb8a6..9164ee5700a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -27,7 +27,7 @@ url = https://github.com/Maratyszcza/FXdiv.git [submodule "backends/xnnpack/third-party/XNNPACK"] path = backends/xnnpack/third-party/XNNPACK - url = https://github.com/kirklandsign/XNNPACK.git + url = https://github.com/digantdesai/XNNPACK.git [submodule "backends/arm/third-party/serialization_lib"] path = backends/arm/third-party/serialization_lib url = https://review.mlplatform.org/tosa/serialization_lib diff --git a/backends/xnnpack/third-party/XNNPACK b/backends/xnnpack/third-party/XNNPACK index 6dd7e0b25f2..1d139a3b4b7 160000 --- a/backends/xnnpack/third-party/XNNPACK +++ b/backends/xnnpack/third-party/XNNPACK @@ -1 +1 @@ -Subproject commit 6dd7e0b25f29143bd9e329875d723b904206de1d +Subproject commit 1d139a3b4b7155889c88c31f370a82c48e7ca89c From 652e269cd9d4e6020d51f2093097bb907faa4207 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 23 Jul 2024 08:15:34 -0700 Subject: [PATCH 5/6] lint --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 878b46c03e2..cfa43760a63 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -91,7 +91,7 @@ jobs: # like https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b-0717.zip when this is # updated to run Llama3 tokenizer: [bpe] - with: + with: device-type: android runner: linux.2xlarge test-infra-ref: '' From 0912fc84f8351758e9af75c7c81a9f4c07f9cdb0 Mon Sep 17 00:00:00 2001 From: Hansong <107070759+kirklandsign@users.noreply.github.com> Date: Tue, 23 Jul 2024 10:25:30 -0700 Subject: [PATCH 6/6] Update android.yml --- .github/workflows/android.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index cfa43760a63..1533bec3fd2 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -104,7 +104,6 @@ jobs: android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug-androidTest.apk # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/abd86868-fa63-467e-a5c7-218194665a77 - # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. # Among the input, this is the biggest file, so it is cached on AWS to make the test faster. Note that the file is deleted by AWS after 30 # days and the job will automatically re-upload the file when that happens. extra-data: https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b-0717.zip