Skip to content

Commit 379c7d8

Browse files
zingozingo
authored
Merge branch 'main' into Arm-backend-Add-support-for-ETDump-of-outputs
2 parents 21c2086 + 897b0d5 commit 379c7d8

File tree

176 files changed

+23436
-1177
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

176 files changed

+23436
-1177
lines changed

‎.ci/scripts/unittest-buck2.sh

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ set -eux
99
# TODO: expand this to //...
1010
# TODO: can't query cadence & vulkan backends
1111
# TODO: can't query //kernels/prim_ops because of non-buckified stuff in OSS.
12-
buck2 query "//backends/apple/... + //backends/example/... + \
12+
# TODO: Make //backends/arm tests use runtime wrapper so we can just query //backends/arm/...
13+
buck2 query "//backends/apple/... + //backends/arm: + //backends/arm/debug/... + \
14+
//backends/arm/operator_support/... + //backends/arm/operators/... + \
15+
//backends/arm/_passes/... + //backends/arm/runtime/... + //backends/arm/tosa/... \
16+
+ //backends/example/... + \
1317
//backends/mediatek/... + //backends/transforms/... + \
1418
//backends/xnnpack/... + //configurations/... + //extension/flat_tensor: + \
1519
//extension/llm/runner: + //kernels/aten/... + //kernels/optimized/... + \

‎.github/workflows/_get-changed-files.yml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: Get Changed Files
2+
3+
on:
4+
workflow_call:
5+
outputs:
6+
changed-files:
7+
description: "List of changed files (space-separated) or '*' if not in a PR"
8+
value: ${{ jobs.get-changed-files.outputs.changed-files }}
9+
10+
jobs:
11+
get-changed-files:
12+
runs-on: ubuntu-latest
13+
outputs:
14+
changed-files: ${{ steps.get-files.outputs.changed-files }}
15+
16+
steps:
17+
- name: Get changed files
18+
id: get-files
19+
env:
20+
GH_TOKEN: ${{ github.token }}
21+
run: |
22+
# Check if we're in a pull request context
23+
if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "pull_request_target" ]; then
24+
echo "Running in PR context"
25+
26+
# Get the PR number from the github context
27+
PR_NUMBER="${{ github.event.number }}"
28+
29+
# Use gh CLI to get changed files in the PR with explicit repo
30+
CHANGED_FILES=$(gh api repos/${{ github.repository }}/pulls/$PR_NUMBER/files --paginate --jq '.[] | select(.status != "removed") | .filename' | tr '\n' ' ' | sed 's/ $//')
31+
32+
if [ -z "$CHANGED_FILES" ]; then
33+
echo "No changed files found, setting to '*'"
34+
CHANGED_FILES="*"
35+
fi
36+
37+
echo "Changed files: $CHANGED_FILES"
38+
echo "changed-files=$CHANGED_FILES" >> "$GITHUB_OUTPUT"
39+
40+
else
41+
echo "Not in PR context, setting changed files to '*'"
42+
echo "changed-files=*" >> "$GITHUB_OUTPUT"
43+
fi

‎.github/workflows/lint.yml

Lines changed: 68 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,23 @@ concurrency:
1515
cancel-in-progress: true
1616

1717
jobs:
18-
lintrunner:
18+
get-changed-files:
19+
if: github.repository_owner == 'pytorch'
20+
name: Get changed files
21+
uses: ./.github/workflows/_get-changed-files.yml
22+
23+
lintrunner-mypy:
1924
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
25+
needs: [get-changed-files]
2026
permissions:
2127
id-token: write
2228
contents: read
29+
if: |
30+
github.repository_owner == 'pytorch' && (
31+
needs.get-changed-files.outputs.changed-files == '*' ||
32+
contains(needs.get-changed-files.outputs.changed-files, '.py') ||
33+
contains(needs.get-changed-files.outputs.changed-files, '.pyi')
34+
)
2335
with:
2436
runner: linux.2xlarge
2537
docker-image: ci-image:executorch-ubuntu-22.04-linter
@@ -45,14 +57,64 @@ jobs:
4557
cp -r "${CACHE_DIRECTORY}" . || true
4658
fi
4759
48-
# This has already been cached in the docker image
49-
lintrunner init
60+
RC=0
61+
# Run lintrunner on all files. pytorch/pytorch notes that mypy
62+
# in particular needs this.
63+
if ! lintrunner --force-color --all-files --take MYPY --tee-json=lint.json 2> /dev/null; then
64+
echo ""
65+
echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner --take MYPY\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m"
66+
echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m"
67+
RC=1
68+
fi
69+
70+
# Use jq to massage the JSON lint output into GitHub Actions workflow commands.
71+
jq --raw-output \
72+
'"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \
73+
lint.json || true
74+
75+
exit $RC
76+
77+
lintrunner:
78+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
79+
needs: [get-changed-files]
80+
permissions:
81+
id-token: write
82+
contents: read
83+
with:
84+
runner: linux.2xlarge
85+
docker-image: ci-image:executorch-ubuntu-22.04-linter
86+
submodules: false
87+
fetch-depth: 0
88+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
89+
timeout: 90
90+
script: |
91+
# The generic Linux job chooses to use base env, not the one setup by the image
92+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
93+
conda activate "${CONDA_ENV}"
94+
95+
# Not sure why this isn't set up in the docker
96+
# image. lintrunner-mypy seems to work becaus setup-linux.sh
97+
# does this as part of install_executorch.
98+
pip install -r requirements-dev.txt
99+
100+
CACHE_DIRECTORY="/tmp/.lintbin"
101+
# Try to recover the cached binaries
102+
if [[ -d "${CACHE_DIRECTORY}" ]]; then
103+
# It's ok to fail this as lintrunner init would download these binaries
104+
# again if they do not exist
105+
cp -r "${CACHE_DIRECTORY}" . || true
106+
fi
50107
51108
RC=0
52-
# Run lintrunner on all files
53-
if ! lintrunner --force-color --all-files --tee-json=lint.json 2> /dev/null; then
109+
CHANGED_FILES="${{ needs.get-changed-files.outputs.changed-files }}"
110+
if [ "$CHANGED_FILES" = '*' ]; then
111+
LINTRUNNER_FILES="--all-files"
112+
else
113+
LINTRUNNER_FILES="${CHANGED_FILES}"
114+
fi
115+
if ! lintrunner --force-color ${LINTRUNNER_FILES} --skip MYPY --tee-json=lint.json 2> /dev/null; then
54116
echo ""
55-
echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m"
117+
echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner --skip MYPY\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m"
56118
echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m"
57119
RC=1
58120
fi

‎.github/workflows/pending_user_response.py

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import datetime
2+
import os
3+
4+
from github import Github
5+
6+
REPO_NAME = "pytorch/executorch"
7+
LABEL = "need-user-input"
8+
REMINDER_MARKER = "<!-- executorch-auto-reminder -->"
9+
REMINDER_COMMENT = (
10+
f"{REMINDER_MARKER}\nHi @{0}, this issue/PR has been marked as 'need-user-input'. "
11+
"Please respond or provide input. If we don't hear back in 30 days, this will be closed."
12+
)
13+
CLOSE_COMMENT = (
14+
f"{REMINDER_MARKER}\nClosing due to no response after 30 days. "
15+
"If you still need help, feel free to re-open or comment again!"
16+
)
17+
DAYS_BEFORE_REMINDER = 30
18+
DAYS_BEFORE_CLOSE = 30
19+
REMINDER_COOLDOWN_DAYS = 7 # Don't post another reminder within 7 days
20+
21+
22+
def main():
23+
g = Github(os.environ["GH_TOKEN"])
24+
repo = g.get_repo(REPO_NAME)
25+
26+
print("[VALIDATION] Would connect to Github and fetch repo:", REPO_NAME)
27+
issues = repo.get_issues(state="open", labels=[LABEL])
28+
print(f"[VALIDATION] Would fetch open issues with label '{LABEL}'.")
29+
30+
now = datetime.datetime.utcnow()
31+
32+
for issue in issues:
33+
print(f"[VALIDATION] Would fetch comments for issue/PR #{issue.number}.")
34+
comments = [] # Replace with mock comments if needed
35+
last_comment = comments[-1] if comments else None
36+
37+
# Find automation comments
38+
auto_comments = [c for c in comments if REMINDER_MARKER in c.body]
39+
user_comments = [c for c in comments if REMINDER_MARKER not in c.body]
40+
41+
# ---- REMINDER LOGIC ----
42+
# Only remind if NO reminder in last 7 days
43+
recent_auto_reminder = any(
44+
(now - c.created_at).days < REMINDER_COOLDOWN_DAYS for c in auto_comments
45+
)
46+
47+
if not auto_comments:
48+
if (
49+
last_comment
50+
and (now - last_comment.created_at).days >= DAYS_BEFORE_REMINDER
51+
):
52+
user = issue.user.login
53+
print(f"[VALIDATION] Would remind {user} on issue/PR #{issue.number}")
54+
elif auto_comments and not recent_auto_reminder:
55+
# Only post new reminder if last was > REMINDER_COOLDOWN_DAYS ago
56+
last_auto = auto_comments[-1]
57+
user = issue.user.login
58+
if (now - last_auto.created_at).days >= REMINDER_COOLDOWN_DAYS:
59+
print(
60+
f"[VALIDATION] Would remind {user} again on issue/PR #{issue.number}"
61+
)
62+
63+
# ---- EXISTING CLOSE/REMOVE LABEL LOGIC ----
64+
if auto_comments:
65+
last_auto = auto_comments[-1]
66+
user_responded = any(
67+
c.created_at > last_auto.created_at and c.user.login == issue.user.login
68+
for c in user_comments
69+
)
70+
if not user_responded:
71+
if (now - last_auto.created_at).days >= DAYS_BEFORE_CLOSE:
72+
print(
73+
f"[VALIDATION] Would close issue/PR #{issue.number} due to inactivity."
74+
)
75+
else:
76+
print(
77+
f"[VALIDATION] Would remove label from issue/PR #{issue.number} after user response."
78+
)
79+
80+
81+
if __name__ == "__main__":
82+
main()

‎.github/workflows/pending_user_response.yml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Needs User Input Automation
2+
3+
on:
4+
schedule:
5+
- cron: '0 8 * * 1' # runs every Monday at 8:00 UTC
6+
workflow_dispatch:
7+
8+
jobs:
9+
needs-user-input:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout repo
13+
uses: actions/checkout@v4
14+
15+
- name: Set up Python
16+
uses: actions/setup-python@v5
17+
with:
18+
python-version: '3.11'
19+
20+
- name: Install dependencies
21+
run: pip install PyGithub
22+
23+
- name: Run needs-user-input script
24+
env:
25+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
26+
run: python .github/scripts/pending_user_response.py

‎backends/arm/TARGETS

Lines changed: 37 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -6,28 +6,6 @@
66
# @noautodeps
77
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
88

9-
runtime.python_library(
10-
name = "ethosu_partitioner",
11-
srcs = [
12-
"ethosu/__init__.py",
13-
"ethosu/backend.py",
14-
"ethosu/partitioner.py"
15-
],
16-
deps = [
17-
":arm_partitioner",
18-
]
19-
)
20-
runtime.python_library(
21-
name = "vgf_partitioner",
22-
srcs = [
23-
"vgf/__init__.py",
24-
"vgf/backend.py",
25-
"vgf/partitioner.py"
26-
],
27-
deps = [
28-
":arm_partitioner",
29-
]
30-
)
319
runtime.python_library(
3210
name = "constants",
3311
srcs = [
@@ -44,52 +22,65 @@ runtime.python_library(
4422
"common/debug.py",
4523
],
4624
deps = [
47-
"fbsource//third-party/tosa_tools/v0.80/serialization_lib/python/serializer:serializer",
48-
"fbsource//third-party/tosa_tools/v1.00/serialization_lib/python/serializer:serializer",
25+
"fbsource//third-party/tosa_tools:serializer",
4926
"//caffe2:torch",
5027
"//executorch/exir:lib",
5128
],
5229
)
30+
5331
runtime.python_library(
54-
name = "arm_partitioner",
32+
name = "arm_compile_spec",
5533
srcs = [
56-
"tosa/backend.py",
57-
"tosa/partitioner.py",
34+
"common/arm_compile_spec.py",
5835
],
5936
deps = [
60-
":arm_backend",
61-
":constants",
62-
"//executorch/backends/arm/debug:schema",
63-
"//executorch/backends/arm/operator_support:operator_support",
37+
"fbsource//third-party/pypi/flatbuffers:flatbuffers",
38+
"fbsource//third-party/pypi/ml-dtypes:ml-dtypes",
39+
"fbsource//third-party/tosa_tools:serializer",
40+
"fbsource//third-party/tosa_tools:tosa",
41+
":process_node",
42+
"//executorch/exir/backend:compile_spec_schema",
43+
"//executorch/backends/arm/operators:lib",
44+
"//executorch/backends/arm/operators:node_visitor",
6445
"//executorch/backends/arm/_passes:passes",
65-
"//executorch/exir:lib",
6646
],
6747
)
6848
runtime.python_library(
69-
name = "arm_backend",
49+
name = "ethosu",
7050
srcs = [
71-
"arm_backend.py",
51+
"ethosu/__init__.py",
52+
"ethosu/backend.py",
53+
"ethosu/compile_spec.py",
54+
"ethosu/partitioner.py",
7255
],
7356
deps = [
74-
"fbsource//third-party/pypi/flatbuffers:flatbuffers",
75-
"fbsource//third-party/pypi/ml-dtypes:ml-dtypes",
76-
"fbsource//third-party/tosa_tools/v0.80/serialization_lib/python/serializer:serializer",
77-
"fbsource//third-party/tosa_tools/v1.00/serialization_lib/python/serializer:serializer",
78-
"fbsource//third-party/tosa_tools/v0.80/serialization_lib/python/tosa:tosa",
79-
"fbsource//third-party/tosa_tools/v1.00/serialization_lib/python/tosa:tosa",
57+
":arm_compile_spec",
8058
":arm_vela",
81-
":process_node",
82-
"//executorch/backends/arm/operators:lib",
83-
"//executorch/backends/arm/operators:node_visitor",
84-
"//executorch/backends/arm/_passes:passes",
59+
"//executorch/backends/arm/tosa:specification",
60+
"//executorch/backends/arm/tosa:partitioner",
8561
],
8662
)
63+
64+
runtime.python_library(
65+
name = "vgf",
66+
srcs = [
67+
"vgf/__init__.py",
68+
"vgf/backend.py",
69+
"vgf/compile_spec.py",
70+
"vgf/partitioner.py",
71+
],
72+
deps = [
73+
":arm_compile_spec",
74+
"//executorch/backends/arm/tosa:specification",
75+
"//executorch/backends/arm/tosa:partitioner",
76+
],
77+
)
78+
8779
runtime.python_library(
8880
name = "process_node",
8981
srcs = ["process_node.py"],
9082
deps = [
91-
"fbsource//third-party/tosa_tools/v0.80/serialization_lib/python/tosa:tosa",
92-
"fbsource//third-party/tosa_tools/v1.00/serialization_lib/python/tosa:tosa",
83+
"fbsource//third-party/tosa_tools:tosa",
9384
"//executorch/backends/arm/operators:node_visitor",
9485
"//executorch/backends/arm/tosa:mapping",
9586
"//executorch/backends/arm/tosa:quant_utils",

0 commit comments

Comments
 (0)