From f167a896bd71eabc4dc1216119b6da9d6fc75642 Mon Sep 17 00:00:00 2001 From: Mengwei Liu Date: Tue, 1 Apr 2025 16:14:17 -0700 Subject: [PATCH 1/6] A short term fix for editable mode install failed to import root level module such as exir Summary: Fixes #9558. The `src/executorch/exir` file exists primarily due to the limitations of `pip install` in editable mode. Specifically, `pip install -e .` does not recognize `/exir` (or any root level directory with a `__init__.py`) as a valid package module because of the presence of `/exir/__init__.py`. See the following GitHub issue for details: [Issue #9558](https://github.com/pytorch/executorch/issues/9558). To work around this limitation, a symlink is used. With this symlink and this package entry in `pyproject.toml`: ```toml [tool.setuptools.package-dir] ... "executorch" = "src/executorch" ``` We are telling `pip install -e .` to treat `src/executorch` as the root of the `executorch` package and hence mapping `executorch.exir` to `src/executorch/exir`. This allows us to perform `pip install -e .` successfully and enables the execution of the following command: ```bash python -c "from executorch.exir import CaptureConfig" ``` Test Plan: ```bash ./install_executorch.sh --pybind --editable python -c "from executorch.exir import CaptureConfig" ``` Reviewers: Subscribers: Tasks: Tags: --- pyproject.toml | 2 +- src/executorch/exir | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 120000 src/executorch/exir diff --git a/pyproject.toml b/pyproject.toml index ad805ad50ad..cbcd4b8b446 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ flatc = "executorch.data.bin:flatc" "executorch.examples.apple.coreml.llama" = "examples/apple/coreml/llama" "executorch.examples.llm_pte_finetuning" = "examples/llm_pte_finetuning" "executorch.examples.models" = "examples/models" -"executorch.exir" = "exir" +"executorch" = "src/executorch" "executorch.extension" = "extension" "executorch.kernels.quantized" = "kernels/quantized" "executorch.schema" = "schema" diff --git a/src/executorch/exir b/src/executorch/exir new file mode 120000 index 00000000000..6ad8bdf218e --- /dev/null +++ b/src/executorch/exir @@ -0,0 +1 @@ +../../exir \ No newline at end of file From 0589ceac8e1a316cc70c6d3aa8649b45b517bcef Mon Sep 17 00:00:00 2001 From: Mengwei Liu Date: Tue, 1 Apr 2025 16:19:12 -0700 Subject: [PATCH 2/6] Add README.md Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- src/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 src/README.md diff --git a/src/README.md b/src/README.md new file mode 100644 index 00000000000..59957b4c48c --- /dev/null +++ b/src/README.md @@ -0,0 +1,20 @@ +# Why We Have a src/executorch/exir File + +The `src/executorch/exir` file exists primarily due to the limitations of `pip install` in editable mode. Specifically, `pip install -e .` does not recognize `/exir` (or any root level directory with a `__init__.py`) as a valid package module because of the presence of `/exir/__init__.py`. See the following GitHub issue for details: [Issue #9558](https://github.com/pytorch/executorch/issues/9558). + +## The Symlink Solution + +To work around this limitation, a symlink is used. With this symlink and this package entry in `pyproject.toml`: + +```toml +[tool.setuptools.package-dir] +... +"executorch" = "src/executorch" +``` +We are telling `pip install -e .` to treat `src/executorch` as the root of the `executorch` package and hence mapping `executorch.exir` to `src/executorch/exir`. + +This allows us to perform `pip install -e .` successfully and enables the execution of the following command: + +```bash +python -c "from executorch.exir import CaptureConfig" +``` From b55ecc71c4ff57bf15bdf744c880de8be3763561 Mon Sep 17 00:00:00 2001 From: Mengwei Liu Date: Wed, 2 Apr 2025 11:06:44 -0700 Subject: [PATCH 3/6] Fix version.py Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- pyproject.toml | 8 +++++++- setup.py | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cbcd4b8b446..39abb2b18bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,13 @@ Changelog = "https://github.com/pytorch/executorch/releases" [project.scripts] flatc = "executorch.data.bin:flatc" +# Tell setuptools to follow the symlink: src/executorch/exir -> exir. +# Doing this allows us to import from executorch.exir directly in +# editable mode. +[tool.setuptools.packages.find] +where = ["src/"] +include = ["executorch.exir"] + # TODO(dbort): Could use py_modules to restrict the set of modules we # package, and package_data to restrict the set up non-python files we # include. See also setuptools/discovery.py for custom finders. @@ -97,7 +104,6 @@ flatc = "executorch.data.bin:flatc" "executorch.examples.apple.coreml.llama" = "examples/apple/coreml/llama" "executorch.examples.llm_pte_finetuning" = "examples/llm_pte_finetuning" "executorch.examples.models" = "examples/models" -"executorch" = "src/executorch" "executorch.extension" = "extension" "executorch.kernels.quantized" = "kernels/quantized" "executorch.schema" = "schema" diff --git a/setup.py b/setup.py index eac28e8e26c..c0cdc3a2c88 100644 --- a/setup.py +++ b/setup.py @@ -575,8 +575,7 @@ def run(self): # In editable mode, the package directory is the original source directory dst_root = self.get_package_dir(".") else: - dst_root = os.path.join(self.build_lib, self.get_package_dir("executorch")) - + dst_root = os.path.join(self.build_lib, "executorch") # Create the version file. Version.write_to_python_file(os.path.join(dst_root, "version.py")) From 45eee85a9de23da8b2d46daefee598afbfacaf70 Mon Sep 17 00:00:00 2001 From: Mengwei Liu Date: Wed, 2 Apr 2025 11:16:03 -0700 Subject: [PATCH 4/6] Include all submodules of exir Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- pyproject.toml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 39abb2b18bf..77c5b342e2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,17 +84,14 @@ Changelog = "https://github.com/pytorch/executorch/releases" [project.scripts] flatc = "executorch.data.bin:flatc" -# Tell setuptools to follow the symlink: src/executorch/exir -> exir. -# Doing this allows us to import from executorch.exir directly in -# editable mode. -[tool.setuptools.packages.find] -where = ["src/"] -include = ["executorch.exir"] - # TODO(dbort): Could use py_modules to restrict the set of modules we # package, and package_data to restrict the set up non-python files we # include. See also setuptools/discovery.py for custom finders. [tool.setuptools.package-dir] +# Tell setuptools to follow the symlink: src/executorch/exir -> exir. +# Doing this allows us to import from executorch.exir directly in +# editable mode. +"executorch" = "src/executorch" "executorch.backends" = "backends" "executorch.codegen" = "codegen" "executorch.data.bin" = "data/bin" From 73981c2b69929ed30537c1128132aaf04b304189 Mon Sep 17 00:00:00 2001 From: Mengwei Liu Date: Wed, 2 Apr 2025 13:37:31 -0700 Subject: [PATCH 5/6] Add all first level modules into src/executorch as symlink Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- pyproject.toml | 25 ++++++---------------- src/README.md | 10 +++++++-- src/executorch/backends | 1 + src/executorch/codegen | 1 + src/executorch/data | 1 + src/executorch/devtools | 1 + src/executorch/examples/apple | 1 + src/executorch/examples/llm_pte_finetuning | 1 + src/executorch/examples/models | 1 + src/executorch/extension | 1 + src/executorch/kernels/quantized | 1 + src/executorch/runtime | 1 + src/executorch/schema | 1 + src/executorch/util | 1 + 14 files changed, 27 insertions(+), 20 deletions(-) create mode 120000 src/executorch/backends create mode 120000 src/executorch/codegen create mode 120000 src/executorch/data create mode 120000 src/executorch/devtools create mode 120000 src/executorch/examples/apple create mode 120000 src/executorch/examples/llm_pte_finetuning create mode 120000 src/executorch/examples/models create mode 120000 src/executorch/extension create mode 120000 src/executorch/kernels/quantized create mode 120000 src/executorch/runtime create mode 120000 src/executorch/schema create mode 120000 src/executorch/util diff --git a/pyproject.toml b/pyproject.toml index 77c5b342e2d..2d7a1a0de07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,26 +88,15 @@ flatc = "executorch.data.bin:flatc" # package, and package_data to restrict the set up non-python files we # include. See also setuptools/discovery.py for custom finders. [tool.setuptools.package-dir] -# Tell setuptools to follow the symlink: src/executorch/exir -> exir. -# Doing this allows us to import from executorch.exir directly in +# Tell setuptools to follow the symlink: src/executorch/* -> * for all first level +# modules such as src/executorch/exir -> exir. This helps us to semi-compliant with +# the "src layout" convention for python packages, which is also discussed in +# https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/. +# In the long term we should move all the modules under the src/executorch/ folder. +# +# Doing this also allows us to import from executorch.exir directly in # editable mode. "executorch" = "src/executorch" -"executorch.backends" = "backends" -"executorch.codegen" = "codegen" -"executorch.data.bin" = "data/bin" -# TODO(mnachin T180504136): Do not put examples/models -# into core pip packages. Refactor out the necessary utils -# or core models files into a separate package. -"executorch.examples.apple.coreml.llama" = "examples/apple/coreml/llama" -"executorch.examples.llm_pte_finetuning" = "examples/llm_pte_finetuning" -"executorch.examples.models" = "examples/models" -"executorch.extension" = "extension" -"executorch.kernels.quantized" = "kernels/quantized" -"executorch.schema" = "schema" -"executorch.devtools" = "devtools" -"executorch.devtools.bundled_program" = "devtools/bundled_program" -"executorch.runtime" = "runtime" -"executorch.util" = "util" [tool.setuptools.package-data] # TODO(dbort): Prune /test[s]/ dirs, /third-party/ dirs, yaml files that we diff --git a/src/README.md b/src/README.md index 59957b4c48c..193fe6f9dd7 100644 --- a/src/README.md +++ b/src/README.md @@ -1,6 +1,6 @@ -# Why We Have a src/executorch/exir File +# Why Do We Have These Symlinks -The `src/executorch/exir` file exists primarily due to the limitations of `pip install` in editable mode. Specifically, `pip install -e .` does not recognize `/exir` (or any root level directory with a `__init__.py`) as a valid package module because of the presence of `/exir/__init__.py`. See the following GitHub issue for details: [Issue #9558](https://github.com/pytorch/executorch/issues/9558). +The `src/executorch/*` files exist primarily due to the limitations of `pip install` in editable mode. Specifically, `pip install -e .` does not recognize `/exir` (or any root level directory with a `__init__.py`) as a valid package module because of the presence of `/exir/__init__.py`. See the following GitHub issue for details: [Issue #9558](https://github.com/pytorch/executorch/issues/9558). ## The Symlink Solution @@ -18,3 +18,9 @@ This allows us to perform `pip install -e .` successfully and enables the execut ```bash python -c "from executorch.exir import CaptureConfig" ``` + +## Long Term Solution + +We should start to move directories from / to /src/ and remove the symlinks. Issue [#8699](https://github.com/pytorch/executorch/issues/8699) to track this effort. This will require a lot of work internally. + +TODO(mnachin T180504136): Do not put examples/models into core pip packages. Refactor out the necessary utils or core models files into a separate package. diff --git a/src/executorch/backends b/src/executorch/backends new file mode 120000 index 00000000000..fc7aa6622a8 --- /dev/null +++ b/src/executorch/backends @@ -0,0 +1 @@ +../../backends \ No newline at end of file diff --git a/src/executorch/codegen b/src/executorch/codegen new file mode 120000 index 00000000000..a87ec5b1201 --- /dev/null +++ b/src/executorch/codegen @@ -0,0 +1 @@ +../../codegen \ No newline at end of file diff --git a/src/executorch/data b/src/executorch/data new file mode 120000 index 00000000000..e67b4559091 --- /dev/null +++ b/src/executorch/data @@ -0,0 +1 @@ +../../data \ No newline at end of file diff --git a/src/executorch/devtools b/src/executorch/devtools new file mode 120000 index 00000000000..121e39d8d10 --- /dev/null +++ b/src/executorch/devtools @@ -0,0 +1 @@ +../../devtools \ No newline at end of file diff --git a/src/executorch/examples/apple b/src/executorch/examples/apple new file mode 120000 index 00000000000..2b2c3df1898 --- /dev/null +++ b/src/executorch/examples/apple @@ -0,0 +1 @@ +../../../examples/apple \ No newline at end of file diff --git a/src/executorch/examples/llm_pte_finetuning b/src/executorch/examples/llm_pte_finetuning new file mode 120000 index 00000000000..f5fb4659860 --- /dev/null +++ b/src/executorch/examples/llm_pte_finetuning @@ -0,0 +1 @@ +../../../examples/llm_pte_finetuning \ No newline at end of file diff --git a/src/executorch/examples/models b/src/executorch/examples/models new file mode 120000 index 00000000000..eec620ef37c --- /dev/null +++ b/src/executorch/examples/models @@ -0,0 +1 @@ +../../../examples/models \ No newline at end of file diff --git a/src/executorch/extension b/src/executorch/extension new file mode 120000 index 00000000000..d7e7128aa2a --- /dev/null +++ b/src/executorch/extension @@ -0,0 +1 @@ +../../extension \ No newline at end of file diff --git a/src/executorch/kernels/quantized b/src/executorch/kernels/quantized new file mode 120000 index 00000000000..1d046e7ac56 --- /dev/null +++ b/src/executorch/kernels/quantized @@ -0,0 +1 @@ +../../../kernels/quantized \ No newline at end of file diff --git a/src/executorch/runtime b/src/executorch/runtime new file mode 120000 index 00000000000..a4f074e95d5 --- /dev/null +++ b/src/executorch/runtime @@ -0,0 +1 @@ +../../runtime \ No newline at end of file diff --git a/src/executorch/schema b/src/executorch/schema new file mode 120000 index 00000000000..56c062f725d --- /dev/null +++ b/src/executorch/schema @@ -0,0 +1 @@ +../../schema \ No newline at end of file diff --git a/src/executorch/util b/src/executorch/util new file mode 120000 index 00000000000..8b27964e55f --- /dev/null +++ b/src/executorch/util @@ -0,0 +1 @@ +../../util \ No newline at end of file From f599c9fdc657579d8c56ea91e280bb64e6a8827d Mon Sep 17 00:00:00 2001 From: Mengwei Liu Date: Wed, 2 Apr 2025 13:41:06 -0700 Subject: [PATCH 6/6] Readme Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: --- src/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/README.md b/src/README.md index 193fe6f9dd7..c718e53ad9c 100644 --- a/src/README.md +++ b/src/README.md @@ -8,10 +8,10 @@ To work around this limitation, a symlink is used. With this symlink and this pa ```toml [tool.setuptools.package-dir] -... +# ... "executorch" = "src/executorch" ``` -We are telling `pip install -e .` to treat `src/executorch` as the root of the `executorch` package and hence mapping `executorch.exir` to `src/executorch/exir`. +We are telling `pip install -e .` to treat `src/executorch` as the root of the `executorch` package and hence mapping `executorch.*.*` to `src/executorch/*/*`. This effectively gets modules like `exir` out from the root level package. This allows us to perform `pip install -e .` successfully and enables the execution of the following command: