diff --git a/pyproject.toml b/pyproject.toml index ad805ad50ad..2d7a1a0de07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,23 +88,15 @@ flatc = "executorch.data.bin:flatc" # package, and package_data to restrict the set up non-python files we # include. See also setuptools/discovery.py for custom finders. [tool.setuptools.package-dir] -"executorch.backends" = "backends" -"executorch.codegen" = "codegen" -"executorch.data.bin" = "data/bin" -# TODO(mnachin T180504136): Do not put examples/models -# into core pip packages. Refactor out the necessary utils -# or core models files into a separate package. -"executorch.examples.apple.coreml.llama" = "examples/apple/coreml/llama" -"executorch.examples.llm_pte_finetuning" = "examples/llm_pte_finetuning" -"executorch.examples.models" = "examples/models" -"executorch.exir" = "exir" -"executorch.extension" = "extension" -"executorch.kernels.quantized" = "kernels/quantized" -"executorch.schema" = "schema" -"executorch.devtools" = "devtools" -"executorch.devtools.bundled_program" = "devtools/bundled_program" -"executorch.runtime" = "runtime" -"executorch.util" = "util" +# Tell setuptools to follow the symlink: src/executorch/* -> * for all first level +# modules such as src/executorch/exir -> exir. This helps us to semi-compliant with +# the "src layout" convention for python packages, which is also discussed in +# https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/. +# In the long term we should move all the modules under the src/executorch/ folder. +# +# Doing this also allows us to import from executorch.exir directly in +# editable mode. +"executorch" = "src/executorch" [tool.setuptools.package-data] # TODO(dbort): Prune /test[s]/ dirs, /third-party/ dirs, yaml files that we diff --git a/setup.py b/setup.py index eac28e8e26c..c0cdc3a2c88 100644 --- a/setup.py +++ b/setup.py @@ -575,8 +575,7 @@ def run(self): # In editable mode, the package directory is the original source directory dst_root = self.get_package_dir(".") else: - dst_root = os.path.join(self.build_lib, self.get_package_dir("executorch")) - + dst_root = os.path.join(self.build_lib, "executorch") # Create the version file. Version.write_to_python_file(os.path.join(dst_root, "version.py")) diff --git a/src/README.md b/src/README.md new file mode 100644 index 00000000000..c718e53ad9c --- /dev/null +++ b/src/README.md @@ -0,0 +1,26 @@ +# Why Do We Have These Symlinks + +The `src/executorch/*` files exist primarily due to the limitations of `pip install` in editable mode. Specifically, `pip install -e .` does not recognize `/exir` (or any root level directory with a `__init__.py`) as a valid package module because of the presence of `/exir/__init__.py`. See the following GitHub issue for details: [Issue #9558](https://github.com/pytorch/executorch/issues/9558). + +## The Symlink Solution + +To work around this limitation, a symlink is used. With this symlink and this package entry in `pyproject.toml`: + +```toml +[tool.setuptools.package-dir] +# ... +"executorch" = "src/executorch" +``` +We are telling `pip install -e .` to treat `src/executorch` as the root of the `executorch` package and hence mapping `executorch.*.*` to `src/executorch/*/*`. This effectively gets modules like `exir` out from the root level package. + +This allows us to perform `pip install -e .` successfully and enables the execution of the following command: + +```bash +python -c "from executorch.exir import CaptureConfig" +``` + +## Long Term Solution + +We should start to move directories from / to /src/ and remove the symlinks. Issue [#8699](https://github.com/pytorch/executorch/issues/8699) to track this effort. This will require a lot of work internally. + +TODO(mnachin T180504136): Do not put examples/models into core pip packages. Refactor out the necessary utils or core models files into a separate package. diff --git a/src/executorch/backends b/src/executorch/backends new file mode 120000 index 00000000000..fc7aa6622a8 --- /dev/null +++ b/src/executorch/backends @@ -0,0 +1 @@ +../../backends \ No newline at end of file diff --git a/src/executorch/codegen b/src/executorch/codegen new file mode 120000 index 00000000000..a87ec5b1201 --- /dev/null +++ b/src/executorch/codegen @@ -0,0 +1 @@ +../../codegen \ No newline at end of file diff --git a/src/executorch/data b/src/executorch/data new file mode 120000 index 00000000000..e67b4559091 --- /dev/null +++ b/src/executorch/data @@ -0,0 +1 @@ +../../data \ No newline at end of file diff --git a/src/executorch/devtools b/src/executorch/devtools new file mode 120000 index 00000000000..121e39d8d10 --- /dev/null +++ b/src/executorch/devtools @@ -0,0 +1 @@ +../../devtools \ No newline at end of file diff --git a/src/executorch/examples/apple b/src/executorch/examples/apple new file mode 120000 index 00000000000..2b2c3df1898 --- /dev/null +++ b/src/executorch/examples/apple @@ -0,0 +1 @@ +../../../examples/apple \ No newline at end of file diff --git a/src/executorch/examples/llm_pte_finetuning b/src/executorch/examples/llm_pte_finetuning new file mode 120000 index 00000000000..f5fb4659860 --- /dev/null +++ b/src/executorch/examples/llm_pte_finetuning @@ -0,0 +1 @@ +../../../examples/llm_pte_finetuning \ No newline at end of file diff --git a/src/executorch/examples/models b/src/executorch/examples/models new file mode 120000 index 00000000000..eec620ef37c --- /dev/null +++ b/src/executorch/examples/models @@ -0,0 +1 @@ +../../../examples/models \ No newline at end of file diff --git a/src/executorch/exir b/src/executorch/exir new file mode 120000 index 00000000000..6ad8bdf218e --- /dev/null +++ b/src/executorch/exir @@ -0,0 +1 @@ +../../exir \ No newline at end of file diff --git a/src/executorch/extension b/src/executorch/extension new file mode 120000 index 00000000000..d7e7128aa2a --- /dev/null +++ b/src/executorch/extension @@ -0,0 +1 @@ +../../extension \ No newline at end of file diff --git a/src/executorch/kernels/quantized b/src/executorch/kernels/quantized new file mode 120000 index 00000000000..1d046e7ac56 --- /dev/null +++ b/src/executorch/kernels/quantized @@ -0,0 +1 @@ +../../../kernels/quantized \ No newline at end of file diff --git a/src/executorch/runtime b/src/executorch/runtime new file mode 120000 index 00000000000..a4f074e95d5 --- /dev/null +++ b/src/executorch/runtime @@ -0,0 +1 @@ +../../runtime \ No newline at end of file diff --git a/src/executorch/schema b/src/executorch/schema new file mode 120000 index 00000000000..56c062f725d --- /dev/null +++ b/src/executorch/schema @@ -0,0 +1 @@ +../../schema \ No newline at end of file diff --git a/src/executorch/util b/src/executorch/util new file mode 120000 index 00000000000..8b27964e55f --- /dev/null +++ b/src/executorch/util @@ -0,0 +1 @@ +../../util \ No newline at end of file