Skip to content

[BE] Move submodule check out from setup.py #8315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 7, 2025
Merged

[BE] Move submodule check out from setup.py #8315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8fc3234
[BE] Move submodule check out from setup.py
larryliu0820 Feb 7, 2025
614ced3
lint
larryliu0820 Feb 7, 2025
5192049
Move submodule handling to before install requirements
larryliu0820 Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 89 additions & 2 deletions install_executorch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import argparse
import glob
import itertools
import logging
import os
import shutil
import subprocess
Expand All @@ -20,6 +21,12 @@
TORCH_NIGHTLY_URL,
)

# Set up logging
logging.basicConfig(
level=logging.INFO, format="%(asctime)s [ExecuTorch] %(levelname)s: %(message)s"
)
logger = logging.getLogger()


def clean():
print("Cleaning build artifacts...")
Expand All @@ -35,6 +42,83 @@ def clean():
VALID_PYBINDS = ["coreml", "mps", "xnnpack", "training"]


################################################################################
# Git submodules
################################################################################
# The following submodules are required to be able to build ExecuTorch. If any of
# these folders are missing or missing CMakeLists.txt, we will run
# `git submodule update` to try to fix it. If the command fails, we will raise an
# error.
# An alternative to this would be to run `git submodule status` and run
# `git submodule update` if there's any local changes. However this is a bit
# too restrictive for users who modifies and tests the dependencies locally.

# keep sorted
REQUIRED_SUBMODULES = {
"ao": "LICENSE", # No CMakeLists.txt, choose a sort of stable file to check.
"cpuinfo": "CMakeLists.txt",
"eigen": "CMakeLists.txt",
"flatbuffers": "CMakeLists.txt",
"FP16": "CMakeLists.txt",
"FXdiv": "CMakeLists.txt",
"gflags": "CMakeLists.txt",
"prelude": "BUCK",
"pthreadpool": "CMakeLists.txt",
"pybind11": "CMakeLists.txt",
"XNNPACK": "CMakeLists.txt",
}


def get_required_submodule_paths():
gitmodules_path = os.path.join(os.getcwd(), ".gitmodules")

if not os.path.isfile(gitmodules_path):
logger.error(".gitmodules file not found.")
exit(1)

with open(gitmodules_path, "r") as file:
lines = file.readlines()

# Extract paths of required submodules
required_paths = {}
for line in lines:
if line.strip().startswith("path ="):
path = line.split("=")[1].strip()
for submodule, file_name in REQUIRED_SUBMODULES.items():
if submodule in path:
required_paths[path] = file_name
return required_paths


def check_and_update_submodules():
def check_folder(folder: str, file: str) -> bool:
return os.path.isdir(folder) and os.path.isfile(os.path.join(folder, file))

# Check if the directories exist for each required submodule
missing_submodules = {}
for path, file in get_required_submodule_paths().items():
if not check_folder(path, file):
missing_submodules[path] = file

# If any required submodule directories are missing, update them
if missing_submodules:
logger.warning("Some required submodules are missing. Updating submodules...")
try:
subprocess.check_call(["git", "submodule", "sync"])
subprocess.check_call(["git", "submodule", "update", "--init"])
except subprocess.CalledProcessError as e:
logger.error(f"Error updating submodules: {e}")
exit(1)

# After updating submodules, check again
for path, file in missing_submodules.items():
if not check_folder(path, file):
logger.error(f"{file} not found in {path}.")
logger.error("Please run `git submodule update --init`.")
exit(1)
logger.info("All required submodules are present.")


def main(args):
if not python_is_compatible():
sys.exit(1)
Expand Down Expand Up @@ -95,8 +179,6 @@ def main(args):
# latest PT commit otherwise
use_pytorch_nightly = False

install_requirements(use_pytorch_nightly)

# If --pybind is not set explicitly for backends (e.g., --pybind xnnpack)
# or is not turned off explicitly (--pybind off)
# then install XNNPACK by default.
Expand All @@ -120,6 +202,11 @@ def main(args):
os.environ["EXECUTORCH_BUILD_PYBIND"] = EXECUTORCH_BUILD_PYBIND
os.environ["CMAKE_ARGS"] = CMAKE_ARGS

# Check if the required submodules are present and update them if not
check_and_update_submodules()

install_requirements(use_pytorch_nightly)

# Run the pip install command
subprocess.run(
[
Expand Down
150 changes: 28 additions & 122 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@
# Import this before distutils so that setuptools can intercept the distuils
# imports.
import setuptools # noqa: F401 # usort: skip
import logging
import subprocess

from distutils import log
Expand All @@ -68,91 +67,6 @@
from setuptools.command.build_ext import build_ext
from setuptools.command.build_py import build_py

# Set up logging
logging.basicConfig(
level=logging.INFO, format="%(asctime)s [ExecuTorch] %(levelname)s: %(message)s"
)
logger = logging.getLogger()

# For information on setuptools Command subclassing see
# https://setuptools.pypa.io/en/latest/userguide/extension.html

################################################################################
# Git submodules
################################################################################
# The following submodules are required to be able to build ExecuTorch. If any of
# these folders are missing or missing CMakeLists.txt, we will run
# `git submodule update` to try to fix it. If the command fails, we will raise an
# error.
# An alternative to this would be to run `git submodule status` and run
# `git submodule update` if there's any local changes. However this is a bit
# too restrictive for users who modifies and tests the dependencies locally.

# keep sorted
REQUIRED_SUBMODULES = {
"ao": "LICENSE", # No CMakeLists.txt, choose a sort of stable file to check.
"cpuinfo": "CMakeLists.txt",
"eigen": "CMakeLists.txt",
"flatbuffers": "CMakeLists.txt",
"FP16": "CMakeLists.txt",
"FXdiv": "CMakeLists.txt",
"gflags": "CMakeLists.txt",
"prelude": "BUCK",
"pthreadpool": "CMakeLists.txt",
"pybind11": "CMakeLists.txt",
"XNNPACK": "CMakeLists.txt",
}


def get_required_submodule_paths():
gitmodules_path = os.path.join(os.getcwd(), ".gitmodules")

if not os.path.isfile(gitmodules_path):
logger.error(".gitmodules file not found.")
exit(1)

with open(gitmodules_path, "r") as file:
lines = file.readlines()

# Extract paths of required submodules
required_paths = {}
for line in lines:
if line.strip().startswith("path ="):
path = line.split("=")[1].strip()
for submodule, file_name in REQUIRED_SUBMODULES.items():
if submodule in path:
required_paths[path] = file_name
return required_paths


def check_and_update_submodules():
def check_folder(folder: str, file: str) -> bool:
return os.path.isdir(folder) and os.path.isfile(os.path.join(folder, file))

# Check if the directories exist for each required submodule
missing_submodules = {}
for path, file in get_required_submodule_paths().items():
if not check_folder(path, file):
missing_submodules[path] = file

# If any required submodule directories are missing, update them
if missing_submodules:
logger.warning("Some required submodules are missing. Updating submodules...")
try:
subprocess.check_call(["git", "submodule", "sync"])
subprocess.check_call(["git", "submodule", "update", "--init"])
except subprocess.CalledProcessError as e:
logger.error(f"Error updating submodules: {e}")
exit(1)

# After updating submodules, check again
for path, file in missing_submodules.items():
if not check_folder(path, file):
logger.error(f"{file} not found in {path}.")
logger.error("Please run `git submodule update --init`.")
exit(1)
logger.info("All required submodules are present.")


class ShouldBuild:
"""Indicates whether to build various components."""
Expand Down Expand Up @@ -825,39 +739,31 @@ def get_ext_modules() -> List[Extension]:
return ext_modules


def main():
# Check submodules
check_and_update_submodules()

setup(
version=Version.string(),
# TODO(dbort): Could use py_modules to restrict the set of modules we
# package, and package_data to restrict the set up non-python files we
# include. See also setuptools/discovery.py for custom finders.
package_dir={
"executorch/backends": "backends",
"executorch/codegen": "codegen",
# TODO(mnachin T180504136): Do not put examples/models
# into core pip packages. Refactor out the necessary utils
# or core models files into a separate package.
"executorch/examples/models": "examples/models",
"executorch/exir": "exir",
"executorch/extension": "extension",
"executorch/kernels/quantized": "kernels/quantized",
"executorch/schema": "schema",
"executorch/devtools": "devtools",
"executorch/devtools/bundled_program": "devtools/bundled_program",
"executorch/runtime": "runtime",
"executorch/util": "util",
},
cmdclass={
"build": CustomBuild,
"build_ext": InstallerBuildExt,
"build_py": CustomBuildPy,
},
ext_modules=get_ext_modules(),
)


if __name__ == "__main__":
main()
setup(
version=Version.string(),
# TODO(dbort): Could use py_modules to restrict the set of modules we
# package, and package_data to restrict the set up non-python files we
# include. See also setuptools/discovery.py for custom finders.
package_dir={
"executorch/backends": "backends",
"executorch/codegen": "codegen",
# TODO(mnachin T180504136): Do not put examples/models
# into core pip packages. Refactor out the necessary utils
# or core models files into a separate package.
"executorch/examples/models": "examples/models",
"executorch/exir": "exir",
"executorch/extension": "extension",
"executorch/kernels/quantized": "kernels/quantized",
"executorch/schema": "schema",
"executorch/devtools": "devtools",
"executorch/devtools/bundled_program": "devtools/bundled_program",
"executorch/runtime": "runtime",
"executorch/util": "util",
},
cmdclass={
"build": CustomBuild,
"build_ext": InstallerBuildExt,
"build_py": CustomBuildPy,
},
ext_modules=get_ext_modules(),
)
Loading