Skip to content

Commit e7e9e06

Browse files
kirklandsignlarryliu0820
kirklandsign
and
larryliu0820
authored
Cherry-pick commits for executorch_no_prim_ops (#3025)
* Add executorch_no_prim_ops target (#2934) Summary: Pull Request resolved: #2934 Currently `libexecutorch.a` always contain prim ops. This becomes a problem when a binary contains 2 "versions" of `libexecutorch.a`, causing a double registration of the prim ops. For example, `libA.so` depends on `libexecutorch.a` and a binary `B` depends on both `libA.so` and `libexecutorch.a`. Since both `libexecutorch.a` and `libA.so` contains prim ops, they will be registered twice. In this PR I created another library `executorch_no_prim_ops` for `libA.so` to depend on. Reviewed By: cccclai, kirklandsign Differential Revision: D55907752 fbshipit-source-id: 755a9b8d5f6f7cf44d011b83bfdc18be6da1aa05 (cherry picked from commit d309e9d) * Fix failing CI jobs caused by #2934 (#2961) Summary: Pull Request resolved: #2961 Fix these 3 CI job failures caused by #2934 (D55907752): * Apple / build-frameworks-ios / macos-job * trunk / test-arm-backend-delegation / linux-job * trunk / test-coreml-delegate / macos-job Reviewed By: kirklandsign Differential Revision: D55950023 fbshipit-source-id: 6166d9112e6d971d042df1400442395d8044c3b3 (cherry picked from commit d993797) * [NOT-CLEAN-CP] Fix 3 CI jobs (#3006) Summary: * [NOT APPLICABLE IN RELEASE] Apple / build-frameworks-ios / macos-job We removed libcustom_ops_lib.a in #2916 so need to remove it from `build_apple_frameworks.sh`. * [NOT APPLICABLE IN RELEASE] Lint / lintrunner / linux-job Remove extra line in backends/qualcomm/quantizer/utils.py * pull / unittest / macos (buck2) / macos-job Fix it by using `executorch_no_prim_ops` instead of `executorch` in MPS and CoreML. Pull Request resolved: #3006 Reviewed By: lucylq Differential Revision: D56048430 Pulled By: larryliu0820 fbshipit-source-id: 9dcb476eea446ea3aba566d595167c691fb00eec (cherry picked from commit 5b7c4ba) --------- Co-authored-by: Mengwei Liu <[email protected]> Co-authored-by: Mengwei Liu <[email protected]>
1 parent 87d3748 commit e7e9e06

File tree

12 files changed

+93
-51
lines changed

12 files changed

+93
-51
lines changed

‎CMakeLists.txt

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -352,23 +352,27 @@ add_subdirectory(schema)
352352
# Only contains primitive operators; does not contain portable kernels or other
353353
# full operators. Does not contain any backends.
354354
#
355-
356-
add_library(executorch ${_executorch__srcs})
357-
target_link_libraries(executorch PRIVATE program_schema)
358-
target_link_options_shared_lib(executorch)
355+
add_library(executorch_no_prim_ops ${_executorch_no_prim_ops__srcs})
356+
target_link_libraries(executorch_no_prim_ops PRIVATE program_schema)
359357
# Check if dl exists for this toolchain and only then link it.
360358
find_library(DL_LIBRARY_EXISTS NAMES dl)
361359
# Check if the library was found
362360
if(DL_LIBRARY_EXISTS)
363-
target_link_libraries(executorch PRIVATE dl) # For dladdr()
361+
target_link_libraries(executorch_no_prim_ops PRIVATE dl) # For dladdr()
364362
endif()
365-
target_include_directories(executorch PUBLIC ${_common_include_directories})
366-
target_compile_options(executorch PUBLIC ${_common_compile_options})
363+
target_include_directories(executorch_no_prim_ops PUBLIC ${_common_include_directories})
364+
target_compile_options(executorch_no_prim_ops PUBLIC ${_common_compile_options})
367365
if(MAX_KERNEL_NUM)
368-
target_compile_definitions(executorch
366+
target_compile_definitions(executorch_no_prim_ops
369367
PRIVATE MAX_KERNEL_NUM=${MAX_KERNEL_NUM})
370368
endif()
371369

370+
add_library(executorch ${_executorch__srcs})
371+
target_link_libraries(executorch PRIVATE executorch_no_prim_ops)
372+
target_include_directories(executorch PUBLIC ${_common_include_directories})
373+
target_compile_options(executorch PUBLIC ${_common_compile_options})
374+
target_link_options_shared_lib(executorch)
375+
372376
#
373377
# portable_ops_lib: A library to register core ATen ops using portable kernels,
374378
# see kernels/portable/CMakeLists.txt.
@@ -406,7 +410,7 @@ endif()
406410
# Install `executorch` library as well as `executorch-config.cmake` under
407411
# ${CMAKE_INSTALL_PREFIX}/
408412
install(
409-
TARGETS executorch
413+
TARGETS executorch executorch_no_prim_ops
410414
DESTINATION lib
411415
INCLUDES
412416
DESTINATION ${_common_include_directories})

‎backends/apple/coreml/CMakeLists.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ target_include_directories(
144144
)
145145
target_link_libraries(
146146
coremldelegate PRIVATE
147-
executorch
147+
executorch_no_prim_ops
148148
)
149149

150150
if(EXECUTORCH_BUILD_SDK)
@@ -174,7 +174,7 @@ find_library(SQLITE_LIBRARY sqlite3)
174174

175175
target_link_libraries(coremldelegate
176176
PRIVATE
177-
executorch
177+
executorch_no_prim_ops
178178
${ACCELERATE_FRAMEWORK}
179179
${COREML_FRAMEWORK}
180180
${FOUNDATION_FRAMEWORK}
@@ -185,7 +185,7 @@ target_compile_options(coremldelegate PRIVATE "-fobjc-arc")
185185
target_compile_options(coremldelegate PRIVATE "-fno-exceptions")
186186

187187
if(EXECUTORCH_BUILD_SDK)
188-
target_compile_options(executorch PUBLIC -DET_EVENT_TRACER_ENABLED)
188+
target_compile_options(executorch_no_prim_ops PUBLIC -DET_EVENT_TRACER_ENABLED)
189189
target_compile_options(coremldelegate PRIVATE "-frtti")
190190
target_compile_options(libprotobuf-lite PRIVATE "-frtti")
191191
else()

‎backends/apple/coreml/runtime/workspace/executorchcoreml.xcodeproj/project.pbxproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@
100100
C9E7D7952AB3F9BF00CCAE5D /* ETCoreMLModelManagerTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = C9E7D78D2AB3F9BF00CCAE5D /* ETCoreMLModelManagerTests.mm */; };
101101
C9E7D7962AB3F9BF00CCAE5D /* KeyValueStoreTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = C9E7D78E2AB3F9BF00CCAE5D /* KeyValueStoreTests.mm */; };
102102
C9E7D7A22AB3FBB200CCAE5D /* CoreMLBackendDelegateTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = C9E7D7A12AB3FBB200CCAE5D /* CoreMLBackendDelegateTests.mm */; };
103+
F24817E52BC655E100E80D98 /* libexecutorch_no_prim_ops.a in Frameworks */ = {isa = PBXBuildFile; fileRef = F24817E42BC655E100E80D98 /* libexecutorch_no_prim_ops.a */; };
103104
/* End PBXBuildFile section */
104105

105106
/* Begin PBXCopyFilesBuildPhase section */
@@ -297,6 +298,7 @@
297298
C9EA3DB22B71A2B200B7D7BD /* CoreML.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreML.framework; path = System/Library/Frameworks/CoreML.framework; sourceTree = SDKROOT; };
298299
C9EA3FDE2B73EEA000B7D7BD /* libsqlite3.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libsqlite3.tbd; path = usr/lib/libsqlite3.tbd; sourceTree = SDKROOT; };
299300
C9EA3FE52B73EF6300B7D7BD /* Accelerate.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Accelerate.framework; path = System/Library/Frameworks/Accelerate.framework; sourceTree = SDKROOT; };
301+
F24817E42BC655E100E80D98 /* libexecutorch_no_prim_ops.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libexecutorch_no_prim_ops.a; path = ../libraries/libexecutorch_no_prim_ops.a; sourceTree = "<group>"; };
300302
/* End PBXFileReference section */
301303

302304
/* Begin PBXFrameworksBuildPhase section */
@@ -305,6 +307,7 @@
305307
buildActionMask = 2147483647;
306308
files = (
307309
C94D510F2ABDF87500AF47FD /* Accelerate.framework in Frameworks */,
310+
F24817E52BC655E100E80D98 /* libexecutorch_no_prim_ops.a in Frameworks */,
308311
C94D510E2ABDF86800AF47FD /* libsqlite3.tbd in Frameworks */,
309312
C94D50D92ABD7B2400AF47FD /* CoreML.framework in Frameworks */,
310313
C99883862B95AD7D000953A3 /* libprotobuf-lite.a in Frameworks */,
@@ -523,6 +526,7 @@
523526
C96560942AABFDCE005F8126 /* libsqlite3.tbd */,
524527
C96560922AABF992005F8126 /* CoreML.framework */,
525528
C96560902AABF982005F8126 /* Accelerate.framework */,
529+
F24817E42BC655E100E80D98 /* libexecutorch_no_prim_ops.a */,
526530
C965608D2AABF72A005F8126 /* libexecutorch.a */,
527531
);
528532
name = "Recovered References";

‎backends/apple/coreml/scripts/build_tests.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ cmake --build "$CMAKE_PROTOBUF_BUILD_DIR_PATH" -j9 -t libprotobuf-lite
5959
echo "ExecuTorch: Copying libraries"
6060
mkdir "$LIBRARIES_DIR_PATH"
6161
cp -f "$CMAKE_EXECUTORCH_BUILD_DIR_PATH/libexecutorch.a" "$LIBRARIES_DIR_PATH"
62+
cp -f "$CMAKE_EXECUTORCH_BUILD_DIR_PATH/libexecutorch_no_prim_ops.a" "$LIBRARIES_DIR_PATH"
6263
cp -f "$CMAKE_PROTOBUF_BUILD_DIR_PATH/libprotobuf-lite.a" "$LIBRARIES_DIR_PATH"
6364

6465
#Copy ExecuTorch headers

‎backends/apple/mps/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ target_link_libraries(mpsdelegate
7070
PRIVATE
7171
bundled_program
7272
mps_schema
73-
${_executor_runner_libs}
73+
executorch_no_prim_ops
7474
${FOUNDATION_FRAMEWORK}
7575
${METAL_FRAMEWORK}
7676
${MPS_FRAMEWORK}

‎build/build_apple_frameworks.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ PORTABLE=OFF
2525
QUANTIZED=OFF
2626
XNNPACK=OFF
2727
HEADERS_PATH="include"
28-
EXECUTORCH_FRAMEWORK="executorch:libexecutorch.a,libextension_apple.a,libextension_data_loader.a,libextension_module.a:$HEADERS_PATH"
28+
EXECUTORCH_FRAMEWORK="executorch:libexecutorch.a,libexecutorch_no_prim_ops.a,libextension_apple.a,libextension_data_loader.a,libextension_module.a:$HEADERS_PATH"
2929
COREML_FRAMEWORK="coreml_backend:libcoremldelegate.a:"
3030
CUSTOM_FRAMEWORK="custom_backend:libcustom_ops.a,libcustom_ops_lib.a:"
3131
MPS_FRAMEWORK="mps_backend:libmpsdelegate.a:"

‎build/cmake_deps.toml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,18 @@ excludes = [
1919
buck_targets = [
2020
"//runtime/executor:program",
2121
]
22+
deps = [
23+
"executorch_no_prim_ops",
24+
]
25+
filters = [
26+
".cpp$",
27+
]
28+
29+
30+
[targets.executorch_no_prim_ops]
31+
buck_targets = [
32+
"//runtime/executor:program_no_prim_ops",
33+
]
2234
deps = [
2335
"program_schema",
2436
]
@@ -43,6 +55,7 @@ excludes = [
4355
]
4456
deps = [
4557
"executorch",
58+
"executorch_no_prim_ops",
4659
]
4760

4861
[targets.optimized_kernels]
@@ -59,6 +72,7 @@ excludes = [
5972
]
6073
deps = [
6174
"executorch",
75+
"executorch_no_prim_ops",
6276
"portable_kernels",
6377
]
6478

@@ -76,6 +90,7 @@ excludes = [
7690
]
7791
deps = [
7892
"executorch",
93+
"executorch_no_prim_ops",
7994
"portable_kernels",
8095
]
8196

@@ -97,6 +112,7 @@ filters = [
97112
excludes = [
98113
]
99114
deps = [
115+
"executorch_no_prim_ops",
100116
"executorch",
101117
]
102118

@@ -113,6 +129,7 @@ filters = [
113129
".cpp$",
114130
]
115131
deps = [
132+
"executorch_no_prim_ops",
116133
"executorch",
117134
]
118135

@@ -125,6 +142,7 @@ filters = [
125142
]
126143
deps = [
127144
"executorch",
145+
"executorch_no_prim_ops",
128146
"extension_data_loader",
129147
]
130148

@@ -137,6 +155,7 @@ filters = [
137155
]
138156
deps = [
139157
"executorch",
158+
"executorch_no_prim_ops",
140159
]
141160

142161
# ---------------------------------- extension end ----------------------------------
@@ -154,6 +173,7 @@ excludes = [
154173
]
155174
deps = [
156175
"executorch",
176+
"executorch_no_prim_ops",
157177
"portable_kernels",
158178
"quantized_kernels",
159179
]
@@ -169,6 +189,7 @@ excludes = [
169189
"^codegen",
170190
]
171191
deps = [
192+
"executorch_no_prim_ops",
172193
"executorch",
173194
]
174195
# ---------------------------------- binary end ----------------------------------
@@ -185,6 +206,7 @@ excludes = [
185206
]
186207
deps = [
187208
"executorch",
209+
"executorch_no_prim_ops",
188210
"portable_kernels",
189211
]
190212

@@ -197,6 +219,7 @@ filters = [
197219
]
198220
deps = [
199221
"executorch",
222+
"executorch_no_prim_ops",
200223
]
201224

202225
[targets.mps_schema]
@@ -222,6 +245,7 @@ excludes = [
222245
]
223246
deps = [
224247
"executorch",
248+
"executorch_no_prim_ops",
225249
"xnnpack_backend",
226250
"portable_kernels",
227251
]
@@ -235,6 +259,7 @@ filters = [
235259
]
236260
deps = [
237261
"executorch",
262+
"executorch_no_prim_ops",
238263
]
239264

240265
[targets.xnnpack_dynamic_quant_utils]
@@ -275,6 +300,7 @@ excludes = [
275300
]
276301
deps = [
277302
"executorch",
303+
"executorch_no_prim_ops",
278304
"optimized_kernels",
279305
"xnnpack_backend",
280306
]
@@ -292,6 +318,7 @@ excludes = [
292318
deps = [
293319
"custom_ops",
294320
"executorch",
321+
"executorch_no_prim_ops",
295322
"extension_data_loader",
296323
"extension_module",
297324
"portable_kernels",

‎build/executorch-config.cmake

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -13,27 +13,20 @@
1313
cmake_minimum_required(VERSION 3.19)
1414

1515
set(_root "${CMAKE_CURRENT_LIST_DIR}/../..")
16-
add_library(executorch STATIC IMPORTED)
17-
find_library(
18-
EXECUTORCH_LIBRARY_PATH executorch
19-
HINTS "${_root}"
20-
CMAKE_FIND_ROOT_PATH_BOTH
21-
)
22-
set_target_properties(
23-
executorch PROPERTIES IMPORTED_LOCATION "${EXECUTORCH_LIBRARY_PATH}"
24-
)
25-
target_include_directories(executorch INTERFACE ${_root})
16+
set(required_lib_list executorch executorch_no_prim_ops portable_kernels)
17+
foreach(lib ${required_lib_list})
18+
set(lib_var "LIB_${lib}")
19+
add_library(${lib} STATIC IMPORTED)
20+
find_library(
21+
${lib_var} ${lib} HINTS "${_root}" CMAKE_FIND_ROOT_PATH_BOTH
22+
)
23+
set_target_properties(
24+
${lib} PROPERTIES IMPORTED_LOCATION "${${lib_var}}"
25+
)
26+
target_include_directories(${lib} INTERFACE ${_root})
27+
endforeach()
2628

27-
add_library(portable_kernels STATIC IMPORTED)
28-
find_library(
29-
PORTABLE_KERNELS_PATH portable_kernels
30-
HINTS "${_root}"
31-
CMAKE_FIND_ROOT_PATH_BOTH
32-
)
33-
set_target_properties(
34-
portable_kernels PROPERTIES IMPORTED_LOCATION "${PORTABLE_KERNELS_PATH}"
35-
)
36-
target_include_directories(portable_kernels INTERFACE ${_root})
29+
target_link_libraries(executorch INTERFACE executorch_no_prim_ops)
3730

3831
if(CMAKE_BUILD_TYPE MATCHES "Debug")
3932
set(FLATCCRT_LIB flatccrt_d)

‎examples/apple/coreml/executor_runner/coreml_executor_runner.xcodeproj/project.pbxproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
C94D51662ACFCBCB00AF47FD /* Accelerate.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C94D51652ACFCBCB00AF47FD /* Accelerate.framework */; };
1717
C94D51682ACFCC7100AF47FD /* libcoremldelegate.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C94D51672ACFCC7100AF47FD /* libcoremldelegate.a */; };
1818
C988D69D2B998CDE00979CF6 /* libprotobuf-lite.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C988D69C2B998CD700979CF6 /* libprotobuf-lite.a */; };
19+
F24817E72BC65B2000E80D98 /* libexecutorch_no_prim_ops.a in Frameworks */ = {isa = PBXBuildFile; fileRef = F24817E62BC65B2000E80D98 /* libexecutorch_no_prim_ops.a */; };
1920
/* End PBXBuildFile section */
2021

2122
/* Begin PBXCopyFilesBuildPhase section */
@@ -41,6 +42,7 @@
4142
C94D51652ACFCBCB00AF47FD /* Accelerate.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Accelerate.framework; path = System/Library/Frameworks/Accelerate.framework; sourceTree = SDKROOT; };
4243
C94D51672ACFCC7100AF47FD /* libcoremldelegate.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libcoremldelegate.a; path = libraries/libcoremldelegate.a; sourceTree = "<group>"; };
4344
C988D69C2B998CD700979CF6 /* libprotobuf-lite.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "libprotobuf-lite.a"; path = "libraries/libprotobuf-lite.a"; sourceTree = "<group>"; };
45+
F24817E62BC65B2000E80D98 /* libexecutorch_no_prim_ops.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = libexecutorch_no_prim_ops.a; path = libraries/libexecutorch_no_prim_ops.a; sourceTree = "<group>"; };
4446
/* End PBXFileReference section */
4547

4648
/* Begin PBXFrameworksBuildPhase section */
@@ -49,6 +51,7 @@
4951
buildActionMask = 2147483647;
5052
files = (
5153
38626BB52B225A890059413D /* libetdump.a in Frameworks */,
54+
F24817E72BC65B2000E80D98 /* libexecutorch_no_prim_ops.a in Frameworks */,
5255
38626BB42B225A560059413D /* libflatccrt.a in Frameworks */,
5356
C94D51682ACFCC7100AF47FD /* libcoremldelegate.a in Frameworks */,
5457
C94D51662ACFCBCB00AF47FD /* Accelerate.framework in Frameworks */,
@@ -90,6 +93,7 @@
9093
C94D515C2ACFCBA000AF47FD /* libexecutorch.a */,
9194
C94D51612ACFCBBA00AF47FD /* libsqlite3.tbd */,
9295
C94D51672ACFCC7100AF47FD /* libcoremldelegate.a */,
96+
F24817E62BC65B2000E80D98 /* libexecutorch_no_prim_ops.a */,
9397
);
9498
name = Frameworks;
9599
sourceTree = "<group>";

‎examples/apple/coreml/scripts/build_executor_runner.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ cp -rf "$COREML_DIR_PATH/runtime/include/" "$INCLUDE_DIR_PATH"
6161
echo "ExecuTorch: Copying libraries"
6262
mkdir "$LIBRARIES_DIR_PATH"
6363
find "$CMAKE_BUILD_DIR_PATH/" -name 'libexecutorch.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH" \;
64+
find "$CMAKE_BUILD_DIR_PATH/" -name 'libexecutorch_no_prim_ops.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH" \;
6465
find "$CMAKE_BUILD_DIR_PATH/" -name 'libetdump.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH" \;
6566
find "$CMAKE_BUILD_DIR_PATH/" -name 'libcoremldelegate.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH" \;
6667
find "$CMAKE_BUILD_DIR_PATH/" -name 'libprotobuf-lite.a' -exec cp -f "{}" "$LIBRARIES_DIR_PATH" \;

‎examples/arm/executor_runner/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ add_library(executorch STATIC IMPORTED)
4343
set_property(TARGET executorch PROPERTY IMPORTED_LOCATION
4444
"${ET_BUILD_DIR_PATH}/libexecutorch.a")
4545

46+
add_library(executorch_no_prim_ops STATIC IMPORTED)
47+
set_property(TARGET executorch_no_prim_ops PROPERTY IMPORTED_LOCATION
48+
"${ET_BUILD_DIR_PATH}/libexecutorch_no_prim_ops.a")
49+
target_link_libraries(executorch INTERFACE executorch_no_prim_ops)
50+
4651
add_library(executorch_delegate_ethos_u STATIC IMPORTED)
4752
set_property(TARGET executorch_delegate_ethos_u PROPERTY IMPORTED_LOCATION
4853
"${ET_BUILD_DIR_PATH}/backends/arm/libexecutorch_delegate_ethos_u.a")

0 commit comments

Comments
 (0)