merge bitcoin#21778: LLD based macOS toolchain

Author: kwvg

contrib/containers/ci/Dockerfile

@@ -58,20 +58,24 @@ RUN set -ex; \
 ARG LLVM_VERSION=16
 RUN set -ex; \
     echo "Installing LLVM and Clang ${LLVM_VERSION}..."; \
-    curl -sL https://apt.llvm.org/llvm.sh | bash -s "${LLVM_VERSION}"; \
+    curl -sL https://apt.llvm.org/llvm.sh | bash -s -- "${LLVM_VERSION}"; \
     echo "Installing additional packages..."; \
     apt-get update && apt-get install $APT_ARGS \
     "clang-format-${LLVM_VERSION}" \
     "clang-tidy-${LLVM_VERSION}" \
     "libc++-${LLVM_VERSION}-dev" \
     "libc++abi-${LLVM_VERSION}-dev" \
-    "libclang-rt-${LLVM_VERSION}-dev"; \
+    "libclang-rt-${LLVM_VERSION}-dev" \
+    "lld-${LLVM_VERSION}"; \
     rm -rf /var/lib/apt/lists/*; \
     echo "Setting defaults..."; \
     lldbUpdAltArgs="update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${LLVM_VERSION} 100"; \
-    for binName in clang clang++ clang-format clang-tidy clangd ld.lld lldb lldb-server; do \
+    for binName in clang clang++ clang-format clang-tidy clangd dsymutil lld lldb lldb-server llvm-ar llvm-cov llvm-nm llvm-objdump llvm-ranlib llvm-strip; do \
         lldbUpdAltArgs="${lldbUpdAltArgs} --slave /usr/bin/${binName} ${binName} /usr/bin/${binName}-${LLVM_VERSION}"; \
     done; \
+    for binName in ld64.lld ld.lld lld-link wasm-ld; do \
+        lldbUpdAltArgs="${lldbUpdAltArgs} --slave /usr/bin/${binName} ${binName} /usr/bin/lld-${LLVM_VERSION}"; \
+    done; \
     sh -c "${lldbUpdAltArgs}";
 # LD_LIBRARY_PATH is empty by default, this is the first entry
 ENV LD_LIBRARY_PATH="/usr/lib/llvm-${LLVM_VERSION}/lib"

contrib/devtools/symbol-check.py

@@ -252,7 +252,7 @@ def check_MACHO_sdk(binary) -> bool:
     return False
 
 def check_MACHO_ld64(binary) -> bool:
-    if binary.build_version.tools[0].version == [711, 0, 0]:
+    if binary.build_version.tools[0].version == [17, 0, 6]:
         return True
     return False
 

contrib/devtools/test-security-check.py

@@ -121,21 +121,15 @@ def test_MACHO(self):
         arch = get_arch(cc, source, executable)
 
         if arch == lief.ARCHITECTURES.X86:
-            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector', '-Wl,-no_fixup_chains']),
-                (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS PIE NX CONTROL_FLOW'))
-            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector', '-Wl,-fixup_chains']),
-                (1, executable+': failed NOUNDEFS Canary PIE NX CONTROL_FLOW'))
-            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all', '-Wl,-fixup_chains']),
-                (1, executable+': failed NOUNDEFS PIE NX CONTROL_FLOW'))
-            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']),
-                (1, executable+': failed NOUNDEFS PIE CONTROL_FLOW'))
-            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']),
-                (1, executable+': failed PIE CONTROL_FLOW'))
-            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-Wl,-fixup_chains']),
-                (1, executable+': failed PIE CONTROL_FLOW'))
-            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']),
-                (1, executable+': failed PIE'))
-            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']),
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-no_fixup_chains']),
+                (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS PIE CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains']),
+                (1, executable+': failed NOUNDEFS Canary CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']),
+                (1, executable+': failed NOUNDEFS CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains']),
+                (1, executable+': failed CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-fcf-protection=full', '-Wl,-fixup_chains']),
                 (0, ''))
         else:
             # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks

contrib/guix/libexec/build.sh

@@ -135,18 +135,7 @@ for p in "${PATHS[@]}"; do
 done
 
 # Disable Guix ld auto-rpath behavior
-case "$HOST" in
-    *darwin*)
-        # The auto-rpath behavior is necessary for darwin builds as some native
-        # tools built by depends refer to and depend on Guix-built native
-        # libraries
-        #
-        # After the native packages in depends are built, the ld wrapper should
-        # no longer affect our build, as clang would instead reach for
-        # x86_64-apple-darwin-ld from cctools
-        ;;
-    *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;;
-esac
+export GUIX_LD_WRAPPER_DISABLE_RPATH=yes
 
 # Make /usr/bin if it doesn't exist
 [ -e /usr/bin ] || mkdir -p /usr/bin
@@ -175,16 +164,6 @@ esac
 # Environment variables for determinism
 export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name"
 export TZ="UTC"
-case "$HOST" in
-    *darwin*)
-        # cctools AR, unlike GNU binutils AR, does not have a deterministic mode
-        # or a configure flag to enable determinism by default, it only
-        # understands if this env-var is set or not. See:
-        #
-        # https://github.com/tpoechtrager/cctools-port/blob/55562e4073dea0fbfd0b20e0bf69ffe6390c7f97/cctools/ar/archive.c#L334
-        export ZERO_AR_DATE=yes
-        ;;
-esac
 
 ####################
 # Depends Building #

contrib/guix/manifest.scm

@@ -607,8 +607,9 @@ inspecting signatures in Mach-O binaries.")
           ((string-contains target "darwin")
            (list ;; Native GCC 11 toolchain
                  gcc-toolchain-11
-                 binutils
                  clang-toolchain-17
+                 lld-17
+                 (make-lld-wrapper lld-17 #:lld-as-ld? #t)
                  python-signapple
                  zip))
           (else '())))))

contrib/macdeploy/README.md

@@ -56,28 +56,13 @@ The `sha256sum` should be `c0c2e7bb92c1fee0c4e9f3a485e4530786732d6c6dd9e9f418c28
 
 ## Deterministic macOS App Notes
 
-macOS Applications are created in Linux by combining a recent `clang` and the Apple
-`binutils` (`ld`, `ar`, etc).
+macOS Applications are created in Linux using a recent LLVM.
 
 Apple uses `clang` extensively for development and has upstreamed the necessary
 functionality so that a vanilla clang can take advantage. It supports the use of `-F`,
 `-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when
 building for macOS.
 
-Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the
-FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several
-other tools are needed as well. These do not build under Linux, so they have been patched to
-do so. The work here was used as a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4).
-
-In order to build a working toolchain, the following source packages are needed from
-Apple: `cctools`, `dyld`, and `ld64`.
-
-These tools inject timestamps by default, which produce non-deterministic binaries. The
-`ZERO_AR_DATE` environment variable is used to disable that.
-
-This version of `cctools` has been patched to use the current version of `clang`'s headers
-and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`.
-
 To complicate things further, all builds must target an Apple SDK. These SDKs are free to
 download, but not redistributable. See the SDK Extraction notes above for how to obtain it.
 

depends/Makefile

@@ -178,7 +178,6 @@ all_packages = $(packages) $(native_packages)
 
 meta_depends = Makefile config.guess config.sub funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
 
-$(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils)
 $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
 
 include funcs.mk
@@ -210,9 +209,8 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
 #      tool needs to be available in $PATH at all times.
 #
 #   2. If the tool is _**not**_ expected to be available in $PATH at all times
-#      (such as is the case for our native_cctools binutils tools), it needs to
-#      be referred to by its absolute path, such as would be output by the
-#      AC_PATH_{PROG,TOOL} macros.
+#      it needs to be referred to by its absolute path, such as would be output
+#      by the AC_PATH_{PROG,TOOL} macros.
 #
 # Minor note: it is also okay to refer to tools by their absolute path even if
 # we expect them to be available in $PATH at all times, more specificity does

depends/builders/darwin.mk

@@ -18,7 +18,6 @@ darwin_STRIP:=$(shell xcrun -f strip)
 darwin_OBJDUMP:=$(shell xcrun -f objdump)
 darwin_NM:=$(shell xcrun -f nm)
 darwin_DSYMUTIL:=$(shell xcrun -f dsymutil)
-darwin_native_binutils=
 darwin_native_toolchain=
 
 x86_64_darwin_CFLAGS += -arch x86_64

depends/funcs.mk

@@ -46,7 +46,7 @@ endef
 
 define int_get_build_id
 $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies))
-$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies)))
+$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies)))
 $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash)))
 $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id))
 $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)))
@@ -299,4 +299,4 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$
 $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package))))
 
 #special exception: if a toolchain package exists, all non-native packages depend on it
-$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) $($($(host_arch)_$(host_os)_native_binutils)_cached) ))
+$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) ))

depends/gen_id

@@ -50,7 +50,6 @@
     echo "BEGIN AR"
     bash -c "${AR} --version"
     env | grep '^AR_'
-    echo "ZERO_AR_DATE=${ZERO_AR_DATE}"
     echo "END AR"
 
     echo "BEGIN RANLIB"

depends/hosts/darwin.mk

@@ -6,24 +6,28 @@ LD64_VERSION=711
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
 
-darwin_native_binutils=native_cctools
-
-cctools_TOOLS=AR RANLIB STRIP NM
-
 ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang
+# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned LLVM
 # from llvm.org
 
-# Clang is a dependency of native_cctools when FORCE_USE_SYSTEM_CLANG is empty
-darwin_native_toolchain=native_cctools
-
-cctools_TOOLS+=" DSYMUTIL"
+darwin_native_toolchain=native_llvm
 
 clang_prog=$(build_prefix)/bin/clang
 clangxx_prog=$(clang_prog)++
 llvm_config_prog=$(build_prefix)/bin/llvm-config
 
-darwin_OBJDUMP=$(build_prefix)/bin/$(host)-objdump
+llvm_TOOLS=AR NM OBJDUMP RANLIB STRIP
+
+# Make-only lowercase function
+lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
+
+# For well-known tools provided by LLVM, make sure that their well-known
+# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG}
+# would.
+$(foreach TOOL,$(llvm_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/llvm-$(call lc,$(TOOL))))
+
+# Clang expects dsymutil to be called dsymutil
+darwin_DSYMUTIL=$(build_prefix)/bin/dsymutil
 
 else
 # FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's
@@ -44,18 +48,14 @@ llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config")
 
 llvm_lib_dir=$(shell $(llvm_config_prog) --libdir)
 
+darwin_AR=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ar")
 darwin_DSYMUTIL=$(shell $(SHELL) $(.SHELLFLAGS) "command -v dsymutil")
+darwin_NM=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-nm")
 darwin_OBJDUMP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-objdump")
+darwin_RANLIB=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ranlib")
+darwin_STRIP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-strip")
 endif
 
-# Make-only lowercase function
-lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
-
-# For well-known tools provided by cctools, make sure that their well-known
-# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG}
-# would.
-$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL))))
-
 # Flag explanations:
 #
 #     -mlinker-version
@@ -65,7 +65,7 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(
 #
 #     -B$(build_prefix)/bin
 #
-#         Explicitly point to our binaries (e.g. cctools) so that they are
+#         Explicitly point to our binaries so that they are
 #         ensured to be found and preferred over other possibilities.
 #
 #     -isysroot$(OSX_SDK) -nostdlibinc
@@ -82,6 +82,11 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(
 #
 #         Indicate to the linker the platform, the oldest supported version,
 #         and the SDK used.
+#
+#     -no_adhoc_codesign
+#
+#         Disable adhoc codesigning (for now) when using LLVM tooling, to avoid
+#         non-determinism issues with the Identifier field.
 
 darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
               -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
@@ -107,6 +112,7 @@ darwin_LDFLAGS=-Wl,-platform_version,macos,$(OSX_MIN_VERSION),$(OSX_SDK_VERSION)
 ifneq ($(build_os),darwin)
 darwin_CFLAGS += -mlinker-version=$(LD64_VERSION)
 darwin_CXXFLAGS += -mlinker-version=$(LD64_VERSION)
+darwin_LDFLAGS += -Wl,-no_adhoc_codesign -fuse-ld=lld
 endif
 
 darwin_release_CFLAGS=-O2

depends/packages/native_cctools.mk

@@ -16,9 +16,14 @@ define $(package)_stage_cmds
   mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \
   cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \
   cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \
-  cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \
+  cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/dsymutil && \
+  cp bin/ld64.lld $($(package)_staging_prefix_dir)/bin/ld64.lld && \
+  cp bin/llvm-ar $($(package)_staging_prefix_dir)/bin/llvm-ar && \
   cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \
-  cp bin/llvm-objdump $($(package)_staging_prefix_dir)/bin/$(host)-objdump && \
+  cp bin/llvm-nm $($(package)_staging_prefix_dir)/bin/llvm-nm && \
+  cp bin/llvm-objdump $($(package)_staging_prefix_dir)/bin/llvm-objdump && \
+  cp bin/llvm-ranlib $($(package)_staging_prefix_dir)/bin/llvm-ranlib && \
+  cp bin/llvm-strip $($(package)_staging_prefix_dir)/bin/llvm-strip && \
   cp include/llvm-c/ExternC.h $($(package)_staging_prefix_dir)/include/llvm-c && \
   cp include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c && \
   cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \

depends/packages/native_libtapi.mk

@@ -27,7 +27,6 @@ usdt_linux_packages=systemtap
 darwin_native_packages =
 
 ifneq ($(build_os),darwin)
-darwin_native_packages += native_cctools native_libtapi
 
 ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 darwin_native_packages+= native_llvm