Enable rust powerpc-unknown-linux-gnuspe for qoriq (#5879)

* py311, sabnzbd, homeassistant: Enable debug_symbols for qoriq

* py311, sabnzbd, homeassistant: Enable debug_symbols also for x64

* py311: Force compiler LTO optimizations with exceptions
  - NOT when testing all wheels
  - NOT when including debug_info symbols
  - NOT when using PPC arch (fails on qoriq)

* py311+sabnzbd+homeassistant: Enable debug_info symbols for all archs

* rustc: Add RUSTFLAGS environment variable option

* qoriq: Tentatively use additional RUSTFLAGS for arch

* toolchain qoriq-6.2.4: Revert RUSTFLAGS and create standard links

- Removing from RUSTFLAGS target-feature=-hard-float,-spe as unsupported
- Create symbolic links using standard "target triple" naming in order
  to avoid confusion due to the usage from Synology of e500v2 specific

* rust: Add support for powerpc-unknown-linux-gnuspe tier 3 target

* rust: Use RUSTUP_TOOLCHAIN env variable to define used toolchain

* rust: Fix default toolchain checking

* duplicity: Update bcrypt wheel from version 4.0.1 to 4.1.1

* py310-py311: Update bcrypt wheel from version 4.0.1 to 4.1.1

* homeassistant: Update bcrypt 4.1.1 and cryptography 41.0.3

* rust: Enforce using newer native/cmake when building toolchain

* rust: Fix typo when building native/cmake

* rust: Tentative to fix online issue relatively to cmake

* rust: Different approach to enforce using native/cmake

* rust: Enforcing --sysroot at link time for tier-3 toolchain builds

* rust: Enforce sysroot using extra RUSTFLAGS variable

* rust: Downgrade to stage1 level compiler and disable toolkit

* rust: Rename custom toolchain to match RUSTUP_TOOLCHAIN for clarity

* rust: Revert back at using stage2 compiler as it fails online

* rust: Clean-up stage0 and stage1 after building stage2

* rust: Add ADDITIONAL_RUSTFLAGS to target-tripple rustflags env var

* rust.mk: Use RUSTUP_TOOLCHAIN overriding +stable|$(toolchain)

* rust: First working version for powerpc-*-gnuspe for qoriq arch

NOTE: requires clean-up and review of every extra variable set

* rust: Removed unused build flags

* rust: Enforce deleting config.toml prior to setting up compiler

* rust: Either use native/llvm-9.0 or include clang-llvm to docker

* rust: Attempt at fixing LLVM error on github-action

* rust: Fix typo in precedent commit

* rust: Remove previous LLVM attempts

* rust: Add llvm-13 and clang-13 part of the base install

* rust: Revert commit related to llvm-13 and clang-13 in Dockerfile

* rust: Enforce build to x86_64-unknown-linux-gnu

* rust: Revert enforcing build triplet

* rust: Disable download-ci-llvm to enforce building of LLVM

* rust: Don't enforce building x86_64 and remove llvm to use default

* rust: Enforce building LLVM

* rust: tentative typo fixing for llvm building

* rust: Change default stage to 2

Hoping to solve github-action error message:
failed: x.py should be run with `--stage 2` on CI, but was run with `--stage 1`

* rust: Re-try downloading LLVM using if-unchanged option

* rust: llvm "if-unchanged" seem to reuquire quotes

* rust: re-introduce host target as otherwise missing crates

This was fixed earlier by ensuring to build host rust as well
part of that toolchain otherwise get errors such as:
error[E0463]: can't find crate for `core`
error[E0463]: can't find crate for `compiler_builtins`

* rust: Specify RUSTUP_TOOLCHAIN so python wheel can build

* native/rust-qoriq: Ability to download and use pre-built toolchain

* rust: Enable usage of native/rust-qoriq

* rust: Fix toolchain setting-up dues to TC_ARCH vs ARCH usages

* rust: "ARCH" detection for toolchain independantly from location

When called from cross or spk usage of $(ARCH)
When called from toolchain:
   - Usage of $(TC_ARCH) when $(TC_NAME) does not exist
   - Usage of $(TC_NAME) for generic archs

* python311: Remove duplicate code kept from rebasing

* homeassistant: Disable GCC_DEBUG_INFO

* python311: Disable GCC_DEBUG_INFO

* sabnzbd: Disable GCC_DEBUG_INFO

* qoriq-6.2.4: convert test to -L and using ln -sf (@hgy59)

* rust: Enable RUST_BUILD_TOOLCHAIN=1 to test rebuild still works

* python311: Fix typo relatively to failure on old ppc only

* tc-rust.mk: Removing left-over exit 2 for debugging purpose

* rust-qoriq: Disable toolchain testing building in favor of native

Author: th0ma7

mk/spksrc.cross-rust-env.mk

@@ -1,41 +1,59 @@
 # Configuration for rust compiler
-# 
+#
 
 # Add cargo for rust compiler to default PATH
 ifneq ($(BASE_DISTRIB_DIR),)
+# Enforce newer cmake when building tier-3 toolchains
+CMAKE_PATH = $(abspath $(WORK_DIR)/../../../native/cmake/work-native/install/usr/local/bin)
 export CARGO_HOME=$(BASE_DISTRIB_DIR)/cargo
 export RUSTUP_HOME=$(BASE_DISTRIB_DIR)/rustup
-export PATH:=$(BASE_DISTRIB_DIR)/cargo/bin:$(PATH)
+export PATH:=$(abspath $(BASE_DISTRIB_DIR)/cargo/bin):$(CMAKE_PATH):$(PATH)
 endif
 
-ifeq ($(RUST_TOOLCHAIN),)
-RUST_TOOLCHAIN = stable
+ifeq ($(RUSTUP_DEFAULT_TOOLCHAIN),)
+RUSTUP_DEFAULT_TOOLCHAIN = stable
 endif
 
-RUST_TARGET =
+# When calling directly from toolchain/syno-<arch>-<version>
+# ARCH variable is still unset thus using $(TC_ARCH) although
+# in generic archs we must rely on $(TC_NANE)
+RUST_ARCH = $(or $(ARCH),$(lastword $(subst -, ,$(TC_NAME))),$(TC_ARCH))
+
+# When building toolchain Tier-3 arch support
+#   While stage-2 is the truly current compiler, stage-1 suffice our needs
+#   https://rustc-dev-guide.rust-lang.org/building/bootstrapping.html#stage-2-the-truly-current-compiler
+RUSTUP_DEFAULT_TOOLCHAIN_STAGE = 2
+
 # map archs to rust targets
-ifeq ($(findstring $(ARCH), $(x64_ARCHS)),$(ARCH))
-RUST_TARGET = x86_64-unknown-linux-gnu
-endif
-ifeq ($(findstring $(ARCH), $(i686_ARCHS)),$(ARCH))
-RUST_TARGET = i686-unknown-linux-gnu
-endif
-ifeq ($(findstring $(ARCH), $(ARMv5_ARCHS)),$(ARCH))
-# may be not supported for cargo
+ifeq ($(findstring $(RUST_ARCH), $(ARMv5_ARCHS)),$(RUST_ARCH))
 RUST_TARGET = armv5te-unknown-linux-gnueabi
 endif
-ifeq ($(findstring $(ARCH), $(ARMv7_ARCHS)),$(ARCH))
+ifeq ($(findstring $(RUST_ARCH), $(ARMv7_ARCHS)),$(RUST_ARCH))
 RUST_TARGET = armv7-unknown-linux-gnueabihf
 endif
-ifeq ($(findstring $(ARCH), $(ARMv7L_ARCHS)),$(ARCH))
+ifeq ($(findstring $(RUST_ARCH), $(ARMv7L_ARCHS)),$(RUST_ARCH))
 RUST_TARGET = armv7-unknown-linux-gnueabi
 endif
-ifeq ($(findstring $(ARCH), $(ARMv8_ARCHS)),$(ARCH))
+ifeq ($(findstring $(RUST_ARCH), $(ARMv8_ARCHS)),$(RUST_ARCH))
 RUST_TARGET = aarch64-unknown-linux-gnu
 endif
-ifeq ($(findstring $(ARCH), $(PPC_ARCHS)),$(ARCH))
-RUST_TARGET = powerpc-unknown-linux-gnu
+ifeq ($(findstring $(RUST_ARCH), $(PPC_ARCHS)),$(RUST_ARCH))
+RUST_BUILD_TOOLCHAIN = 0
+RUST_TARGET = powerpc-unknown-linux-gnuspe
+TC_RUSTUP_TOOLCHAIN = stage$(RUSTUP_DEFAULT_TOOLCHAIN_STAGE)-$(RUST_TARGET)
 endif
+ifeq ($(findstring $(RUST_ARCH), $(x64_ARCHS)),$(RUST_ARCH))
+RUST_TARGET = x86_64-unknown-linux-gnu
+endif
+ifeq ($(findstring $(RUST_ARCH), $(i686_ARCHS)),$(RUST_ARCH))
+RUST_TARGET = i686-unknown-linux-gnu
+endif
+
 ifeq ($(RUST_TARGET),)
-$(error Arch $(ARCH) not supported)
+$(error Arch $(RUST_ARCH) not supported)
+endif
+
+# By default use the default toolchain if unset
+ifeq ($(TC_RUSTUP_TOOLCHAIN),)
+TC_RUSTUP_TOOLCHAIN = $(RUSTUP_DEFAULT_TOOLCHAIN)
 endif

mk/spksrc.cross-rust.mk

@@ -59,10 +59,14 @@ CARGO_INSTALL_ARGS += $(CARGO_BUILD_ARGS)
 endif
 
 # Default build with rust and install with cargo
+# The cargo call uses tc_vars.mk RUSTUP_TOOLCHAIN variable
+# overriding definition using +stable or +$(RUSTUP_TOOLCHAIN)
+# https://rust-lang.github.io/rustup/environment-variables.html 
 rust_install_target:
-	@echo "  ==> Cargo install rust package $(PKG_NAME) ($(shell rustc --version); $(RUST_TOOLCHAIN))"
-	@$(RUN) cargo +$(RUST_TOOLCHAIN) install $(CARGO_INSTALL_ARGS)
-
+	@echo "  ==> Cargo install rust package $(PKG_NAME) (rustc +$(TC_RUSTUP_TOOLCHAIN) -vV)"
+	@$(RUN) rustc +$(TC_RUSTUP_TOOLCHAIN) -vV
+	@$(RUN) echo cargo +$(TC_RUSTUP_TOOLCHAIN) install $(CARGO_INSTALL_ARGS) --target $(RUST_TARGET)
+	@$(RUN) cargo +$(TC_RUSTUP_TOOLCHAIN) install $(CARGO_INSTALL_ARGS) --target $(RUST_TARGET)
 
 #####
 

mk/spksrc.tc-rust.mk

@@ -6,6 +6,14 @@
 #  rustc_target       (override with RUSTC_TARGET)
 #  post_rustc_target  (override with POST_RUSTC_TARGET)
 
+# Define rustc configuration toml file location
+# when rebuilding for unsupported archs (i.e. Tier 3)
+ifeq ($(strip $(TC_NAME)),)
+TC_LOCAL_VARS_RUST = $(WORK_DIR)/$(TC_ARCH).toml
+else
+TC_LOCAL_VARS_RUST = $(WORK_DIR)/$(lastword $(subst -, ,$(TC_NAME))).toml
+endif
+
 # Configure file descriptor lock timeout
 ifeq ($(strip $(FLOCK_TIMEOUT)),)
 FLOCK_TIMEOUT = 300
@@ -32,6 +40,43 @@ endif
 .PHONY: rustc rustc_msg
 .PHONY: $(PRE_RUSTC_TARGET) $(RUSTC_TARGET) $(POST_RUSTC_TARGET)
 
+.PHONY: $(TC_LOCAL_VARS_RUST)
+$(TC_LOCAL_VARS_RUST):
+	env $(MAKE) --no-print-directory rust_toml > $@ 2>/dev/null;
+
+.PHONY: rust_toml
+rust_toml:
+	@echo 'profile = "compiler"' ; \
+	echo
+	@echo "[build]" ; \
+	echo 'target = ["x86_64-unknown-linux-gnu", "$(RUST_TARGET)"]' ; \
+	echo "build-stage = $(RUSTUP_DEFAULT_TOOLCHAIN_STAGE)" ; \
+	echo "doc-stage = 2" ; \
+	echo "docs = false" ; \
+	echo "docs-minification = false" ; \
+	echo "compiler-docs = false" ; \
+	echo
+	@echo "[rust]" ; \
+	echo 'channel = "stable"' ; \
+	echo 'lto = "off"' ; \
+	echo
+	@echo "[llvm]" ; \
+	echo 'download-ci-llvm = "if-unchanged"' ; \
+	echo
+	@echo "[install]" ; \
+	echo
+	@echo "[dist]" ; \
+	echo
+	@echo "[target.x86_64-unknown-linux-gnu]" ; \
+	echo
+	@echo "[target.$(RUST_TARGET)]" ; \
+	echo 'cc = "$(WORK_DIR)/$(TC_TARGET)/bin/$(TC_PREFIX)gcc"' ; \
+	echo 'cxx = "$(WORK_DIR)/$(TC_TARGET)/bin/$(TC_PREFIX)g++"' ; \
+	echo 'ar = "$(WORK_DIR)/$(TC_TARGET)/bin/$(TC_PREFIX)ar"' ; \
+	echo 'ranlib = "$(WORK_DIR)/$(TC_TARGET)/bin/$(TC_PREFIX)ranlib"' ; \
+	echo 'linker = "$(WORK_DIR)/$(TC_TARGET)/bin/$(TC_PREFIX)gcc"' ; \
+	echo
+
 rustc_msg:
 	@$(MSG) "Installing rustc toolchain for $(NAME)"
 	@$(MSG) "- rustup installation PATH: $(RUSTUP_HOME)"
@@ -47,18 +92,46 @@ pre_rustc_target: rustc_msg
 	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path ; \
 	flock -u 5
 
-rustc_target: $(PRE_RUSTC_TARGET)
-	@$(MSG) "rustup toolchain install $(RUST_TOOLCHAIN)" ; \
+rustc_target: $(PRE_RUSTC_TARGET) $(TC_LOCAL_VARS_RUST)
+	@$(MSG) "rustup toolchain install $(RUSTUP_DEFAULT_TOOLCHAIN)" ; \
 	exec 5> /tmp/tc-rustc.lock ; \
 	flock --timeout $(FLOCK_TIMEOUT) --exclusive 5 || exit 1 ; \
 	pid=$$$$ ; \
 	echo "$${pid}" 1>&5 ; \
-	rustup toolchain install $(RUST_TOOLCHAIN) ; \
-	$(MSG) "rustup default $(RUST_TOOLCHAIN)" ; \
-	rustup default $(RUST_TOOLCHAIN) ; \
-	$(MSG) "rustup target add $(RUST_TARGET)" ; \
-	rustup target add $(RUST_TARGET) ; \
+	rustup toolchain install $(RUSTUP_DEFAULT_TOOLCHAIN) ; \
+	$(MSG) "rustup default $(RUSTUP_DEFAULT_TOOLCHAIN)" ; \
+	rustup default $(RUSTUP_DEFAULT_TOOLCHAIN) ; \
 	flock -u 5
+ifeq ($(TC_RUSTUP_TOOLCHAIN),$(RUSTUP_DEFAULT_TOOLCHAIN))
+	@$(MSG) "rustup target add $(RUST_TARGET)"
+	rustup override set stable
+	rustup target add $(RUST_TARGET)
+	rustup show
+else
+	@$(MSG) "Target $(RUST_TARGET) unavailable..."
+ifeq ($(RUST_BUILD_TOOLCHAIN),1)
+	@$(MSG) "Build rust target $(RUST_TARGET) from sources"
+	@$(MSG) "Enforce usage of CMake 3.20.0 or higher"
+	@$(MAKE) -C ../../native/cmake
+	@$(MSG) "Building Tier-3 rust target: $(RUST_TARGET)"
+	@(cd $(WORK_DIR) && [ ! -d rust ] && git clone --depth 1 https://github.com/rust-lang/rust.git || true)
+	@(cd $(WORK_DIR)/rust && rm -f config.toml && ./x setup compiler)
+	(cd $(WORK_DIR)/rust && \
+	   CFLAGS_$(subst -,_,$(RUST_TARGET))="$(TC_EXTRA_CFLAGS)" \
+	   CARGO_TARGET_$(shell echo $(RUST_TARGET) | tr - _ | tr a-z A-Z)_RUSTFLAGS="$(TC_RUSTFLAGS)" \
+	   RUST_BACKTRACE=full \
+	   ./x build --config $(TC_LOCAL_VARS_RUST))
+	rustup toolchain link $(TC_RUSTUP_TOOLCHAIN) $(WORK_DIR)/rust/build/host/stage$(RUSTUP_DEFAULT_TOOLCHAIN_STAGE)
+	@for i in 0 1 2 ; do \
+	   [ $${i} -lt $(RUSTUP_DEFAULT_TOOLCHAIN_STAGE) ] && (cd $(WORK_DIR)/rust && ./x clean --stage $${i}) || true ; \
+	done
+	@$(MSG) "Building Tier 3 rust target: $(RUST_TARGET) - stage$(RUSTUP_DEFAULT_TOOLCHAIN_STAGE) complete"
+	rustup show
+else
+	@$(MSG) "Install rust target $(RUST_TARGET) from native"
+	@$(MAKE) -C ../../native/rust-qoriq
+endif
+endif
 
 post_rustc_target: $(RUSTC_TARGET)
 

mk/spksrc.tc.mk

@@ -154,9 +154,11 @@ tc_vars: flag
 	echo TC_ENV += LDFLAGS=\"$(LDFLAGS) $$\(ADDITIONAL_LDFLAGS\)\" ; \
 	echo TC_ENV += CARGO_HOME=\"$(realpath $(CARGO_HOME))\" ; \
 	echo TC_ENV += RUSTUP_HOME=\"$(realpath $(RUSTUP_HOME))\" ; \
+	echo TC_ENV += RUSTUP_TOOLCHAIN=\"$(TC_RUSTUP_TOOLCHAIN)\" ; \
 	echo TC_ENV += CARGO_BUILD_TARGET=\"$(RUST_TARGET)\" ; \
 	echo TC_ENV += CARGO_TARGET_$(shell echo $(RUST_TARGET) | tr - _ | tr a-z A-Z)_AR=\"$(WORK_DIR)/$(TC_TARGET)/bin/$(TC_PREFIX)ar\" ; \
 	echo TC_ENV += CARGO_TARGET_$(shell echo $(RUST_TARGET) | tr - _ | tr a-z A-Z)_LINKER=\"$(WORK_DIR)/$(TC_TARGET)/bin/$(TC_PREFIX)gcc\" ; \
+	echo TC_ENV += CARGO_TARGET_$(shell echo $(RUST_TARGET) | tr - _ | tr a-z A-Z)_RUSTFLAGS=\"$(TC_RUSTFLAGS) $$\(ADDITIONAL_RUSTFLAGS\)\" ; \
 	echo TC_CONFIGURE_ARGS := --host=$(TC_TARGET) --build=i686-pc-linux ; \
 	echo TC_TYPE := $(TC_TYPE) ; \
 	echo TC_SYSROOT := $(WORK_DIR)/$(TC_TARGET)/$(TC_SYSROOT) ; \

native/rust-qoriq/Makefile

@@ -0,0 +1,29 @@
+PKG_NAME = rust-qoriq
+PKG_VERS = 1.77.0-nightly
+PKG_EXT = txz
+PKG_DIST_NAME = stage2_powerpc-unknown-linux-gnuspe.txz
+PKG_DIST_SITE = https://github.com/SynoCommunity/spksrc/releases/download/toolchains%2Frust%2F1.77.0-nightly
+EXTRACT_PATH = $(INSTALL_DIR)/$(INSTALL_PREFIX)
+
+DEPENDS = 
+
+HOMEPAGE = https://www.rust-lang.org/
+COMMENT = A language empowering everyone to build reliable and efficient software.
+LICENSE  = Apache-2.0, MIT licenses
+
+# extracted directly into install folder
+INSTALL_TARGET = nop
+POST_INSTALL_TARGET = rust-qoriq-postinstall
+
+CARGO_PATH=$(abspath $(BASE_DISTRIB_DIR)/cargo/bin)
+RUSTUP_HOME=$(abspath $(BASE_DISTRIB_DIR)/rustup)
+
+include ../../mk/spksrc.native-install.mk
+
+.PHONY: rust-qoriq-postinstall
+rust-qoriq-postinstall:
+	export PATH="$(CARGO_PATH):$${PATH}" ; \
+	export RUSTUP_HOME="$(RUSTUP_HOME)" ; \
+	which rustup ; \
+	rustup toolchain link stage2-powerpc-unknown-linux-gnuspe $(abspath $(INSTALL_DIR)/$(INSTALL_PREFIX)) ; \
+	rustup show

native/rust-qoriq/digests

@@ -0,0 +1,3 @@
+stage2_powerpc-unknown-linux-gnuspe.txz SHA1 377911465d0364e13136fa50d19bfd0a3660a373
+stage2_powerpc-unknown-linux-gnuspe.txz SHA256 e02030225a6e6e1f680f4bdc4e902506f8742a73ba02673a80640d9aaf336fea
+stage2_powerpc-unknown-linux-gnuspe.txz MD5 1129e9a88ccfaa94217a18e26d8adc00

spk/duplicity/src/requirements-crossenv.txt

@@ -7,7 +7,7 @@ duplicity==1.2.3
 # Using launchpad source as alternate
 #https://launchpad.net/duplicity/0.8-series/0.8.23/+download/duplicity-0.8.23.tar.gz#egg=duplicity==0.8.23
 
-bcrypt==4.0.1
+bcrypt==4.1.1
 cffi==1.15.1
 coverage==7.2.7
 #cryptography               ==> cross/cryptography

spk/homeassistant/src/requirements-crossenv.txt

@@ -1,5 +1,5 @@
 aiohttp==3.8.5
-bcrypt==4.0.1
+bcrypt==4.1.1
 bitstruct==8.17.0
 bluetooth_data_tools==1.3.0
 Brotli==1.0.9
@@ -9,7 +9,7 @@ chacha20poly1305_reuseable==0.4.1
 ciso8601==2.3.0
 coverage==7.2.7
 crcmod==1.7
-cryptography==41.0.1
+cryptography==41.0.3
 dbus_fast==1.86.0
 ephem==4.1.2
 faust_cchardet==2.1.18

spk/python310/src/requirements-crossenv.txt

@@ -8,7 +8,7 @@
 # Require environment variables
 #  PYO3_CROSS_LIB_DIR=$(STAGING_INSTALL_PREFIX)/lib/
 #  PYO3_CROSS_INCLUDE_DIR=$(STAGING_INSTALL_PREFIX)/include/
-bcrypt==4.0.1
+bcrypt==4.1.1
 
 # [cryptography]
 # Mandatory of using OPENSSL_*_DIR starting with version >= 40

spk/python311/Makefile

@@ -104,10 +104,10 @@ include ../../mk/spksrc.common.mk
 # Force compiler LTO optimizations except:
 #   - when testing all wheels
 #   - when including debug_info symbols
-#   - when using PPC arch (fails on qoriq)
+#   - when using OLD_PPC_ARCHS arch
 ifneq ($(strip $(WHEELS_TEST_ALL)),1)
 ifneq ($(strip $(GCC_DEBUG_INFO)),1)
-ifneq ($(findstring $(ARCH),$(PPC_ARCHS)),$(ARCH))
+ifneq ($(findstring $(ARCH),$(OLD_PPC_ARCHS)),$(ARCH))
 ENV += PYTHON_OPTIMIZE=1
 endif
 endif