From 51bec29f18850b0c2c9496b9ea3cf3fd8798a5f6 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Aug 2025 13:27:23 +1000 Subject: [PATCH 1/6] MDBF-1076: Create MSAN Debug sequence builder (for aarch64) --- .../builders/sequences/sanitizers.py | 121 ++++++++++++++++++ .../steps/generators/cmake/options.py | 5 + constants.py | 1 + master-migration/master.cfg | 42 +++++- 4 files changed, 168 insertions(+), 1 deletion(-) diff --git a/configuration/builders/sequences/sanitizers.py b/configuration/builders/sequences/sanitizers.py index a2d0ac520..52e18c972 100644 --- a/configuration/builders/sequences/sanitizers.py +++ b/configuration/builders/sequences/sanitizers.py @@ -180,3 +180,124 @@ def asan_ubsan( sequence.add_step(step) return sequence + + +def msan( + config: DockerConfig, + jobs: int, + isDebugBuildType: bool, +): + sequence = BuildSequence() + + sequence.add_step(ShellStep(command=PrintEnvironmentDetails())) + + sequence.add_step( + InContainer( + docker_environment=config, + container_commit=False, + step=ShellStep( + command=FetchTarball(workdir=PurePath("src")), + options=StepOptions(descriptionDone="Fetch tarball"), + ), + ) + ) + + flags = [ + CMakeOption(WITH.MSAN, True), + CMakeOption( + CMAKE.EXE_LINKER_FLAGS, "-L${MSAN_LIBDIR} -Wl,-rpath=${MSAN_LIBDIR}" + ), + CMakeOption( + CMAKE.MODULE_LINKER_FLAGS, "-L${MSAN_LIBDIR} -Wl,-rpath=${MSAN_LIBDIR}" + ), + CMakeOption(WITH.UNIT_TESTS, False), + CMakeOption(WITH.ZLIB, "bundled"), + CMakeOption(WITH.SYSTEMD, "no"), + CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False), + CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, False), + CMakeOption(PLUGIN.ROCKSDB_STORAGE_ENGINE, False), + CMakeOption(PLUGIN.OQGRAPH_STORAGE_ENGINE, False), + ] + if isDebugBuildType: + flags.append(CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG)) + flags.append(CMakeOption(WITH.DBUG_TRACE, False)) + + sequence.add_step( + InContainer( + docker_environment=config, + step=ShellStep( + command=ConfigureMariaDBCMake( + name="configure", + workdir=PurePath("bld"), + cmake_generator=CMakeGenerator( + use_ccache=True, + flags=flags, + source_path="../src", + compiler=ClangCompiler(), + ), + ), + options=StepOptions(descriptionDone="Configure"), + ), + ) + ) + + sequence.add_step( + InContainer( + docker_environment=config, + step=ShellStep( + command=CompileCMakeCommand( + builddir="bld", + jobs=jobs, + verbose=True, + ), + options=StepOptions(descriptionDone="compile"), + ), + ) + ) + + env_vars = [ + ( + "MSAN_OPTIONS", + "abort_on_error=1:poison_in_dtor=0", + ), + ("MTR_FEEDBACK_PLUGIN", "1"), + ] + + ## ADD MTR TESTS + for step in ( + get_mtr_normal_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + path_to_test_runner=PurePath("bld", "mysql-test"), + additional_mtr_options=[MTROption(MTR.BIG_TEST, True)], + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ) + + get_mtr_s3_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + additional_mtr_options=[MTROption(MTR.BIG_TEST, True)], + path_to_test_runner=PurePath("bld", "mysql-test"), + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ) + + [ + save_mtr_logs( + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ), + mtr_junit_reporter( + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ), + ] + ): + sequence.add_step(step) + + return sequence diff --git a/configuration/steps/generators/cmake/options.py b/configuration/steps/generators/cmake/options.py index 94aca5085..b7e5bfdef 100644 --- a/configuration/steps/generators/cmake/options.py +++ b/configuration/steps/generators/cmake/options.py @@ -24,8 +24,10 @@ class CMAKE(StrEnum): C_FLAGS = "C_FLAGS" C_COMPILER_LAUNCHER = "C_COMPILER_LAUNCHER" CXX_COMPILER_LAUNCHER = "CXX_COMPILER_LAUNCHER" + EXE_LINKER_FLAGS = "EXE_LINKER_FLAGS" INSTALL_PREFIX = "INSTALL_PREFIX" LIBRARY_PATH = "LIBRARY_PATH" + MODULE_LINKER_FLAGS = "MODULE_LINKER_FLAGS" def __str__(self): return f"CMAKE_{self.value}" @@ -71,13 +73,16 @@ class WITH(StrEnum): EMBEDDED_SERVER = "EMBEDDED_SERVER" EXTRA_CHARSETS = "EXTRA_CHARSETS" JEMALLOC = "JEMALLOC" + MSAN = "MSAN" NONE = "NONE" SAFEMALLOC = "SAFEMALLOC" SSL = "SSL" + SYSTEMD = "SYSTEMD" UBSAN = "UBSAN" UNIT_TESTS = "UNIT_TESTS" VALGRIND = "VALGRIND" WSREP = "WSREP" + ZLIB = "ZLIB" def __str__(self): return f"WITH_{self.value}" diff --git a/constants.py b/constants.py index 22e121d24..86aad1a3c 100644 --- a/constants.py +++ b/constants.py @@ -169,6 +169,7 @@ "aarch64-debian-12", "aarch64-fedora-41", "aarch64-fedora-42", + "aarch64-msan-clang-20-debug", "aarch64-rhel-10", "aarch64-ubuntu-2404", "amd64-centos-stream10", diff --git a/master-migration/master.cfg b/master-migration/master.cfg index 11f884ec2..b2fec33d9 100644 --- a/master-migration/master.cfg +++ b/master-migration/master.cfg @@ -15,7 +15,7 @@ from configuration.builders.sequences.compile_only import ( ) from configuration.builders.sequences.debug import openssl_fips from configuration.builders.sequences.release import deb_autobake, rpm_autobake -from configuration.builders.sequences.sanitizers import asan_ubsan +from configuration.builders.sequences.sanitizers import asan_ubsan, msan from configuration.reporters import github_summary from configuration.workers import worker from master_common import base_master_config, IS_CHECKCONFIG @@ -276,6 +276,46 @@ for builder in ["amd64-ubasan-clang-20", "amd64-ubasan-clang-20-debug"]: c["builders"].append(ubasan_builder(name=builder, debug=builder.endswith("debug"))) +def msan_builder(name: str, debug: bool) -> GenericBuilder: + tags_msan = ("Debian", "clang", "msan", "big") + if debug: + tag_msan = tags_msan + ("debug",) + + return GenericBuilder( + name=name, + sequences=[ + msan( + jobs=12, + config=DockerConfig( + repository=os.environ["CONTAINER_REGISTRY_URL"], + image_tag="debian12-msan-clang-20", + workdir=PurePath("/home/buildbot"), + bind_mounts=[ + ("/srv/buildbot/ccache", "/mnt/ccache"), + (f'{os.environ["MASTER_PACKAGES_DIR"]}/', "/packages"), + ], + shm_size="24g", + env_vars=[ + ("CCACHE_DIR", "/mnt/ccache"), + ("ARTIFACTS_URL", os.environ["ARTIFACTS_URL"]), + ], + memlock_limit=memlock_limit, + ), + isDebugBuildType=debug, + ) + ], + ).get_config( + workers=WORKER_POOL.get_workers_for_arch(arch="aarch64"), + next_build=nextBuild, + can_start_build=canStartBuild, + tags=list(tags_msan), + jobs=12, + ) + + +builder = "aarch64-msan-clang-20-debug" +c["builders"].append(msan_builder(name=builder, debug=builder.endswith("debug"))) + ## ------------------------------------------------------------------- ## ## REPORTERS ## From 8f4afe2f0d169262868ee7f779faf434cd16f9ea Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Aug 2025 16:43:31 +1000 Subject: [PATCH 2/6] msan debug- amd64 for now qemu insufficient to run aarch64 compiles --- constants.py | 2 +- master-migration/master.cfg | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/constants.py b/constants.py index 86aad1a3c..5d059f3e3 100644 --- a/constants.py +++ b/constants.py @@ -169,7 +169,6 @@ "aarch64-debian-12", "aarch64-fedora-41", "aarch64-fedora-42", - "aarch64-msan-clang-20-debug", "aarch64-rhel-10", "aarch64-ubuntu-2404", "amd64-centos-stream10", @@ -178,6 +177,7 @@ "amd64-debian-12-debug-embedded", "amd64-fedora-41", "amd64-fedora-42", + "amd64-msan-clang-20-debug", "amd64-opensuse-1506", "amd64-rhel-10", "amd64-sles-1506", diff --git a/master-migration/master.cfg b/master-migration/master.cfg index b2fec33d9..350b6aeef 100644 --- a/master-migration/master.cfg +++ b/master-migration/master.cfg @@ -305,7 +305,7 @@ def msan_builder(name: str, debug: bool) -> GenericBuilder: ) ], ).get_config( - workers=WORKER_POOL.get_workers_for_arch(arch="aarch64"), + workers=WORKER_POOL.get_workers_for_arch(arch="amd64"), next_build=nextBuild, can_start_build=canStartBuild, tags=list(tags_msan), @@ -313,7 +313,7 @@ def msan_builder(name: str, debug: bool) -> GenericBuilder: ) -builder = "aarch64-msan-clang-20-debug" +builder = "amd64-msan-clang-20-debug" c["builders"].append(msan_builder(name=builder, debug=builder.endswith("debug"))) From 1d78bae18262fa4e3da530c35e4320017ce25b48 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 12 Sep 2025 17:25:38 +1000 Subject: [PATCH 3/6] MDBF-1076: MSAN Debug builder master.cfg --- master-migration/master.cfg | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/master-migration/master.cfg b/master-migration/master.cfg index 350b6aeef..787ceada1 100644 --- a/master-migration/master.cfg +++ b/master-migration/master.cfg @@ -278,14 +278,16 @@ for builder in ["amd64-ubasan-clang-20", "amd64-ubasan-clang-20-debug"]: def msan_builder(name: str, debug: bool) -> GenericBuilder: tags_msan = ("Debian", "clang", "msan", "big") + jobs = 12 if debug: tag_msan = tags_msan + ("debug",) + jobs = 6 return GenericBuilder( name=name, sequences=[ msan( - jobs=12, + jobs=jobs, config=DockerConfig( repository=os.environ["CONTAINER_REGISTRY_URL"], image_tag="debian12-msan-clang-20", @@ -309,14 +311,13 @@ def msan_builder(name: str, debug: bool) -> GenericBuilder: next_build=nextBuild, can_start_build=canStartBuild, tags=list(tags_msan), - jobs=12, + jobs=jobs, ) builder = "amd64-msan-clang-20-debug" c["builders"].append(msan_builder(name=builder, debug=builder.endswith("debug"))) - ## ------------------------------------------------------------------- ## ## REPORTERS ## ## ------------------------------------------------------------------- ## From 02c43ccb725f99d99337953c37b588633081522a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 5 Sep 2025 13:20:53 +1000 Subject: [PATCH 4/6] MDEV-36723: spider msan testing (debug) Until MDEV-36723 is merged up Spider is Debug only. --- .../builders/sequences/sanitizers.py | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/configuration/builders/sequences/sanitizers.py b/configuration/builders/sequences/sanitizers.py index 52e18c972..6be395bdf 100644 --- a/configuration/builders/sequences/sanitizers.py +++ b/configuration/builders/sequences/sanitizers.py @@ -214,7 +214,7 @@ def msan( CMakeOption(WITH.ZLIB, "bundled"), CMakeOption(WITH.SYSTEMD, "no"), CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False), - CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, False), + CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType), CMakeOption(PLUGIN.ROCKSDB_STORAGE_ENGINE, False), CMakeOption(PLUGIN.OQGRAPH_STORAGE_ENGINE, False), ] @@ -264,40 +264,46 @@ def msan( ] ## ADD MTR TESTS - for step in ( - get_mtr_normal_steps( + steps = get_mtr_normal_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + path_to_test_runner=PurePath("bld", "mysql-test"), + additional_mtr_options=[MTROption(MTR.BIG_TEST, True)], + step_wrapping_fn=lambda step: InContainer(docker_environment=config, step=step), + ) + get_mtr_s3_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + additional_mtr_options=[MTROption(MTR.BIG_TEST, True)], + path_to_test_runner=PurePath("bld", "mysql-test"), + step_wrapping_fn=lambda step: InContainer(docker_environment=config, step=step), + ) + if isDebugBuildType: + steps += get_mtr_spider_steps( jobs=jobs, env_vars=env_vars, halt_on_failure=False, - path_to_test_runner=PurePath("bld", "mysql-test"), additional_mtr_options=[MTROption(MTR.BIG_TEST, True)], + path_to_test_runner=PurePath("bld", "mysql-test"), step_wrapping_fn=lambda step: InContainer( docker_environment=config, step=step ), ) - + get_mtr_s3_steps( - jobs=jobs, - env_vars=env_vars, - halt_on_failure=False, - additional_mtr_options=[MTROption(MTR.BIG_TEST, True)], - path_to_test_runner=PurePath("bld", "mysql-test"), + steps += [ + save_mtr_logs( step_wrapping_fn=lambda step: InContainer( docker_environment=config, step=step ), - ) - + [ - save_mtr_logs( - step_wrapping_fn=lambda step: InContainer( - docker_environment=config, step=step - ), - ), - mtr_junit_reporter( - step_wrapping_fn=lambda step: InContainer( - docker_environment=config, step=step - ), + ), + mtr_junit_reporter( + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step ), - ] - ): + ), + ] + + for step in steps: sequence.add_step(step) return sequence From b90fa1f46daab9f461ed41fd4c14df7cc6c20998 Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Wed, 17 Sep 2025 11:48:18 +0300 Subject: [PATCH 5/6] MDBF-1076 - Remove clang-20-debug builder from DockerLatent master Since it was defined as a sequence in master-migration, its definition is no longer needed in `master-docker-nonstandard-2` --- master-docker-nonstandard-2/master.cfg | 32 ++++++++------------------ 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index ce1a52ee1..6c22dd3e7 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -103,15 +103,15 @@ for full_test_worker in ["debian-12-32-bit", "debian-12-32-bit-debug"]: shm_size="30G", ) -for msan_worker in ["msan-clang-20", "msan-clang-20-debug"]: - for worker_id in [1,4,5]: # Hetzner designated workers for protected builds - addWorker( - "hz-bbw", - worker_id, - msan_worker, - os.environ["CONTAINER_REGISTRY_URL"] + "debian12-msan-clang-20", - jobs=20, - save_packages=False, +msan_worker = "msan-clang-20" +for worker_id in [1,4,5]: # Hetzner designated workers for protected builds + addWorker( + "hz-bbw", + worker_id, + msan_worker, + os.environ["CONTAINER_REGISTRY_URL"] + "debian12-msan-clang-20", + jobs=20, + save_packages=False, ) addWorker( @@ -696,20 +696,6 @@ c["builders"].append( ) ) -c["builders"].append( - util.BuilderConfig( - name="amd64-msan-clang-20-debug", - workernames=workers["x64-bbw-docker-msan-clang-20-debug"], - tags=["Debian", "clang", "msan", "debug"], - collapseRequests=True, - nextBuild=nextBuild, - canStartBuild=canStartBuild, - properties={"c_compiler": "clang", "cxx_compiler": "clang++", "build_type": "Debug"}, - locks=getLocks, - factory=f_msan_build, - ) -) - c["builders"].append( util.BuilderConfig( name="amd64-ubuntu-2204-jepsen-mariadb", From a3f25875b7aa8aacff187cf27f5acbd87e2f0d5e Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Wed, 17 Sep 2025 15:45:01 +0300 Subject: [PATCH 6/6] MDBF-1076 - Increase jobs for clang-20 debug builder Tests showed 2 hours for parallel = 6 during MTR: https://buildbot.dev.mariadb.org/#/builders/535/builds/7 Quite unacceptable. Increasing parallel for the builder will decrease a host capacity to build a whole push if max_worker_jobs < requested_jobs Current CPU/MEM utilisation over 1M for hz-bbw8 and 9 is: - CPU Max around 70 % - MEM Max around 30 % This means we can allocate more jobs than CPU's available. Increased to 110 so that a host will continue to build a whole push. --- master-migration/master.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/master-migration/master.cfg b/master-migration/master.cfg index 787ceada1..ca3760b76 100644 --- a/master-migration/master.cfg +++ b/master-migration/master.cfg @@ -33,10 +33,10 @@ c = BuildmasterConfig = base_master_config(config) WORKER_POOL = worker.WorkerPool() WORKER_POOL.add( - arch="amd64", worker=worker.NonLatent(name="hz-bbw8", config=config, total_jobs=96) + arch="amd64", worker=worker.NonLatent(name="hz-bbw8", config=config, total_jobs=110) ) WORKER_POOL.add( - arch="amd64", worker=worker.NonLatent(name="hz-bbw9", config=config, total_jobs=96) + arch="amd64", worker=worker.NonLatent(name="hz-bbw9", config=config, total_jobs=110) ) @@ -281,7 +281,7 @@ def msan_builder(name: str, debug: bool) -> GenericBuilder: jobs = 12 if debug: tag_msan = tags_msan + ("debug",) - jobs = 6 + jobs = 20 return GenericBuilder( name=name,