master-nonlatent move to common class

Author: cvicentiu

master-nonlatent/master.cfg

@@ -1,66 +1,53 @@
 # -*- python -*-
 # ex: set filetype=python:
 
-from buildbot.plugins import *
-from buildbot.process.properties import Property, Properties
-from buildbot.steps.shell import ShellCommand, Compile, Test, SetPropertyFromCommand
-from buildbot.steps.mtrlogobserver import MTR, MtrLogObserver
-from buildbot.steps.source.github import GitHub
-from buildbot.process.remotecommand import RemoteCommand
-from datetime import timedelta
-from twisted.internet import defer
-
-import docker
 import os
-import sys
-
-sys.setrecursionlimit(10000)
+from datetime import timedelta
 
-sys.path.insert(0, "/srv/buildbot/master")
-from common_factories import *
-from locks import *
-from schedulers_definition import SCHEDULERS
-from utils import *
+from buildbot.plugins import steps, util, worker
+from buildbot.process.properties import Property
+from common_factories import addWinTests, getQuickBuildFactory
+from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES
+from master_common import base_master_config
+from utils import (
+    canStartBuild,
+    createVar,
+    getHTMLLogString,
+    getSourceTarball,
+    hasFailed,
+    ls2list,
+    moveMTRLogs,
+    mtrJobsMultiplier,
+    nextBuild,
+    prioritizeBuilders,
+    savePackage,
+)
 
 ####### VARIABLES
 vsWarningPattern = "^.*: warning C.*$"
 
-# This is the dictionary that the buildmaster pays attention to. We also use
-# a shorter alias to save typing.
-c = BuildmasterConfig = {}
-
 # Load the slave, database passwords and 3rd-party tokens from an external private file, so
 # that the rest of the configuration can be public.
 config = {"private": {}}
 exec(open("master-private.cfg").read(), config, {})
 
-####### PROJECT IDENTITY
-
-# the 'title' string will appear at the top of this buildbot installation's
-# home pages (linked to the 'titleURL').
-c["title"] = os.getenv("TITLE", default="MariaDB CI")
-c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server")
-
-# the 'buildbotURL' string should point to the location where the buildbot's
-# internal web server is visible. This typically uses the port number set in
-# the 'www' entry below, but with an externally-visible host name which the
-# buildbot cannot figure out without some help.
-c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/")
-
-# 'protocols' contains information about protocols which master will use for
-# communicating with workers. You must define at least 'port' option that workers
-# could connect to your master with this protocol.
-# 'port' must match the value configured into the workers (with their
-# --master option)
-port = int(os.getenv("PORT", default="9989"))
-c["protocols"] = {"pb": {"port": port}}
-
-####### DB URL
+# This is the dictionary that the buildmaster pays attention to. We also use
+# a shorter alias to save typing.
+c = BuildmasterConfig = base_master_config(
+    title=os.getenv("TITLE", default="MariaDB CI"),
+    title_url=os.getenv("TITLE_URL", default="https://github.com/MariaDB/server"),
+    buildbot_url=os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/"),
+    github_access_token=config["private"]["gh_mdbci"]["access_token"],
+    # TODO(cvicentiu) undo this hardcoding
+    secrets_provider_file=os.getenv(
+        "MASTER_CREDENTIALS_DIR",
+        default="/srv/buildbot/master/master-credential-provider",
+    ),
+    master_port=int(os.getenv("PORT", default="9989")),
+    db_url=config["private"]["db_url"],
+    mq_router_url=os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"),
+)
 
-c["db"] = {
-    # This specifies what database buildbot uses to store its state.
-    "db_url": config["private"]["db_url"]
-}
 
 mtrDbPool = util.EqConnectionPool(
     "MySQLdb",
@@ -70,29 +57,10 @@ mtrDbPool = util.EqConnectionPool(
     config["private"]["db_mtr_db"],
 )
 
-####### Disable net usage reports from being sent to buildbot.net
-c["buildbotNetUsageData"] = None
 
-####### Services
-c["services"] = []
-context = util.Interpolate("buildbot/%(prop:buildername)s")
-gs = reporters.GitHubStatusPush(
-    token=config["private"]["gh_mdbci"]["access_token"],
-    context=context,
-    startDescription="Build started.",
-    endDescription="Build done.",
-    verbose=True,
-    builders=GITHUB_STATUS_BUILDERS,
-)
-c["services"].append(gs)
-c['secretsProviders'] = [secrets.SecretInAFile(dirname=os.getenv("MASTER_CREDENTIALS_DIR", default="/srv/buildbot/master/master-credential-provider"))]
 ####### Builder priority
 c["prioritizeBuilders"] = prioritizeBuilders
 
-####### SCHEDULERS
-
-# Configure the Schedulers, which decide how to react to incoming changes.
-c["schedulers"] = SCHEDULERS
 
 ####### WORKERS
 
@@ -159,9 +127,6 @@ f_windows_env = {
 }
 f_windows_env.update(MTR_ENV)
 
-# f_quick_build = getQuickBuildFactory(mtrDbPool)
-f_rpm_autobake = getRpmAutobakeFactory(mtrDbPool)
-
 ## f_windows
 f_windows = util.BuildFactory()
 f_windows.addStep(
@@ -268,23 +233,22 @@ f_windows.addStep(
 )
 
 windows_tests = {
-    "nm": {
-        "create_scripts": True
-    },
+    "nm": {"create_scripts": True},
     "connect": {
         "suites": ["connect"],
     },
 }
 
 for typ in windows_tests:
-    addWinTests(f_windows,
-            mtr_test_type=typ,
-            mtr_env=f_windows_env,
-            mtr_additional_args=util.Property("mtr_additional_args", default=""),
-            mtr_step_db_pool=mtrDbPool,
-            mtr_suites=windows_tests[typ].get('suites',["default"]),
-            create_scripts=windows_tests[typ].get('create_scripts',False)
-            )
+    addWinTests(
+        f_windows,
+        mtr_test_type=typ,
+        mtr_env=f_windows_env,
+        mtr_additional_args=util.Property("mtr_additional_args", default=""),
+        mtr_step_db_pool=mtrDbPool,
+        mtr_suites=windows_tests[typ].get("suites", ["default"]),
+        create_scripts=windows_tests[typ].get("create_scripts", False),
+    )
 
 
 f_windows.addStep(
@@ -300,13 +264,13 @@ f_windows.addStep(
         ),
         url=util.Interpolate(
             f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}'
-            '/'
+            "/"
             "%(prop:tarbuildnum)s"
-            '/'
-            'logs'
-            '/'
-            '%(prop:buildername)s'
-            '/'
+            "/"
+            "logs"
+            "/"
+            "%(prop:buildername)s"
+            "/"
         ),
     )
 )
@@ -331,8 +295,16 @@ f_windows.addStep(
 ## f_windows_msi
 
 f_windows_msi_env = {
-    "TMP": util.Interpolate("{0}\\%(prop:buildername)s\\build\\tmpdir".format('D:\\DEV\\Buildbot' if os.getenv('ENVIRON') == 'DEV' else 'D:\\Buildbot')),
-    "TEMP": util.Interpolate("{0}\\%(prop:buildername)s\\build\\tmpdir".format('D:\\DEV\\Buildbot' if os.getenv('ENVIRON') == 'DEV' else 'D:\\Buildbot')),
+    "TMP": util.Interpolate(
+        "{0}\\%(prop:buildername)s\\build\\tmpdir".format(
+            "D:\\DEV\\Buildbot" if os.getenv("ENVIRON") == "DEV" else "D:\\Buildbot"
+        )
+    ),
+    "TEMP": util.Interpolate(
+        "{0}\\%(prop:buildername)s\\build\\tmpdir".format(
+            "D:\\DEV\\Buildbot" if os.getenv("ENVIRON") == "DEV" else "D:\\Buildbot"
+        )
+    ),
 }
 f_windows_msi_env.update(MTR_ENV)
 
@@ -457,23 +429,22 @@ f_windows_msi.addStep(
     )
 )
 windows_msi_tests = {
-    "nm": {
-        "create_scripts": True
-    },
+    "nm": {"create_scripts": True},
     "connect": {
         "suites": ["connect"],
     },
 }
 
 for typ in windows_msi_tests:
-    addWinTests(f_windows_msi,
-            mtr_test_type=typ,
-            mtr_env=f_windows_msi_env,
-            mtr_additional_args=util.Property("mtr_additional_args", default=""),
-            mtr_step_db_pool=mtrDbPool,
-            mtr_suites=windows_msi_tests[typ].get('suites',["default"]),
-            create_scripts=windows_msi_tests[typ].get('create_scripts',False)
-            )
+    addWinTests(
+        f_windows_msi,
+        mtr_test_type=typ,
+        mtr_env=f_windows_msi_env,
+        mtr_additional_args=util.Property("mtr_additional_args", default=""),
+        mtr_step_db_pool=mtrDbPool,
+        mtr_suites=windows_msi_tests[typ].get("suites", ["default"]),
+        create_scripts=windows_msi_tests[typ].get("create_scripts", False),
+    )
 
 f_windows_msi.addStep(
     steps.DirectoryUpload(
@@ -488,13 +459,13 @@ f_windows_msi.addStep(
         ),
         url=util.Interpolate(
             f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}'
-            '/'
+            "/"
             "%(prop:tarbuildnum)s"
-            '/'
-            'logs'
-            '/'
-            '%(prop:buildername)s'
-            '/'
+            "/"
+            "logs"
+            "/"
+            "%(prop:buildername)s"
+            "/"
         ),
     )
 )
@@ -528,12 +499,11 @@ f_windows_msi.addStep(
         mode=0o755,
         url=util.Interpolate(
             f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}'
-            '/'
-            '%(prop:tarbuildnum)s'
-            '/'
-            '%(prop:buildername)s'
-            '/'
-
+            "/"
+            "%(prop:tarbuildnum)s"
+            "/"
+            "%(prop:buildername)s"
+            "/"
         ),
         doStepIf=lambda step: savePackage(step, SAVED_PACKAGE_BRANCHES),
     )
@@ -551,11 +521,11 @@ f_windows_msi.addStep(
         mode=0o755,
         url=util.Interpolate(
             f'{os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org")}'
-            '/'
-            '%(prop:tarbuildnum)s'
-            '/'
-            '%(prop:buildername)s'
-            '/'
+            "/"
+            "%(prop:tarbuildnum)s"
+            "/"
+            "%(prop:buildername)s"
+            "/"
         ),
         doStepIf=lambda step: savePackage(step, SAVED_PACKAGE_BRANCHES),
     )
@@ -739,22 +709,26 @@ f_dockerlibrary.addStep(
 f_dockerlibrary.addStep(
     steps.SetPropertyFromCommand(
         name="Extract the last tag created",
-	command=['bash','-c','if [ -f last_tag ]; then cat last_tag; else echo ""; fi'],
+        command=[
+            "bash",
+            "-c",
+            'if [ -f last_tag ]; then cat last_tag; else echo ""; fi',
+        ],
         property="lasttag",
     )
 )
 f_dockerlibrary.addStep(
     steps.SetPropertyFromCommand(
         name="Determine sha for the last tag",
         command=[
-           "bash",
+            "bash",
             "-xc",
             util.Interpolate(
                 'curl -s "https://quay.io/api/v1/repository/mariadb-foundation/mariadb-devel/tag/?filter_tag_name=eq:%(prop:lasttag)s&onlyActiveTags=true" | jq -r .tags[0].manifest_digest'
             ),
         ],
         property="lastsha",
-        doStepIf=lambda step: (str(step.getProperty("lasttag")) != "")
+        doStepIf=lambda step: (str(step.getProperty("lasttag")) != ""),
     )
 )
 f_dockerlibrary.addStep(
@@ -765,7 +739,9 @@ f_dockerlibrary.addStep(
             "-xc",
             "gh auth login --with-token < ~/gh_auth",
         ],
-        doStepIf=lambda step: (str(step.getProperty("lastsha")) != "null" and step.hasProperty("lastsha")),
+        doStepIf=lambda step: (
+            str(step.getProperty("lastsha")) != "null" and step.hasProperty("lastsha")
+        ),
     )
 )
 
@@ -1132,24 +1108,12 @@ c["builders"].append(
             "cxx_compiler": "clang++",
             "mtr_env": {
                 "WSREP_PROVIDER": "/usr/local/lib/libgalera_smm.so",
-                }
             },
+        },
         nextBuild=nextBuild,
         factory=f_freebsd,
     )
 )
 
 # Add a Janitor configurator that removes old logs
 c["configurators"] = [util.JanitorConfigurator(logHorizon=timedelta(weeks=6), hour=23)]
-
-c["logEncoding"] = "utf-8"
-
-c["multiMaster"] = True
-
-c["mq"] = {  # Need to enable multimaster aware mq. Wamp is the only option for now.
-    "type": "wamp",
-    "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"),
-    "realm": "realm1",
-    # valid are: none, critical, error, warn, info, debug, trace
-    "wamp_debug_level": "info",
-}