Fix modifying cross versions in mill (#114)

WojciechMazur · web-flow · commit c372eee520b8 · 2022-09-28T00:47:16.000+02:00
* Add extra cross versions instead of replacing it

* Jenkins: add plugins missing on prod

* Fix typos in crossVersion mapping

* Try fix CI

* Filter out tapir-loom - no JDK 19 support yet

* Fix name clashes

* Try fix CI

* Don't build accessory images in CI

* Try fix CI

* Update configs

* Don't publish artifact in the CI

* Revert skiping image builing

* Fix cli

* Bump version to 0.1.1
diff --git a/README.md b/README.md
@@ -93,19 +93,19 @@ eval $(minikube -p minikube docker-env)
 Most likely you'll need to build the base image only once (it doesn't get modified too often but building it takes quite a lot of time), e.g.:
 
 ```shell
-scripts/build-builder-base.sh v0.1.0
+scripts/build-builder-base.sh v0.1.1
 ```
 
 Build all the remaining images
 
 ```shell
-scripts/build-quick.sh v0.1.0
+scripts/build-quick.sh v0.1.1
 ```
 
 or (re)build each image separately e.g.
 
 ```shell
-scripts/build-mvn-repo.sh v0.1.0
+scripts/build-mvn-repo.sh v0.1.1
 ```
 
 ### Deploying and debugging in k8s
diff --git a/cli/scb-cli.scala b/cli/scb-cli.scala
@@ -25,7 +25,7 @@ given ExecutionContext = ExecutionContext.Implicits.global
 
 class FailedProjectException(msg: String) extends RuntimeException(msg) with NoStackTrace
 
-val communityBuildVersion = sys.props.getOrElse("communitybuild.version", "v0.1.0")
+val communityBuildVersion = sys.props.getOrElse("communitybuild.version", "v0.1.1")
 private val CBRepoName = "VirtusLab/community-build3"
 val projectBuilderUrl = s"https://raw.githubusercontent.com/$CBRepoName/master/project-builder"
 lazy val communityBuildDir = sys.props
@@ -41,7 +41,8 @@ case class Config(
     reproducer: JenkinsReproducerConfig = JenkinsReproducerConfig(),
     customRun: CustomBuildConfig = CustomBuildConfig(null, null),
     minikube: Config.MinikubeConfig = Config.MinikubeConfig(),
-    redirectLogs: Boolean = true
+    redirectLogs: Boolean = true,
+    publishArtifacts: Boolean = true
 ):
   def withMinikube(fn: Config.MinikubeConfig => Config.MinikubeConfig) =
     copy(minikube = fn(minikube))
@@ -106,6 +107,10 @@ object Config:
         .action { (_, c) => c.copy(redirectLogs = false) }
         .text("Do not redirect runners logs to file")
         .hidden(),
+      opt[Unit]("noPublishArtifacts")
+        .action { (_, c) => c.copy(publishArtifacts = false) }
+        .text("Don't publish artifacts of the build")
+        .hidden(),
       // Commands
       cmd("reproduce")
         .action { (_, c) => c.copy(command = Config.Command.ReproduceJenkinsBuild) }
@@ -819,7 +824,7 @@ object MinikubeReproducer:
                   params.repositoryUrl,
                   params.repositoryRevision.getOrElse(""),
                   buildInfo.scalaVersion,
-                  params.version.getOrElse(""),
+                  if !config.publishArtifacts then "" else params.version.getOrElse(""),
                   project.effectiveTargets.mkString(" "),
                   params.mavenRepositoryUrl,
                   params.enforcedSbtVersion.getOrElse(config.command match {
@@ -1103,7 +1108,7 @@ class LocalReproducer(using config: Config, build: BuildInfo):
       if !os.exists(sbtProjectDir) then os.makeDir(sbtProjectDir)
       project.params.enforcedSbtVersion match {
         case Some(version) =>
-           os.write.over(sbtBuildProperties, s"sbt.version=$version")
+          os.write.over(sbtBuildProperties, s"sbt.version=$version")
         case _ =>
           if belowMinimalSbtVersion then
             println(
diff --git a/env/prod/config/filtered-projects.txt b/env/prod/config/filtered-projects.txt
@@ -28,6 +28,7 @@ wireapp:wire-signals:.*
 # Missing module moduleMappings
 busymachines:pureharm-db-core:.*
 eed3si9n:sbt-coursier:.*
+neandertech:jsonrpclib:.*
 
 # Not an official release version
 com-lihaoyi:ammonite:.* 
@@ -66,6 +67,22 @@ sciss:swingplus:.*
 joan38:kubernetes-client:.*
 carlosedp:riscvassembler:.*
 
+## Missing artifacts in public repo
+# sbt-beangle-parent:0.6.0
+beangle:boot:.*
+beangle:cache:.*
+beangle:cdi:.*
+beangle:commons:.*
+beangle:data:.*
+beangle:security:.*
+beangle:serializer:.*
+beangle:template:.*
+beangle:web:.*
+beangle:webmvc:.*
+
+## Misconfigured + needs ruby on path
+makenowjust-labs:lite:.*
+
 ## Publishing artifact leads to failurs in  downstream projects
 ### Publishes only pom, no artifacts
 msgpack4z:msgpack4z-native:.*
@@ -134,15 +151,18 @@ zio:zio-aws:.*
 
 # Depends on not published lib with CrossVersion.full
 ## wartremover
+atedeg:mdm:.*
 cognitedata:cognite-sdk-scala:.*
 kevin-lee:just-semver:.* 
 kevin-lee:just-sysprocess:.*
+medeia:medeia:.*
 xuwei-k:wartremover-scalikejdbc:.*
 
 ## better2string
 kubukoz:drops:.*
 kubukoz:spotify-next:.*
 kubukoz:sup:.* 
+polyvariant:scodec-java-classfile:.*
 
 # Depends on no longer available pre-release
 oleg-py:shironeko:0.2.0-M.*
@@ -167,3 +187,6 @@ japgolly:nyaya.*
 ## errors4s_errors4s-core
 errors4s:errors4s-core-scalacheck:.*
 
+# Unsupported minimal JDK version
+## JDK 19
+softwaremill:tapir-loom:.*
diff --git a/jenkins/seeds/buildCommunityProject.groovy b/jenkins/seeds/buildCommunityProject.groovy
@@ -57,7 +57,7 @@ pipeline {
                   shareProcessNamespace: true
                   containers:
                   - name: project-builder
-                    image: virtuslab/scala-community-build-project-builder:jdk${params.javaVersion?: 11}-v0.1.0
+                    image: virtuslab/scala-community-build-project-builder:jdk${params.javaVersion?: 11}-v0.1.1
                     imagePullPolicy: IfNotPresent
                     volumeMounts:
                     - name: mvn-repo-cert
@@ -164,7 +164,7 @@ pipeline {
               containers: [
                 containerTemplate(
                   name: 'reporter',
-                  image: 'virtuslab/scala-community-build-project-builder:jdk11-v0.1.0',
+                  image: 'virtuslab/scala-community-build-project-builder:jdk11-v0.1.1',
                   command: 'sleep',
                   args: '15m',
                   resourceRequestMemory: '250M',
diff --git a/jenkins/seeds/buildCompiler.groovy b/jenkins/seeds/buildCompiler.groovy
@@ -34,7 +34,7 @@ pipeline {
                               name: mvn-repo-cert
                           containers:
                           - name: compiler-builder
-                            image: virtuslab/scala-community-build-compiler-builder:v0.1.0
+                            image: virtuslab/scala-community-build-compiler-builder:v0.1.1
                             imagePullPolicy: IfNotPresent
                             volumeMounts:
                             - name: mvn-repo-cert
diff --git a/jenkins/seeds/computeBuildPlan.groovy b/jenkins/seeds/computeBuildPlan.groovy
@@ -25,7 +25,7 @@ pipeline {
                         spec:
                           containers:
                           - name: coordinator
-                            image: virtuslab/scala-community-build-coordinator:v0.1.0
+                            image: virtuslab/scala-community-build-coordinator:v0.1.1
                             imagePullPolicy: IfNotPresent
                             command:
                             - cat
diff --git a/jenkins/seeds/runBuild.groovy b/jenkins/seeds/runBuild.groovy
@@ -188,7 +188,7 @@ pipeline {
         podTemplate(
           containers: [
             // Any container having a curl or pre-installed scala-cli would work 
-            containerTemplate(name: 'reporter', image: 'virtuslab/scala-community-build-project-builder:jdk17-v0.1.0', command: 'sleep', args: '15m'),
+            containerTemplate(name: 'reporter', image: 'virtuslab/scala-community-build-project-builder:jdk17-v0.1.1', command: 'sleep', args: '15m'),
           ],
           envVars: [
             envVar(key: 'ELASTIC_USERNAME', value: params.elasticSearchUserName),
diff --git a/k8s/jenkins.yaml b/k8s/jenkins.yaml
@@ -146,6 +146,10 @@ jenkins:
       version: "1.3"
     - name: timestamper
       version: "1.20"
+    - name: workflow-cps
+      version: "2729.vea_17b_79ed57a_"
+    - name: pipeline-model-extensions
+      version: "2.2114.v2654ca_721309"
 
   # roles define a list of extra RBAC roles for the Jenkins Master pod service account
   roles: []
diff --git a/k8s/mvn-repo.yaml b/k8s/mvn-repo.yaml
@@ -20,7 +20,7 @@ spec:
           secretName: mvn-repo-keystore
       containers:
       - name: mvn-repo
-        image: virtuslab/scala-community-build-mvn-repo:v0.1.0
+        image: virtuslab/scala-community-build-mvn-repo:v0.1.1
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 8081
diff --git a/k8s/project-builder-mill-test.yaml b/k8s/project-builder-mill-test.yaml
@@ -5,23 +5,10 @@ metadata:
 spec:
   template:
     spec:
-      volumes:
-      - name: mvn-repo-cert
-        configMap:
-          name: mvn-repo-cert
       containers:
       - name: project-builder-mill-test
         image: virtuslab/scala-community-build-project-builder:jdk17-test
         imagePullPolicy: IfNotPresent
-        volumeMounts:
-        - name: mvn-repo-cert
-          mountPath: /usr/local/share/ca-certificates/mvn-repo.crt
-          subPath: mvn-repo.crt
-          readOnly: true
-        lifecycle:
-          postStart:
-            exec:
-              command: ["update-ca-certificates"]
         command:
         - "bash"
         - "-c"
@@ -30,7 +17,7 @@ spec:
               https://github.com/com-lihaoyi/upickle.git \
               1.5.0 \
               3.1.0 \
-              1.5.0 \
+              "" \
               "com.lihaoyi%ujson com.lihaoyi%upickle com.lihaoyi%upickle-core com.lihaoyi%upack com.lihaoyi%upickle-implicits dummy.org%dummy-project" \
               https://mvn-repo:8081/maven2/test \
               1.5.5 \
diff --git a/k8s/project-builder-sbt-test.yaml b/k8s/project-builder-sbt-test.yaml
@@ -5,26 +5,13 @@ metadata:
 spec:
   template:
     spec:
-      volumes:
-        - name: mvn-repo-cert
-          configMap:
-            name: mvn-repo-cert
       containers:
         - name: project-builder-sbt-test
           image: virtuslab/scala-community-build-project-builder:jdk11-test
           imagePullPolicy: IfNotPresent
-          volumeMounts:
-            - name: mvn-repo-cert
-              mountPath: /usr/local/share/ca-certificates/mvn-repo.crt
-              subPath: mvn-repo.crt
-              readOnly: true
           resources:
             requests:
               memory: 4G
-          lifecycle:
-            postStart:
-              exec:
-                command: ["update-ca-certificates"]
           command:
             - "bash"
             - "-c"
@@ -33,7 +20,7 @@ spec:
                 https://github.com/typelevel/shapeless-3.git \
                 "v3.0.4" \
                 3.1.0 \
-                3.0.4 \
+                "" \
                 "org.typelevel%shapeless3-deriving org.typelevel%shapeless3-test org.typelevel%shapeless3-typeable dummy.org%dummy-project" \
                 https://mvn-repo:8081/maven2/test \
                 1.5.5 \
diff --git a/project-builder/build-revision.sh b/project-builder/build-revision.sh
@@ -23,6 +23,7 @@ $scriptDir/checkout.sh "$repoUrl" "$rev" repo
 # especially in cli immediately after starting minikube
 for i in {1..10}; do
   if errMsg=$(curl $mvnRepoUrl 2>&1); then
+    echo "Connected with $mvnRepoUrl."
     break
   else
     echo "${errMsg}"
diff --git a/project-builder/mill/MillCommunityBuild.sc b/project-builder/mill/MillCommunityBuild.sc
@@ -1,5 +1,6 @@
 import $file.CommunityBuildCore,
 CommunityBuildCore.Scala3CommunityBuild.{TestingMode => _, ProjectBuildConfig => _, ProjectOverrides => _, _}
+import CommunityBuildCore.Scala3CommunityBuild.Utils._
 // Make sure that following classes are in sync with the ones defined in CommunityBuildcore,
 //  upickle has problems with classess imported from other file when creating readers
 case class ProjectBuildConfig(
@@ -129,11 +130,12 @@ def mapCrossVersions(
         (crossVersion, buildScalaVersion)
       case _ => crossEntry
     }
+    version <- Seq(crossEntry, mappedCrossVersion).distinct
   } yield {
-    if (mappedCrossVersion != crossEntry) {
-      println(s"Use cross-version $mappedCrossVersion instead of $crossEntry")
+    if (version != crossEntry) {
+      println(s"Use cross-version $version instead of $crossEntry")
     }
-    mappedCrossVersion
+    version
   }
 }
 
@@ -187,7 +189,7 @@ def runBuild(configJson: String, targets: Seq[String])(implicit ctx: Ctx) = {
   println(s"Build config: ${configJson}")
   val config = read[ProjectBuildConfig](configJson)
   println(s"Parsed config: ${config}")
-  val filteredTargets = Utils.filterTargets(targets, config.projects.exclude.map(_.r))
+  val filteredTargets = filterTargets(targets, config.projects.exclude.map(_.r))
   val mappings = checkedModuleMappings(filteredTargets.toSet)
   val topLevelModules = mappings.collect {
     case (target, info) if filteredTargets.contains(target) => info
diff --git a/project-builder/sbt/CommunityBuildPlugin.scala b/project-builder/sbt/CommunityBuildPlugin.scala
@@ -9,6 +9,7 @@ import xsbti.{Severity, CompileFailed, Problem}
 import scala.collection.JavaConverters._
 
 import Scala3CommunityBuild._
+import Scala3CommunityBuild.Utils._
 import TaskEvaluator.EvalResult
 
 class SbtTaskEvaluator(val project: ProjectRef, private var state: State)
@@ -310,7 +311,7 @@ object CommunityBuildPlugin extends AutoPlugin {
 
   lazy val ourVersion =
     Option(sys.props("communitybuild.version")).filter(_.nonEmpty)
-  lazy val dualVersioning = Utils.DualVersioningType.resolve
+  lazy val dualVersioning = DualVersioningType.resolve
   lazy val publishVersions = ourVersion.toList.map { version =>
     version :: dualVersioning.flatMap(_.apply(version)).map(_.render).toList
   }
@@ -400,7 +401,7 @@ object CommunityBuildPlugin extends AutoPlugin {
         simplifiedModuleId(key) -> value
       }
 
-      val filteredIds = Utils.filterTargets(ids, config.projects.exclude.map(_.r))
+      val filteredIds = filterTargets(ids, config.projects.exclude.map(_.r))
 
       println("Starting build...")
       // Find projects that matches maven
diff --git a/scripts/build-all.sh b/scripts/build-all.sh
@@ -7,7 +7,7 @@ if [ $# -ne 1 ]; then
 fi
 
 VERSION="$1"
-export PREV_CB_VERSION="v0.1.0"
+export PREV_CB_VERSION="v0.1.1"
 
 javaDefault=11
 javaAccessoryVersions=(8 17)
diff --git a/scripts/start-mvn-repo.sh b/scripts/start-mvn-repo.sh
@@ -8,9 +8,9 @@ export MVN_REPO_KEYSTORE_PASSWORD=$(openssl rand -base64 32)
 
 $scriptDir/generate-secrets.sh
 
-scbk create secret generic mvn-repo-keystore --from-file=$scriptDir/../secrets/mvn-repo.p12
-scbk create secret generic mvn-repo-passwords --from-literal=keystore-password="$MVN_REPO_KEYSTORE_PASSWORD"
-scbk create cm mvn-repo-cert --from-file=$scriptDir/../secrets/mvn-repo.crt
+scbk create secret generic mvn-repo-keystore --from-file=$scriptDir/../secrets/mvn-repo.p12 || echo "secret mvn-repo-keystore already exists"
+scbk create secret generic mvn-repo-passwords --from-literal=keystore-password="$MVN_REPO_KEYSTORE_PASSWORD"  || echo "secret mvn-repo-passwords already exists"
+scbk create cm mvn-repo-cert --from-file=$scriptDir/../secrets/mvn-repo.crt || echo "config-map mvn-repo-cert already exists" 
 scbk apply -f $scriptDir/../k8s/mvn-repo-data.yaml
 
 mvnRepoYaml=$scriptDir/../k8s/mvn-repo.yaml
diff --git a/scripts/test-build.sh b/scripts/test-build.sh
@@ -10,10 +10,8 @@ kubectl delete namespace $testNamespace --ignore-not-found=true
 kubectl create namespace $testNamespace
 
 CB_VERSION="test" \
-CB_K8S_NAMESPACE="${testNamespace}" \
-$scriptDir/start-mvn-repo.sh
-# Wait until ready
-sleep 30
+  CB_K8S_NAMESPACE="${testNamespace}" \
+  $scriptDir/start-mvn-repo.sh
 
 function projectBuilderFailed() {
   jobName="$1"
diff --git a/scripts/test-cli.sh b/scripts/test-cli.sh
@@ -8,17 +8,17 @@ cd $scriptDir/../cli
 # Installation of scala-cli in the GH actions workflow was not very effective, and might have lead to missing binary on the PATH when executing this script
 
 testNamespace=scala3-community-build-test
-cliRunCmd="run scb-cli.scala --jvm=11 --java-prop communitybuild.version=test --java-prop communitybuild.local.dir=$scriptDir/.. --quiet -- "
-commonOpts="--namespace=$testNamespace --keepCluster --keepMavenRepo --noRedirectLogs"
-sbtProject=typelevel/shapeless-3
-millProject=com-lihaoyi/os-lib
+cliRunCmd="run scb-cli.scala --jvm=11 --java-prop communitybuild.version=test --java-prop communitybuild.local.dir=$scriptDir/..  -- "
+commonOpts="--namespace=$testNamespace --noRedirectLogs"
+sbtProject="typelevel/shapeless-3 --revision=v3.1.0"
+millProject="com-lihaoyi/os-lib --revision=0.8.1" 
 scalaVersion=3.1.1
 
 echo "Test sbt custom build in minikube"
-scala-cli $cliRunCmd run $sbtProject $scalaVersion $commonOpts
+scala-cli $cliRunCmd run $sbtProject $scalaVersion $commonOpts --noPublishArtifacts
 echo
 echo "Test mill custom build in minikube"
-scala-cli $cliRunCmd run $millProject $scalaVersion $commonOpts
+scala-cli $cliRunCmd run $millProject $scalaVersion $commonOpts --noPublishArtifacts
 
 echo
 echo "Test sbt custom build locally"
@@ -27,5 +27,4 @@ echo
 echo "Test mill custom build locally"
 scala-cli $cliRunCmd run $millProject $scalaVersion $commonOpts --locally
 
-
 echo "Tests passed"
