Skip to content

Upgrade to Scala v2.13 #425

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 27, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
language: scala
dist: xenial
scala:
- 2.12.12
- 2.13.4

jdk:
- openjdk8
Expand Down
3 changes: 1 addition & 2 deletions bfg-benchmark/build.sbt
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import Dependencies._

libraryDependencies ++= Seq(
libraryDependencies ++= guava ++ Seq(
madgagCompress,
scalaIoFile,
textmatching,
scopt
)
38 changes: 17 additions & 21 deletions bfg-benchmark/src/main/scala/Benchmark.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@ import lib.Timing.measureTask
import lib._
import model._

import java.nio.file.Files
import java.nio.file.Files.isDirectory
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent._
import scala.concurrent.duration.Duration
import scala.jdk.StreamConverters._
import scala.sys.process._
import scalax.file.PathMatcher.IsDirectory
import scalax.io.Codec

/*
* Vary BFG runs by:
Expand All @@ -17,24 +18,22 @@ import scalax.io.Codec
*/
object Benchmark extends App {

implicit val codec = Codec.UTF8

BenchmarkConfig.parser.parse(args, BenchmarkConfig()) map {
config =>
println(s"Using resources dir : ${config.resourcesDir.path}")
println(s"Using resources dir : ${config.resourcesDir}")

require(config.resourcesDir.exists, s"Resources dir not found : ${config.resourcesDir.path}")
require(config.jarsDir.exists, s"Jars dir not found : ${config.jarsDir.path}")
require(config.reposDir.exists, s"Repos dir not found : ${config.reposDir.path}")
require(Files.exists(config.resourcesDir), s"Resources dir not found : ${config.resourcesDir}")
require(Files.exists(config.jarsDir), s"Jars dir not found : ${config.jarsDir}")
require(Files.exists(config.reposDir), s"Repos dir not found : ${config.reposDir}")

val missingJars = config.bfgJars.filterNot(_.exists).map(_.toAbsolute.path)
val missingJars = config.bfgJars.filterNot(Files.exists(_))
require(missingJars.isEmpty, s"Missing BFG jars : ${missingJars.mkString(",")}")

val tasksFuture = for {
bfgInvocableEngineSet <- bfgInvocableEngineSet(config)
} yield {
val gfbInvocableEngineSetOpt =
if (config.onlyBfg) None else Some(InvocableEngineSet[GFBInvocation](GitFilterBranch, Seq(InvocableGitFilterBranch)))
Option.when(!config.onlyBfg)(InvocableEngineSet[GFBInvocation](GitFilterBranch, Seq(InvocableGitFilterBranch)))
boogaloo(config, new RepoExtractor(config.scratchDir), Seq(bfgInvocableEngineSet) ++ gfbInvocableEngineSetOpt.toSeq)
}

Expand All @@ -59,27 +58,24 @@ object Benchmark extends App {
def boogaloo(config: BenchmarkConfig, repoExtractor: RepoExtractor, invocableEngineSets: Seq[InvocableEngineSet[_ <: EngineInvocation]]) = {

for {
repoSpecDir <- config.repoSpecDirs.toList
availableCommandDirs = (repoSpecDir / "commands").children().filter(IsDirectory).toList
repoSpecDir <- config.repoSpecDirs
availableCommandDirs = Files.list(repoSpecDir.resolve("commands")).toScala(Seq).filter(isDirectory(_))
// println(s"Available commands for $repoName : ${availableCommandDirs.map(_.name).mkString(", ")}")
commandDir <- availableCommandDirs.filter(p => config.commands(p.name))
commandDir <- availableCommandDirs.filter(p => config.commands(p.getFileName.toString))
} yield {

val repoName = repoSpecDir.name

val commandName = commandDir.name
val commandName: String = commandDir.getFileName.toString

commandName -> (for {
invocableEngineSet <- invocableEngineSets
} yield for {
(invocable, processMaker) <- invocableEngineSet.invocationsFor(commandDir)
} yield {
val cleanRepoDir = repoExtractor.extractRepoFrom(repoSpecDir / "repo.git.zip")
commandDir.children().foreach(p => p.copyTo(cleanRepoDir / p.name))
val process = processMaker(cleanRepoDir)
val cleanRepoDir = repoExtractor.extractRepoFrom(repoSpecDir.resolve("repo.git.zip"))
Files.list(commandDir).toScala(Seq).foreach(p => Files.copy(p, cleanRepoDir.resolve(p.getFileName)))
val process = processMaker(cleanRepoDir.toFile)

val duration = measureTask(s"$commandName - $invocable") {
process ! ProcessLogger(_ => Unit)
process ! ProcessLogger(_ => ())
}

if (config.dieIfTaskTakesLongerThan.exists(_ < duration.toMillis)) {
Expand Down
23 changes: 10 additions & 13 deletions bfg-benchmark/src/main/scala/BenchmarkConfig.scala
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
import java.io.File

import com.madgag.textmatching.{Glob, TextMatcher}
import scopt.OptionParser

import scalax.file.ImplicitConversions._
import scalax.file.Path
import scalax.file.defaultfs.DefaultPath
import java.nio.file.{Path, Paths}

object BenchmarkConfig {
val parser = new OptionParser[BenchmarkConfig]("benchmark") {
opt[File]("resources-dir").text("benchmark resources folder - contains jars and repos").action {
(v, c) => c.copy(resourcesDirOption = v)
(v, c) => c.copy(resourcesDirOption = v.toPath)
}
opt[String]("java").text("Java command paths").action {
(v, c) => c.copy(javaCmds = v.split(',').toSeq)
Expand All @@ -28,27 +25,27 @@ object BenchmarkConfig {
(v, c) => c.copy(commands = TextMatcher(v, defaultType = Glob))
}
opt[File]("scratch-dir").text("Temp-dir for job runs - preferably ramdisk, eg tmpfs.").action {
(v, c) => c.copy(scratchDir = v)
(v, c) => c.copy(scratchDir = v.toPath)
}
opt[Unit]("only-bfg") action { (_, c) => c.copy(onlyBfg = true) } text "Don't benchmark git-filter-branch"
}
}
case class BenchmarkConfig(resourcesDirOption: Path = Path.fromString(System.getProperty("user.dir")) / "bfg-benchmark" / "resources",
scratchDir: DefaultPath = Path.fromString("/dev/shm/"),
case class BenchmarkConfig(resourcesDirOption: Path = Paths.get(System.getProperty("user.dir"), "bfg-benchmark", "resources"),
scratchDir: Path = Paths.get("/dev/shm/"),
javaCmds: Seq[String] = Seq("java"),
bfgVersions: Seq[String] = Seq.empty,
commands: TextMatcher = Glob("*"),
onlyBfg: Boolean = false,
dieIfTaskTakesLongerThan: Option[Int] = None,
repoNames: Seq[String] = Seq.empty) {

lazy val resourcesDir = Path.fromString(resourcesDirOption.path).toAbsolute
lazy val resourcesDir: Path = resourcesDirOption.toAbsolutePath

lazy val jarsDir = resourcesDir / "jars"
lazy val jarsDir: Path = resourcesDir.resolve("jars")

lazy val reposDir = resourcesDir / "repos"
lazy val reposDir: Path = resourcesDir.resolve("repos")

lazy val bfgJars = bfgVersions.map(version => jarsDir / s"bfg-$version.jar")
lazy val bfgJars: Seq[Path] = bfgVersions.map(version => jarsDir.resolve(s"bfg-$version.jar"))

lazy val repoSpecDirs = repoNames.map(reposDir / _)
lazy val repoSpecDirs: Seq[Path] = repoNames.map(reposDir.resolve)
}
21 changes: 12 additions & 9 deletions bfg-benchmark/src/main/scala/lib/Repo.scala
Original file line number Diff line number Diff line change
@@ -1,23 +1,26 @@
package lib

import com.google.common.io.MoreFiles
import com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE
import com.madgag.compress.CompressUtil._

import scalax.file.ImplicitConversions._
import scalax.file.Path
import scalax.file.defaultfs.DefaultPath
import java.nio.file.{Files, Path}
import scala.util.Using

class RepoExtractor(scratchDir: DefaultPath) {
class RepoExtractor(scratchDir: Path) {

val repoDir = scratchDir / "repo.git"
val repoDir = scratchDir.resolve( "repo.git")

def extractRepoFrom(zipPath: Path) = {
repoDir.deleteRecursively(force = true)
if (Files.exists(repoDir)) MoreFiles.deleteRecursively(repoDir, ALLOW_INSECURE)

repoDir.createDirectory()
Files.createDirectories(repoDir)

println(s"Extracting repo to ${repoDir.toAbsolute.path}")
println(s"Extracting repo to ${repoDir.toAbsolutePath}")

zipPath.inputStream.acquireFor { stream => unzip(stream, repoDir) }
Using(Files.newInputStream(zipPath)) {
stream => unzip(stream, repoDir.toFile)
}

repoDir
}
Expand Down
2 changes: 1 addition & 1 deletion bfg-benchmark/src/main/scala/model/BFGJar.scala
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package model

import scalax.file.Path
import java.nio.file.Path

object BFGJar {
def from(path: Path) = BFGJar(path, Map.empty)
Expand Down
29 changes: 16 additions & 13 deletions bfg-benchmark/src/main/scala/model/InvocableEngine.scala
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package model

import com.google.common.io.CharSource
import com.google.common.io.Files.asCharSource

import java.io.File
import java.nio.charset.StandardCharsets.UTF_8
import java.nio.file.{Files, Path}
import scala.jdk.StreamConverters._
import scala.sys.process.{Process, ProcessBuilder}
import scalax.file.ImplicitConversions._
import scalax.file.Path
import scalax.file.defaultfs.DefaultPath
import scalax.io.Input

trait EngineInvocation

Expand All @@ -15,19 +18,19 @@ case class GFBInvocation(args: Seq[String]) extends EngineInvocation

trait InvocableEngine[InvocationArgs <: EngineInvocation] {

def processFor(invocation: InvocationArgs)(repoPath: DefaultPath): ProcessBuilder
def processFor(invocation: InvocationArgs)(repoPath: File): ProcessBuilder
}

case class InvocableBFG(java: Java, bfgJar: BFGJar) extends InvocableEngine[BFGInvocation] {

def processFor(invocation: BFGInvocation)(repoPath: DefaultPath) =
Process(s"${java.javaCmd} -jar ${bfgJar.path.path} ${invocation.args}", repoPath)
def processFor(invocation: BFGInvocation)(repoPath: File) =
Process(s"${java.javaCmd} -jar ${bfgJar.path} ${invocation.args}", repoPath)

}

object InvocableGitFilterBranch extends InvocableEngine[GFBInvocation] {

def processFor(invocation: GFBInvocation)(repoPath: DefaultPath) =
def processFor(invocation: GFBInvocation)(repoPath: File) =
Process(Seq("git", "filter-branch") ++ invocation.args, repoPath)
}

Expand All @@ -42,24 +45,24 @@ We want to allow the user to vary:
trait EngineType[InvocationType <: EngineInvocation] {
val configName: String

def argsFor(config: Input): InvocationType
def argsFor(config: CharSource): InvocationType

def argsOptsFor(commandDir: Path): Option[InvocationType] = {
val paramsPath = commandDir / s"$configName.txt"
if (paramsPath.exists) Some(argsFor(paramsPath)) else None
val paramsPath = commandDir.resolve(s"$configName.txt")
if (Files.exists(paramsPath)) Some(argsFor(asCharSource(paramsPath.toFile, UTF_8))) else None
}
}

case object BFG extends EngineType[BFGInvocation] {

val configName = "bfg"

def argsFor(config: Input) = BFGInvocation(config.string)
def argsFor(config: CharSource) = BFGInvocation(config.read())
}

case object GitFilterBranch extends EngineType[GFBInvocation] {

val configName = "gfb"

def argsFor(config: Input) = GFBInvocation(config.lines().toSeq)
def argsFor(config: CharSource) = GFBInvocation(config.lines().toScala(Seq))
}
6 changes: 3 additions & 3 deletions bfg-benchmark/src/main/scala/model/InvocableEngineSet.scala
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
package model

import scalax.file.Path
import scalax.file.defaultfs.DefaultPath
import java.io.File
import java.nio.file.Path

case class InvocableEngineSet[InvocationArgs <: EngineInvocation](
engineType: EngineType[InvocationArgs],
invocableEngines: Seq[InvocableEngine[InvocationArgs]]
) {

def invocationsFor(commandDir: Path): Seq[(InvocableEngine[InvocationArgs], DefaultPath => scala.sys.process.ProcessBuilder)] = {
def invocationsFor(commandDir: Path): Seq[(InvocableEngine[InvocationArgs], File => scala.sys.process.ProcessBuilder)] = {
for {
args <- engineType.argsOptsFor(commandDir).toSeq
invocable <- invocableEngines
Expand Down
4 changes: 3 additions & 1 deletion bfg-benchmark/src/test/scala/JavaVersionSpec.scala
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import org.scalatest.{FlatSpec, Matchers, OptionValues}
import org.scalatest.OptionValues
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

object JavaVersionSpec extends AnyFlatSpec with OptionValues with Matchers {
"version" should "parse an example line" in {
Expand Down
12 changes: 11 additions & 1 deletion bfg-library/build.sbt
Original file line number Diff line number Diff line change
@@ -1,4 +1,14 @@
import Dependencies._

libraryDependencies ++= guava :+ scalaIoFile :+ textmatching :+ scalaGit :+ jgit :+ slf4jSimple :+ scalaGitTest % "test"
libraryDependencies ++= guava ++ Seq(
parCollections,
scalaCollectionPlus,
textmatching,
scalaGit,
jgit,
slf4jSimple,
lineSplitting,
scalaGitTest % Test,
"org.apache.commons" % "commons-text" % "1.9" % Test
)

Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

package com.madgag.collection.concurrent

import com.madgag.scala.collection.decorators._

class ConcurrentMultiMap[A, B] {

Expand All @@ -34,5 +35,5 @@ class ConcurrentMultiMap[A, B] {
this
}

def toMap: Map[A, Set[B]] = m.toMap.mapValues(_.toSet)
def toMap: Map[A, Set[B]] = m.toMap.mapV(_.toSet)
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,29 +20,53 @@

package com.madgag.collection.concurrent

import scala.collection.mutable.{Set, SetLike}
import scala.collection.mutable.{AbstractSet, SetOps}
import scala.collection.{IterableFactory, IterableFactoryDefaults, mutable}


class ConcurrentSet[A] extends Set[A] with SetLike[A, ConcurrentSet[A]] {
class ConcurrentSet[A]()
extends AbstractSet[A]
with SetOps[A, ConcurrentSet, ConcurrentSet[A]]
with IterableFactoryDefaults[A, ConcurrentSet]
{

val m: collection.concurrent.Map[A, Boolean] = collection.concurrent.TrieMap.empty

override def +=(elem: A): this.type = {
override def iterableFactory: IterableFactory[ConcurrentSet] = ConcurrentSet

override def clear(): Unit = m.clear()

override def addOne(elem: A): ConcurrentSet.this.type = {
m.put(elem, true)
this
}

override def -=(elem: A): this.type = {
override def subtractOne(elem: A): ConcurrentSet.this.type = {
m.remove(elem)
this
}

override def empty: this.type = {
m.empty
this
}

override def contains(elem: A): Boolean = m.contains(elem)

override def iterator: Iterator[A] = m.keysIterator

}

object ConcurrentSet extends IterableFactory[ConcurrentSet] {

@transient
private final val EmptySet = new ConcurrentSet()

def empty[A]: ConcurrentSet[A] = EmptySet.asInstanceOf[ConcurrentSet[A]]

def from[A](source: collection.IterableOnce[A]): ConcurrentSet[A] =
source match {
case hs: ConcurrentSet[A] => hs
case _ if source.knownSize == 0 => empty[A]
case _ => (newBuilder[A] ++= source).result()
}

/** Create a new Builder which can be reused after calling `result()` without an
* intermediate call to `clear()` in order to build multiple related results.
*/
def newBuilder[A]: mutable.Builder[A, ConcurrentSet[A]] = ???
}
Loading