Merge pull request #11343 from dotty-staging/tasty-version-scheme

new TASTy header format - add experimental version and tooling version fields

Author: bishabosha

compiler/src/dotty/tools/dotc/config/Properties.scala

@@ -64,18 +64,24 @@ trait PropertiesTrait {
    */
   def versionNumberString: String = scalaPropOrEmpty("version.number")
 
-  /** The version number of the jar this was loaded from plus "version " prefix,
-   *  or "version (unknown)" if it cannot be determined.
+  /** The version number of the jar this was loaded from,
+   *  or `"(unknown)"` if it cannot be determined.
    */
-  val versionString: String = {
+  val simpleVersionString: String = {
     val v = scalaPropOrElse("version.number", "(unknown)")
-    "version " + scalaPropOrElse("version.number", "(unknown)") + {
+    v + (
       if (v.contains("SNAPSHOT") || v.contains("NIGHTLY"))
         "-git-" + scalaPropOrElse("git.hash", "(unknown)")
-      else ""
-    }
+      else
+        ""
+    )
   }
 
+  /** The version number of the jar this was loaded from plus `"version "` prefix,
+   *  or `"version (unknown)"` if it cannot be determined.
+   */
+  val versionString: String = "version " + simpleVersionString
+
   /** Whether the current version of compiler is experimental
    *
    *  1. Snapshot and nightly releases are experimental.

compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala

@@ -12,6 +12,15 @@ import core.Symbols.{Symbol, ClassSymbol}
 import ast.tpd
 import Decorators._
 
+object TastyPickler {
+
+  private val versionStringBytes = {
+    val compilerString = s"Scala ${config.Properties.simpleVersionString}"
+    compilerString.getBytes(java.nio.charset.StandardCharsets.UTF_8)
+  }
+
+}
+
 class TastyPickler(val rootCls: ClassSymbol) {
 
   private val sections = new mutable.ArrayBuffer[(NameRef, TastyBuffer)]
@@ -37,10 +46,13 @@ class TastyPickler(val rootCls: ClassSymbol) {
     val uuidHi: Long = otherSectionHashes.fold(0L)(_ ^ _)
 
     val headerBuffer = {
-      val buf = new TastyBuffer(header.length + 24)
+      val buf = new TastyBuffer(header.length + TastyPickler.versionStringBytes.length + 32)
       for (ch <- header) buf.writeByte(ch.toByte)
       buf.writeNat(MajorVersion)
       buf.writeNat(MinorVersion)
+      buf.writeNat(ExperimentalVersion)
+      buf.writeNat(TastyPickler.versionStringBytes.length)
+      buf.writeBytes(TastyPickler.versionStringBytes, TastyPickler.versionStringBytes.length)
       buf.writeUncompressedLong(uuidLow)
       buf.writeUncompressedLong(uuidHi)
       buf

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -248,7 +248,7 @@ class CompilationTests {
           //  "-source", "future",  // TODO: re-enable once we allow : @unchecked in pattern definitions. Right now, lots of narrowing pattern definitions fail.
           ))(libGroup)
 
-    val tastyCoreSources = sources(Paths.get("tasty/src")) ++ sources(Paths.get("tasty/src-bootstrapped"))
+    val tastyCoreSources = sources(Paths.get("tasty/src"))
     val tastyCore = compileList("tastyCore", tastyCoreSources, opt)(tastyCoreGroup)
 
     val compilerSources = sources(Paths.get("compiler/src")) ++ sources(Paths.get("compiler/src-bootstrapped"))

tasty/src-bootstrapped/dotty/tools/tasty/TastyHeaderUnpickler.scala

@@ -19,10 +19,11 @@ Micro-syntax:
 
 Macro-format:
 
-  File          = Header majorVersion_Nat minorVersion_Nat UUID
+  File          = Header majorVersion_Nat minorVersion_Nat experimentalVersion_Nat VersionString UUID
                   nameTable_Length Name* Section*
   Header        = 0x5CA1AB1F
   UUID          = Byte*16                 -- random UUID
+  VersionString = Length UTF8-CodePoint*  -- string that represents the compiler that produced the TASTy
 
   Section       = NameRef Length Bytes
   Length        = Nat                     -- length of rest of entry in bytes
@@ -262,9 +263,100 @@ Standard Section: "Comments" Comment*
 
 object TastyFormat {
 
+  /** The first four bytes of a TASTy file, followed by four values:
+    * - `MajorVersion: Int` - see definition in `TastyFormat`
+    * - `MinorVersion: Int` - see definition in `TastyFormat`
+    * - `ExperimentalVersion: Int` - see definition in `TastyFormat`
+    * - `ToolingVersion: String` - arbitrary length string representing the tool that produced the TASTy.
+    */
   final val header: Array[Int] = Array(0x5C, 0xA1, 0xAB, 0x1F)
-  val MajorVersion: Int = 27
-  val MinorVersion: Int = 0
+
+  /**Natural number. Each increment of the `MajorVersion` begins a
+   * new series of backward compatible TASTy versions.
+   *
+   * A TASTy file in either the preceeding or succeeding series is
+   * incompatible with the current value.
+   */
+  final val MajorVersion: Int = 28
+
+  /**Natural number. Each increment of the `MinorVersion`, within
+   * a series declared by the `MajorVersion`, breaks forward
+   * compatibility, but remains backwards compatible, with all
+   * preceeding `MinorVersion`.
+   */
+  final val MinorVersion: Int = 0
+
+  /**Natural Number. The `ExperimentalVersion` allows for
+   * experimentation with changes to TASTy without committing
+   * to any guarantees of compatibility.
+   *
+   * A zero value indicates that the TASTy version is from a
+   * stable, final release.
+   *
+   * A strictly positive value indicates that the TASTy
+   * version is experimental. An experimental TASTy file
+   * can only be read by a tool with the same version.
+   * However, tooling with an experimental TASTy version
+   * is able to read final TASTy documents if the file's
+   * `MinorVersion` is strictly less than the current value.
+   */
+  final val ExperimentalVersion: Int = 1
+
+  /**This method implements a binary relation (`<:<`) between two TASTy versions.
+   * We label the lhs `file` and rhs `compiler`.
+   * if `file <:< compiler` then the TASTy file is valid to be read.
+   *
+   * TASTy versions have a partial order,
+   * for example `a <:< b` and `b <:< a` are both false if `a` and `b` have different major versions.
+   *
+   * We follow the given algorithm:
+   * ```
+   * if file.major != compiler.major then
+   *   return incompatible
+   * if compiler.experimental == 0 then
+   *   if file.experimental != 0 then
+   *     return incompatible
+   *   if file.minor > compiler.minor then
+   *     return incompatible
+   *   else
+   *     return compatible
+   * else invariant[compiler.experimental != 0]
+   *   if file.experimental == compiler.experimental then
+   *     if file.minor == compiler.minor then
+   *       return compatible (all fields equal)
+   *     else
+   *       return incompatible
+   *   else if file.experimental == 0,
+   *     if file.minor < compiler.minor then
+   *       return compatible (an experimental version can read a previous released version)
+   *     else
+   *       return incompatible (an experimental version cannot read its own minor version or any later version)
+   *   else invariant[file.experimental is non-0 and different than compiler.experimental]
+   *     return incompatible
+   * ```
+   */
+  def isVersionCompatible(
+    fileMajor: Int,
+    fileMinor: Int,
+    fileExperimental: Int,
+    compilerMajor: Int,
+    compilerMinor: Int,
+    compilerExperimental: Int
+  ): Boolean = (
+    fileMajor == compilerMajor && (
+      if (fileExperimental == compilerExperimental) {
+        if (compilerExperimental == 0) {
+          fileMinor <= compilerMinor
+        }
+        else {
+          fileMinor == compilerMinor
+        }
+      }
+      else {
+        fileExperimental == 0 && fileMinor < compilerMinor
+      }
+    )
+  )
 
   final val ASTsSection = "ASTs"
   final val PositionsSection = "Positions"

tasty/src-non-bootstrapped/dotty/tools/tasty/TastyHeaderUnpickler.scala

@@ -0,0 +1,128 @@
+package dotty.tools.tasty
+
+import java.util.UUID
+
+import TastyFormat.{MajorVersion, MinorVersion, ExperimentalVersion, header}
+
+/**
+ * The Tasty Header consists of four fields:
+ * - uuid
+ *   - contains a hash of the sections of the TASTy file
+ * - majorVersion
+ *   - matching the TASTy format version that last broke backwards compatibility
+ * - minorVersion
+ *   - matching the TASTy format version that last broke forward compatibility
+ * - experimentalVersion
+ *   - 0 for final compiler version
+ *   - positive for between minor versions and forward compatibility
+ *     is broken since the previous stable version.
+ * - toolingVersion
+ *   - arbitrary string representing the tooling that produced the TASTy
+ */
+sealed abstract case class TastyHeader(
+  uuid: UUID,
+  majorVersion: Int,
+  minorVersion: Int,
+  experimentalVersion: Int,
+  toolingVersion: String
+)
+
+class TastyHeaderUnpickler(reader: TastyReader) {
+  import TastyHeaderUnpickler._
+  import reader._
+
+  def this(bytes: Array[Byte]) = this(new TastyReader(bytes))
+
+  /** reads and verifies the TASTy version, extracting the UUID */
+  def readHeader(): UUID =
+    readFullHeader().uuid
+
+  /** reads and verifies the TASTy version, extracting the whole header */
+  def readFullHeader(): TastyHeader = {
+
+    for (i <- 0 until header.length)
+      check(readByte() == header(i), "not a TASTy file")
+    val fileMajor = readNat()
+    if (fileMajor <= 27) { // old behavior before `tasty-core` 3.0.0-M4
+      val fileMinor = readNat()
+      val signature = signatureString(fileMajor, fileMinor, 0)
+      throw new UnpickleException(signature + backIncompatAddendum + toolingAddendum)
+    }
+    else {
+      val fileMinor = readNat()
+      val fileExperimental = readNat()
+      val toolingVersion = {
+        val length = readNat()
+        val start = currentAddr
+        val end = start + length
+        goto(end)
+        new String(bytes, start.index, length)
+      }
+
+      val validVersion = TastyFormat.isVersionCompatible(
+        fileMajor            = fileMajor,
+        fileMinor            = fileMinor,
+        fileExperimental     = fileExperimental,
+        compilerMajor        = MajorVersion,
+        compilerMinor        = MinorVersion,
+        compilerExperimental = ExperimentalVersion
+      )
+
+      check(validVersion, {
+        val signature = signatureString(fileMajor, fileMinor, fileExperimental)
+        val producedByAddendum = s"\nThe TASTy file was produced by $toolingVersion.$toolingAddendum"
+        val msg = (
+          if (fileExperimental != 0) unstableAddendum
+          else if (fileMajor < MajorVersion) backIncompatAddendum
+          else forwardIncompatAddendum
+        )
+        signature + msg + producedByAddendum
+      })
+
+      val uuid = new UUID(readUncompressedLong(), readUncompressedLong())
+      new TastyHeader(uuid, fileMajor, fileMinor, fileExperimental, toolingVersion) {}
+    }
+  }
+
+  def isAtEnd: Boolean = reader.isAtEnd
+
+  private def check(cond: Boolean, msg: => String): Unit = {
+    if (!cond) throw new UnpickleException(msg)
+  }
+}
+
+object TastyHeaderUnpickler {
+
+  private def toolingAddendum = (
+    if (ExperimentalVersion > 0)
+      "\nNote that your tooling is currently using an unstable TASTy version."
+    else
+      ""
+  )
+
+  private def signatureString(fileMajor: Int, fileMinor: Int, fileExperimental: Int) = {
+    def showMinorVersion(min: Int, exp: Int) = {
+      val expStr = if (exp == 0) "" else s" [unstable release: $exp]"
+      s"$min$expStr"
+    }
+    val minorVersion = showMinorVersion(MinorVersion, ExperimentalVersion)
+    val fileMinorVersion = showMinorVersion(fileMinor, fileExperimental)
+    s"""TASTy signature has wrong version.
+      | expected: {majorVersion: $MajorVersion, minorVersion: $minorVersion}
+      | found   : {majorVersion: $fileMajor, minorVersion: $fileMinorVersion}
+      |
+      |""".stripMargin
+  }
+
+  private def unstableAddendum =
+    """This TASTy file was produced by an unstable release.
+      |To read this TASTy file, your tooling must be at the same version.""".stripMargin
+
+  private def backIncompatAddendum =
+    """This TASTy file was produced by an earlier release that is not supported anymore.
+      |Please recompile this TASTy with a later version.""".stripMargin
+
+  private def forwardIncompatAddendum =
+    """This TASTy file was produced by a more recent, forwards incompatible release.
+      |To read this TASTy file, please upgrade your tooling.""".stripMargin
+}