Breaking: new RoAST representation of module.comments (#1987)

A comment is just text at a given location, so perfectly representable as just
a location. Doing this also allows us to avoid base64-decoding each one, as Regal
can instead get the original text from the source file.

While this is a breaking change for anyone referencing `input.comments` directly,
users are advised to instead reference `data.regal.ast.comments_decoded`, which'll
provide them a list of comment loctations in the usual location form.

Also:
- Played some with using the new `not` form in a few places, mostly so that we're
  invested in that from the start.

Signed-off-by: Anders Eknert <anders.eknert@apple.com>

Author: anderseknert

.regal/config.yaml

@@ -142,7 +142,7 @@ rules:
         files:
           # This package is used to build the LSP initialize
           # response, and names must conform to the specification
-          - "bundle/regal/lsp/initialize/initialize.rego"
+          - "**/bundle/regal/lsp/initialize/initialize.rego"
   imports:
     unresolved-reference:
       level: error

build/capabilities.json

@@ -5712,7 +5712,7 @@
       "decl": {
         "args": [
           {
-            "description": "a duration like \"3m\"; see the [Go `time` package documentation](https://golang.org/pkg/time/#ParseDuration) for more details",
+            "description": "a duration like \"3m\"; see the [OPA `Duration Parsing` documentation](https://www.openpolicyagent.org/docs/latest/policy-reference/builtins/time#duration-parsing) for more details",
             "name": "duration",
             "type": "string"
           }
@@ -6131,6 +6131,52 @@
         "type": "function"
       }
     },
+    {
+      "name": "uri.is_valid",
+      "description": "Returns true if the input can be parsed as a URI.",
+      "decl": {
+        "args": [
+          {
+            "description": "the URI string to validate",
+            "name": "uri",
+            "type": "string"
+          }
+        ],
+        "result": {
+          "description": "true if `uri` is a valid URI, false otherwise",
+          "name": "result",
+          "type": "boolean"
+        },
+        "type": "function"
+      }
+    },
+    {
+      "name": "uri.parse",
+      "description": "Parses a URI and returns an object containing its components according to RFC 3986. Empty components are omitted. In addition to the standard components, `raw_query` is returned for use with `urlquery` builtins, and `raw_path` is returned to allow detection of path-based exploits using percent-encoded characters.",
+      "decl": {
+        "args": [
+          {
+            "description": "the URI string to parse",
+            "name": "uri",
+            "type": "string"
+          }
+        ],
+        "result": {
+          "description": "object containing URI components",
+          "dynamic": {
+            "key": {
+              "type": "string"
+            },
+            "value": {
+              "type": "string"
+            }
+          },
+          "name": "output",
+          "type": "object"
+        },
+        "type": "function"
+      }
+    },
     {
       "name": "urlquery.decode",
       "description": "Decodes a URL-encoded input string.",
@@ -6403,6 +6449,9 @@
       }
     }
   ],
+  "future_keywords": [
+    "not"
+  ],
   "features": [
     "keywords_in_refs",
     "rego_v1",

bundle/regal/ast/ast_test.rego

@@ -38,14 +38,14 @@ allow := true
 
 	blocks == [
 		[
-			{"location": {"col": 1, "end": {"col": 15, "row": 5}, "row": 3, "text": "# METADATA"}, "text": " METADATA"},
-			{"location": {"col": 1, "end": {"col": 13, "row": 4}, "row": 4, "text": "# title: foo"}, "text": " title: foo"},
-			{"location": {"col": 1, "end": {"col": 15, "row": 5}, "row": 5, "text": "# bar: invalid"}, "text": " bar: invalid"},
+			{"col": 1, "end": {"col": 15, "row": 5}, "row": 3, "text": "# METADATA"},
+			{"col": 1, "end": {"col": 13, "row": 4}, "row": 4, "text": "# title: foo"},
+			{"col": 1, "end": {"col": 15, "row": 5}, "row": 5, "text": "# bar: invalid"},
 		],
-		[{"location": {"col": 1, "end": {"col": 15, "row": 8}, "row": 8, "text": "# not metadata"}, "text": " not metadata"}],
+		[{"col": 1, "end": {"col": 15, "row": 8}, "row": 8, "text": "# not metadata"}],
 		[
-			{"location": {"col": 1, "end": {"col": 10, "row": 10}, "row": 10, "text": "# another"}, "text": " another"},
-			{"location": {"col": 1, "end": {"col": 8, "row": 11}, "row": 11, "text": "# block"}, "text": " block"},
+			{"col": 1, "end": {"col": 10, "row": 10}, "row": 10, "text": "# another"},
+			{"col": 1, "end": {"col": 8, "row": 11}, "row": 11, "text": "# block"},
 		],
 	]
 }

bundle/regal/ast/comments.rego

@@ -1,24 +1,23 @@
 package regal.ast
 
 # METADATA
-# description: all comments in the input AST with their `text` attribute base64 decoded
+# description: all comments in the input AST
 comments_decoded := [decoded |
-	some comment in _comments
+	some location in _comments
 
-	text_decoded := base64.decode(comment.text)
-	[row_str, col_str, end_row_str, end_col_str] := split(comment.location, ":")
+	[row_str, col_str, end_row_str, end_col_str] := split(location, ":")
+	row := to_number(row_str)
+	col := to_number(col_str)
+	text := substring(input.regal.file.lines[row - 1], col - 1, -1)
 
 	decoded := {
-		"text": text_decoded,
-		"location": {
-			"row": to_number(row_str),
-			"col": to_number(col_str),
-			"end": {
-				"row": to_number(end_row_str),
-				"col": to_number(end_col_str),
-			},
-			"text": $"#{text_decoded}",
+		"row": row,
+		"col": col,
+		"end": {
+			"row": to_number(end_row_str),
+			"col": to_number(end_col_str),
 		},
+		"text": text,
 	}
 ]
 
@@ -46,7 +45,7 @@ comments["metadata_attributes"] := {
 # METADATA
 # description: true if comment matches a metadata annotation attribute
 comments["annotation_match"](str) if regex.match(
-	`^\s*(scope|title|description|labels|related_resources|authors|organizations|schemas|entrypoint|custom|compile)\s*:`,
+	`^#?\s*(scope|title|description|labels|related_resources|authors|organizations|schemas|entrypoint|custom|compile)\s*:`,
 	str,
 )
 
@@ -55,13 +54,13 @@ comments["annotation_match"](str) if regex.match(
 #   map of all ignore directive comments, like ("# regal ignore:line-length")
 #   found in input AST, indexed by the row they're at
 ignore_directives[row] := rules if {
-	some comment in comments_decoded
+	some location in comments_decoded
 
-	contains(comment.text, "regal ignore:")
+	contains(location.text, "regal ignore:")
 
-	row := comment.location.row + 1
+	row := location.row + 1
 
-	rules := regex.split(`,\s*`, trim_space(regex.replace(comment.text, `^.*regal ignore:\s*(\S+)`, "$1")))
+	rules := regex.split(`,\s*`, trim_space(regex.replace(location.text, `^.*regal ignore:\s*(\S+)`, "$1")))
 }
 
 # METADATA
@@ -71,7 +70,7 @@ ignore_directives[row] := rules if {
 #   one before is considered to be part of a block.
 comment_blocks(comments_decoded) := blocks if {
 	row_partitions := [partition |
-		rows := [comment.location.row | some comment in comments_decoded]
+		rows := [location.row | some location in comments_decoded]
 		breaks := _splits(rows)
 
 		some j, k in breaks
@@ -85,12 +84,12 @@ comment_blocks(comments_decoded) := blocks if {
 	blocks := [block |
 		some row_partition in row_partitions
 		some block in {col: partition |
-			some comment in row_partition
+			some location in row_partition
 
-			col := comment.location.col # regal ignore:comprehension-term-assignment
-			partition := [partition_comment |
-				some partition_comment in row_partition
-				partition_comment.location.col == col
+			col := location.col # regal ignore:comprehension-term-assignment
+			partition := [partition_location |
+				some partition_location in row_partition
+				partition_location.col == col
 			]
 		}
 	]

bundle/regal/lsp/documentlink/documentlink.rego

@@ -14,7 +14,7 @@
 #   - input.params: schema.regal.lsp.textdocument
 package regal.lsp.documentlink
 
-import data.regal.util
+import data.regal.ast
 
 # METADATA
 # entrypoint: true
@@ -24,19 +24,15 @@ result["response"] := items
 # description: Set of links in document
 # entrypoint: true
 items contains item if {
-	module := data.workspace.parsed[input.params.textDocument.uri]
+	some location in ast.comments_decoded
 
-	some encoded in module.comments
+	contains(location.text, "regal ignore:")
 
-	text := base64.decode(encoded.text)
-	contains(text, "regal ignore:")
+	row := location.row - 1
 
-	loc := util.to_location_no_text(encoded.location)
-	row := loc.row - 1
+	some rule in regex.split(`,\s*`, trim_space(regex.replace(location.text, `^.*regal ignore:\s*(\S+)`, "$1")))
 
-	some rule in regex.split(`,\s*`, trim_space(regex.replace(text, `^.*regal ignore:\s*(\S+)`, "$1")))
-
-	col := loc.col + indexof(text, rule)
+	col := (location.col + indexof(location.text, rule)) - 1
 
 	item := {
 		"target": $"https://www.openpolicyagent.org/projects/regal/rules/{_category_for[rule]}/{rule}",

bundle/regal/lsp/documentlink/documentlink_test.rego

@@ -3,13 +3,17 @@ package regal.lsp.documentlink_test
 import data.regal.lsp.documentlink
 
 test_documentlink_ranges_in_inline_ignores if {
-	items := documentlink.items with input as {"params": {"textDocument": {"uri": "file://p.rego"}}}
-		with data.workspace.parsed["file://p.rego"] as regal.parse_module("p.rego", concat("\n", [
-			"package p",
-			"",
-			"# regal ignore:messy-rule,unresolved-reference",
-			"ignored if directives",
-		]))
+	lines := [
+		"package p",
+		"",
+		"# regal ignore:messy-rule,unresolved-reference",
+		"ignored if directives",
+	]
+
+	items := documentlink.items
+		with input.params.textDocument.uri as "file://p.rego"
+		with input.regal.file.lines as lines
+		with data.workspace.parsed["file://p.rego"] as regal.parse_module("p.rego", concat("\n", lines))
 		with data.workspace.config.rules as {
 			"style": {"messy-rule": {}},
 			"imports": {"unresolved-reference": {}},

bundle/regal/lsp/foldingrange/foldingrange.rego

@@ -38,7 +38,7 @@ ranges contains range if {
 	count(block) > 1
 
 	# note: the comment locations have already been deserialized in ast.comments.blocks
-	range := _block_range(block[0].location, regal.last(block).location, only_lines)
+	range := _block_range(block[0], regal.last(block), only_lines)
 }
 
 # METADATA

bundle/regal/lsp/foldingrange/foldingrange_test.rego

@@ -82,6 +82,7 @@ import data.h
 test_folding_comment_blocks if {
 	result := foldingrange.result
 		with input.params.textDocument.uri as "file:///p.rego"
+		with input.regal.file.lines as comment_blocks_module.regal.file.lines
 		with data.workspace.parsed["file:///p.rego"] as comment_blocks_module
 
 	result.response == {
@@ -105,6 +106,7 @@ test_folding_comment_blocks if {
 test_folding_comment_blocks_only_lines if {
 	result := foldingrange.result
 		with input.params.textDocument.uri as "file:///p.rego"
+		with input.regal.file.lines as comment_blocks_module.regal.file.lines
 		with data.client.capabilities.textDocument.foldingRange.lineFoldingOnly as true
 		with data.workspace.parsed["file:///p.rego"] as comment_blocks_module
 
@@ -236,6 +238,7 @@ test_folding_limit_honored[limit] if {
 
 	result := foldingrange.result
 		with input.params.textDocument.uri as "file:///p.rego"
+		with input.regal.file.lines as comment_blocks_module.regal.file.lines
 		with data.client.capabilities.textDocument.foldingRange.rangeLimit as limit
 		with data.workspace.parsed["file:///p.rego"] as comment_blocks_module
 

bundle/regal/lsp/hover/hover_test.rego

@@ -226,14 +226,14 @@ test_builtin_url_override if {
 test_keyword_hover if {
 	res := hover.result.response
 		with input as {
-			"params": {"position": {
-				"line": 0,
-				"character": 3,
-			}},
-			"regal": {"file": {
-				"lines": ["package foo", ""],
-				"uri": "file:///p.rego",
-			}},
+			"params": {
+				"textDocument": {"uri": "file:///p.rego"},
+				"position": {
+					"line": 0,
+					"character": 3,
+				},
+			},
+			"regal": {"file": {"lines": ["package foo", ""]}},
 		}
 		with data.workspace.parsed["file:///p.rego"] as regal.parse_module("p.rego", "package foo\n")
 

bundle/regal/lsp/hover/keywords/keywords.rego

@@ -1,8 +1,13 @@
+# METADATA
+# schemas:
+#   - input:        schema.regal.lsp.common
+#   - input.params: schema.regal.lsp.textdocument
 package regal.lsp.hover.keywords
 
+import data.regal.ast
 import data.regal.util
 
-_module := data.workspace.parsed[input.regal.file.uri]
+_module := data.workspace.parsed[input.params.textDocument.uri]
 
 # METADATA
 # description: collects keywords from input module by the line that they appear on
@@ -116,4 +121,4 @@ _in_on_row(row) := [keyword |
 	}
 ]
 
-_comment_row_index contains util.to_location_object(comment.location).row if some comment in _module.comments
+_comment_row_index contains location.row if some location in ast.comments_decoded