gopls/internal/golang: don't panic when findKeyword fails

As the existing comment attests, this can happen in the wild.
Just skip it and move on.

+ a test

Fixes golang/go#68205

Change-Id: I3227b0ce7ffacf3c8b4bbf2180a10e218bf87aa3
Reviewed-on: https://go-review.googlesource.com/c/tools/+/595117
Reviewed-by: Robert Findley <[email protected]>
Auto-Submit: Alan Donovan <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: adonovan

gopls/internal/golang/semtok.go

@@ -259,13 +259,12 @@ func (tv *tokenVisitor) comment(c *ast.Comment, importByName map[string]*types.P
 
 // token emits a token of the specified extent and semantics.
 func (tv *tokenVisitor) token(start token.Pos, length int, typ semtok.TokenType, modifiers []string) {
-	if length <= 0 {
-		return // vscode doesn't like 0-length Tokens
-	}
 	if !start.IsValid() {
-		// This is not worth reporting. TODO(pjw): does it still happen?
 		return
 	}
+	if length <= 0 {
+		return // vscode doesn't like 0-length Tokens
+	}
 	end := start + token.Pos(length)
 	if start >= tv.end || end <= tv.start {
 		return
@@ -849,19 +848,21 @@ func (tv *tokenVisitor) multiline(start, end token.Pos, tok semtok.TokenType) {
 }
 
 // findKeyword returns the position of a keyword by searching within
-// the specified range, for when its cannot be exactly known from the AST.
+// the specified range, for when it cannot be exactly known from the AST.
+// It returns NoPos if the keyword was not present in the source due to parse error.
 func (tv *tokenVisitor) findKeyword(keyword string, start, end token.Pos) token.Pos {
 	// TODO(adonovan): use safetoken.Offset.
 	offset := int(start) - tv.pgf.Tok.Base()
 	last := int(end) - tv.pgf.Tok.Base()
 	buf := tv.pgf.Src
 	idx := bytes.Index(buf[offset:last], []byte(keyword))
-	if idx != -1 {
-		return start + token.Pos(idx)
+	if idx < 0 {
+		// Ill-formed code may form syntax trees without their usual tokens.
+		// For example, "type _ <-<-chan int" parses as <-chan (chan int),
+		// with two nested ChanTypes but only one chan keyword.
+		return token.NoPos
 	}
-	//(in unparsable programs: type _ <-<-chan int)
-	tv.errorf("not found:%s %v", keyword, safetoken.StartPosition(tv.fset, start))
-	return token.NoPos
+	return start + token.Pos(idx)
 }
 
 func (tv *tokenVisitor) importSpec(spec *ast.ImportSpec) {

gopls/internal/test/marker/testdata/token/illformed.txt

@@ -0,0 +1,15 @@
+This test checks semanticTokens on ill-formed code.
+(Regression test for #68205.)
+
+-- settings.json --
+{
+	"semanticTokens": true
+}
+
+-- flags --
+-ignore_extra_diags
+
+-- a.go --
+package p
+
+type _ <-<-chan int //@ token("<-", "operator", ""), token("chan", "keyword", "")