vendor/language: v20.2.6: mysterious index out of bounds in golang.org/x/text/internal/language

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/2336852841/?referrer=webhooks_plugin

Panic message:

*barriers.barrierError
catch.go:29: *withstack.withStack (top exception)
*assert.withAssertionFailure

Stacktrace (expand for inline code snippets):

cockroach/pkg/util/errorutil/catch.go

Lines 28 to 30 in 4da6dc3

	// get reported to Sentry.
	err = errors.HandleAsAssertionFailure(err)
	}

in pkg/util/errorutil.ShouldCatch

cockroach/pkg/sql/opt/optbuilder/builder.go

Lines 164 to 166 in 4da6dc3

	// manipulate locks.
	if ok, e := errorutil.ShouldCatch(r); ok {
	err = e

in pkg/sql/opt/optbuilder.(*Builder).Build.func1
/usr/local/go/src/runtime/panic.go#L678-L680 in runtime.gopanic
/usr/local/go/src/runtime/panic.go#L102-L104 in runtime.goPanicSliceB
https://github.com/cockroachdb/cockroach/blob/4da6dc378d563472d59166167ce760756e76f893/vendor/golang.org/x/text/internal/language/parse.go#L141-L143 in golang.org/x/text/internal/language.(*scanner).resizeRange
https://github.com/cockroachdb/cockroach/blob/4da6dc378d563472d59166167ce760756e76f893/vendor/golang.org/x/text/internal/language/parse.go#L150-L152 in golang.org/x/text/internal/language.(*scanner).replace
https://github.com/cockroachdb/cockroach/blob/4da6dc378d563472d59166167ce760756e76f893/vendor/golang.org/x/text/internal/language/parse.go#L295-L297 in golang.org/x/text/internal/language.parseTag
https://github.com/cockroachdb/cockroach/blob/4da6dc378d563472d59166167ce760756e76f893/vendor/golang.org/x/text/internal/language/parse.go#L551-L553 in golang.org/x/text/internal/language.parseExtension
https://github.com/cockroachdb/cockroach/blob/4da6dc378d563472d59166167ce760756e76f893/vendor/golang.org/x/text/internal/language/parse.go#L450-L452 in golang.org/x/text/internal/language.parseExtensions
https://github.com/cockroachdb/cockroach/blob/4da6dc378d563472d59166167ce760756e76f893/vendor/golang.org/x/text/internal/language/parse.go#L267-L269 in golang.org/x/text/internal/language.parse
https://github.com/cockroachdb/cockroach/blob/4da6dc378d563472d59166167ce760756e76f893/vendor/golang.org/x/text/internal/language/parse.go#L249-L251 in golang.org/x/text/internal/language.Parse
https://github.com/cockroachdb/cockroach/blob/4da6dc378d563472d59166167ce760756e76f893/vendor/golang.org/x/text/language/parse.go#L45-L47 in golang.org/x/text/language.CanonType.Parse
https://github.com/cockroachdb/cockroach/blob/4da6dc378d563472d59166167ce760756e76f893/vendor/golang.org/x/text/language/parse.go#L33-L35 in golang.org/x/text/language.Parse

cockroach/pkg/sql/sem/tree/type_check.go

Lines 569 to 571 in 4da6dc3

	) (TypedExpr, error) {
	_, err := language.Parse(expr.Locale)
	if err != nil {

in pkg/sql/sem/tree.(*CollateExpr).TypeCheck

cockroach/pkg/sql/sem/tree/type_check.go

Lines 253 to 255 in 4da6dc3

	return expr.TypeCheck(ctx, semaCtx, desired)
	}

in pkg/sql/sem/tree.TypeCheck

cockroach/pkg/sql/opt/optbuilder/scope.go

Lines 438 to 440 in 4da6dc3

	expr = s.walkExprTree(expr)
	texpr, err := tree.TypeCheck(s.builder.ctx, expr, s.builder.semaCtx, desired)
	if err != nil {

in pkg/sql/opt/optbuilder.(*scope).resolveType

cockroach/pkg/sql/opt/optbuilder/project.go

Lines 153 to 155 in 4da6dc3

	texpr = inScope.resolveType(e.Expr, desired)
	}

in pkg/sql/opt/optbuilder.(*Builder).analyzeSelectList

cockroach/pkg/sql/opt/optbuilder/project.go

Lines 87 to 89 in 4da6dc3

	b.analyzeSelectList(selects, desiredTypes, inScope, outScope)
	}

in pkg/sql/opt/optbuilder.(*Builder).analyzeProjectionList

cockroach/pkg/sql/opt/optbuilder/select.go

Lines 1061 to 1063 in 4da6dc3

	// aggInScope and aggOutScope.
	b.analyzeProjectionList(sel.Exprs, desiredTypes, fromScope, projectionsScope)

in pkg/sql/opt/optbuilder.(*Builder).buildSelectClause

cockroach/pkg/sql/opt/optbuilder/select.go

Lines 998 to 1000 in 4da6dc3

	case *tree.SelectClause:
	outScope = b.buildSelectClause(t, orderBy, locking, desiredTypes, inScope)

in pkg/sql/opt/optbuilder.(*Builder).buildSelectStmtWithoutParens

cockroach/pkg/sql/opt/optbuilder/select.go

Lines 971 to 973 in 4da6dc3

	return b.processWiths(with, inScope, func(inScope *scope) *scope {
	return b.buildSelectStmtWithoutParens(
	wrapped, orderBy, limit, locking, desiredTypes, inScope,

in pkg/sql/opt/optbuilder.(*Builder).buildSelect.func1

cockroach/pkg/sql/opt/optbuilder/with.go

Lines 28 to 30 in 4da6dc3

	inScope.atRoot = false
	outScope := buildStmt(inScope)
	inScope.atRoot = prevAtRoot

in pkg/sql/opt/optbuilder.(*Builder).processWiths

cockroach/pkg/sql/opt/optbuilder/select.go

Lines 970 to 972 in 4da6dc3

	return b.processWiths(with, inScope, func(inScope *scope) *scope {
	return b.buildSelectStmtWithoutParens(

in pkg/sql/opt/optbuilder.(*Builder).buildSelect

cockroach/pkg/sql/opt/optbuilder/builder.go

Lines 264 to 266 in 4da6dc3

	case *tree.Select:
	return b.buildSelect(stmt, noRowLocking, desiredTypes, inScope)

in pkg/sql/opt/optbuilder.(*Builder).buildStmt

cockroach/pkg/sql/opt/optbuilder/builder.go

Lines 228 to 230 in 4da6dc3

	return b.buildStmt(stmt, desiredTypes, inScope)
	}

in pkg/sql/opt/optbuilder.(*Builder).buildStmtAtRoot

cockroach/pkg/sql/opt/optbuilder/builder.go

Lines 199 to 201 in 4da6dc3

	// and physical properties.
	outScope := b.buildStmtAtRoot(b.stmt, nil /* desiredTypes */, b.allocScope())

in pkg/sql/opt/optbuilder.(*Builder).Build

cockroach/pkg/sql/plan_opt.go

Lines 511 to 513 in 4da6dc3

	bld := optbuilder.New(ctx, &p.semaCtx, p.EvalContext(), &opc.catalog, f, opc.p.stmt.AST)
	if err := bld.Build(); err != nil {
	return nil, err

in pkg/sql.(*optPlanningCtx).buildExecMemo

cockroach/pkg/sql/plan_opt.go

Lines 194 to 196 in 4da6dc3

	execMemo, err := opc.buildExecMemo(ctx)
	if err != nil {

in pkg/sql.(*planner).makeOptimizerPlan

cockroach/pkg/sql/conn_executor_exec.go

Lines 900 to 902 in 4da6dc3

	if err := planner.makeOptimizerPlan(ctx); err != nil {
	log.VEventf(ctx, 1, "optimizer plan failed: %v", err)

in pkg/sql.(*connExecutor).makeExecPlan

cockroach/pkg/sql/conn_executor_exec.go

Lines 779 to 781 in 4da6dc3

	// between here and there needs to happen even if there's an error.
	err := ex.makeExecPlan(ctx, planner)
	// We'll be closing the plan manually below after execution; this

in pkg/sql.(*connExecutor).dispatchToExecutionEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 638 to 640 in 4da6dc3

	p.autoCommit = os.ImplicitTxn.Get() && !ex.server.cfg.TestingKnobs.DisableAutoCommit
	if err := ex.dispatchToExecutionEngine(ctx, p, res); err != nil {
	return nil, nil, err

in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 113 to 115 in 4da6dc3

	} else {
	ev, payload, err = ex.execStmtInOpenState(ctx, stmt, res, pinfo)
	}

in pkg/sql.(*connExecutor).execStmt

pkg/util/errorutil/catch.go in pkg/util/errorutil.ShouldCatch at line 29
pkg/sql/opt/optbuilder/builder.go in pkg/sql/opt/optbuilder.(*Builder).Build.func1 at line 165
/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 679
/usr/local/go/src/runtime/panic.go in runtime.goPanicSliceB at line 103
vendor/golang.org/x/text/internal/language/parse.go in golang.org/x/text/internal/language.(*scanner).resizeRange at line 142
vendor/golang.org/x/text/internal/language/parse.go in golang.org/x/text/internal/language.(*scanner).replace at line 151
vendor/golang.org/x/text/internal/language/parse.go in golang.org/x/text/internal/language.parseTag at line 296
vendor/golang.org/x/text/internal/language/parse.go in golang.org/x/text/internal/language.parseExtension at line 552
vendor/golang.org/x/text/internal/language/parse.go in golang.org/x/text/internal/language.parseExtensions at line 451
vendor/golang.org/x/text/internal/language/parse.go in golang.org/x/text/internal/language.parse at line 268
vendor/golang.org/x/text/internal/language/parse.go in golang.org/x/text/internal/language.Parse at line 250
vendor/golang.org/x/text/language/parse.go in golang.org/x/text/language.CanonType.Parse at line 46
vendor/golang.org/x/text/language/parse.go in golang.org/x/text/language.Parse at line 34
pkg/sql/sem/tree/type_check.go in pkg/sql/sem/tree.(*CollateExpr).TypeCheck at line 570
pkg/sql/sem/tree/type_check.go in pkg/sql/sem/tree.TypeCheck at line 254
pkg/sql/opt/optbuilder/scope.go in pkg/sql/opt/optbuilder.(*scope).resolveType at line 439
pkg/sql/opt/optbuilder/project.go in pkg/sql/opt/optbuilder.(*Builder).analyzeSelectList at line 154
pkg/sql/opt/optbuilder/project.go in pkg/sql/opt/optbuilder.(*Builder).analyzeProjectionList at line 88
pkg/sql/opt/optbuilder/select.go in pkg/sql/opt/optbuilder.(*Builder).buildSelectClause at line 1062
pkg/sql/opt/optbuilder/select.go in pkg/sql/opt/optbuilder.(*Builder).buildSelectStmtWithoutParens at line 999
pkg/sql/opt/optbuilder/select.go in pkg/sql/opt/optbuilder.(*Builder).buildSelect.func1 at line 972
pkg/sql/opt/optbuilder/with.go in pkg/sql/opt/optbuilder.(*Builder).processWiths at line 29
pkg/sql/opt/optbuilder/select.go in pkg/sql/opt/optbuilder.(*Builder).buildSelect at line 971
pkg/sql/opt/optbuilder/builder.go in pkg/sql/opt/optbuilder.(*Builder).buildStmt at line 265
pkg/sql/opt/optbuilder/builder.go in pkg/sql/opt/optbuilder.(*Builder).buildStmtAtRoot at line 229
pkg/sql/opt/optbuilder/builder.go in pkg/sql/opt/optbuilder.(*Builder).Build at line 200
pkg/sql/plan_opt.go in pkg/sql.(*optPlanningCtx).buildExecMemo at line 512
pkg/sql/plan_opt.go in pkg/sql.(*planner).makeOptimizerPlan at line 195
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).makeExecPlan at line 901
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 780
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 639
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 114

Tag	Value
Cockroach Release	v20.2.6
Cockroach SHA:	4da6dc3
Platform	darwin amd64
Distribution	CCL
Environment	v20.2.6
Command	demo
Go Version	``
# of CPUs
# of Goroutines

[rytaft]

cc @bdarnell and @knz this looks related to the conversation about xtext upgrades that you have been having. Any thoughts?

[RaduBerinde]

This seems like a Go issue that was fixed: golang/text@4482a91#diff-6c5a66b686c190b74f1faad2ed43c7eb3824eaa700d8e9fb81d818084ec1115c
golang/go#42536

[bdarnell]

Yeah, this looks like the one that @RaduBerinde found. But there's another panic in this area fixed by golang/text@e3aa4ad, which we don't have yet. The panic gets caught so it doesn't crash the cockroach process (Do those still report to sentry?)

[RaduBerinde]

Usually caught panics get wrapped in assertion errors which do get reported to sentry even if they don't crash the server.

[knz]

The panic gets caught so it doesn't crash the cockroach process

Where do you see this?

Usually caught panics get wrapped in assertion errors which do get reported to sentry even if they don't crash the server.

I agree that caught panics get reported to sentry, but only if they were converted to a crdb-style assertion error. If the conversion of panic to error occurs with e.g. fmt.Errorf we don't see anything. See my question above.

[RaduBerinde]

The stacktrace shows it is caught at the optbuilder boundary.

[knz]

so do we want to fold this issue into #63559?

[bdarnell]

The panic gets caught so it doesn't crash the cockroach process

Where do you see this?

Using a test case from golang/text@e3aa4ad as a collation:

# Server version: CockroachDB CCL v20.2.6 (x86_64-apple-darwin14, built 2021/03/15 18:01:29, go1.13.14) (same version as client)
# Cluster ID: bd4ce26c-99ae-4414-baf7-d9fafc7f4730
#
# Enter \? for a brief introduction.
#
[email protected]:58094/defaultdb> select 'a' collate "es-u-000-00-v-00";
*
* ERROR: [n1,client=127.0.0.1:58095,hostssl,user=root] Queued as error 5b4d94d5a10e48a7b6e3e5f34f13242a
*
ERROR: internal error: runtime error: index out of range [17] with length 14
SQLSTATE: XX000
DETAIL: stack trace:
/go/src/github.com/cockroachdb/cockroach/pkg/util/errorutil/catch.go:29: ShouldCatch()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/builder.go:165: func1()
/usr/local/go/src/runtime/panic.go:679: gopanic()
/usr/local/go/src/runtime/panic.go:75: goPanicIndex()
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/text/internal/language/language.go:456: findTypeForKey()
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/text/internal/language/language.go:307: TypeForKey()
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/text/internal/language/compact/language.go:38: Make()
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/text/language/language.go:25: makeTag()
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/text/language/parse.go:48: Parse()
/go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/text/language/parse.go:34: Parse()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/type_check.go:570: TypeCheck()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sem/tree/type_check.go:254: TypeCheck()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/scope.go:439: resolveType()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/project.go:154: analyzeSelectList()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/project.go:88: analyzeProjectionList()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/select.go:1062: buildSelectClause()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/select.go:999: buildSelectStmtWithoutParens()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/select.go:972: func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/with.go:29: processWiths()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/select.go:971: buildSelect()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/builder.go:265: buildStmt()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/builder.go:229: buildStmtAtRoot()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/builder.go:200: Build()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:512: buildExecMemo()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:195: makeOptimizerPlan()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:901: makeExecPlan()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:780: dispatchToExecutionEngine()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:639: execStmtInOpenState()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:114: execStmt()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1465: func1()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1467: execCmd()
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1391: run()

HINT: You have encountered an unexpected error.

Please check the public issue tracker to check whether this problem is
already tracked. If you cannot find it there, please report the error
with details by creating a new issue.

If you would rather not post publicly, please contact us directly
using the support form.

We appreciate your feedback.

so do we want to fold this issue into #63559?

Yes, I think so.

[knz]

cool thanks