all: update to protobuf 27.0-rc1 and regenerate protos

This change required some changes to the editions default handling code
because the descriptor.proto changed upstream [2]. The defaults are no
longer one feature set but are split into overridable and
not-overridable features which have to be merged.

I had to do bootstraping in 4 phases but the results should be correct:

1. generate everything depending on descriptor.proto
2. generate new defaults binary proto
3. adjust all code that works with defaults (*/edition.go files)
4. generate everything else

The was required because 1. is a prerequisite for 3. while 2. and 3. are
a prerequisite for 4. (2. and 3. can probably be done in parallel).

The new release also introduced new conformance tests. The go
implementation is not yet conformant and the tests will be fixed in a
follow up change because they require changes to the protojson and
protoext encoders.

[1] protocolbuffers/protobuf@e5502c7

Change-Id: Iddf248f6582a0402ab31256f6e64755d870ed82c
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/582635
Auto-Submit: Lasse Folger <lassefolger@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Nicolas Hillegeer <aktau@google.com>

Author: lfolger

integration_test.go

@@ -34,7 +34,7 @@ var (
 	regenerate   = flag.Bool("regenerate", false, "regenerate files")
 	buildRelease = flag.Bool("buildRelease", false, "build release binaries")
 
-	protobufVersion = "26.0-rc2"
+	protobufVersion = "27.0-rc1"
 
 	golangVersions = func() []string {
 		// Version policy: same version as is in the x/ repos' go.mod.
@@ -500,6 +500,12 @@ func mustHaveCopyrightHeader(t *testing.T, files []string) {
 	var bad []string
 File:
 	for _, file := range files {
+		if strings.HasSuffix(file, "internal/testprotos/conformance/editions/test_messages_edition2023.pb.go") {
+			// TODO(lassefolger) the underlying proto file is checked into
+			// the protobuf repo without a copyright header. Fix is pending but
+			// might require a release.
+			continue
+		}
 		b, err := os.ReadFile(file)
 		if err != nil {
 			t.Fatal(err)

internal/cmd/generate-protos/main.go

@@ -111,9 +111,9 @@ func generateEditionsDefaults() {
 	maxEdition := strings.TrimPrefix(fmt.Sprint(editionssupport.Maximum), "EDITION_")
 	cmd := exec.Command(
 		"protoc",
-		"--experimental_edition_defaults_out", dest,
-		"--experimental_edition_defaults_minimum", minEdition,
-		"--experimental_edition_defaults_maximum", maxEdition,
+		"--edition_defaults_out", dest,
+		"--edition_defaults_minimum", minEdition,
+		"--edition_defaults_maximum", maxEdition,
 		"-I"+filepath.Join(protoRoot, "src"), "-I"+filepath.Join(repoRoot, "src"),
 		srcDescriptorProto, srcGoFeatures,
 	)
@@ -230,8 +230,9 @@ func generateRemoteProtos() {
 		{"", "conformance/conformance.proto", "google.golang.org/protobuf/internal/testprotos/conformance;conformance"},
 		{"src", "google/protobuf/test_messages_proto2.proto", "google.golang.org/protobuf/internal/testprotos/conformance;conformance"},
 		{"src", "google/protobuf/test_messages_proto3.proto", "google.golang.org/protobuf/internal/testprotos/conformance;conformance"},
-		{"src", "google/protobuf/editions/golden/test_messages_proto2_editions.proto", "google.golang.org/protobuf/internal/testprotos/conformance/editions;editions"},
-		{"src", "google/protobuf/editions/golden/test_messages_proto3_editions.proto", "google.golang.org/protobuf/internal/testprotos/conformance/editions;editions"},
+		{"src", "editions/golden/test_messages_proto2_editions.proto", "google.golang.org/protobuf/internal/testprotos/conformance/editionsmigration;editions"},
+		{"src", "editions/golden/test_messages_proto3_editions.proto", "google.golang.org/protobuf/internal/testprotos/conformance/editionsmigration;editions"},
+		{"", "conformance/test_protos/test_messages_edition2023.proto", "google.golang.org/protobuf/internal/testprotos/conformance/editions;editions"},
 
 		// Benchmark protos.
 		// TODO: The protobuf repo no longer includes benchmarks.
@@ -267,7 +268,7 @@ func generateRemoteProtos() {
 		}
 	}
 	for _, f := range files {
-		protoc("-I"+filepath.Join(protoRoot, f.prefix), "--go_out="+opts+":"+tmpDir, f.path)
+		protoc("-I"+protoRoot, "-I"+filepath.Join(protoRoot, f.prefix), "--go_out="+opts+":"+tmpDir, f.path)
 	}
 
 	syncOutput(repoRoot, tmpDir)

internal/conformance/conformance_test.go

@@ -7,6 +7,7 @@ package conformance_test
 import (
 	"encoding/binary"
 	"flag"
+	"fmt"
 	"io"
 	"log"
 	"os"
@@ -20,6 +21,7 @@ import (
 
 	pb "google.golang.org/protobuf/internal/testprotos/conformance"
 	epb "google.golang.org/protobuf/internal/testprotos/conformance/editions"
+	empb "google.golang.org/protobuf/internal/testprotos/conformance/editionsmigration"
 )
 
 func init() {
@@ -102,10 +104,14 @@ func handle(req *pb.ConformanceRequest) (res *pb.ConformanceResponse) {
 		msg = &pb.TestAllTypesProto3{}
 	case "protobuf_test_messages.proto2.TestAllTypesProto2":
 		msg = &pb.TestAllTypesProto2{}
+	case "protobuf_test_messages.editions.TestAllTypesEdition2023":
+		msg = &epb.TestAllTypesEdition2023{}
 	case "protobuf_test_messages.editions.proto3.TestAllTypesProto3":
-		msg = &epb.TestAllTypesProto3{}
+		msg = &empb.TestAllTypesProto3{}
 	case "protobuf_test_messages.editions.proto2.TestAllTypesProto2":
-		msg = &epb.TestAllTypesProto2{}
+		msg = &empb.TestAllTypesProto2{}
+	default:
+		panic(fmt.Sprintf("unknown message type: %s", req.GetMessageType()))
 	}
 
 	// Unmarshal the test message.

internal/conformance/failing_tests.txt

@@ -0,0 +1 @@
+

internal/conformance/failing_tests_text_format.txt

@@ -8,3 +8,13 @@ Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairB
 Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
 Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
 Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
+Required.Editions.TextFormatInput.DelimitedFieldLowercased.ProtobufOutput
+Required.Editions.TextFormatInput.DelimitedFieldLowercased.TextFormatOutput
+Required.Editions_Proto2.TextFormatInput.GroupFieldLowercased.ProtobufOutput
+Required.Editions_Proto2.TextFormatInput.GroupFieldLowercased.TextFormatOutput
+Required.Editions_Proto2.TextFormatInput.GroupFieldLowercasedMultiWord.ProtobufOutput
+Required.Editions_Proto2.TextFormatInput.GroupFieldLowercasedMultiWord.TextFormatOutput
+Required.Proto2.TextFormatInput.GroupFieldLowercased.ProtobufOutput
+Required.Proto2.TextFormatInput.GroupFieldLowercased.TextFormatOutput
+Required.Proto2.TextFormatInput.GroupFieldLowercasedMultiWord.ProtobufOutput
+Required.Proto2.TextFormatInput.GroupFieldLowercasedMultiWord.TextFormatOutput

internal/editiondefaults/editions_defaults.binpb

@@ -108,7 +108,9 @@ func unmarshalEditionDefault(b []byte) {
 			v, m := protowire.ConsumeBytes(b)
 			b = b[m:]
 			switch num {
-			case genid.FeatureSetDefaults_FeatureSetEditionDefault_Features_field_number:
+			case genid.FeatureSetDefaults_FeatureSetEditionDefault_FixedFeatures_field_number:
+				fs = unmarshalFeatureSet(v, fs)
+			case genid.FeatureSetDefaults_FeatureSetEditionDefault_OverridableFeatures_field_number:
 				fs = unmarshalFeatureSet(v, fs)
 			}
 		}