Skip to content

Commit 1001acf

Browse files
shollymanshollyman
authored
feat(bigquery): add external dataset reference (#8545)
Adds preview functionality for manipulating external dataset references.
1 parent 7f62291 commit 1001acf

File tree

4 files changed

+123
-100
lines changed

4 files changed

+123
-100
lines changed

‎bigquery/dataset.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,9 @@ type DatasetMetadata struct {
5858
// More information: https://cloud.google.com/bigquery/docs/reference/standard-sql/collation-concepts
5959
DefaultCollation string
6060

61+
// For externally defined datasets, contains information about the configuration.
62+
ExternalDatasetReference *ExternalDatasetReference
63+
6164
// MaxTimeTravel represents the number of hours for the max time travel for all tables
6265
// in the dataset. Durations are rounded towards zero for the nearest hourly value.
6366
MaxTimeTravel time.Duration
@@ -135,6 +138,9 @@ type DatasetMetadataToUpdate struct {
135138
// created in the dataset.
136139
DefaultCollation optional.String
137140

141+
// For externally defined datasets, contains information about the configuration.
142+
ExternalDatasetReference *ExternalDatasetReference
143+
138144
// MaxTimeTravel represents the number of hours for the max time travel for all tables
139145
// in the dataset. Durations are rounded towards zero for the nearest hourly value.
140146
MaxTimeTravel optional.Duration
@@ -239,6 +245,9 @@ func (dm *DatasetMetadata) toBQ() (*bq.Dataset, error) {
239245
if dm.DefaultEncryptionConfig != nil {
240246
ds.DefaultEncryptionConfiguration = dm.DefaultEncryptionConfig.toBQ()
241247
}
248+
if dm.ExternalDatasetReference != nil {
249+
ds.ExternalDatasetReference = dm.ExternalDatasetReference.toBQ()
250+
}
242251
return ds, nil
243252
}
244253

@@ -304,6 +313,7 @@ func bqToDatasetMetadata(d *bq.Dataset, c *Client) (*DatasetMetadata, error) {
304313
DefaultTableExpiration: time.Duration(d.DefaultTableExpirationMs) * time.Millisecond,
305314
DefaultPartitionExpiration: time.Duration(d.DefaultPartitionExpirationMs) * time.Millisecond,
306315
DefaultCollation: d.DefaultCollation,
316+
ExternalDatasetReference: bqToExternalDatasetReference(d.ExternalDatasetReference),
307317
MaxTimeTravel: time.Duration(d.MaxTimeTravelHours) * time.Hour,
308318
StorageBillingModel: d.StorageBillingModel,
309319
DefaultEncryptionConfig: bqToEncryptionConfig(d.DefaultEncryptionConfiguration),
@@ -395,6 +405,10 @@ func (dm *DatasetMetadataToUpdate) toBQ() (*bq.Dataset, error) {
395405
ds.DefaultCollation = optional.ToString(dm.DefaultCollation)
396406
forceSend("DefaultCollation")
397407
}
408+
if dm.ExternalDatasetReference != nil {
409+
ds.ExternalDatasetReference = dm.ExternalDatasetReference.toBQ()
410+
forceSend("ExternalDatasetReference")
411+
}
398412
if dm.MaxTimeTravel != nil {
399413
dur := optional.ToDuration(dm.MaxTimeTravel)
400414
if dur == 0 {
@@ -936,3 +950,33 @@ func bqToDatasetAccessEntry(entry *bq.DatasetAccessEntry, c *Client) *DatasetAcc
936950
TargetTypes: entry.TargetTypes,
937951
}
938952
}
953+
954+
// ExternalDatasetReference provides information about external dataset metadata.
955+
type ExternalDatasetReference struct {
956+
//The connection id that is used to access the external_source.
957+
// Format: projects/{project_id}/locations/{location_id}/connections/{connection_id}
958+
Connection string
959+
960+
// External source that backs this dataset.
961+
ExternalSource string
962+
}
963+
964+
func bqToExternalDatasetReference(bq *bq.ExternalDatasetReference) *ExternalDatasetReference {
965+
if bq == nil {
966+
return nil
967+
}
968+
return &ExternalDatasetReference{
969+
Connection: bq.Connection,
970+
ExternalSource: bq.ExternalSource,
971+
}
972+
}
973+
974+
func (edr *ExternalDatasetReference) toBQ() *bq.ExternalDatasetReference {
975+
if edr == nil {
976+
return nil
977+
}
978+
return &bq.ExternalDatasetReference{
979+
Connection: edr.Connection,
980+
ExternalSource: edr.ExternalSource,
981+
}
982+
}

‎bigquery/dataset_test.go‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,10 @@ func TestDatasetToBQ(t *testing.T) {
327327
DefaultEncryptionConfig: &EncryptionConfig{
328328
KMSKeyName: "some_key",
329329
},
330+
ExternalDatasetReference: &ExternalDatasetReference{
331+
Connection: "conn",
332+
ExternalSource: "external_src",
333+
},
330334
Location: "EU",
331335
Labels: map[string]string{"x": "y"},
332336
Access: []*AccessEntry{
@@ -348,6 +352,10 @@ func TestDatasetToBQ(t *testing.T) {
348352
DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
349353
KmsKeyName: "some_key",
350354
},
355+
ExternalDatasetReference: &bq.ExternalDatasetReference{
356+
Connection: "conn",
357+
ExternalSource: "external_src",
358+
},
351359
Location: "EU",
352360
Labels: map[string]string{"x": "y"},
353361
Access: []*bq.DatasetAccess{
@@ -404,6 +412,10 @@ func TestBQToDatasetMetadata(t *testing.T) {
404412
DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
405413
KmsKeyName: "some_key",
406414
},
415+
ExternalDatasetReference: &bq.ExternalDatasetReference{
416+
Connection: "conn",
417+
ExternalSource: "external_src",
418+
},
407419
Location: "EU",
408420
Labels: map[string]string{"x": "y"},
409421
Access: []*bq.DatasetAccess{
@@ -436,6 +448,10 @@ func TestBQToDatasetMetadata(t *testing.T) {
436448
DefaultEncryptionConfig: &EncryptionConfig{
437449
KMSKeyName: "some_key",
438450
},
451+
ExternalDatasetReference: &ExternalDatasetReference{
452+
Connection: "conn",
453+
ExternalSource: "external_src",
454+
},
439455
StorageBillingModel: LogicalStorageBillingModel,
440456
Location: "EU",
441457
Labels: map[string]string{"x": "y"},
@@ -476,6 +492,10 @@ func TestDatasetMetadataToUpdateToBQ(t *testing.T) {
476492
DefaultEncryptionConfig: &EncryptionConfig{
477493
KMSKeyName: "some_key",
478494
},
495+
ExternalDatasetReference: &ExternalDatasetReference{
496+
Connection: "conn",
497+
ExternalSource: "external_src",
498+
},
479499
}
480500
dm.SetLabel("label", "value")
481501
dm.DeleteLabel("del")
@@ -495,8 +515,12 @@ func TestDatasetMetadataToUpdateToBQ(t *testing.T) {
495515
KmsKeyName: "some_key",
496516
ForceSendFields: []string{"KmsKeyName"},
497517
},
518+
ExternalDatasetReference: &bq.ExternalDatasetReference{
519+
Connection: "conn",
520+
ExternalSource: "external_src",
521+
},
498522
Labels: map[string]string{"label": "value"},
499-
ForceSendFields: []string{"Description", "FriendlyName", "StorageBillingModel"},
523+
ForceSendFields: []string{"Description", "FriendlyName", "ExternalDatasetReference", "StorageBillingModel"},
500524
NullFields: []string{"Labels.del"},
501525
}
502526
if diff := testutil.Diff(got, want); diff != "" {

‎bigquery/go.mod‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,29 +3,29 @@ module cloud.google.com/go/bigquery
33
go 1.19
44

55
require (
6-
cloud.google.com/go v0.110.2
7-
cloud.google.com/go/datacatalog v1.14.0
8-
cloud.google.com/go/iam v1.1.0
6+
cloud.google.com/go v0.110.6
7+
cloud.google.com/go/datacatalog v1.16.0
8+
cloud.google.com/go/iam v1.1.1
99
cloud.google.com/go/storage v1.30.1
1010
github.com/apache/arrow/go/v12 v12.0.0
1111
github.com/google/go-cmp v0.5.9
1212
github.com/google/uuid v1.3.0
1313
github.com/googleapis/gax-go/v2 v2.12.0
1414
go.opencensus.io v0.24.0
15-
golang.org/x/sync v0.2.0
15+
golang.org/x/sync v0.3.0
1616
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
17-
google.golang.org/api v0.128.0
18-
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc
19-
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc
20-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc
21-
google.golang.org/grpc v1.56.1
17+
google.golang.org/api v0.139.0
18+
google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5
19+
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5
20+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d
21+
google.golang.org/grpc v1.57.0
2222
google.golang.org/protobuf v1.31.0
2323
)
2424

2525
require (
26-
cloud.google.com/go/compute v1.19.3 // indirect
26+
cloud.google.com/go/compute v1.23.0 // indirect
2727
cloud.google.com/go/compute/metadata v0.2.3 // indirect
28-
cloud.google.com/go/longrunning v0.4.2 // indirect
28+
cloud.google.com/go/longrunning v0.5.1 // indirect
2929
github.com/andybalholm/brotli v1.0.4 // indirect
3030
github.com/apache/thrift v0.16.0 // indirect
3131
github.com/goccy/go-json v0.9.11 // indirect
@@ -34,8 +34,8 @@ require (
3434
github.com/golang/snappy v0.0.4 // indirect
3535
github.com/google/flatbuffers v2.0.8+incompatible // indirect
3636
github.com/google/martian/v3 v3.3.2 // indirect
37-
github.com/google/s2a-go v0.1.4 // indirect
38-
github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
37+
github.com/google/s2a-go v0.1.7 // indirect
38+
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
3939
github.com/klauspost/asmfmt v1.3.2 // indirect
4040
github.com/klauspost/compress v1.15.9 // indirect
4141
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
@@ -44,12 +44,12 @@ require (
4444
github.com/pierrec/lz4/v4 v4.1.15 // indirect
4545
github.com/stretchr/testify v1.8.2 // indirect
4646
github.com/zeebo/xxh3 v1.0.2 // indirect
47-
golang.org/x/crypto v0.9.0 // indirect
47+
golang.org/x/crypto v0.12.0 // indirect
4848
golang.org/x/mod v0.10.0 // indirect
49-
golang.org/x/net v0.10.0 // indirect
50-
golang.org/x/oauth2 v0.8.0 // indirect
51-
golang.org/x/sys v0.8.0 // indirect
52-
golang.org/x/text v0.9.0 // indirect
49+
golang.org/x/net v0.14.0 // indirect
50+
golang.org/x/oauth2 v0.11.0 // indirect
51+
golang.org/x/sys v0.11.0 // indirect
52+
golang.org/x/text v0.12.0 // indirect
5353
golang.org/x/tools v0.9.1 // indirect
5454
google.golang.org/appengine v1.6.7 // indirect
5555
)

0 commit comments

Comments
 (0)