Skip to content

GODRIVER-3123 [master] QE Range Protocol V2 (#1723) #1763

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 20, 2024
Merged

GODRIVER-3123 [master] QE Range Protocol V2 (#1723) #1763

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1401d90
GODRIVER-3123 QE Range Protocol V2 (#1723)
qingyang-hu Aug 19, 2024
22caf58
address build errors
blink1073 Aug 20, 2024
fe3d04d
syntax
blink1073 Aug 20, 2024
131de83
syntax
blink1073 Aug 20, 2024
36c9982
syntax
blink1073 Aug 20, 2024
7c6b6f9
syntax
blink1073 Aug 20, 2024
d0cd04d
syntax
blink1073 Aug 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions etc/install-libmongocrypt.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# This script installs libmongocrypt into an "install" directory.
set -eux

LIBMONGOCRYPT_TAG="1.8.2"
LIBMONGOCRYPT_TAG="1.11.0"

# Install libmongocrypt based on OS.
if [ "Windows_NT" = "${OS:-}" ]; then
Expand All @@ -13,11 +13,11 @@ if [ "Windows_NT" = "${OS:-}" ]; then
mkdir libmongocrypt-all
cd libmongocrypt-all
# The following URL is published from the upload-all task in the libmongocrypt Evergreen project.
curl https://mciuploads.s3.amazonaws.com/libmongocrypt/all/$LIBMONGOCRYPT_TAG/libmongocrypt-all.tar.gz -o libmongocrypt-all.tar.gz
curl -L https://github.com/mongodb/libmongocrypt/releases/download/$LIBMONGOCRYPT_TAG/libmongocrypt-windows-x86_64-$LIBMONGOCRYPT_TAG.tar.gz -o libmongocrypt-all.tar.gz
tar -xf libmongocrypt-all.tar.gz
cd ..
cp libmongocrypt-all/windows-test/bin/mongocrypt.dll c:/libmongocrypt/bin
cp libmongocrypt-all/windows-test/include/mongocrypt/*.h c:/libmongocrypt/include
cp libmongocrypt-all/bin/mongocrypt.dll c:/libmongocrypt/bin
cp libmongocrypt-all/include/mongocrypt/*.h c:/libmongocrypt/include

rm -rf libmongocrypt-all
echo "fetching build for Windows ... end"
Expand Down
112 changes: 90 additions & 22 deletions internal/integration/client_side_encryption_prose_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2455,8 +2455,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
}
})

// GODRIVER-3123. When we implement this feature, lower the min server version to 8.0.1
qeRunOpts22 := qeRunOpts.MaxServerVersion("7.99.99")
qeRunOpts22 := qeRunOpts.MinServerVersion("8.0")
mt.RunOpts("22. range explicit encryption", qeRunOpts22, func(mt *mtest.T) {
type testcase struct {
typeStr string
Expand All @@ -2470,6 +2469,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
twoHundredOne bson.RawValue
}

trimFactor := int32(1)
sparsity := int64(1)
precision := int32(2)

d128_0, err := bson.ParseDecimal128("0")
Expand Down Expand Up @@ -2497,7 +2498,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
typeStr: "DecimalNoPrecision",
field: "encryptedDecimalNoPrecision",
typeBson: bson.TypeDecimal128,
rangeOpts: options.Range().SetSparsity(1),
rangeOpts: options.Range().SetTrimFactor(trimFactor).SetSparsity(sparsity),
zero: bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_0h, d128_0l)},
six: bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_6h, d128_6l)},
thirty: bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_30h, d128_30l)},
Expand All @@ -2511,7 +2512,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
rangeOpts: options.Range().
SetMin(bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_0h, d128_0l)}).
SetMax(bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_200h, d128_200l)}).
SetSparsity(1).
SetTrimFactor(trimFactor).
SetSparsity(sparsity).
SetPrecision(precision),
zero: bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_0h, d128_0l)},
six: bson.RawValue{Type: bson.TypeDecimal128, Value: bsoncore.AppendDecimal128(nil, d128_6h, d128_6l)},
Expand All @@ -2523,7 +2525,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
typeStr: "DoubleNoPrecision",
field: "encryptedDoubleNoPrecision",
typeBson: bson.TypeDouble,
rangeOpts: options.Range().SetSparsity(1),
rangeOpts: options.Range().SetTrimFactor(trimFactor).SetSparsity(sparsity),
zero: bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 0)},
six: bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 6)},
thirty: bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 30)},
Expand All @@ -2537,7 +2539,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
rangeOpts: options.Range().
SetMin(bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 200)}).
SetSparsity(1).
SetTrimFactor(trimFactor).
SetSparsity(sparsity).
SetPrecision(precision),
zero: bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 0)},
six: bson.RawValue{Type: bson.TypeDouble, Value: bsoncore.AppendDouble(nil, 6)},
Expand All @@ -2552,7 +2555,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
rangeOpts: options.Range().
SetMin(bson.RawValue{Type: bson.TypeDateTime, Value: bsoncore.AppendDateTime(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeDateTime, Value: bsoncore.AppendDateTime(nil, 200)}).
SetSparsity(1),
SetTrimFactor(trimFactor).
SetSparsity(sparsity),
zero: bson.RawValue{Type: bson.TypeDateTime, Value: bsoncore.AppendDateTime(nil, 0)},
six: bson.RawValue{Type: bson.TypeDateTime, Value: bsoncore.AppendDateTime(nil, 6)},
thirty: bson.RawValue{Type: bson.TypeDateTime, Value: bsoncore.AppendDateTime(nil, 30)},
Expand All @@ -2566,7 +2570,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
rangeOpts: options.Range().
SetMin(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 200)}).
SetSparsity(1),
SetTrimFactor(trimFactor).
SetSparsity(sparsity),
zero: bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 0)},
six: bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 6)},
thirty: bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 30)},
Expand All @@ -2580,7 +2585,8 @@ func TestClientSideEncryptionProse(t *testing.T) {
rangeOpts: options.Range().
SetMin(bson.RawValue{Type: bson.TypeInt64, Value: bsoncore.AppendInt64(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeInt64, Value: bsoncore.AppendInt64(nil, 200)}).
SetSparsity(1),
SetTrimFactor(trimFactor).
SetSparsity(sparsity),
zero: bson.RawValue{Type: bson.TypeInt64, Value: bsoncore.AppendInt64(nil, 0)},
six: bson.RawValue{Type: bson.TypeInt64, Value: bsoncore.AppendInt64(nil, 6)},
thirty: bson.RawValue{Type: bson.TypeInt64, Value: bsoncore.AppendInt64(nil, 30)},
Expand Down Expand Up @@ -2635,7 +2641,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
// Insert 0, 6, 30, and 200.
coll := encryptedClient.Database("db").Collection("explicit_encryption")
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetRangeOptions(test.rangeOpts)
Expand Down Expand Up @@ -2682,7 +2688,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetRangeOptions(test.rangeOpts)
Expand All @@ -2698,10 +2704,10 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetQueryType("rangePreview").
SetQueryType("range").
SetRangeOptions(test.rangeOpts)

expr := bson.M{
Expand Down Expand Up @@ -2740,10 +2746,10 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetQueryType("rangePreview").
SetQueryType("range").
SetRangeOptions(test.rangeOpts)

expr := bson.M{
Expand Down Expand Up @@ -2782,10 +2788,10 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetQueryType("rangePreview").
SetQueryType("range").
SetRangeOptions(test.rangeOpts)

expr := bson.M{
Expand Down Expand Up @@ -2819,10 +2825,10 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetQueryType("rangePreview").
SetQueryType("range").
SetRangeOptions(test.rangeOpts)

expr := bson.M{
Expand Down Expand Up @@ -2858,7 +2864,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetRangeOptions(test.rangeOpts)
Expand All @@ -2872,7 +2878,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
defer clientEncryption.Close(context.Background())
defer encryptedClient.Disconnect(context.Background())
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetRangeOptions(test.rangeOpts)
Expand All @@ -2899,7 +2905,7 @@ func TestClientSideEncryptionProse(t *testing.T) {
ro := test.rangeOpts
ro.SetPrecision(2)
eo := options.Encrypt().
SetAlgorithm("RangePreview").
SetAlgorithm("Range").
SetKeyID(key1ID).
SetContentionFactor(0).
SetRangeOptions(ro)
Expand All @@ -2911,6 +2917,68 @@ func TestClientSideEncryptionProse(t *testing.T) {
})
}
})

mt.RunOpts("22. range explicit encryption applies defaults", qeRunOpts22, func(mt *mtest.T) {
err := mt.Client.Database("keyvault").Collection("datakeys").Drop(context.Background())
assert.Nil(mt, err, "error on Drop: %v", err)

testVal := bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 123)}

keyVaultClient, err := mongo.Connect(options.Client().ApplyURI(mtest.ClusterURI()))
assert.Nil(mt, err, "error on Connect: %v", err)

ceo := options.ClientEncryption().
SetKeyVaultNamespace("keyvault.datakeys").
SetKmsProviders(fullKmsProvidersMap)
clientEncryption, err := mongo.NewClientEncryption(keyVaultClient, ceo)
assert.Nil(mt, err, "error on NewClientEncryption: %v", err)

dkOpts := options.DataKey()
keyID, err := clientEncryption.CreateDataKey(context.Background(), "local", dkOpts)
assert.Nil(mt, err, "error in CreateDataKey: %v", err)

eo := options.Encrypt().
SetAlgorithm("Range").
SetKeyID(keyID).
SetContentionFactor(0).
SetRangeOptions(options.Range().
SetMin(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 1000)}))
payloadDefaults, err := clientEncryption.Encrypt(context.Background(), testVal, eo)
assert.Nil(mt, err, "error in Encrypt: %v", err)

mt.Run("Case 1: uses libmongocrypt defaults", func(mt *mtest.T) {
trimFactor := int32(6)
sparsity := int64(2)
eo := options.Encrypt().
SetAlgorithm("Range").
SetKeyID(keyID).
SetContentionFactor(0).
SetRangeOptions(options.Range().
SetMin(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 1000)}).
SetTrimFactor(trimFactor).
SetSparsity(sparsity))
payload, err := clientEncryption.Encrypt(context.Background(), testVal, eo)
assert.Nil(mt, err, "error in Encrypt: %v", err)
assert.Equalf(mt, len(payload.Data), len(payloadDefaults.Data), "the returned payload size is expected to be %d", len(payloadDefaults.Data))
})

mt.Run("Case 2: accepts trimFactor 0", func(mt *mtest.T) {
trimFactor := int32(0)
eo := options.Encrypt().
SetAlgorithm("Range").
SetKeyID(keyID).
SetContentionFactor(0).
SetRangeOptions(options.Range().
SetMin(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 0)}).
SetMax(bson.RawValue{Type: bson.TypeInt32, Value: bsoncore.AppendInt32(nil, 1000)}).
SetTrimFactor(trimFactor))
payload, err := clientEncryption.Encrypt(context.Background(), testVal, eo)
assert.Nil(mt, err, "error in Encrypt: %v", err)
assert.Greater(t, len(payload.Data), len(payloadDefaults.Data), "the returned payload size is expected to be greater than %d", len(payloadDefaults.Data))
})
})
}

func getWatcher(mt *mtest.T, streamType mongo.StreamType, cpt *cseProseTest) watcher {
Expand Down
17 changes: 8 additions & 9 deletions mongo/client_encryption.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,12 @@ func transformExplicitEncryptionOptions(opts ...options.Lister[options.EncryptOp
if rangeArgs.Precision != nil {
transformedRange.Precision = rangeArgs.Precision
}
transformedRange.Sparsity = rangeArgs.Sparsity
if rangeArgs.Sparsity != nil {
transformedRange.Sparsity = rangeArgs.Sparsity
}
if rangeArgs.TrimFactor != nil {
transformedRange.TrimFactor = rangeArgs.TrimFactor
}
transformed.SetRangeOptions(transformedRange)
}
return transformed
Expand Down Expand Up @@ -250,14 +255,8 @@ func (ce *ClientEncryption) Encrypt(
// 2. An Aggregate Expression of this form:
// {$and: [{$gt: [<fieldpath>, <value1>]}, {$lt: [<fieldpath>, <value2>]}]
// $gt may also be $gte. $lt may also be $lte.
// Only supported for queryType "rangePreview"
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
func (ce *ClientEncryption) EncryptExpression(
ctx context.Context,
expr interface{},
result interface{},
opts ...options.Lister[options.EncryptOptions],
) error {
// Only supported for queryType "range"
func (ce *ClientEncryption) EncryptExpression(ctx context.Context, expr interface{}, result interface{}, opts ...options.Lister[options.EncryptOptions]) error {
transformed := transformExplicitEncryptionOptions(opts...)

exprDoc, err := marshal(expr, nil, nil)
Expand Down
34 changes: 20 additions & 14 deletions mongo/options/encryptoptions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,13 @@ const (
QueryTypeEquality string = "equality"
)

// RangeOptions specifies index options for a Queryable Encryption field
// supporting "rangePreview" queries. Beta: The Range algorithm is experimental
// only. It is not intended for public use. It is subject to breaking changes.
// RangeOptions specifies index options for a Queryable Encryption field supporting "range" queries.
type RangeOptions struct {
Min *bson.RawValue
Max *bson.RawValue
Sparsity int64
Precision *int32
Min *bson.RawValue
Max *bson.RawValue
Sparsity *int64
TrimFactor *int32
Precision *int32
}

// RangeOptionsBuilder contains options to configure Rangeopts for queryeable
Expand Down Expand Up @@ -96,6 +95,7 @@ func (e *EncryptOptionsBuilder) SetKeyAltName(keyAltName string) *EncryptOptions
// - AEAD_AES_256_CBC_HMAC_SHA_512-Random
// - Indexed
// - Unindexed
// - Range
// This is required.
// Indexed and Unindexed are used for Queryable Encryption.
func (e *EncryptOptionsBuilder) SetAlgorithm(algorithm string) *EncryptOptionsBuilder {
Expand Down Expand Up @@ -134,8 +134,7 @@ func (e *EncryptOptionsBuilder) SetContentionFactor(contentionFactor int64) *Enc
return e
}

// SetRangeOptions specifies the options to use for explicit encryption with range. It is only valid to set if algorithm is "rangePreview".
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
// SetRangeOptions specifies the options to use for explicit encryption with range. It is only valid to set if algorithm is "range".
func (e *EncryptOptionsBuilder) SetRangeOptions(ro *RangeOptionsBuilder) *EncryptOptionsBuilder {
e.Opts = append(e.Opts, func(opts *EncryptOptions) error {
opts.RangeOptions = ro
Expand All @@ -147,7 +146,6 @@ func (e *EncryptOptionsBuilder) SetRangeOptions(ro *RangeOptionsBuilder) *Encryp
}

// SetMin sets the range index minimum value.
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
func (ro *RangeOptionsBuilder) SetMin(min bson.RawValue) *RangeOptionsBuilder {
ro.Opts = append(ro.Opts, func(opts *RangeOptions) error {
opts.Min = &min
Expand All @@ -159,7 +157,6 @@ func (ro *RangeOptionsBuilder) SetMin(min bson.RawValue) *RangeOptionsBuilder {
}

// SetMax sets the range index maximum value.
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
func (ro *RangeOptionsBuilder) SetMax(max bson.RawValue) *RangeOptionsBuilder {
ro.Opts = append(ro.Opts, func(opts *RangeOptions) error {
opts.Max = &max
Expand All @@ -171,10 +168,20 @@ func (ro *RangeOptionsBuilder) SetMax(max bson.RawValue) *RangeOptionsBuilder {
}

// SetSparsity sets the range index sparsity.
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
func (ro *RangeOptionsBuilder) SetSparsity(sparsity int64) *RangeOptionsBuilder {
ro.Opts = append(ro.Opts, func(opts *RangeOptions) error {
opts.Sparsity = sparsity
opts.Sparsity = &sparsity

return nil
})

return ro
}

// SetTrimFactor sets the range index trim factor.
func (ro *RangeOptionsBuilder) SetTrimFactor(trimFactor int32) *RangeOptionsBuilder {
ro.Opts = append(ro.Opts, func(opts *RangeOptions) error {
opts.TrimFactor = &trimFactor

return nil
})
Expand All @@ -183,7 +190,6 @@ func (ro *RangeOptionsBuilder) SetSparsity(sparsity int64) *RangeOptionsBuilder
}

// SetPrecision sets the range index precision.
// Beta: The Range algorithm is experimental only. It is not intended for public use. It is subject to breaking changes.
func (ro *RangeOptionsBuilder) SetPrecision(precision int32) *RangeOptionsBuilder {
ro.Opts = append(ro.Opts, func(opts *RangeOptions) error {
opts.Precision = &precision
Expand Down
Loading
Loading