Skip to content

Commit 092bac7

Browse files
author
thienlk1
committed
fix validator index format
1 parent 6b92f33 commit 092bac7

File tree

6 files changed

+122
-11
lines changed

6 files changed

+122
-11
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,4 @@ tags
2424
TESTS*.xml
2525
.tox/
2626
.vagrant/
27-
.vscode
27+
.vscode

‎core/ledger/kvledger/txmgmt/statedb/statemongodb/commit_handling.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,6 @@ func (c *committer) commitUpdates() error {
186186
// IF INDIVIDUAL DOCUMENTS IN THE BULK UPDATE DID NOT SUCCEED, TRY THEM INDIVIDUALLY
187187
// iterate through the response from MongoDB by document
188188
for _, resp := range responses {
189-
logger.Debugf("commitUpdates resp.Error :%s", resp.Error)
190189
// If the document returned an error, retry the individual document
191190
if resp.Error != "" || resp.ID == "" {
192191
rev := string(resp.Rev)

‎core/ledger/kvledger/txmgmt/statedb/statemongodb/mongodb.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func (d *mongoDoc) key() (string, error) {
5555
type BinaryDataInfo struct {
5656
Name string
5757
Length uint64
58-
Binarydata []byte
58+
Binarydata []byte `bson:"Binarydata"`
5959
}
6060

6161
//MongoConnectionDef contains parameters
@@ -357,7 +357,11 @@ func (dbclient *mongoDatabase) readDoc(id string) (*mongoDoc, string, error) {
357357
jsonValue[key] = int64(value.Value().Int32())
358358
}
359359
} else {
360-
jsonValue[key] = value.Value().String()
360+
if _, ok := value.Value().StringValueOK(); ok {
361+
jsonValue[key] = value.Value().StringValue()
362+
} else {
363+
jsonValue[key] = value.Value().String()
364+
}
361365
}
362366
}
363367

@@ -373,6 +377,8 @@ func (dbclient *mongoDatabase) readDoc(id string) (*mongoDoc, string, error) {
373377
}
374378
mongoDoc.jsonValue = data
375379
logger.Debugf("Database Name : [%s] Collection Name : [%s] Exiting readDoc()", dbName, colName)
380+
logger.Debugf("jsonValue : %+v", jsonValue)
381+
logger.Debugf("mongoDoc : %s", string(mongoDoc.jsonValue))
376382
return &mongoDoc, revision, nil
377383
}
378384

‎internal/ccmetadata/validators.go‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,19 @@ type fileValidator func(fileName string, fileBytes []byte) error
2525
// AllowedCharsCollectionName captures the regex pattern for a valid collection name
2626
const AllowedCharsCollectionName = "[A-Za-z0-9_-]+"
2727

28-
// Currently, the only metadata expected and allowed is for META-INF/statedb/couchdb/indexes.
28+
// Currently, the only metadata expected and allowed is for META-INF/statedb/[couchdb, mongodb]/indexes.
2929
var fileValidators = map[*regexp.Regexp]fileValidator{
3030
regexp.MustCompile("^META-INF/statedb/couchdb/indexes/.*[.]json"): couchdbIndexFileValidator,
3131
regexp.MustCompile("^META-INF/statedb/couchdb/collections/" + AllowedCharsCollectionName + "/indexes/.*[.]json"): couchdbIndexFileValidator,
32+
regexp.MustCompile("^META-INF/statedb/mongodb/indexes/.*[.]json"): mongodbIndexFileValidator,
33+
regexp.MustCompile("^META-INF/statedb/mongodb/collections/" + AllowedCharsCollectionName + "/indexes/.*[.]json"): mongodbIndexFileValidator,
3234
}
3335

3436
var collectionNameValid = regexp.MustCompile("^" + AllowedCharsCollectionName)
3537

3638
var fileNameValid = regexp.MustCompile("^.*[.]json")
3739

38-
var validDatabases = []string{"couchdb"}
40+
var validDatabases = []string{"couchdb", "mongodb"}
3941

4042
// UnhandledDirectoryError is returned for metadata files in unhandled directories
4143
type UnhandledDirectoryError struct {
@@ -77,7 +79,7 @@ func ValidateMetadataFile(filePathName string, fileBytes []byte) error {
7779
}
7880

7981
func buildMetadataFileErrorMessage(filePathName string) string {
80-
82+
logger.Debugf("buildMetadataFileErrorMessage")
8183
dir, filename := filepath.Split(filePathName)
8284

8385
if !strings.HasPrefix(filePathName, "META-INF/statedb") {
@@ -89,8 +91,9 @@ func buildMetadataFileErrorMessage(filePathName string) string {
8991
return fmt.Sprintf("metadata file path must include a database and index directory: %s", dir)
9092
}
9193
// validate the database type
94+
logger.Debugf("validDatabases: %s", validDatabases)
9295
if !contains(validDatabases, directoryArray[2]) {
93-
return fmt.Sprintf("database name [%s] is not supported, valid options: %s", directoryArray[2], validDatabases)
96+
return fmt.Sprintf("DATABASE name [%s] is not supported, valid options: %s", directoryArray[2], validDatabases)
9497
}
9598
// verify "indexes" is under the database name
9699
if len(directoryArray) == 4 && directoryArray[3] != "indexes" {
@@ -156,6 +159,24 @@ func couchdbIndexFileValidator(fileName string, fileBytes []byte) error {
156159

157160
}
158161

162+
// mongodbIndexFileValidator implements fileValidator
163+
func mongodbIndexFileValidator(fileName string, fileBytes []byte) error {
164+
// if the content does not validate as JSON, return err to invalidate the file
165+
boolIsJSON, indexDefinition := isJSON(fileBytes)
166+
if !boolIsJSON {
167+
return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid JSON", fileName)}
168+
}
169+
170+
//validate the index definition
171+
err := validateIndexJSON(indexDefinition)
172+
if err != nil {
173+
return &InvalidIndexContentError{fmt.Sprintf("Index metadata file [%s] is not a valid index definition: %s", fileName, err)}
174+
}
175+
176+
return nil
177+
178+
}
179+
159180
// isJSON tests a string to determine if it can be parsed as valid JSON
160181
func isJSON(s []byte) (bool, map[string]interface{}) {
161182
var js map[string]interface{}
@@ -297,10 +318,16 @@ func validateFieldMap(jsonFragment map[string]interface{}) error {
297318
//Ensure the sort is either "asc" or "desc"
298319
jv := strings.ToLower(jsonValue)
299320
if jv != "asc" && jv != "desc" {
300-
return fmt.Errorf("Sort must be either \"asc\" or \"desc\". \"%s\" was found.", jsonValue)
321+
return fmt.Errorf("Sort must be either \"asc\",\"desc\",1 or -1. \"%s\" was found.", jsonValue)
322+
}
323+
logger.Debugf("Found index field name: \"%s\":\"%s\"", jsonKey, jsonValue)
324+
case float64:
325+
//Ensure the sort is either 1 or -1
326+
jv := int(jsonValue)
327+
if jv != 1 && jv != -1 {
328+
return fmt.Errorf("Sort must be either \"asc\",\"desc\",1 or -1. \"%d\" was found.", jsonValue)
301329
}
302330
logger.Debugf("Found index field name: \"%s\":\"%s\"", jsonKey, jsonValue)
303-
304331
default:
305332
return fmt.Errorf("Invalid field definition, fields must be in the form \"fieldname\":\"sort\"")
306333
}

‎internal/ccmetadata/validators_test.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ func TestGoodIndexJSON(t *testing.T) {
2727

2828
err := ValidateMetadataFile(fileName, fileBytes)
2929
assert.NoError(t, err, "Error validating a good index")
30+
31+
fileName = "META-INF/statedb/mongodb/indexes/myIndex.json"
32+
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner"}`)
33+
34+
err = ValidateMetadataFile(fileName, fileBytes)
35+
assert.NoError(t, err, "Error validating a good index")
3036
}
3137

3238
func TestBadIndexJSON(t *testing.T) {
@@ -46,6 +52,19 @@ func TestBadIndexJSON(t *testing.T) {
4652
assert.True(t, ok, "Should have received an InvalidIndexContentError")
4753

4854
t.Log("SAMPLE ERROR STRING:", err.Error())
55+
56+
fileName = "META-INF/statedb/mongodb/indexes/myIndex.json"
57+
fileBytes = []byte("invalid json")
58+
59+
err = ValidateMetadataFile(fileName, fileBytes)
60+
61+
assert.Error(t, err, "Should have received an InvalidIndexContentError")
62+
63+
// Type assertion on InvalidIndexContentError
64+
_, ok = err.(*InvalidIndexContentError)
65+
assert.True(t, ok, "Should have received an InvalidIndexContentError")
66+
67+
t.Log("SAMPLE ERROR STRING:", err.Error())
4968
}
5069

5170
func TestIndexWrongLocation(t *testing.T) {
@@ -64,6 +83,18 @@ func TestIndexWrongLocation(t *testing.T) {
6483
assert.True(t, ok, "Should have received an UnhandledDirectoryError")
6584

6685
t.Log("SAMPLE ERROR STRING:", err.Error())
86+
87+
fileName = "META-INF/statedb/mongodb/myIndex.json"
88+
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner"`)
89+
90+
err = ValidateMetadataFile(fileName, fileBytes)
91+
assert.Error(t, err, "Should have received an UnhandledDirectoryError")
92+
93+
// Type assertion on UnhandledDirectoryError
94+
_, ok = err.(*UnhandledDirectoryError)
95+
assert.True(t, ok, "Should have received an UnhandledDirectoryError")
96+
97+
t.Log("SAMPLE ERROR STRING:", err.Error())
6798
}
6899

69100
func TestInvalidMetadataType(t *testing.T) {
@@ -80,6 +111,7 @@ func TestInvalidMetadataType(t *testing.T) {
80111
// Type assertion on UnhandledDirectoryError
81112
_, ok := err.(*UnhandledDirectoryError)
82113
assert.True(t, ok, "Should have received an UnhandledDirectoryError")
114+
83115
}
84116

85117
func TestBadMetadataExtension(t *testing.T) {
@@ -108,6 +140,14 @@ func TestBadFilePaths(t *testing.T) {
108140
fmt.Println(err)
109141
assert.Error(t, err, "Should have received an error for bad META-INF directory")
110142

143+
// Test bad META-INF
144+
fileName = "META-INF1/statedb/mongodb/indexes/test1.json"
145+
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
146+
147+
err = ValidateMetadataFile(fileName, fileBytes)
148+
fmt.Println(err)
149+
assert.Error(t, err, "Should have received an error for bad META-INF directory")
150+
111151
// Test bad path length
112152
fileName = "META-INF/statedb/test1.json"
113153
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
@@ -116,6 +156,14 @@ func TestBadFilePaths(t *testing.T) {
116156
fmt.Println(err)
117157
assert.Error(t, err, "Should have received an error for bad length")
118158

159+
// Test invalid indexes directory name
160+
fileName = "META-INF/statedb/mongodb/index/test1.json"
161+
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
162+
163+
err = ValidateMetadataFile(fileName, fileBytes)
164+
fmt.Println(err)
165+
assert.Error(t, err, "Should have received an error for invalid indexes directory")
166+
119167
// Test invalid database name
120168
fileName = "META-INF/statedb/goleveldb/indexes/test1.json"
121169
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
@@ -140,6 +188,14 @@ func TestBadFilePaths(t *testing.T) {
140188
fmt.Println(err)
141189
assert.Error(t, err, "Should have received an error for invalid collections directory")
142190

191+
// Test invalid collections directory name
192+
fileName = "META-INF/statedb/mongodb/collection/testcoll/indexes/test1.json"
193+
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
194+
195+
err = ValidateMetadataFile(fileName, fileBytes)
196+
fmt.Println(err)
197+
assert.Error(t, err, "Should have received an error for invalid collections directory")
198+
143199
// Test valid collections name
144200
fileName = "META-INF/statedb/couchdb/collections/testcoll/indexes/test1.json"
145201
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
@@ -148,6 +204,14 @@ func TestBadFilePaths(t *testing.T) {
148204
fmt.Println(err)
149205
assert.NoError(t, err, "Error should not have been thrown for a valid collection name")
150206

207+
// Test valid collections name
208+
fileName = "META-INF/statedb/mongodb/collections/testcoll/indexes/test1.json"
209+
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
210+
211+
err = ValidateMetadataFile(fileName, fileBytes)
212+
fmt.Println(err)
213+
assert.NoError(t, err, "Error should not have been thrown for a valid collection name")
214+
151215
// Test invalid collections name
152216
fileName = "META-INF/statedb/couchdb/collections/#testcoll/indexes/test1.json"
153217
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
@@ -156,6 +220,14 @@ func TestBadFilePaths(t *testing.T) {
156220
fmt.Println(err)
157221
assert.Error(t, err, "Should have received an error for an invalid collection name")
158222

223+
// Test invalid collections name
224+
fileName = "META-INF/statedb/mongodb/collections/#testcoll/indexes/test1.json"
225+
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
226+
227+
err = ValidateMetadataFile(fileName, fileBytes)
228+
fmt.Println(err)
229+
assert.Error(t, err, "Should have received an error for an invalid collection name")
230+
159231
// Test invalid collections name
160232
fileName = "META-INF/statedb/couchdb/collections/testcoll/indexes/test1.txt"
161233
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
@@ -164,6 +236,13 @@ func TestBadFilePaths(t *testing.T) {
164236
fmt.Println(err)
165237
assert.Error(t, err, "Should have received an error for an invalid file name")
166238

239+
// Test invalid collections name
240+
fileName = "META-INF/statedb/mongodb/collections/testcoll/indexes/test1.txt"
241+
fileBytes = []byte(`{"index":{"fields":["data.docType","data.owner"]},"name":"indexOwner","type":"json"}`)
242+
243+
err = ValidateMetadataFile(fileName, fileBytes)
244+
fmt.Println(err)
245+
assert.Error(t, err, "Should have received an error for an invalid file name")
167246
}
168247

169248
func TestIndexValidation(t *testing.T) {

‎internal/peer/node/config.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ func ledgerConfig() *ledger.Config {
116116
//RequestTimeout: viper.GetDuration("ledger.state.mongoDBConfig.requestTimeout"),
117117
MaxRetries: 3,
118118
MaxRetriesOnStartup: 3,
119-
RequestTimeout: 35000000000,
119+
RequestTimeout: 350000000000,
120120
QueryLimit: internalQueryLimit,
121121
MaxBatchUpdateSize: maxBatchUpdateSize,
122122
WarmIndexesAfterNBlocks: warmAfterNBlocks,

0 commit comments

Comments
 (0)