Skip to content

Commit 957852a

Browse files
abouroumineabouroumine
authored
Issue 524 (#529)
Root hash calculation optimization
1 parent e35c51f commit 957852a

31 files changed

+546
-400
lines changed

‎code/go/0chain.net/blobber/integration_tests.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
//go:build integration_tests
12
// +build integration_tests
23

34
package main

‎code/go/0chain.net/blobber/stub.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
//go:build !integration_tests
12
// +build !integration_tests
23

34
package main

‎code/go/0chain.net/blobbercore/allocation/attributesfilechange.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ func (ac *AttributesChange) ApplyChange(ctx context.Context, _ *AllocationChange
2828

2929
// root reference
3030
ref, err = reference.GetReferencePath(ctx, ac.AllocationID, ac.Path)
31+
3132
if err != nil {
3233
return nil, common.NewErrorf("process_attrs_update",
3334
"getting root reference path: %v", err)
@@ -38,13 +39,14 @@ func (ac *AttributesChange) ApplyChange(ctx context.Context, _ *AllocationChange
3839
dirRef = ref
3940
treelevel = 0
4041
)
41-
42+
dirRef.HashToBeComputed = true
4243
for treelevel < len(tSubDirs) {
4344
var found bool
4445
for _, child := range dirRef.Children {
4546
if child.Type == reference.DIRECTORY && treelevel < len(tSubDirs) {
4647
if child.Name == tSubDirs[treelevel] {
4748
dirRef, found = child, true
49+
dirRef.HashToBeComputed = true
4850
break
4951
}
5052
}
@@ -78,7 +80,9 @@ func (ac *AttributesChange) ApplyChange(ctx context.Context, _ *AllocationChange
7880
"setting new attributes: %v", err)
7981
}
8082

81-
if _, err = ref.CalculateHash(ctx, true); err != nil {
83+
existingRef.HashToBeComputed = true
84+
85+
if _, err := ref.CalculateHash(ctx, true); err != nil {
8286
return nil, common.NewErrorf("process_attrs_update",
8387
"saving updated reference: %v", err)
8488
}

‎code/go/0chain.net/blobbercore/allocation/copyfilechange.go

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,19 +28,18 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, change *AllocationCha
2828
if err != nil {
2929
return nil, err
3030
}
31-
31+
affectedRef.HashToBeComputed = true
3232
if rf.DestPath == "/" {
3333
destRef, err := reference.GetRefWithSortedChildren(ctx, rf.AllocationID, rf.DestPath)
3434
if err != nil || destRef.Type != reference.DIRECTORY {
3535
return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.")
3636
}
37+
destRef.HashToBeComputed = true
3738
fileRefs := rf.processCopyRefs(ctx, affectedRef, destRef, allocationRoot)
38-
3939
_, err = destRef.CalculateHash(ctx, true)
4040
if err != nil {
4141
return nil, err
4242
}
43-
4443
for _, fileRef := range fileRefs {
4544
stats.NewFileCreated(ctx, fileRef.ID)
4645
}
@@ -53,21 +52,19 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, change *AllocationCha
5352
if err != nil || destRef.Type != reference.DIRECTORY {
5453
return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.")
5554
}
56-
5755
destRef, err = reference.GetRefWithSortedChildren(ctx, rf.AllocationID, rf.DestPath)
5856
if err != nil || destRef.Type != reference.DIRECTORY {
5957
return nil, common.NewError("invalid_parameters", "Invalid destination path. Should be a valid directory.")
6058
}
61-
59+
destRef.HashToBeComputed = true
6260
path, _ := filepath.Split(rf.DestPath)
6361
path = filepath.Clean(path)
6462
tSubDirs := reference.GetSubDirsFromPath(path)
65-
6663
rootRef, err := reference.GetReferencePath(ctx, rf.AllocationID, path)
6764
if err != nil {
6865
return nil, err
6966
}
70-
67+
rootRef.HashToBeComputed = true
7168
dirRef := rootRef
7269
treelevel := 0
7370
for treelevel < len(tSubDirs) {
@@ -76,6 +73,7 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, change *AllocationCha
7673
if child.Type == reference.DIRECTORY && treelevel < len(tSubDirs) {
7774
if child.Name == tSubDirs[treelevel] {
7875
dirRef = child
76+
dirRef.HashToBeComputed = true
7977
found = true
8078
break
8179
}
@@ -102,7 +100,6 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, change *AllocationCha
102100
dirRef.RemoveChild(childIndex)
103101
filerefs := rf.processCopyRefs(ctx, affectedRef, destRef, allocationRoot)
104102
dirRef.AddChild(destRef)
105-
106103
_, err = rootRef.CalculateHash(ctx, true)
107104
if err != nil {
108105
return nil, err
@@ -111,7 +108,6 @@ func (rf *CopyFileChange) ApplyChange(ctx context.Context, change *AllocationCha
111108
for _, fileRef := range filerefs {
112109
stats.NewFileCreated(ctx, fileRef.ID)
113110
}
114-
115111
return rootRef, err
116112
}
117113

@@ -127,6 +123,7 @@ func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef, dest
127123
newRef.Name = affectedRef.Name
128124
newRef.LookupHash = reference.GetReferenceLookup(newRef.AllocationID, newRef.Path)
129125
newRef.Attributes = datatypes.JSON(string(affectedRef.Attributes))
126+
newRef.HashToBeComputed = true
130127
destRef.AddChild(newRef)
131128

132129
for _, childRef := range affectedRef.Children {
@@ -154,7 +151,7 @@ func (rf *CopyFileChange) processCopyRefs(ctx context.Context, affectedRef, dest
154151
newFile.EncryptedKey = affectedRef.EncryptedKey
155152
newFile.Attributes = datatypes.JSON(string(affectedRef.Attributes))
156153
newFile.ChunkSize = affectedRef.ChunkSize
157-
154+
newFile.HashToBeComputed = true
158155
destRef.AddChild(newFile)
159156

160157
files = append(files, newFile)

‎code/go/0chain.net/blobbercore/allocation/file_changer_update.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, change *Allocation
3131
if err != nil {
3232
return nil, err
3333
}
34-
34+
rootRef.HashToBeComputed = true
3535
dirRef := rootRef
3636
treelevel := 0
3737
for treelevel < len(tSubDirs) {
@@ -40,6 +40,7 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, change *Allocation
4040
if child.Type == reference.DIRECTORY && treelevel < len(tSubDirs) {
4141
if child.Name == tSubDirs[treelevel] {
4242
dirRef = child
43+
dirRef.HashToBeComputed = true
4344
found = true
4445
break
4546
}
@@ -63,15 +64,14 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, change *Allocation
6364
return nil, common.NewError("file_not_found", "File to update not found in blobber")
6465
}
6566
existingRef := dirRef.Children[idx]
66-
// remove changed thumbnail and files
67+
existingRef.HashToBeComputed = true
6768
nf.deleteHash = make(map[string]bool)
6869
if existingRef.ThumbnailHash != "" && existingRef.ThumbnailHash != nf.ThumbnailHash {
6970
nf.deleteHash[existingRef.ThumbnailHash] = true
7071
}
7172
if existingRef.ContentHash != "" && existingRef.ContentHash != nf.Hash {
7273
nf.deleteHash[existingRef.ContentHash] = true
7374
}
74-
7575
existingRef.ActualFileHash = nf.ActualHash
7676
existingRef.ActualFileSize = nf.ActualSize
7777
existingRef.MimeType = nf.MimeType
@@ -91,7 +91,6 @@ func (nf *UpdateFileChanger) ApplyChange(ctx context.Context, change *Allocation
9191
return nil, common.NewErrorf("process_update_file_change",
9292
"setting file attributes: %v", err)
9393
}
94-
9594
_, err = rootRef.CalculateHash(ctx, true)
9695
stats.FileUpdated(ctx, existingRef.ID)
9796
return rootRef, err

‎code/go/0chain.net/blobbercore/allocation/file_changer_upload.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,16 @@ func (nf *UploadFileChanger) ApplyChange(ctx context.Context, change *Allocation
2828
if err != nil {
2929
return nil, err
3030
}
31-
3231
dirRef := rootRef
32+
rootRef.HashToBeComputed = true
3333
treelevel := 0
3434
for {
3535
found := false
3636
for _, child := range dirRef.Children {
3737
if child.Type == reference.DIRECTORY && treelevel < len(tSubDirs) {
3838
if child.Name == tSubDirs[treelevel] {
3939
dirRef = child
40+
dirRef.HashToBeComputed = true
4041
found = true
4142
break
4243
}
@@ -53,6 +54,7 @@ func (nf *UploadFileChanger) ApplyChange(ctx context.Context, change *Allocation
5354
newRef.ParentPath = "/" + strings.Join(tSubDirs[:treelevel], "/")
5455
newRef.Name = tSubDirs[treelevel]
5556
newRef.LookupHash = reference.GetReferenceLookup(dirRef.AllocationID, newRef.Path)
57+
newRef.HashToBeComputed = true
5658
dirRef.AddChild(newRef)
5759
dirRef = newRef
5860
treelevel++
@@ -82,12 +84,11 @@ func (nf *UploadFileChanger) ApplyChange(ctx context.Context, change *Allocation
8284
newFile.ActualThumbnailSize = nf.ActualThumbnailSize
8385
newFile.EncryptedKey = nf.EncryptedKey
8486
newFile.ChunkSize = nf.ChunkSize
85-
87+
newFile.HashToBeComputed = true
8688
if err = newFile.SetAttributes(&nf.Attributes); err != nil {
8789
return nil, common.NewErrorf("process_new_file_change",
8890
"setting file attributes: %v", err)
8991
}
90-
9192
dirRef.AddChild(newFile)
9293
if _, err := rootRef.CalculateHash(ctx, true); err != nil {
9394
return nil, err

‎code/go/0chain.net/blobbercore/allocation/newfilechange.go

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,11 @@ type NewFileChange struct {
5555
func (nf *NewFileChange) CreateDir(ctx context.Context, allocationID, dirName, allocationRoot string) (*reference.Ref, error) {
5656
path := filepath.Clean(dirName)
5757
tSubDirs := reference.GetSubDirsFromPath(path)
58-
59-
rootRef, err := reference.GetReferencePath(ctx, allocationID, nf.Path)
58+
rootRef, err := reference.GetReferencePath(ctx, nf.AllocationID, nf.Path)
6059
if err != nil {
6160
return nil, err
6261
}
63-
62+
rootRef.HashToBeComputed = true
6463
dirRef := rootRef
6564
treelevel := 0
6665
for {
@@ -69,6 +68,7 @@ func (nf *NewFileChange) CreateDir(ctx context.Context, allocationID, dirName, a
6968
if child.Type == reference.DIRECTORY && treelevel < len(tSubDirs) {
7069
if child.Name == tSubDirs[treelevel] {
7170
dirRef = child
71+
dirRef.HashToBeComputed = true
7272
found = true
7373
break
7474
}
@@ -85,6 +85,7 @@ func (nf *NewFileChange) CreateDir(ctx context.Context, allocationID, dirName, a
8585
newRef.ParentPath = "/" + strings.Join(tSubDirs[:treelevel], "/")
8686
newRef.Name = tSubDirs[treelevel]
8787
newRef.LookupHash = reference.GetReferenceLookup(dirRef.AllocationID, newRef.Path)
88+
newRef.HashToBeComputed = true
8889
dirRef.AddChild(newRef)
8990
dirRef = newRef
9091
treelevel++
@@ -105,6 +106,7 @@ func (nf *NewFileChange) CreateDir(ctx context.Context, allocationID, dirName, a
105106
newDir.NumBlocks = 0
106107
newDir.ParentPath = dirRef.Path
107108
newDir.WriteMarker = allocationRoot
109+
newDir.HashToBeComputed = true
108110
dirRef.AddChild(newDir)
109111

110112
if _, err := rootRef.CalculateHash(ctx, true); err != nil {
@@ -136,7 +138,7 @@ func (nf *NewFileChange) ApplyChange(ctx context.Context, change *AllocationChan
136138
if err != nil {
137139
return nil, err
138140
}
139-
141+
rootRef.HashToBeComputed = true
140142
dirRef := rootRef
141143
treelevel := 0
142144
for {
@@ -145,6 +147,7 @@ func (nf *NewFileChange) ApplyChange(ctx context.Context, change *AllocationChan
145147
if child.Type == reference.DIRECTORY && treelevel < len(tSubDirs) {
146148
if child.Name == tSubDirs[treelevel] {
147149
dirRef = child
150+
dirRef.HashToBeComputed = true
148151
found = true
149152
break
150153
}
@@ -161,6 +164,7 @@ func (nf *NewFileChange) ApplyChange(ctx context.Context, change *AllocationChan
161164
newRef.ParentPath = "/" + strings.Join(tSubDirs[:treelevel], "/")
162165
newRef.Name = tSubDirs[treelevel]
163166
newRef.LookupHash = reference.GetReferenceLookup(dirRef.AllocationID, newRef.Path)
167+
newRef.HashToBeComputed = true
164168
dirRef.AddChild(newRef)
165169
dirRef = newRef
166170
treelevel++
@@ -190,13 +194,15 @@ func (nf *NewFileChange) ApplyChange(ctx context.Context, change *AllocationChan
190194
newFile.ActualThumbnailSize = nf.ActualThumbnailSize
191195
newFile.EncryptedKey = nf.EncryptedKey
192196
newFile.ChunkSize = nf.ChunkSize
197+
newFile.HashToBeComputed = true
193198

194199
if err = newFile.SetAttributes(&nf.Attributes); err != nil {
195200
return nil, common.NewErrorf("process_new_file_change",
196201
"setting file attributes: %v", err)
197202
}
198203

199204
dirRef.AddChild(newFile)
205+
200206
if _, err := rootRef.CalculateHash(ctx, true); err != nil {
201207
return nil, err
202208
}

‎code/go/0chain.net/blobbercore/allocation/renamefilechange.go

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,12 @@ func (rf *RenameFileChange) DeleteTempFile() error {
2525
}
2626

2727
func (rf *RenameFileChange) ApplyChange(ctx context.Context, change *AllocationChange, allocationRoot string) (*reference.Ref, error) {
28-
isFilePresent, _ := reference.PathExists(ctx, rf.AllocationID, rf.NewName)
28+
29+
isFilePresent, err := reference.IsRefExist(ctx, rf.AllocationID, rf.NewName)
30+
if err != nil {
31+
Logger.Info("invalid_reference_path", zap.Error(err))
32+
}
33+
2934
if isFilePresent {
3035
return nil, common.NewError("invalid_reference_path", "file already exists")
3136
}
@@ -34,7 +39,7 @@ func (rf *RenameFileChange) ApplyChange(ctx context.Context, change *AllocationC
3439
if err != nil {
3540
return nil, err
3641
}
37-
42+
affectedRef.HashToBeComputed = true
3843
path, _ := filepath.Split(affectedRef.Path)
3944
path = filepath.Clean(path)
4045
affectedRef.Name = rf.NewName
@@ -54,7 +59,7 @@ func (rf *RenameFileChange) ApplyChange(ctx context.Context, change *AllocationC
5459
if err != nil {
5560
return nil, err
5661
}
57-
62+
rootRef.HashToBeComputed = true
5863
dirRef := rootRef
5964
treelevel := 0
6065
for treelevel < len(tSubDirs) {
@@ -63,6 +68,7 @@ func (rf *RenameFileChange) ApplyChange(ctx context.Context, change *AllocationC
6368
if child.Type == reference.DIRECTORY && treelevel < len(tSubDirs) {
6469
if child.Name == tSubDirs[treelevel] {
6570
dirRef = child
71+
dirRef.HashToBeComputed = true
6672
found = true
6773
break
6874
}
@@ -74,7 +80,6 @@ func (rf *RenameFileChange) ApplyChange(ctx context.Context, change *AllocationC
7480
return nil, common.NewError("invalid_reference_path", "Invalid reference path from the blobber")
7581
}
7682
}
77-
7883
if len(dirRef.Children) == 0 {
7984
Logger.Error("no files in root folder", zap.Any("change", rf))
8085
return nil, common.NewError("file_not_found", "No files in root folder")
@@ -91,11 +96,9 @@ func (rf *RenameFileChange) ApplyChange(ctx context.Context, change *AllocationC
9196
Logger.Error("error in file rename", zap.Any("change", rf))
9297
return nil, common.NewError("file_not_found", "File to rename not found in blobber")
9398
}
94-
//dirRef.Children[idx] = affectedRef
9599
dirRef.RemoveChild(idx)
96100
dirRef.AddChild(affectedRef)
97101
_, err = rootRef.CalculateHash(ctx, true)
98-
99102
return rootRef, err
100103
}
101104

0 commit comments

Comments
 (0)