Skip to content

Commit a736b74

Browse files
efirsefirs
committed
fix: Enable race detector in tests
* Fix detected races in tests * Further disable not used nested mount feature * Some lints fixes
1 parent bb683fc commit a736b74

20 files changed

+418
-280
lines changed

‎.github/workflows/xfstests.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,5 @@ jobs:
1616
run: env CGO_ENABLED=0 go build -o geesefs-linux-amd64 -v && ln -s geesefs-linux-amd64 geesefs
1717

1818
- name: Run xfstests
19-
# Often crashes in CI due to memory limits
20-
continue-on-error: true
2119
run: make run-xfstests
2220
timeout-minutes: 10

‎Makefile‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
export CGO_ENABLED=0
22

3-
run-test: s3proxy.jar build
3+
run-test: s3proxy.jar build-debug
44
./test/run-tests.sh
55

6-
run-xfstests: s3proxy.jar xfstests
6+
run-xfstests: s3proxy.jar xfstests build-debug
77
./test/run-xfstests.sh
88

9-
run-cluster-test: s3proxy.jar
9+
run-cluster-test: s3proxy.jar build-debug
1010
./test/cluster/test_random.sh
1111

1212
xfstests:
@@ -22,10 +22,18 @@ get-deps: s3proxy.jar
2222
build:
2323
go build -ldflags "-X main.Version=`git rev-parse HEAD`"
2424

25+
build-debug:
26+
CGO_ENABLED=1 go build -race -ldflags "-X main.Version=`git rev-parse HEAD`"
27+
2528
install:
2629
go install -ldflags "-X main.Version=`git rev-parse HEAD`"
2730

2831

2932
.PHONY: protoc
3033
protoc:
3134
protoc --go_out=. --experimental_allow_proto3_optional --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative core/pb/*.proto
35+
36+
clean:
37+
rm -f geesefs
38+
rm -f core/mount_GoofysTest.*log
39+
findmnt -t fuse.geesefs -n -o TARGET|xargs -r umount

‎core/backend_s3.go‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -445,32 +445,32 @@ func withHeader(req *request.Request, key, value string) {
445445

446446
func (s *S3Backend) ListObjectsV2(params *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, string, error) {
447447
/*
448-
if s.config.ListV1Ext {
449-
in := s3.ListObjectsV1ExtInput(*params)
450-
req, resp := s.S3.ListObjectsV1ExtRequest(&in)
451-
if s.flags.TigrisPrefetch {
452-
withHeader(req, "X-Tigris-Prefetch", "true")
453-
}
454-
err := req.Send()
455-
if err != nil {
456-
if awsErr, ok := err.(awserr.Error); ok {
457-
if awsErr.Code() == "InvalidArgument" || awsErr.Code() == "NotImplemented" {
458-
// Fallback to list v1
459-
s.config.ListV1Ext = false
460-
return s.ListObjectsV2(params)
448+
if s.config.ListV1Ext {
449+
in := s3.ListObjectsV1ExtInput(*params)
450+
req, resp := s.S3.ListObjectsV1ExtRequest(&in)
451+
if s.flags.TigrisPrefetch {
452+
withHeader(req, "X-Tigris-Prefetch", "true")
453+
}
454+
err := req.Send()
455+
if err != nil {
456+
if awsErr, ok := err.(awserr.Error); ok {
457+
if awsErr.Code() == "InvalidArgument" || awsErr.Code() == "NotImplemented" {
458+
// Fallback to list v1
459+
s.config.ListV1Ext = false
460+
return s.ListObjectsV2(params)
461+
}
461462
}
463+
return nil, "", err
462464
}
463-
return nil, "", err
464-
}
465-
out := s3.ListObjectsV2Output(*resp)
466-
for _, obj := range out.Contents {
467-
// Make non-nil maps for all objects so that we know metadata is empty
468-
if obj.UserMetadata == nil {
469-
obj.UserMetadata = make(map[string]*string)
465+
out := s3.ListObjectsV2Output(*resp)
466+
for _, obj := range out.Contents {
467+
// Make non-nil maps for all objects so that we know metadata is empty
468+
if obj.UserMetadata == nil {
469+
obj.UserMetadata = make(map[string]*string)
470+
}
470471
}
471-
}
472-
return &out, s.getRequestId(req), nil
473-
} else
472+
return &out, s.getRequestId(req), nil
473+
} else
474474
*/
475475
if s.config.ListV2 {
476476
req, resp := s.S3.ListObjectsV2Request(params)
@@ -606,7 +606,7 @@ func (s *S3Backend) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) {
606606
LastModified: i.LastModified,
607607
Size: uint64(*i.Size),
608608
StorageClass: i.StorageClass,
609-
// Metadata: i.UserMetadata,
609+
// Metadata: i.UserMetadata,
610610
})
611611
}
612612

@@ -1028,33 +1028,33 @@ func (s *S3Backend) selectStorageClass(size *uint64) *string {
10281028
func (s *S3Backend) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error) {
10291029
return nil, fmt.Errorf("not implemented")
10301030
/*
1031-
patch := &s3.PatchObjectInput{
1032-
Bucket: &s.bucket,
1033-
Key: &param.Key,
1034-
ContentRange: PString(fmt.Sprintf("bytes %d-%d/*", param.Offset, param.Offset+param.Size-1)),
1035-
Body: param.Body,
1036-
}
1037-
if param.AppendPartSize > 0 {
1038-
patch.PatchAppendPartSize = &param.AppendPartSize
1039-
}
1031+
patch := &s3.PatchObjectInput{
1032+
Bucket: &s.bucket,
1033+
Key: &param.Key,
1034+
ContentRange: PString(fmt.Sprintf("bytes %d-%d/*", param.Offset, param.Offset+param.Size-1)),
1035+
Body: param.Body,
1036+
}
1037+
if param.AppendPartSize > 0 {
1038+
patch.PatchAppendPartSize = &param.AppendPartSize
1039+
}
10401040
1041-
req, resp := s.PatchObjectRequest(patch)
1042-
err := req.Send()
1043-
if err != nil {
1044-
if awsErr, ok := err.(awserr.Error); ok {
1045-
if awsErr.Code() == "NotImplemented" {
1046-
return nil, syscall.ENOSYS
1041+
req, resp := s.PatchObjectRequest(patch)
1042+
err := req.Send()
1043+
if err != nil {
1044+
if awsErr, ok := err.(awserr.Error); ok {
1045+
if awsErr.Code() == "NotImplemented" {
1046+
return nil, syscall.ENOSYS
1047+
}
10471048
}
1049+
return nil, err
10481050
}
1049-
return nil, err
1050-
}
10511051
1052-
return &PatchBlobOutput{
1053-
ETag: resp.Object.ETag,
1054-
LastModified: resp.Object.LastModified,
1055-
RequestId: s.getRequestId(req),
1056-
}, nil
1057-
*/
1052+
return &PatchBlobOutput{
1053+
ETag: resp.Object.ETag,
1054+
LastModified: resp.Object.LastModified,
1055+
RequestId: s.getRequestId(req),
1056+
}, nil
1057+
*/
10581058
}
10591059

10601060
func (s *S3Backend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) {

‎core/backend_test.go‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
package core
1818

19+
import "sync"
20+
1921
type TestBackend struct {
2022
StorageBackend
2123
ListBlobsFunc func(param *ListBlobsInput) (*ListBlobsOutput, error)
@@ -25,16 +27,36 @@ type TestBackend struct {
2527
MultipartBlobCommitFunc func(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)
2628
capabilities *Capabilities
2729
err error
30+
lock sync.RWMutex
31+
}
32+
33+
func NewTestBackend(init *TestBackend) *TestBackend {
34+
t := &TestBackend{}
35+
t.lock.Lock()
36+
defer t.lock.Unlock()
37+
t.StorageBackend = init.StorageBackend
38+
t.ListBlobsFunc = init.ListBlobsFunc
39+
t.HeadBlobFunc = init.HeadBlobFunc
40+
t.MultipartBlobAddFunc = init.MultipartBlobAddFunc
41+
t.MultipartBlobCopyFunc = init.MultipartBlobCopyFunc
42+
t.MultipartBlobCommitFunc = init.MultipartBlobCommitFunc
43+
t.capabilities = init.capabilities
44+
t.err = init.err
45+
return t
2846
}
2947

3048
func (s *TestBackend) Init(bucket string) error {
49+
s.lock.RLock()
50+
defer s.lock.RUnlock()
3151
if s.StorageBackend == nil {
3252
return nil
3353
}
3454
return s.StorageBackend.Init(bucket)
3555
}
3656

3757
func (s *TestBackend) Capabilities() *Capabilities {
58+
s.lock.RLock()
59+
defer s.lock.RUnlock()
3860
if s.StorageBackend == nil {
3961
if s.capabilities == nil {
4062
s.capabilities = &Capabilities{
@@ -47,11 +69,19 @@ func (s *TestBackend) Capabilities() *Capabilities {
4769
return s.StorageBackend.Capabilities()
4870
}
4971

72+
func (s *TestBackend) SetErr(err error) {
73+
s.lock.Lock()
74+
defer s.lock.Unlock()
75+
s.err = err
76+
}
77+
5078
func (s *TestBackend) Delegate() interface{} {
5179
return s
5280
}
5381

5482
func (s *TestBackend) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) {
83+
s.lock.RLock()
84+
defer s.lock.RUnlock()
5585
if s.HeadBlobFunc != nil {
5686
return s.HeadBlobFunc(param)
5787
}
@@ -62,6 +92,8 @@ func (s *TestBackend) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) {
6292
}
6393

6494
func (s *TestBackend) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) {
95+
s.lock.RLock()
96+
defer s.lock.RUnlock()
6597
if s.ListBlobsFunc != nil {
6698
return s.ListBlobsFunc(param)
6799
}
@@ -72,55 +104,71 @@ func (s *TestBackend) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)
72104
}
73105

74106
func (s *TestBackend) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) {
107+
s.lock.RLock()
108+
defer s.lock.RUnlock()
75109
if s.err != nil {
76110
return nil, s.err
77111
}
78112
return s.StorageBackend.DeleteBlob(param)
79113
}
80114

81115
func (s *TestBackend) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) {
116+
s.lock.RLock()
117+
defer s.lock.RUnlock()
82118
if s.err != nil {
83119
return nil, s.err
84120
}
85121
return s.StorageBackend.DeleteBlobs(param)
86122
}
87123

88124
func (s *TestBackend) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) {
125+
s.lock.RLock()
126+
defer s.lock.RUnlock()
89127
if s.err != nil {
90128
return nil, s.err
91129
}
92130
return s.StorageBackend.RenameBlob(param)
93131
}
94132

95133
func (s *TestBackend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) {
134+
s.lock.RLock()
135+
defer s.lock.RUnlock()
96136
if s.err != nil {
97137
return nil, s.err
98138
}
99139
return s.StorageBackend.CopyBlob(param)
100140
}
101141

102142
func (s *TestBackend) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) {
143+
s.lock.RLock()
144+
defer s.lock.RUnlock()
103145
if s.err != nil {
104146
return nil, s.err
105147
}
106148
return s.StorageBackend.GetBlob(param)
107149
}
108150

109151
func (s *TestBackend) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) {
152+
s.lock.RLock()
153+
defer s.lock.RUnlock()
110154
if s.err != nil {
111155
return nil, s.err
112156
}
113157
return s.StorageBackend.PutBlob(param)
114158
}
115159

116160
func (s *TestBackend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) {
161+
s.lock.RLock()
162+
defer s.lock.RUnlock()
117163
if s.err != nil {
118164
return nil, s.err
119165
}
120166
return s.StorageBackend.MultipartBlobBegin(param)
121167
}
122168

123169
func (s *TestBackend) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) {
170+
s.lock.RLock()
171+
defer s.lock.RUnlock()
124172
if s.MultipartBlobAddFunc != nil {
125173
return s.MultipartBlobAddFunc(param)
126174
}
@@ -131,6 +179,8 @@ func (s *TestBackend) MultipartBlobAdd(param *MultipartBlobAddInput) (*Multipart
131179
}
132180

133181
func (s *TestBackend) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error) {
182+
s.lock.RLock()
183+
defer s.lock.RUnlock()
134184
if s.MultipartBlobCopyFunc != nil {
135185
return s.MultipartBlobCopyFunc(param)
136186
}
@@ -141,13 +191,17 @@ func (s *TestBackend) MultipartBlobCopy(param *MultipartBlobCopyInput) (*Multipa
141191
}
142192

143193
func (s *TestBackend) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) {
194+
s.lock.RLock()
195+
defer s.lock.RUnlock()
144196
if s.err != nil {
145197
return nil, s.err
146198
}
147199
return s.StorageBackend.MultipartBlobAbort(param)
148200
}
149201

150202
func (s *TestBackend) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) {
203+
s.lock.RLock()
204+
defer s.lock.RUnlock()
151205
if s.MultipartBlobCommitFunc != nil {
152206
return s.MultipartBlobCommitFunc(param)
153207
}
@@ -158,6 +212,8 @@ func (s *TestBackend) MultipartBlobCommit(param *MultipartBlobCommitInput) (*Mul
158212
}
159213

160214
func (s *TestBackend) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) {
215+
s.lock.RLock()
216+
defer s.lock.RUnlock()
161217
if s.err != nil {
162218
return nil, s.err
163219
}

0 commit comments

Comments
 (0)