Skip to content

Commit 0455094

Browse files
efirsefirs
committed
fix: Fix cluster mode to pass the tests
1 parent eee47f0 commit 0455094

File tree

13 files changed

+146
-70
lines changed

13 files changed

+146
-70
lines changed

‎.github/workflows/cluster_test.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: Test
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
9+
jobs:
10+
cluster-tests:
11+
runs-on: ubuntu-latest
12+
steps:
13+
- uses: actions/checkout@v4
14+
15+
- name: Install test dependencies
16+
run: sudo apt-get install -y s3cmd
17+
18+
- name: Build linux amd64
19+
run: env CGO_ENABLED=0 go build -o geesefs-linux-amd64 -v && ln -s geesefs-linux-amd64 geesefs
20+
21+
- name: Run cluster tests
22+
run: NUM_ITER=100 SAME_PROCESS_MOUNT=1 make run-cluster-test
23+
timeout-minutes: 12

‎.github/workflows/test.yml‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,26 +7,14 @@ on:
77
- main
88

99
jobs:
10-
11-
build:
10+
unittests:
1211
runs-on: ubuntu-latest
1312
steps:
14-
- uses: actions/checkout@v2
15-
16-
- name: Set up Go
17-
uses: actions/setup-go@v2
18-
with:
19-
go-version: 1.22
13+
- uses: actions/checkout@v4
2014

2115
- name: Build linux amd64
2216
run: env CGO_ENABLED=0 go build -o geesefs-linux-amd64 -v && ln -s geesefs-linux-amd64 geesefs
2317

2418
- name: Run tests
2519
run: SAME_PROCESS_MOUNT=1 make run-test
2620
timeout-minutes: 12
27-
28-
- name: Run xfstests
29-
# Often crashes in CI due to memory limits
30-
continue-on-error: true
31-
run: make run-xfstests
32-
timeout-minutes: 10

‎.github/workflows/xfstests.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: Test
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
9+
jobs:
10+
xfstests:
11+
runs-on: ubuntu-latest
12+
steps:
13+
- uses: actions/checkout@v4
14+
15+
- name: Build linux amd64
16+
run: env CGO_ENABLED=0 go build -o geesefs-linux-amd64 -v && ln -s geesefs-linux-amd64 geesefs
17+
18+
- name: Run xfstests
19+
# Often crashes in CI due to memory limits
20+
continue-on-error: true
21+
run: make run-xfstests
22+
timeout-minutes: 10

‎Makefile‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
export CGO_ENABLED=0
22

3-
run-test: s3proxy.jar
3+
run-test: s3proxy.jar build
44
./test/run-tests.sh
55

66
run-xfstests: s3proxy.jar xfstests
77
./test/run-xfstests.sh
88

9+
run-cluster-test: s3proxy.jar
10+
./test/cluster/test_random.sh
11+
912
xfstests:
1013
git clone --depth=1 https://github.com/kdave/xfstests
1114
cd xfstests && patch -p1 -l < ../test/xfstests.diff
@@ -25,4 +28,4 @@ install:
2528

2629
.PHONY: protoc
2730
protoc:
28-
protoc --go_out=. --experimental_allow_proto3_optional --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative internal/pb/*.proto
31+
protoc --go_out=. --experimental_allow_proto3_optional --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative internal/pb/*.proto

‎internal/cluster_fs.go‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -516,9 +516,7 @@ func (fs *ClusterFs) lookUpInode2(inode *Inode) (pbAttr *pb.Attributes, err erro
516516

517517
// REQUIRED_LOCK(inode.KeepOwnerLock)
518518
func (fs *ClusterFs) getInodeAttributes(inode *Inode, size *uint64, mtime *time.Time, ctime *time.Time, mode *os.FileMode) {
519-
inode.mu.Lock()
520519
attr := inode.GetAttributes()
521-
inode.mu.Unlock()
522520

523521
*size = attr.Size
524522
*mtime = attr.Mtime
@@ -530,6 +528,9 @@ func (fs *ClusterFs) getInodeAttributes(inode *Inode, size *uint64, mtime *time.
530528
func (fs *ClusterFs) setInodeAttributes(inode *Inode, size *uint64, mtime *time.Time, ctime *time.Time, mode *os.FileMode) error {
531529
modified := false
532530

531+
inode.mu.Lock()
532+
defer inode.mu.Unlock()
533+
533534
if size != nil && inode.Attributes.Size != *size {
534535
if *size > fs.Goofys.getMaxFileSize() {
535536
return syscall.EFBIG
@@ -560,7 +561,7 @@ func (fs *ClusterFs) setInodeAttributes(inode *Inode, size *uint64, mtime *time.
560561
inode.fs.WakeupFlusher()
561562
}
562563

563-
attr := inode.GetAttributes()
564+
attr := inode.InflateAttributes()
564565

565566
*size = attr.Size
566567
*mtime = attr.Mtime
@@ -660,6 +661,7 @@ func (fs *ClusterFs) applyStolenInode(inode *Inode, stolenInode *pb.StolenInode)
660661
inode.Attributes.Mtime = stolenInode.Attr.Mtime.AsTime()
661662
inode.Attributes.Ctime = stolenInode.Attr.Ctime.AsTime()
662663
inode.Attributes.Mode = iofs.FileMode(stolenInode.Attr.Mode)
664+
inode.knownSize = stolenInode.Attr.Size
663665

664666
inode.refcnt = stolenInode.Refcnt
665667

@@ -729,6 +731,7 @@ func (fs *ClusterFs) tryYield(inode *Inode, newOwner NodeId) *pb.StolenInode {
729731
} else {
730732
fuseLog.Infof("could not yield inode %v: len(inode.dir.DeletedChildren) == %v",
731733
inode.Id, len(inode.dir.DeletedChildren))
734+
inode.TryFlush(MAX_FLUSH_PRIORITY)
732735
return nil
733736
}
734737
} else {
@@ -754,6 +757,7 @@ func (fs *ClusterFs) tryYield(inode *Inode, newOwner NodeId) *pb.StolenInode {
754757
} else {
755758
fuseLog.Infof("could not yield inode %v: inode.CacheState == %v inode.fileHandles == %v",
756759
inode.Id, inode.CacheState, inode.fileHandles)
760+
inode.TryFlush(MAX_FLUSH_PRIORITY)
757761
return nil
758762
}
759763
}

‎internal/cluster_fs_fuse.go‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@ package internal
44

55
import (
66
"context"
7-
iofs "io/fs"
87
"fmt"
9-
"syscall"
8+
iofs "io/fs"
109
"sync/atomic"
10+
"syscall"
1111

12-
"github.com/yandex-cloud/geesefs/internal/cfg"
1312
"github.com/jacobsa/fuse"
1413
"github.com/jacobsa/fuse/fuseops"
1514
"github.com/jacobsa/fuse/fuseutil"
16-
"github.com/yandex-cloud/geesefs/internal/pb"
1715
"github.com/sirupsen/logrus"
16+
"github.com/yandex-cloud/geesefs/internal/cfg"
17+
"github.com/yandex-cloud/geesefs/internal/pb"
1818
"google.golang.org/grpc"
1919
"google.golang.org/protobuf/types/known/timestamppb"
2020
)
@@ -66,8 +66,8 @@ func (fs *ClusterFsFuse) CreateFile(ctx context.Context, op *fuseops.CreateFileO
6666
op.Entry.Attributes.Mtime = attr.Mtime.AsTime()
6767
op.Entry.Attributes.Ctime = attr.Ctime.AsTime()
6868
op.Entry.Attributes.Mode = iofs.FileMode(attr.Mode)
69-
op.Entry.Attributes.Uid = 1000
70-
op.Entry.Attributes.Gid = 1001
69+
op.Entry.Attributes.Uid = fs.Flags.Uid
70+
op.Entry.Attributes.Gid = fs.Flags.Gid
7171
op.Handle = fuseops.HandleID(handleId)
7272
},
7373
func(parent *Inode, parentOwner NodeId) *pb.Owner {
@@ -117,8 +117,8 @@ func (fs *ClusterFsFuse) CreateFile(ctx context.Context, op *fuseops.CreateFileO
117117
op.Entry.Attributes.Mtime = resp.Attr.Mtime.AsTime()
118118
op.Entry.Attributes.Ctime = resp.Attr.Ctime.AsTime()
119119
op.Entry.Attributes.Mode = iofs.FileMode(resp.Attr.Mode)
120-
op.Entry.Attributes.Uid = 1000
121-
op.Entry.Attributes.Gid = 1001
120+
op.Entry.Attributes.Uid = fs.Flags.Uid
121+
op.Entry.Attributes.Gid = fs.Flags.Gid
122122
op.Handle = fuseops.HandleID(resp.HandleId)
123123

124124
return nil
@@ -336,8 +336,8 @@ func (fs *ClusterFsFuse) CreateSymlink(ctx context.Context, op *fuseops.CreateSy
336336
op.Entry.Attributes.Mtime = attr.Mtime.AsTime()
337337
op.Entry.Attributes.Ctime = attr.Ctime.AsTime()
338338
op.Entry.Attributes.Mode = iofs.FileMode(attr.Mode)
339-
op.Entry.Attributes.Uid = 1000
340-
op.Entry.Attributes.Gid = 1001
339+
op.Entry.Attributes.Uid = fs.Flags.Uid
340+
op.Entry.Attributes.Gid = fs.Flags.Gid
341341
},
342342
func(parent *Inode, parentOwner NodeId) *pb.Owner {
343343
var resp *pb.CreateSymlinkResponse
@@ -381,8 +381,8 @@ func (fs *ClusterFsFuse) CreateSymlink(ctx context.Context, op *fuseops.CreateSy
381381
op.Entry.Attributes.Mtime = resp.Attr.Mtime.AsTime()
382382
op.Entry.Attributes.Ctime = resp.Attr.Ctime.AsTime()
383383
op.Entry.Attributes.Mode = iofs.FileMode(resp.Attr.Mode)
384-
op.Entry.Attributes.Uid = 1000
385-
op.Entry.Attributes.Gid = 1001
384+
op.Entry.Attributes.Uid = fs.Flags.Uid
385+
op.Entry.Attributes.Gid = fs.Flags.Gid
386386

387387
return nil
388388
},
@@ -450,8 +450,8 @@ func (fs *ClusterFsFuse) MkDir(ctx context.Context, op *fuseops.MkDirOp) (err er
450450
op.Entry.Attributes.Mtime = attr.Mtime.AsTime()
451451
op.Entry.Attributes.Ctime = attr.Ctime.AsTime()
452452
op.Entry.Attributes.Mode = iofs.FileMode(attr.Mode)
453-
op.Entry.Attributes.Uid = 1000
454-
op.Entry.Attributes.Gid = 1001
453+
op.Entry.Attributes.Uid = fs.Flags.Uid
454+
op.Entry.Attributes.Gid = fs.Flags.Gid
455455
},
456456
func(parent *Inode, parentOwner NodeId) *pb.Owner {
457457
// 1st phase
@@ -496,8 +496,8 @@ func (fs *ClusterFsFuse) MkDir(ctx context.Context, op *fuseops.MkDirOp) (err er
496496
op.Entry.Attributes.Mtime = resp.Attr.Mtime.AsTime()
497497
op.Entry.Attributes.Ctime = resp.Attr.Ctime.AsTime()
498498
op.Entry.Attributes.Mode = iofs.FileMode(resp.Attr.Mode)
499-
op.Entry.Attributes.Uid = 1000
500-
op.Entry.Attributes.Gid = 1001
499+
op.Entry.Attributes.Uid = fs.Flags.Uid
500+
op.Entry.Attributes.Gid = fs.Flags.Gid
501501

502502
return nil
503503
},
@@ -643,8 +643,8 @@ func (fs *ClusterFsFuse) LookUpInode(ctx context.Context, op *fuseops.LookUpInod
643643
op.Entry.Attributes.Mtime = pbAttr.Mtime.AsTime()
644644
op.Entry.Attributes.Ctime = pbAttr.Ctime.AsTime()
645645
op.Entry.Attributes.Mode = iofs.FileMode(pbAttr.Mode)
646-
op.Entry.Attributes.Uid = 1000
647-
op.Entry.Attributes.Gid = 1001
646+
op.Entry.Attributes.Uid = fs.Flags.Uid
647+
op.Entry.Attributes.Gid = fs.Flags.Gid
648648
},
649649
func(parent *Inode, parentOwner NodeId) *pb.Owner {
650650
// 1st phase
@@ -688,8 +688,8 @@ func (fs *ClusterFsFuse) LookUpInode(ctx context.Context, op *fuseops.LookUpInod
688688
op.Entry.Attributes.Mtime = resp.Attr.Mtime.AsTime()
689689
op.Entry.Attributes.Ctime = resp.Attr.Ctime.AsTime()
690690
op.Entry.Attributes.Mode = iofs.FileMode(resp.Attr.Mode)
691-
op.Entry.Attributes.Uid = 1000
692-
op.Entry.Attributes.Gid = 1001
691+
op.Entry.Attributes.Uid = fs.Flags.Uid
692+
op.Entry.Attributes.Gid = fs.Flags.Gid
693693

694694
return nil
695695
},
@@ -749,8 +749,8 @@ func (fs *ClusterFsFuse) GetInodeAttributes(ctx context.Context, op *fuseops.Get
749749
} else {
750750
op.Attributes.Nlink = 1
751751
}
752-
op.Attributes.Uid = 1000
753-
op.Attributes.Gid = 1001
752+
op.Attributes.Uid = fs.Flags.Uid
753+
op.Attributes.Gid = fs.Flags.Gid
754754
},
755755
func(inode *Inode, inodeOwner NodeId) *pb.Owner {
756756
var resp *pb.GetInodeAttributesResponse
@@ -780,8 +780,8 @@ func (fs *ClusterFsFuse) GetInodeAttributes(ctx context.Context, op *fuseops.Get
780780
op.Attributes.Mtime = resp.Attributes.Mtime.AsTime()
781781
op.Attributes.Ctime = resp.Attributes.Ctime.AsTime()
782782
op.Attributes.Mode = iofs.FileMode(resp.Attributes.Mode)
783-
op.Attributes.Uid = 1000
784-
op.Attributes.Gid = 1001
783+
op.Attributes.Uid = fs.Flags.Uid
784+
op.Attributes.Gid = fs.Flags.Gid
785785

786786
return nil
787787
},
@@ -801,8 +801,8 @@ func (fs *ClusterFsFuse) SetInodeAttributes(ctx context.Context, op *fuseops.Set
801801
} else {
802802
op.Attributes.Nlink = 1
803803
}
804-
op.Attributes.Uid = 1000
805-
op.Attributes.Gid = 1001
804+
op.Attributes.Uid = fs.Flags.Uid
805+
op.Attributes.Gid = fs.Flags.Gid
806806
},
807807
func(inode *Inode, inodeOwner NodeId) *pb.Owner {
808808
var mtime *timestamppb.Timestamp
@@ -839,8 +839,8 @@ func (fs *ClusterFsFuse) SetInodeAttributes(ctx context.Context, op *fuseops.Set
839839
op.Attributes.Mtime = resp.Attributes.Mtime.AsTime()
840840
op.Attributes.Ctime = resp.Attributes.Ctime.AsTime()
841841
op.Attributes.Mode = iofs.FileMode(resp.Attributes.Mode)
842-
op.Attributes.Uid = 1000
843-
op.Attributes.Gid = 1001
842+
op.Attributes.Uid = fs.Flags.Uid
843+
op.Attributes.Gid = fs.Flags.Gid
844844

845845
return nil
846846
},

‎internal/dir.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1546,9 +1546,12 @@ func (inode *Inode) SetCacheState(state int32) {
15461546

15471547
func (parent *Inode) addModified(inc int64) {
15481548
for parent != nil {
1549-
n := atomic.AddInt64(&parent.dir.ModifiedChildren, inc)
1550-
if n < 0 {
1551-
log.Errorf("BUG: ModifiedChildren of %v < 0", parent.FullName())
1549+
if atomic.LoadInt64(&parent.dir.ModifiedChildren) > 0 || inc > 0 {
1550+
n := atomic.AddInt64(&parent.dir.ModifiedChildren, inc)
1551+
if n < 0 {
1552+
parent.DumpTree("add_modified", true, true)
1553+
panic(fmt.Errorf("BUG: ModifiedChildren of %v < 0, n=%v, inc=%v", parent.FullName(), n, inc))
1554+
}
15521555
}
15531556
parent = parent.Parent
15541557
}
@@ -2008,6 +2011,9 @@ func (parent *Inode) LookUp(name string, doSlurp bool) (*Inode, error) {
20082011
}
20092012

20102013
func (parent *Inode) LookUpInodeMaybeDir(name string) (*BlobItemOutput, error) {
2014+
parent.mu.Lock()
2015+
defer parent.mu.Unlock()
2016+
20112017
cloud, parentKey := parent.cloud()
20122018
if cloud == nil {
20132019
panic("s3 disabled")

‎internal/goofys_unix_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ func (s *GoofysTest) TestPythonCopyTree(t *C) {
365365

366366
mountPoint := s.tmp + "/mnt" + s.fs.bucket
367367

368-
s.runFuseTest(t, mountPoint, true, "python", "-c",
368+
s.runFuseTest(t, mountPoint, true, "python3", "-c",
369369
"import shutil; shutil.copytree('dir2', 'dir5')",
370370
mountPoint)
371371
}

0 commit comments

Comments
 (0)