Skip to content

Commit becd84e

Browse files
YunhuiChenYunhuiChen
committed
Merge branch 'quota_test' of https://github.com/juicedata/juicefs into quota_test
2 parents a9560f4 + eb55dab commit becd84e

File tree

14 files changed

+359
-124
lines changed

14 files changed

+359
-124
lines changed

‎.github/scripts/perf/compare_mdtest_fio.sh‎

Lines changed: 69 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,18 @@ CURRENT_RESULTS=$1
55
OLD_RESULTS=$2
66
FILTER_OPS=("File read" "File stat" "File removal" "Tree removal" "Tree creation")
77

8-
# New function to extract time from time output
9-
extract_time() {
10-
local time_output=$1
11-
local real_time=$(grep -oP 'real\s+\d+m\d+\.\d+s' <<< "$time_output" | awk '{print $2}')
12-
local minutes=$(echo "$real_time" | grep -oP '\d+(?=m)')
13-
local seconds=$(echo "$real_time" | grep -oP '\d+\.\d+(?=s)' | head -1)
14-
echo "$minutes * 60 + $seconds" | bc -l
8+
# Function to extract files/s from built-in mdtest output
9+
extract_files_per_sec() {
10+
local mdtest_output=$1
11+
local files_per_sec=$(grep -oP 'Created .*files.*\(\K[0-9]+(\.[0-9]+)?(?= files/s\))' <<< "$mdtest_output" | head -1)
12+
if [[ -z "$files_per_sec" ]]; then
13+
files_per_sec=$(grep -oP '\(\K[0-9]+(\.[0-9]+)?(?= files/s\))' <<< "$mdtest_output" | head -1)
14+
fi
15+
if [[ -z "$files_per_sec" ]]; then
16+
echo "0"
17+
else
18+
echo "$files_per_sec"
19+
fi
1520
}
1621

1722
# Function to extract IOPS from fio output
@@ -30,6 +35,16 @@ extract_iops() {
3035
fi
3136
}
3237

38+
extract_bw() {
39+
local fio_output=$1
40+
local bw=$(grep -oP 'BW=\K[^, ]+' <<< "$fio_output" | head -1)
41+
if [[ -z "$bw" ]]; then
42+
echo "N/A"
43+
else
44+
echo "$bw"
45+
fi
46+
}
47+
3348
extract_metrics() {
3449
awk '{
3550
op_description=$1;
@@ -54,6 +69,7 @@ compare_with_tolerance() {
5469
local current=$1
5570
local old=$2
5671
local op_type=$3
72+
local direction=${4:-higher}
5773
tolerance=$(echo "$old * 0.2" | bc -l)
5874
lower_bound=$(echo "$old - $tolerance" | bc -l)
5975
upper_bound=$(echo "$old + $tolerance" | bc -l)
@@ -63,10 +79,20 @@ compare_with_tolerance() {
6379
echo "skip"
6480
elif (( $(echo "$current <= $upper_bound && $current >= $lower_bound" | bc -l) )); then
6581
echo "same"
66-
elif (( $(echo "$current > $old" | bc -l) )); then
67-
echo "better"
6882
else
69-
echo "worse"
83+
if [[ "$direction" == "lower" ]]; then
84+
if (( $(echo "$current < $old" | bc -l) )); then
85+
echo "better"
86+
else
87+
echo "worse"
88+
fi
89+
else
90+
if (( $(echo "$current > $old" | bc -l) )); then
91+
echo "better"
92+
else
93+
echo "worse"
94+
fi
95+
fi
7096
fi
7197
}
7298

@@ -86,7 +112,7 @@ compare_scenario() {
86112
echo "Command is : mpirun --use-hwthread-cpus --allow-run-as-root -np 4 mdtest -F -w 102400 -I 3000 -z 0"
87113
;;
88114
"scenario3")
89-
echo "Command is : cmd/mount/mount mdtest --threads 100 --dirs 3 --depth 5 --files 100 --create"
115+
echo "Command is : ./juicefs mdtest <meta-url> /mdtest_perf --threads 10 --dirs 3 --depth 3 --files 100"
90116
;;
91117
"fio_scenario4")
92118
echo "Command is : fio --name=big-write --directory=/mnt/fio --group_reporting --rw=write --direct=1 --bs=64k --end_fsync=1 --numjobs=8 --nrfiles=1 --size=2G --runtime=120"
@@ -114,15 +140,20 @@ compare_scenario() {
114140

115141
# Handle built-in mdtest scenario (scenario3)
116142
if [[ "$scenario" == "scenario3" ]]; then
117-
printf "%-30s %-12s %-12s %-12s %-12s %-12s\n" "Operation" "Current Time" "Old Time" "Diff" "Status" "Variance"
143+
printf "%-30s %-12s %-12s %-12s %-12s %-12s\n" "Operation" "Current files/s" "Old files/s" "Diff" "Status" "Variance"
118144
echo "--------------------------------------------------------------------"
119145

120-
current_time=$(extract_time "$(cat "${current_file}")")
121-
old_time=$(extract_time "$(cat "${old_file}")")
146+
current_files_per_sec=$(extract_files_per_sec "$(cat "${current_file}")")
147+
old_files_per_sec=$(extract_files_per_sec "$(cat "${old_file}")")
122148

123-
diff=$(echo "$current_time - $old_time" | bc -l)
124-
variance=$(echo "scale=2; ($current_time - $old_time)*100/$old_time" | bc -l)
125-
comparison=$(compare_with_tolerance $current_time $old_time "builtin_mdtest")
149+
diff=$(echo "$current_files_per_sec - $old_files_per_sec" | bc -l)
150+
if (( $(echo "$old_files_per_sec == 0" | bc -l) )); then
151+
variance="N/A"
152+
comparison="same"
153+
else
154+
variance=$(echo "scale=2; ($current_files_per_sec - $old_files_per_sec)*100/$old_files_per_sec" | bc -l)
155+
comparison=$(compare_with_tolerance $current_files_per_sec $old_files_per_sec "builtin_mdtest")
156+
fi
126157

127158
case $comparison in
128159
"worse") status="❌ Worse" ;;
@@ -132,8 +163,13 @@ compare_scenario() {
132163
*) status="⚠️ Unknown" ;;
133164
esac
134165

135-
printf "%-30s %-12.2f %-12.2f %-12.2f %-12s %-12s%%\n" \
136-
"Built-in mdtest" "$current_time" "$old_time" "$diff" "$status" "$variance"
166+
if [[ "$variance" == "N/A" ]]; then
167+
printf "%-30s %-12.2f %-12.2f %-12.2f %-12s %-12s\n" \
168+
"Built-in mdtest" "$current_files_per_sec" "$old_files_per_sec" "$diff" "$status" "$variance"
169+
else
170+
printf "%-30s %-12.2f %-12.2f %-12.2f %-12s %-12s%%\n" \
171+
"Built-in mdtest" "$current_files_per_sec" "$old_files_per_sec" "$diff" "$status" "$variance"
172+
fi
137173

138174
# Handle fio scenarios
139175
elif [[ "$scenario" =~ ^fio ]]; then
@@ -142,6 +178,8 @@ compare_scenario() {
142178

143179
current_iops=$(extract_iops "$(cat "${current_file}")")
144180
old_iops=$(extract_iops "$(cat "${old_file}")")
181+
current_bw=$(extract_bw "$(cat "${current_file}")")
182+
old_bw=$(extract_bw "$(cat "${old_file}")")
145183

146184
diff=$(echo "$current_iops - $old_iops" | bc -l)
147185
variance=$(echo "scale=2; ($current_iops - $old_iops)*100/$old_iops" | bc -l)
@@ -157,7 +195,8 @@ compare_scenario() {
157195

158196
printf "%-30s %-12.2f %-12.2f %-12.2f %-12s %-12s%%\n" \
159197
"FIO ${scenario}" "$current_iops" "$old_iops" "$diff" "$status" "$variance"
160-
198+
printf "%-30s %-12s %-12s\n" "Bandwidth" "$current_bw" "$old_bw"
199+
161200
# Handle mdtest scenarios
162201
else
163202
printf "%-30s %-12s %-12s %-12s %-12s %-12s\n" "Operation" "Current Max" "Old Max" "Diff" "Status" "Variance"
@@ -214,13 +253,17 @@ check_regression() {
214253

215254
# Handle built-in mdtest scenario (scenario3)
216255
if [[ "$scenario" == "scenario3" ]]; then
217-
current_time=$(extract_time "$(cat "${current_file}")")
218-
old_time=$(extract_time "$(cat "${old_file}")")
219-
comparison=$(compare_with_tolerance $current_time $old_time "builtin_mdtest")
256+
current_files_per_sec=$(extract_files_per_sec "$(cat "${current_file}")")
257+
old_files_per_sec=$(extract_files_per_sec "$(cat "${old_file}")")
258+
if (( $(echo "$old_files_per_sec == 0" | bc -l) )); then
259+
comparison="same"
260+
else
261+
comparison=$(compare_with_tolerance $current_files_per_sec $old_files_per_sec "builtin_mdtest")
262+
fi
220263

221264
if [ "$comparison" == "worse" ]; then
222-
variance=$(echo "scale=2; ($current_time - $old_time)*100/$old_time" | bc -l)
223-
echo "Regression detected in $scenario for built-in mdtest: Current $current_time vs Old $old_time (Variance: ${variance}%)"
265+
variance=$(echo "scale=2; ($current_files_per_sec - $old_files_per_sec)*100/$old_files_per_sec" | bc -l)
266+
echo "Regression detected in $scenario for built-in mdtest (files/s): Current $current_files_per_sec vs Old $old_files_per_sec (Variance: ${variance}%)"
224267
regression_detected=1
225268
fi
226269

@@ -235,7 +278,7 @@ check_regression() {
235278
echo "Regression detected in $scenario: Current $current_iops IOPS vs Old $old_iops IOPS (Variance: ${variance}%)"
236279
regression_detected=1
237280
fi
238-
281+
239282
# Handle mdtest scenarios
240283
else
241284
while IFS= read -r current_line && IFS= read -r old_line <&3; do

‎.github/scripts/perf/mdtest_fio.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@ set -e
44
MNT_POINT=$1
55
RESULTS_FILE=$2
66
VERSION=$3
7+
META_URL=$4
8+
9+
if [[ -z "$META_URL" ]]; then
10+
echo "ERROR: META_URL is required as 4th argument for built-in mdtest scenario"
11+
exit 1
12+
fi
713

814
mkdir -p "$(dirname "$RESULTS_FILE")"
915

@@ -123,7 +129,7 @@ done
123129
echo "Running scenario 3 (built-in mdtest)..."
124130
output_file="${RESULTS_FILE}.scenario3.run1"
125131
echo 3 | sudo tee /proc/sys/vm/drop_caches
126-
{ time sudo ./juicefs mdtest "$MNT_POINT" --threads 10 --dirs 3 --depth 3 --files 100 --create; } 2>&1 | tee "$output_file"
132+
{ time sudo ./juicefs mdtest "$META_URL" /mdtest_perf --threads 10 --dirs 3 --depth 3 --files 100; } 2>&1 | tee "$output_file"
127133
process_run "$output_file" "scenario3" 1
128134

129135
# Fio Scenario 4: Concurrent sequential write of 1 big file per thread (16 threads)

‎.github/workflows/command-win.yml‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,19 @@ jobs:
8282
.\install.ps1 -RunAsAdmin
8383
echo $env:USERNAME
8484
scoop
85-
scoop install redis
86-
scoop reset redis
87-
$env:Path += ";$env:USERPROFILE\scoop\shims"
85+
$redisUrl = "https://github.com/tporadowski/redis/releases/download/v5.0.14.1/Redis-x64-5.0.14.1.zip"
86+
$redisRoot = Join-Path $env:USERPROFILE "scoop\apps\redis\current"
87+
New-Item -ItemType Directory -Force -Path $redisRoot | Out-Null
88+
Invoke-WebRequest -Uri $redisUrl -OutFile redis.zip
89+
Expand-Archive -Path redis.zip -DestinationPath $redisRoot -Force
90+
$redisCli = Join-Path $redisRoot "redis-cli.exe"
91+
if (-not (Test-Path $redisCli)) {
92+
throw "redis-cli.exe not found after downloading from $redisUrl"
93+
}
94+
$env:Path += ";$redisRoot"
95+
$redisRoot | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
8896
echo "Updated PATH: $env:Path"
89-
redis-cli --version
97+
& $redisCli --version
9098
scoop install minio@2021-12-10T23-03-39Z
9199
scoop install runasti
92100
@@ -98,15 +106,26 @@ jobs:
98106
- name: Start Redis
99107
run: |
100108
copy winsw.exe redis-service.exe
101-
echo "<service>" >> redis-service.xml
102-
echo "<id>redisredis</id>" >> redis-service.xml
103-
echo "<name>redisredis</name>" >> redis-service.xml
104-
echo "<description>redisredis</description>" >> redis-service.xml
105-
echo "<executable>C:\Users\$env:USERNAME\scoop\shims\redis-server.exe</executable>" >> redis-service.xml
106-
echo "<logmode>rotate</logmode>" >> redis-service.xml
107-
echo "</service>" >> redis-service.xml
109+
$redisRoot = Join-Path $env:USERPROFILE "scoop\apps\redis\current"
110+
$redisExe = Join-Path $redisRoot "redis-server.exe"
111+
if (-not (Test-Path $redisExe)) {
112+
throw "redis-server.exe not found: $redisExe"
113+
}
114+
@"
115+
<service>
116+
<id>redisredis</id>
117+
<name>redisredis</name>
118+
<description>redisredis</description>
119+
<executable>$redisExe</executable>
120+
<arguments>--bind 127.0.0.1 --port 6379 --save \"\" --appendonly no</arguments>
121+
<logmode>rotate</logmode>
122+
</service>
123+
"@ | Set-Content redis-service.xml -Encoding utf8
108124
.\redis-service.exe install
109125
net start redisredis
126+
Start-Sleep -Seconds 2
127+
$redisCli = Join-Path $redisRoot "redis-cli.exe"
128+
& $redisCli -h 127.0.0.1 -p 6379 ping
110129
111130
- name: Download MinGW
112131
run: |

‎.github/workflows/perf-test.yml‎

Lines changed: 48 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ jobs:
4141
strategy:
4242
fail-fast: false
4343
matrix:
44-
meta: ['redis', 'mysql', 'tikv']
45-
cases: ["ai","mdtest_fio"]
44+
meta: ['redis', 'redis-nocache', 'mysql', 'tikv']
45+
cases: ["mdtest_fio"]
4646
runs-on: ubuntu-22.04
4747

4848
steps:
@@ -76,8 +76,12 @@ jobs:
7676
run: |
7777
chmod +x .github/scripts/start_meta_engine.sh
7878
source .github/scripts/start_meta_engine.sh
79-
start_meta_engine ${{matrix.meta}}
80-
meta_url=$(get_meta_url ${{matrix.meta}})
79+
engine_meta=${{matrix.meta}}
80+
if [[ "$engine_meta" == "redis-nocache" ]]; then
81+
engine_meta="redis"
82+
fi
83+
start_meta_engine $engine_meta
84+
meta_url=$(get_meta_url $engine_meta)
8185
create_database $meta_url
8286
8387
- name: Install mdtest & fio
@@ -113,34 +117,42 @@ jobs:
113117
sudo chmod 777 /tmp/jfs
114118
115119
# Mount current version
116-
meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url ${{matrix.meta}})
120+
engine_meta=${{matrix.meta}}
121+
if [[ "$engine_meta" == "redis-nocache" ]]; then
122+
engine_meta="redis"
123+
fi
124+
meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url $engine_meta)
117125
sudo chmod 777 /mnt
118-
if [[ "${{matrix.meta}}" == "mysql" ]]; then
126+
if [[ "${{matrix.meta}}" == "redis-nocache" ]]; then
119127
meta_url=${meta_url%%\?*}
120-
echo "Removed query parameters from MySQL URL"
128+
echo "Removed redis query parameters for redis-nocache profile"
121129
fi
122130
./juicefs format $meta_url current-version-test --trash-days 0 --bucket=/mnt/jfs
123131
./juicefs mount -d "$meta_url" /tmp/jfs --no-usage-report
124132
125133
# Run tests
126134
chmod +x .github/scripts/perf/*.sh
127-
.github/scripts/perf/${{matrix.cases}}.sh /tmp/jfs "./results/current_${{matrix.meta}}" "current"
135+
.github/scripts/perf/${{matrix.cases}}.sh /tmp/jfs "./results/current_${{matrix.meta}}" "current" "$meta_url"
128136
129137
- name: Cleanup current version
130138
run: |
131139
source .github/scripts/common/common.sh
132140
source .github/scripts/start_meta_engine.sh
133-
meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url ${{matrix.meta}})
134-
if [[ "${{matrix.meta}}" == "mysql" ]]; then
141+
engine_meta=${{matrix.meta}}
142+
if [[ "$engine_meta" == "redis-nocache" ]]; then
143+
engine_meta="redis"
144+
fi
145+
meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url $engine_meta)
146+
if [[ "${{matrix.meta}}" == "redis-nocache" ]]; then
135147
meta_url=${meta_url%%\?*}
136-
echo "Removed query parameters from MySQL URL"
148+
echo "Removed redis query parameters for redis-nocache profile"
137149
fi
138150
META_URL=$meta_url
139151
uuid=$(./juicefs status $meta_url | grep UUID | cut -d '"' -f 4)
140152
./juicefs destroy --force $meta_url $uuid
141153
./juicefs umount /tmp/jfs
142154
rm -rf /mnt/jfs
143-
start_meta_engine ${{matrix.meta}}
155+
start_meta_engine $engine_meta
144156
create_database $meta_url
145157
prepare_test
146158
@@ -153,12 +165,16 @@ jobs:
153165
git clone $GITHUB_SERVER_URL/$GITHUB_REPOSITORY .
154166
git checkout ${{ inputs.old_version_commit }}
155167
make
156-
meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url ${{matrix.meta}})
168+
engine_meta=${{matrix.meta}}
169+
if [[ "$engine_meta" == "redis-nocache" ]]; then
170+
engine_meta="redis"
171+
fi
172+
meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url $engine_meta)
157173
mkdir -p /tmp/jfs_old/mdtest
158174
sudo chmod 777 /tmp/jfs_old
159-
if [[ "${{matrix.meta}}" == "mysql" ]]; then
175+
if [[ "$engine_meta" == "mysql" || "$engine_meta" == "redis" ]]; then
160176
meta_url=${meta_url%%\?*}
161-
echo "Removed query parameters from MySQL URL"
177+
echo "Removed query parameters for old version compatibility"
162178
fi
163179
./juicefs format "$meta_url" old-version-test --trash-days 0 --bucket=/mnt/jfs
164180
./juicefs mount -d "$meta_url" /tmp/jfs_old --no-usage-report
@@ -169,10 +185,14 @@ jobs:
169185
run: |
170186
curl -sSL https://d.juicefs.com/install | sh -
171187
JFS_LATEST_TAG=$(curl -s https://api.github.com/repos/juicedata/juicefs/releases/latest | grep 'tag_name' | cut -d '"' -f 4 | tr -d 'v')
172-
meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url ${{matrix.meta}})
173-
if [[ "${{matrix.meta}}" == "mysql" ]]; then
188+
engine_meta=${{matrix.meta}}
189+
if [[ "$engine_meta" == "redis-nocache" ]]; then
190+
engine_meta="redis"
191+
fi
192+
meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url $engine_meta)
193+
if [[ "$engine_meta" == "mysql" || "$engine_meta" == "redis" ]]; then
174194
meta_url=${meta_url%%\?*}
175-
echo "Removed query parameters from MySQL URL"
195+
echo "Removed query parameters for old version compatibility"
176196
fi
177197
juicefs format "$meta_url" old-version-test --trash-days 0 --bucket=/mnt/jfs
178198
mkdir -p /tmp/jfs_old/mdtest
@@ -181,7 +201,16 @@ jobs:
181201
182202
- name: Run benchmark with old version
183203
run: |
184-
.github/scripts/perf/${{matrix.cases}}.sh /tmp/jfs_old "./results/old_${{matrix.meta}}" "old"
204+
engine_meta=${{matrix.meta}}
205+
if [[ "$engine_meta" == "redis-nocache" ]]; then
206+
engine_meta="redis"
207+
fi
208+
meta_url=$(source .github/scripts/start_meta_engine.sh; get_meta_url $engine_meta)
209+
if [[ "$engine_meta" == "mysql" || "$engine_meta" == "redis" ]]; then
210+
meta_url=${meta_url%%\?*}
211+
echo "Removed query parameters for old version compatibility"
212+
fi
213+
.github/scripts/perf/${{matrix.cases}}.sh /tmp/jfs_old "./results/old_${{matrix.meta}}" "old" "$meta_url"
185214
186215
- name: Compare results
187216
run: |

‎.github/workflows/unit-random-tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ jobs:
6969
if [[ ${{matrix.meta}} == "tikv" ]]; then
7070
number=1500
7171
fi
72-
make unit-random-test seed=$RANDOM checks=${number} steps=200 meta=$meta_url
72+
make unit-random-test seed=$RANDOM checks=${number} steps=200 meta="$meta_url"
7373
7474
- name: print failfile content
7575
if: failure()

0 commit comments

Comments
 (0)