Skip to content

Commit efd1890

Browse files
potetopoteto
committed
[ci] Parallelize yarn build
ghstack-source-id: c93d81e Pull Request resolved: #30071
1 parent ceee12e commit efd1890

File tree

5 files changed

+978
-44
lines changed

5 files changed

+978
-44
lines changed

‎.circleci/config.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ jobs:
9797
steps:
9898
- checkout
9999
- setup_node_modules
100-
- run: yarn build
100+
- run: yarn build --ci=circleci
101101
- persist_to_workspace:
102102
root: .
103103
paths:

‎.github/workflows/runtime_build.yml‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ jobs:
1111
build_and_lint:
1212
name: yarn build and lint
1313
runs-on: ubuntu-latest
14+
strategy:
15+
matrix:
16+
# yml is dumb. update the --total arg to yarn build if you change the number of workers
17+
worker_id: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
18+
release_channel: [stable, experimental]
1419
steps:
1520
- uses: actions/checkout@v4
1621
- uses: actions/setup-node@v4
@@ -25,11 +30,18 @@ jobs:
2530
path: "**/node_modules"
2631
key: ${{ runner.arch }}-${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
2732
- run: yarn install --frozen-lockfile
28-
- run: yarn build
29-
- run: yarn lint-build
30-
- name: Cache build
31-
uses: actions/cache@v4
32-
id: build_cache
33+
- run: yarn build --index=${{ matrix.worker_id }} --total=20 --r=${{ matrix.release_channel }} --ci=github
34+
env:
35+
CI: github
36+
RELEASE_CHANNEL: ${{ matrix.release_channel }}
37+
NODE_INDEX: ${{ matrix.worker_id }}
38+
- name: Lint build
39+
run: yarn lint-build
40+
- name: Display structure of build
41+
run: ls -R build
42+
- name: Archive build
43+
uses: actions/upload-artifact@v4
3344
with:
34-
path: "build/**"
35-
key: yarn-build-${{ runner.arch }}-${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
45+
name: build_${{ matrix.worker_id }}_${{ matrix.release_channel }}
46+
path: |
47+
build

‎scripts/rollup/build-all-release-channels.js‎

Lines changed: 84 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ const {
1515
canaryChannelLabel,
1616
rcNumber,
1717
} = require('../../ReactVersions');
18+
const yargs = require('yargs');
19+
const {buildEverything} = require('./build-ghaction');
1820

1921
// Runs the build script for both stable and experimental release channels,
2022
// by configuring an environment variable.
@@ -53,44 +55,88 @@ fs.writeFileSync(
5355
`export default '${PLACEHOLDER_REACT_VERSION}';\n`
5456
);
5557

56-
if (process.env.CIRCLE_NODE_TOTAL) {
57-
// In CI, we use multiple concurrent processes. Allocate half the processes to
58-
// build the stable channel, and the other half for experimental. Override
59-
// the environment variables to "trick" the underlying build script.
60-
const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
61-
const halfTotal = Math.floor(total / 2);
62-
const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
63-
if (index < halfTotal) {
64-
const nodeTotal = halfTotal;
65-
const nodeIndex = index;
66-
buildForChannel('stable', nodeTotal, nodeIndex);
67-
processStable('./build');
58+
const argv = yargs.wrap(yargs.terminalWidth()).options({
59+
releaseChannel: {
60+
alias: 'r',
61+
describe: 'Build the given release channel.',
62+
requiresArg: true,
63+
type: 'string',
64+
choices: ['experimental', 'stable'],
65+
},
66+
index: {
67+
alias: 'i',
68+
describe: 'Worker id.',
69+
requiresArg: true,
70+
type: 'number',
71+
},
72+
total: {
73+
alias: 't',
74+
describe: 'Total number of workers.',
75+
requiresArg: true,
76+
type: 'number',
77+
},
78+
ci: {
79+
describe: 'Run tests in CI',
80+
requiresArg: false,
81+
type: 'choices',
82+
choices: ['circleci', 'github'],
83+
},
84+
}).argv;
85+
86+
async function main() {
87+
if (argv.ci === 'github') {
88+
await buildEverything(argv.index, argv.total);
89+
switch (argv.releaseChannel) {
90+
case 'stable': {
91+
processStable('./build');
92+
break;
93+
}
94+
case 'experimental': {
95+
processExperimental('./build');
96+
break;
97+
}
98+
default:
99+
throw new Error(`Unknown release channel ${argv.releaseChannel}`);
100+
}
101+
} else if (argv.ci === 'circleci') {
102+
// In CI, we use multiple concurrent processes. Allocate half the processes to
103+
// build the stable channel, and the other half for experimental. Override
104+
// the environment variables to "trick" the underlying build script.
105+
const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
106+
const halfTotal = Math.floor(total / 2);
107+
const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
108+
if (index < halfTotal) {
109+
const nodeTotal = halfTotal;
110+
const nodeIndex = index;
111+
buildForChannel('stable', nodeTotal, nodeIndex);
112+
processStable('./build');
113+
} else {
114+
const nodeTotal = total - halfTotal;
115+
const nodeIndex = index - halfTotal;
116+
buildForChannel('experimental', nodeTotal, nodeIndex);
117+
processExperimental('./build');
118+
}
68119
} else {
69-
const nodeTotal = total - halfTotal;
70-
const nodeIndex = index - halfTotal;
71-
buildForChannel('experimental', nodeTotal, nodeIndex);
72-
processExperimental('./build');
120+
// Running locally, no concurrency. Move each channel's build artifacts into
121+
// a temporary directory so that they don't conflict.
122+
buildForChannel('stable', '', '');
123+
const stableDir = tmp.dirSync().name;
124+
crossDeviceRenameSync('./build', stableDir);
125+
processStable(stableDir);
126+
buildForChannel('experimental', '', '');
127+
const experimentalDir = tmp.dirSync().name;
128+
crossDeviceRenameSync('./build', experimentalDir);
129+
processExperimental(experimentalDir);
130+
131+
// Then merge the experimental folder into the stable one. processExperimental
132+
// will have already removed conflicting files.
133+
//
134+
// In CI, merging is handled automatically by CircleCI's workspace feature.
135+
mergeDirsSync(experimentalDir + '/', stableDir + '/');
136+
137+
// Now restore the combined directory back to its original name
138+
crossDeviceRenameSync(stableDir, './build');
73139
}
74-
} else {
75-
// Running locally, no concurrency. Move each channel's build artifacts into
76-
// a temporary directory so that they don't conflict.
77-
buildForChannel('stable', '', '');
78-
const stableDir = tmp.dirSync().name;
79-
crossDeviceRenameSync('./build', stableDir);
80-
processStable(stableDir);
81-
buildForChannel('experimental', '', '');
82-
const experimentalDir = tmp.dirSync().name;
83-
crossDeviceRenameSync('./build', experimentalDir);
84-
processExperimental(experimentalDir);
85-
86-
// Then merge the experimental folder into the stable one. processExperimental
87-
// will have already removed conflicting files.
88-
//
89-
// In CI, merging is handled automatically by CircleCI's workspace feature.
90-
mergeDirsSync(experimentalDir + '/', stableDir + '/');
91-
92-
// Now restore the combined directory back to its original name
93-
crossDeviceRenameSync(stableDir, './build');
94140
}
95141

96142
function buildForChannel(channel, nodeTotal, nodeIndex) {
@@ -457,3 +503,5 @@ function mergeDirsSync(source, destination) {
457503
}
458504
}
459505
}
506+
507+
main();

0 commit comments

Comments
 (0)