Skip to content

Commit 4cc30fe

Browse files
fengmk2fengmk2
authored
feat: support block sync packages (#929)
<!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added configurable block lists to prevent specific scopes and packages from syncing. * **Bug Fixes** * Enhanced validation to detect blocked and invalid packages early, with improved error messages and task finalization. * Improved error logging for sync operations involving blocked packages or unsupported versions. <sub>✏️ Tip: You can customize this high-level summary in your review settings.</sub> <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent bb580b4 commit 4cc30fe

File tree

4 files changed

+85
-16
lines changed

4 files changed

+85
-16
lines changed

‎app/core/service/PackageSyncerService.ts‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,20 @@ taskQueue: ${taskQueueLength}/${taskQueueHighWaterSize} 🚧🚧🚧🚧🚧`,
521521
return;
522522
}
523523

524+
if (await this.packageVersionFileService.isPackageBlockedToSync(scope, name)) {
525+
task.error = `stop sync by block list, see ${UNPKG_WHITE_LIST_URL}`;
526+
logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
527+
logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
528+
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
529+
this.logger.info(
530+
'[PackageSyncerService.executeTask:fail-package-blocked-to-sync] taskId: %s, targetName: %s, %s',
531+
task.taskId,
532+
task.targetName,
533+
task.error,
534+
);
535+
return;
536+
}
537+
524538
let registryFetchResult: RegistryResponse<Buffer>;
525539
try {
526540
registryFetchResult = await this.npmRegistry.getFullManifestsBuffer(fullname, {
@@ -869,17 +883,23 @@ data sample: ${remoteData.subarray(0, 200).toString()}`;
869883
}
870884
const size = dist.size ?? dist.unpackedSize;
871885
if (size && size > this.config.cnpmcore.largePackageVersionSize) {
872-
const isAllowLargePackageVersion = await this.packageVersionFileService.isAllowLargePackageVersion(
886+
const isAllowLargePackageVersion = await this.packageVersionFileService.isLargePackageVersionAllowed(
873887
scope,
874888
name,
875889
version,
876890
);
877891
if (!isAllowLargePackageVersion) {
878-
lastErrorMessage = `large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}`;
879-
logs.push(`[${isoNow()}] ❌ [${syncIndex}] Synced version ${version} fail, ${lastErrorMessage}`);
880-
await this.taskService.appendTaskLog(task, logs.join('\n'));
881-
logs = [];
882-
continue;
892+
task.error = `Synced version ${version} fail, large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}`;
893+
logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
894+
logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
895+
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
896+
this.logger.info(
897+
'[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s',
898+
task.taskId,
899+
task.targetName,
900+
task.error,
901+
);
902+
return;
883903
}
884904
logs.push(
885905
`[${isoNow()}] 🚧 [${syncIndex}] Synced version ${version} size: ${size} too large, it is allowed to sync by unpkg white list`,
@@ -1367,17 +1387,23 @@ ${diff.addedVersions.length} added, ${diff.removedVersions.length} removed, calc
13671387

13681388
const size = dist.size ?? dist.unpackedSize;
13691389
if (size && size > this.config.cnpmcore.largePackageVersionSize) {
1370-
const isAllowLargePackageVersion = await this.packageVersionFileService.isAllowLargePackageVersion(
1390+
const isAllowLargePackageVersion = await this.packageVersionFileService.isLargePackageVersionAllowed(
13711391
scope,
13721392
name,
13731393
version,
13741394
);
13751395
if (!isAllowLargePackageVersion) {
1376-
lastErrorMessage = `large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}`;
1377-
logs.push(`[${isoNow()}] ❌ [${syncIndex}] Synced version ${version} fail, ${lastErrorMessage}`);
1378-
await this.taskService.appendTaskLog(task, logs.join('\n'));
1379-
logs = [];
1380-
continue;
1396+
task.error = `Synced version ${version} fail, large package version size: ${size}, allow size: ${this.config.cnpmcore.largePackageVersionSize}, see ${UNPKG_WHITE_LIST_URL}`;
1397+
logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
1398+
logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
1399+
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
1400+
this.logger.info(
1401+
'[PackageSyncerService.executeTask:fail-large-package-version-size] taskId: %s, targetName: %s, %s',
1402+
task.taskId,
1403+
task.targetName,
1404+
task.error,
1405+
);
1406+
return;
13811407
}
13821408
logs.push(
13831409
`[${isoNow()}] 🚧 [${syncIndex}] Synced version ${version} size: ${size} too large, it is allowed to sync by unpkg white list`,

‎app/core/service/PackageVersionFileService.ts‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ export class PackageVersionFileService extends AbstractService {
6060
> = {};
6161
// allow large package scopes, e.g. ['@foo', '@bar']
6262
#unpkgWhiteListAllowLargeScopes: string[] = [];
63+
// block sync scopes, e.g. ['@foo', '@bar']
64+
#unpkgWhiteListBlockSyncScopes: string[] = [];
65+
// block sync packages, e.g. ['@foo/foo', '@foo/bar']
66+
#unpkgWhiteListBlockSyncPackages: string[] = [];
6367

6468
async listPackageVersionFiles(pkgVersion: PackageVersion, directory: string) {
6569
await this.#ensurePackageVersionFilesSync(pkgVersion);
@@ -119,16 +123,20 @@ export class PackageVersionFileService extends AbstractService {
119123
this.#unpkgWhiteListAllowScopes = manifest.allowScopes ?? ([] as any);
120124
this.#unpkgWhiteListAllowLargePackages = manifest.allowLargePackages ?? ({} as any);
121125
this.#unpkgWhiteListAllowLargeScopes = manifest.allowLargeScopes ?? ([] as any);
126+
this.#unpkgWhiteListBlockSyncScopes = manifest.blockSyncScopes ?? ([] as any);
127+
this.#unpkgWhiteListBlockSyncPackages = manifest.blockSyncPackages ?? ([] as any);
122128
this.logger.info(
123-
'[PackageVersionFileService.updateUnpkgWhiteList] version:%s, total %s packages, %s scopes, %s large packages',
129+
'[PackageVersionFileService.updateUnpkgWhiteList] version:%s, total %s packages, %s scopes, %s large packages, %s block sync scopes, %s block sync packages',
124130
whiteListPackageVersion,
125131
Object.keys(this.#unpkgWhiteListAllowPackages).length,
126132
this.#unpkgWhiteListAllowScopes.length,
127133
Object.keys(this.#unpkgWhiteListAllowLargePackages).length,
134+
this.#unpkgWhiteListBlockSyncScopes.length,
135+
this.#unpkgWhiteListBlockSyncPackages.length,
128136
);
129137
}
130138

131-
async isAllowLargePackageVersion(pkgScope: string, pkgName: string, pkgVersion: string) {
139+
async isLargePackageVersionAllowed(pkgScope: string, pkgName: string, pkgVersion: string) {
132140
if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return false;
133141
await this.#updateUnpkgWhiteList();
134142

@@ -143,6 +151,24 @@ export class PackageVersionFileService extends AbstractService {
143151
});
144152
}
145153

154+
/**
155+
* Check if the package is blocked to sync
156+
* @param pkgScope - The scope of the package
157+
* @param pkgName - The name of the package
158+
* @returns True if the package is blocked to sync, false otherwise
159+
*/
160+
async isPackageBlockedToSync(pkgScope: string, pkgName: string) {
161+
if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return false;
162+
await this.#updateUnpkgWhiteList();
163+
164+
// check block scopes
165+
if (this.#unpkgWhiteListBlockSyncScopes.includes(pkgScope)) return true;
166+
167+
// check block packages
168+
const fullname = getFullname(pkgScope, pkgName);
169+
return this.#unpkgWhiteListBlockSyncPackages.includes(fullname);
170+
}
171+
146172
async checkPackageVersionInUnpkgWhiteList(pkgScope: string, pkgName: string, pkgVersion: string) {
147173
if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return;
148174
await this.#updateUnpkgWhiteList();

‎test/core/service/PackageSyncerService/executeTask.test.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2043,6 +2043,23 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
20432043
assert.ok(log.includes('❌ stop sync by block list: ["cnpmcore-test-sync-blocklist","foo"]'));
20442044
});
20452045

2046+
it('should stop sync by block list, see https://github.com/cnpm/unpkg-white-list', async () => {
2047+
const name = 'cnpmcore-test-sync-blocklist';
2048+
mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true);
2049+
mock(PackageVersionFileService.prototype, 'isPackageBlockedToSync', async () => true);
2050+
await packageSyncerService.createTask(name);
2051+
const task = await packageSyncerService.findExecuteTask();
2052+
assert.ok(task);
2053+
assert.equal(task.targetName, name);
2054+
await packageSyncerService.executeTask(task);
2055+
const stream = await packageSyncerService.findTaskLog(task);
2056+
assert.ok(stream);
2057+
const log = await TestUtil.readStreamToLog(stream);
2058+
// console.log(log);
2059+
assert.ok(log.includes(`❌❌❌❌❌ ${name} ❌❌❌❌❌`));
2060+
assert.ok(log.includes('❌ stop sync by block list, see https://github.com/cnpm/unpkg-white-list'));
2061+
});
2062+
20462063
it('should sync upper case "D" success', async () => {
20472064
app.mockHttpclient('https://registry.npmjs.org/D', 'GET', {
20482065
data: '{"_id":"D","_rev":"9-83b2794fe968ab4d4dc5c72475afe1ed","name":"D","dist-tags":{"latest":"1.0.0"},"versions":{"0.0.1":{"name":"D","version":"0.0.1","description":"","main":"index.js","scripts":{"test":"echo \\"Error: no test specified\\" && exit 1"},"author":{"name":"zhengzk"},"license":"MIT","_id":"[email protected]","_shasum":"292f54dbd43f36e4853f6a09e77617c23c46228b","_from":".","_npmVersion":"2.15.1","_nodeVersion":"4.4.3","_npmUser":{"name":"zhengzk","email":"[email protected]"},"dist":{"shasum":"292f54dbd43f36e4853f6a09e77617c23c46228b","tarball":"https://registry.npmjs.org/D/-/D-0.0.1.tgz","integrity":"sha512-8LsXYQFO1mko5MQ7qh72xjbCtB6PiNFC+q97eEZ85vrGL5HwMb27CuhWJKbcN6LrL8/zEwQYolHSV/jMyZ4zYg==","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCICqSbDSXvMFKDfikUr0uttJJGWnrYFHfbOab67m0qOtZAiEAimvItLP13BkE5ex448H7eRtfAzOfQ1lCqNd1ygihn7A="}]},"maintainers":[{"name":"zhengzk","email":"[email protected]"}],"_npmOperationalInternal":{"host":"packages-16-east.internal.npmjs.com","tmp":"tmp/D-0.0.1.tgz_1464672428722_0.6098439614288509"},"deprecated":"Package no longer supported. Contact [email protected] for more info.","directories":{}},"1.0.0":{"name":"D","version":"1.0.0","description":"This package is no longer supported and has been deprecated. To avoid malicious use, npm is hanging on to the package name.","main":"index.js","scripts":{"test":"echo \\"Error: no test specified\\" && exit 1"},"repository":{"type":"git","url":"git+https://github.com/npm/deprecate-holder.git"},"author":"","license":"ISC","bugs":{"url":"https://github.com/npm/deprecate-holder/issues"},"homepage":"https://github.com/npm/deprecate-holder#readme","_id":"[email protected]","_npmVersion":"5.3.0","_nodeVersion":"8.2.1","_npmUser":{"name":"lisayu","email":"[email protected]"},"dist":{"integrity":"sha512-nQvrCBu7K2pSSEtIM0EEF03FVjcczCXInMt3moLNFbjlWx6bZrX72uT6/1uAXDbnzGUAx9gTyDiQ+vrFi663oA==","shasum":"c348a4e034f72847be51206fc530fc089e9cc2a9","tarball":"https://registry.npmjs.org/D/-/D-1.0.0.tgz","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCICZ8ljQHo15g72enGrhsxRggI5u1HmTzqaSwDdKK7pWMAiBophHVaH8mkvZjw45S2C65XmgOEqaKxKqv59mniAjosw=="}]},"maintainers":[{"email":"[email protected]","name":"lisayu"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/D-1.0.0.tgz_1502418423486_0.45665086852386594"},"deprecated":"Package no longer supported. Contact [email protected] for more info.","directories":{}}},"readme":"# Deprecated Package\\n\\nThis package is no longer supported and has been deprecated. To avoid malicious use, npm is hanging on to the package name.\\n\\nPlease contact [email protected] if you have questions about this package. \\n","maintainers":[{"email":"[email protected]","name":"npm"}],"time":{"modified":"2022-06-13T02:13:35.883Z","created":"2016-05-31T05:27:12.203Z","0.0.1":"2016-05-31T05:27:12.203Z","1.0.0":"2017-08-11T02:27:03.593Z"},"license":"ISC","readmeFilename":"README.md","description":"This package is no longer supported and has been deprecated. To avoid malicious use, npm is hanging on to the package name.","homepage":"https://github.com/npm/deprecate-holder#readme","repository":{"type":"git","url":"git+https://github.com/npm/deprecate-holder.git"},"bugs":{"url":"https://github.com/npm/deprecate-holder/issues"},"users":{"subbarao":true}}',
@@ -2272,7 +2289,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
22722289
});
22732290
mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true);
22742291
mock(app.config.cnpmcore, 'largePackageVersionSize', 100 * 1024 * 1024);
2275-
mock(PackageVersionFileService.prototype, 'isAllowLargePackageVersion', async () => true);
2292+
mock(PackageVersionFileService.prototype, 'isLargePackageVersionAllowed', async () => true);
22762293
const name = 'cnpmcore-test-sync-deprecated';
22772294
await packageSyncerService.createTask(name);
22782295
const task = await packageSyncerService.findExecuteTask();

‎test/core/service/PackageSyncerService/executeTaskWithPackument.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2248,7 +2248,7 @@ describe('test/core/service/PackageSyncerService/executeTaskWithPackument.test.t
22482248
});
22492249
mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true);
22502250
mock(app.config.cnpmcore, 'largePackageVersionSize', 100 * 1024 * 1024);
2251-
mock(PackageVersionFileService.prototype, 'isAllowLargePackageVersion', async () => true);
2251+
mock(PackageVersionFileService.prototype, 'isLargePackageVersionAllowed', async () => true);
22522252
const name = 'cnpmcore-test-sync-deprecated';
22532253
await packageSyncerService.createTask(name);
22542254
const task = await packageSyncerService.findExecuteTask();

0 commit comments

Comments
 (0)