feat(ruleset-migrator): use Content-Type header to detect ruleset format (stoplightio#2317)

Author: P0lip

packages/ruleset-migrator/src/__tests__/ruleset.test.ts

@@ -18,6 +18,11 @@ afterAll(() => {
   vol.reset();
 });
 
+function createFetchMockSandbox() {
+  // something is off with default module interop in Karma :man_shrugging:
+  return ((fetchMock as { default?: typeof import('fetch-mock') }).default ?? fetchMock).sandbox();
+}
+
 const scenarios = Object.keys(fixtures)
   .filter(key => path.basename(key) === 'output.mjs')
   .map(key => path.dirname(key));
@@ -99,8 +104,7 @@ describe('migrator', () => {
   });
 
   it('should accept custom fetch implementation', async () => {
-    // something is off with default module interop in Karma :man_shrugging:
-    const fetch = ((fetchMock as { default?: typeof import('fetch-mock') }).default ?? fetchMock).sandbox();
+    const fetch = createFetchMockSandbox();
 
     await vol.promises.writeFile(
       path.join(cwd, 'ruleset.json'),
@@ -123,6 +127,9 @@ describe('migrator', () => {
           },
         },
       },
+      headers: {
+        'Content-Type': 'application/json; charset=utf-8',
+      },
     });
 
     expect(
@@ -218,6 +225,36 @@ export default {
 `);
   });
 
+  it('should use Content-Type detection', async () => {
+    const fetch = createFetchMockSandbox();
+
+    await vol.promises.writeFile(
+      path.join(cwd, 'ruleset.json'),
+      JSON.stringify({
+        extends: ['https://spectral.stoplight.io/ruleset'],
+      }),
+    );
+
+    fetch.get('https://spectral.stoplight.io/ruleset', {
+      body: `export default { rules: {} }`,
+      headers: {
+        'Content-Type': 'application/javascript; charset=utf-8',
+      },
+    });
+
+    expect(
+      await migrateRuleset(path.join(cwd, 'ruleset.json'), {
+        format: 'esm',
+        fs: vol as any,
+        fetch,
+      }),
+    ).toEqual(`import ruleset_ from "https://spectral.stoplight.io/ruleset";
+export default {
+  "extends": [ruleset_]
+};
+`);
+  });
+
   describe('custom npm registry', () => {
     it('should be supported', async () => {
       serveAssets({

packages/ruleset-migrator/src/index.ts

@@ -10,6 +10,8 @@ import { ExpressionKind } from 'ast-types/gen/kinds';
 import { assertRuleset } from './validation';
 import { Ruleset } from './validation/types';
 
+export { isBasicRuleset } from './utils/isBasicRuleset';
+
 async function read(filepath: string, fs: MigrationOptions['fs'], fetch: Fetch): Promise<Ruleset> {
   const input = isURL(filepath) ? await (await fetch(filepath)).text() : await fs.promises.readFile(filepath, 'utf8');
 

packages/ruleset-migrator/src/transformers/extends.ts

@@ -3,15 +3,14 @@ import * as path from '@stoplight/path';
 import { Transformer, TransformerCtx } from '../types';
 import { Ruleset } from '../validation/types';
 import { assertArray } from '../validation';
-import { process } from '..';
+import { process } from '../index';
+import { isBasicRuleset } from '../utils/isBasicRuleset';
 
 const REPLACEMENTS = {
   'spectral:oas': 'oas',
   'spectral:asyncapi': 'asyncapi',
 };
 
-const KNOWN_JS_EXTS = /^\.[cm]?js$/;
-
 export { transformer as default };
 
 async function processExtend(
@@ -24,7 +23,7 @@ async function processExtend(
 
   const filepath = ctx.tree.resolveModule(name, ctx, 'ruleset');
 
-  if (KNOWN_JS_EXTS.test(path.extname(filepath))) {
+  if (!(await isBasicRuleset(filepath, ctx.opts.fetch))) {
     return ctx.tree.addImport(`${path.basename(filepath, true)}_${path.extname(filepath)}`, filepath, true);
   }
 

packages/ruleset-migrator/src/utils/__tests__/isBasicRuleset.spec.ts

@@ -0,0 +1,54 @@
+import { isBasicRuleset } from '../isBasicRuleset';
+
+describe('isBasicRuleset util', () => {
+  it.concurrent.each(['json', 'yaml', 'yml'])('given %s extension, should return true', async ext => {
+    const fetch = jest.fn();
+    await expect(isBasicRuleset(`/ruleset.${ext}`, fetch)).resolves.toBe(true);
+    expect(fetch).not.toBeCalled();
+  });
+
+  it.concurrent.each(['js', 'mjs', 'cjs'])('given %s extension, should return false', async ext => {
+    const fetch = jest.fn();
+    await expect(isBasicRuleset(`/ruleset.${ext}`, fetch)).resolves.toBe(false);
+    expect(fetch).not.toBeCalled();
+  });
+
+  it.concurrent('given an URL with query, should strip query prior to the lookup', async () => {
+    const fetch = jest.fn();
+    await expect(isBasicRuleset(`https://stoplight.io/ruleset.yaml?token=test`, fetch)).resolves.toBe(true);
+    expect(fetch).not.toBeCalled();
+  });
+
+  it.concurrent.each([
+    'application/json',
+    'application/yaml',
+    'text/json',
+    'text/yaml',
+    'application/yaml; charset=utf-8',
+    'application/json; charset=utf-8',
+    'text/yaml; charset=utf-16',
+  ])('given %s Content-Type, should return true', async input => {
+    const fetch = jest.fn().mockResolvedValue({
+      headers: new Map([['Content-Type', input]]),
+    });
+
+    await expect(isBasicRuleset('https://stoplight.io', fetch)).resolves.toBe(true);
+  });
+
+  it.concurrent.each(['application/javascript', 'application/x-yaml', 'application/yaml-', 'something/yaml'])(
+    'given %s Content-Type, should return false',
+    async input => {
+      const fetch = jest.fn().mockResolvedValue({
+        headers: new Map([['Content-Type', input]]),
+      });
+
+      await expect(isBasicRuleset('https://stoplight.io', fetch)).resolves.toBe(false);
+    },
+  );
+
+  it.concurrent('given fetch failure, should return false', async () => {
+    const fetch = jest.fn().mockRejectedValueOnce(new Error());
+
+    await expect(isBasicRuleset('https://stoplight.io', fetch)).resolves.toBe(false);
+  });
+});

packages/ruleset-migrator/src/utils/isBasicRuleset.ts

@@ -0,0 +1,34 @@
+import { fetch as defaultFetch } from '@stoplight/spectral-runtime';
+import { isURL, extname } from '@stoplight/path';
+import type { Fetch } from '../types';
+
+function stripSearchFromUrl(url: string): string {
+  try {
+    const { href, search } = new URL(url);
+    return href.slice(0, href.length - search.length);
+  } catch {
+    return url;
+  }
+}
+
+const CONTENT_TYPE_REGEXP = /^(?:application|text)\/(?:yaml|json)(?:;|$)/i;
+const EXT_REGEXP = /\.(json|ya?ml)$/i;
+
+export async function isBasicRuleset(uri: string, fetch: Fetch = defaultFetch): Promise<boolean> {
+  const ext = extname(isURL(uri) ? stripSearchFromUrl(uri) : uri);
+
+  if (EXT_REGEXP.test(ext)) {
+    return true;
+  }
+
+  if (!isURL(uri)) {
+    return false;
+  }
+
+  try {
+    const contentType = (await fetch(uri)).headers.get('Content-Type');
+    return contentType !== null && CONTENT_TYPE_REGEXP.test(contentType);
+  } catch {
+    return false;
+  }
+}