feat: support index based filename naming strategy

bin/node-pg-migrate.ts

@@ -173,7 +173,7 @@ const parser = yargs(process.argv.slice(2))
     },
     [migrationFilenameFormatArg]: {
       defaultDescription: '"timestamp"',
-      choices: ['timestamp', 'utc'],
+      choices: ['timestamp', 'utc', 'index'],
       describe:
         'Prefix type of migration filename (Only valid with the create action)',
       type: 'string',
@@ -425,7 +425,8 @@ function readJson(json: unknown): void {
       MIGRATIONS_FILENAME_FORMAT,
       migrationFilenameFormatArg,
       json,
-      (val): val is FilenameFormat => val === 'timestamp' || val === 'utc'
+      (val): val is FilenameFormat =>
+        val === 'timestamp' || val === 'utc' || val === 'index'
     );
     TEMPLATE_FILE_NAME = applyIf(
       TEMPLATE_FILE_NAME,

docs/src/cli.md

@@ -93,7 +93,7 @@ You can adjust defaults by passing arguments to `node-pg-migrate`:
 | `create-migrations-schema`  |         | `false`                         | Create the configured migrations schema if it doesn't exist                                                                                                                                                                                                                             |
 | `migrations-table`          | `t`     | `pgmigrations`                  | The table storing which migrations have been run                                                                                                                                                                                                                                        |
 | `ignore-pattern`            |         | `undefined`                     | Regex pattern for file names to ignore (ignores files starting with `.` by default). Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern and set `--use-glob`. Note: enabling glob will read both, `--migrations-dir` _and_ `--ignore-pattern` as glob patterns |
-| `migration-filename-format` |         | `timestamp`                     | Choose prefix of file, `utc` (`20200605075829074`) or `timestamp` (`1591343909074`)                                                                                                                                                                                                     |
+| `migration-filename-format` |         | `timestamp`                     | Choose prefix of file, `utc` (`20200605075829074`), `timestamp` (`1591343909074`), or `index` (`0012`)                                                                                                                                                                                  |
 | `migration-file-language`   | `j`     | `js`                            | Language of the migration file to create (`js`, `ts` or `sql`)                                                                                                                                                                                                                          |
 | `template-file-name`        |         | `undefined`                     | Utilize a custom migration template file with language inferred from its extension. The file should export the up method, accepting a MigrationBuilder instance.                                                                                                                        |
 | `tsconfig`                  |         | `undefined`                     | Path to tsconfig.json. Used to setup transpiling of TS migration files. (Also sets `migration-file-language` to typescript, if not overridden)                                                                                                                                          |

src/migration.ts

@@ -33,6 +33,7 @@
 export const FilenameFormat = Object.freeze({
   timestamp: 'timestamp',
   utc: 'utc',
+  index: 'index',
 });
 
 export type FilenameFormat =
@@ -49,6 +50,7 @@
 
 export type CreateOptions = {
   filenameFormat?: FilenameFormat;
+  ignorePattern?: string;
 } & (CreateOptionsTemplate | CreateOptionsDefault);
 
 const SEPARATOR = '_';
@@ -187,8 +189,11 @@
 }
 
 /**
- * extracts numeric value from everything in `filename` before `SEPARATOR`.
- * 17 digit numbers are interpreted as utc date and converted to the number representation of that date.
+ * Extracts numeric value from everything in `filename` before `SEPARATOR`.
+ * 17 digit numbers are interpreted as UTC date and converted to the number
+ * representation of that date. 1...4 digit numbers are interpreted as index
+ * based naming scheme.
+ *
  * @param filename filename to extract the prefix from
  * @param logger Redirect messages to this logger object, rather than `console`.
  * @returns numeric value of the filename prefix (everything before `SEPARATOR`).
@@ -198,29 +203,29 @@
   logger: Logger = console
 ): number {
   const prefix = filename.split(SEPARATOR)[0];
-  if (prefix && /^\d+$/.test(prefix)) {
-    if (prefix.length === 13) {
-      // timestamp: 1391877300255
-      return Number(prefix);
-    }
+  const value = Number(prefix);
 
-    if (prefix && prefix.length === 17) {
-      // utc: 20200513070724505
-      const year = prefix.slice(0, 4);
-      const month = prefix.slice(4, 6);
-      const date = prefix.slice(6, 8);
-      const hours = prefix.slice(8, 10);
-      const minutes = prefix.slice(10, 12);
-      const seconds = prefix.slice(12, 14);
-      const ms = prefix.slice(14, 17);
-      return new Date(
-        `${year}-${month}-${date}T${hours}:${minutes}:${seconds}.${ms}Z`
-      ).valueOf();
-    }
+  if (!/^\d+$/.test(prefix) || Number.isNaN(value)) {
+    logger.error(`Cannot determine numeric prefix for "${prefix}"`);
+    throw new Error(`Cannot determine numeric prefix for "${prefix}"`);
   }
 
-  logger.error(`Can't determine timestamp for ${prefix}`);
-  return Number(prefix) || 0;
+  // Special case for UTC timestamp
+  if (prefix.startsWith('20') && prefix.length === 17) {
+    // utc: 20200513070724505
+    const year = prefix.slice(0, 4);
+    const month = prefix.slice(4, 6);
+    const date = prefix.slice(6, 8);
+    const hours = prefix.slice(8, 10);
+    const minutes = prefix.slice(10, 12);
+    const seconds = prefix.slice(12, 14);
+    const ms = prefix.slice(14, 17);
+    return new Date(
+      `${year}-${month}-${date}T${hours}:${minutes}:${seconds}.${ms}Z`
+    ).valueOf();
+  }
+
+  return value;
 }
 
 async function resolveSuffix(
@@ -232,22 +237,80 @@
 }
 
 export class Migration implements RunMigration {
-  // class method that creates a new migration file by cloning the migration template
+  /**
+   * Get file prefix for a new migrations file
+   *
+   * @method Migration.getFilePrefix
+   * @param filenameFormat Filename format
+   * @param directory Migrations directory
+   * @param [ignorePattern] Glob ignore pattern
+   * @returns string New file prefix
+   */
+  static async getFilePrefix(
+    filenameFormat: 'timestamp' | 'utc',
+    directory: string
+  ): Promise<string>;
+
+  static async getFilePrefix(
+    filenameFormat: 'index',
+    directory: string,
+    ignorePattern?: string
+  ): Promise<string>;
+
+  static async getFilePrefix(
+    filenameFormat: string,
+    directory: string,
+    ignorePattern?: string
+  ): Promise<string> {
+    if (filenameFormat === FilenameFormat.index) {
+      const filePaths = await getMigrationFilePaths(directory, {
+        ignorePattern,
+        useGlob: /\*/.test(directory) || /\*/.test(ignorePattern || ''),
+      });
+
+      // Get the minimum last found prefix as the total number of matching files
+      let lastPrefix = filePaths.length;
+
+      // Index can be used only when there are no mismatching filenames, so all
+      // the filenames have to be verified first against "index" naming pattern
+      for (const filenamePath of filePaths) {
+        const filename = basename(filenamePath);
+        if (!/^\d{1,4}\D/.test(filename)) {
+          throw new Error(
+            `Cannot deduce index for previously created file "${filenamePath}"`
+          );
+        }
+
+        lastPrefix = Math.max(lastPrefix, getNumericPrefix(filename));
+      }
+
+      // Next prefix is one more than the last found prefix
+      return `${lastPrefix + 1}`.padStart(4, '0');
+    }
+
+    return filenameFormat === FilenameFormat.utc
+      ? new Date().toISOString().replace(/\D/g, '')
+      : Date.now().toString();
+  }
+
+  // Class method that creates a new migration file by cloning the migration template
   static async create(
     name: string,
     directory: string,
     options: CreateOptions = {}
   ): Promise<string> {
-    const { filenameFormat = FilenameFormat.timestamp } = options;
+    const { filenameFormat = FilenameFormat.timestamp, ignorePattern } =
+      options;
 
-    // ensure the migrations directory exists
+    // Ensure the migrations directory exists
     await mkdir(directory, { recursive: true });
 
-    const now = new Date();
-    const time =
-      filenameFormat === FilenameFormat.utc
-        ? now.toISOString().replace(/\D/g, '')
-        : now.valueOf();
+    // Get prefix based on the configured naming scheme
+    const prefix = await Migration.getFilePrefix(
+      filenameFormat,
+      directory,
+      ignorePattern
+    );
 
     const templateFileName =
       'templateFileName' in options
@@ -262,7 +325,7 @@
     const suffix = getSuffixFromFileName(templateFileName);
 
     // file name looks like migrations/1391877300255_migration-title.js
-    const newFile = join(directory, `${time}${SEPARATOR}${name}.${suffix}`);
+    const newFile = join(directory, `${prefix}${SEPARATOR}${name}.${suffix}`);
 
     // copy the default migration template to the new file location
     await new Promise<void>((resolve, reject) => {
@@ -304,12 +367,20 @@
     this.db = db;
     this.path = migrationPath;
     this.name = basename(migrationPath, extname(migrationPath));
-    this.timestamp = getNumericPrefix(this.name, logger);
     this.up = up;
     this.down = down;
     this.options = options;
     this.typeShorthands = typeShorthands;
     this.logger = logger;
+
+    // Backwards compatibility patch: getNumericPrefix failed silently earlier
+    // so this try-catch block is to simulate it. The constructor should fail
+    // with invalid prefix rather than allow to continue.
+    try {
+      this.timestamp = getNumericPrefix(this.name, logger);
+    } catch {
+      this.timestamp = 0;
+    }
   }
 
   _getMarkAsRun(action: MigrationAction): string {

test/invalid-migrations/invalid-prefix.js

@@ -0,0 +1,8 @@
+export const up = (pgm) => {
+  pgm.createExtension('uuid-ossp', { ifNotExists: true });
+  pgm.dropExtension('uuid-ossp');
+  pgm.createExtension('uuid-ossp');
+  pgm.dropExtension('uuid-ossp');
+};
+
+export const down = () => null;

test/migration.spec.ts

@@ -5,6 +5,7 @@ import type { RunnerOption } from '../src';
 import type { DBConnection } from '../src/db';
 import type { Logger } from '../src/logger';
 import {
+  FilenameFormat,
   getMigrationFilePaths,
   getNumericPrefix,
   Migration,
@@ -40,9 +41,15 @@ describe('migration', () => {
   });
 
   describe('getNumericPrefix', () => {
+    it('should fail with a non-numeric value', () => {
+      const prefix = 'invalid-prefix';
+      expect(() => getNumericPrefix(prefix, logger)).toThrow(
+        new Error(`Cannot determine numeric prefix for "${prefix}"`)
+      );
+    });
+
     it('should get timestamp for normal timestamp', () => {
       const now = Date.now();
-
       expect(getNumericPrefix(String(now), logger)).toBe(now);
     });
 
@@ -53,6 +60,11 @@ describe('migration', () => {
         getNumericPrefix(now.toISOString().replace(/\D/g, ''), logger)
       ).toBe(now.valueOf());
     });
+
+    it('should get prefix for index strings', () => {
+      expect(getNumericPrefix('0001', logger)).toBe(1);
+      expect(getNumericPrefix('1234', logger)).toBe(1234);
+    });
   });
 
   describe('getMigrationFilePaths', () => {
@@ -126,6 +138,40 @@ describe('migration', () => {
         expect(isAbsolute(filePath)).toBeTruthy();
       }
     });
+
+    it('should resolve the next index for file paths', async () => {
+      const dir = 'test/{cockroach,migrations}/**';
+      // ignores those files that have `test` in their name (not in the path, just filename)
+      const ignorePattern = '*/cockroach/*test*';
+
+      const nextPrefix = await Migration.getFilePrefix(
+        FilenameFormat.index,
+        dir,
+        ignorePattern
+      );
+
+      // There are 106 files matching the pattern
+      expect(nextPrefix).toEqual('0107');
+    });
+
+    it('should fail to get the next index with invalid filenames', async () => {
+      const prefix = Migration.getFilePrefix(
+        FilenameFormat.index,
+        'test/invalid-migrations/invalid-prefix.*'
+      );
+
+      await expect(prefix).rejects.toThrow();
+    });
+
+    it('should get a normalized UTC as an epoch timestamp', async () => {
+      const now = Number.parseInt(new Date().toISOString().replace(/\D/g, ''));
+
+      const dir = 'test/migrations/**';
+      const prefix = await Migration.getFilePrefix('utc', dir);
+
+      // Checking against asynchronous code: prefix should be within 1000 ms from now
+      expect(Number.parseInt(prefix) - now < 1000).toEqual(true);
+    });
   });
 
   describe('self.applyUp', () => {
@@ -220,20 +266,21 @@ describe('migration', () => {
     });
 
     it('should fail with an error message if the migration is invalid', () => {
-      const invalidMigrationName = 'invalid-migration';
-
-      const migration = new Migration(
-        dbMock,
-        invalidMigrationName,
-        {},
-        options,
-        {},
-        logger
-      );
-
       const direction = 'up';
+      const invalidMigrationName = 'invalid-migration';
 
-      expect(() => migration.apply(direction)).toThrow(
+      expect(() => {
+        const migration = new Migration(
+          dbMock,
+          invalidMigrationName,
+          {},
+          options,
+          {},
+          logger
+        );
+
+        migration.apply(direction);
+      }).toThrow(
         new Error(
           `Unknown value for direction: ${direction}. Is the migration ${invalidMigrationName} exporting a '${direction}' function?`
         )