From 61b8c01d14f856ca71fcae070ec03f67d308724f Mon Sep 17 00:00:00 2001 From: Neil Gilmour Date: Tue, 8 Apr 2025 12:13:27 +0100 Subject: [PATCH 1/7] Test it can exclude an array of tables --- tests/DumperTest.php | 37 ++++++++++++++++--- ...uded_array_of_tables_from_allTables__1.txt | 3 ++ 2 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 tests/__snapshots__/DumperTest__it_does_remove_excluded_array_of_tables_from_allTables__1.txt diff --git a/tests/DumperTest.php b/tests/DumperTest.php index ca20e52..bea9973 100644 --- a/tests/DumperTest.php +++ b/tests/DumperTest.php @@ -6,7 +6,6 @@ use BeyondCode\LaravelMaskedDumper\LaravelMaskedDumpServiceProvider; use BeyondCode\LaravelMaskedDumper\TableDefinitions\TableDefinition; use Faker\Generator; -use Illuminate\Auth\Authenticatable; use Illuminate\Support\Facades\DB; use Orchestra\Testbench\TestCase; use Spatie\Snapshots\MatchesSnapshots; @@ -197,6 +196,32 @@ public function it_does_remove_excluded_tables_from_allTables() $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); } + /** @test */ + public function it_does_remove_excluded_array_of_tables_from_allTables() + { + $this->loadLaravelMigrations(); + + DB::table('users') + ->insert([ + 'name' => 'Marcel', + 'email' => 'marcel@beyondco.de', + 'password' => 'test', + 'created_at' => '2021-01-01 00:00:00', + 'updated_at' => '2021-01-01 00:00:00', + ]); + + $outputFile = base_path('test.sql'); + + $this->app['config']['masked-dump.default'] = DumpSchema::define() + ->allTables() + ->exclude(['users']); + + $this->artisan('db:masked-dump', [ + 'output' => $outputFile + ]); + + $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); + } /** @test */ public function it_creates_chunked_insert_statements_for_a_table() { @@ -224,12 +249,12 @@ public function it_creates_chunked_insert_statements_for_a_table() ]); $outputFile = base_path('test.sql'); - + $this->app['config']['masked-dump.default'] = DumpSchema::define() - ->allTables() - ->table('users', function($table) { - return $table->outputInChunksOf(3); - }); + ->allTables() + ->table('users', function($table) { + return $table->outputInChunksOf(3); + }); $this->artisan('db:masked-dump', [ 'output' => $outputFile diff --git a/tests/__snapshots__/DumperTest__it_does_remove_excluded_array_of_tables_from_allTables__1.txt b/tests/__snapshots__/DumperTest__it_does_remove_excluded_array_of_tables_from_allTables__1.txt new file mode 100644 index 0000000..e1088c5 --- /dev/null +++ b/tests/__snapshots__/DumperTest__it_does_remove_excluded_array_of_tables_from_allTables__1.txt @@ -0,0 +1,3 @@ +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('1', '0001_01_01_000000_testbench_create_users_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('2', '0001_01_01_000001_testbench_create_cache_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('3', '0001_01_01_000002_testbench_create_jobs_table', '1'); From 24a02742b4b71857808dde624f32ef001d14a18d Mon Sep 17 00:00:00 2001 From: Neil Gilmour Date: Tue, 8 Apr 2025 12:13:57 +0100 Subject: [PATCH 2/7] Add method to allow excluding an array of tables at once. --- src/DumpSchema.php | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/DumpSchema.php b/src/DumpSchema.php index 5a7c623..1422561 100644 --- a/src/DumpSchema.php +++ b/src/DumpSchema.php @@ -42,9 +42,17 @@ public function allTables() return $this; } - public function exclude(string $tableName) + /** + * @param string|string[] $tableName Table name(s) to exclude from the dump + * @return $this + */ + public function exclude(string|array $tableName) { - $this->excludedTables[] = $tableName; + collect($tableName) + ->flatten() + ->unique() + ->filter(fn ($table) => is_string($table)) + ->each(fn ($table) => $this->excludedTables[] = $table); return $this; } From 4670688a6bc91063de5fdfd0e212ff9eccf4d846 Mon Sep 17 00:00:00 2001 From: Neil Gilmour Date: Tue, 8 Apr 2025 13:13:20 +0100 Subject: [PATCH 3/7] Update docs --- docs/schema-definition.md | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/docs/schema-definition.md b/docs/schema-definition.md index 89f250b..3f1e8bb 100644 --- a/docs/schema-definition.md +++ b/docs/schema-definition.md @@ -75,7 +75,7 @@ return [ * NOTE: This approach is not compatible with Laravel's config caching. */ 'default' => DumpSchema::define() - ->allTables() + ->allTables() ->table('users', function (TableDefinition $table) { $table->replace('name', function (Faker $faker) { return $faker->name; @@ -96,19 +96,30 @@ This ensures that all of your database tables will be represented in the dump. Y ```php return [ 'default' => DumpSchema::define() - ->allTables(), + ->allTables(), ]; ``` -## Exclude specific tables from dumps +## Excluding tables from dumps -The `exclude()` method allows you to exclude specific tables from the dump. This can be useful if you want to exclude certain tables from the dump: +The `exclude()` method allows you to exclude specific tables from the dump. This can be useful if you have certain tables which aren't required for this dump. ```php return [ 'default' => DumpSchema::define() - ->allTables() - ->exclude('password_resets'), + ->allTables() + ->exclude('password_resets'), +]; +``` + +Consider the case where you have a set of tables that you never want to dump. Perhaps they contain complex nested JSON that is too complex to anonymise, or huge amounts of analytic data that just won't be necessary. Well, you can also pass an array to `exclude()`: + +```php +return [ + 'default' => DumpSchema::define() + ->allTables() + ->exclude(['password_resets', 'secrets', 'lies', 'cat_birthdays']), + ->exclude(config('database.forbidden_tables')), ]; ``` @@ -170,7 +181,7 @@ When dumping your data, the dump will now contain a safe, randomly generated ema ## Optimizing large datasets -The method TableDefinition::outputInChunksOf(int $chunkSize) allows for chunked inserts for large datasets, +The method `TableDefinition::outputInChunksOf(int $chunkSize)` allows for chunked inserts for large datasets, improving performance and reducing memory consumption during the dump process. ```php @@ -191,7 +202,7 @@ You can pass the connection to the `DumpSchema::define` method, in order to spec ```php return [ 'default' => DumpSchema::define('sqlite') - ->allTables() + ->allTables() ]; ``` @@ -203,10 +214,10 @@ The key in the configuration array is the identifier that will be used when you ```php return [ 'default' => DumpSchema::define() - ->allTables(), + ->allTables(), 'sqlite' => DumpSchema::define('sqlite') - ->schemaOnly('custom_table'), + ->schemaOnly('custom_table'), ]; ``` From 2e1df9a6520d742b5cfaf700cee6fc45475205d0 Mon Sep 17 00:00:00 2001 From: Neil Gilmour Date: Fri, 11 Apr 2025 21:20:43 +0100 Subject: [PATCH 4/7] Add an `include()` method to allow including tables individually/in an array. --- src/DumpSchema.php | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/DumpSchema.php b/src/DumpSchema.php index 1422561..7714b75 100644 --- a/src/DumpSchema.php +++ b/src/DumpSchema.php @@ -8,6 +8,8 @@ use Doctrine\DBAL\Types\Types; use Illuminate\Support\Facades\Schema; +use function collect; + class DumpSchema { protected $connectionName; @@ -57,6 +59,31 @@ public function exclude(string|array $tableName) return $this; } + public function include(string|array $tableName) + { + // We're kinda fooling the `load()` and `loadAvailableTables()` method here; + // by setting `loadAllTables` to true, and adding tables directly to `availableTables`, + // the `load()` method still calls `loadAvailableTables()` but that returns early + // because there's already our tables in `availableTables`. Then the `load()` + // method sees that `loadAllTables` is true, and loads the tables from + // `availableTables` (which we just put there!) into `dumpTables`. + $this->loadAllTables = true; + $tables = collect($tableName) + ->flatten() + ->unique() + ->filter(fn ($table) => is_string($table)) + ->filter(fn ($table) => $this->getBuilder()->hasTable($table)) + ->map(fn ($table) => ['name' => $table]) + ->toArray(); + + $doctrineTables = $this->createDoctrineTables($tables); + foreach ($doctrineTables as $doctrineTable) { + $this->availableTables[] = $doctrineTable; + } + + return $this; + } + /** * @return \Illuminate\Database\Schema\Builder */ From 7d37a813482e6d7bf10f3c3004a599ccda4c4e9b Mon Sep 17 00:00:00 2001 From: Neil Gilmour Date: Fri, 11 Apr 2025 21:31:55 +0100 Subject: [PATCH 5/7] Add tests for including unmodified tables --- tests/DumperTest.php | 52 +++++++++++++++++++ ...clude_an_array_of_unmodified_tables__1.txt | 4 ++ ...nclude_individual_unmodified_tables__1.txt | 1 + 3 files changed, 57 insertions(+) create mode 100644 tests/__snapshots__/DumperTest__it_can_include_an_array_of_unmodified_tables__1.txt create mode 100644 tests/__snapshots__/DumperTest__it_can_include_individual_unmodified_tables__1.txt diff --git a/tests/DumperTest.php b/tests/DumperTest.php index bea9973..23ab428 100644 --- a/tests/DumperTest.php +++ b/tests/DumperTest.php @@ -262,4 +262,56 @@ public function it_creates_chunked_insert_statements_for_a_table() $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); } + + /** @test */ + public function it_can_include_individual_unmodified_tables() + { + $this->loadLaravelMigrations(); + + DB::table('users') + ->insert([ + 'name' => 'Marcel', + 'email' => 'marcel@beyondco.de', + 'password' => 'test', + 'created_at' => '2021-01-01 00:00:00', + 'updated_at' => '2021-01-01 00:00:00', + ]); + + $outputFile = base_path('test.sql'); + + $this->app['config']['masked-dump.default'] = DumpSchema::define() + ->include('users'); + + $this->artisan('db:masked-dump', [ + 'output' => $outputFile + ]); + + $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); + } + + /** @test */ + public function it_can_include_an_array_of_unmodified_tables() + { + $this->loadLaravelMigrations(); + + DB::table('users') + ->insert([ + 'name' => 'Marcel', + 'email' => 'marcel@beyondco.de', + 'password' => 'test', + 'created_at' => '2021-01-01 00:00:00', + 'updated_at' => '2021-01-01 00:00:00', + ]); + + $outputFile = base_path('test.sql'); + + $this->app['config']['masked-dump.default'] = DumpSchema::define() + ->include(['users', 'migrations']); + + $this->artisan('db:masked-dump', [ + 'output' => $outputFile + ]); + + $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); + } } diff --git a/tests/__snapshots__/DumperTest__it_can_include_an_array_of_unmodified_tables__1.txt b/tests/__snapshots__/DumperTest__it_can_include_an_array_of_unmodified_tables__1.txt new file mode 100644 index 0000000..6eabb3f --- /dev/null +++ b/tests/__snapshots__/DumperTest__it_can_include_an_array_of_unmodified_tables__1.txt @@ -0,0 +1,4 @@ +INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('1', 'Marcel', 'marcel@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('1', '0001_01_01_000000_testbench_create_users_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('2', '0001_01_01_000001_testbench_create_cache_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('3', '0001_01_01_000002_testbench_create_jobs_table', '1'); diff --git a/tests/__snapshots__/DumperTest__it_can_include_individual_unmodified_tables__1.txt b/tests/__snapshots__/DumperTest__it_can_include_individual_unmodified_tables__1.txt new file mode 100644 index 0000000..1fcebe4 --- /dev/null +++ b/tests/__snapshots__/DumperTest__it_can_include_individual_unmodified_tables__1.txt @@ -0,0 +1 @@ +INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('1', 'Marcel', 'marcel@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00'); From 909e69f938b4459bcdbf433c5282023b24a005b7 Mon Sep 17 00:00:00 2001 From: Neil Gilmour Date: Fri, 11 Apr 2025 21:45:17 +0100 Subject: [PATCH 6/7] Update docs with include() method. --- docs/schema-definition.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/schema-definition.md b/docs/schema-definition.md index 3f1e8bb..e438b32 100644 --- a/docs/schema-definition.md +++ b/docs/schema-definition.md @@ -100,9 +100,17 @@ return [ ]; ``` -## Excluding tables from dumps +If you don't want to dump all of your tables, you may add specific tables that you wish to include in the dump by using the `include()` method. You don't need to add a table here if you will be customizing it. -The `exclude()` method allows you to exclude specific tables from the dump. This can be useful if you have certain tables which aren't required for this dump. +```php +return [ + 'default' => DumpSchema::define() + ->include('audit_logs') + ->include(['user_logins', 'failed_jobs']), +]; +``` + +If you've started out by adding all tables with `allTables()`, you can remove some of them with the `exclude()` method. This can be useful if you have certain tables which aren't required for this dump. ```php return [ From 063a080924d17b398a6117c7021115e3e74e3a16 Mon Sep 17 00:00:00 2001 From: Neil Gilmour Date: Fri, 11 Apr 2025 21:46:41 +0100 Subject: [PATCH 7/7] Test it can handle multiple uses of include() Basically we're making sure that `include()` adds to the `availableTables` array, and doesn't replace it. --- tests/DumperTest.php | 27 +++++++++++++++++++ ...tables_with_repeated_use_of_include__1.txt | 4 +++ 2 files changed, 31 insertions(+) create mode 100644 tests/__snapshots__/DumperTest__it_still_includes_tables_with_repeated_use_of_include__1.txt diff --git a/tests/DumperTest.php b/tests/DumperTest.php index 23ab428..558360a 100644 --- a/tests/DumperTest.php +++ b/tests/DumperTest.php @@ -314,4 +314,31 @@ public function it_can_include_an_array_of_unmodified_tables() $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); } + + /** @test */ + public function it_still_includes_tables_with_repeated_use_of_include() + { + $this->loadLaravelMigrations(); + + DB::table('users') + ->insert([ + 'name' => 'Marcel', + 'email' => 'marcel@beyondco.de', + 'password' => 'test', + 'created_at' => '2021-01-01 00:00:00', + 'updated_at' => '2021-01-01 00:00:00', + ]); + + $outputFile = base_path('test.sql'); + + $this->app['config']['masked-dump.default'] = DumpSchema::define() + ->include('users') + ->include('migrations'); + + $this->artisan('db:masked-dump', [ + 'output' => $outputFile + ]); + + $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); + } } diff --git a/tests/__snapshots__/DumperTest__it_still_includes_tables_with_repeated_use_of_include__1.txt b/tests/__snapshots__/DumperTest__it_still_includes_tables_with_repeated_use_of_include__1.txt new file mode 100644 index 0000000..6eabb3f --- /dev/null +++ b/tests/__snapshots__/DumperTest__it_still_includes_tables_with_repeated_use_of_include__1.txt @@ -0,0 +1,4 @@ +INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('1', 'Marcel', 'marcel@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('1', '0001_01_01_000000_testbench_create_users_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('2', '0001_01_01_000001_testbench_create_cache_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('3', '0001_01_01_000002_testbench_create_jobs_table', '1');