Merge pull request #22 from keboola/COM-882-ondra-export-structure-only

Export structure only
keboola · Jun 28, 2021 · 833005a · 833005a
2 parents a98db38 + 7618e97
commit 833005a
Show file tree

Hide file tree

Showing 5 changed files with 167 additions and 5 deletions.
diff --git a/src/Application.php b/src/Application.php
@@ -55,11 +55,13 @@ public function run(): void
 
         $backup->backupTablesMetadata();
 
-        $tables = $sapi->listTables();
-        $tablesCount = count($tables);
-        foreach ($tables as $i => $table) {
-            $this->logger->info(sprintf('Table %d/%d', $i + 1, $tablesCount));
-            $backup->backupTable($table['id']);
+        if (!$this->config->exportStructureOnly()) {
+            $tables = $sapi->listTables();
+            $tablesCount = count($tables);
+            foreach ($tables as $i => $table) {
+                $this->logger->info(sprintf('Table %d/%d', $i + 1, $tablesCount));
+                $backup->backupTable($table['id']);
+            }
         }
 
         $backup->backupConfigs(false);

diff --git a/src/Config/Config.php b/src/Config/Config.php
@@ -26,6 +26,11 @@ public function getStorageBackendType(): string
         return $this->getImageParameters()['storageBackendType'];
     }
 
+    public function exportStructureOnly(): bool
+    {
+        return $this->getValue(['parameters', 'exportStructureOnly'], false);
+    }
+
     public function isUserDefinedCredentials(): bool
     {
         $storageBackendType = $this->getValue(['parameters', 'storageBackendType'], '');

diff --git a/src/Config/ConfigDefinition.php b/src/Config/ConfigDefinition.php
@@ -79,6 +79,7 @@ protected function getParametersDefinition(): ArrayNodeDefinition
             ->children()
                 ->scalarNode('backupId')->isRequired()->end()
                 ->scalarNode('backupPath')->end()
+                ->booleanNode('exportStructureOnly')->end()
                 ->scalarNode('storageBackendType')->end()
                 ->scalarNode('accountName')->end()
                 ->scalarNode('#accountKey')->end()

diff --git a/tests/phpunit/FunctionalAbsTest.php b/tests/phpunit/FunctionalAbsTest.php
@@ -5,6 +5,7 @@
 namespace Keboola\App\ProjectBackup\Tests;
 
 use Keboola\App\ProjectBackup\Config\Config;
+use Keboola\Csv\CsvFile;
 use Keboola\StorageApi\Client as StorageApi;
 use Keboola\StorageApi\Components;
 use Keboola\StorageApi\Options\Components\Configuration;
@@ -159,6 +160,81 @@ public function testSuccessfulRun(): void
         self::assertGreaterThan(0, count($events));
     }
 
+    public function testSuccessfulRunOnlyStructure(): void
+    {
+        $events = $this->sapiClient->listEvents(['runId' => $this->testRunId]);
+        self::assertCount(0, $events);
+
+        $tmp = new Temp();
+        $tmp->initRunFolder();
+
+        $file = $tmp->createFile('testStructureOnly.csv');
+        file_put_contents($file->getPathname(), 'a,b,c,d,e,f');
+
+        $csvFile = new CsvFile($file);
+
+        $this->sapiClient->createBucket('test-bucket', 'out');
+        $this->sapiClient->createTable('out.c-test-bucket', 'test-table', $csvFile);
+
+        $fileSystem = new Filesystem();
+
+        // create backupId
+        $fileSystem->dumpFile(
+            $this->temp->getTmpFolder() . '/config.json',
+            (string) json_encode([
+                'action' => 'generate-read-credentials',
+                'image_parameters' => [
+                    'storageBackendType' => Config::STORAGE_BACKEND_ABS,
+                    'accountName' => getenv('TEST_AZURE_ACCOUNT_NAME'),
+                    '#accountKey' => getenv('TEST_AZURE_ACCOUNT_KEY'),
+                    'region' => getenv('TEST_AZURE_REGION'),
+                ],
+            ])
+        );
+
+        $runProcess = $this->createTestProcess();
+        $runProcess->mustRun();
+
+        $this->assertEmpty($runProcess->getErrorOutput());
+
+        $output = $runProcess->getOutput();
+        $outputData = json_decode($output, true);
+
+        $this->assertArrayHasKey('backupId', $outputData);
+
+        // run backup
+        $fileSystem->dumpFile(
+            $this->temp->getTmpFolder() . '/config.json',
+            (string) json_encode([
+                'action' => 'run',
+                'parameters' => [
+                    'backupId' => $outputData['backupId'],
+                    'exportStructureOnly' => true,
+                ],
+                'image_parameters' => [
+                    'storageBackendType' => Config::STORAGE_BACKEND_ABS,
+                    'accountName' => getenv('TEST_AZURE_ACCOUNT_NAME'),
+                    '#accountKey' => getenv('TEST_AZURE_ACCOUNT_KEY'),
+                    'region' => getenv('TEST_AZURE_REGION'),
+                ],
+            ])
+        );
+
+        $runProcess = $this->createTestProcess();
+        $runProcess->mustRun();
+
+        $this->assertEmpty($runProcess->getErrorOutput());
+
+        $output = $runProcess->getOutput();
+        $this->assertContains('Exporting buckets', $output);
+        $this->assertContains('Exporting tables', $output);
+        $this->assertContains('Exporting configurations', $output);
+        $this->assertNotContains('Table ', $output);
+
+        $events = $this->sapiClient->listEvents(['runId' => $this->testRunId]);
+        self::assertGreaterThan(0, count($events));
+    }
+
     public function testBadBackupIdRun(): void
     {
         $fileSystem = new Filesystem();

diff --git a/tests/phpunit/FunctionalS3Test.php b/tests/phpunit/FunctionalS3Test.php
@@ -8,6 +8,7 @@
 use Aws\S3\S3Client;
 use Aws\S3\S3UriParser;
 use Keboola\App\ProjectBackup\Config\Config;
+use Keboola\Csv\CsvFile;
 use Keboola\StorageApi\Client as StorageApi;
 use Keboola\StorageApi\Components;
 use Keboola\StorageApi\Options\Components\Configuration;
@@ -201,6 +202,83 @@ public function testSuccessfulRun(): void
         self::assertGreaterThan(0, count($events));
     }
 
+    public function testSuccessfulRunOnlyStructure(): void
+    {
+        $events = $this->sapiClient->listEvents(['runId' => $this->testRunId]);
+        self::assertCount(0, $events);
+
+        $tmp = new Temp();
+        $tmp->initRunFolder();
+
+        $file = $tmp->createFile('testStructureOnly.csv');
+        file_put_contents($file->getPathname(), 'a,b,c,d,e,f');
+
+        $csvFile = new CsvFile($file);
+
+        $this->sapiClient->createBucket('test-bucket', 'out');
+        $this->sapiClient->createTable('out.c-test-bucket', 'test-table', $csvFile);
+
+        $fileSystem = new Filesystem();
+
+        // create backupId
+        $fileSystem->dumpFile(
+            $this->temp->getTmpFolder() . '/config.json',
+            (string) json_encode([
+                'action' => 'generate-read-credentials',
+                'image_parameters' => [
+                    'storageBackendType' => Config::STORAGE_BACKEND_S3,
+                    'access_key_id' => getenv('TEST_AWS_ACCESS_KEY_ID'),
+                    '#secret_access_key' => getenv('TEST_AWS_SECRET_ACCESS_KEY'),
+                    'region' => getenv('TEST_AWS_REGION'),
+                    '#bucket' => getenv('TEST_AWS_S3_BUCKET'),
+                ],
+            ])
+        );
+
+        $runProcess = $this->createTestProcess();
+        $runProcess->mustRun();
+
+        $this->assertEmpty($runProcess->getErrorOutput());
+
+        $output = $runProcess->getOutput();
+        $outputData = \json_decode($output, true);
+
+        $this->assertArrayHasKey('backupId', $outputData);
+
+        // run backup
+        $fileSystem->dumpFile(
+            $this->temp->getTmpFolder() . '/config.json',
+            (string) json_encode([
+                'action' => 'run',
+                'parameters' => [
+                    'backupId' => $outputData['backupId'],
+                    'exportStructureOnly' => true,
+                ],
+                'image_parameters' => [
+                    'storageBackendType' => Config::STORAGE_BACKEND_S3,
+                    'access_key_id' => getenv('TEST_AWS_ACCESS_KEY_ID'),
+                    '#secret_access_key' => getenv('TEST_AWS_SECRET_ACCESS_KEY'),
+                    'region' => getenv('TEST_AWS_REGION'),
+                    '#bucket' => getenv('TEST_AWS_S3_BUCKET'),
+                ],
+            ])
+        );
+
+        $runProcess = $this->createTestProcess();
+        $runProcess->mustRun();
+
+        $this->assertEmpty($runProcess->getErrorOutput());
+
+        $output = $runProcess->getOutput();
+        $this->assertContains('Exporting buckets', $output);
+        $this->assertContains('Exporting tables', $output);
+        $this->assertContains('Exporting configurations', $output);
+        $this->assertNotContains('Table ', $output);
+
+        $events = $this->sapiClient->listEvents(['runId' => $this->testRunId]);
+        self::assertGreaterThan(0, count($events));
+    }
+
     public function testBadBackupIdRun(): void
     {
         $fileSystem = new Filesystem();