Merge pull request #344 from keboola/erik-disable-synapse-part2

[output-mapping] abs tests moved to own suite for azure
keboola · Jan 3, 2025 · c8fbc6d · c8fbc6d
2 parents 1257e63 + d891366
commit c8fbc6d
Show file tree

Hide file tree

Showing 7 changed files with 41 additions and 86 deletions.
diff --git a/azure-pipelines.tests.yml b/azure-pipelines.tests.yml
@@ -91,6 +91,22 @@ jobs:
         STORAGE_API_TOKEN_MASTER: $(OUTPUT_MAPPING__STORAGE_API_TOKEN_MASTER_AWS)
         SYNAPSE_STORAGE_API_TOKEN: $(OUTPUT_MAPPING__SYNAPSE_STORAGE_API_TOKEN)
 
+  - template: ../../azure-pipelines/jobs/run-tests.yml
+    parameters:
+      jobName: tests_php81_tableWriter_synapse
+      condition: eq(variables.RUN_SYNAPSE_TESTS, '1')
+      dependsOn: [cs]
+      displayName: Synapse Writer Tests
+      serviceName: dev-output-mapping
+      testCommand: 'composer install && vendor/bin/phpunit --testsuite synapse-writer-tests'
+      variables:
+        STORAGE_API_URL: $(STORAGE_API_URL_AZURE)
+        SYNAPSE_STORAGE_API_URL: $(STORAGE_API_URL_AZURE)
+      secrets:
+        STORAGE_API_TOKEN: $(OUTPUT_MAPPING__STORAGE_API_TOKEN_AZURE)
+        STORAGE_API_TOKEN_MASTER: $(OUTPUT_MAPPING__STORAGE_API_TOKEN_MASTER_AZURE)
+        SYNAPSE_STORAGE_API_TOKEN: $(OUTPUT_MAPPING__SYNAPSE_STORAGE_API_TOKEN)
+
   - template: ../../azure-pipelines/jobs/run-tests.yml
     parameters:
       jobName: tests_php81_tableWriter_nativeTypes

diff --git a/phpunit.xml.dist b/phpunit.xml.dist
@@ -21,6 +21,9 @@
             <exclude>tests/Writer/TableDefinitionV2Test.php</exclude>
             <exclude>tests/Writer/StorageApiLocalTableWriterTest.php</exclude>
             <exclude>tests/Writer/StorageApiHeadlessWriterTest.php</exclude>
+            <exclude>tests/Writer/File/Strategy/ABSWorkspaceTest.php</exclude>
+            <exclude>tests/Writer/Table/Source/AbsWorkspaceItemSourceFactoryTest.php</exclude>
+            <exclude>tests/Writer/Table/Strategy/AbsWorkspaceTableStrategyTest.php</exclude>
         </testsuite>
         <testsuite name="main-writer-tests-1">
             <file>tests/Writer/StorageApiLocalTableWriterTest.php</file>
@@ -43,6 +46,8 @@
         </testsuite>
         <testsuite name="workspace-writer-tests">
             <directory>tests/Writer/Workspace</directory>
+            <exclude>tests/Writer/Workspace/AbsWriterWorkspaceTest.php</exclude>
+            <exclude>tests/Writer/Workspace/SynapseWriterWorkspaceTest.php</exclude>
         </testsuite>
         <testsuite name="native-types">
             <file>tests/Writer/TableDefinitionTest.php</file>
@@ -54,6 +59,15 @@
             <directory>tests/Writer/Table</directory>
             <file>tests/Writer/StorageApiLocalTableWriterTest.php</file>
             <file>tests/Writer/StorageApiSlicedWriterTest.php</file>
+            <exclude>tests/Writer/Table/Source/AbsWorkspaceItemSourceFactoryTest.php</exclude>
+            <exclude>tests/Writer/Table/Strategy/AbsWorkspaceTableStrategyTest.php</exclude>
+        </testsuite>
+        <testsuite name="synapse-writer-tests">
+            <file>tests/Writer/File/Strategy/ABSWorkspaceTest.php</file>
+            <file>tests/Writer/Table/Source/AbsWorkspaceItemSourceFactoryTest.php</file>
+            <file>tests/Writer/Table/Strategy/AbsWorkspaceTableStrategyTest.php</file>
+            <file>tests/Writer/Workspace/AbsWriterWorkspaceTest.php</file>
+            <file>tests/Writer/Workspace/SynapseWriterWorkspaceTest.php</file>
         </testsuite>
     </testsuites>
     <coverage processUncoveredFiles="true">

diff --git a/tests/InitSynapseStorageClientTrait.php b/tests/InitSynapseStorageClientTrait.php
@@ -4,26 +4,11 @@
 
 namespace Keboola\OutputMapping\Tests;
 
-use Keboola\StorageApi\Exception;
 use Keboola\StorageApiBranch\ClientWrapper;
 use Keboola\StorageApiBranch\Factory\ClientOptions;
 
 trait InitSynapseStorageClientTrait
 {
-    protected function checkSynapseTests(): bool
-    {
-        if (!getenv('RUN_SYNAPSE_TESTS')) {
-            return false;
-        }
-        if (getenv('SYNAPSE_STORAGE_API_TOKEN') === false) {
-            throw new Exception('SYNAPSE_STORAGE_API_TOKEN must be set for synapse tests');
-        }
-        if (getenv('SYNAPSE_STORAGE_API_URL') === false) {
-            throw new Exception('SYNAPSE_STORAGE_API_URL must be set for synapse tests');
-        }
-        return true;
-    }
-
     protected function getSynapseClientWrapper(): ClientWrapper
     {
         $clientOptions = (new ClientOptions())

diff --git a/tests/Writer/File/Strategy/ABSWorkspaceTest.php b/tests/Writer/File/Strategy/ABSWorkspaceTest.php
@@ -26,14 +26,6 @@ protected function initClient(?string $branchId = ''): void
         $this->clientWrapper = $this->getSynapseClientWrapper();
     }
 
-    public function setUp(): void
-    {
-        if (!$this->checkSynapseTests()) {
-            self::markTestSkipped('Synapse tests disabled.');
-        }
-        parent::setUp();
-    }
-
     private function getProvider(array $data = []): ProviderInterface
     {
         $mock = self::getMockBuilder(NullProvider::class)

diff --git a/tests/Writer/Table/Source/AbsWorkspaceItemSourceFactoryTest.php b/tests/Writer/Table/Source/AbsWorkspaceItemSourceFactoryTest.php
@@ -24,14 +24,6 @@ protected function initClient(?string $branchId = ''): void
         $this->clientWrapper = $this->getSynapseClientWrapper();
     }
 
-    public function setUp(): void
-    {
-        if (!$this->checkSynapseTests()) {
-            self::markTestSkipped('Synapse tests disabled.');
-        }
-        parent::setUp();
-    }
-
     public function testCreateSource(): void
     {
         $workspaces = new Workspaces($this->clientWrapper->getBranchClient());

diff --git a/tests/Writer/Workspace/AbsWriterWorkspaceTest.php b/tests/Writer/Workspace/AbsWriterWorkspaceTest.php
@@ -25,9 +25,6 @@ class AbsWriterWorkspaceTest extends AbstractTestCase
 
     public function setUp(): void
     {
-        if (!$this->checkSynapseTests()) {
-            self::markTestSkipped('Synapse tests disabled.');
-        }
         parent::setUp();
         $this->clearFileUploads([self::FILE_TAG]);
     }

diff --git a/tests/Writer/Workspace/SynapseWriterWorkspaceTest.php b/tests/Writer/Workspace/SynapseWriterWorkspaceTest.php
@@ -10,63 +10,22 @@
 use Keboola\OutputMapping\Tests\InitSynapseStorageClientTrait;
 use Keboola\OutputMapping\Tests\Needs\NeedsEmptyInputBucket;
 use Keboola\OutputMapping\Tests\Needs\NeedsEmptyOutputBucket;
-use Keboola\OutputMapping\Tests\Needs\NeedsTestTables;
-use Keboola\OutputMapping\Tests\Needs\TestSatisfyer;
 use Keboola\OutputMapping\Writer\TableWriter;
-use Keboola\StorageApi\Client;
-use Keboola\StorageApi\TableExporter;
 
 class SynapseWriterWorkspaceTest extends AbstractTestCase
 {
     use InitSynapseStorageClientTrait;
 
-    public function setUp(): void
-    {
-        if (!$this->checkSynapseTests()) {
-            self::markTestSkipped('Synapse tests disabled.');
-        }
-        parent::setUp();
-    }
-
     protected function initClient(?string $branchId = null): void
     {
         $this->clientWrapper = $this->getSynapseClientWrapper();
     }
 
     #[NeedsEmptyOutputBucket]
+    #[NeedsEmptyInputBucket]
     public function testSynapseTableOutputMapping(): void
     {
         // snowflake bucket does not work - https://keboola.atlassian.net/browse/KBC-228
-        $bucketName = 'testSynapseTableOutputMapping';
-
-        $outBucketId = TestSatisfyer::getBucketIdByDisplayName($this->clientWrapper, $bucketName, Client::STAGE_OUT);
-        if ($outBucketId !== null) {
-            $tables = $this->clientWrapper->getTableAndFileStorageClient()->listTables($outBucketId, ['include' => '']);
-            foreach ($tables as $table) {
-                $this->clientWrapper->getTableAndFileStorageClient()->dropTable($table['id']);
-            }
-        } else {
-            $outBucketId  = $this->clientWrapper->getTableAndFileStorageClient()->createBucket(
-                name: $bucketName,
-                stage: Client::STAGE_OUT,
-                backend: 'synapse',
-            );
-        }
-
-        $bucketId = TestSatisfyer::getBucketIdByDisplayName($this->clientWrapper, $bucketName, Client::STAGE_IN);
-        if ($bucketId !== null) {
-            $tables = $this->clientWrapper->getTableAndFileStorageClient()->listTables($bucketId, ['include' => '']);
-            foreach ($tables as $table) {
-                $this->clientWrapper->getTableAndFileStorageClient()->dropTable($table['id']);
-            }
-        } else {
-            $bucketId  = $this->clientWrapper->getTableAndFileStorageClient()->createBucket(
-                name: $bucketName,
-                stage: Client::STAGE_IN,
-                backend: 'synapse',
-            );
-        }
-
         $csv = new CsvFile($this->temp->getTmpFolder() . DIRECTORY_SEPARATOR . 'upload.csv');
         $csv->writeRow(['Id', 'Name', 'foo', 'bar']);
         $csv->writeRow(['id1', 'name1', 'foo1', 'bar1']);
@@ -77,7 +36,7 @@ public function testSynapseTableOutputMapping(): void
         // Create table
         for ($i = 0; $i < 2; $i++) {
             $tableIds[$i] = $this->clientWrapper->getTableAndFileStorageClient()->createTableAsync(
-                $bucketId,
+                $this->emptyInputBucketId,
                 'test' . ($i + 1),
                 $csv,
             );
@@ -93,16 +52,16 @@ public function testSynapseTableOutputMapping(): void
         $factory->getTableOutputStrategy(AbstractStrategyFactory::WORKSPACE_SYNAPSE)
             ->getDataStorage()->getWorkspaceId();
         $root = $this->temp->getTmpFolder();
-        $this->prepareWorkspaceWithTables($bucketId);
+        $this->prepareWorkspaceWithTables($this->emptyInputBucketId);
         $configs = [
             [
                 'source' => 'table1a',
-                'destination' => $outBucketId . '.table1a',
+                'destination' => $this->emptyOutputBucketId . '.table1a',
                 'distribution_key' => [],
             ],
             [
                 'source' => 'table2a',
-                'destination' => $outBucketId . '.table2a',
+                'destination' => $this->emptyOutputBucketId . '.table2a',
                 'distribution_key' => ['Id'],
             ],
         ];
@@ -131,21 +90,21 @@ public function testSynapseTableOutputMapping(): void
         $jobIds = $tableQueue->waitForAll();
         self::assertCount(2, $jobIds);
 
-        $tables = $this->clientWrapper->getTableAndFileStorageClient()->listTables($outBucketId);
+        $tables = $this->clientWrapper->getTableAndFileStorageClient()->listTables($this->emptyOutputBucketId);
         self::assertCount(2, $tables);
         $sortedTables = [$tables[0]['id'] => $tables[0], $tables[1]['id'] => $tables[1]];
         ksort($sortedTables);
         self::assertEquals(
-            [$outBucketId . '.table1a', $outBucketId . '.table2a'],
+            [$this->emptyOutputBucketId . '.table1a', $this->emptyOutputBucketId . '.table2a'],
             array_keys($sortedTables),
         );
-        self::assertArrayHasKey('distributionKey', $sortedTables[$outBucketId . '.table2a']);
-        self::assertEquals(['Id'], $sortedTables[$outBucketId . '.table2a']['distributionKey']);
+        self::assertArrayHasKey('distributionKey', $sortedTables[$this->emptyOutputBucketId . '.table2a']);
+        self::assertEquals(['Id'], $sortedTables[$this->emptyOutputBucketId . '.table2a']['distributionKey']);
         self::assertCount(2, $jobIds);
         self::assertNotEmpty($jobIds[0]);
         self::assertNotEmpty($jobIds[1]);
-        self::assertTableRowsEquals(
-            $outBucketId . '.table1a',
+        $this->assertTableRowsEquals(
+            $this->emptyOutputBucketId . '.table1a',
             [
                 '"id","name","foo","bar"',
                 '"id1","name1","foo1","bar1"',