Merge pull request #362 from keboola/erik-PST-2398-split2

[output-mapping] delete where - part2 - filters values_from_set, values_from_workspace
keboola · Jan 30, 2025 · fa68f2f · fa68f2f
2 parents a5a2560 + 328438b
commit fa68f2f
Show file tree

Hide file tree

Showing 13 changed files with 1,102 additions and 20 deletions.
diff --git a/src/Mapping/MappingFromConfigurationDeleteWhereFilterFromWorkspace.php b/src/Mapping/MappingFromConfigurationDeleteWhereFilterFromWorkspace.php
@@ -16,8 +16,8 @@ public function getWorkspaceTable(): string
         return $this->mapping['values_from_workspace']['table'];
     }
 
-    public function getWorkspaceColumn(): string
+    public function getWorkspaceColumn(): ?string
     {
-        return $this->mapping['values_from_workspace']['column'];
+        return $this->mapping['values_from_workspace']['column'] ?? null;
     }
 }
diff --git a/src/Mapping/MappingFromProcessedConfiguration.php b/src/Mapping/MappingFromProcessedConfiguration.php
@@ -208,4 +208,21 @@ public function getItemSourceType(): SourceType
     {
         return $this->source->getSourceType();
     }
+
+    /**
+     * @return MappingFromConfigurationDeleteWhere[]|null
+     */
+    public function getDeleteWhere(): ?array
+    {
+        if (!isset($this->mapping['delete_where'])) {
+            return null;
+        }
+
+        return array_map(
+            function (array $deleteWhere): MappingFromConfigurationDeleteWhere {
+                return new MappingFromConfigurationDeleteWhere($deleteWhere);
+            },
+            $this->mapping['delete_where'],
+        );
+    }
 }
diff --git a/src/Storage/DeleteTableRowsOptionsFactory.php b/src/Storage/DeleteTableRowsOptionsFactory.php
@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Keboola\OutputMapping\Storage;
+
+use Keboola\OutputMapping\Mapping\MappingFromConfigurationDeleteWhere;
+use Keboola\OutputMapping\Mapping\MappingFromConfigurationDeleteWhereFilterFromSet;
+use Keboola\OutputMapping\Mapping\MappingFromConfigurationDeleteWhereFilterFromWorkspace;
+
+class DeleteTableRowsOptionsFactory
+{
+    public static function createFromLegacyDeleteWhereColumn(
+        string $column,
+        string $operator,
+        array $values,
+    ): array {
+        return [
+            'whereColumn' => $column,
+            'whereOperator' => $operator,
+            'whereValues' => $values,
+        ];
+    }
+
+    public static function createFromDeleteWhere(
+        MappingFromConfigurationDeleteWhere $deleteWhere,
+    ): ?array {
+        $options = [];
+
+        if ($deleteWhere->getChangedSince()) {
+            $options['changedSince'] = $deleteWhere->getChangedSince();
+        }
+
+        if ($deleteWhere->getChangedUntil()) {
+            $options['changedUntil'] = $deleteWhere->getChangedUntil();
+        }
+
+        $whereFilters = self::createWhereFilters($deleteWhere);
+        if ($whereFilters !== []) {
+            $options['whereFilters'] = $whereFilters;
+        }
+
+        return $options === [] ? null : $options;
+    }
+
+    private static function createWhereFilters(MappingFromConfigurationDeleteWhere $deleteWhere): array
+    {
+        if (!$deleteWhere->getWhereFilters()) {
+            return [];
+        }
+
+        $whereFilters = [];
+        foreach ($deleteWhere->getWhereFilters() as $deleteFilter) {
+            $whereFilter = [
+                'column' => $deleteFilter->getColumn(),
+                'operator' => $deleteFilter->getOperator(),
+            ];
+
+            if ($deleteFilter instanceof MappingFromConfigurationDeleteWhereFilterFromSet) {
+                $whereFilter['values'] = $deleteFilter->getValues();
+                $whereFilters[] = $whereFilter;
+            }
+            if ($deleteFilter instanceof MappingFromConfigurationDeleteWhereFilterFromWorkspace) {
+                $whereFilter['valuesByTableInWorkspace'] = [
+                    'workspaceId' => $deleteFilter->getWorkspaceId(),
+                    'table' => $deleteFilter->getWorkspaceTable(),
+                    'column' => $deleteFilter->getWorkspaceColumn() ?: $deleteFilter->getColumn(),
+                ];
+                $whereFilters[] = $whereFilter;
+            }
+        }
+
+        return $whereFilters;
+    }
+}
diff --git a/src/Storage/TableDataModifier.php b/src/Storage/TableDataModifier.php
@@ -5,6 +5,7 @@
 namespace Keboola\OutputMapping\Storage;
 
 use Keboola\OutputMapping\Exception\InvalidOutputException;
+use Keboola\OutputMapping\Mapping\MappingFromConfigurationDeleteWhere;
 use Keboola\OutputMapping\Mapping\MappingFromProcessedConfiguration;
 use Keboola\OutputMapping\Writer\Table\MappingDestination;
 use Keboola\StorageApi\ClientException;
@@ -19,13 +20,7 @@ public function __construct(
 
     public function updateTableData(MappingFromProcessedConfiguration $source, MappingDestination $destination): void
     {
-        if (!is_null($source->getDeleteWhereColumn())) {
-            // Delete rows
-            $deleteOptions = [
-                'whereColumn' => $source->getDeleteWhereColumn(),
-                'whereOperator' => $source->getDeleteWhereOperator(),
-                'whereValues' => $source->getDeleteWhereValues(),
-            ];
+        foreach ($this->prepareDeleteOptionsList($source) as $deleteOptions) {
             try {
                 $this->clientWrapper->getTableAndFileStorageClient()->deleteTableRows(
                     $destination->getTableId(),
@@ -44,4 +39,29 @@ public function updateTableData(MappingFromProcessedConfiguration $source, Mappi
             }
         }
     }
+
+    private function prepareDeleteOptionsList(MappingFromProcessedConfiguration $source): array
+    {
+        if ($source->getDeleteWhere() !== null) {
+            return array_filter(
+                array_map(
+                    function (MappingFromConfigurationDeleteWhere $deleteWhere) {
+                        return DeleteTableRowsOptionsFactory::createFromDeleteWhere($deleteWhere);
+                    },
+                    $source->getDeleteWhere(),
+                ),
+            );
+        }
+        if ($source->getDeleteWhereColumn() !== null) {
+            return [
+                DeleteTableRowsOptionsFactory::createFromLegacyDeleteWhereColumn(
+                    $source->getDeleteWhereColumn(),
+                    $source->getDeleteWhereOperator(),
+                    $source->getDeleteWhereValues(),
+                ),
+            ];
+        }
+
+        return [];
+    }
 }
diff --git a/src/Writer/Helper/DeleteWhereHelper.php b/src/Writer/Helper/DeleteWhereHelper.php
@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Keboola\OutputMapping\Writer\Helper;
+
+class DeleteWhereHelper
+{
+    private static function hasWhereFilters(array $deleteWhere): bool
+    {
+        return isset($deleteWhere['where_filters']) && is_array($deleteWhere['where_filters']);
+    }
+
+    private static function isSourceWorkspaceIdNeeded(array $whereFilter): bool
+    {
+        return isset($whereFilter['values_from_workspace'])
+            && empty($whereFilter['values_from_workspace']['workspace_id']);
+    }
+
+    public static function addWorkspaceIdToValuesFromWorkspaceIfMissing(array $deleteWhere, string $workspaceId): array
+    {
+        if (self::hasWhereFilters($deleteWhere)) {
+            $deleteWhere['where_filters'] = array_map(
+                function (array $whereFilter) use ($workspaceId): array {
+                    if (self::isSourceWorkspaceIdNeeded($whereFilter)) {
+                        $whereFilter['values_from_workspace']['workspace_id'] = $workspaceId;
+                    }
+                    return $whereFilter;
+                },
+                $deleteWhere['where_filters'],
+            );
+        }
+
+        return $deleteWhere;
+    }
+}
diff --git a/src/Writer/Table/TableConfigurationResolver.php b/src/Writer/Table/TableConfigurationResolver.php
@@ -10,8 +10,10 @@
 use Keboola\OutputMapping\OutputMappingSettings;
 use Keboola\OutputMapping\SystemMetadata;
 use Keboola\OutputMapping\Writer\Helper\ConfigurationMerger;
+use Keboola\OutputMapping\Writer\Helper\DeleteWhereHelper;
 use Keboola\OutputMapping\Writer\Helper\PrimaryKeyHelper;
 use Keboola\OutputMapping\Writer\Helper\TagsHelper;
+use Keboola\OutputMapping\Writer\Table\Source\SourceType;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\Config\Definition\Exception\InvalidConfigurationException;
 
@@ -36,6 +38,7 @@ public function resolveTableConfiguration(
             $mappingFromManifest['destination'] ?? null,
             $mappingFromConfiguration['destination'] ?? null,
         );
+
         if (isset($config['primary_key'])) {
             $config['primary_key'] = PrimaryKeyHelper::normalizeKeyArray($this->logger, $config['primary_key']);
         }
@@ -46,6 +49,20 @@ public function resolveTableConfiguration(
             );
         }
 
+        if (isset($config['delete_where'])) {
+            if ($source->getSourceType() === SourceType::WORKSPACE) {
+                $config['delete_where'] = array_map(
+                    function (array $deleteWhere) use ($source): array {
+                        return DeleteWhereHelper::addWorkspaceIdToValuesFromWorkspaceIfMissing(
+                            $deleteWhere,
+                            $source->getWorkspaceId(),
+                        );
+                    },
+                    $config['delete_where'],
+                );
+            }
+        }
+
         if ($configuration->hasTagStagingFilesFeature()) {
             $config = TagsHelper::addSystemTags($config, $systemMetadata);
         }

diff --git a/tests/Mapping/MappingFromConfigurationDeleteWhereFilterFromWorkspaceTest.php b/tests/Mapping/MappingFromConfigurationDeleteWhereFilterFromWorkspaceTest.php
@@ -4,14 +4,33 @@
 
 namespace Keboola\OutputMapping\Tests\Mapping;
 
+use Generator;
 use Keboola\OutputMapping\Mapping\MappingFromConfigurationDeleteWhereFilterFromWorkspace;
 use PHPUnit\Framework\TestCase;
 
 class MappingFromConfigurationDeleteWhereFilterFromWorkspaceTest extends TestCase
 {
-    public function testGetters(): void
+    public static function configurationProvider(): Generator
     {
-        $whereFilterFromSet = new MappingFromConfigurationDeleteWhereFilterFromWorkspace(
+        yield 'minimal configuration' => [
+            [
+                'column' => 'columnName',
+                'operator' => 'ne',
+                'values_from_workspace' => [
+                    'workspace_id' => '123',
+                    'table' => 'workspaceTable',
+                ],
+            ],
+            [
+                'column' => 'columnName',
+                'operator' => 'ne',
+                'workspace_id' => '123',
+                'table' => 'workspaceTable',
+                'workspace_column' => null,
+            ],
+        ];
+
+        yield 'full configuration' => [
             [
                 'column' => 'columnName',
                 'operator' => 'ne',
@@ -21,12 +40,27 @@ public function testGetters(): void
                     'column' => 'workspaceColumn',
                 ],
             ],
-        );
+            [
+                'column' => 'columnName',
+                'operator' => 'ne',
+                'workspace_id' => '123',
+                'table' => 'workspaceTable',
+                'workspace_column' => 'workspaceColumn',
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider configurationProvider
+     */
+    public function testGetters(array $config, array $expected): void
+    {
+        $whereFilterFromSet = new MappingFromConfigurationDeleteWhereFilterFromWorkspace($config);
 
-        self::assertSame('columnName', $whereFilterFromSet->getColumn());
-        self::assertSame('ne', $whereFilterFromSet->getOperator());
-        self::assertSame('123', $whereFilterFromSet->getWorkspaceId());
-        self::assertSame('workspaceTable', $whereFilterFromSet->getWorkspaceTable());
-        self::assertSame('workspaceColumn', $whereFilterFromSet->getWorkspaceColumn());
+        self::assertSame($expected['column'], $whereFilterFromSet->getColumn());
+        self::assertSame($expected['operator'], $whereFilterFromSet->getOperator());
+        self::assertSame($expected['workspace_id'], $whereFilterFromSet->getWorkspaceId());
+        self::assertSame($expected['table'], $whereFilterFromSet->getWorkspaceTable());
+        self::assertSame($expected['workspace_column'], $whereFilterFromSet->getWorkspaceColumn());
     }
 }
diff --git a/tests/Mapping/MappingFromProcessedConfigurationTest.php b/tests/Mapping/MappingFromProcessedConfigurationTest.php
@@ -69,6 +69,7 @@ public function testBasic(): void
         self::assertFalse($mapping->isIncremental());
         self::assertEquals(SourceType::WORKSPACE, $mapping->getItemSourceType());
         self::assertInstanceOf(MappingDestination::class, $mapping->getDestination());
+        self::assertNull($mapping->getDeleteWhere());
     }
 
     public function testTableMetadata(): void
@@ -277,4 +278,31 @@ public function testPrimaryKeyAndColumsAreConvertedToStrings(): void
         self::assertSame(['', '123', 'col1', 'col2'], $mapping->getColumns());
         self::assertSame(['', '123', 'col1'], $mapping->getPrimaryKey());
     }
+
+    public function testGetDeleteWhere(): void
+    {
+        $mapping = [
+            'destination' => 'in.c-main.table',
+            'delete_where' => [
+                [
+                    'changed_since' => '-7 days',
+                ],
+                [
+                    'changed_until' => '-2 days',
+                ],
+            ],
+        ];
+
+        $mapping = new MappingFromProcessedConfiguration(
+            $mapping,
+            $this->createMock(MappingFromRawConfigurationAndPhysicalDataWithManifest::class),
+        );
+
+        $deleteWhere = $mapping->getDeleteWhere();
+        self::assertNotNull($deleteWhere);
+        self::assertCount(2, $deleteWhere);
+
+        self::assertSame('-7 days', $deleteWhere[0]->getChangedSince());
+        self::assertSame('-2 days', $deleteWhere[1]->getChangedUntil());
+    }
 }
diff --git a/tests/Needs/TestSatisfyer.php b/tests/Needs/TestSatisfyer.php
@@ -238,7 +238,7 @@ public static function satisfyTestNeeds(
 
                 // Create table
                 $propNames = ['firstTableId', 'secondTableId', 'thirdTableId'];
-                for ($i = 0; $i < max($tableCount, count($propNames)); $i++) {
+                for ($i = 0; $i < $tableCount; $i++) {
                     $tableIds[$i] = $clientWrapper->getTableAndFileStorageClient()->createTableAsync(
                         $testBucketId,
                         'test' . ($i + 1),
@@ -254,7 +254,7 @@ public static function satisfyTestNeeds(
 
                 // Create table
                 $propNames = ['firstTableId', 'secondTableId', 'thirdTableId'];
-                for ($i = 0; $i < max($tableCount, count($propNames)); $i++) {
+                for ($i = 0; $i < $tableCount; $i++) {
                     $tableName = 'test' . ($i + 1);
                     $tableId = $clientWrapper->getTableAndFileStorageClient()->createTableDefinition(
                         $testBucketId,