create typeform-test-gcp

keboola · Feb 6, 2025 · bd488bf · bd488bf
1 parent 76baa29
commit bd488bf
Show file tree

Hide file tree

Showing 89 changed files with 2,566 additions and 1 deletion.
diff --git a/.github/actions/export-kbc-projects/action.yml b/.github/actions/export-kbc-projects/action.yml
@@ -0,0 +1,25 @@
+name: 'Export kbc projects with secrets'
+description: 'Exports kbc projects from repository environments including secrets'
+inputs:
+  secrets:
+    description: workflow secrets
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Generate projects.json file, replace secrets
+      shell: bash
+      run: |
+        # Filter secrets by the prefix, set key=value ENVs
+        jqQuery='to_entries[] | select(.key | startswith("TEST_KBC_PROJECT_")) | "\(.key)=\(.value|tostring) "'
+        export $(
+        (jq -r -j "$jqQuery") << EndOfSecrets
+          ${{ inputs.secrets }}
+        EndOfSecrets
+        )
+        
+        # Replace Storage API tokens
+        envsubst < .github/projects.json > projects.json
+        
+        # Fix tests caching
+        touch -d '1970-01-01T00:00:01' projects.json
diff --git a/.github/projects.json b/.github/projects.json
@@ -0,0 +1,10 @@
+[
+  {
+    "host": "connection.us-east4.gcp.keboola.com",
+    "project": 242,
+    "stagingStorage": "s3",
+    "backend": "snowflake",
+    "token": "242-199776-YT92zlG6fgBzMKFk7EQ7XXlXmmXeax7XbH5xV2Qx",
+    "legacyTransformation": true
+  }
+]
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -0,0 +1,6 @@
+Jira: XXXXX
+
+**Changes:**
+- ...
+
+-----------
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,3 +10,7 @@ jobs:
     secrets: inherit
     uses: ./.github/workflows/json-validation.yml
 
+  local-test:
+    name: "Run local tests"
+    secrets: inherit
+    uses: ./.github/workflows/local-tests.yml
diff --git a/.github/workflows/json-validation.yml b/.github/workflows/json-validation.yml
@@ -18,4 +18,3 @@ jobs:
 #          Ignore validation for manifest.json
 #          Manifest.json contains placeholders, that we want to ignore.
           json_exclude_regex: '^__[A-Z_]+__$'
-
diff --git a/.github/workflows/local-tests.yml b/.github/workflows/local-tests.yml
@@ -0,0 +1,60 @@
+name: "Local tests"
+on:
+  workflow_call:
+
+jobs:
+  local-tests:
+    name: local tests
+    runs-on: ubuntu-latest
+    steps:
+          # Checkout repository
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download keboola CLI
+        id: kbc_download_latest_release
+        uses: robinraju/release-downloader@v1.11
+        with:
+          repository: 'keboola/keboola-as-code'
+          latest: true
+          fileName: '*_linux_amd64.zip'
+
+      - name: Install keboola CLI
+        run: |
+          mkdir "${{ runner.temp }}/kbc"
+          latest_tag=${{steps.kbc_download_latest_release.outputs.tag_name}}
+          latest_version=${latest_tag:1}
+          
+          release_zip="keboola-cli_${latest_version}_linux_amd64.zip"
+          if [ ! -f "$release_zip" ]; then
+          echo "Could not download keboola-cli_${latest_version}_linux_amd64.zip from the latest release."
+          exit 1
+          fi
+          
+          # Unzip the CLI binary
+          release_bin="/usr/local/bin/kbc"
+          unzip "$release_zip" -d /usr/local/bin
+          chmod +x "$release_bin"
+          echo "Keboola as Code CLI installed: $release_bin"
+          kbc --version
+        shell: bash
+
+      - name: Set TEST_KBC_PROJECTS_FILE entries
+        uses: ./.github/actions/export-kbc-projects
+        with:
+          secrets: ${{ toJSON(secrets) }}
+
+      - name: Run tests
+        run: |
+          set -eo pipefail
+          kbc template test run --local-only
+        shell: bash
+        env:
+          TEST_KBC_PROJECTS_FILE: '${{ github.workspace }}/${{ vars.TEST_KBC_PROJECTS_FILE }}'
+          KBC_SECRET_TOKEN: ${{ secrets.KBC_SECRET_TOKEN }}
+          KBC_SECRET_BIGQUERY_PRIVATE_KEY: ${{ secrets.KBC_SECRET_BIGQUERY_PRIVATE_KEY }}
+          KBC_SECRET_BIGQUERY_PRIVATE_KEY_ID: ${{ secrets.KBC_SECRET_BIGQUERY_PRIVATE_KEY_ID }}
+          KBC_SECRET_POSTGRES_PASSWORD: ${{ secrets.KBC_SECRET_POSTGRES_PASSWORD }}
+          KBC_SECRET_POSTGRES_DB_USER: ${{ secrets.KBC_SECRET_POSTGRES_DB_USER }}
+          KBC_SECRET_SNOWFLAKE_PASSWORD: ${{ secrets.KBC_SECRET_SNOWFLAKE_PASSWORD }}
+          KBC_SECRET_SNOWFLAKE_DB_USER: ${{ secrets.KBC_SECRET_SNOWFLAKE_DB_USER }}
diff --git a/keboola-typeform/v1/tests/typeform-test-gcp/expected-out/.keboola/manifest.json b/keboola-typeform/v1/tests/typeform-test-gcp/expected-out/.keboola/manifest.json
@@ -0,0 +1,195 @@
+{
+  "version": 2,
+  "project": {
+    "id": __PROJECT_ID__,
+    "apiHost": "__STORAGE_API_HOST__"
+  },
+  "allowTargetEnv": false,
+  "sortBy": "id",
+  "naming": {
+    "branch": "{branch_name}",
+    "config": "{component_type}/{component_id}/{config_name}",
+    "configRow": "rows/{config_row_name}",
+    "schedulerConfig": "schedules/{config_name}",
+    "sharedCodeConfig": "_shared/{target_component_id}",
+    "sharedCodeConfigRow": "codes/{config_row_name}",
+    "variablesConfig": "variables",
+    "variablesValuesRow": "values/{config_row_name}",
+    "dataAppConfig": "app/{component_id}/{config_name}"
+  },
+  "allowedBranches": [
+    "*"
+  ],
+  "ignoredComponents": [],
+  "templates": {
+    "repositories": [
+      {
+        "type": "dir",
+        "name": "keboola",
+        "url": "../repository",
+        "ref": "main"
+      }
+    ]
+  },
+  "branches": [
+    {
+      "id": __MAIN_BRANCH_ID__,
+      "path": "main",
+      "metadata": {
+        "KBC.KAC.templates.instances": "[{\"instanceId\":\"%s\",\"instanceName\":\"test\",\"templateId\":\"keboola-typeform\",\"repositoryName\":\"keboola\",\"version\":\"1.0.1\",\"created\":{\"date\":\"%s\",\"tokenId\":\"%s\"},\"updated\":{\"date\":\"%s\",\"tokenId\":\"%s\"},\"mainConfig\":{\"configId\":\"%s\",\"componentId\":\"keboola.orchestrator\"}}]"
+      }
+    }
+  ],
+  "configurations": [
+    {
+      "branchId": __MAIN_BRANCH_ID__,
+      "componentId": "fisa.ex-typeform",
+      "id": "%s",
+      "path": "extractor/fisa.ex-typeform/data-source-in-typeform-instance",
+      "metadata": {
+        "KBC.KAC.templates.configId": "{\"idInTemplate\":\"data-source-in-typeform\"}",
+        "KBC.KAC.templates.configInputs": "[{\"input\":\"ex-typeform-config-token\",\"key\":\"parameters.config.#token\"}]",
+        "KBC.KAC.templates.instanceId": "%s",
+        "KBC.KAC.templates.repository": "keboola",
+        "KBC.KAC.templates.templateId": "keboola-typeform"
+      },
+      "rows": []
+    },
+    {
+      "branchId": __MAIN_BRANCH_ID__,
+      "componentId": "keboola.orchestrator",
+      "id": "%s",
+      "path": "other/keboola.orchestrator/flow-typeform-instance",
+      "metadata": {
+        "KBC.KAC.templates.configId": "{\"idInTemplate\":\"flow-typeform\"}",
+        "KBC.KAC.templates.instanceId": "%s",
+        "KBC.KAC.templates.repository": "keboola",
+        "KBC.KAC.templates.templateId": "keboola-typeform"
+      },
+      "rows": []
+    },
+    {
+      "branchId": __MAIN_BRANCH_ID__,
+      "componentId": "keboola.snowflake-transformation",
+      "id": "%s",
+      "path": "transformation/keboola.snowflake-transformation/transformation-in-typeform-instance",
+      "metadata": {
+        "KBC.KAC.templates.configId": "{\"idInTemplate\":\"transformation-in-typeform\"}",
+        "KBC.KAC.templates.instanceId": "%s",
+        "KBC.KAC.templates.repository": "keboola",
+        "KBC.KAC.templates.templateId": "keboola-typeform"
+      },
+      "rows": []
+    },
+    {
+      "branchId": __MAIN_BRANCH_ID__,
+      "componentId": "keboola.wr-db-pgsql",
+      "id": "%s",
+      "path": "writer/keboola.wr-db-pgsql/data-destination-out-crm-postgresql-instance",
+      "metadata": {
+        "KBC.KAC.templates.configId": "{\"idInTemplate\":\"out-typeform-postgresql-writer\"}",
+        "KBC.KAC.templates.configInputs": "[{\"input\":\"wr-postgresql-db-password\",\"key\":\"parameters.db.#password\"},{\"input\":\"wr-postgresql-db-database\",\"key\":\"parameters.db.database\"},{\"input\":\"wr-postgresql-db-driver\",\"key\":\"parameters.db.driver\"},{\"input\":\"wr-postgresql-db-hostname\",\"key\":\"parameters.db.host\"},{\"input\":\"wr-postgresql-db-port\",\"key\":\"parameters.db.port\"},{\"input\":\"wr-postgresql-db-schema\",\"key\":\"parameters.db.schema\"},{\"input\":\"wr-postgresql-db-user\",\"key\":\"parameters.db.user\"}]",
+        "KBC.KAC.templates.instanceId": "%s",
+        "KBC.KAC.templates.repository": "keboola",
+        "KBC.KAC.templates.rowsIds": "[{\"idInProject\":\"%s\",\"idInTemplate\":\"answer-option\"},{\"idInProject\":\"%s\",\"idInTemplate\":\"question\"},{\"idInProject\":\"%s\",\"idInTemplate\":\"responses\"},{\"idInProject\":\"%s\",\"idInTemplate\":\"survey\"}]",
+        "KBC.KAC.templates.templateId": "keboola-typeform"
+      },
+      "rows": [
+        {
+          "id": "%s",
+          "path": "rows/answer-option"
+        },
+        {
+          "id": "%s",
+          "path": "rows/question"
+        },
+        {
+          "id": "%s",
+          "path": "rows/responses"
+        },
+        {
+          "id": "%s",
+          "path": "rows/survey"
+        }
+      ]
+    },
+    {
+      "branchId": __MAIN_BRANCH_ID__,
+      "componentId": "keboola.wr-db-snowflake-gcs-s3",
+      "id": "%s",
+      "path": "writer/keboola.wr-db-snowflake-gcs-s3/data-destination-out-typeform-snowflake-instance",
+      "metadata": {
+        "KBC.KAC.templates.configId": "{\"idInTemplate\":\"data-destination-out-typeform-snowflake\"}",
+        "KBC.KAC.templates.configInputs": "[{\"input\":\"wr-snowflake-db-password\",\"key\":\"parameters.db.#password\"},{\"input\":\"wr-snowflake-db-database\",\"key\":\"parameters.db.database\"},{\"input\":\"wr-snowflake-db-host\",\"key\":\"parameters.db.host\"},{\"input\":\"wr-snowflake-db-port\",\"key\":\"parameters.db.port\"},{\"input\":\"wr-snowflake-db-schema\",\"key\":\"parameters.db.schema\"},{\"input\":\"wr-snowflake-db-user\",\"key\":\"parameters.db.user\"},{\"input\":\"wr-snowflake-db-warehouse\",\"key\":\"parameters.db.warehouse\"}]",
+        "KBC.KAC.templates.instanceId": "%s",
+        "KBC.KAC.templates.repository": "keboola",
+        "KBC.KAC.templates.rowsIds": "[{\"idInProject\":\"%s\",\"idInTemplate\":\"answer-option-001\"},{\"idInProject\":\"%s\",\"idInTemplate\":\"question-001\"},{\"idInProject\":\"%s\",\"idInTemplate\":\"responses-001\"},{\"idInProject\":\"%s\",\"idInTemplate\":\"survey-001\"}]",
+        "KBC.KAC.templates.templateId": "keboola-typeform"
+      },
+      "rows": [
+        {
+          "id": "%s",
+          "path": "rows/answer-option"
+        },
+        {
+          "id": "%s",
+          "path": "rows/question"
+        },
+        {
+          "id": "%s",
+          "path": "rows/responses"
+        },
+        {
+          "id": "%s",
+          "path": "rows/survey"
+        }
+      ]
+    },
+    {
+      "branchId": __MAIN_BRANCH_ID__,
+      "componentId": "keboola.wr-google-bigquery-v2",
+      "id": "%s",
+      "path": "writer/keboola.wr-google-bigquery-v2/data-destination-out-typeform-bigquery-instance",
+      "metadata": {
+        "KBC.KAC.templates.configId": "{\"idInTemplate\":\"data-destination-out-typeform-bigquery\"}",
+        "KBC.KAC.templates.configInputs": "[{\"input\":\"wr-google-bigquery-v2-service-account-private-key\",\"key\":\"parameters.service_account.#private_key\"},{\"input\":\"wr-google-bigquery-v2-service-account-client-email\",\"key\":\"parameters.service_account.client_email\"},{\"input\":\"wr-google-bigquery-v2-service-account-client-id\",\"key\":\"parameters.service_account.client_id\"},{\"input\":\"wr-google-bigquery-v2-service-account-private-key-id\",\"key\":\"parameters.service_account.private_key_id\"},{\"input\":\"wr-google-bigquery-v2-service-account-project-id\",\"key\":\"parameters.service_account.project_id\"}]",
+        "KBC.KAC.templates.instanceId": "%s",
+        "KBC.KAC.templates.repository": "keboola",
+        "KBC.KAC.templates.rowsIds": "[{\"idInProject\":\"%s\",\"idInTemplate\":\"answer-option\"},{\"idInProject\":\"%s\",\"idInTemplate\":\"question\"},{\"idInProject\":\"%s\",\"idInTemplate\":\"responses\"},{\"idInProject\":\"%s\",\"idInTemplate\":\"survey\"}]",
+        "KBC.KAC.templates.templateId": "keboola-typeform"
+      },
+      "rows": [
+        {
+          "id": "%s",
+          "path": "rows/answer-option"
+        },
+        {
+          "id": "%s",
+          "path": "rows/question"
+        },
+        {
+          "id": "%s",
+          "path": "rows/responses"
+        },
+        {
+          "id": "%s",
+          "path": "rows/survey"
+        }
+      ]
+    },
+    {
+      "branchId": __MAIN_BRANCH_ID__,
+      "componentId": "keboola.wr-google-sheets",
+      "id": "%s",
+      "path": "writer/keboola.wr-google-sheets/data-destination-out-typeform-gsheet-instance",
+      "metadata": {
+        "KBC.KAC.templates.configId": "{\"idInTemplate\":\"data-destination-out-typeform-gsheet\"}",
+        "KBC.KAC.templates.configInputs": "[{\"input\":\"oauth-google-sheets\",\"key\":\"authorization.oauth_api\"},{\"input\":\"google-sheet-id\",\"key\":\"parameters.tables[0].fileId\"},{\"input\":\"google-sheet-id\",\"key\":\"parameters.tables[1].fileId\"},{\"input\":\"google-sheet-id\",\"key\":\"parameters.tables[2].fileId\"},{\"input\":\"google-sheet-id\",\"key\":\"parameters.tables[3].fileId\"}]",
+        "KBC.KAC.templates.instanceId": "%s",
+        "KBC.KAC.templates.repository": "keboola",
+        "KBC.KAC.templates.templateId": "keboola-typeform"
+      },
+      "rows": []
+    }
+  ]
+}
diff --git a/keboola-typeform/v1/tests/typeform-test-gcp/expected-out/description.md b/keboola-typeform/v1/tests/typeform-test-gcp/expected-out/description.md
@@ -0,0 +1 @@
+
diff --git a/keboola-typeform/v1/tests/typeform-test-gcp/expected-out/main/description.md b/keboola-typeform/v1/tests/typeform-test-gcp/expected-out/main/description.md
@@ -0,0 +1 @@
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -18,4 +18,3 @@ jobs:
		# Ignore validation for manifest.json
		# Manifest.json contains placeholders, that we want to ignore.
		json_exclude_regex: '^__[A-Z_]+__$'