diff --git a/Makefile b/Makefile index 456cbee0c..2f41bc80c 100644 --- a/Makefile +++ b/Makefile @@ -22,21 +22,20 @@ SUBDIRS_ALL=$(SUBDIRS_HAS_UNIT) integration/ end_to_end/ GOLANG_LINTER=$(GOPATH)/bin/golangci-lint GINKGO=$(GOPATH)/bin/ginkgo GOIMPORTS=$(GOPATH)/bin/goimports -GO_ENV=GO111MODULE=on # ensures the project still compiles in $GOPATH/src using golang versions 1.12 and below -GO_BUILD=$(GO_ENV) go build -mod=readonly +GO_BUILD=go build -mod=readonly DEBUG=-gcflags=all="-N -l" CUSTOM_BACKUP_DIR ?= "/tmp" helper_path ?= $(BIN_DIR)/$(HELPER) depend : - $(GO_ENV) go mod download + go mod download -$(GINKGO) : - $(GO_ENV) go install github.com/onsi/ginkgo/ginkgo +$(GINKGO) : # v1.14.0 is compatible with centos6 default gcc version + go install github.com/onsi/ginkgo/ginkgo@v1.14.0 $(GOIMPORTS) : - $(GO_ENV) go install golang.org/x/tools/cmd/goimports + go install golang.org/x/tools/cmd/goimports@latest format : $(GOIMPORTS) @goimports -w $(shell find . -type f -name '*.go' -not -path "./vendor/*") @@ -52,21 +51,21 @@ lint : $(GOLANG_LINTER) golangci-lint run --tests=false unit : $(GINKGO) - $(GO_ENV) ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 + ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 unit_all_gpdb_versions : $(GINKGO) - TEST_GPDB_VERSION=4.3.999 $(GO_ENV) ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 - TEST_GPDB_VERSION=5.999.0 $(GO_ENV) ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 - TEST_GPDB_VERSION=6.999.0 $(GO_ENV) ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 - TEST_GPDB_VERSION=7.0.0 $(GO_ENV) ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 # GPDB master + TEST_GPDB_VERSION=4.3.999 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 + TEST_GPDB_VERSION=5.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 + TEST_GPDB_VERSION=6.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 + TEST_GPDB_VERSION=7.0.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 # GPDB master integration : $(GINKGO) - $(GO_ENV) ginkgo $(GINKGO_FLAGS) integration 2>&1 + ginkgo $(GINKGO_FLAGS) integration 2>&1 test : build unit integration end_to_end : $(GINKGO) - $(GO_ENV) ginkgo $(GINKGO_FLAGS) -slowSpecThreshold=10 end_to_end -- --custom_backup_dir $(CUSTOM_BACKUP_DIR) 2>&1 + ginkgo $(GINKGO_FLAGS) -slowSpecThreshold=10 end_to_end -- --custom_backup_dir $(CUSTOM_BACKUP_DIR) 2>&1 coverage : @./show_coverage.sh @@ -108,6 +107,10 @@ clean : rm -f $(BIN_DIR)/$(BACKUP) $(BACKUP) $(BIN_DIR)/$(RESTORE) $(RESTORE) $(BIN_DIR)/$(HELPER) $(HELPER) # Test artifacts rm -rf /tmp/go-build* /tmp/gexec_artifacts* /tmp/ginkgo* + docker stop s3-minio # stop minio before removing its data directories + docker rm s3-minio + rm -rf /tmp/minio + rm -f /tmp/minio_config.yaml # Code coverage files rm -rf /tmp/cover* /tmp/unit* go clean -i -r -x -testcache -modcache @@ -126,3 +129,12 @@ info-report: @echo "Info and verbose messaging:" @echo "" @ag "gplog.Info|gplog.Verbose" --ignore "*_test*" + +test-s3-local: build install + ${PWD}/plugins/generate_minio_config.sh + mkdir -p /tmp/minio/gpbackup-s3-test + docker run -d --name s3-minio -p 9000:9000 -p 9001:9001 -v /tmp/minio:/data/minio quay.io/minio/minio server /data/minio --console-address ":9001" + sleep 2 # Wait for minio server to start up + ${PWD}/plugins/plugin_test.sh $(BIN_DIR)/gpbackup_s3_plugin /tmp/minio_config.yaml + docker stop s3-minio + docker rm s3-minio diff --git a/README.md b/README.md index f479f31b2..edacb235c 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ The `build` target will put the `gpbackup` and `gprestore` binaries in `$HOME/go This will also attempt to copy `gpbackup_helper` to the greenplum segments (retrieving hostnames from `gp_segment_configuration`). Pay attention to the output as it will indicate whether this operation was successful. -`make build_linux` and `make build_mac` are for cross compiling between macOS and Linux +`make build_linux` is for cross compiling on macOS, and the target is Linux. `make install` will scp the `gpbackup_helper` binary (used with -single-data-file flag) to all hosts diff --git a/arenadata/run_gpbackup_tests.bash b/arenadata/run_gpbackup_tests.bash index 174ff9d18..ccea91d1d 100644 --- a/arenadata/run_gpbackup_tests.bash +++ b/arenadata/run_gpbackup_tests.bash @@ -14,6 +14,6 @@ source /usr/local/greenplum-db-devel/greenplum_path.sh; source ~/gpdb_src/gpAux/gpdemo/gpdemo-env.sh; gpconfig -c shared_preload_libraries -v dummy_seclabel; gpstop -ar; -wget https://golang.org/dl/go1.14.15.linux-amd64.tar.gz; -tar -C ~/ -xzf go1.14.15.linux-amd64.tar.gz; +wget https://golang.org/dl/go1.17.6.linux-amd64.tar.gz; +tar -C ~/ -xzf go1.17.6.linux-amd64.tar.gz; PATH=$PATH:~/go/bin GOPATH=~/go make depend build install test end_to_end -C go/src/github.com/greenplum-db/gpbackup/" diff --git a/backup/backup.go b/backup/backup.go index 9a8b485e3..7557faca7 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -1,6 +1,7 @@ package backup import ( + "encoding/json" "errors" "fmt" "os" @@ -522,3 +523,77 @@ func CreateInitialSegmentPipes(oidList []string, c *cluster.Cluster, connectionP } return maxPipes } + +type TableLocks struct { + Oid uint32 + Database string + Relation string + Mode string + Application string + Granted string + User string + Pid string +} + +func getTableLocks(table Table) []TableLocks { + conn := dbconn.NewDBConnFromEnvironment(MustGetFlagString(options.DBNAME)) + conn.MustConnect(1) + var query string + defer conn.Close() + if conn.Version.Before("6") { + query = fmt.Sprintf(` + SELECT c.oid as oid, + coalesce(a.datname, '') as database, + n.nspname || '.' || l.relation as relation, + l.mode, + l.GRANTED as granted, + coalesce(a.application_name, '') as application, + coalesce(a.usename, '') as user, + a.procpid as pid + FROM pg_stat_activity a + JOIN pg_locks l ON l.pid = a.procpid + JOIN pg_class c on c.oid = l.relation + JOIN pg_namespace n on n.oid=c.relnamespace + WHERE (a.datname = '%s' OR a.datname IS NULL) + AND NOT a.procpid = pg_backend_pid() + AND relation = '%s'::regclass + AND mode = 'AccessExclusiveLock' + ORDER BY a.query_start; + `, conn.DBName, table.FQN()) + } else { + query = fmt.Sprintf(` + SELECT c.oid as oid, + coalesce(a.datname, '') as database, + n.nspname || '.' || l.relation relation, + l.mode, + l.GRANTED as granted, + coalesce(a.application_name, '') as application, + coalesce(a.usename, '') as user, + a.pid + FROM pg_stat_activity a + JOIN pg_locks l ON l.pid = a.pid + JOIN pg_class c on c.oid = l.relation + JOIN pg_namespace n on n.oid=c.relnamespace + WHERE (a.datname = '%s' OR a.datname IS NULL) + AND NOT a.pid = pg_backend_pid() + AND relation = '%s'::regclass + AND mode = 'AccessExclusiveLock' + ORDER BY a.query_start; + `, conn.DBName, table.FQN()) + } + + locksResults := make([]TableLocks, 0) + err := conn.Select(&locksResults, query) + if err != nil { + gplog.FatalOnError(err) + } + + return locksResults +} + +func logTableLocks(table Table, whichConn int) { + locks := getTableLocks(table) + jsonData, _ := json.Marshal(&locks) + gplog.Warn("Worker %d could not acquire AccessShareLock for table %s. Terminating worker and deferring table to main worker thread.", whichConn,table.FQN()) + gplog.Warn("Locks held on table %s: %s", table.FQN(), jsonData) +} diff --git a/backup/data.go b/backup/data.go index 04e20a7d1..a6e3887e7 100644 --- a/backup/data.go +++ b/backup/data.go @@ -172,9 +172,8 @@ func backupDataForAllTablesCopyQueue(tables []Table) []map[uint32]int64 { // the following WARN message is nicely outputted. fmt.Printf("\n") } - gplog.Warn("Worker %d could not acquire AccessShareLock for table %s. Terminating worker and deferring table to main worker thread.", - whichConn, table.FQN()) - + // Log locks held on the table + logTableLocks(table, whichConn) oidMap.Store(table.Oid, Deferred) // Rollback transaction since it's in an aborted state connectionPool.MustRollback(whichConn) @@ -300,9 +299,8 @@ func backupDataForAllTables(tables []Table) []map[uint32]int64 { // the following WARN message is nicely outputted. fmt.Printf("\n") } - gplog.Warn("Worker %d could not acquire AccessShareLock for table %s. Terminating worker and deferring table to main worker thread.", - whichConn, table.FQN()) - + // Log locks held on the table + logTableLocks(table, whichConn) // Defer table to main worker thread deferredTablesMutex.Lock() deferredTables = append(deferredTables, table) @@ -395,6 +393,5 @@ func LockTableNoWait(dataTable Table, connNum int) error { if err != nil { return err } - return nil } diff --git a/ci/gpbackup-generated.yml b/ci/gpbackup-generated.yml index bf0063c15..8cda3ccec 100644 --- a/ci/gpbackup-generated.yml +++ b/ci/gpbackup-generated.yml @@ -12,7 +12,7 @@ ## file (example: templates/gpbackup-tpl.yml) and regenerate the pipeline ## using appropriate tool (example: gen_pipeline.py -p gpbackup-release). ## ---------------------------------------------------------------------- -## Generated by gen_pipeline.py at: 2021-07-27 15:59:55.830445 +## Generated by gen_pipeline.py at: 2022-01-28 05:33:23.346490 ## Template file: gpbackup-tpl.yml ## Pipeline Name: gpbackup ## Nightly Trigger: True @@ -26,7 +26,6 @@ groups: - build_gppkgs - GPDB4.3 - GPDB5 - - GPDB5-oracle7 - GPDB5-sles11 - GPDB6-ubuntu - ddboost_plugin_and_boostfs_tests_43 @@ -49,7 +48,6 @@ groups: - name: GPDB5 jobs: - GPDB5 - - GPDB5-oracle7 - GPDB5-sles11 - scale-5x - ddboost_plugin_and_boostfs_tests_5x @@ -82,70 +80,43 @@ groups: ##### Anchors ##### -ccp_default_params_anchor: &ccp_default_params +anchors: +- &ccp_default_params action: create delete_on_failure: true generate_random_name: true plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 terraform_source: ccp_src/google/ -ccp_aws_default_params_anchor: &ccp_aws_default_params - action: create - delete_on_failure: true - generate_random_name: true - plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 - terraform_source: ccp_src/aws/ - -ccp_vars_anchor: &ccp_default_vars +- &ccp_default_vars instance_type: n1-standard-1 PLATFORM: centos6 -ccp_vars_anchor_aws: &ccp_default_vars_aws - aws_instance-node-instance_type: t2.medium - PLATFORM: centos6 - -ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params - AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} - AWS_DEFAULT_REGION: {{aws-region}} +- &ccp_gen_cluster_default_params + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + AWS_DEFAULT_REGION: ((aws-region)) BUCKET_PATH: clusters-google/ - BUCKET_NAME: {{tf-bucket-name}} + BUCKET_NAME: ((tf-bucket-name)) CLOUD_PROVIDER: google -ddboost_params_anchor: &ddboost_params - DD_SOURCE_HOST: {{datadomain_source_host}} - DD_USER: {{datadomain_user}} - DD_PW: {{datadomain_password}} - -ddboost_test_params_anchor: &ddboost_test_params - <<: *ddboost_params - DD_DEST_HOST: {{datadomain_dest_host}} - DD_ENCRYPTED_PW: {{encrypted_datadomain_password}} +- &destroy_params + action: destroy + plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 + terraform_source: ccp_src/google/ + env_name_file: terraform/name + vars: + aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required. + aws_ebs_volume_type: standard -ccp_destroy_anchor: &ccp_destroy +- &ccp_destroy put: terraform - params: &destroy_params - action: destroy - env_name_file: terraform/name - plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 - terraform_source: ccp_src/google/ - vars: - aws_instance-node-instance_type: t2.micro - aws_ebs_volume_type: standard - instance_type: n1-standard-1 - get_params: - action: destroy - -ccp_destroy_anchor_aws: &ccp_destroy_aws - put: terraform_aws params: <<: *destroy_params - env_name_file: terraform_aws/name - terraform_source: ccp_src/aws/ get_params: action: destroy -ccp_nvme_destroy_anchor: &ccp_destroy_nvme +- &ccp_destroy_nvme put: terraform params: <<: *destroy_params @@ -153,144 +124,64 @@ ccp_nvme_destroy_anchor: &ccp_destroy_nvme get_params: action: destroy -ddboost_ccp_destroy_anchor: &ddboost_ccp_destroy - put: ddboost_terraform - params: - <<: *destroy_params - env_name_file: ddboost_terraform/name - terraform_source: ccp_src/aws/ - get_params: - action: destroy - -debug_sleep_anchor: &debug_sleep - task: debug_sleep - config: - platform: linux - image_resource: - type: registry-image - source: - repository: alpine - tag: latest - run: - path: 'sh' - args: ['-c', 'sleep 2h'] - ensure: - <<: *ddboost_ccp_destroy - -set_failed_anchor: &set_failed +- &set_failed do: - task: on_failure_set_failed config: platform: linux - image_resource: &failed_image_resource + image_resource: type: registry-image source: repository: gcr.io/data-gpdb-public-images/ccp - tag: latest + tag: "latest" inputs: - name: ccp_src - name: terraform run: path: 'ccp_src/google/ccp_failed_test.sh' - params: &failed_params - GOOGLE_CREDENTIALS: {{google-service-account-key}} - GOOGLE_PROJECT_ID: {{google-project-id}} - GOOGLE_ZONE: {{google-zone}} - GOOGLE_SERVICE_ACCOUNT: {{google-service-account}} - AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} - AWS_DEFAULT_REGION: {{tf-machine-region}} + params: + GOOGLE_CREDENTIALS: ((google-service-account-key)) + GOOGLE_PROJECT_ID: ((google-project-id)) + GOOGLE_ZONE: ((google-zone)) + GOOGLE_SERVICE_ACCOUNT: ((google-service-account)) + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + AWS_DEFAULT_REGION: ((tf-machine-region)) BUCKET_PATH: clusters-google/ - BUCKET_NAME: {{tf-bucket-name}} + BUCKET_NAME: ((tf-bucket-name)) -set_failed_anchor_aws: &set_failed_aws - do: - - task: on_failure_set_failed - config: - platform: linux - image_resource: - <<: *failed_image_resource - inputs: - - name: ccp_src - - name: terraform_aws - path: terraform - run: - path: 'ccp_src/aws/ccp_failed_test.sh' - params: - <<: *failed_params - BUCKET_PATH: clusters-aws/ +- &terraform_cluster + type: terraform + source: + env: + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + GOOGLE_CREDENTIALS: ((google-service-account-key)) + vars: + project_id: ((google-project-id)) + storage: + access_key_id: ((tf-machine-access-key-id)) + secret_access_key: ((tf-machine-secret-access-key)) + region_name: ((aws-region)) + # This is not parameterized, on purpose. All tfstates will go to this spot, + # and different teams will place there clusters' tfstate files under different paths + bucket: gpdb5-pipeline-dynamic-terraform + bucket_path: clusters-google/ + +- &ddboost_params + DD_SOURCE_HOST: ((datadomain_source_host_gcp)) + DD_USER: ((datadomain_user)) + DD_PW: ((datadomain_password_gcp)) + DD_DEST_HOST: ((datadomain_dest_host_gcp)) + DD_ENCRYPTED_PW: ((encrypted_datadomain_password_gcp)) -slack_alert_anchor: &slack_alert +- &slack_alert put: slack-alert params: text: | [gpbackup/$BUILD_JOB_NAME] failed: https://prod.ci.gpdb.pivotal.io/teams/main/pipelines/gpbackup/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME -ddboost_plugin_and_boostfs_anchor: &ddboost_plugin_and_boostfs - in_parallel: - - get: centos7-image - - get: gpbackup_ddboost_plugin - passed: [build_gppkgs] - - get: gpbackup - passed: [build_gppkgs] - - get: ccp_src - - get: boostfs_installer - - get: gppkgs - passed: [build_gppkgs] - trigger: true - - get: terraform.d - params: - unpack: true - -ddboost_plugin_and_boostfs_tests_anchor: &ddboost_plugin_and_boostfs_tests - do: - - put: ddboost_terraform - tags: ["ddboost"] - params: - <<: *ccp_aws_default_params - vars: - aws_instance-node-instance_type: t2.medium - PLATFORM: centos6 - - task: gen_cluster - tags: ["ddboost"] - params: - <<: *ccp_gen_cluster_default_params - BUCKET_PATH: clusters-aws/ - CLOUD_PROVIDER: aws - file: ccp_src/ci/tasks/gen_cluster.yml - input_mapping: - terraform: ddboost_terraform - - task: gpinitsystem - tags: ["ddboost"] - file: ccp_src/ci/tasks/gpinitsystem.yml - - task: setup-centos-env - tags: ["ddboost"] - image: centos7-image - file: gpbackup/ci/tasks/setup-centos-env.yml - - task: boostfs_installation - tags: ["ddboost"] - image: centos7-image - file: gpbackup/ci/tasks/boostfs-install.yml - params: - <<: *ddboost_params - on_failure: - do: - - *debug_sleep - - task: run_tests - tags: ["ddboost"] - image: centos7-image - file: gpbackup/ci/tasks/ddboost-plugin-tests.yml - params: - <<: *ddboost_test_params - on_failure: - do: - - *slack_alert - - *debug_sleep - ensure: - <<: *ddboost_ccp_destroy - - resource_types: - name: terraform type: registry-image @@ -565,33 +456,7 @@ resources: uri: {{ccp-git-remote}} - name: terraform - type: terraform - source: - env: - AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} - GOOGLE_CREDENTIALS: {{google-service-account-key}} - vars: - project_id: {{google-project-id}} - storage: - access_key_id: {{tf-machine-access-key-id}} - secret_access_key: {{tf-machine-secret-access-key}} - region_name: {{aws-region}} - bucket: {{tf-bucket-name}} - bucket_path: clusters-google/ - -- name: ddboost_terraform - type: terraform - source: - env: - AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} - storage: - access_key_id: {{tf-machine-access-key-id}} - secret_access_key: {{tf-machine-secret-access-key}} - region_name: {{aws-region}} - bucket: {{tf-bucket-name}} - bucket_path: clusters-aws/ + <<: *terraform_cluster - name: terraform.d type: s3 @@ -661,24 +526,6 @@ resources: versioned_file: gppkgs/intermediates/gpbackup-gppkgs.tar.gz -- name: terraform_aws - type: terraform - source: - env: - AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} - GOOGLE_CREDENTIALS: {{google-service-account-key}} - vars: - project_id: {{google-project-id}} - storage: - access_key_id: {{tf-machine-access-key-id}} - secret_access_key: {{tf-machine-secret-access-key}} - region_name: {{aws-region}} - # This is not parameterized, on purpose. All tfstates will go to this spot, - # and different teams will place there clusters' tfstate files under different paths - bucket: gpdb5-pipeline-dynamic-terraform - bucket_path: clusters-aws/ - - name: gpbackup-release-license type: gcs @@ -692,7 +539,6 @@ jobs: - name: build_binaries plan: - in_parallel: - - get: centos6-image - get: gpbackup_s3_plugin trigger: true - get: gpbackup_manager_src @@ -701,7 +547,6 @@ jobs: trigger: true - get: pivnet_release_cache - task: build-go-binaries - image: centos6-image file: gpbackup/ci/tasks/build-go-binaries.yml - put: gpbackup-go-components params: @@ -875,7 +720,7 @@ jobs: number_of_nodes: 4 segments_per_host: 4 instance_type: n1-standard-8 - ccp_reap_minutes: 240 + ccp_reap_minutes: 360 - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: @@ -928,8 +773,50 @@ jobs: - get: gpdb_binary resource: bin_gpdb_43_stable - get: pgcrypto43 - - <<: *ddboost_plugin_and_boostfs - - <<: *ddboost_plugin_and_boostfs_tests + - get: centos6-image + - get: gpbackup_ddboost_plugin + passed: [build_gppkgs] + - get: gpbackup + passed: [build_gppkgs] + - get: boostfs_installer + - get: ccp_src + - get: gppkgs + passed: [build_gppkgs] + trigger: true + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + terraform_source: ccp_src/google-nvme-block-device/ + vars: + instance_type: n1-standard-8 + PLATFORM: centos6 + - task: gen_cluster + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: centos6 + file: ccp_src/ci/tasks/gen_cluster.yml + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: setup-centos-env + file: gpbackup/ci/tasks/setup-centos-env.yml + image: centos6-image + - task: install-boostfs + file: gpbackup/ci/tasks/boostfs-install.yml + image: centos6-image + params: + <<: *ddboost_params + - task: run_tests + image: centos6-image + file: gpbackup/ci/tasks/ddboost-plugin-tests.yml + params: + <<: *ddboost_params + on_success: + <<: *ccp_destroy_nvme + ensure: + <<: *set_failed - name: ddboost_plugin_and_boostfs_tests_5x plan: @@ -938,8 +825,50 @@ jobs: resource: gpdb5_src - get: gpdb_binary resource: bin_gpdb_5x_stable_centos6 - - <<: *ddboost_plugin_and_boostfs - - <<: *ddboost_plugin_and_boostfs_tests + - get: centos6-image + - get: gpbackup_ddboost_plugin + passed: [build_gppkgs] + - get: gpbackup + passed: [build_gppkgs] + - get: boostfs_installer + - get: ccp_src + - get: gppkgs + passed: [build_gppkgs] + trigger: true + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + terraform_source: ccp_src/google-nvme-block-device/ + vars: + instance_type: n1-standard-8 + PLATFORM: centos6 + - task: gen_cluster + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: centos6 + file: ccp_src/ci/tasks/gen_cluster.yml + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: setup-centos-env + file: gpbackup/ci/tasks/setup-centos-env.yml + image: centos6-image + - task: install-boostfs + file: gpbackup/ci/tasks/boostfs-install.yml + image: centos6-image + params: + <<: *ddboost_params + - task: run_tests + image: centos6-image + file: gpbackup/ci/tasks/ddboost-plugin-tests.yml + params: + <<: *ddboost_params + on_success: + <<: *ccp_destroy_nvme + ensure: + <<: *set_failed - name: ddboost_plugin_and_boostfs_tests_6x plan: @@ -950,61 +879,104 @@ jobs: resource: bin_gpdb_6x_stable_centos6 - get: dummy_seclabel resource: dummy_seclabel_linux_gpdb6 - - <<: *ddboost_plugin_and_boostfs - - <<: *ddboost_plugin_and_boostfs_tests + - get: centos6-image + - get: gpbackup_ddboost_plugin + passed: [build_gppkgs] + - get: gpbackup + passed: [build_gppkgs] + - get: boostfs_installer + - get: ccp_src + - get: gppkgs + passed: [build_gppkgs] + trigger: true + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + terraform_source: ccp_src/google-nvme-block-device/ + vars: + instance_type: n1-standard-8 + PLATFORM: centos6 + - task: gen_cluster + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: centos6 + file: ccp_src/ci/tasks/gen_cluster.yml + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: setup-centos-env + file: gpbackup/ci/tasks/setup-centos-env.yml + image: centos6-image + - task: install-boostfs + file: gpbackup/ci/tasks/boostfs-install.yml + image: centos6-image + params: + <<: *ddboost_params + - task: run_tests + image: centos6-image + file: gpbackup/ci/tasks/ddboost-plugin-tests.yml + params: + <<: *ddboost_params + on_success: + <<: *ccp_destroy_nvme + ensure: + <<: *set_failed - name: ddboost_plugin_perf plan: - in_parallel: + - get: centos6-image + - get: gpbackup_ddboost_plugin + passed: [build_gppkgs] + - get: gpbackup + passed: [build_gppkgs] + - get: boostfs_installer + - get: ccp_src - get: gpdb_src resource: gpdb6_src - get: gpdb_binary resource: bin_gpdb_6x_stable_centos6 - - do: - - <<: *ddboost_plugin_and_boostfs - - put: ddboost_terraform - tags: ["ddboost"] - params: - <<: *ccp_aws_default_params - PLATFORM: centos6 - vars: - number_of_nodes: 4 - segments_per_host: 4 - aws_instance-node-instance_type: i3.xlarge - ccp_reap_minutes: 240 - - task: gen_cluster - tags: ["ddboost"] - params: - <<: *ccp_gen_cluster_default_params - BUCKET_PATH: clusters-aws/ - CLOUD_PROVIDER: aws - file: ccp_src/ci/tasks/gen_cluster.yml - input_mapping: - terraform: ddboost_terraform - - task: gpinitsystem - tags: ["ddboost"] - file: ccp_src/ci/tasks/gpinitsystem.yml + - get: gppkgs + passed: [build_gppkgs] + trigger: true + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + terraform_source: ccp_src/google-nvme-block-device/ + vars: + instance_type: n1-standard-8 + number_of_nodes: 4 + segments_per_host: 4 + PLATFORM: centos6 + - task: gen_cluster + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: centos6 + file: ccp_src/ci/tasks/gen_cluster.yml + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml - task: setup-centos-env - tags: ["ddboost"] - image: centos7-image + image: centos6-image file: gpbackup/ci/tasks/setup-centos-env.yml - task: run_perf - tags: ["ddboost"] - image: centos7-image + image: centos6-image file: gpbackup/ci/tasks/ddboost-plugin-perf.yml params: - <<: *ddboost_test_params + <<: *ddboost_params REGION: us-west-2 - AWS_ACCESS_KEY_ID: {{bucket-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{bucket-secret-access-key}} - BUCKET: {{gpbackup-s3-plugin-test-bucket}} + AWS_ACCESS_KEY_ID: ((bucket-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((bucket-secret-access-key)) + BUCKET: ((gpbackup-s3-plugin-test-bucket)) SCALE_FACTOR: 1 - on_failure: - do: - - *slack_alert - - *debug_sleep + on_success: + <<: *ccp_destroy_nvme ensure: - <<: *ddboost_ccp_destroy + <<: *set_failed - name: GPDB5-sles11 plan: @@ -1050,62 +1022,6 @@ jobs: ensure: <<: *set_failed -- name: GPDB5-oracle7 - plan: - - in_parallel: - - get: centos6-image - - get: gpbackup - passed: [build_gppkgs] - - get: gpdb_binary - resource: bin_gpdb_5x_stable_centos7 - trigger: true - - get: ccp_src - - get: gpdb_src - resource: gpdb5_src - - get: gppkgs - trigger: true - passed: [build_gppkgs] - - get: terraform.d - params: - unpack: true - - put: terraform_aws - params: - <<: *ccp_aws_default_params - vars: - <<: *ccp_default_vars_aws - PLATFORM: oracle7 - default_ami_user: clckwrk - disk_size: 30 - - task: gen_cluster - tags: ['aws'] - file: ccp_src/ci/tasks/gen_cluster.yml - input_mapping: - terraform: terraform_aws - params: - <<: *ccp_gen_cluster_default_params - BUCKET_PATH: clusters-aws/ - CLOUD_PROVIDER: aws - PLATFORM: oracle7 - - task: gpinitsystem - tags: ['aws'] - file: ccp_src/ci/tasks/gpinitsystem.yml - - task: setup-oracle-env - tags: ['aws'] - image: centos6-image - file: gpbackup/ci/tasks/setup-centos-env.yml - params: - default_ami_user: clckwrk - - task: integration-tests - tags: ['aws'] - image: centos6-image - file: gpbackup/ci/tasks/all-tests.yml - on_success: - <<: *ccp_destroy_aws - on_failure: - *slack_alert - ensure: - <<: *set_failed_aws - - name: GPDB6-ubuntu plan: - in_parallel: @@ -1279,7 +1195,7 @@ jobs: - get: gpbackup passed: [build_gppkgs] - get: gpdb_binary - resource: bin_gpdb_5x_stable_centos6 + resource: bin_gpdb_5x_stable_centos7 trigger: true - get: bin_gpbackup_1.0.0_and_1.7.1 - get: ccp_src @@ -1296,15 +1212,17 @@ jobs: <<: *ccp_default_params vars: <<: *ccp_default_vars + PLATFORM: centos7 - task: gen_cluster params: <<: *ccp_gen_cluster_default_params + PLATFORM: centos7 file: ccp_src/ci/tasks/gen_cluster.yml - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: setup-centos-env - image: centos6-image file: gpbackup/ci/tasks/setup-centos-env.yml + image: centos6-image - task: backup-1.0.0-restore-latest image: centos6-image file: gpbackup/ci/tasks/all-tests.yml @@ -1315,10 +1233,8 @@ jobs: file: gpbackup/ci/tasks/all-tests.yml params: GPBACKUP_VERSION: "1.7.1" - on_success: - <<: *ccp_destroy - on_failure: - *slack_alert + on_success: + <<: *ccp_destroy ensure: <<: *set_failed @@ -1333,41 +1249,17 @@ jobs: trigger: true - get: gpbackup passed: [build_gppkgs] - - get: gpdb_binary + - get: bin_gpdb resource: bin_gpdb_5x_stable_intermediate trigger: true - - get: ccp_src - get: gpdb_src resource: gpdb5_src - get: gpbackup_1.12.1_dependencies - - get: terraform.d - params: - unpack: true - - put: terraform - params: - <<: *ccp_default_params - vars: - <<: *ccp_default_vars - - task: gen_cluster - params: - <<: *ccp_gen_cluster_default_params - file: ccp_src/ci/tasks/gen_cluster.yml - - task: gpinitsystem - file: ccp_src/ci/tasks/gpinitsystem.yml - - task: setup-centos-env - image: centos6-image - file: gpbackup/ci/tasks/setup-centos-env.yml - task: gpbackup-1.12.1-test image: centos6-image file: gpbackup/ci/tasks/integration-tests-fixed-version.yml params: GPBACKUP_VERSION: "1.12.1" - on_success: - <<: *ccp_destroy - on_failure: - *slack_alert - ensure: - <<: *set_failed - name: scale-master plan: @@ -1564,7 +1456,6 @@ jobs: - s3_plugin_tests - backward-compatibility - 5X-head-gpbackup-fixed-test - - GPDB5-oracle7 - GPDB5-sles11 - GPDB6-ubuntu - ddboost_plugin_and_boostfs_tests_43 diff --git a/ci/gpbackup-release-generated.yml b/ci/gpbackup-release-generated.yml index c1edcfd7a..28a115a46 100644 --- a/ci/gpbackup-release-generated.yml +++ b/ci/gpbackup-release-generated.yml @@ -12,7 +12,7 @@ ## file (example: templates/gpbackup-tpl.yml) and regenerate the pipeline ## using appropriate tool (example: gen_pipeline.py -p gpbackup-release). ## ---------------------------------------------------------------------- -## Generated by gen_pipeline.py at: 2021-07-27 15:59:55.866024 +## Generated by gen_pipeline.py at: 2022-01-28 05:33:23.363325 ## Template file: gpbackup-tpl.yml ## Pipeline Name: gpbackup-release ## Nightly Trigger: True @@ -26,7 +26,6 @@ groups: - build_gppkgs - GPDB4.3 - GPDB5 - - GPDB5-oracle7 - GPDB5-sles11 - GPDB6-ubuntu - ddboost_plugin_and_boostfs_tests_43 @@ -49,7 +48,6 @@ groups: - name: GPDB5 jobs: - GPDB5 - - GPDB5-oracle7 - GPDB5-sles11 - scale-5x - ddboost_plugin_and_boostfs_tests_5x @@ -82,70 +80,43 @@ groups: - push-to-pivnet ##### Anchors ##### -ccp_default_params_anchor: &ccp_default_params +anchors: +- &ccp_default_params action: create delete_on_failure: true generate_random_name: true plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 terraform_source: ccp_src/google/ -ccp_aws_default_params_anchor: &ccp_aws_default_params - action: create - delete_on_failure: true - generate_random_name: true - plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 - terraform_source: ccp_src/aws/ - -ccp_vars_anchor: &ccp_default_vars +- &ccp_default_vars instance_type: n1-standard-1 PLATFORM: centos6 -ccp_vars_anchor_aws: &ccp_default_vars_aws - aws_instance-node-instance_type: t2.medium - PLATFORM: centos6 - -ccp_gen_cluster_default_params_anchor: &ccp_gen_cluster_default_params - AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} - AWS_DEFAULT_REGION: {{aws-region}} +- &ccp_gen_cluster_default_params + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + AWS_DEFAULT_REGION: ((aws-region)) BUCKET_PATH: clusters-google/ - BUCKET_NAME: {{tf-bucket-name}} + BUCKET_NAME: ((tf-bucket-name)) CLOUD_PROVIDER: google -ddboost_params_anchor: &ddboost_params - DD_SOURCE_HOST: {{datadomain_source_host}} - DD_USER: {{datadomain_user}} - DD_PW: {{datadomain_password}} - -ddboost_test_params_anchor: &ddboost_test_params - <<: *ddboost_params - DD_DEST_HOST: {{datadomain_dest_host}} - DD_ENCRYPTED_PW: {{encrypted_datadomain_password}} +- &destroy_params + action: destroy + plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 + terraform_source: ccp_src/google/ + env_name_file: terraform/name + vars: + aws_instance-node-instance_type: t2.micro #t2.micro is ignored in destroy, but aws_instance-node-instance_type is required. + aws_ebs_volume_type: standard -ccp_destroy_anchor: &ccp_destroy +- &ccp_destroy put: terraform - params: &destroy_params - action: destroy - env_name_file: terraform/name - plugin_dir: ../../terraform.d/plugin-cache/linux_amd64 - terraform_source: ccp_src/google/ - vars: - aws_instance-node-instance_type: t2.micro - aws_ebs_volume_type: standard - instance_type: n1-standard-1 - get_params: - action: destroy - -ccp_destroy_anchor_aws: &ccp_destroy_aws - put: terraform_aws params: <<: *destroy_params - env_name_file: terraform_aws/name - terraform_source: ccp_src/aws/ get_params: action: destroy -ccp_nvme_destroy_anchor: &ccp_destroy_nvme +- &ccp_destroy_nvme put: terraform params: <<: *destroy_params @@ -153,143 +124,63 @@ ccp_nvme_destroy_anchor: &ccp_destroy_nvme get_params: action: destroy -ddboost_ccp_destroy_anchor: &ddboost_ccp_destroy - put: ddboost_terraform - params: - <<: *destroy_params - env_name_file: ddboost_terraform/name - terraform_source: ccp_src/aws/ - get_params: - action: destroy - -debug_sleep_anchor: &debug_sleep - task: debug_sleep - config: - platform: linux - image_resource: - type: registry-image - source: - repository: alpine - tag: latest - run: - path: 'sh' - args: ['-c', 'sleep 2h'] - ensure: - <<: *ddboost_ccp_destroy - -set_failed_anchor: &set_failed +- &set_failed do: - task: on_failure_set_failed config: platform: linux - image_resource: &failed_image_resource + image_resource: type: registry-image source: repository: gcr.io/data-gpdb-public-images/ccp - tag: latest + tag: "latest" inputs: - name: ccp_src - name: terraform run: path: 'ccp_src/google/ccp_failed_test.sh' - params: &failed_params - GOOGLE_CREDENTIALS: {{google-service-account-key}} - GOOGLE_PROJECT_ID: {{google-project-id}} - GOOGLE_ZONE: {{google-zone}} - GOOGLE_SERVICE_ACCOUNT: {{google-service-account}} - AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} - AWS_DEFAULT_REGION: {{tf-machine-region}} + params: + GOOGLE_CREDENTIALS: ((google-service-account-key)) + GOOGLE_PROJECT_ID: ((google-project-id)) + GOOGLE_ZONE: ((google-zone)) + GOOGLE_SERVICE_ACCOUNT: ((google-service-account)) + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + AWS_DEFAULT_REGION: ((tf-machine-region)) BUCKET_PATH: clusters-google/ - BUCKET_NAME: {{tf-bucket-name}} + BUCKET_NAME: ((tf-bucket-name)) -set_failed_anchor_aws: &set_failed_aws - do: - - task: on_failure_set_failed - config: - platform: linux - image_resource: - <<: *failed_image_resource - inputs: - - name: ccp_src - - name: terraform_aws - path: terraform - run: - path: 'ccp_src/aws/ccp_failed_test.sh' - params: - <<: *failed_params - BUCKET_PATH: clusters-aws/ +- &terraform_cluster + type: terraform + source: + env: + AWS_ACCESS_KEY_ID: ((tf-machine-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((tf-machine-secret-access-key)) + GOOGLE_CREDENTIALS: ((google-service-account-key)) + vars: + project_id: ((google-project-id)) + storage: + access_key_id: ((tf-machine-access-key-id)) + secret_access_key: ((tf-machine-secret-access-key)) + region_name: ((aws-region)) + # This is not parameterized, on purpose. All tfstates will go to this spot, + # and different teams will place there clusters' tfstate files under different paths + bucket: gpdb5-pipeline-dynamic-terraform + bucket_path: clusters-google/ + +- &ddboost_params + DD_SOURCE_HOST: ((datadomain_source_host_gcp)) + DD_USER: ((datadomain_user)) + DD_PW: ((datadomain_password_gcp)) + DD_DEST_HOST: ((datadomain_dest_host_gcp)) + DD_ENCRYPTED_PW: ((encrypted_datadomain_password_gcp)) -slack_alert_anchor: &slack_alert +- &slack_alert put: slack-alert params: text: | - [gpbackup-release/$BUILD_JOB_NAME] failed: - https://prod.ci.gpdb.pivotal.io/teams/main/pipelines/gpbackup-release/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME - -ddboost_plugin_and_boostfs_anchor: &ddboost_plugin_and_boostfs - in_parallel: - - get: centos7-image - - get: gpbackup_ddboost_plugin - passed: [build_gppkgs] - - get: gpbackup - passed: [build_gppkgs] - - get: ccp_src - - get: boostfs_installer - - get: gppkgs - passed: [build_gppkgs] - trigger: true - - get: terraform.d - params: - unpack: true - -ddboost_plugin_and_boostfs_tests_anchor: &ddboost_plugin_and_boostfs_tests - do: - - put: ddboost_terraform - tags: ["ddboost"] - params: - <<: *ccp_aws_default_params - vars: - aws_instance-node-instance_type: t2.medium - PLATFORM: centos6 - - task: gen_cluster - tags: ["ddboost"] - params: - <<: *ccp_gen_cluster_default_params - BUCKET_PATH: clusters-aws/ - CLOUD_PROVIDER: aws - file: ccp_src/ci/tasks/gen_cluster.yml - input_mapping: - terraform: ddboost_terraform - - task: gpinitsystem - tags: ["ddboost"] - file: ccp_src/ci/tasks/gpinitsystem.yml - - task: setup-centos-env - tags: ["ddboost"] - image: centos7-image - file: gpbackup/ci/tasks/setup-centos-env.yml - - task: boostfs_installation - tags: ["ddboost"] - image: centos7-image - file: gpbackup/ci/tasks/boostfs-install.yml - params: - <<: *ddboost_params - on_failure: - do: - - *debug_sleep - - task: run_tests - tags: ["ddboost"] - image: centos7-image - file: gpbackup/ci/tasks/ddboost-plugin-tests.yml - params: - <<: *ddboost_test_params - on_failure: - do: - - *slack_alert - - *debug_sleep - ensure: - <<: *ddboost_ccp_destroy - + [gpbackup/$BUILD_JOB_NAME] failed: + https://prod.ci.gpdb.pivotal.io/teams/main/pipelines/gpbackup/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME resource_types: - name: terraform @@ -521,33 +412,7 @@ resources: uri: {{ccp-git-remote}} - name: terraform - type: terraform - source: - env: - AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} - GOOGLE_CREDENTIALS: {{google-service-account-key}} - vars: - project_id: {{google-project-id}} - storage: - access_key_id: {{tf-machine-access-key-id}} - secret_access_key: {{tf-machine-secret-access-key}} - region_name: {{aws-region}} - bucket: {{tf-bucket-name}} - bucket_path: clusters-google/ - -- name: ddboost_terraform - type: terraform - source: - env: - AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} - storage: - access_key_id: {{tf-machine-access-key-id}} - secret_access_key: {{tf-machine-secret-access-key}} - region_name: {{aws-region}} - bucket: {{tf-bucket-name}} - bucket_path: clusters-aws/ + <<: *terraform_cluster - name: terraform.d type: s3 @@ -653,24 +518,6 @@ resources: access_token: {{gpbackup-git-access-token}} release: ((dpm-enable-release)) -- name: terraform_aws - type: terraform - source: - env: - AWS_ACCESS_KEY_ID: {{tf-machine-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{tf-machine-secret-access-key}} - GOOGLE_CREDENTIALS: {{google-service-account-key}} - vars: - project_id: {{google-project-id}} - storage: - access_key_id: {{tf-machine-access-key-id}} - secret_access_key: {{tf-machine-secret-access-key}} - region_name: {{aws-region}} - # This is not parameterized, on purpose. All tfstates will go to this spot, - # and different teams will place there clusters' tfstate files under different paths - bucket: gpdb5-pipeline-dynamic-terraform - bucket_path: clusters-aws/ - - name: pivnet-upload type: pivnet source: @@ -696,13 +543,11 @@ jobs: - name: build_binaries plan: - in_parallel: - - get: centos6-image - get: gpbackup_s3_plugin - get: gpbackup_manager_src - get: gpbackup - get: pivnet_release_cache - task: build-go-binaries - image: centos6-image file: gpbackup/ci/tasks/build-go-binaries.yml - put: gpbackup-go-components params: @@ -878,7 +723,7 @@ jobs: number_of_nodes: 4 segments_per_host: 4 instance_type: n1-standard-8 - ccp_reap_minutes: 240 + ccp_reap_minutes: 360 - task: gen_cluster file: ccp_src/ci/tasks/gen_cluster.yml params: @@ -931,8 +776,50 @@ jobs: - get: gpdb_binary resource: bin_gpdb_43_stable - get: pgcrypto43 - - <<: *ddboost_plugin_and_boostfs - - <<: *ddboost_plugin_and_boostfs_tests + - get: centos6-image + - get: gpbackup_ddboost_plugin + passed: [build_gppkgs] + - get: gpbackup + passed: [build_gppkgs] + - get: boostfs_installer + - get: ccp_src + - get: gppkgs + passed: [build_gppkgs] + trigger: true + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + terraform_source: ccp_src/google-nvme-block-device/ + vars: + instance_type: n1-standard-8 + PLATFORM: centos6 + - task: gen_cluster + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: centos6 + file: ccp_src/ci/tasks/gen_cluster.yml + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: setup-centos-env + file: gpbackup/ci/tasks/setup-centos-env.yml + image: centos6-image + - task: install-boostfs + file: gpbackup/ci/tasks/boostfs-install.yml + image: centos6-image + params: + <<: *ddboost_params + - task: run_tests + image: centos6-image + file: gpbackup/ci/tasks/ddboost-plugin-tests.yml + params: + <<: *ddboost_params + on_success: + <<: *ccp_destroy_nvme + ensure: + <<: *set_failed - name: ddboost_plugin_and_boostfs_tests_5x plan: @@ -941,8 +828,50 @@ jobs: resource: gpdb5_src - get: gpdb_binary resource: bin_gpdb_5x_stable_centos6 - - <<: *ddboost_plugin_and_boostfs - - <<: *ddboost_plugin_and_boostfs_tests + - get: centos6-image + - get: gpbackup_ddboost_plugin + passed: [build_gppkgs] + - get: gpbackup + passed: [build_gppkgs] + - get: boostfs_installer + - get: ccp_src + - get: gppkgs + passed: [build_gppkgs] + trigger: true + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + terraform_source: ccp_src/google-nvme-block-device/ + vars: + instance_type: n1-standard-8 + PLATFORM: centos6 + - task: gen_cluster + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: centos6 + file: ccp_src/ci/tasks/gen_cluster.yml + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: setup-centos-env + file: gpbackup/ci/tasks/setup-centos-env.yml + image: centos6-image + - task: install-boostfs + file: gpbackup/ci/tasks/boostfs-install.yml + image: centos6-image + params: + <<: *ddboost_params + - task: run_tests + image: centos6-image + file: gpbackup/ci/tasks/ddboost-plugin-tests.yml + params: + <<: *ddboost_params + on_success: + <<: *ccp_destroy_nvme + ensure: + <<: *set_failed - name: ddboost_plugin_and_boostfs_tests_6x plan: @@ -953,61 +882,104 @@ jobs: resource: bin_gpdb_6x_stable_centos6 - get: dummy_seclabel resource: dummy_seclabel_linux_gpdb6 - - <<: *ddboost_plugin_and_boostfs - - <<: *ddboost_plugin_and_boostfs_tests + - get: centos6-image + - get: gpbackup_ddboost_plugin + passed: [build_gppkgs] + - get: gpbackup + passed: [build_gppkgs] + - get: boostfs_installer + - get: ccp_src + - get: gppkgs + passed: [build_gppkgs] + trigger: true + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + terraform_source: ccp_src/google-nvme-block-device/ + vars: + instance_type: n1-standard-8 + PLATFORM: centos6 + - task: gen_cluster + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: centos6 + file: ccp_src/ci/tasks/gen_cluster.yml + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + - task: setup-centos-env + file: gpbackup/ci/tasks/setup-centos-env.yml + image: centos6-image + - task: install-boostfs + file: gpbackup/ci/tasks/boostfs-install.yml + image: centos6-image + params: + <<: *ddboost_params + - task: run_tests + image: centos6-image + file: gpbackup/ci/tasks/ddboost-plugin-tests.yml + params: + <<: *ddboost_params + on_success: + <<: *ccp_destroy_nvme + ensure: + <<: *set_failed - name: ddboost_plugin_perf plan: - in_parallel: + - get: centos6-image + - get: gpbackup_ddboost_plugin + passed: [build_gppkgs] + - get: gpbackup + passed: [build_gppkgs] + - get: boostfs_installer + - get: ccp_src - get: gpdb_src resource: gpdb6_src - get: gpdb_binary resource: bin_gpdb_6x_stable_centos6 - - do: - - <<: *ddboost_plugin_and_boostfs - - put: ddboost_terraform - tags: ["ddboost"] - params: - <<: *ccp_aws_default_params - PLATFORM: centos6 - vars: - number_of_nodes: 4 - segments_per_host: 4 - aws_instance-node-instance_type: i3.xlarge - ccp_reap_minutes: 240 - - task: gen_cluster - tags: ["ddboost"] - params: - <<: *ccp_gen_cluster_default_params - BUCKET_PATH: clusters-aws/ - CLOUD_PROVIDER: aws - file: ccp_src/ci/tasks/gen_cluster.yml - input_mapping: - terraform: ddboost_terraform - - task: gpinitsystem - tags: ["ddboost"] - file: ccp_src/ci/tasks/gpinitsystem.yml + - get: gppkgs + passed: [build_gppkgs] + trigger: true + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + terraform_source: ccp_src/google-nvme-block-device/ + vars: + instance_type: n1-standard-8 + number_of_nodes: 4 + segments_per_host: 4 + PLATFORM: centos6 + - task: gen_cluster + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: centos6 + file: ccp_src/ci/tasks/gen_cluster.yml + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml - task: setup-centos-env - tags: ["ddboost"] - image: centos7-image + image: centos6-image file: gpbackup/ci/tasks/setup-centos-env.yml - task: run_perf - tags: ["ddboost"] - image: centos7-image + image: centos6-image file: gpbackup/ci/tasks/ddboost-plugin-perf.yml params: - <<: *ddboost_test_params + <<: *ddboost_params REGION: us-west-2 - AWS_ACCESS_KEY_ID: {{bucket-access-key-id}} - AWS_SECRET_ACCESS_KEY: {{bucket-secret-access-key}} - BUCKET: {{gpbackup-s3-plugin-test-bucket}} + AWS_ACCESS_KEY_ID: ((bucket-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((bucket-secret-access-key)) + BUCKET: ((gpbackup-s3-plugin-test-bucket)) SCALE_FACTOR: 1 - on_failure: - do: - - *slack_alert - - *debug_sleep + on_success: + <<: *ccp_destroy_nvme ensure: - <<: *ddboost_ccp_destroy + <<: *set_failed - name: GPDB5-sles11 plan: @@ -1052,61 +1024,6 @@ jobs: ensure: <<: *set_failed -- name: GPDB5-oracle7 - plan: - - in_parallel: - - get: centos6-image - - get: gpbackup - passed: [build_gppkgs] - - get: gpdb_binary - resource: bin_gpdb_5x_stable_centos7 - - get: ccp_src - - get: gpdb_src - resource: gpdb5_src - - get: gppkgs - trigger: true - passed: [build_gppkgs] - - get: terraform.d - params: - unpack: true - - put: terraform_aws - params: - <<: *ccp_aws_default_params - vars: - <<: *ccp_default_vars_aws - PLATFORM: oracle7 - default_ami_user: clckwrk - disk_size: 30 - - task: gen_cluster - tags: ['aws'] - file: ccp_src/ci/tasks/gen_cluster.yml - input_mapping: - terraform: terraform_aws - params: - <<: *ccp_gen_cluster_default_params - BUCKET_PATH: clusters-aws/ - CLOUD_PROVIDER: aws - PLATFORM: oracle7 - - task: gpinitsystem - tags: ['aws'] - file: ccp_src/ci/tasks/gpinitsystem.yml - - task: setup-oracle-env - tags: ['aws'] - image: centos6-image - file: gpbackup/ci/tasks/setup-centos-env.yml - params: - default_ami_user: clckwrk - - task: integration-tests - tags: ['aws'] - image: centos6-image - file: gpbackup/ci/tasks/all-tests.yml - on_success: - <<: *ccp_destroy_aws - on_failure: - *slack_alert - ensure: - <<: *set_failed_aws - - name: GPDB6-ubuntu plan: - in_parallel: @@ -1252,7 +1169,7 @@ jobs: - get: gpbackup passed: [build_gppkgs] - get: gpdb_binary - resource: bin_gpdb_5x_stable_centos6 + resource: bin_gpdb_5x_stable_centos7 - get: bin_gpbackup_1.0.0_and_1.7.1 - get: ccp_src - get: gpdb_src @@ -1268,15 +1185,17 @@ jobs: <<: *ccp_default_params vars: <<: *ccp_default_vars + PLATFORM: centos7 - task: gen_cluster params: <<: *ccp_gen_cluster_default_params + PLATFORM: centos7 file: ccp_src/ci/tasks/gen_cluster.yml - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml - task: setup-centos-env - image: centos6-image file: gpbackup/ci/tasks/setup-centos-env.yml + image: centos6-image - task: backup-1.0.0-restore-latest image: centos6-image file: gpbackup/ci/tasks/all-tests.yml @@ -1287,10 +1206,8 @@ jobs: file: gpbackup/ci/tasks/all-tests.yml params: GPBACKUP_VERSION: "1.7.1" - on_success: - <<: *ccp_destroy - on_failure: - *slack_alert + on_success: + <<: *ccp_destroy ensure: <<: *set_failed @@ -1435,7 +1352,6 @@ jobs: - build_gppkgs - GPDB4.3 - GPDB5 - - GPDB5-oracle7 - GPDB5-sles11 - GPDB6-ubuntu - GPDB6 diff --git a/ci/regression/regression_pipeline.yml b/ci/regression/regression_pipeline.yml index faa224f29..c13a46ae2 100644 --- a/ci/regression/regression_pipeline.yml +++ b/ci/regression/regression_pipeline.yml @@ -13,13 +13,13 @@ groups: resource_types: - name: terraform - type: docker-image + type: registry-image source: repository: ljfranklin/terraform-resource tag: 0.11.14 - name: gcs - type: docker-image + type: registry-image source: repository: frodenas/gcs-resource @@ -69,13 +69,13 @@ resources: versioned_file: gpdb6/icw_gporca_centos6/gpbackup_all.tar.gz - name: centos-gpdb-dev-5 - type: docker-image + type: registry-image source: repository: pivotaldata/centos-gpdb-dev tag: '5' - name: centos6-image - type: docker-image + type: registry-image source: repository: pivotaldata/centos-gpdb-dev tag: '6-gcc6.2-llvm3.7' @@ -391,7 +391,7 @@ set_failed_anchor: &set_failed config: platform: linux image_resource: - type: docker-image + type: registry-image source: repository: pivotaldata/ccp tag: "7" diff --git a/ci/scripts/all-tests.bash b/ci/scripts/all-tests.bash index d907ae57c..6baf87e15 100755 --- a/ci/scripts/all-tests.bash +++ b/ci/scripts/all-tests.bash @@ -24,7 +24,9 @@ cat <