From 185c86d7f919bfdbce2e28ed9f2fbc5e5f119a7a Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Mon, 3 Mar 2025 20:54:15 +0800 Subject: [PATCH 1/6] Trigger unit test based on the changed code & extract the duplicate code as a command --- .circleci/config.yml | 17 ++--- .circleci/unittest-config.yml | 119 ++++++++++++++++++---------------- 2 files changed, 73 insertions(+), 63 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b102de61..098c02b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,13 +54,14 @@ workflows: base-revision: main config-path: .circleci/unittest-config.yml mapping: | - api/.* run_unittest true - bazel/.* run_unittest true - cmd/.* run_unittest true - engine/.* run_unittest true - pkg/.* run_unittest true - .bazelrc run_unittest true - .circleci/unittest-config.yml run_unittest true + api/.* run_scdb_ut true + cmd/.* run_scdb_ut true + pkg/.* run_scdb_ut true + contrib/.* run_scdb_ut true + bazel/.* run_engine_ut true + engine/.* run_engine_ut true + .bazelrc run_engine_ut true + .circleci/unittest-config.yml run_scdb_ut true run_engine_ut true run_regtest: when: << pipeline.parameters.enable_regtest >> jobs: @@ -84,4 +85,4 @@ workflows: # This is mandatory to trigger a pipeline when pushing a tag filters: tags: - only: /.*(? ../gcs.data + ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json --min_download + run_scdb_test: + description: "Run scdb tests" + steps: + - run: + name: "SCDB Test" + command: | + set +e + go mod tidy + go test -timeout=30m -v -short ./pkg/... ./contrib/... + run_engine_test: + description: "Run engine tests and collect artifacts" + parameters: + extra_bazel_args: + type: string + default: "" + find_executable_flag: + type: string + default: "-executable" + steps: + - run: + name: "Engine Test" + command: | + set +e + declare -i test_status + bazel --host_jvm_args=-Xmx8g test //engine/... << parameters.extra_bazel_args >> --jobs=32 --ui_event_filters=-info,-debug,-warning --test_output=errors | tee test_result.log; test_status=${PIPESTATUS[0]} + + sh ../devtools/rename-junit-xml.sh + find bazel-bin/ << parameters.find_executable_flag >> -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz + find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz + exit ${test_status} + # Define a job to be invoked later in a workflow. # See: https://circleci.com/docs/2.0/configuration-reference/#jobs @@ -40,12 +86,7 @@ jobs: # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: - checkout - - run: - name: "SCDB Test" - command: | - set +e - go mod tidy - go test -timeout=30m -v -short ./pkg/... + - run_scdb_test linux_engine_ut: docker: - image: secretflow/scql-ci:latest @@ -57,25 +98,10 @@ jobs: # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: - checkout - - run: - name: "Checkout devtools" - command: git clone https://github.com/secretflow/devtools.git ../devtools - - run: - name: Setup GCS - command: | - echo ${gcs_content} > ../gcs.data - ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json --min_download - - run: - name: "Engine Test" - command: | - set +e - declare -i test_status - bazelisk --host_jvm_args=-Xmx8g test //engine/... -c opt --jobs=32 --ui_event_filters=-info,-debug,-warning --test_output=errors | tee test_result.log; test_status=${PIPESTATUS[0]} - - sh ../devtools/rename-junit-xml.sh - find bazel-bin/ -executable -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz - find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz - exit ${test_status} + - setup_engine + - run_engine_test: + extra_bazel_args: "-c opt" + find_executable_flag: "-executable" - store_test_results: path: test-results - store_artifacts: @@ -92,12 +118,7 @@ jobs: name: "Install homebrew dependencies" command: | brew install wget go - - run: - name: "SCDB Test" - command: | - set +e - go mod tidy - go test -timeout=30m -v -short ./pkg/... + - run_scdb_test macOS_engine_ut: macos: xcode: 16.0.0 @@ -111,30 +132,15 @@ jobs: echo "Canceling workflow as too much time has elapsed" curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}" - checkout - - run: - name: "Checkout devtools" - command: git clone https://github.com/secretflow/devtools.git ../devtools - run: name: "Install homebrew dependencies" command: | brew install bazelisk cmake ninja libomp wget go@1.24 md5sha1sum brew link go@1.24 - - run: - name: Setup GCS - command: | - echo ${gcs_content} > ../gcs.data - ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json --min_download - - run: - name: "Engine Test" - command: | - set +e - declare -i test_status - bazelisk --host_jvm_args=-Xmx8g test //engine/... --jobs=32 --ui_event_filters=-info,-debug,-warning --test_output=errors| tee test_result.log; test_status=${PIPESTATUS[0]} - - sh ../devtools/rename-junit-xml.sh - find bazel-bin/ -perm +111 -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz - find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz - exit ${test_status} + - engine_setup + - run_engine_test: + extra_bazel_args: "" + find_executable_flag: "-perm +111" - store_test_results: path: test-results - store_artifacts: @@ -145,16 +151,19 @@ jobs: # Invoke jobs via workflows # See: https://circleci.com/docs/2.0/configuration-reference/#workflows workflows: - unittest: - when: << pipeline.parameters.run_unittest >> + run_scdb_ut: + when: << pipeline.parameters.run_scdb_ut >> jobs: - linux_scdb_ut: matrix: parameters: resource_class: ["2xlarge", "arm-xlarge"] + - macOS_scdb_ut + run_engine_ut: + when: << pipeline.parameters.run_engine_ut >> + jobs: - linux_engine_ut: matrix: parameters: resource_class: ["2xlarge", "arm-xlarge"] - - macOS_scdb_ut - - macOS_engine_ut + - macOS_engine_ut \ No newline at end of file From 5d36e23aba26024914a6c712cda180a006b24056 Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Mon, 3 Mar 2025 21:48:02 +0800 Subject: [PATCH 2/6] change bazel to bazelisk --- .circleci/unittest-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/unittest-config.yml b/.circleci/unittest-config.yml index 4e718346..8562b3df 100644 --- a/.circleci/unittest-config.yml +++ b/.circleci/unittest-config.yml @@ -62,7 +62,7 @@ commands: command: | set +e declare -i test_status - bazel --host_jvm_args=-Xmx8g test //engine/... << parameters.extra_bazel_args >> --jobs=32 --ui_event_filters=-info,-debug,-warning --test_output=errors | tee test_result.log; test_status=${PIPESTATUS[0]} + bazelisk --host_jvm_args=-Xmx8g test //engine/... << parameters.extra_bazel_args >> --jobs=32 --ui_event_filters=-info,-debug,-warning --test_output=errors | tee test_result.log; test_status=${PIPESTATUS[0]} sh ../devtools/rename-junit-xml.sh find bazel-bin/ << parameters.find_executable_flag >> -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz From 72e1d4762fc4fe2fa0dfe979b673bdff53f24f0d Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Tue, 4 Mar 2025 10:08:03 +0800 Subject: [PATCH 3/6] Split path-filtering/filter parameters --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 098c02b7..0d2f6feb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,7 +61,8 @@ workflows: bazel/.* run_engine_ut true engine/.* run_engine_ut true .bazelrc run_engine_ut true - .circleci/unittest-config.yml run_scdb_ut true run_engine_ut true + .circleci/unittest-config.yml run_scdb_ut true + .circleci/unittest-config.yml run_engine_ut true run_regtest: when: << pipeline.parameters.enable_regtest >> jobs: From 82ce4fe7f5651bc87be5309b43b4399d35f67128 Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Tue, 4 Mar 2025 10:11:21 +0800 Subject: [PATCH 4/6] fix typo --- .circleci/unittest-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/unittest-config.yml b/.circleci/unittest-config.yml index 8562b3df..80772ffb 100644 --- a/.circleci/unittest-config.yml +++ b/.circleci/unittest-config.yml @@ -137,7 +137,7 @@ jobs: command: | brew install bazelisk cmake ninja libomp wget go@1.24 md5sha1sum brew link go@1.24 - - engine_setup + - setup_engine - run_engine_test: extra_bazel_args: "" find_executable_flag: "-perm +111" From 32dc15c12185d2488485c53ebecd928d15eca209 Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Tue, 4 Mar 2025 10:49:39 +0800 Subject: [PATCH 5/6] rename scdb_ut/engine_ut to go_ut/cpp_ut --- .circleci/config.yml | 18 ++++++------ .circleci/unittest-config.yml | 54 +++++++++++++++++------------------ 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0d2f6feb..e3310157 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,15 +54,15 @@ workflows: base-revision: main config-path: .circleci/unittest-config.yml mapping: | - api/.* run_scdb_ut true - cmd/.* run_scdb_ut true - pkg/.* run_scdb_ut true - contrib/.* run_scdb_ut true - bazel/.* run_engine_ut true - engine/.* run_engine_ut true - .bazelrc run_engine_ut true - .circleci/unittest-config.yml run_scdb_ut true - .circleci/unittest-config.yml run_engine_ut true + api/.* run_go_ut true + cmd/.* run_go_ut true + pkg/.* run_go_ut true + contrib/.* run_go_ut true + bazel/.* run_cpp_ut true + engine/.* run_cpp_ut true + .bazelrc run_cpp_ut true + .circleci/unittest-config.yml run_go_ut true + .circleci/unittest-config.yml run_cpp_ut true run_regtest: when: << pipeline.parameters.enable_regtest >> jobs: diff --git a/.circleci/unittest-config.yml b/.circleci/unittest-config.yml index 80772ffb..e87b2209 100644 --- a/.circleci/unittest-config.yml +++ b/.circleci/unittest-config.yml @@ -20,15 +20,15 @@ parameters: enable_regtest: type: boolean default: false - run_scdb_ut: + run_go_ut: type: boolean default: false - run_engine_ut: + run_cpp_ut: type: boolean default: false commands: - setup_engine: + setup_cpp_test: steps: - run: name: "Checkout devtools" @@ -38,17 +38,17 @@ commands: command: | echo ${gcs_content} > ../gcs.data ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json --min_download - run_scdb_test: - description: "Run scdb tests" + run_go_test: + description: "Run go tests" steps: - run: - name: "SCDB Test" + name: "Go Test" command: | set +e go mod tidy go test -timeout=30m -v -short ./pkg/... ./contrib/... - run_engine_test: - description: "Run engine tests and collect artifacts" + run_cpp_test: + description: "Run cpp tests and collect artifacts" parameters: extra_bazel_args: type: string @@ -58,7 +58,7 @@ commands: default: "-executable" steps: - run: - name: "Engine Test" + name: "Cpp Test" command: | set +e declare -i test_status @@ -73,7 +73,7 @@ commands: # Define a job to be invoked later in a workflow. # See: https://circleci.com/docs/2.0/configuration-reference/#jobs jobs: - linux_scdb_ut: + linux_go_ut: # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: @@ -86,8 +86,8 @@ jobs: # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: - checkout - - run_scdb_test - linux_engine_ut: + - run_go_test + linux_cpp_ut: docker: - image: secretflow/scql-ci:latest parameters: @@ -98,8 +98,8 @@ jobs: # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: - checkout - - setup_engine - - run_engine_test: + - setup_cpp_test + - run_cpp_test: extra_bazel_args: "-c opt" find_executable_flag: "-executable" - store_test_results: @@ -108,7 +108,7 @@ jobs: path: test_binary.tar.gz - store_artifacts: path: test_logs.tar.gz - macOS_scdb_ut: + macOS_go_ut: macos: xcode: 16.0.0 resource_class: macos.m1.large.gen1 @@ -118,8 +118,8 @@ jobs: name: "Install homebrew dependencies" command: | brew install wget go - - run_scdb_test - macOS_engine_ut: + - run_go_test + macOS_cpp_ut: macos: xcode: 16.0.0 resource_class: macos.m1.large.gen1 @@ -137,8 +137,8 @@ jobs: command: | brew install bazelisk cmake ninja libomp wget go@1.24 md5sha1sum brew link go@1.24 - - setup_engine - - run_engine_test: + - setup_cpp_test + - run_cpp_test: extra_bazel_args: "" find_executable_flag: "-perm +111" - store_test_results: @@ -151,19 +151,19 @@ jobs: # Invoke jobs via workflows # See: https://circleci.com/docs/2.0/configuration-reference/#workflows workflows: - run_scdb_ut: - when: << pipeline.parameters.run_scdb_ut >> + run_go_ut: + when: << pipeline.parameters.run_go_ut >> jobs: - - linux_scdb_ut: + - linux_go_ut: matrix: parameters: resource_class: ["2xlarge", "arm-xlarge"] - - macOS_scdb_ut - run_engine_ut: - when: << pipeline.parameters.run_engine_ut >> + - macOS_go_ut + run_cpp_ut: + when: << pipeline.parameters.run_cpp_ut >> jobs: - - linux_engine_ut: + - linux_cpp_ut: matrix: parameters: resource_class: ["2xlarge", "arm-xlarge"] - - macOS_engine_ut \ No newline at end of file + - macOS_cpp_ut \ No newline at end of file From 36922381970ae13cc2d2325303f20554c2b356f8 Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Tue, 4 Mar 2025 18:31:22 +0800 Subject: [PATCH 6/6] add trigger file for run_cpp_ut --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e3310157..5f10a67c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,10 +58,12 @@ workflows: cmd/.* run_go_ut true pkg/.* run_go_ut true contrib/.* run_go_ut true + .circleci/unittest-config.yml run_go_ut true bazel/.* run_cpp_ut true engine/.* run_cpp_ut true .bazelrc run_cpp_ut true - .circleci/unittest-config.yml run_go_ut true + .bazeliskrc run_cpp_ut true + MODULE.bazel run_cpp_ut true .circleci/unittest-config.yml run_cpp_ut true run_regtest: when: << pipeline.parameters.enable_regtest >>