Merge branch 'main' of github.com:singularity-data/risingwave into yu…

…hao/use-jsonschema

Makefile.toml

@@ -1359,7 +1359,7 @@ echo "All processes has exited."
 
 [tasks.slt]
 category = "RiseDev - Test - SQLLogicTest"
-install_crate = { min_version = "0.26.3", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
+install_crate = { min_version = "0.27.1", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
     "--help",
 ], install_command = "binstall" }
 dependencies = ["check-and-load-risedev-env-file"]

README.md

@@ -9,7 +9,7 @@
 
 <div align="center">
 
-### 🌊 Ride the Wave of Real-Time Data.
+### 🌊 Ride the Wave of Streaming Data.
 
 </div>
 <p align="center">

ci/Dockerfile

@@ -70,7 +70,7 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 RUN cargo binstall -y --locked cargo-llvm-cov cargo-nextest cargo-sort cargo-cache cargo-machete \
     cargo-make@0.37.9 \
-    sqllogictest-bin@0.26.3 \
+    sqllogictest-bin@0.27.1 \
     sccache@0.7.4 \
     && cargo cache -a \
     && rm -rf "/root/.cargo/registry/index" \

ci/build-ci-image.sh

@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20250115
+export BUILD_ENV_VERSION=v20250218
 
 export BUILD_TAG="public.ecr.aws/w1p7b4n3/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -90,7 +90,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250115
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250218
     depends_on:
       - mysql
       - mysql-meta
@@ -106,7 +106,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250115
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250218
     depends_on:
       - mysql
       - mysql-meta
@@ -129,28 +129,28 @@ services:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250115
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250218
     volumes:
       - ..:/risingwave
 
   # Standard environment for CI, including MySQL and Postgres for metadata.
   ci-standard-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250115
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250218
     depends_on:
       - mysql-meta
       - db
     volumes:
       - ..:/risingwave
 
   iceberg-engine-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250115
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250218
     depends_on:
       - db
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250115
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250218
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -161,7 +161,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250115
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20250218
     depends_on:
       db:
         condition: service_healthy

ci/scripts/e2e-test-parallel.sh

@@ -26,34 +26,34 @@ download_and_prepare_rw "$profile" common
 echo "--- Download artifacts"
 download-and-decompress-artifact e2e_test_generated ./
 
+start_cluster() {
+    echo "--- Start cluster"
+    risedev ci-start ci-3streaming-2serving-3fe
+}
+
 kill_cluster() {
-  echo "--- Kill cluster"
-  risedev ci-kill
+    echo "--- Kill cluster"
+    risedev ci-kill
 }
 
 host_args=(-h localhost -p 4565 -h localhost -p 4566 -h localhost -p 4567)
 
-RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info,risingwave_storage::hummock::compactor::compactor_runner=warn"
-
-echo "--- e2e, ci-3streaming-2serving-3fe, streaming"
-RUST_LOG=$RUST_LOG \
-risedev ci-start ci-3streaming-2serving-3fe
-sqllogictest "${host_args[@]}" -d dev './e2e_test/streaming/**/*.slt' -j 16 --junit "parallel-streaming-${profile}" --label "parallel"
-
+echo "--- e2e, parallel, streaming"
+RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info,risingwave_storage::hummock::compactor::compactor_runner=warn" \
+start_cluster
+risedev slt "${host_args[@]}" -d dev './e2e_test/streaming/**/*.slt' -j 16 --junit "parallel-streaming-${profile}" --label "parallel"
 kill_cluster
 
-echo "--- e2e, ci-3streaming-2serving-3fe, batch"
-RUST_LOG=$RUST_LOG \
-risedev ci-start ci-3streaming-2serving-3fe
+echo "--- e2e, parallel, batch"
+RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info,risingwave_storage::hummock::compactor::compactor_runner=warn" \
+start_cluster
 # Exclude files that contain ALTER SYSTEM commands
 find ./e2e_test/ddl -name "*.slt" -type f -exec grep -L "ALTER SYSTEM" {} \; | xargs -r sqllogictest "${host_args[@]}" -d dev --junit "parallel-batch-ddl-${profile}" --label "parallel"
-sqllogictest "${host_args[@]}" -d dev './e2e_test/visibility_mode/*.slt' -j 16 --junit "parallel-batch-${profile}" --label "parallel"
-
+risedev slt "${host_args[@]}" -d dev './e2e_test/visibility_mode/*.slt' -j 16 --junit "parallel-batch-${profile}" --label "parallel"
 kill_cluster
 
-echo "--- e2e, ci-3streaming-2serving-3fe, generated"
-RUST_LOG=$RUST_LOG \
-risedev ci-start ci-3streaming-2serving-3fe
-sqllogictest "${host_args[@]}" -d dev './e2e_test/generated/**/*.slt' -j 16 --junit "parallel-generated-${profile}" --label "parallel"
-
+echo "--- e2e, parallel, generated"
+RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info,risingwave_storage::hummock::compactor::compactor_runner=warn" \
+start_cluster
+risedev slt "${host_args[@]}" -d dev './e2e_test/generated/**/*.slt' -j 16 --junit "parallel-generated-${profile}" --label "parallel"
 kill_cluster

ci/scripts/e2e-test-serial.sh

@@ -84,8 +84,7 @@ echo "--- e2e, $mode, streaming"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info,risingwave_stream::common::table::state_table=warn" \
 cluster_start
 # Please make sure the regression is expected before increasing the timeout.
-sqllogictest -p 4566 -d dev './e2e_test/streaming/**/*.slt' --junit "streaming-${profile}"
-risedev slt -p 4566 -d dev './e2e_test/queryable_internal_state/**/*.slt' --junit "queryable-internal-state-${profile}"
+risedev slt -p 4566 -d dev './e2e_test/streaming/**/*.slt' --junit "streaming-${profile}"
 sqllogictest -p 4566 -d dev './e2e_test/backfill/sink/different_pk_and_dist_key.slt'
 
 echo "--- Kill cluster"
@@ -145,14 +144,6 @@ sqllogictest -p 4566 -d dev './e2e_test/udf/python_udf.slt'
 echo "--- Kill cluster"
 cluster_stop
 
-echo "--- e2e, $mode, generated"
-RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
-cluster_start
-sqllogictest -p 4566 -d dev './e2e_test/generated/**/*.slt' --junit "generated-${profile}"
-
-echo "--- Kill cluster"
-cluster_stop
-
 # only run if mode is not single-node or standalone
 if [[ "$mode" != "single-node" && "$mode" != "standalone" ]]; then
   echo "--- e2e, ci-3cn-1fe-with-recovery, error ui"

e2e_test/batch/functions/cast.slt.part

@@ -38,3 +38,19 @@ select date(1);
 
 query error unexpected arguments number
 select date();
+
+query ?
+select cast('a' as char(1));
+----
+a
+
+# since we don't support char type, cast to char(2) will be the same as cast to varchar, so its behavior is the same as cast to varchar and not the same as postgresql.
+query ?
+select 'a'::char(2) = 'a '::char(2);
+----
+f
+
+query ?
+select 'a'::varchar = 'a '::varchar;
+----
+f

e2e_test/commands/internal_table.mjs

@@ -4,19 +4,23 @@
 // https://google.github.io/zx/
 
 const {
+  db: db_name,
   name: job_name,
   type: table_type,
   count: count,
 } = minimist(process.argv.slice(3), {
-  string: ["name", "type"],
+  string: ["db", "name", "type"],
   boolean: ["count"],
+  default: {
+    "db": "dev",
+  }
 });
 
 // Return an array of CSV string
-async function psql(query) {
+async function psql(db_name, query) {
   return (
     await $`
-psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev \
+psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d ${db_name} \
 --csv --tuples-only --quiet -c ${query}
 `
   )
@@ -28,9 +32,10 @@ psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root
 
 // If `table_type` is null, return all internal tables for the job.
 // If `job_name` is null, return all jobs' internal tables.
-async function select_internal_table(job_name, table_type) {
+async function select_internal_table(db_name, job_name, table_type) {
   // Note: if we have `t1`, and `t1_balabala`, the latter one will also be matched 😄.
   const internal_tables = await psql(
+    db_name,
     `select name from rw_internal_tables where name like '__internal_${job_name}_%_${table_type}_%'`
   );
   if (internal_tables.length == 0) {
@@ -42,15 +47,15 @@ async function select_internal_table(job_name, table_type) {
   const res = new Map(
     await Promise.all(
       internal_tables.map(async (t) => {
-        let rows = await psql(`select * from ${t}`);
+        let rows = await psql(db_name, `select * from ${t}`);
         return [t, rows];
       })
     )
   );
   return res;
 }
 
-const tables = await select_internal_table(job_name, table_type);
+const tables = await select_internal_table(db_name, job_name, table_type);
 for (const [table_name, rows] of tables) {
   if (tables.size > 1) {
     console.log(`Table: ${table_name}`);

e2e_test/source_inline/kafka/alter_table_drop_connector.slt

@@ -95,7 +95,7 @@ FORMAT PLAIN ENCODE AVRO (
 
 sleep 1s
 
-query ??
+query ?? retry 3 backoff 5s
 select foo, bar from avro_drop_table_connector_test_table
 ----
 ABC	1

e2e_test/source_legacy/cdc/cdc.share_stream.slt

@@ -161,6 +161,17 @@ create table non_exist ( id INT,
  PRIMARY KEY (id)
 ) from mysql_mytest table 'mytest.non_exist';
 
+statement error Not supported: Non-generated column found after a generated column.
+create table orders_test (
+   next_order_id int as order_id + 1,
+   order_id int,
+   order_date timestamp,
+   customer_name string,
+   price decimal,
+   product_id int,
+   order_status smallint,
+   PRIMARY KEY (order_id)
+) from mysql_mytest table 'mytest.orders';
 
 statement ok
 create table orders_test (
@@ -170,6 +181,7 @@ create table orders_test (
    price decimal,
    product_id int,
    order_status smallint,
+   next_order_id int as order_id + 1,
    PRIMARY KEY (order_id)
 ) from mysql_mytest table 'mytest.orders';
 
@@ -305,11 +317,11 @@ Milk Milk is a white liquid food
 Juice 100ml Juice
 
 query ITTT
-SELECT order_id,order_date,customer_name,product_id FROM orders_test order by order_id limit 3
+SELECT order_id,order_date,customer_name,product_id,next_order_id FROM orders_test order by order_id limit 3
 ----
-10001 2020-07-30 10:08:22 Jark 102
-10002 2020-07-30 10:11:09 Sally 105
-10003 2020-07-30 12:00:30 Edward 106
+10001 2020-07-30 10:08:22 Jark 102 10002
+10002 2020-07-30 10:11:09 Sally 105 10003
+10003 2020-07-30 12:00:30 Edward 106 10004
 
 query IIIIITTTTTTTTT
 SELECT c_boolean, c_bit, c_tinyint, c_smallint, c_mediumint, c_integer, c_bigint, c_decimal, c_float, c_double, c_char_255, c_varchar_10000, c_date, c_time, c_datetime, c_timestamp FROM mysql_all_types order by c_bigint;
@@ -393,13 +405,15 @@ CREATE TABLE upper_orders_shared (
     name varchar
 ) FROM pg_source TABLE 'public.Orders';
 
+# FIXME(kexiang): Currently, the generated rows (next_id in this case) must be at the end of schema, otherwise the frontend will throw an error.
 statement ok
 CREATE TABLE person_new (
     id int,
     name varchar,
     email_address varchar,
     credit_card varchar,
     city varchar,
+    next_id int as id + 1,
     PRIMARY KEY (id)
 ) INCLUDE TIMESTAMP AS commit_ts
 INCLUDE DATABASE_NAME as database_name
@@ -459,22 +473,22 @@ cdc_test public person
 
 
 query ITTTT
-SELECT id,name,email_address,credit_card,city from person_new order by id;
+SELECT id,name,email_address,credit_card,city,next_id from person_new order by id;
 ----
-1000 vicky noris yplkvgz@qbxfg.com 7878 5821 1864 2539 cheyenne
-1001 peter white myckhsp@xpmpe.com 1781 2313 8157 6974 boise
-1002 sarah spencer wipvdbm@dkaap.com 3453 4987 9481 6270 los angeles
-1100 noris ypl@qbxfg.com 1864 2539 enne
-1101 white myc@xpmpe.com 8157 6974 se
-1102 spencer wip@dkaap.com 9481 6270 angeles
+1000 vicky noris yplkvgz@qbxfg.com 7878 5821 1864 2539 cheyenne 1001
+1001 peter white myckhsp@xpmpe.com 1781 2313 8157 6974 boise 1002
+1002 sarah spencer wipvdbm@dkaap.com 3453 4987 9481 6270 los angeles 1003
+1100 noris ypl@qbxfg.com 1864 2539 enne 1101
+1101 white myc@xpmpe.com 8157 6974 se 1102
+1102 spencer wip@dkaap.com 9481 6270 angeles 1103
 
 # historical data
 query ITTTT
-SELECT id,name,email_address,credit_card,city from person_new where commit_ts = '1970-01-01 00:00:00+00:00' order by id;
+SELECT id,name,email_address,credit_card,city,next_id from person_new where commit_ts = '1970-01-01 00:00:00+00:00' order by id;
 ----
-1000 vicky noris yplkvgz@qbxfg.com 7878 5821 1864 2539 cheyenne
-1001 peter white myckhsp@xpmpe.com 1781 2313 8157 6974 boise
-1002 sarah spencer wipvdbm@dkaap.com 3453 4987 9481 6270 los angeles
+1000 vicky noris yplkvgz@qbxfg.com 7878 5821 1864 2539 cheyenne 1001
+1001 peter white myckhsp@xpmpe.com 1781 2313 8157 6974 boise 1002
+1002 sarah spencer wipvdbm@dkaap.com 3453 4987 9481 6270 los angeles 1003
 
 # incremental data
 query ITTTT

e2e_test/source_legacy/cdc/cdc.validate.postgres.slt

@@ -169,6 +169,26 @@ create table shipments (
  slot.name = 'shipments'
 ) format canal encode csv;
 
+statement error Not supported: Non-generated column found after a generated column.
+create table shipments (
+  next_shipment_id INTEGER as shipment + 1,
+  shipment_id INTEGER,
+  order_id INTEGER,
+  origin STRING,
+  destination STRING,
+  is_arrived boolean,
+ PRIMARY KEY (shipment_id)
+) with (
+ connector = 'postgres-cdc',
+ hostname = '${PGHOST:localhost}',
+ port = '${PGPORT:5432}',
+ username = '${PGUSER:$USER}',
+ password = '${PGPASSWORD:}',
+ database.name = '${PGDATABASE:postgres}',
+ table.name = 'shipments',
+ slot.name = 'shipments'
+);
+
 statement ok
 explain create table numeric_to_rw_int256 (
   id int,