disable dictionary encoding

Signed-off-by: chenxu <chenxu@dmetasoul.com>
lakesoul-io · Jan 24, 2025 · 68ade11 · 68ade11
1 parent af84785
commit 68ade11
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/rust/lakesoul-io/src/async_writer/multipart_writer.rs b/rust/lakesoul-io/src/async_writer/multipart_writer.rs
@@ -104,7 +104,7 @@ impl MultiPartAsyncWriter {
                     .set_max_row_group_size(max_row_group_size)
                     .set_write_batch_size(config.batch_size)
                     .set_compression(Compression::SNAPPY)
-                    // .set_statistics_enabled(parquet::file::properties::EnabledStatistics::None)
+                    .set_dictionary_enabled(false)
                     .build(),
             ),
         )?;

diff --git a/rust/lakesoul-io/src/lakesoul_io_config.rs b/rust/lakesoul-io/src/lakesoul_io_config.rs
@@ -577,6 +577,7 @@ pub fn create_session_context_with_planner(
     sess_conf.options_mut().optimizer.prefer_hash_join = false; //if true, panicked at 'range end out of bounds'
     sess_conf.options_mut().execution.parquet.pushdown_filters = config.parquet_filter_pushdown;
     sess_conf.options_mut().execution.target_partitions = 1;
+    sess_conf.options_mut().execution.parquet.dictionary_enabled = Some(false);
     // sess_conf.options_mut().execution.sort_in_place_threshold_bytes = 16 * 1024;
     // sess_conf.options_mut().execution.sort_spill_reservation_bytes = 2 * 1024 * 1024;
     // sess_conf.options_mut().catalog.default_catalog = "lakesoul".into();