Skip to content

Commit

Permalink
enhance row filter for Go Parquet compatibility with improved type co…
Browse files Browse the repository at this point in the history
…ercion handling
  • Loading branch information
kosiew committed Jan 28, 2025
1 parent 88ad42d commit d021d58
Showing 1 changed file with 6 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -131,18 +131,17 @@ impl DatafusionArrowPredicate {
println!("==> Row filter: Parquet file version {}", version);
let physical_expr = if version >= 2 {
println!("==> Row filter: Applying version 2.6+ type handling");
// For version 2.6 files, we need to ensure proper type handling
let expr = reassign_predicate_columns(candidate.expr, &schema, true)?;
// For version 2.6+ files, we need to ensure proper type handling
let mut expr = reassign_predicate_columns(candidate.expr, &schema, true)?;
// Apply additional type coercion for version 2.6 compatibility
// This ensures proper handling of type differences between parquet-go and arrow
// For Go-generated Parquet files, we need to handle type conversion more carefully
let expr = if version == 2 {
if version == 2 {
// For version 2.0 files (like those from Go), apply special type handling
println!("==> Row filter: Applying Go Parquet compatibility handling");
reassign_predicate_columns(expr.clone(), &schema, true)?
} else {
expr
};
// Apply type coercion twice to handle potential nested type conversions
expr = reassign_predicate_columns(expr.clone(), &schema, true)?;
}
expr
} else {
println!("==> Row filter: Using standard type handling");
Expand Down

0 comments on commit d021d58

Please sign in to comment.