Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GH-35331: [Python] Expose Parquet sorting metadata #37665

Merged
merged 18 commits into from
Dec 20, 2023
+394 −13
Merged

GH-35331: [Python] Expose Parquet sorting metadata #37665

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
94b437f
feat: expose parquet sort metadata
wjones127 May 5, 2023
23aa525
refactor: simplify API
wjones127 May 10, 2023
fe0c7c0
docs: add a nice example
wjones127 May 11, 2023
f92869b
fix: address nested schemas
wjones127 May 11, 2023
ad128a6
fix the repr
wjones127 May 11, 2023
2ed9b1b
docs: fix doc formatting
wjones127 May 12, 2023
67584e5
docs: update repr in examples
wjones127 May 12, 2023
bba1a12
remove sort order aggregation
wjones127 Jun 10, 2023
3a487e3
make repr deterministic
wjones127 Jun 18, 2023
2c98b04
fix doctest
wjones127 Jun 19, 2023
d994b42
pr feedback
wjones127 Jul 6, 2023
dda4fed
Add `sorting_columns` to `ParquetFileWriteOptions`
judahrand Sep 11, 2023
0de1e68
Format for reduced git diff
judahrand Sep 11, 2023
280cfee
Remove unused helper
judahrand Oct 24, 2023
0249bb6
Move import to where it is needed!
judahrand Oct 30, 2023
7c8849f
Remove unnecessary exception
judahrand Oct 30, 2023
842107f
Fix rebase error
judahrand Dec 10, 2023
44dff9d
Rename methods
judahrand Dec 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/source/python/api/formats.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ Parquet Metadata

FileMetaData
RowGroupMetaData
SortingColumn
ColumnChunkMetaData
Statistics
ParquetSchema
Expand Down
2 changes: 2 additions & 0 deletions python/pyarrow/_dataset_parquet.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -608,6 +608,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions):
dictionary_pagesize_limit=self._properties["dictionary_pagesize_limit"],
write_page_index=self._properties["write_page_index"],
write_page_checksum=self._properties["write_page_checksum"],
sorting_columns=self._properties["sorting_columns"],
)

def _set_arrow_properties(self):
Expand Down Expand Up @@ -658,6 +659,7 @@ cdef class ParquetFileWriteOptions(FileWriteOptions):
write_page_index=False,
encryption_config=None,
write_page_checksum=False,
sorting_columns=None,
)

self._set_properties()
Expand Down
24 changes: 17 additions & 7 deletions python/pyarrow/_parquet.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,11 +328,17 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
optional[ParquetIndexLocation] GetColumnIndexLocation() const
optional[ParquetIndexLocation] GetOffsetIndexLocation() const

struct CSortingColumn" parquet::SortingColumn":
int column_idx
c_bool descending
c_bool nulls_first

cdef cppclass CRowGroupMetaData" parquet::RowGroupMetaData":
c_bool Equals(const CRowGroupMetaData&) const
int num_columns()
int64_t num_rows()
int64_t total_byte_size()
int num_columns() const
int64_t num_rows() const
int64_t total_byte_size() const
vector[CSortingColumn] sorting_columns() const
unique_ptr[CColumnChunkMetaData] ColumnChunk(int i) const

cdef cppclass CFileMetaData" parquet::FileMetaData":
Expand Down Expand Up @@ -419,6 +425,7 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
Builder* disable_dictionary()
Builder* enable_dictionary()
Builder* enable_dictionary(const c_string& path)
Builder* set_sorting_columns(vector[CSortingColumn] sorting_columns)
Builder* disable_statistics()
Builder* enable_statistics()
Builder* enable_statistics(const c_string& path)
Expand Down Expand Up @@ -515,8 +522,8 @@ cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:

CStatus ToParquetSchema(
const CSchema* arrow_schema,
const ArrowReaderProperties& properties,
const shared_ptr[const CKeyValueMetadata]& key_value_metadata,
const WriterProperties& properties,
const ArrowWriterProperties& arrow_properties,
shared_ptr[SchemaDescriptor]* out)


Expand Down Expand Up @@ -582,7 +589,9 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
write_batch_size=*,
dictionary_pagesize_limit=*,
write_page_index=*,
write_page_checksum=*) except *
write_page_checksum=*,
sorting_columns=*,
) except *


cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties(
Expand All @@ -591,7 +600,8 @@ cdef shared_ptr[ArrowWriterProperties] _create_arrow_writer_properties(
allow_truncated_timestamps=*,
writer_engine_version=*,
use_compliant_nested_type=*,
store_schema=*) except *
store_schema=*,
) except *

cdef class ParquetSchema(_Weakrefable):
cdef:
Expand Down
Loading