fix(python): reuse table state in write engine (#2453)

# Description
Instead of reusing the table state, it was being instantiated every time
you call write with the rust engine.

- https://github.com/delta-io/delta-rs/discussions/2448
This commit is contained in:
Ion Koutsouris 2024-04-25 15:37:31 +02:00 committed by GitHub
parent f55ddc64a3
commit 6a7c684d9b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 10 additions and 4 deletions

View File

@ -1,6 +1,6 @@
[package]
name = "deltalake-python"
version = "0.17.1"
version = "0.17.2"
authors = ["Qingping Hou <dave2008713@gmail.com>", "Will Jones <willjones127@gmail.com>"]
homepage = "https://github.com/delta-io/delta-rs"
license = "Apache-2.0"

View File

@ -174,6 +174,7 @@ def write_to_deltalake(
data: pyarrow.RecordBatchReader,
partition_by: Optional[List[str]],
mode: str,
table: Optional[RawDeltaTable],
schema_mode: Optional[str],
predicate: Optional[str],
name: Optional[str],

View File

@ -318,6 +318,7 @@ def write_deltalake(
data=data,
partition_by=partition_by,
mode=mode,
table=table._table if table is not None else None,
schema_mode=schema_mode,
predicate=predicate,
name=name,

View File

@ -1410,6 +1410,7 @@ fn write_to_deltalake(
table_uri: String,
data: PyArrowType<ArrowArrayStreamReader>,
mode: String,
table: Option<&RawDeltaTable>,
schema_mode: Option<String>,
partition_by: Option<Vec<String>>,
predicate: Option<String>,
@ -1425,11 +1426,14 @@ fn write_to_deltalake(
let save_mode = mode.parse().map_err(PythonError::from)?;
let options = storage_options.clone().unwrap_or_default();
let table = rt()
.block_on(DeltaOps::try_from_uri_with_storage_options(
let table = if let Some(table) = table {
DeltaOps(table._table.clone())
} else {
rt().block_on(DeltaOps::try_from_uri_with_storage_options(
&table_uri, options,
))
.map_err(PythonError::from)?;
.map_err(PythonError::from)?
};
let mut builder = table.write(batches).with_save_mode(save_mode);
if let Some(schema_mode) = schema_mode {