Prepare for DF52 release

Cargo.toml

@@ -57,10 +57,10 @@ pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] }
 pyo3-log = "0.13.2"
 arrow = { version = "57", features = ["pyarrow"] }
 arrow-select = { version = "57" }
-datafusion = { version = "51", features = ["avro", "unicode_expressions"] }
-datafusion-substrait = { version = "51", optional = true }
-datafusion-proto = { version = "51" }
-datafusion-ffi = { version = "51" }
+datafusion = { version = "52", features = ["avro", "unicode_expressions"] }
+datafusion-substrait = { version = "52", optional = true }
+datafusion-proto = { version = "52" }
+datafusion-ffi = { version = "52" }
 prost = "0.14.1" # keep in line with `datafusion-substrait`
 uuid = { version = "1.18", features = ["v4"] }
 mimalloc = { version = "0.1", optional = true, default-features = false, features = [

benchmarks/tpch/tpch.py

@@ -23,7 +23,7 @@
 
 
 def bench(data_path, query_path) -> None:
-    with Path.open("results.csv", "w") as results:
+    with Path("results.csv").open("w") as results:
         # register tables
         start = time.time()
         total_time_millis = 0
@@ -46,7 +46,7 @@ def bench(data_path, query_path) -> None:
         print("Configuration:\n", ctx)
 
         # register tables
-        with Path.open("create_tables.sql") as f:
+        with Path("create_tables.sql").open() as f:
             sql = ""
             for line in f.readlines():
                 if line.startswith("--"):
@@ -66,7 +66,7 @@ def bench(data_path, query_path) -> None:
 
         # run queries
         for query in range(1, 23):
-            with Path.open(f"{query_path}/q{query}.sql") as f:
+            with Path(f"{query_path}/q{query}.sql").open() as f:
                 text = f.read()
                 tmp = text.split(";")
                 queries = [s.strip() for s in tmp if len(s.strip()) > 0]

docs/source/contributor-guide/ffi.rst

@@ -15,6 +15,8 @@
 .. specific language governing permissions and limitations
 .. under the License.
 
+.. _ffi:
+
 Python Extensions
 =================
 

docs/source/index.rst

@@ -77,6 +77,7 @@ Example
    user-guide/io/index
    user-guide/configuration
    user-guide/sql
+   user-guide/upgrade-guides
 
 
 .. _toc.contributor_guide:

docs/source/user-guide/upgrade-guides.rst

@@ -0,0 +1,106 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Upgrade Guides
+==============
+
+DataFusion 52.0.0
+-----------------
+
+This version includes a major update to the :ref:`ffi` due to upgrades
+to the `Foreign Function Interface <https://doc.rust-lang.org/nomicon/ffi.html>`_.
+Users who contribute their own ``CatalogProvider``, ``SchemaProvider``,
+``TableProvider`` or ``TableFunction`` via FFI must now provide access to a
+``LogicalExtensionCodec`` and a ``TaskContextProvider``. The most convenient
+way to provide these is from the ``datafusion-python`` ``SessionContext`` Python
+object. The ``SessionContext`` now has a method to export a
+``FFI_LogicalExtensionCodec``, which can satisfy this new requirement.
+
+A complete example can be found in the `FFI example <https://github.com/apache/datafusion-python/tree/main/examples/datafusion-ffi-example>`_.
+The constructor for your provider needs to take as an input the ``SessionContext``
+python object. Instead of calling ``FFI_CatalogProvider::new`` you can use the
+added method ``FFI_CatalogProvider::new_with_ffi_codec`` as follows:
+
+.. code-block:: rust
+
+    #[pymethods]
+    impl MyCatalogProvider {
+        #[new]
+        pub fn new(session: &Bound<PyAny>) -> PyResult<Self> {
+            let logical_codec = ffi_logical_codec_from_pycapsule(session)?;
+            let inner = Arc::new(MemoryCatalogProvider::new());
+
+            Ok(Self {
+                inner,
+                logical_codec,
+            })
+        }
+
+        pub fn __datafusion_catalog_provider__<'py>(
+            &self,
+            py: Python<'py>,
+        ) -> PyResult<Bound<'py, PyCapsule>> {
+            let name = cr"datafusion_catalog_provider".into();
+            let codec = self.logical_codec.clone();
+            let catalog_provider =
+                FFI_CatalogProvider::new_with_ffi_codec(Arc::new(self.clone()), None, codec);
+
+            PyCapsule::new(py, catalog_provider, Some(name))
+        }
+    }
+
+To extract the logical extension codec FFI object from the ``SessionContext`` you
+can implement a helper method such as:
+
+.. code-block:: rust
+
+    pub(crate) fn ffi_logical_codec_from_pycapsule(
+        obj: &Bound<PyAny>,
+    ) -> PyResult<FFI_LogicalExtensionCodec> {
+        let attr_name = "__datafusion_logical_extension_codec__";
+
+        if obj.hasattr(attr_name)? {
+            let capsule = obj.getattr(attr_name)?.call0()?;
+            let capsule = capsule.downcast::<PyCapsule>()?;
+            validate_pycapsule(capsule, "datafusion_logical_extension_codec")?;
+
+            let provider = unsafe { capsule.reference::<FFI_LogicalExtensionCodec>() };
+
+            Ok(provider.clone())
+        } else {
+            Err(PyValueError::new_err(
+                "Expected PyCapsule object for FFI_LogicalExtensionCodec, but attribute does not exist",
+            ))
+        }
+    }
+
+The DataFusion FFI interface updates no longer depend directly on the
+``datafusion`` core crate. You can improve your build times and potentially
+reduce your library binary size by removing this dependency and instead
+using the specific datafusion project crates.
+
+For example, instead of including expressions like:
+
+.. code-block:: rust
+
+    use datafusion::catalog::MemTable;
+
+Instead you can now write:
+
+.. code-block:: rust
+
+    use datafusion_catalog::MemTable;