More doc

Author: friendlymatthew

datafusion/core/src/datasource/file_format/mod.rs

@@ -89,7 +89,7 @@ pub(crate) mod test_util {
                 .build();
 
         let source = format.file_source(config);
-        Ok(format.create_physical_plan(state, source).await?)
+        format.create_physical_plan(state, source).await
     }
 }
 

datafusion/datasource/src/file_scan_config.rs

@@ -15,8 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! [`FileScanConfig`] to configure scanning of possibly partitioned
-//! file sources.
+//! [`FileScanConfig`] for configuring file-based data sources that implement [`DataSource`].
+//!
+//! [`DataSource`]: crate::source::DataSource
 
 use std::{
     borrow::Cow, collections::HashMap, fmt::Debug, fmt::Formatter,
@@ -54,10 +55,12 @@ use datafusion_physical_plan::{
 
 use log::{debug, warn};
 
-/// The base configurations for a [`DataSourceExec`], the a physical plan for
-/// any given file format.
+/// Configuration for file-based data sources.
 ///
-/// Use [`DataSourceExec::from_data_source`] to create a [`DataSourceExec`] from a ``FileScanConfig`.
+/// Owned by file sources that implement [`DataSource`]. Use [`DataSourceExec::from_data_source`]
+/// to create a [`DataSourceExec`] from a data source that owns this config.
+///
+/// [`DataSource`]: crate::source::DataSource
 ///
 /// # Example
 /// ```
@@ -88,7 +91,7 @@ use log::{debug, warn};
 /// #[derive(Clone)]
 /// # struct ParquetSource {
 /// #    projected_statistics: Option<Statistics>,
-/// #    schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>
+/// #    schema_adapter_factory: Option<Arc<dyn SchemaAdapterFactory>>,
 /// #    config: FileScanConfig,
 /// # };
 /// # impl Debug for ParquetSource {fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { unimplemented!() }}
@@ -120,8 +123,8 @@ use log::{debug, warn};
 ///    PartitionedFile::new("file2.parquet", 56),
 ///    PartitionedFile::new("file3.parquet", 78),
 ///   ])).build();
-/// // create an execution plan from the config
-/// let plan: Arc<dyn ExecutionPlan> =DataSourceExec::from_data_source(ParquetSource::new(config.clone()));
+/// // create an execution plan from the data source that owns the config
+/// let plan: Arc<dyn ExecutionPlan> = DataSourceExec::from_data_source(ParquetSource::new(config));
 /// ```
 #[derive(Clone)]
 pub struct FileScanConfig {
@@ -137,12 +140,10 @@ pub struct FileScanConfig {
     /// [`RuntimeEnv::register_object_store`]: datafusion_execution::runtime_env::RuntimeEnv::register_object_store
     /// [`RuntimeEnv::object_store`]: datafusion_execution::runtime_env::RuntimeEnv::object_store
     pub object_store_url: ObjectStoreUrl,
-    /// Schema before `projection` is applied. It contains the all columns that may
-    /// appear in the files. It does not include table partition columns
-    /// that may be added.
-    /// Note that this is **not** the schema of the physical files.
-    /// This is the schema that the physical file schema will be
-    /// mapped onto, and the schema that the [`DataSourceExec`] will return.
+    /// Schema before `projection` is applied. Contains all columns that may
+    /// appear in the files, excluding table partition columns that may be added.
+    /// This is the logical schema that the physical file schema will be
+    /// mapped onto, and the schema that data sources will expose.
     pub file_schema: SchemaRef,
     /// List of files to be processed, grouped into partitions
     ///
@@ -180,7 +181,7 @@ pub struct FileScanConfig {
     pub file_source_projected_statistics: Statistics,
 }
 
-/// A builder for [`FileScanConfig`]'s.
+/// A builder for [`FileScanConfig`].
 ///
 /// Example:
 ///