diff --git a/Cargo.lock b/Cargo.lock index 68afca862da2..b2a18343da91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1843,6 +1843,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", + "object_store", "parking_lot", "tokio", ] @@ -2006,13 +2007,9 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", - "log", "object_store", - "rand 0.8.5", "regex", "tokio", - "url", ] [[package]] diff --git a/datafusion/catalog/Cargo.toml b/datafusion/catalog/Cargo.toml index 864749411198..4e1555b7827e 100644 --- a/datafusion/catalog/Cargo.toml +++ b/datafusion/catalog/Cargo.toml @@ -42,6 +42,7 @@ datafusion-sql = { workspace = true } futures = { workspace = true } itertools = { workspace = true } log = { workspace = true } +object_store = { workspace = true } parking_lot = { workspace = true } [dev-dependencies] diff --git a/datafusion/catalog/src/lib.rs b/datafusion/catalog/src/lib.rs index a339d4916b8d..cf7dd4b067dd 100644 --- a/datafusion/catalog/src/lib.rs +++ b/datafusion/catalog/src/lib.rs @@ -26,18 +26,9 @@ //! Implementations //! * Information schema: [`information_schema`] //! * Simple memory based catalog: [`MemoryCatalogProviderList`], [`MemoryCatalogProvider`], [`MemorySchemaProvider`] +//! * Listing schema: [`listing_schema`] pub mod memory; -#[deprecated( - since = "46.0.0", - note = "use datafusion_sql::resolve::resolve_table_references" -)] -pub use datafusion_sql::resolve::resolve_table_references; -#[deprecated( - since = "46.0.0", - note = "use datafusion_common::{ResolvedTableReference, TableReference}" -)] -pub use datafusion_sql::{ResolvedTableReference, TableReference}; pub use memory::{ MemoryCatalogProvider, MemoryCatalogProviderList, MemorySchemaProvider, }; @@ -45,6 +36,7 @@ mod r#async; mod catalog; mod dynamic_file; pub mod information_schema; +pub mod listing_schema; mod schema; mod session; mod table; diff --git a/datafusion/core/src/catalog_common/listing_schema.rs b/datafusion/catalog/src/listing_schema.rs similarity index 97% rename from datafusion/core/src/catalog_common/listing_schema.rs rename to datafusion/catalog/src/listing_schema.rs index dc55a07ef82d..cc2c2ee606b3 100644 --- a/datafusion/core/src/catalog_common/listing_schema.rs +++ b/datafusion/catalog/src/listing_schema.rs @@ -22,9 +22,9 @@ use std::collections::HashSet; use std::path::Path; use std::sync::{Arc, Mutex}; -use crate::catalog::{SchemaProvider, TableProvider, TableProviderFactory}; -use crate::execution::context::SessionState; +use crate::{SchemaProvider, TableProvider, TableProviderFactory}; +use crate::Session; use datafusion_common::{ Constraints, DFSchema, DataFusionError, HashMap, TableReference, }; @@ -88,7 +88,7 @@ impl ListingSchemaProvider { } /// Reload table information from ObjectStore - pub async fn refresh(&self, state: &SessionState) -> datafusion_common::Result<()> { + pub async fn refresh(&self, state: &dyn Session) -> datafusion_common::Result<()> { let entries: Vec<_> = self.store.list(Some(&self.path)).try_collect().await?; let base = Path::new(self.path.as_ref()); let mut tables = HashSet::new(); diff --git a/datafusion/core/src/catalog_common/mod.rs b/datafusion/core/src/catalog_common/mod.rs deleted file mode 100644 index 213afb32405e..000000000000 --- a/datafusion/core/src/catalog_common/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Interfaces and default implementations of catalogs and schemas. -//! -//! Implementations -//! * Listing schema: [`listing_schema`] - -pub mod listing_schema; -pub use crate::catalog::{CatalogProvider, CatalogProviderList, SchemaProvider}; diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index ad0993ed43ca..beefca6d572d 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -25,10 +25,10 @@ use std::sync::{Arc, Weak}; use super::options::ReadOptions; use crate::{ + catalog::listing_schema::ListingSchemaProvider, catalog::{ CatalogProvider, CatalogProviderList, TableProvider, TableProviderFactory, }, - catalog_common::listing_schema::ListingSchemaProvider, dataframe::DataFrame, datasource::listing::{ ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index f4b0fd0c125f..002220f93e49 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -436,16 +436,16 @@ impl SessionState { /// Resolve all table references in the SQL statement. Does not include CTE references. /// - /// See [`datafusion_catalog::resolve_table_references`] for more information. + /// See [`datafusion_sql::resolve::resolve_table_references`] for more information. /// - /// [`datafusion_catalog::resolve_table_references`]: datafusion_catalog::resolve_table_references + /// [`datafusion_sql::resolve::resolve_table_references`]: datafusion_sql::resolve::resolve_table_references pub fn resolve_table_references( &self, statement: &Statement, ) -> datafusion_common::Result> { let enable_ident_normalization = self.config.options().sql_parser.enable_ident_normalization; - let (table_refs, _) = datafusion_catalog::resolve_table_references( + let (table_refs, _) = datafusion_sql::resolve::resolve_table_references( statement, enable_ident_normalization, )?; diff --git a/datafusion/core/src/execution/session_state_defaults.rs b/datafusion/core/src/execution/session_state_defaults.rs index b48ef90f2bd5..a241738bd3a4 100644 --- a/datafusion/core/src/execution/session_state_defaults.rs +++ b/datafusion/core/src/execution/session_state_defaults.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. +use crate::catalog::listing_schema::ListingSchemaProvider; use crate::catalog::{CatalogProvider, TableProviderFactory}; -use crate::catalog_common::listing_schema::ListingSchemaProvider; use crate::datasource::file_format::arrow::ArrowFormatFactory; #[cfg(feature = "avro")] use crate::datasource::file_format::avro::AvroFormatFactory; diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rs index b4d5f9740baa..c2ac2f76013d 100644 --- a/datafusion/core/src/lib.rs +++ b/datafusion/core/src/lib.rs @@ -699,7 +699,6 @@ pub const DATAFUSION_VERSION: &str = env!("CARGO_PKG_VERSION"); extern crate core; extern crate sqlparser; -pub mod catalog_common; pub mod dataframe; pub mod datasource; pub mod error; diff --git a/datafusion/datasource-csv/Cargo.toml b/datafusion/datasource-csv/Cargo.toml index 689531758cad..b95c51cbbeff 100644 --- a/datafusion/datasource-csv/Cargo.toml +++ b/datafusion/datasource-csv/Cargo.toml @@ -44,13 +44,9 @@ datafusion-physical-expr = { workspace = true } datafusion-physical-expr-common = { workspace = true } datafusion-physical-plan = { workspace = true } futures = { workspace = true } -itertools = { workspace = true } -log = { workspace = true } object_store = { workspace = true } -rand = { workspace = true } regex = { workspace = true } tokio = { workspace = true } -url = { workspace = true } [lints] workspace = true