diff --git a/Cargo.lock b/Cargo.lock index c3b90e1..dfe36bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,9 +169,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2b10dcb159faf30d3f81f6d56c1211a5bea2ca424eabe477648a44b993320e" +checksum = "602268ce9f569f282cedb9a9f6bac569b680af47b9b077d515900c03c5d190da" dependencies = [ "arrow-arith", "arrow-array", @@ -190,9 +190,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "288015089e7931843c80ed4032c5274f02b37bcb720c4a42096d50b390e70372" +checksum = "cd53c6bf277dea91f136ae8e3a5d7041b44b5e489e244e637d00ae302051f56f" dependencies = [ "arrow-array", "arrow-buffer", @@ -204,9 +204,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ca404ea6191e06bf30956394173337fa9c35f445bd447fe6c21ab944e1a23c" +checksum = "e53796e07a6525edaf7dc28b540d477a934aff14af97967ad1d5550878969b9e" dependencies = [ "ahash", "arrow-buffer", @@ -223,9 +223,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.3.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" +checksum = "f2c1a85bb2e94ee10b76531d8bc3ce9b7b4c0d508cabfb17d477f63f2617bd20" dependencies = [ "bytes", "half", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8e372ed52bd4ee88cc1e6c3859aa7ecea204158ac640b10e187936e7e87074" +checksum = "89fb245db6b0e234ed8e15b644edb8664673fefe630575e94e62cd9d489a8a26" dependencies = [ "arrow-array", "arrow-buffer", @@ -257,9 +257,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e4100b729fe656f2e4fb32bc5884f14acf9118d4ad532b7b33c1132e4dce896" +checksum = "d374882fb465a194462527c0c15a93aa19a554cf690a6b77a26b2a02539937a7" dependencies = [ "arrow-array", "arrow-cast", @@ -272,9 +272,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf87f4ff5fc13290aa47e499a8b669a82c5977c6a1fedce22c7f542c1fd5a597" +checksum = "189d210bc4244c715fa3ed9e6e22864673cccb73d5da28c2723fb2e527329b33" dependencies = [ "arrow-buffer", "arrow-schema", @@ -285,9 +285,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3ca63edd2073fcb42ba112f8ae165df1de935627ead6e203d07c99445f2081" +checksum = "7968c2e5210c41f4909b2ef76f6e05e172b99021c2def5edf3cc48fdd39d1d6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -301,9 +301,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a36b2332559d3310ebe3e173f75b29989b4412df4029a26a30cc3f7da0869297" +checksum = "92111dba5bf900f443488e01f00d8c4ddc2f47f5c50039d18120287b580baa22" dependencies = [ "arrow-array", "arrow-buffer", @@ -325,9 +325,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c4e0530272ca755d6814218dffd04425c5b7854b87fa741d5ff848bf50aa39" +checksum = "211136cb253577ee1a6665f741a13136d4e563f64f5093ffd6fb837af90b9495" dependencies = [ "arrow-array", "arrow-buffer", @@ -338,9 +338,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b07f52788744cc71c4628567ad834cadbaeb9f09026ff1d7a4120f69edf7abd3" +checksum = "8e0f20145f9f5ea3fe383e2ba7a7487bf19be36aa9dbf5dd6a1f92f657179663" dependencies = [ "arrow-array", "arrow-buffer", @@ -351,9 +351,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb63203e8e0e54b288d0d8043ca8fa1013820822a27692ef1b78a977d879f2c" +checksum = "1b47e0ca91cc438d2c7879fe95e0bca5329fff28649e30a88c6f760b1faeddcb" dependencies = [ "serde_core", "serde_json", @@ -361,9 +361,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96d8a1c180b44ecf2e66c9a2f2bbcb8b1b6f14e165ce46ac8bde211a363411b" +checksum = "750a7d1dda177735f5e82a314485b6915c7cccdbb278262ac44090f4aba4a325" dependencies = [ "ahash", "arrow-array", @@ -375,9 +375,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8ad6a81add9d3ea30bf8374ee8329992c7fd246ffd8b7e2f48a3cea5aa0cc9a" +checksum = "e1eab1208bc4fe55d768cdc9b9f3d9df5a794cdb3ee2586bf89f9b30dc31ad8c" dependencies = [ "arrow-array", "arrow-buffer", @@ -870,8 +870,6 @@ dependencies = [ [[package]] name = "datafusion" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d12ee9fdc6cdb5898c7691bb994f0ba606c4acc93a2258d78bb9f26ff8158bb3" dependencies = [ "arrow", "arrow-schema", @@ -926,8 +924,6 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "462dc9ef45e5d688aeaae49a7e310587e81b6016b9d03bace5626ad0043e5a9e" dependencies = [ "arrow", "async-trait", @@ -951,8 +947,6 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b96dbf1d728fc321817b744eb5080cdd75312faa6980b338817f68f3caa4208" dependencies = [ "arrow", "async-trait", @@ -974,8 +968,6 @@ dependencies = [ [[package]] name = "datafusion-common" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3237a6ff0d2149af4631290074289cae548c9863c885d821315d54c6673a074a" dependencies = [ "ahash", "apache-avro", @@ -985,6 +977,7 @@ dependencies = [ "half", "hashbrown 0.16.1", "indexmap", + "itertools 0.14.0", "libc", "log", "object_store", @@ -999,8 +992,6 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70b5e34026af55a1bfccb1ef0a763cf1f64e77c696ffcf5a128a278c31236528" dependencies = [ "futures", "log", @@ -1010,8 +1001,6 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b2a6be734cc3785e18bbf2a7f2b22537f6b9fb960d79617775a51568c281842" dependencies = [ "arrow", "async-compression", @@ -1045,8 +1034,6 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1739b9b07c9236389e09c74f770e88aff7055250774e9def7d3f4f56b3dcc7be" dependencies = [ "arrow", "arrow-ipc", @@ -1069,8 +1056,6 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "828088c2fb681cc0e06fb42f541f76c82a0c10278f9fd6334e22c8d1e3574ee7" dependencies = [ "apache-avro", "arrow", @@ -1089,8 +1074,6 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c73bc54b518bbba7c7650299d07d58730293cfba4356f6f428cc94c20b7600" dependencies = [ "arrow", "async-trait", @@ -1112,8 +1095,6 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37812c8494c698c4d889374ecfabbff780f1f26d9ec095dd1bddfc2a8ca12559" dependencies = [ "arrow", "async-trait", @@ -1128,14 +1109,14 @@ dependencies = [ "datafusion-session", "futures", "object_store", + "serde_json", "tokio", + "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2210937ecd9f0e824c397e73f4b5385c97cd1aff43ab2b5836fcfd2d321523fb" dependencies = [ "arrow", "async-trait", @@ -1164,21 +1145,19 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c825f969126bc2ef6a6a02d94b3c07abff871acf4d6dd759ce1255edb7923ce" [[package]] name = "datafusion-execution" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa03ef05a2c2f90dd6c743e3e111078e322f4b395d20d4b4d431a245d79521ae" dependencies = [ "arrow", + "arrow-buffer", "async-trait", "chrono", "dashmap", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr-common", "futures", "log", "object_store", @@ -1191,8 +1170,6 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef33934c1f98ee695cc51192cc5f9ed3a8febee84fdbcd9131bf9d3a9a78276f" dependencies = [ "arrow", "async-trait", @@ -1214,8 +1191,6 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "000c98206e3dd47d2939a94b6c67af4bfa6732dd668ac4fafdbde408fd9134ea" dependencies = [ "arrow", "datafusion-common", @@ -1227,8 +1202,6 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379b01418ab95ca947014066248c22139fe9af9289354de10b445bd000d5d276" dependencies = [ "arrow", "arrow-buffer", @@ -1247,6 +1220,7 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", + "memchr", "num-traits", "rand 0.9.2", "regex", @@ -1258,8 +1232,6 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd00d5454ba4c3f8ebbd04bd6a6a9dc7ced7c56d883f70f2076c188be8459e4c" dependencies = [ "ahash", "arrow", @@ -1273,14 +1245,13 @@ dependencies = [ "datafusion-physical-expr-common", "half", "log", + "num-traits", "paste", ] [[package]] name = "datafusion-functions-aggregate-common" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aec06b380729a87210a4e11f555ec2d729a328142253f8d557b87593622ecc9f" dependencies = [ "ahash", "arrow", @@ -1292,8 +1263,6 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904f48d45e0f1eb7d0eb5c0f80f2b5c6046a85454364a6b16a2e0b46f62e7dff" dependencies = [ "arrow", "arrow-ord", @@ -1307,6 +1276,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", + "hashbrown 0.16.1", "itertools 0.14.0", "log", "paste", @@ -1315,8 +1285,6 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9a0d20e2b887e11bee24f7734d780a2588b925796ac741c3118dd06d5aa77f0" dependencies = [ "arrow", "async-trait", @@ -1331,8 +1299,6 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3414b0a07e39b6979fe3a69c7aa79a9f1369f1d5c8e52146e66058be1b285ee" dependencies = [ "arrow", "datafusion-common", @@ -1349,8 +1315,6 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf2feae63cd4754e31add64ce75cae07d015bce4bb41cd09872f93add32523a" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1359,8 +1323,6 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe888aeb6a095c4bcbe8ac1874c4b9a4c7ffa2ba849db7922683ba20875aaf" dependencies = [ "datafusion-doc", "quote", @@ -1370,8 +1332,6 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a6527c063ae305c11be397a86d8193936f4b84d137fe40bd706dfc178cf733c" dependencies = [ "arrow", "chrono", @@ -1390,8 +1350,6 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb028323dd4efd049dd8a78d78fe81b2b969447b39c51424167f973ac5811d9" dependencies = [ "ahash", "arrow", @@ -1414,8 +1372,6 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78fe0826aef7eab6b4b61533d811234a7a9e5e458331ebbf94152a51fc8ab433" dependencies = [ "arrow", "datafusion-common", @@ -1429,8 +1385,6 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfccd388620734c661bd8b7ca93c44cdd59fecc9b550eea416a78ffcbb29475f" dependencies = [ "ahash", "arrow", @@ -1446,8 +1400,6 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde5fa10e73259a03b705d5fddc136516814ab5f441b939525618a4070f5a059" dependencies = [ "arrow", "datafusion-common", @@ -1465,8 +1417,6 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1098760fb29127c24cc9ade3277051dc73c9ed0ac0131bd7bcd742e0ad7470" dependencies = [ "ahash", "arrow", @@ -1488,6 +1438,7 @@ dependencies = [ "indexmap", "itertools 0.14.0", "log", + "num-traits", "parking_lot", "pin-project-lite", "tokio", @@ -1496,8 +1447,6 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d0fef4201777b52951edec086c21a5b246f3c82621569ddb4a26f488bc38a9" dependencies = [ "arrow", "datafusion-common", @@ -1513,8 +1462,6 @@ dependencies = [ [[package]] name = "datafusion-session" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f71f1e39e8f2acbf1c63b0e93756c2e970a64729dab70ac789587d6237c4fde0" dependencies = [ "async-trait", "datafusion-common", @@ -1527,37 +1474,38 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556c431f5f2259620c8223254c0ef57aa9a85c576d4da0166157260f71eb0e25" dependencies = [ "arrow", "bigdecimal", "chrono", "crc32fast", + "datafusion", "datafusion-catalog", "datafusion-common", "datafusion-execution", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "datafusion-functions-nested", "log", "percent-encoding", "rand 0.9.2", + "serde_json", "sha1", + "sha2", "url", ] [[package]] name = "datafusion-sql" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f44693cfcaeb7a9f12d71d1c576c3a6dc025a12cef209375fa2d16fb3b5670ee" dependencies = [ "arrow", "bigdecimal", "chrono", "datafusion-common", "datafusion-expr", + "datafusion-functions-nested", "indexmap", "log", "recursive", @@ -1568,8 +1516,6 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d388fec80647198ae041d314dd7d9e2305207836ecec3ad48908eac6844cdef" dependencies = [ "arrow", "async-trait", @@ -1584,7 +1530,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "sqllogictest", + "sqllogictest 0.29.1", "sqlparser", "tempfile", "thiserror", @@ -1594,8 +1540,6 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "52.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6042adacd0bd64e56c22f6a7f9ce0ce1793dd367c899d868179d029f110d9215" dependencies = [ "async-recursion", "async-trait", @@ -1609,7 +1553,6 @@ dependencies = [ "substrait", "tokio", "url", - "uuid", ] [[package]] @@ -1631,7 +1574,7 @@ dependencies = [ "parquet-variant-json", "rustyline", "serde_json", - "sqllogictest", + "sqllogictest 0.28.4", "tokio", ] @@ -2566,9 +2509,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.5" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +checksum = "c2858065e55c148d294a9f3aae3b0fa9458edadb41a108397094566f4e3c0dfb" dependencies = [ "async-trait", "bytes", @@ -2640,14 +2583,13 @@ dependencies = [ [[package]] name = "parquet" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6a2926a30477c0b95fea6c28c3072712b139337a242c2cc64817bdc20a8854" +checksum = "3f491d0ef1b510194426ee67ddc18a9b747ef3c42050c19322a2cd2e1666c29b" dependencies = [ "ahash", "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-schema", @@ -2677,9 +2619,9 @@ dependencies = [ [[package]] name = "parquet-variant" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c254fac16af78ad96aa442290cb6504951c4d484fdfcfe58f4588033d30e4c8f" +checksum = "00ba4e5dcbc8ad65882b7337a95c12a0f9cbb6add237c53d93b803b7d7f70f02" dependencies = [ "arrow-schema", "chrono", @@ -2691,9 +2633,9 @@ dependencies = [ [[package]] name = "parquet-variant-compute" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2178772f1c5ad7e5da8b569d986d3f5cbb4a4cee915925f28fdc700dbb2e80cf" +checksum = "9ec4cfb8da15565c8d211b6bc51e8eb481ea65d19132462af3f948b150ac8efe" dependencies = [ "arrow", "arrow-schema", @@ -2702,14 +2644,15 @@ dependencies = [ "indexmap", "parquet-variant", "parquet-variant-json", + "serde_json", "uuid", ] [[package]] name = "parquet-variant-json" -version = "57.2.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a1510daa121c04848368f9c38d0be425b9418c70be610ecc0aa8071738c0ef3" +checksum = "3668ff00a6aeb29d172ba15f9d8fedf1675d79bff7d1916daa333efdeaa13e46" dependencies = [ "arrow-schema", "base64", @@ -3394,11 +3337,36 @@ dependencies = [ "tracing", ] +[[package]] +name = "sqllogictest" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d03b2262a244037b0b510edbd25a8e6c9fb8d73ee0237fc6cc95a54c16f94a82" +dependencies = [ + "async-trait", + "educe", + "fs-err", + "futures", + "glob", + "humantime", + "itertools 0.13.0", + "libtest-mimic", + "md-5", + "owo-colors", + "rand 0.8.5", + "regex", + "similar", + "subst", + "tempfile", + "thiserror", + "tracing", +] + [[package]] name = "sqlparser" -version = "0.59.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" dependencies = [ "log", "recursive", @@ -3407,9 +3375,9 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", @@ -3607,6 +3575,18 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "tokio-util" version = "0.7.18" diff --git a/Cargo.toml b/Cargo.toml index dbe2fce..a34ac37 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,24 +4,23 @@ version = "0.1.0" edition = "2024" [dependencies] -datafusion = "52.1.0" -arrow = "=57.2.0" -arrow-schema = "=57.2.0" -parquet-variant-compute = "=57.2.0" -parquet-variant-json = "=57.2.0" -parquet-variant = "=57.2.0" +datafusion = { path = "../datafusion/datafusion/core/" } +# datafusion = { path = "https://github.com/apache/datafusion.git", branch = "main" } +arrow = "58.0.0" +arrow-schema = "58.0.0" +parquet-variant-compute = "58.0.0" +parquet-variant-json = "58.0.0" +parquet-variant = "58.0.0" [dev-dependencies] anyhow = "1.0.100" -arrow-cast = "=57.2.0" +arrow-cast = "58.0.0" serde_json = "1.0.145" flate2 = "1.0" tokio = { version = "1.0", features = ["rt-multi-thread", "macros"] } -datafusion-sqllogictest = "52.1.0" +datafusion-sqllogictest = { path = "../datafusion/datafusion/sqllogictest/" } sqllogictest = "0.28" indicatif = "0.18" env_logger = "0.11" insta = "1.43.2" rustyline = { version = "14.0", features = ["derive"] } - - diff --git a/examples/cli.rs b/examples/cli.rs index 624d868..8cbc9c8 100644 --- a/examples/cli.rs +++ b/examples/cli.rs @@ -1,12 +1,13 @@ use anyhow::{Context, Result}; use arrow::array::{ArrayRef, RecordBatch, StringArray}; use arrow::datatypes::{DataType, Field, Schema}; +use datafusion::execution::FunctionRegistry; use datafusion::logical_expr::ScalarUDF; use datafusion::prelude::*; use datafusion_variant::{ - CastToVariantUdf, IsVariantNullUdf, JsonToVariantUdf, VariantGetUdf, VariantListConstruct, - VariantListInsert, VariantObjectConstruct, VariantObjectInsert, VariantPretty, - VariantToJsonUdf, + CastToVariantUdf, IsVariantNullUdf, JsonToVariantUdf, VariantExprPlanner, VariantGetUdf, + VariantListConstruct, VariantListInsert, VariantObjectConstruct, VariantObjectInsert, + VariantPretty, VariantToJsonUdf, }; use flate2::read::GzDecoder; use rustyline::error::ReadlineError; @@ -97,7 +98,7 @@ async fn main() -> Result<()> { let ctx = { let setup_start = Instant::now(); - let ctx = SessionContext::new(); + let mut ctx = SessionContext::new(); let schema = Schema::new(vec![Field::new("json_data", DataType::Utf8, false)]); let string_array: ArrayRef = Arc::new(StringArray::from(json_strings)); let batch = RecordBatch::try_new(Arc::new(schema), vec![string_array])?; @@ -118,6 +119,7 @@ async fn main() -> Result<()> { ctx.register_udf(ScalarUDF::new_from_impl(VariantListConstruct::default())); ctx.register_udf(ScalarUDF::new_from_impl(VariantListInsert::default())); ctx.register_udf(ScalarUDF::new_from_impl(VariantObjectInsert::default())); + ctx.register_expr_planner(Arc::new(VariantExprPlanner))?; let setup_duration = setup_start.elapsed(); println!( diff --git a/src/expr_planner.rs b/src/expr_planner.rs new file mode 100644 index 0000000..ebf19b8 --- /dev/null +++ b/src/expr_planner.rs @@ -0,0 +1,36 @@ +use std::sync::Arc; + +use datafusion::{ + common::DFSchema, + error::DataFusionError, + logical_expr::{ + ScalarUDF, + expr::ScalarFunction, + planner::{ExprPlanner, PlannerResult, RawBinaryExpr}, + }, + prelude::Expr, + sql::sqlparser::ast::BinaryOperator, +}; + +use crate::VariantGetUdf; + +#[derive(Debug)] +pub struct VariantExprPlanner; + +impl ExprPlanner for VariantExprPlanner { + fn plan_binary_op( + &self, + expr: RawBinaryExpr, + _schema: &DFSchema, + ) -> Result, DataFusionError> { + match &expr.op { + BinaryOperator::Custom(s) if s == ":" => Ok(PlannerResult::Planned( + Expr::ScalarFunction(ScalarFunction::new_udf( + Arc::new(ScalarUDF::new_from_impl(VariantGetUdf::default())), + vec![expr.left, expr.right], + )), + )), + _ => Ok(PlannerResult::Original(expr)), + } + } +} diff --git a/src/lib.rs b/src/lib.rs index da2a616..529b938 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ mod shared; mod cast_to_variant; +mod expr_planner; mod is_variant_null; mod json_to_variant; mod variant_get; @@ -16,6 +17,7 @@ mod variant_pretty; mod variant_to_json; pub use cast_to_variant::*; +pub use expr_planner::*; pub use is_variant_null::*; pub use json_to_variant::*; pub use variant_get::*; diff --git a/src/variant_get.rs b/src/variant_get.rs index 43e06e9..54ed6a7 100644 --- a/src/variant_get.rs +++ b/src/variant_get.rs @@ -140,7 +140,7 @@ impl ScalarUDFImpl for VariantGetUdf { let res = variant_get( variant_array, - build_get_options(VariantPath::from(variant_path), &type_field), + build_get_options(VariantPath::try_from(variant_path)?, &type_field), )?; ColumnarValue::Array(res) @@ -158,7 +158,7 @@ impl ScalarUDFImpl for VariantGetUdf { let res = variant_get( &variant_array, - build_get_options(VariantPath::from(variant_path), &type_field), + build_get_options(VariantPath::try_from(variant_path)?, &type_field), )?; let scalar = ScalarValue::try_from_array(res.as_ref(), 0)?; @@ -185,7 +185,10 @@ impl ScalarUDFImpl for VariantGetUdf { let res = variant_get( &arr, - build_get_options(VariantPath::from(path.unwrap_or_default()), &type_field), + build_get_options( + VariantPath::try_from(path.unwrap_or_default())?, + &type_field, + ), )?; out.push(res); @@ -208,7 +211,7 @@ impl ScalarUDFImpl for VariantGetUdf { let path = path.unwrap_or_default(); let res = variant_get( &variant_array, - build_get_options(VariantPath::from(path), &type_field), + build_get_options(VariantPath::try_from(path)?, &type_field), )?; out.push(res);