diff --git a/Cargo.lock b/Cargo.lock index 43181baf..126dfeb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aes" version = "0.8.4" @@ -935,6 +941,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.25.2" @@ -983,6 +995,15 @@ dependencies = [ "shlex 2.0.1", ] +[[package]] +name = "cedarwood" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0524a528a6a0288df1863c3c20fe92c301875b4941e7b6c4b394ab08c5a4c55" +dependencies = [ + "smallvec", +] + [[package]] name = "census" version = "0.4.2" @@ -1042,7 +1063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "phf", + "phf 0.12.1", ] [[package]] @@ -1449,6 +1470,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + [[package]] name = "dashmap" version = "6.2.1" @@ -2191,6 +2218,12 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + [[package]] name = "defmt" version = "1.1.1" @@ -2319,6 +2352,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + [[package]] name = "dtype_dispatch" version = "0.2.1" @@ -2378,6 +2417,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -2564,6 +2614,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix 1.1.4", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2827,6 +2887,17 @@ dependencies = [ "foldhash 0.1.5", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -3201,6 +3272,39 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "include-flate" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f173716febb1ad596c16ea5637b5f1790ea32de8e627493ff82bc73b0876ce" +dependencies = [ + "include-flate-codegen", + "include-flate-compress", +] + +[[package]] +name = "include-flate-codegen" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a7875b62a72ad3f3203cdd8950d4cf9947db036030b974b8b37ceae90c8d8c0" +dependencies = [ + "include-flate-compress", + "proc-macro-error3", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "include-flate-compress" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fbb9c5ccb9a5b67b4afa2974c27e5507ea1bf6d22828cef418e4dfaeca51dd" +dependencies = [ + "libflate", + "zstd", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -3306,6 +3410,30 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jieba-macros" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34904340bc65749a9e9a02fcc7f3368e675427c18447b9bbe02df52c15c9a36a" +dependencies = [ + "phf_codegen", +] + +[[package]] +name = "jieba-rs" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb5bdea4dc241d589e179f39d2a778f31490f3370aa2f626223dbd930ebc5c9d" +dependencies = [ + "bytecount", + "cedarwood", + "include-flate", + "jieba-macros", + "phf 0.13.1", + "regex", + "rustc-hash 2.1.3", +] + [[package]] name = "jiff" version = "0.2.34" @@ -3560,6 +3688,30 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libflate" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df" +dependencies = [ + "adler32", + "crc32fast", + "dary_heap", + "libflate_lz77", + "no_std_io2", +] + +[[package]] +name = "libflate_lz77" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd" +dependencies = [ + "hashbrown 0.16.1", + "no_std_io2", + "rle-decode-fast", +] + [[package]] name = "libloading" version = "0.8.9" @@ -3660,6 +3812,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -3765,6 +3926,15 @@ dependencies = [ "log", ] +[[package]] +name = "measure_time" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" +dependencies = [ + "log", +] + [[package]] name = "memchr" version = "2.8.3" @@ -3889,6 +4059,15 @@ version = "6.6.666" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf5a574dadd7941adeaa71823ecba5e28331b8313fb2e1c6a5c7e5981ea53ad6" +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + [[package]] name = "nom" version = "7.1.3" @@ -4380,6 +4559,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-multimap" version = "0.7.3" @@ -4399,6 +4587,15 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "paimon" version = "0.3.0" @@ -4440,6 +4637,7 @@ dependencies = [ "opendal-service-oss", "opendal-service-s3", "orc-rust", + "paimon-ftindex-core", "paimon-mosaic-core", "paimon-vindex-core", "parquet", @@ -4457,7 +4655,7 @@ dependencies = [ "sha2 0.10.9", "snafu 0.9.1", "snap", - "tantivy", + "tantivy 0.22.1", "tempfile", "tokio", "tokio-util", @@ -4509,6 +4707,24 @@ dependencies = [ "uuid", ] +[[package]] +name = "paimon-ftindex-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c9b761618093284c768eacaa2a93c965c697c60ae1d323b55b44de35a59ae12" +dependencies = [ + "levenshtein_automata", + "roaring", + "serde", + "serde_json", + "tantivy 0.26.1", + "tantivy-common 0.11.0", + "tantivy-fst", + "tantivy-jieba", + "tempfile", + "thiserror 1.0.69", +] + [[package]] name = "paimon-integration-tests" version = "0.3.0" @@ -4716,7 +4932,37 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_shared", + "phf_shared 0.12.1", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared 0.13.1", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", ] [[package]] @@ -4728,6 +4974,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -4903,6 +5158,28 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error-attr3" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34e4dd828515431dd6c4a030d26f7eaed7dd4778226e9d2bb968d65ca4ec3d4d" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error3" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee475e440453418ff1335189eddf7101ba502cd818ab7ae04209bc83aa925aa" +dependencies = [ + "proc-macro-error-attr3", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -5609,6 +5886,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + [[package]] name = "roaring" version = "0.11.4" @@ -6171,6 +6454,9 @@ name = "sketches-ddsketch" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea" +dependencies = [ + "serde", +] [[package]] name = "slab" @@ -6486,17 +6772,17 @@ dependencies = [ "census", "crc32fast", "crossbeam-channel", - "downcast-rs", + "downcast-rs 1.2.1", "fastdivide", "fnv", - "fs4", + "fs4 0.8.4", "htmlescape", "itertools 0.12.1", "levenshtein_automata", "log", - "lru", + "lru 0.12.5", "lz4_flex 0.11.6", - "measure_time", + "measure_time 0.8.3", "memmap2", "num_cpus", "once_cell", @@ -6509,13 +6795,13 @@ dependencies = [ "serde_json", "sketches-ddsketch 0.2.2", "smallvec", - "tantivy-bitpacker", - "tantivy-columnar", - "tantivy-common", + "tantivy-bitpacker 0.6.0", + "tantivy-columnar 0.3.0", + "tantivy-common 0.7.0", "tantivy-fst", - "tantivy-query-grammar", - "tantivy-stacker", - "tantivy-tokenizer-api", + "tantivy-query-grammar 0.22.0", + "tantivy-stacker 0.3.0", + "tantivy-tokenizer-api 0.3.0", "tempfile", "thiserror 1.0.69", "time", @@ -6523,6 +6809,59 @@ dependencies = [ "winapi", ] +[[package]] +name = "tantivy" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "bon", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "datasketches", + "downcast-rs 2.0.2", + "fastdivide", + "fnv", + "fs4 0.13.1", + "htmlescape", + "itertools 0.14.0", + "levenshtein_automata", + "log", + "lru 0.16.4", + "lz4_flex 0.13.1", + "measure_time 0.9.0", + "memmap2", + "once_cell", + "oneshot 0.1.13", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash 2.1.3", + "serde", + "serde_json", + "sketches-ddsketch 0.4.0", + "smallvec", + "tantivy-bitpacker 0.10.0", + "tantivy-columnar 0.7.0", + "tantivy-common 0.11.0", + "tantivy-fst", + "tantivy-query-grammar 0.26.0", + "tantivy-stacker 0.7.0", + "tantivy-tokenizer-api 0.7.0", + "tempfile", + "thiserror 2.0.19", + "time", + "typetag", + "uuid", + "winapi", +] + [[package]] name = "tantivy-bitpacker" version = "0.6.0" @@ -6532,20 +6871,45 @@ dependencies = [ "bitpacking", ] +[[package]] +name = "tantivy-bitpacker" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" +dependencies = [ + "bitpacking", +] + [[package]] name = "tantivy-columnar" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" dependencies = [ - "downcast-rs", + "downcast-rs 1.2.1", "fastdivide", "itertools 0.12.1", "serde", - "tantivy-bitpacker", - "tantivy-common", - "tantivy-sstable", - "tantivy-stacker", + "tantivy-bitpacker 0.6.0", + "tantivy-common 0.7.0", + "tantivy-sstable 0.3.0", + "tantivy-stacker 0.3.0", +] + +[[package]] +name = "tantivy-columnar" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" +dependencies = [ + "downcast-rs 2.0.2", + "fastdivide", + "itertools 0.14.0", + "serde", + "tantivy-bitpacker 0.10.0", + "tantivy-common 0.11.0", + "tantivy-sstable 0.7.0", + "tantivy-stacker 0.7.0", ] [[package]] @@ -6556,7 +6920,20 @@ checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" dependencies = [ "async-trait", "byteorder", - "ownedbytes", + "ownedbytes 0.7.0", + "serde", + "time", +] + +[[package]] +name = "tantivy-common" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes 0.9.0", "serde", "time", ] @@ -6572,6 +6949,17 @@ dependencies = [ "utf8-ranges", ] +[[package]] +name = "tantivy-jieba" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3392170e86f1c387170aba7d171a466ffdc98a8b55b006e19ac64b123a7b690a" +dependencies = [ + "jieba-rs", + "lazy_static", + "tantivy-tokenizer-api 0.7.0", +] + [[package]] name = "tantivy-query-grammar" version = "0.22.0" @@ -6581,14 +6969,41 @@ dependencies = [ "nom", ] +[[package]] +name = "tantivy-query-grammar" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" +dependencies = [ + "fnv", + "nom", + "ordered-float 5.3.0", + "serde", + "serde_json", +] + [[package]] name = "tantivy-sstable" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" dependencies = [ - "tantivy-bitpacker", - "tantivy-common", + "tantivy-bitpacker 0.6.0", + "tantivy-common 0.7.0", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-sstable" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" +dependencies = [ + "futures-util", + "itertools 0.14.0", + "tantivy-bitpacker 0.10.0", + "tantivy-common 0.11.0", "tantivy-fst", "zstd", ] @@ -6601,7 +7016,17 @@ checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" dependencies = [ "murmurhash32", "rand_distr", - "tantivy-common", + "tantivy-common 0.7.0", +] + +[[package]] +name = "tantivy-stacker" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" +dependencies = [ + "murmurhash32", + "tantivy-common 0.11.0", ] [[package]] @@ -6613,6 +7038,15 @@ dependencies = [ "serde", ] +[[package]] +name = "tantivy-tokenizer-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" +dependencies = [ + "serde", +] + [[package]] name = "tap" version = "1.0.1" @@ -6703,7 +7137,7 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" dependencies = [ "byteorder", "integer-encoding", - "ordered-float", + "ordered-float 2.10.1", ] [[package]] @@ -6983,12 +7417,42 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/DEPENDENCIES.rust.tsv b/DEPENDENCIES.rust.tsv index 7c66bcb0..166d7ce6 100644 --- a/DEPENDENCIES.rust.tsv +++ b/DEPENDENCIES.rust.tsv @@ -1,5 +1,6 @@ crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 EPL-2.0 ISC LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense Zlib bzip2-1.0.6 zlib-acknowledgement adler2@2.0.1 X X X +adler32@1.2.0 X aes@0.8.4 X X ahash@0.8.12 X X aho-corasick@1.1.4 X X @@ -77,12 +78,14 @@ bon-macros@3.9.3 X X brotli@8.0.4 X X brotli-decompressor@5.0.3 X X bumpalo@3.20.3 X X +bytecount@0.6.9 X X bytemuck@1.25.2 X X X byteorder@1.5.0 X X bytes@1.12.1 X bzip2@0.6.1 X X cbc@0.1.2 X X cc@1.3.0 X X +cedarwood@0.5.0 X census@0.4.2 X cexpr@0.6.0 X X cfg-if@1.0.4 X X @@ -134,6 +137,7 @@ custom-labels@0.4.6 X darling@0.23.0 X darling_core@0.23.0 X darling_macro@0.23.0 X +dary_heap@0.3.9 X X dashmap@6.2.1 X datafusion@54.0.0 X datafusion-catalog@54.0.0 X @@ -169,6 +173,7 @@ datafusion-proto-common@54.0.0 X datafusion-pruning@54.0.0 X datafusion-session@54.0.0 X datafusion-sql@54.0.0 X +datasketches@0.2.0 X der@0.7.10 X X deranged@0.5.8 X X des@0.8.1 X X @@ -179,6 +184,7 @@ displaydoc@0.2.6 X X dlv-list@0.5.2 X X dns-lookup@3.0.1 X X downcast-rs@1.2.1 X X +downcast-rs@2.0.2 X X dtype_dispatch@0.2.1 X dunce@1.0.5 X X X either@1.16.0 X X @@ -186,6 +192,7 @@ encoding_rs@0.8.35 X X X enum-iterator@2.3.0 X enum-iterator-derive@1.5.0 X equivalent@1.0.2 X X +erased-serde@0.4.10 X X errno@0.3.14 X X event-listener@5.4.1 X X event-listener-strategy@0.5.4 X X @@ -206,6 +213,7 @@ foldhash@0.2.0 X foreign-types@0.3.2 X X foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X +fs4@0.13.1 X X fs4@0.8.4 X X fs_extra@1.3.0 X fsst-rs@0.5.11 X @@ -232,6 +240,7 @@ h2@0.4.15 X half@2.7.1 X X hashbrown@0.14.5 X X hashbrown@0.15.5 X X +hashbrown@0.16.1 X X hashbrown@0.17.1 X X hdfs-native@0.13.5 X heck@0.5.0 X X @@ -264,6 +273,9 @@ icu_provider@2.2.0 X ident_case@1.0.1 X X idna@1.1.0 X X idna_adapter@1.2.2 X X +include-flate@0.3.4 X +include-flate-codegen@0.3.4 X +include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X instant@0.1.13 X @@ -275,6 +287,8 @@ itertools@0.12.1 X X itertools@0.13.0 X X itertools@0.14.0 X X itoa@1.0.18 X X +jieba-macros@0.10.3 X +jieba-rs@0.10.3 X jiff@0.2.34 X X jiff-core@0.1.0 X X jiff-tzdb@0.1.8 X X @@ -299,6 +313,8 @@ lexical-write-float@1.0.6 X X lexical-write-integer@1.0.6 X X libbz2-rs-sys@0.2.5 X libc@0.2.186 X X +libflate@2.3.0 X +libflate_lz77@2.3.0 X libloading@0.8.9 X libloading@0.9.0 X liblzma@0.4.7 X X @@ -311,6 +327,7 @@ litemap@0.8.2 X lock_api@0.4.14 X X log@0.4.33 X X lru@0.12.5 X +lru@0.16.4 X lz4_flex@0.11.6 X lz4_flex@0.13.1 X lzokay-native@0.1.0 X @@ -322,6 +339,7 @@ md-5@0.10.6 X X md-5@0.11.0 X X mea@0.6.4 X measure_time@0.8.3 X +measure_time@0.9.0 X memchr@2.8.3 X X memmap2@0.9.11 X X mime@0.3.17 X X @@ -334,6 +352,7 @@ nalgebra@0.33.3 X nalgebra-macros@0.2.2 X native-tls@0.2.18 X X never-say-never@6.6.666 X X X +no_std_io2@0.9.4 X X nom@7.1.3 X nougat@0.2.4 X X X nougat-proc_macros@0.2.4 X X X @@ -372,11 +391,14 @@ openssl-probe@0.2.1 X X openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X +ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.7.0 X +ownedbytes@0.9.0 X paimon@0.3.0 X paimon-c@0.3.0 X paimon-datafusion@0.3.0 X +paimon-ftindex-core@0.1.0 X paimon-integration-tests@0.3.0 X paimon-mosaic-core@0.2.0 X paimon-rest-server@0.3.0 X @@ -394,7 +416,11 @@ pem-rfc7468@0.7.0 X X percent-encoding@2.3.2 X X petgraph@0.8.3 X X phf@0.12.1 X +phf@0.13.1 X +phf_codegen@0.13.1 X +phf_generator@0.13.1 X phf_shared@0.12.1 X +phf_shared@0.13.1 X pin-project@1.1.13 X X pin-project-internal@1.1.13 X X pin-project-lite@0.2.17 X X @@ -414,6 +440,8 @@ ppv-lite86@0.2.21 X X pretty_assertions@1.4.1 X X prettyplease@0.2.37 X X proc-macro-crate@3.5.0 X X +proc-macro-error-attr3@3.0.2 X X +proc-macro-error3@3.0.2 X X proc-macro2@1.0.107 X X prost@0.13.5 X prost@0.14.4 X @@ -464,6 +492,7 @@ reqsign-huaweicloud-obs@3.0.2 X reqsign-tencent-cos@3.0.2 X reqwest@0.12.28 X X reqwest@0.13.4 X X +rle-decode-fast@1.0.3 X X roaring@0.11.4 X X roxmltree@0.21.1 X X rsa@0.9.10 X X @@ -551,14 +580,23 @@ system-configuration@0.7.0 X X system-configuration-sys@0.6.0 X X tagptr@0.2.0 X X tantivy@0.22.1 X +tantivy@0.26.1 X +tantivy-bitpacker@0.10.0 X tantivy-bitpacker@0.6.0 X tantivy-columnar@0.3.0 X +tantivy-columnar@0.7.0 X +tantivy-common@0.11.0 X tantivy-common@0.7.0 X tantivy-fst@0.5.0 X X +tantivy-jieba@0.20.0 X tantivy-query-grammar@0.22.0 X +tantivy-query-grammar@0.26.0 X tantivy-sstable@0.3.0 X +tantivy-sstable@0.7.0 X tantivy-stacker@0.3.0 X +tantivy-stacker@0.7.0 X tantivy-tokenizer-api@0.3.0 X +tantivy-tokenizer-api@0.7.0 X tap@1.0.1 X target-lexicon@0.13.5 X tempfile@3.27.0 X X @@ -592,7 +630,10 @@ try-lock@0.2.5 X twox-hash@2.1.3 X typed-builder@0.19.1 X X typed-builder-macro@0.19.1 X X +typeid@1.0.3 X X typenum@1.20.1 X X +typetag@0.2.23 X X +typetag-impl@0.2.23 X X unicode-ident@1.0.24 X X X unicode-segmentation@1.13.2 X X unicode-width@0.2.2 X X diff --git a/bindings/python/DEPENDENCIES.rust.tsv b/bindings/python/DEPENDENCIES.rust.tsv index 76426cb4..041777ad 100644 --- a/bindings/python/DEPENDENCIES.rust.tsv +++ b/bindings/python/DEPENDENCIES.rust.tsv @@ -1,5 +1,6 @@ crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 EPL-2.0 ISC LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense Zlib bzip2-1.0.6 zlib-acknowledgement adler2@2.0.1 X X X +adler32@1.2.0 X aes@0.8.4 X X ahash@0.8.12 X X aho-corasick@1.1.4 X X @@ -54,12 +55,14 @@ bon-macros@3.9.3 X X brotli@8.0.4 X X brotli-decompressor@5.0.3 X X bumpalo@3.20.3 X X +bytecount@0.6.9 X X bytemuck@1.25.2 X X X byteorder@1.5.0 X X bytes@1.12.1 X bzip2@0.6.1 X X cbc@0.1.2 X X cc@1.3.0 X X +cedarwood@0.5.0 X census@0.4.2 X cfg-if@1.0.4 X X chrono@0.4.45 X X @@ -99,6 +102,7 @@ ctutils@0.4.2 X X darling@0.23.0 X darling_core@0.23.0 X darling_macro@0.23.0 X +dary_heap@0.3.9 X X dashmap@6.2.1 X datafusion@54.0.0 X datafusion-catalog@54.0.0 X @@ -134,6 +138,7 @@ datafusion-proto-common@54.0.0 X datafusion-pruning@54.0.0 X datafusion-session@54.0.0 X datafusion-sql@54.0.0 X +datasketches@0.2.0 X der@0.7.10 X X deranged@0.5.8 X X des@0.8.1 X X @@ -144,10 +149,12 @@ displaydoc@0.2.6 X X dlv-list@0.5.2 X X dns-lookup@3.0.1 X X downcast-rs@1.2.1 X X +downcast-rs@2.0.2 X X dunce@1.0.5 X X X either@1.16.0 X X encoding_rs@0.8.35 X X X equivalent@1.0.2 X X +erased-serde@0.4.10 X X errno@0.3.14 X X fallible-streaming-iterator@0.1.9 X X fastdivide@0.4.2 X X @@ -162,6 +169,7 @@ foldhash@0.2.0 X foreign-types@0.3.2 X X foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X +fs4@0.13.1 X X fs4@0.8.4 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -185,6 +193,7 @@ h2@0.4.15 X half@2.7.1 X X hashbrown@0.14.5 X X hashbrown@0.15.5 X X +hashbrown@0.16.1 X X hashbrown@0.17.1 X X hdfs-native@0.13.5 X heck@0.5.0 X X @@ -216,14 +225,20 @@ icu_provider@2.2.0 X ident_case@1.0.1 X X idna@1.1.0 X X idna_adapter@1.2.2 X X +include-flate@0.3.4 X +include-flate-codegen@0.3.4 X +include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X instant@0.1.13 X integer-encoding@3.0.4 X +inventory@0.3.24 X X ipnet@2.12.0 X X itertools@0.12.1 X X itertools@0.14.0 X X itoa@1.0.18 X X +jieba-macros@0.10.3 X +jieba-rs@0.10.3 X jiff@0.2.34 X X jiff-core@0.1.0 X X jiff-tzdb@0.1.8 X X @@ -244,6 +259,8 @@ lexical-write-float@1.0.6 X X lexical-write-integer@1.0.6 X X libbz2-rs-sys@0.2.5 X libc@0.2.186 X X +libflate@2.3.0 X +libflate_lz77@2.3.0 X libloading@0.9.0 X liblzma@0.4.7 X X liblzma-sys@0.4.7 X X @@ -255,6 +272,7 @@ litemap@0.8.2 X lock_api@0.4.14 X X log@0.4.33 X X lru@0.12.5 X +lru@0.16.4 X lz4_flex@0.11.6 X lz4_flex@0.13.1 X lzokay-native@0.1.0 X @@ -263,6 +281,7 @@ md-5@0.10.6 X X md-5@0.11.0 X X mea@0.6.4 X measure_time@0.8.3 X +measure_time@0.9.0 X memchr@2.8.3 X X memmap2@0.9.11 X X mime@0.3.17 X X @@ -273,6 +292,7 @@ murmurhash32@0.3.1 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X native-tls@0.2.18 X X +no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X num-bigint@0.4.8 X X @@ -305,10 +325,13 @@ openssl-probe@0.2.1 X X openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X +ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.7.0 X +ownedbytes@0.9.0 X paimon@0.3.0 X paimon-datafusion@0.3.0 X +paimon-ftindex-core@0.1.0 X paimon-mosaic-core@0.2.0 X paimon-vindex-core@0.2.0 X parking_lot@0.12.5 X X @@ -321,7 +344,11 @@ pem-rfc7468@0.7.0 X X percent-encoding@2.3.2 X X petgraph@0.8.3 X X phf@0.12.1 X +phf@0.13.1 X +phf_codegen@0.13.1 X +phf_generator@0.13.1 X phf_shared@0.12.1 X +phf_shared@0.13.1 X pin-project@1.1.13 X X pin-project-internal@1.1.13 X X pin-project-lite@0.2.17 X X @@ -338,6 +365,8 @@ ppv-lite86@0.2.21 X X pretty_assertions@1.4.1 X X prettyplease@0.2.37 X X proc-macro-crate@3.5.0 X X +proc-macro-error-attr3@3.0.2 X X +proc-macro-error3@3.0.2 X X proc-macro2@1.0.107 X X prost@0.13.5 X prost@0.14.4 X @@ -384,12 +413,14 @@ reqsign-huaweicloud-obs@3.0.2 X reqsign-tencent-cos@3.0.2 X reqwest@0.12.28 X X reqwest@0.13.4 X X +rle-decode-fast@1.0.3 X X roaring@0.11.4 X X roxmltree@0.21.1 X X rsa@0.9.10 X X rust-ini@0.21.3 X rust-stemmers@1.2.0 X X rustc-hash@1.1.0 X X +rustc-hash@2.1.3 X X rustc_version@0.4.1 X X rustix@0.38.44 X X X rustix@1.1.4 X X X @@ -433,6 +464,7 @@ simd_cesu8@1.2.0 X X simdutf8@0.1.5 X X siphasher@1.0.3 X X sketches-ddsketch@0.2.2 X +sketches-ddsketch@0.4.0 X slab@0.4.12 X smallvec@1.15.2 X X snafu@0.8.9 X X @@ -462,14 +494,23 @@ synstructure@0.13.2 X system-configuration@0.7.0 X X system-configuration-sys@0.6.0 X X tantivy@0.22.1 X +tantivy@0.26.1 X +tantivy-bitpacker@0.10.0 X tantivy-bitpacker@0.6.0 X tantivy-columnar@0.3.0 X +tantivy-columnar@0.7.0 X +tantivy-common@0.11.0 X tantivy-common@0.7.0 X tantivy-fst@0.5.0 X X +tantivy-jieba@0.20.0 X tantivy-query-grammar@0.22.0 X +tantivy-query-grammar@0.26.0 X tantivy-sstable@0.3.0 X +tantivy-sstable@0.7.0 X tantivy-stacker@0.3.0 X +tantivy-stacker@0.7.0 X tantivy-tokenizer-api@0.3.0 X +tantivy-tokenizer-api@0.7.0 X target-lexicon@0.13.5 X tempfile@3.27.0 X X thiserror@1.0.69 X X @@ -501,7 +542,10 @@ try-lock@0.2.5 X twox-hash@2.1.3 X typed-builder@0.19.1 X X typed-builder-macro@0.19.1 X X +typeid@1.0.3 X X typenum@1.20.1 X X +typetag@0.2.23 X X +typetag-impl@0.2.23 X X unicode-ident@1.0.24 X X X unicode-segmentation@1.13.2 X X unicode-width@0.2.2 X X diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv index 7a3621e9..f230597d 100644 --- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv +++ b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv @@ -1,5 +1,6 @@ crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense Zlib bzip2-1.0.6 zlib-acknowledgement adler2@2.0.1 X X X +adler32@1.2.0 X ahash@0.8.12 X X aho-corasick@1.1.4 X X alloc-no-stdlib@2.0.4 X @@ -64,11 +65,13 @@ bon-macros@3.9.3 X X brotli@8.0.4 X X brotli-decompressor@5.0.3 X X bumpalo@3.20.3 X X +bytecount@0.6.9 X X bytemuck@1.25.2 X X X byteorder@1.5.0 X X bytes@1.12.1 X bzip2@0.6.1 X X cc@1.3.0 X X +cedarwood@0.5.0 X census@0.4.2 X cexpr@0.6.0 X X cfg-if@1.0.4 X X @@ -109,6 +112,7 @@ custom-labels@0.4.6 X darling@0.23.0 X darling_core@0.23.0 X darling_macro@0.23.0 X +dary_heap@0.3.9 X X dashmap@6.2.1 X datafusion@54.0.0 X datafusion-catalog@54.0.0 X @@ -141,6 +145,7 @@ datafusion-physical-plan@54.0.0 X datafusion-pruning@54.0.0 X datafusion-session@54.0.0 X datafusion-sql@54.0.0 X +datasketches@0.2.0 X deranged@0.5.8 X X diff@0.1.13 X X digest@0.10.7 X X @@ -148,6 +153,7 @@ digest@0.11.3 X X displaydoc@0.2.6 X X dlv-list@0.5.2 X X downcast-rs@1.2.1 X X +downcast-rs@2.0.2 X X dtype_dispatch@0.2.1 X dunce@1.0.5 X X X either@1.16.0 X X @@ -155,6 +161,7 @@ encoding_rs@0.8.35 X X X enum-iterator@2.3.0 X enum-iterator-derive@1.5.0 X equivalent@1.0.2 X X +erased-serde@0.4.10 X X errno@0.3.14 X X event-listener@5.4.1 X X event-listener-strategy@0.5.4 X X @@ -175,6 +182,7 @@ foldhash@0.2.0 X foreign-types@0.3.2 X X foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X +fs4@0.13.1 X X fs4@0.8.4 X X fs_extra@1.3.0 X fsst-rs@0.5.11 X @@ -198,6 +206,7 @@ h2@0.4.15 X half@2.7.1 X X hashbrown@0.14.5 X X hashbrown@0.15.5 X X +hashbrown@0.16.1 X X hashbrown@0.17.1 X X heck@0.5.0 X X hermit-abi@0.5.2 X X @@ -229,6 +238,9 @@ icu_provider@2.2.0 X ident_case@1.0.1 X X idna@1.1.0 X X idna_adapter@1.2.2 X X +include-flate@0.3.4 X +include-flate-codegen@0.3.4 X +include-flate-compress@0.3.4 X indexmap@2.14.0 X X instant@0.1.13 X integer-encoding@3.0.4 X @@ -238,6 +250,8 @@ itertools@0.12.1 X X itertools@0.13.0 X X itertools@0.14.0 X X itoa@1.0.18 X X +jieba-macros@0.10.3 X +jieba-rs@0.10.3 X jiff@0.2.34 X X jiff-core@0.1.0 X X jiff-tzdb@0.1.8 X X @@ -250,6 +264,7 @@ jobserver@0.1.35 X X js-sys@0.3.103 X X kanal@0.1.1 X lasso@0.7.3 X X +lazy_static@1.5.0 X X lending-iterator@0.1.7 X X X lending-iterator-proc_macros@0.1.7 X X X levenshtein_automata@0.2.1 X @@ -261,6 +276,8 @@ lexical-write-float@1.0.6 X X lexical-write-integer@1.0.6 X X libbz2-rs-sys@0.2.5 X libc@0.2.186 X X +libflate@2.3.0 X +libflate_lz77@2.3.0 X libloading@0.8.9 X libloading@0.9.0 X liblzma@0.4.7 X X @@ -272,6 +289,7 @@ litemap@0.8.2 X lock_api@0.4.14 X X log@0.4.33 X X lru@0.12.5 X +lru@0.16.4 X lz4_flex@0.11.6 X lz4_flex@0.13.1 X lzokay-native@0.1.0 X @@ -282,6 +300,7 @@ md-5@0.10.6 X X md-5@0.11.0 X X mea@0.6.4 X measure_time@0.8.3 X +measure_time@0.9.0 X memchr@2.8.3 X X memmap2@0.9.11 X X mime@0.3.17 X X @@ -294,6 +313,7 @@ nalgebra@0.33.3 X nalgebra-macros@0.2.2 X native-tls@0.2.18 X X never-say-never@6.6.666 X X X +no_std_io2@0.9.4 X X nom@7.1.3 X nougat@0.2.4 X X X nougat-proc_macros@0.2.4 X X X @@ -323,10 +343,13 @@ openssl-probe@0.2.1 X X openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X +ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.7.0 X +ownedbytes@0.9.0 X paimon@0.3.0 X paimon-datafusion@0.3.0 X +paimon-ftindex-core@0.1.0 X paimon-mosaic-core@0.2.0 X paimon-vindex-core@0.2.0 X parking@2.2.1 X X @@ -338,7 +361,11 @@ pco@1.0.2 X percent-encoding@2.3.2 X X petgraph@0.8.3 X X phf@0.12.1 X +phf@0.13.1 X +phf_codegen@0.13.1 X +phf_generator@0.13.1 X phf_shared@0.12.1 X +phf_shared@0.13.1 X pin-project@1.1.13 X X pin-project-internal@1.1.13 X X pin-project-lite@0.2.17 X X @@ -353,6 +380,8 @@ powerfmt@0.2.0 X X ppv-lite86@0.2.21 X X pretty_assertions@1.4.1 X X prettyplease@0.2.37 X X +proc-macro-error-attr3@3.0.2 X X +proc-macro-error3@3.0.2 X X proc-macro2@1.0.107 X X prost@0.13.5 X prost@0.14.4 X @@ -391,6 +420,7 @@ reqsign-core@3.1.0 X reqsign-file-read-tokio@3.0.2 X reqwest@0.12.28 X X reqwest@0.13.4 X X +rle-decode-fast@1.0.3 X X roaring@0.11.4 X X rust-ini@0.21.3 X rust-stemmers@1.2.0 X X @@ -465,14 +495,23 @@ system-configuration@0.7.0 X X system-configuration-sys@0.6.0 X X tagptr@0.2.0 X X tantivy@0.22.1 X +tantivy@0.26.1 X +tantivy-bitpacker@0.10.0 X tantivy-bitpacker@0.6.0 X tantivy-columnar@0.3.0 X +tantivy-columnar@0.7.0 X +tantivy-common@0.11.0 X tantivy-common@0.7.0 X tantivy-fst@0.5.0 X X +tantivy-jieba@0.20.0 X tantivy-query-grammar@0.22.0 X +tantivy-query-grammar@0.26.0 X tantivy-sstable@0.3.0 X +tantivy-sstable@0.7.0 X tantivy-stacker@0.3.0 X +tantivy-stacker@0.7.0 X tantivy-tokenizer-api@0.3.0 X +tantivy-tokenizer-api@0.7.0 X tap@1.0.1 X tempfile@3.27.0 X X termtree@1.0.0 X @@ -502,7 +541,10 @@ try-lock@0.2.5 X twox-hash@2.1.3 X typed-builder@0.19.1 X X typed-builder-macro@0.19.1 X X +typeid@1.0.3 X X typenum@1.20.1 X X +typetag@0.2.23 X X +typetag-impl@0.2.23 X X unicode-ident@1.0.24 X X X unicode-segmentation@1.13.2 X X unicode-width@0.2.2 X X diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml index 5349896c..dee0b630 100644 --- a/crates/paimon/Cargo.toml +++ b/crates/paimon/Cargo.toml @@ -43,7 +43,7 @@ storage-all = [ "storage-gcs", "storage-hdfs", ] -fulltext = ["tantivy", "tempfile"] +fulltext = ["tantivy", "tempfile", "dep:paimon-ftindex-core"] vortex = ["dep:vortex"] storage-memory = ["opendal/services-memory"] @@ -115,6 +115,7 @@ uuid = { version = "1", features = ["v4"] } urlencoding = "2.1" tantivy = { version = "0.22", optional = true } tempfile = { version = "3", optional = true } +paimon-ftindex-core = { version = "0.1.0", optional = true } paimon-mosaic-core = "0.2.0" paimon-vindex-core = "0.2.0" vortex = { version = "0.75.0", features = ["tokio"], optional = true } diff --git a/crates/paimon/DEPENDENCIES.rust.tsv b/crates/paimon/DEPENDENCIES.rust.tsv index b60dd8de..b032f87b 100644 --- a/crates/paimon/DEPENDENCIES.rust.tsv +++ b/crates/paimon/DEPENDENCIES.rust.tsv @@ -1,5 +1,6 @@ crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense Zlib zlib-acknowledgement adler2@2.0.1 X X X +adler32@1.2.0 X aes@0.8.4 X X ahash@0.8.12 X X aho-corasick@1.1.4 X X @@ -61,11 +62,13 @@ bon-macros@3.9.3 X X brotli@8.0.4 X X brotli-decompressor@5.0.3 X X bumpalo@3.20.3 X X +bytecount@0.6.9 X X bytemuck@1.25.2 X X X byteorder@1.5.0 X X bytes@1.12.1 X cbc@0.1.2 X X cc@1.3.0 X X +cedarwood@0.5.0 X census@0.4.2 X cexpr@0.6.0 X X cfg-if@1.0.4 X X @@ -109,7 +112,9 @@ custom-labels@0.4.6 X darling@0.23.0 X darling_core@0.23.0 X darling_macro@0.23.0 X +dary_heap@0.3.9 X X dashmap@6.2.1 X +datasketches@0.2.0 X der@0.7.10 X X deranged@0.5.8 X X des@0.8.1 X X @@ -120,6 +125,7 @@ displaydoc@0.2.6 X X dlv-list@0.5.2 X X dns-lookup@3.0.1 X X downcast-rs@1.2.1 X X +downcast-rs@2.0.2 X X dtype_dispatch@0.2.1 X dunce@1.0.5 X X X either@1.16.0 X X @@ -127,6 +133,7 @@ encoding_rs@0.8.35 X X X enum-iterator@2.3.0 X enum-iterator-derive@1.5.0 X equivalent@1.0.2 X X +erased-serde@0.4.10 X X errno@0.3.14 X X event-listener@5.4.1 X X event-listener-strategy@0.5.4 X X @@ -146,6 +153,7 @@ foldhash@0.2.0 X foreign-types@0.3.2 X X foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X +fs4@0.13.1 X X fs4@0.8.4 X X fs_extra@1.3.0 X fsst-rs@0.5.11 X @@ -172,6 +180,7 @@ h2@0.4.15 X half@2.7.1 X X hashbrown@0.14.5 X X hashbrown@0.15.5 X X +hashbrown@0.16.1 X X hashbrown@0.17.1 X X hdfs-native@0.13.5 X heck@0.5.0 X X @@ -203,6 +212,9 @@ icu_provider@2.2.0 X ident_case@1.0.1 X X idna@1.1.0 X X idna_adapter@1.2.2 X X +include-flate@0.3.4 X +include-flate-codegen@0.3.4 X +include-flate-compress@0.3.4 X indexmap@2.14.0 X X inout@0.1.4 X X instant@0.1.13 X @@ -213,6 +225,8 @@ itertools@0.12.1 X X itertools@0.13.0 X X itertools@0.14.0 X X itoa@1.0.18 X X +jieba-macros@0.10.3 X +jieba-rs@0.10.3 X jiff@0.2.34 X X jiff-core@0.1.0 X X jiff-tzdb@0.1.8 X X @@ -236,6 +250,8 @@ lexical-util@1.0.7 X X lexical-write-float@1.0.6 X X lexical-write-integer@1.0.6 X X libc@0.2.186 X X +libflate@2.3.0 X +libflate_lz77@2.3.0 X libloading@0.8.9 X libloading@0.9.0 X libm@0.2.16 X @@ -246,6 +262,7 @@ litemap@0.8.2 X lock_api@0.4.14 X X log@0.4.33 X X lru@0.12.5 X +lru@0.16.4 X lz4_flex@0.11.6 X lz4_flex@0.13.1 X lzokay-native@0.1.0 X @@ -256,6 +273,7 @@ md-5@0.10.6 X X md-5@0.11.0 X X mea@0.6.4 X measure_time@0.8.3 X +measure_time@0.9.0 X memchr@2.8.3 X X memmap2@0.9.11 X X mime@0.3.17 X X @@ -268,6 +286,7 @@ nalgebra@0.33.3 X nalgebra-macros@0.2.2 X native-tls@0.2.18 X X never-say-never@6.6.666 X X X +no_std_io2@0.9.4 X X nom@7.1.3 X nougat@0.2.4 X X X nougat-proc_macros@0.2.4 X X X @@ -303,9 +322,12 @@ openssl-probe@0.2.1 X X openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X +ordered-float@5.3.0 X ordered-multimap@0.7.3 X ownedbytes@0.7.0 X +ownedbytes@0.9.0 X paimon@0.3.0 X +paimon-ftindex-core@0.1.0 X paimon-mosaic-core@0.2.0 X paimon-vindex-core@0.2.0 X parking@2.2.1 X X @@ -319,7 +341,11 @@ pem@3.0.6 X pem-rfc7468@0.7.0 X X percent-encoding@2.3.2 X X phf@0.12.1 X +phf@0.13.1 X +phf_codegen@0.13.1 X +phf_generator@0.13.1 X phf_shared@0.12.1 X +phf_shared@0.13.1 X pin-project-lite@0.2.17 X X piper@0.2.5 X X pkcs1@0.7.5 X X @@ -336,6 +362,8 @@ powerfmt@0.2.0 X X ppv-lite86@0.2.21 X X pretty_assertions@1.4.1 X X prettyplease@0.2.37 X X +proc-macro-error-attr3@3.0.2 X X +proc-macro-error3@3.0.2 X X proc-macro2@1.0.107 X X prost@0.13.5 X prost@0.14.4 X @@ -377,6 +405,7 @@ reqsign-huaweicloud-obs@3.0.2 X reqsign-tencent-cos@3.0.2 X reqwest@0.12.28 X X reqwest@0.13.4 X X +rle-decode-fast@1.0.3 X X roaring@0.11.4 X X roxmltree@0.21.1 X X rsa@0.9.10 X X @@ -456,14 +485,23 @@ system-configuration@0.7.0 X X system-configuration-sys@0.6.0 X X tagptr@0.2.0 X X tantivy@0.22.1 X +tantivy@0.26.1 X +tantivy-bitpacker@0.10.0 X tantivy-bitpacker@0.6.0 X tantivy-columnar@0.3.0 X +tantivy-columnar@0.7.0 X +tantivy-common@0.11.0 X tantivy-common@0.7.0 X tantivy-fst@0.5.0 X X +tantivy-jieba@0.20.0 X tantivy-query-grammar@0.22.0 X +tantivy-query-grammar@0.26.0 X tantivy-sstable@0.3.0 X +tantivy-sstable@0.7.0 X tantivy-stacker@0.3.0 X +tantivy-stacker@0.7.0 X tantivy-tokenizer-api@0.3.0 X +tantivy-tokenizer-api@0.7.0 X tap@1.0.1 X tempfile@3.27.0 X X termtree@1.0.0 X @@ -492,7 +530,10 @@ try-lock@0.2.5 X twox-hash@2.1.3 X typed-builder@0.19.1 X X typed-builder-macro@0.19.1 X X +typeid@1.0.3 X X typenum@1.20.1 X X +typetag@0.2.23 X X +typetag-impl@0.2.23 X X unicode-ident@1.0.24 X X X unicode-segmentation@1.13.2 X X unicode-width@0.2.2 X X diff --git a/crates/paimon/src/ftindex/mod.rs b/crates/paimon/src/ftindex/mod.rs new file mode 100644 index 00000000..0a56c9f1 --- /dev/null +++ b/crates/paimon/src/ftindex/mod.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Full-text index reader backed by the shared `paimon-ftindex-core` engine +//! (the same native core the Java/Python bindings use), reading the v1 +//! archive format. + +pub mod reader; + +// Re-export the core I/O types that appear in this module's public API so that +// consumers can implement `SeekRead` (e.g. a remote range-reader) or name the +// in-memory reader while depending only on `paimon`, without pulling in +// `paimon-ftindex-core` directly. +pub use paimon_ftindex_core::io::{ReadRequest, SeekRead, SliceReader}; +pub use reader::{BytesReader, FullTextArchiveReader, FullTextHits}; diff --git a/crates/paimon/src/ftindex/reader.rs b/crates/paimon/src/ftindex/reader.rs new file mode 100644 index 00000000..c7dda64d --- /dev/null +++ b/crates/paimon/src/ftindex/reader.rs @@ -0,0 +1,282 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Reader over the `paimon-ftindex-core` v1 archive format. + +use std::io; + +use bytes::Bytes; +use paimon_ftindex_core::io::{ReadRequest, SeekRead}; +use paimon_ftindex_core::{FullTextIndexReader, FullTextSearchResult}; +use roaring::RoaringTreemap; + +use crate::io::InputFile; + +/// Search hits from a full-text archive: parallel row-id and score arrays, +/// ordered best-score-first (as returned by the engine). +#[derive(Debug, Clone)] +pub struct FullTextHits { + pub row_ids: Vec, + pub scores: Vec, +} + +impl From for FullTextHits { + fn from(r: FullTextSearchResult) -> Self { + Self { + row_ids: r.row_ids, + scores: r.scores, + } + } +} + +/// Reader over a single full-text index archive, backed by the shared +/// `paimon-ftindex-core` engine (v1 archive format). +/// +/// Generic over any `SeekRead` implementation, allowing both whole-file +/// (via `BytesReader`/`SliceReader`) and streaming/range-read strategies. +pub struct FullTextArchiveReader { + inner: FullTextIndexReader, +} + +impl FullTextArchiveReader { + /// Open a full-text archive reader over any `SeekRead` implementation. + /// + /// Use this for streaming/range-read strategies (e.g., remote FileIO with + /// bounded concurrency). For convenience when reading whole files into + /// memory, see [`from_input_file`](Self::from_input_file). + pub fn from_seek_read(reader: R) -> crate::Result { + let inner = FullTextIndexReader::open(reader).map_err(map_ft_err)?; + Ok(Self { inner }) + } + + /// Search with a JSON DSL query (see the engine's query spec), returning up + /// to `limit` best-scoring hits. + pub fn search(&self, query_json: &str, limit: usize) -> crate::Result { + self.inner + .search(query_json, limit) + .map(Into::into) + .map_err(map_ft_err) + } + + /// Like [`search`](Self::search) but restricts results to `include_row_ids` + /// (the live-row allow-list). The bitmap is only serialized, never retained, + /// so it is borrowed to spare callers a clone when reusing an allow-list + /// across archives. An empty allow-list admits no rows, so this returns + /// empty hits immediately without running the query. + pub fn search_with_include( + &self, + query_json: &str, + limit: usize, + include_row_ids: &RoaringTreemap, + ) -> crate::Result { + if include_row_ids.is_empty() { + return Ok(FullTextHits { + row_ids: Vec::new(), + scores: Vec::new(), + }); + } + let mut filter_bytes = Vec::with_capacity(include_row_ids.serialized_size()); + include_row_ids + .serialize_into(&mut filter_bytes) + .map_err(|e| crate::Error::UnexpectedError { + message: format!("failed to serialize row-id filter: {e}"), + source: Some(Box::new(e)), + })?; + self.inner + .search_with_roaring_filter(query_json, limit, &filter_bytes) + .map(Into::into) + .map_err(map_ft_err) + } +} + +impl FullTextArchiveReader { + /// Read the whole archive from `input` into memory and open the engine + /// reader over it. The bytes returned by `input.read()` are held directly + /// by a [`BytesReader`], so no second copy is made and peak memory stays at + /// roughly one archive. + /// + /// This is a convenience wrapper over [`from_seek_read`](Self::from_seek_read) + /// for the common whole-file case. For large or remote archives that should + /// not be fully buffered, call [`from_seek_read`](Self::from_seek_read) with + /// a range-reading [`SeekRead`] implementation instead. + pub async fn from_input_file(input: &InputFile) -> crate::Result { + let bytes = input.read().await?; + Self::from_seek_read(BytesReader::new(bytes)) + } +} + +/// A whole-archive [`SeekRead`] backed by an in-memory [`Bytes`] buffer. +/// +/// Unlike the core `SliceReader` (which owns a `Vec`), this holds the +/// `Bytes` returned by `InputFile::read()` directly, so opening a reader from a +/// whole-file read does not copy the archive a second time. +pub struct BytesReader { + data: Bytes, +} + +impl BytesReader { + /// Wrap an already-read archive buffer. + pub fn new(data: Bytes) -> Self { + Self { data } + } +} + +impl SeekRead for BytesReader { + fn pread(&self, ranges: &mut [ReadRequest<'_>]) -> io::Result<()> { + for range in ranges { + let start = usize::try_from(range.pos) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "offset overflow"))?; + let end = start + .checked_add(range.buf.len()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "range overflow"))?; + if end > self.data.len() { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "read past end of archive", + )); + } + range.buf.copy_from_slice(&self.data[start..end]); + } + Ok(()) + } +} + +fn map_ft_err(e: paimon_ftindex_core::FtIndexError) -> crate::Error { + crate::Error::UnexpectedError { + message: format!("full-text index engine error: {e}"), + source: Some(Box::new(e)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::io::FileIOBuilder; + use paimon_ftindex_core::io::PosWriter; + use paimon_ftindex_core::{FullTextIndexConfig, FullTextIndexWriter}; + + // Build a tiny archive in memory using the core writer, return its bytes. + fn build_archive(docs: &[(i64, &str)]) -> Vec { + let mut writer = FullTextIndexWriter::new(FullTextIndexConfig::new()).unwrap(); + for (row_id, text) in docs { + writer.add_document(*row_id, (*text).to_string()).unwrap(); + } + let mut out = PosWriter::new(Vec::::new()); + writer.write(&mut out).unwrap(); + out.into_inner() + } + + async fn archive_input(bytes: Vec) -> crate::io::InputFile { + let file_io = FileIOBuilder::new("memory").build().unwrap(); + let path = "memory:/ft-pr1-roundtrip.archive"; + let output = file_io.new_output(path).unwrap(); + output.write(Bytes::from(bytes)).await.unwrap(); + output.to_input_file() + } + + #[tokio::test] + async fn test_round_trip_search_returns_expected_rows() { + let bytes = build_archive(&[ + (0, "alpha bravo charlie"), + (1, "bravo delta"), + (2, "echo foxtrot"), + ]); + let input = archive_input(bytes).await; + let reader = FullTextArchiveReader::from_input_file(&input) + .await + .unwrap(); + + let hits = reader.search(r#"{"match":{"query":"bravo"}}"#, 10).unwrap(); + let mut ids = hits.row_ids.clone(); + ids.sort_unstable(); + assert_eq!(ids, vec![0, 1]); + assert_eq!(hits.scores.len(), hits.row_ids.len()); + } + + #[tokio::test] + async fn test_search_with_include_restricts_to_allow_list() { + let bytes = build_archive(&[ + (0, "shared token here"), + (1, "shared token here"), + (2, "shared token here"), + ]); + let input = archive_input(bytes).await; + let reader = FullTextArchiveReader::from_input_file(&input) + .await + .unwrap(); + + // All three match, but restrict to row-ids {0, 2}. + let mut include = roaring::RoaringTreemap::new(); + include.insert(0); + include.insert(2); + + let hits = reader + .search_with_include(r#"{"match":{"query":"token"}}"#, 10, &include) + .unwrap(); + let mut ids = hits.row_ids.clone(); + ids.sort_unstable(); + assert_eq!(ids, vec![0, 2]); + } + + #[tokio::test] + async fn test_search_with_include_empty_or_disjoint_returns_no_hits() { + let bytes = build_archive(&[ + (0, "shared token here"), + (1, "shared token here"), + (2, "shared token here"), + ]); + let input = archive_input(bytes).await; + let reader = FullTextArchiveReader::from_input_file(&input) + .await + .unwrap(); + + // Empty allow-list: an empty include-set admits no rows (not all rows). + let empty = roaring::RoaringTreemap::new(); + let hits = reader + .search_with_include(r#"{"match":{"query":"token"}}"#, 10, &empty) + .unwrap(); + assert!( + hits.row_ids.is_empty(), + "empty include-set must admit no rows" + ); + + // Allow-list disjoint from the matches (row 99 never indexed): still nothing. + let mut disjoint = roaring::RoaringTreemap::new(); + disjoint.insert(99); + let hits = reader + .search_with_include(r#"{"match":{"query":"token"}}"#, 10, &disjoint) + .unwrap(); + assert!( + hits.row_ids.is_empty(), + "include-set disjoint from matches must admit no rows" + ); + } + + #[test] + fn test_from_seek_read_opens_archive_directly() { + // Exercise the generic constructor directly (the key API the remote + // FileIO path in #571 depends on), bypassing `from_input_file`. + let bytes = build_archive(&[(0, "alpha bravo"), (1, "bravo charlie")]); + let reader = + FullTextArchiveReader::from_seek_read(BytesReader::new(Bytes::from(bytes))).unwrap(); + + let hits = reader.search(r#"{"match":{"query":"bravo"}}"#, 10).unwrap(); + let mut ids = hits.row_ids.clone(); + ids.sort_unstable(); + assert_eq!(ids, vec![0, 1]); + } +} diff --git a/crates/paimon/src/lib.rs b/crates/paimon/src/lib.rs index ddf207d3..85ad6b51 100644 --- a/crates/paimon/src/lib.rs +++ b/crates/paimon/src/lib.rs @@ -30,6 +30,8 @@ pub mod btree; pub mod catalog; mod deletion_vector; pub mod file_index; +#[cfg(feature = "fulltext")] +pub mod ftindex; pub mod io; pub mod lumina; mod predicate_stats;