diff --git a/Cargo.lock b/Cargo.lock index ab5236b..b8dd8cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2057,6 +2057,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "daachorse" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2" + [[package]] name = "darling" version = "0.11.0" @@ -3000,7 +3006,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", - "tokenizers 0.22.2", + "tokenizers 0.23.1", "tokio", "tracing", ] @@ -3059,7 +3065,7 @@ dependencies = [ "serde_json", "sysinfo 0.38.4", "thiserror 2.0.18", - "tokenizers 0.22.2", + "tokenizers 0.23.1", "tokio", "tracing", "tracing-subscriber", @@ -8549,6 +8555,39 @@ dependencies = [ "derive_builder", "esaxx-rs", "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.4", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "tokenizers" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112" +dependencies = [ + "ahash 0.8.12", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", "indicatif 0.18.4", "itertools", "log", diff --git a/Cargo.toml b/Cargo.toml index b345b77..2246ec6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,4 +105,4 @@ candle-core = { version = "0.10", default-features = false } candle-nn = { version = "0.10", default-features = false } candle-transformers = { version = "0.10", default-features = false } hf-hub = { version = "0.4", default-features = false, features = ["ureq"] } -tokenizers = { version = "0.22", default-features = false, features = ["onig"] } +tokenizers = { version = "0.23", default-features = false, features = ["onig"] } diff --git a/crates/finima-llm/Cargo.toml b/crates/finima-llm/Cargo.toml index 7b5de8d..c6cccc1 100644 --- a/crates/finima-llm/Cargo.toml +++ b/crates/finima-llm/Cargo.toml @@ -34,7 +34,7 @@ tracing-subscriber = { workspace = true, optional = true } # Candle/mistral.rs backend (in-process inference) mistralrs = { version = "0.8", optional = true, default-features = false } hf-hub = { version = "0.5", optional = true } -tokenizers = { version = "0.22", optional = true } +tokenizers = { version = "0.23", optional = true } [[bin]] name = "download_model"