diff --git a/embeddings/manticoresearch_text_embeddings.h b/embeddings/manticoresearch_text_embeddings.h index 3a8d5ef3..8241c865 100644 --- a/embeddings/manticoresearch_text_embeddings.h +++ b/embeddings/manticoresearch_text_embeddings.h @@ -23,7 +23,8 @@ using LoadModelFn = TextModelResult(*)(const char*, const char*, uintptr_t, int32_t, - bool); + bool, + int32_t); using FreeModelResultFn = void(*)(TextModelResult); diff --git a/embeddings/src/error_handling_test.rs b/embeddings/src/error_handling_test.rs index 3b2d3b70..b61e5eb6 100644 --- a/embeddings/src/error_handling_test.rs +++ b/embeddings/src/error_handling_test.rs @@ -41,6 +41,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); // Should fail due to empty API key (basic validation) @@ -82,6 +83,7 @@ mod tests { api_url_c.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); // All should fail and have proper error messages @@ -241,6 +243,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); // Should fail due to empty API key (basic validation) @@ -316,6 +319,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); if should_be_valid { @@ -406,6 +410,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); if should_be_valid && model_id.starts_with("openai/") { @@ -469,6 +474,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); // Should fail with invalid API key diff --git a/embeddings/src/ffi.rs b/embeddings/src/ffi.rs index be53847e..302dcaa0 100644 --- a/embeddings/src/ffi.rs +++ b/embeddings/src/ffi.rs @@ -14,6 +14,7 @@ type LoadModelFn = extern "C" fn( usize, // api_url length i32, // api_timeout: timeout in seconds (0 means use default, positive value is timeout in seconds) bool, // use_gpu flag + i32, // max_input_tokens: cap on tokens per input text (0 means the model's own limit) ) -> TextModelResult; type FreeModelResultFn = extern "C" fn(TextModelResult); @@ -62,7 +63,7 @@ pub struct EmbedLib { const VERSION_STR: &[u8] = concat!(env!("EMBEDDINGS_VERSION_STR"), "\0").as_bytes(); const LIB: EmbedLib = EmbedLib { - version: 4usize, + version: 5usize, version_str: VERSION_STR.as_ptr() as *const c_char, load_model: TextModelWrapper::load_model, free_model_result: TextModelWrapper::free_model_result, diff --git a/embeddings/src/integration_test.rs b/embeddings/src/integration_test.rs index 5ac143be..a54bcdca 100644 --- a/embeddings/src/integration_test.rs +++ b/embeddings/src/integration_test.rs @@ -29,6 +29,7 @@ mod tests { api_url: None, api_timeout: None, use_gpu: Some(false), + max_input_tokens: None, }; let result = create_model(options); @@ -58,6 +59,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); if result.model.is_null() { @@ -91,6 +93,7 @@ mod tests { api_url: None, api_timeout: None, use_gpu: Some(false), + max_input_tokens: None, }; let result = create_model(options); @@ -113,6 +116,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); assert!(ffi_result.model.is_null()); @@ -142,6 +146,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); // Should have error due to empty API key (basic validation) @@ -180,6 +185,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); if !model_result.model.is_null() { @@ -268,6 +274,7 @@ mod tests { api_url: None, api_timeout: None, use_gpu: Some(use_gpu), + max_input_tokens: None, }; let result = create_model(options); @@ -316,6 +323,7 @@ mod tests { api_url: None, api_timeout: None, use_gpu: Some(false), + max_input_tokens: None, }; let result = create_model(options); @@ -402,6 +410,7 @@ mod tests { api_url_c.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); // Should fail for all these cases diff --git a/embeddings/src/model/create_model_test.rs b/embeddings/src/model/create_model_test.rs index 4e2dd3be..1b8c90b8 100644 --- a/embeddings/src/model/create_model_test.rs +++ b/embeddings/src/model/create_model_test.rs @@ -9,6 +9,7 @@ fn test_create_model_allows_custom_openai_model_when_custom_api_url_is_set() { api_url: Some("http://localhost:8080/v1/embeddings".to_string()), api_timeout: None, use_gpu: None, + max_input_tokens: None, }); assert!(model.is_ok()); @@ -28,6 +29,7 @@ fn test_create_model_with_custom_url_still_uses_prefixed_jina_as_remote_signal() api_url: Some("http://localhost:8080/v1/embeddings".to_string()), api_timeout: None, use_gpu: None, + max_input_tokens: None, }); assert!(model.is_ok()); @@ -47,6 +49,7 @@ fn test_create_model_supports_explicit_openai_colon_syntax() { api_url: Some("http://localhost:8080/v1/embeddings".to_string()), api_timeout: None, use_gpu: None, + max_input_tokens: None, }); assert!(model.is_ok()); @@ -66,6 +69,7 @@ fn test_create_model_supports_explicit_openai_colon_syntax_with_simple_model() { api_url: Some("http://localhost:8080/v1/embeddings".to_string()), api_timeout: None, use_gpu: None, + max_input_tokens: None, }); assert!(model.is_ok()); diff --git a/embeddings/src/model/ffi_test.rs b/embeddings/src/model/ffi_test.rs index 322012fa..89472e30 100644 --- a/embeddings/src/model/ffi_test.rs +++ b/embeddings/src/model/ffi_test.rs @@ -40,6 +40,7 @@ mod tests { api_url.as_bytes().len(), 0, // Unlimited timeout false, + 0, // max_input_tokens: model's own limit ); if result.model.is_null() { @@ -173,6 +174,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); // Should fail with invalid model @@ -208,6 +210,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); // Should fail with empty API key (basic validation) @@ -243,6 +246,7 @@ mod tests { api_url.as_bytes().len(), 0, // Use default timeout false, + 0, // max_input_tokens: model's own limit ); // Should fail with missing API key @@ -475,6 +479,7 @@ mod tests { api_url: None, api_timeout: None, use_gpu: Some(true), + max_input_tokens: None, }; let options2 = ModelOptions { @@ -484,6 +489,7 @@ mod tests { api_url: None, api_timeout: None, use_gpu: None, + max_input_tokens: None, }; assert_eq!(options1.model_id, "test-model"); diff --git a/embeddings/src/model/local.rs b/embeddings/src/model/local.rs index 429a6fb4..6378f0bf 100644 --- a/embeddings/src/model/local.rs +++ b/embeddings/src/model/local.rs @@ -1118,11 +1118,42 @@ pub enum LocalModel { } impl LocalModel { + /// Loads a local model. `max_input_tokens` caps the tokens taken from each input + /// text; `None` keeps the model's own limit. The cap can only lower the limit. pub fn new( model_id: &str, cache_path: PathBuf, use_gpu: bool, hf_token: Option<&str>, + max_input_tokens: Option, + ) -> Result> { + let mut model = Self::new_uncapped(model_id, cache_path, use_gpu, hf_token)?; + if let Some(cap) = max_input_tokens { + model.cap_max_input_len(cap); + } + Ok(model) + } + + /// Lowers `max_input_len` of the loaded architecture to `cap` (never raises it). + /// All predict paths truncate on that field, so the cap applies everywhere. + fn cap_max_input_len(&mut self, cap: usize) { + if cap == 0 { + return; + } + match self { + LocalModel::Bert(m) => m.max_input_len = m.max_input_len.min(cap), + LocalModel::T5(m) => m.max_input_len = m.max_input_len.min(cap), + LocalModel::Causal(m) => m.max_input_len = m.max_input_len.min(cap), + LocalModel::Quantized(m) => m.max_input_len = m.max_input_len.min(cap), + LocalModel::Onnx(m) => m.max_input_len = m.max_input_len.min(cap), + } + } + + fn new_uncapped( + model_id: &str, + cache_path: PathBuf, + use_gpu: bool, + hf_token: Option<&str>, ) -> Result> { let model_info = build_model_info(cache_path, model_id, "main", hf_token)?; let config = std::fs::read_to_string(&model_info.config_path) diff --git a/embeddings/src/model/local_test.rs b/embeddings/src/model/local_test.rs index 6bd9dad1..787deed1 100644 --- a/embeddings/src/model/local_test.rs +++ b/embeddings/src/model/local_test.rs @@ -23,12 +23,55 @@ mod tests { // Note: These tests require actual model files to run successfully // They are designed to test the structure and error handling + /// MAX_INPUT_TOKENS (manticoresearch#4816): the cap lowers the model's input limit, + /// never raises it, and really truncates what gets embedded. + #[test] + fn test_max_input_tokens_cap() { + let model_id = "sentence-transformers/all-MiniLM-L6-v2"; + let cache_path = test_cache_path(); + + let uncapped = match LocalModel::new(model_id, cache_path.clone(), false, None, None) { + Ok(m) => m, + Err(e) => { + println!("Model not available, skipping: {}", e); + return; + } + }; + let model_limit = uncapped.get_max_input_len(); + assert!(model_limit > 6); + + // A cap below the model limit is applied as-is + let capped = LocalModel::new(model_id, cache_path.clone(), false, None, Some(6)).unwrap(); + assert_eq!(capped.get_max_input_len(), 6); + + // A cap above the model limit never raises it; zero means "no cap" + let too_big = + LocalModel::new(model_id, cache_path.clone(), false, None, Some(100_000)).unwrap(); + assert_eq!(too_big.get_max_input_len(), model_limit); + let zero = LocalModel::new(model_id, cache_path, false, None, Some(0)).unwrap(); + assert_eq!(zero.get_max_input_len(), model_limit); + + // With the cap (6 tokens = [CLS] a red apple on the), a text and the same text + // plus a long tail embed identically + let head = "a red apple on the table"; + let tail: String = (1..=300) + .map(|i| format!(" extra word number {i}")) + .collect(); + let long = format!("{head}{tail}"); + let capped_vecs = capped.predict(&[head, long.as_str()], 0).unwrap(); + assert_eq!(capped_vecs[0], capped_vecs[1]); + + // Without the cap the tail changes the vector (control) + let uncapped_vecs = uncapped.predict(&[head, long.as_str()], 0).unwrap(); + assert_ne!(uncapped_vecs[0], uncapped_vecs[1]); + } + #[test] fn test_local_model_creation_invalid_path() { let model_id = "sentence-transformers/all-MiniLM-L6-v2"; let cache_path = PathBuf::from("/nonexistent/path"); - let result = LocalModel::new(model_id, cache_path, false, None); + let result = LocalModel::new(model_id, cache_path, false, None, None); // Should fail with invalid path assert!(result.is_err()); @@ -43,7 +86,7 @@ mod tests { let model_id = ""; let cache_path = PathBuf::from("/tmp/test_cache"); - let result = LocalModel::new(model_id, cache_path, false, None); + let result = LocalModel::new(model_id, cache_path, false, None, None); // Should fail with empty model ID assert!(result.is_err()); @@ -55,10 +98,10 @@ mod tests { let cache_path = PathBuf::from("/tmp/test_cache"); // Test with GPU enabled (will likely fail without CUDA, but tests the path) - let result_gpu = LocalModel::new(model_id, cache_path.clone(), true, None); + let result_gpu = LocalModel::new(model_id, cache_path.clone(), true, None, None); // Test with GPU disabled - let result_cpu = LocalModel::new(model_id, cache_path, false, None); + let result_cpu = LocalModel::new(model_id, cache_path, false, None, None); // Both should fail without actual model files, but for different reasons if result_gpu.is_err() && result_cpu.is_err() { @@ -147,7 +190,7 @@ mod tests { ]; for model_id in model_ids { - let result = LocalModel::new(model_id, cache_path.clone(), false, None); + let result = LocalModel::new(model_id, cache_path.clone(), false, None, None); // All should fail without actual model files if result.is_err() { @@ -177,7 +220,7 @@ mod tests { ]; for cache_path in cache_paths { - let result = LocalModel::new(model_id, cache_path.clone(), false, None); + let result = LocalModel::new(model_id, cache_path.clone(), false, None, None); // Should handle different path formats gracefully if result.is_err() { @@ -197,7 +240,7 @@ mod tests { let model_id = "sentence-transformers/all-MiniLM-L6-v2"; let cache_path = PathBuf::from("/tmp/test_cache"); - let result = LocalModel::new(model_id, cache_path, false, None); + let result = LocalModel::new(model_id, cache_path, false, None, None); if result.is_err() { let error_str = if let Err(error) = result { @@ -246,7 +289,7 @@ mod tests { // Test both GPU and CPU modes for use_gpu in [true, false] { - let result = LocalModel::new(model_id, cache_path.clone(), use_gpu, None); + let result = LocalModel::new(model_id, cache_path.clone(), use_gpu, None, None); if result.is_err() { let error_str = if let Err(error) = result { @@ -367,7 +410,7 @@ mod tests { let model_id = "sentence-transformers/all-MiniLM-L6-v2"; let cache_path = PathBuf::from(format!("/tmp/test_cache_{}", i)); - let result = LocalModel::new(model_id, cache_path, false, None); + let result = LocalModel::new(model_id, cache_path, false, None, None); // Should handle concurrent access gracefully if result.is_err() { @@ -395,7 +438,7 @@ mod tests { let model_id = "sentence-transformers/all-MiniLM-L6-v2"; let cache_path = PathBuf::from("/tmp/test_cache"); - let result = LocalModel::new(model_id, cache_path, false, None); + let result = LocalModel::new(model_id, cache_path, false, None, None); // Even if creation fails, it should be memory safe if result.is_err() { @@ -416,7 +459,8 @@ mod tests { ]; for sentence in &test_sentences { - let local_model = LocalModel::new(model_id, cache_path.clone(), false, None).unwrap(); + let local_model = + LocalModel::new(model_id, cache_path.clone(), false, None, None).unwrap(); let embedding = local_model.predict(&[sentence], 0).unwrap(); check_embedding_properties(&embedding[0], local_model.get_hidden_size()); } @@ -426,7 +470,7 @@ mod tests { fn test_embedding_consistency() { let model_id = "sentence-transformers/all-MiniLM-L6-v2"; let cache_path = test_cache_path(); - let local_model = LocalModel::new(model_id, cache_path, false, None).unwrap(); + let local_model = LocalModel::new(model_id, cache_path, false, None, None).unwrap(); let sentence = &["This is a test sentence."]; let embedding1 = local_model.predict(sentence, 0).unwrap(); @@ -441,7 +485,7 @@ mod tests { fn test_hidden_size() { let model_id = "sentence-transformers/all-MiniLM-L6-v2"; let cache_path = test_cache_path(); - let local_model = LocalModel::new(model_id, cache_path, false, None).unwrap(); + let local_model = LocalModel::new(model_id, cache_path, false, None, None).unwrap(); assert_eq!(local_model.get_hidden_size(), 384); } @@ -449,7 +493,7 @@ mod tests { fn test_max_input_len() { let model_id = "sentence-transformers/all-MiniLM-L6-v2"; let cache_path = test_cache_path(); - let local_model = LocalModel::new(model_id, cache_path, false, None).unwrap(); + let local_model = LocalModel::new(model_id, cache_path, false, None, None).unwrap(); assert_eq!(local_model.get_max_input_len(), 512); } @@ -459,7 +503,7 @@ mod tests { let model_id = "Qwen/Qwen3-Embedding-0.6B"; let cache_path = test_cache_path(); - let local_model = LocalModel::new(model_id, cache_path.clone(), false, None) + let local_model = LocalModel::new(model_id, cache_path.clone(), false, None, None) .expect("Qwen model should load successfully"); assert_eq!(local_model.get_hidden_size(), 1024); assert_eq!(local_model.get_max_input_len(), 32768); @@ -478,7 +522,7 @@ mod tests { let model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"; let cache_path = test_cache_path(); - let local_model = LocalModel::new(model_id, cache_path.clone(), false, None) + let local_model = LocalModel::new(model_id, cache_path.clone(), false, None, None) .expect("Llama model should load"); let test_text = &["This is a test sentence for Llama embedding model."]; @@ -493,7 +537,7 @@ mod tests { let model_id = "Locutusque/TinyMistral-248M-v2"; let cache_path = test_cache_path(); - let local_model = LocalModel::new(model_id, cache_path.clone(), false, None) + let local_model = LocalModel::new(model_id, cache_path.clone(), false, None, None) .expect("Mistral model should load"); let test_text = &["This is a test sentence for Mistral embedding model."]; let embeddings = local_model.predict(test_text, 0).unwrap(); @@ -506,7 +550,7 @@ mod tests { let model_id = "h2oai/embeddinggemma-300m"; let cache_path = test_cache_path(); - let local_model = LocalModel::new(model_id, cache_path.clone(), false, None) + let local_model = LocalModel::new(model_id, cache_path.clone(), false, None, None) .expect("Gemma model should load"); let test_text = &["This is a test sentence for Gemma embedding model."]; @@ -520,7 +564,7 @@ mod tests { let model_id = "Qwen/Qwen3-Embedding-0.6B"; let cache_path = test_cache_path(); - let result = LocalModel::new(model_id, cache_path.clone(), false, None); + let result = LocalModel::new(model_id, cache_path.clone(), false, None, None); let local_model = match result { Ok(m) => m, @@ -584,7 +628,7 @@ mod tests { let cache_path = test_cache_path(); // Should work with None token for non-gated models - let result = LocalModel::new(model_id, cache_path.clone(), false, None); + let result = LocalModel::new(model_id, cache_path.clone(), false, None, None); if result.is_err() { // If model isn't cached, skip this test println!("Skipping test - model not cached"); @@ -639,7 +683,7 @@ mod tests { let model_id = "ai-forever/FRIDA"; let cache_path = test_cache_path(); - let result = LocalModel::new(model_id, cache_path.clone(), false, None); + let result = LocalModel::new(model_id, cache_path.clone(), false, None, None); let local_model = match result { Ok(m) => m, Err(e) => { @@ -677,7 +721,13 @@ mod tests { // Try to get HF token from environment let hf_token = std::env::var("HF_TOKEN").ok(); - let result = LocalModel::new(model_id, cache_path.clone(), false, hf_token.as_deref()); + let result = LocalModel::new( + model_id, + cache_path.clone(), + false, + hf_token.as_deref(), + None, + ); let local_model = match result { Ok(m) => m, @@ -726,7 +776,7 @@ mod tests { let model_id = "onnx-models/all-MiniLM-L12-v2-onnx"; let cache_path = test_cache_path(); - let result = LocalModel::new(model_id, cache_path.clone(), false, None); + let result = LocalModel::new(model_id, cache_path.clone(), false, None, None); let local_model = match result { Ok(m) => m, Err(e) => { @@ -751,7 +801,7 @@ mod tests { let model_id = "onnx-models/all-MiniLM-L12-v2-onnx"; let cache_path = test_cache_path(); - let result = LocalModel::new(model_id, cache_path, false, None); + let result = LocalModel::new(model_id, cache_path, false, None, None); let local_model = match result { Ok(m) => m, Err(e) => { @@ -774,7 +824,7 @@ mod tests { let model_id = "onnx-models/all-MiniLM-L12-v2-onnx"; let cache_path = test_cache_path(); - let result = LocalModel::new(model_id, cache_path, false, None); + let result = LocalModel::new(model_id, cache_path, false, None, None); let local_model = match result { Ok(m) => m, Err(e) => { @@ -827,6 +877,7 @@ mod tests { cache_path.clone(), false, None, + None, ) { Ok(m) => m, Err(e) => { @@ -841,6 +892,7 @@ mod tests { cache_path, false, None, + None, ) { Ok(m) => m, Err(e) => { diff --git a/embeddings/src/model/mod.rs b/embeddings/src/model/mod.rs index cfa48d72..986fba69 100644 --- a/embeddings/src/model/mod.rs +++ b/embeddings/src/model/mod.rs @@ -46,6 +46,10 @@ pub struct ModelOptions { pub api_url: Option, pub api_timeout: Option, // Timeout in seconds (None means use default: 10 seconds) pub use_gpu: Option, + /// Cap on the number of tokens taken from each input text. None = the model's own + /// context limit (max_position_embeddings), which for long-context models is far + /// beyond what is practical on CPU (see manticoresearch#4816). + pub max_input_tokens: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -198,6 +202,7 @@ pub fn create_model(options: ModelOptions) -> Result> { cache_path, options.use_gpu.unwrap_or(false), hf_token, + options.max_input_tokens, )?; Ok(Model::Local(Box::new(model))) diff --git a/embeddings/src/model/text_model_wrapper.rs b/embeddings/src/model/text_model_wrapper.rs index 396de4a6..f273db82 100644 --- a/embeddings/src/model/text_model_wrapper.rs +++ b/embeddings/src/model/text_model_wrapper.rs @@ -97,6 +97,7 @@ impl TextModelWrapper { api_url_len: usize, api_timeout: i32, // 0 = unlimited, >0 = timeout in seconds use_gpu: bool, + max_input_tokens: i32, // 0 = model's own limit, >0 = cap on tokens per input text ) -> TextModelResult { panic_guard::catch_panic(|| { let name = unsafe { @@ -142,6 +143,11 @@ impl TextModelWrapper { None // Unlimited (no timeout) }, use_gpu: Some(use_gpu), + max_input_tokens: if max_input_tokens > 0 { + Some(max_input_tokens as usize) + } else { + None // Model's own limit + }, }; match create_model(options) { diff --git a/knn/embeddings.cpp b/knn/embeddings.cpp index 9b3a1f48..eddcbb25 100644 --- a/knn/embeddings.cpp +++ b/knn/embeddings.cpp @@ -184,6 +184,7 @@ std::string ToKey ( const ModelSettings_t & tSettings ) sKey += tSettings.m_sAPIUrl; sKey += std::to_string ( tSettings.m_iAPITimeout ); sKey += std::to_string ( tSettings.m_bUseGPU ? 1 : 0 ); + sKey += std::to_string ( tSettings.m_iMaxInputTokens ); return sKey; } @@ -217,7 +218,7 @@ bool TextToEmbeddings_c::Initialize ( std::shared_ptr pLib, std::st auto * pFuncs = m_pLib->GetLibFuncs(); assert(pFuncs); - TextModelResult tResult = pFuncs->load_model ( m_tSettings.m_sModelName.c_str(), m_tSettings.m_sModelName.length(), m_tSettings.m_sCachePath.c_str(), m_tSettings.m_sCachePath.length(), m_tSettings.m_sAPIKey.c_str(), m_tSettings.m_sAPIKey.length(), m_tSettings.m_sAPIUrl.c_str(), m_tSettings.m_sAPIUrl.length(), m_tSettings.m_iAPITimeout, m_tSettings.m_bUseGPU ); + TextModelResult tResult = pFuncs->load_model ( m_tSettings.m_sModelName.c_str(), m_tSettings.m_sModelName.length(), m_tSettings.m_sCachePath.c_str(), m_tSettings.m_sCachePath.length(), m_tSettings.m_sAPIKey.c_str(), m_tSettings.m_sAPIKey.length(), m_tSettings.m_sAPIUrl.c_str(), m_tSettings.m_sAPIUrl.length(), m_tSettings.m_iAPITimeout, m_tSettings.m_bUseGPU, m_tSettings.m_iMaxInputTokens ); if ( tResult.m_szError ) { sError = tResult.m_szError; @@ -312,7 +313,7 @@ knn::EmbeddingsLib_i * LoadEmbeddingsLib ( const std::string & sLibPath, std::st if ( !pLib->Load(sError) ) return nullptr; - const int SUPPORTED_EMBEDDINGS_LIB_VER = 4; + const int SUPPORTED_EMBEDDINGS_LIB_VER = 5; if ( pLib->GetVersion()!=SUPPORTED_EMBEDDINGS_LIB_VER ) { sError = util::FormatStr ( "Unsupported embeddings library version %d (expected %d)", pLib->GetVersion(), SUPPORTED_EMBEDDINGS_LIB_VER ); diff --git a/knn/knn.h b/knn/knn.h index edb43c94..00a44cf7 100644 --- a/knn/knn.h +++ b/knn/knn.h @@ -63,6 +63,7 @@ struct ModelSettings_t std::string m_sAPIUrl; int m_iAPITimeout = 10; // 0 = unlimited, >0 = timeout in seconds (default: 10) bool m_bUseGPU = false; + int m_iMaxInputTokens = 0; // 0 = model's own limit, >0 = cap on tokens taken from each input text }; struct AttrWithSettings_t : public common::SchemaAttr_t, public IndexSettings_t {};