[BUG] Remove hard coded variables and better class extraction from KB

docs/mindsdb_sql/knowledge_bases/insert_data.mdx

@@ -35,14 +35,25 @@ FROM information_schema.queries;
 </Tip>
 
 <Info>
-To speed up the initial loading of a knowledge base with data, run the `INSERT INTO` command with the `kb_no_upsert` flag. It is recommended to use this flag only when the knowledge base is empty, that is, only for the initial data insertion.
+To speed up data insertion, you can use these performance optimization flags:
 
+**Skip duplicate checking (kb_no_upsert)**
 ```sql
 INSERT INTO my_kb
 SELECT *
 FROM table_name
 USING kb_no_upsert = true;
 ```
+This skips all duplicate checking and directly inserts data. Use only when the knowledge base is empty (initial data load).
+
+**Skip existing items (kb_skip_existing)**
+```sql
+INSERT INTO my_kb
+SELECT *
+FROM table_name
+USING kb_skip_existing = true;
+```
+This checks for existing items and skips them entirely, including avoiding embedding calculation for existing content. More efficient than upsert when you only want to insert new items.
 </Info>
 
 <Note>
@@ -86,6 +97,12 @@ Knowledge bases uniquely identify data rows using an ID column, which prevents f
 
 Ensure the `id_column` uniquely identifies each row to avoid unintentional data loss due to duplicate ID skipping.
 
+**Performance optimization for duplicate handling**
+
+For better performance when handling duplicates, you can use:
+- `kb_skip_existing = true`: Checks for existing IDs and skips them completely (no embedding calculation, more efficient)
+- `kb_no_upsert = true`: Skips duplicate checking entirely (fastest, use only for initial load into empty KB)
+
 </Note>
 
 ### Update Existing Data

mindsdb/integrations/libs/vectordatabase_handler.py

@@ -593,6 +593,32 @@ def hybrid_search(
         """
         raise NotImplementedError(f"Hybrid search not supported for VectorStoreHandler {self.name}")
 
+    def check_existing_ids(self, table_name: str, ids: List[str]) -> List[str]:
+        """
+        Check which IDs from the provided list already exist in the table.
+
+        Args:
+            table_name (str): Name of the table to check
+            ids (List[str]): List of IDs to check for existence
+
+        Returns:
+            List[str]: List of IDs that already exist in the table
+        """
+        if not ids:
+            return []
+
+        try:
+            # Query existing IDs
+            df_existing = self.select(
+                table_name,
+                columns=[TableField.ID.value],
+                conditions=[FilterCondition(column=TableField.ID.value, op=FilterOperator.IN, value=ids)],
+            )
+            return list(df_existing[TableField.ID.value]) if not df_existing.empty else []
+        except Exception:
+            # If select fails for any reason, return empty list to be safe
+            return []
+
     def create_index(self, *args, **kwargs):
         """
         Create an index on the specified table.

mindsdb/integrations/utilities/rag/rerankers/base_reranker.py

@@ -13,7 +13,15 @@
 from openai import AsyncOpenAI, AsyncAzureOpenAI
 from pydantic import BaseModel
 
-from mindsdb.integrations.utilities.rag.settings import DEFAULT_RERANKING_MODEL, DEFAULT_LLM_ENDPOINT
+from mindsdb.integrations.utilities.rag.settings import (
+    DEFAULT_RERANKING_MODEL,
+    DEFAULT_LLM_ENDPOINT,
+    DEFAULT_RERANKER_N,
+    DEFAULT_RERANKER_LOGPROBS,
+    DEFAULT_RERANKER_TOP_LOGPROBS,
+    DEFAULT_RERANKER_MAX_TOKENS,
+    DEFAULT_VALID_CLASS_TOKENS,
+)
 from mindsdb.integrations.libs.base import BaseMLEngine
 
 log = logging.getLogger(__name__)
@@ -38,6 +46,11 @@ class BaseLLMReranker(BaseModel, ABC):
     request_timeout: float = 20.0  # Timeout for API requests
     early_stop: bool = True  # Whether to enable early stopping
     early_stop_threshold: float = 0.8  # Confidence threshold for early stopping
+    n: int = DEFAULT_RERANKER_N  # Number of completions to generate
+    logprobs: bool = DEFAULT_RERANKER_LOGPROBS  # Whether to include log probabilities
+    top_logprobs: int = DEFAULT_RERANKER_TOP_LOGPROBS  # Number of top log probabilities to include
+    max_tokens: int = DEFAULT_RERANKER_MAX_TOKENS  # Maximum tokens to generate
+    valid_class_tokens: List[str] = DEFAULT_VALID_CLASS_TOKENS
 
     class Config:
         arbitrary_types_allowed = True
@@ -234,6 +247,28 @@ async def search_relevancy_no_logprob(self, query: str, document: str) -> Any:
         return rerank_data
 
     async def search_relevancy_score(self, query: str, document: str) -> Any:
+        """
+        This method is used to score the relevance of a document to a query.
+
+        Args:
+            query: The query to score the relevance of.
+            document: The document to score the relevance of.
+
+        Returns:
+            A dictionary with the document and the relevance score.
+        """
+
+        log.debug("Start search_relevancy_score")
+        log.debug(f"Reranker query: {query[:5]}")
+        log.debug(f"Reranker document: {document[:50]}")
+        log.debug(f"Reranker model: {self.model}")
+        log.debug(f"Reranker temperature: {self.temperature}")
+        log.debug(f"Reranker n: {self.n}")
+        log.debug(f"Reranker logprobs: {self.logprobs}")
+        log.debug(f"Reranker top_logprobs: {self.top_logprobs}")
+        log.debug(f"Reranker max_tokens: {self.max_tokens}")
+        log.debug(f"Reranker valid_class_tokens: {self.valid_class_tokens}")
+
         response = await self.client.chat.completions.create(
             model=self.model,
             messages=[
@@ -306,17 +341,30 @@ async def search_relevancy_score(self, query: str, document: str) -> Any:
                 },
             ],
             temperature=self.temperature,
-            n=1,
-            logprobs=True,
-            top_logprobs=4,
-            max_tokens=3,
+            n=self.n,
+            logprobs=self.logprobs,
+            top_logprobs=self.top_logprobs,
+            max_tokens=self.max_tokens,
         )
 
         # Extract response and logprobs
         token_logprobs = response.choices[0].logprobs.content
-        # Reconstruct the prediction and extract the top logprobs from the final token (e.g., "1")
-        final_token_logprob = token_logprobs[-1]
-        top_logprobs = final_token_logprob.top_logprobs
+
+        # Find the token that contains the class number
+        # Instead of just taking the last token, search for the actual class number token
+        class_token_logprob = None
+        for token_logprob in reversed(token_logprobs):
+            if token_logprob.token in self.valid_class_tokens:
+                class_token_logprob = token_logprob
+                break
+
+        # If we couldn't find a class token, fall back to the last non-empty token
+        if class_token_logprob is None:
+            log.warning("No class token logprob found, using the last token as fallback")
+            class_token_logprob = token_logprobs[-1]
+
+        top_logprobs = class_token_logprob.top_logprobs
+
         # Create a map of 'class_1' -> probability, using token combinations
         class_probs = {}
         for top_token in top_logprobs:
@@ -337,6 +385,8 @@ async def search_relevancy_score(self, query: str, document: str) -> Any:
                 score = 0.0
 
         rerank_data = {"document": document, "relevance_score": score}
+        log.debug(f"Reranker score: {score}")
+        log.debug("End search_relevancy_score")
         return rerank_data
 
     def get_scores(self, query: str, documents: list[str]):