diff --git a/.github/workflows/build_deploy_dev.yml b/.github/workflows/build_deploy_dev.yml index 8bbdcba80c1..63df83ed8c7 100644 --- a/.github/workflows/build_deploy_dev.yml +++ b/.github/workflows/build_deploy_dev.yml @@ -4,10 +4,13 @@ permissions: contents: read on: - pull_request_target: + pull_request: types: [opened, reopened, synchronize, labeled] branches: - 'main' + - 'develop' + - '25-8-2-revert' + - 'feat/FQE-1654' jobs: @@ -30,9 +33,6 @@ jobs: get-deploy-labels: name: Get Deploy Envs runs-on: mdb-dev - concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-labels - cancel-in-progress: true environment: name: ${{ github.event.pull_request.head.repo.fork && 'manual-approval' || '' }} outputs: @@ -52,9 +52,6 @@ jobs: runs-on: mdb-dev needs: [get-deploy-labels] if: ${{ needs.get-deploy-labels.outputs.deploy-envs != '[]' }} - concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-build - cancel-in-progress: true steps: - uses: actions/checkout@v4 with: @@ -78,9 +75,6 @@ jobs: name: Push Docker Cache runs-on: mdb-dev needs: [build] - concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-cache - cancel-in-progress: true steps: - uses: actions/checkout@v4 with: @@ -120,9 +114,6 @@ jobs: fail-fast: false matrix: deploy-env: ${{ fromJson(needs.get-deploy-labels.outputs.deploy-envs) }} - concurrency: - group: deploy-${{ matrix.deploy-env }} - cancel-in-progress: false uses: ./.github/workflows/test_on_deploy.yml with: git-sha: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2692ea37d40..33b17bbb160 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,7 +19,7 @@ on: required: true REPO_DISPATCH_PAT_TOKEN: required: true - MINDSDB_DB_CON: + MINDSDB_DB_URI: required: true jobs: @@ -39,7 +39,7 @@ jobs: group: deploy-${{ matrix.deploy-env }} cancel-in-progress: false env: - MINDSDB_DB_CON: ${{ secrets.MINDSDB_DB_CON }} + MINDSDB_DB_URI: ${{ secrets.MINDSDB_DB_URI }} UV_LINK_MODE: "symlink" steps: - uses: actions/checkout@v4 @@ -110,4 +110,4 @@ jobs: env-name: ${{ matrix.deploy-env }} env-url: ${{ vars.ENV_URL }} slack-token: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }} - update-message-id: ${{ steps.slack.outputs.ts }} \ No newline at end of file + update-message-id: ${{ steps.slack.outputs.ts }} diff --git a/.github/workflows/matrix_includes.json b/.github/workflows/matrix_includes.json index fbb02ccf939..bc266bf9c17 100644 --- a/.github/workflows/matrix_includes.json +++ b/.github/workflows/matrix_includes.json @@ -9,11 +9,6 @@ "python-version": "3.11", "runOnBranch": "main" }, - { - "runs_on": "windows-latest", - "python-version": 3.11, - "runOnBranch": "always" - }, { "runs_on": "macos-latest", "python-version": 3.11, diff --git a/.github/workflows/test_on_deploy.yml b/.github/workflows/test_on_deploy.yml index eb5c04b5f81..36b36bf5681 100644 --- a/.github/workflows/test_on_deploy.yml +++ b/.github/workflows/test_on_deploy.yml @@ -84,5 +84,4 @@ jobs: make integration_tests_slow env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INTERNAL_URL: ${{ vars.INTERNAL_URL }} - + INTERNAL_URL: ${{ vars.MINDSDB_INTERNAL_URL }} diff --git a/.github/workflows/test_on_push.yml b/.github/workflows/test_on_push.yml index 9d92f07d03b..95b95f84179 100644 --- a/.github/workflows/test_on_push.yml +++ b/.github/workflows/test_on_push.yml @@ -5,7 +5,11 @@ permissions: on: pull_request: - branches: [main] + branches: + - 'main' + - 'develop' + - '25-8-2-revert' + - 'feat/FQE-1654' workflow_dispatch: defaults: diff --git a/docker/mindsdb.Dockerfile b/docker/mindsdb.Dockerfile index c35bb75adad..07c8816873e 100644 --- a/docker/mindsdb.Dockerfile +++ b/docker/mindsdb.Dockerfile @@ -8,7 +8,7 @@ WORKDIR /mindsdb # This will almost always invalidate the cache for this stage COPY . . # Find every FILE that is not a requirements file and delete it -RUN find ./ -type f -not -name "requirements*.txt" -print | xargs rm -f \ +RUN find ./ -type f -not -name "requirements*.txt" -print0 | xargs -0 rm -f \ # Find every empty directory and delete it && find ./ -type d -empty -delete # Copy setup.py and everything else used by setup.py diff --git a/mindsdb/__main__.py b/mindsdb/__main__.py index 07006a9955c..4793e0af8e6 100644 --- a/mindsdb/__main__.py +++ b/mindsdb/__main__.py @@ -391,6 +391,7 @@ def start_process(trunc_process_data: TrunkProcessData) -> None: logger.info(f"Version: {mindsdb_version}") logger.info(f"Configuration file: {config.config_path or 'absent'}") logger.info(f"Storage path: {config.paths['root']}") + log.log_system_info(logger) logger.debug(f"User config: {config.user_config}") logger.debug(f"System config: {config.auto_config}") logger.debug(f"Env config: {config.env_config}") @@ -606,6 +607,7 @@ async def gather_apis(): ioloop.run_until_complete(wait_apis_start()) threading.Thread(target=do_clean_process_marks, name="clean_process_marks").start() + threading.Thread(target=log.log_resources_thread, args=(_stop_event,), name="log_resources").start() ioloop.run_until_complete(gather_apis()) ioloop.close() diff --git a/mindsdb/api/http/namespaces/sql.py b/mindsdb/api/http/namespaces/sql.py index f702e55a851..8566f9f0f1d 100644 --- a/mindsdb/api/http/namespaces/sql.py +++ b/mindsdb/api/http/namespaces/sql.py @@ -1,5 +1,6 @@ -from http import HTTPStatus +import time import traceback +from http import HTTPStatus from flask import request from flask_restx import Resource @@ -18,6 +19,7 @@ from mindsdb.utilities import log from mindsdb.utilities.config import Config from mindsdb.utilities.context import context as ctx +from mindsdb.utilities.functions import mark_process logger = log.getLogger(__name__) @@ -29,18 +31,16 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @ns_conf.doc("query") - @api_endpoint_metrics('POST', '/sql/query') + @api_endpoint_metrics("POST", "/sql/query") + @mark_process(name="http_sql") def post(self): + start_time = time.time() query = request.json["query"] context = request.json.get("context", {}) if isinstance(query, str) is False or isinstance(context, dict) is False: - return http_error( - HTTPStatus.BAD_REQUEST, - 'Wrong arguments', - 'Please provide "query" with the request.' - ) - logger.debug(f'Incoming query: {query}') + return http_error(HTTPStatus.BAD_REQUEST, "Wrong arguments", 'Please provide "query" with the request. ') + logger.debug(f"Incoming query: {query}") if context.get("profiling") is True: profiler.enable() @@ -50,9 +50,7 @@ def post(self): error_text = None error_traceback = None - profiler.set_meta( - query=query, api="http", environment=Config().get("environment") - ) + profiler.set_meta(query=query, api="http", environment=Config().get("environment")) with profiler.Context("http_query_processing"): mysql_proxy = FakeMysqlProxy() mysql_proxy.set_context(context) @@ -107,6 +105,14 @@ def post(self): error_text=error_text, traceback=error_traceback, ) + end_time = time.time() + log_msg = f"SQL processed in {(end_time - start_time):.2f}s ({end_time:.2f}-{start_time:.2f}), result is {query_response['type']}" + if query_response["type"] is SQL_RESPONSE_TYPE.TABLE: + log_msg += f" ({len(query_response['data'])} rows), " + elif query_response["type"] is SQL_RESPONSE_TYPE.ERROR: + log_msg += f" ({query_response['error_message']}), " + log_msg += f"used handlers {ctx.used_handlers}" + logger.info(log_msg) return query_response, 200 @@ -115,7 +121,7 @@ def post(self): @ns_conf.param("list_databases", "lists databases of mindsdb") class ListDatabases(Resource): @ns_conf.doc("list_databases") - @api_endpoint_metrics('GET', '/sql/list_databases') + @api_endpoint_metrics("GET", "/sql/list_databases") def get(self): listing_query = "SHOW DATABASES" mysql_proxy = FakeMysqlProxy() @@ -133,15 +139,18 @@ def get(self): listing_query_response = {"type": "ok"} elif result.type == SQL_RESPONSE_TYPE.TABLE: listing_query_response = { - "data": [{ - "name": db_row[0], - "tables": [ - table_row[0] - for table_row in mysql_proxy.process_query( - "SHOW TABLES FROM `{}`".format(db_row[0]) - ).result_set.to_lists() - ] - } for db_row in result.result_set.to_lists()] + "data": [ + { + "name": db_row[0], + "tables": [ + table_row[0] + for table_row in mysql_proxy.process_query( + "SHOW TABLES FROM `{}`".format(db_row[0]) + ).result_set.to_lists() + ], + } + for db_row in result.result_set.to_lists() + ] } except Exception as e: listing_query_response = { diff --git a/mindsdb/integrations/handlers/statsforecast_handler/requirements.txt b/mindsdb/integrations/handlers/statsforecast_handler/requirements.txt index 1188c719a17..a0a6ca31b91 100644 --- a/mindsdb/integrations/handlers/statsforecast_handler/requirements.txt +++ b/mindsdb/integrations/handlers/statsforecast_handler/requirements.txt @@ -1,2 +1,3 @@ statsforecast==1.6.0 scipy==1.15.3 +numba >=0.55.0, <=0.61.0 diff --git a/mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt b/mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt index 1188c719a17..a0a6ca31b91 100644 --- a/mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt +++ b/mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt @@ -1,2 +1,3 @@ statsforecast==1.6.0 scipy==1.15.3 +numba >=0.55.0, <=0.61.0 diff --git a/mindsdb/interfaces/database/integrations.py b/mindsdb/interfaces/database/integrations.py index 9d5fe8437da..aea38f9831a 100644 --- a/mindsdb/interfaces/database/integrations.py +++ b/mindsdb/interfaces/database/integrations.py @@ -515,6 +515,7 @@ def get_data_handler(self, name: str, case_sensitive: bool = False, connect=True """ handler = self.handlers_cache.get(name) if handler is not None: + ctx.used_handlers.add(getattr(handler.__class__, "name", handler.__class__.__name__)) return handler integration_record = self._get_integration_record(name, case_sensitive) @@ -588,6 +589,7 @@ def get_data_handler(self, name: str, case_sensitive: bool = False, connect=True if connect: self.handlers_cache.set(handler) + ctx.used_handlers.add(getattr(handler.__class__, "name", handler.__class__.__name__)) return handler def reload_handler_module(self, handler_name): diff --git a/mindsdb/utilities/context.py b/mindsdb/utilities/context.py index 6993068d0c3..93329ce94f2 100644 --- a/mindsdb/utilities/context.py +++ b/mindsdb/utilities/context.py @@ -1,33 +1,30 @@ - from contextvars import ContextVar from typing import Any from copy import deepcopy class Context: - ''' Thread independent storage - ''' - __slots__ = ('_storage',) + """Thread independent storage""" + + __slots__ = ("_storage",) def __init__(self, storage) -> None: - object.__setattr__(self, '_storage', storage) + object.__setattr__(self, "_storage", storage) self.set_default() def set_default(self) -> None: - self._storage.set({ - 'user_id': None, - 'company_id': None, - 'session_id': "", - 'task_id': None, - 'user_class': 0, - 'profiling': { - 'level': 0, - 'enabled': False, - 'pointer': None, - 'tree': None - }, - 'email_confirmed': 0, - }) + self._storage.set( + { + "user_id": None, + "company_id": None, + "session_id": "", + "task_id": None, + "user_class": 0, + "profiling": {"level": 0, "enabled": False, "pointer": None, "tree": None}, + "email_confirmed": 0, + "used_handlers": set(), + } + ) def __getattr__(self, name: str) -> Any: storage = self._storage.get({}) @@ -44,7 +41,7 @@ def __delattr__(self, name: str) -> None: storage = deepcopy(self._storage.get({})) if name not in storage: raise AttributeError(name) - del storage['name'] + del storage["name"] self._storage.set(storage) def dump(self) -> dict: @@ -56,13 +53,13 @@ def load(self, storage: dict) -> None: def get_metadata(self, **kwargs) -> dict: return { - 'user_id': self.user_id or "", - 'company_id': self.company_id or "", - 'session_id': self.session_id, - 'user_class': self.user_class, - **kwargs + "user_id": self.user_id or "", + "company_id": self.company_id or "", + "session_id": self.session_id, + "user_class": self.user_class, + **kwargs, } -_context_var = ContextVar('mindsdb.context') +_context_var = ContextVar("mindsdb.context") context = Context(_context_var) diff --git a/mindsdb/utilities/log.py b/mindsdb/utilities/log.py index 83a42445dc1..385a9a16918 100644 --- a/mindsdb/utilities/log.py +++ b/mindsdb/utilities/log.py @@ -1,3 +1,4 @@ +import os import json import logging from logging.config import dictConfig @@ -169,3 +170,340 @@ def getLogger(name=None): """ initialize_logging() return logging.getLogger(name) + + +def log_ram_info(logger: logging.Logger) -> None: + """Log RAM/memory information to the provided logger. + This function logs memory usage information: total, available, used memory in GB and memory + usage percentage. The logging only occurs if the logger is enabled for DEBUG level. + Args: + logger (logging.Logger): The logger instance to use for outputting memory information. + """ + if logger.isEnabledFor(logging.DEBUG) is False: + return + + try: + import psutil + + memory = psutil.virtual_memory() + total_memory_gb = memory.total / (1024**3) + available_memory_gb = memory.available / (1024**3) + used_memory_gb = memory.used / (1024**3) + memory_percent = memory.percent + logger.debug( + f"Memory: {total_memory_gb:.1f}GB total, {available_memory_gb:.1f}GB available, {used_memory_gb:.1f}GB used ({memory_percent:.1f}%)" + ) + except Exception as e: + logger.debug(f"Failed to get memory information: {e}") + + +def log_system_info(logger: logging.Logger) -> None: + """Log detailed system information for debugging purposes. + The function only logs system information (if the logger is configured for DEBUG level): + - Operating system details (OS type, version, distribution, architecture) + - CPU information (processor type, physical and logical core counts) + - Memory information (total, available, used memory in GB and percentage) + - GPU information (NVIDIA, AMD, Intel graphics cards with memory details) + Args: + logger (logging.Logger): The logger instance to use for outputting system information. + Must be configured for DEBUG level to see the output. + Returns: + None + Note: + - For Linux systems, attempts to detect distribution via /etc/os-release, /etc/issue, or lsb_release + - For Windows systems, uses wmic commands to get detailed OS and GPU information + - For macOS systems, uses sw_vers and system_profiler commands + - GPU detection supports NVIDIA (via nvidia-smi), AMD (via rocm-smi), and fallback methods + - All subprocess calls have timeout protection to prevent hanging + - If any system information gathering fails, it logs the error and continues + """ + if logger.isEnabledFor(logging.DEBUG) is False: + return + + try: + import os + import shutil + import psutil + import platform + import subprocess + + # region OS information + os_system = platform.system() + os_release = platform.release() + os_machine = platform.machine() + + os_details = [] + + if os_system == "Linux": + # Try to detect Linux distribution + distro_info = "Unknown Linux" + try: + # Check for /etc/os-release (most modern distributions) + if os.path.exists("/etc/os-release"): + with open("/etc/os-release", "r") as f: + os_release_data = {} + for line in f: + if "=" in line: + key, value = line.strip().split("=", 1) + os_release_data[key] = value.strip('"') + + if "PRETTY_NAME" in os_release_data: + distro_info = os_release_data["PRETTY_NAME"] + elif "NAME" in os_release_data and "VERSION" in os_release_data: + distro_info = f"{os_release_data['NAME']} {os_release_data['VERSION']}" + elif "ID" in os_release_data: + distro_info = os_release_data["ID"].title() + # Fallback to /etc/issue + elif os.path.exists("/etc/issue"): + with open("/etc/issue", "r") as f: + issue_content = f.read().strip() + if issue_content: + distro_info = issue_content.split("\n")[0] + # Fallback to lsb_release + else: + try: + result = subprocess.run(["lsb_release", "-d"], capture_output=True, text=True, timeout=2) + if result.returncode == 0: + distro_info = result.stdout.split(":")[-1].strip() + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + except Exception: + pass + + os_details.append(f"{distro_info} (kernel {os_release})") + + elif os_system == "Windows": + os_name = "Windows" + os_version = "unknown" + try: + result = subprocess.run( + ["wmic", "os", "get", "Caption,Version", "/format:list"], capture_output=True, text=True, timeout=3 + ) + if result.returncode == 0: + windows_info = {} + for line in result.stdout.strip().split("\n"): + if "=" in line: + key, value = line.strip().split("=", 1) + windows_info[key] = value.strip() + + if "Caption" in windows_info and "Version" in windows_info: + os_name = windows_info["Caption"] + os_version = windows_info["Version"] + except Exception: + pass + os_details.append(f"{os_name} {os_release} (version {os_version})") + + elif os_system == "Darwin": # macOS + os_name = "macOS" + os_version = "unknown" + try: + result = subprocess.run( + ["sw_vers", "-productName", "-productVersion"], capture_output=True, text=True, timeout=3 + ) + if result.returncode == 0: + lines = result.stdout.strip().split("\n") + if len(lines) >= 2: + os_name = lines[0].strip() + os_version = lines[1].strip() + except Exception: + pass + os_details.append(f"{os_name} {os_release} (version {os_version})") + else: + os_details.append(f"{os_system} {os_release}") + + os_details.append(f"({os_machine})") + os_info = " ".join(os_details) + logger.debug(f"Operating System: {os_info}") + # endregion + + # region CPU information + cpu_info = platform.processor() + if not cpu_info or cpu_info == "": + cpu_info = platform.machine() + cpu_count = psutil.cpu_count(logical=False) + cpu_count_logical = psutil.cpu_count(logical=True) + logger.debug(f"CPU: {cpu_info} ({cpu_count} physical cores, {cpu_count_logical} logical cores)") + # endregion + + # memory information + log_ram_info(logger) + + # region GPU information + gpu_info = [] + try: + # Check for NVIDIA GPU (works on Linux, Windows, macOS) + nvidia_smi_path = shutil.which("nvidia-smi") + if nvidia_smi_path: + try: + result = subprocess.run( + [nvidia_smi_path, "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"], + capture_output=True, + text=True, + timeout=3, + ) + if result.returncode == 0: + for line in result.stdout.strip().split("\n"): + if line.strip(): + parts = line.split(", ") + if len(parts) >= 2: + gpu_name = parts[0].strip() + gpu_memory = parts[1].strip() + gpu_info.append(f"{gpu_name} ({gpu_memory}MB)") + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + + # Check for AMD GPU (rocm-smi on Linux, wmic on Windows) + if not gpu_info: # Only check AMD if no NVIDIA GPU found + if platform.system() == "Windows": + # Use wmic on Windows to detect AMD GPU + try: + result = subprocess.run( + ["wmic", "path", "win32_VideoController", "get", "name"], + capture_output=True, + text=True, + timeout=3, + ) + if result.returncode == 0: + for line in result.stdout.strip().split("\n"): + line = line.strip() + if line and line != "Name" and "AMD" in line.upper(): + gpu_info.append(line) + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + else: + # Use rocm-smi on Linux/macOS + rocm_smi_path = shutil.which("rocm-smi") + if rocm_smi_path: + try: + result = subprocess.run( + [rocm_smi_path, "--showproductname"], capture_output=True, text=True, timeout=3 + ) + if result.returncode == 0: + for line in result.stdout.strip().split("\n"): + if "Product Name" in line: + gpu_name = line.split(":")[-1].strip() + gpu_info.append(gpu_name) + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + + # Fallback: Try to detect any GPU using platform-specific methods + if not gpu_info: + if platform.system() == "Windows": + try: + # Use wmic to get all video controllers + result = subprocess.run( + ["wmic", "path", "win32_VideoController", "get", "name"], + capture_output=True, + text=True, + timeout=3, + ) + if result.returncode == 0: + for line in result.stdout.strip().split("\n"): + line = line.strip() + if ( + line + and line != "Name" + and any( + keyword in line.upper() + for keyword in ["NVIDIA", "AMD", "INTEL", "RADEON", "GEFORCE"] + ) + ): + gpu_info.append(line) + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + elif platform.system() == "Darwin": # macOS + try: + # Use system_profiler on macOS + result = subprocess.run( + ["system_profiler", "SPDisplaysDataType"], capture_output=True, text=True, timeout=3 + ) + if result.returncode == 0: + for line in result.stdout.strip().split("\n"): + if "Chipset Model:" in line: + gpu_name = line.split(":")[-1].strip() + gpu_info.append(gpu_name) + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + pass + + except Exception: + pass + + if gpu_info: + logger.debug(f"GPU: {', '.join(gpu_info)}") + else: + logger.debug("GPU: Not detected or not supported") + # endregion + + except Exception as e: + logger.debug(f"Failed to get system information: {e}") + + +def log_resources_thread(stop_event): + from mindsdb.utilities.fs import get_tmp_dir + + logger = getLogger(__name__) + i = 0 + timeout = 3 + interval = 60 + while stop_event.wait(timeout=timeout) is False: + i = (i + timeout) % interval + if i != 0: + continue + try: + import psutil + + main_process = psutil.Process(os.getpid()) + children = main_process.children(recursive=True) + + total_memory_info = { + "main_process": { + "pid": main_process.pid, + "name": main_process.name(), + "memory_info": main_process.memory_info(), + "memory_percent": main_process.memory_percent(), + }, + "children": [], + "total_memory": {"rss": 0, "vms": 0, "percent": 0}, + } + + for child in children: + try: + child_info = { + "pid": child.pid, + "name": child.name(), + "memory_info": child.memory_info(), + "memory_percent": child.memory_percent(), + } + total_memory_info["children"].append(child_info) + + total_memory_info["total_memory"]["rss"] += child.memory_info().rss + total_memory_info["total_memory"]["vms"] += child.memory_info().vms + total_memory_info["total_memory"]["percent"] += child.memory_percent() + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + + total_memory_info["total_memory"]["rss"] += main_process.memory_info().rss + total_memory_info["total_memory"]["vms"] += main_process.memory_info().vms + total_memory_info["total_memory"]["percent"] += main_process.memory_percent() + + memory = psutil.virtual_memory() + total_memory_gb = memory.total / (1024**3) + available_memory_gb = memory.available / (1024**3) + used_memory_gb = memory.used / (1024**3) + memory_percent = memory.percent + cpu_usage = psutil.cpu_percent() + + active_http_queries = 0 + p = get_tmp_dir().joinpath("processes/http_sql/") + if p.exists() and p.is_dir(): + for f in p.iterdir(): + active_http_queries += 1 + + logger.info( + f"RAM: {total_memory_gb:.1f}GB total, {available_memory_gb:.1f}GB available, {used_memory_gb:.1f}GB used ({memory_percent:.1f}%)\n" + f"Consumed RAM: {total_memory_info['total_memory']['rss'] / (1024**2):.1f}Mb, {total_memory_info['total_memory']['percent']:.2f}%\n" + f"CPU usage: {cpu_usage}% {interval}s\n" + f"Active HTTP SQL: {active_http_queries}" + ) + except Exception as e: + logger.debug(f"Failed to get memory information: {e}") diff --git a/mindsdb/utilities/render/sqlalchemy_render.py b/mindsdb/utilities/render/sqlalchemy_render.py index 784b4ac0848..64d668951bf 100644 --- a/mindsdb/utilities/render/sqlalchemy_render.py +++ b/mindsdb/utilities/render/sqlalchemy_render.py @@ -98,9 +98,9 @@ def __init__(self, dialect_name): dialect = dialect_name # override dialect's preparer - if hasattr(dialect, "preparer"): + if hasattr(dialect, "preparer") and dialect.preparer.__name__ != "MDBPreparer": - class Preparer(dialect.preparer): + class MDBPreparer(dialect.preparer): def _requires_quotes(self, value: str) -> bool: # check force-quote flag if isinstance(value, AttributedStr): @@ -116,7 +116,7 @@ def _requires_quotes(self, value: str) -> bool: # or (lc_value != value) ) - dialect.preparer = Preparer + dialect.preparer = MDBPreparer # remove double percent signs # https://docs.sqlalchemy.org/en/14/faq/sqlexpressions.html#why-are-percent-signs-being-doubled-up-when-stringifying-sql-statements diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 2e49ab96996..530721716df 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -17,7 +17,7 @@ walrus==0.9.3 flask-compress >= 1.0.0 appdirs >= 1.0.0 mindsdb-sql-parser ~= 0.11.3 -pydantic == 2.9.2 +pydantic == 2.11.4 mindsdb-evaluator == 0.0.18 duckdb ~= 1.3.2 requests == 2.32.4 @@ -70,4 +70,4 @@ typing-extensions==4.13.2 python-dotenv==1.1.0 jwcrypto==1.5.6 pyjwt==2.10.1 -pydantic_core>=2.23.2 +pydantic_core>=2.33.2 diff --git a/tests/integration/flows/test_knowledge_base.py b/tests/integration/flows/test_knowledge_base.py index 7da2ccd3da5..43a96ff7e8e 100644 --- a/tests/integration/flows/test_knowledge_base.py +++ b/tests/integration/flows/test_knowledge_base.py @@ -181,7 +181,7 @@ def create_kb(self, name, storage, embedding_model, reranking_model=None, params """) -class TestKB(KBTestBase): +class Disable_TestKB(KBTestBase): @pytest.mark.parametrize("storage, embedding_model", get_configurations()) def test_base_syntax(self, storage, embedding_model): self.create_kb("test_kb_crm", storage, embedding_model) diff --git a/tests/integration/flows/test_mysql_api.py b/tests/integration/flows/test_mysql_api.py index 8d81d53c50a..74582ec1769 100644 --- a/tests/integration/flows/test_mysql_api.py +++ b/tests/integration/flows/test_mysql_api.py @@ -352,6 +352,10 @@ def test_response_types(self, use_binary, table_name): assert abs(row[column_name] - expected_values[column_name]) < 1e-5, ( f"Expected value {expected_values[column_name]} for column {column_name}, but got {row[column_name]}, use_binary={self.use_binary}, table_name={table_name}" ) + elif column_name in ("t_json", "t_jsonb"): + assert json.loads(row[column_name]) == json.loads(expected_values[column_name]), ( + f"Expected value {expected_values[column_name]} for column {column_name}, but got {row[column_name]}, use_binary={self.use_binary}, table_name={table_name}" + ) else: assert row[column_name] == expected_values[column_name], ( f"Expected value {expected_values[column_name]} for column {column_name}, but got {row[column_name]}, use_binary={self.use_binary}, table_name={table_name}" diff --git a/tests/scripts/check_requirements.py b/tests/scripts/check_requirements.py index ca113b1af6c..c801d3b4a21 100644 --- a/tests/scripts/check_requirements.py +++ b/tests/scripts/check_requirements.py @@ -101,6 +101,9 @@ def get_requirements_from_file(path): CHROMADB_EP002_IGNORE_HANDLER_DEPS = ["onnxruntime"] +# upper version of numba is fixed in statsforecast handler to prevent installing numba==0.62.0 (its import fails on windows) +STATSFORECAST_EP002_IGNORE_HANDLER_DEPS = ["numba"] + # The `pyarrow` package is used only if it is installed. # The handler can work without it. SNOWFLAKE_DEP003_IGNORE_HANDLER_DEPS = ["pyarrow"] @@ -119,6 +122,7 @@ def get_requirements_from_file(path): + LANGCHAIN_EMBEDDING_DEP002_IGNORE_HANDLER_DEPS + OPENAI_DEP002_IGNORE_HANDLER_DEPS + CHROMADB_EP002_IGNORE_HANDLER_DEPS + + STATSFORECAST_EP002_IGNORE_HANDLER_DEPS ) )