Additional logging for HTTP /sql endpoint

.github/workflows/build_deploy_dev.yml

@@ -4,10 +4,13 @@ permissions:
   contents: read
 
 on:
-  pull_request_target:
+  pull_request:
     types: [opened, reopened, synchronize, labeled]
     branches:
       - 'main'
+      - 'develop'
+      - '25-8-2-revert'
+      - 'feat/FQE-1654'
 
 jobs:
 
@@ -30,9 +33,6 @@ jobs:
   get-deploy-labels:
     name: Get Deploy Envs
     runs-on: mdb-dev
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-labels
-      cancel-in-progress: true
     environment:
       name: ${{ github.event.pull_request.head.repo.fork && 'manual-approval' || '' }}
     outputs:
@@ -52,9 +52,6 @@ jobs:
     runs-on: mdb-dev
     needs: [get-deploy-labels]
     if: ${{ needs.get-deploy-labels.outputs.deploy-envs != '[]' }}
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-build
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
         with:
@@ -78,9 +75,6 @@ jobs:
     name: Push Docker Cache
     runs-on: mdb-dev
     needs: [build]
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}-cache
-      cancel-in-progress: true
     steps:
       - uses: actions/checkout@v4
         with:
@@ -120,9 +114,6 @@ jobs:
       fail-fast: false
       matrix:
         deploy-env: ${{ fromJson(needs.get-deploy-labels.outputs.deploy-envs) }}
-    concurrency:
-      group: deploy-${{ matrix.deploy-env }}
-      cancel-in-progress: false
     uses: ./.github/workflows/test_on_deploy.yml
     with:
       git-sha: ${{ github.event.pull_request.head.sha }}

.github/workflows/deploy.yml

@@ -19,7 +19,7 @@ on:
         required: true
       REPO_DISPATCH_PAT_TOKEN:
         required: true
-      MINDSDB_DB_CON:
+      MINDSDB_DB_URI:
         required: true
 
 jobs:
@@ -39,7 +39,7 @@ jobs:
       group: deploy-${{ matrix.deploy-env }}
       cancel-in-progress: false
     env:
-      MINDSDB_DB_CON: ${{ secrets.MINDSDB_DB_CON }}
+      MINDSDB_DB_URI: ${{ secrets.MINDSDB_DB_URI }}
       UV_LINK_MODE: "symlink"
     steps:
       - uses: actions/checkout@v4
@@ -110,4 +110,4 @@ jobs:
         env-name: ${{ matrix.deploy-env }}
         env-url: ${{ vars.ENV_URL }}
         slack-token: ${{ secrets.GH_ACTIONS_SLACK_BOT_TOKEN }}
-        update-message-id: ${{ steps.slack.outputs.ts }}
+        update-message-id: ${{ steps.slack.outputs.ts }}

.github/workflows/matrix_includes.json

@@ -9,11 +9,6 @@
     "python-version": "3.11",
     "runOnBranch": "main"
   },
-  {
-     "runs_on": "windows-latest",
-     "python-version": 3.11,
-     "runOnBranch": "always"
-  },
   {
      "runs_on": "macos-latest",
      "python-version": 3.11,

.github/workflows/test_on_deploy.yml

@@ -84,5 +84,4 @@ jobs:
           make integration_tests_slow
       env:
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        INTERNAL_URL: ${{ vars.INTERNAL_URL }}
-
+        INTERNAL_URL: ${{ vars.MINDSDB_INTERNAL_URL }}

.github/workflows/test_on_push.yml

@@ -5,7 +5,11 @@ permissions:
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - 'main'
+      - 'develop'
+      - '25-8-2-revert'
+      - 'feat/FQE-1654'
   workflow_dispatch:
 
 defaults:

docker/mindsdb.Dockerfile

@@ -8,7 +8,7 @@ WORKDIR /mindsdb
 # This will almost always invalidate the cache for this stage
 COPY . .
 # Find every FILE that is not a requirements file and delete it
-RUN find ./ -type f -not -name "requirements*.txt" -print | xargs rm -f \
+RUN find ./ -type f -not -name "requirements*.txt" -print0 | xargs -0 rm -f \
 # Find every empty directory and delete it
     && find ./ -type d -empty -delete
 # Copy setup.py and everything else used by setup.py

mindsdb/__main__.py

@@ -391,6 +391,7 @@ def start_process(trunc_process_data: TrunkProcessData) -> None:
     logger.info(f"Version: {mindsdb_version}")
     logger.info(f"Configuration file: {config.config_path or 'absent'}")
     logger.info(f"Storage path: {config.paths['root']}")
+    log.log_system_info(logger)
     logger.debug(f"User config: {config.user_config}")
     logger.debug(f"System config: {config.auto_config}")
     logger.debug(f"Env config: {config.env_config}")
@@ -606,6 +607,7 @@ async def gather_apis():
     ioloop.run_until_complete(wait_apis_start())
 
     threading.Thread(target=do_clean_process_marks, name="clean_process_marks").start()
+    threading.Thread(target=log.log_resources_thread, args=(_stop_event,), name="log_resources").start()
 
     ioloop.run_until_complete(gather_apis())
     ioloop.close()

mindsdb/api/http/namespaces/sql.py

@@ -1,5 +1,6 @@
-from http import HTTPStatus
+import time
 import traceback
+from http import HTTPStatus
 
 from flask import request
 from flask_restx import Resource
@@ -18,6 +19,7 @@
 from mindsdb.utilities import log
 from mindsdb.utilities.config import Config
 from mindsdb.utilities.context import context as ctx
+from mindsdb.utilities.functions import mark_process
 
 logger = log.getLogger(__name__)
 
@@ -29,18 +31,16 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
     @ns_conf.doc("query")
-    @api_endpoint_metrics('POST', '/sql/query')
+    @api_endpoint_metrics("POST", "/sql/query")
+    @mark_process(name="http_sql")
     def post(self):
+        start_time = time.time()
         query = request.json["query"]
         context = request.json.get("context", {})
 
         if isinstance(query, str) is False or isinstance(context, dict) is False:
-            return http_error(
-                HTTPStatus.BAD_REQUEST,
-                'Wrong arguments',
-                'Please provide "query" with the request.'
-            )
-        logger.debug(f'Incoming query: {query}')
+            return http_error(HTTPStatus.BAD_REQUEST, "Wrong arguments", 'Please provide "query" with the request. ')
+        logger.debug(f"Incoming query: {query}")
 
         if context.get("profiling") is True:
             profiler.enable()
@@ -50,9 +50,7 @@ def post(self):
         error_text = None
         error_traceback = None
 
-        profiler.set_meta(
-            query=query, api="http", environment=Config().get("environment")
-        )
+        profiler.set_meta(query=query, api="http", environment=Config().get("environment"))
         with profiler.Context("http_query_processing"):
             mysql_proxy = FakeMysqlProxy()
             mysql_proxy.set_context(context)
@@ -107,6 +105,14 @@ def post(self):
             error_text=error_text,
             traceback=error_traceback,
         )
+        end_time = time.time()
+        log_msg = f"SQL processed in {(end_time - start_time):.2f}s ({end_time:.2f}-{start_time:.2f}), result is {query_response['type']}"
+        if query_response["type"] is SQL_RESPONSE_TYPE.TABLE:
+            log_msg += f" ({len(query_response['data'])} rows), "
+        elif query_response["type"] is SQL_RESPONSE_TYPE.ERROR:
+            log_msg += f" ({query_response['error_message']}), "
+        log_msg += f"used handlers {ctx.used_handlers}"
+        logger.info(log_msg)
 
         return query_response, 200
 
@@ -115,7 +121,7 @@ def post(self):
 @ns_conf.param("list_databases", "lists databases of mindsdb")
 class ListDatabases(Resource):
     @ns_conf.doc("list_databases")
-    @api_endpoint_metrics('GET', '/sql/list_databases')
+    @api_endpoint_metrics("GET", "/sql/list_databases")
     def get(self):
         listing_query = "SHOW DATABASES"
         mysql_proxy = FakeMysqlProxy()
@@ -133,15 +139,18 @@ def get(self):
                 listing_query_response = {"type": "ok"}
             elif result.type == SQL_RESPONSE_TYPE.TABLE:
                 listing_query_response = {
-                    "data": [{
-                        "name": db_row[0],
-                        "tables": [
-                            table_row[0]
-                            for table_row in mysql_proxy.process_query(
-                                "SHOW TABLES FROM `{}`".format(db_row[0])
-                            ).result_set.to_lists()
-                        ]
-                    } for db_row in result.result_set.to_lists()]
+                    "data": [
+                        {
+                            "name": db_row[0],
+                            "tables": [
+                                table_row[0]
+                                for table_row in mysql_proxy.process_query(
+                                    "SHOW TABLES FROM `{}`".format(db_row[0])
+                                ).result_set.to_lists()
+                            ],
+                        }
+                        for db_row in result.result_set.to_lists()
+                    ]
                 }
         except Exception as e:
             listing_query_response = {

mindsdb/integrations/handlers/statsforecast_handler/requirements.txt

@@ -1,2 +1,3 @@
 statsforecast==1.6.0
 scipy==1.15.3
+numba >=0.55.0, <=0.61.0

mindsdb/integrations/handlers/statsforecast_handler/requirements_extra.txt

@@ -1,2 +1,3 @@
 statsforecast==1.6.0
 scipy==1.15.3
+numba >=0.55.0, <=0.61.0

mindsdb/interfaces/database/integrations.py

@@ -515,6 +515,7 @@ def get_data_handler(self, name: str, case_sensitive: bool = False, connect=True
         """
         handler = self.handlers_cache.get(name)
         if handler is not None:
+            ctx.used_handlers.add(getattr(handler.__class__, "name", handler.__class__.__name__))
             return handler
 
         integration_record = self._get_integration_record(name, case_sensitive)
@@ -588,6 +589,7 @@ def get_data_handler(self, name: str, case_sensitive: bool = False, connect=True
         if connect:
             self.handlers_cache.set(handler)
 
+        ctx.used_handlers.add(getattr(handler.__class__, "name", handler.__class__.__name__))
         return handler
 
     def reload_handler_module(self, handler_name):

mindsdb/utilities/context.py

@@ -1,33 +1,30 @@
-
 from contextvars import ContextVar
 from typing import Any
 from copy import deepcopy
 
 
 class Context:
-    ''' Thread independent storage
-    '''
-    __slots__ = ('_storage',)
+    """Thread independent storage"""
+
+    __slots__ = ("_storage",)
 
     def __init__(self, storage) -> None:
-        object.__setattr__(self, '_storage', storage)
+        object.__setattr__(self, "_storage", storage)
         self.set_default()
 
     def set_default(self) -> None:
-        self._storage.set({
-            'user_id': None,
-            'company_id': None,
-            'session_id': "",
-            'task_id': None,
-            'user_class': 0,
-            'profiling': {
-                'level': 0,
-                'enabled': False,
-                'pointer': None,
-                'tree': None
-            },
-            'email_confirmed': 0,
-        })
+        self._storage.set(
+            {
+                "user_id": None,
+                "company_id": None,
+                "session_id": "",
+                "task_id": None,
+                "user_class": 0,
+                "profiling": {"level": 0, "enabled": False, "pointer": None, "tree": None},
+                "email_confirmed": 0,
+                "used_handlers": set(),
+            }
+        )
 
     def __getattr__(self, name: str) -> Any:
         storage = self._storage.get({})
@@ -44,7 +41,7 @@ def __delattr__(self, name: str) -> None:
         storage = deepcopy(self._storage.get({}))
         if name not in storage:
             raise AttributeError(name)
-        del storage['name']
+        del storage["name"]
         self._storage.set(storage)
 
     def dump(self) -> dict:
@@ -56,13 +53,13 @@ def load(self, storage: dict) -> None:
 
     def get_metadata(self, **kwargs) -> dict:
         return {
-            'user_id': self.user_id or "",
-            'company_id': self.company_id or "",
-            'session_id': self.session_id,
-            'user_class': self.user_class,
-            **kwargs
+            "user_id": self.user_id or "",
+            "company_id": self.company_id or "",
+            "session_id": self.session_id,
+            "user_class": self.user_class,
+            **kwargs,
         }
 
 
-_context_var = ContextVar('mindsdb.context')
+_context_var = ContextVar("mindsdb.context")
 context = Context(_context_var)