Python bindings built on top of C-API

.github/workflows/ci.yml

@@ -7,12 +7,128 @@ on:
   push:
     branches: [main]
 
+permissions:
+  actions: read
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  python-ci:
+  python-ci-capi:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout ladybug
+        uses: actions/checkout@v4
+        with:
+          repository: LadybugDB/ladybug
+          fetch-depth: 1
+          path: ladybug
+
+      - name: Update submodules
+        working-directory: ladybug
+        run: git submodule update --init --recursive dataset
+
+      - name: Checkout ladybug-python into ladybug/tools/python_api
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          path: ladybug/tools/python_api
+
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          key: python-${{ runner.os }}-${{ runner.arch }}-${{ github.ref }}
+          max-size: 2G
+          create-symlink: true
+          restore-keys: |
+            python-${{ runner.os }}-${{ runner.arch }}-refs/heads/main
+            python-${{ runner.os }}-${{ runner.arch }}-
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        working-directory: ladybug/tools/python_api
+        run: |
+          uv venv .venv
+          uv pip install -e .[dev]
+
+      - name: Resolve compatible lbug artifact run
+        working-directory: ladybug
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          SHA="$(git rev-parse HEAD)"
+          API_URL="https://api.github.com/repos/LadybugDB/ladybug/actions/workflows/build-and-deploy.yml/runs"
+          AUTH_HEADER="Authorization: Bearer $GITHUB_TOKEN"
+          ACCEPT_HEADER="Accept: application/vnd.github+json"
+          VERSION_HEADER="X-GitHub-Api-Version: 2022-11-28"
+
+          RUN_ID="$(
+            curl -fsSL \
+              -H "$AUTH_HEADER" \
+              -H "$ACCEPT_HEADER" \
+              -H "$VERSION_HEADER" \
+              "$API_URL?head_sha=$SHA&status=success&per_page=1" \
+            | python -c 'import json,sys; data=json.load(sys.stdin); runs=data.get("workflow_runs") or []; print(runs[0]["id"] if runs else "")'
+          )"
+
+          if [ -z "$RUN_ID" ]; then
+            RUN_ID="$(
+              curl -fsSL \
+                -H "$AUTH_HEADER" \
+                -H "$ACCEPT_HEADER" \
+                -H "$VERSION_HEADER" \
+                "$API_URL?branch=main&status=success&per_page=1" \
+              | python -c 'import json,sys; data=json.load(sys.stdin); runs=data.get("workflow_runs") or []; print(runs[0]["id"] if runs else "")'
+            )"
+          fi
+
+          if [ -z "$RUN_ID" ]; then
+            echo "Could not find a successful LadybugDB/ladybug build-and-deploy run." >&2
+            exit 1
+          fi
+
+          echo "Using Ladybug build-and-deploy RUN_ID=$RUN_ID for SHA=$SHA"
+          echo "LBUG_BUILD_RUN_ID=$RUN_ID" >> "$GITHUB_ENV"
+
+      - name: Download shared lbug library
+        working-directory: ladybug/tools/python_api
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh --version
+          LBUG_PRECOMPILED_RUN_ID="$LBUG_BUILD_RUN_ID" LBUG_LIB_KIND=shared bash scripts/download_lbug.sh .cache/lbug-capi.env
+          cat .cache/lbug-capi.env >> "$GITHUB_ENV"
+
+      - name: Check formatting (black)
+        working-directory: ladybug/tools/python_api
+        run: |
+          uv pip install black
+          .venv/bin/black --check src_py test
+
+      - name: Run ruff check
+        working-directory: ladybug/tools/python_api
+        run: |
+          .venv/bin/ruff check src_py test
+
+      - name: Run pytest (C API backend)
+        working-directory: ladybug/tools/python_api
+        env:
+          LBUG_PYTHON_BACKEND: capi
+        run: |
+          .venv/bin/python -m pytest -vv ./test
+
+  python-ci-pybind:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout ladybug
@@ -79,8 +195,10 @@ jobs:
           make python
           cp tools/python_api/src_py/*.py tools/python_api/build/ladybug/
 
-      - name: Run pytest
+      - name: Run pytest (pybind backend)
         working-directory: ladybug/tools/python_api
+        env:
+          LBUG_PYTHON_BACKEND: pybind
         run: |
           export PYTHONPATH=./build
           .venv/bin/python -m pytest -vv ./test

.gitignore

@@ -1,4 +1,6 @@
 build/
 *.egg-info/
 **/__pycache__/
+.cache/
+scripts/download-liblbug.sh
 uv.lock

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "dataset"]
+	path = dataset
+	url = https://github.com/ladybugdb/dataset.git

CMakeLists.txt

@@ -1,7 +1,37 @@
+cmake_minimum_required(VERSION 3.15)
+
 include(FetchContent)
-project(_lbug)
+project(_lbug LANGUAGES CXX C)
 
 set(CMAKE_CXX_STANDARD 20)
+set(LBUG_SOURCE_DIR "" CACHE PATH "Path to the Ladybug source tree used for pybind builds")
+
+if(NOT TARGET pybind11::module)
+    if(LBUG_SOURCE_DIR)
+        add_subdirectory("${LBUG_SOURCE_DIR}/third_party/pybind11" "${CMAKE_BINARY_DIR}/third_party/pybind11" EXCLUDE_FROM_ALL)
+    else()
+        find_package(pybind11 CONFIG REQUIRED)
+    endif()
+endif()
+
+if(NOT LBUG_API_USE_PRECOMPILED_LIB AND NOT TARGET lbug)
+    if(NOT LBUG_SOURCE_DIR)
+        message(FATAL_ERROR "LBUG_SOURCE_DIR must be set when building the pybind extension from Ladybug sources.")
+    endif()
+
+    set(BUILD_BENCHMARK FALSE CACHE BOOL "" FORCE)
+    set(BUILD_EXAMPLES FALSE CACHE BOOL "" FORCE)
+    set(BUILD_EXTENSION_TESTS FALSE CACHE BOOL "" FORCE)
+    set(BUILD_JAVA FALSE CACHE BOOL "" FORCE)
+    set(BUILD_NODEJS FALSE CACHE BOOL "" FORCE)
+    set(BUILD_PYTHON FALSE CACHE BOOL "" FORCE)
+    set(BUILD_SHELL FALSE CACHE BOOL "" FORCE)
+    set(BUILD_TESTS FALSE CACHE BOOL "" FORCE)
+    set(BUILD_WAL_DUMP FALSE CACHE BOOL "" FORCE)
+    set(BUILD_WASM FALSE CACHE BOOL "" FORCE)
+
+    add_subdirectory("${LBUG_SOURCE_DIR}" "${CMAKE_BINARY_DIR}/lbug-source" EXCLUDE_FROM_ALL)
+endif()
 
 file(GLOB SOURCE_PY
         "src_py/*")
@@ -60,6 +90,23 @@ target_include_directories(
         PUBLIC
         src_cpp/include)
 
+if(TARGET lbug)
+    get_target_property(LBUG_INCLUDE_DIRECTORIES lbug INCLUDE_DIRECTORIES)
+    if(LBUG_INCLUDE_DIRECTORIES)
+        target_include_directories(_lbug PRIVATE ${LBUG_INCLUDE_DIRECTORIES})
+    endif()
+
+    get_target_property(LBUG_COMPILE_DEFINITIONS lbug COMPILE_DEFINITIONS)
+    if(LBUG_COMPILE_DEFINITIONS)
+        target_compile_definitions(_lbug PRIVATE ${LBUG_COMPILE_DEFINITIONS})
+    endif()
+
+    get_target_property(LBUG_COMPILE_OPTIONS lbug COMPILE_OPTIONS)
+    if(LBUG_COMPILE_OPTIONS)
+        target_compile_options(_lbug PRIVATE ${LBUG_COMPILE_OPTIONS})
+    endif()
+endif()
+
 get_target_property(PYTHON_DEST _lbug LIBRARY_OUTPUT_DIRECTORY)
 
 file(COPY ${SOURCE_PY} DESTINATION ${PYTHON_DEST})

Makefile

@@ -1,14 +1,16 @@
 .DEFAULT_GOAL := help
 # Explicit targets to avoid conflict with files of the same name.
 .PHONY: \
-	requirements \
+	requirements sync \
 	lint check format \
-	build test \
+	build bootstrap-capi build-pybind-subdir test test-pybind-subdir \
 	help
 
 PYTHONPATH=
 SHELL=/usr/bin/env bash
 VENV=.venv
+UV_CACHE_DIR?=$(CURDIR)/.cache/uv
+LBUG_SOURCE_DIR?=$(abspath ../ladybug)
 
 ifeq ($(OS),Windows_NT)
 	VENV_BIN=$(VENV)/Scripts
@@ -17,11 +19,14 @@ else
 endif
 
 .venv:  ## Set up a Python virtual environment and install dev packages
-	uv venv $(VENV)
+	UV_CACHE_DIR="$(UV_CACHE_DIR)" uv venv $(VENV)
 
 requirements: .venv ## Install/update Python dev packages
 	@unset CONDA_PREFIX \
-	&& uv pip install -e .[dev]
+	&& UV_CACHE_DIR="$(UV_CACHE_DIR)" uv pip install -e .[dev]
+
+sync: bootstrap-capi ## Sync project + dev dependencies for uv run / pytest
+	UV_CACHE_DIR="$(UV_CACHE_DIR)" uv sync --extra dev
 
 pytest: requirements
 ifeq ($(OS),Windows_NT)
@@ -42,13 +47,23 @@ check: requirements
 format: requirements
 	$(VENV_BIN)/ruff format src_py test
 
-build:  ## Compile ladybug (and install in 'build') for Python
-	$(MAKE) -C ../../ python
-	cp src_py/*.py build/ladybug/
+CAPI_ENV_FILE=.cache/lbug-capi.env
+
+build: bootstrap-capi ## Prepare standalone C-API runtime assets
+	@echo "Standalone package loads from src_py via editable install; shared lib cached under .cache/lbug-prebuilt."
+
+build-pybind-subdir: requirements ## Build pybind from this repo using Ladybug sources at LBUG_SOURCE_DIR
+	bash scripts/build_pybind_from_subdir.sh "$(LBUG_SOURCE_DIR)"
+
+test-pybind-subdir: build-pybind-subdir ## Run tests against pybind build produced from ./ladybug
+	export PYTHONPATH=./build
+	$(VENV_BIN)/pytest -q
+
+bootstrap-capi: ## Download latest shared C-API binary and emit runtime env file
+	LBUG_LIB_KIND=shared bash scripts/download_lbug.sh $(CAPI_ENV_FILE)
 
-test: requirements  ## Run the Python unit tests
-	cp src_py/*.py build/ladybug/ && cd build
-	$(VENV_BIN)/pytest test
+test: requirements build ## Run the standalone Python unit tests
+	$(VENV_BIN)/pytest -q
 
 help:  ## Display this help information
 	@echo -e "\033[1mAvailable commands:\033[0m"

README.md

@@ -1,3 +1,49 @@
 # Python APIs
 
-## Build
+## Build
+
+### C-API backend (default)
+
+```bash
+make sync
+```
+
+This downloads the latest shared `liblbug` binary (via upstream
+`download-liblbug.sh`) and syncs the project with dev dependencies.
+The Python package is installed directly from `src_py/`, so the standalone
+workflow no longer depends on `./build/ladybug`.
+
+Run tests with:
+
+```bash
+uv run pytest
+```
+
+### Pybind backend from inverted layout
+
+If your checkout layout is:
+
+- `ladybug-python/` (this repo)
+- `../ladybug/` (main Ladybug repo as a sibling checkout)
+
+then build the pybind extension through the Ladybug top-level build with:
+
+```bash
+make build-pybind-subdir
+```
+
+This uses `LBUG_SOURCE_DIR` (default: `../ladybug`) to configure this repo's
+CMake build against the Ladybug source checkout and writes `_lbug*` into
+`./build/ladybug`.
+
+Run tests against that pybind build with:
+
+```bash
+make test-pybind-subdir
+```
+
+Override the source tree location when needed:
+
+```bash
+make build-pybind-subdir LBUG_SOURCE_DIR=/path/to/ladybug
+```

plan.md

@@ -0,0 +1,54 @@
+# Plan: Full C-API Python backend + Node-style memory ownership
+
+## Goal
+
+Move `ladybug-python` fully to `lbug.h` C-API, with no backend knob, while preserving public Python API behavior and stability.
+
+## Memory Management Strategy (authoritative)
+
+### Ownership model
+
+- **All heap memory returned by C-API result-reading calls is owned by the backend `QueryResult` object**.
+- Memory is released when `result.close()` is called (or when GC triggers close), matching Node-style lifetime semantics.
+- This includes:
+  - `char*` returned through result paths (column names, string/uuid/decimal rendering, etc.)
+  - blob buffers returned from result values
+
+### Lifecycle ordering
+
+- Normal close order remains:
+  1. `result.close()`
+  2. `conn.close()`
+  3. `db.close()`
+
+### Out-of-order safety
+
+- Out-of-order close must never crash.
+- We enforce safe parent/child close behavior in Python wrappers:
+  - Database tracks live connections; closes them before destroying DB handle.
+  - Connection tracks live query results; closes them before destroying connection handle.
+  - QueryResult methods detect closed parent DB/connection and raise Python exceptions, not segfault.
+
+## Execution Steps
+
+1. Make C-API backend the only backend path.
+2. Add QueryResult-owned allocation tracking and deferred free-on-close.
+3. Add parent-child tracking across Database/Connection/QueryResult.
+4. Ensure out-of-order close behavior is idempotent and crash-safe.
+5. Add/adjust tests for:
+   - normal close ordering
+   - out-of-order close safety
+   - C-API smoke and parameter binding.
+
+## Transitional pybind usage (tracking subsection)
+
+Use pybind only where C-API does not currently expose equivalent functionality.
+
+- Keep C-API as default for duplicated core functionality (`Database`, `Connection`,
+  `PreparedStatement`, `QueryResult` lifecycle/query execution semantics).
+- Route to pybind for non-duplicated features:
+  - Python object scan replacement (`LOAD/COPY ... FROM df/tab`)
+  - Arrow memory-backed table APIs (`create_arrow_table`, `create_arrow_rel_table`, `drop_arrow_table`)
+  - UDF registration/removal (until C-API equivalent is available)
+- Track and reduce duplication over time by migrating pybind-only features to C-API upstream,
+  then removing fallback paths.