Locks: report shared lock-table exhaustion as ENOLCK

docs/posix-status.md

@@ -93,8 +93,8 @@ Kandelo uses a single kernel Wasm instance that holds a `ProcessTable` and serve
 | `F_GETFL` | Full | Returns status flags + access mode. Use O_ACCMODE mask. |
 | `F_SETFL` | Full | Only O_APPEND, O_NONBLOCK modifiable. Access mode bits preserved. |
 | `F_GETLK` | Full | Advisory record locking. Returns blocking lock info or F_UNLCK if no conflict. Locks released on close() and exit() per POSIX. |
-| `F_SETLK` | Full | Non-blocking lock acquisition. Returns EAGAIN on conflict. Read/write access mode validated. Locks released on close() and exit() per POSIX. |
-| `F_SETLKW` | Partial | Blocking lock acquisition. Host-backed locks and in-kernel fallback locks are coordinated across processes; blocking conflicts use an internal EAGAIN retry path in the host worker until the lock is available. No deadlock detection. |
+| `F_SETLK` | Full | Non-blocking lock acquisition. Returns EAGAIN on conflict and ENOLCK when the fixed-size shared host lock table is full. Read/write access mode validated. Locks released on close() and exit() per POSIX. |
+| `F_SETLKW` | Partial | Blocking lock acquisition. Host-backed locks and in-kernel fallback locks are coordinated across processes; blocking conflicts use an internal EAGAIN retry path in the host worker until the lock is available. Shared host lock-table exhaustion returns ENOLCK instead of retrying. No deadlock detection. |
 | `F_GETOWN` | Full | Returns async I/O owner PID from OFD. Default 0. |
 | `F_SETOWN` | Full | Sets async I/O owner PID on OFD. SIGIO delivery deferred to signal delivery phase. |
 

host/src/kernel.ts

@@ -2706,12 +2706,18 @@ export class WasmPosixKernel {
           return 0;
         }
         case WasmPosixKernel.F_SETLK: {
-          const ok = this.sharedLockTable.setLock(pathHash, pid, lockType, start, len);
-          return ok ? 0 : -11; // -EAGAIN
+          const result = this.sharedLockTable.setLockResult(pathHash, pid, lockType, start, len);
+          if (result === "acquired") return 0;
+          if (result === "blocked") return -11; // -EAGAIN
+          return -37; // -ENOLCK
         }
         case WasmPosixKernel.F_SETLKW: {
-          const ok = this.sharedLockTable.setLock(pathHash, pid, lockType, start, len);
-          return ok ? 0 : -11; // -EAGAIN, kernel-worker retries blocking fcntl
+          const result = this.sharedLockTable.setLockResult(pathHash, pid, lockType, start, len);
+          // The worker retries EAGAIN for a blocking lock. ENOLCK is a real
+          // capacity failure and must be returned instead of spinning.
+          if (result === "acquired") return 0;
+          if (result === "blocked") return -11; // -EAGAIN
+          return -37; // -ENOLCK
         }
         default:
           return -22; // -EINVAL

host/src/shared-lock-table.ts

@@ -55,6 +55,8 @@ export interface LockInfo {
   len: bigint;
 }
 
+export type LockSetResult = "acquired" | "blocked" | "no-space";
+
 export class SharedLockTable {
   private view: Int32Array;
   private sab: SharedArrayBuffer;
@@ -231,7 +233,8 @@ export class SharedLockTable {
 
   /**
    * Set a lock (non-blocking). For F_UNLCK, removes matching locks.
-   * Returns true on success, false if conflicting lock exists (EAGAIN).
+   * Returns true on success, false if the lock conflicts or the table is full.
+   * Errno-producing callers must use setLockResult() to preserve the cause.
    */
   setLock(
     pathHash: number,
@@ -240,6 +243,23 @@ export class SharedLockTable {
     start: bigint,
     len: bigint,
   ): boolean {
+    return (
+      this.setLockResult(pathHash, pid, lockType, start, len) === "acquired"
+    );
+  }
+
+  /**
+   * Set a lock and preserve the reason it could not be installed. Callers
+   * that translate the result to an errno must distinguish a conflicting
+   * lock (EAGAIN) from an exhausted system lock table (ENOLCK).
+   */
+  setLockResult(
+    pathHash: number,
+    pid: number,
+    lockType: number,
+    start: bigint,
+    len: bigint,
+  ): LockSetResult {
     this.acquire();
     try {
       return this._setLockUnsafe(pathHash, pid, lockType, start, len);
@@ -254,7 +274,7 @@ export class SharedLockTable {
     lockType: number,
     start: bigint,
     len: bigint,
-  ): boolean {
+  ): LockSetResult {
     // For unlock: remove overlapping locks from same pid on same path, then wake waiters
     if (lockType === F_UNLCK) {
       let i = 0;
@@ -274,12 +294,12 @@ export class SharedLockTable {
       // Wake any F_SETLKW waiters
       Atomics.add(this.view, WAKE_COUNTER, 1);
       Atomics.notify(this.view, WAKE_COUNTER);
-      return true;
+      return "acquired";
     }
 
     // Check for conflicts
     if (this._getBlockingLockUnsafe(pathHash, lockType, start, len, pid)) {
-      return false; // caller should return EAGAIN
+      return "blocked";
     }
 
     // Remove overlapping locks from same pid on same path (upgrade/replace)
@@ -301,24 +321,25 @@ export class SharedLockTable {
     const count = this.view[COUNT];
     const capacity = this.view[CAPACITY];
     if (count >= capacity) {
-      return false; // table full — treat as EAGAIN
+      return "no-space";
     }
     this.writeEntry(count, { pathHash, pid, lockType, start, len });
     this.view[COUNT] = count + 1;
-    return true;
+    return "acquired";
   }
 
   /**
    * Set a lock, blocking until it can be acquired (F_SETLKW).
    * Uses Atomics.wait on wake_counter to sleep between retries.
+   * Returns no-space instead of waiting when the fixed-size table is full.
    */
   setLockWait(
     pathHash: number,
     pid: number,
     lockType: number,
     start: bigint,
     len: bigint,
-  ): void {
+  ): "acquired" | "no-space" {
     while (true) {
       this.acquire();
       const blocker = this._getBlockingLockUnsafe(
@@ -329,9 +350,10 @@ export class SharedLockTable {
         pid,
       );
       if (!blocker) {
-        this._setLockUnsafe(pathHash, pid, lockType, start, len);
+        const result = this._setLockUnsafe(pathHash, pid, lockType, start, len);
         this.release();
-        return;
+        if (result !== "blocked") return result;
+        continue;
       }
       const wakeCount = Atomics.load(this.view, WAKE_COUNTER);
       this.release();

host/test/kernel-fcntl-lock.test.ts

@@ -6,6 +6,7 @@ const F_SETLK = 13;
 const F_SETLKW = 14;
 const F_WRLCK = 1;
 const EAGAIN = 11;
+const ENOLCK = 37;
 
 type LockCall = [number, number, number, bigint, bigint];
 
@@ -54,9 +55,9 @@ describe("WasmPosixKernel fcntl locking import", () => {
     const setLockCalls: LockCall[] = [];
     let setLockWaitCalled = false;
     const { kernel, path } = makeKernel({
-      setLock: (...args: LockCall) => {
+      setLockResult: (...args: LockCall) => {
         setLockCalls.push(args);
-        return false;
+        return "blocked";
       },
       setLockWait: () => {
         setLockWaitCalled = true;
@@ -75,9 +76,9 @@ describe("WasmPosixKernel fcntl locking import", () => {
     const setLockCalls: LockCall[] = [];
     let setLockWaitCalled = false;
     const { kernel, path } = makeKernel({
-      setLock: (...args: LockCall) => {
+      setLockResult: (...args: LockCall) => {
         setLockCalls.push(args);
-        return false;
+        return "blocked";
       },
       setLockWait: () => {
         setLockWaitCalled = true;
@@ -96,9 +97,9 @@ describe("WasmPosixKernel fcntl locking import", () => {
     const setLockCalls: LockCall[] = [];
     let setLockWaitCalled = false;
     const { kernel, path } = makeKernel({
-      setLock: (...args: LockCall) => {
+      setLockResult: (...args: LockCall) => {
         setLockCalls.push(args);
-        return true;
+        return "acquired";
       },
       setLockWait: () => {
         setLockWaitCalled = true;
@@ -112,4 +113,20 @@ describe("WasmPosixKernel fcntl locking import", () => {
     expect(setLockCalls[0].slice(1)).toEqual([2, F_WRLCK, 32n, 64n]);
     expect(setLockWaitCalled).toBe(false);
   });
+
+  it.each([F_SETLK, F_SETLKW])(
+    "returns ENOLCK when command %i exhausts the shared table",
+    (cmd) => {
+      const setLockCalls: LockCall[] = [];
+      const { kernel, path } = makeKernel({
+        setLockResult: (...args: LockCall) => {
+          setLockCalls.push(args);
+          return "no-space";
+        },
+      });
+
+      expect(hostFcntlLock(kernel, path, cmd)).toBe(-ENOLCK);
+      expect(setLockCalls).toHaveLength(1);
+    },
+  );
 });

host/test/shared-lock-table.test.ts

@@ -98,6 +98,24 @@ describe("SharedLockTable", () => {
     expect(result).toBe(false);
   });
 
+  it("distinguishes table exhaustion from a conflicting lock", () => {
+    const table = SharedLockTable.create(1);
+
+    expect(table.setLockResult(100, 1, 1, 0n, 1n)).toBe("acquired");
+    expect(table.setLockResult(100, 2, 1, 0n, 1n)).toBe("blocked");
+    expect(table.setLockResult(200, 2, 1, 0n, 1n)).toBe("no-space");
+    // Keep the existing boolean API compatible for callers that only need a
+    // success/failure answer.
+    expect(table.setLock(200, 2, 1, 0n, 1n)).toBe(false);
+  });
+
+  it("does not wait when a blocking request exhausts the table", () => {
+    const table = SharedLockTable.create(1);
+
+    expect(table.setLock(100, 1, 1, 0n, 1n)).toBe(true);
+    expect(table.setLockWait(200, 2, 1, 0n, 1n)).toBe("no-space");
+  });
+
   it("should removeLocksByPid", () => {
     const table = SharedLockTable.create();
     table.setLock(100, 1, 1, 0n, 50n);