From 3b470d3b26059e998d18072ced4d331977ca6255 Mon Sep 17 00:00:00 2001 From: Brandon Payton Date: Sat, 25 Jul 2026 16:53:06 -0400 Subject: [PATCH] [Kernel/POSIX] Distinguish terminals from character devices Character-special mode is broader than terminal identity. Kandelo encoded host terminal stdio and virtual devices such as /dev/null, framebuffer, audio, and DRM with FileType::CharDevice, so terminal probes could mistake every virtual character device for an interactive terminal. In particular, musl implements isatty() with TIOCGWINSZ, causing redirected Bash output to observe false terminal behavior. Derive terminal identity from the open file description: dedicated PTY master/slave types are terminals, and legacy host stdio is a terminal only when its stable canonical stdio path and host handle agree. Reuse that classification across isatty, termios, ioctl namespace gating, and fpathconf. Keep generic FION* requests ahead of terminal gating and preserve device-specific ioctl dispatch. This is a compatible semantic correction under ABI 42. It changes no syscall numbers, marshalling, exported signatures, structure layouts, generated bindings, or VFS ABI metadata, so existing binaries remain valid and need no rebuild. Validation: - focused non-terminal character-device terminal matrix, including the exact musl TIOCGWINSZ isatty path: 1 passed - full kernel unit suite: 1,253 passed - ABI snapshot and generated bindings check - git diff --check --- crates/kernel/src/ofd.rs | 22 ++++ crates/kernel/src/syscalls.rs | 208 +++++++++++++++++++++++++++++----- docs/posix-status.md | 8 +- 3 files changed, 207 insertions(+), 31 deletions(-) diff --git a/crates/kernel/src/ofd.rs b/crates/kernel/src/ofd.rs index 2de44573db..07651d7241 100644 --- a/crates/kernel/src/ofd.rs +++ b/crates/kernel/src/ofd.rs @@ -311,6 +311,28 @@ pub(crate) struct PendingDirEntry { } impl OpenFileDesc { + /// Whether this open description denotes a terminal endpoint. + /// + /// Host-backed standard streams predate the dedicated PTY file types, so + /// they remain encoded as `CharDevice` OFDs with their canonical stdio + /// paths and non-negative host stream handles. Kernel-owned character + /// devices use negative handles instead (`/dev/null`, framebuffer, DRM, + /// and so on) and must not acquire terminal semantics merely because + /// `stat(2)` reports `S_IFCHR`. + pub(crate) fn is_terminal(&self) -> bool { + match self.file_type { + FileType::PtyMaster | FileType::PtySlave => true, + FileType::CharDevice => { + self.host_handle >= 0 + && matches!( + self.path.as_slice(), + b"/dev/stdin" | b"/dev/stdout" | b"/dev/stderr" + ) + } + _ => false, + } + } + /// Drop process-local directory-iterator state while preserving the /// guest-visible position at which a newly inherited or transferred /// descriptor must resume. diff --git a/crates/kernel/src/syscalls.rs b/crates/kernel/src/syscalls.rs index 0526fe9fb3..7aa0a11454 100644 --- a/crates/kernel/src/syscalls.rs +++ b/crates/kernel/src/syscalls.rs @@ -7793,16 +7793,13 @@ pub fn sys_mremap( } /// Check if a file descriptor refers to a terminal. -/// Returns 1 if it's a terminal (CharDevice, PtyMaster, or PtySlave), Err(ENOTTY) otherwise. +/// Returns 1 if it is a host terminal or PTY, Err(ENOTTY) otherwise. pub fn sys_isatty(proc: &Process, fd: i32) -> Result { let entry = proc.fd_table.get(fd)?; let ofd_idx = entry.ofd_ref.0; let ofd = proc.ofd_table.get(ofd_idx).ok_or(Errno::EBADF)?; - if matches!( - ofd.file_type, - FileType::CharDevice | FileType::PtyMaster | FileType::PtySlave - ) { + if ofd.is_terminal() { Ok(1) } else { Err(Errno::ENOTTY) @@ -12338,10 +12335,7 @@ pub fn sys_tcgetattr(proc: &mut Process, fd: i32, buf: &mut [u8]) -> Result<(), let entry = proc.fd_table.get(fd)?; let ofd_idx = entry.ofd_ref.0; let ofd = proc.ofd_table.get(ofd_idx).ok_or(Errno::EBADF)?; - if !matches!( - ofd.file_type, - FileType::CharDevice | FileType::PtyMaster | FileType::PtySlave - ) { + if !ofd.is_terminal() { return Err(Errno::ENOTTY); } if buf.len() < 48 { @@ -12371,10 +12365,7 @@ pub fn sys_tcsetattr(proc: &mut Process, fd: i32, action: u32, buf: &[u8]) -> Re let entry = proc.fd_table.get(fd)?; let ofd_idx = entry.ofd_ref.0; let ofd = proc.ofd_table.get(ofd_idx).ok_or(Errno::EBADF)?; - if !matches!( - ofd.file_type, - FileType::CharDevice | FileType::PtyMaster | FileType::PtySlave - ) { + if !ofd.is_terminal() { return Err(Errno::ENOTTY); } if buf.len() < 48 { @@ -12414,9 +12405,17 @@ pub fn sys_tcsetattr(proc: &mut Process, fd: i32, action: u32, buf: &[u8]) -> Re Ok(()) } +fn is_terminal_ioctl_request(request: u32) -> bool { + // Linux reserves ioctl type 'T' for tty/termios and 'K' for VT keyboard + // controls. Classify the namespaces instead of duplicating today's + // request list, so a newly implemented terminal request cannot bypass + // non-terminal gating merely because this helper was not updated. + matches!((request >> 8) & 0xff, 0x54 | 0x4B) +} + /// ioctl -- device control. /// Supports generic ioctls (FIONREAD, FIONBIO, FIOCLEX, FIONCLEX) on any fd type, -/// plus terminal ioctls (TIOCGWINSZ, TIOCSWINSZ) on CharDevice fds only. +/// plus terminal ioctls (TIOCGWINSZ, TIOCSWINSZ) on host terminals and PTYs. pub fn sys_ioctl( proc: &mut Process, host: &mut dyn HostIO, @@ -12447,6 +12446,7 @@ pub fn sys_ioctl( if ofd.is_path_only() { return Err(Errno::EBADF); } + let is_terminal = ofd.is_terminal(); // FIONBIO — toggle O_NONBLOCK on the OFD status_flags if request == 0x5421 { @@ -12555,6 +12555,14 @@ pub fn sys_ioctl( return Ok(()); } + // Device-specific handlers intentionally own unknown-ioctl errno policy, + // but a terminal request on a non-terminal must consistently be ENOTTY. + // Gate that shared namespace before framebuffer/audio/DRM dispatch so a + // broad device handler cannot reinterpret TCGETS or a VT probe. + if is_terminal_ioctl_request(request) && !is_terminal { + return Err(Errno::ENOTTY); + } + // --- PTY-specific ioctls (work on PtyMaster only) --- { let ofd = proc.ofd_table.get(ofd_idx).ok_or(Errno::EBADF)?; @@ -12640,6 +12648,9 @@ pub fn sys_ioctl( match request { // KDGKBTYPE — return KB_101 (0x02) as a single byte. 0x4B33 => { + if !is_terminal { + return Err(Errno::ENOTTY); + } if buf.is_empty() { return Err(Errno::EINVAL); } @@ -12648,6 +12659,9 @@ pub fn sys_ioctl( } // KDGKBMODE — return K_XLATE (1) as i32. 0x4B44 => { + if !is_terminal { + return Err(Errno::ENOTTY); + } if buf.len() < 4 { return Err(Errno::EINVAL); } @@ -12655,25 +12669,26 @@ pub fn sys_ioctl( return Ok(()); } // KDSKBMODE — accept any mode, no-op success. - 0x4B45 => return Ok(()), + 0x4B45 => { + if !is_terminal { + return Err(Errno::ENOTTY); + } + return Ok(()); + } _ => {} } - // --- Terminal ioctls (work on CharDevice, PtyMaster, PtySlave) --- + // --- Terminal ioctls (work on host terminals and PTYs) --- let ofd = proc.ofd_table.get(ofd_idx).ok_or(Errno::EBADF)?; let file_type = ofd.file_type; let host_handle = ofd.host_handle; - let is_terminal = matches!( - file_type, - FileType::CharDevice | FileType::PtyMaster | FileType::PtySlave - ); if !is_terminal { return Err(Errno::ENOTTY); } // Helper: get mutable reference to the appropriate TerminalState. - // For PTY fds → PTY pair's terminal state; for CharDevice → process terminal state. + // For PTY fds → PTY pair's terminal state; for host stdio → process terminal state. // We handle this by dispatching per-request below. use crate::terminal::*; @@ -13930,6 +13945,7 @@ pub fn sys_fpathconf( validate_pathconf_name(name)?; let entry = proc.fd_table.get(fd)?; let ofd = proc.ofd_table.get(entry.ofd_ref.0).ok_or(Errno::EBADF)?; + let is_terminal = ofd.is_terminal(); let file_type = ofd.file_type; let host_handle = ofd.host_handle; let path = ofd.path.clone(); @@ -13967,9 +13983,7 @@ pub fn sys_fpathconf( } FileType::MemFd => filesystem_pathconf_value(name, false, None), FileType::Regular | FileType::Directory | FileType::CharDevice => { - if file_type == FileType::CharDevice - && matches!(path.as_slice(), b"/dev/stdin" | b"/dev/stdout" | b"/dev/stderr") - { + if is_terminal { terminal_pathconf_value(name) } else if is_procfs_namespace_path(&path) || (is_devfs_namespace_path(&path) && !is_host_backed_devfs_path(&path)) @@ -21845,9 +21859,149 @@ mod tests { } #[test] - fn test_isatty_stdin() { - let proc = terminal_process(1); - assert_eq!(sys_isatty(&proc, 0), Ok(1)); + fn test_isatty_distinguishes_host_terminal_from_captured_stdio() { + let terminal = terminal_process(1); + let captured = Process::new(2); + + for fd in 0..=2 { + assert_eq!(sys_isatty(&terminal, fd), Ok(1)); + assert_eq!(sys_isatty(&captured, fd), Err(Errno::ENOTTY)); + } + } + + #[test] + fn test_host_terminal_and_both_pty_endpoints_accept_terminal_operations() { + fn assert_terminal_surface(proc: &mut Process, host: &mut MockHostIO, fd: i32) { + assert_eq!(sys_isatty(proc, fd), Ok(1)); + + let mut legacy_attrs = [0; 48]; + assert_eq!(sys_tcgetattr(proc, fd, &mut legacy_attrs), Ok(())); + assert_eq!( + sys_tcsetattr(proc, fd, crate::terminal::TCSANOW, &legacy_attrs), + Ok(()), + ); + + let mut ioctl_attrs = [0; crate::terminal::TERMIOS_SIZE]; + assert_eq!( + sys_ioctl( + proc, + host, + fd, + crate::terminal::TCGETS, + &mut ioctl_attrs, + ), + Ok(()), + ); + let mut keyboard_type = [0]; + assert_eq!( + sys_ioctl(proc, host, fd, 0x4B33, &mut keyboard_type), + Ok(()), + ); + assert_eq!(keyboard_type, [0x02]); + } + + let mut terminal = terminal_process(1); + let mut terminal_host = MockHostIO::new(); + for fd in 0..=2 { + assert_terminal_surface(&mut terminal, &mut terminal_host, fd); + } + + let mut fixture = PtyFixture::new(); + for fd in [fixture.master_fd, fixture.slave_fd] { + assert_terminal_surface(&mut fixture.proc, &mut fixture.host, fd); + } + } + + #[test] + fn test_virtual_character_devices_reject_terminal_operations() { + let mut proc = Process::new(1); + let mut host = MockHostIO::new(); + let devices: &[(&[u8], i64)] = &[ + (b"/dev/null", VirtualDevice::Null.host_handle()), + (b"/dev/console", VirtualDevice::Null.host_handle()), + (b"/dev/zero", VirtualDevice::Zero.host_handle()), + (b"/dev/urandom", VirtualDevice::Urandom.host_handle()), + (b"/dev/random", VirtualDevice::Urandom.host_handle()), + (b"/dev/full", VirtualDevice::Full.host_handle()), + (b"/dev/fb0", VirtualDevice::Fb0.host_handle()), + (b"/dev/input/mice", VirtualDevice::Mice.host_handle()), + (b"/dev/dsp", VirtualDevice::Dsp.host_handle()), + ( + b"/dev/dri/renderD128", + VirtualDevice::DriRenderD128.host_handle(), + ), + (b"/dev/dri/card0", VirtualDevice::DriCard0.host_handle()), + // Prime fds are kernel-owned CharDevices outside the named + // VirtualDevice range and obey the same non-terminal contract. + (b"/dev/dri/prime-test", -200), + // A future host-backed CharDevice must opt into terminal identity + // rather than inheriting it from a non-negative handle. + (b"/dev/other-char-device", 77), + ]; + + for &(path, host_handle) in devices { + let ofd_idx = + proc.ofd_table + .create(FileType::CharDevice, O_RDWR, host_handle, path.to_vec()); + let fd = proc + .fd_table + .alloc(crate::fd::OpenFileDescRef(ofd_idx), 0) + .unwrap(); + + assert_eq!(sys_isatty(&proc, fd), Err(Errno::ENOTTY), "{path:?}"); + assert_eq!( + sys_tcgetattr(&mut proc, fd, &mut [0; 48]), + Err(Errno::ENOTTY), + "{path:?}", + ); + assert_eq!( + sys_tcsetattr(&mut proc, fd, crate::terminal::TCSANOW, &[0; 48]), + Err(Errno::ENOTTY), + "{path:?}", + ); + assert_eq!( + sys_ioctl( + &mut proc, + &mut host, + fd, + crate::terminal::TCGETS, + &mut [0; crate::terminal::TERMIOS_SIZE], + ), + Err(Errno::ENOTTY), + "{path:?}", + ); + // WHY: musl implements isatty() with TIOCGWINSZ rather than + // Kandelo's legacy direct isatty syscall. Keep the public libc + // path in this matrix so a future ioctl refactor cannot restore + // false terminal identity for character devices. + assert_eq!( + sys_ioctl( + &mut proc, + &mut host, + fd, + crate::terminal::TIOCGWINSZ, + &mut [0; 8], + ), + Err(Errno::ENOTTY), + "{path:?}", + ); + assert_eq!( + sys_ioctl(&mut proc, &mut host, fd, 0x4B33, &mut [0; 1]), + Err(Errno::ENOTTY), + "{path:?}", + ); + } + } + + #[test] + fn test_redirecting_terminal_stdout_to_dev_null_clears_terminal_identity() { + let mut proc = terminal_process(1); + let mut host = MockHostIO::new(); + let null_fd = sys_open(&mut proc, &mut host, b"/dev/null", O_WRONLY, 0).unwrap(); + + assert_eq!(sys_isatty(&proc, 1), Ok(1)); + assert_eq!(sys_dup2(&mut proc, &mut host, null_fd, 1), Ok(1)); + assert_eq!(sys_isatty(&proc, 1), Err(Errno::ENOTTY)); } #[test] diff --git a/docs/posix-status.md b/docs/posix-status.md index 8bb0f936d9..31d4e197f3 100644 --- a/docs/posix-status.md +++ b/docs/posix-status.md @@ -350,9 +350,9 @@ shortcuts. | Function | Status | Notes | |----------|--------|-------| -| `isatty()` | Full | Returns 1 for CharDevice, PtyMaster, and PtySlave fds; ENOTTY for others. | -| `tcgetattr()` / `tcsetattr()` | Partial | CharDevice and PTY fds round-trip musl's 60-byte termios layout, including all four flag words, `c_line`, `c_cc`, and input/output speeds; custom syscalls 70/71 retain the older 48-byte flags-plus-`c_cc` layout. `TCSANOW` and `TCSADRAIN` preserve unread input across `ICANON` transitions: completed lines and the current edited partial line become raw-readable in byte order, while unread raw bytes become immediately readable if the mode changes back, matching Linux EOF-push behavior. `TCSAFLUSH` discards unread input before applying the change. PTY writes synchronously enter the output queue, so there is no deferred device transmission to await. Implemented line discipline includes `VERASE`, `VKILL`, non-empty-line `VEOF`, ICRNL/INLCR/IGNCR, and ECHO/ECHOE/ECHOK/ECHONL. Remaining gaps: `VMIN`/`VTIME` values round-trip but raw-read timing is approximated, an empty canonical `VEOF` does not create a queued EOF event, a canonical `read()` can return bytes from multiple completed lines instead of stopping after one line, `VWERASE` is not implemented, and exposed input/output flags outside the listed subset do not all have data-path semantics. | -| `ioctl()` | Full | 16 terminal ioctls: TCGETS/TCSETS/TCSETSW/TCSETSF (termios), TIOCGPTN (PTY number), TIOCSPTLCK (unlock PTY), TIOCGPGRP/TIOCSPGRP (foreground pgid), TIOCGWINSZ/TIOCSWINSZ (window size + SIGWINCH), TCSBRK/TCXONC/TCFLSH, TIOCGSID/TIOCSCTTY/TIOCNOTTY (session/controlling terminal). Generic: FIONREAD, FIONBIO, FIOCLEX/FIONCLEX, FIOASYNC. Works on CharDevice, PtyMaster, and PtySlave fds. | +| `isatty()` | Full | Returns 1 for host terminal stdio and PTY master/slave fds; returns ENOTTY for pipes, files, and non-terminal character devices such as `/dev/null`, `/dev/zero`, framebuffer, audio, and DRM nodes. | +| `tcgetattr()` / `tcsetattr()` | Partial | Host terminal and PTY fds round-trip musl's 60-byte termios layout, including all four flag words, `c_line`, `c_cc`, and input/output speeds; custom syscalls 70/71 retain the older 48-byte flags-plus-`c_cc` layout. Non-terminal character devices return ENOTTY. `TCSANOW` and `TCSADRAIN` preserve unread input across `ICANON` transitions: completed lines and the current edited partial line become raw-readable in byte order, while unread raw bytes become immediately readable if the mode changes back, matching Linux EOF-push behavior. `TCSAFLUSH` discards unread input before applying the change. PTY writes synchronously enter the output queue, so there is no deferred device transmission to await. Implemented line discipline includes `VERASE`, `VKILL`, non-empty-line `VEOF`, ICRNL/INLCR/IGNCR, and ECHO/ECHOE/ECHOK/ECHONL. Remaining gaps: `VMIN`/`VTIME` values round-trip but raw-read timing is approximated, an empty canonical `VEOF` does not create a queued EOF event, a canonical `read()` can return bytes from multiple completed lines instead of stopping after one line, `VWERASE` is not implemented, and exposed input/output flags outside the listed subset do not all have data-path semantics. | +| `ioctl()` | Full | 16 terminal ioctls: TCGETS/TCSETS/TCSETSW/TCSETSF (termios), TIOCGPTN (PTY number), TIOCSPTLCK (unlock PTY), TIOCGPGRP/TIOCSPGRP (foreground pgid), TIOCGWINSZ/TIOCSWINSZ (window size + SIGWINCH), TCSBRK/TCXONC/TCFLSH, TIOCGSID/TIOCSCTTY/TIOCNOTTY (session/controlling terminal). Generic: FIONREAD, FIONBIO, FIOCLEX/FIONCLEX, FIOASYNC. Terminal and Linux-VT requests work on host terminals and PTYs and return ENOTTY on other character devices. | | `posix_openpt()` | Full | Opens `/dev/ptmx`, allocates PTY pair, returns master fd. | | `grantpt()` / `unlockpt()` | Full | `grantpt()` is a no-op (no permissions to set). `unlockpt()` clears the lock flag on the PTY pair. | | `ptsname()` | Full | Returns `/dev/pts/N` path for the slave side. | @@ -375,7 +375,7 @@ shortcuts. | `/dev/tty` | Partial | Uses the first open PTY-slave OFD as the current controlling-terminal heuristic. When none is open, it currently falls back to fd 0 rather than returning ENXIO; `pathconf()` follows that same OFD selection and therefore does not advertise terminal variables for the captured, pipe-backed case. | | `/dev/ptmx` | Full | PTY master multiplexer. `open()` allocates a new PTY pair, returns master fd. | | `/dev/pts/*` | Full | PTY slave devices. `posix_openpt()` + `grantpt()` + `unlockpt()` + `ptsname()`. Full line discipline, canonical/raw mode, OPOST/ONLCR, 16 terminal ioctls. | -| `/dev/fb0` | Full | Linux fbdev framebuffer. Single-open (`EBUSY` for second opener). 640×400 BGRA32 packed-pixel. ioctls: `FBIOGET_VSCREENINFO`, `FBIOGET_FSCREENINFO`, `FBIOPAN_DISPLAY` (no-op success), `FBIOPUT_VSCREENINFO` (validates geometry). `mmap` returns a region in process memory and notifies the host (`bind_framebuffer` callback) so the browser canvas can mirror pixels. `munmap`/`exit`/`exec` discard the image mapping; a surviving fd retains device ownership across exec. Ownership is released after both the final fd and any live mapping are gone, since a mapping remains valid after `close()`. Linux-VT keyboard ioctls (`KDGKBTYPE`/`KDGKBMODE`/`KDSKBMODE`) accepted with sensible defaults so fbDOOM-style software works unmodified. | +| `/dev/fb0` | Full | Linux fbdev framebuffer. Single-open (`EBUSY` for second opener). 640×400 BGRA32 packed-pixel. ioctls: `FBIOGET_VSCREENINFO`, `FBIOGET_FSCREENINFO`, `FBIOPAN_DISPLAY` (no-op success), `FBIOPUT_VSCREENINFO` (validates geometry). `mmap` returns a region in process memory and notifies the host (`bind_framebuffer` callback) so the browser canvas can mirror pixels. `munmap`/`exit`/`exec` discard the image mapping; a surviving fd retains device ownership across exec. Ownership is released after both the final fd and any live mapping are gone, since a mapping remains valid after `close()`. Linux-VT keyboard ioctls (`KDGKBTYPE`/`KDGKBMODE`/`KDSKBMODE`) are accepted on the process's terminal fd so fbDOOM-style software works unmodified; `/dev/fb0` itself is not a terminal. | | `/dev/input/mice` | Full | Linux `mousedev` PS/2 mouse stream. Single-open (`EBUSY` for second pid). 3-byte packets: byte0 button bits + sign/overflow flags, bytes 1..2 signed dx/dy with positive-up dy. Host pushes events via `kernel_inject_mouse_event(dx, dy, buttons)`; the kernel buffers up to 4096 packets (whole-packet drop on overflow). `read()` drains queued bytes; returns `EAGAIN` when empty. `poll()` reports `POLLIN` only when bytes are queued. Ownership and queued packets survive exec with a non-CLOEXEC fd; last close or exit releases and clears them. No IMPS/2 wheel protocol, no `evdev`/`/dev/input/eventN`. | | `/dev/dsp` | Full (write-only) | OSS-style PCM audio sink. Single-open (`EBUSY` for second pid). `write()` accepts interleaved 16-bit-LE PCM and buffers it in a 256 KiB ring; the host drains via the `kernel_drain_audio` wasm export and feeds a Web Audio `AudioContext`. ioctls: `SNDCTL_DSP_RESET`, `SNDCTL_DSP_SYNC`, `SNDCTL_DSP_SPEED` (clamp 4000–192000 Hz), `SNDCTL_DSP_STEREO` / `SNDCTL_DSP_CHANNELS` (1 or 2), `SNDCTL_DSP_SETFMT` (only `AFMT_S16_LE`), `SNDCTL_DSP_GETFMTS`, `SNDCTL_DSP_SETFRAGMENT` (accept-and-acknowledge). On overflow drops the *oldest whole frame* — never tears L/R alignment. Ownership and queued samples survive exec with a non-CLOEXEC fd; last close or exit releases and flushes them. `read()` returns 0 (EOF-like). `poll()` reports `POLLOUT` always, never `POLLIN`. No record path, no `mmap`-based zero-copy; DOOM's mixer is in user space. | | `/dev/shm/*` | Partial | POSIX shm objects are regular files used by `shm_open()`. Stable-identity backends support host-coordinated `MAP_SHARED` across processes at syscall boundaries; this is not immediate shared linear memory and does not make process-shared futexes work. |