Move clipboard, drag-and-drop, and file opening onto a shared multi-format data transfer type (#4945)

* Generalize data transfer systems

* Move clipboard items to ingest as application data

* Carry application data as parsed items

* Accept image drags on desktop

* Fix tests

Author: timon-schelling

desktop/src/app.rs

@@ -8,13 +8,14 @@ use std::sync::mpsc::{Receiver, Sender};
 use std::thread;
 use std::time::{Duration, Instant};
 use winit::application::ApplicationHandler;
-use winit::data_transfer::{DataTransferSendBuilder, TypeHint};
+use winit::data_transfer::TypeHint;
 use winit::dpi::PhysicalSize;
 use winit::event::{ElementState, MouseButton, StartCause, WindowEvent};
 use winit::event_loop::run_on_demand::EventLoopExtRunOnDemand;
-use winit::event_loop::{ActiveEventLoop, AsyncRequestSerial, ControlFlow, DndAction, EventLoop};
+use winit::event_loop::{ActiveEventLoop, ControlFlow, DndAction, EventLoop};
 use winit::window::WindowId;
 
+use crate::data_transfer::{self, TransferReads};
 use crate::dirs;
 use crate::event::{AppEvent, AppEventScheduler};
 use crate::input::InputState;
@@ -24,7 +25,7 @@ use crate::render::{RenderError, RenderState};
 use crate::spawner::TokioSpawner;
 use crate::ui::{Acceleration, InputEvent, UiCommand, UiConfig, UiInstance};
 use crate::window::Window;
-use crate::wrapper::messages::{DesktopFrontendMessage, DesktopWrapperMessage, IngestAction};
+use crate::wrapper::messages::{DataTransfer, DesktopFrontendMessage, DesktopWrapperMessage, IngestAction};
 use crate::wrapper::{DesktopWrapper, FolderStore, MmapResourceStorage, WgpuContext, WgpuExecutor, serialize_frontend_messages};
 
 pub(crate) struct App {
@@ -36,8 +37,7 @@ pub(crate) struct App {
 	window_maximized: bool,
 	window_fullscreen: bool,
 	window_pending_drag: bool,
-	pending_dnd_fetch: Option<AsyncRequestSerial>,
-	pending_clipboard_fetch: Option<AsyncRequestSerial>,
+	transfer_reads: TransferReads,
 	input_state: InputState,
 	app_event_receiver: Receiver<AppEvent>,
 	app_event_scheduler: AppEventScheduler,
@@ -133,8 +133,7 @@ impl App {
 			window_maximized: false,
 			window_fullscreen: false,
 			window_pending_drag: false,
-			pending_dnd_fetch: None,
-			pending_clipboard_fetch: None,
+			transfer_reads: TransferReads::default(),
 			input_state: InputState::new(),
 			app_event_receiver,
 			app_event_scheduler,
@@ -256,16 +255,10 @@ impl App {
 						futures::executor::block_on(dialog.pick_file()).into_iter().collect()
 					};
 
-					for handle in handles {
-						let path = handle.path().to_path_buf();
-						match fs::read(&path) {
-							Ok(content) => {
-								let action = action.clone();
-								let message = DesktopWrapperMessage::IngestFile { path, content, action };
-								app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
-							}
-							Err(e) => tracing::error!("Failed to read file {}: {}", path.display(), e),
-						}
+					let files = data_transfer::read_files(handles.iter().map(|handle| handle.path().to_path_buf()));
+					if !files.is_empty() {
+						let message = DesktopWrapperMessage::Ingest { content: DataTransfer(files), action };
+						app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
 					}
 				});
 			}
@@ -380,8 +373,8 @@ impl App {
 			DesktopFrontendMessage::ClipboardRead => {
 				self.app_event_scheduler.schedule(AppEvent::ClipboardRead);
 			}
-			DesktopFrontendMessage::ClipboardWrite { content, svg } => {
-				self.app_event_scheduler.schedule(AppEvent::ClipboardWrite { content, svg });
+			DesktopFrontendMessage::ClipboardWrite(content) => {
+				self.app_event_scheduler.schedule(AppEvent::ClipboardWrite(content));
 			}
 			DesktopFrontendMessage::PointerLock => {
 				self.input_state.lock_pointer();
@@ -502,31 +495,15 @@ impl App {
 				}
 			}
 			AppEvent::ClipboardRead => {
-				let result = event_loop.clipboard().and_then(|id| {
-					let Some(id) = id else { return Ok(None) };
-					let data_transfer = event_loop.data_transfer(id)?;
-					if !data_transfer.has_type(&TypeHint::Plaintext) {
-						return Ok(None);
-					}
-					event_loop.fetch_data_transfer(id, &TypeHint::Plaintext).map(Some)
-				});
-				match result {
-					Ok(Some(serial)) => self.pending_clipboard_fetch = Some(serial),
-					Ok(None) => self.dispatch_desktop_wrapper_message(DesktopWrapperMessage::ClipboardReadResult { content: None }),
-					Err(e) => {
-						tracing::error!("Failed to read from clipboard: {e}");
-						self.dispatch_desktop_wrapper_message(DesktopWrapperMessage::ClipboardReadResult { content: None });
-					}
+				let read = event_loop
+					.clipboard()
+					.and_then(|id| id.map_or(Ok(()), |id| self.transfer_reads.start(event_loop, id, IngestAction::Paste)));
+				if let Err(e) = read {
+					tracing::error!("Failed to read from clipboard: {e}");
 				}
 			}
-			AppEvent::ClipboardWrite { content, svg } => {
-				let has_svg = svg.is_some();
-				let mut builder = DataTransferSendBuilder::new((content, svg)).with_type(TypeHint::Plaintext, |(content, _), _| Some(content.clone()));
-				if has_svg {
-					builder = builder.with_type(TypeHint::Image { extension_hint: Some("svg") }, |(_, svg), _| svg.as_ref().map(|svg| svg.as_bytes().to_vec()));
-				}
-				let send_data = builder.build();
-				if let Err(e) = event_loop.set_clipboard(send_data) {
+			AppEvent::ClipboardWrite(content) => {
+				if let Err(e) = event_loop.set_clipboard(data_transfer::send(content)) {
 					tracing::error!("Failed to write to clipboard: {e}");
 				}
 			}
@@ -559,18 +536,16 @@ impl App {
 				}
 				let app_event_scheduler = self.app_event_scheduler.clone();
 				let _ = thread::spawn(move || {
-					for path in paths {
+					for path in &paths {
 						tracing::info!("Opening file: {}", path.display());
-						if let Ok(content) = fs::read(&path) {
-							let message = DesktopWrapperMessage::IngestFile {
-								path,
-								content,
-								action: IngestAction::Open,
-							};
-							app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
-						} else {
-							tracing::error!("Failed to read file: {}", path.display());
-						}
+					}
+					let files = data_transfer::read_files(paths);
+					if !files.is_empty() {
+						let message = DesktopWrapperMessage::Ingest {
+							content: DataTransfer(files),
+							action: IngestAction::Open,
+						};
+						app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
 					}
 				});
 			}
@@ -670,57 +645,25 @@ impl ApplicationHandler for App {
 				}
 			}
 			WindowEvent::DragEntered { id, .. } => {
-				let accepts = event_loop.data_transfer(id).is_ok_and(|data_transfer| data_transfer.has_type(&TypeHint::UriList));
+				let accepts = event_loop.data_transfer(id).is_ok_and(|data_transfer| {
+					data_transfer.has_type(&TypeHint::UriList) || data_transfer.has_type(&TypeHint::Image { extension_hint: None }) || data_transfer.has_type(&TypeHint::Plaintext)
+				});
 				let actions: &[DndAction] = if accepts { &[DndAction::Copy] } else { &[] };
 				if let Err(e) = event_loop.set_valid_dnd_actions(id, actions) {
 					tracing::error!("Failed to set valid drag and drop actions: {e}");
 				}
 			}
-			WindowEvent::DragDropped { id, .. } => match event_loop.fetch_data_transfer(id, &TypeHint::UriList) {
-				Ok(serial) => self.pending_dnd_fetch = Some(serial),
-				Err(e) => tracing::error!("Failed to fetch dropped data: {e}"),
-			},
-			WindowEvent::DataTransferReceived { serial, ref value, .. } if self.pending_clipboard_fetch == Some(serial) => match value.try_as_string() {
-				Ok(content) => {
-					self.pending_clipboard_fetch = None;
-					let message = DesktopWrapperMessage::ClipboardReadResult { content: Some(content) };
-					self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
+			WindowEvent::DragDropped { id, .. } => {
+				if let Err(e) = self.transfer_reads.start(event_loop, id, IngestAction::Import) {
+					tracing::error!("Failed to fetch dropped data: {e}");
 				}
-				Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {}
-				Err(e) => {
-					self.pending_clipboard_fetch = None;
-					tracing::error!("Failed to read from clipboard: {e}");
-					let message = DesktopWrapperMessage::ClipboardReadResult { content: None };
+			}
+			WindowEvent::DataTransferReceived { serial, ref value, .. } => {
+				if let Some((content, action)) = self.transfer_reads.receive(serial, value.as_ref()) {
+					let message = DesktopWrapperMessage::Ingest { content, action };
 					self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
 				}
-			},
-			WindowEvent::DataTransferReceived { serial, ref value, .. } if self.pending_dnd_fetch == Some(serial) => match value.try_as_uris() {
-				Ok(uris) => {
-					self.pending_dnd_fetch = None;
-					let paths = uris.iter().filter_map(|uri| url::Url::parse(uri).ok()?.to_file_path().ok());
-					for path in paths {
-						match fs::read(&path) {
-							Ok(content) => {
-								let message = DesktopWrapperMessage::IngestFile {
-									path,
-									content,
-									action: IngestAction::Import,
-								};
-								self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
-							}
-							Err(e) => {
-								tracing::error!("Failed to read dropped file {}: {}", path.display(), e);
-								return;
-							}
-						};
-					}
-				}
-				Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {}
-				Err(e) => {
-					self.pending_dnd_fetch = None;
-					tracing::error!("Failed to read dropped data: {e}");
-				}
-			},
+			}
 
 			WindowEvent::PointerMoved { .. } | WindowEvent::PointerLeft { position: Some(_), .. } | WindowEvent::PointerEntered { .. }
 				if !self.input_state.pointer_locked() && self.window_pending_drag =>

desktop/src/data_transfer.rs

@@ -0,0 +1,120 @@
+use std::fs;
+use std::io;
+use std::path::PathBuf;
+use winit::data_transfer::{DataTransferId, DataTransferSend, DataTransferSendBuilder, SendData, TypeHint, TypedData};
+use winit::error::TransferError;
+use winit::event_loop::{ActiveEventLoop, AsyncRequestSerial};
+
+use crate::wrapper::messages::{DataTransfer, DataTransferItem, DataType, IngestAction, RawData};
+
+fn type_hint(data_type: DataType) -> Option<TypeHint> {
+	match data_type {
+		DataType::Text => Some(TypeHint::Plaintext),
+		DataType::Svg | DataType::Raster(_) => Some(TypeHint::Image {
+			extension_hint: Some(data_type.extensions().first().copied()?),
+		}),
+		_ => None,
+	}
+}
+
+fn data_type(hint: TypeHint) -> Option<DataType> {
+	match hint {
+		TypeHint::Plaintext => Some(DataType::Text),
+		TypeHint::Image { extension_hint: Some(extension) } => match DataType::from_extension(extension) {
+			data_type @ (DataType::Svg | DataType::Raster(_)) => Some(data_type),
+			_ => None,
+		},
+		_ => None,
+	}
+}
+
+pub(crate) fn send(content: DataTransfer) -> Box<dyn DataTransferSend> {
+	let mut builder = DataTransferSendBuilder::new(());
+	for item in content.0 {
+		let Some((data_type, data)) = item.into_raw() else { continue };
+		let Some(hint) = type_hint(data_type) else { continue };
+		let data = match (hint, data) {
+			(TypeHint::Plaintext, RawData::Text(text)) => SendData::String(text),
+			(TypeHint::Plaintext, RawData::Bytes(_)) => continue,
+			(_, data) => SendData::Bytes(data.into_bytes()),
+		};
+		builder.add_type(hint, move |_, _| Some(data.clone()));
+	}
+	builder.build()
+}
+
+pub(crate) fn read_files(paths: impl IntoIterator<Item = PathBuf>) -> Vec<DataTransferItem> {
+	paths
+		.into_iter()
+		.filter_map(|path| match fs::read(&path) {
+			Ok(data) => Some(DataTransferItem::from_file(path, None, data)),
+			Err(e) => {
+				tracing::error!("Failed to read file {}: {e}", path.display());
+				None
+			}
+		})
+		.collect()
+}
+
+#[derive(Default)]
+pub(crate) struct TransferReads(Vec<TransferRead>);
+
+struct TransferRead {
+	action: IngestAction,
+	fetches: Vec<(AsyncRequestSerial, TypeHint)>,
+	items: Vec<DataTransferItem>,
+}
+
+impl TransferReads {
+	pub(crate) fn start(&mut self, event_loop: &dyn ActiveEventLoop, id: DataTransferId, action: IngestAction) -> Result<(), TransferError> {
+		let mut hints = Vec::new();
+		for hint in event_loop.data_transfer(id)?.available_types().iter().filter_map(|transfer_type| transfer_type.hint()) {
+			if (hint == TypeHint::UriList || data_type(hint).is_some()) && !hints.contains(&hint) {
+				hints.push(hint);
+			}
+		}
+
+		let mut fetches = Vec::new();
+		for hint in hints {
+			fetches.push((event_loop.fetch_data_transfer(id, &hint)?, hint));
+		}
+
+		if !fetches.is_empty() {
+			self.0.push(TransferRead { action, fetches, items: Vec::new() });
+		}
+		Ok(())
+	}
+
+	pub(crate) fn receive(&mut self, serial: AsyncRequestSerial, value: &dyn TypedData) -> Option<(DataTransfer, IngestAction)> {
+		let (index, fetch) = self
+			.0
+			.iter()
+			.enumerate()
+			.find_map(|(index, read)| Some((index, read.fetches.iter().position(|(fetch, _)| *fetch == serial)?)))?;
+		let read = &mut self.0[index];
+		let hint = read.fetches[fetch].1;
+
+		let received = match hint {
+			TypeHint::UriList => value.try_as_uris().map(|uris| {
+				let paths = uris.iter().filter_map(|uri| url::Url::parse(uri).ok()?.to_file_path().ok());
+				read.items.extend(read_files(paths));
+			}),
+			TypeHint::Plaintext => value.try_as_string().map(|text| read.items.extend(DataTransferItem::from_raw(DataType::Text, RawData::Text(text)))),
+			_ => value
+				.try_as_bytes()
+				.map(|bytes| read.items.extend(data_type(hint).and_then(|data_type| DataTransferItem::from_raw(data_type, RawData::Bytes(bytes))))),
+		};
+		match received {
+			Err(e) if e.kind() == io::ErrorKind::WouldBlock => return None,
+			Err(e) => tracing::error!("Failed to read transferred data: {e}"),
+			Ok(()) => {}
+		}
+		read.fetches.remove(fetch);
+		if !read.fetches.is_empty() {
+			return None;
+		}
+
+		let read = self.0.remove(index);
+		Some((DataTransfer(read.items), read.action))
+	}
+}

desktop/src/event.rs

@@ -1,6 +1,6 @@
 use crate::ui::Cursor;
 use crate::wrapper::Texture;
-use crate::wrapper::messages::DesktopWrapperMessage;
+use crate::wrapper::messages::{DataTransfer, DesktopWrapperMessage};
 
 pub(crate) enum AppEvent {
 	UiUpdate(wgpu::Texture),
@@ -9,10 +9,7 @@ pub(crate) enum AppEvent {
 	DesktopWrapperMessage(DesktopWrapperMessage),
 	ViewportTexture(Option<Texture>),
 	ClipboardRead,
-	ClipboardWrite {
-		content: String,
-		svg: Option<String>,
-	},
+	ClipboardWrite(DataTransfer),
 	Exit,
 	UiCrashed,
 	OpenFiles(Vec<std::path::PathBuf>),

desktop/src/lib.rs

@@ -14,6 +14,7 @@ pub(crate) use graphite_desktop_wrapper as wrapper;
 
 mod app;
 mod cli;
+mod data_transfer;
 mod dirs;
 mod event;
 mod gpu_context;

desktop/wrapper/src/handle_desktop_wrapper_message.rs

@@ -1,4 +1,3 @@
-use graphite_editor::messages::clipboard::utility_types::ClipboardContentRaw;
 use graphite_editor::messages::prelude::*;
 
 use super::DesktopWrapperMessageDispatcher;
@@ -15,13 +14,8 @@ pub(super) fn handle_desktop_wrapper_message(dispatcher: &mut DesktopWrapperMess
 		DesktopWrapperMessage::Input(message) => {
 			dispatcher.queue_editor_message(EditorMessage::InputPreprocessor(message));
 		}
-		DesktopWrapperMessage::IngestFile { path, content, action } => {
-			dispatcher.queue_editor_message(IngestMessage::Ingest {
-				data: content,
-				action,
-				mime_type: String::new(),
-				path: Some(path),
-			});
+		DesktopWrapperMessage::Ingest { content, action } => {
+			dispatcher.queue_editor_message(IngestMessage::Ingest { content, action });
 		}
 		DesktopWrapperMessage::SaveFileDialogResult { path, context } => match context {
 			SaveFileDialogContext::Document { document_id, content } => {
@@ -61,14 +55,6 @@ pub(super) fn handle_desktop_wrapper_message(dispatcher: &mut DesktopWrapperMess
 		}
 		#[cfg(not(target_os = "macos"))]
 		DesktopWrapperMessage::MenuEvent { id: _ } => {}
-		DesktopWrapperMessage::ClipboardReadResult { content } => {
-			if let Some(content) = content {
-				let message = ClipboardMessage::ReadClipboard {
-					content: ClipboardContentRaw::Text(content),
-				};
-				dispatcher.queue_editor_message(message);
-			}
-		}
 		DesktopWrapperMessage::PointerLockMove { x, y } => {
 			let message = AppWindowMessage::PointerLockMove { x, y };
 			dispatcher.queue_editor_message(message);