Desktop: Fix Eyedropper tool (#3764)

Author: timon-schelling

desktop/src/app.rs

@@ -74,7 +74,7 @@ impl App {
 			loop {
 				let result = runtime.block_on(DesktopWrapper::execute_node_graph());
 				rendering_app_event_scheduler.schedule(AppEvent::NodeGraphExecutionResult(result));
-				let _ = start_render_receiver.recv();
+				let _ = start_render_receiver.recv_timeout(Duration::from_millis(10));
 			}
 		});
 

editor/src/dispatcher.rs

@@ -51,6 +51,7 @@ const SIDE_EFFECT_FREE_MESSAGES: &[MessageDiscriminant] = &[
 		NodeGraphMessageDiscriminant::RunDocumentGraph,
 	))),
 	MessageDiscriminant::Portfolio(PortfolioMessageDiscriminant::SubmitActiveGraphRender),
+	MessageDiscriminant::Portfolio(PortfolioMessageDiscriminant::SubmitEyedropperPreviewRender),
 	MessageDiscriminant::Frontend(FrontendMessageDiscriminant::TriggerFontDataLoad),
 	MessageDiscriminant::Frontend(FrontendMessageDiscriminant::UpdateUIScale),
 ];

editor/src/messages/portfolio/portfolio_message_handler.rs

@@ -1185,6 +1185,7 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 				let Some(document) = self.documents.get_mut(&document_id) else { return };
 
 				let resolution = glam::UVec2::splat(EYEDROPPER_PREVIEW_AREA_RESOLUTION);
+				let scale = viewport.scale();
 
 				let preview_offset_in_viewport = ipp.mouse.position - (glam::DVec2::splat(EYEDROPPER_PREVIEW_AREA_RESOLUTION as f64 / 2.));
 				let preview_offset_in_viewport = DAffine2::from_translation(preview_offset_in_viewport);
@@ -1196,7 +1197,7 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 
 				let result = self
 					.executor
-					.submit_eyedropper_preview(document_id, preview_transform, pointer_position, resolution, timing_information);
+					.submit_eyedropper_preview(document_id, preview_transform, pointer_position, resolution, scale, timing_information);
 
 				match result {
 					Err(description) => {

editor/src/messages/tool/tool_messages/eyedropper_tool.rs

@@ -48,10 +48,10 @@ impl<'a> MessageHandler<ToolMessage, &mut ToolActionMessageContext<'a>> for Eyed
 		if let ToolMessage::Eyedropper(EyedropperToolMessage::PreviewImage { data, width, height }) = message {
 			let image = EyedropperPreviewImage { data, width, height };
 
-			update_cursor_preview_common(responses, Some(image), context.input, context.global_tool_data, self.data.color_choice.clone());
-
 			if !self.data.preview {
 				disable_cursor_preview(responses, &mut self.data);
+			} else {
+				update_cursor_preview_common(responses, Some(image), context.input, context.global_tool_data, self.data.color_choice.clone());
 			}
 			return;
 		}

editor/src/node_graph_executor.rs

@@ -7,6 +7,7 @@ use graph_craft::proto::GraphErrors;
 use graph_craft::wasm_application_io::EditorPreferences;
 use graphene_std::application_io::{NodeGraphUpdateMessage, RenderConfig};
 use graphene_std::application_io::{SurfaceFrame, TimingInformation};
+use graphene_std::raster::{CPU, Raster};
 use graphene_std::renderer::{RenderMetadata, format_transform_matrix};
 use graphene_std::text::FontCache;
 use graphene_std::transform::Footprint;
@@ -44,6 +45,7 @@ pub struct CompilationResponse {
 pub enum NodeGraphUpdate {
 	ExecutionResponse(ExecutionResponse),
 	CompilationResponse(CompilationResponse),
+	EyedropperPreview(Raster<CPU>),
 	NodeGraphUpdateMessage(NodeGraphUpdateMessage),
 }
 
@@ -59,7 +61,6 @@ pub struct NodeGraphExecutor {
 
 #[derive(Debug, Clone)]
 struct ExecutionContext {
-	render_config: RenderConfig,
 	export_config: Option<ExportConfig>,
 	document_id: DocumentId,
 }
@@ -164,14 +165,7 @@ impl NodeGraphExecutor {
 		// Execute the node graph
 		let execution_id = self.queue_execution(render_config);
 
-		self.futures.push_back((
-			execution_id,
-			ExecutionContext {
-				render_config,
-				export_config: None,
-				document_id,
-			},
-		));
+		self.futures.push_back((execution_id, ExecutionContext { export_config: None, document_id }));
 
 		Ok(DeferMessage::SetGraphSubmissionIndex { execution_id }.into())
 	}
@@ -194,15 +188,23 @@ impl NodeGraphExecutor {
 	}
 
 	#[cfg(not(target_family = "wasm"))]
-	pub(crate) fn submit_eyedropper_preview(&mut self, document_id: DocumentId, transform: DAffine2, pointer: DVec2, resolution: UVec2, time: TimingInformation) -> Result<Message, String> {
+	pub(crate) fn submit_eyedropper_preview(
+		&mut self,
+		document_id: DocumentId,
+		transform: DAffine2,
+		pointer: DVec2,
+		viewport_resolution: UVec2,
+		viewport_scale: f64,
+		time: TimingInformation,
+	) -> Result<Message, String> {
 		let viewport = Footprint {
 			transform,
-			resolution,
+			resolution: viewport_resolution,
 			..Default::default()
 		};
 		let render_config = RenderConfig {
 			viewport,
-			scale: 1.,
+			scale: viewport_scale,
 			time,
 			pointer,
 			export_format: graphene_std::application_io::ExportFormat::Raster,
@@ -215,14 +217,7 @@ impl NodeGraphExecutor {
 		// Execute the node graph
 		let execution_id = self.queue_execution(render_config);
 
-		self.futures.push_back((
-			execution_id,
-			ExecutionContext {
-				render_config,
-				export_config: None,
-				document_id,
-			},
-		));
+		self.futures.push_back((execution_id, ExecutionContext { export_config: None, document_id }));
 
 		Ok(DeferMessage::SetGraphSubmissionIndex { execution_id }.into())
 	}
@@ -272,7 +267,6 @@ impl NodeGraphExecutor {
 		self.futures.push_back((
 			execution_id,
 			ExecutionContext {
-				render_config,
 				export_config: Some(export_config),
 				document_id,
 			},
@@ -325,9 +319,6 @@ impl NodeGraphExecutor {
 					if let Some(export_config) = execution_context.export_config {
 						// Special handling for exporting the artwork
 						self.process_export(node_graph_output, export_config, responses)?;
-					} else if execution_context.render_config.for_eyedropper {
-						// Special handling for Eyedropper tool preview
-						self.process_eyedropper_preview(node_graph_output, responses)?;
 					} else {
 						self.process_node_graph_output(node_graph_output, responses)?;
 					}
@@ -371,6 +362,10 @@ impl NodeGraphExecutor {
 					});
 					responses.add(NodeGraphMessage::SendGraph);
 				}
+				NodeGraphUpdate::EyedropperPreview(raster) => {
+					let (data, width, height) = raster.to_flat_u8();
+					responses.add(EyedropperToolMessage::PreviewImage { data, width, height });
+				}
 			}
 		}
 
@@ -431,23 +426,6 @@ impl NodeGraphExecutor {
 		Ok(())
 	}
 
-	fn process_eyedropper_preview(&self, node_graph_output: TaggedValue, responses: &mut VecDeque<Message>) -> Result<(), String> {
-		match node_graph_output {
-			#[cfg(feature = "gpu")]
-			TaggedValue::RenderOutput(RenderOutput {
-				data: RenderOutputType::Buffer { data, width, height },
-				..
-			}) => {
-				responses.add(EyedropperToolMessage::PreviewImage { data, width, height });
-			}
-			_ => {
-				// TODO: Support Eyedropper preview in SVG mode on desktop
-			}
-		};
-
-		Ok(())
-	}
-
 	fn process_export(&self, node_graph_output: TaggedValue, export_config: ExportConfig, responses: &mut VecDeque<Message>) -> Result<(), String> {
 		let ExportConfig {
 			file_type,

editor/src/node_graph_executor/runtime.rs

@@ -99,6 +99,10 @@ impl InternalNodeGraphUpdateSender {
 	fn send_execution_response(&self, response: ExecutionResponse) {
 		self.0.send(NodeGraphUpdate::ExecutionResponse(response)).expect("Failed to send response")
 	}
+
+	fn send_eyedropper_preview(&self, raster: Raster<CPU>) {
+		self.0.send(NodeGraphUpdate::EyedropperPreview(raster)).expect("Failed to send response")
+	}
 }
 
 impl NodeGraphUpdateSender for InternalNodeGraphUpdateSender {
@@ -159,13 +163,22 @@ impl NodeRuntime {
 		let mut font = None;
 		let mut preferences = None;
 		let mut graph = None;
+		let mut eyedropper = None;
 		let mut execution = None;
 		for request in self.receiver.try_iter() {
 			match request {
 				GraphRuntimeRequest::GraphUpdate(_) => graph = Some(request),
 				GraphRuntimeRequest::ExecutionRequest(ref execution_request) => {
+					if execution_request.render_config.for_eyedropper {
+						eyedropper = Some(request);
+
+						continue;
+					}
+
 					let for_export = execution_request.render_config.for_export;
+
 					execution = Some(request);
+
 					// If we get an export request we always execute it immedeatly otherwise it could get deduplicated
 					if for_export {
 						break;
@@ -175,7 +188,16 @@ impl NodeRuntime {
 				GraphRuntimeRequest::EditorPreferencesUpdate(_) => preferences = Some(request),
 			}
 		}
-		let requests = [font, preferences, graph, execution].into_iter().flatten();
+
+		// Eydropper should use the same time and pointer to not invalidate the cache
+		if let Some(GraphRuntimeRequest::ExecutionRequest(eyedropper)) = &mut eyedropper
+			&& let Some(GraphRuntimeRequest::ExecutionRequest(execution)) = &execution
+		{
+			eyedropper.render_config.time = execution.render_config.time;
+			eyedropper.render_config.pointer = execution.render_config.pointer;
+		}
+
+		let requests = [font, preferences, graph, eyedropper, execution].into_iter().flatten();
 
 		for request in requests {
 			match request {
@@ -236,7 +258,9 @@ impl NodeRuntime {
 					let result = self.execute_network(render_config).await;
 					let mut responses = VecDeque::new();
 					// TODO: Only process monitor nodes if the graph has changed, not when only the Footprint changes
-					self.process_monitor_nodes(&mut responses, self.update_thumbnails);
+					if !render_config.for_eyedropper {
+						self.process_monitor_nodes(&mut responses, self.update_thumbnails);
+					}
 					self.update_thumbnails = false;
 
 					// Resolve the result from the inspection by accessing the monitor node
@@ -246,7 +270,7 @@ impl NodeRuntime {
 						Ok(TaggedValue::RenderOutput(RenderOutput {
 							data: RenderOutputType::Texture(image_texture),
 							metadata,
-						})) if render_config.for_export || render_config.for_eyedropper => {
+						})) if render_config.for_export => {
 							let executor = self
 								.editor_api
 								.application_io
@@ -267,6 +291,23 @@ impl NodeRuntime {
 								None,
 							)
 						}
+						Ok(TaggedValue::RenderOutput(RenderOutput {
+							data: RenderOutputType::Texture(image_texture),
+							metadata: _,
+						})) if render_config.for_eyedropper => {
+							let executor = self
+								.editor_api
+								.application_io
+								.as_ref()
+								.unwrap()
+								.gpu_executor()
+								.expect("GPU executor should be available when we receive a texture");
+
+							let raster_cpu = Raster::new_gpu(image_texture.texture).convert(Footprint::BOUNDLESS, executor).await;
+
+							self.sender.send_eyedropper_preview(raster_cpu);
+							continue;
+						}
 						#[cfg(all(target_family = "wasm", feature = "gpu"))]
 						Ok(TaggedValue::RenderOutput(RenderOutput {
 							data: RenderOutputType::Texture(image_texture),

node-graph/libraries/wgpu-executor/src/texture_conversion.rs

@@ -100,12 +100,15 @@ impl RasterGpuToRasterCpuConverter {
 		}
 	}
 
-	async fn convert(self) -> Result<Raster<CPU>, wgpu::BufferAsyncError> {
+	async fn convert(self, device: &std::sync::Arc<wgpu::Device>) -> Result<Raster<CPU>, wgpu::BufferAsyncError> {
 		let buffer_slice = self.buffer.slice(..);
 		let (sender, receiver) = futures::channel::oneshot::channel();
 		buffer_slice.map_async(wgpu::MapMode::Read, move |result| {
 			let _ = sender.send(result);
 		});
+
+		let _ = device.poll(wgpu::wgt::PollType::wait_indefinitely());
+
 		receiver.await.expect("Failed to receive map result")?;
 
 		let view = buffer_slice.get_mapped_range();
@@ -215,7 +218,7 @@ impl<'i> Convert<Table<Raster<CPU>>, &'i WgpuExecutor> for Table<Raster<GPU>> {
 
 		let mut map_futures = Vec::new();
 		for converter in converters {
-			map_futures.push(converter.convert());
+			map_futures.push(converter.convert(device));
 		}
 
 		let map_results = futures::future::try_join_all(map_futures)
@@ -250,7 +253,7 @@ impl<'i> Convert<Raster<CPU>, &'i WgpuExecutor> for Raster<GPU> {
 
 		queue.submit([encoder.finish()]);
 
-		converter.convert().await.expect("Failed to download texture data")
+		converter.convert(device).await.expect("Failed to download texture data")
 	}
 }