PRJ-57 Draw notifications and floating toolbar on top of client markdown/JCEF component

projector-agent-common/src/main/kotlin/org/jetbrains/projector/agent/common/Misc.kt

@@ -75,6 +75,8 @@ public fun Constructor<*>.toGetDeclaredMethodFormat(): String {
 
 public fun loadClassWithProjectorLoader(clazz: Class<*>): String = loadClassWithProjectorLoader(clazz.name, true)
 
+public fun loadClassWithProjectorLoader(className: String): String = loadClassWithProjectorLoader(className, true)
+
 private fun loadClassWithProjectorLoader(className: String, trim: Boolean): String = """
         $commonClassLoadCode
           .loadClass("$className")

projector-agent-ij-injector/src/main/kotlin/org/jetbrains/projector/agent/ijInjector/IjJcefTransformer.kt

@@ -23,6 +23,8 @@
  */
 package org.jetbrains.projector.agent.ijInjector
 
+import com.intellij.openapi.actionSystem.ActionGroup
+import com.intellij.openapi.fileEditor.LayoutActionsFloatingToolbar
 import com.intellij.ui.jcef.*
 import javassist.*
 import javassist.expr.*
@@ -38,6 +40,7 @@ import org.jetbrains.projector.common.intellij.buildAtLeast
 import org.jetbrains.projector.ij.jcef.CefHandlers
 import org.jetbrains.projector.ij.jcef.ProjectorCefBrowser
 import org.jetbrains.projector.util.loading.state.IdeState
+import javax.swing.JComponent
 
 internal object IjJcefTransformer : IdeTransformerSetup<IjInjector.AgentParameters>() {
 
@@ -64,6 +67,13 @@ internal object IjJcefTransformer : IdeTransformerSetup<IjInjector.AgentParamete
       transformations += JBCefBrowserBase::class.java to ::transformJBCefBrowserBase
     }
 
+    @Suppress("UnstableApiUsage")
+    transformations += HwFacadeHelper::class.java to ::transformHwFacadeHelper
+
+    if (buildAtLeast("213")) {
+      transformations += LayoutActionsFloatingToolbar::class.java to ::transformLayoutActionsFloatingToolbar
+    }
+
     return transformations
   }
 
@@ -74,6 +84,95 @@ internal object IjJcefTransformer : IdeTransformerSetup<IjInjector.AgentParamete
   override val loadingState: IdeState
     get() = IdeState.CONFIGURATION_STORE_INITIALIZED
 
+  private fun createPWindow(component: String) = """
+    {
+      Class pWindowClass = ${loadClassWithProjectorLoader("org.jetbrains.projector.awt.PWindow")};
+      Object pWindow = pWindowClass
+        .getDeclaredConstructor(new Class[]{java.awt.Component.class, boolean.class})
+        .newInstance(new Object[] { $component, Boolean.valueOf($isAgent) });
+    }
+  """.trimIndent()
+
+  private fun disposePWindow(component: String) = """
+    {
+      Class pWindowClass = ${loadClassWithProjectorLoader("org.jetbrains.projector.awt.PWindow")};
+      pWindowClass
+        .getDeclaredMethod("disposeWindow", new Class[]{java.awt.Component.class})
+        .invoke(null, new Object[] { $component });
+    }
+  """.trimIndent()
+
+  private fun useGraphics(component: String, useGraphics: (String) -> String) = """
+    {
+      Class pWindowClass = ${loadClassWithProjectorLoader("org.jetbrains.projector.awt.PWindow")};
+      Object pWindow = pWindowClass
+        .getDeclaredMethod("getWindow", new Class[]{java.awt.Component.class})
+        .invoke(null, new Object[]{$component});
+      if (pWindow != null) {
+        java.awt.Graphics2D windowGraphics = (java.awt.Graphics2D) pWindowClass
+          .getDeclaredMethod("getGraphics", new Class[]{})
+          .invoke(pWindow, new Object[]{});
+        ${useGraphics("windowGraphics")}  
+      }
+    }
+  """.trimIndent()
+
+  private fun transformLayoutActionsFloatingToolbar(ctClass: CtClass): ByteArray {
+
+    ctClass
+      .getDeclaredConstructor(JComponent::class.java, ActionGroup::class.java)
+      .insertAfter(createPWindow("this"))
+
+    ctClass
+      .getDeclaredMethod("dispose")
+      .insertBefore(disposePWindow("this"))
+
+    listOf(
+      "paintComponent",
+      "paintChildren",
+    ).forEach { methodName ->
+      ctClass
+        .getDeclaredMethod(methodName)
+        .apply {
+          if (isAgent) {
+            insertAfter(useGraphics("this") { g ->
+              """
+                if ($g != $1) {
+                  $methodName($g);
+                }
+              """.trimIndent()
+            })
+          } else {
+            insertBefore(useGraphics("this") { g -> "$1 = $g;" })
+          }
+        }
+    }
+
+    return ctClass.toBytecode()
+  }
+
+  private fun transformHwFacadeHelper(clazz: CtClass): ByteArray {
+
+    clazz
+      .getDeclaredMethod("paint")
+      .setBodyOrInsertAfter(useGraphics("myTarget") { g ->
+        """
+          $g.clearRect(0, 0, myTarget.getWidth(), myTarget.getHeight());
+          $2.accept($g);
+        """.trimIndent()
+      })
+
+    clazz
+      .getDeclaredMethod("addNotify")
+      .setBody(createPWindow("myTarget"))
+
+    clazz
+      .getDeclaredMethod("removeNotify")
+      .setBody(disposePWindow("myTarget"))
+
+    return clazz.toBytecode()
+  }
+
   private fun transformJBSchemeHandlerFactory(clazz: CtClass): ByteArray {
 
     if (isAgent) {

projector-client-common/src/commonMain/kotlin/org/jetbrains/projector/client/common/Renderer.kt

@@ -388,6 +388,14 @@ class Renderer(private val renderingSurface: RenderingSurface) {
     }
   }
 
+  fun clearRect(x: Double, y: Double, width: Double, height: Double) {
+    ensureClip()
+    ensureTransform()
+    ensureComposite()
+
+    ctx.clearRect(x, y, width, height)
+  }
+
   fun paintPolygon(paintType: PaintType, points: List<Point>) {
     ensureClip()
     ensureTransform()

projector-client-common/src/commonMain/kotlin/org/jetbrains/projector/client/common/SingleRenderingSurfaceProcessor.kt

@@ -129,6 +129,13 @@ class SingleRenderingSurfaceProcessor(private val renderingSurface: RenderingSur
             r2 = arcHeight.toDouble()
           )
 
+          is ServerClearRectEvent -> renderer.clearRect(
+            x = x,
+            y = y,
+            width = width,
+            height = height,
+          )
+
           is ServerDrawImageEvent -> {
             val image = imageCacher.getImageData(imageId)
 

projector-client-common/src/commonMain/kotlin/org/jetbrains/projector/client/common/canvas/buffering/DoubleBufferedRenderingSurface.kt

@@ -49,7 +49,11 @@ class DoubleBufferedRenderingSurface(bufferCanvasFactory: CanvasFactory, private
     // optimization: flush only if the buffer is changed
     if (buffer.changed) {
       buffer.resetChanged()
-      target.context2d.drawImage(buffer.imageSource, 0.0, 0.0)
+      target.context2d.apply {
+        // clear canvas so that semi-transparent parts won't stack on top of each other
+        clearRect(0.0, 0.0, buffer.width.toDouble(), buffer.height.toDouble())
+        drawImage(buffer.imageSource, 0.0, 0.0)
+      }
     }
   }
 

projector-client-common/src/jsMain/kotlin/org/jetbrains/projector/client/common/canvas/DomContext2d.kt

@@ -95,7 +95,7 @@ internal class DomContext2d(private val myContext2d: CanvasRenderingContext2D) :
       CompositeOperationType.CLEAR,
       CompositeOperationType.DST,
       -> "source-over".also {
-        logger.info { "Missing implementation for $this, applying source-over" }
+        logger.info { "Missing implementation for $type, applying $it" }
       }
     }
   }

projector-client-web/src/main/kotlin/org/jetbrains/projector/client/web/component/ClientComponent.kt

@@ -25,16 +25,24 @@ package org.jetbrains.projector.client.web.component
 
 import kotlinx.browser.document
 import org.w3c.dom.HTMLIFrameElement
+import org.w3c.dom.events.Event
+import org.w3c.dom.events.EventListener
+import org.w3c.dom.events.MouseEvent
+import org.w3c.dom.events.MouseEventInit
 
 abstract class ClientComponent(
   protected val id: Int,
+  private val onMouseMove: (Event) -> Unit,
 ) {
 
+  private val contentDocumentListeners = mutableMapOf<String, EventListener>()
+
   val iFrame: HTMLIFrameElement = createIFrame(id)
 
   var windowId: Int? = null
 
   fun dispose() {
+    contentDocumentListeners.forEach { iFrame.contentDocument?.removeEventListener(it.key, it.value) }
     iFrame.remove()
   }
 
@@ -60,6 +68,10 @@ abstract class ClientComponent(
     }
 
     contentDocument!!.oncontextmenu = { false }
+
+    addEventListener("load", EventListener {
+      onContentChanged()
+    })
   }
 
   protected fun setLinkProcessor(linkProcessor: (String) -> Unit) {
@@ -92,6 +104,49 @@ abstract class ClientComponent(
     }
   }
 
+  private fun onContentChanged() {
+    contentDocumentListeners.forEach { iFrame.contentDocument!!.addEventListener(it.key, it.value) }
+  }
+
   private fun getIFrameId(browserId: Int) = "${this::class.simpleName}$browserId"
 
+  init {
+    contentDocumentListeners["mousemove"] = EventListener {
+
+      val mouseEventInit = with(it as MouseEvent) {
+        MouseEventInit(
+          screenX = screenX,
+          screenY = screenY,
+          clientX = clientX + iFrame.offsetLeft,
+          clientY = clientY + iFrame.offsetTop,
+          button = button,
+          buttons = buttons,
+          relatedTarget = relatedTarget,
+          region = region,
+          ctrlKey = ctrlKey,
+          shiftKey = shiftKey,
+          altKey = altKey,
+          metaKey = metaKey,
+          modifierAltGraph = getModifierState("AltGraph"),
+          modifierCapsLock = getModifierState("CapsLock"),
+          modifierFn = getModifierState("Fn"),
+          modifierFnLock = getModifierState("FnLock"),
+          modifierHyper = getModifierState("Hyper"),
+          modifierNumLock = getModifierState("NumLock"),
+          modifierScrollLock = getModifierState("ScrollLock"),
+          modifierSuper = getModifierState("Super"),
+          modifierSymbol = getModifierState("Symbol"),
+          modifierSymbolLock = getModifierState("SymbolLock"),
+          view = view,
+          detail = detail,
+          bubbles = bubbles,
+          cancelable = cancelable,
+          composed = composed,
+        )
+      }
+      onMouseMove(MouseEvent(it.type, mouseEventInit))
+
+    }
+    onContentChanged()
+  }
 }

projector-client-web/src/main/kotlin/org/jetbrains/projector/client/web/component/EmbeddedBrowserManager.kt

@@ -26,14 +26,17 @@ package org.jetbrains.projector.client.web.component
 import kotlinx.browser.document
 import org.jetbrains.projector.client.web.UriHandler
 import org.w3c.dom.HTMLScriptElement
+import org.w3c.dom.events.Event
+import org.w3c.dom.events.EventListener
 import org.w3c.dom.events.MouseEvent
 
 class EmbeddedBrowserManager(
   zIndexByWindowIdGetter: (Int) -> Int?,
+  private val onMouseMove: (Event) -> Unit,
   private val openInExternalBrowser: (String) -> Unit,
 ) : ClientComponentManager<EmbeddedBrowserManager.EmbeddedBrowserPanel>(zIndexByWindowIdGetter) {
 
-  class EmbeddedBrowserPanel(id: Int, private val openInExternalBrowser: (String) -> Unit) : ClientComponent(id) {
+  inner class EmbeddedBrowserPanel(id: Int, private val openInExternalBrowser: (String) -> Unit) : ClientComponent(id, onMouseMove) {
 
     var wasLoaded = false
 
@@ -44,17 +47,15 @@ class EmbeddedBrowserManager(
     private val jsQueue = ArrayDeque<String>()
 
     init {
-      iFrame.apply {
-        onload = {
-          setOpenLinksInExternalBrowser(openLinksInExternalBrowser, true)
-
-          wasLoaded = true
-          while (jsQueue.isNotEmpty()) {
-            val code = jsQueue.removeFirst()
-            executeJsImpl(code)
-          }
+      iFrame.addEventListener("load", EventListener {
+        setOpenLinksInExternalBrowser(openLinksInExternalBrowser, true)
+
+        wasLoaded = true
+        while (jsQueue.isNotEmpty()) {
+          val code = jsQueue.removeFirst()
+          executeJsImpl(code)
         }
-      }
+      })
     }
 
     fun setHtml(html: String) {

projector-client-web/src/main/kotlin/org/jetbrains/projector/client/web/component/MarkdownPanelManager.kt

@@ -25,19 +25,18 @@ package org.jetbrains.projector.client.web.component
 
 import kotlinx.dom.clear
 import org.jetbrains.projector.util.logging.Logger
-import org.w3c.dom.HTMLElement
-import org.w3c.dom.HTMLIFrameElement
-import org.w3c.dom.Node
-import org.w3c.dom.get
+import org.w3c.dom.*
+import org.w3c.dom.events.Event
 import org.w3c.dom.parsing.DOMParser
 import kotlin.math.absoluteValue
 
 class MarkdownPanelManager(
   zIndexByWindowIdGetter: (Int) -> Int?,
+  private val onMouseMove: (Event) -> Unit,
   private val openInExternalBrowser: (String) -> Unit,
 ) : ClientComponentManager<MarkdownPanelManager.MarkdownPanel>(zIndexByWindowIdGetter) {
 
-  class MarkdownPanel(id: Int, openInExternalBrowser: (String) -> Unit) : ClientComponent(id) {
+  inner class MarkdownPanel(id: Int, openInExternalBrowser: (String) -> Unit) : ClientComponent(id, onMouseMove) {
 
     init {
       setLinkProcessor(openInExternalBrowser)

projector-client-web/src/main/kotlin/org/jetbrains/projector/client/web/input/InputController.kt

@@ -75,7 +75,7 @@ class InputController(
   private var lastTouchX = 1
   private var lastTouchY = 1
 
-  private fun handleMouseMoveEvent(event: Event) {
+  fun handleMouseMoveEvent(event: Event) {
     require(event is MouseEvent)
 
     val topWindow = windowManager.getTopWindow(event.clientX, event.clientY)

projector-client-web/src/main/kotlin/org/jetbrains/projector/client/web/protocol/ManualJsonToClientMessageDecoder.kt

@@ -158,6 +158,7 @@ object ManualJsonToClientMessageDecoder : ToClientMessageDecoder {
       "a" -> WindowType.WINDOW
       "b" -> WindowType.POPUP
       "c" -> WindowType.IDEA_WINDOW
+      "d" -> WindowType.FAKE_WINDOW
       else -> throw IllegalArgumentException("Unsupported window type: $this")
     }
   }
@@ -286,6 +287,13 @@ object ManualJsonToClientMessageDecoder : ToClientMessageDecoder {
 
       "u" -> Flush
 
+      "v" -> ServerClearRectEvent(
+        content["a"] as Double,
+        content["b"] as Double,
+        content["c"] as Double,
+        content["d"] as Double,
+      )
+
       else -> throw IllegalArgumentException("Unsupported event type: ${JSON.stringify(this)}")
     }
   }

projector-client-web/src/main/kotlin/org/jetbrains/projector/client/web/state/ClientState.kt

@@ -408,11 +408,11 @@ sealed class ClientState {
       true -> Typing.SpeculativeTyping(windowManager::getWindowCanvas)
     }
 
-    private val markdownPanelManager = MarkdownPanelManager(windowManager::getWindowZIndex) { link ->
+    private val markdownPanelManager = MarkdownPanelManager(windowManager::getWindowZIndex, inputController::handleMouseMoveEvent) { link ->
       stateMachine.fire(ClientAction.AddEvent(ClientOpenLinkEvent(link)))
     }
 
-    private val embeddedBrowserManager = EmbeddedBrowserManager(windowManager::getWindowZIndex) { link ->
+    private val embeddedBrowserManager = EmbeddedBrowserManager(windowManager::getWindowZIndex, inputController::handleMouseMoveEvent) { link ->
       stateMachine.fire(ClientAction.AddEvent(ClientOpenLinkEvent(link)))
     }