WIP fxgl-intelligence module

fxgl-core/src/main/kotlin/com/almasb/fxgl/core/reflect/ReflectionFunctionCaller.kt

@@ -26,7 +26,7 @@ class ReflectionFunctionCaller {
     private val stringToObject = hashMapOf<Class<*>, (String) -> Any>()
 
     var defaultFunctionHandler: BiFunction<String, List<String>, Any> = BiFunction { name, args ->
-        throw RuntimeException("No function handler for $name with $args")
+        throw RuntimeException("No function handler for $name with args $args")
     }
 
     val methods: List<Method>

fxgl-intelligence/src/main/java/com/almasb/fxgl/intelligence/WebAPI.java

@@ -15,7 +15,7 @@
 public final class WebAPI {
 
     public static final String TEXT_TO_SPEECH_API = "https://almasb.github.io/web-api/text-to-speech-v1/";
-    public static final String SPEECH_RECOGNITION_API = "https://almasb.github.io/web-api/speech-recog/";
+    public static final String SPEECH_RECOGNITION_API = "https://almasb.github.io/web-api/speech-recog-v1/";
     public static final String GESTURE_RECOGNITION_API = "https://almasb.github.io/web-api/gesture-recog-v1/";
 
     public static final int TEXT_TO_SPEECH_PORT = 55550;

fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/gesturerecog/HandTrackingService.kt

@@ -10,7 +10,6 @@ import com.almasb.fxgl.core.EngineService
 import com.almasb.fxgl.core.concurrent.Async
 import com.almasb.fxgl.intelligence.WebAPI
 import com.almasb.fxgl.logging.Logger
-import com.almasb.fxgl.speechrecog.SpeechRecognitionService
 import com.almasb.fxgl.net.ws.LocalWebSocketServer
 import javafx.geometry.Point3D
 import org.openqa.selenium.WebDriver
@@ -25,7 +24,7 @@ import java.util.function.Consumer
  */
 class HandTrackingService : EngineService() {
 
-    private val log = Logger.get(SpeechRecognitionService::class.java)
+    private val log = Logger.get(HandTrackingService::class.java)
     private val server = LocalWebSocketServer("HandTrackingServer", WebAPI.GESTURE_RECOGNITION_PORT)
 
     private var webDriver: WebDriver? = null

fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/intelligence/WebAPIService.kt

@@ -29,12 +29,28 @@ abstract class WebAPIService(server: LocalWebSocketServer, private val apiURL: S
 
     var isReady: Boolean
         get() = readyProp.value
-        protected set(value) { readyProp.value = value }
+        private set(value) { readyProp.value = value }
 
+    /**
+     * @return a property that tracks whether this service is ready to be used
+     * all changes to the property are notified on the JavaFX thread
+     */
     fun readyProperty(): ReadOnlyBooleanProperty {
         return readyProp.readOnlyProperty
     }
 
+    protected fun setReady() {
+        Async.startAsyncFX {
+            isReady = true
+        }
+    }
+
+    protected fun setNotReady() {
+        Async.startAsyncFX {
+            isReady = false
+        }
+    }
+
     private var webDriver: WebDriver? = null
 
     /**
@@ -69,6 +85,8 @@ abstract class WebAPIService(server: LocalWebSocketServer, private val apiURL: S
      * No-op if it has not started via start() before.
      */
     fun stop() {
+        setNotReady()
+
         try {
             if (webDriver != null) {
                 webDriver!!.quit()
@@ -79,6 +97,11 @@ abstract class WebAPIService(server: LocalWebSocketServer, private val apiURL: S
         }
     }
 
+    override fun onExit() {
+        stop()
+        super.onExit()
+    }
+
     /**
      * Called after the web driver has loaded the page.
      */

fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/intelligence/speechrecog/SpeechRecognitionService.kt

@@ -0,0 +1,54 @@
+/*
+ * FXGL - JavaFX Game Library. The MIT License (MIT).
+ * Copyright (c) AlmasB ([email protected]).
+ * See LICENSE for details.
+ */
+
+package com.almasb.fxgl.intelligence.speechrecog
+
+import com.almasb.fxgl.core.concurrent.Async
+import com.almasb.fxgl.intelligence.WebAPI
+import com.almasb.fxgl.intelligence.WebAPIService
+import com.almasb.fxgl.logging.Logger
+import com.almasb.fxgl.net.ws.LocalWebSocketServer
+import java.util.function.BiConsumer
+
+/**
+ * @author Almas Baim (https://github.com/AlmasB)
+ */
+class SpeechRecognitionService : WebAPIService(
+        LocalWebSocketServer("SpeechRecogServer", WebAPI.SPEECH_RECOGNITION_PORT),
+        WebAPI.SPEECH_RECOGNITION_API
+) {
+
+    private val log = Logger.get(SpeechRecognitionService::class.java)
+
+    private val speechInputHandlers = arrayListOf<BiConsumer<String, Double>>()
+
+    /**
+     * Add a [handler] for incoming speech input.
+     */
+    fun addInputHandler(handler: BiConsumer<String, Double>) {
+        speechInputHandlers += handler
+    }
+
+    /**
+     * Remove an existing input handler.
+     */
+    fun removeInputHandler(handler: BiConsumer<String, Double>) {
+        speechInputHandlers -= handler
+    }
+
+    private fun initService() {
+        setReady()
+    }
+
+    // TODO: expand API to also include alternative text options
+    private fun onSpeechInput(text: String, confidence: Double) {
+        log.debug("Received speech input ($confidence): $text")
+
+        Async.startAsyncFX {
+            speechInputHandlers.forEach { it.accept(text, confidence) }
+        }
+    }
+}

fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/intelligence/tts/TextToSpeechService.kt

@@ -42,9 +42,7 @@ class TextToSpeechService : WebAPIService(
             selectedVoice = synthVoices[0]
         }
 
-        Async.startAsyncFX {
-            isReady = true
-        }
+        setReady()
     }
 
     fun speak(text: String) {
@@ -53,11 +51,6 @@ class TextToSpeechService : WebAPIService(
 
         rpcRun("speak", selectedVoice.name, text)
     }
-
-    override fun onExit() {
-        stop()
-        super.onExit()
-    }
 }
 
 data class Voice internal constructor(val name: String)

fxgl-intelligence/src/main/kotlin/com/almasb/fxgl/net/ws/RPCService.kt

@@ -39,6 +39,8 @@ abstract class RPCService(
                 val funcName = message.substringAfter(FUNCTION_CALL_TAG).substringBefore(SEPARATOR)
                 val args = message.substringAfter(SEPARATOR)
                         .split(SEPARATOR)
+                        // TODO: we should just remove the last empty item
+                        // otherwise this could filter empty String that is a genuine argument
                         .filter { it.isNotEmpty() }
 
                 rfc.call(funcName, args)

fxgl-samples/src/main/java/sandbox/net/SpeechRecogSample.java

@@ -6,9 +6,10 @@
 
 package sandbox.net;
 
+import com.almasb.fxgl.app.ApplicationMode;
 import com.almasb.fxgl.app.GameApplication;
 import com.almasb.fxgl.app.GameSettings;
-import com.almasb.fxgl.speechrecog.SpeechRecognitionService;
+import com.almasb.fxgl.intelligence.speechrecog.SpeechRecognitionService;
 import com.almasb.fxgl.ui.FontType;
 import javafx.scene.control.TextArea;
 
@@ -26,14 +27,17 @@ public class SpeechRecogSample extends GameApplication {
     @Override
     protected void initSettings(GameSettings settings) {
         settings.addEngineService(SpeechRecognitionService.class);
+        settings.setApplicationMode(ApplicationMode.DEBUG);
     }
 
     @Override
     protected void initGame() {
-        getService(SpeechRecognitionService.class).addInputHandler(input -> {
-            getExecutor().startAsyncFX(() -> {
-                output.appendText(input + "\n");
-            });
+        getService(SpeechRecognitionService.class).readyProperty().addListener((o, old, isReady) -> {
+            System.out.println("Updated readyProperty: " + isReady);
+        });
+
+        getService(SpeechRecognitionService.class).addInputHandler((input, confidence) -> {
+            output.appendText(String.format("(confidence %.0f) %s\n", confidence*100, input));
         });
 
         getService(SpeechRecognitionService.class).start();