Skip to content

Commit

Permalink
feat: added WebAPI and RPC services
Browse files Browse the repository at this point in the history
  • Loading branch information
AlmasB committed Feb 10, 2024
1 parent 17236fb commit fe19464
Show file tree
Hide file tree
Showing 4 changed files with 221 additions and 66 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* FXGL - JavaFX Game Library. The MIT License (MIT).
* Copyright (c) AlmasB ([email protected]).
* See LICENSE for details.
*/

package com.almasb.fxgl.intelligence

import com.almasb.fxgl.core.concurrent.Async
import com.almasb.fxgl.logging.Logger
import com.almasb.fxgl.net.ws.LocalWebSocketServer
import com.almasb.fxgl.net.ws.RPCService
import javafx.beans.property.ReadOnlyBooleanProperty
import javafx.beans.property.ReadOnlyBooleanWrapper
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.chrome.ChromeOptions

/**
* Provides access to JS-driven implementation.
*
* @author Almas Baim (https://github.com/AlmasB)
*/
abstract class WebAPIService(server: LocalWebSocketServer, private val apiURL: String) : RPCService(server) {

private val log = Logger.get(WebAPIService::class.java)

private val readyProp = ReadOnlyBooleanWrapper(false)

var isReady: Boolean
get() = readyProp.value
protected set(value) { readyProp.value = value }

fun readyProperty(): ReadOnlyBooleanProperty {
return readyProp.readOnlyProperty
}

private var webDriver: WebDriver? = null

/**
* Starts this service in a background thread.
* Can be called after stop() to restart the service.
* If the service has already started, then calls stop() and restarts it.
*/
fun start() {
Async.startAsync {
try {
if (webDriver != null) {
stop()
}

val options = ChromeOptions()
options.addArguments("--headless=new")
options.addArguments("--use-fake-ui-for-media-stream")

webDriver = ChromeDriver(options)
webDriver!!.get(apiURL)

onWebDriverLoaded(webDriver!!)
} catch (e: Exception) {
log.warning("Failed to start Chrome web driver. Ensure Chrome is installed in default location")
log.warning("Error data", e)
}
}
}

/**
* Stops this service.
* No-op if it has not started via start() before.
*/
fun stop() {
try {
if (webDriver != null) {
webDriver!!.quit()
webDriver = null
}
} catch (e: Exception) {
log.warning("Failed to quit web driver", e)
}
}

/**
* Called after the web driver has loaded the page.
*/
protected open fun onWebDriverLoaded(webDriver: WebDriver) { }
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,85 +6,58 @@

package com.almasb.fxgl.intelligence.tts

import com.almasb.fxgl.core.EngineService
import com.almasb.fxgl.core.concurrent.Async
import com.almasb.fxgl.intelligence.WebAPI
import com.almasb.fxgl.intelligence.WebAPIService
import com.almasb.fxgl.logging.Logger
import com.almasb.fxgl.net.ws.LocalWebSocketServer
import com.almasb.fxgl.speechrecog.SpeechRecognitionService
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.chrome.ChromeOptions

/**
* TODO: remove duplicate code
*
* @author Almas Baim (https://github.com/AlmasB)
*/
class TextToSpeechService : EngineService() {
class TextToSpeechService : WebAPIService(
LocalWebSocketServer("TTSServer", WebAPI.TEXT_TO_SPEECH_PORT),
WebAPI.TEXT_TO_SPEECH_API
) {

private val log = Logger.get(TextToSpeechService::class.java)
private val server = LocalWebSocketServer("TTSServer", WebAPI.TEXT_TO_SPEECH_PORT)

private var webDriver: WebDriver? = null
private val synthVoices = arrayListOf<Voice>()
var selectedVoice: Voice = Voice("NULL")

override fun onInit() {
server.start()
}

/**
* Starts this service in a background thread.
* Can be called after stop() to restart the service.
* If the service has already started, then calls stop() and restarts it.
*/
fun start() {
Async.startAsync {
try {
if (webDriver != null) {
stop()
}
val voices: List<Voice>
get() = synthVoices.toList()

val options = ChromeOptions()
options.addArguments("--headless=new")
options.addArguments("--use-fake-ui-for-media-stream")

webDriver = ChromeDriver(options)
webDriver!!.get(WebAPI.TEXT_TO_SPEECH_API)
override fun onWebDriverLoaded(webDriver: WebDriver) {
// force it to play, so the audio output is initialized
webDriver.findElement(By.id("play")).click()
}

// TODO: update web-api impl
// force it to play, so the audio output is initialized
webDriver!!.findElement(By.id("play")).click()
private fun initVoices(voiceNames: List<String>) {
synthVoices += voiceNames.map { Voice(it) }

// we are ready to use the web api service
} catch (e: Exception) {
log.warning("Failed to start Chrome web driver. Ensure Chrome is installed in default location")
log.warning("Error data", e)
}
if (synthVoices.isNotEmpty()) {
selectedVoice = synthVoices[0]
}
}

/**
* Stops this service.
* No-op if it has not started via start() before.
*/
fun stop() {
try {
if (webDriver != null) {
webDriver!!.quit()
webDriver = null
}
} catch (e: Exception) {
log.warning("Failed to quit web driver", e)
Async.startAsyncFX {
isReady = true
}
}

fun speak(text: String) {
server.send(text)
if (!isReady || synthVoices.isEmpty())
return

rpcRun("speak", selectedVoice.name, text)
}

override fun onExit() {
stop()
server.stop()
super.onExit()
}
}
}

data class Voice internal constructor(val name: String)
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*
* FXGL - JavaFX Game Library. The MIT License (MIT).
* Copyright (c) AlmasB ([email protected]).
* See LICENSE for details.
*/

package com.almasb.fxgl.net.ws

import com.almasb.fxgl.core.EngineService
import com.almasb.fxgl.core.reflect.ReflectionFunctionCaller
import com.almasb.fxgl.logging.Logger

private const val SEPARATOR = "*,,*"
private const val FUNCTION_CALL_TAG = "F_CALL:"
private const val FUNCTION_RETURN_TAG = "F_RETURN:"

/**
* Allows a remote application (possibly written in a different language)
* to issue function calls to and accept function calls from subclasses of this service.
*
* @author Almas Baim (https://github.com/AlmasB)
*/
abstract class RPCService(

/**
* The server to which clients connect.
* Maintenance responsibility of the server object lies with this RPC service.
*/
protected val server: LocalWebSocketServer
) : EngineService() {

private val log = Logger.get(RPCService::class.java)

private val rfc = ReflectionFunctionCaller()

init {
server.addMessageHandler { message ->
if (message.startsWith(FUNCTION_CALL_TAG)) {
val funcName = message.substringAfter(FUNCTION_CALL_TAG).substringBefore(SEPARATOR)
val args = message.substringAfter(SEPARATOR)
.split(SEPARATOR)
.filter { it.isNotEmpty() }

rfc.call(funcName, args)
}

if (message.startsWith(FUNCTION_RETURN_TAG)) {
// TODO:
}
}
}

override fun onInit() {
rfc.addFunctionCallTarget(this)
log.debug("Added ${javaClass.simpleName} methods: ${rfc.methods.map { it.name }}")

server.start()
}

fun rpcRun(funcName: String, vararg args: String) {
rpcRun(funcName, args.toList())
}

fun rpcRun(funcName: String, args: List<String>) {
var argsString = ""

args.forEach { argsString += it + SEPARATOR }

if (argsString.isNotEmpty()) {
argsString = argsString.removeSuffix(SEPARATOR)
}

server.send("$FUNCTION_CALL_TAG$funcName$SEPARATOR$argsString")
}

private fun rpcReturn() {
// TODO:
}

override fun onExit() {
server.stop()
}
}
37 changes: 25 additions & 12 deletions fxgl-samples/src/main/java/sandbox/net/TTSSample.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@
import com.almasb.fxgl.app.GameApplication;
import com.almasb.fxgl.app.GameSettings;
import com.almasb.fxgl.intelligence.tts.TextToSpeechService;
import com.almasb.fxgl.speechrecog.SpeechRecognitionService;
import com.almasb.fxgl.ui.FontType;
import javafx.collections.FXCollections;
import javafx.scene.control.Button;
import javafx.scene.control.TextArea;
import javafx.scene.text.Font;

import static com.almasb.fxgl.dsl.FXGL.*;

Expand All @@ -27,38 +28,50 @@ public class TTSSample extends GameApplication {

@Override
protected void initSettings(GameSettings settings) {
settings.addEngineService(SpeechRecognitionService.class);
settings.setWidth(1280);
settings.setHeight(720);
settings.addEngineService(TextToSpeechService.class);
}

@Override
protected void initGame() {
getService(SpeechRecognitionService.class).addInputHandler(input -> {
if (input.isEmpty() || input.trim().isEmpty())
return;

getService(TextToSpeechService.class).speak(input);
});

//getService(SpeechRecognitionService.class).start();
getService(TextToSpeechService.class).start();
}

@Override
protected void initUI() {
output = new TextArea();
output.setWrapText(true);
output.setPrefSize(getAppWidth(), getAppHeight() - 200);
output.setPrefSize(getAppWidth() - 200, getAppHeight() - 200);
output.setFont(getUIFactoryService().newFont(FontType.MONO, 18));

addUINode(output);

var btn = new Button("Speak");
btn.setFont(Font.font(16.0));
btn.setOnAction(e -> {
getService(TextToSpeechService.class).speak(output.getText());
});

addUINode(btn, 50, getAppHeight() - 150);
addUINode(btn, 50, getAppHeight() - 100);

getService(TextToSpeechService.class).readyProperty().addListener((o, old, isReady) -> {
if (isReady) {
System.out.println("TTS service is ready");

var cb = getUIFactoryService().newChoiceBox(FXCollections.observableArrayList(getService(TextToSpeechService.class).getVoices()));
cb.setPrefWidth(400);
cb.getSelectionModel().selectedItemProperty().addListener((observable, oldValue, newValue) -> {
getService(TextToSpeechService.class).setSelectedVoice(newValue);
});

if (!cb.getItems().isEmpty()) {
cb.getSelectionModel().selectFirst();
}

addUINode(cb, 50, getAppHeight() - 150);
}
});
}

public static void main(String[] args) {
Expand Down

0 comments on commit fe19464

Please sign in to comment.