diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ad46b30 --- /dev/null +++ b/.gitignore @@ -0,0 +1,61 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + +# next.js build output +.next diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0a04128 --- /dev/null +++ b/LICENSE @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/googleassistant.js b/googleassistant.js new file mode 100644 index 0000000..1011907 --- /dev/null +++ b/googleassistant.js @@ -0,0 +1,52 @@ +'use strict'; + +const path = require('path'); +const grpc = require('grpc'); +const protoFiles = require('google-proto-files'); +const GoogleAuth = require('google-auth-library'); + +//Import proto files +const PROTO_ROOT_DIR = protoFiles('..'); + +const embedded_assistant_pb = grpc.load({ + root: PROTO_ROOT_DIR, + file: path.relative(PROTO_ROOT_DIR, protoFiles.embeddedAssistant.v1alpha2) +}).google.assistant.embedded.v1alpha2; + +class GoogleAssistant { + constructor(credentials) { + GoogleAssistant.prototype.endpoint_ = "embeddedassistant.googleapis.com"; + this.client = this.createClient_(credentials); + this.locale = "en-US"; + this.deviceModelId = 'default'; + this.deviceInstanceId = 'default'; + } + + createClient_(credentials) { + const sslCreds = grpc.credentials.createSsl(); + // https://github.com/google/google-auth-library-nodejs/blob/master/ts/lib/auth/refreshclient.ts + const auth = new GoogleAuth(); + const refresh = new auth.UserRefreshClient(); + refresh.fromJSON(credentials, function (res) { }); + const callCreds = grpc.credentials.createFromGoogleCredential(refresh); + const combinedCreds = grpc.credentials.combineChannelCredentials(sslCreds, callCreds); + const client = new embedded_assistant_pb.EmbeddedAssistant(this.endpoint_, combinedCreds); + return client; + } +} + +// GA auth +const homedir = require('homedir') +const deviceCredentials = require(`${homedir()}/.config/google-oauthlib-tool/credentials.json`); // Path to the Google credentials + +const CREDENTIALS = { + client_id: deviceCredentials.client_id, + client_secret: deviceCredentials.client_secret, + refresh_token: deviceCredentials.refresh_token, + type: "authorized_user" +}; + +var assistant = new GoogleAssistant(CREDENTIALS); + +exports.assistant = assistant; +exports.embedded_assistant_pb = embedded_assistant_pb; \ No newline at end of file diff --git a/index.js b/index.js new file mode 100644 index 0000000..a55edb0 --- /dev/null +++ b/index.js @@ -0,0 +1,181 @@ +'use strict'; + +var GoogleAssistant = require('./googleassistant.js') + +var net = require('net'); +/* [DEBUG] Save audio chunks +const FileWriter = require('wav').FileWriter; +var stream = require('stream'); +*/ + +//Server configuration +const HOST = '0.0.0.0' +const PORT = 4000 + +//GA configuration +const inputSampleRate = 16000; +const inputEncoding = 1; +const outputSampleRate = 16000; +const outputEncoding = 1; + +const assistant = GoogleAssistant.assistant; +const embedded_assistant_pb = GoogleAssistant.embedded_assistant_pb; + +var server = net.createServer(function (client) { + console.log('Client connected:' + client.remoteAddress + ':' + client.remotePort); + + client.setTimeout(3000); + client.setKeepAlive(true); + + //GA start + const config = new embedded_assistant_pb.AssistConfig(); + + //Input + config.setAudioInConfig(new embedded_assistant_pb.AudioInConfig()); + config.getAudioInConfig().setEncoding(inputEncoding); + config.getAudioInConfig().setSampleRateHertz(inputSampleRate); + + //Output + config.setAudioOutConfig(new embedded_assistant_pb.AudioOutConfig()); + config.getAudioOutConfig().setEncoding(outputEncoding); + config.getAudioOutConfig().setSampleRateHertz(outputSampleRate); + config.getAudioOutConfig().setVolumePercentage(100); + + config.setDialogStateIn(new embedded_assistant_pb.DialogStateIn()); + config.setDeviceConfig(new embedded_assistant_pb.DeviceConfig()); + config.getDialogStateIn().setLanguageCode(assistant.locale); + config.getDeviceConfig().setDeviceId(assistant.deviceInstanceId); + config.getDeviceConfig().setDeviceModelId(assistant.deviceModelId); + const request = new embedded_assistant_pb.AssistRequest(); + request.setConfig(config); + delete request.audio_in; + + const conversation = assistant.client.assist(); + + // Send config to GA + conversation.write(request); + + // GA response JSON + let response = {}; + + //[DEBUG] Audio bytes + //let audioTCP = null; + + /* GA data start */ + conversation.on('data', (data) => { + if (data.device_action) { + // Device action + response.deviceAction = JSON.parse(data.device_action.device_request_json); + } else if (data.dialog_state_out !== null && data.dialog_state_out.supplemental_display_text) { + // Response text + response.text = data.dialog_state_out.supplemental_display_text; + response.microphone_mode = data.dialog_state_out.microphone_mode; + response.conversation_state = data.dialog_state_out.conversation_state; + } else if (data.speech_results != [] && data.speech_results.length == 1 && data.speech_results[0].stability == 1) { + // Audio transcript + response.transcript = data.speech_results[0].transcript; + }else if (data.event_type == "END_OF_UTTERANCE") { + // End of utterance + var bf = Buffer.from("0END_OF_UTTERANCE"); + + client.write(bf); + console.log("END_OF_UTTERANCE"); + } + + //[DEBUG] AssistResponse + console.log(data); + }); + + conversation.on('end', (error) => { + // Send json with response to the client + var bf1 = Buffer.from("START_JSON"); + var bf2 = Buffer.from(JSON.stringify(response)); + var bf3 = Buffer.from("STOP_JSON"); + client.write(Buffer.concat([bf1, bf2, bf3])); + }); + + conversation.on('error', (error) => { + console.log(error); + }); + /* GA data end */ + + /* Server start */ + client.on('data', function (data) { + //console.log('Read: ' + client.bytesRead); + + //[DEBUG] Concat audio buffer for save the final audio + /*if(audioTCP == null){ + audioTCP = data + }else{ + audioTCP = Buffer.concat([audioTCP, data]) + }*/ + + // Split audio in smaller chunks if needed + const buf_length = 1024; + var buf_start = 0; + var buf_end = buf_length; + while (true) { + //Create GA request + const audio = new embedded_assistant_pb.AssistRequest(); + audio.setAudioIn(data.slice(buf_start, buf_end)); + + //Send audio to GA + conversation.write(audio); + + buf_start = buf_end; + buf_end += buf_length; + + if (buf_start >= data.length) { + break; + } + } + }); + + client.on('error', (error) => { + console.log(error); + }); + + client.on('end', function () { + console.log('Client disconnect.'); + + //[DEBUG] Save final audio file + /*var outputFileStream = new FileWriter('out.wav', { + sampleRate: 16000, + channels: 1 + }); + + var bufferStream = new stream.PassThrough(); + bufferStream.end(audioTCP); + bufferStream.pipe(outputFileStream)*/ + + conversation.end(); + + server.getConnections(function (err, count) { + if (!err) { + console.log('There are ' + count +' connections.'); + } else { + console.error(JSON.stringify(err)); + } + }); + }); + + client.on('timeout', function () { + console.log('Client request time out. '); + }) + /* Server end */ +}); + + +// TCP server listening +server.listen(PORT, HOST, function () { + + console.log('TCP server listen on address : ' + HOST + ':' + PORT); + + server.on('close', function () { + console.log('TCP server socket is closed.'); + }); + + server.on('error', function (error) { + console.error(JSON.stringify(error)); + }); +}); diff --git a/package.json b/package.json new file mode 100644 index 0000000..8950fca --- /dev/null +++ b/package.json @@ -0,0 +1,21 @@ +{ + "scripts": { + "start": "node index.js", + "demo": "", + "test": "" + }, + "name": "google-assistant-server", + "description": "A simple TCP server that forwards audio chunk to Google Assistant", + "version": "1.0.0", + "dependencies": { + "async": "^1.5.2", + "google-auth-library": "0.12.0", + "google-proto-files": "^0.15.0", + "google-protobuf": "^3.3.0", + "grpc": "^1.0.0", + "homedir": "^0.6.0", + "promise.prototype.finally": "3.1.0", + "resolve": "1.5.0", + "wav": "^1.0.2" + } +} diff --git a/test/test_send_file.py b/test/test_send_file.py new file mode 100644 index 0000000..7eaa55c --- /dev/null +++ b/test/test_send_file.py @@ -0,0 +1,40 @@ +import socket +import re +import json +import wave +import struct + +# TCP client +TCP_IP = '127.0.0.1' +TCP_PORT = 32768 + +waveFile = wave.open("whoisobama.wav", 'rb') + +dataList = [] + +length = waveFile.getnframes() +for i in range(0, length): + data = struct.unpack("