-
Notifications
You must be signed in to change notification settings - Fork 3
Expand file tree
/
Copy pathmain.html
More file actions
233 lines (192 loc) · 8.33 KB
/
main.html
File metadata and controls
233 lines (192 loc) · 8.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
<!DOCTYPE html>
<html lang="en">
<head>
<title>speech-nl-demo</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{{ url_for('static', filename='bootstrap.min.css') }}">
<link rel="stylesheet" href="{{ url_for('static', filename='nprogress.min.css') }}">
<script src="{{ url_for('static', filename='jquery.min.js') }}"></script>
<script src="{{ url_for('static', filename='mustache.min.js') }}"></script>
<script src="{{ url_for('static', filename='MediaStreamRecorder.min.js') }}"></script>
<script src="{{ url_for('static', filename='bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='nprogress.min.js') }}"></script>
<script>
$(document).ready(function() {
$(this).ajaxStart(function() { NProgress.start() });
$(this).ajaxStop(function() { NProgress.done() });
var recordClickCounter = 0;
var stream;
var recorder;
// start/stop recording every other time the button is clicked
$("#record").click(function() {
if(recordClickCounter++ % 2 == 0) {
// user started recording, restyle into 'stop' button
$(this).text("Stop recording");
$(this).addClass("btn-danger");
// set up audio recorder and connect to backend service
navigator.getUserMedia({audio: true},
function(innerStream) {
stream = innerStream;
recorder = new MediaStreamRecorder(stream);
// todo: probably room for optimizations (ideally convert to flac)
recorder.mimeType = "audio/wav";
recorder.sampleRate = 44100;
recorder.audioChannels = 1;
// this event is fired whenever time's up or stop() is called
recorder.ondataavailable = function(audio) {
// wrap audio blob in form data in order to post it to backend
var data = new FormData();
data.append("file", audio);
// post audio blob to backend
$.ajax({
url: "speech",
type: "POST",
data: data,
contentType: false,
processData: false,
success: function(resp) {
// todo: implement proper error handling
var transcript;
try {
transcript = resp.results[0].alternatives[0].transcript;
} catch(error) {
transcript = "\"\"";
}
// display transcript
$("#output").prepend(Mustache.render(
$("#snippet-tmpl").html(),
{ snippet: transcript }
));
// create modal showing raw json response
$("#output blockquote:first div:first").append(Mustache.render(
$("#modal-tmpl").html(),
{ buttonTitle: "speech json",
buttonClass: "btn-success",
modalId: "speech-modal-" + $("#output blockquote").length,
modalTitle: "Cloud Speech API raw response",
json: JSON.stringify(resp, null, 2) }
));
// enable analyze button now when there's something to analyze
$("#analyze").prop("disabled", false);
}
});
};
// start the actual recording, run for 60 secs max
recorder.start(60000);
},
function(e) {
console.error("Couldn't connect to user's audio input", e);
}
);
} else {
// user stopped recording, restyle into 'start' button
$(this).text("Record audio");
$(this).removeClass("btn-danger");
// kill recording and stop hogging user's microphone
recorder.stop();
stream.stop();
}
});
$("#analyze").click(function() {
// grab last available text snippet
var string = $("#output blockquote:first p").text();
$.post("language", {string: string}, function(resp) {
// disable analyze button to prevent analysis on the analysis
$("#analyze").prop("disabled", true);
// build a temporary helper structure to help annotate text snippet
var replacements = [];
for (let entity of resp.entities) {
for (let mention of entity.mentions) {
replacements.push({
offset: mention.text.beginOffset,
content: mention.text.content,
type: entity.type
});
}
}
// sort temporary structure as the loop below require ascending order
// todo: can we assume NL API outputs entities in the order they were found?
replacements = replacements.sort(function(a, b) { return a.offset > b.offset; })
// add <mark> tags to annotate snippet according to temp structure
var acc_diff = 0;
for (let repl of replacements) {
new_string = string.substr(0, repl.offset + acc_diff) +
Mustache.render($("#markup-tmpl").html(), repl) +
string.substr(repl.offset + repl.content.length + acc_diff);
acc_diff = acc_diff + (new_string.length - string.length);
string = new_string;
}
// replace the original text with the new annotated version
$("#output blockquote:first p").html(string);
// create modal showing raw json response
$("#output blockquote:first div:first").append(Mustache.render(
$("#modal-tmpl").html(),
{ buttonTitle: "language json",
buttonClass: "btn-success",
modalId: "language-modal-" + $("#output blockquote").length,
modalTitle: "Cloud Natural Language API raw response",
json: JSON.stringify(resp, null, 2) }
));
// print sentiment analysis
$("#output blockquote:first div:last").append(
Mustache.render($("#sentiment-tmpl").html(), resp.documentSentiment)
);
// activate jquery plugin for showing annotation tooltips
$("[data-toggle='tooltip']").tooltip();
});
});
});
</script>
<script id="snippet-tmpl" type="x-tmpl-mustache">
<blockquote>
<div class="pull-right"></div>
<p>{{ '{{ snippet }}' }}</p>
<div class="small"></div>
</blockquote>
</script>
<script id="modal-tmpl" type="x-tmpl-mustache">
<button class="btn btn-xs {{ '{{ buttonClass }}' }}" data-toggle="modal" data-target="#{{ '{{ modalId }}' }}">
{{ '{{ buttonTitle }}' }}
</button>
<div class="modal fade" id="{{ '{{ modalId }}' }}" role="dialog" data-keyboard="true" tabindex="-1">
<div class="modal-dialog modal-lg">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal">×</button>
<h4 class="modal-title">{{ '{{ modalTitle }}' }}</h4>
</div>
<div class="modal-body">
<pre>{{ '{{ json }}' }}</pre>
</div>
</div>
</div>
</div>
</script>
<script id="markup-tmpl" type="x-tmpl-mustache">
<mark data-toggle="tooltip" title="{{ '{{ type }}' }}">{{ '{{ content }}' }}</mark>
</script>
<script id="sentiment-tmpl" type="x-tmpl-mustache">
Polarity <span class="badge">{{ '{{ polarity }}' }}</span>
Magnitude <span class="badge">{{ '{{ magnitude }}' }}</span>
</script>
</head>
<body>
<div class="container-fluid">
<h1>speech-nl-demo.appspot.com</h1>
<p>
This is a demo of the <a href="https://cloud.google.com/speech/">Cloud Speech API</a>
and the <a href="https://cloud.google.com/natural-language/">Natural Language API</a>.
Record speech, do speech-to-text conversion and optionally run natural language analysis
(sentiment, entity recognition) on the last recorded audio snippet. N.B. the demo is
currently hard-coded to assume American English - other languages and accents will not work
equally well. This is a limitation of this demo, not the Speech API.
</p>
<p>
<button id="record" class="btn btn-lg btn-primary">Record audio</button>
<button id="analyze" class="btn btn-lg btn-primary" disabled>Analyze text</button>
</p>
<div id="output"></div>
</div>
</body>
</html>