|
| 1 | +/* |
| 2 | + * Copyright 2018 Google LLC |
| 3 | + * |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + * you may not use this file except in compliance with the License. |
| 6 | + * You may obtain a copy of the License at |
| 7 | + * |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + * |
| 10 | + * Unless required by applicable law or agreed to in writing, software |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + * See the License for the specific language governing permissions and |
| 14 | + * limitations under the License. |
| 15 | + */ |
| 16 | + |
| 17 | +package com.example.speech; |
| 18 | + |
| 19 | +// [START speech_transcribe_infinite_streaming] |
| 20 | + |
| 21 | +import com.google.api.gax.rpc.ClientStream; |
| 22 | +import com.google.api.gax.rpc.ResponseObserver; |
| 23 | +import com.google.api.gax.rpc.StreamController; |
| 24 | +import com.google.cloud.speech.v1p1beta1.RecognitionConfig; |
| 25 | +import com.google.cloud.speech.v1p1beta1.SpeechClient; |
| 26 | +import com.google.cloud.speech.v1p1beta1.SpeechRecognitionAlternative; |
| 27 | +import com.google.cloud.speech.v1p1beta1.StreamingRecognitionConfig; |
| 28 | +import com.google.cloud.speech.v1p1beta1.StreamingRecognitionResult; |
| 29 | +import com.google.cloud.speech.v1p1beta1.StreamingRecognizeRequest; |
| 30 | +import com.google.cloud.speech.v1p1beta1.StreamingRecognizeResponse; |
| 31 | +import com.google.protobuf.ByteString; |
| 32 | +import com.google.protobuf.Duration; |
| 33 | +import java.text.DecimalFormat; |
| 34 | +import java.util.ArrayList; |
| 35 | +import java.util.concurrent.BlockingQueue; |
| 36 | +import java.util.concurrent.LinkedBlockingQueue; |
| 37 | +import java.util.concurrent.TimeUnit; |
| 38 | +import javax.sound.sampled.AudioFormat; |
| 39 | +import javax.sound.sampled.AudioSystem; |
| 40 | +import javax.sound.sampled.DataLine; |
| 41 | +import javax.sound.sampled.DataLine.Info; |
| 42 | +import javax.sound.sampled.TargetDataLine; |
| 43 | + |
| 44 | +public class InfiniteStreamRecognize { |
| 45 | + |
| 46 | + private static final int STREAMING_LIMIT = 290000; // ~5 minutes |
| 47 | + |
| 48 | + public static final String RED = "\033[0;31m"; |
| 49 | + public static final String GREEN = "\033[0;32m"; |
| 50 | + public static final String YELLOW = "\033[0;33m"; |
| 51 | + |
| 52 | + // Creating shared object |
| 53 | + private static volatile BlockingQueue<byte[]> sharedQueue = new LinkedBlockingQueue(); |
| 54 | + private static TargetDataLine targetDataLine; |
| 55 | + private static int BYTES_PER_BUFFER = 6400; // buffer size in bytes |
| 56 | + |
| 57 | + private static int restartCounter = 0; |
| 58 | + private static ArrayList<ByteString> audioInput = new ArrayList<ByteString>(); |
| 59 | + private static ArrayList<ByteString> lastAudioInput = new ArrayList<ByteString>(); |
| 60 | + private static int resultEndTimeInMS = 0; |
| 61 | + private static int isFinalEndTime = 0; |
| 62 | + private static int finalRequestEndTime = 0; |
| 63 | + private static boolean newStream = true; |
| 64 | + private static double bridgingOffset = 0; |
| 65 | + private static boolean lastTranscriptWasFinal = false; |
| 66 | + private static StreamController referenceToStreamController; |
| 67 | + private static ByteString tempByteString; |
| 68 | + |
| 69 | + public static void main(String... args) { |
| 70 | + InfiniteStreamRecognizeOptions options = InfiniteStreamRecognizeOptions.fromFlags(args); |
| 71 | + if (options == null) { |
| 72 | + // Could not parse. |
| 73 | + System.out.println("Failed to parse options."); |
| 74 | + System.exit(1); |
| 75 | + } |
| 76 | + |
| 77 | + try { |
| 78 | + infiniteStreamingRecognize(options.langCode); |
| 79 | + } catch (Exception e) { |
| 80 | + System.out.println("Exception caught: " + e); |
| 81 | + } |
| 82 | + } |
| 83 | + |
| 84 | + public static String convertMillisToDate(double milliSeconds) { |
| 85 | + long millis = (long) milliSeconds; |
| 86 | + DecimalFormat format = new DecimalFormat(); |
| 87 | + format.setMinimumIntegerDigits(2); |
| 88 | + return String.format( |
| 89 | + "%s:%s /", |
| 90 | + format.format(TimeUnit.MILLISECONDS.toMinutes(millis)), |
| 91 | + format.format( |
| 92 | + TimeUnit.MILLISECONDS.toSeconds(millis) |
| 93 | + - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis)))); |
| 94 | + } |
| 95 | + |
| 96 | + /** Performs infinite streaming speech recognition */ |
| 97 | + public static void infiniteStreamingRecognize(String languageCode) throws Exception { |
| 98 | + |
| 99 | + // Microphone Input buffering |
| 100 | + class MicBuffer implements Runnable { |
| 101 | + |
| 102 | + @Override |
| 103 | + public void run() { |
| 104 | + System.out.println(YELLOW); |
| 105 | + System.out.println("Start speaking...Press Ctrl-C to stop"); |
| 106 | + targetDataLine.start(); |
| 107 | + byte[] data = new byte[BYTES_PER_BUFFER]; |
| 108 | + while (targetDataLine.isOpen()) { |
| 109 | + try { |
| 110 | + int numBytesRead = targetDataLine.read(data, 0, data.length); |
| 111 | + if ((numBytesRead <= 0) && (targetDataLine.isOpen())) { |
| 112 | + continue; |
| 113 | + } |
| 114 | + sharedQueue.put(data.clone()); |
| 115 | + } catch (InterruptedException e) { |
| 116 | + System.out.println("Microphone input buffering interrupted : " + e.getMessage()); |
| 117 | + } |
| 118 | + } |
| 119 | + } |
| 120 | + } |
| 121 | + |
| 122 | + // Creating microphone input buffer thread |
| 123 | + MicBuffer micrunnable = new MicBuffer(); |
| 124 | + Thread micThread = new Thread(micrunnable); |
| 125 | + ResponseObserver<StreamingRecognizeResponse> responseObserver = null; |
| 126 | + try (SpeechClient client = SpeechClient.create()) { |
| 127 | + ClientStream<StreamingRecognizeRequest> clientStream; |
| 128 | + responseObserver = |
| 129 | + new ResponseObserver<StreamingRecognizeResponse>() { |
| 130 | + |
| 131 | + ArrayList<StreamingRecognizeResponse> responses = new ArrayList<>(); |
| 132 | + |
| 133 | + public void onStart(StreamController controller) { |
| 134 | + referenceToStreamController = controller; |
| 135 | + } |
| 136 | + |
| 137 | + public void onResponse(StreamingRecognizeResponse response) { |
| 138 | + responses.add(response); |
| 139 | + StreamingRecognitionResult result = response.getResultsList().get(0); |
| 140 | + Duration resultEndTime = result.getResultEndTime(); |
| 141 | + resultEndTimeInMS = |
| 142 | + (int) |
| 143 | + ((resultEndTime.getSeconds() * 1000) + (resultEndTime.getNanos() / 1000000)); |
| 144 | + double correctedTime = |
| 145 | + resultEndTimeInMS - bridgingOffset + (STREAMING_LIMIT * restartCounter); |
| 146 | + |
| 147 | + SpeechRecognitionAlternative alternative = result.getAlternativesList().get(0); |
| 148 | + if (result.getIsFinal()) { |
| 149 | + System.out.print(GREEN); |
| 150 | + System.out.print("\033[2K\r"); |
| 151 | + System.out.printf( |
| 152 | + "%s: %s [confidence: %.2f]\n", |
| 153 | + convertMillisToDate(correctedTime), |
| 154 | + alternative.getTranscript(), |
| 155 | + alternative.getConfidence()); |
| 156 | + isFinalEndTime = resultEndTimeInMS; |
| 157 | + lastTranscriptWasFinal = true; |
| 158 | + } else { |
| 159 | + System.out.print(RED); |
| 160 | + System.out.print("\033[2K\r"); |
| 161 | + System.out.printf( |
| 162 | + "%s: %s", convertMillisToDate(correctedTime), alternative.getTranscript()); |
| 163 | + lastTranscriptWasFinal = false; |
| 164 | + } |
| 165 | + } |
| 166 | + |
| 167 | + public void onComplete() {} |
| 168 | + |
| 169 | + public void onError(Throwable t) {} |
| 170 | + }; |
| 171 | + clientStream = client.streamingRecognizeCallable().splitCall(responseObserver); |
| 172 | + |
| 173 | + RecognitionConfig recognitionConfig = |
| 174 | + RecognitionConfig.newBuilder() |
| 175 | + .setEncoding(RecognitionConfig.AudioEncoding.LINEAR16) |
| 176 | + .setLanguageCode(languageCode) |
| 177 | + .setSampleRateHertz(16000) |
| 178 | + .build(); |
| 179 | + |
| 180 | + StreamingRecognitionConfig streamingRecognitionConfig = |
| 181 | + StreamingRecognitionConfig.newBuilder() |
| 182 | + .setConfig(recognitionConfig) |
| 183 | + .setInterimResults(true) |
| 184 | + .build(); |
| 185 | + |
| 186 | + StreamingRecognizeRequest request = |
| 187 | + StreamingRecognizeRequest.newBuilder() |
| 188 | + .setStreamingConfig(streamingRecognitionConfig) |
| 189 | + .build(); // The first request in a streaming call has to be a config |
| 190 | + |
| 191 | + clientStream.send(request); |
| 192 | + |
| 193 | + try { |
| 194 | + // SampleRate:16000Hz, SampleSizeInBits: 16, Number of channels: 1, Signed: true, |
| 195 | + // bigEndian: false |
| 196 | + AudioFormat audioFormat = new AudioFormat(16000, 16, 1, true, false); |
| 197 | + DataLine.Info targetInfo = |
| 198 | + new Info( |
| 199 | + TargetDataLine.class, |
| 200 | + audioFormat); // Set the system information to read from the microphone audio |
| 201 | + // stream |
| 202 | + |
| 203 | + if (!AudioSystem.isLineSupported(targetInfo)) { |
| 204 | + System.out.println("Microphone not supported"); |
| 205 | + System.exit(0); |
| 206 | + } |
| 207 | + // Target data line captures the audio stream the microphone produces. |
| 208 | + targetDataLine = (TargetDataLine) AudioSystem.getLine(targetInfo); |
| 209 | + targetDataLine.open(audioFormat); |
| 210 | + micThread.start(); |
| 211 | + |
| 212 | + long startTime = System.currentTimeMillis(); |
| 213 | + |
| 214 | + while (true) { |
| 215 | + |
| 216 | + long estimatedTime = System.currentTimeMillis() - startTime; |
| 217 | + |
| 218 | + if (estimatedTime >= STREAMING_LIMIT) { |
| 219 | + |
| 220 | + clientStream.closeSend(); |
| 221 | + referenceToStreamController.cancel(); // remove Observer |
| 222 | + |
| 223 | + if (resultEndTimeInMS > 0) { |
| 224 | + finalRequestEndTime = isFinalEndTime; |
| 225 | + } |
| 226 | + resultEndTimeInMS = 0; |
| 227 | + |
| 228 | + lastAudioInput = null; |
| 229 | + lastAudioInput = audioInput; |
| 230 | + audioInput = new ArrayList<ByteString>(); |
| 231 | + |
| 232 | + restartCounter++; |
| 233 | + |
| 234 | + if (!lastTranscriptWasFinal) { |
| 235 | + System.out.print('\n'); |
| 236 | + } |
| 237 | + |
| 238 | + newStream = true; |
| 239 | + |
| 240 | + clientStream = client.streamingRecognizeCallable().splitCall(responseObserver); |
| 241 | + |
| 242 | + request = |
| 243 | + StreamingRecognizeRequest.newBuilder() |
| 244 | + .setStreamingConfig(streamingRecognitionConfig) |
| 245 | + .build(); |
| 246 | + |
| 247 | + System.out.println(YELLOW); |
| 248 | + System.out.printf("%d: RESTARTING REQUEST\n", restartCounter * STREAMING_LIMIT); |
| 249 | + |
| 250 | + startTime = System.currentTimeMillis(); |
| 251 | + |
| 252 | + } else { |
| 253 | + |
| 254 | + if ((newStream) && (lastAudioInput.size() > 0)) { |
| 255 | + // if this is the first audio from a new request |
| 256 | + // calculate amount of unfinalized audio from last request |
| 257 | + // resend the audio to the speech client before incoming audio |
| 258 | + double chunkTime = STREAMING_LIMIT / lastAudioInput.size(); |
| 259 | + // ms length of each chunk in previous request audio arrayList |
| 260 | + if (chunkTime != 0) { |
| 261 | + if (bridgingOffset < 0) { |
| 262 | + // bridging Offset accounts for time of resent audio |
| 263 | + // calculated from last request |
| 264 | + bridgingOffset = 0; |
| 265 | + } |
| 266 | + if (bridgingOffset > finalRequestEndTime) { |
| 267 | + bridgingOffset = finalRequestEndTime; |
| 268 | + } |
| 269 | + int chunksFromMs = |
| 270 | + (int) Math.floor((finalRequestEndTime - bridgingOffset) / chunkTime); |
| 271 | + // chunks from MS is number of chunks to resend |
| 272 | + bridgingOffset = |
| 273 | + (int) Math.floor((lastAudioInput.size() - chunksFromMs) * chunkTime); |
| 274 | + // set bridging offset for next request |
| 275 | + for (int i = chunksFromMs; i < lastAudioInput.size(); i++) { |
| 276 | + request = |
| 277 | + StreamingRecognizeRequest.newBuilder() |
| 278 | + .setAudioContent(lastAudioInput.get(i)) |
| 279 | + .build(); |
| 280 | + clientStream.send(request); |
| 281 | + } |
| 282 | + } |
| 283 | + newStream = false; |
| 284 | + } |
| 285 | + |
| 286 | + tempByteString = ByteString.copyFrom(sharedQueue.take()); |
| 287 | + |
| 288 | + request = |
| 289 | + StreamingRecognizeRequest.newBuilder().setAudioContent(tempByteString).build(); |
| 290 | + |
| 291 | + audioInput.add(tempByteString); |
| 292 | + } |
| 293 | + |
| 294 | + clientStream.send(request); |
| 295 | + } |
| 296 | + } catch (Exception e) { |
| 297 | + System.out.println(e); |
| 298 | + } |
| 299 | + } |
| 300 | + } |
| 301 | +} |
| 302 | +// [END speech_transcribe_infinite_streaming] |
0 commit comments