From 390503b7ef68c4807fd2f0fed64333d74603c58f Mon Sep 17 00:00:00 2001 From: GiviMAD Date: Sat, 29 Nov 2025 21:07:55 +0000 Subject: [PATCH] [audio] Add pcm audio websocket with dialog support (#4032) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [audio] Add pcm audio websocket with dialog support * improve KSEdgeService support * fix/spotless voice provider Signed-off-by: Miguel Álvarez --- bom/openhab-core/pom.xml | 6 + bundles/org.openhab.core.audio/pom.xml | 12 + .../core/audio/AudioDialogProvider.java | 39 ++ .../.classpath | 42 ++ .../.project | 23 ++ .../NOTICE | 14 + .../pom.xml | 30 ++ .../audio/internal/PCMWebSocketAdapter.java | 148 +++++++ .../audio/internal/PCMWebSocketAudioSink.java | 261 ++++++++++++ .../internal/PCMWebSocketAudioSource.java | 195 +++++++++ .../audio/internal/PCMWebSocketAudioUtil.java | 57 +++ .../internal/PCMWebSocketConnection.java | 375 ++++++++++++++++++ .../internal/PCMWebSocketStreamIdUtil.java | 72 ++++ .../openhab/core/voice/BasicDTService.java | 31 ++ .../org/openhab/core/voice/DTErrorEvent.java | 46 +++ .../java/org/openhab/core/voice/DTEvent.java | 27 ++ .../org/openhab/core/voice/DTException.java | 64 +++ .../org/openhab/core/voice/DTListener.java | 32 ++ .../org/openhab/core/voice/DTService.java | 41 ++ .../openhab/core/voice/DTServiceHandle.java | 28 ++ .../openhab/core/voice/DTTriggeredEvent.java | 24 ++ .../org/openhab/core/voice/DialogContext.java | 17 +- .../org/openhab/core/voice/KSEdgeService.java | 44 -- .../org/openhab/core/voice/KSErrorEvent.java | 18 +- .../java/org/openhab/core/voice/KSEvent.java | 2 +- .../org/openhab/core/voice/KSException.java | 2 +- .../org/openhab/core/voice/KSListener.java | 2 +- .../org/openhab/core/voice/KSService.java | 19 +- .../openhab/core/voice/KSServiceHandle.java | 2 +- .../org/openhab/core/voice/KSpottedEvent.java | 2 +- .../org/openhab/core/voice/VoiceManager.java | 4 +- .../internal/AudioDialogProviderImpl.java | 115 ++++++ .../core/voice/internal/DialogProcessor.java | 71 ++-- .../VoiceConsoleCommandExtension.java | 2 +- .../core/voice/internal/VoiceManagerImpl.java | 20 +- bundles/pom.xml | 1 + .../openhab-core/src/main/feature/feature.xml | 7 + 37 files changed, 1772 insertions(+), 123 deletions(-) create mode 100644 bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioDialogProvider.java create mode 100644 bundles/org.openhab.core.io.websocket.audio/.classpath create mode 100644 bundles/org.openhab.core.io.websocket.audio/.project create mode 100644 bundles/org.openhab.core.io.websocket.audio/NOTICE create mode 100644 bundles/org.openhab.core.io.websocket.audio/pom.xml create mode 100644 bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAdapter.java create mode 100644 bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioSink.java create mode 100644 bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioSource.java create mode 100644 bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioUtil.java create mode 100644 bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketConnection.java create mode 100644 bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketStreamIdUtil.java create mode 100644 bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/BasicDTService.java create mode 100644 bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTErrorEvent.java create mode 100644 bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTEvent.java create mode 100644 bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTException.java create mode 100644 bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTListener.java create mode 100644 bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTService.java create mode 100644 bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTServiceHandle.java create mode 100644 bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTTriggeredEvent.java delete mode 100644 bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSEdgeService.java create mode 100644 bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/AudioDialogProviderImpl.java diff --git a/bom/openhab-core/pom.xml b/bom/openhab-core/pom.xml index 2beba0b65..7c2ed6892 100644 --- a/bom/openhab-core/pom.xml +++ b/bom/openhab-core/pom.xml @@ -286,6 +286,12 @@ ${project.version} compile + + org.openhab.core.bundles + org.openhab.core.io.websocket.audio + ${project.version} + compile + org.openhab.core.bundles org.openhab.core.io.jetty.certificate diff --git a/bundles/org.openhab.core.audio/pom.xml b/bundles/org.openhab.core.audio/pom.xml index 4ff3e1b1a..b0c2267d8 100644 --- a/bundles/org.openhab.core.audio/pom.xml +++ b/bundles/org.openhab.core.audio/pom.xml @@ -38,6 +38,18 @@ ${project.version} compile + + org.openhab.core.bundles + org.openhab.core.io.http + ${project.version} + compile + + + org.openhab.core.bundles + org.openhab.core.io.websocket + ${project.version} + compile + org.openhab.core.bundles org.openhab.core.test diff --git a/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioDialogProvider.java b/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioDialogProvider.java new file mode 100644 index 000000000..4a922e1df --- /dev/null +++ b/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioDialogProvider.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.audio; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +/** + * This interface is designed so the voice bundle can inject the start dialog functionality for audio bundle to consume, + * which a programmatic way of trigger the dialog execution. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public interface AudioDialogProvider { + /** + * Starts a dialog and returns a runnable that triggers it or null if the dialog initialization fails + * + * @param audioSink the audio sink to play sound + * @param audioSource the audio source to capture sound + * @param locationItem an optional Item name to scope dialog commands + * @param listeningItem an optional Item name to toggle while dialog is listening, overwrites default + * @param onAbort an optional {@link Runnable} instance to call on abort. + * @return a {@link Runnable} instance to trigger dialog processing or null if the dialog initialization fails + */ + @Nullable + Runnable startDialog(AudioSink audioSink, AudioSource audioSource, @Nullable String locationItem, + @Nullable String listeningItem, @Nullable Runnable onAbort); +} diff --git a/bundles/org.openhab.core.io.websocket.audio/.classpath b/bundles/org.openhab.core.io.websocket.audio/.classpath new file mode 100644 index 000000000..4b18de0f9 --- /dev/null +++ b/bundles/org.openhab.core.io.websocket.audio/.classpath @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bundles/org.openhab.core.io.websocket.audio/.project b/bundles/org.openhab.core.io.websocket.audio/.project new file mode 100644 index 000000000..1065d0aca --- /dev/null +++ b/bundles/org.openhab.core.io.websocket.audio/.project @@ -0,0 +1,23 @@ + + + org.openhab.core.io.websocket.audio + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/bundles/org.openhab.core.io.websocket.audio/NOTICE b/bundles/org.openhab.core.io.websocket.audio/NOTICE new file mode 100644 index 000000000..6c17d0d8a --- /dev/null +++ b/bundles/org.openhab.core.io.websocket.audio/NOTICE @@ -0,0 +1,14 @@ +This content is produced and maintained by the openHAB project. + +* Project home: https://www.openhab.org + +== Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License 2.0 which is available at +https://www.eclipse.org/legal/epl-2.0/. + +== Source Code + +https://github.com/openhab/openhab-core + diff --git a/bundles/org.openhab.core.io.websocket.audio/pom.xml b/bundles/org.openhab.core.io.websocket.audio/pom.xml new file mode 100644 index 000000000..24ab2abcd --- /dev/null +++ b/bundles/org.openhab.core.io.websocket.audio/pom.xml @@ -0,0 +1,30 @@ + + + + 4.0.0 + + + org.openhab.core.bundles + org.openhab.core.reactor.bundles + 5.1.0-SNAPSHOT + + + org.openhab.core.io.websocket.audio + + openHAB Core :: Bundles :: Audio PCM Websocket Interface + + + + org.openhab.core.bundles + org.openhab.core.audio + ${project.version} + + + org.openhab.core.bundles + org.openhab.core.io.websocket + ${project.version} + + + + diff --git a/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAdapter.java b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAdapter.java new file mode 100644 index 000000000..b9f44549a --- /dev/null +++ b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAdapter.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.io.websocket.audio.internal; + +import static java.nio.ByteBuffer.wrap; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.jetty.websocket.servlet.ServletUpgradeRequest; +import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; +import org.openhab.core.audio.AudioDialogProvider; +import org.openhab.core.audio.AudioManager; +import org.openhab.core.common.ThreadPoolManager; +import org.openhab.core.io.websocket.WebSocketAdapter; +import org.osgi.framework.BundleContext; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Deactivate; +import org.osgi.service.component.annotations.Reference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The {@link PCMWebSocketAdapter} creates instances of {@link PCMWebSocketConnection} to handle PCM audio streams. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +@Component(immediate = true, service = { PCMWebSocketAdapter.class, WebSocketAdapter.class }) +public class PCMWebSocketAdapter implements WebSocketAdapter { + public static final String ADAPTER_ID = "audio-pcm"; + + private final Logger logger = LoggerFactory.getLogger(PCMWebSocketAdapter.class); + private final ScheduledExecutorService executor = ThreadPoolManager.getScheduledPool("audio-pcm-websocket"); + protected final BundleContext bundleContext; + protected final AudioManager audioManager; + protected final AudioDialogProvider audioDialogProvider; + private final ScheduledFuture pingTask; + private final Set webSocketConnections = Collections.synchronizedSet(new HashSet<>()); + + @Activate + public PCMWebSocketAdapter(BundleContext bundleContext, final @Reference AudioManager audioManager, + final @Reference AudioDialogProvider audioDialogProvider) { + this.bundleContext = bundleContext; + this.audioManager = audioManager; + this.audioDialogProvider = audioDialogProvider; + this.pingTask = executor.scheduleWithFixedDelay(this::pingHandlers, 10, 5, TimeUnit.SECONDS); + } + + protected void onSpeakerConnected(PCMWebSocketConnection speaker) throws IllegalStateException { + synchronized (webSocketConnections) { + if (getSpeakerConnection(speaker.getId()) != null) { + throw new IllegalStateException("Another speaker with the same id is already connected"); + } + webSocketConnections.add(speaker); + logger.debug("connected speakers {}", webSocketConnections.size()); + } + } + + protected void onClientDisconnected(PCMWebSocketConnection connection) { + logger.debug("speaker disconnected '{}'", connection.getId()); + synchronized (webSocketConnections) { + webSocketConnections.remove(connection); + logger.debug("connected speakers {}", webSocketConnections.size()); + } + } + + public @Nullable PCMWebSocketConnection getSpeakerConnection(String id) { + synchronized (webSocketConnections) { + return webSocketConnections.stream() + .filter(speakerConnection -> speakerConnection.getId().equalsIgnoreCase(id)).findAny().orElse(null); + } + } + + @Override + public String getId() { + return ADAPTER_ID; + } + + @Override + public Object createWebSocket(ServletUpgradeRequest servletUpgradeRequest, + ServletUpgradeResponse servletUpgradeResponse) { + logger.debug("creating connection"); + return new PCMWebSocketConnection(this, executor); + } + + @Deactivate + @SuppressWarnings("unused") + public synchronized void deactivate() { + logger.debug("stopping connection check"); + pingTask.cancel(true); + disconnectAll(); + } + + private void pingHandlers() { + ArrayList handlers = new ArrayList<>(webSocketConnections); + for (var handler : handlers) { + if (handler != null) { + boolean pinged = false; + var remote = handler.getRemote(); + if (remote != null) { + try { + remote.sendPing(wrap("oh".getBytes(StandardCharsets.UTF_8))); + pinged = true; + } catch (IOException ignored) { + } + } + if (!pinged) { + logger.debug("Ping failed, disconnecting speaker '{}'", handler.getId()); + var session = handler.getSession(); + if (session != null) { + session.close(); + } + } + } + + } + } + + private void disconnectAll() { + logger.debug("Disconnecting {} clients...", webSocketConnections.size()); + ArrayList connections = new ArrayList<>(webSocketConnections); + for (var connection : connections) { + onClientDisconnected(connection); + connection.disconnect(); + } + } +} diff --git a/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioSink.java b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioSink.java new file mode 100644 index 000000000..d38f075a2 --- /dev/null +++ b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioSink.java @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.io.websocket.audio.internal; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.core.audio.*; +import org.openhab.core.audio.utils.AudioWaveUtils; +import org.openhab.core.library.types.PercentType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is an {@link AudioSink} implementation connected to the {@link PCMWebSocketConnection} that allows to + * transmit concurrent PCM audio lines through WebSocket. + *

+ * To identify the different audio lines the data chucks are prefixed by a header added by the + * {@link PCMWebSocketOutputStream} class. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public class PCMWebSocketAudioSink implements AudioSink { + /** + * Byte sent after the last chunk for a stream to indicate the stream has ended, so the client can dispose resources + * associated with the stream. + * It's sent on a finally clause that covers the audio transmission execution so it gets sent even if some + * exception interrupts the audio transmission. + */ + private static final byte STREAM_TERMINATION_BYTE = (byte) 254; + private static final Set SUPPORTED_FORMATS = Set.of(AudioFormat.WAV, AudioFormat.PCM_SIGNED); + private static final Set> SUPPORTED_STREAMS = Set.of(FixedLengthAudioStream.class, + PipedAudioStream.class); + + private final Logger logger = LoggerFactory.getLogger(PCMWebSocketAudioSink.class); + + private final String sinkId; + private final String sinkLabel; + private final PCMWebSocketConnection websocket; + private PercentType sinkVolume = new PercentType(100); + @Nullable + private Integer forceSampleRate; + @Nullable + private Integer forceBitDepth; + @Nullable + private Integer forceChannels; + + public PCMWebSocketAudioSink(String id, String label, PCMWebSocketConnection websocket, + @Nullable Integer forceSampleRate, @Nullable Integer forceBitDepth, @Nullable Integer forceChannels) { + this.sinkId = id; + this.sinkLabel = label; + this.websocket = websocket; + this.forceSampleRate = forceSampleRate; + this.forceBitDepth = forceBitDepth; + this.forceChannels = forceChannels; + } + + @Override + public String getId() { + return this.sinkId; + } + + @Override + public @Nullable String getLabel(@Nullable Locale locale) { + return this.sinkLabel; + } + + @Override + public void process(@Nullable AudioStream audioStream) + throws UnsupportedAudioFormatException, UnsupportedAudioStreamException { + if (audioStream == null) { + return; + } + OutputStream outputStream = null; + try { + long duration = -1; + if (AudioFormat.CONTAINER_WAVE.equals(audioStream.getFormat().getContainer())) { + logger.debug("Removing wav container from data"); + try { + AudioWaveUtils.removeFMT(audioStream); + } catch (IOException e) { + logger.warn("IOException trying to remove wav header: {}", e.getMessage()); + } + } + var audioFormat = audioStream.getFormat(); + if (audioStream instanceof SizeableAudioStream sizeableAudioStream) { + long byteLength = sizeableAudioStream.length(); + long bytesPerSecond = (Objects.requireNonNull(audioFormat.getBitDepth()) / 8) + * Objects.requireNonNull(audioFormat.getFrequency()) + * Objects.requireNonNull(audioFormat.getChannels()); + float durationInSeconds = (float) byteLength / bytesPerSecond; + duration = Math.round(durationInSeconds * 1000); + logger.debug("Duration of input stream : {}", duration); + } + AtomicBoolean transferenceAborted = new AtomicBoolean(false); + if (audioStream instanceof PipedAudioStream pipedAudioStream) { + pipedAudioStream.onClose(() -> transferenceAborted.set(true)); + } + int sampleRate = Objects.requireNonNull(audioFormat.getFrequency()).intValue(); + int bitDepth = Objects.requireNonNull(audioFormat.getBitDepth()); + int channels = Objects.requireNonNull(audioFormat.getChannels()); + int targetSampleRate = Objects.requireNonNullElse(forceSampleRate, sampleRate); + Integer targetBitDepth = Objects.requireNonNullElse(forceBitDepth, bitDepth); + Integer targetChannels = Objects.requireNonNullElse(forceChannels, channels); + outputStream = new PCMWebSocketOutputStream(websocket, targetSampleRate, targetBitDepth.byteValue(), + targetChannels.byteValue()); + InputStream finalAudioStream; + if ( // + (forceSampleRate != null && !forceSampleRate.equals(sampleRate)) || // + (forceBitDepth != null && !forceBitDepth.equals(bitDepth)) || // + (forceChannels != null && !forceChannels.equals(channels)) // + ) { + logger.debug("Sound is not in the target format. Trying to re-encode it"); + finalAudioStream = PCMWebSocketAudioUtil.getPCMStreamNormalized(audioStream, sampleRate, bitDepth, + channels, targetSampleRate, targetBitDepth, targetChannels); + } else { + finalAudioStream = audioStream; + } + int bytesPer500ms = (targetSampleRate * (targetBitDepth / 8) * targetChannels) / 2; + transferAudio(finalAudioStream, outputStream, bytesPer500ms, duration, transferenceAborted); + } catch (InterruptedIOException ignored) { + } catch (IOException e) { + logger.warn("IOException: {}", e.getMessage()); + } catch (InterruptedException e) { + logger.warn("InterruptedException: {}", e.getMessage()); + } finally { + try { + audioStream.close(); + } catch (IOException e) { + logger.warn("IOException: {}", e.getMessage(), e); + } + try { + if (outputStream != null) { + outputStream.close(); + } + } catch (IOException e) { + logger.warn("IOException: {}", e.getMessage(), e); + } + } + } + + private void transferAudio(InputStream inputStream, OutputStream outputStream, int chunkSize, long duration, + AtomicBoolean aborted) throws IOException, InterruptedException { + Instant start = Instant.now(); + long transferred = 0; + try { + byte[] buffer = new byte[chunkSize]; + int read; + while (!aborted.get() && (read = inputStream.read(buffer, 0, chunkSize)) >= 0) { + outputStream.write(buffer, 0, read); + transferred += read; + } + } finally { + try { + // send a byte indicating this stream has ended, so it can be tear down on the client + outputStream.write(new byte[] { STREAM_TERMINATION_BYTE }, 0, 1); + } catch (IOException e) { + logger.warn("Unable to send termination byte to sink {}", sinkId); + } + } + logger.debug("Sent {} bytes of audio", transferred); + if (duration != -1) { + Instant end = Instant.now(); + long millisSecondTimedToSendAudioData = Duration.between(start, end).toMillis(); + if (millisSecondTimedToSendAudioData < duration) { + long timeToSleep = duration - millisSecondTimedToSendAudioData; + logger.debug("Sleep time to let the system play sound : {}ms", timeToSleep); + Thread.sleep(timeToSleep); + } + } + } + + @Override + public Set getSupportedFormats() { + return SUPPORTED_FORMATS; + } + + @Override + public Set> getSupportedStreams() { + return SUPPORTED_STREAMS; + } + + @Override + public PercentType getVolume() throws IOException { + return this.sinkVolume; + } + + @Override + public void setVolume(PercentType percentType) throws IOException { + this.sinkVolume = percentType; + websocket.setSinkVolume(percentType.intValue()); + } + + /** + * This is an {@link OutputStream} implementation for writing binary data to the websocket that + * will prefix each chunk with a header composed of 8 bytes. + * Header: 2 bytes (stream id) + 4 byte (stream sample rate) + 1 byte (stream bit depth) + 1 byte (channels). + */ + protected static class PCMWebSocketOutputStream extends OutputStream { + private final byte[] header; + private final PCMWebSocketConnection websocket; + private boolean closed = false; + + public PCMWebSocketOutputStream(PCMWebSocketConnection websocket, int sampleRate, byte bitDepth, + byte channels) { + this.websocket = websocket; + this.header = PCMWebSocketStreamIdUtil.generateAudioPacketHeader(sampleRate, bitDepth, channels).array(); + } + + @Override + public void write(int b) throws IOException { + write(ByteBuffer.allocate(4).putInt(b).array()); + } + + @Override + public void write(byte @Nullable [] b) throws IOException { + if (closed) { + throw new IOException("Stream closed"); + } + if (b != null) { + websocket.sendAudio(header, b); + } + } + + @Override + public void write(byte @Nullable [] b, int off, int len) throws IOException { + if (b != null) { + write(Arrays.copyOfRange(b, off, off + len)); + } + } + + @Override + public void close() throws IOException { + closed = true; + super.close(); + } + } +} diff --git a/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioSource.java b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioSource.java new file mode 100644 index 000000000..d32aedbd4 --- /dev/null +++ b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioSource.java @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.io.websocket.audio.internal; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.Arrays; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.core.audio.*; +import org.openhab.core.common.ThreadPoolManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is an {@link AudioSource} implementation connected to a {@link PCMWebSocketConnection} supporting + * a single PCM audio line through a WebSocket connection and shared across all active {@link AudioStream} instances. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public class PCMWebSocketAudioSource implements AudioSource { + private static final int SUPPORTED_BIT_DEPTH = 16; + private static final int SUPPORTED_SAMPLE_RATE = 16000; + private static final int SUPPORTED_CHANNELS = 1; + public static AudioFormat supportedFormat = new AudioFormat(AudioFormat.CONTAINER_WAVE, + AudioFormat.CODEC_PCM_SIGNED, false, SUPPORTED_BIT_DEPTH, null, (long) SUPPORTED_SAMPLE_RATE, + SUPPORTED_CHANNELS); + private final Logger logger = LoggerFactory.getLogger(PCMWebSocketAudioSource.class); + private final String sourceId; + private final String sourceLabel; + private final PCMWebSocketConnection websocket; + private @Nullable PipedOutputStream sourceAudioPipedOutput; + private @Nullable PipedInputStream sourceAudioPipedInput; + private @Nullable InputStream sourceAudioStream; + private final PipedAudioStream.Group streamGroup = PipedAudioStream.newGroup(supportedFormat); + private @Nullable Future sourceWriteTask; + private final ScheduledExecutorService scheduler = ThreadPoolManager.getScheduledPool("pcm-audio-source"); + private byte @Nullable [] streamId; + + public PCMWebSocketAudioSource(String id, String label, PCMWebSocketConnection websocket) { + this.sourceId = id; + this.sourceLabel = label; + this.websocket = websocket; + } + + @Override + public String getId() { + return this.sourceId; + } + + @Override + public String getLabel(@Nullable Locale locale) { + return this.sourceLabel; + } + + @Override + public Set getSupportedFormats() { + return Set.of(supportedFormat); + } + + @Override + public AudioStream getInputStream(AudioFormat audioFormat) throws AudioException { + try { + final PipedAudioStream stream = streamGroup.getAudioStreamInGroup(); + synchronized (streamGroup) { + if (this.streamGroup.size() == 1) { + logger.debug("Send start listening {}", getId()); + this.streamId = null; + websocket.setListening(true); + } + } + stream.onClose(this::onStreamClose); + return stream; + } catch (IOException e) { + throw new AudioException(e); + } + } + + public void close() throws Exception { + streamGroup.close(); + } + + public void writeToStreams(byte[] id, int sampleRate, int bitDepth, int channels, byte[] payload) { + if (streamGroup.isEmpty()) { + logger.debug("Source already disposed, ignoring data"); + return; + } + if (this.streamId == null) { + this.streamId = id; + } else if (!Arrays.equals(this.streamId, id)) { + logger.warn("Only one concurrent data line is supported, ignoring data from source stream {}", id); + return; + } + boolean needsConvert = sampleRate != SUPPORTED_SAMPLE_RATE || bitDepth != SUPPORTED_BIT_DEPTH + || channels != SUPPORTED_CHANNELS; + if (!needsConvert) { + streamGroup.write(payload); + return; + } + if (this.sourceAudioPipedOutput == null || this.sourceAudioStream == null) { + try { + this.sourceAudioPipedOutput = new PipedOutputStream(); + var sourceAudioPipedInput = this.sourceAudioPipedInput = new PipedInputStream( + this.sourceAudioPipedOutput, (sampleRate * (bitDepth / 8) * channels) * 2); + logger.debug( + "Enabling converting pcm audio for the audio source stream: sample rate {}, bit depth {}, channels {} => sample rate {}, bit depth {}, channels {}", + sampleRate, bitDepth, channels, SUPPORTED_SAMPLE_RATE, SUPPORTED_BIT_DEPTH, SUPPORTED_CHANNELS); + this.sourceAudioStream = PCMWebSocketAudioUtil.getPCMStreamNormalized(sourceAudioPipedInput, sampleRate, + bitDepth, channels, SUPPORTED_SAMPLE_RATE, SUPPORTED_BIT_DEPTH, SUPPORTED_CHANNELS); + sourceWriteTask = scheduler.submit(() -> { + int bytesPer250ms = (SUPPORTED_SAMPLE_RATE * (SUPPORTED_BIT_DEPTH / 8) * SUPPORTED_CHANNELS) / 4; + while (true) { + byte[] convertedPayload; + try { + convertedPayload = this.sourceAudioStream.readNBytes(bytesPer250ms); + Thread.sleep(0); + } catch (InterruptedIOException | InterruptedException e) { + continue; + } catch (IOException e) { + if (e.getMessage().contains("Pipe closed")) { + return; + } + logger.error("Error reading converted audio data", e); + continue; + } + streamGroup.write(convertedPayload); + } + }); + } catch (IOException e) { + logger.error("Unable to setup audio source stream", e); + return; + } + } + try { + this.sourceAudioPipedOutput.write(payload); + } catch (IOException e) { + logger.error("Error converting source audio format", e); + } + } + + private void onStreamClose() { + logger.debug("Unregister source audio stream for '{}'", getId()); + synchronized (streamGroup) { + if (streamGroup.isEmpty()) { + logger.debug("Send stop listening {}", getId()); + websocket.setListening(false); + if (this.sourceWriteTask != null) { + this.sourceWriteTask.cancel(true); + this.sourceWriteTask = null; + } + if (this.sourceAudioStream != null) { + try { + this.sourceAudioStream.close(); + } catch (IOException ignored) { + } + this.sourceAudioStream = null; + } + if (this.sourceAudioPipedOutput != null) { + try { + this.sourceAudioPipedOutput.close(); + } catch (IOException ignored) { + } + this.sourceAudioPipedOutput = null; + } + if (this.sourceAudioPipedInput != null) { + try { + this.sourceAudioPipedInput.close(); + } catch (IOException ignored) { + } + this.sourceAudioPipedInput = null; + } + this.streamId = null; + } + } + } +} diff --git a/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioUtil.java b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioUtil.java new file mode 100644 index 000000000..31f455f02 --- /dev/null +++ b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketAudioUtil.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.io.websocket.audio.internal; + +import java.io.InputStream; + +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * Audio utils. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public final class PCMWebSocketAudioUtil { + private PCMWebSocketAudioUtil() { + } + + /** + * Ensure right PCM format by converting if needed (sample rate, channel) + * + * @param sampleRate Stream sample rate + * @param stream PCM input stream + * @return A PCM normalized stream at the desired format + */ + public static AudioInputStream getPCMStreamNormalized(InputStream stream, int sampleRate, int bitDepth, + int channels, int targetSampleRate, int targetBitDepth, int targetChannels) { + javax.sound.sampled.AudioFormat jFormat = new javax.sound.sampled.AudioFormat( // + (float) sampleRate, // + bitDepth, // + channels, // + true, // + false // + ); + javax.sound.sampled.AudioFormat fixedJFormat = new javax.sound.sampled.AudioFormat( // + (float) targetSampleRate, // + targetBitDepth, // + targetChannels, // + true, // + false // + ); + return AudioSystem.getAudioInputStream(fixedJFormat, new AudioInputStream(stream, jFormat, -1)); + } +} diff --git a/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketConnection.java b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketConnection.java new file mode 100644 index 000000000..ee3436388 --- /dev/null +++ b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketConnection.java @@ -0,0 +1,375 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.io.websocket.audio.internal; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.eclipse.jetty.websocket.api.RemoteEndpoint; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketListener; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.openhab.core.audio.AudioSink; +import org.openhab.core.audio.AudioSource; +import org.osgi.framework.ServiceRegistration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * The {@link PCMWebSocketConnection} represents a WebSocket connection used to transmit PCM audio. + *

+ * The websocket uses the text protocol for send commands represented by {@link WebSocketCommand} and the binary + * protocol to transmit the audio data. + *

+ * The websocket supports only one line for the {@link PCMWebSocketAudioSource} (the audio is shared on the server), + * the data transmission is instructed by the server (START_LISTENING and STOP_LISTENING commands). + *

+ * The websocket supports multiple lines for the {@link PCMWebSocketAudioSink} (to accomplish that, the outgoing data + * chucks are prefixed with a 6 byte header to transmit the identity and format specification, check + * {@link PCMWebSocketAudioSink.PCMWebSocketOutputStream}) + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@WebSocket +@NonNullByDefault +@SuppressWarnings("unused") +public class PCMWebSocketConnection implements WebSocketListener { + private final Logger logger = LoggerFactory.getLogger(PCMWebSocketConnection.class); + protected final Map> audioComponentRegistrations = new ConcurrentHashMap<>(); + private volatile @Nullable Session session; + private @Nullable RemoteEndpoint remote; + private final PCMWebSocketAdapter wsAdapter; + private final ScheduledExecutorService executor; + private @Nullable ScheduledFuture scheduledDisconnection; + + private boolean initialized = false; + private @Nullable Runnable dialogTrigger = null; + + private final ObjectMapper jsonMapper = new ObjectMapper(); + private String id = ""; + private @Nullable PCMWebSocketAudioSource audioSource = null; + + public PCMWebSocketConnection(PCMWebSocketAdapter wsAdapter, ScheduledExecutorService executor) { + this.wsAdapter = wsAdapter; + this.executor = executor; + } + + public void sendAudio(byte[] id, byte[] b) { + try { + var remote = getRemote(); + if (remote != null) { + // concat stream identifier and send + ByteBuffer buff = ByteBuffer.wrap(new byte[id.length + b.length]); + buff.put(id); + buff.put(b); + remote.sendBytesByFuture(ByteBuffer.wrap(buff.array())); + } + } catch (IllegalStateException ignored) { + logger.warn("Unable to send audio buffer"); + } + } + + public void setListening(boolean listening) { + sendClientCommand(new WebSocketCommand(listening ? WebSocketCommand.OutputCommands.START_LISTENING + : WebSocketCommand.OutputCommands.STOP_LISTENING)); + } + + public void disconnect() { + var session = getSession(); + if (session != null) { + try { + session.disconnect(); + } catch (IOException ignored) { + } + } + } + + @Override + public void onWebSocketConnect(@Nullable Session sess) { + if (sess == null) { + // never + return; + } + this.session = sess; + this.remote = sess.getRemote(); + logger.debug("New client connected."); + scheduledDisconnection = executor.schedule(() -> { + try { + sess.disconnect(); + } catch (IOException ignored) { + } + }, 5, TimeUnit.SECONDS); + } + + private void sendClientCommand(T msg) { + var remote = getRemote(); + if (remote != null) { + try { + remote.sendStringByFuture(new ObjectMapper().writeValueAsString(msg)); + } catch (JsonProcessingException e) { + logger.warn("JsonProcessingException writing JSON message", e); + } + } + } + + @Override + public void onWebSocketBinary(byte @Nullable [] payload, int offset, int len) { + logger.trace("Received binary data of length {}", len); + PCMWebSocketAudioSource audioSource = this.audioSource; + if (payload != null && audioSource != null) { + PCMWebSocketStreamIdUtil.AudioPacketData streamData; + try { + streamData = PCMWebSocketStreamIdUtil.parseAudioPacket(payload); + } catch (IOException e) { + logger.warn("Exception processing binary message: {}", e.getMessage()); + return; + } + audioSource.writeToStreams(streamData.id(), streamData.sampleRate(), streamData.bitDepth(), + streamData.channels(), streamData.audioData()); + } + } + + @Override + public void onWebSocketText(@Nullable String message) { + try { + JsonNode rootMessageNode = jsonMapper.readTree(message); + if (rootMessageNode.has("cmd")) { + String cmd = rootMessageNode.get("cmd").asText().trim().toUpperCase(); + try { + logger.debug("Handling msg '{}'", cmd); + var messageType = WebSocketCommand.InputCommands.valueOf(cmd); + switch (messageType) { + case INITIALIZE -> { + wsAdapter.onSpeakerConnected(this); + JsonNode argsNode = rootMessageNode.get("args"); + var clientOptions = jsonMapper.treeToValue(argsNode, ConnectionOptions.class); + var scheduledDisconnection = this.scheduledDisconnection; + if (scheduledDisconnection != null) { + scheduledDisconnection.cancel(true); + } + // update connection settings + id = clientOptions.id; + registerSpeakerComponents(id, clientOptions); + sendClientCommand(new WebSocketCommand(WebSocketCommand.OutputCommands.INITIALIZED)); + } + case ON_SPOT -> onRemoteSpot(); + } + } catch (IOException | IllegalStateException e) { + logger.warn("Error handing command '{}' with message: {}. Disconnecting client", cmd, + e.getMessage()); + disconnect(); + } + } + } catch (JsonProcessingException e) { + logger.warn("Exception parsing JSON message.", e); + logger.warn("Disconnecting client."); + disconnect(); + } + } + + @Override + public void onWebSocketError(@Nullable Throwable cause) { + logger.warn("WebSocket Error", cause); + } + + @Override + public void onWebSocketClose(int statusCode, @Nullable String reason) { + this.session = null; + this.remote = null; + logger.debug("Session closed with code {}: {}", statusCode, reason); + wsAdapter.onClientDisconnected(this); + unregisterSpeakerComponents(id); + } + + public void setSinkVolume(int value) { + if (initialized) { + sendClientCommand( + new WebSocketCommand(WebSocketCommand.OutputCommands.SINK_VOLUME, Map.of("value", value))); + } + } + + public void setSourceVolume(int value) { + if (initialized) { + sendClientCommand( + new WebSocketCommand(WebSocketCommand.OutputCommands.SOURCE_VOLUME, Map.of("value", value))); + } + } + + public @Nullable RemoteEndpoint getRemote() { + return this.remote; + } + + public @Nullable Session getSession() { + return this.session; + } + + public boolean isConnected() { + Session sess = this.session; + return sess != null && sess.isOpen(); + } + + private synchronized void registerSpeakerComponents(String id, ConnectionOptions clientOptions) throws IOException { + if (id.isBlank()) { + throw new IOException("Unable to register audio components"); + } + String label = "PCM Audio WebSocket (" + id + ")"; + logger.debug("Registering dialog components for '{}'", id); + this.initialized = true; + // register source + var audioSource = this.audioSource = new PCMWebSocketAudioSource(getSourceId(id), label, this); + logger.debug("Registering audio source {}", this.audioSource.getId()); + audioComponentRegistrations.put(this.audioSource.getId(), wsAdapter.bundleContext + .registerService(AudioSource.class.getName(), this.audioSource, new Hashtable<>())); + // register sink + var audioSink = new PCMWebSocketAudioSink(getSinkId(id), label, this, clientOptions.forceSampleRate, + clientOptions.forceBitDepth, clientOptions.forceChannels); + logger.debug("Registering audio sink {}", audioSink.getId()); + audioComponentRegistrations.put(audioSink.getId(), + wsAdapter.bundleContext.registerService(AudioSink.class.getName(), audioSink, new Hashtable<>())); + // init dialog + if (clientOptions.startDialog) { + var dialogProvider = this.wsAdapter.audioDialogProvider; + if (dialogProvider == null) { + throw new IOException("Voice functionality is not ready"); + } + dialogTrigger = dialogProvider.startDialog(audioSink, audioSource, + !clientOptions.locationItem.isBlank() ? clientOptions.locationItem : null, + !clientOptions.listeningItem.isBlank() ? clientOptions.listeningItem : null, () -> disconnect()); + } else { + dialogTrigger = null; + } + } + + private synchronized void unregisterSpeakerComponents(String id) { + initialized = false; + dialogTrigger = null; + var source = wsAdapter.audioManager.getSource(getSourceId(id)); + if (source instanceof PCMWebSocketAudioSource hsAudioSource) { + try { + hsAudioSource.close(); + } catch (Exception ignored) { + } + } + if (source != null) { + ServiceRegistration sourceReg = audioComponentRegistrations.remove(source.getId()); + if (sourceReg != null) { + logger.debug("Unregistering audio source {}", source.getId()); + sourceReg.unregister(); + } + } + var sink = wsAdapter.audioManager.getSink(getSinkId(id)); + if (sink != null) { + ServiceRegistration sinkReg = audioComponentRegistrations.remove(sink.getId()); + if (sinkReg != null) { + logger.debug("Unregistering audio sink {}", sink.getId()); + sinkReg.unregister(); + } + } + } + + private void onRemoteSpot() { + var dialogTrigger = this.dialogTrigger; + if (dialogTrigger != null) { + dialogTrigger.run(); + } + } + + private String getSinkId(String id) { + return "pcm::" + id + "::sink"; + } + + private String getSourceId(String id) { + return "pcm::" + id + "::source"; + } + + public String getId() { + return id; + } + + private static class WebSocketCommand { + public String cmd = ""; + public Map args; + + public WebSocketCommand(OutputCommands cmd) { + this(cmd, new HashMap<>()); + } + + public WebSocketCommand(OutputCommands cmd, Map args) { + this.cmd = cmd.name(); + this.args = args; + } + + public enum OutputCommands { + INITIALIZED, + START_LISTENING, + STOP_LISTENING, + SINK_VOLUME, + SOURCE_VOLUME, + } + + public enum InputCommands { + INITIALIZE, + ON_SPOT, + } + } + + /** + * The {@link ConnectionOptions} represents the options provided by the ws client. + */ + public static class ConnectionOptions { + /** + * Identifier to concatenate to related services (dialog, source and sick) + */ + public String id = ""; + /** + * Force sink audio sample rate (resample in server) + */ + public @Nullable Integer forceSampleRate; + /** + * Force sink audio bit depth (resample in server) + */ + public @Nullable Integer forceBitDepth; + /** + * Force sink audio channels (resample in server) + */ + public @Nullable Integer forceChannels; + /** + * Start a dialog processor using the registered audio components + */ + public boolean startDialog = false; + /** + * Listening item for the dialog + */ + public String listeningItem = ""; + /** + * Location item for the dialog + */ + public String locationItem = ""; + + public ConnectionOptions() { + } + } +} diff --git a/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketStreamIdUtil.java b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketStreamIdUtil.java new file mode 100644 index 000000000..bc0bb36f4 --- /dev/null +++ b/bundles/org.openhab.core.io.websocket.audio/src/main/java/org/openhab/core/io/websocket/audio/internal/PCMWebSocketStreamIdUtil.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.io.websocket.audio.internal; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.security.SecureRandom; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * Utils to read/write the audio packets send though the websocket binary protocol. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public final class PCMWebSocketStreamIdUtil { + private PCMWebSocketStreamIdUtil() { + } + + /** + * Packet header length in bytes: + *

    + *
  • 2 for id
    • + *
  • 4 for sample rate as int little-endian
    • + *
  • 1 for bitDepth
    • + *
  • 1 for channels
    • + *
+ */ + public static final int PACKET_HEADER_BYTE_LENGTH = 2 + 4 + 1 + 1; + + public static AudioPacketData parseAudioPacket(byte[] bytes) throws IOException { + if (bytes.length < PACKET_HEADER_BYTE_LENGTH) { + throw new IOException("Audio packet byte length is too small, invalid data."); + } + var byteBuffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); + byte[] idBytes = new byte[] { byteBuffer.get(), byteBuffer.get() }; + int sampleRate = byteBuffer.getInt(); + byte bitDepth = byteBuffer.get(); + byte channels = byteBuffer.get(); + byte[] audioBytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(audioBytes); + return new AudioPacketData(idBytes, sampleRate, bitDepth, channels, audioBytes); + } + + public static ByteBuffer generateAudioPacketHeader(int sampleRate, byte bitDepth, byte channels) { + var byteBuffer = ByteBuffer.allocate(PACKET_HEADER_BYTE_LENGTH).order(ByteOrder.LITTLE_ENDIAN); + SecureRandom sr = new SecureRandom(); + byte[] rndBytes = new byte[2]; + sr.nextBytes(rndBytes); + byteBuffer.put(rndBytes[0]); + byteBuffer.put(rndBytes[1]); + byteBuffer.putInt(sampleRate); + byteBuffer.put(bitDepth); + byteBuffer.put(channels); + return byteBuffer; + } + + public record AudioPacketData(byte[] id, int sampleRate, byte bitDepth, byte channels, byte[] audioData) { + } +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/BasicDTService.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/BasicDTService.java new file mode 100644 index 000000000..0952ba76e --- /dev/null +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/BasicDTService.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.voice; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * This is the interface for a dialog trigger service that only allows to register a {@link DTEvent} listener. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public interface BasicDTService extends DTService { + /** + * Used to register the dialog trigger events listener + * + * @param dtListener Non-null {@link DTListener} that {@link DTEvent} events target + * @throws DTException if the listener cannot be registered due to an internal error or invalid state + */ + DTServiceHandle registerListener(DTListener dtListener) throws DTException; +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTErrorEvent.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTErrorEvent.java new file mode 100644 index 000000000..55fe87564 --- /dev/null +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTErrorEvent.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.voice; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * A {@link DTEvent} fired when the {@link DTService} encounters an error. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public class DTErrorEvent implements DTEvent { + /** + * The message describing the error + */ + private final String message; + + /** + * Constructs an instance with the passed {@code message}. + * + * @param message The message describing the error + */ + public DTErrorEvent(String message) { + this.message = message; + } + + /** + * Gets the message describing this error + * + * @return The message describing this error + */ + public String getMessage() { + return this.message; + } +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTEvent.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTEvent.java new file mode 100644 index 000000000..840d6d6f5 --- /dev/null +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTEvent.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.voice; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * A tagging interface for dialog trigger events. + * + * This interface is intended to represent events that can trigger a dialog, + * without being specific to any particular type of event such as keyword spotting. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public interface DTEvent { +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTException.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTException.java new file mode 100644 index 000000000..607655ce3 --- /dev/null +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTException.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.voice; + +import java.io.Serial; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * General purpose dialog trigger exception + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public class DTException extends Exception { + + @Serial + private static final long serialVersionUID = 1L; + + /** + * Constructs a new exception with null as its detail message. + */ + public DTException() { + super(); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param message Detail message + * @param cause The cause + */ + public DTException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new exception with the specified detail message. + * + * @param message Detail message + */ + public DTException(String message) { + super(message); + } + + /** + * Constructs a new exception with the specified cause. + * + * @param cause The cause + */ + public DTException(Throwable cause) { + super(cause); + } +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTListener.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTListener.java new file mode 100644 index 000000000..c49fa5eba --- /dev/null +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTListener.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.voice; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * The listener interface for receiving {@link DTEvent} events. + * + * A class interested in processing {@link DTEvent} events implements this interface. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public interface DTListener { + /** + * Invoked when a {@link DTEvent} event occurs. + * + * @param dtEvent The {@link DTEvent} fired by the {@link DTService} + */ + void dtEventReceived(DTEvent dtEvent); +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTService.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTService.java new file mode 100644 index 000000000..ccb4911e6 --- /dev/null +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTService.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.voice; + +import java.util.Locale; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +/** + * This is the interface that a dialog trigger service has to implement. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public interface DTService { + /** + * Returns a simple string that uniquely identifies this service + * + * @return an id that identifies this service + */ + String getId(); + + /** + * Returns a localized human-readable label that can be used within UIs. + * + * @param locale the locale to provide the label for + * @return a localized string to be used in UIs + */ + String getLabel(@Nullable Locale locale); +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTServiceHandle.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTServiceHandle.java new file mode 100644 index 000000000..9872fda29 --- /dev/null +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTServiceHandle.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.voice; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * A handle to a {@link DTService}. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public interface DTServiceHandle { + /** + * Aborts execution in the associated {@link DTServiceHandle} + */ + void abort(); +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTTriggeredEvent.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTTriggeredEvent.java new file mode 100644 index 000000000..8c3923e6d --- /dev/null +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DTTriggeredEvent.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.voice; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * A {@link DTEvent} fired when the {@link DTService} wants to trigger the dialog. + * + * @author Miguel Álvarez Díez - Initial contribution + */ +@NonNullByDefault +public class DTTriggeredEvent implements DTEvent { +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DialogContext.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DialogContext.java index a49d54beb..b67ee3bfa 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DialogContext.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DialogContext.java @@ -29,7 +29,7 @@ import org.openhab.core.voice.text.HumanLanguageInterpreter; * @author Miguel Álvarez - Initial contribution */ @NonNullByDefault -public record DialogContext(@Nullable KSService ks, @Nullable String keyword, STTService stt, TTSService tts, +public record DialogContext(@Nullable DTService dt, @Nullable String keyword, STTService stt, TTSService tts, @Nullable Voice voice, List hlis, AudioSource source, AudioSink sink, Locale locale, String dialogGroup, @Nullable String locationItem, @Nullable String listeningItem, @Nullable String listeningMelody) { @@ -42,7 +42,7 @@ public record DialogContext(@Nullable KSService ks, @Nullable String keyword, ST // services private @Nullable AudioSource source; private @Nullable AudioSink sink; - private @Nullable KSService ks; + private @Nullable DTService dt; private @Nullable STTService stt; private @Nullable TTSService tts; private @Nullable Voice voice; @@ -72,7 +72,14 @@ public record DialogContext(@Nullable KSService ks, @Nullable String keyword, ST public Builder withKS(@Nullable KSService service) { if (service != null) { - this.ks = service; + this.dt = service; + } + return this; + } + + public Builder withDT(@Nullable DTService service) { + if (service != null) { + this.dt = service; } return this; } @@ -165,7 +172,7 @@ public record DialogContext(@Nullable KSService ks, @Nullable String keyword, ST * @throws IllegalStateException if a required dialog component is missing */ public DialogContext build() throws IllegalStateException { - KSService ksService = ks; + DTService dtService = dt; STTService sttService = stt; TTSService ttsService = tts; List hliServices = hlis; @@ -191,7 +198,7 @@ public record DialogContext(@Nullable KSService ks, @Nullable String keyword, ST } throw new IllegalStateException("Cannot build dialog context: " + String.join(", ", errors) + "."); } else { - return new DialogContext(ksService, keyword, sttService, ttsService, voice, hliServices, audioSource, + return new DialogContext(dtService, keyword, sttService, ttsService, voice, hliServices, audioSource, audioSink, locale, dialogGroup, locationItem, listeningItem, listeningMelody); } } diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSEdgeService.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSEdgeService.java deleted file mode 100644 index 8c7f43380..000000000 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSEdgeService.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2010-2025 Contributors to the openHAB project - * - * See the NOTICE file(s) distributed with this work for additional - * information. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - */ -package org.openhab.core.voice; - -import java.util.Locale; - -import org.eclipse.jdt.annotation.NonNullByDefault; -import org.openhab.core.audio.AudioStream; - -/** - * This is the interface that an edge keyword spotting service has to implement. - * Used to register a keyword spotting service that is running on a remote device. - * - * @author Miguel Álvarez - Initial contribution - */ -@NonNullByDefault -public interface KSEdgeService extends KSService { - - /** - * This method links the remote keyword spotting process to a consumer. - * - * The method is supposed to return fast. - * - * @param ksListener Non-null {@link KSListener} that {@link KSEvent} events target - * @throws KSException if any parameter is invalid or a problem occurs - */ - KSServiceHandle spot(KSListener ksListener) throws KSException; - - @Override - default KSServiceHandle spot(KSListener ksListener, AudioStream audioStream, Locale locale, String keyword) - throws KSException { - throw new KSException("An edge keyword spotter is not meant to process audio in the server"); - } -} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSErrorEvent.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSErrorEvent.java index c74d04f6f..713c5d875 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSErrorEvent.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSErrorEvent.java @@ -20,27 +20,13 @@ import org.eclipse.jdt.annotation.NonNullByDefault; * @author Kelly Davis - Initial contribution */ @NonNullByDefault -public class KSErrorEvent implements KSEvent { - /** - * The message describing the error - */ - private final String message; - +public class KSErrorEvent extends DTErrorEvent implements KSEvent { /** * Constructs an instance with the passed {@code message}. * * @param message The message describing the error */ public KSErrorEvent(String message) { - this.message = message; - } - - /** - * Gets the message describing this error - * - * @return The message describing this error - */ - public String getMessage() { - return this.message; + super(message); } } diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSEvent.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSEvent.java index 3f0a50141..6dd8f28f6 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSEvent.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSEvent.java @@ -20,5 +20,5 @@ import org.eclipse.jdt.annotation.NonNullByDefault; * @author Kelly Davis - Initial contribution */ @NonNullByDefault -public interface KSEvent { +public interface KSEvent extends DTEvent { } diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSException.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSException.java index 1f878a330..59b3624fd 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSException.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSException.java @@ -22,7 +22,7 @@ import org.eclipse.jdt.annotation.NonNullByDefault; * @author Kelly Davis - Initial contribution */ @NonNullByDefault -public class KSException extends Exception { +public class KSException extends DTException { @Serial private static final long serialVersionUID = 1L; diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSListener.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSListener.java index 4ed83f144..608d40766 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSListener.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSListener.java @@ -25,7 +25,7 @@ import org.eclipse.jdt.annotation.NonNullByDefault; * @author Kelly Davis - Initial contribution */ @NonNullByDefault -public interface KSListener { +public interface KSListener extends DTListener { /** * Invoked when a {@link KSEvent} event occurs during keyword spotting. * diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSService.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSService.java index 7626b6004..3488b23da 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSService.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSService.java @@ -16,7 +16,6 @@ import java.util.Locale; import java.util.Set; import org.eclipse.jdt.annotation.NonNullByDefault; -import org.eclipse.jdt.annotation.Nullable; import org.openhab.core.audio.AudioFormat; import org.openhab.core.audio.AudioStream; @@ -27,23 +26,7 @@ import org.openhab.core.audio.AudioStream; * @author Kai Kreuzer - Refactored to use AudioStream */ @NonNullByDefault -public interface KSService { - - /** - * Returns a simple string that uniquely identifies this service - * - * @return an id that identifies this service - */ - String getId(); - - /** - * Returns a localized human readable label that can be used within UIs. - * - * @param locale the locale to provide the label for - * @return a localized string to be used in UIs - */ - String getLabel(@Nullable Locale locale); - +public interface KSService extends DTService { /** * Obtain the Locales available from this KSService * diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSServiceHandle.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSServiceHandle.java index 7554c51a2..8b42c6b08 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSServiceHandle.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSServiceHandle.java @@ -20,7 +20,7 @@ import org.eclipse.jdt.annotation.NonNullByDefault; * @author Kelly Davis - Initial contribution */ @NonNullByDefault -public interface KSServiceHandle { +public interface KSServiceHandle extends DTServiceHandle { /** * Aborts keyword spotting in the associated {@link KSService} */ diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSpottedEvent.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSpottedEvent.java index 565d6ec8d..b8a1d7f92 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSpottedEvent.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/KSpottedEvent.java @@ -21,5 +21,5 @@ import org.eclipse.jdt.annotation.NonNullByDefault; * @author Yannick Schaus - Removed AudioSource information */ @NonNullByDefault -public class KSpottedEvent implements KSEvent { +public class KSpottedEvent extends DTTriggeredEvent implements KSEvent { } diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/VoiceManager.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/VoiceManager.java index 133d0c4ba..203ed0c38 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/VoiceManager.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/VoiceManager.java @@ -173,8 +173,10 @@ public interface VoiceManager { * @param context with the configured services and options for the dialog * @throws IllegalStateException if required services are not compatible or the provided locale is not supported * by all these services or a dialog is already started for this audio source + * @return a {@link DTServiceHandle} implementation if the dialog was started correctly. */ - void startDialog(DialogContext context) throws IllegalStateException; + @Nullable + DTServiceHandle startDialog(DialogContext context) throws IllegalStateException; /** * Stop the dialog associated to an audio source diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/AudioDialogProviderImpl.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/AudioDialogProviderImpl.java new file mode 100644 index 000000000..e3a3024fc --- /dev/null +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/AudioDialogProviderImpl.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2010-2025 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.voice.internal; + +import java.util.Locale; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.core.audio.AudioDialogProvider; +import org.openhab.core.audio.AudioSink; +import org.openhab.core.audio.AudioSource; +import org.openhab.core.voice.BasicDTService; +import org.openhab.core.voice.DTListener; +import org.openhab.core.voice.DTService; +import org.openhab.core.voice.DTServiceHandle; +import org.openhab.core.voice.DTTriggeredEvent; +import org.openhab.core.voice.VoiceManager; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Reference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Allows the audio bundle to register a dialog that can be triggered programmatically. + * + * @author Miguel Álvarez - Initial contribution + */ +@Component(service = AudioDialogProvider.class) +@NonNullByDefault +public class AudioDialogProviderImpl implements AudioDialogProvider { + private final Logger logger = LoggerFactory.getLogger(AudioDialogProviderImpl.class); + private final VoiceManager voiceManager; + + @Activate + public AudioDialogProviderImpl(@Reference VoiceManager voiceManager) { + this.voiceManager = voiceManager; + } + + @Override + public @Nullable Runnable startDialog(AudioSink audioSink, AudioSource audioSource, @Nullable String locationItem, + @Nullable String listeningItem, @Nullable Runnable onAbort) { + DTService dt = new TriggerService(onAbort); + TriggerServiceHandle triggerHandle = (TriggerServiceHandle) voiceManager.startDialog( // + voiceManager.getDialogContextBuilder() // + .withSource(audioSource) // + .withSink(audioSink) // + .withLocationItem(locationItem) // + .withListeningItem(listeningItem) // + .withDT(dt) // + .build() // + ); + if (triggerHandle == null) { + return null; + } + return triggerHandle::trigger; + } + + private static class TriggerService implements BasicDTService { + @Nullable + Runnable onAbort; + + public TriggerService(@Nullable Runnable onAbort) { + this.onAbort = onAbort; + } + + @Override + public DTServiceHandle registerListener(DTListener dtListener) { + return new TriggerServiceHandle(dtListener, onAbort); + } + + @Override + public String getId() { + return "audiodialog::anonymous::trigger"; + } + + @Override + public String getLabel(@Nullable Locale locale) { + // never shown + return "Anonymous"; + } + } + + private static class TriggerServiceHandle implements DTServiceHandle { + public final DTListener dtListener; + public @Nullable Runnable abortCallback; + + public TriggerServiceHandle(DTListener dtListener, @Nullable Runnable abortCallback) { + this.dtListener = dtListener; + this.abortCallback = abortCallback; + } + + public void trigger() { + dtListener.dtEventReceived(new DTTriggeredEvent()); + } + + @Override + public void abort() { + if (this.abortCallback != null) { + this.abortCallback.run(); + this.abortCallback = null; + } + } + } +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java index d6288da82..0240c7f36 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java @@ -33,14 +33,19 @@ import org.openhab.core.i18n.TranslationProvider; import org.openhab.core.items.ItemUtil; import org.openhab.core.items.events.ItemEventFactory; import org.openhab.core.library.types.OnOffType; +import org.openhab.core.voice.BasicDTService; +import org.openhab.core.voice.DTErrorEvent; +import org.openhab.core.voice.DTEvent; +import org.openhab.core.voice.DTException; +import org.openhab.core.voice.DTService; +import org.openhab.core.voice.DTServiceHandle; +import org.openhab.core.voice.DTTriggeredEvent; import org.openhab.core.voice.DialogContext; -import org.openhab.core.voice.KSEdgeService; import org.openhab.core.voice.KSErrorEvent; import org.openhab.core.voice.KSEvent; import org.openhab.core.voice.KSException; import org.openhab.core.voice.KSListener; import org.openhab.core.voice.KSService; -import org.openhab.core.voice.KSServiceHandle; import org.openhab.core.voice.KSpottedEvent; import org.openhab.core.voice.RecognitionStartEvent; import org.openhab.core.voice.RecognitionStopEvent; @@ -98,7 +103,7 @@ public class DialogProcessor implements KSListener, STTListener { */ private boolean isSTTServerAborting = false; - private @Nullable KSServiceHandle ksServiceHandle; + private @Nullable DTServiceHandle dtServiceHandle; private @Nullable STTServiceHandle sttServiceHandle; private @Nullable AudioStream streamKS; @@ -113,8 +118,8 @@ public class DialogProcessor implements KSListener, STTListener { this.i18nProvider = i18nProvider; this.activeDialogGroups = activeDialogGroups; this.bundle = bundle; - var ks = context.ks(); - this.ksFormat = ks != null + var dt = context.dt(); + this.ksFormat = dt instanceof KSService ks ? VoiceManagerImpl.getBestMatch(context.source().getSupportedFormats(), ks.getSupportedFormats()) : null; this.sttFormat = VoiceManagerImpl.getBestMatch(context.source().getSupportedFormats(), @@ -151,34 +156,42 @@ public class DialogProcessor implements KSListener, STTListener { * Starts a persistent dialog * * @throws IllegalStateException if keyword spot service is misconfigured + * @return {@link DTServiceHandle} or null if dialog fails to start */ - public void start() throws IllegalStateException { - KSService ksService = dialogContext.ks(); + public @Nullable DTServiceHandle start() throws IllegalStateException { + DTService dtService = dialogContext.dt(); String keyword = dialogContext.keyword(); - if (ksService != null && keyword != null) { + if (dtService != null && keyword != null) { abortKS(); closeStreamKS(); - AudioFormat fmt = ksFormat; - if (fmt == null) { - logger.warn("No compatible audio format found for ks '{}' and source '{}'", ksService.getId(), - dialogContext.source().getId()); - return; - } try { - if (ksService instanceof KSEdgeService service) { - service.spot(this); - } else { + if (dtService instanceof KSService ksService) { + AudioFormat fmt = ksFormat; + if (fmt == null) { + logger.warn("No compatible audio format found for ks '{}' and source '{}'", ksService.getId(), + dialogContext.source().getId()); + return null; + } AudioStream stream = dialogContext.source().getInputStream(fmt); streamKS = stream; - ksServiceHandle = ksService.spot(this, stream, dialogContext.locale(), keyword); + dtServiceHandle = ksService.spot(this, stream, dialogContext.locale(), keyword); + } else if (dtService instanceof BasicDTService basicDTService) { + dtServiceHandle = basicDTService.registerListener(this); + } else { + logger.warn("Voice manager is not able to handle this DTService implementation '{}'", + dtService.getClass().getName()); } playStartSound(); + return dtServiceHandle; } catch (AudioException e) { logger.warn("Encountered audio error: {}", e.getMessage()); } catch (KSException e) { logger.warn("Encountered error calling spot: {}", e.getMessage()); closeStreamKS(); + } catch (DTException e) { + logger.warn("Encountered error starting the dialog trigger: {}", e.getMessage()); } + return null; } else { throw new IllegalStateException("Unable to run persistent dialog ks service is not configured"); } @@ -258,10 +271,10 @@ public class DialogProcessor implements KSListener, STTListener { } private void abortKS() { - KSServiceHandle handle = ksServiceHandle; + DTServiceHandle handle = dtServiceHandle; if (handle != null) { handle.abort(); - ksServiceHandle = null; + dtServiceHandle = null; } } @@ -314,22 +327,28 @@ public class DialogProcessor implements KSListener, STTListener { } @Override - public void ksEventReceived(KSEvent ksEvent) { + public void dtEventReceived(DTEvent dtEvent) { isSTTServerAborting = false; - if (ksEvent instanceof KSpottedEvent) { - logger.debug("KSpottedEvent event received"); + if (dtEvent instanceof DTTriggeredEvent) { + logger.debug("{} event received", + (dtEvent instanceof KSpottedEvent) ? "KSpottedEvent" : "DTTriggeredEvent"); try { startSimpleDialog(); } catch (IllegalStateException e) { logger.warn("{}", e.getMessage()); } - } else if (ksEvent instanceof KSErrorEvent kse) { - logger.debug("KSErrorEvent event received"); + } else if (dtEvent instanceof DTErrorEvent dte) { + logger.debug("{} event received", dte instanceof KSErrorEvent ? "KSErrorEvent" : "DTErrorEvent"); String text = i18nProvider.getText(bundle, "error.ks-error", null, dialogContext.locale()); - say(text == null ? kse.getMessage() : text.replace("{0}", kse.getMessage())); + say(text == null ? dte.getMessage() : text.replace("{0}", dte.getMessage())); } } + @Override + public void ksEventReceived(KSEvent ksEvent) { + dtEventReceived(ksEvent); + } + @Override public synchronized void sttEventReceived(STTEvent sttEvent) { if (sttEvent instanceof SpeechRecognitionEvent sre) { diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceConsoleCommandExtension.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceConsoleCommandExtension.java index 658d29110..c84eb766f 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceConsoleCommandExtension.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceConsoleCommandExtension.java @@ -406,7 +406,7 @@ public class VoiceConsoleCommandExtension extends AbstractConsoleCommandExtensio Collection dialogContexts = voiceManager.getDialogsContexts(); if (!dialogContexts.isEmpty()) { dialogContexts.stream().sorted(comparing(s -> s.source().getId())).forEach(c -> { - var ks = c.ks(); + var ks = c.dt(); String ksText = ks != null ? String.format(", KS: %s, Keyword: %s", ks.getId(), c.keyword()) : ""; String locationText = c.locationItem() != null ? String.format(" Location: %s", c.locationItem()) : ""; console.println(String.format( diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java index 7db8de5c7..e980008d6 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java @@ -57,6 +57,7 @@ import org.openhab.core.i18n.TranslationProvider; import org.openhab.core.library.types.PercentType; import org.openhab.core.storage.Storage; import org.openhab.core.storage.StorageService; +import org.openhab.core.voice.DTServiceHandle; import org.openhab.core.voice.DialogContext; import org.openhab.core.voice.DialogRegistration; import org.openhab.core.voice.KSService; @@ -608,10 +609,14 @@ public class VoiceManagerImpl implements VoiceManager, ConfigOptionProvider, Dia } @Override - public void startDialog(DialogContext context) throws IllegalStateException { - var ksService = context.ks(); + public @Nullable DTServiceHandle startDialog(DialogContext context) throws IllegalStateException { + var dtService = context.dt(); var ksKeyword = context.keyword(); - if (ksService == null || ksKeyword == null) { + if (dtService == null) { + throw new IllegalStateException( + "Invalid dialog context for persistent dialog: missing dialog trigger implementation"); + } + if (dtService instanceof KSService && (ksKeyword == null || ksKeyword.isEmpty())) { throw new IllegalStateException( "Invalid dialog context for persistent dialog: missing keyword spot configuration"); } @@ -619,7 +624,8 @@ public class VoiceManagerImpl implements VoiceManager, ConfigOptionProvider, Dia if (b == null) { throw new IllegalStateException("Bundle is not (yet?) set."); } - if (!checkLocales(ksService.getSupportedLocales(), context.locale()) + if ((dtService instanceof KSService ksService + && !checkLocales(ksService.getSupportedLocales(), context.locale())) || !checkLocales(context.stt().getSupportedLocales(), context.locale()) || !context.hlis().stream() .allMatch(interpreter -> checkLocales(interpreter.getSupportedLocales(), context.locale()))) { throw new IllegalStateException("Cannot start dialog as provided locale is not supported by all services."); @@ -631,7 +637,7 @@ public class VoiceManagerImpl implements VoiceManager, ConfigOptionProvider, Dia processor = new DialogProcessor(context, this, this.eventPublisher, this.activeDialogGroups, this.i18nProvider, b); dialogProcessors.put(context.source().getId(), processor); - processor.start(); + return processor.start(); } else { throw new IllegalStateException( String.format("Cannot start dialog as a dialog is already started for audio source '%s'.", @@ -785,8 +791,8 @@ public class VoiceManagerImpl implements VoiceManager, ConfigOptionProvider, Dia protected void removeKSService(KSService ksService) { this.ksServices.remove(ksService.getId()); stopDialogs(dialog -> { - var ks = dialog.dialogContext.ks(); - return ks != null && ks.getId().equals(ksService.getId()); + var dt = dialog.dialogContext.dt(); + return dt != null && dt.getId().equals(ksService.getId()); }); } diff --git a/bundles/pom.xml b/bundles/pom.xml index d10d7dd64..2d05fa0e6 100644 --- a/bundles/pom.xml +++ b/bundles/pom.xml @@ -87,6 +87,7 @@ org.openhab.core.io.transport.serial.rxtx.rfc2217 org.openhab.core.io.transport.upnp org.openhab.core.io.websocket + org.openhab.core.io.websocket.audio org.openhab.core.io.jetty.certificate org.openhab.core.model.core org.openhab.core.model.item diff --git a/features/karaf/openhab-core/src/main/feature/feature.xml b/features/karaf/openhab-core/src/main/feature/feature.xml index 491d3129c..0847ffc79 100644 --- a/features/karaf/openhab-core/src/main/feature/feature.xml +++ b/features/karaf/openhab-core/src/main/feature/feature.xml @@ -239,6 +239,12 @@ mvn:org.openhab.core.bundles/org.openhab.core.io.websocket/${project.version} + + openhab-core-base + openhab-core-io-websocket + mvn:org.openhab.core.bundles/org.openhab.core.io.websocket.audio/${project.version} + + openhab-core-base @@ -466,6 +472,7 @@ openhab-core-io-rest-transform openhab-core-io-rest-voice openhab-core-io-websocket + openhab-core-io-websocket-audio openhab-core-model-lsp openhab-core-model-item openhab-core-model-persistence