diff --git a/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioManager.java b/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioManager.java index b9fe949e8..b0cd7e19b 100644 --- a/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioManager.java +++ b/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/AudioManager.java @@ -112,6 +112,45 @@ public interface AudioManager { */ void stream(@Nullable String url, @Nullable String sinkId) throws AudioException; + /** + * Parse and synthesize a melody and play it into the default sink. + * + * The melody should be a spaced separated list of note names or silences (character 0 or O). + * You can optionally add the character "'" to increase the note one octave. + * You can optionally add ":ms" where ms is an int value to customize the note/silence milliseconds duration + * (defaults to 200ms). + * + * @param melody The url to stream from or null if streaming should be stopped. + */ + void playMelody(String melody); + + /** + * Parse and synthesize a melody and play it into the given sink. + * + * The melody should be a spaced separated list of note names or silences (character 0 or O). + * You can optionally add the character "'" to increase the note one octave. + * You can optionally add ":ms" where ms is an int value to customize the note/silence milliseconds duration + * (defaults to 200ms). + * + * @param melody The url to stream from or null if streaming should be stopped. + * @param sinkId The id of the audio sink to use or null for the default. + */ + void playMelody(String melody, @Nullable String sinkId); + + /** + * Parse and synthesize a melody and play it into the given sink at the desired volume. + * + * The melody should be a spaced separated list of note names or silences (character 0 or O). + * You can optionally add the character "'" to increase the note one octave. + * You can optionally add ":ms" where ms is an int value to customize the note/silence milliseconds duration + * (defaults to 200ms). + * + * @param melody The url to stream from or null if streaming should be stopped. + * @param sinkId The id of the audio sink to use or null for the default. + * @param volume The volume to be used or null if the default notification volume should be used + */ + void playMelody(String melody, @Nullable String sinkId, @Nullable PercentType volume); + /** * Retrieves the current volume of a sink * diff --git a/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/internal/AudioConsoleCommandExtension.java b/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/internal/AudioConsoleCommandExtension.java index c8c6986bb..833b8c92a 100644 --- a/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/internal/AudioConsoleCommandExtension.java +++ b/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/internal/AudioConsoleCommandExtension.java @@ -49,6 +49,7 @@ public class AudioConsoleCommandExtension extends AbstractConsoleCommandExtensio static final String SUBCMD_PLAY = "play"; static final String SUBCMD_STREAM = "stream"; + static final String SUBCMD_SYNTHESIZE = "synthesize"; static final String SUBCMD_SOURCES = "sources"; static final String SUBCMD_SINKS = "sinks"; @@ -72,6 +73,11 @@ public class AudioConsoleCommandExtension extends AbstractConsoleCommandExtensio "plays a sound file from the sounds folder through the specified audio sink(s) with the specified volume"), buildCommandUsage(SUBCMD_STREAM + " [] ", "streams the sound from the url through the optionally specified audio sink(s)"), + buildCommandUsage(SUBCMD_SYNTHESIZE + " [] \"\"", + "synthesize a tone melody and play it through the optionally specified audio sink(s) (" + + SUBCMD_SYNTHESIZE + " \"A O A O A\")"), + buildCommandUsage(SUBCMD_SYNTHESIZE + " \"\" ", + "synthesize a tone melody and play it through the optionally specified audio sink(s) with the specified volume"), buildCommandUsage(SUBCMD_SOURCES, "lists the audio sources"), buildCommandUsage(SUBCMD_SINKS, "lists the audio sinks") }); } @@ -96,6 +102,14 @@ public class AudioConsoleCommandExtension extends AbstractConsoleCommandExtensio console.println("Specify url to stream from, and optionally the sink(s) to use"); } return; + case SUBCMD_SYNTHESIZE: + if (args.length > 1) { + synthesizeMelody(Arrays.copyOfRange(args, 1, args.length), console); + } else { + console.println( + "Specify file to play, and optionally the sink(s) to use (e.g. 'play javasound A B C:1000')"); + } + return; case SUBCMD_SOURCES: listSources(console); return; @@ -161,6 +175,40 @@ public class AudioConsoleCommandExtension extends AbstractConsoleCommandExtensio } } + private void synthesizeMelody(String[] args, Console console) { + switch (args.length) { + case 1: + playMelodyOnSink(null, args[0], null, console); + break; + case 2: + playMelodyOnSinks(args[0], args[1], null, console); + break; + case 3: + PercentType volume = null; + try { + volume = PercentType.valueOf(args[2]); + } catch (Exception e) { + console.println("Specify volume as percentage between 0 and 100"); + break; + } + playMelodyOnSink(args[0], args[1], volume, console); + break; + default: + break; + } + } + + private void playMelodyOnSinks(String pattern, String melody, @Nullable PercentType volume, Console console) { + for (String sinkId : audioManager.getSinkIds(pattern)) { + playMelodyOnSink(sinkId, melody, volume, console); + } + } + + private void playMelodyOnSink(@Nullable String sinkId, String melody, @Nullable PercentType volume, + Console console) { + audioManager.playMelody(melody, sinkId, volume); + } + private void playOnSinks(String pattern, String fileName, @Nullable PercentType volume, Console console) { for (String sinkId : audioManager.getSinkIds(pattern)) { playOnSink(sinkId, fileName, volume, console); diff --git a/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/internal/AudioManagerImpl.java b/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/internal/AudioManagerImpl.java index acfbdabc5..d8e794849 100644 --- a/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/internal/AudioManagerImpl.java +++ b/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/internal/AudioManagerImpl.java @@ -18,6 +18,7 @@ import static java.util.stream.Collectors.toList; import java.io.File; import java.io.IOException; import java.net.URI; +import java.text.ParseException; import java.util.Collection; import java.util.HashSet; import java.util.Locale; @@ -30,6 +31,7 @@ import org.eclipse.jdt.annotation.NonNullByDefault; import org.eclipse.jdt.annotation.Nullable; import org.openhab.core.OpenHAB; import org.openhab.core.audio.AudioException; +import org.openhab.core.audio.AudioFormat; import org.openhab.core.audio.AudioManager; import org.openhab.core.audio.AudioSink; import org.openhab.core.audio.AudioSource; @@ -38,6 +40,7 @@ import org.openhab.core.audio.FileAudioStream; import org.openhab.core.audio.URLAudioStream; import org.openhab.core.audio.UnsupportedAudioFormatException; import org.openhab.core.audio.UnsupportedAudioStreamException; +import org.openhab.core.audio.utils.ToneSynthesizer; import org.openhab.core.config.core.ConfigOptionProvider; import org.openhab.core.config.core.ConfigurableService; import org.openhab.core.config.core.ParameterOption; @@ -191,6 +194,37 @@ public class AudioManagerImpl implements AudioManager, ConfigOptionProvider { play(audioStream, sinkId, null); } + @Override + public void playMelody(String melody) { + playMelody(melody, null); + } + + @Override + public void playMelody(String melody, @Nullable String sinkId) { + playMelody(melody, sinkId, null); + } + + @Override + public void playMelody(String melody, @Nullable String sinkId, @Nullable PercentType volume) { + AudioSink sink = getSink(sinkId); + if (sink == null) { + logger.warn("Failed playing melody as no audio sink {} was found.", sinkId); + return; + } + var synthesizerFormat = AudioFormat.getBestMatch(ToneSynthesizer.getSupportedFormats(), + sink.getSupportedFormats()); + if (synthesizerFormat == null) { + logger.warn("Failed playing melody as sink {} does not support wav.", sinkId); + return; + } + try { + var audioStream = new ToneSynthesizer(synthesizerFormat).getStream(ToneSynthesizer.parseMelody(melody)); + play(audioStream, sinkId, volume); + } catch (IOException | ParseException e) { + logger.warn("Failed playing melody: {}", e.getMessage()); + } + } + @Override public PercentType getVolume(@Nullable String sinkId) throws IOException { AudioSink sink = getSink(sinkId); diff --git a/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/utils/ToneSynthesizer.java b/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/utils/ToneSynthesizer.java new file mode 100644 index 000000000..2160bd3b5 --- /dev/null +++ b/bundles/org.openhab.core.audio/src/main/java/org/openhab/core/audio/utils/ToneSynthesizer.java @@ -0,0 +1,260 @@ +/** + * Copyright (c) 2010-2022 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.audio.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import javax.sound.sampled.AudioFileFormat; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.openhab.core.audio.AudioFormat; +import org.openhab.core.audio.AudioStream; +import org.openhab.core.audio.ByteArrayAudioStream; + +/** + * An audio tone synthesizer. A utility to sent tone melodies to audio sinks. + * Limited to wav little endian streams. + * + * @author Miguel Álvarez - Initial contribution + * + */ +@NonNullByDefault +public class ToneSynthesizer { + private final long sampleRate; + private final int bitDepth; + private final int bitRate; + private final int channels; + private final boolean bigEndian; + + public static Set getSupportedFormats() { + return Set.of(new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, null, null, null, + null)); + } + + /** + * Parses a tone melody into a list of {@link Tone} instances. + * The melody should be a spaced separated list of note names or silences (character 0 or O). + * You can optionally add the character "'" to increase the note one octave. + * You can optionally add ":ms" where ms is an int value to customize the note/silence milliseconds duration + * (defaults to 200ms). + * + * @param melody to be parsed. + * @return list of {@link Tone} instances. + * @throws ParseException if melody can not be played. + */ + public static List parseMelody(String melody) throws ParseException { + var melodySounds = new ArrayList(); + var noteTextList = melody.split("\\s"); + var melodyTextIndex = 0; + for (var i = 0; i < noteTextList.length; i++) { + var noteText = noteTextList[i]; + var noteTextParts = noteText.split(":"); + var soundMillis = 200; + switch (noteTextParts.length) { + case 2: + try { + soundMillis = Integer.parseInt(noteTextParts[1]); + } catch (NumberFormatException e) { + throw new ParseException("Unable to parse note duration " + noteText, melodyTextIndex); + } + case 1: + var note = noteTextParts[0]; + int octaves = (int) note.chars().filter(ch -> ch == '\'').count(); + note = note.replaceAll("'", ""); + var noteObj = Note.fromString(note); + if (noteObj.isPresent()) { + melodySounds.add(noteTone(noteObj.get(), soundMillis, octaves)); + break; + } else if (note.equals("O") || note.equals("0")) { + melodySounds.add(silenceTone(soundMillis)); + break; + } + default: + throw new ParseException("Unable to parse note " + noteText, melodyTextIndex); + } + melodyTextIndex += noteText.length() + 1; + } + return melodySounds; + } + + public static Tone noteTone(Note note, long millis) { + return noteTone(note, millis, 0); + } + + public static Tone noteTone(Note note, long millis, int octaves) { + return new Tone(note.getFrequency() * (octaves + 1), millis); + } + + public static Tone silenceTone(long millis) { + return new Tone(0.0, millis); + } + + public ToneSynthesizer(AudioFormat audioFormat) { + assert audioFormat.getFrequency() != null; + this.sampleRate = audioFormat.getFrequency(); + assert audioFormat.getBitDepth() != null; + this.bitDepth = audioFormat.getBitDepth(); + assert audioFormat.getBitRate() != null; + this.bitRate = audioFormat.getBitRate(); + assert audioFormat.getChannels() != null; + this.channels = audioFormat.getChannels(); + var bigEndian = audioFormat.isBigEndian(); + assert bigEndian != null; + this.bigEndian = bigEndian; + } + + /** + * Synthesize a list of {@link Tone} into a wav audio stream + * + * @param tones the list of {@link Tone} + * @return an audio stream with the synthesized tones + * @throws IOException in case of problems writing the audio stream + */ + public AudioStream getStream(List tones) throws IOException { + int byteRate = (int) (sampleRate * bitDepth * channels / 8); + byte[] audioBuffer = new byte[0]; + var fixedTones = new ArrayList<>(tones); + fixedTones.add(silenceTone(100)); + for (var sound : fixedTones) { + var frequency = sound.frequency; + var millis = sound.millis; + int samplesPerChannel = (int) Math.ceil(sampleRate * (((double) millis) / 1000)); + int byteSize = (int) Math.ceil(byteRate * ((double) millis / 1000)); + byte[] audioPart = getAudioBytes(frequency, samplesPerChannel); + audioBuffer = ByteBuffer.allocate(audioBuffer.length + audioPart.length).put(audioBuffer).put(audioPart) + .array(); + } + // ensure min audio size + int minByteSize = (int) Math.ceil(byteRate * 0.5); + if (audioBuffer.length < minByteSize) { + // ensure min duration of half second, prevents issue with pulseaudio sink + byte[] padBytes = new byte[minByteSize - audioBuffer.length]; + audioBuffer = ByteBuffer.allocate(minByteSize).put(padBytes).put(audioBuffer).array(); + } + if (!bigEndian) { + // for little endian add the RIFF header to the stream to increase compatibility + return getAudioStreamWithRIFFHeader(audioBuffer); + } else { + return getAudioStream(audioBuffer); + } + } + + private double getSample(double frequency, int sampleNum) { + return 0xfff * Math.sin(frequency * (2 * Math.PI) * sampleNum / sampleRate); + } + + private ByteArrayAudioStream getAudioStreamWithRIFFHeader(byte[] audioBytes) throws IOException { + var jAudioFormat = new javax.sound.sampled.AudioFormat(sampleRate, bitDepth, channels, true, bigEndian); + AudioInputStream audioInputStreamTemp = new AudioInputStream(new ByteArrayInputStream(audioBytes), jAudioFormat, + (long) Math.ceil(((double) audioBytes.length) / jAudioFormat.getFrameSize())); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + AudioSystem.write(audioInputStreamTemp, AudioFileFormat.Type.WAVE, outputStream); + return getAudioStream(outputStream.toByteArray()); + } + + private ByteArrayAudioStream getAudioStream(byte[] audioBytes) { + return new ByteArrayAudioStream(audioBytes, new AudioFormat(AudioFormat.CONTAINER_WAVE, + AudioFormat.CODEC_PCM_SIGNED, bigEndian, bitDepth, bitRate, sampleRate, channels)); + } + + private byte[] getAudioBytes(double frequency, int samplesPerChannel) { + var audioBuffer = ByteBuffer.allocate(samplesPerChannel * (bitDepth / 8) * channels); + audioBuffer.order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < samplesPerChannel; i++) { + short sample = (short) getSample(frequency, i); + for (int c = 0; c < channels; c++) { + switch (bitDepth) { + case 8: + audioBuffer.put((byte) (sample & 0xff)); + break; + case 16: + audioBuffer.putShort(sample); + break; + case 24: + putInt24Bits(audioBuffer, ((int) sample) << 8); + break; + case 32: + audioBuffer.putInt(((int) sample) << 16); + break; + } + } + } + return audioBuffer.array(); + } + + private void putInt24Bits(ByteBuffer buffer, int value) { + if (bigEndian) { + buffer.put((byte) ((value >> 16) & 0xff)); + buffer.put((byte) ((value >> 8) & 0xff)); + buffer.put((byte) (value & 0xff)); + } else { + buffer.put((byte) (value & 0xff)); + buffer.put((byte) ((value >> 8) & 0xff)); + buffer.put((byte) ((value >> 16) & 0xff)); + } + } + + public static class Tone { + private final double frequency; + private final long millis; + + private Tone(double frequency, long millis) { + this.frequency = frequency; + this.millis = millis; + } + } + + public enum Note { + B(List.of("B", "Si"), 493.88), + Bb(List.of("A#", "Bb", "LA#", "SIb"), 466.16), + A(List.of("A", "LA"), 440.0), + Ab(List.of("G#", "Ab", "SOL#", "LAb"), 415.30), + G(List.of("G", "SOL"), 392.0), + Gb(List.of("F#", "Gb", "FA#", "SOLb"), 369.99), + F(List.of("F", "FA"), 349.23), + E(List.of("E", "MI"), 329.63), + Eb(List.of("D#", "Eb", "RE#", "MIb"), 311.13), + D(List.of("D", "RE"), 293.66), + Cb(List.of("C#", "Db", "DO#", "REb"), 277.18), + C(List.of("C", "DO"), 261.63); + + private final List names; + private final double frequency; + + Note(List names, double frequency) { + this.names = names; + this.frequency = frequency; + } + + public double getFrequency() { + return frequency; + } + + public static Optional fromString(String note) { + return Arrays.stream(Note.values()) + .filter(note1 -> note1.names.stream().filter(note::equalsIgnoreCase).count() == 1).findAny(); + } + } +} diff --git a/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/MediaActionTypeProvider.java b/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/MediaActionTypeProvider.java index 4a7757e5b..be8ca0127 100644 --- a/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/MediaActionTypeProvider.java +++ b/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/MediaActionTypeProvider.java @@ -41,7 +41,7 @@ import org.osgi.service.component.annotations.Component; import org.osgi.service.component.annotations.Reference; /** - * This class dynamically provides the Play and Say action types. + * This class dynamically provides the Play, Say and Synthesize action types. * This is necessary since there is no other way to provide dynamic config param options for module types. * * @author Kai Kreuzer - Initial contribution @@ -67,6 +67,8 @@ public class MediaActionTypeProvider implements ModuleTypeProvider { return getPlayActionType(locale); case SayActionHandler.TYPE_ID: return getSayActionType(locale); + case SynthesizeActionHandler.TYPE_ID: + return getSynthesizeActionType(locale); default: return null; } @@ -88,6 +90,12 @@ public class MediaActionTypeProvider implements ModuleTypeProvider { null, null); } + private ModuleType getSynthesizeActionType(@Nullable Locale locale) { + return new ActionType(SynthesizeActionHandler.TYPE_ID, getConfigSynthesizeDesc(locale), + "synthesize a tone melody", "Synthesize the given melody text and play it. Optionally sets the volume.", + null, Visibility.VISIBLE, null, null); + } + private List getConfigPlayDesc(@Nullable Locale locale) { return List.of( ConfigDescriptionParameterBuilder.create(PlayActionHandler.PARAM_SOUND, Type.TEXT).withRequired(true) @@ -103,6 +111,14 @@ public class MediaActionTypeProvider implements ModuleTypeProvider { getAudioSinkConfigDescParam(locale), getVolumeConfigDescParam(locale)); } + private List getConfigSynthesizeDesc(@Nullable Locale locale) { + return List.of( + ConfigDescriptionParameterBuilder.create(SynthesizeActionHandler.PARAM_MELODY, Type.TEXT) + .withRequired(true).withLabel("Melody") + .withDescription("the melody as spaced separated note names").build(), + getAudioSinkConfigDescParam(locale), getVolumeConfigDescParam(locale)); + } + private ConfigDescriptionParameter getAudioSinkConfigDescParam(@Nullable Locale locale) { ConfigDescriptionParameter param2 = ConfigDescriptionParameterBuilder .create(SayActionHandler.PARAM_SINK, Type.TEXT).withRequired(false).withLabel("Sink") diff --git a/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/MediaModuleHandlerFactory.java b/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/MediaModuleHandlerFactory.java index 46868ed28..91c56e063 100644 --- a/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/MediaModuleHandlerFactory.java +++ b/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/MediaModuleHandlerFactory.java @@ -68,6 +68,8 @@ public class MediaModuleHandlerFactory extends BaseModuleHandlerFactory { return new SayActionHandler((Action) module, voiceManager); case PlayActionHandler.TYPE_ID: return new PlayActionHandler((Action) module, audioManager); + case SynthesizeActionHandler.TYPE_ID: + return new SynthesizeActionHandler((Action) module, audioManager); default: break; } diff --git a/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/SynthesizeActionHandler.java b/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/SynthesizeActionHandler.java new file mode 100644 index 000000000..58e4d3954 --- /dev/null +++ b/bundles/org.openhab.core.automation.module.media/src/main/java/org/openhab/core/automation/module/media/internal/SynthesizeActionHandler.java @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2010-2022 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.core.automation.module.media.internal; + +import java.math.BigDecimal; +import java.util.Map; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.core.audio.AudioManager; +import org.openhab.core.automation.Action; +import org.openhab.core.automation.handler.BaseActionModuleHandler; +import org.openhab.core.library.types.PercentType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is an ModuleHandler implementation for Actions that synthesize a tone melody. + * + * @author Miguel Álvarez - Initial contribution + */ +@NonNullByDefault +public class SynthesizeActionHandler extends BaseActionModuleHandler { + + public static final String TYPE_ID = "media.SynthesizeAction"; + public static final String PARAM_MELODY = "melody"; + public static final String PARAM_SINK = "sink"; + public static final String PARAM_VOLUME = "volume"; + + private final Logger logger = LoggerFactory.getLogger(SynthesizeActionHandler.class); + + private final AudioManager audioManager; + private final String melody; + private final @Nullable String sink; + private final @Nullable PercentType volume; + + public SynthesizeActionHandler(Action module, AudioManager audioManager) { + super(module); + this.audioManager = audioManager; + + this.melody = module.getConfiguration().get(PARAM_MELODY).toString(); + + Object sinkParam = module.getConfiguration().get(PARAM_SINK); + this.sink = sinkParam != null ? sinkParam.toString() : null; + + Object volumeParam = module.getConfiguration().get(PARAM_VOLUME); + this.volume = volumeParam instanceof BigDecimal ? new PercentType((BigDecimal) volumeParam) : null; + } + + @Override + public @Nullable Map execute(Map context) { + audioManager.playMelody(melody, sink, volume); + return null; + } +} diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DialogContext.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DialogContext.java index e05dded56..abd58d45a 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DialogContext.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/DialogContext.java @@ -40,10 +40,11 @@ public class DialogContext { private final AudioSink sink; private final Locale locale; private final @Nullable String listeningItem; + private final @Nullable String listeningMelody; public DialogContext(@Nullable KSService ks, @Nullable String keyword, STTService stt, TTSService tts, @Nullable Voice voice, List hlis, AudioSource source, AudioSink sink, - Locale locale, @Nullable String listeningItem) { + Locale locale, @Nullable String listeningItem, @Nullable String listeningMelody) { this.ks = ks; this.keyword = keyword; this.stt = stt; @@ -54,6 +55,7 @@ public class DialogContext { this.sink = sink; this.locale = locale; this.listeningItem = listeningItem; + this.listeningMelody = listeningMelody; } public @Nullable KSService ks() { @@ -96,6 +98,10 @@ public class DialogContext { return listeningItem; } + public @Nullable String listeningMelody() { + return listeningMelody; + } + /** * Builder for {@link DialogContext} * Allows to describe a dialog context without requiring the involved services to be loaded @@ -111,6 +117,7 @@ public class DialogContext { private List hlis = List.of(); // options private @Nullable String listeningItem; + private @Nullable String listeningMelody; private String keyword; private Locale locale; @@ -168,6 +175,11 @@ public class DialogContext { return this; } + public Builder withMelody(@Nullable String listeningMelody) { + this.listeningMelody = listeningMelody; + return this; + } + public Builder withLocale(Locale locale) { this.locale = locale; return this; @@ -206,7 +218,7 @@ public class DialogContext { throw new IllegalStateException("Cannot build dialog context: " + String.join(", ", errors) + "."); } return new DialogContext(ksService, keyword, sttService, ttsService, voice, hliServices, audioSource, - audioSink, locale, listeningItem); + audioSink, locale, listeningItem, listeningMelody); } } } diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java index 33c83ee17..25e7c307b 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/DialogProcessor.java @@ -13,8 +13,12 @@ package org.openhab.core.voice.internal; import java.io.IOException; +import java.text.ParseException; import java.util.HashSet; +import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.eclipse.jdt.annotation.NonNullByDefault; import org.eclipse.jdt.annotation.Nullable; @@ -23,6 +27,7 @@ import org.openhab.core.audio.AudioFormat; import org.openhab.core.audio.AudioStream; import org.openhab.core.audio.UnsupportedAudioFormatException; import org.openhab.core.audio.UnsupportedAudioStreamException; +import org.openhab.core.audio.utils.ToneSynthesizer; import org.openhab.core.events.EventPublisher; import org.openhab.core.i18n.TranslationProvider; import org.openhab.core.items.ItemUtil; @@ -63,6 +68,7 @@ import org.slf4j.LoggerFactory; * @author Laurent Garnier - Added stop() + null annotations + resources releasing * @author Miguel Álvarez - Close audio streams + use RecognitionStartEvent * @author Miguel Álvarez - Use dialog context + * @author Miguel Álvarez - Add sounds * */ @NonNullByDefault @@ -71,6 +77,7 @@ public class DialogProcessor implements KSListener, STTListener { private final Logger logger = LoggerFactory.getLogger(DialogProcessor.class); private final DialogContext dialogContext; + private @Nullable List listeningMelody; private final EventPublisher eventPublisher; private final TranslationProvider i18nProvider; private final Bundle bundle; @@ -95,6 +102,7 @@ public class DialogProcessor implements KSListener, STTListener { private @Nullable AudioStream streamKS; private @Nullable AudioStream streamSTT; + private @Nullable ToneSynthesizer toneSynthesizer; public DialogProcessor(DialogContext context, DialogEventListener eventListener, EventPublisher eventPublisher, TranslationProvider i18nProvider, Bundle bundle) { @@ -111,6 +119,30 @@ public class DialogProcessor implements KSListener, STTListener { context.stt().getSupportedFormats()); this.ttsFormat = VoiceManagerImpl.getBestMatch(context.tts().getSupportedFormats(), context.sink().getSupportedFormats()); + initToneSynthesizer(context.listeningMelody()); + } + + private void initToneSynthesizer(@Nullable String listeningMelodyText) { + @Nullable + List listeningMelody = null; + ToneSynthesizer toneSynthesizer = null; + if (listeningMelodyText != null && !listeningMelodyText.isBlank()) { + try { + listeningMelody = ToneSynthesizer.parseMelody(listeningMelodyText); + var synthesizerFormat = VoiceManagerImpl.getBestMatch(ToneSynthesizer.getSupportedFormats(), + dialogContext.sink().getSupportedFormats()); + if (synthesizerFormat != null) { + toneSynthesizer = new ToneSynthesizer(synthesizerFormat); + logger.debug("Sounds enabled"); + } else { + logger.warn("Sounds disabled, synthesizer is not compatible with this sink"); + } + } catch (ParseException e) { + logger.warn("Sounds disabled, unable to parse 'listening' melody: {}", e.getMessage()); + } + } + this.toneSynthesizer = toneSynthesizer; + this.listeningMelody = listeningMelody; } /** @@ -134,6 +166,7 @@ public class DialogProcessor implements KSListener, STTListener { AudioStream stream = dialogContext.source().getInputStream(fmt); streamKS = stream; ksServiceHandle = ksService.spot(this, stream, dialogContext.locale(), keyword); + playStartSound(); } catch (AudioException e) { logger.warn("Encountered audio error: {}", e.getMessage()); } catch (KSException e) { @@ -158,6 +191,7 @@ public class DialogProcessor implements KSListener, STTListener { dialogContext.source().getId()); return; } + playOnListeningSound(); try { AudioStream stream = dialogContext.source().getInputStream(fmt); streamSTT = stream; @@ -185,6 +219,7 @@ public class DialogProcessor implements KSListener, STTListener { abortKS(); closeStreamKS(); toggleProcessing(false); + playStopSound(); } /** @@ -366,6 +401,39 @@ public class DialogProcessor implements KSListener, STTListener { } } + private void playStartSound() { + playNotes(Stream.of(ToneSynthesizer.Note.G, ToneSynthesizer.Note.A, ToneSynthesizer.Note.B) + .map(note -> ToneSynthesizer.noteTone(note, 100L)).collect(Collectors.toList())); + } + + private void playStopSound() { + playNotes(Stream.of(ToneSynthesizer.Note.B, ToneSynthesizer.Note.A, ToneSynthesizer.Note.G) + .map(note -> ToneSynthesizer.noteTone(note, 100L)).collect(Collectors.toList())); + } + + private void playOnListeningSound() { + var listeningMelody = this.listeningMelody; + if (listeningMelody != null) { + playNotes(listeningMelody); + } + } + + private void playNotes(List notes) { + var toneSynthesizer = this.toneSynthesizer; + if (toneSynthesizer != null) { + try (AudioStream stream = toneSynthesizer.getStream(notes)) { + var sink = dialogContext.sink(); + if (sink.getSupportedStreams().stream().anyMatch(clazz -> clazz.isInstance(stream))) { + sink.process(stream); + } else { + logger.warn("Failed playing synthesizer sound as audio sink doesn't support it."); + } + } catch (UnsupportedAudioFormatException | UnsupportedAudioStreamException | IOException e) { + logger.warn("{} playing synthesizer sound: {}", e.getClass().getName(), e.getMessage()); + } + } + } + /** * Check if other DialogProcessor instance have same configuration ignoring the keyword spotting configuration * diff --git a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java index 4ffe52b18..81aef2516 100644 --- a/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java +++ b/bundles/org.openhab.core.voice/src/main/java/org/openhab/core/voice/internal/VoiceManagerImpl.java @@ -94,6 +94,7 @@ public class VoiceManagerImpl implements VoiceManager, ConfigOptionProvider, Dia protected static final String CONFIG_URI = "system:voice"; private static final String CONFIG_KEYWORD = "keyword"; private static final String CONFIG_LISTENING_ITEM = "listeningItem"; + private static final String CONFIG_LISTENING_MELODY = "listeningMelody"; private static final String CONFIG_DEFAULT_HLI = "defaultHLI"; private static final String CONFIG_DEFAULT_KS = "defaultKS"; private static final String CONFIG_DEFAULT_STT = "defaultSTT"; @@ -121,6 +122,7 @@ public class VoiceManagerImpl implements VoiceManager, ConfigOptionProvider, Dia */ private String keyword = DEFAULT_KEYWORD; private @Nullable String listeningItem; + private @Nullable String listeningMelody; private @Nullable String defaultTTS; private @Nullable String defaultSTT; private @Nullable String defaultKS; @@ -160,6 +162,9 @@ public class VoiceManagerImpl implements VoiceManager, ConfigOptionProvider, Dia this.listeningItem = config.containsKey(CONFIG_LISTENING_ITEM) ? config.get(CONFIG_LISTENING_ITEM).toString() : null; + this.listeningMelody = config.containsKey(CONFIG_LISTENING_MELODY) + ? config.get(CONFIG_LISTENING_MELODY).toString() + : null; this.defaultTTS = config.containsKey(CONFIG_DEFAULT_TTS) ? config.get(CONFIG_DEFAULT_TTS).toString() : null; this.defaultSTT = config.containsKey(CONFIG_DEFAULT_STT) ? config.get(CONFIG_DEFAULT_STT).toString() : null; this.defaultKS = config.containsKey(CONFIG_DEFAULT_KS) ? config.get(CONFIG_DEFAULT_KS).toString() : null; @@ -494,6 +499,7 @@ public class VoiceManagerImpl implements VoiceManager, ConfigOptionProvider, Dia .withTTS(this.getTTS()) // .withHLIs(this.getHLIs()) // .withVoice(this.getDefaultVoice()) // + .withMelody(listeningMelody) // .withListeningItem(listeningItem); } diff --git a/bundles/org.openhab.core.voice/src/main/resources/OH-INF/config/voice.xml b/bundles/org.openhab.core.voice/src/main/resources/OH-INF/config/voice.xml index ea2fde09c..25d9bac28 100644 --- a/bundles/org.openhab.core.voice/src/main/resources/OH-INF/config/voice.xml +++ b/bundles/org.openhab.core.voice/src/main/resources/OH-INF/config/voice.xml @@ -38,6 +38,17 @@ If provided, the item will be switched on during the period when the dialog processor has spotted the keyword and is listening for commands. + + + + + + + false + + A melody to be played to advertise the user the dialog processing is going to start, leave empty to + disable it. (Spaced separated list of notes. Example: "A O:100 A':50") + diff --git a/bundles/org.openhab.core.voice/src/main/resources/OH-INF/i18n/voice.properties b/bundles/org.openhab.core.voice/src/main/resources/OH-INF/i18n/voice.properties index 440e83115..cc63e3d4b 100644 --- a/bundles/org.openhab.core.voice/src/main/resources/OH-INF/i18n/voice.properties +++ b/bundles/org.openhab.core.voice/src/main/resources/OH-INF/i18n/voice.properties @@ -1,20 +1,24 @@ -system.config.voice.defaultTTS.label = Default Text-to-Speech -system.config.voice.defaultTTS.description = The default text-to-speech service to use if no other is specified. -system.config.voice.defaultSTT.label = Default Speech-to-Text -system.config.voice.defaultSTT.description = The default speech-to-text service to use if no other is specified. -system.config.voice.defaultVoice.label = Default Voice -system.config.voice.defaultVoice.description = The default voice to use if no specific TTS service or voice is specified. system.config.voice.defaultHLI.label = Default Human Language Interpreter system.config.voice.defaultHLI.description = The default human language interpreter to use if no other is specified. system.config.voice.defaultKS.label = Default Keyword Spotter system.config.voice.defaultKS.description = The default keyword spotting service to use if no other is specified. +system.config.voice.defaultSTT.label = Default Speech-to-Text +system.config.voice.defaultSTT.description = The default speech-to-text service to use if no other is specified. +system.config.voice.defaultTTS.label = Default Text-to-Speech +system.config.voice.defaultTTS.description = The default text-to-speech service to use if no other is specified. +system.config.voice.defaultVoice.label = Default Voice +system.config.voice.defaultVoice.description = The default voice to use if no specific TTS service or voice is specified. system.config.voice.keyword.label = Magic Word system.config.voice.keyword.description = The magic word to spot before initiating a dialog. system.config.voice.listeningItem.label = Listening Switch system.config.voice.listeningItem.description = If provided, the item will be switched on during the period when the dialog processor has spotted the keyword and is listening for commands. - -service.system.voice.label = Voice +system.config.voice.listeningMelody.label = Listening Melody +system.config.voice.listeningMelody.description = A melody to be played to advertise the user the dialog processing is going to start, leave empty to disable it. (Spaced separated list of notes. Example: "A O:100 A':50") +system.config.voice.listeningMelody.option.Bb = Bb +system.config.voice.listeningMelody.option.F# = F# +system.config.voice.listeningMelody.option.E = E error.ks-error = Encountered error while spotting keywords, {0} error.stt-error = Encountered error while recognizing text, {0} error.stt-exception = Error during recognition, {0} +service.system.voice.label = Voice