Auditory scanning Features - #568

app/lang/i18n.en.json

@@ -1,6 +1,7 @@
 {
   "GridActionSpeak": "Speak label",
   "GridActionSpeakCustom": "Speak custom text",
+  "GridActionSpeakLetters": "Speak text letter by letter",
   "GridActionAudio": "Play recorded audio",
   "GridActionWordForm": "Change word forms",
   "GridActionNavigate": "Navigate to other grid",
@@ -121,6 +122,8 @@
   "language": "Language",
   "automaticCurrentLanguage": "automatic (current language)",
   "textToSpeak": "Text to speak",
+  "leaveEmptyToUseElementLabel": "Leave empty to use element label",
+  "pauseBetweenLettersMs": "Pause between letters (ms)",
   "navigateToHomeGrid": "Navigate to home grid",
   "navigateToLastOpenedGrid": "Navigate to last opened grid",
   "navigateToGrid": "Navigate to grid",
@@ -969,6 +972,8 @@
   "automaticTimedSequentialInput": "Automatic (timed) sequential input",
   "acousticFeedbackOptions": "Acoustic Feedback Options",
   "speedForReadingActiveElement": "Speed for reading active element",
+  "fastReadElementCount": "Number of elements to read fast during row/column scanning",
+  "fastReadSpeed": "Speed for fast reading during scanning",
   "enableAcousticFeedbackUsingBeepingSounds": "Enable acoustic feedback using beeping sounds",
   "colon": "colon",
   "period": "period",
@@ -979,6 +984,7 @@
   "hyphen": "hyphen",
   "space": "space",
   "readElementActionsInAdditionToLabel": "Read element actions in addition to label",
+  "readCollectElementsLetterByLetter": "Read collect elements letter-by-letter (until word completion)",
   "generalInputSettings": "General input settings",
   "minimumPauseForCollectingAndSpeakingTheSameCellSev": "Minimum pause for collecting and speaking the same cell several times in a row",
   "backupFileDoesntContainData": "Backup file doesn't contain grid configuration!",

src/js/model/GridActionSpeakLetters.js

@@ -0,0 +1,32 @@
+import { modelUtil } from '../util/modelUtil';
+import { constants } from '../util/constants';
+import { Model } from '../externals/objectmodel';
+import { i18nService } from '../service/i18nService';
+
+class GridActionSpeakLetters extends Model({
+    id: String,
+    modelName: String,
+    modelVersion: String,
+    speakLanguage: [String, null, undefined],
+    speakText: [Object, String], //map locale -> translation, e.g. "de" => LabelDE
+    pauseDurationMs: [Number] // pause duration between letters in milliseconds
+}) {
+    constructor(properties, elementToCopy) {
+        properties = modelUtil.setDefaults(properties, elementToCopy, GridActionSpeakLetters);
+        super(properties);
+        this.id = this.id || modelUtil.generateId('grid-action-speak-letters');
+    }
+
+    static getModelName() {
+        return 'GridActionSpeakLetters';
+    }
+}
+
+GridActionSpeakLetters.defaults({
+    id: '', //will be replaced by constructor
+    modelName: GridActionSpeakLetters.getModelName(),
+    modelVersion: constants.MODEL_VERSION,
+    pauseDurationMs: 200 // default 200ms pause between letters
+});
+
+export { GridActionSpeakLetters };

src/js/model/GridElement.js

@@ -2,6 +2,7 @@ import { modelUtil } from '../util/modelUtil';
 import { GridImage } from './GridImage';
 import { GridActionSpeak } from './GridActionSpeak';
 import { GridActionSpeakCustom } from './GridActionSpeakCustom';
+import { GridActionSpeakLetters } from './GridActionSpeakLetters';
 import { GridActionNavigate } from './GridActionNavigate';
 import { GridActionARE } from './GridActionARE';
 import { GridActionOpenHAB } from './GridActionOpenHAB';
@@ -72,6 +73,7 @@ class GridElement extends Model({
             GridActionSpeak,
             GridActionNavigate,
             GridActionSpeakCustom,
+            GridActionSpeakLetters,
             GridActionAudio,
             GridActionWordForm,
             GridActionPredict,

src/js/model/InputConfig.js

@@ -12,6 +12,9 @@ class InputConfig extends Model({
     globalReadActive: [Boolean], //read out loud active element(s)?
     globalReadActiveRate: [Number],
     globalReadAdditionalActions: [Boolean],
+    globalReadActiveFastCount: [Number], //number of elements to read fast during row/column scanning
+    globalReadActiveFastRate: [Number], //rate for fast reading during scanning
+    globalReadCollectLetters: [Boolean], //read collect elements letter-by-letter until word completion
     globalBeepFeedback: [Boolean],
     globalMinPauseCollectSpeak: [Number],
     scanEnabled: [Boolean],
@@ -125,6 +128,9 @@ InputConfig.defaults({
     modelName: InputConfig.getModelName(),
     modelVersion: constants.MODEL_VERSION,
     globalReadActiveRate: 1,
+    globalReadActiveFastCount: 3,
+    globalReadActiveFastRate: 2,
+    globalReadCollectLetters: false,
     globalMinPauseCollectSpeak: 0,
     scanAuto: false,
     scanTimeoutMs: 1000,

src/js/service/actionService.js

@@ -18,6 +18,7 @@ import $ from '../externals/jquery.js';
 import { GridActionAudio } from '../model/GridActionAudio.js';
 import { GridActionSpeak } from '../model/GridActionSpeak.js';
 import { GridActionSpeakCustom } from '../model/GridActionSpeakCustom.js';
+import { GridActionSpeakLetters } from '../model/GridActionSpeakLetters.js';
 import {audioUtil} from "../util/audioUtil.js";
 import {MainVue} from "../vue/mainVue.js";
 import {stateService} from "./stateService.js";
@@ -106,7 +107,7 @@ async function doActions(gridElement, gridId) {
     }
     metadata = metadata || (await dataService.getMetadata());
     let actionTypes = actions.map((a) => a.modelName);
-    let navBackActions = [GridActionAudio.getModelName(), GridActionChangeLang.getModelName(), GridActionSpeak.getModelName(), GridActionSpeakCustom.getModelName()];
+    let navBackActions = [GridActionAudio.getModelName(), GridActionChangeLang.getModelName(), GridActionSpeak.getModelName(), GridActionSpeakCustom.getModelName(), GridActionSpeakLetters.getModelName()];
     let noNavBackActions = GridElement.getActionTypeModelNames().filter((name) => !navBackActions.includes(name));
     if (
         metadata.toHomeAfterSelect &&
@@ -168,6 +169,28 @@ async function doAction(gridElement, action, options = {}) {
                 });
             }
             break;
+        case 'GridActionSpeakLetters':
+            log.debug('action speak letters');
+            let speakLettersText = action.speakText;
+            if (!speakLettersText) {
+                // If no custom text, use element label
+                speakLettersText = stateService.getSpeakTextAllLangs(gridElement.id);
+                if (gridElement.type === GridElement.ELEMENT_TYPE_PREDICTION) {
+                    speakLettersText[i18nService.getContentLang()] = predictionService.getLastAppliedPrediction();
+                }
+                if (gridElement.type === GridElement.ELEMENT_TYPE_LIVE) {
+                    speakLettersText[i18nService.getContentLang()] = liveElementService.getLastValue(gridElement.id);
+                }
+            } else if (gridElement.type === GridElement.ELEMENT_TYPE_LIVE) {
+                let text = JSON.parse(JSON.stringify(speakLettersText));
+                text[i18nService.getContentLang()] = liveElementService.replacePlaceholder(gridElement, text[i18nService.getContentLang()]);
+                speakLettersText = text;
+            }
+            speechService.speakLetters(speakLettersText, {
+                lang: action.speakLanguage,
+                pauseDurationMs: action.pauseDurationMs || 200
+            });
+            break;
         case 'GridActionAudio':
             if (action.dataBase64) {
                 audioUtil.stopAudio();

src/js/service/speechService.js

@@ -207,6 +207,65 @@ speechService.speakArray = async function (array, progressFn, index) {
     speechService.speakArray(currentSpeakArray, progressFn, index + 1);
 };
 
+/**
+ * Speaks text letter by letter with configurable pause between letters
+ * @param textOrObject text to speak or object with language mappings
+ * @param options options object
+ * @param options.pauseDurationMs pause duration between letters in milliseconds (default: 200)
+ * @param options.lang language to use for speaking
+ * @param options.speakSecondary if true, speak secondary language as well
+ * @param options.rate speech rate
+ * @param options.progressFn function called with current letter index
+ */
+speechService.speakLetters = async function (textOrObject, options = {}) {
+    options = options || {};
+    let pauseDurationMs = options.pauseDurationMs || 200;
+    let text = null;
+    let isString = typeof textOrObject === 'string';
+
+    if (!textOrObject || (!isString && Object.keys(textOrObject).length === 0)) {
+        return;
+    }
+
+    if (isString) {
+        text = textOrObject;
+    } else {
+        let langToUse = options.lang || i18nService.getContentLang();
+        text = textOrObject[langToUse] || textOrObject[Object.keys(textOrObject)[0]] || '';
+    }
+
+    if (!text) {
+        return;
+    }
+
+    // Split text into individual characters, preserving spaces
+    let letters = text.split('');
+
+    for (let i = 0; i < letters.length; i++) {
+        let letter = letters[i];
+
+        // Call progress function if provided
+        if (options.progressFn) {
+            options.progressFn(i);
+        }
+
+        // Skip speaking spaces but still pause
+        if (letter.trim() !== '') {
+            speechService.speak(letter, {
+                lang: options.lang,
+                rate: options.rate,
+                dontStop: true
+            });
+            await speechService.waitForFinishedSpeaking();
+        }
+
+        // Add pause between letters (except for the last one)
+        if (i < letters.length - 1) {
+            await new Promise(resolve => setTimeout(resolve, pauseDurationMs));
+        }
+    }
+};
+
 speechService.stopSpeaking = function () {
     currentSpeakArray = [];
     isSpeakingNative = false;

src/vue-components/modals/editAction.vue

@@ -37,6 +37,31 @@
             <input class="eight columns" id="inCustomText" type="text" v-model="action.speakText[getCurrentSpeakLang(action)]" :placeholder="gridElement.type === GridElement.ELEMENT_TYPE_LIVE ? $t('canIncludePlaceholderLike') : ''"/>
         </div>
     </div>
+    <div v-if="action.modelName == 'GridActionSpeakLetters'">
+        <div class="srow">
+            <div class="four columns">
+                <label for="selectLang3" class="normal-text">{{ $t('language') }}</label>
+            </div>
+            <select class="eight columns" id="selectLang3" v-model="action.speakLanguage">
+                <option :value="undefined">{{ $t('automaticCurrentLanguage') }}</option>
+                <option v-for="lang in voiceLangs" :value="lang.code">
+                    {{lang | extractTranslation}}
+                </option>
+            </select>
+        </div>
+        <div class="srow">
+            <div class="four columns">
+                <label for="inLettersText" class="normal-text">{{ $t('textToSpeak') }}</label>
+            </div>
+            <input class="eight columns" id="inLettersText" type="text" v-model="action.speakText[getCurrentSpeakLang(action)]" :placeholder="$t('leaveEmptyToUseElementLabel')"/>
+        </div>
+        <div class="srow">
+            <div class="four columns">
+                <label for="inPauseDuration" class="normal-text">{{ $t('pauseBetweenLettersMs') }}</label>
+            </div>
+            <input class="eight columns" id="inPauseDuration" type="number" min="0" max="5000" step="50" v-model.number="action.pauseDurationMs"/>
+        </div>
+    </div>
     <div v-if="action.modelName == 'GridActionAudio'">
         <edit-audio-action :action="action" :grid-data="gridData"></edit-audio-action>
     </div>
@@ -329,6 +354,7 @@
     import { gridUtil } from '../../js/util/gridUtil';
     import { GridActionSystem } from '../../js/model/GridActionSystem';
     import { GridActionChangeLang } from '../../js/model/GridActionChangeLang';
+    import { GridActionSpeakLetters } from '../../js/model/GridActionSpeakLetters';
     import EditPredefinedAction from './editActionsSub/editPredefinedAction.vue';
     import EditMatrixAction from './editActionsSub/editMatrixAction.vue';
     import EditPodcastAction from './editActionsSub/editPodcastAction.vue';

src/vue-components/modals/input/globalInputOptions.vue

@@ -14,6 +14,12 @@
             <div class="srow my-2">
                 <slider-input :label="$t('speedForReadingActiveElement')" id="readActiveRate" min="0.1" max="10" step="0.1" decimals="1" v-model.number="inputConfig.globalReadActiveRate" @change="changed"/>
             </div>
+            <div class="srow my-2">
+                <slider-input :label="$t('fastReadElementCount')" id="fastReadCount" min="1" max="10" step="1" decimals="0" v-model.number="inputConfig.globalReadActiveFastCount" @change="changed"/>
+            </div>
+            <div class="srow my-2">
+                <slider-input :label="$t('fastReadSpeed')" id="fastReadRate" min="0.5" max="5" step="0.1" decimals="1" v-model.number="inputConfig.globalReadActiveFastRate" @change="changed"/>
+            </div>
             <div class="srow">
                 <div class="twelve columns">
                     <input type="checkbox" id="beepFeedback" v-model="inputConfig.globalBeepFeedback" @change="changed"/>
@@ -26,6 +32,12 @@
                     <label for="readAdditional">{{ $t('readElementActionsInAdditionToLabel') }}</label>
                 </div>
             </div>
+            <div class="srow">
+                <div class="twelve columns">
+                    <input type="checkbox" id="readCollectLetters" v-model="inputConfig.globalReadCollectLetters" @change="changed"/>
+                    <label for="readCollectLetters">{{ $t('readCollectElementsLetterByLetter') }}</label>
+                </div>
+            </div>
         </div>
     </div>
 </template>