snsr-debug¶

/* Sensory Confidential
 * Copyright (C)2016-2025 Sensory, Inc. https://sensory.com/
 *------------------------------------------------------------------------------
 */

package com.sensory.speech.snsr.demo.snsrdebug;

import android.os.Handler;
import android.os.HandlerThread;
import android.os.Message;
import android.util.Log;

import com.sensory.speech.snsr.Snsr;
import com.sensory.speech.snsr.SnsrDataFormat;
import com.sensory.speech.snsr.SnsrRC;
import com.sensory.speech.snsr.SnsrSession;
import com.sensory.speech.snsr.SnsrStream;

import java.io.File;
import java.io.IOException;
import java.util.Locale;

@SuppressWarnings({"SameParameterValue", "CanBeFinal", "UnusedReturnValue"})
class PhraseSpot implements SnsrSession.Listener {
    private final String TAG = "PhraseSpot";
    private final Boolean VERBOSE = false; // set to true for additional event callbacks
    private final RunMode RUNMODE = RunMode.PUSH; // set to PUSH or PULL
    private final int BLOCKSIZE = 480; // size in bytes of a 15 mS audio block captured at 16 KHz

    // add for push mode HandlerThread
    private static final int MSG_RESET = 1;
    private static final int MSG_PUSH = 2;
    private static final int MSG_STOP = 3;

    private Thread mRecogThread;
    private Handler mPushHandler;
    private final RecogMode mRecogMode;
    private String mLogPath;
    private final double mTimeout;
    private int mSampleRate;
    private double mSamples;
    private double mSamplesTimeoutBegin;
    private final MainActivity mUi;
    private boolean mDebugging;
    private volatile boolean mRunning = false, mStopping = false;
    private HandlerThread mPushHandlerThread;

    PhraseSpot(MainActivity mainActivity, RecogMode recogMode, double timeout) {
        mUi = mainActivity;
        mRecogMode = recogMode;
        mTimeout = timeout;
        mSamples = mSamplesTimeoutBegin = 0;
        mLogPath = null;
        mDebugging = false;
    }

    public void enableDebugging(String logPath) {
        mDebugging = true;
        mLogPath = logPath;
    }

    public synchronized void start() {
        if (mRecogThread == null) {
            mRunning = true;
            Log.d(TAG, "Starting recognition thread.");
            mRecogThread = new Thread(new Runnable() {
                @Override
                public void run() {
                    doPhraseSpot();
                }
            });
            mRecogThread.start();
        }
    }

    public synchronized void stop() {
        if (mRecogThread != null && mRecogThread.isAlive()) {
            Log.d(TAG, "Stopping recognition thread.");
            mRunning = false;
            try {
                mRecogThread.join();
                mRecogThread = null;
            } catch (InterruptedException ignored) {}
        }
    }

    private SnsrRC doPhraseSpot() {
        SnsrStream audio = SnsrStream.fromAudioDevice();
        SnsrSession session = new SnsrSession();
        String prompt;
        try {
            if (mRecogMode == RecogMode.WW_ONLY) {
                session.load(assetToString(BuildConfig.TRIGGER_MODEL));
                prompt = "Say 'Voice Genie'";

            } else if (mRecogMode == RecogMode.WW_CMDS) {
                session.load(assetToString(BuildConfig.SEQUENTIAL_TEMPLATE));
                session.setStream(Snsr.SLOT_0, SnsrStream.fromFileName(assetToString(BuildConfig.TRIGGER_MODEL),"r"));
                session.setStream(Snsr.SLOT_1, SnsrStream.fromFileName(assetToString(BuildConfig.COMMAND_MODEL),"r"));
                prompt = "Say 'Voice Genie' followed by one of: \n'Play music'\n'Pause music'\n'Stop music'\n'Next song'\n'Previous song'";

            } else if (BuildConfig.SDK_TYPE.equals("tnl") && ((mRecogMode == RecogMode.STT_ONLY || mRecogMode == RecogMode.WW_STT))) {
                session.load(assetToString(BuildConfig.OPT_SPOT_VAD_LVCSR_TEMPLATE));
                session.setStream(Snsr.PHRASESPOT, SnsrStream.fromFileName(assetToString(BuildConfig.TRIGGER_MODEL),"r"));
                session.setStream(Snsr.LVCSR, SnsrStream.fromFileName(assetToString(BuildConfig.STT_MODEL),"r"));
                session.setInt(Snsr.INCLUDE_LEADING_SILENCE, 1);
                if (mRecogMode == RecogMode.WW_STT) {
                    session.setString(Snsr.SLOT, Snsr.SLOT_0);
                    prompt = "Say 'Voice Genie' followed by an automotive command (eg 'Turn on the radio' or 'open the rear hatch')";
                } else {
                    session.setString(Snsr.SLOT, Snsr.SLOT_1);
                    prompt = "Say an automotive command (eg 'set the AC to 72' or 'roll down the driver's window')";
                }

            } else {
                throw new Exception("Unknown recognition mode");
            }

            if (mDebugging) {
                // Create debug session
                SnsrSession debug = new SnsrSession();
                debug.load(assetToString(BuildConfig.DEBUG_TEMPLATE));
                debug.setString(Snsr.DEBUG_LOG_FILE, mLogPath);
                debug.setInt(Snsr.INCLUDE_MODEL, 0);
                // Load existing session into the debug model
                SnsrStream modelData = SnsrStream.fromBuffer(1<<20, 1<<30);
                session.save(SnsrDataFormat.CONFIG, modelData);
                session.release();
                debug.setStream(Snsr.SLOT_0, modelData);
                modelData.release();
                // Replace session with the the same model wrapped in the tpl-spot-debug template
                session = debug;

                // Show the debug log file name in the UI
                File file = new File(mLogPath);
                mUi.logToConsole("\nAudio will be logged to " + file.getName());
            }

            // Main result handler
            session.setHandler(Snsr.RESULT_EVENT, this);

            // Get sample rate in case timeout was set
            mSampleRate = session.getInt(Snsr.SAMPLE_RATE);
            session.setHandler(Snsr.SAMPLES_EVENT, this);

            // These events exist only for a subset of the models, we therefore
            // ignore any errors while attempting to set them.
            try {
                if (mDebugging) {
                    session.setHandler(Snsr.SLOT_0 + Snsr.SLOT_0 + Snsr.RESULT_EVENT, this);
                } else {
                    session.setHandler(Snsr.SLOT_0 + Snsr.RESULT_EVENT, this);
                }
            } catch (Exception ignored) {}
            try {
                session.setHandler(Snsr.NLU_INTENT_EVENT, this);
            } catch (Exception ignored) {}
            if (VERBOSE) {
                try {
                    session.setHandler(Snsr.LISTEN_BEGIN_EVENT, this);
                } catch (Exception ignored) {}
                try {
                    session.setHandler(Snsr.LISTEN_END_EVENT, this);
                } catch (Exception ignored) {}
                try {
                    session.setHandler(Snsr.BEGIN_EVENT, this);
                } catch (Exception ignored) {}
                try {
                    session.setHandler(Snsr.END_EVENT, this);
                } catch (Exception ignored) {}
            }
            session.reset(); // clear error codes reported by rC()

            mUi.logToConsole("\n" + prompt +"\n");

            if (RUNMODE == RunMode.PULL) {
                // pull mode - the session will read audio and process it internally
                session.setStream(Snsr.SOURCE_AUDIO_PCM, audio);
                session.run();
            } else {
                // push mode - the application code reads the audio and passes it to the session
                startHandlerThread(session);
                do {
                    byte[] buffer;
                    buffer = new byte[BLOCKSIZE];
                    long bytesRead = audio.read(buffer);
                    // since audio.read blocks execution while it waits for an audio block to
                    // become available, move session.push into its own Handler
                    mPushHandler.sendMessage(Message.obtain(null, MSG_PUSH, buffer));
                } while (session.rC() == SnsrRC.OK && audio.rC() == SnsrRC.OK);
                mPushHandler.sendMessage(Message.obtain(null, MSG_STOP));
                try {
                    mPushHandlerThread.join();
                    mPushHandlerThread = null;
                } catch (InterruptedException ignored) {}
                mPushHandler = null;
            }
        } catch (IOException e) {
            Log.e(TAG, "Error loading and starting model", e);
            mUi.logToConsole("ERROR: " + e.getMessage());
        } catch (Exception e) {
            Log.e(TAG, "Initialization error" + e);
            mUi.logToConsole("ERROR: " + e.getMessage());
        }
        this.onEvent(session, "stopped");

        SnsrRC rc = session.rC();
        // Release the underlying native handles immediately, rather than waiting for GC.
        session.release();
        audio.release();
        return rc;
    }

    // Format a BuildConfig model filename to an "assets/models" string
    private String assetToString(String assetName) {
        return new File("assets/models", assetName.replace(':', '-')).toString();
    }

    // in push mode, run the SnsrSession in its own HandlerThread.
    private void startHandlerThread(SnsrSession session) {
        mPushHandlerThread = new HandlerThread("pushMode");
        mPushHandlerThread.start();
        mPushHandler = new Handler(mPushHandlerThread.getLooper()) {
                @Override
                public void handleMessage(Message msg) {
                    switch (msg.what) {
                        case MSG_PUSH:
                            byte[] data = (byte[]) msg.obj;
                            //Log.d(TAG, "stt.push called with " + data.length + " bytes");
                            session.push(Snsr.SOURCE_AUDIO_PCM, data);
                            break;

                        case MSG_STOP:
                            session.stop();
                            mPushHandlerThread.quit();
                            break;
                    }
                }
            };
    }

    @Override
    public SnsrRC onEvent(SnsrSession s, String key) {
        if (!Snsr.SAMPLES_EVENT.equals(key))
            Log.i(TAG, "SNSR Event: " + key);
        switch (key) {
            case Snsr.SAMPLES_EVENT:
                // used to implement timeout after 30 seconds of no speech
                if (!mRunning)
                    return SnsrRC.STOP;
                if (mTimeout == 0)
                    return SnsrRC.OK;
                mSamples = s.getDouble(Snsr.RES_SAMPLES);
                double elapsedSamples = mSamples - mSamplesTimeoutBegin;
                // Log.d(TAG, "elapsedSamples = " + elapsedSamples);
                if (elapsedSamples > mTimeout * mSampleRate) {
                    if (!mStopping)
                        mUi.logToConsole("Phrase spot timed out.\n");
                    mStopping = true;
                    return SnsrRC.TIMED_OUT;
                }
                else
                    return SnsrRC.OK;
            case Snsr.SLOT_0 + Snsr.RESULT_EVENT:
                // callback for a wakeword result in WW_CMDS mode
                mUi.logToConsole(String.format(Locale.US, "Wakeword: '%s'",
                        s.getString(Snsr.SLOT_0 + Snsr.RES_TEXT)));
                return SnsrRC.OK;
            case Snsr.SLOT_0 + Snsr.SLOT_0 + Snsr.RESULT_EVENT:
                // callback for a wakeword result in WW_STT mode
                mUi.logToConsole(String.format(Locale.US, "Wakeword: '%s'",
                        s.getString(Snsr.SLOT_0 + Snsr.SLOT_0 + Snsr.RES_TEXT)));
                return SnsrRC.OK;
            case Snsr.RESULT_EVENT:
                // Reset timeout counter after a result
                mSamplesTimeoutBegin = mSamples;
                mUi.logToConsole(String.format(Locale.US, "Result: '%s'\n", s.getString(Snsr.RES_TEXT)));
                if (VERBOSE) {
                    // print individual words in the result
                    // Try changing this to Snsr.PHONE_LIST for phonemes
                    s.forEach(Snsr.WORD_LIST, new SnsrSession.Listener() {
                        @Override
                        public SnsrRC onEvent(SnsrSession s, String key) {
                            mUi.logToConsole(String.format(Locale.US, "  [%4.0f, %4.0f] %s",
                                    s.getDouble(Snsr.RES_BEGIN_MS),
                                    s.getDouble(Snsr.RES_END_MS),
                                    s.getString(Snsr.RES_TEXT)));
                            return SnsrRC.OK;
                        }
                    });
                }
                return SnsrRC.OK;
            case Snsr.LISTEN_BEGIN_EVENT:
            case Snsr.LISTEN_END_EVENT:
            case Snsr.BEGIN_EVENT:
            case Snsr.END_EVENT:
                // misc sequential and VAD events (VAD requires TrulyNatural SDK)
                mUi.logToConsole(String.format(Locale.US, "Event: '%s'", key));
                return SnsrRC.OK;
            case Snsr.NLU_INTENT_EVENT:
                // NLU intents for STT_ONLY and WW_STT recogModes (Requires TrulyNatural SDK)
                mUi.logToConsole(String.format(Locale.US, "Intent: '%s' = '%s'",
                        s.getString(Snsr.RES_NLU_INTENT_NAME),
                        s.getString(Snsr.RES_NLU_INTENT_VALUE)));
                s.forEach(Snsr.NLU_ENTITY_LIST, new SnsrSession.Listener() {
                    @Override
                    public SnsrRC onEvent(SnsrSession s, String key) {
                        mUi.logToConsole(String.format(Locale.US, "Entity: '%s' = '%s'",
                                s.getString(Snsr.RES_NLU_ENTITY_NAME),
                                s.getString(Snsr.RES_NLU_ENTITY_VALUE)));
                        return SnsrRC.OK;
                    }
                });
                return SnsrRC.OK;
            case "stopped":
                // custom event callback to reset screen buttons and checkboxes
                mUi.notify(UiState.NOT_TALKING);
                return SnsrRC.OK;
            default:
                Log.e(TAG, "Failed to implement handler for: " + key);
                return SnsrRC.OK;
        }
    }
}