Rewrite audio DSP plan around fingerprint-first architecture

engine/audio/src/CMakeLists.txt

@@ -5,10 +5,12 @@ add_library(${module_name}
     audio.cpp audio.h
     audiocapture.cpp audiocapture.h
     audiodecoder.cpp audiodecoder.h
+    audiofeatures.h
     audioparameters.cpp audioparameters.h
     audioplugincache.cpp audioplugincache.h
     audiorenderer.cpp audiorenderer.h
     beattracker.cpp beattracker.h
+    liveaudioanalyzer.cpp liveaudioanalyzer.h
 )
 set_property(TARGET ${module_name} PROPERTY POSITION_INDEPENDENT_CODE ON)
 target_include_directories(${module_name} PUBLIC

engine/audio/src/audiocapture.cpp

@@ -18,6 +18,8 @@
   limitations under the License.
 */
 
+#include <algorithm>
+
 #include <QSettings>
 #include <QDateTime>
 #include <QDebug>
@@ -31,6 +33,9 @@
 
 #define M_2PI       6.28318530718           /* 2*pi */
 
+static_assert(AUDIO_FEATURE_BANDS == FREQ_SUBBANDS_MAX_NUMBER,
+              "AudioFeatures and AudioCapture must use the same live band count");
+
 AudioCapture::AudioCapture (QObject* parent)
     : QThread (parent)
     , m_userStop(true)
@@ -123,6 +128,12 @@ double AudioCapture::bandMaxMagnitude(int numBands) const
     return maxVal;
 }
 
+AudioFeatures AudioCapture::audioFeatures() const
+{
+    QMutexLocker locker(&m_mutex);
+    return m_audioFeatures;
+}
+
 int AudioCapture::lowCutBin(int N)
 {
     if (N < 3) return 0;
@@ -202,6 +213,36 @@ void AudioCapture::stop()
 
 double AudioCapture::fillBandsData(int number)
 {
+    auto it = m_fftMagnitudeMap.find(number);
+    if (it == m_fftMagnitudeMap.end())
+        return 0.0;
+
+    QVector<double> &bands = it.value().m_fftMagnitudeBuffer;
+    if (bands.size() != number)
+        bands = QVector<double>(number);
+
+    return fillLogBands(number, bands.data());
+}
+
+double AudioCapture::fillLogBands(int number, QVector<double> &bands) const
+{
+    if (number <= 0)
+    {
+        bands.clear();
+        return 0.0;
+    }
+
+    if (bands.size() != number)
+        bands = QVector<double>(number);
+
+    return fillLogBands(number, bands.data());
+}
+
+double AudioCapture::fillLogBands(int number, double *bands) const
+{
+    if (number <= 0 || bands == nullptr)
+        return 0.0;
+
     // m_fftOutputBuffer contains the real and imaginary data of a spectrum
     // representing all the frequencies from 0 to m_sampleRate Hz.
     // Consider the configured spectrum range and calculate average magnitude
@@ -215,10 +256,9 @@ double AudioCapture::fillBandsData(int number)
     const double maxFreq = qMin(double(SPECTRUM_MAX_FREQUENCY), nyquist);
     const double logRange = (maxFreq > minFreq) ? qLn(maxFreq / minFreq) : 0.0;
 
-    if (number <= 0 || maxBin <= 1 || logRange <= 0.0)
+    if (maxBin <= 1 || logRange <= 0.0)
     {
-        if (number > 0 && m_fftMagnitudeMap.contains(number))
-            m_fftMagnitudeMap[number].m_fftMagnitudeBuffer.fill(0.0);
+        std::fill_n(bands, number, 0.0);
         return 0.0;
     }
 
@@ -243,16 +283,21 @@ double AudioCapture::fillBandsData(int number)
 
         const int bandWidth = endBin - startBin;
         const double bandMagnitude = magnitudeSum / (double(bandWidth) * M_2PI);
-        m_fftMagnitudeMap[number].m_fftMagnitudeBuffer[b] = bandMagnitude;
+        bands[b] = bandMagnitude;
         if (maxMagnitude < bandMagnitude)
             maxMagnitude = bandMagnitude;
     }
 #else
-    Q_UNUSED(number)
+    std::fill_n(bands, number, 0.0);
 #endif
     return maxMagnitude;
 }
 
+double AudioCapture::fillAudioFeatureBands(std::array<double, AUDIO_FEATURE_BANDS> &bands) const
+{
+    return fillLogBands(AUDIO_FEATURE_BANDS, bands.data());
+}
+
 void AudioCapture::processData()
 {
     unsigned int i, j;
@@ -288,10 +333,12 @@ void AudioCapture::processData()
 
     // Remove DC, compute RMS in one pass (normalize to [-1,1])
     double sumSq = 0.0;
+    double peak = 0.0;
     for (i = 0; i < m_bufferSize; ++i)
     {
         const double x = (double(m_audioMixdown[i]) - mean) / 32768.0;
         sumSq += x * x;
+        peak = qMax(peak, qAbs(x));
         m_fftInputBuffer[i] = x; // will be windowed right below
     }
     const double rms = qSqrt(sumSq / double(m_bufferSize));
@@ -304,6 +351,7 @@ void AudioCapture::processData()
         double maxMagnitude = 0.0;
         quint32 power = smoothPower(0.0);
         m_signalPower = power;
+        m_audioFeatures = m_liveAnalyzer.analyzeSilence();
         for (int barsNumber : m_fftMagnitudeMap.keys())
         {
             // Ensure the buffer exists and is zeroed
@@ -356,6 +404,9 @@ void AudioCapture::processData()
     // 5) Fill per-band magnitudes and compute power
     double pwrSum = 0.;
     double maxMagnitude = 0.;
+    std::array<double, AUDIO_FEATURE_BANDS> featureBands {};
+    const double featureMaxMagnitude = fillAudioFeatureBands(featureBands);
+    m_audioFeatures = m_liveAnalyzer.analyze(rms, peak, featureBands, featureMaxMagnitude);
     for (int barsNumber : m_fftMagnitudeMap.keys())
     {
         maxMagnitude = fillBandsData(barsNumber); // fills & returns max per-band
@@ -389,10 +440,28 @@ void AudioCapture::run()
         {
             if (readAudio(m_captureSize) == true)
             {
-                QMutexLocker locker(&m_mutex);
-                processData();
-
-                if (m_beatTracker->processAudio(m_audioBuffer, m_captureSize))
+                bool hasBeat = false;
+                {
+                    QMutexLocker locker(&m_mutex);
+                    processData();
+
+                    hasBeat = m_beatTracker->processAudio(m_audioBuffer, m_captureSize);
+                    // BeatTracker may keep returning a stable BPM even during silence.
+                    // If processData() gated the frame as silent, force beat/BPM back to zero.
+                    if (m_audioFeatures.rmsDb <= -90.0f)
+                    {
+                        m_audioFeatures.beat = false;
+                        m_audioFeatures.bpm = 0.0;
+                    }
+                    else
+                    {
+                        m_audioFeatures.beat = hasBeat;
+                        m_audioFeatures.bpm = m_beatTracker->getCurrentBpm();
+                    }
+                }
+                emit audioFeaturesChanged();
+
+                if (hasBeat)
                     emit beatDetected();
             }
             else

engine/audio/src/audiocapture.h

@@ -22,11 +22,15 @@
 #define AUDIOCAPTURE_H
 
 #include <stdint.h>
+#include <array>
 #include <QThread>
 #include <QVector>
 #include <QMutex>
 #include <QMap>
 
+#include "audiofeatures.h"
+#include "liveaudioanalyzer.h"
+
 #ifdef HAS_FFTW3
 #include "fftw3.h"
 #endif
@@ -142,6 +146,9 @@ class AudioCapture : public QThread
     /** Get the maximum magnitude across all bands in a registered band set. */
     double bandMaxMagnitude(int numBands) const;
 
+    /** Get the latest live audio feature frame. */
+    AudioFeatures audioFeatures() const;
+
 protected:
     void stop();
 
@@ -155,6 +162,13 @@ class AudioCapture : public QThread
     /** This is called at every processData to fill a single BandsData structure */
     double fillBandsData(int number);
 
+    /** Fill logarithmic FFT bands without registering a legacy consumer. */
+    double fillLogBands(int number, QVector<double> &bands) const;
+    double fillLogBands(int number, double *bands) const;
+
+    /** Fill the fixed-size bands used by AudioFeatures. */
+    double fillAudioFeatureBands(std::array<double, AUDIO_FEATURE_BANDS> &bands) const;
+
     /** This is the method where captured audio data is processed in this order
      *  1) calculates the signal power, which will be the volume bar
      *  2) perform the FFT
@@ -164,11 +178,12 @@ class AudioCapture : public QThread
 
 signals:
     void dataProcessed(double *spectrumBands, int size, double maxMagnitude, quint32 power);
+    void audioFeaturesChanged();
     void volumeChanged(int volume);
     void beatDetected();
 
 protected:
-    QMutex m_mutex;
+    mutable QMutex m_mutex;
 
     bool m_userStop, m_pause;
     unsigned int m_bufferSize, m_captureSize, m_sampleRate, m_channels;
@@ -192,6 +207,10 @@ class AudioCapture : public QThread
 
     /** Reference to the beat tracking processor */
     BeatTracker *m_beatTracker;
+
+    /** Unified live audio feature state */
+    LiveAudioAnalyzer m_liveAnalyzer;
+    AudioFeatures m_audioFeatures;
 };
 
 /** @} */

engine/audio/src/audiofeatures.h

@@ -0,0 +1,71 @@
+/*
+  Q Light Controller Plus
+  audiofeatures.h
+
+  Copyright (c) Massimo Callegari
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0.txt
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+#ifndef AUDIOFEATURES_H
+#define AUDIOFEATURES_H
+
+#include <array>
+#include <QtGlobal>
+
+static constexpr int AUDIO_FEATURE_BANDS = 32;
+
+struct AudioFeatures final
+{
+    enum Source
+    {
+        Live,
+        Cached
+    };
+
+    struct PerceptualBands
+    {
+        float sub = 0.0f;
+        float bass = 0.0f;
+        float lowMid = 0.0f;
+        float mid = 0.0f;
+        float high = 0.0f;
+    };
+
+    Source source = Live;
+    qint64 trackId = -1;
+    double positionMs = 0.0;
+
+    float rmsDb = -96.0f;
+    float peakDb = -96.0f;
+    float crestFactor = 0.0f;
+
+    std::array<float, AUDIO_FEATURE_BANDS> bandsLog {};
+    std::array<float, AUDIO_FEATURE_BANDS> bandsDb {};
+    std::array<float, AUDIO_FEATURE_BANDS> bandsNormalized {};
+    PerceptualBands bands;
+
+    float spectralCentroidHz = 0.0f;
+    float spectralRolloffHz = 0.0f;
+    float spectralFlatness = 0.0f;
+    float spectralFlux = 0.0f;
+
+    bool onset = false;
+    bool beat = false;
+    double bpm = 0.0;
+
+    bool matchLocked = false;
+    double matchConfidence = 0.0;
+};
+
+#endif // AUDIOFEATURES_H