neural-codec: add Q4 format (6.8 kbit/s), fix NCA -> NAC (Neural Audio Codec)

ServeurpersoCom · ServeurpersoCom · commit aeff0b813fc5 · 2026-03-07T00:53:18.000+01:00
diff --git a/README.md b/README.md
@@ -354,11 +354,11 @@ Models are loaded once and reused across all requests.
 
 GGML-native neural audio codec based on the Oobleck VAE encoder and decoder.
 Serves two purposes: validating the precision of the full VAE chain (encode +
-decode roundtrip), and compressing music at 1.6 KB/s with CD quality and no
-perceptible difference from the original.
+decode roundtrip), and compressing music at ~850 B/s with no perceptible
+difference from the original.
 
 ```
-Usage: neural-codec --vae <gguf> --encode|--decode -i <input> [-o <o>] [--q8]
+Usage: neural-codec --vae <gguf> --encode|--decode -i <input> [-o <o>] [--q8|--q4]
 
 Required:
   --vae <path>            VAE GGUF file
@@ -367,9 +367,10 @@ Required:
 
 Output:
   -o <path>               Output file (auto-named if omitted)
-  --q8                    Quantize latent to int8 (~13 kbit/s vs ~51 kbit/s f32)
+  --q8                    Quantize latent to int8 (~13 kbit/s)
+  --q4                    Quantize latent to int4 (~6.8 kbit/s)
 
-Output naming: song.wav -> song.latent (f32) or song.nca8 (Q8)
+Output naming: song.wav -> song.latent (f32) or song.nac8 (Q8) or song.nac4 (Q4)
                song.latent -> song.wav
 
 VAE tiling (memory control):
@@ -378,24 +379,38 @@ VAE tiling (memory control):
 
 Latent formats (decode auto-detects):
   f32:  flat [T, 64] f32, no header. ~51 kbit/s.
-  NCA8: header + per-frame Q8. ~13 kbit/s.
+  NAC8: header + per-frame Q8. ~13 kbit/s.
+  NAC4: header + per-frame Q4. ~6.8 kbit/s.
 ```
 
 The encoder is the symmetric mirror of the decoder: same snake activations,
 same residual units, strided conv1d for downsampling instead of transposed
 conv1d for upsampling. No new GGML ops. Downsample 2x4x4x6x10 = 1920x.
 
 48kHz stereo audio is compressed to 64-dimensional latent frames at 25 Hz.
-With Q8 quantization, each frame is 66 bytes (2B scale + 64B int8), giving
-~13 kbit/s. The quantization error is 39 dB below the VAE reconstruction
-error, meaning the Q8 step is perceptually free.
+Three output formats, decode auto-detects from file content:
+
+| Format | Frame size | Bitrate | 3 min song | vs f32 (cossim) |
+|--------|-----------|---------|------------|-----------------|
+| f32    | 256B      | 51 kbit/s | 1.1 MB   | baseline        |
+| NAC8   | 66B       | 13 kbit/s | 290 KB   | 0.9999          |
+| NAC4   | 34B       | 6.8 kbit/s | 150 KB  | 0.989           |
+
+NAC = Neural Audio Codec. The NAC8 and NAC4 file formats are headerless
+except for a 4-byte magic (`NAC8` or `NAC4`) and a uint32 frame count.
+Q8 quantization error is 39 dB below the VAE reconstruction error (free).
+Q4 quantization error is 16 dB below the VAE reconstruction error (inaudible
+on most material).
 
 ```bash
-# encode
-neural-codec --vae models/vae-BF16.gguf --encode --q8 -i song.wav -o song.nca8
+# encode (Q4: 6.8 kbit/s, ~150 KB for 3 minutes)
+neural-codec --vae models/vae-BF16.gguf --encode --q4 -i song.wav -o song.nac4
+
+# encode (Q8: 13 kbit/s, ~290 KB for 3 minutes)
+neural-codec --vae models/vae-BF16.gguf --encode --q8 -i song.wav -o song.nac8
 
-# decode
-neural-codec --vae models/vae-BF16.gguf --decode -i song.nca8 -o song_decoded.wav
+# decode (auto-detects format)
+neural-codec --vae models/vae-BF16.gguf --decode -i song.nac4 -o song_decoded.wav
 
 # roundtrip validation: compare song.wav and song_decoded.wav with your ears
 ```
diff --git a/tools/neural-codec.cpp b/tools/neural-codec.cpp
@@ -1,23 +1,29 @@
 // neural-codec.cpp: neural audio codec (Oobleck VAE)
 //
 // 2 modes:
-//   encode: WAV -> latent file (f32 or Q8)
+//   encode: WAV -> latent file (f32, Q8, or Q4)
 //   decode: latent file -> WAV (48kHz stereo)
 //
-// Two latent formats, decode auto-detects:
+// Three latent formats, decode auto-detects:
 //
 //   f32 (default): flat [T, 64] f32, no header.
 //     T = file_size / 256. 25Hz, ~6.4 KB/s, ~51 kbit/s.
 //
 //   Q8 (--q8): symmetric per-frame int8 quantization.
-//     header: "NCA8" magic (4B) + uint32 T_latent (4B)
+//     header: "NAC8" magic (4B) + uint32 T_latent (4B)
 //     frame:  f16 scale (2B) + int8[64] (64B) = 66B
 //     25Hz, ~1.6 KB/s, ~13 kbit/s.
 //
+//   Q4 (--q4): symmetric per-frame 4-bit quantization.
+//     header: "NAC4" magic (4B) + uint32 T_latent (4B)
+//     frame:  f16 scale (2B) + nibbles[32] (32B) = 34B
+//     25Hz, ~850 B/s, ~6.8 kbit/s.
+//
 // Usage:
 //   neural-codec --vae model.gguf --encode -i song.wav -o song.latent
-//   neural-codec --vae model.gguf --encode --q8 -i song.wav -o song.nca8
-//   neural-codec --vae model.gguf --decode -i song.nca8 -o song.wav
+//   neural-codec --vae model.gguf --encode --q8 -i song.wav -o song.nac8
+//   neural-codec --vae model.gguf --encode --q4 -i song.wav -o song.nac4
+//   neural-codec --vae model.gguf --decode -i song.nac4 -o song.wav
 
 #include "vae.h"
 #include "vae-enc.h"
@@ -149,16 +155,16 @@ static bool write_wav(const char * path, const float * audio, int T_audio, int s
 }
 
 // Q8 format constants
-static const char NCA8_MAGIC[4] = {'N', 'C', 'A', '8'};
-static const int NCA8_HEADER = 8;   // 4B magic + 4B T_latent
-static const int NCA8_FRAME  = 66;  // 2B f16 scale + 64B int8
+static const char NAC8_MAGIC[4] = {'N', 'A', 'C', '8'};
+static const int NAC8_HEADER = 8;   // 4B magic + 4B T_latent
+static const int NAC8_FRAME  = 66;  // 2B f16 scale + 64B int8
 
 // Write Q8 quantized latent
 static bool write_latent_q8(const char * path, const float * data, int T_latent) {
     FILE * f = fopen(path, "wb");
     if (!f) return false;
 
-    fwrite(NCA8_MAGIC, 1, 4, f);
+    fwrite(NAC8_MAGIC, 1, 4, f);
     uint32_t t = (uint32_t)T_latent;
     fwrite(&t, 4, 1, f);
 
@@ -186,14 +192,65 @@ static bool write_latent_q8(const char * path, const float * data, int T_latent)
     }
     fclose(f);
 
-    size_t bytes = NCA8_HEADER + (size_t)T_latent * NCA8_FRAME;
+    size_t bytes = NAC8_HEADER + (size_t)T_latent * NAC8_FRAME;
     float duration = (float)T_latent * 1920.0f / 48000.0f;
     float kbps = (float)bytes * 8.0f / (duration * 1000.0f);
     fprintf(stderr, "[Latent] Wrote %s: Q8, %d frames (%.2fs, %.1f KB, %.1f kbit/s)\n",
             path, T_latent, duration, (float)bytes / 1024.0f, kbps);
     return true;
 }
 
+// Q4 format constants
+static const char NAC4_MAGIC[4] = {'N', 'A', 'C', '4'};
+static const int NAC4_HEADER = 8;   // 4B magic + 4B T_latent
+static const int NAC4_FRAME  = 34;  // 2B f16 scale + 32B packed nibbles
+
+// Write Q4 quantized latent
+// Symmetric 4-bit: range [-7, 7], scale = amax / 7.0
+// Packing: byte = (low & 0x0F) | (high << 4), two signed nibbles per byte
+static bool write_latent_q4(const char * path, const float * data, int T_latent) {
+    FILE * f = fopen(path, "wb");
+    if (!f) return false;
+
+    fwrite(NAC4_MAGIC, 1, 4, f);
+    uint32_t t = (uint32_t)T_latent;
+    fwrite(&t, 4, 1, f);
+
+    for (int i = 0; i < T_latent; i++) {
+        const float * frame = data + i * 64;
+
+        // find max abs for symmetric quant
+        float amax = 0.0f;
+        for (int j = 0; j < 64; j++) {
+            float a = fabsf(frame[j]);
+            if (a > amax) amax = a;
+        }
+        float scale = amax / 7.0f;
+        ggml_fp16_t scale_f16 = ggml_fp32_to_fp16(scale);
+        fwrite(&scale_f16, 2, 1, f);
+
+        // quantize and pack pairs into bytes
+        float inv = (scale > 0.0f) ? 7.0f / amax : 0.0f;
+        uint8_t packed[32];
+        for (int j = 0; j < 32; j++) {
+            int lo = (int)roundf(frame[j * 2 + 0] * inv);
+            int hi = (int)roundf(frame[j * 2 + 1] * inv);
+            lo = lo < -7 ? -7 : (lo > 7 ? 7 : lo);
+            hi = hi < -7 ? -7 : (hi > 7 ? 7 : hi);
+            packed[j] = (uint8_t)((lo & 0x0F) | (hi << 4));
+        }
+        fwrite(packed, 1, 32, f);
+    }
+    fclose(f);
+
+    size_t bytes = NAC4_HEADER + (size_t)T_latent * NAC4_FRAME;
+    float duration = (float)T_latent * 1920.0f / 48000.0f;
+    float kbps = (float)bytes * 8.0f / (duration * 1000.0f);
+    fprintf(stderr, "[Latent] Wrote %s: Q4, %d frames (%.2fs, %.1f KB, %.1f kbit/s)\n",
+            path, T_latent, duration, (float)bytes / 1024.0f, kbps);
+    return true;
+}
+
 // Write f32 raw latent (no header)
 static bool write_latent_f32(const char * path, const float * data, int T_latent) {
     FILE * f = fopen(path, "wb");
@@ -208,8 +265,8 @@ static bool write_latent_f32(const char * path, const float * data, int T_latent
     return true;
 }
 
-// Read latent, auto-detect format (NCA8 magic -> Q8, else f32).
-// Returns [T_latent, 64] f32 (dequantized if Q8). Caller frees.
+// Read latent, auto-detect format (NAC8 -> Q8, NAC4 -> Q4, else f32).
+// Returns [T_latent, 64] f32 (dequantized if quantized). Caller frees.
 static float * read_latent(const char * path, int * T_latent) {
     FILE * f = fopen(path, "rb");
     if (!f) { fprintf(stderr, "[Latent] Cannot open %s\n", path); return NULL; }
@@ -221,13 +278,13 @@ static float * read_latent(const char * path, int * T_latent) {
     char magic[4] = {};
     if (fsize >= 8) fread(magic, 1, 4, f);
 
-    if (memcmp(magic, NCA8_MAGIC, 4) == 0) {
+    if (memcmp(magic, NAC8_MAGIC, 4) == 0) {
         // Q8 format
         uint32_t t;
         fread(&t, 4, 1, f);
         *T_latent = (int)t;
 
-        long expected = NCA8_HEADER + (long)t * NCA8_FRAME;
+        long expected = NAC8_HEADER + (long)t * NAC8_FRAME;
         if (fsize != expected) {
             fprintf(stderr, "[Latent] Q8 size mismatch: expected %ld, got %ld\n", expected, fsize);
             fclose(f); return NULL;
@@ -255,6 +312,47 @@ static float * read_latent(const char * path, int * T_latent) {
         return data;
     }
 
+    if (memcmp(magic, NAC4_MAGIC, 4) == 0) {
+        // Q4 format
+        uint32_t t;
+        fread(&t, 4, 1, f);
+        *T_latent = (int)t;
+
+        long expected = NAC4_HEADER + (long)t * NAC4_FRAME;
+        if (fsize != expected) {
+            fprintf(stderr, "[Latent] Q4 size mismatch: expected %ld, got %ld\n", expected, fsize);
+            fclose(f); return NULL;
+        }
+
+        float * data = (float *)malloc((size_t)t * 64 * sizeof(float));
+        for (int i = 0; i < (int)t; i++) {
+            ggml_fp16_t scale_f16;
+            fread(&scale_f16, 2, 1, f);
+            float scale = ggml_fp16_to_fp32(scale_f16);
+
+            uint8_t packed[32];
+            fread(packed, 1, 32, f);
+
+            // unpack signed nibbles
+            float * frame = data + i * 64;
+            for (int j = 0; j < 32; j++) {
+                int lo = (int)(packed[j] & 0x0F);
+                int hi = (int)(packed[j] >> 4);
+                if (lo >= 8) lo -= 16;
+                if (hi >= 8) hi -= 16;
+                frame[j * 2 + 0] = (float)lo * scale;
+                frame[j * 2 + 1] = (float)hi * scale;
+            }
+        }
+        fclose(f);
+
+        float duration = (float)(*T_latent) * 1920.0f / 48000.0f;
+        float kbps = (float)fsize * 8.0f / (duration * 1000.0f);
+        fprintf(stderr, "[Latent] Read %s: Q4, %d frames (%.2fs, %.1f KB, %.1f kbit/s)\n",
+                path, *T_latent, duration, (float)fsize / 1024.0f, kbps);
+        return data;
+    }
+
     // f32 format (no header, rewind)
     fseek(f, 0, SEEK_SET);
     if (fsize % (64 * (int)sizeof(float)) != 0) {
@@ -277,22 +375,24 @@ static float * read_latent(const char * path, int * T_latent) {
 
 static void print_usage(const char * prog) {
     fprintf(stderr,
-        "Usage: %s --vae <gguf> --encode|--decode -i <input> [-o <output>] [--q8]\n\n"
+        "Usage: %s --vae <gguf> --encode|--decode -i <input> [-o <output>] [--q8|--q4]\n\n"
         "Required:\n"
         "  --vae <path>            VAE GGUF file\n"
         "  --encode | --decode     Encode WAV to latent, or decode latent to WAV\n"
         "  -i <path>               Input (WAV for encode, latent for decode)\n\n"
         "Output:\n"
         "  -o <path>               Output file (auto-named if omitted)\n"
-        "  --q8                    Quantize latent to int8 (~13 kbit/s vs ~51 kbit/s f32)\n\n"
-        "Output naming: song.wav -> song.latent (f32) or song.nca8 (Q8)\n"
+        "  --q8                    Quantize latent to int8 (~13 kbit/s)\n"
+        "  --q4                    Quantize latent to int4 (~6.8 kbit/s)\n\n"
+        "Output naming: song.wav -> song.latent (f32) or song.nac8 (Q8) or song.nac4 (Q4)\n"
         "               song.latent -> song.wav\n\n"
         "VAE tiling (memory control):\n"
         "  --vae-chunk <N>         Latent frames per tile (default: 256)\n"
         "  --vae-overlap <N>       Overlap frames per side (default: 64)\n\n"
         "Latent formats (decode auto-detects):\n"
         "  f32:  flat [T, 64] f32, no header. ~51 kbit/s.\n"
-        "  NCA8: header + per-frame Q8. ~13 kbit/s.\n",
+        "  NAC8: header + per-frame Q8. ~13 kbit/s.\n"
+        "  NAC4: header + per-frame Q4. ~6.8 kbit/s.\n",
         prog);
 }
 
@@ -311,7 +411,7 @@ int main(int argc, char ** argv) {
     int chunk_size = 256;
     int overlap = 64;
     int mode = -1;  // 0 = encode, 1 = decode
-    bool use_q8 = false;
+    int quant = 0;  // 0 = f32, 8 = q8, 4 = q4
 
     for (int i = 1; i < argc; i++) {
         if      (strcmp(argv[i], "--vae") == 0 && i + 1 < argc)     vae_path = argv[++i];
@@ -323,7 +423,8 @@ int main(int argc, char ** argv) {
         else if (strcmp(argv[i], "--vae-overlap") == 0 && i + 1 < argc) overlap = atoi(argv[++i]);
         else if (strcmp(argv[i], "--encode") == 0)  mode = 0;
         else if (strcmp(argv[i], "--decode") == 0)  mode = 1;
-        else if (strcmp(argv[i], "--q8") == 0)      use_q8 = true;
+        else if (strcmp(argv[i], "--q8") == 0)      quant = 8;
+        else if (strcmp(argv[i], "--q4") == 0)      quant = 4;
         else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
             print_usage(argv[0]); return 0;
         } else {
@@ -339,15 +440,21 @@ int main(int argc, char ** argv) {
     // Auto output names
     std::string out_str;
     if (!output_path) {
-        if (mode == 0)
-            out_str = auto_output(input_path, use_q8 ? ".nca8" : ".latent");
-        else
+        if (mode == 0) {
+            const char * ext = ".latent";
+            if (quant == 8) ext = ".nac8";
+            if (quant == 4) ext = ".nac4";
+            out_str = auto_output(input_path, ext);
+        } else {
             out_str = auto_output(input_path, ".wav");
+        }
         output_path = out_str.c_str();
     }
 
-    fprintf(stderr, "\n[VAE] Mode: %s%s\n", mode == 0 ? "encode" : "decode",
-            (mode == 0 && use_q8) ? " (Q8)" : "");
+    const char * quant_str = "";
+    if (mode == 0 && quant == 8) quant_str = " (Q8)";
+    if (mode == 0 && quant == 4) quant_str = " (Q4)";
+    fprintf(stderr, "\n[VAE] Mode: %s%s\n", mode == 0 ? "encode" : "decode", quant_str);
     fprintf(stderr, "[VAE] Input:  %s\n", input_path);
     fprintf(stderr, "[VAE] Output: %s\n\n", output_path);
 
@@ -372,8 +479,10 @@ int main(int argc, char ** argv) {
         free(audio);
         if (T_latent < 0) { vae_enc_free(&enc); return 1; }
 
-        if (use_q8)
+        if (quant == 8)
             write_latent_q8(output_path, latent.data(), T_latent);
+        else if (quant == 4)
+            write_latent_q4(output_path, latent.data(), T_latent);
         else
             write_latent_f32(output_path, latent.data(), T_latent);
 
@@ -382,7 +491,7 @@ int main(int argc, char ** argv) {
         return 0;
     }
 
-    // DECODE (auto-detects f32 vs Q8 from file content)
+    // DECODE (auto-detects f32 vs Q8 vs Q4 from file content)
     {
         int T_latent = 0;
         float * latent = read_latent(input_path, &T_latent);
