feat(openchat): Mistral-7B inference engine (GQA + RoPE + RMSNorm + SiLU)

OpenChat 3.5 / Mistral-7B architecture, fully distinct from GPT-2:
- GQA: 32 query heads share 8 KV heads (4:1 ratio, 75% KV cache savings)
- RoPE: rotary positional embedding (no learned positions)
- RMSNorm: simpler norm without mean subtraction (both in models::layers)
- SiLU: gated MLP (gate * up → down) with F32x16 element-wise SIMD
- GGUF weight loading via hpc::gguf (Q4_K_M + Q4_0 dequantization added)
- CausalEdge64 emission from attention patterns
- OpenChat chat template (GPT4 Correct User/Assistant markers)
- /v1/chat/completions API types

All ops through crate::simd::F32x16 via models::layers.
No weights stored — loaded at runtime from user-provided GGUF.
15 tests passing. 77 total across new modules.

https://claude.ai/code/session_01Y69Vnw751w75iVSBRws7o7

Author: claude

src/hpc/gguf.rs

@@ -225,6 +225,12 @@ pub fn read_tensor_f32<R: Read + Seek>(
         GgmlType::Q8_0 => {
             dequantize_q8_0(reader, n_elements)
         }
+        GgmlType::Q4_0 => {
+            dequantize_q4_0(reader, n_elements)
+        }
+        GgmlType::Q4_K => {
+            dequantize_q4_k(reader, n_elements)
+        }
         other => Err(format!("Unsupported dtype for dequantization: {:?}", other)),
     }
 }
@@ -317,6 +323,90 @@ fn dequantize_q8_0<R: Read>(r: &mut R, n_elements: usize) -> Result<Vec<f32>, St
     Ok(result)
 }
 
+/// Dequantize Q4_0: each block = 2 bytes scale (f16) + 16 bytes (32 nibbles).
+fn dequantize_q4_0<R: Read>(r: &mut R, n_elements: usize) -> Result<Vec<f32>, String> {
+    let block_size = 32;
+    let n_blocks = (n_elements + block_size - 1) / block_size;
+    let mut result = Vec::with_capacity(n_elements);
+
+    for _ in 0..n_blocks {
+        let mut scale_buf = [0u8; 2];
+        r.read_exact(&mut scale_buf).map_err(|e| e.to_string())?;
+        let scale = f16_to_f32(u16::from_le_bytes(scale_buf));
+
+        let mut nibbles = [0u8; 16];
+        r.read_exact(&mut nibbles).map_err(|e| e.to_string())?;
+
+        for &byte in &nibbles {
+            let lo = (byte & 0x0F) as i8 - 8;
+            let hi = ((byte >> 4) & 0x0F) as i8 - 8;
+            result.push(lo as f32 * scale);
+            result.push(hi as f32 * scale);
+        }
+    }
+
+    result.truncate(n_elements);
+    Ok(result)
+}
+
+/// Dequantize Q4_K: super-blocks of 256 elements.
+///
+/// Q4_K block layout (144 bytes for 256 elements):
+/// - 2 bytes: d (f16 scale)
+/// - 2 bytes: dmin (f16 min)
+/// - 12 bytes: scales (6-bit per sub-block, packed)
+/// - 128 bytes: 256 4-bit quantized values (nibbles)
+fn dequantize_q4_k<R: Read>(r: &mut R, n_elements: usize) -> Result<Vec<f32>, String> {
+    let block_size = 256;
+    let n_blocks = (n_elements + block_size - 1) / block_size;
+    let mut result = Vec::with_capacity(n_elements);
+
+    for _ in 0..n_blocks {
+        // Read d and dmin (f16)
+        let mut d_buf = [0u8; 2];
+        let mut dmin_buf = [0u8; 2];
+        r.read_exact(&mut d_buf).map_err(|e| e.to_string())?;
+        r.read_exact(&mut dmin_buf).map_err(|e| e.to_string())?;
+        let d = f16_to_f32(u16::from_le_bytes(d_buf));
+        let dmin = f16_to_f32(u16::from_le_bytes(dmin_buf));
+
+        // Read scales (12 bytes = 8 sub-block scales + 8 sub-block mins, 6-bit packed)
+        let mut scales_raw = [0u8; 12];
+        r.read_exact(&mut scales_raw).map_err(|e| e.to_string())?;
+
+        // Decode 8 scale/min pairs from 12 bytes (6 bits each)
+        let mut sc = [0u8; 8];
+        let mut mn = [0u8; 8];
+        for i in 0..4 {
+            sc[i] = scales_raw[i] & 0x3F;
+            mn[i] = scales_raw[i + 4] & 0x3F;
+            sc[i + 4] = ((scales_raw[i + 8] & 0x0F) << 2) | (scales_raw[i] >> 6);
+            mn[i + 4] = ((scales_raw[i + 8] >> 4) << 2) | (scales_raw[i + 4] >> 6);
+        }
+
+        // Read 128 bytes of nibbles (256 4-bit values)
+        let mut nibbles = [0u8; 128];
+        r.read_exact(&mut nibbles).map_err(|e| e.to_string())?;
+
+        // Dequantize: each sub-block of 32 elements
+        for j in 0..8 {
+            let sub_d = d * sc[j] as f32;
+            let sub_m = dmin * mn[j] as f32;
+            let nib_offset = j * 16;
+            for k in 0..16 {
+                let byte = nibbles[nib_offset + k];
+                let lo = (byte & 0x0F) as f32;
+                let hi = ((byte >> 4) & 0x0F) as f32;
+                result.push(lo * sub_d - sub_m);
+                result.push(hi * sub_d - sub_m);
+            }
+        }
+    }
+
+    result.truncate(n_elements);
+    Ok(result)
+}
+
 /// Convert f16 bit pattern to f32.
 fn f16_to_f32(bits: u16) -> f32 {
     let sign = ((bits >> 15) & 1) as u32;

src/hpc/mod.rs

@@ -182,6 +182,10 @@ pub mod gpt2;
 #[allow(missing_docs)]
 pub mod stable_diffusion;
 
+/// OpenChat 3.5 inference — Mistral-7B architecture (GQA + RoPE + RMSNorm + SiLU).
+#[allow(missing_docs)]
+pub mod openchat;
+
 // jitson: JSON config → scan pipeline (parser, validator, template, precompile, packed)
 // Always available — no Cranelift dependency.
 #[allow(missing_docs)]

src/hpc/models/layers.rs

@@ -213,6 +213,72 @@ pub fn dot_product(a: &[f32], b: &[f32]) -> f32 {
     sum
 }
 
+/// RMS normalization (Mistral/Llama style): `x = x * weight / sqrt(mean(x²) + eps)`
+///
+/// No mean subtraction, no bias. Simpler and faster than LayerNorm.
+/// Used by OpenChat 3.5, Mistral, Llama 2/3.
+pub fn rms_norm(x: &mut [f32], weight: &[f32], eps: f32) {
+    let n = x.len();
+    let chunks = n / 16;
+
+    // Mean of squares (SIMD)
+    let mut sq_acc = F32x16::splat(0.0);
+    for c in 0..chunks {
+        let off = c * 16;
+        let v = F32x16::from_slice(&x[off..off + 16]);
+        sq_acc = v.mul_add(v, sq_acc);
+    }
+    let mut mean_sq = sq_acc.reduce_sum();
+    for i in (chunks * 16)..n {
+        mean_sq += x[i] * x[i];
+    }
+    mean_sq /= n as f32;
+
+    let inv_rms = 1.0 / (mean_sq + eps).sqrt();
+    let inv_rms_vec = F32x16::splat(inv_rms);
+
+    // Normalize × weight (SIMD)
+    for c in 0..chunks {
+        let off = c * 16;
+        let v = F32x16::from_slice(&x[off..off + 16]);
+        let w = F32x16::from_slice(&weight[off..off + 16]);
+        let result = v * inv_rms_vec * w;
+        result.copy_to_slice(&mut x[off..off + 16]);
+    }
+    for i in (chunks * 16)..n {
+        x[i] = x[i] * inv_rms * weight[i];
+    }
+}
+
+/// Apply Rotary Positional Embedding (RoPE) to Q and K vectors.
+///
+/// Rotates pairs of dimensions by position-dependent angles:
+/// `(q[2i], q[2i+1]) = R(θ_i × pos) × (q[2i], q[2i+1])`
+/// where θ_i = 10000^(-2i/d).
+///
+/// Used by Mistral, Llama, OpenChat (replaces learned positional embeddings).
+pub fn rope_apply(q: &mut [f32], k: &mut [f32], head_dim: usize, position: usize, rope_theta: f32) {
+    let half = head_dim / 2;
+    for i in 0..half {
+        let theta = rope_theta.powf(-(2.0 * i as f32) / head_dim as f32);
+        let angle = position as f32 * theta;
+        let cos_a = angle.cos();
+        let sin_a = angle.sin();
+
+        // Apply to Q
+        let q0 = q[2 * i];
+        let q1 = q[2 * i + 1];
+        q[2 * i] = q0 * cos_a - q1 * sin_a;
+        q[2 * i + 1] = q0 * sin_a + q1 * cos_a;
+
+        // Apply to K
+        let k0 = k[2 * i];
+        let k1 = k[2 * i + 1];
+        k[2 * i] = k0 * cos_a - k1 * sin_a;
+        k[2 * i + 1] = k0 * sin_a + k1 * cos_a;
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -305,4 +371,62 @@ mod tests {
         assert!((output[0] - 3.0).abs() < 1e-5);
         assert!((output[1] - 7.0).abs() < 1e-5);
     }
+
+    #[test]
+    fn test_rms_norm_unit_weight() {
+        let mut x = vec![3.0, 4.0]; // rms = sqrt((9+16)/2) = sqrt(12.5) ≈ 3.536
+        let w = vec![1.0; 2];
+        rms_norm(&mut x, &w, 1e-5);
+        let rms = (12.5f32).sqrt();
+        assert!((x[0] - 3.0 / rms).abs() < 0.01);
+        assert!((x[1] - 4.0 / rms).abs() < 0.01);
+    }
+
+    #[test]
+    fn test_rms_norm_scaling() {
+        let mut x = vec![1.0, 1.0, 1.0, 1.0];
+        let w = vec![2.0; 4];
+        rms_norm(&mut x, &w, 1e-5);
+        // rms = 1.0, so result = 1.0 * 2.0 = 2.0
+        assert!((x[0] - 2.0).abs() < 0.01);
+    }
+
+    #[test]
+    fn test_rope_position_zero_identity() {
+        let mut q = vec![1.0, 2.0, 3.0, 4.0];
+        let mut k = vec![5.0, 6.0, 7.0, 8.0];
+        let orig_q = q.clone();
+        let orig_k = k.clone();
+        rope_apply(&mut q, &mut k, 4, 0, 10000.0);
+        // At position 0, angle = 0, cos=1, sin=0 → identity
+        for i in 0..4 {
+            assert!((q[i] - orig_q[i]).abs() < 1e-5);
+            assert!((k[i] - orig_k[i]).abs() < 1e-5);
+        }
+    }
+
+    #[test]
+    fn test_rope_changes_with_position() {
+        let mut q1 = vec![1.0, 0.0, 1.0, 0.0];
+        let mut k1 = vec![1.0, 0.0, 1.0, 0.0];
+        let mut q2 = q1.clone();
+        let mut k2 = k1.clone();
+        rope_apply(&mut q1, &mut k1, 4, 1, 10000.0);
+        rope_apply(&mut q2, &mut k2, 4, 100, 10000.0);
+        // Different positions should give different results
+        let diff: f32 = q1.iter().zip(&q2).map(|(a, b)| (a - b).abs()).sum();
+        assert!(diff > 0.01, "different positions should produce different embeddings");
+    }
+
+    #[test]
+    fn test_rope_preserves_norm() {
+        let mut q = vec![3.0, 4.0, 1.0, 2.0];
+        let mut k = vec![0.0; 4];
+        let norm_before: f32 = q.iter().map(|x| x * x).sum::<f32>().sqrt();
+        rope_apply(&mut q, &mut k, 4, 42, 10000.0);
+        let norm_after: f32 = q.iter().map(|x| x * x).sum::<f32>().sqrt();
+        // RoPE is a rotation — should preserve L2 norm
+        assert!((norm_before - norm_after).abs() < 0.01,
+            "RoPE should preserve norm: {} vs {}", norm_before, norm_after);
+    }
 }