Feat: Delta Sparse Attention

configs/mixed_landmark_0814_no_extend_qsa.json

@@ -2,10 +2,11 @@
     "__head_reduce": 0,
     "__last_dense": -1,
     "__using_landmark": false,
-    "__seq_thresh_fa3": 131072,
-    "__delta_attention_args": "window_0-diff_1-w_8-dense_decode-smooth",
+    "__using_dense_prefill": true,
+    "__seq_thresh_fa3": 0,
+    "__delta_attention_args": "window_0-diff_1-w_16-dense_decode-smooth",
     "using_extend": false,
-    "dense_layers": [0],
+    "dense_layers": [0, 1, 2, 47, 46, 45],
     "mask_refresh_interval": [96],
     "layers": [
         {

configs/qwen3_30b_a3b_qsa.json

@@ -0,0 +1,65 @@
+{
+    "__delta_attention_args": "window_0-diff_1-w_16-dense_decode-smooth",
+    "__using_dense_prefill": true,
+    "__head_reduce": 2,
+    "using_extend": false,
+    "dense_layers": [
+        0,
+        1,
+        2,
+        3,
+        4,
+        5,
+        8,
+        11,
+        14,
+        17,
+        20,
+        23,
+        26,
+        29,
+        30,
+        33,
+        36,
+        39,
+        41,
+        42,
+        43,
+        44,
+        45,
+        46,
+        47
+    ],
+    "mask_refresh_interval": [64, 24, 8],
+    "prefill_layers": [
+        {},
+        {
+            "second_stage_k": 2048,
+            "sliding_window_size": 512,
+            "sink_token_size": 64,
+            "stages": [
+                {
+                    "stage_block_size_q": 64,
+                    "stage_block_stride_q": 2,
+                    "stage_chunk_size": 64,
+                    "stage_k": null,
+                    "stage_stride": 1
+                },
+                {
+                    "stage_block_size_q": 64,
+                    "stage_block_stride_q": 2,
+                    "stage_chunk_size": 32,
+                    "stage_k": 16384,
+                    "stage_stride": 1
+                },
+                {
+                    "stage_block_size_q": 64,
+                    "stage_block_stride_q": 1,
+                    "stage_chunk_size": 16,
+                    "stage_k": 4096,
+                    "stage_stride": 1
+                }
+            ]
+        }
+    ]
+}

configs/rebuttal/llama31_extend.json

@@ -5,7 +5,7 @@
     "__seq_thresh_fa3": 0,
     "using_extend": true,
     "self_extend_scale": 36,
-    "dense_layers": [0,1,2],
+    "dense_layers": [0, 1, 2],
     "mask_refresh_interval": [32, 16, 8],
     "layers": [
         {

configs/rebuttal/llama31_noextend_dense_decode.json

@@ -0,0 +1,66 @@
+{
+    "__head_reduce": 0,
+    "__using_dense_prefill": true,
+    "__last_dense": -1,
+    "__using_landmark": false,
+    "__seq_thresh_fa3": 0,
+    "__delta_attention_args": "window_0-diff_2-w_256-dense_decode",
+    "using_extend": false,
+    "self_extend_scale": 2,
+    "dense_layers": [0,1,2],
+    "mask_refresh_interval": [32, 16, 8],
+    "layers": [
+        {
+            "sliding_window_size": 131072,
+            "second_stage_k": 0,
+            "sink_token_size": 0,
+            "sa_extend_backend": "self_extend"
+        },
+        {
+            "sliding_window_size": 131072,
+            "second_stage_k": 0,
+            "sink_token_size": 0,
+            "sa_extend_backend": "self_extend"
+        }
+    ],
+    "prefill_layers": [
+        {
+            "sliding_window_size": 131072,
+            "second_stage_k": 0,
+            "sink_token_size": 0,
+            "sa_extend_backend": "self_extend"
+        },
+        {
+            "sliding_window_size": 1024,
+            "second_stage_k": 2048,
+            "sink_token_size": 256,
+            "sa_extend_backend": "self_extend",
+            "scan_extend_backend": "none",
+            "stages": [
+                {
+                    "stage_block_size_q":64,
+                    "stage_block_stride_q":2,
+                    "stage_chunk_size":128,
+                    "stage_k":null,
+                    "stage_stride":1
+                },
+                {
+                    "stage_block_size_q":64,
+                    "stage_block_stride_q":2,
+                    "stage_chunk_size":32,
+                    "stage_k":32768,
+                    "stage_stride":1,
+                    "using_landmark":true
+                },
+                {
+                    "stage_block_size_q":64,
+                    "stage_block_stride_q":1,
+                    "stage_chunk_size":8,
+                    "stage_k":8192,
+                    "stage_stride":1,
+                    "using_landmark":true
+                }
+            ]
+        }
+    ]
+}

hip-research/src/hip_research/main/chat.py

@@ -152,6 +152,7 @@
             "model": "anything",
             "messages": [{"role": "system", "content": sys_prompt}] + chat_log,
             "stream": True,
+            "max_tokens": 32768,
             "temperature": 0.7,
             "top_p": 0.8,
             "top_k": 20,