RTL8814AU: chip-init + TX descriptor + USB-layer parity with aircrack-ng OOT driver

src/HalModule.cpp

@@ -29,7 +29,11 @@ constexpr uint32_t HPQ_PGNUM_8814A = 0x20; /* 32 pages per queue (USB) */
 constexpr uint32_t LPQ_PGNUM_8814A = 0x20;
 constexpr uint32_t NPQ_PGNUM_8814A = 0x20;
 constexpr uint32_t EPQ_PGNUM_8814A = 0x20;
-constexpr uint32_t BCNQ_PAGE_NUM_8814 = 0x08;
+/* BCNQ_PAGE_NUM_8814 is documented as 0x08 in upstream's rtl8814a_hal.h, but
+ * the aircrack-ng OOT driver running in our oracle VM shows boundary regs at
+ * 0x07F6 (= 2038), implying BCNQ reserves 10 pages, not 8. Use 0x0A to match
+ * the OOT post-init state. */
+constexpr uint32_t BCNQ_PAGE_NUM_8814 = 0x0A;
 constexpr uint32_t WOWLAN_PAGE_NUM_8814 = 0x00;
 constexpr uint32_t TXPKT_PGNUM_8814A =
     2048 - BCNQ_PAGE_NUM_8814 - WOWLAN_PAGE_NUM_8814;
@@ -315,12 +319,32 @@ bool HalModule::rtl8812au_hal_init() {
   /* enable Tx report. */
   _device.rtw_write8(REG_FWHW_TXQ_CTRL + 1, 0x0F);
 
-  /* Suggested by SD1 pisa. Added by tynli. 2011.10.21. */
-  _device.rtw_write8(REG_EARLY_MODE_CONTROL_8812 + 3,
-                     0x01); /* Pretx_en, for WEP/TKIP SEC */
+  if (is_8814a) {
+    /* Port of upstream _InitRetryFunction_8814A: set EN_AMPDU_RTY_NEW (bit 7
+     * of REG_FWHW_TXQ_CTRL byte 0) and REG_ACKTO = 0x80. Also clear BIT6 of
+     * byte 2 (EN_BCN_FUNCTION-within-TXQ_CTRL, not the BIT3 of REG_BCN_CTRL
+     * which is a separate per-port flag). OOT post-init shows
+     * REG_FWHW_TXQ_CTRL = 0x03310F80; we were at 0x03711F00 without these. */
+    uint8_t txqctl_b0 = _device.rtw_read8(REG_FWHW_TXQ_CTRL);
+    _device.rtw_write8(REG_FWHW_TXQ_CTRL, (uint8_t)(txqctl_b0 | 0x80 /* EN_AMPDU_RTY_NEW */));
+    uint8_t txqctl_b2 = _device.rtw_read8(REG_FWHW_TXQ_CTRL + 2);
+    _device.rtw_write8(REG_FWHW_TXQ_CTRL + 2, (uint8_t)(txqctl_b2 & ~0x40));
+    _device.rtw_write8(REG_ACKTO, 0x80);
+  }
 
-  /* tynli_test_tx_report. */
-  _device.rtw_write16(REG_TX_RPT_TIME, 0x3DF0);
+  /* Suggested by SD1 pisa. Added by tynli. 2011.10.21.
+   * Upstream rtl8814au's usb_halinit.c explicitly comments BOTH of these
+   * out for 8814 — the REG_EARLY_MODE_CONTROL_8812+3 = 0x01 write
+   * (Pretx_en for WEP/TKIP SEC) plus unconfigured security state may
+   * be holding TX frames in a "wait for encryption key" state on 8814.
+   * Gate to 8812-only. */
+  if (!is_8814a) {
+    _device.rtw_write8(REG_EARLY_MODE_CONTROL_8812 + 3,
+                       0x01); /* Pretx_en, for WEP/TKIP SEC */
+
+    /* tynli_test_tx_report. */
+    _device.rtw_write16(REG_TX_RPT_TIME, 0x3DF0);
+  }
 
   /* Reset USB mode switch setting */
   _device.rtw_write8(REG_SDIO_CTRL_8812, 0x0);
@@ -330,8 +354,12 @@ bool HalModule::rtl8812au_hal_init() {
 
   // TODO:
   ///* ack for xmit mgmt frames. */
-  _device.rtw_write32(REG_FWHW_TXQ_CTRL,
-                      _device.rtw_read32(REG_FWHW_TXQ_CTRL) | BIT12);
+  if (!is_8814a) {
+    /* OOT-driver register readback shows the 8814 doesn't set BIT12 (ack
+     * for xmit mgmt frames). Skip for 8814. */
+    _device.rtw_write32(REG_FWHW_TXQ_CTRL,
+                        _device.rtw_read32(REG_FWHW_TXQ_CTRL) | BIT12);
+  }
 
   /* Final safety re-write of MAC/RX enable bits. Some of the post-fwdl
    * init helpers can overwrite REG_CR back to only MACTXEN|MACRXEN
@@ -350,6 +378,20 @@ bool HalModule::rtl8812au_hal_init() {
   uint16_t cr_min = (uint16_t)(HCI_TXDMA_EN | HCI_RXDMA_EN | TXDMA_EN |
                                RXDMA_EN | PROTOCOL_EN | SCHEDULE_EN |
                                MACTXEN | MACRXEN);
+  if (is_8814a) {
+    /* OOT driver register dump shows REG_CR=0x06FF on a working 8814 init —
+     * ENSEC (BIT9) + CALTMR_EN (BIT10) are set in addition to the low byte.
+     * The upstream _InitPowerOn_8814AU OR-mask also includes both. Our
+     * earlier init skips InitPowerOn for 8814 (since fwdl does it via the
+     * RSVD-page path), so these high bits never get set. Add them here. */
+    cr_min |= ENSEC | CALTMR_EN;
+    /* OOT-driver readback shows REG_TXDMA_OFFSET_CHK = 0x0FFD0200 — bit 9
+     * (DROP_DATA_EN) is set. Our existing _InitHardwareDropIncorrectBulkOut
+     * runs pre-fwdl and gets cleared by the chip reset during fwdl. Re-apply
+     * here post-fwdl. */
+    uint32_t dma_chk = _device.rtw_read32(REG_TXDMA_OFFSET_CHK);
+    _device.rtw_write32(REG_TXDMA_OFFSET_CHK, dma_chk | DROP_DATA_EN);
+  }
   uint16_t cr_final = static_cast<uint16_t>(cr_observed | cr_min);
   _device.rtw_write16(REG_CR, cr_final);
   _device.rtw_write16(REG_RXFLTMAP2, 0xFFFF);
@@ -1185,11 +1227,22 @@ void HalModule::_InitQueueReservedPage_8814AUsb() {
    * page counts and 16-bit boundary registers. The 8812 8-bit REG_RQPN /
    * REG_BCNQ_BDNY equivalents don't exist on 8814, so reusing the 8812
    * path leaves HPQ/NPQ/LPQ with zero pages and TX bulk OUT stalls. */
-  _device.rtw_write32(REG_FIFOPAGE_INFO_1_8814A, HPQ_PGNUM_8814A);
-  _device.rtw_write32(REG_FIFOPAGE_INFO_2_8814A, LPQ_PGNUM_8814A);
-  _device.rtw_write32(REG_FIFOPAGE_INFO_3_8814A, NPQ_PGNUM_8814A);
-  _device.rtw_write32(REG_FIFOPAGE_INFO_4_8814A, EPQ_PGNUM_8814A);
-  _device.rtw_write32(REG_FIFOPAGE_INFO_5_8814A, PUB_PGNUM_8814A);
+  /* 8814 has dual MAC ports — these 32-bit FIFOPAGE_INFO regs carry the
+   * per-queue page count for port 0 in the low 16 bits and port 1 in the
+   * high 16 bits. Upstream rtl8814a_hal_init.c writes the raw page count
+   * (low half only), and the OOT-driver readback shows the chip mirrors it
+   * into the high half automatically — but on our path the high half stays
+   * at 0, so MAC port 1 has zero pages allocated. Write BOTH halves
+   * explicitly to match the OOT-driver post-init state (FIFOPAGE_INFO_1 =
+   * 0x00200020, FIFOPAGE_INFO_5 = 0x07760776). */
+  auto dup16 = [](uint32_t v) -> uint32_t {
+    return (v & 0xFFFF) | ((v & 0xFFFF) << 16);
+  };
+  _device.rtw_write32(REG_FIFOPAGE_INFO_1_8814A, dup16(HPQ_PGNUM_8814A));
+  _device.rtw_write32(REG_FIFOPAGE_INFO_2_8814A, dup16(LPQ_PGNUM_8814A));
+  _device.rtw_write32(REG_FIFOPAGE_INFO_3_8814A, dup16(NPQ_PGNUM_8814A));
+  _device.rtw_write32(REG_FIFOPAGE_INFO_4_8814A, dup16(EPQ_PGNUM_8814A));
+  _device.rtw_write32(REG_FIFOPAGE_INFO_5_8814A, dup16(PUB_PGNUM_8814A));
 
   _device.rtw_write32(REG_RQPN_CTRL_2_8814A, 0x80000000);
 

src/RtlJaguarDevice.cpp

@@ -137,7 +137,12 @@ bool RtlJaguarDevice::send_packet(const uint8_t *packet, size_t length) {
 
   SET_TX_DESC_DATA_BW_8812(usb_frame, BWSettingOfDesc);
 
-  SET_TX_DESC_FIRST_SEG_8812(usb_frame, 1);
+  /* Upstream rtl8814a_xmit.c sets ONLY LAST_SEG=1 (not FIRST_SEG) for a
+   * single-fragment frame, and MACID = bmc_camid which is 0 for the
+   * broadcast/default CAM entry. Our previous values (FIRST_SEG=1, MACID=1)
+   * caused the chip to silently reject every bulk-OUT — verified by
+   * usbmon trace comparing the OOT-driver's working TX descriptor to
+   * ours at byte offsets 3 (flags) and 4 (MACID). */
   SET_TX_DESC_LAST_SEG_8812(usb_frame, 1);
   SET_TX_DESC_OWN_8812(usb_frame, 1);
 
@@ -147,7 +152,7 @@ bool RtlJaguarDevice::send_packet(const uint8_t *packet, size_t length) {
   SET_TX_DESC_OFFSET_8812(usb_frame,
                           static_cast<uint8_t>(TXDESC_SIZE + OFFSET_SZ));
 
-  SET_TX_DESC_MACID_8812(usb_frame, static_cast<uint8_t>(0x01));
+  SET_TX_DESC_MACID_8812(usb_frame, static_cast<uint8_t>(0x00));
 
   if (!vht) {
     rate_id = 7;
@@ -161,15 +166,21 @@ bool RtlJaguarDevice::send_packet(const uint8_t *packet, size_t length) {
       static_cast<uint8_t>(rate_id));
 
   SET_TX_DESC_QUEUE_SEL_8812(usb_frame, 0x12);
-  SET_TX_DESC_HWSEQ_EN_8812(
-      usb_frame, static_cast<uint8_t>(0));
-  SET_TX_DESC_SEQ_8812(
-      usb_frame,
-      GetSequence(packet +
-                  radiotap_length));
-  SET_TX_DESC_RETRY_LIMIT_ENABLE_8812(usb_frame, static_cast<uint8_t>(1));
-
-  SET_TX_DESC_DATA_RETRY_LIMIT_8812(usb_frame, static_cast<uint8_t>(0));
+  /* Upstream 8814 uses HWSEQ_EN=1 (chip auto-fills the 802.11 SEQ number)
+   * with the descriptor SEQ field zero. Verified by usbmon trace: OOT
+   * driver sets bit 15 of word 8 (byte 33 = 0x80). Our previous path set
+   * HWSEQ_EN=0 + SEQ field manually — the chip's HWSEQ-disabled path may
+   * require additional driver-side sequence handling we don't have. */
+  SET_TX_DESC_HWSEQ_EN_8812(usb_frame, static_cast<uint8_t>(1));
+  /* OOT-driver descriptor has SPE_RPT=1 (bit 19 of word 2 = byte 10 bit 3).
+   * Special TX report — chip may use this to signal TX-complete back to the
+   * driver. Without it, the TX queue may stall waiting for a status ack we
+   * never enable. */
+  SET_TX_DESC_SPE_RPT_8812(usb_frame, static_cast<uint8_t>(1));
+  /* OOT-driver descriptor has RETRY_LIMIT_ENABLE=0 (let chip use its
+   * default retry policy). Setting RETRY_LIMIT_ENABLE=1 with
+   * DATA_RETRY_LIMIT=0 means "give up after 0 retries" — the chip drops
+   * the frame and never even attempts to TX. */
   if (sgi) {
     _logger->info("short gi enabled,set sgi");
     SET_TX_DESC_DATA_SHORT_8812(usb_frame, 1);

src/RtlUsbAdapter.cpp

@@ -351,6 +351,26 @@ bool RtlUsbAdapter::send_packet(uint8_t *packet, size_t length) {
     return false;
   }
 
+  /* On the FIRST send only, dump the bulk-OUT bytes to compare against
+   * the OOT-driver wire trace. */
+  static bool first_pkt_dump = true;
+  if (first_pkt_dump) {
+    first_pkt_dump = false;
+    /* Clear any HALT state on the TX EP. The fwdl process can leave EP
+     * 0x02 in a stalled state from the chip side; without clear_halt the
+     * USB controller would NAK every subsequent bulk OUT URB. */
+    int chr = libusb_clear_halt(_dev_handle, 0x02);
+    _logger->info("libusb_clear_halt(EP 0x02) rc={}", chr);
+    size_t dump_len = std::min<size_t>(length, 64);
+    char hex[64 * 2 + 1] = {0};
+    for (size_t k = 0; k < dump_len; ++k) {
+      static const char hd[] = "0123456789abcdef";
+      hex[2*k]   = hd[packet[k] >> 4];
+      hex[2*k+1] = hd[packet[k] & 0xF];
+    }
+    _logger->info("first TX bulk-OUT len={} bytes: {}", length, hex);
+  }
+
   /* On the FIRST send only, dump chip state via vendor reads. Surfaces any
    * register clobber between init-end and first TX (e.g. SetMonitorChannel
    * could be resetting REG_CR or related). */
@@ -382,6 +402,13 @@ bool RtlUsbAdapter::send_packet(uint8_t *packet, size_t length) {
   libusb_fill_bulk_transfer(transfer, _dev_handle, tx_ep, packet, length,
                             transfer_callback, (void *)(_logger.get()),
                             USB_TIMEOUT);
+  /* Upstream OOT (rtl8814a/usb/rtl8814au_xmit.c) sets URB_ZERO_PACKET on
+   * every TX URB. libusb equivalent: LIBUSB_TRANSFER_ADD_ZERO_PACKET.
+   * Without it the chip's SuperSpeed bulk OUT controller can wait
+   * indefinitely for transfer-end signaling and NAK every URB until libusb
+   * cancels — matches the usbmon trace we captured: 6977 submitted URBs,
+   * every completion with status=-2 (ENOENT/cancelled), data_len=0. */
+  transfer->flags |= LIBUSB_TRANSFER_ADD_ZERO_PACKET;
   auto start = std::chrono::high_resolution_clock::now();
   int rc = rc = libusb_submit_transfer(transfer);
   auto end = std::chrono::high_resolution_clock::now();

txdemo/main.cpp

@@ -175,7 +175,10 @@ int main(int argc, char **argv) {
   uint8_t beacon_frame[] = {
       0x00, 0x00, 0x0d, 0x00, 0x00, 0x80, 0x08, 0x00, 0x08, 0x00, 0x37,
       0x00, 0x01, // radiotap header
-      0x08, 0x01, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x57,
+      /* Mgmt / probe-request frame (FC=0x40 0x00). Was DATA / ToDS=1
+       * (FC=0x08 0x01) which requires an AP context the chip doesn't
+       * have in monitor mode — the chip silently NAKed every bulk OUT. */
+      0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x57,
       0x42, 0x75, 0x05, 0xd6, 0x00, 0x57, 0x42, 0x75, 0x05, 0xd6, 0x00,
       0x80, 0x00, // 80211 header
       0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x24, 0x4f,