From 196a8595cda938d503c0555fb2a924a97d4451ba Mon Sep 17 00:00:00 2001 From: Sunil Muthuswamy Date: Fri, 15 May 2026 17:18:32 -0700 Subject: [PATCH 1/2] gdma: Support selective revoke of vtl0 vf EQE 135 now indicates whether to revoke vtl0 VF or not using the EQE data field. Also added a new feature flag to report to HWC about this supported feature. --- openhcl/underhill_core/src/emuplat/netvsp.rs | 13 +++++---- vm/devices/net/gdma_defs/src/lib.rs | 12 ++++++++ vm/devices/net/mana_driver/src/gdma_driver.rs | 29 +++++++++++-------- vm/devices/net/mana_driver/src/mana.rs | 9 +++--- vm/devices/net/mana_driver/src/resources.rs | 2 +- vm/devices/net/mana_driver/src/tests.rs | 6 ++-- 6 files changed, 45 insertions(+), 26 deletions(-) diff --git a/openhcl/underhill_core/src/emuplat/netvsp.rs b/openhcl/underhill_core/src/emuplat/netvsp.rs index 497bff811f..d59d44fb85 100644 --- a/openhcl/underhill_core/src/emuplat/netvsp.rs +++ b/openhcl/underhill_core/src/emuplat/netvsp.rs @@ -341,7 +341,7 @@ struct HclNetworkVFManagerWorker { #[inspect(skip)] dma_clients: VfioDmaClients, #[inspect(skip)] - vf_reset_request_receiver: Option>, + vf_reset_request_receiver: Option>, #[inspect(skip)] network_adapter_index: NetworkAdapterIndex, } @@ -1032,10 +1032,11 @@ impl HclNetworkVFManagerWorker { async fn reconfigure_vf( &mut self, vtl2_device_state: &mut Vtl2DeviceState, + revoke_vtl0_vf: bool, ) -> Option { // Remove VTL0 VF if present *self.guest_state.vtl0_vfid.lock().await = None; - if self.guest_state.is_offered_to_guest().await { + if revoke_vtl0_vf && self.guest_state.is_offered_to_guest().await { tracing::warn!( vtl2_vfid = vtl2_vfid_from_bus_control(&self.vtl2_bus_control), vtl0_vfid = vtl0_vfid_from_bus_control(&self.vtl0_bus_control), @@ -1185,7 +1186,7 @@ impl HclNetworkVFManagerWorker { ManagerMessage(HclNetworkVfManagerMessage), ManaDeviceArrived, ManaDeviceRemoved, - VfReconfig, + VfReconfig(bool), VfReconfigRestart, ExitWorker, } @@ -1236,7 +1237,7 @@ impl HclNetworkVFManagerWorker { .vf_reset_request_receiver .as_mut() .unwrap() - .map(|()| NextWorkItem::VfReconfig); + .map(NextWorkItem::VfReconfig); let reconfig_restart_deadline = vf_reconfig_backoff.map(|backoff| backoff.deadline); let wait_for_reconfig = futures::stream::once(async { match reconfig_restart_deadline { @@ -1368,7 +1369,7 @@ impl HclNetworkVFManagerWorker { // Exit worker thread. return; } - NextWorkItem::VfReconfig => { + NextWorkItem::VfReconfig(revoke_vtl0_vf) => { if self.is_shutdown_active || matches!(vtl2_device_state, Vtl2DeviceState::Missing) { @@ -1383,7 +1384,7 @@ impl HclNetworkVFManagerWorker { } vf_reconfig_backoff = self - .reconfigure_vf(&mut vtl2_device_state) + .reconfigure_vf(&mut vtl2_device_state, revoke_vtl0_vf) .instrument(tracing::info_span!( "VTL2 VF reconfiguration requested", vtl2_vfid diff --git a/vm/devices/net/gdma_defs/src/lib.rs b/vm/devices/net/gdma_defs/src/lib.rs index 248cac24ce..4f113ce6fa 100644 --- a/vm/devices/net/gdma_defs/src/lib.rs +++ b/vm/devices/net/gdma_defs/src/lib.rs @@ -288,6 +288,17 @@ pub struct EqeDataReconfig { pub reserved1: [u8; 8], } +#[bitfield(u32)] +#[derive(IntoBytes, Immutable, KnownLayout, FromBytes)] +pub struct EqeVfReset { + #[bits(1)] + pub revoke_vtl0_vf: bool, + #[bits(7)] + pub reserved1: u8, + #[bits(24)] + pub reserved2: u32, +} + pub const HWC_INIT_DATA_CQID: u8 = 1; pub const HWC_INIT_DATA_RQID: u8 = 2; pub const HWC_INIT_DATA_SQID: u8 = 3; @@ -455,6 +466,7 @@ pub const DRIVER_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT: u64 = 0x20; pub const DRIVER_CAP_FLAG_1_HW_VPORT_LINK_AWARE: u64 = 0x40; pub const DRIVER_CAP_FLAG_1_SELF_RESET_ON_EQE_NOTIFICATION: u64 = 0x4000; pub const DRIVER_CAP_FLAG_1_VTL2_REVOKE_SUB_ON_RESET_EQE: u64 = 0x10000; +pub const DRIVER_CAP_FLAG_1_VTL2_SELECTIVE_REVOKE_SUB_ON_RESET_EQE: u64 = 0x8000000; pub const OS_TYPE_OHCL: u32 = 0x60; diff --git a/vm/devices/net/mana_driver/src/gdma_driver.rs b/vm/devices/net/mana_driver/src/gdma_driver.rs index d3246044fd..37de4265b7 100644 --- a/vm/devices/net/mana_driver/src/gdma_driver.rs +++ b/vm/devices/net/mana_driver/src/gdma_driver.rs @@ -20,7 +20,9 @@ use gdma_defs::DRIVER_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG; use gdma_defs::DRIVER_CAP_FLAG_1_SELF_RESET_ON_EQE_NOTIFICATION; use gdma_defs::DRIVER_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT; use gdma_defs::DRIVER_CAP_FLAG_1_VTL2_REVOKE_SUB_ON_RESET_EQE; +use gdma_defs::DRIVER_CAP_FLAG_1_VTL2_SELECTIVE_REVOKE_SUB_ON_RESET_EQE; use gdma_defs::EqeDataReconfig; +use gdma_defs::EqeVfReset; use gdma_defs::EstablishHwc; use gdma_defs::GDMA_EQE_COMPLETION; use gdma_defs::GDMA_EQE_HWC_INIT_DATA; @@ -163,7 +165,9 @@ pub struct GdmaDriver { hwc_failure: bool, db_id: u32, state_saved: bool, - reset_request_pending: bool, + // The option will be set if there is a pending VF reset event. The + // option value indicates whether to remove the subordinate VF or not. + reset_request_pending: Option, } const EQ_PAGE: usize = 0; @@ -212,7 +216,7 @@ impl Drop for GdmaDriver { fn drop(&mut self) { tracing::info!(?self.state_saved, ?self.hwc_failure, ?self.reset_request_pending, "dropping gdma driver"); - if self.reset_request_pending { + if self.reset_request_pending.is_some() { return; } @@ -498,7 +502,7 @@ impl GdmaDriver { hwc_failure: false, state_saved: false, db_id, - reset_request_pending: false, + reset_request_pending: None, }; this.push_rqe(); @@ -528,7 +532,7 @@ impl GdmaDriver { anyhow::bail!("cannot save/restore after HWC failure"); } - if self.reset_request_pending { + if self.reset_request_pending.is_some() { anyhow::bail!("cannot save/restore with HWC reset request pending"); } @@ -677,7 +681,7 @@ impl GdmaDriver { hwc_failure: false, state_saved: false, db_id: db_id as u32, - reset_request_pending: false, + reset_request_pending: None, }; this.eq.arm(); @@ -693,7 +697,7 @@ impl GdmaDriver { ms_elapsed: u32, ) { // Don't report timeout once HWC reset request is pending, SoC will not respond. - if self.reset_request_pending { + if self.reset_request_pending.is_some() { return; } // Perform initial check for ownership, failing without wait if device @@ -790,7 +794,7 @@ impl GdmaDriver { self.link_toggle.drain(..).collect() } - pub fn get_reset_request_pending(&self) -> bool { + pub fn get_reset_request_pending(&self) -> Option { self.reset_request_pending } @@ -846,7 +850,7 @@ impl GdmaDriver { dev_id: GdmaDevId, req: Req, ) -> anyhow::Result<(Resp, u32)> { - if self.reset_request_pending { + if self.reset_request_pending.is_some() { anyhow::bail!("HWC reset request pending"); } if self.hwc_failure { @@ -1040,9 +1044,9 @@ impl GdmaDriver { } } GDMA_EQE_HWC_RESET_REQUEST => { - // No data is supplied for HWC reset request events. - tracing::info!("HWC reset request event"); - self.reset_request_pending = true; + let data = EqeVfReset::read_from_prefix(&eqe.data[..]).unwrap().0; + tracing::info!("HWC VF reconfiguration event"); + self.reset_request_pending = Some(data.revoke_vtl0_vf()); } ty => tracing::error!(ty, "unknown eq event"), } @@ -1259,7 +1263,8 @@ impl GdmaDriver { | DRIVER_CAP_FLAG_1_HW_VPORT_LINK_AWARE | DRIVER_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG | DRIVER_CAP_FLAG_1_SELF_RESET_ON_EQE_NOTIFICATION - | DRIVER_CAP_FLAG_1_VTL2_REVOKE_SUB_ON_RESET_EQE, + | DRIVER_CAP_FLAG_1_VTL2_REVOKE_SUB_ON_RESET_EQE + | DRIVER_CAP_FLAG_1_VTL2_SELECTIVE_REVOKE_SUB_ON_RESET_EQE, os_type: gdma_defs::OS_TYPE_OHCL, os_ver_major: ver.major(), os_ver_minor: ver.minor(), diff --git a/vm/devices/net/mana_driver/src/mana.rs b/vm/devices/net/mana_driver/src/mana.rs index 62f14af288..82b74f7450 100644 --- a/vm/devices/net/mana_driver/src/mana.rs +++ b/vm/devices/net/mana_driver/src/mana.rs @@ -69,7 +69,7 @@ struct Inner { dev_config: ManaQueryDeviceCfgResp, doorbell: Arc, vport_link_status: Arc>>, - vf_reset_request_sender: Arc>>>, + vf_reset_request_sender: Arc>>>, } impl ManaDevice { @@ -245,12 +245,13 @@ impl ManaDevice { ); } } - if gdma.get_reset_request_pending() { + if let Some(revoke_vtl0_vf) = gdma.get_reset_request_pending() { // `reset_request_pending` stays true until destruction. // Take the sender so we only notify once per lifetime. if let Some(sender) = inner.vf_reset_request_sender.lock().await.take() { - sender.send(()); + sender.send(revoke_vtl0_vf); + } } } @@ -296,7 +297,7 @@ impl ManaDevice { /// Subscribes to HWC reset request events. /// Returned receiver will receive a message on HWC reset request events. - pub async fn subscribe_vf_reset_request(&self) -> mesh::Receiver<()> { + pub async fn subscribe_vf_reset_request(&self) -> mesh::Receiver { tracing::debug!("subscribing to HWC reset request events"); let mut reset_request_sender = self.inner.vf_reset_request_sender.lock().await; assert!( diff --git a/vm/devices/net/mana_driver/src/resources.rs b/vm/devices/net/mana_driver/src/resources.rs index baa1475649..5ab7811d62 100644 --- a/vm/devices/net/mana_driver/src/resources.rs +++ b/vm/devices/net/mana_driver/src/resources.rs @@ -63,7 +63,7 @@ impl ResourceArena { } pub(crate) async fn destroy(mut self, gdma: &mut GdmaDriver) { - let skip_hwc = gdma.get_reset_request_pending(); + let skip_hwc = gdma.get_reset_request_pending().is_some(); if skip_hwc { tracing::info!( count = self.resources.len(), diff --git a/vm/devices/net/mana_driver/src/tests.rs b/vm/devices/net/mana_driver/src/tests.rs index 5938c6d809..0ada132b0b 100644 --- a/vm/devices/net/mana_driver/src/tests.rs +++ b/vm/devices/net/mana_driver/src/tests.rs @@ -337,7 +337,7 @@ async fn test_gdma_reset_request(driver: DefaultDriver) { .unwrap(); assert!( - !gdma.get_reset_request_pending(), + !gdma.get_reset_request_pending().is_some(), "reset_request_pending should be false" ); @@ -355,7 +355,7 @@ async fn test_gdma_reset_request(driver: DefaultDriver) { gdma.generate_reset_request_eqe().await.unwrap(); assert!( - gdma.get_reset_request_pending(), + gdma.get_reset_request_pending().is_some(), "reset_request_pending should be true after reset request" ); @@ -368,7 +368,7 @@ async fn test_gdma_reset_request(driver: DefaultDriver) { "unexpected error: {err_msg}" ); assert!( - gdma.get_reset_request_pending(), + gdma.get_reset_request_pending().is_some(), "reset_request_pending should remain true after deregister_device" ); } From 0cc93cd082eab74c8d551e358a1dbf990e25743c Mon Sep 17 00:00:00 2001 From: Sunil Muthuswamy Date: Thu, 28 May 2026 23:57:43 +0000 Subject: [PATCH 2/2] PR feedback --- openhcl/underhill_core/src/emuplat/netvsp.rs | 4 +- vm/devices/net/gdma/src/hwc.rs | 11 ++- vm/devices/net/gdma_defs/src/lib.rs | 7 ++ vm/devices/net/mana_driver/src/gdma_driver.rs | 13 ++- vm/devices/net/mana_driver/src/mana.rs | 1 - vm/devices/net/mana_driver/src/tests.rs | 84 +++++++++++++++++-- 6 files changed, 100 insertions(+), 20 deletions(-) diff --git a/openhcl/underhill_core/src/emuplat/netvsp.rs b/openhcl/underhill_core/src/emuplat/netvsp.rs index d59d44fb85..061f96a64f 100644 --- a/openhcl/underhill_core/src/emuplat/netvsp.rs +++ b/openhcl/underhill_core/src/emuplat/netvsp.rs @@ -1034,9 +1034,9 @@ impl HclNetworkVFManagerWorker { vtl2_device_state: &mut Vtl2DeviceState, revoke_vtl0_vf: bool, ) -> Option { - // Remove VTL0 VF if present - *self.guest_state.vtl0_vfid.lock().await = None; + // Remove VTL0 VF if requested. if revoke_vtl0_vf && self.guest_state.is_offered_to_guest().await { + *self.guest_state.vtl0_vfid.lock().await = None; tracing::warn!( vtl2_vfid = vtl2_vfid_from_bus_control(&self.vtl2_bus_control), vtl0_vfid = vtl0_vfid_from_bus_control(&self.vtl0_bus_control), diff --git a/vm/devices/net/gdma/src/hwc.rs b/vm/devices/net/gdma/src/hwc.rs index 3c130a8e87..1797487639 100644 --- a/vm/devices/net/gdma/src/hwc.rs +++ b/vm/devices/net/gdma/src/hwc.rs @@ -20,6 +20,7 @@ use gdma_defs::GdmaCreateQueueResp; use gdma_defs::GdmaDevId; use gdma_defs::GdmaDevType; use gdma_defs::GdmaDisableQueueReq; +use gdma_defs::GdmaGenerateResetEventReq; use gdma_defs::GdmaGenerateTestEventReq; use gdma_defs::GdmaListDevicesResp; use gdma_defs::GdmaQueryMaxResourcesResp; @@ -317,11 +318,13 @@ impl HwControl { 0 } GdmaRequestType::GDMA_GENERATE_RESET_REQUEST_EQE => { - let req: GdmaGenerateTestEventReq = + let req: GdmaGenerateResetEventReq = read.read_plain().context("reading reset request EQE")?; - self.state - .queues - .post_eq(req.queue_index, GDMA_EQE_HWC_RESET_REQUEST, &[]); + self.state.queues.post_eq( + req.queue_index, + GDMA_EQE_HWC_RESET_REQUEST, + req.data.as_bytes(), + ); 0 } GdmaRequestType::GDMA_VERIFY_VF_DRIVER_VERSION => { diff --git a/vm/devices/net/gdma_defs/src/lib.rs b/vm/devices/net/gdma_defs/src/lib.rs index 4f113ce6fa..e5c6a472c8 100644 --- a/vm/devices/net/gdma_defs/src/lib.rs +++ b/vm/devices/net/gdma_defs/src/lib.rs @@ -615,3 +615,10 @@ pub struct GdmaChangeMsixVectorIndexForEq { pub reserved1: u32, pub reserved2: u32, } + +#[repr(C)] +#[derive(Debug, IntoBytes, Immutable, KnownLayout, FromBytes)] +pub struct GdmaGenerateResetEventReq { + pub queue_index: u32, + pub data: EqeVfReset, +} diff --git a/vm/devices/net/mana_driver/src/gdma_driver.rs b/vm/devices/net/mana_driver/src/gdma_driver.rs index 37de4265b7..5ffbf3e400 100644 --- a/vm/devices/net/mana_driver/src/gdma_driver.rs +++ b/vm/devices/net/mana_driver/src/gdma_driver.rs @@ -42,6 +42,8 @@ use gdma_defs::GdmaCreateQueueResp; use gdma_defs::GdmaDestroyDmaRegionReq; use gdma_defs::GdmaDevId; use gdma_defs::GdmaDisableQueueReq; +#[cfg(test)] +use gdma_defs::GdmaGenerateResetEventReq; use gdma_defs::GdmaGenerateTestEventReq; use gdma_defs::GdmaListDevicesResp; use gdma_defs::GdmaMsgHdr; @@ -1045,8 +1047,9 @@ impl GdmaDriver { } GDMA_EQE_HWC_RESET_REQUEST => { let data = EqeVfReset::read_from_prefix(&eqe.data[..]).unwrap().0; - tracing::info!("HWC VF reconfiguration event"); - self.reset_request_pending = Some(data.revoke_vtl0_vf()); + let revoke_vtl0_vf = data.revoke_vtl0_vf(); + tracing::info!(revoke_vtl0_vf, "HWC VF reset request"); + self.reset_request_pending = Some(revoke_vtl0_vf); } ty => tracing::error!(ty, "unknown eq event"), } @@ -1240,12 +1243,14 @@ impl GdmaDriver { #[cfg(test)] #[tracing::instrument(skip(self), level = "debug", err)] - pub async fn generate_reset_request_eqe(&mut self) -> anyhow::Result<()> { + pub async fn generate_reset_request_eqe(&mut self, revoke_vtl0_vf: bool) -> anyhow::Result<()> { + let reset = EqeVfReset::new().with_revoke_vtl0_vf(revoke_vtl0_vf); self.request::<_, ()>( GdmaRequestType::GDMA_GENERATE_RESET_REQUEST_EQE.0, HWC_DEV_ID, - GdmaGenerateTestEventReq { + GdmaGenerateResetEventReq { queue_index: self.eq.id(), + data: reset, }, ) .await?; diff --git a/vm/devices/net/mana_driver/src/mana.rs b/vm/devices/net/mana_driver/src/mana.rs index 82b74f7450..c00718bec3 100644 --- a/vm/devices/net/mana_driver/src/mana.rs +++ b/vm/devices/net/mana_driver/src/mana.rs @@ -251,7 +251,6 @@ impl ManaDevice { if let Some(sender) = inner.vf_reset_request_sender.lock().await.take() { sender.send(revoke_vtl0_vf); - } } } diff --git a/vm/devices/net/mana_driver/src/tests.rs b/vm/devices/net/mana_driver/src/tests.rs index 0ada132b0b..8781af7afa 100644 --- a/vm/devices/net/mana_driver/src/tests.rs +++ b/vm/devices/net/mana_driver/src/tests.rs @@ -336,9 +336,10 @@ async fn test_gdma_reset_request(driver: DefaultDriver) { .await .unwrap(); - assert!( - !gdma.get_reset_request_pending().is_some(), - "reset_request_pending should be false" + assert_eq!( + gdma.get_reset_request_pending(), + None, + "reset_request_pending should be unset before reset request" ); // Get the device ID while HWC is still alive (needed for deregister later). @@ -352,11 +353,12 @@ async fn test_gdma_reset_request(driver: DefaultDriver) { .unwrap(); // Trigger the reset event (EQE 135). - gdma.generate_reset_request_eqe().await.unwrap(); + gdma.generate_reset_request_eqe(false).await.unwrap(); - assert!( - gdma.get_reset_request_pending().is_some(), - "reset_request_pending should be true after reset request" + assert_eq!( + gdma.get_reset_request_pending(), + Some(false), + "reset_request_pending should capture revoke_vtl0_vf=false" ); // Deregister should fail immediately because reset_request_pending is set. @@ -367,8 +369,72 @@ async fn test_gdma_reset_request(driver: DefaultDriver) { err_msg.contains("HWC reset request pending"), "unexpected error: {err_msg}" ); + assert_eq!( + gdma.get_reset_request_pending(), + Some(false), + "reset_request_pending should remain revoke_vtl0_vf=false after deregister_device" + ); +} + +#[async_test] +async fn test_gdma_reset_request_with_revoke(driver: DefaultDriver) { + let mem = DeviceTestMemory::new(128, false, "test_gdma"); + let msi_conn = MsiConnection::new(AssignedBusRange::new(), 0); + let device = gdma::GdmaDevice::new( + &VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())), + mem.guest_memory(), + msi_conn.target(), + vec![VportConfig { + mac_address: [1, 2, 3, 4, 5, 6].into(), + endpoint: Box::new(NullEndpoint::new()), + }], + &mut ExternallyManagedMmioIntercepts, + ); + let dma_client = mem.dma_client(); + let device = EmulatedDevice::new(device, msi_conn, dma_client); + let dma_client = device.dma_client(); + let buffer = dma_client.allocate_dma_buffer(6 * PAGE_SIZE).unwrap(); + + let mut gdma = GdmaDriver::new(&driver, device, 1, Some(buffer)) + .await + .unwrap(); + + assert_eq!( + gdma.get_reset_request_pending(), + None, + "reset_request_pending should be unset before reset request" + ); + + // Get the device ID while HWC is still alive (needed for deregister later). + let dev_id = gdma + .list_devices() + .await + .unwrap() + .iter() + .copied() + .find(|dev_id| dev_id.ty == GdmaDevType::GDMA_DEVICE_MANA) + .unwrap(); + + // Trigger the reset event (EQE 135) with vtl0 VF revoke. + gdma.generate_reset_request_eqe(true).await.unwrap(); + + assert_eq!( + gdma.get_reset_request_pending(), + Some(true), + "reset_request_pending should capture revoke_vtl0_vf=true" + ); + + // Deregister should fail immediately because reset_request_pending is set. + let deregister_result = gdma.deregister_device(dev_id).await; + let err = deregister_result.expect_err("deregister_device should fail after EQE 135"); + let err_msg = format!("{err:#}"); assert!( - gdma.get_reset_request_pending().is_some(), - "reset_request_pending should remain true after deregister_device" + err_msg.contains("HWC reset request pending"), + "unexpected error: {err_msg}" + ); + assert_eq!( + gdma.get_reset_request_pending(), + Some(true), + "reset_request_pending should remain revoke_vtl0_vf=true after deregister_device" ); }