Skip to content

mana: Support selective revoke of vtl0 vf #3571

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sunilmut merged 2 commits into from
May 29, 2026
Merged

mana: Support selective revoke of vtl0 vf #3571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
196a859
gdma: Support selective revoke of vtl0 vf
sunilmut May 16, 2026
0cc93cd
PR feedback
sunilmut May 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions openhcl/underhill_core/src/emuplat/netvsp.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ struct HclNetworkVFManagerWorker {
#[inspect(skip)]
dma_clients: VfioDmaClients,
#[inspect(skip)]
vf_reset_request_receiver: Option<mesh::Receiver<()>>,
vf_reset_request_receiver: Option<mesh::Receiver<bool>>,
#[inspect(skip)]
network_adapter_index: NetworkAdapterIndex,
}
Expand Down Expand Up @@ -1032,10 +1032,11 @@ impl HclNetworkVFManagerWorker {
async fn reconfigure_vf(
&mut self,
vtl2_device_state: &mut Vtl2DeviceState,
revoke_vtl0_vf: bool,
) -> Option<VfReconfigBackoff> {
// Remove VTL0 VF if present
*self.guest_state.vtl0_vfid.lock().await = None;
if self.guest_state.is_offered_to_guest().await {
// Remove VTL0 VF if requested.
if revoke_vtl0_vf && self.guest_state.is_offered_to_guest().await {
*self.guest_state.vtl0_vfid.lock().await = None;
tracing::warn!(
vtl2_vfid = vtl2_vfid_from_bus_control(&self.vtl2_bus_control),
vtl0_vfid = vtl0_vfid_from_bus_control(&self.vtl0_bus_control),
Expand Down Expand Up @@ -1185,7 +1186,7 @@ impl HclNetworkVFManagerWorker {
ManagerMessage(HclNetworkVfManagerMessage),
ManaDeviceArrived,
ManaDeviceRemoved,
VfReconfig,
VfReconfig(bool),
VfReconfigRestart,
ExitWorker,
}
Expand Down Expand Up @@ -1236,7 +1237,7 @@ impl HclNetworkVFManagerWorker {
.vf_reset_request_receiver
.as_mut()
.unwrap()
.map(|()| NextWorkItem::VfReconfig);
.map(NextWorkItem::VfReconfig);
let reconfig_restart_deadline = vf_reconfig_backoff.map(|backoff| backoff.deadline);
let wait_for_reconfig = futures::stream::once(async {
match reconfig_restart_deadline {
Expand Down Expand Up @@ -1368,7 +1369,7 @@ impl HclNetworkVFManagerWorker {
// Exit worker thread.
return;
}
NextWorkItem::VfReconfig => {
NextWorkItem::VfReconfig(revoke_vtl0_vf) => {
if self.is_shutdown_active
|| matches!(vtl2_device_state, Vtl2DeviceState::Missing)
{
Expand All @@ -1383,7 +1384,7 @@ impl HclNetworkVFManagerWorker {
}

vf_reconfig_backoff = self
.reconfigure_vf(&mut vtl2_device_state)
.reconfigure_vf(&mut vtl2_device_state, revoke_vtl0_vf)
.instrument(tracing::info_span!(
"VTL2 VF reconfiguration requested",
vtl2_vfid
Expand Down
11 changes: 7 additions & 4 deletions vm/devices/net/gdma/src/hwc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use gdma_defs::GdmaCreateQueueResp;
use gdma_defs::GdmaDevId;
use gdma_defs::GdmaDevType;
use gdma_defs::GdmaDisableQueueReq;
use gdma_defs::GdmaGenerateResetEventReq;
use gdma_defs::GdmaGenerateTestEventReq;
use gdma_defs::GdmaListDevicesResp;
use gdma_defs::GdmaQueryMaxResourcesResp;
Expand Down Expand Up @@ -317,11 +318,13 @@ impl HwControl {
0
}
GdmaRequestType::GDMA_GENERATE_RESET_REQUEST_EQE => {
let req: GdmaGenerateTestEventReq =
let req: GdmaGenerateResetEventReq =
read.read_plain().context("reading reset request EQE")?;
self.state
.queues
.post_eq(req.queue_index, GDMA_EQE_HWC_RESET_REQUEST, &[]);
self.state.queues.post_eq(
req.queue_index,
GDMA_EQE_HWC_RESET_REQUEST,
req.data.as_bytes(),
);
0
}
GdmaRequestType::GDMA_VERIFY_VF_DRIVER_VERSION => {
Expand Down
19 changes: 19 additions & 0 deletions vm/devices/net/gdma_defs/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,17 @@ pub struct EqeDataReconfig {
pub reserved1: [u8; 8],
}

#[bitfield(u32)]
#[derive(IntoBytes, Immutable, KnownLayout, FromBytes)]
pub struct EqeVfReset {
#[bits(1)]
pub revoke_vtl0_vf: bool,
#[bits(7)]
pub reserved1: u8,
#[bits(24)]
pub reserved2: u32,
}

pub const HWC_INIT_DATA_CQID: u8 = 1;
pub const HWC_INIT_DATA_RQID: u8 = 2;
pub const HWC_INIT_DATA_SQID: u8 = 3;
Expand Down Expand Up @@ -455,6 +466,7 @@ pub const DRIVER_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT: u64 = 0x20;
pub const DRIVER_CAP_FLAG_1_HW_VPORT_LINK_AWARE: u64 = 0x40;
pub const DRIVER_CAP_FLAG_1_SELF_RESET_ON_EQE_NOTIFICATION: u64 = 0x4000;
pub const DRIVER_CAP_FLAG_1_VTL2_REVOKE_SUB_ON_RESET_EQE: u64 = 0x10000;
pub const DRIVER_CAP_FLAG_1_VTL2_SELECTIVE_REVOKE_SUB_ON_RESET_EQE: u64 = 0x8000000;

pub const OS_TYPE_OHCL: u32 = 0x60;

Expand Down Expand Up @@ -603,3 +615,10 @@ pub struct GdmaChangeMsixVectorIndexForEq {
pub reserved1: u32,
pub reserved2: u32,
}

#[repr(C)]
#[derive(Debug, IntoBytes, Immutable, KnownLayout, FromBytes)]
pub struct GdmaGenerateResetEventReq {
pub queue_index: u32,
pub data: EqeVfReset,
}
38 changes: 24 additions & 14 deletions vm/devices/net/mana_driver/src/gdma_driver.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ use gdma_defs::DRIVER_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG;
use gdma_defs::DRIVER_CAP_FLAG_1_SELF_RESET_ON_EQE_NOTIFICATION;
use gdma_defs::DRIVER_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT;
use gdma_defs::DRIVER_CAP_FLAG_1_VTL2_REVOKE_SUB_ON_RESET_EQE;
use gdma_defs::DRIVER_CAP_FLAG_1_VTL2_SELECTIVE_REVOKE_SUB_ON_RESET_EQE;
use gdma_defs::EqeDataReconfig;
use gdma_defs::EqeVfReset;
use gdma_defs::EstablishHwc;
use gdma_defs::GDMA_EQE_COMPLETION;
use gdma_defs::GDMA_EQE_HWC_INIT_DATA;
Expand All @@ -40,6 +42,8 @@ use gdma_defs::GdmaCreateQueueResp;
use gdma_defs::GdmaDestroyDmaRegionReq;
use gdma_defs::GdmaDevId;
use gdma_defs::GdmaDisableQueueReq;
#[cfg(test)]
use gdma_defs::GdmaGenerateResetEventReq;
use gdma_defs::GdmaGenerateTestEventReq;
use gdma_defs::GdmaListDevicesResp;
use gdma_defs::GdmaMsgHdr;
Expand Down Expand Up @@ -163,7 +167,9 @@ pub struct GdmaDriver<T: DeviceBacking> {
hwc_failure: bool,
db_id: u32,
state_saved: bool,
reset_request_pending: bool,
// The option will be set if there is a pending VF reset event. The
// option value indicates whether to remove the subordinate VF or not.
reset_request_pending: Option<bool>,
}

const EQ_PAGE: usize = 0;
Expand Down Expand Up @@ -212,7 +218,7 @@ impl<T: DeviceBacking> Drop for GdmaDriver<T> {
fn drop(&mut self) {
tracing::info!(?self.state_saved, ?self.hwc_failure, ?self.reset_request_pending, "dropping gdma driver");

if self.reset_request_pending {
if self.reset_request_pending.is_some() {
return;
}

Expand Down Expand Up @@ -498,7 +504,7 @@ impl<T: DeviceBacking> GdmaDriver<T> {
hwc_failure: false,
state_saved: false,
db_id,
reset_request_pending: false,
reset_request_pending: None,
};

this.push_rqe();
Expand Down Expand Up @@ -528,7 +534,7 @@ impl<T: DeviceBacking> GdmaDriver<T> {
anyhow::bail!("cannot save/restore after HWC failure");
}

if self.reset_request_pending {
if self.reset_request_pending.is_some() {
anyhow::bail!("cannot save/restore with HWC reset request pending");
}

Expand Down Expand Up @@ -677,7 +683,7 @@ impl<T: DeviceBacking> GdmaDriver<T> {
hwc_failure: false,
state_saved: false,
db_id: db_id as u32,
reset_request_pending: false,
reset_request_pending: None,
};

this.eq.arm();
Expand All @@ -693,7 +699,7 @@ impl<T: DeviceBacking> GdmaDriver<T> {
ms_elapsed: u32,
) {
// Don't report timeout once HWC reset request is pending, SoC will not respond.
if self.reset_request_pending {
if self.reset_request_pending.is_some() {
return;
}
// Perform initial check for ownership, failing without wait if device
Expand Down Expand Up @@ -790,7 +796,7 @@ impl<T: DeviceBacking> GdmaDriver<T> {
self.link_toggle.drain(..).collect()
}

pub fn get_reset_request_pending(&self) -> bool {
pub fn get_reset_request_pending(&self) -> Option<bool> {
self.reset_request_pending
}

Expand Down Expand Up @@ -846,7 +852,7 @@ impl<T: DeviceBacking> GdmaDriver<T> {
dev_id: GdmaDevId,
req: Req,
) -> anyhow::Result<(Resp, u32)> {
if self.reset_request_pending {
if self.reset_request_pending.is_some() {
anyhow::bail!("HWC reset request pending");
}
if self.hwc_failure {
Expand Down Expand Up @@ -1040,9 +1046,10 @@ impl<T: DeviceBacking> GdmaDriver<T> {
}
}
GDMA_EQE_HWC_RESET_REQUEST => {
// No data is supplied for HWC reset request events.
tracing::info!("HWC reset request event");
self.reset_request_pending = true;
let data = EqeVfReset::read_from_prefix(&eqe.data[..]).unwrap().0;
let revoke_vtl0_vf = data.revoke_vtl0_vf();
tracing::info!(revoke_vtl0_vf, "HWC VF reset request");
self.reset_request_pending = Some(revoke_vtl0_vf);
}
ty => tracing::error!(ty, "unknown eq event"),
}
Expand Down Expand Up @@ -1236,12 +1243,14 @@ impl<T: DeviceBacking> GdmaDriver<T> {

#[cfg(test)]
#[tracing::instrument(skip(self), level = "debug", err)]
pub async fn generate_reset_request_eqe(&mut self) -> anyhow::Result<()> {
pub async fn generate_reset_request_eqe(&mut self, revoke_vtl0_vf: bool) -> anyhow::Result<()> {
let reset = EqeVfReset::new().with_revoke_vtl0_vf(revoke_vtl0_vf);
self.request::<_, ()>(
GdmaRequestType::GDMA_GENERATE_RESET_REQUEST_EQE.0,
HWC_DEV_ID,
GdmaGenerateTestEventReq {
GdmaGenerateResetEventReq {
queue_index: self.eq.id(),
data: reset,
},
)
.await?;
Expand All @@ -1259,7 +1268,8 @@ impl<T: DeviceBacking> GdmaDriver<T> {
| DRIVER_CAP_FLAG_1_HW_VPORT_LINK_AWARE
| DRIVER_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG
| DRIVER_CAP_FLAG_1_SELF_RESET_ON_EQE_NOTIFICATION
| DRIVER_CAP_FLAG_1_VTL2_REVOKE_SUB_ON_RESET_EQE,
| DRIVER_CAP_FLAG_1_VTL2_REVOKE_SUB_ON_RESET_EQE
| DRIVER_CAP_FLAG_1_VTL2_SELECTIVE_REVOKE_SUB_ON_RESET_EQE,
os_type: gdma_defs::OS_TYPE_OHCL,
os_ver_major: ver.major(),
os_ver_minor: ver.minor(),
Expand Down
8 changes: 4 additions & 4 deletions vm/devices/net/mana_driver/src/mana.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ struct Inner<T: DeviceBacking> {
dev_config: ManaQueryDeviceCfgResp,
doorbell: Arc<dyn Doorbell>,
vport_link_status: Arc<Mutex<Vec<LinkStatus>>>,
vf_reset_request_sender: Arc<Mutex<Option<mesh::Sender<()>>>>,
vf_reset_request_sender: Arc<Mutex<Option<mesh::Sender<bool>>>>,
}

impl<T: DeviceBacking> ManaDevice<T> {
Expand Down Expand Up @@ -245,12 +245,12 @@ impl<T: DeviceBacking> ManaDevice<T> {
);
}
}
if gdma.get_reset_request_pending() {
if let Some(revoke_vtl0_vf) = gdma.get_reset_request_pending() {
// `reset_request_pending` stays true until destruction.
// Take the sender so we only notify once per lifetime.
if let Some(sender) = inner.vf_reset_request_sender.lock().await.take()
{
sender.send(());
sender.send(revoke_vtl0_vf);
}
}
}
Expand Down Expand Up @@ -296,7 +296,7 @@ impl<T: DeviceBacking> ManaDevice<T> {

/// Subscribes to HWC reset request events.
/// Returned receiver will receive a message on HWC reset request events.
pub async fn subscribe_vf_reset_request(&self) -> mesh::Receiver<()> {
pub async fn subscribe_vf_reset_request(&self) -> mesh::Receiver<bool> {
tracing::debug!("subscribing to HWC reset request events");
let mut reset_request_sender = self.inner.vf_reset_request_sender.lock().await;
assert!(
Expand Down
2 changes: 1 addition & 1 deletion vm/devices/net/mana_driver/src/resources.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ impl ResourceArena {
}

pub(crate) async fn destroy<T: DeviceBacking>(mut self, gdma: &mut GdmaDriver<T>) {
let skip_hwc = gdma.get_reset_request_pending();
let skip_hwc = gdma.get_reset_request_pending().is_some();
if skip_hwc {
tracing::info!(
count = self.resources.len(),
Expand Down
Loading
Loading