Skip to content

Construct and wire up lapics for VTL 1 #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
smalis-msft merged 5 commits into from
Oct 14, 2024
Merged

Construct and wire up lapics for VTL 1 #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
06e3878
Construct and wire up lapics for VTL 1
smalis-msft Oct 4, 2024
cc3d0af
Fix handling of INIT/SIPI with more than 1 VTL
smalis-msft Oct 14, 2024
09e65b0
Revert "Fix handling of INIT/SIPI with more than 1 VTL"
smalis-msft Oct 14, 2024
412661a
Better fix for init/sipi
smalis-msft Oct 14, 2024
3d8a081
Bring back the asserts too
smalis-msft Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 23 additions & 15 deletions openhcl/virt_mshv_vtl/src/processor/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,9 +207,10 @@ mod private {
stop: &mut StopVp<'_>,
) -> impl Future<Output = Result<(), VpHaltReason<UhRunVpError>>>;

/// Returns true if the VP is ready to run, false if it is halted.
/// Returns true if the VP is ready to run the given VTL, false if it is halted.
fn poll_apic(
this: &mut UhProcessor<'_, Self>,
vtl: Vtl,
scan_irr: bool,
) -> Result<bool, UhRunVpError>;

Expand Down Expand Up @@ -616,7 +617,7 @@ impl<'p, T: Backing> Processor for UhProcessor<'p, T> {
Ordering::Relaxed,
);

let mut scan_irr = true;
let mut first_scan_irr = true;

loop {
// Process VP activity and wait for the VP to be ready.
Expand All @@ -639,22 +640,28 @@ impl<'p, T: Backing> Processor for UhProcessor<'p, T> {
}

// Process wakes.
if self.inner.wake_reasons.load(Ordering::Relaxed) != 0 {
scan_irr = self.handle_wake().map_err(VpHaltReason::Hypervisor)?;
let scan_irr = if self.inner.wake_reasons.load(Ordering::Relaxed) != 0 {
self.handle_wake().map_err(VpHaltReason::Hypervisor)?
} else {
[false, false].into()
};

if self.untrusted_synic.is_some() {
self.update_synic(Vtl::Vtl0, true);
}

// TODO CVM GUEST VSM: Split ready into two to track per-vtl
let mut ready = false;
for vtl in [Vtl::Vtl1, Vtl::Vtl0] {
// Process interrupts.
if self.hv(vtl).is_some() {
self.update_synic(vtl, false);
}
}
if self.untrusted_synic.is_some() {
self.update_synic(Vtl::Vtl0, true);
}

let ready = T::poll_apic(self, scan_irr).map_err(VpHaltReason::Hypervisor)?;
scan_irr = false;
ready |= T::poll_apic(self, vtl, scan_irr[vtl] || first_scan_irr)
.map_err(VpHaltReason::Hypervisor)?;
}
first_scan_irr = false;

// Arm the timer.
if let Some(timeout) = self.vmtime.get_timeout() {
Expand Down Expand Up @@ -696,8 +703,10 @@ impl<'p, T: Backing> Processor for UhProcessor<'p, T> {
fn flush_async_requests(&mut self) -> Result<(), Self::RunVpError> {
if self.inner.wake_reasons.load(Ordering::Relaxed) != 0 {
let scan_irr = self.handle_wake()?;
if scan_irr {
T::poll_apic(self, true)?;
for vtl in [Vtl::Vtl1, Vtl::Vtl0] {
if scan_irr[vtl] {
T::poll_apic(self, vtl, true)?;
}
}
}
Ok(())
Expand Down Expand Up @@ -778,7 +787,7 @@ impl<'a, T: Backing> UhProcessor<'a, T> {
}

/// Returns true if the interrupt controller has work to do.
fn handle_wake(&mut self) -> Result<bool, UhRunVpError> {
fn handle_wake(&mut self) -> Result<VtlArray<bool, 2>, UhRunVpError> {
let wake_reasons_raw = self.inner.wake_reasons.swap(0, Ordering::SeqCst);
let wake_reasons_vtl: [WakeReason; 2] = zerocopy::transmute!(wake_reasons_raw);
for (vtl, wake_reasons) in [
Expand Down Expand Up @@ -843,7 +852,6 @@ impl<'a, T: Backing> UhProcessor<'a, T> {
// Should not have already initialized the hv emulator for this vtl
assert!(self.hv(vtl).is_none());

// TODO GUEST_VSM construct VTL 1 lapics
self.hv[vtl] = Some(
self.partition
.hv
Expand All @@ -864,7 +872,7 @@ impl<'a, T: Backing> UhProcessor<'a, T> {
}
}

Ok(wake_reasons_vtl[0].intcon())
Ok(wake_reasons_vtl.map(|w| w.intcon()).into())
}

fn request_sint_notifications(&mut self, vtl: Vtl, sints: u16) {
Expand Down
41 changes: 25 additions & 16 deletions openhcl/virt_mshv_vtl/src/processor/mshv/apic.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,21 @@ use x86defs::RFlags;
#[derive(Inspect)]
pub(super) struct UhApicState {
lapic: LocalApic,
#[inspect(debug)]
vtl: Vtl,
pub(super) halted: bool,
pub(super) startup_suspend: bool,
nmi_pending: bool,
}

impl UhApicState {
pub fn new(lapic: LocalApic, vp_info: &VpInfo) -> Self {
pub fn new(lapic: LocalApic, vtl: Vtl, vp_info: &VpInfo) -> Self {
Self {
lapic,
vtl,
halted: false,
nmi_pending: false,
startup_suspend: !vp_info.is_bsp(),
startup_suspend: vtl == Vtl::Vtl0 && !vp_info.is_bsp(),
}
}

Expand All @@ -69,6 +72,7 @@ impl UhApicState {
runner,
dev,
vmtime,
vtl: self.vtl,
})
.mmio_write(address, data);
}
Expand All @@ -88,6 +92,7 @@ impl UhApicState {
runner,
dev,
vmtime,
vtl: self.vtl,
})
.mmio_read(address, data);
}
Expand All @@ -107,6 +112,7 @@ impl UhApicState {
runner,
dev,
vmtime,
vtl: self.vtl,
})
.msr_write(msr, value)
}
Expand All @@ -125,6 +131,7 @@ impl UhApicState {
runner,
dev,
vmtime,
vtl: self.vtl,
})
.msr_read(msr)
}
Expand Down Expand Up @@ -254,10 +261,12 @@ impl UhApicState {

impl UhProcessor<'_, HypervisorBackedX86> {
/// Returns true if the VP is ready to run, false if it is halted.
pub(super) fn poll_apic(&mut self, scan_irr: bool) -> Result<bool, UhRunVpError> {
let Some(lapic) = self.backing.lapic.as_mut() else {
pub(super) fn poll_apic(&mut self, vtl: Vtl, scan_irr: bool) -> Result<bool, UhRunVpError> {
let Some(lapics) = self.backing.lapics.as_mut() else {
return Ok(true);
};

let lapic = &mut lapics[vtl];
let ApicWork {
init,
extint,
Expand Down Expand Up @@ -286,33 +295,31 @@ impl UhProcessor<'_, HypervisorBackedX86> {
todo!();
}

// TODO WHP GUEST VSM: An INIT/SIPI targeted at a VP with more than one guest VTL enabled is ignored.
if init {
self.handle_init()?;
self.handle_init(vtl)?;
}

if let Some(vector) = sipi {
self.handle_sipi(vector)?;
self.handle_sipi(vtl, vector)?;
}

let lapic = self.backing.lapic.as_ref().unwrap();
let lapic = &self.backing.lapics.as_ref().unwrap()[vtl];
if lapic.halted || lapic.startup_suspend {
return Ok(false);
}

Ok(true)
}

fn handle_init(&mut self) -> Result<(), UhRunVpError> {
fn handle_init(&mut self, vtl: Vtl) -> Result<(), UhRunVpError> {
let vp_info = self.inner.vp_info;
{
let mut access = self.access_state(Vtl::Vtl0);
virt::x86::vp::x86_init(&mut access, &vp_info).map_err(UhRunVpError::State)?;
}
Ok(())
let mut access = self.access_state(vtl);
virt::x86::vp::x86_init(&mut access, &vp_info).map_err(UhRunVpError::State)
}

fn handle_sipi(&mut self, vector: u8) -> Result<(), UhRunVpError> {
let lapic = self.backing.lapic.as_mut().unwrap();
fn handle_sipi(&mut self, vtl: Vtl, vector: u8) -> Result<(), UhRunVpError> {
let lapic = &mut self.backing.lapics.as_mut().unwrap()[vtl];
if lapic.startup_suspend {
let address = (vector as u64) << 12;
let cs: hvdef::HvX64SegmentRegister = hvdef::HvX64SegmentRegister {
Expand All @@ -339,6 +346,7 @@ struct UhApicClient<'a, 'b, T> {
runner: &'a mut ProcessorRunner<'b, MshvX64>,
dev: &'a T,
vmtime: &'a VmTimeAccess,
vtl: Vtl,
}

impl<T: CpuIo> ApicClient for UhApicClient<'_, '_, T> {
Expand All @@ -365,10 +373,11 @@ impl<T: CpuIo> ApicClient for UhApicClient<'_, '_, T> {
self.partition
.vp(vp_index)
.unwrap()
.wake(Vtl::Vtl0, WakeReason::INTCON);
.wake(self.vtl, WakeReason::INTCON);
}

fn eoi(&mut self, vector: u8) {
debug_assert_eq!(self.vtl, Vtl::Vtl0);
self.dev.handle_eoi(vector.into())
}

Expand Down
11 changes: 6 additions & 5 deletions openhcl/virt_mshv_vtl/src/processor/mshv/arm64.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,11 +194,12 @@ impl BackingPrivate for HypervisorBackedArm64 {
Ok(())
}

fn poll_apic(this: &mut UhProcessor<'_, Self>, scan_irr: bool) -> Result<bool, UhRunVpError> {
{
let _ = (this, scan_irr);
Ok(true)
}
fn poll_apic(
_this: &mut UhProcessor<'_, Self>,
_vtl: Vtl,
_scan_irr: bool,
) -> Result<bool, UhRunVpError> {
Ok(true)
}

fn request_extint_readiness(this: &mut UhProcessor<'_, Self>) {
Expand Down
62 changes: 44 additions & 18 deletions openhcl/virt_mshv_vtl/src/processor/mshv/x64.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ use virt_support_x86emu::emulate::EmuCheckVtlAccessError;
use virt_support_x86emu::emulate::EmuTranslateError;
use virt_support_x86emu::emulate::EmuTranslateResult;
use virt_support_x86emu::emulate::EmulatorSupport;
use vtl_array::VtlArray;
use vtl_array::VtlSet;
use x86defs::xsave::Fxsave;
use x86defs::xsave::XsaveHeader;
Expand All @@ -74,7 +75,11 @@ use zerocopy::FromZeroes;
#[derive(InspectMut)]
pub struct HypervisorBackedX86 {
/// Underhill APIC state
pub(super) lapic: Option<apic::UhApicState>,
pub(super) lapics: Option<VtlArray<apic::UhApicState, 2>>,
// TODO WHP GUEST VSM: To be completely correct here, when emulating the APICs
// we would need two sets of deliverability notifications too. However currently
// we don't support VTL 1 on WHP, and on the hypervisor we don't emulate the APIC,
// so this can wait.
#[inspect(with = "|x| inspect::AsHex(u64::from(*x))")]
deliverability_notifications: HvDeliverabilityNotificationsRegister,
/// Next set of deliverability notifications. See register definition for details.
Expand Down Expand Up @@ -124,21 +129,27 @@ impl BackingPrivate for HypervisorBackedX86 {
reserved: [0; 384],
};

let lapic = params.partition.lapic.as_ref().map(|arr| {
let lapic_set = &arr[Vtl::Vtl0];
let lapics = params.partition.lapic.as_ref().map(|arr| {
// Initialize APIC base to match the current VM state.
let apic_base = params
.runner
.get_vp_register(HvX64RegisterName::ApicBase)
.unwrap()
.as_u64();
let mut lapic = lapic_set.add_apic(params.vp_info);
lapic.set_apic_base(apic_base).unwrap();
apic::UhApicState::new(lapic, &params.vp_info.base)
let mut lapic0 = arr[Vtl::Vtl0].add_apic(params.vp_info);
lapic0.set_apic_base(apic_base).unwrap();
let mut lapic1 = arr[Vtl::Vtl1].add_apic(params.vp_info);
lapic1.set_apic_base(apic_base).unwrap();

[
apic::UhApicState::new(lapic0, Vtl::Vtl0, &params.vp_info.base),
apic::UhApicState::new(lapic1, Vtl::Vtl1, &params.vp_info.base),
]
.into()
});

Ok(Self {
lapic,
lapics,
deliverability_notifications: Default::default(),
next_deliverability_notifications: Default::default(),
stats: Default::default(),
Expand Down Expand Up @@ -275,8 +286,12 @@ impl BackingPrivate for HypervisorBackedX86 {
Ok(())
}

fn poll_apic(this: &mut UhProcessor<'_, Self>, scan_irr: bool) -> Result<bool, UhRunVpError> {
this.poll_apic(scan_irr)
fn poll_apic(
this: &mut UhProcessor<'_, Self>,
vtl: Vtl,
scan_irr: bool,
) -> Result<bool, UhRunVpError> {
this.poll_apic(vtl, scan_irr)
}

fn request_extint_readiness(this: &mut UhProcessor<'_, Self>) {
Expand Down Expand Up @@ -591,14 +606,21 @@ impl UhProcessor<'_, HypervisorBackedX86> {
hvdef::HvX64MsrInterceptMessage::ref_from_prefix(self.runner.exit_message().payload())
.unwrap();
let rip = next_rip(&message.header);
let last_vtl = self.last_vtl();

tracing::trace!(msg = %format_args!("{:x?}", message), "msr");

let msr = message.msr_number;
match message.header.intercept_access_type {
HvInterceptAccessType::READ => {
let r = if let Some(lapic) = &mut self.backing.lapic {
lapic.msr_read(self.partition, &mut self.runner, &self.vmtime, dev, msr)
let r = if let Some(lapics) = &mut self.backing.lapics {
lapics[last_vtl].msr_read(
self.partition,
&mut self.runner,
&self.vmtime,
dev,
msr,
)
} else {
Err(MsrError::Unknown)
};
Expand All @@ -622,8 +644,8 @@ impl UhProcessor<'_, HypervisorBackedX86> {
}
HvInterceptAccessType::WRITE => {
let value = (message.rax & 0xffff_ffff) | (message.rdx << 32);
let r = if let Some(lapic) = &mut self.backing.lapic {
lapic.msr_write(
let r = if let Some(lapic) = &mut self.backing.lapics {
lapic[last_vtl].msr_write(
self.partition,
&mut self.runner,
&self.vmtime,
Expand Down Expand Up @@ -687,7 +709,8 @@ impl UhProcessor<'_, HypervisorBackedX86> {
}

fn handle_halt(&mut self) -> Result<(), VpHaltReason<UhRunVpError>> {
self.backing.lapic.as_mut().unwrap().halt();
let last_vtl = self.last_vtl();
self.backing.lapics.as_mut().unwrap()[last_vtl].halt();
Ok(())
}

Expand Down Expand Up @@ -1116,15 +1139,17 @@ impl<T: CpuIo> EmulatorSupport for UhEmulationState<'_, '_, T, HypervisorBackedX
}

fn lapic_base_address(&self) -> Option<u64> {
let last_vtl = self.vp.last_vtl();
self.vp
.backing
.lapic
.lapics
.as_ref()
.and_then(|lapic| lapic.base_address())
.and_then(|lapic| lapic[last_vtl].base_address())
}

fn lapic_read(&mut self, address: u64, data: &mut [u8]) {
self.vp.backing.lapic.as_mut().unwrap().mmio_read(
let last_vtl = self.vp.last_vtl();
self.vp.backing.lapics.as_mut().unwrap()[last_vtl].mmio_read(
self.vp.partition,
&mut self.vp.runner,
&self.vp.vmtime,
Expand All @@ -1135,7 +1160,8 @@ impl<T: CpuIo> EmulatorSupport for UhEmulationState<'_, '_, T, HypervisorBackedX
}

fn lapic_write(&mut self, address: u64, data: &[u8]) {
self.vp.backing.lapic.as_mut().unwrap().mmio_write(
let last_vtl = self.vp.last_vtl();
self.vp.backing.lapics.as_mut().unwrap()[last_vtl].mmio_write(
self.vp.partition,
&mut self.vp.runner,
&self.vp.vmtime,
Expand Down
Loading