Message-ID: <ZMyJIq4CgXxudJED@chao-email>
Date:   Fri, 4 Aug 2023 13:14:10 +0800
From:   Chao Gao <chao.gao@...el.com>
To:     Yang Weijiang <weijiang.yang@...el.com>
CC:     <seanjc@...gle.com>, <pbonzini@...hat.com>, <peterz@...radead.org>,
        <john.allen@....com>, <kvm@...r.kernel.org>,
        <linux-kernel@...r.kernel.org>, <rick.p.edgecombe@...el.com>,
        <binbin.wu@...ux.intel.com>
Subject: Re: [PATCH v5 11/19] KVM:VMX: Emulate read and write to CET MSRs

On Thu, Aug 03, 2023 at 12:27:24AM -0400, Yang Weijiang wrote:
>Add emulation interface for CET MSR read and write.
>The emulation code is split into common part and vendor specific
>part, the former resides in x86.c to benefic different x86 CPU
>vendors, the latter for VMX is implemented in this patch.
>
>Signed-off-by: Yang Weijiang <weijiang.yang@...el.com>
>---
> arch/x86/kvm/vmx/vmx.c |  27 +++++++++++
> arch/x86/kvm/x86.c     | 104 +++++++++++++++++++++++++++++++++++++----
> arch/x86/kvm/x86.h     |  18 +++++++
> 3 files changed, 141 insertions(+), 8 deletions(-)
>
>diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
>index 6aa76124e81e..ccf750e79608 100644
>--- a/arch/x86/kvm/vmx/vmx.c
>+++ b/arch/x86/kvm/vmx/vmx.c
>@@ -2095,6 +2095,18 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 		else
> 			msr_info->data = vmx->pt_desc.guest.addr_a[index / 2];
> 		break;
>+	case MSR_IA32_S_CET:
>+	case MSR_KVM_GUEST_SSP:
>+	case MSR_IA32_INT_SSP_TAB:
>+		if (kvm_get_msr_common(vcpu, msr_info))
>+			return 1;
>+		if (msr_info->index == MSR_KVM_GUEST_SSP)
>+			msr_info->data = vmcs_readl(GUEST_SSP);
>+		else if (msr_info->index == MSR_IA32_S_CET)
>+			msr_info->data = vmcs_readl(GUEST_S_CET);
>+		else if (msr_info->index == MSR_IA32_INT_SSP_TAB)
>+			msr_info->data = vmcs_readl(GUEST_INTR_SSP_TABLE);

This if-else-if suggests that they are focibly grouped together to just
share the call of kvm_get_msr_common(). For readability, I think it is better
to handle them separately.

e.g.,
	case MSR_IA32_S_CET:
		if (kvm_get_msr_common(vcpu, msr_info))
			return 1;
		msr_info->data = vmcs_readl(GUEST_S_CET);
		break;

	case MSR_KVM_GUEST_SSP:
		if (kvm_get_msr_common(vcpu, msr_info))
			return 1;
		msr_info->data = vmcs_readl(GUEST_SSP);
		break;

	...


>+		break;
> 	case MSR_IA32_DEBUGCTLMSR:
> 		msr_info->data = vmcs_read64(GUEST_IA32_DEBUGCTL);
> 		break;
>@@ -2404,6 +2416,18 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 		else
> 			vmx->pt_desc.guest.addr_a[index / 2] = data;
> 		break;
>+	case MSR_IA32_S_CET:
>+	case MSR_KVM_GUEST_SSP:
>+	case MSR_IA32_INT_SSP_TAB:
>+		if (kvm_set_msr_common(vcpu, msr_info))
>+			return 1;
>+		if (msr_index == MSR_KVM_GUEST_SSP)
>+			vmcs_writel(GUEST_SSP, data);
>+		else if (msr_index == MSR_IA32_S_CET)
>+			vmcs_writel(GUEST_S_CET, data);
>+		else if (msr_index == MSR_IA32_INT_SSP_TAB)
>+			vmcs_writel(GUEST_INTR_SSP_TABLE, data);

ditto

>+		break;
> 	case MSR_IA32_PERF_CAPABILITIES:
> 		if (data && !vcpu_to_pmu(vcpu)->version)
> 			return 1;
>@@ -4864,6 +4888,9 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
> 		vmcs_write64(GUEST_BNDCFGS, 0);
> 
> 	vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
>+	vmcs_writel(GUEST_SSP, 0);
>+	vmcs_writel(GUEST_S_CET, 0);
>+	vmcs_writel(GUEST_INTR_SSP_TABLE, 0);

where are MSR_IA32_PL3_SSP and MSR_IA32_U_CET reset?

I thought that guest FPU would be reset in kvm_vcpu_reset(). But it turns out
only MPX states are reset in KVM while other FPU states are unchanged. This
is aligned with "Table 10.1 IA-32 and Intel® 64 Processor States Following
Power-up, Reset, or INIT"

Could you double confirm the hardware beahavior that CET states are reset to 0
on INIT? If CET states are reset, we need to handle CET_IA32_PL3_SSP and
MSR_IA32_U_CET like MPX.

> 
> 	kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
> 
>diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
>index 5b63441fd2d2..98f3ff6078e6 100644
>--- a/arch/x86/kvm/x86.c
>+++ b/arch/x86/kvm/x86.c
>@@ -3627,6 +3627,39 @@ static bool kvm_is_msr_to_save(u32 msr_index)
> 	return false;
> }
> 
>+static inline bool is_shadow_stack_msr(u32 msr)
>+{
>+	return msr == MSR_IA32_PL0_SSP ||
>+		msr == MSR_IA32_PL1_SSP ||
>+		msr == MSR_IA32_PL2_SSP ||
>+		msr == MSR_IA32_PL3_SSP ||
>+		msr == MSR_IA32_INT_SSP_TAB ||
>+		msr == MSR_KVM_GUEST_SSP;
>+}
>+
>+static bool kvm_cet_is_msr_accessible(struct kvm_vcpu *vcpu,
>+				      struct msr_data *msr)
>+{
>+	if (is_shadow_stack_msr(msr->index)) {
>+		if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK))
>+			return false;
>+
>+		if (msr->index == MSR_KVM_GUEST_SSP)
>+			return msr->host_initiated;
>+
>+		return msr->host_initiated ||
>+			guest_cpuid_has(vcpu, X86_FEATURE_SHSTK);
>+	}
>+
>+	if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) &&
>+	    !kvm_cpu_cap_has(X86_FEATURE_IBT))
>+		return false;
>+
>+	return msr->host_initiated ||
>+		guest_cpuid_has(vcpu, X86_FEATURE_IBT) ||
>+		guest_cpuid_has(vcpu, X86_FEATURE_SHSTK);
>+}
>+
> int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> {
> 	u32 msr = msr_info->index;
>@@ -3981,6 +4014,45 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 		vcpu->arch.guest_fpu.xfd_err = data;
> 		break;
> #endif
>+#define CET_EXCLUSIVE_BITS		(CET_SUPPRESS | CET_WAIT_ENDBR)
>+#define CET_CTRL_RESERVED_BITS		GENMASK(9, 6)
>+#define CET_SHSTK_MASK_BITS		GENMASK(1, 0)
>+#define CET_IBT_MASK_BITS		(GENMASK_ULL(5, 2) | \
>+					 GENMASK_ULL(63, 10))
>+#define CET_LEG_BITMAP_BASE(data)	((data) >> 12)
>+	case MSR_IA32_U_CET:
>+	case MSR_IA32_S_CET:
>+		if (!kvm_cet_is_msr_accessible(vcpu, msr_info))
>+			return 1;
>+		if (!!(data & CET_CTRL_RESERVED_BITS))
>+			return 1;
>+		if (!guest_can_use(vcpu, X86_FEATURE_SHSTK) &&
>+		    (data & CET_SHSTK_MASK_BITS))
>+			return 1;
>+		if (!guest_can_use(vcpu, X86_FEATURE_IBT) &&
>+		    (data & CET_IBT_MASK_BITS))
>+			return 1;
>+		if (!IS_ALIGNED(CET_LEG_BITMAP_BASE(data), 4) ||
>+		    (data & CET_EXCLUSIVE_BITS) == CET_EXCLUSIVE_BITS)
>+			return 1;
>+		if (msr == MSR_IA32_U_CET)

can you add a comment before this if() statement like?
		/* MSR_IA32_S_CET is handled by vendor code */

>+	case MSR_KVM_GUEST_SSP:
>+	case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB:
>+		if (!kvm_cet_is_msr_accessible(vcpu, msr_info))
>+			return 1;
>+		if (is_noncanonical_address(data, vcpu))
>+			return 1;
>+		if (!IS_ALIGNED(data, 4))
>+			return 1;
>+		if (msr == MSR_IA32_PL0_SSP || msr == MSR_IA32_PL1_SSP ||
>+		    msr == MSR_IA32_PL2_SSP) {
>+			vcpu->arch.cet_s_ssp[msr - MSR_IA32_PL0_SSP] = data;
>+		} else if (msr == MSR_IA32_PL3_SSP) {
>+			kvm_set_xsave_msr(msr_info);
>+		}

brackets are not needed.

also add a comment for MSR_KVM_GUEST_SSP.

>+		break;
> 	default:
> 		if (kvm_pmu_is_valid_msr(vcpu, msr))
> 			return kvm_pmu_set_msr(vcpu, msr_info);
>@@ -4051,7 +4123,9 @@ static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
> 
> int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> {
>-	switch (msr_info->index) {
>+	u32 msr = msr_info->index;
>+
>+	switch (msr) {
> 	case MSR_IA32_PLATFORM_ID:
> 	case MSR_IA32_EBL_CR_POWERON:
> 	case MSR_IA32_LASTBRANCHFROMIP:
>@@ -4086,7 +4160,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
> 	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
> 	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
>-		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
>+		if (kvm_pmu_is_valid_msr(vcpu, msr))
> 			return kvm_pmu_get_msr(vcpu, msr_info);
> 		msr_info->data = 0;
> 		break;
>@@ -4137,7 +4211,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 	case MSR_MTRRcap:
> 	case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000:
> 	case MSR_MTRRdefType:
>-		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
>+		return kvm_mtrr_get_msr(vcpu, msr, &msr_info->data);
> 	case 0xcd: /* fsb frequency */
> 		msr_info->data = 3;
> 		break;
>@@ -4159,7 +4233,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 		msr_info->data = kvm_get_apic_base(vcpu);
> 		break;
> 	case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
>-		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
>+		return kvm_x2apic_msr_read(vcpu, msr, &msr_info->data);
> 	case MSR_IA32_TSC_DEADLINE:
> 		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
> 		break;
>@@ -4253,7 +4327,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 	case MSR_IA32_MCG_STATUS:
> 	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
> 	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
>-		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
>+		return get_msr_mce(vcpu, msr, &msr_info->data,
> 				   msr_info->host_initiated);
> 	case MSR_IA32_XSS:
> 		if (!msr_info->host_initiated &&
>@@ -4284,7 +4358,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 	case HV_X64_MSR_TSC_EMULATION_STATUS:
> 	case HV_X64_MSR_TSC_INVARIANT_CONTROL:
> 		return kvm_hv_get_msr_common(vcpu,
>-					     msr_info->index, &msr_info->data,
>+					     msr, &msr_info->data,
> 					     msr_info->host_initiated);
> 	case MSR_IA32_BBL_CR_CTL3:
> 		/* This legacy MSR exists but isn't fully documented in current
>@@ -4337,8 +4411,22 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 		msr_info->data = vcpu->arch.guest_fpu.xfd_err;
> 		break;
> #endif
>+	case MSR_IA32_U_CET:
>+	case MSR_IA32_S_CET:
>+	case MSR_KVM_GUEST_SSP:
>+	case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB:
>+		if (!kvm_cet_is_msr_accessible(vcpu, msr_info))
>+			return 1;
>+		if (msr == MSR_IA32_PL0_SSP || msr == MSR_IA32_PL1_SSP ||
>+		    msr == MSR_IA32_PL2_SSP) {
>+			msr_info->data =
>+				vcpu->arch.cet_s_ssp[msr - MSR_IA32_PL0_SSP];
>+		} else if (msr == MSR_IA32_U_CET || msr == MSR_IA32_PL3_SSP) {
>+			kvm_get_xsave_msr(msr_info);
>+		}

Again, for readability and clarity, how about:

	case MSR_IA32_U_CET:
	case MSR_IA32_PL3_SSP:
		if (!kvm_cet_is_msr_accessible(vcpu, msr_info))
			return 1;
		kvm_get_xsave_msr(msr_info);
		break;
	case MSR_IA32_PL0_SSP ... MSR_IA32_PL2_SSP:
		if (!kvm_cet_is_msr_accessible(vcpu, msr_info))
			return 1;
		msr_info->data = vcpu->arch.cet_s_ssp[msr - MSR_IA32_PL0_SSP];
		break;
	case MSR_IA32_S_CET:
	case MSR_KVM_GUEST_SSP:
		if (!kvm_cet_is_msr_accessible(vcpu, msr_info))
			return 1;
		/* Further handling in vendor code */
		break;

>+		break;
> 	default:
>-		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
>+		if (kvm_pmu_is_valid_msr(vcpu, msr))
> 			return kvm_pmu_get_msr(vcpu, msr_info);
> 
> 		/*
>@@ -4346,7 +4434,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> 		 * to-be-saved, even if an MSR isn't fully supported.
> 		 */
> 		if (msr_info->host_initiated &&
>-		    kvm_is_msr_to_save(msr_info->index)) {
>+		    kvm_is_msr_to_save(msr)) {
> 			msr_info->data = 0;
> 			break;
> 		}
>diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
>index c69fc027f5ec..3b79d6db2f83 100644
>--- a/arch/x86/kvm/x86.h
>+++ b/arch/x86/kvm/x86.h
>@@ -552,4 +552,22 @@ int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,
> 			 unsigned int port, void *data,  unsigned int count,
> 			 int in);
> 
>+/*
>+ * Guest xstate MSRs have been loaded in __msr_io(), disable preemption before
>+ * access the MSRs to avoid MSR content corruption.
>+ */

I think it is better to describe what the function does prior to jumping into
details like where guest FPU is loaded.

/*
 * Lock and/or reload guest FPU and access xstate MSRs. For accesses initiated
 * by host, guest FPU is loaded in __msr_io(). For accesses initiated by guest,
 * guest FPU should have been loaded already.
 */
>+static inline void kvm_get_xsave_msr(struct msr_data *msr_info)
>+{
>+	kvm_fpu_get();
>+	rdmsrl(msr_info->index, msr_info->data);
>+	kvm_fpu_put();
>+}
>+
>+static inline void kvm_set_xsave_msr(struct msr_data *msr_info)
>+{
>+	kvm_fpu_get();
>+	wrmsrl(msr_info->index, msr_info->data);
>+	kvm_fpu_put();
>+}

Can you rename functions to kvm_get/set_xstate_msr() to align with the comment
and patch 6? And if there is no user outside x86.c, you can just put these two
functions right after the is_xstate_msr() added in patch 6.

>+
> #endif
>-- 
>2.27.0
>

Powered by blists - more mailing lists