| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <c7b2a64d65f1911e84891eba92fd139e24cc9862.1705965958.git.isaku.yamahata@intel.com>
Date: Mon, 22 Jan 2024 16:22:24 -0800
From: isaku.yamahata@...el.com
To: kvm@...r.kernel.org,
linux-kernel@...r.kernel.org
Cc: isaku.yamahata@...el.com,
isaku.yamahata@...il.com,
Paolo Bonzini <pbonzini@...hat.com>,
erdemaktas@...gle.com,
Sean Christopherson <seanjc@...gle.com>,
Sagi Shahar <sagis@...gle.com>,
Kai Huang <kai.huang@...el.com>,
chen.bo@...el.com,
hang.yuan@...el.com,
tina.zhang@...el.com,
Xiaoyao Li <xiaoyao.li@...el.com>
Subject: [PATCH v7 09/13] KVM: x86/tdp_mmu, TDX: Split a large page when 4KB page within it converted to shared
From: Xiaoyao Li <xiaoyao.li@...el.com>
When mapping the shared page for TDX, it needs to zap private alias.
In the case that private page is mapped as large page (2MB), it can be
removed directly only when the whole 2MB is converted to shared.
Otherwise, it has to split 2MB page into 512 4KB page, and only remove
the pages that converted to shared.
When a present large leaf spte switches to present non-leaf spte, TDX needs
to split the corresponding SEPT page to reflect it.
Signed-off-by: Xiaoyao Li <xiaoyao.li@...el.com>
Signed-off-by: Isaku Yamahata <isaku.yamahata@...el.com>
---
v7:
- catch up for tdx_seamcall() change
- typo in a comment of __set_private_spte_present()
- improved a comment in tdx_sept_split_private_spt()
v6:
- repeat TDH.MEM.PAGE.DEMOTE on TDX_INTERRUPTED_RESTARTABLE
---
arch/x86/include/asm/kvm-x86-ops.h | 1 +
arch/x86/include/asm/kvm_host.h | 2 ++
arch/x86/kvm/mmu/tdp_mmu.c | 21 ++++++++++++++++-----
arch/x86/kvm/vmx/tdx.c | 27 +++++++++++++++++++++++++--
arch/x86/kvm/vmx/tdx_arch.h | 1 +
arch/x86/kvm/vmx/tdx_errno.h | 1 +
arch/x86/kvm/vmx/tdx_ops.h | 13 +++++++++++++
7 files changed, 59 insertions(+), 7 deletions(-)
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index 527db174d6b5..08c55c3d6e5b 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -105,6 +105,7 @@ KVM_X86_OP_OPTIONAL_RET0(get_mt_mask)
KVM_X86_OP(load_mmu_pgd)
KVM_X86_OP_OPTIONAL(link_private_spt)
KVM_X86_OP_OPTIONAL(free_private_spt)
+KVM_X86_OP_OPTIONAL(split_private_spt)
KVM_X86_OP_OPTIONAL(set_private_spte)
KVM_X86_OP_OPTIONAL(remove_private_spte)
KVM_X86_OP_OPTIONAL(zap_private_spte)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 3a2237ed9dba..8123fad88750 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1783,6 +1783,8 @@ struct kvm_x86_ops {
void *private_spt);
int (*free_private_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
void *private_spt);
+ int (*split_private_spt)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
+ void *private_spt);
int (*set_private_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
kvm_pfn_t pfn);
int (*remove_private_spte)(struct kvm *kvm, gfn_t gfn, enum pg_level level,
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 98de2c093815..3f7307938982 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -588,23 +588,34 @@ static int __must_check __set_private_spte_present(struct kvm *kvm, tdp_ptep_t s
{
bool was_present = is_shadow_present_pte(old_spte);
bool is_present = is_shadow_present_pte(new_spte);
+ bool was_leaf = was_present && is_last_spte(old_spte, level);
bool is_leaf = is_present && is_last_spte(new_spte, level);
kvm_pfn_t new_pfn = spte_to_pfn(new_spte);
+ void *private_spt;
int ret = 0;
lockdep_assert_held(&kvm->mmu_lock);
- /* TDP MMU doesn't change present -> present */
- KVM_BUG_ON(was_present, kvm);
/*
* Use different call to either set up middle level
* private page table, or leaf.
*/
- if (is_leaf)
+ if (level > PG_LEVEL_4K && was_leaf && !is_leaf) {
+ /*
+ * splitting large page into 4KB.
+ * tdp_mmu_split_huge_page() => tdp_mmu_link_sp()
+ */
+ private_spt = get_private_spt(gfn, new_spte, level);
+ KVM_BUG_ON(!private_spt, kvm);
+ ret = static_call(kvm_x86_zap_private_spte)(kvm, gfn, level);
+ kvm_flush_remote_tlbs(kvm);
+ if (!ret)
+ ret = static_call(kvm_x86_split_private_spt)(kvm, gfn,
+ level, private_spt);
+ } else if (is_leaf)
ret = static_call(kvm_x86_set_private_spte)(kvm, gfn, level, new_pfn);
else {
- void *private_spt = get_private_spt(gfn, new_spte, level);
-
+ private_spt = get_private_spt(gfn, new_spte, level);
KVM_BUG_ON(!private_spt, kvm);
ret = static_call(kvm_x86_link_private_spt)(kvm, gfn, level, private_spt);
}
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index 747152af0882..10dbe4a4db7a 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -1718,6 +1718,30 @@ static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn,
return 0;
}
+static int tdx_sept_split_private_spt(struct kvm *kvm, gfn_t gfn,
+ enum pg_level level, void *private_spt)
+{
+ int tdx_level = pg_level_to_tdx_sept_level(level);
+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
+ gpa_t gpa = gfn_to_gpa(gfn) & KVM_HPAGE_MASK(level);
+ hpa_t hpa = __pa(private_spt);
+ struct tdx_module_args out;
+ u64 err;
+
+ /* See comment in tdx_sept_set_private_spte() to pin pages. */
+ do {
+ err = tdh_mem_page_demote(kvm_tdx->tdr_pa, gpa, tdx_level, hpa, &out);
+ } while (err == TDX_INTERRUPTED_RESTARTABLE);
+ if (unlikely(err == TDX_ERROR_SEPT_BUSY))
+ return -EAGAIN;
+ if (KVM_BUG_ON(err, kvm)) {
+ pr_tdx_error(TDH_MEM_PAGE_DEMOTE, err, &out);
+ return -EIO;
+ }
+
+ return 0;
+}
+
static int tdx_sept_zap_private_spte(struct kvm *kvm, gfn_t gfn,
enum pg_level level)
{
@@ -1731,8 +1755,6 @@ static int tdx_sept_zap_private_spte(struct kvm *kvm, gfn_t gfn,
if (unlikely(!is_hkid_assigned(kvm_tdx)))
return 0;
- /* For now large page isn't supported yet. */
- WARN_ON_ONCE(level != PG_LEVEL_4K);
err = tdh_mem_range_block(kvm_tdx->tdr_pa, gpa, tdx_level, &out);
if (unlikely(err == TDX_ERROR_SEPT_BUSY))
return -EAGAIN;
@@ -3286,6 +3308,7 @@ int __init tdx_hardware_setup(struct kvm_x86_ops *x86_ops)
x86_ops->link_private_spt = tdx_sept_link_private_spt;
x86_ops->free_private_spt = tdx_sept_free_private_spt;
+ x86_ops->split_private_spt = tdx_sept_split_private_spt;
x86_ops->set_private_spte = tdx_sept_set_private_spte;
x86_ops->remove_private_spte = tdx_sept_remove_private_spte;
x86_ops->zap_private_spte = tdx_sept_zap_private_spte;
diff --git a/arch/x86/kvm/vmx/tdx_arch.h b/arch/x86/kvm/vmx/tdx_arch.h
index eb62b8804cb4..e663abaa3aa0 100644
--- a/arch/x86/kvm/vmx/tdx_arch.h
+++ b/arch/x86/kvm/vmx/tdx_arch.h
@@ -21,6 +21,7 @@
#define TDH_MNG_CREATE 9
#define TDH_VP_CREATE 10
#define TDH_MNG_RD 11
+#define TDH_MEM_PAGE_DEMOTE 15
#define TDH_MR_EXTEND 16
#define TDH_MR_FINALIZE 17
#define TDH_VP_FLUSH 18
diff --git a/arch/x86/kvm/vmx/tdx_errno.h b/arch/x86/kvm/vmx/tdx_errno.h
index bb093e292fef..d08b4d14e57b 100644
--- a/arch/x86/kvm/vmx/tdx_errno.h
+++ b/arch/x86/kvm/vmx/tdx_errno.h
@@ -11,6 +11,7 @@
*/
#define TDX_NON_RECOVERABLE_VCPU 0x4000000100000000ULL
#define TDX_INTERRUPTED_RESUMABLE 0x8000000300000000ULL
+#define TDX_INTERRUPTED_RESTARTABLE 0x8000000400000000ULL
#define TDX_OPERAND_INVALID 0xC000010000000000ULL
#define TDX_OPERAND_BUSY 0x8000020000000000ULL
#define TDX_PREVIOUS_TLB_EPOCH_BUSY 0x8000020100000000ULL
diff --git a/arch/x86/kvm/vmx/tdx_ops.h b/arch/x86/kvm/vmx/tdx_ops.h
index ce722e917d14..772e2e7d61e7 100644
--- a/arch/x86/kvm/vmx/tdx_ops.h
+++ b/arch/x86/kvm/vmx/tdx_ops.h
@@ -249,6 +249,19 @@ static inline u64 tdh_mng_rd(hpa_t tdr, u64 field, struct tdx_module_args *out)
return tdx_seamcall(TDH_MNG_RD, &in, out);
}
+static inline u64 tdh_mem_page_demote(hpa_t tdr, gpa_t gpa, int level, hpa_t page,
+ struct tdx_module_args *out)
+{
+ struct tdx_module_args in = {
+ .rcx = gpa | level,
+ .rdx = tdr,
+ .r8 = page,
+ };
+
+ tdx_clflush_page(page, PG_LEVEL_4K);
+ return tdx_seamcall_sept(TDH_MEM_PAGE_DEMOTE, &in, out);
+}
+
static inline u64 tdh_mr_extend(hpa_t tdr, gpa_t gpa,
struct tdx_module_args *out)
{
--
2.25.1
Powered by blists - more mailing lists