linux-kernel - [PATCH 11/12] KVM: MMU: simplify last_pte

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Date:	Wed, 24 Feb 2016 14:17:52 +0100
From:	Paolo Bonzini <pbonzini@...hat.com>
To:	linux-kernel@...r.kernel.org, kvm@...r.kernel.org
Cc:	yoshikawa_takuya_b1@....ntt.co.jp, guangrong.xiao@...ux.intel.com,
	mtosatti@...hat.com
Subject: [PATCH 11/12] KVM: MMU: simplify last_pte_bitmap

Branch-free code is fun and everybody knows how much Avi loves it,
but last_pte_bitmap takes it a bit to the extreme.  A logical | is
still branch-free and simpler than building the whole truth table into
last_pte_bitmap.

Signed-off-by: Paolo Bonzini <pbonzini@...hat.com>
---
 arch/x86/include/asm/kvm_host.h |  6 +-----
 arch/x86/kvm/mmu.c              | 40 +++++++++++++++++++++++++---------------
 2 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 7b5459982433..3b4c0b98f285 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -338,11 +338,7 @@ struct kvm_mmu {
 
 	struct rsvd_bits_validate guest_rsvd_check;
 
-	/*
-	 * Bitmap: bit set = last pte in walk
-	 * index[0:1]: level (zero-based)
-	 * index[2]: pte.ps
-	 */
+	/* bit [7-n] = can have large pages at (one-based) level n */
 	u8 last_pte_bitmap;
 
 	bool nx;
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 6d47b5c43246..6bf8c36b681b 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -3566,13 +3566,21 @@ static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
 	return false;
 }
 
-static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)
+static inline bool is_last_gpte(struct kvm_mmu *mmu,
+				unsigned level, unsigned gpte)
 {
-	unsigned index;
+	unsigned m1, m2;
+
+	/* PT_PAGE_TABLE_LEVEL always terminates.  m1 has bit 7 set iff
+	 * level <= PT_PAGE_TABLE_LEVEL, which for our purpose means
+	 * level == PT_PAGE_TABLE_LEVEL (level is never zero).  */
+	m1 = level - (PT_PAGE_TABLE_LEVEL + 1);
 
-	index = level - 1;
-	index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
-	return mmu->last_pte_bitmap & (1 << index);
+	/* last_pte_bitmap is built so that shifting it left by level
+	 * produces a 1-bit mask for gpte's PT_PAGE_SIZE_MASK bit.
+	 */
+	m2 = gpte & (mmu->last_pte_bitmap << level);
+	return (m1 | m2) & PT_PAGE_SIZE_MASK;
 }
 
 #define PTTYPE_EPT 18 /* arbitrary */
@@ -3848,16 +3856,18 @@ static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
 {
 	u8 map;
 	unsigned level, root_level = mmu->root_level;
-	const unsigned ps_set_index = 1 << 2;  /* bit 2 of index: ps */
-
-	if (root_level == PT32E_ROOT_LEVEL)
-		--root_level;
-	/* PT_PAGE_TABLE_LEVEL always terminates */
-	map = 1 | (1 << ps_set_index);
-	for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
-		if (level <= PT_PDPE_LEVEL
-		    && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
-			map |= 1 << (ps_set_index | (level - 1));
+
+	map = 0;
+	if (root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)) {
+		/* Apart from PSE, there are never large pages at the top level
+		 * (512GB for 64-bit, 1GB for PAE).
+		 */
+		if (root_level != PT32_ROOT_LEVEL)
+			--root_level;
+
+		BUILD_BUG_ON(PT_PAGE_SIZE_MASK > 255);
+		for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level)
+			map |= PT_PAGE_SIZE_MASK >> level;
 	}
 	mmu->last_pte_bitmap = map;
 }
-- 
1.8.3.1