Message-Id: <1443767639-62457-4-git-send-email-fenghua.yu@intel.com>
Date:	Thu,  1 Oct 2015 23:33:57 -0700
From:	Fenghua Yu <fenghua.yu@...el.com>
To:	"H Peter Anvin" <hpa@...or.com>, "Ingo Molnar" <mingo@...hat.com>,
	"Thomas Gleixner" <tglx@...utronix.de>,
	"Peter Zijlstra" <peterz@...radead.org>
Cc:	"linux-kernel" <linux-kernel@...r.kernel.org>,
	"x86" <x86@...nel.org>, "Fenghua Yu" <fenghua.yu@...el.com>,
	"Vikas Shivappa" <vikas.shivappa@...ux.intel.com>
Subject: [PATCH V2 3/5] x86/intel_rdt: Class of service and capacity bitmask management for CDP

Add support to manage CLOSid(CLass Of Service id) and capacity
bitmask(cbm) for code data prioritization(CDP).

Closid management includes changes to allocating, freeing closid and
closid_get and closid_put and changes to closid availability map during
mode switch. CDP has a separate cbm for code and data.

Once the mode is switched to cdp, the number of CLOSids is halved.
The clos_cbm_table is reused to store dcache_cbm and
icache_cbm entries and the index is calculated as below:
index of dcache_cbm for a CLOSid 'n' = (n << 1)
index of icache_cbm for a CLOSid 'n' = (n << 1) + 1.
The offset of the IA32_L3_MASK_n MSRs from the base is related to the
CLOSid 'n' in the same way.

In other words, each closid is mapped to a (dcache_cbm, icache_cbm) pair
when cdp mode is enabled. Support for setting up of the
clos_cbm_table and closmap when the mode switch happens is added.

Signed-off-by: Vikas Shivappa <vikas.shivappa@...ux.intel.com>
Signed-off-by: Fenghua Yu <fenghua.yu@...el.com>
---
 arch/x86/kernel/cpu/intel_rdt.c | 194 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 190 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/cpu/intel_rdt.c b/arch/x86/kernel/cpu/intel_rdt.c
index 54a8e29..eab9d73 100644
--- a/arch/x86/kernel/cpu/intel_rdt.c
+++ b/arch/x86/kernel/cpu/intel_rdt.c
@@ -30,7 +30,13 @@
 #include <asm/intel_rdt.h>
 
 /*
- * cctable maintains 1:1 mapping between CLOSid and cache bitmask.
+ * During cache alloc mode cctable maintains 1:1 mapping between
+ * CLOSid and l3_cbm.
+ *
+ * During CDP mode, the cctable maintains a 1:2 mapping between the closid
+ * and (dcache_cbm, icache_cbm) pair.
+ * index of a dcache_cbm for CLOSid 'n' = n << 1.
+ * index of a icache_cbm for CLOSid 'n' = n << 1 + 1
  */
 static struct clos_cbm_table *cctable;
 /*
@@ -53,6 +59,13 @@ static bool cdp_enabled;
 
 #define __DCBM_TABLE_INDEX(x)	(x << 1)
 #define __ICBM_TABLE_INDEX(x)	((x << 1) + 1)
+#define __DCBM_MSR_INDEX(x)			\
+	CBM_FROM_INDEX(__DCBM_TABLE_INDEX(x))
+#define __ICBM_MSR_INDEX(x)			\
+	CBM_FROM_INDEX(__ICBM_TABLE_INDEX(x))
+
+#define DCBM_TABLE_INDEX(x)	(x << cdp_enabled)
+#define ICBM_TABLE_INDEX(x)	((x << cdp_enabled) + cdp_enabled)
 
 static struct intel_rdt rdt_root_group;
 #define rdt_for_each_child(pos_css, parent_ir)		\
@@ -133,9 +146,12 @@ static inline void closid_tasks_sync(void)
 	on_each_cpu_mask(cpu_online_mask, __intel_rdt_sched_in, NULL, 1);
 }
 
+/*
+ * When cdp mode is enabled, refcnt is maintained in the dcache_cbm entry.
+ */
 static inline void closid_get(u32 closid)
 {
-	struct clos_cbm_table *cct = &cctable[closid];
+	struct clos_cbm_table *cct = &cctable[DCBM_TABLE_INDEX(closid)];
 
 	lockdep_assert_held(&rdt_group_mutex);
 
@@ -165,7 +181,7 @@ static int closid_alloc(u32 *closid)
 static inline void closid_free(u32 closid)
 {
 	clear_bit(closid, cconfig.closmap);
-	cctable[closid].l3_cbm = 0;
+	cctable[DCBM_TABLE_INDEX(closid)].l3_cbm = 0;
 
 	if (WARN_ON(!cconfig.closids_used))
 		return;
@@ -175,7 +191,7 @@ static inline void closid_free(u32 closid)
 
 static void closid_put(u32 closid)
 {
-	struct clos_cbm_table *cct = &cctable[closid];
+	struct clos_cbm_table *cct = &cctable[DCBM_TABLE_INDEX(closid)];
 
 	lockdep_assert_held(&rdt_group_mutex);
 	if (WARN_ON(!cct->clos_refcnt))
@@ -259,6 +275,30 @@ static bool cbm_search(unsigned long cbm, u32 *closid)
 	return false;
 }
 
+static bool cbm_pair_search(unsigned long dcache_cbm, unsigned long icache_cbm,
+			    u32 *closid)
+{
+	u32 maxid = cconfig.max_closid;
+	unsigned long dcbm, icbm;
+	u32 i, dindex, iindex;
+
+	for (i = 0; i < maxid; i++) {
+		dindex = __DCBM_TABLE_INDEX(i);
+		iindex = __ICBM_TABLE_INDEX(i);
+		dcbm = cctable[dindex].l3_cbm;
+		icbm = cctable[iindex].l3_cbm;
+
+		if (cctable[dindex].clos_refcnt &&
+		    bitmap_equal(&dcache_cbm, &dcbm, MAX_CBM_LENGTH) &&
+		    bitmap_equal(&icache_cbm, &icbm, MAX_CBM_LENGTH)) {
+			*closid = i;
+			return true;
+		}
+	}
+
+	return false;
+}
+
 static void closcbm_map_dump(void)
 {
 	u32 i;
@@ -289,6 +329,93 @@ static inline void msr_update_all(int msr, u64 val)
 	on_each_cpu_mask(&rdt_cpumask, msr_cpu_update, &info, 1);
 }
 
+/*
+ * clos_cbm_table_df() - Defragments the clos_cbm_table entries
+ * @ct: The clos_cbm_table to which the defragmented entries are copied.
+ *
+ * The max entries in ct is never > original max closids / 2.
+ */
+static void clos_cbm_table_df(struct clos_cbm_table *ct)
+{
+	u32 orig_maxid = boot_cpu_data.x86_cache_max_closid;
+	int i, j;
+
+	for (i = 0, j = 0; i < orig_maxid; i++) {
+		if (cctable[i].clos_refcnt) {
+			ct[j] = cctable[i];
+			set_bit(j, cconfig.closmap);
+			j++;
+		}
+	}
+}
+
+/*
+ * post_cdp_enable() - This sets up the clos_cbm_table and
+ * IA32_L3_MASK_n MSRs before starting to use CDP.
+ *
+ * The existing l3_cbm entries are retained as dcache_cbm and
+ * icache_cbm entries. The IA32_L3_QOS_n MSRs are also updated
+ * as they were reset to all 1s before mode change.
+ */
+static int post_cdp_enable(void)
+{
+	u32 orig_maxid = boot_cpu_data.x86_cache_max_closid;
+	u32 maxid = cconfig.max_closid;
+	int size, dindex, iindex, i;
+	struct clos_cbm_table *ct;
+
+	maxid = cconfig.max_closid;
+	size = maxid * sizeof(struct clos_cbm_table);
+	ct = kzalloc(size, GFP_KERNEL);
+	if (!ct)
+		return -ENOMEM;
+
+	bitmap_zero(cconfig.closmap, orig_maxid);
+	clos_cbm_table_df(ct);
+
+	for (i = 0; i < maxid; i++) {
+		if (ct[i].clos_refcnt) {
+			msr_update_all(__DCBM_MSR_INDEX(i), ct[i].l3_cbm);
+			msr_update_all(__ICBM_MSR_INDEX(i), ct[i].l3_cbm);
+		}
+		dindex = __DCBM_TABLE_INDEX(i);
+		iindex = __ICBM_TABLE_INDEX(i);
+		cctable[dindex] = cctable[iindex] = ct[i];
+	}
+	kfree(ct);
+
+	return 0;
+}
+
+/*
+ * post_cdp_disable() - Set the state of closmap and clos_cbm_table
+ * before using the cache alloc mode.
+ *
+ * The existing dcache_cbm entries are retained as l3_cbm entries.
+ * The IA32_L3_QOS_n MSRs are also updated
+ * as they were reset to all 1s before mode change.
+ */
+static void post_cdp_disable(void)
+{
+	int dindex, maxid, i;
+
+	maxid = cconfig.max_closid >> 1;
+	for (i = 0; i < maxid; i++) {
+		dindex = __DCBM_TABLE_INDEX(i);
+		if (cctable[dindex].clos_refcnt)
+			msr_update_all(CBM_FROM_INDEX(i),
+					 cctable[dindex].l3_cbm);
+
+		cctable[i] = cctable[dindex];
+	}
+
+	/*
+	 * We updated half of the clos_cbm_table entries, initialize the
+	 * rest of the clos_cbm_table entries.
+	 */
+	memset(&cctable[maxid], 0, maxid * sizeof(struct clos_cbm_table));
+}
+
 static bool code_data_mask_equal(void)
 {
 	int i, dindex, iindex;
@@ -304,6 +431,65 @@ static bool code_data_mask_equal(void)
 	return true;
 }
 
+static void __switch_mode_cdp(bool cdpenable)
+{
+	u32 max_cbm_len = boot_cpu_data.x86_cache_max_cbm_len;
+	u32 orig_maxid = boot_cpu_data.x86_cache_max_closid;
+	u32 max_cbm, i;
+
+	max_cbm = (1ULL << max_cbm_len) - 1;
+	for (i = 0; i < orig_maxid; i++)
+		msr_update_all(CBM_FROM_INDEX(i), max_cbm);
+
+	msr_update_all(MSR_IA32_PQOS_CFG, cdpenable);
+
+	if (cdpenable)
+		cconfig.max_closid = cconfig.max_closid >> 1;
+	else
+		cconfig.max_closid = cconfig.max_closid << 1;
+	cdp_enabled = cdpenable;
+}
+
+/*
+ * switch_mode_cdp() - switch between legacy cache alloc and cdp modes
+ * @cdpmode: '0' to disable cdp and '1' to enable cdp
+ *
+ * cdp is enabled only when the number of closids used is less than half
+ * of available closids. Switch to legacy cache alloc mode when
+ * for each (dcache_cbm,icache_cbm) pair, the dcache_cbm = icache_cbm.
+ */
+static int switch_mode_cdp(bool cdpenable)
+{
+	struct cpuinfo_x86 *c = &boot_cpu_data;
+	u32 maxid = cconfig.max_closid;
+	int err = 0;
+
+	lockdep_assert_held(&rdt_group_mutex);
+
+	if (!cpu_has(c, X86_FEATURE_CDP_L3) || cdpenable == cdp_enabled)
+		return -EINVAL;
+
+	if ((cdpenable && (cconfig.closids_used >= (maxid >> 1))) ||
+	     (!cdpenable && !code_data_mask_equal()))
+		return -ENOSPC;
+
+	__switch_mode_cdp(cdpenable);
+
+	/*
+	 * After mode switch to cdp, the index for IA32_L3_MASK_n from the base
+	 * for a CLOSid 'n' is:
+	 * dcache_cbm_index (n) = (n << 1)
+	 * icache_cbm_index (n) = (n << 1) +1
+	*/
+	if (cdpenable)
+		err = post_cdp_enable();
+	else
+		post_cdp_disable();
+	closcbm_map_dump();
+
+	return err;
+}
+
 static inline bool rdt_cpumask_update(int cpu)
 {
 	cpumask_and(&tmp_cpumask, &rdt_cpumask, topology_core_cpumask(cpu));
-- 
1.8.1.2

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

Powered by blists - more mailing lists