linux-cxl.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Ben Widawsky <ben.widawsky@intel.com>
To: linux-cxl@vger.kernel.org, Chet Douglas <chet.r.douglas@intel.com>
Cc: Ben Widawsky <ben.widawsky@intel.com>,
	Alison Schofield <alison.schofield@intel.com>,
	Dan Williams <dan.j.williams@intel.com>,
	Ira Weiny <ira.weiny@intel.com>,
	Jonathan Cameron <Jonathan.Cameron@Huawei.com>,
	Vishal Verma <vishal.l.verma@intel.com>
Subject: [RFC PATCH 21/27] cxl/region: Implement XHB verification
Date: Fri, 15 Oct 2021 22:15:25 -0700	[thread overview]
Message-ID: <20211016051531.622613-22-ben.widawsky@intel.com> (raw)
In-Reply-To: <20211016051531.622613-1-ben.widawsky@intel.com>

Cross host bridge verification primarily determines if the requested
interleave ordering can be achieved by the root decoder, which isn't as
programmable as other decoders.

The algorithm implemented here is based on the CXL Type 3 Memory Device
Software Guide, chapter 2.13.14

Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
---
 .clang-format        |  1 +
 drivers/cxl/region.c | 81 +++++++++++++++++++++++++++++++++++++++++++-
 drivers/cxl/trace.h  |  3 ++
 3 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/.clang-format b/.clang-format
index cb7c46371465..55f628f21722 100644
--- a/.clang-format
+++ b/.clang-format
@@ -169,6 +169,7 @@ ForEachMacros:
   - 'for_each_cpu_and'
   - 'for_each_cpu_not'
   - 'for_each_cpu_wrap'
+  - 'for_each_cxl_decoder_target'
   - 'for_each_cxl_endpoint'
   - 'for_each_dapm_widgets'
   - 'for_each_dev_addr'
diff --git a/drivers/cxl/region.c b/drivers/cxl/region.c
index cd9d696fdd4a..024d562ad161 100644
--- a/drivers/cxl/region.c
+++ b/drivers/cxl/region.c
@@ -30,6 +30,11 @@
 	for (idx = 0, ep = (region)->targets[idx]; idx < region_ways(region);  \
 	     idx++, ep = (region)->targets[idx])
 
+#define for_each_cxl_decoder_target(target, decoder, idx)                      \
+	for (idx = 0, target = (decoder)->target[idx];                         \
+	     idx < (decoder)->nr_targets;                                      \
+	     idx++, target = (decoder)->target[idx])
+
 #define region_ways(region) ((region)->eniw)
 #define region_ig(region) (ilog2((region)->ig))
 
@@ -165,6 +170,28 @@ static bool qtg_match(const struct cxl_decoder *cfmws,
 	return true;
 }
 
+static int get_unique_hostbridges(const struct cxl_region *region,
+				  struct cxl_port **hbs)
+{
+	struct cxl_memdev *ep;
+	int i, hb_count = 0;
+
+	for_each_cxl_endpoint(ep, region, i) {
+		struct cxl_port *hb = get_hostbridge(ep);
+		bool found = false;
+		int j;
+
+		for (j = 0; j < hb_count; j++) {
+			if (hbs[j] == hb)
+				found = true;
+		}
+		if (!found)
+			hbs[hb_count++] = hb;
+	}
+
+	return hb_count;
+}
+
 /**
  * region_xhb_config_valid() - determine cross host bridge validity
  * @cfmws: The CFMWS to check against
@@ -178,7 +205,59 @@ static bool qtg_match(const struct cxl_decoder *cfmws,
 static bool region_xhb_config_valid(const struct cxl_region *region,
 				    const struct cxl_decoder *cfmws)
 {
-	/* TODO: */
+	struct cxl_port *hbs[CXL_DECODER_MAX_INTERLEAVE];
+	int cfmws_ig, i;
+	struct cxl_dport *target;
+
+	/* Are all devices in this region on the same CXL host bridge */
+	if (get_unique_hostbridges(region, hbs) == 1)
+		return true;
+
+	cfmws_ig = cfmws->interleave_granularity;
+
+	/* CFMWS.HBIG >= Device.Label.IG */
+	if (cfmws_ig < region_ig(region)) {
+		trace_xhb_valid(region,
+				"granularity does not support the region interleave granularity\n");
+		return false;
+	}
+
+	/* ((2^(CFMWS.HBIG - Device.RLabel.IG) * (2^CFMWS.ENIW)) > Device.RLabel.NLabel) */
+	if (1 << (cfmws_ig - region_ig(region)) * (1 << cfmws->interleave_ways) >
+	    region_ways(region)) {
+		trace_xhb_valid(region,
+				"granularity to device granularity ratio requires a larger number of devices than currently configured");
+		return false;
+	}
+
+	/* Check that endpoints are hooked up in the correct order */
+	for_each_cxl_decoder_target(target, cfmws, i) {
+		struct cxl_memdev *endpoint = region->targets[i];
+
+		if (get_hostbridge(endpoint) != target->port) {
+			trace_xhb_valid(region, "device ordering bad\n");
+			return false;
+		}
+	}
+
+	/*
+	 * CFMWS.InterleaveTargetList[n] must contain all devices, x where:
+	 *	(Device[x],RegionLabel.Position >> (CFMWS.HBIG -
+	 *	Device[x].RegionLabel.InterleaveGranularity)) &
+	 *	((2^CFMWS.ENIW) - 1) = n
+	 *
+	 * Linux notes: All devices are known to have the same interleave
+	 * granularity at this point.
+	 */
+	for_each_cxl_decoder_target(target, cfmws, i) {
+		if (((i >> (cfmws_ig - region_ig(region)))) &
+		    (((1 << cfmws->interleave_ways) - 1) != target->port_id)) {
+			trace_xhb_valid(region,
+					"One or more devices are not connected to the correct hostbridge.");
+			return false;
+		}
+	}
+
 	return true;
 }
 
diff --git a/drivers/cxl/trace.h b/drivers/cxl/trace.h
index a53f00ba5d0e..4de47d1111ac 100644
--- a/drivers/cxl/trace.h
+++ b/drivers/cxl/trace.h
@@ -38,6 +38,9 @@ DEFINE_EVENT(cxl_region_template, sanitize_failed,
 DEFINE_EVENT(cxl_region_template, allocation_failed,
 	     TP_PROTO(const struct cxl_region *region, char *status),
 	     TP_ARGS(region, status));
+DEFINE_EVENT(cxl_region_template, xhb_valid,
+	     TP_PROTO(const struct cxl_region *region, char *status),
+	     TP_ARGS(region, status));
 
 #endif /* if !defined (__CXL_TRACE_H__) || defined(TRACE_HEADER_MULTI_READ) */
 
-- 
2.33.1


  parent reply	other threads:[~2021-10-16  5:15 UTC|newest]

Thread overview: 32+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-10-16  5:15 [RFC PATCH 00/27] CXL Region Creation / HDM decoder programming Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 01/27] cxl: Rename CXL_MEM to CXL_PCI Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 02/27] cxl: Move register block enumeration to core Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 03/27] cxl/acpi: Map component registers for Root Ports Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 04/27] cxl: Add helper for new drivers Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 05/27] cxl/core: Convert decoder range to resource Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 06/27] cxl: Introduce endpoint decoders Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 07/27] cxl/port: Introduce a port driver Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 08/27] cxl/acpi: Map single port host bridge component registers Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 09/27] cxl/core: Store global list of root ports Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 10/27] cxl/acpi: Rescan bus at probe completion Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 11/27] cxl/core: Store component register base for memdevs Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 12/27] cxl: Flesh out register names Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 13/27] cxl/core: Introduce API to scan switch ports Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 14/27] cxl: Introduce cxl_mem driver Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 15/27] cxl: Disable switch hierarchies for now Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 16/27] cxl/region: Add region creation ABI Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 17/27] cxl/region: Introduce concept of region configuration Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 18/27] cxl/region: Introduce a cxl_region driver Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 19/27] cxl/acpi: Handle address space allocation Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 20/27] cxl/region: Address " Ben Widawsky
2021-10-16  5:15 ` Ben Widawsky [this message]
2021-10-16  5:15 ` [RFC PATCH 22/27] cxl/region: HB port config verification Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 23/27] cxl/region: Record host bridge target list Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 24/27] cxl/mem: Store the endpoint's uport Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 25/27] cxl/region: Gather HDM decoder resources Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 26/27] cxl: Program decoders for regions Ben Widawsky
2021-10-18 23:30   ` [RFC v2 " Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 27/27] dont-merge: My QEMU CFMWS is wrong Ben Widawsky
2021-10-18 23:36   ` Ben Widawsky
2021-10-18  0:15 ` [RFC PATCH 00/27] CXL Region Creation / HDM decoder programming Ben Widawsky
2021-10-21 14:29 ` Ben Widawsky

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20211016051531.622613-22-ben.widawsky@intel.com \
    --to=ben.widawsky@intel.com \
    --cc=Jonathan.Cameron@Huawei.com \
    --cc=alison.schofield@intel.com \
    --cc=chet.r.douglas@intel.com \
    --cc=dan.j.williams@intel.com \
    --cc=ira.weiny@intel.com \
    --cc=linux-cxl@vger.kernel.org \
    --cc=vishal.l.verma@intel.com \
    --subject='Re: [RFC PATCH 21/27] cxl/region: Implement XHB verification' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).