linux-cxl.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Ben Widawsky <ben.widawsky@intel.com>
To: linux-cxl@vger.kernel.org, Chet Douglas <chet.r.douglas@intel.com>
Cc: Ben Widawsky <ben.widawsky@intel.com>,
	Alison Schofield <alison.schofield@intel.com>,
	Dan Williams <dan.j.williams@intel.com>,
	Ira Weiny <ira.weiny@intel.com>,
	Jonathan Cameron <Jonathan.Cameron@Huawei.com>,
	Vishal Verma <vishal.l.verma@intel.com>
Subject: [RFC PATCH 25/27] cxl/region: Gather HDM decoder resources
Date: Fri, 15 Oct 2021 22:15:29 -0700	[thread overview]
Message-ID: <20211016051531.622613-26-ben.widawsky@intel.com> (raw)
In-Reply-To: <20211016051531.622613-1-ben.widawsky@intel.com>

Prepare for HDM decoder programming by iterating through all components
and obtaining a cxl_decoder for each.

Programming a CXL region to accept memory transactions over a set of
devices requires programming HDM decoders for every component that is
part of the hierarchy enabling the region (host bridges, switches, and
endpoints). For this to be possible, each of these components must have
an available HDM decoder, which is a limited hardware resource.

Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
---
 drivers/cxl/core/bus.c |  8 ++++++
 drivers/cxl/cxl.h      |  3 ++
 drivers/cxl/port.c     | 53 +++++++++++++++++++++++++++++++++++
 drivers/cxl/region.c   | 63 +++++++++++++++++++++++++++++++++++++++---
 drivers/cxl/region.h   |  4 +++
 5 files changed, 127 insertions(+), 4 deletions(-)

diff --git a/drivers/cxl/core/bus.c b/drivers/cxl/core/bus.c
index edde7df70fdf..6b0cd1cbd9f7 100644
--- a/drivers/cxl/core/bus.c
+++ b/drivers/cxl/core/bus.c
@@ -301,6 +301,14 @@ static bool is_endpoint_decoder(struct device *dev)
 	return dev->type == &cxl_decoder_endpoint_type;
 }
 
+bool is_cxl_decoder(struct device *dev)
+{
+	return dev->type == &cxl_decoder_switch_type ||
+	       dev->type == &cxl_decoder_endpoint_type ||
+	       dev->type == &cxl_decoder_root_type;
+}
+EXPORT_SYMBOL_GPL(is_cxl_decoder);
+
 bool is_root_decoder(struct device *dev)
 {
 	return dev->type == &cxl_decoder_root_type;
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index a2062e370c85..24e4a14b531c 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -331,11 +331,14 @@ int cxl_add_dport(struct cxl_port *port, struct device *dport, int port_id,
 struct cxl_dport *cxl_get_root_dport(struct device *dev);
 struct cxl_dport *cxl_find_dport_by_dev(struct cxl_port *port,
 					struct device *dev);
+struct cxl_decoder *cxl_pop_decoder(struct cxl_port *port);
+void cxl_push_decoder(struct cxl_decoder *cxld);
 
 struct cxl_decoder *to_cxl_decoder(struct device *dev);
 bool is_root_decoder(struct device *dev);
 struct cxl_decoder *cxl_decoder_alloc(struct cxl_port *port,
 				      unsigned int nr_targets);
+bool is_cxl_decoder(struct device *dev);
 int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map);
 int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld);
 
diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c
index 4d65560eb739..87b36565fa6a 100644
--- a/drivers/cxl/port.c
+++ b/drivers/cxl/port.c
@@ -35,6 +35,59 @@
  * presenting APIs to other drivers to utilize the decoders.
  */
 
+static int unused_decoder(struct device *dev, void *data)
+{
+	struct cxl_decoder *cxld;
+
+	if (!is_cxl_decoder(dev))
+		return 0;
+
+	if (dev_WARN_ONCE(dev, is_root_decoder(dev),
+			  "Root decoders can't be present"))
+		return 0;
+
+	cxld = to_cxl_decoder(dev);
+	if (cxld->flags & CXL_DECODER_F_EN)
+		return 0;
+
+	/*
+	 * Mark this decoder as enabled to prevent other entities from thinking
+	 * it's available.
+	 */
+	cxld->flags |= CXL_DECODER_F_EN;
+
+	return 1;
+}
+
+/**
+ * cxl_pop_decoder() - Obtains an available decoder resource
+ * @port: Owner of the decoder resource
+ */
+struct cxl_decoder *cxl_pop_decoder(struct cxl_port *port)
+{
+	struct device *cxldd;
+
+	cxldd = device_find_child(&port->dev, NULL, unused_decoder);
+	if (!cxldd)
+		return ERR_PTR(-EBUSY);
+
+	/* Keep the reference */
+
+	return to_cxl_decoder(cxldd);
+}
+EXPORT_SYMBOL_GPL(cxl_pop_decoder);
+
+/**
+ * cxl_push_decoder() - Restores decoder resource to the port
+ * @cxld: the decoder resource to replace
+ */
+void cxl_push_decoder(struct cxl_decoder *cxld)
+{
+	cxld->flags &= ~CXL_DECODER_F_EN;
+	put_device(&cxld->dev);
+}
+EXPORT_SYMBOL_GPL(cxl_push_decoder);
+
 struct cxl_port_data {
 	struct cxl_component_regs regs;
 
diff --git a/drivers/cxl/region.c b/drivers/cxl/region.c
index d5a326c7e369..8849633afb23 100644
--- a/drivers/cxl/region.c
+++ b/drivers/cxl/region.c
@@ -450,9 +450,32 @@ static struct cxl_decoder *find_cfmws(const struct cxl_region *region,
 	return NULL;
 }
 
+static void put_all_decoders(struct decoder_programming *p)
+{
+	int i;
+
+	for (i = 0; i < CXL_DECODER_MAX_INTERLEAVE; i++) {
+		if (p->hbs[i].cxld) {
+			cxl_push_decoder(p->hbs[i].cxld);
+			p->hbs[i].cxld = NULL;
+		}
+
+		if (p->ep_cxld[i]) {
+			cxl_push_decoder(p->ep_cxld[i]);
+			p->ep_cxld[i] = NULL;
+		}
+	}
+}
+
+static void release_decoders(void *p)
+{
+	put_all_decoders(p);
+}
+
 /**
  * gather_hdm_decoders() - Amass all HDM decoders in the hierarchy
  * @region: The region to be programmed
+ * @p: Programming state that will gather decoders
  *
  * Programming the hardware such that the correct set of devices receive the
  * correct memory traffic requires all connected components in the hierarchy to
@@ -461,10 +484,42 @@ static struct cxl_decoder *find_cfmws(const struct cxl_region *region,
  * Returns 0 if an HDM decoder was obtained for each component, else returns a
  * negative error code.
  */
-static int gather_hdm_decoders(const struct cxl_region *region)
+static int gather_hdm_decoders(const struct cxl_region *region, struct decoder_programming *p)
 {
-	/* TODO: */
-	return 0;
+	struct cxl_memdev *ep;
+	struct cxl_port *hbs[CXL_DECODER_MAX_INTERLEAVE];
+	int i, hb_count = get_unique_hostbridges(region, hbs);
+
+	for_each_cxl_endpoint(ep, region, i) {
+		struct cxl_port *port = ep->uport;
+
+		p->ep_cxld[i] = cxl_pop_decoder(port);
+		if (IS_ERR(p->ep_cxld[i])) {
+			int err = PTR_ERR(p->ep_cxld[i]);
+
+			p->ep_cxld[i] = NULL;
+			put_all_decoders(p);
+			return err;
+		}
+	}
+
+	/* TODO: Switches */
+
+	for (i = 0; i < hb_count; i++) {
+		struct cxl_port *hb = hbs[i];
+
+		p->hbs[i].cxld = cxl_pop_decoder(hb);
+		if (IS_ERR(p->hbs[i].cxld)) {
+			int err = PTR_ERR(p->hbs[i].cxld);
+
+			p->hbs[i].cxld = NULL;
+			put_all_decoders(p);
+			return err;
+		}
+	}
+
+	return devm_add_action_or_reset((struct device *)&region->dev,
+					release_decoders, p);
 }
 
 static int bind_region(const struct cxl_region *region)
@@ -513,7 +568,7 @@ static int cxl_region_probe(struct device *dev)
 	if (ours)
 		put_device(&ours->dev);
 
-	ret = gather_hdm_decoders(region);
+	ret = gather_hdm_decoders(region, &region->state);
 	if (ret)
 		return ret;
 
diff --git a/drivers/cxl/region.h b/drivers/cxl/region.h
index 51f442636364..4c0b94c3c001 100644
--- a/drivers/cxl/region.h
+++ b/drivers/cxl/region.h
@@ -23,6 +23,8 @@
  * @state.hbs: Host bridge state. One per hostbridge in the interleave set.
  * @state.hbs.rp_count: Count of root ports for this region
  * @state.hbs.rp_target_list: Ordered list of downstream root ports.
+ * @state.hbs.cxld: an available decoder to set up the programming.
+ * @state.ep_cxld: available decoders for endpoint programming.
  */
 struct cxl_region {
 	struct device dev;
@@ -44,7 +46,9 @@ struct cxl_region {
 		struct {
 			int rp_count;
 			struct cxl_dport *rp_target_list[CXL_DECODER_MAX_INTERLEAVE];
+			struct cxl_decoder *cxld;
 		} hbs[CXL_DECODER_MAX_INTERLEAVE];
+		struct cxl_decoder *ep_cxld[CXL_DECODER_MAX_INTERLEAVE];
 	} state;
 };
 
-- 
2.33.1


  parent reply	other threads:[~2021-10-16  5:15 UTC|newest]

Thread overview: 32+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-10-16  5:15 [RFC PATCH 00/27] CXL Region Creation / HDM decoder programming Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 01/27] cxl: Rename CXL_MEM to CXL_PCI Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 02/27] cxl: Move register block enumeration to core Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 03/27] cxl/acpi: Map component registers for Root Ports Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 04/27] cxl: Add helper for new drivers Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 05/27] cxl/core: Convert decoder range to resource Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 06/27] cxl: Introduce endpoint decoders Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 07/27] cxl/port: Introduce a port driver Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 08/27] cxl/acpi: Map single port host bridge component registers Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 09/27] cxl/core: Store global list of root ports Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 10/27] cxl/acpi: Rescan bus at probe completion Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 11/27] cxl/core: Store component register base for memdevs Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 12/27] cxl: Flesh out register names Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 13/27] cxl/core: Introduce API to scan switch ports Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 14/27] cxl: Introduce cxl_mem driver Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 15/27] cxl: Disable switch hierarchies for now Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 16/27] cxl/region: Add region creation ABI Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 17/27] cxl/region: Introduce concept of region configuration Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 18/27] cxl/region: Introduce a cxl_region driver Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 19/27] cxl/acpi: Handle address space allocation Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 20/27] cxl/region: Address " Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 21/27] cxl/region: Implement XHB verification Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 22/27] cxl/region: HB port config verification Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 23/27] cxl/region: Record host bridge target list Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 24/27] cxl/mem: Store the endpoint's uport Ben Widawsky
2021-10-16  5:15 ` Ben Widawsky [this message]
2021-10-16  5:15 ` [RFC PATCH 26/27] cxl: Program decoders for regions Ben Widawsky
2021-10-18 23:30   ` [RFC v2 " Ben Widawsky
2021-10-16  5:15 ` [RFC PATCH 27/27] dont-merge: My QEMU CFMWS is wrong Ben Widawsky
2021-10-18 23:36   ` Ben Widawsky
2021-10-18  0:15 ` [RFC PATCH 00/27] CXL Region Creation / HDM decoder programming Ben Widawsky
2021-10-21 14:29 ` Ben Widawsky

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20211016051531.622613-26-ben.widawsky@intel.com \
    --to=ben.widawsky@intel.com \
    --cc=Jonathan.Cameron@Huawei.com \
    --cc=alison.schofield@intel.com \
    --cc=chet.r.douglas@intel.com \
    --cc=dan.j.williams@intel.com \
    --cc=ira.weiny@intel.com \
    --cc=linux-cxl@vger.kernel.org \
    --cc=vishal.l.verma@intel.com \
    --subject='Re: [RFC PATCH 25/27] cxl/region: Gather HDM decoder resources' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).