>From 353ab1ff93a483154273cd2ded85e5f2190bcedb Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Mon, 15 Sep 2025 14:36:28 -0700 Subject: [PATCH] drm/xe: Move rebar support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create a xe_pci_rebar.c to contain the logic about resizing the lmembar. Trying to do a resize later inside xe_vram is way too late. At that time the BAR0 will already be mapped for register access and the driver would need to remap to continue working. All the rebar logic is more pci-specific than xe-specific and can be done very early in the probe sequence, resulting in messages like below, [] xe 0000:03:00.0: vgaarb: deactivate vga console [] xe 0000:03:00.0: [drm] Attempting to resize bar from 8192MiB -> 16384MiB [] xe 0000:03:00.0: BAR 0 [mem 0x83000000-0x83ffffff 64bit]: releasing [] xe 0000:03:00.0: BAR 2 [mem 0x4000000000-0x41ffffffff 64bit pref]: releasing [] pcieport 0000:02:01.0: bridge window [mem 0x4000000000-0x41ffffffff 64bit pref]: releasing [] pcieport 0000:01:00.0: bridge window [mem 0x4000000000-0x41ffffffff 64bit pref]: releasing [] pcieport 0000:01:00.0: bridge window [mem 0x4000000000-0x43ffffffff 64bit pref]: assigned [] pcieport 0000:02:01.0: bridge window [mem 0x4000000000-0x43ffffffff 64bit pref]: assigned [] xe 0000:03:00.0: BAR 2 [mem 0x4000000000-0x43ffffffff 64bit pref]: assigned [] xe 0000:03:00.0: BAR 0 [mem 0x83000000-0x83ffffff 64bit]: assigned [] pcieport 0000:00:01.0: PCI bridge to [bus 01-04] [] pcieport 0000:00:01.0: bridge window [mem 0x83000000-0x840fffff] [] pcieport 0000:00:01.0: bridge window [mem 0x4000000000-0x44007fffff 64bit pref] [] pcieport 0000:01:00.0: PCI bridge to [bus 02-04] [] pcieport 0000:01:00.0: bridge window [mem 0x83000000-0x840fffff] [] pcieport 0000:01:00.0: bridge window [mem 0x4000000000-0x43ffffffff 64bit pref] [] pcieport 0000:02:01.0: PCI bridge to [bus 03] [] pcieport 0000:02:01.0: bridge window [mem 0x83000000-0x83ffffff] [] pcieport 0000:02:01.0: bridge window [mem 0x4000000000-0x43ffffffff 64bit pref] [] xe 0000:03:00.0: [drm] BAR2 resized to 16384M [] xe 0000:03:00.0: [drm:xe_pci_probe [xe]] BATTLEMAGE e221:0000 dgfx:1 gfx:Xe2_HPG (20.02) ... As shown above, it happens even before we try to read any register for platform identification. With that move, the BAR0 won't be in use by the driver and it can be released prior to the resize, avoiding the shortcomings of how pbus_reassign_bridge_resources() handles that case. Improving pbus_reassign_bridge_resources() is something to be done, but it wouldn't be able to fix this case since xe was already using BAR0. Cc: Ilpo Järvinen Cc: # 6.12+ Link: https://lore.kernel.org/intel-xe/fafda2a3-fc63-ce97-d22b-803f771a4d19@linux.intel.com Signed-off-by: Lucas De Marchi --- Tested with some BAR resizes prior to loading xe and also with a SBR which makes the BMG's BAR2 to come back as 256M when rescanning the bus --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_pci.c | 3 + drivers/gpu/drm/xe/xe_pci_rebar.c | 123 ++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_pci_rebar.h | 13 ++++ drivers/gpu/drm/xe/xe_vram.c | 98 +----------------------- 5 files changed, 141 insertions(+), 97 deletions(-) create mode 100644 drivers/gpu/drm/xe/xe_pci_rebar.c create mode 100644 drivers/gpu/drm/xe/xe_pci_rebar.h diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 7a065c98a3b85..5a66d7a53d0db 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -95,6 +95,7 @@ xe-y += xe_bb.o \ xe_observation.o \ xe_pat.o \ xe_pci.o \ + xe_pci_rebar.o \ xe_pcode.o \ xe_pm.o \ xe_preempt_fence.o \ diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 701ba9baa9d7e..6cc5e7b6901a8 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -27,6 +27,7 @@ #include "xe_macros.h" #include "xe_mmio.h" #include "xe_module.h" +#include "xe_pci_rebar.h" #include "xe_pci_sriov.h" #include "xe_pci_types.h" #include "xe_pm.h" @@ -866,6 +867,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) return err; + xe_pci_rebar(xe); + err = xe_device_probe_early(xe); /* * In Boot Survivability mode, no drm card is exposed and driver diff --git a/drivers/gpu/drm/xe/xe_pci_rebar.c b/drivers/gpu/drm/xe/xe_pci_rebar.c new file mode 100644 index 0000000000000..3487845596d88 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci_rebar.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_pci_rebar.h" + +#include +#include + +#include + +#include "regs/xe_bars.h" +#include "xe_device_types.h" +#include "xe_module.h" + +#define BAR_SIZE_SHIFT 20 + +static void release_bars(struct pci_dev *pdev) +{ + int resno; + + for (resno = PCI_STD_RESOURCES; resno < PCI_STD_RESOURCE_END; resno++) { + if (pci_resource_len(pdev, resno)) + pci_release_resource(pdev, resno); + } +} + +static void resize_bar(struct xe_device *xe, int resno, resource_size_t size) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int bar_size = pci_rebar_bytes_to_size(size); + int ret; + + release_bars(pdev); + + ret = pci_resize_resource(pdev, resno, bar_size); + if (ret) { + drm_info(&xe->drm, "Failed to resize BAR%d to %dM (%pe). Consider enabling 'Resizable BAR' support in your BIOS\n", + resno, 1 << bar_size, ERR_PTR(ret)); + return; + } + + drm_info(&xe->drm, "BAR%d resized to %dM\n", resno, 1 << bar_size); +} + +void xe_pci_rebar(struct xe_device *xe) +{ + int force_vram_bar_size = xe_modparam.force_vram_bar_size; + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct pci_bus *root = pdev->bus; + resource_size_t current_size; + resource_size_t rebar_size; + struct resource *root_res; + int max_size, i; + u32 pci_cmd; + + /* gather some relevant info */ + current_size = pci_resource_len(pdev, LMEM_BAR); + + /* + * Handle force_vram_bar_size: + * - negative: resize is disabled + * - 0: try to resize to maximum possible + * - positive: resize to specific value + */ + + if (force_vram_bar_size < 0) + return; + + /* set to a specific size? */ + if (force_vram_bar_size) { + rebar_size = pci_rebar_bytes_to_size(force_vram_bar_size * + (resource_size_t)SZ_1M); + + if (!pci_rebar_size_supported(pdev, LMEM_BAR, rebar_size)) { + drm_info(&xe->drm, + "Requested size: %lluMiB is not supported by rebar sizes: 0x%llx. Leaving default: %lluMiB\n", + (u64)pci_rebar_size_to_bytes(rebar_size) >> 20, + pci_rebar_get_possible_sizes(pdev, LMEM_BAR), + (u64)current_size >> 20); + return; + } + + rebar_size = pci_rebar_size_to_bytes(rebar_size); + if (rebar_size == current_size) + return; + } else { + max_size = pci_rebar_get_max_size(pdev, LMEM_BAR); + if (max_size < 0) + return; + rebar_size = pci_rebar_size_to_bytes(max_size); + + /* only resize if larger than current */ + if (rebar_size <= current_size) + return; + } + + drm_info(&xe->drm, "Attempting to resize bar from %lluMiB -> %lluMiB\n", + (u64)current_size >> 20, (u64)rebar_size >> 20); + + while (root->parent) + root = root->parent; + + pci_bus_for_each_resource(root, root_res, i) { + if (root_res && root_res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && + (u64)root_res->start > 0x100000000ul) + break; + } + + if (!root_res) { + drm_info(&xe->drm, "Can't resize VRAM BAR - platform support is missing. Consider enabling 'Resizable BAR' support in your BIOS\n"); + return; + } + + pci_read_config_dword(pdev, PCI_COMMAND, &pci_cmd); + pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd & ~PCI_COMMAND_MEMORY); + + resize_bar(xe, LMEM_BAR, rebar_size); + + pci_assign_unassigned_bus_resources(pdev->bus); + pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd); +} diff --git a/drivers/gpu/drm/xe/xe_pci_rebar.h b/drivers/gpu/drm/xe/xe_pci_rebar.h new file mode 100644 index 0000000000000..c87aa58aee718 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci_rebar.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PCI_REBAR_H_ +#define _XE_PCI_REBAR_H_ + +struct xe_device; + +void xe_pci_rebar(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index f182326688106..989cf705f1b17 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -24,101 +24,6 @@ #include "xe_vram.h" #include "xe_vram_types.h" -static void -_resize_bar(struct xe_device *xe, int resno, resource_size_t size) -{ - struct pci_dev *pdev = to_pci_dev(xe->drm.dev); - int bar_size = pci_rebar_bytes_to_size(size); - int ret; - - if (pci_resource_len(pdev, resno)) - pci_release_resource(pdev, resno); - - ret = pci_resize_resource(pdev, resno, bar_size); - if (ret) { - drm_info(&xe->drm, "Failed to resize BAR%d to %dM (%pe). Consider enabling 'Resizable BAR' support in your BIOS\n", - resno, 1 << bar_size, ERR_PTR(ret)); - return; - } - - drm_info(&xe->drm, "BAR%d resized to %dM\n", resno, 1 << bar_size); -} - -/* - * if force_vram_bar_size is set, attempt to set to the requested size - * else set to maximum possible size - */ -static void resize_vram_bar(struct xe_device *xe) -{ - int force_vram_bar_size = xe_modparam.force_vram_bar_size; - struct pci_dev *pdev = to_pci_dev(xe->drm.dev); - struct pci_bus *root = pdev->bus; - resource_size_t current_size; - resource_size_t rebar_size; - struct resource *root_res; - int max_size, i; - u32 pci_cmd; - - /* gather some relevant info */ - current_size = pci_resource_len(pdev, LMEM_BAR); - - if (force_vram_bar_size < 0) - return; - - /* set to a specific size? */ - if (force_vram_bar_size) { - rebar_size = pci_rebar_bytes_to_size(force_vram_bar_size * - (resource_size_t)SZ_1M); - - if (!pci_rebar_size_supported(pdev, LMEM_BAR, rebar_size)) { - drm_info(&xe->drm, - "Requested size: %lluMiB is not supported by rebar sizes: 0x%llx. Leaving default: %lluMiB\n", - (u64)pci_rebar_size_to_bytes(rebar_size) >> 20, - pci_rebar_get_possible_sizes(pdev, LMEM_BAR), - (u64)current_size >> 20); - return; - } - - rebar_size = pci_rebar_size_to_bytes(rebar_size); - if (rebar_size == current_size) - return; - } else { - max_size = pci_rebar_get_max_size(pdev, LMEM_BAR); - if (max_size < 0) - return; - rebar_size = pci_rebar_size_to_bytes(max_size); - - /* only resize if larger than current */ - if (rebar_size <= current_size) - return; - } - - drm_info(&xe->drm, "Attempting to resize bar from %lluMiB -> %lluMiB\n", - (u64)current_size >> 20, (u64)rebar_size >> 20); - - while (root->parent) - root = root->parent; - - pci_bus_for_each_resource(root, root_res, i) { - if (root_res && root_res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && - (u64)root_res->start > 0x100000000ul) - break; - } - - if (!root_res) { - drm_info(&xe->drm, "Can't resize VRAM BAR - platform support is missing. Consider enabling 'Resizable BAR' support in your BIOS\n"); - return; - } - - pci_read_config_dword(pdev, PCI_COMMAND, &pci_cmd); - pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd & ~PCI_COMMAND_MEMORY); - - _resize_bar(xe, LMEM_BAR, rebar_size); - - pci_assign_unassigned_bus_resources(pdev->bus); - pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd); -} - static bool resource_is_valid(struct pci_dev *pdev, int bar) { if (!pci_resource_flags(pdev, bar)) @@ -142,8 +47,6 @@ static int determine_lmem_bar_size(struct xe_device *xe, struct xe_vram_region * return -ENXIO; } - resize_vram_bar(xe); - lmem_bar->io_start = pci_resource_start(pdev, LMEM_BAR); lmem_bar->io_size = pci_resource_len(pdev, LMEM_BAR); if (!lmem_bar->io_size) @@ -366,6 +269,7 @@ int xe_vram_probe(struct xe_device *xe) err = determine_lmem_bar_size(xe, &lmem_bar); if (err) return err; + drm_info(&xe->drm, "VISIBLE VRAM: %pa, %pa\n", &lmem_bar.io_start, &lmem_bar.io_size); remain_io_size = lmem_bar.io_size; -- 2.50.1