From c093fd7539cb8bd6e76f84ad2c5f13af21e68d28 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Sat, 4 Jul 2026 01:41:43 +0300 Subject: drm/bridge: Replace outdated forward declaration The drm_bridge.h header forward-declares struct edid, whose last usage in the file was removed in commit 27b8f91c08d9 ("drm/bridge: remove ->get_edid callback"). Commit 11f6c4b1b259 ("drm/bridge: Add connector-related bridge operations and data") then introduced usage of struct drm_edid, without a corresponding forward declaration. Fix those two issues by replacing the struct edid forward declaration with struct drm_edid. Signed-off-by: Laurent Pinchart Reviewed-by: Luca Ceresoli Link: https://patch.msgid.link/20260703224143.3886069-1-laurent.pinchart+renesas@ideasonboard.com Signed-off-by: Luca Ceresoli --- include/drm/drm_bridge.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/drm') diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h index 3ac84ed57ab2..58fff047f43b 100644 --- a/include/drm/drm_bridge.h +++ b/include/drm/drm_bridge.h @@ -40,9 +40,9 @@ struct drm_bridge; struct drm_bridge_timings; struct drm_connector; struct drm_display_info; +struct drm_edid; struct drm_minor; struct drm_panel; -struct edid; struct hdmi_codec_daifmt; struct hdmi_codec_params; struct i2c_adapter; -- cgit From 4e6d9bc1586494358e368f96e61d5689a17b31e0 Mon Sep 17 00:00:00 2001 From: Natalie Vock Date: Tue, 4 Aug 2026 10:25:19 +0200 Subject: drm/ttm: Split cgroup charge and resource allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Coupling resource allocation and cgroup charging is racy when charging succeeds, but subsequent resource allocation fails. Certain eviction decisions are made on the basis of whether the allocating cgroup is protected, i.e. within its min/low limits, but with the charge being tied to resource allocation (and uncharged when the resource allocation fails), this check is done at a point where the allocation is not actually charged to the cgroup. This is subtly wrong if the allocation were to cause the cgroup to exceed the min/low protection, but it's even more wrong if the same cgroup tries allocating multiple buffers concurrently: In this case, the min/low protection may pass for all allocation attempts when the real min/low protection covers only some, or potentially none of the allocated buffers. Instead, charge the allocation to the cgroup once and keep the charge for as long as we try to allocate a ttm_resource, and only undo the charge if allocating the resource is ultimately unsuccessful and we move on to a different ttm_place. Reviewed-by: Maarten Lankhorst Reviewed-by: Timur Kristóf Signed-off-by: Natalie Vock Link: https://patch.msgid.link/20260804-dmemcg-aggressive-protect-v8-4-07af96681bf8@gmx.de --- drivers/gpu/drm/ttm/ttm_bo.c | 57 ++++++++++++++++++++++++++++++-------- drivers/gpu/drm/ttm/ttm_resource.c | 51 +++++++++++++++++++++++----------- include/drm/ttm/ttm_resource.h | 6 +++- 3 files changed, 86 insertions(+), 28 deletions(-) (limited to 'include/drm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 1cdb2172d592..3becddd2c43b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -489,8 +489,12 @@ out_no_ref: } struct ttm_bo_alloc_state { + /** @charge_pool: The memory pool the resource is charged to */ + struct dmem_cgroup_pool_state *charge_pool; /** @limit_pool: Which pool limit we should test against */ struct dmem_cgroup_pool_state *limit_pool; + /** @in_evict: Whether we are currently evicting buffers */ + bool in_evict; }; /** @@ -518,18 +522,39 @@ static int ttm_bo_alloc_at_place(struct ttm_buffer_object *bo, bool may_evict; int ret; - may_evict = force_space && place->mem_type != TTM_PL_SYSTEM; - - ret = ttm_resource_alloc(bo, place, res, - force_space ? &alloc_state->limit_pool : NULL); + may_evict = !alloc_state->in_evict && force_space && + place->mem_type != TTM_PL_SYSTEM; + if (!alloc_state->charge_pool) { + ret = ttm_resource_try_charge(bo, place, &alloc_state->charge_pool, + force_space ? &alloc_state->limit_pool + : NULL); + if (ret) { + /* + * -EAGAIN means the charge failed, which we treat + * like an allocation failure. Therefore, return an + * error code indicating the allocation failed - + * either -EBUSY if the allocation should be + * retried with eviction, or -ENOSPC if there should + * be no second attempt. + */ + if (ret == -EAGAIN) + ret = may_evict ? -EBUSY : -ENOSPC; + return ret; + } + } + ret = ttm_resource_alloc(bo, place, res, alloc_state->charge_pool); if (ret) { if (ret == -ENOSPC && may_evict) - return -EBUSY; - + ret = -EBUSY; return ret; } + /* + * Ownership of charge_pool has been transferred to the TTM resource, + * don't make the caller think we still hold a reference to it. + */ + alloc_state->charge_pool = NULL; return 0; } @@ -584,8 +609,10 @@ static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object * evict_walk->evicted++; if (evict_walk->res) - lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place, - evict_walk->res, NULL); + lret = ttm_bo_alloc_at_place(evict_walk->evictor, + evict_walk->place, false, + evict_walk->res, + evict_walk->alloc_state); if (lret == 0) return 1; out: @@ -624,6 +651,8 @@ static int ttm_bo_evict_alloc(struct ttm_device *bdev, }; s64 lret; + state->in_evict = true; + evict_walk.walk.arg.trylock_only = true; lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); @@ -654,6 +683,7 @@ retry: goto retry; } out: + state->in_evict = false; if (lret < 0) return lret; if (lret == 0) @@ -786,6 +816,7 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, &alloc_state); if (ret == -ENOSPC) { + dmem_cgroup_uncharge(alloc_state.charge_pool, bo->base.size); dmem_cgroup_pool_state_put(alloc_state.limit_pool); continue; } else if (ret == -EBUSY) { @@ -794,11 +825,15 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, dmem_cgroup_pool_state_put(alloc_state.limit_pool); - if (ret == -EBUSY) - continue; - else if (ret) + if (ret) { + dmem_cgroup_uncharge(alloc_state.charge_pool, + bo->base.size); + if (ret == -EBUSY) + continue; return ret; + } } else if (ret) { + dmem_cgroup_uncharge(alloc_state.charge_pool, bo->base.size); dmem_cgroup_pool_state_put(alloc_state.limit_pool); return ret; } diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 154d6739256f..02eca679cb68 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -386,33 +386,52 @@ void ttm_resource_fini(struct ttm_resource_manager *man, } EXPORT_SYMBOL(ttm_resource_fini); +/** + * ttm_resource_try_charge - charge a resource manager's cgroup pool + * @bo: buffer for which an allocation should be charged + * @place: where the allocation is attempted to be placed + * @ret_pool: on charge success, the pool that was charged + * @ret_limit_pool: on charge failure, the pool responsible for the failure + * + * Should be used to charge cgroups before attempting resource allocation. + * When charging succeeds, the value of ret_pool should be passed to + * ttm_resource_alloc. + * + * Returns: 0 on charge success, negative errno on failure. + */ +int ttm_resource_try_charge(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct dmem_cgroup_pool_state **ret_pool, + struct dmem_cgroup_pool_state **ret_limit_pool) +{ + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, place->mem_type); + + if (!man->cg) { + *ret_pool = NULL; + if (ret_limit_pool) + *ret_limit_pool = NULL; + return 0; + } + + return dmem_cgroup_try_charge(man->cg, bo->base.size, ret_pool, + ret_limit_pool); +} + int ttm_resource_alloc(struct ttm_buffer_object *bo, const struct ttm_place *place, struct ttm_resource **res_ptr, - struct dmem_cgroup_pool_state **ret_limit_pool) + struct dmem_cgroup_pool_state *charge_pool) { struct ttm_resource_manager *man = ttm_manager_type(bo->bdev, place->mem_type); - struct dmem_cgroup_pool_state *pool = NULL; int ret; - if (man->cg) { - ret = dmem_cgroup_try_charge(man->cg, bo->base.size, &pool, ret_limit_pool); - if (ret) { - if (ret == -EAGAIN) - ret = -ENOSPC; - return ret; - } - } - ret = man->func->alloc(man, bo, place, res_ptr); - if (ret) { - if (pool) - dmem_cgroup_uncharge(pool, bo->base.size); + if (ret) return ret; - } - (*res_ptr)->css = pool; + (*res_ptr)->css = charge_pool; spin_lock(&bo->bdev->lru_lock); ttm_resource_add_bulk_move(*res_ptr, bo); diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h index a5d386583fb6..e567b7ec8218 100644 --- a/include/drm/ttm/ttm_resource.h +++ b/include/drm/ttm/ttm_resource.h @@ -458,10 +458,14 @@ void ttm_resource_init(struct ttm_buffer_object *bo, void ttm_resource_fini(struct ttm_resource_manager *man, struct ttm_resource *res); +int ttm_resource_try_charge(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct dmem_cgroup_pool_state **ret_pool, + struct dmem_cgroup_pool_state **ret_limit_pool); int ttm_resource_alloc(struct ttm_buffer_object *bo, const struct ttm_place *place, struct ttm_resource **res, - struct dmem_cgroup_pool_state **ret_limit_pool); + struct dmem_cgroup_pool_state *charge_pool); void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res); bool ttm_resource_intersects(struct ttm_device *bdev, struct ttm_resource *res, -- cgit From e11640b9cea4e0c448df1d31b9e950fa52cf19af Mon Sep 17 00:00:00 2001 From: Thomas Hellström Date: Sat, 25 Jul 2026 12:00:32 +0200 Subject: cgroup/dmem: Introduce struct dmem_cgroup_init for region initialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the bare u64 size argument to dmem_cgroup_register_region() and drmm_cgroup_register_region() with a const struct dmem_cgroup_init * pointer. The struct currently carries only the size field, but using a struct makes the API extensible: future callers can supply additional initialization parameters without adding more positional arguments. Update all in-tree callers (amdgpu, xe) to use a compound-literal initializer. v5: - Commit introduced. Assisted-by: GitHub_Copilot:claude-sonnet-4.6 Signed-off-by: Thomas Hellström Reviewed-by: Maarten Lankhorst Tested-by: Thadeu Lima de Souza Cascardo Link: https://patch.msgid.link/20260725100036.2372-3-thomas.hellstrom@linux.intel.com Acked-by: Dave Airlie Acked-by: Christian König Signed-off-by: Maarten Lankhorst --- drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 6 +++++- drivers/gpu/drm/drm_drv.c | 8 +++++--- drivers/gpu/drm/nouveau/nouveau_ttm.c | 5 ++++- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 7 ++++++- include/drm/drm_drv.h | 4 +++- include/linux/cgroup_dmem.h | 16 +++++++++++++--- kernel/cgroup/dmem.c | 10 ++++++---- 7 files changed, 42 insertions(+), 14 deletions(-) (limited to 'include/drm') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index ac3f71d77140..08f05c3aed1d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -23,6 +23,7 @@ */ #include +#include #include #include #include @@ -932,7 +933,10 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev) if (err) return err; - man->cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram", adev->gmc.real_vram_size); + man->cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram", + &(struct dmem_cgroup_init){ + .size = adev->gmc.real_vram_size, + }); if (IS_ERR(man->cg)) return PTR_ERR(man->cg); diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c index 1ff0bf7cba6a..3c570f9393b9 100644 --- a/drivers/gpu/drm/drm_drv.c +++ b/drivers/gpu/drm/drm_drv.c @@ -960,17 +960,19 @@ static void drmm_cg_unregister_region(struct drm_device *dev, void *arg) * drmm_cgroup_register_region - Register a region of a DRM device to cgroups * @dev: device for region * @region_name: Region name for registering - * @size: Size of region in bytes + * @init: Initialization parameters for the region. * * This decreases the ref-count of @dev by one. The device is destroyed if the * ref-count drops to zero. */ -struct dmem_cgroup_region *drmm_cgroup_register_region(struct drm_device *dev, const char *region_name, u64 size) +struct dmem_cgroup_region * +drmm_cgroup_register_region(struct drm_device *dev, const char *region_name, + const struct dmem_cgroup_init *init) { struct dmem_cgroup_region *region; int ret; - region = dmem_cgroup_register_region(size, "drm/%s/%s", dev->unique, region_name); + region = dmem_cgroup_register_region(init, "drm/%s/%s", dev->unique, region_name); if (IS_ERR_OR_NULL(region)) return region; diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index 860bca7e3ce2..cfc0868e2dd3 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -23,6 +23,7 @@ * USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#include #include #include @@ -189,7 +190,9 @@ nouveau_ttm_init_vram(struct nouveau_drm *drm) man->func = &nouveau_vram_manager; man->cg = drmm_cgroup_register_region(drm->dev, "vram", - drm->gem.vram_available); + &(struct dmem_cgroup_init){ + .size = drm->gem.vram_available + }); if (IS_ERR(man->cg)) return PTR_ERR(man->cg); diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index b518f7dec680..308fda4248eb 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -4,6 +4,8 @@ * Copyright (C) 2021-2022 Red Hat */ +#include + #include #include #include @@ -303,7 +305,10 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, int err; name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; - man->cg = drmm_cgroup_register_region(&xe->drm, name, size); + man->cg = drmm_cgroup_register_region(&xe->drm, name, + &(struct dmem_cgroup_init){ + .size = size, + }); if (IS_ERR(man->cg)) return PTR_ERR(man->cg); diff --git a/include/drm/drm_drv.h b/include/drm/drm_drv.h index e09559495c5b..b23830494ed4 100644 --- a/include/drm/drm_drv.h +++ b/include/drm/drm_drv.h @@ -34,6 +34,7 @@ #include +struct dmem_cgroup_init; struct dmem_cgroup_region; struct drm_fb_helper; struct drm_fb_helper_surface_size; @@ -433,7 +434,8 @@ void *__devm_drm_dev_alloc(struct device *parent, struct dmem_cgroup_region * drmm_cgroup_register_region(struct drm_device *dev, - const char *region_name, u64 size); + const char *region_name, + const struct dmem_cgroup_init *init); /** * devm_drm_dev_alloc - Resource managed allocation of a &drm_device instance diff --git a/include/linux/cgroup_dmem.h b/include/linux/cgroup_dmem.h index 9d72457c4cb9..0575bab38d24 100644 --- a/include/linux/cgroup_dmem.h +++ b/include/linux/cgroup_dmem.h @@ -14,8 +14,18 @@ struct dmem_cgroup_pool_state; /* Opaque definition of a cgroup region, used internally */ struct dmem_cgroup_region; +/** + * struct dmem_cgroup_init - Initialization parameters for a dmem cgroup region. + * @size: Size of the region in bytes. + */ +struct dmem_cgroup_init { + u64 size; +}; + #if IS_ENABLED(CONFIG_CGROUP_DMEM) -struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size, const char *name_fmt, ...) __printf(2,3); +struct dmem_cgroup_region * +dmem_cgroup_register_region(const struct dmem_cgroup_init *init, + const char *name_fmt, ...) __printf(2, 3); void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region); int dmem_cgroup_try_charge(struct dmem_cgroup_region *region, u64 size, struct dmem_cgroup_pool_state **ret_pool, @@ -33,8 +43,8 @@ struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgrou void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool); #else -static inline __printf(2,3) struct dmem_cgroup_region * -dmem_cgroup_register_region(u64 size, const char *name_fmt, ...) +static inline __printf(2, 3) struct dmem_cgroup_region * +dmem_cgroup_register_region(const struct dmem_cgroup_init *init, const char *name_fmt, ...) { return NULL; } diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c index 1af08b0464b8..aea21e39180a 100644 --- a/kernel/cgroup/dmem.c +++ b/kernel/cgroup/dmem.c @@ -502,7 +502,7 @@ EXPORT_SYMBOL_GPL(dmem_cgroup_unregister_region); /** * dmem_cgroup_register_region() - Register a regions for dev cgroup. - * @size: Size of region to register, in bytes. + * @init: Initialization parameters for the region. * @fmt: Region parameters to register * * This function registers a node in the dmem cgroup with the @@ -511,13 +511,15 @@ EXPORT_SYMBOL_GPL(dmem_cgroup_unregister_region); * * Return: NULL or a struct on success, PTR_ERR on failure. */ -struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size, const char *fmt, ...) +struct dmem_cgroup_region * +dmem_cgroup_register_region(const struct dmem_cgroup_init *init, + const char *fmt, ...) { struct dmem_cgroup_region *ret; char *region_name; va_list ap; - if (!size) + if (!init || !init->size) return NULL; va_start(ap, fmt); @@ -534,7 +536,7 @@ struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size, const char *fmt INIT_LIST_HEAD(&ret->pools); ret->name = region_name; - ret->size = size; + ret->size = init->size; kref_init(&ret->ref); spin_lock(&dmemcg_lock); -- cgit From 425a783e1bc18bebd6f278b3ca55929e81f0fe0d Mon Sep 17 00:00:00 2001 From: Thomas Hellström Date: Sat, 25 Jul 2026 12:00:34 +0200 Subject: drm/ttm: Hook up a cgroup-aware reclaim callback for the dmem controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ttm_bo_evict_cgroup() to evict buffer objects charged to a specific dmem cgroup pool from a resource manager's LRU until a byte target is met. Add ttm_resource_manager_set_dmem_region() to associate a dmem cgroup region with a resource manager; drivers supply their own dmem_cgroup_ops with ttm_resource_manager_dmem_reclaim as the reclaim function and the manager pointer as reclaim_priv in the dmem_cgroup_init to wire up TTM eviction as the reclaim callback. The eviction context is interruptible; signals abort the operation and propagate back through the write() syscall. Introduce a new mode for the bo LRU walker so that sleeping locks can be taken. This can be used when the caller doesn't hold any previous dma_resv locks, and where it intends to hold at most one lock at a time. Like the rest of the TTM eviction this should sooner than later be converted to full WW transactions. v3: - Fix ttm_resource_manager_set_dmem_region() storing an error pointer in man->cg unconditionally. (Sashiko-bot) - Fix kernel-doc function name format for ttm_bo_evict_cgroup() and ttm_resource_manager_set_dmem_region(). v5: - Rebased on the introduction of struct dmem_cgroup_init. - Handle NULL region in ttm_resource_manager_set_dmem_region() to clear the reclaim callback, preventing use-after-free when the manager is torn down while the dmem region outlives it. (Sashiko-bot) - Return 0 on any progress (even partial eviction), -ENOSPC only when nothing was freed; fixes callers that expected 0 on partial success. - Document that the reclaim callback should return 0 if some progress was made, -ENOSPC if no progress at all, or another error for fatal failures. v8: - Fix ttm_resource_manager_set_dmem_region() using IS_ERR_OR_NULL(), which skipped the assignment for a NULL region and thus never cleared man->cg. Use IS_ERR() so that a NULL region detaches the region as the kernel-doc and the v5 changelog intended. (Sashiko-bot) v9: - Don't leak cgroup charges for bos that may have survived dmemcg region fini. - Drop the misleading "Capture size before eviction in case res is cleared" comment in ttm_bo_evict_cb(). (Maarten Lankhorst) Assisted-by: GitHub_Copilot:claude-sonnet-4.6 Signed-off-by: Thomas Hellström Reviewed-by: Maarten Lankhorst #v7 Tested-by: Thadeu Lima de Souza Cascardo Link: https://patch.msgid.link/20260725100036.2372-5-thomas.hellstrom@linux.intel.com Signed-off-by: Maarten Lankhorst --- drivers/gpu/drm/ttm/ttm_bo.c | 97 ++++++++++++++++++++++++++++++++++++-- drivers/gpu/drm/ttm/ttm_bo_util.c | 3 +- drivers/gpu/drm/ttm/ttm_resource.c | 54 ++++++++++++++++++++- include/drm/ttm/ttm_bo.h | 10 ++++ include/drm/ttm/ttm_resource.h | 7 +++ 5 files changed, 166 insertions(+), 5 deletions(-) (limited to 'include/drm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 79c96aba4bf4..ef56c18ded1b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -629,6 +629,7 @@ static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object * struct ttm_bo_evict_walk *evict_walk = container_of(walk, typeof(*evict_walk), walk); struct dmem_cgroup_pool_state *limit_pool, *ancestor = NULL; + s64 bo_size = bo->base.size; bool evict_valuable; s64 lret; @@ -672,6 +673,12 @@ static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object * if (!evict_valuable) return 0; + /* + * evict_walk->place is NULL in cgroup drain mode. Drivers' + * eviction_valuable() callbacks must handle a NULL place, treating it + * as "any placement": the TTM base implementation already does so via + * ttm_resource_intersects(). + */ if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, evict_walk->place)) return 0; @@ -687,13 +694,17 @@ static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object * goto out; evict_walk->evicted++; - if (evict_walk->res) + if (evict_walk->res) { lret = ttm_bo_alloc_at_place(evict_walk->evictor, evict_walk->place, false, evict_walk->res, evict_walk->alloc_state); - if (lret == 0) - return 1; + if (lret == 0) + return 1; + } else { + /* Cgroup drain: return bytes freed for byte-denominated progress. */ + return bo_size; + } out: /* Errors that should terminate the walk. */ if (lret == -ENOSPC) @@ -775,6 +786,86 @@ out: return 0; } +/** + * ttm_bo_evict_cgroup() - Evict buffer objects charged to a specific cgroup. + * @bdev: The TTM device. + * @man: The resource manager whose LRU to walk. + * @limit_pool: The cgroup pool state whose members should be evicted. + * @target_bytes: Number of bytes to free. + * @ctx: The TTM operation context. + * + * Walk the LRU of @man and evict buffer objects that are charged to the + * cgroup identified by @limit_pool, until at least @target_bytes have been + * freed. Mirrors the two-pass (trylock -> sleeping-lock, low-watermark) + * strategy used by ttm_bo_evict_alloc(). + * + * Return: >= @target_bytes on full success, 0..target_bytes-1 if partial, + * negative error code on fatal error. + */ +s64 ttm_bo_evict_cgroup(struct ttm_device *bdev, + struct ttm_resource_manager *man, + struct dmem_cgroup_pool_state *limit_pool, + s64 target_bytes, + struct ttm_operation_ctx *ctx) +{ + struct ttm_bo_evict_walk evict_walk = { + .walk = { + .ops = &ttm_evict_walk_ops, + .arg = { .ctx = ctx }, + }, + .alloc_state = &(struct ttm_bo_alloc_state) { + .limit_pool = limit_pool, + .in_evict = true, + }, + /* place, evictor, res left NULL: selects cgroup drain mode */ + }; + s64 lret, pass; + + evict_walk.walk.arg.trylock_only = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, target_bytes); + if (lret < 0 || lret >= target_bytes) + return lret; + + /* Second pass: also evict BOs at the low watermark. */ + if (evict_walk.hit_low) { + evict_walk.try_low = true; + pass = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, + target_bytes - lret); + if (pass < 0) + return pass; + lret += pass; + if (lret >= target_bytes) + return lret; + } + + /* Full sleeping-lock pass for remaining target. */ + evict_walk.try_low = evict_walk.hit_low = false; + evict_walk.walk.arg.trylock_only = false; + +retry: + evict_walk.walk.arg.sleeping_lock = true; + do { + evict_walk.evicted = 0; + pass = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, + target_bytes - lret); + if (pass < 0) { + lret = pass; + goto out; + } + lret += pass; + } while (lret < target_bytes && evict_walk.evicted); + + /* One more attempt if we hit the low limit during sleeping-lock pass. */ + if (lret < target_bytes && evict_walk.hit_low && !evict_walk.try_low) { + evict_walk.try_low = true; + goto retry; + } + +out: + return lret; +} +EXPORT_SYMBOL(ttm_bo_evict_cgroup); + /** * ttm_bo_pin - Pin the buffer object. * @bo: The buffer object to pin diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 3e3c201a0222..bd0b23ac2cc4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -999,7 +999,8 @@ __ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) bo = res->bo; if (ttm_lru_walk_trylock(curs, bo)) bo_locked = true; - else if (!arg->ticket || arg->ctx->no_wait_gpu || arg->trylock_only) + else if ((!arg->ticket && !arg->sleeping_lock) || arg->ctx->no_wait_gpu || + arg->trylock_only) continue; if (!ttm_bo_get_unless_zero(bo)) { diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 02eca679cb68..4e6d3d658d85 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -456,7 +456,7 @@ void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res) man = ttm_manager_type(bo->bdev, (*res)->mem_type); man->func->free(man, *res); *res = NULL; - if (man->cg) + if (pool) dmem_cgroup_uncharge(pool, bo->base.size); } EXPORT_SYMBOL(ttm_resource_free); @@ -972,3 +972,55 @@ void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, #endif } EXPORT_SYMBOL(ttm_resource_manager_create_debugfs); + +/** + * ttm_resource_manager_dmem_reclaim() - dmem cgroup reclaim callback for TTM + * resource managers. + * @pool: The dmem cgroup pool state for the cgroup being reclaimed. + * @target_bytes: Number of bytes to try to free. + * @priv: The &ttm_resource_manager pointer, passed as @init.reclaim_priv to + * dmem_cgroup_register_region(). + * + * Drivers should use this as the @reclaim member of their own + * &struct dmem_cgroup_ops, with the &ttm_resource_manager pointer as + * @init.reclaim_priv. + * + * Return: 0 if some memory was freed, -ENOSPC if nothing was freed, or + * another negative error code on fatal failure. + */ +int ttm_resource_manager_dmem_reclaim(struct dmem_cgroup_pool_state *pool, + u64 target_bytes, void *priv) +{ + struct ttm_resource_manager *man = priv; + struct ttm_operation_ctx ctx = { .interruptible = true }; + s64 freed; + + freed = ttm_bo_evict_cgroup(man->bdev, man, pool, target_bytes, &ctx); + if (freed < 0) + return freed; + + return freed > 0 ? 0 : -ENOSPC; +} +EXPORT_SYMBOL(ttm_resource_manager_dmem_reclaim); + +/** + * ttm_resource_manager_set_dmem_region() - Associate a dmem cgroup region with a + * resource manager. + * @man: The resource manager. + * @region: The dmem cgroup region to associate, may be NULL or IS_ERR(). + * + * When @region is valid, stores it in @man->cg so that TTM can look up the + * associated pool during charging and eviction-target selection. When + * @region is %NULL, clears @man->cg to detach the region before teardown. + * An IS_ERR() @region is ignored, leaving @man->cg unchanged. + * The reclaim callback must be wired up using ttm_resource_manager_dmem_reclaim() + * in the driver's own &struct dmem_cgroup_ops, with the manager pointer as + * @init.reclaim_priv. + */ +void ttm_resource_manager_set_dmem_region(struct ttm_resource_manager *man, + struct dmem_cgroup_region *region) +{ + if (!IS_ERR(region)) + man->cg = region; +} +EXPORT_SYMBOL(ttm_resource_manager_set_dmem_region); diff --git a/include/drm/ttm/ttm_bo.h b/include/drm/ttm/ttm_bo.h index 8310bc3d55f9..32791c4db2a9 100644 --- a/include/drm/ttm/ttm_bo.h +++ b/include/drm/ttm/ttm_bo.h @@ -226,6 +226,11 @@ struct ttm_lru_walk_arg { struct ww_acquire_ctx *ticket; /** @trylock_only: Only use trylock for locking. */ bool trylock_only; + /** + * @sleeping_lock: Use sleeping locks even with %NULL @ticket. + * @trylock_only has precedence over this field. + */ + bool sleeping_lock; }; /** @@ -431,6 +436,11 @@ void ttm_bo_unpin(struct ttm_buffer_object *bo); int ttm_bo_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, struct ttm_operation_ctx *ctx); +s64 ttm_bo_evict_cgroup(struct ttm_device *bdev, + struct ttm_resource_manager *man, + struct dmem_cgroup_pool_state *limit_pool, + s64 target_bytes, + struct ttm_operation_ctx *ctx); int ttm_bo_access(struct ttm_buffer_object *bo, unsigned long offset, void *buf, int len, int write); vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h index e567b7ec8218..3f2812743a7e 100644 --- a/include/drm/ttm/ttm_resource.h +++ b/include/drm/ttm/ttm_resource.h @@ -39,6 +39,7 @@ struct dentry; struct dmem_cgroup_device; +struct dmem_cgroup_region; struct drm_printer; struct ttm_device; struct ttm_resource_manager; @@ -481,6 +482,12 @@ void ttm_resource_manager_init(struct ttm_resource_manager *man, struct ttm_device *bdev, uint64_t size); +void ttm_resource_manager_set_dmem_region(struct ttm_resource_manager *man, + struct dmem_cgroup_region *region); + +int ttm_resource_manager_dmem_reclaim(struct dmem_cgroup_pool_state *pool, + u64 target_bytes, void *priv); + int ttm_resource_manager_evict_all(struct ttm_device *bdev, struct ttm_resource_manager *man); -- cgit