From: Rob Clark robdclark@chromium.org

So we don't have to duplicate the boilerplate for eviction.

This also lets us re-use the main scan loop for vmap shrinker.

Signed-off-by: Rob Clark robdclark@chromium.org --- drivers/gpu/drm/msm/msm_gem_shrinker.c | 94 +++++++++++++------------- 1 file changed, 46 insertions(+), 48 deletions(-)

diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c index 33a49641ef30..38bf919f8508 100644 --- a/drivers/gpu/drm/msm/msm_gem_shrinker.c +++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c @@ -17,21 +17,35 @@ msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) return priv->shrinkable_count; }

+static bool +purge(struct msm_gem_object *msm_obj) +{ + if (!is_purgeable(msm_obj)) + return false; + + /* + * This will move the obj out of still_in_list to + * the purged list + */ + msm_gem_purge(&msm_obj->base); + + return true; +} + static unsigned long -msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) +scan(struct msm_drm_private *priv, unsigned nr_to_scan, struct list_head *list, + bool (*shrink)(struct msm_gem_object *msm_obj)) { - struct msm_drm_private *priv = - container_of(shrinker, struct msm_drm_private, shrinker); + unsigned freed = 0; struct list_head still_in_list; - unsigned long freed = 0;

INIT_LIST_HEAD(&still_in_list);

mutex_lock(&priv->mm_lock);

- while (freed < sc->nr_to_scan) { + while (freed < nr_to_scan) { struct msm_gem_object *msm_obj = list_first_entry_or_null( - &priv->inactive_dontneed, typeof(*msm_obj), mm_list); + list, typeof(*msm_obj), mm_list);

if (!msm_obj) break; @@ -62,14 +76,9 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) if (!msm_gem_trylock(&msm_obj->base)) goto tail;

- if (is_purgeable(msm_obj)) { - /* - * This will move the obj out of still_in_list to - * the purged list - */ - msm_gem_purge(&msm_obj->base); + if (shrink(msm_obj)) freed += msm_obj->base.size >> PAGE_SHIFT; - } + msm_gem_unlock(&msm_obj->base);

tail: @@ -77,16 +86,25 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) mutex_lock(&priv->mm_lock); }

- list_splice_tail(&still_in_list, &priv->inactive_dontneed); + list_splice_tail(&still_in_list, list); mutex_unlock(&priv->mm_lock);

- if (freed > 0) { + return freed; +} + +static unsigned long +msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) +{ + struct msm_drm_private *priv = + container_of(shrinker, struct msm_drm_private, shrinker); + unsigned long freed; + + freed = scan(priv, sc->nr_to_scan, &priv->inactive_dontneed, purge); + + if (freed > 0) trace_msm_gem_purge(freed << PAGE_SHIFT); - } else { - return SHRINK_STOP; - }

- return freed; + return (freed > 0) ? freed : SHRINK_STOP; }

/* since we don't know any better, lets bail after a few @@ -95,29 +113,15 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) */ static const int vmap_shrink_limit = 15;

-static unsigned -vmap_shrink(struct list_head *mm_list) +static bool +vmap_shrink(struct msm_gem_object *msm_obj) { - struct msm_gem_object *msm_obj; - unsigned unmapped = 0; + if (!is_vunmapable(msm_obj)) + return false;

- list_for_each_entry(msm_obj, mm_list, mm_list) { - /* Use trylock, because we cannot block on a obj that - * might be trying to acquire mm_lock - */ - if (!msm_gem_trylock(&msm_obj->base)) - continue; - if (is_vunmapable(msm_obj)) { - msm_gem_vunmap(&msm_obj->base); - unmapped++; - } - msm_gem_unlock(&msm_obj->base); + msm_gem_vunmap(&msm_obj->base);

- if (++unmapped >= vmap_shrink_limit) - break; - } - - return unmapped; + return true; }

static int @@ -133,17 +137,11 @@ msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) }; unsigned idx, unmapped = 0;

- mutex_lock(&priv->mm_lock); - - for (idx = 0; mm_lists[idx]; idx++) { - unmapped += vmap_shrink(mm_lists[idx]); - - if (unmapped >= vmap_shrink_limit) - break; + for (idx = 0; mm_lists[idx] && unmapped < vmap_shrink_limit; idx++) { + unmapped += scan(priv, vmap_shrink_limit - unmapped, + mm_lists[idx], vmap_shrink); }

- mutex_unlock(&priv->mm_lock); - *(unsigned long *)ptr += unmapped;

if (unmapped > 0)

From: Rob Clark robdclark@chromium.org

Currently these always go together, either when we purge MADV_WONTNEED objects or when the object is freed. But for unpin, we want to be able to purge (unmap from iommu) the vma, while keeping the iova range allocated (so we can remap back to the same GPU virtual address when the object is re-pinned.

Signed-off-by: Rob Clark robdclark@chromium.org --- drivers/gpu/drm/msm/msm_gem.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 71530a89b675..5f0647adc29d 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -357,9 +357,14 @@ static void del_vma(struct msm_gem_vma *vma) kfree(vma); }

-/* Called with msm_obj locked */ +/** + * If close is true, this also closes the VMA (releasing the allocated + * iova range) in addition to removing the iommu mapping. In the eviction + * case (!close), we keep the iova allocated, but only remove the iommu + * mapping. + */ static void -put_iova_spaces(struct drm_gem_object *obj) +put_iova_spaces(struct drm_gem_object *obj, bool close) { struct msm_gem_object *msm_obj = to_msm_bo(obj); struct msm_gem_vma *vma; @@ -369,7 +374,8 @@ put_iova_spaces(struct drm_gem_object *obj) list_for_each_entry(vma, &msm_obj->vmas, list) { if (vma->aspace) { msm_gem_purge_vma(vma->aspace, vma); - msm_gem_close_vma(vma->aspace, vma); + if (close) + msm_gem_close_vma(vma->aspace, vma); } } } @@ -711,7 +717,8 @@ void msm_gem_purge(struct drm_gem_object *obj) GEM_WARN_ON(!is_purgeable(msm_obj)); GEM_WARN_ON(obj->import_attach);

- put_iova_spaces(obj); + /* Get rid of any iommu mapping(s): */ + put_iova_spaces(obj, true);

msm_gem_vunmap(obj);

@@ -1013,7 +1020,7 @@ void msm_gem_free_object(struct drm_gem_object *obj) /* object should not be on active list: */ GEM_WARN_ON(is_active(msm_obj));

- put_iova_spaces(obj); + put_iova_spaces(obj, true);

if (obj->import_attach) { GEM_WARN_ON(msm_obj->vaddr);

From: Rob Clark robdclark@chromium.org

Objects that are potential for swapping out are (1) willneed (ie. if they are purgable/MADV_WONTNEED we can just free the pages without them having to land in swap), (2) not on an active list, (3) not dma-buf imported or exported, and (4) not vmap'd. This repurposes the purged list for objects that do not have backing pages (either because they have not been pinned for the first time yet, or in a later patch because they have been unpinned/evicted.

Signed-off-by: Rob Clark robdclark@chromium.org --- drivers/gpu/drm/msm/msm_drv.c | 2 +- drivers/gpu/drm/msm/msm_drv.h | 13 ++++++---- drivers/gpu/drm/msm/msm_gem.c | 44 ++++++++++++++++++++++++++-------- drivers/gpu/drm/msm/msm_gem.h | 45 +++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 15 deletions(-)

diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index e12d5fbd0a34..d3d6c743b7af 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -451,7 +451,7 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv)

INIT_LIST_HEAD(&priv->inactive_willneed); INIT_LIST_HEAD(&priv->inactive_dontneed); - INIT_LIST_HEAD(&priv->inactive_purged); + INIT_LIST_HEAD(&priv->inactive_unpinned); mutex_init(&priv->mm_lock);

/* Teach lockdep about lock ordering wrt. shrinker: */ diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 6a42cdf4cf7e..2668941df529 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -182,11 +182,15 @@ struct msm_drm_private { struct mutex obj_lock;

/** - * Lists of inactive GEM objects. Every bo is either in one of the + * LRUs of inactive GEM objects. Every bo is either in one of the * inactive lists (depending on whether or not it is shrinkable) or * gpu->active_list (for the gpu it is active on[1]), or transiently * on a temporary list as the shrinker is running. * + * Note that inactive_willneed also contains pinned and vmap'd bos, + * but the number of pinned-but-not-active objects is small (scanout + * buffers, ringbuffer, etc). + * * These lists are protected by mm_lock (which should be acquired * before per GEM object lock). One should *not* hold mm_lock in * get_pages()/vmap()/etc paths, as they can trigger the shrinker. @@ -194,10 +198,11 @@ struct msm_drm_private { * [1] if someone ever added support for the old 2d cores, there could be * more than one gpu object */ - struct list_head inactive_willneed; /* inactive + !shrinkable */ - struct list_head inactive_dontneed; /* inactive + shrinkable */ - struct list_head inactive_purged; /* inactive + purged */ + struct list_head inactive_willneed; /* inactive + potentially unpin/evictable */ + struct list_head inactive_dontneed; /* inactive + shrinkable */ + struct list_head inactive_unpinned; /* inactive + purged or unpinned */ long shrinkable_count; /* write access under mm_lock */ + long evictable_count; /* write access under mm_lock */ struct mutex mm_lock;

struct workqueue_struct *wq; diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 9ff37904ec2b..9ac89951080c 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -130,6 +130,9 @@ static struct page **get_pages(struct drm_gem_object *obj) */ if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED)) sync_for_device(msm_obj); + + GEM_WARN_ON(msm_obj->active_count); + update_inactive(msm_obj); }

return msm_obj->pages; @@ -428,7 +431,7 @@ static int msm_gem_pin_iova(struct drm_gem_object *obj, struct msm_gem_object *msm_obj = to_msm_bo(obj); struct msm_gem_vma *vma; struct page **pages; - int prot = IOMMU_READ; + int ret, prot = IOMMU_READ;

if (!(msm_obj->flags & MSM_BO_GPU_READONLY)) prot |= IOMMU_WRITE; @@ -449,8 +452,13 @@ static int msm_gem_pin_iova(struct drm_gem_object *obj, if (IS_ERR(pages)) return PTR_ERR(pages);

- return msm_gem_map_vma(aspace, vma, prot, + ret = msm_gem_map_vma(aspace, vma, prot, msm_obj->sgt, obj->size >> PAGE_SHIFT); + + if (!ret) + msm_obj->pin_count++; + + return ret; }

static int get_and_pin_iova_range_locked(struct drm_gem_object *obj, @@ -542,14 +550,21 @@ uint64_t msm_gem_iova(struct drm_gem_object *obj, void msm_gem_unpin_iova_locked(struct drm_gem_object *obj, struct msm_gem_address_space *aspace) { + struct msm_gem_object *msm_obj = to_msm_bo(obj); struct msm_gem_vma *vma;

GEM_WARN_ON(!msm_gem_is_locked(obj));

vma = lookup_vma(obj, aspace);

- if (!GEM_WARN_ON(!vma)) + if (!GEM_WARN_ON(!vma)) { msm_gem_unmap_vma(aspace, vma); + + msm_obj->pin_count--; + GEM_WARN_ON(msm_obj->pin_count < 0); + + update_inactive(msm_obj); + } }

/* @@ -800,9 +815,12 @@ void msm_gem_active_get(struct drm_gem_object *obj, struct msm_gpu *gpu) GEM_WARN_ON(!msm_gem_is_locked(obj)); GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED); GEM_WARN_ON(msm_obj->dontneed); + GEM_WARN_ON(!msm_obj->sgt);

if (msm_obj->active_count++ == 0) { mutex_lock(&priv->mm_lock); + if (msm_obj->evictable) + mark_unevictable(msm_obj); list_del(&msm_obj->mm_list); list_add_tail(&msm_obj->mm_list, &gpu->active_list); mutex_unlock(&priv->mm_lock); @@ -825,21 +843,28 @@ static void update_inactive(struct msm_gem_object *msm_obj) { struct msm_drm_private *priv = msm_obj->base.dev->dev_private;

+ GEM_WARN_ON(!msm_gem_is_locked(&msm_obj->base)); + + if (msm_obj->active_count != 0) + return; + mutex_lock(&priv->mm_lock); - GEM_WARN_ON(msm_obj->active_count != 0);

if (msm_obj->dontneed) mark_unpurgable(msm_obj); + if (msm_obj->evictable) + mark_unevictable(msm_obj);

list_del(&msm_obj->mm_list); - if (msm_obj->madv == MSM_MADV_WILLNEED) { + if ((msm_obj->madv == MSM_MADV_WILLNEED) && msm_obj->sgt) { list_add_tail(&msm_obj->mm_list, &priv->inactive_willneed); + mark_evictable(msm_obj); } else if (msm_obj->madv == MSM_MADV_DONTNEED) { list_add_tail(&msm_obj->mm_list, &priv->inactive_dontneed); mark_purgable(msm_obj); } else { - GEM_WARN_ON(msm_obj->madv != __MSM_MADV_PURGED); - list_add_tail(&msm_obj->mm_list, &priv->inactive_purged); + GEM_WARN_ON((msm_obj->madv != __MSM_MADV_PURGED) && msm_obj->sgt); + list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned); }

mutex_unlock(&priv->mm_lock); @@ -1201,8 +1226,7 @@ static struct drm_gem_object *_msm_gem_new(struct drm_device *dev, }

mutex_lock(&priv->mm_lock); - /* Initially obj is idle, obj->madv == WILLNEED: */ - list_add_tail(&msm_obj->mm_list, &priv->inactive_willneed); + list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned); mutex_unlock(&priv->mm_lock);

mutex_lock(&priv->obj_lock); @@ -1276,7 +1300,7 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev, msm_gem_unlock(obj);

mutex_lock(&priv->mm_lock); - list_add_tail(&msm_obj->mm_list, &priv->inactive_willneed); + list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned); mutex_unlock(&priv->mm_lock);

mutex_lock(&priv->obj_lock); diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h index e13a9301b616..39b2e5584f97 100644 --- a/drivers/gpu/drm/msm/msm_gem.h +++ b/drivers/gpu/drm/msm/msm_gem.h @@ -60,6 +60,11 @@ struct msm_gem_object { */ bool dontneed : 1;

+ /** + * Is object evictable (ie. counted in priv->evictable_count)? + */ + bool evictable : 1; + /** * count of active vmap'ing */ @@ -103,6 +108,7 @@ struct msm_gem_object { char name[32]; /* Identifier to print for the debugfs files */

int active_count; + int pin_count; }; #define to_msm_bo(x) container_of(x, struct msm_gem_object, base)

@@ -263,7 +269,46 @@ static inline void mark_unpurgable(struct msm_gem_object *msm_obj) msm_obj->dontneed = false; }

+static inline bool is_unevictable(struct msm_gem_object *msm_obj) +{ + return is_unpurgable(msm_obj) || msm_obj->pin_count || msm_obj->vaddr; +} + +static inline void mark_evictable(struct msm_gem_object *msm_obj) +{ + struct msm_drm_private *priv = msm_obj->base.dev->dev_private; + + WARN_ON(!mutex_is_locked(&priv->mm_lock)); + + if (is_unevictable(msm_obj)) + return; + + if (WARN_ON(msm_obj->evictable)) + return; + + priv->evictable_count += msm_obj->base.size >> PAGE_SHIFT; + msm_obj->evictable = true; +} + +static inline void mark_unevictable(struct msm_gem_object *msm_obj) +{ + struct msm_drm_private *priv = msm_obj->base.dev->dev_private; + + WARN_ON(!mutex_is_locked(&priv->mm_lock)); + + if (is_unevictable(msm_obj)) + return; + + if (WARN_ON(!msm_obj->evictable)) + return; + + priv->evictable_count -= msm_obj->base.size >> PAGE_SHIFT; + WARN_ON(priv->evictable_count < 0); + msm_obj->evictable = false; +} + void msm_gem_purge(struct drm_gem_object *obj); +void msm_gem_evict(struct drm_gem_object *obj); void msm_gem_vunmap(struct drm_gem_object *obj);

/* Created per submit-ioctl, to track bo's and cmdstream bufs, etc,

From: Rob Clark robdclark@chromium.org

Now that tracking is wired up for potentially evictable GEM objects, wire up shrinker and the remaining GEM bits for unpinning backing pages of inactive objects.

Signed-off-by: Rob Clark robdclark@chromium.org --- drivers/gpu/drm/msm/msm_gem.c | 23 ++++++++++++++++ drivers/gpu/drm/msm/msm_gem_shrinker.c | 37 +++++++++++++++++++++++++- drivers/gpu/drm/msm/msm_gpu_trace.h | 13 +++++++++ 3 files changed, 72 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 163a1d30b5c9..2b731cf42294 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -759,6 +759,29 @@ void msm_gem_purge(struct drm_gem_object *obj) 0, (loff_t)-1); }

+/** + * Unpin the backing pages and make them available to be swapped out. + */ +void msm_gem_evict(struct drm_gem_object *obj) +{ + struct drm_device *dev = obj->dev; + struct msm_gem_object *msm_obj = to_msm_bo(obj); + + GEM_WARN_ON(!msm_gem_is_locked(obj)); + GEM_WARN_ON(is_unevictable(msm_obj)); + GEM_WARN_ON(!msm_obj->evictable); + GEM_WARN_ON(msm_obj->active_count); + + /* Get rid of any iommu mapping(s): */ + put_iova_spaces(obj, false); + + drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); + + put_pages(obj); + + update_inactive(msm_obj); +} + void msm_gem_vunmap(struct drm_gem_object *obj) { struct msm_gem_object *msm_obj = to_msm_bo(obj); diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c index 38bf919f8508..52828028b9d4 100644 --- a/drivers/gpu/drm/msm/msm_gem_shrinker.c +++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c @@ -9,12 +9,26 @@ #include "msm_gpu.h" #include "msm_gpu_trace.h"

+bool enable_swap = true; +MODULE_PARM_DESC(enable_swap, "Enable swappable GEM buffers"); +module_param(enable_swap, bool, 0600); + +static bool can_swap(void) +{ + return enable_swap && get_nr_swap_pages() > 0; +} + static unsigned long msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) { struct msm_drm_private *priv = container_of(shrinker, struct msm_drm_private, shrinker); - return priv->shrinkable_count; + unsigned count = priv->shrinkable_count; + + if (can_swap()) + count += priv->evictable_count; + + return count; }

static bool @@ -32,6 +46,17 @@ purge(struct msm_gem_object *msm_obj) return true; }

+static bool +evict(struct msm_gem_object *msm_obj) +{ + if (is_unevictable(msm_obj)) + return false; + + msm_gem_evict(&msm_obj->base); + + return true; +} + static unsigned long scan(struct msm_drm_private *priv, unsigned nr_to_scan, struct list_head *list, bool (*shrink)(struct msm_gem_object *msm_obj)) @@ -104,6 +129,16 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) if (freed > 0) trace_msm_gem_purge(freed << PAGE_SHIFT);

+ if (can_swap() && freed < sc->nr_to_scan) { + int evicted = scan(priv, sc->nr_to_scan - freed, + &priv->inactive_willneed, evict); + + if (evicted > 0) + trace_msm_gem_evict(evicted << PAGE_SHIFT); + + freed += evicted; + } + return (freed > 0) ? freed : SHRINK_STOP; }

diff --git a/drivers/gpu/drm/msm/msm_gpu_trace.h b/drivers/gpu/drm/msm/msm_gpu_trace.h index 03e0c2536b94..ca0b08d7875b 100644 --- a/drivers/gpu/drm/msm/msm_gpu_trace.h +++ b/drivers/gpu/drm/msm/msm_gpu_trace.h @@ -128,6 +128,19 @@ TRACE_EVENT(msm_gem_purge, );

+TRACE_EVENT(msm_gem_evict, + TP_PROTO(u32 bytes), + TP_ARGS(bytes), + TP_STRUCT__entry( + __field(u32, bytes) + ), + TP_fast_assign( + __entry->bytes = bytes; + ), + TP_printk("Evicting %u bytes", __entry->bytes) +); + + TRACE_EVENT(msm_gem_purge_vmaps, TP_PROTO(u32 unmapped), TP_ARGS(unmapped),