[PATCH 2/2] drm/amdgpu: add gart recovery by gtt list

[Christian König]

Am 30.08.2016 um 10:54 schrieb Chunming Zhou:
> Change-Id: I8daf687d43b51d69f9be36cae8b265deb88e5754
> Signed-off-by: Chunming Zhou <David1.Zhou at amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 15 +++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  3 +++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c   | 20 ++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    | 14 +-------------
>   4 files changed, 39 insertions(+), 13 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 18bbfea..6ee9987 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -607,6 +607,20 @@ struct amdgpu_gart {
>   	const struct amdgpu_gart_funcs *gart_funcs;
>   };
>   
> +struct amdgpu_ttm_tt {
> +	struct ttm_dma_tt	ttm;
> +	struct amdgpu_device	*adev;
> +	u64			offset;
> +	uint32_t                pte_flags;
> +	uint64_t		userptr;
> +	struct mm_struct	*usermm;
> +	uint32_t		userflags;
> +	spinlock_t              guptasklock;
> +	struct list_head        guptasks;
> +	atomic_t		mmu_invalidations;
> +	struct list_head        list;
> +};
> +
>   int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev);
>   void amdgpu_gart_table_ram_free(struct amdgpu_device *adev);
>   int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev);
> @@ -620,6 +634,7 @@ void amdgpu_gart_unbind(struct amdgpu_device *adev, unsigned offset,
>   int amdgpu_gart_bind(struct amdgpu_device *adev, unsigned offset,
>   		     int pages, struct page **pagelist,
>   		     dma_addr_t *dma_addr, uint32_t flags);
> +int amdgpu_gart_recovery(struct amdgpu_device *adev);
>   
>   /*
>    * GPU MC structures, functions & helpers
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 6cd485a..0e475ed 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -2247,6 +2247,9 @@ retry:
>   			struct fence *fence = NULL, *next = NULL;
>   
>   			DRM_INFO("recover vram bo from shadow\n");
> +			r = amdgpu_gart_recovery(adev);
> +			if (r)
> +				DRM_ERROR("gart recovery failed!!!\n");
>   			mutex_lock(&adev->shadow_list_lock);
>   			list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
>   				amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
> index 921bce2..02695a6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
> @@ -302,6 +302,26 @@ int amdgpu_gart_bind(struct amdgpu_device *adev, unsigned offset,
>   	return 0;
>   }
>   
> +int amdgpu_gart_recovery(struct amdgpu_device *adev)
> +{
> +	struct amdgpu_ttm_tt *gtt, *tmp;
> +	int r;
> +
> +	mutex_lock(&adev->gtt_list_lock);
> +	list_for_each_entry_safe(gtt, tmp, &adev->gtt_list, list) {
> +		r = amdgpu_gart_bind(adev, gtt->offset, gtt->ttm.ttm.num_pages,
> +				     gtt->ttm.ttm.pages, gtt->ttm.dma_address,
> +				     gtt->pte_flags);
> +		if (r) {
> +			mutex_unlock(&adev->gtt_list_lock);
> +			DRM_ERROR("failed to bind %lu pages at 0x%08X\n",
> +				  gtt->ttm.ttm.num_pages, (unsigned)gtt->offset);
> +			return r;
> +		}
> +	}
> +	mutex_unlock(&adev->gtt_list_lock);
> +	return 0;
> +}

This function would better be in amdgpu_ttm.c, cause it uses the TTM 
handling internal structures for the recovery.

Call it something like amdgpu_ttm_recover_gart(), this way you also 
don't need to move the structure.

>   /**
>    * amdgpu_gart_init - init the driver info for managing the gart
>    *
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index efeb9a5..431fe62 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -511,19 +511,6 @@ struct amdgpu_ttm_gup_task_list {
>   	struct task_struct	*task;
>   };
>   
> -struct amdgpu_ttm_tt {
> -	struct ttm_dma_tt	ttm;
> -	struct amdgpu_device	*adev;
> -	u64			offset;
> -	uint64_t		userptr;
> -	struct mm_struct	*usermm;
> -	uint32_t		userflags;
> -	spinlock_t              guptasklock;
> -	struct list_head        guptasks;
> -	atomic_t		mmu_invalidations;
> -	struct list_head        list;
> -};
> -
>   int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
>   {
>   	struct amdgpu_ttm_tt *gtt = (void *)ttm;
> @@ -668,6 +655,7 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
>   			  ttm->num_pages, (unsigned)gtt->offset);
>   		return r;
>   	}
> +	gtt->pte_flags = flags;

No need to save the flags here. When the BO is bound we know that it's 
mem type is TTM_PL_TT.

Just use a fixed ttm_mem_reg in the backup function with the mem_type 
set accordingly.

Regards,
Christian.

>   	mutex_lock(&gtt->adev->gtt_list_lock);
>   	list_add_tail(&gtt->list, &gtt->adev->gtt_list);
>   	mutex_unlock(&gtt->adev->gtt_list_lock);