[Libva] [PATCH] CreateSurfaces: clear right and bottom border for nv12

[Guangxin.Xu at intel.com]

From: XuGuangxin <Guangxin.Xu at intel.com>

VME hardware will search reference pixel in 16 pixel aligned way.
If you encode a 1920x1080 surface, it will search in 1920x1088 range.
Pixels from 1080 to 1088 will introduce run to run issue, since they are random.
This patch will fix run to run issue when you encode B/P frame.

Signed-off-by: Xu Guangxin <guangxin.xu at intel.com>
---
 src/i965_drv_video.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 61 insertions(+), 2 deletions(-)

diff --git a/src/i965_drv_video.c b/src/i965_drv_video.c
index 259bfc0..8038faf 100644
--- a/src/i965_drv_video.c
+++ b/src/i965_drv_video.c
@@ -3895,6 +3895,51 @@ i965_CreateImage(VADriverContextP ctx,
     return va_status;
 }
 
+VAStatus clear_border(struct object_surface *obj_surface)
+{
+    int width[3], height[3], hstride[3], vstride[3]; /* in byte */
+    int planes;
+    unsigned char* p;
+    int i,j;
+    if (obj_surface->fourcc == VA_FOURCC_NV12) {
+        planes = 2;
+        width[0] = width[1] = obj_surface->orig_width;
+        height[0] = obj_surface->orig_height;
+        height[1] = obj_surface->orig_height / 2;
+        hstride[0] = hstride[1] = obj_surface->width;
+        vstride[0]= obj_surface->height;
+        vstride[1] = obj_surface->height / 2;
+
+    } else {
+        return VA_STATUS_ERROR_INVALID_PARAMETER;
+    }
+    drm_intel_gem_bo_map_gtt(obj_surface->bo);
+
+    p = (unsigned char*)obj_surface->bo->virtual;
+    if (!p)
+        return VA_STATUS_ERROR_INVALID_SURFACE;
+
+    for (i = 0; i < planes; i++) {
+        int w = width[i];
+        int h = height[i];
+        int hs = hstride[i];
+        int vs = vstride[i];
+        /* right */
+        for (j = 0; j < h; j++) {
+            memset(p + w, 0, hs - w);
+            p += hs;
+        }
+        /* bottom */
+        for (/* nothing */; j < vs; j++) {
+            memset(p, 0, hs);
+            p += hs;
+        }
+
+    }
+    drm_intel_gem_bo_unmap_gtt(obj_surface->bo);
+    return VA_STATUS_SUCCESS;
+}
+
 VAStatus
 i965_check_alloc_surface_bo(VADriverContextP ctx,
                             struct object_surface *obj_surface,
@@ -3904,6 +3949,7 @@ i965_check_alloc_surface_bo(VADriverContextP ctx,
 {
     struct i965_driver_data *i965 = i965_driver_data(ctx);
     int region_width, region_height;
+    VAStatus status;
 
     if (obj_surface->bo) {
         ASSERT_RET(obj_surface->fourcc, VA_STATUS_ERROR_INVALID_SURFACE);
@@ -4135,6 +4181,9 @@ i965_check_alloc_surface_bo(VADriverContextP ctx,
         }
     }
 
+    obj_surface->fourcc = fourcc;
+    obj_surface->subsampling = subsampling;
+
     obj_surface->size = ALIGN(region_width * region_height, 0x1000);
 
     if ((tiled && !obj_surface->user_disable_tiling)) {
@@ -4151,6 +4200,18 @@ i965_check_alloc_surface_bo(VADriverContextP ctx,
                                                    0);
         assert(tiling_mode == I915_TILING_Y);
         assert(pitch == obj_surface->width);
+
+        /* vme hardware will search pixel in 16 aligned way,
+         * suppose you crate a 1920x1080 surface, it will search in 1920x1088
+         * we'd better clean border, else it will have run to run issue
+         * when border pixel have random value.
+         */
+        /* TODO: support VA_FOURCC_P010 */
+        if (fourcc == VA_FOURCC_NV12) {
+            status = clear_border(obj_surface);
+            if (status != VA_STATUS_SUCCESS)
+                return status;
+        }
     } else {
         obj_surface->bo = dri_bo_alloc(i965->intel.bufmgr,
                                        "vaapi surface",
@@ -4158,8 +4219,6 @@ i965_check_alloc_surface_bo(VADriverContextP ctx,
                                        0x1000);
     }
 
-    obj_surface->fourcc = fourcc;
-    obj_surface->subsampling = subsampling;
     assert(obj_surface->bo);
     return VA_STATUS_SUCCESS;
 }
-- 
1.9.1