[Libva] [PATCH] FIX:HEVC enc cause GPU hang when lcu_size=32 with 720x480 and 176x144

[Qu,Pengfei]

Signed-off-by: Qu,Pengfei <Pengfei.Qu at intel.com>
---
 src/gen9_mfc_hevc.c | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/gen9_mfc_hevc.c b/src/gen9_mfc_hevc.c
index 940146b..5fe34da 100644
--- a/src/gen9_mfc_hevc.c
+++ b/src/gen9_mfc_hevc.c
@@ -1307,9 +1307,11 @@ gen9_hcpe_hevc_fill_indirect_cu_inter(VADriverContextP ctx,
     int cu_size = 1;
     int tu_size = 0x55;
     int tu_count = 4;
+    int inter_mode = 0;
 
     unsigned int *mv_ptr;
     {
+        inter_mode = (msg[0] & AVC_INTER_MODE_MASK);
         submb_pre_mode = (msg[1] & AVC_INTER_SUBMB_PRE_MODE_MASK) >> 16;
 #define MSG_MV_OFFSET   4
         mv_ptr = msg + MSG_MV_OFFSET;
@@ -1319,7 +1321,7 @@ gen9_hcpe_hevc_fill_indirect_cu_inter(VADriverContextP ctx,
         */
         /* 0/2/4/6/8... ： l0, 1/3/5/7...: l1 ; now it only support 16x16,16x8,8x16,8x8*/
 
-        if ((msg[0] & AVC_INTER_MODE_MASK) == AVC_INTER_16X16) {
+        if (inter_mode == AVC_INTER_16X16) {
             mv_ptr[4] = mv_ptr[0];
             mv_ptr[5] = mv_ptr[1];
             mv_ptr[2] = mv_ptr[0];
@@ -1330,7 +1332,7 @@ gen9_hcpe_hevc_fill_indirect_cu_inter(VADriverContextP ctx,
             cu_size = 1;
             tu_size = 0x55;
             tu_count = 4;
-        } else if ((msg[0] & AVC_INTER_MODE_MASK) == AVC_INTER_8X16) {
+        } else if (inter_mode == AVC_INTER_8X16) {
             mv_ptr[4] = mv_ptr[0];
             mv_ptr[5] = mv_ptr[1];
             mv_ptr[2] = mv_ptr[8];
@@ -1341,7 +1343,7 @@ gen9_hcpe_hevc_fill_indirect_cu_inter(VADriverContextP ctx,
             cu_size = 1;
             tu_size = 0x55;
             tu_count = 4;
-        } else if ((msg[0] & AVC_INTER_MODE_MASK) == AVC_INTER_16X8) {
+        } else if (inter_mode == AVC_INTER_16X8) {
             mv_ptr[2] = mv_ptr[0];
             mv_ptr[3] = mv_ptr[1];
             mv_ptr[4] = mv_ptr[16];
@@ -1352,7 +1354,7 @@ gen9_hcpe_hevc_fill_indirect_cu_inter(VADriverContextP ctx,
             cu_size = 1;
             tu_size = 0x55;
             tu_count = 4;
-        }else if((msg[0] & AVC_INTER_MODE_MASK) == AVC_INTER_8X8) {
+        }else if(inter_mode == AVC_INTER_8X8) {
             mv_ptr[0] = mv_ptr[index * 8 + 0 ];
             mv_ptr[1] = mv_ptr[index * 8 + 1 ];
             mv_ptr[2] = mv_ptr[index * 8 + 0 ];
@@ -1592,8 +1594,9 @@ gen9_hcpe_hevc_pipeline_slice_programing(VADriverContextP ctx,
     int num_mb_in_ctb = ctb_width_in_mb * ctb_width_in_mb;
     int i_ctb, ctb_x, ctb_y;
     unsigned int split_coding_unit_flag = 0;
-    int width_in_mbs = (mfc_context->surface_state.width + 15) / 16;
+    int width_in_mbs = (pSequenceParameter->pic_width_in_luma_samples + 15) / 16;
     int row_pad_flag = (pSequenceParameter->pic_height_in_luma_samples % ctb_size)> 0 ? 1:0;
+    int col_pad_flag = (pSequenceParameter->pic_width_in_luma_samples % ctb_size)> 0 ? 1:0;
 
     int is_intra = (slice_type == HEVC_SLICE_I);
     unsigned int *msg = NULL;
@@ -1655,25 +1658,26 @@ gen9_hcpe_hevc_pipeline_slice_programing(VADriverContextP ctx,
 
     for (i_ctb = pSliceParameter->slice_segment_address;i_ctb < pSliceParameter->slice_segment_address + pSliceParameter->num_ctu_in_slice; i_ctb++) {
         int last_ctb = (i_ctb == (pSliceParameter->slice_segment_address + pSliceParameter->num_ctu_in_slice - 1));
-        int ctb_height_in_mb = ctb_width_in_mb;
+        int ctb_height_in_mb_internal = ctb_width_in_mb;
+        int ctb_width_in_mb_internal = ctb_width_in_mb;
         ctb_x = i_ctb % width_in_ctb;
         ctb_y = i_ctb / width_in_ctb;
-        if(ctb_y == (height_in_ctb - 1) && row_pad_flag)  ctb_height_in_mb = 1;
+        if(ctb_y == (height_in_ctb - 1) && row_pad_flag)  ctb_height_in_mb_internal = 1;
+        if(ctb_x == (width_in_ctb - 1) && col_pad_flag)  ctb_width_in_mb_internal = 1;
 
         mb_x = 0;
         mb_y = 0;
-        macroblock_address = (i_ctb - ctb_x) * num_mb_in_ctb + ctb_x * ctb_width_in_mb;
+        macroblock_address = ctb_y * width_in_mbs * ctb_width_in_mb + ctb_x * ctb_width_in_mb;
         split_coding_unit_flag = ((ctb_width_in_mb == 2) ? HEVC_SPLIT_CU_FLAG_32_32 : HEVC_SPLIT_CU_FLAG_16_16);
         cu_count = 1;
         cu_index = 0;
         mb_addr = 0;
         msg = NULL;
-        for (mb_y = 0; mb_y < ctb_height_in_mb; mb_y++)
+        for (mb_y = 0; mb_y < ctb_height_in_mb_internal; mb_y++) 
         {
             mb_addr = macroblock_address + mb_y * width_in_mbs ;
-            for (mb_x = 0; mb_x < ctb_width_in_mb; mb_x++)
+            for (mb_x = 0; mb_x < ctb_width_in_mb_internal; mb_x++) 
             {
-                split_coding_unit_flag = ((ctb_width_in_mb == 2) ? HEVC_SPLIT_CU_FLAG_32_32 : HEVC_SPLIT_CU_FLAG_16_16);
                 /* get the mb info from the vme out */
                 msg = (unsigned int *)(msg_ptr + mb_addr * vme_context->vme_output.size_block);
 
-- 
1.9.1