[RFC] drm: return int error code from mode_fixup

[Grace An]

When CONFIG_PROVE_LOCKING is defined, the kernel randomly injects
-EDEADLK errors for all the ww_mutex. This results in
drm_atomic_get_private_obj_state randomly returning -EDEADLK.
However, the mode_fixup functions do not propagate these error
codes and return false, causing the atomic commit to fail with
-EINVAL instead of retrying.

Change encoder, crtc, and bridge mode_fixup functions to return
an int instead of a boolean to indicate success or failure. If
any of these functions fail, the mode_fixup function now returns
the provided integer error code instead of -EINVAL.

This change needs modifications across drivers, but before submitting
the entire change, we want to get feedback on this RFC.

Signed-off-by: Grace An <gracan at codeaurora.org>
---
 drivers/gpu/drm/drm_atomic_helper.c      | 8 ++++----
 drivers/gpu/drm/drm_bridge.c             | 4 ++--
 include/drm/drm_bridge.h                 | 2 +-
 include/drm/drm_modeset_helper_vtables.h | 4 ++--
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index f2b3e28..d75f09a 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -457,10 +457,10 @@ mode_fixup(struct drm_atomic_state *state)
 		} else if (funcs && funcs->mode_fixup) {
 			ret = funcs->mode_fixup(encoder, &new_crtc_state->mode,
 						&new_crtc_state->adjusted_mode);
-			if (!ret) {
+			if (ret) {
 				DRM_DEBUG_ATOMIC("[ENCODER:%d:%s] fixup failed\n",
 						 encoder->base.id, encoder->name);
-				return -EINVAL;
+				return ret;
 			}
 		}
 	}
@@ -481,10 +481,10 @@ mode_fixup(struct drm_atomic_state *state)
 
 		ret = funcs->mode_fixup(crtc, &new_crtc_state->mode,
 					&new_crtc_state->adjusted_mode);
-		if (!ret) {
+		if (ret) {
 			DRM_DEBUG_ATOMIC("[CRTC:%d:%s] fixup failed\n",
 					 crtc->base.id, crtc->name);
-			return -EINVAL;
+			return ret;
 		}
 	}
 
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index 64f0eff..3ad16b5 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -736,9 +736,9 @@ static int drm_atomic_bridge_check(struct drm_bridge *bridge,
 		if (ret)
 			return ret;
 	} else if (bridge->funcs->mode_fixup) {
-		if (!bridge->funcs->mode_fixup(bridge, &crtc_state->mode,
+		if (bridge->funcs->mode_fixup(bridge, &crtc_state->mode,
 					       &crtc_state->adjusted_mode))
-			return -EINVAL;
+			return ret;
 	}
 
 	return 0;
diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h
index 2195daa..5d02dfc 100644
--- a/include/drm/drm_bridge.h
+++ b/include/drm/drm_bridge.h
@@ -153,7 +153,7 @@ struct drm_bridge_funcs {
 	 * True if an acceptable configuration is possible, false if the modeset
 	 * operation should be rejected.
 	 */
-	bool (*mode_fixup)(struct drm_bridge *bridge,
+	int (*mode_fixup)(struct drm_bridge *bridge,
 			   const struct drm_display_mode *mode,
 			   struct drm_display_mode *adjusted_mode);
 	/**
diff --git a/include/drm/drm_modeset_helper_vtables.h b/include/drm/drm_modeset_helper_vtables.h
index f3a4b47..e305c97 100644
--- a/include/drm/drm_modeset_helper_vtables.h
+++ b/include/drm/drm_modeset_helper_vtables.h
@@ -184,7 +184,7 @@ struct drm_crtc_helper_funcs {
 	 * True if an acceptable configuration is possible, false if the modeset
 	 * operation should be rejected.
 	 */
-	bool (*mode_fixup)(struct drm_crtc *crtc,
+	int (*mode_fixup)(struct drm_crtc *crtc,
 			   const struct drm_display_mode *mode,
 			   struct drm_display_mode *adjusted_mode);
 
@@ -599,7 +599,7 @@ struct drm_encoder_helper_funcs {
 	 * True if an acceptable configuration is possible, false if the modeset
 	 * operation should be rejected.
 	 */
-	bool (*mode_fixup)(struct drm_encoder *encoder,
+	int (*mode_fixup)(struct drm_encoder *encoder,
 			   const struct drm_display_mode *mode,
 			   struct drm_display_mode *adjusted_mode);
 
-- 
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum,
a Linux Foundation Collaborative Project