[Intel-gfx] [PATCH 1/3] drm/i915: Move watermark handling to intel_wm.c
Matt Roper
matthew.d.roper at intel.com
Fri Feb 27 08:54:19 PST 2015
Move watermark handling from intel_pm.c to intel_wm.c and add a little
bit of kerneldoc to exported functions. We also add a new
intel_init_wm() function to setup memory timing information and
initialize the relevant watermark vfuncs.
Jira: VIZ-4612
Signed-off-by: Matt Roper <matthew.d.roper at intel.com>
---
drivers/gpu/drm/i915/Makefile | 3 +-
drivers/gpu/drm/i915/intel_display.c | 1 +
drivers/gpu/drm/i915/intel_drv.h | 3 +
drivers/gpu/drm/i915/intel_pm.c | 3450 ---------------------------------
drivers/gpu/drm/i915/intel_wm.c | 3525 ++++++++++++++++++++++++++++++++++
5 files changed, 3531 insertions(+), 3451 deletions(-)
create mode 100644 drivers/gpu/drm/i915/intel_wm.c
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index f025e7f..5fd2257 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -55,7 +55,8 @@ i915-y += intel_audio.o \
intel_overlay.o \
intel_psr.o \
intel_sideband.o \
- intel_sprite.o
+ intel_sprite.o \
+ intel_wm.o
i915-$(CONFIG_ACPI) += intel_acpi.o intel_opregion.o
i915-$(CONFIG_DRM_I915_FBDEV) += intel_fbdev.o
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index fad5f76..370c173 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -13213,6 +13213,7 @@ void intel_modeset_init(struct drm_device *dev)
intel_init_quirks(dev);
intel_init_pm(dev);
+ intel_init_wm(dev);
if (INTEL_INFO(dev)->num_pipes == 0)
return;
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 1fb1529..995d030 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -1258,6 +1258,9 @@ void intel_pre_disable_primary(struct drm_crtc *crtc);
/* intel_tv.c */
void intel_tv_init(struct drm_device *dev);
+/* intel_wm.c */
+void intel_init_wm(struct drm_device *dev);
+
/* intel_atomic.c */
int intel_atomic_check(struct drm_device *dev,
struct drm_atomic_state *state);
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 7dcb5b6..f906b05 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -94,3402 +94,6 @@ static void skl_init_clock_gating(struct drm_device *dev)
GEN8_LQSC_RO_PERF_DIS);
}
-static void i915_pineview_get_mem_freq(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- u32 tmp;
-
- tmp = I915_READ(CLKCFG);
-
- switch (tmp & CLKCFG_FSB_MASK) {
- case CLKCFG_FSB_533:
- dev_priv->fsb_freq = 533; /* 133*4 */
- break;
- case CLKCFG_FSB_800:
- dev_priv->fsb_freq = 800; /* 200*4 */
- break;
- case CLKCFG_FSB_667:
- dev_priv->fsb_freq = 667; /* 167*4 */
- break;
- case CLKCFG_FSB_400:
- dev_priv->fsb_freq = 400; /* 100*4 */
- break;
- }
-
- switch (tmp & CLKCFG_MEM_MASK) {
- case CLKCFG_MEM_533:
- dev_priv->mem_freq = 533;
- break;
- case CLKCFG_MEM_667:
- dev_priv->mem_freq = 667;
- break;
- case CLKCFG_MEM_800:
- dev_priv->mem_freq = 800;
- break;
- }
-
- /* detect pineview DDR3 setting */
- tmp = I915_READ(CSHRDDR3CTL);
- dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
-}
-
-static void i915_ironlake_get_mem_freq(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- u16 ddrpll, csipll;
-
- ddrpll = I915_READ16(DDRMPLL1);
- csipll = I915_READ16(CSIPLL0);
-
- switch (ddrpll & 0xff) {
- case 0xc:
- dev_priv->mem_freq = 800;
- break;
- case 0x10:
- dev_priv->mem_freq = 1066;
- break;
- case 0x14:
- dev_priv->mem_freq = 1333;
- break;
- case 0x18:
- dev_priv->mem_freq = 1600;
- break;
- default:
- DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
- ddrpll & 0xff);
- dev_priv->mem_freq = 0;
- break;
- }
-
- dev_priv->ips.r_t = dev_priv->mem_freq;
-
- switch (csipll & 0x3ff) {
- case 0x00c:
- dev_priv->fsb_freq = 3200;
- break;
- case 0x00e:
- dev_priv->fsb_freq = 3733;
- break;
- case 0x010:
- dev_priv->fsb_freq = 4266;
- break;
- case 0x012:
- dev_priv->fsb_freq = 4800;
- break;
- case 0x014:
- dev_priv->fsb_freq = 5333;
- break;
- case 0x016:
- dev_priv->fsb_freq = 5866;
- break;
- case 0x018:
- dev_priv->fsb_freq = 6400;
- break;
- default:
- DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
- csipll & 0x3ff);
- dev_priv->fsb_freq = 0;
- break;
- }
-
- if (dev_priv->fsb_freq == 3200) {
- dev_priv->ips.c_m = 0;
- } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
- dev_priv->ips.c_m = 1;
- } else {
- dev_priv->ips.c_m = 2;
- }
-}
-
-static const struct cxsr_latency cxsr_latency_table[] = {
- {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
- {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
- {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
- {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
- {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
-
- {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
- {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
- {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
- {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
- {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
-
- {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
- {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
- {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
- {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
- {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
-
- {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
- {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
- {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
- {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
- {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
-
- {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
- {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
- {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
- {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
- {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
-
- {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
- {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
- {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
- {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
- {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
-};
-
-static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
- int is_ddr3,
- int fsb,
- int mem)
-{
- const struct cxsr_latency *latency;
- int i;
-
- if (fsb == 0 || mem == 0)
- return NULL;
-
- for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
- latency = &cxsr_latency_table[i];
- if (is_desktop == latency->is_desktop &&
- is_ddr3 == latency->is_ddr3 &&
- fsb == latency->fsb_freq && mem == latency->mem_freq)
- return latency;
- }
-
- DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
-
- return NULL;
-}
-
-void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
-{
- struct drm_device *dev = dev_priv->dev;
- u32 val;
-
- if (IS_VALLEYVIEW(dev)) {
- I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
- } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) {
- I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
- } else if (IS_PINEVIEW(dev)) {
- val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
- val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
- I915_WRITE(DSPFW3, val);
- } else if (IS_I945G(dev) || IS_I945GM(dev)) {
- val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
- _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
- I915_WRITE(FW_BLC_SELF, val);
- } else if (IS_I915GM(dev)) {
- val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
- _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
- I915_WRITE(INSTPM, val);
- } else {
- return;
- }
-
- DRM_DEBUG_KMS("memory self-refresh is %s\n",
- enable ? "enabled" : "disabled");
-}
-
-/*
- * Latency for FIFO fetches is dependent on several factors:
- * - memory configuration (speed, channels)
- * - chipset
- * - current MCH state
- * It can be fairly high in some situations, so here we assume a fairly
- * pessimal value. It's a tradeoff between extra memory fetches (if we
- * set this value too high, the FIFO will fetch frequently to stay full)
- * and power consumption (set it too low to save power and we might see
- * FIFO underruns and display "flicker").
- *
- * A value of 5us seems to be a good balance; safe for very low end
- * platforms but not overly aggressive on lower latency configs.
- */
-static const int pessimal_latency_ns = 5000;
-
-static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- uint32_t dsparb = I915_READ(DSPARB);
- int size;
-
- size = dsparb & 0x7f;
- if (plane)
- size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
-
- DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
- plane ? "B" : "A", size);
-
- return size;
-}
-
-static int i830_get_fifo_size(struct drm_device *dev, int plane)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- uint32_t dsparb = I915_READ(DSPARB);
- int size;
-
- size = dsparb & 0x1ff;
- if (plane)
- size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
- size >>= 1; /* Convert to cachelines */
-
- DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
- plane ? "B" : "A", size);
-
- return size;
-}
-
-static int i845_get_fifo_size(struct drm_device *dev, int plane)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- uint32_t dsparb = I915_READ(DSPARB);
- int size;
-
- size = dsparb & 0x7f;
- size >>= 2; /* Convert to cachelines */
-
- DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
- plane ? "B" : "A",
- size);
-
- return size;
-}
-
-/* Pineview has different values for various configs */
-static const struct intel_watermark_params pineview_display_wm = {
- .fifo_size = PINEVIEW_DISPLAY_FIFO,
- .max_wm = PINEVIEW_MAX_WM,
- .default_wm = PINEVIEW_DFT_WM,
- .guard_size = PINEVIEW_GUARD_WM,
- .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params pineview_display_hplloff_wm = {
- .fifo_size = PINEVIEW_DISPLAY_FIFO,
- .max_wm = PINEVIEW_MAX_WM,
- .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
- .guard_size = PINEVIEW_GUARD_WM,
- .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params pineview_cursor_wm = {
- .fifo_size = PINEVIEW_CURSOR_FIFO,
- .max_wm = PINEVIEW_CURSOR_MAX_WM,
- .default_wm = PINEVIEW_CURSOR_DFT_WM,
- .guard_size = PINEVIEW_CURSOR_GUARD_WM,
- .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
- .fifo_size = PINEVIEW_CURSOR_FIFO,
- .max_wm = PINEVIEW_CURSOR_MAX_WM,
- .default_wm = PINEVIEW_CURSOR_DFT_WM,
- .guard_size = PINEVIEW_CURSOR_GUARD_WM,
- .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params g4x_wm_info = {
- .fifo_size = G4X_FIFO_SIZE,
- .max_wm = G4X_MAX_WM,
- .default_wm = G4X_MAX_WM,
- .guard_size = 2,
- .cacheline_size = G4X_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params g4x_cursor_wm_info = {
- .fifo_size = I965_CURSOR_FIFO,
- .max_wm = I965_CURSOR_MAX_WM,
- .default_wm = I965_CURSOR_DFT_WM,
- .guard_size = 2,
- .cacheline_size = G4X_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params valleyview_wm_info = {
- .fifo_size = VALLEYVIEW_FIFO_SIZE,
- .max_wm = VALLEYVIEW_MAX_WM,
- .default_wm = VALLEYVIEW_MAX_WM,
- .guard_size = 2,
- .cacheline_size = G4X_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params valleyview_cursor_wm_info = {
- .fifo_size = I965_CURSOR_FIFO,
- .max_wm = VALLEYVIEW_CURSOR_MAX_WM,
- .default_wm = I965_CURSOR_DFT_WM,
- .guard_size = 2,
- .cacheline_size = G4X_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params i965_cursor_wm_info = {
- .fifo_size = I965_CURSOR_FIFO,
- .max_wm = I965_CURSOR_MAX_WM,
- .default_wm = I965_CURSOR_DFT_WM,
- .guard_size = 2,
- .cacheline_size = I915_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params i945_wm_info = {
- .fifo_size = I945_FIFO_SIZE,
- .max_wm = I915_MAX_WM,
- .default_wm = 1,
- .guard_size = 2,
- .cacheline_size = I915_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params i915_wm_info = {
- .fifo_size = I915_FIFO_SIZE,
- .max_wm = I915_MAX_WM,
- .default_wm = 1,
- .guard_size = 2,
- .cacheline_size = I915_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params i830_a_wm_info = {
- .fifo_size = I855GM_FIFO_SIZE,
- .max_wm = I915_MAX_WM,
- .default_wm = 1,
- .guard_size = 2,
- .cacheline_size = I830_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params i830_bc_wm_info = {
- .fifo_size = I855GM_FIFO_SIZE,
- .max_wm = I915_MAX_WM/2,
- .default_wm = 1,
- .guard_size = 2,
- .cacheline_size = I830_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params i845_wm_info = {
- .fifo_size = I830_FIFO_SIZE,
- .max_wm = I915_MAX_WM,
- .default_wm = 1,
- .guard_size = 2,
- .cacheline_size = I830_FIFO_LINE_SIZE,
-};
-
-/**
- * intel_calculate_wm - calculate watermark level
- * @clock_in_khz: pixel clock
- * @wm: chip FIFO params
- * @pixel_size: display pixel size
- * @latency_ns: memory latency for the platform
- *
- * Calculate the watermark level (the level at which the display plane will
- * start fetching from memory again). Each chip has a different display
- * FIFO size and allocation, so the caller needs to figure that out and pass
- * in the correct intel_watermark_params structure.
- *
- * As the pixel clock runs, the FIFO will be drained at a rate that depends
- * on the pixel size. When it reaches the watermark level, it'll start
- * fetching FIFO line sized based chunks from memory until the FIFO fills
- * past the watermark point. If the FIFO drains completely, a FIFO underrun
- * will occur, and a display engine hang could result.
- */
-static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
- const struct intel_watermark_params *wm,
- int fifo_size,
- int pixel_size,
- unsigned long latency_ns)
-{
- long entries_required, wm_size;
-
- /*
- * Note: we need to make sure we don't overflow for various clock &
- * latency values.
- * clocks go from a few thousand to several hundred thousand.
- * latency is usually a few thousand
- */
- entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
- 1000;
- entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
-
- DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
-
- wm_size = fifo_size - (entries_required + wm->guard_size);
-
- DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
-
- /* Don't promote wm_size to unsigned... */
- if (wm_size > (long)wm->max_wm)
- wm_size = wm->max_wm;
- if (wm_size <= 0)
- wm_size = wm->default_wm;
-
- /*
- * Bspec seems to indicate that the value shouldn't be lower than
- * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
- * Lets go for 8 which is the burst size since certain platforms
- * already use a hardcoded 8 (which is what the spec says should be
- * done).
- */
- if (wm_size <= 8)
- wm_size = 8;
-
- return wm_size;
-}
-
-static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
-{
- struct drm_crtc *crtc, *enabled = NULL;
-
- for_each_crtc(dev, crtc) {
- if (intel_crtc_active(crtc)) {
- if (enabled)
- return NULL;
- enabled = crtc;
- }
- }
-
- return enabled;
-}
-
-static void pineview_update_wm(struct drm_crtc *unused_crtc)
-{
- struct drm_device *dev = unused_crtc->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct drm_crtc *crtc;
- const struct cxsr_latency *latency;
- u32 reg;
- unsigned long wm;
-
- latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
- dev_priv->fsb_freq, dev_priv->mem_freq);
- if (!latency) {
- DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
- intel_set_memory_cxsr(dev_priv, false);
- return;
- }
-
- crtc = single_enabled_crtc(dev);
- if (crtc) {
- const struct drm_display_mode *adjusted_mode;
- int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
- int clock;
-
- adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
- clock = adjusted_mode->crtc_clock;
-
- /* Display SR */
- wm = intel_calculate_wm(clock, &pineview_display_wm,
- pineview_display_wm.fifo_size,
- pixel_size, latency->display_sr);
- reg = I915_READ(DSPFW1);
- reg &= ~DSPFW_SR_MASK;
- reg |= wm << DSPFW_SR_SHIFT;
- I915_WRITE(DSPFW1, reg);
- DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
-
- /* cursor SR */
- wm = intel_calculate_wm(clock, &pineview_cursor_wm,
- pineview_display_wm.fifo_size,
- pixel_size, latency->cursor_sr);
- reg = I915_READ(DSPFW3);
- reg &= ~DSPFW_CURSOR_SR_MASK;
- reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
- I915_WRITE(DSPFW3, reg);
-
- /* Display HPLL off SR */
- wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
- pineview_display_hplloff_wm.fifo_size,
- pixel_size, latency->display_hpll_disable);
- reg = I915_READ(DSPFW3);
- reg &= ~DSPFW_HPLL_SR_MASK;
- reg |= wm & DSPFW_HPLL_SR_MASK;
- I915_WRITE(DSPFW3, reg);
-
- /* cursor HPLL off SR */
- wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
- pineview_display_hplloff_wm.fifo_size,
- pixel_size, latency->cursor_hpll_disable);
- reg = I915_READ(DSPFW3);
- reg &= ~DSPFW_HPLL_CURSOR_MASK;
- reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
- I915_WRITE(DSPFW3, reg);
- DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
-
- intel_set_memory_cxsr(dev_priv, true);
- } else {
- intel_set_memory_cxsr(dev_priv, false);
- }
-}
-
-static bool g4x_compute_wm0(struct drm_device *dev,
- int plane,
- const struct intel_watermark_params *display,
- int display_latency_ns,
- const struct intel_watermark_params *cursor,
- int cursor_latency_ns,
- int *plane_wm,
- int *cursor_wm)
-{
- struct drm_crtc *crtc;
- const struct drm_display_mode *adjusted_mode;
- int htotal, hdisplay, clock, pixel_size;
- int line_time_us, line_count;
- int entries, tlb_miss;
-
- crtc = intel_get_crtc_for_plane(dev, plane);
- if (!intel_crtc_active(crtc)) {
- *cursor_wm = cursor->guard_size;
- *plane_wm = display->guard_size;
- return false;
- }
-
- adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
- clock = adjusted_mode->crtc_clock;
- htotal = adjusted_mode->crtc_htotal;
- hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
- pixel_size = crtc->primary->fb->bits_per_pixel / 8;
-
- /* Use the small buffer method to calculate plane watermark */
- entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
- tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
- if (tlb_miss > 0)
- entries += tlb_miss;
- entries = DIV_ROUND_UP(entries, display->cacheline_size);
- *plane_wm = entries + display->guard_size;
- if (*plane_wm > (int)display->max_wm)
- *plane_wm = display->max_wm;
-
- /* Use the large buffer method to calculate cursor watermark */
- line_time_us = max(htotal * 1000 / clock, 1);
- line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
- entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size;
- tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
- if (tlb_miss > 0)
- entries += tlb_miss;
- entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
- *cursor_wm = entries + cursor->guard_size;
- if (*cursor_wm > (int)cursor->max_wm)
- *cursor_wm = (int)cursor->max_wm;
-
- return true;
-}
-
-/*
- * Check the wm result.
- *
- * If any calculated watermark values is larger than the maximum value that
- * can be programmed into the associated watermark register, that watermark
- * must be disabled.
- */
-static bool g4x_check_srwm(struct drm_device *dev,
- int display_wm, int cursor_wm,
- const struct intel_watermark_params *display,
- const struct intel_watermark_params *cursor)
-{
- DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
- display_wm, cursor_wm);
-
- if (display_wm > display->max_wm) {
- DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
- display_wm, display->max_wm);
- return false;
- }
-
- if (cursor_wm > cursor->max_wm) {
- DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
- cursor_wm, cursor->max_wm);
- return false;
- }
-
- if (!(display_wm || cursor_wm)) {
- DRM_DEBUG_KMS("SR latency is 0, disabling\n");
- return false;
- }
-
- return true;
-}
-
-static bool g4x_compute_srwm(struct drm_device *dev,
- int plane,
- int latency_ns,
- const struct intel_watermark_params *display,
- const struct intel_watermark_params *cursor,
- int *display_wm, int *cursor_wm)
-{
- struct drm_crtc *crtc;
- const struct drm_display_mode *adjusted_mode;
- int hdisplay, htotal, pixel_size, clock;
- unsigned long line_time_us;
- int line_count, line_size;
- int small, large;
- int entries;
-
- if (!latency_ns) {
- *display_wm = *cursor_wm = 0;
- return false;
- }
-
- crtc = intel_get_crtc_for_plane(dev, plane);
- adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
- clock = adjusted_mode->crtc_clock;
- htotal = adjusted_mode->crtc_htotal;
- hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
- pixel_size = crtc->primary->fb->bits_per_pixel / 8;
-
- line_time_us = max(htotal * 1000 / clock, 1);
- line_count = (latency_ns / line_time_us + 1000) / 1000;
- line_size = hdisplay * pixel_size;
-
- /* Use the minimum of the small and large buffer method for primary */
- small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
- large = line_count * line_size;
-
- entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
- *display_wm = entries + display->guard_size;
-
- /* calculate the self-refresh watermark for display cursor */
- entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width;
- entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
- *cursor_wm = entries + cursor->guard_size;
-
- return g4x_check_srwm(dev,
- *display_wm, *cursor_wm,
- display, cursor);
-}
-
-static bool vlv_compute_drain_latency(struct drm_crtc *crtc,
- int pixel_size,
- int *prec_mult,
- int *drain_latency)
-{
- struct drm_device *dev = crtc->dev;
- int entries;
- int clock = to_intel_crtc(crtc)->config->base.adjusted_mode.crtc_clock;
-
- if (WARN(clock == 0, "Pixel clock is zero!\n"))
- return false;
-
- if (WARN(pixel_size == 0, "Pixel size is zero!\n"))
- return false;
-
- entries = DIV_ROUND_UP(clock, 1000) * pixel_size;
- if (IS_CHERRYVIEW(dev))
- *prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_32 :
- DRAIN_LATENCY_PRECISION_16;
- else
- *prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_64 :
- DRAIN_LATENCY_PRECISION_32;
- *drain_latency = (64 * (*prec_mult) * 4) / entries;
-
- if (*drain_latency > DRAIN_LATENCY_MASK)
- *drain_latency = DRAIN_LATENCY_MASK;
-
- return true;
-}
-
-/*
- * Update drain latency registers of memory arbiter
- *
- * Valleyview SoC has a new memory arbiter and needs drain latency registers
- * to be programmed. Each plane has a drain latency multiplier and a drain
- * latency value.
- */
-
-static void vlv_update_drain_latency(struct drm_crtc *crtc)
-{
- struct drm_device *dev = crtc->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- int pixel_size;
- int drain_latency;
- enum pipe pipe = intel_crtc->pipe;
- int plane_prec, prec_mult, plane_dl;
- const int high_precision = IS_CHERRYVIEW(dev) ?
- DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_64;
-
- plane_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_PLANE_PRECISION_HIGH |
- DRAIN_LATENCY_MASK | DDL_CURSOR_PRECISION_HIGH |
- (DRAIN_LATENCY_MASK << DDL_CURSOR_SHIFT));
-
- if (!intel_crtc_active(crtc)) {
- I915_WRITE(VLV_DDL(pipe), plane_dl);
- return;
- }
-
- /* Primary plane Drain Latency */
- pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */
- if (vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
- plane_prec = (prec_mult == high_precision) ?
- DDL_PLANE_PRECISION_HIGH :
- DDL_PLANE_PRECISION_LOW;
- plane_dl |= plane_prec | drain_latency;
- }
-
- /* Cursor Drain Latency
- * BPP is always 4 for cursor
- */
- pixel_size = 4;
-
- /* Program cursor DL only if it is enabled */
- if (intel_crtc->cursor_base &&
- vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
- plane_prec = (prec_mult == high_precision) ?
- DDL_CURSOR_PRECISION_HIGH :
- DDL_CURSOR_PRECISION_LOW;
- plane_dl |= plane_prec | (drain_latency << DDL_CURSOR_SHIFT);
- }
-
- I915_WRITE(VLV_DDL(pipe), plane_dl);
-}
-
-#define single_plane_enabled(mask) is_power_of_2(mask)
-
-static void valleyview_update_wm(struct drm_crtc *crtc)
-{
- struct drm_device *dev = crtc->dev;
- static const int sr_latency_ns = 12000;
- struct drm_i915_private *dev_priv = dev->dev_private;
- int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
- int plane_sr, cursor_sr;
- int ignore_plane_sr, ignore_cursor_sr;
- unsigned int enabled = 0;
- bool cxsr_enabled;
-
- vlv_update_drain_latency(crtc);
-
- if (g4x_compute_wm0(dev, PIPE_A,
- &valleyview_wm_info, pessimal_latency_ns,
- &valleyview_cursor_wm_info, pessimal_latency_ns,
- &planea_wm, &cursora_wm))
- enabled |= 1 << PIPE_A;
-
- if (g4x_compute_wm0(dev, PIPE_B,
- &valleyview_wm_info, pessimal_latency_ns,
- &valleyview_cursor_wm_info, pessimal_latency_ns,
- &planeb_wm, &cursorb_wm))
- enabled |= 1 << PIPE_B;
-
- if (single_plane_enabled(enabled) &&
- g4x_compute_srwm(dev, ffs(enabled) - 1,
- sr_latency_ns,
- &valleyview_wm_info,
- &valleyview_cursor_wm_info,
- &plane_sr, &ignore_cursor_sr) &&
- g4x_compute_srwm(dev, ffs(enabled) - 1,
- 2*sr_latency_ns,
- &valleyview_wm_info,
- &valleyview_cursor_wm_info,
- &ignore_plane_sr, &cursor_sr)) {
- cxsr_enabled = true;
- } else {
- cxsr_enabled = false;
- intel_set_memory_cxsr(dev_priv, false);
- plane_sr = cursor_sr = 0;
- }
-
- DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
- "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
- planea_wm, cursora_wm,
- planeb_wm, cursorb_wm,
- plane_sr, cursor_sr);
-
- I915_WRITE(DSPFW1,
- (plane_sr << DSPFW_SR_SHIFT) |
- (cursorb_wm << DSPFW_CURSORB_SHIFT) |
- (planeb_wm << DSPFW_PLANEB_SHIFT) |
- (planea_wm << DSPFW_PLANEA_SHIFT));
- I915_WRITE(DSPFW2,
- (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
- (cursora_wm << DSPFW_CURSORA_SHIFT));
- I915_WRITE(DSPFW3,
- (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
- (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
-
- if (cxsr_enabled)
- intel_set_memory_cxsr(dev_priv, true);
-}
-
-static void cherryview_update_wm(struct drm_crtc *crtc)
-{
- struct drm_device *dev = crtc->dev;
- static const int sr_latency_ns = 12000;
- struct drm_i915_private *dev_priv = dev->dev_private;
- int planea_wm, planeb_wm, planec_wm;
- int cursora_wm, cursorb_wm, cursorc_wm;
- int plane_sr, cursor_sr;
- int ignore_plane_sr, ignore_cursor_sr;
- unsigned int enabled = 0;
- bool cxsr_enabled;
-
- vlv_update_drain_latency(crtc);
-
- if (g4x_compute_wm0(dev, PIPE_A,
- &valleyview_wm_info, pessimal_latency_ns,
- &valleyview_cursor_wm_info, pessimal_latency_ns,
- &planea_wm, &cursora_wm))
- enabled |= 1 << PIPE_A;
-
- if (g4x_compute_wm0(dev, PIPE_B,
- &valleyview_wm_info, pessimal_latency_ns,
- &valleyview_cursor_wm_info, pessimal_latency_ns,
- &planeb_wm, &cursorb_wm))
- enabled |= 1 << PIPE_B;
-
- if (g4x_compute_wm0(dev, PIPE_C,
- &valleyview_wm_info, pessimal_latency_ns,
- &valleyview_cursor_wm_info, pessimal_latency_ns,
- &planec_wm, &cursorc_wm))
- enabled |= 1 << PIPE_C;
-
- if (single_plane_enabled(enabled) &&
- g4x_compute_srwm(dev, ffs(enabled) - 1,
- sr_latency_ns,
- &valleyview_wm_info,
- &valleyview_cursor_wm_info,
- &plane_sr, &ignore_cursor_sr) &&
- g4x_compute_srwm(dev, ffs(enabled) - 1,
- 2*sr_latency_ns,
- &valleyview_wm_info,
- &valleyview_cursor_wm_info,
- &ignore_plane_sr, &cursor_sr)) {
- cxsr_enabled = true;
- } else {
- cxsr_enabled = false;
- intel_set_memory_cxsr(dev_priv, false);
- plane_sr = cursor_sr = 0;
- }
-
- DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
- "B: plane=%d, cursor=%d, C: plane=%d, cursor=%d, "
- "SR: plane=%d, cursor=%d\n",
- planea_wm, cursora_wm,
- planeb_wm, cursorb_wm,
- planec_wm, cursorc_wm,
- plane_sr, cursor_sr);
-
- I915_WRITE(DSPFW1,
- (plane_sr << DSPFW_SR_SHIFT) |
- (cursorb_wm << DSPFW_CURSORB_SHIFT) |
- (planeb_wm << DSPFW_PLANEB_SHIFT) |
- (planea_wm << DSPFW_PLANEA_SHIFT));
- I915_WRITE(DSPFW2,
- (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
- (cursora_wm << DSPFW_CURSORA_SHIFT));
- I915_WRITE(DSPFW3,
- (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
- (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
- I915_WRITE(DSPFW9_CHV,
- (I915_READ(DSPFW9_CHV) & ~(DSPFW_PLANEC_MASK |
- DSPFW_CURSORC_MASK)) |
- (planec_wm << DSPFW_PLANEC_SHIFT) |
- (cursorc_wm << DSPFW_CURSORC_SHIFT));
-
- if (cxsr_enabled)
- intel_set_memory_cxsr(dev_priv, true);
-}
-
-static void valleyview_update_sprite_wm(struct drm_plane *plane,
- struct drm_crtc *crtc,
- uint32_t sprite_width,
- uint32_t sprite_height,
- int pixel_size,
- bool enabled, bool scaled)
-{
- struct drm_device *dev = crtc->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- int pipe = to_intel_plane(plane)->pipe;
- int sprite = to_intel_plane(plane)->plane;
- int drain_latency;
- int plane_prec;
- int sprite_dl;
- int prec_mult;
- const int high_precision = IS_CHERRYVIEW(dev) ?
- DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_64;
-
- sprite_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_SPRITE_PRECISION_HIGH(sprite) |
- (DRAIN_LATENCY_MASK << DDL_SPRITE_SHIFT(sprite)));
-
- if (enabled && vlv_compute_drain_latency(crtc, pixel_size, &prec_mult,
- &drain_latency)) {
- plane_prec = (prec_mult == high_precision) ?
- DDL_SPRITE_PRECISION_HIGH(sprite) :
- DDL_SPRITE_PRECISION_LOW(sprite);
- sprite_dl |= plane_prec |
- (drain_latency << DDL_SPRITE_SHIFT(sprite));
- }
-
- I915_WRITE(VLV_DDL(pipe), sprite_dl);
-}
-
-static void g4x_update_wm(struct drm_crtc *crtc)
-{
- struct drm_device *dev = crtc->dev;
- static const int sr_latency_ns = 12000;
- struct drm_i915_private *dev_priv = dev->dev_private;
- int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
- int plane_sr, cursor_sr;
- unsigned int enabled = 0;
- bool cxsr_enabled;
-
- if (g4x_compute_wm0(dev, PIPE_A,
- &g4x_wm_info, pessimal_latency_ns,
- &g4x_cursor_wm_info, pessimal_latency_ns,
- &planea_wm, &cursora_wm))
- enabled |= 1 << PIPE_A;
-
- if (g4x_compute_wm0(dev, PIPE_B,
- &g4x_wm_info, pessimal_latency_ns,
- &g4x_cursor_wm_info, pessimal_latency_ns,
- &planeb_wm, &cursorb_wm))
- enabled |= 1 << PIPE_B;
-
- if (single_plane_enabled(enabled) &&
- g4x_compute_srwm(dev, ffs(enabled) - 1,
- sr_latency_ns,
- &g4x_wm_info,
- &g4x_cursor_wm_info,
- &plane_sr, &cursor_sr)) {
- cxsr_enabled = true;
- } else {
- cxsr_enabled = false;
- intel_set_memory_cxsr(dev_priv, false);
- plane_sr = cursor_sr = 0;
- }
-
- DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
- "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
- planea_wm, cursora_wm,
- planeb_wm, cursorb_wm,
- plane_sr, cursor_sr);
-
- I915_WRITE(DSPFW1,
- (plane_sr << DSPFW_SR_SHIFT) |
- (cursorb_wm << DSPFW_CURSORB_SHIFT) |
- (planeb_wm << DSPFW_PLANEB_SHIFT) |
- (planea_wm << DSPFW_PLANEA_SHIFT));
- I915_WRITE(DSPFW2,
- (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
- (cursora_wm << DSPFW_CURSORA_SHIFT));
- /* HPLL off in SR has some issues on G4x... disable it */
- I915_WRITE(DSPFW3,
- (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
- (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
-
- if (cxsr_enabled)
- intel_set_memory_cxsr(dev_priv, true);
-}
-
-static void i965_update_wm(struct drm_crtc *unused_crtc)
-{
- struct drm_device *dev = unused_crtc->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct drm_crtc *crtc;
- int srwm = 1;
- int cursor_sr = 16;
- bool cxsr_enabled;
-
- /* Calc sr entries for one plane configs */
- crtc = single_enabled_crtc(dev);
- if (crtc) {
- /* self-refresh has much higher latency */
- static const int sr_latency_ns = 12000;
- const struct drm_display_mode *adjusted_mode =
- &to_intel_crtc(crtc)->config->base.adjusted_mode;
- int clock = adjusted_mode->crtc_clock;
- int htotal = adjusted_mode->crtc_htotal;
- int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
- int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
- unsigned long line_time_us;
- int entries;
-
- line_time_us = max(htotal * 1000 / clock, 1);
-
- /* Use ns/us then divide to preserve precision */
- entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
- pixel_size * hdisplay;
- entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
- srwm = I965_FIFO_SIZE - entries;
- if (srwm < 0)
- srwm = 1;
- srwm &= 0x1ff;
- DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
- entries, srwm);
-
- entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
- pixel_size * to_intel_crtc(crtc)->cursor_width;
- entries = DIV_ROUND_UP(entries,
- i965_cursor_wm_info.cacheline_size);
- cursor_sr = i965_cursor_wm_info.fifo_size -
- (entries + i965_cursor_wm_info.guard_size);
-
- if (cursor_sr > i965_cursor_wm_info.max_wm)
- cursor_sr = i965_cursor_wm_info.max_wm;
-
- DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
- "cursor %d\n", srwm, cursor_sr);
-
- cxsr_enabled = true;
- } else {
- cxsr_enabled = false;
- /* Turn off self refresh if both pipes are enabled */
- intel_set_memory_cxsr(dev_priv, false);
- }
-
- DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
- srwm);
-
- /* 965 has limitations... */
- I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
- (8 << DSPFW_CURSORB_SHIFT) |
- (8 << DSPFW_PLANEB_SHIFT) |
- (8 << DSPFW_PLANEA_SHIFT));
- I915_WRITE(DSPFW2, (8 << DSPFW_CURSORA_SHIFT) |
- (8 << DSPFW_PLANEC_SHIFT_OLD));
- /* update cursor SR watermark */
- I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
-
- if (cxsr_enabled)
- intel_set_memory_cxsr(dev_priv, true);
-}
-
-static void i9xx_update_wm(struct drm_crtc *unused_crtc)
-{
- struct drm_device *dev = unused_crtc->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- const struct intel_watermark_params *wm_info;
- uint32_t fwater_lo;
- uint32_t fwater_hi;
- int cwm, srwm = 1;
- int fifo_size;
- int planea_wm, planeb_wm;
- struct drm_crtc *crtc, *enabled = NULL;
-
- if (IS_I945GM(dev))
- wm_info = &i945_wm_info;
- else if (!IS_GEN2(dev))
- wm_info = &i915_wm_info;
- else
- wm_info = &i830_a_wm_info;
-
- fifo_size = dev_priv->display.get_fifo_size(dev, 0);
- crtc = intel_get_crtc_for_plane(dev, 0);
- if (intel_crtc_active(crtc)) {
- const struct drm_display_mode *adjusted_mode;
- int cpp = crtc->primary->fb->bits_per_pixel / 8;
- if (IS_GEN2(dev))
- cpp = 4;
-
- adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
- planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
- wm_info, fifo_size, cpp,
- pessimal_latency_ns);
- enabled = crtc;
- } else {
- planea_wm = fifo_size - wm_info->guard_size;
- if (planea_wm > (long)wm_info->max_wm)
- planea_wm = wm_info->max_wm;
- }
-
- if (IS_GEN2(dev))
- wm_info = &i830_bc_wm_info;
-
- fifo_size = dev_priv->display.get_fifo_size(dev, 1);
- crtc = intel_get_crtc_for_plane(dev, 1);
- if (intel_crtc_active(crtc)) {
- const struct drm_display_mode *adjusted_mode;
- int cpp = crtc->primary->fb->bits_per_pixel / 8;
- if (IS_GEN2(dev))
- cpp = 4;
-
- adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
- planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
- wm_info, fifo_size, cpp,
- pessimal_latency_ns);
- if (enabled == NULL)
- enabled = crtc;
- else
- enabled = NULL;
- } else {
- planeb_wm = fifo_size - wm_info->guard_size;
- if (planeb_wm > (long)wm_info->max_wm)
- planeb_wm = wm_info->max_wm;
- }
-
- DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
-
- if (IS_I915GM(dev) && enabled) {
- struct drm_i915_gem_object *obj;
-
- obj = intel_fb_obj(enabled->primary->fb);
-
- /* self-refresh seems busted with untiled */
- if (obj->tiling_mode == I915_TILING_NONE)
- enabled = NULL;
- }
-
- /*
- * Overlay gets an aggressive default since video jitter is bad.
- */
- cwm = 2;
-
- /* Play safe and disable self-refresh before adjusting watermarks. */
- intel_set_memory_cxsr(dev_priv, false);
-
- /* Calc sr entries for one plane configs */
- if (HAS_FW_BLC(dev) && enabled) {
- /* self-refresh has much higher latency */
- static const int sr_latency_ns = 6000;
- const struct drm_display_mode *adjusted_mode =
- &to_intel_crtc(enabled)->config->base.adjusted_mode;
- int clock = adjusted_mode->crtc_clock;
- int htotal = adjusted_mode->crtc_htotal;
- int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
- int pixel_size = enabled->primary->fb->bits_per_pixel / 8;
- unsigned long line_time_us;
- int entries;
-
- line_time_us = max(htotal * 1000 / clock, 1);
-
- /* Use ns/us then divide to preserve precision */
- entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
- pixel_size * hdisplay;
- entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
- DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
- srwm = wm_info->fifo_size - entries;
- if (srwm < 0)
- srwm = 1;
-
- if (IS_I945G(dev) || IS_I945GM(dev))
- I915_WRITE(FW_BLC_SELF,
- FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
- else if (IS_I915GM(dev))
- I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
- }
-
- DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
- planea_wm, planeb_wm, cwm, srwm);
-
- fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
- fwater_hi = (cwm & 0x1f);
-
- /* Set request length to 8 cachelines per fetch */
- fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
- fwater_hi = fwater_hi | (1 << 8);
-
- I915_WRITE(FW_BLC, fwater_lo);
- I915_WRITE(FW_BLC2, fwater_hi);
-
- if (enabled)
- intel_set_memory_cxsr(dev_priv, true);
-}
-
-static void i845_update_wm(struct drm_crtc *unused_crtc)
-{
- struct drm_device *dev = unused_crtc->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct drm_crtc *crtc;
- const struct drm_display_mode *adjusted_mode;
- uint32_t fwater_lo;
- int planea_wm;
-
- crtc = single_enabled_crtc(dev);
- if (crtc == NULL)
- return;
-
- adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
- planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
- &i845_wm_info,
- dev_priv->display.get_fifo_size(dev, 0),
- 4, pessimal_latency_ns);
- fwater_lo = I915_READ(FW_BLC) & ~0xfff;
- fwater_lo |= (3<<8) | planea_wm;
-
- DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
-
- I915_WRITE(FW_BLC, fwater_lo);
-}
-
-static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
- struct drm_crtc *crtc)
-{
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- uint32_t pixel_rate;
-
- pixel_rate = intel_crtc->config->base.adjusted_mode.crtc_clock;
-
- /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
- * adjust the pixel_rate here. */
-
- if (intel_crtc->config->pch_pfit.enabled) {
- uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
- uint32_t pfit_size = intel_crtc->config->pch_pfit.size;
-
- pipe_w = intel_crtc->config->pipe_src_w;
- pipe_h = intel_crtc->config->pipe_src_h;
- pfit_w = (pfit_size >> 16) & 0xFFFF;
- pfit_h = pfit_size & 0xFFFF;
- if (pipe_w < pfit_w)
- pipe_w = pfit_w;
- if (pipe_h < pfit_h)
- pipe_h = pfit_h;
-
- pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
- pfit_w * pfit_h);
- }
-
- return pixel_rate;
-}
-
-/* latency must be in 0.1us units. */
-static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
- uint32_t latency)
-{
- uint64_t ret;
-
- if (WARN(latency == 0, "Latency value missing\n"))
- return UINT_MAX;
-
- ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
- ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
-
- return ret;
-}
-
-/* latency must be in 0.1us units. */
-static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
- uint32_t horiz_pixels, uint8_t bytes_per_pixel,
- uint32_t latency)
-{
- uint32_t ret;
-
- if (WARN(latency == 0, "Latency value missing\n"))
- return UINT_MAX;
-
- ret = (latency * pixel_rate) / (pipe_htotal * 10000);
- ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
- ret = DIV_ROUND_UP(ret, 64) + 2;
- return ret;
-}
-
-static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
- uint8_t bytes_per_pixel)
-{
- return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
-}
-
-struct skl_pipe_wm_parameters {
- bool active;
- uint32_t pipe_htotal;
- uint32_t pixel_rate; /* in KHz */
- struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
- struct intel_plane_wm_parameters cursor;
-};
-
-struct ilk_pipe_wm_parameters {
- bool active;
- uint32_t pipe_htotal;
- uint32_t pixel_rate;
- struct intel_plane_wm_parameters pri;
- struct intel_plane_wm_parameters spr;
- struct intel_plane_wm_parameters cur;
-};
-
-struct ilk_wm_maximums {
- uint16_t pri;
- uint16_t spr;
- uint16_t cur;
- uint16_t fbc;
-};
-
-/* used in computing the new watermarks state */
-struct intel_wm_config {
- unsigned int num_pipes_active;
- bool sprites_enabled;
- bool sprites_scaled;
-};
-
-/*
- * For both WM_PIPE and WM_LP.
- * mem_value must be in 0.1us units.
- */
-static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
- uint32_t mem_value,
- bool is_lp)
-{
- uint32_t method1, method2;
-
- if (!params->active || !params->pri.enabled)
- return 0;
-
- method1 = ilk_wm_method1(params->pixel_rate,
- params->pri.bytes_per_pixel,
- mem_value);
-
- if (!is_lp)
- return method1;
-
- method2 = ilk_wm_method2(params->pixel_rate,
- params->pipe_htotal,
- params->pri.horiz_pixels,
- params->pri.bytes_per_pixel,
- mem_value);
-
- return min(method1, method2);
-}
-
-/*
- * For both WM_PIPE and WM_LP.
- * mem_value must be in 0.1us units.
- */
-static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
- uint32_t mem_value)
-{
- uint32_t method1, method2;
-
- if (!params->active || !params->spr.enabled)
- return 0;
-
- method1 = ilk_wm_method1(params->pixel_rate,
- params->spr.bytes_per_pixel,
- mem_value);
- method2 = ilk_wm_method2(params->pixel_rate,
- params->pipe_htotal,
- params->spr.horiz_pixels,
- params->spr.bytes_per_pixel,
- mem_value);
- return min(method1, method2);
-}
-
-/*
- * For both WM_PIPE and WM_LP.
- * mem_value must be in 0.1us units.
- */
-static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
- uint32_t mem_value)
-{
- if (!params->active || !params->cur.enabled)
- return 0;
-
- return ilk_wm_method2(params->pixel_rate,
- params->pipe_htotal,
- params->cur.horiz_pixels,
- params->cur.bytes_per_pixel,
- mem_value);
-}
-
-/* Only for WM_LP. */
-static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
- uint32_t pri_val)
-{
- if (!params->active || !params->pri.enabled)
- return 0;
-
- return ilk_wm_fbc(pri_val,
- params->pri.horiz_pixels,
- params->pri.bytes_per_pixel);
-}
-
-static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
-{
- if (INTEL_INFO(dev)->gen >= 8)
- return 3072;
- else if (INTEL_INFO(dev)->gen >= 7)
- return 768;
- else
- return 512;
-}
-
-static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
- int level, bool is_sprite)
-{
- if (INTEL_INFO(dev)->gen >= 8)
- /* BDW primary/sprite plane watermarks */
- return level == 0 ? 255 : 2047;
- else if (INTEL_INFO(dev)->gen >= 7)
- /* IVB/HSW primary/sprite plane watermarks */
- return level == 0 ? 127 : 1023;
- else if (!is_sprite)
- /* ILK/SNB primary plane watermarks */
- return level == 0 ? 127 : 511;
- else
- /* ILK/SNB sprite plane watermarks */
- return level == 0 ? 63 : 255;
-}
-
-static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
- int level)
-{
- if (INTEL_INFO(dev)->gen >= 7)
- return level == 0 ? 63 : 255;
- else
- return level == 0 ? 31 : 63;
-}
-
-static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
-{
- if (INTEL_INFO(dev)->gen >= 8)
- return 31;
- else
- return 15;
-}
-
-/* Calculate the maximum primary/sprite plane watermark */
-static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
- int level,
- const struct intel_wm_config *config,
- enum intel_ddb_partitioning ddb_partitioning,
- bool is_sprite)
-{
- unsigned int fifo_size = ilk_display_fifo_size(dev);
-
- /* if sprites aren't enabled, sprites get nothing */
- if (is_sprite && !config->sprites_enabled)
- return 0;
-
- /* HSW allows LP1+ watermarks even with multiple pipes */
- if (level == 0 || config->num_pipes_active > 1) {
- fifo_size /= INTEL_INFO(dev)->num_pipes;
-
- /*
- * For some reason the non self refresh
- * FIFO size is only half of the self
- * refresh FIFO size on ILK/SNB.
- */
- if (INTEL_INFO(dev)->gen <= 6)
- fifo_size /= 2;
- }
-
- if (config->sprites_enabled) {
- /* level 0 is always calculated with 1:1 split */
- if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
- if (is_sprite)
- fifo_size *= 5;
- fifo_size /= 6;
- } else {
- fifo_size /= 2;
- }
- }
-
- /* clamp to max that the registers can hold */
- return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
-}
-
-/* Calculate the maximum cursor plane watermark */
-static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
- int level,
- const struct intel_wm_config *config)
-{
- /* HSW LP1+ watermarks w/ multiple pipes */
- if (level > 0 && config->num_pipes_active > 1)
- return 64;
-
- /* otherwise just report max that registers can hold */
- return ilk_cursor_wm_reg_max(dev, level);
-}
-
-static void ilk_compute_wm_maximums(const struct drm_device *dev,
- int level,
- const struct intel_wm_config *config,
- enum intel_ddb_partitioning ddb_partitioning,
- struct ilk_wm_maximums *max)
-{
- max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
- max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
- max->cur = ilk_cursor_wm_max(dev, level, config);
- max->fbc = ilk_fbc_wm_reg_max(dev);
-}
-
-static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
- int level,
- struct ilk_wm_maximums *max)
-{
- max->pri = ilk_plane_wm_reg_max(dev, level, false);
- max->spr = ilk_plane_wm_reg_max(dev, level, true);
- max->cur = ilk_cursor_wm_reg_max(dev, level);
- max->fbc = ilk_fbc_wm_reg_max(dev);
-}
-
-static bool ilk_validate_wm_level(int level,
- const struct ilk_wm_maximums *max,
- struct intel_wm_level *result)
-{
- bool ret;
-
- /* already determined to be invalid? */
- if (!result->enable)
- return false;
-
- result->enable = result->pri_val <= max->pri &&
- result->spr_val <= max->spr &&
- result->cur_val <= max->cur;
-
- ret = result->enable;
-
- /*
- * HACK until we can pre-compute everything,
- * and thus fail gracefully if LP0 watermarks
- * are exceeded...
- */
- if (level == 0 && !result->enable) {
- if (result->pri_val > max->pri)
- DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
- level, result->pri_val, max->pri);
- if (result->spr_val > max->spr)
- DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
- level, result->spr_val, max->spr);
- if (result->cur_val > max->cur)
- DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
- level, result->cur_val, max->cur);
-
- result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
- result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
- result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
- result->enable = true;
- }
-
- return ret;
-}
-
-static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
- int level,
- const struct ilk_pipe_wm_parameters *p,
- struct intel_wm_level *result)
-{
- uint16_t pri_latency = dev_priv->wm.pri_latency[level];
- uint16_t spr_latency = dev_priv->wm.spr_latency[level];
- uint16_t cur_latency = dev_priv->wm.cur_latency[level];
-
- /* WM1+ latency values stored in 0.5us units */
- if (level > 0) {
- pri_latency *= 5;
- spr_latency *= 5;
- cur_latency *= 5;
- }
-
- result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
- result->spr_val = ilk_compute_spr_wm(p, spr_latency);
- result->cur_val = ilk_compute_cur_wm(p, cur_latency);
- result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
- result->enable = true;
-}
-
-static uint32_t
-hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- struct drm_display_mode *mode = &intel_crtc->config->base.adjusted_mode;
- u32 linetime, ips_linetime;
-
- if (!intel_crtc_active(crtc))
- return 0;
-
- /* The WM are computed with base on how long it takes to fill a single
- * row at the given clock rate, multiplied by 8.
- * */
- linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
- mode->crtc_clock);
- ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
- intel_ddi_get_cdclk_freq(dev_priv));
-
- return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
- PIPE_WM_LINETIME_TIME(linetime);
-}
-
-static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
-
- if (IS_GEN9(dev)) {
- uint32_t val;
- int ret, i;
- int level, max_level = ilk_wm_max_level(dev);
-
- /* read the first set of memory latencies[0:3] */
- val = 0; /* data0 to be programmed to 0 for first set */
- mutex_lock(&dev_priv->rps.hw_lock);
- ret = sandybridge_pcode_read(dev_priv,
- GEN9_PCODE_READ_MEM_LATENCY,
- &val);
- mutex_unlock(&dev_priv->rps.hw_lock);
-
- if (ret) {
- DRM_ERROR("SKL Mailbox read error = %d\n", ret);
- return;
- }
-
- wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
- wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
- GEN9_MEM_LATENCY_LEVEL_MASK;
- wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
- GEN9_MEM_LATENCY_LEVEL_MASK;
- wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
- GEN9_MEM_LATENCY_LEVEL_MASK;
-
- /* read the second set of memory latencies[4:7] */
- val = 1; /* data0 to be programmed to 1 for second set */
- mutex_lock(&dev_priv->rps.hw_lock);
- ret = sandybridge_pcode_read(dev_priv,
- GEN9_PCODE_READ_MEM_LATENCY,
- &val);
- mutex_unlock(&dev_priv->rps.hw_lock);
- if (ret) {
- DRM_ERROR("SKL Mailbox read error = %d\n", ret);
- return;
- }
-
- wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
- wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
- GEN9_MEM_LATENCY_LEVEL_MASK;
- wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
- GEN9_MEM_LATENCY_LEVEL_MASK;
- wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
- GEN9_MEM_LATENCY_LEVEL_MASK;
-
- /*
- * WaWmMemoryReadLatency:skl
- *
- * punit doesn't take into account the read latency so we need
- * to add 2us to the various latency levels we retrieve from
- * the punit.
- * - W0 is a bit special in that it's the only level that
- * can't be disabled if we want to have display working, so
- * we always add 2us there.
- * - For levels >=1, punit returns 0us latency when they are
- * disabled, so we respect that and don't add 2us then
- *
- * Additionally, if a level n (n > 1) has a 0us latency, all
- * levels m (m >= n) need to be disabled. We make sure to
- * sanitize the values out of the punit to satisfy this
- * requirement.
- */
- wm[0] += 2;
- for (level = 1; level <= max_level; level++)
- if (wm[level] != 0)
- wm[level] += 2;
- else {
- for (i = level + 1; i <= max_level; i++)
- wm[i] = 0;
-
- break;
- }
- } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
- uint64_t sskpd = I915_READ64(MCH_SSKPD);
-
- wm[0] = (sskpd >> 56) & 0xFF;
- if (wm[0] == 0)
- wm[0] = sskpd & 0xF;
- wm[1] = (sskpd >> 4) & 0xFF;
- wm[2] = (sskpd >> 12) & 0xFF;
- wm[3] = (sskpd >> 20) & 0x1FF;
- wm[4] = (sskpd >> 32) & 0x1FF;
- } else if (INTEL_INFO(dev)->gen >= 6) {
- uint32_t sskpd = I915_READ(MCH_SSKPD);
-
- wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
- wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
- wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
- wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
- } else if (INTEL_INFO(dev)->gen >= 5) {
- uint32_t mltr = I915_READ(MLTR_ILK);
-
- /* ILK primary LP0 latency is 700 ns */
- wm[0] = 7;
- wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
- wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
- }
-}
-
-static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
-{
- /* ILK sprite LP0 latency is 1300 ns */
- if (INTEL_INFO(dev)->gen == 5)
- wm[0] = 13;
-}
-
-static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
-{
- /* ILK cursor LP0 latency is 1300 ns */
- if (INTEL_INFO(dev)->gen == 5)
- wm[0] = 13;
-
- /* WaDoubleCursorLP3Latency:ivb */
- if (IS_IVYBRIDGE(dev))
- wm[3] *= 2;
-}
-
-int ilk_wm_max_level(const struct drm_device *dev)
-{
- /* how many WM levels are we expecting */
- if (IS_GEN9(dev))
- return 7;
- else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
- return 4;
- else if (INTEL_INFO(dev)->gen >= 6)
- return 3;
- else
- return 2;
-}
-
-static void intel_print_wm_latency(struct drm_device *dev,
- const char *name,
- const uint16_t wm[8])
-{
- int level, max_level = ilk_wm_max_level(dev);
-
- for (level = 0; level <= max_level; level++) {
- unsigned int latency = wm[level];
-
- if (latency == 0) {
- DRM_ERROR("%s WM%d latency not provided\n",
- name, level);
- continue;
- }
-
- /*
- * - latencies are in us on gen9.
- * - before then, WM1+ latency values are in 0.5us units
- */
- if (IS_GEN9(dev))
- latency *= 10;
- else if (level > 0)
- latency *= 5;
-
- DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
- name, level, wm[level],
- latency / 10, latency % 10);
- }
-}
-
-static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
- uint16_t wm[5], uint16_t min)
-{
- int level, max_level = ilk_wm_max_level(dev_priv->dev);
-
- if (wm[0] >= min)
- return false;
-
- wm[0] = max(wm[0], min);
- for (level = 1; level <= max_level; level++)
- wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
-
- return true;
-}
-
-static void snb_wm_latency_quirk(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- bool changed;
-
- /*
- * The BIOS provided WM memory latency values are often
- * inadequate for high resolution displays. Adjust them.
- */
- changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
- ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
- ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
-
- if (!changed)
- return;
-
- DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
- intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
- intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
- intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
-}
-
-static void ilk_setup_wm_latency(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
-
- intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
-
- memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
- sizeof(dev_priv->wm.pri_latency));
- memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
- sizeof(dev_priv->wm.pri_latency));
-
- intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
- intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
-
- intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
- intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
- intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
-
- if (IS_GEN6(dev))
- snb_wm_latency_quirk(dev);
-}
-
-static void skl_setup_wm_latency(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
-
- intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
- intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
-}
-
-static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
- struct ilk_pipe_wm_parameters *p)
-{
- struct drm_device *dev = crtc->dev;
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- enum pipe pipe = intel_crtc->pipe;
- struct drm_plane *plane;
-
- if (!intel_crtc_active(crtc))
- return;
-
- p->active = true;
- p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
- p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
- p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8;
- p->cur.bytes_per_pixel = 4;
- p->pri.horiz_pixels = intel_crtc->config->pipe_src_w;
- p->cur.horiz_pixels = intel_crtc->cursor_width;
- /* TODO: for now, assume primary and cursor planes are always enabled. */
- p->pri.enabled = true;
- p->cur.enabled = true;
-
- drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
- struct intel_plane *intel_plane = to_intel_plane(plane);
-
- if (intel_plane->pipe == pipe) {
- p->spr = intel_plane->wm;
- break;
- }
- }
-}
-
-static void ilk_compute_wm_config(struct drm_device *dev,
- struct intel_wm_config *config)
-{
- struct intel_crtc *intel_crtc;
-
- /* Compute the currently _active_ config */
- for_each_intel_crtc(dev, intel_crtc) {
- const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
-
- if (!wm->pipe_enabled)
- continue;
-
- config->sprites_enabled |= wm->sprites_enabled;
- config->sprites_scaled |= wm->sprites_scaled;
- config->num_pipes_active++;
- }
-}
-
-/* Compute new watermarks for the pipe */
-static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
- const struct ilk_pipe_wm_parameters *params,
- struct intel_pipe_wm *pipe_wm)
-{
- struct drm_device *dev = crtc->dev;
- const struct drm_i915_private *dev_priv = dev->dev_private;
- int level, max_level = ilk_wm_max_level(dev);
- /* LP0 watermark maximums depend on this pipe alone */
- struct intel_wm_config config = {
- .num_pipes_active = 1,
- .sprites_enabled = params->spr.enabled,
- .sprites_scaled = params->spr.scaled,
- };
- struct ilk_wm_maximums max;
-
- pipe_wm->pipe_enabled = params->active;
- pipe_wm->sprites_enabled = params->spr.enabled;
- pipe_wm->sprites_scaled = params->spr.scaled;
-
- /* ILK/SNB: LP2+ watermarks only w/o sprites */
- if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
- max_level = 1;
-
- /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
- if (params->spr.scaled)
- max_level = 0;
-
- ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]);
-
- if (IS_HASWELL(dev) || IS_BROADWELL(dev))
- pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
-
- /* LP0 watermarks always use 1/2 DDB partitioning */
- ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
-
- /* At least LP0 must be valid */
- if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
- return false;
-
- ilk_compute_wm_reg_maximums(dev, 1, &max);
-
- for (level = 1; level <= max_level; level++) {
- struct intel_wm_level wm = {};
-
- ilk_compute_wm_level(dev_priv, level, params, &wm);
-
- /*
- * Disable any watermark level that exceeds the
- * register maximums since such watermarks are
- * always invalid.
- */
- if (!ilk_validate_wm_level(level, &max, &wm))
- break;
-
- pipe_wm->wm[level] = wm;
- }
-
- return true;
-}
-
-/*
- * Merge the watermarks from all active pipes for a specific level.
- */
-static void ilk_merge_wm_level(struct drm_device *dev,
- int level,
- struct intel_wm_level *ret_wm)
-{
- const struct intel_crtc *intel_crtc;
-
- ret_wm->enable = true;
-
- for_each_intel_crtc(dev, intel_crtc) {
- const struct intel_pipe_wm *active = &intel_crtc->wm.active;
- const struct intel_wm_level *wm = &active->wm[level];
-
- if (!active->pipe_enabled)
- continue;
-
- /*
- * The watermark values may have been used in the past,
- * so we must maintain them in the registers for some
- * time even if the level is now disabled.
- */
- if (!wm->enable)
- ret_wm->enable = false;
-
- ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
- ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
- ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
- ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
- }
-}
-
-/*
- * Merge all low power watermarks for all active pipes.
- */
-static void ilk_wm_merge(struct drm_device *dev,
- const struct intel_wm_config *config,
- const struct ilk_wm_maximums *max,
- struct intel_pipe_wm *merged)
-{
- int level, max_level = ilk_wm_max_level(dev);
- int last_enabled_level = max_level;
-
- /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
- if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
- config->num_pipes_active > 1)
- return;
-
- /* ILK: FBC WM must be disabled always */
- merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
-
- /* merge each WM1+ level */
- for (level = 1; level <= max_level; level++) {
- struct intel_wm_level *wm = &merged->wm[level];
-
- ilk_merge_wm_level(dev, level, wm);
-
- if (level > last_enabled_level)
- wm->enable = false;
- else if (!ilk_validate_wm_level(level, max, wm))
- /* make sure all following levels get disabled */
- last_enabled_level = level - 1;
-
- /*
- * The spec says it is preferred to disable
- * FBC WMs instead of disabling a WM level.
- */
- if (wm->fbc_val > max->fbc) {
- if (wm->enable)
- merged->fbc_wm_enabled = false;
- wm->fbc_val = 0;
- }
- }
-
- /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
- /*
- * FIXME this is racy. FBC might get enabled later.
- * What we should check here is whether FBC can be
- * enabled sometime later.
- */
- if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
- for (level = 2; level <= max_level; level++) {
- struct intel_wm_level *wm = &merged->wm[level];
-
- wm->enable = false;
- }
- }
-}
-
-static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
-{
- /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
- return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
-}
-
-/* The value we need to program into the WM_LPx latency field */
-static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
-
- if (IS_HASWELL(dev) || IS_BROADWELL(dev))
- return 2 * level;
- else
- return dev_priv->wm.pri_latency[level];
-}
-
-static void ilk_compute_wm_results(struct drm_device *dev,
- const struct intel_pipe_wm *merged,
- enum intel_ddb_partitioning partitioning,
- struct ilk_wm_values *results)
-{
- struct intel_crtc *intel_crtc;
- int level, wm_lp;
-
- results->enable_fbc_wm = merged->fbc_wm_enabled;
- results->partitioning = partitioning;
-
- /* LP1+ register values */
- for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
- const struct intel_wm_level *r;
-
- level = ilk_wm_lp_to_level(wm_lp, merged);
-
- r = &merged->wm[level];
-
- /*
- * Maintain the watermark values even if the level is
- * disabled. Doing otherwise could cause underruns.
- */
- results->wm_lp[wm_lp - 1] =
- (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
- (r->pri_val << WM1_LP_SR_SHIFT) |
- r->cur_val;
-
- if (r->enable)
- results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
-
- if (INTEL_INFO(dev)->gen >= 8)
- results->wm_lp[wm_lp - 1] |=
- r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
- else
- results->wm_lp[wm_lp - 1] |=
- r->fbc_val << WM1_LP_FBC_SHIFT;
-
- /*
- * Always set WM1S_LP_EN when spr_val != 0, even if the
- * level is disabled. Doing otherwise could cause underruns.
- */
- if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
- WARN_ON(wm_lp != 1);
- results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
- } else
- results->wm_lp_spr[wm_lp - 1] = r->spr_val;
- }
-
- /* LP0 register values */
- for_each_intel_crtc(dev, intel_crtc) {
- enum pipe pipe = intel_crtc->pipe;
- const struct intel_wm_level *r =
- &intel_crtc->wm.active.wm[0];
-
- if (WARN_ON(!r->enable))
- continue;
-
- results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
-
- results->wm_pipe[pipe] =
- (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
- (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
- r->cur_val;
- }
-}
-
-/* Find the result with the highest level enabled. Check for enable_fbc_wm in
- * case both are at the same level. Prefer r1 in case they're the same. */
-static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
- struct intel_pipe_wm *r1,
- struct intel_pipe_wm *r2)
-{
- int level, max_level = ilk_wm_max_level(dev);
- int level1 = 0, level2 = 0;
-
- for (level = 1; level <= max_level; level++) {
- if (r1->wm[level].enable)
- level1 = level;
- if (r2->wm[level].enable)
- level2 = level;
- }
-
- if (level1 == level2) {
- if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
- return r2;
- else
- return r1;
- } else if (level1 > level2) {
- return r1;
- } else {
- return r2;
- }
-}
-
-/* dirty bits used to track which watermarks need changes */
-#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
-#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
-#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
-#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
-#define WM_DIRTY_FBC (1 << 24)
-#define WM_DIRTY_DDB (1 << 25)
-
-static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
- const struct ilk_wm_values *old,
- const struct ilk_wm_values *new)
-{
- unsigned int dirty = 0;
- enum pipe pipe;
- int wm_lp;
-
- for_each_pipe(dev_priv, pipe) {
- if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
- dirty |= WM_DIRTY_LINETIME(pipe);
- /* Must disable LP1+ watermarks too */
- dirty |= WM_DIRTY_LP_ALL;
- }
-
- if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
- dirty |= WM_DIRTY_PIPE(pipe);
- /* Must disable LP1+ watermarks too */
- dirty |= WM_DIRTY_LP_ALL;
- }
- }
-
- if (old->enable_fbc_wm != new->enable_fbc_wm) {
- dirty |= WM_DIRTY_FBC;
- /* Must disable LP1+ watermarks too */
- dirty |= WM_DIRTY_LP_ALL;
- }
-
- if (old->partitioning != new->partitioning) {
- dirty |= WM_DIRTY_DDB;
- /* Must disable LP1+ watermarks too */
- dirty |= WM_DIRTY_LP_ALL;
- }
-
- /* LP1+ watermarks already deemed dirty, no need to continue */
- if (dirty & WM_DIRTY_LP_ALL)
- return dirty;
-
- /* Find the lowest numbered LP1+ watermark in need of an update... */
- for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
- if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
- old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
- break;
- }
-
- /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
- for (; wm_lp <= 3; wm_lp++)
- dirty |= WM_DIRTY_LP(wm_lp);
-
- return dirty;
-}
-
-static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
- unsigned int dirty)
-{
- struct ilk_wm_values *previous = &dev_priv->wm.hw;
- bool changed = false;
-
- if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
- previous->wm_lp[2] &= ~WM1_LP_SR_EN;
- I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
- changed = true;
- }
- if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
- previous->wm_lp[1] &= ~WM1_LP_SR_EN;
- I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
- changed = true;
- }
- if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
- previous->wm_lp[0] &= ~WM1_LP_SR_EN;
- I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
- changed = true;
- }
-
- /*
- * Don't touch WM1S_LP_EN here.
- * Doing so could cause underruns.
- */
-
- return changed;
-}
-
-/*
- * The spec says we shouldn't write when we don't need, because every write
- * causes WMs to be re-evaluated, expending some power.
- */
-static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
- struct ilk_wm_values *results)
-{
- struct drm_device *dev = dev_priv->dev;
- struct ilk_wm_values *previous = &dev_priv->wm.hw;
- unsigned int dirty;
- uint32_t val;
-
- dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
- if (!dirty)
- return;
-
- _ilk_disable_lp_wm(dev_priv, dirty);
-
- if (dirty & WM_DIRTY_PIPE(PIPE_A))
- I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
- if (dirty & WM_DIRTY_PIPE(PIPE_B))
- I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
- if (dirty & WM_DIRTY_PIPE(PIPE_C))
- I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
-
- if (dirty & WM_DIRTY_LINETIME(PIPE_A))
- I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
- if (dirty & WM_DIRTY_LINETIME(PIPE_B))
- I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
- if (dirty & WM_DIRTY_LINETIME(PIPE_C))
- I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
-
- if (dirty & WM_DIRTY_DDB) {
- if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
- val = I915_READ(WM_MISC);
- if (results->partitioning == INTEL_DDB_PART_1_2)
- val &= ~WM_MISC_DATA_PARTITION_5_6;
- else
- val |= WM_MISC_DATA_PARTITION_5_6;
- I915_WRITE(WM_MISC, val);
- } else {
- val = I915_READ(DISP_ARB_CTL2);
- if (results->partitioning == INTEL_DDB_PART_1_2)
- val &= ~DISP_DATA_PARTITION_5_6;
- else
- val |= DISP_DATA_PARTITION_5_6;
- I915_WRITE(DISP_ARB_CTL2, val);
- }
- }
-
- if (dirty & WM_DIRTY_FBC) {
- val = I915_READ(DISP_ARB_CTL);
- if (results->enable_fbc_wm)
- val &= ~DISP_FBC_WM_DIS;
- else
- val |= DISP_FBC_WM_DIS;
- I915_WRITE(DISP_ARB_CTL, val);
- }
-
- if (dirty & WM_DIRTY_LP(1) &&
- previous->wm_lp_spr[0] != results->wm_lp_spr[0])
- I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
-
- if (INTEL_INFO(dev)->gen >= 7) {
- if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
- I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
- if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
- I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
- }
-
- if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
- I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
- if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
- I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
- if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
- I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
-
- dev_priv->wm.hw = *results;
-}
-
-static bool ilk_disable_lp_wm(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
-
- return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
-}
-
-/*
- * On gen9, we need to allocate Display Data Buffer (DDB) portions to the
- * different active planes.
- */
-
-#define SKL_DDB_SIZE 896 /* in blocks */
-
-static void
-skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
- struct drm_crtc *for_crtc,
- const struct intel_wm_config *config,
- const struct skl_pipe_wm_parameters *params,
- struct skl_ddb_entry *alloc /* out */)
-{
- struct drm_crtc *crtc;
- unsigned int pipe_size, ddb_size;
- int nth_active_pipe;
-
- if (!params->active) {
- alloc->start = 0;
- alloc->end = 0;
- return;
- }
-
- ddb_size = SKL_DDB_SIZE;
-
- ddb_size -= 4; /* 4 blocks for bypass path allocation */
-
- nth_active_pipe = 0;
- for_each_crtc(dev, crtc) {
- if (!intel_crtc_active(crtc))
- continue;
-
- if (crtc == for_crtc)
- break;
-
- nth_active_pipe++;
- }
-
- pipe_size = ddb_size / config->num_pipes_active;
- alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
- alloc->end = alloc->start + pipe_size;
-}
-
-static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
-{
- if (config->num_pipes_active == 1)
- return 32;
-
- return 8;
-}
-
-static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
-{
- entry->start = reg & 0x3ff;
- entry->end = (reg >> 16) & 0x3ff;
- if (entry->end)
- entry->end += 1;
-}
-
-void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
- struct skl_ddb_allocation *ddb /* out */)
-{
- struct drm_device *dev = dev_priv->dev;
- enum pipe pipe;
- int plane;
- u32 val;
-
- for_each_pipe(dev_priv, pipe) {
- for_each_plane(pipe, plane) {
- val = I915_READ(PLANE_BUF_CFG(pipe, plane));
- skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
- val);
- }
-
- val = I915_READ(CUR_BUF_CFG(pipe));
- skl_ddb_entry_init_from_hw(&ddb->cursor[pipe], val);
- }
-}
-
-static unsigned int
-skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p)
-{
- return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
-}
-
-/*
- * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
- * a 8192x4096 at 32bpp framebuffer:
- * 3 * 4096 * 8192 * 4 < 2^32
- */
-static unsigned int
-skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
- const struct skl_pipe_wm_parameters *params)
-{
- unsigned int total_data_rate = 0;
- int plane;
-
- for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
- const struct intel_plane_wm_parameters *p;
-
- p = ¶ms->plane[plane];
- if (!p->enabled)
- continue;
-
- total_data_rate += skl_plane_relative_data_rate(p);
- }
-
- return total_data_rate;
-}
-
-static void
-skl_allocate_pipe_ddb(struct drm_crtc *crtc,
- const struct intel_wm_config *config,
- const struct skl_pipe_wm_parameters *params,
- struct skl_ddb_allocation *ddb /* out */)
-{
- struct drm_device *dev = crtc->dev;
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- enum pipe pipe = intel_crtc->pipe;
- struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
- uint16_t alloc_size, start, cursor_blocks;
- uint16_t minimum[I915_MAX_PLANES];
- unsigned int total_data_rate;
- int plane;
-
- skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
- alloc_size = skl_ddb_entry_size(alloc);
- if (alloc_size == 0) {
- memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
- memset(&ddb->cursor[pipe], 0, sizeof(ddb->cursor[pipe]));
- return;
- }
-
- cursor_blocks = skl_cursor_allocation(config);
- ddb->cursor[pipe].start = alloc->end - cursor_blocks;
- ddb->cursor[pipe].end = alloc->end;
-
- alloc_size -= cursor_blocks;
- alloc->end -= cursor_blocks;
-
- /* 1. Allocate the mininum required blocks for each active plane */
- for_each_plane(pipe, plane) {
- const struct intel_plane_wm_parameters *p;
-
- p = ¶ms->plane[plane];
- if (!p->enabled)
- continue;
-
- minimum[plane] = 8;
- alloc_size -= minimum[plane];
- }
-
- /*
- * 2. Distribute the remaining space in proportion to the amount of
- * data each plane needs to fetch from memory.
- *
- * FIXME: we may not allocate every single block here.
- */
- total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
-
- start = alloc->start;
- for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
- const struct intel_plane_wm_parameters *p;
- unsigned int data_rate;
- uint16_t plane_blocks;
-
- p = ¶ms->plane[plane];
- if (!p->enabled)
- continue;
-
- data_rate = skl_plane_relative_data_rate(p);
-
- /*
- * promote the expression to 64 bits to avoid overflowing, the
- * result is < available as data_rate / total_data_rate < 1
- */
- plane_blocks = minimum[plane];
- plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
- total_data_rate);
-
- ddb->plane[pipe][plane].start = start;
- ddb->plane[pipe][plane].end = start + plane_blocks;
-
- start += plane_blocks;
- }
-
-}
-
-static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
-{
- /* TODO: Take into account the scalers once we support them */
- return config->base.adjusted_mode.crtc_clock;
-}
-
-/*
- * The max latency should be 257 (max the punit can code is 255 and we add 2us
- * for the read latency) and bytes_per_pixel should always be <= 8, so that
- * should allow pixel_rate up to ~2 GHz which seems sufficient since max
- * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
-*/
-static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
- uint32_t latency)
-{
- uint32_t wm_intermediate_val, ret;
-
- if (latency == 0)
- return UINT_MAX;
-
- wm_intermediate_val = latency * pixel_rate * bytes_per_pixel;
- ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
-
- return ret;
-}
-
-static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
- uint32_t horiz_pixels, uint8_t bytes_per_pixel,
- uint32_t latency)
-{
- uint32_t ret, plane_bytes_per_line, wm_intermediate_val;
-
- if (latency == 0)
- return UINT_MAX;
-
- plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
- wm_intermediate_val = latency * pixel_rate;
- ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
- plane_bytes_per_line;
-
- return ret;
-}
-
-static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
- const struct intel_crtc *intel_crtc)
-{
- struct drm_device *dev = intel_crtc->base.dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
- enum pipe pipe = intel_crtc->pipe;
-
- if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
- sizeof(new_ddb->plane[pipe])))
- return true;
-
- if (memcmp(&new_ddb->cursor[pipe], &cur_ddb->cursor[pipe],
- sizeof(new_ddb->cursor[pipe])))
- return true;
-
- return false;
-}
-
-static void skl_compute_wm_global_parameters(struct drm_device *dev,
- struct intel_wm_config *config)
-{
- struct drm_crtc *crtc;
- struct drm_plane *plane;
-
- list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
- config->num_pipes_active += intel_crtc_active(crtc);
-
- /* FIXME: I don't think we need those two global parameters on SKL */
- list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
- struct intel_plane *intel_plane = to_intel_plane(plane);
-
- config->sprites_enabled |= intel_plane->wm.enabled;
- config->sprites_scaled |= intel_plane->wm.scaled;
- }
-}
-
-static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
- struct skl_pipe_wm_parameters *p)
-{
- struct drm_device *dev = crtc->dev;
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- enum pipe pipe = intel_crtc->pipe;
- struct drm_plane *plane;
- int i = 1; /* Index for sprite planes start */
-
- p->active = intel_crtc_active(crtc);
- if (p->active) {
- p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
- p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config);
-
- /*
- * For now, assume primary and cursor planes are always enabled.
- */
- p->plane[0].enabled = true;
- p->plane[0].bytes_per_pixel =
- crtc->primary->fb->bits_per_pixel / 8;
- p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w;
- p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h;
-
- p->cursor.enabled = true;
- p->cursor.bytes_per_pixel = 4;
- p->cursor.horiz_pixels = intel_crtc->cursor_width ?
- intel_crtc->cursor_width : 64;
- }
-
- list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
- struct intel_plane *intel_plane = to_intel_plane(plane);
-
- if (intel_plane->pipe == pipe &&
- plane->type == DRM_PLANE_TYPE_OVERLAY)
- p->plane[i++] = intel_plane->wm;
- }
-}
-
-static bool skl_compute_plane_wm(struct skl_pipe_wm_parameters *p,
- struct intel_plane_wm_parameters *p_params,
- uint16_t ddb_allocation,
- uint32_t mem_value,
- uint16_t *out_blocks, /* out */
- uint8_t *out_lines /* out */)
-{
- uint32_t method1, method2, plane_bytes_per_line, res_blocks, res_lines;
- uint32_t result_bytes;
-
- if (mem_value == 0 || !p->active || !p_params->enabled)
- return false;
-
- method1 = skl_wm_method1(p->pixel_rate,
- p_params->bytes_per_pixel,
- mem_value);
- method2 = skl_wm_method2(p->pixel_rate,
- p->pipe_htotal,
- p_params->horiz_pixels,
- p_params->bytes_per_pixel,
- mem_value);
-
- plane_bytes_per_line = p_params->horiz_pixels *
- p_params->bytes_per_pixel;
-
- /* For now xtile and linear */
- if (((ddb_allocation * 512) / plane_bytes_per_line) >= 1)
- result_bytes = min(method1, method2);
- else
- result_bytes = method1;
-
- res_blocks = DIV_ROUND_UP(result_bytes, 512) + 1;
- res_lines = DIV_ROUND_UP(result_bytes, plane_bytes_per_line);
-
- if (res_blocks > ddb_allocation || res_lines > 31)
- return false;
-
- *out_blocks = res_blocks;
- *out_lines = res_lines;
-
- return true;
-}
-
-static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
- struct skl_ddb_allocation *ddb,
- struct skl_pipe_wm_parameters *p,
- enum pipe pipe,
- int level,
- int num_planes,
- struct skl_wm_level *result)
-{
- uint16_t latency = dev_priv->wm.skl_latency[level];
- uint16_t ddb_blocks;
- int i;
-
- for (i = 0; i < num_planes; i++) {
- ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
-
- result->plane_en[i] = skl_compute_plane_wm(p, &p->plane[i],
- ddb_blocks,
- latency,
- &result->plane_res_b[i],
- &result->plane_res_l[i]);
- }
-
- ddb_blocks = skl_ddb_entry_size(&ddb->cursor[pipe]);
- result->cursor_en = skl_compute_plane_wm(p, &p->cursor, ddb_blocks,
- latency, &result->cursor_res_b,
- &result->cursor_res_l);
-}
-
-static uint32_t
-skl_compute_linetime_wm(struct drm_crtc *crtc, struct skl_pipe_wm_parameters *p)
-{
- if (!intel_crtc_active(crtc))
- return 0;
-
- return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
-
-}
-
-static void skl_compute_transition_wm(struct drm_crtc *crtc,
- struct skl_pipe_wm_parameters *params,
- struct skl_wm_level *trans_wm /* out */)
-{
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- int i;
-
- if (!params->active)
- return;
-
- /* Until we know more, just disable transition WMs */
- for (i = 0; i < intel_num_planes(intel_crtc); i++)
- trans_wm->plane_en[i] = false;
- trans_wm->cursor_en = false;
-}
-
-static void skl_compute_pipe_wm(struct drm_crtc *crtc,
- struct skl_ddb_allocation *ddb,
- struct skl_pipe_wm_parameters *params,
- struct skl_pipe_wm *pipe_wm)
-{
- struct drm_device *dev = crtc->dev;
- const struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- int level, max_level = ilk_wm_max_level(dev);
-
- for (level = 0; level <= max_level; level++) {
- skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
- level, intel_num_planes(intel_crtc),
- &pipe_wm->wm[level]);
- }
- pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
-
- skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
-}
-
-static void skl_compute_wm_results(struct drm_device *dev,
- struct skl_pipe_wm_parameters *p,
- struct skl_pipe_wm *p_wm,
- struct skl_wm_values *r,
- struct intel_crtc *intel_crtc)
-{
- int level, max_level = ilk_wm_max_level(dev);
- enum pipe pipe = intel_crtc->pipe;
- uint32_t temp;
- int i;
-
- for (level = 0; level <= max_level; level++) {
- for (i = 0; i < intel_num_planes(intel_crtc); i++) {
- temp = 0;
-
- temp |= p_wm->wm[level].plane_res_l[i] <<
- PLANE_WM_LINES_SHIFT;
- temp |= p_wm->wm[level].plane_res_b[i];
- if (p_wm->wm[level].plane_en[i])
- temp |= PLANE_WM_EN;
-
- r->plane[pipe][i][level] = temp;
- }
-
- temp = 0;
-
- temp |= p_wm->wm[level].cursor_res_l << PLANE_WM_LINES_SHIFT;
- temp |= p_wm->wm[level].cursor_res_b;
-
- if (p_wm->wm[level].cursor_en)
- temp |= PLANE_WM_EN;
-
- r->cursor[pipe][level] = temp;
-
- }
-
- /* transition WMs */
- for (i = 0; i < intel_num_planes(intel_crtc); i++) {
- temp = 0;
- temp |= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
- temp |= p_wm->trans_wm.plane_res_b[i];
- if (p_wm->trans_wm.plane_en[i])
- temp |= PLANE_WM_EN;
-
- r->plane_trans[pipe][i] = temp;
- }
-
- temp = 0;
- temp |= p_wm->trans_wm.cursor_res_l << PLANE_WM_LINES_SHIFT;
- temp |= p_wm->trans_wm.cursor_res_b;
- if (p_wm->trans_wm.cursor_en)
- temp |= PLANE_WM_EN;
-
- r->cursor_trans[pipe] = temp;
-
- r->wm_linetime[pipe] = p_wm->linetime;
-}
-
-static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
- const struct skl_ddb_entry *entry)
-{
- if (entry->end)
- I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
- else
- I915_WRITE(reg, 0);
-}
-
-static void skl_write_wm_values(struct drm_i915_private *dev_priv,
- const struct skl_wm_values *new)
-{
- struct drm_device *dev = dev_priv->dev;
- struct intel_crtc *crtc;
-
- list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
- int i, level, max_level = ilk_wm_max_level(dev);
- enum pipe pipe = crtc->pipe;
-
- if (!new->dirty[pipe])
- continue;
-
- I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
-
- for (level = 0; level <= max_level; level++) {
- for (i = 0; i < intel_num_planes(crtc); i++)
- I915_WRITE(PLANE_WM(pipe, i, level),
- new->plane[pipe][i][level]);
- I915_WRITE(CUR_WM(pipe, level),
- new->cursor[pipe][level]);
- }
- for (i = 0; i < intel_num_planes(crtc); i++)
- I915_WRITE(PLANE_WM_TRANS(pipe, i),
- new->plane_trans[pipe][i]);
- I915_WRITE(CUR_WM_TRANS(pipe), new->cursor_trans[pipe]);
-
- for (i = 0; i < intel_num_planes(crtc); i++)
- skl_ddb_entry_write(dev_priv,
- PLANE_BUF_CFG(pipe, i),
- &new->ddb.plane[pipe][i]);
-
- skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
- &new->ddb.cursor[pipe]);
- }
-}
-
-/*
- * When setting up a new DDB allocation arrangement, we need to correctly
- * sequence the times at which the new allocations for the pipes are taken into
- * account or we'll have pipes fetching from space previously allocated to
- * another pipe.
- *
- * Roughly the sequence looks like:
- * 1. re-allocate the pipe(s) with the allocation being reduced and not
- * overlapping with a previous light-up pipe (another way to put it is:
- * pipes with their new allocation strickly included into their old ones).
- * 2. re-allocate the other pipes that get their allocation reduced
- * 3. allocate the pipes having their allocation increased
- *
- * Steps 1. and 2. are here to take care of the following case:
- * - Initially DDB looks like this:
- * | B | C |
- * - enable pipe A.
- * - pipe B has a reduced DDB allocation that overlaps with the old pipe C
- * allocation
- * | A | B | C |
- *
- * We need to sequence the re-allocation: C, B, A (and not B, C, A).
- */
-
-static void
-skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
-{
- struct drm_device *dev = dev_priv->dev;
- int plane;
-
- DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
-
- for_each_plane(pipe, plane) {
- I915_WRITE(PLANE_SURF(pipe, plane),
- I915_READ(PLANE_SURF(pipe, plane)));
- }
- I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
-}
-
-static bool
-skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
- const struct skl_ddb_allocation *new,
- enum pipe pipe)
-{
- uint16_t old_size, new_size;
-
- old_size = skl_ddb_entry_size(&old->pipe[pipe]);
- new_size = skl_ddb_entry_size(&new->pipe[pipe]);
-
- return old_size != new_size &&
- new->pipe[pipe].start >= old->pipe[pipe].start &&
- new->pipe[pipe].end <= old->pipe[pipe].end;
-}
-
-static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
- struct skl_wm_values *new_values)
-{
- struct drm_device *dev = dev_priv->dev;
- struct skl_ddb_allocation *cur_ddb, *new_ddb;
- bool reallocated[I915_MAX_PIPES] = {false, false, false};
- struct intel_crtc *crtc;
- enum pipe pipe;
-
- new_ddb = &new_values->ddb;
- cur_ddb = &dev_priv->wm.skl_hw.ddb;
-
- /*
- * First pass: flush the pipes with the new allocation contained into
- * the old space.
- *
- * We'll wait for the vblank on those pipes to ensure we can safely
- * re-allocate the freed space without this pipe fetching from it.
- */
- for_each_intel_crtc(dev, crtc) {
- if (!crtc->active)
- continue;
-
- pipe = crtc->pipe;
-
- if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
- continue;
-
- skl_wm_flush_pipe(dev_priv, pipe, 1);
- intel_wait_for_vblank(dev, pipe);
-
- reallocated[pipe] = true;
- }
-
-
- /*
- * Second pass: flush the pipes that are having their allocation
- * reduced, but overlapping with a previous allocation.
- *
- * Here as well we need to wait for the vblank to make sure the freed
- * space is not used anymore.
- */
- for_each_intel_crtc(dev, crtc) {
- if (!crtc->active)
- continue;
-
- pipe = crtc->pipe;
-
- if (reallocated[pipe])
- continue;
-
- if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
- skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
- skl_wm_flush_pipe(dev_priv, pipe, 2);
- intel_wait_for_vblank(dev, pipe);
- reallocated[pipe] = true;
- }
- }
-
- /*
- * Third pass: flush the pipes that got more space allocated.
- *
- * We don't need to actively wait for the update here, next vblank
- * will just get more DDB space with the correct WM values.
- */
- for_each_intel_crtc(dev, crtc) {
- if (!crtc->active)
- continue;
-
- pipe = crtc->pipe;
-
- /*
- * At this point, only the pipes more space than before are
- * left to re-allocate.
- */
- if (reallocated[pipe])
- continue;
-
- skl_wm_flush_pipe(dev_priv, pipe, 3);
- }
-}
-
-static bool skl_update_pipe_wm(struct drm_crtc *crtc,
- struct skl_pipe_wm_parameters *params,
- struct intel_wm_config *config,
- struct skl_ddb_allocation *ddb, /* out */
- struct skl_pipe_wm *pipe_wm /* out */)
-{
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-
- skl_compute_wm_pipe_parameters(crtc, params);
- skl_allocate_pipe_ddb(crtc, config, params, ddb);
- skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
-
- if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
- return false;
-
- intel_crtc->wm.skl_active = *pipe_wm;
- return true;
-}
-
-static void skl_update_other_pipe_wm(struct drm_device *dev,
- struct drm_crtc *crtc,
- struct intel_wm_config *config,
- struct skl_wm_values *r)
-{
- struct intel_crtc *intel_crtc;
- struct intel_crtc *this_crtc = to_intel_crtc(crtc);
-
- /*
- * If the WM update hasn't changed the allocation for this_crtc (the
- * crtc we are currently computing the new WM values for), other
- * enabled crtcs will keep the same allocation and we don't need to
- * recompute anything for them.
- */
- if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
- return;
-
- /*
- * Otherwise, because of this_crtc being freshly enabled/disabled, the
- * other active pipes need new DDB allocation and WM values.
- */
- list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
- base.head) {
- struct skl_pipe_wm_parameters params = {};
- struct skl_pipe_wm pipe_wm = {};
- bool wm_changed;
-
- if (this_crtc->pipe == intel_crtc->pipe)
- continue;
-
- if (!intel_crtc->active)
- continue;
-
- wm_changed = skl_update_pipe_wm(&intel_crtc->base,
- ¶ms, config,
- &r->ddb, &pipe_wm);
-
- /*
- * If we end up re-computing the other pipe WM values, it's
- * because it was really needed, so we expect the WM values to
- * be different.
- */
- WARN_ON(!wm_changed);
-
- skl_compute_wm_results(dev, ¶ms, &pipe_wm, r, intel_crtc);
- r->dirty[intel_crtc->pipe] = true;
- }
-}
-
-static void skl_update_wm(struct drm_crtc *crtc)
-{
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- struct drm_device *dev = crtc->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct skl_pipe_wm_parameters params = {};
- struct skl_wm_values *results = &dev_priv->wm.skl_results;
- struct skl_pipe_wm pipe_wm = {};
- struct intel_wm_config config = {};
-
- memset(results, 0, sizeof(*results));
-
- skl_compute_wm_global_parameters(dev, &config);
-
- if (!skl_update_pipe_wm(crtc, ¶ms, &config,
- &results->ddb, &pipe_wm))
- return;
-
- skl_compute_wm_results(dev, ¶ms, &pipe_wm, results, intel_crtc);
- results->dirty[intel_crtc->pipe] = true;
-
- skl_update_other_pipe_wm(dev, crtc, &config, results);
- skl_write_wm_values(dev_priv, results);
- skl_flush_wm_values(dev_priv, results);
-
- /* store the new configuration */
- dev_priv->wm.skl_hw = *results;
-}
-
-static void
-skl_update_sprite_wm(struct drm_plane *plane, struct drm_crtc *crtc,
- uint32_t sprite_width, uint32_t sprite_height,
- int pixel_size, bool enabled, bool scaled)
-{
- struct intel_plane *intel_plane = to_intel_plane(plane);
-
- intel_plane->wm.enabled = enabled;
- intel_plane->wm.scaled = scaled;
- intel_plane->wm.horiz_pixels = sprite_width;
- intel_plane->wm.vert_pixels = sprite_height;
- intel_plane->wm.bytes_per_pixel = pixel_size;
-
- skl_update_wm(crtc);
-}
-
-static void ilk_update_wm(struct drm_crtc *crtc)
-{
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- struct drm_device *dev = crtc->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct ilk_wm_maximums max;
- struct ilk_pipe_wm_parameters params = {};
- struct ilk_wm_values results = {};
- enum intel_ddb_partitioning partitioning;
- struct intel_pipe_wm pipe_wm = {};
- struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
- struct intel_wm_config config = {};
-
- ilk_compute_wm_parameters(crtc, ¶ms);
-
- intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm);
-
- if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
- return;
-
- intel_crtc->wm.active = pipe_wm;
-
- ilk_compute_wm_config(dev, &config);
-
- ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
- ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
-
- /* 5/6 split only in single pipe config on IVB+ */
- if (INTEL_INFO(dev)->gen >= 7 &&
- config.num_pipes_active == 1 && config.sprites_enabled) {
- ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
- ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
-
- best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
- } else {
- best_lp_wm = &lp_wm_1_2;
- }
-
- partitioning = (best_lp_wm == &lp_wm_1_2) ?
- INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
-
- ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
-
- ilk_write_wm_values(dev_priv, &results);
-}
-
-static void
-ilk_update_sprite_wm(struct drm_plane *plane,
- struct drm_crtc *crtc,
- uint32_t sprite_width, uint32_t sprite_height,
- int pixel_size, bool enabled, bool scaled)
-{
- struct drm_device *dev = plane->dev;
- struct intel_plane *intel_plane = to_intel_plane(plane);
-
- intel_plane->wm.enabled = enabled;
- intel_plane->wm.scaled = scaled;
- intel_plane->wm.horiz_pixels = sprite_width;
- intel_plane->wm.vert_pixels = sprite_width;
- intel_plane->wm.bytes_per_pixel = pixel_size;
-
- /*
- * IVB workaround: must disable low power watermarks for at least
- * one frame before enabling scaling. LP watermarks can be re-enabled
- * when scaling is disabled.
- *
- * WaCxSRDisabledForSpriteScaling:ivb
- */
- if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
- intel_wait_for_vblank(dev, intel_plane->pipe);
-
- ilk_update_wm(crtc);
-}
-
-static void skl_pipe_wm_active_state(uint32_t val,
- struct skl_pipe_wm *active,
- bool is_transwm,
- bool is_cursor,
- int i,
- int level)
-{
- bool is_enabled = (val & PLANE_WM_EN) != 0;
-
- if (!is_transwm) {
- if (!is_cursor) {
- active->wm[level].plane_en[i] = is_enabled;
- active->wm[level].plane_res_b[i] =
- val & PLANE_WM_BLOCKS_MASK;
- active->wm[level].plane_res_l[i] =
- (val >> PLANE_WM_LINES_SHIFT) &
- PLANE_WM_LINES_MASK;
- } else {
- active->wm[level].cursor_en = is_enabled;
- active->wm[level].cursor_res_b =
- val & PLANE_WM_BLOCKS_MASK;
- active->wm[level].cursor_res_l =
- (val >> PLANE_WM_LINES_SHIFT) &
- PLANE_WM_LINES_MASK;
- }
- } else {
- if (!is_cursor) {
- active->trans_wm.plane_en[i] = is_enabled;
- active->trans_wm.plane_res_b[i] =
- val & PLANE_WM_BLOCKS_MASK;
- active->trans_wm.plane_res_l[i] =
- (val >> PLANE_WM_LINES_SHIFT) &
- PLANE_WM_LINES_MASK;
- } else {
- active->trans_wm.cursor_en = is_enabled;
- active->trans_wm.cursor_res_b =
- val & PLANE_WM_BLOCKS_MASK;
- active->trans_wm.cursor_res_l =
- (val >> PLANE_WM_LINES_SHIFT) &
- PLANE_WM_LINES_MASK;
- }
- }
-}
-
-static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
-{
- struct drm_device *dev = crtc->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
- enum pipe pipe = intel_crtc->pipe;
- int level, i, max_level;
- uint32_t temp;
-
- max_level = ilk_wm_max_level(dev);
-
- hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
-
- for (level = 0; level <= max_level; level++) {
- for (i = 0; i < intel_num_planes(intel_crtc); i++)
- hw->plane[pipe][i][level] =
- I915_READ(PLANE_WM(pipe, i, level));
- hw->cursor[pipe][level] = I915_READ(CUR_WM(pipe, level));
- }
-
- for (i = 0; i < intel_num_planes(intel_crtc); i++)
- hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
- hw->cursor_trans[pipe] = I915_READ(CUR_WM_TRANS(pipe));
-
- if (!intel_crtc_active(crtc))
- return;
-
- hw->dirty[pipe] = true;
-
- active->linetime = hw->wm_linetime[pipe];
-
- for (level = 0; level <= max_level; level++) {
- for (i = 0; i < intel_num_planes(intel_crtc); i++) {
- temp = hw->plane[pipe][i][level];
- skl_pipe_wm_active_state(temp, active, false,
- false, i, level);
- }
- temp = hw->cursor[pipe][level];
- skl_pipe_wm_active_state(temp, active, false, true, i, level);
- }
-
- for (i = 0; i < intel_num_planes(intel_crtc); i++) {
- temp = hw->plane_trans[pipe][i];
- skl_pipe_wm_active_state(temp, active, true, false, i, 0);
- }
-
- temp = hw->cursor_trans[pipe];
- skl_pipe_wm_active_state(temp, active, true, true, i, 0);
-}
-
-void skl_wm_get_hw_state(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
- struct drm_crtc *crtc;
-
- skl_ddb_get_hw_state(dev_priv, ddb);
- list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
- skl_pipe_wm_get_hw_state(crtc);
-}
-
-static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
-{
- struct drm_device *dev = crtc->dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct ilk_wm_values *hw = &dev_priv->wm.hw;
- struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
- struct intel_pipe_wm *active = &intel_crtc->wm.active;
- enum pipe pipe = intel_crtc->pipe;
- static const unsigned int wm0_pipe_reg[] = {
- [PIPE_A] = WM0_PIPEA_ILK,
- [PIPE_B] = WM0_PIPEB_ILK,
- [PIPE_C] = WM0_PIPEC_IVB,
- };
-
- hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
- if (IS_HASWELL(dev) || IS_BROADWELL(dev))
- hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
-
- active->pipe_enabled = intel_crtc_active(crtc);
-
- if (active->pipe_enabled) {
- u32 tmp = hw->wm_pipe[pipe];
-
- /*
- * For active pipes LP0 watermark is marked as
- * enabled, and LP1+ watermaks as disabled since
- * we can't really reverse compute them in case
- * multiple pipes are active.
- */
- active->wm[0].enable = true;
- active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
- active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
- active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
- active->linetime = hw->wm_linetime[pipe];
- } else {
- int level, max_level = ilk_wm_max_level(dev);
-
- /*
- * For inactive pipes, all watermark levels
- * should be marked as enabled but zeroed,
- * which is what we'd compute them to.
- */
- for (level = 0; level <= max_level; level++)
- active->wm[level].enable = true;
- }
-}
-
-void ilk_wm_get_hw_state(struct drm_device *dev)
-{
- struct drm_i915_private *dev_priv = dev->dev_private;
- struct ilk_wm_values *hw = &dev_priv->wm.hw;
- struct drm_crtc *crtc;
-
- for_each_crtc(dev, crtc)
- ilk_pipe_wm_get_hw_state(crtc);
-
- hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
- hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
- hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
-
- hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
- if (INTEL_INFO(dev)->gen >= 7) {
- hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
- hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
- }
-
- if (IS_HASWELL(dev) || IS_BROADWELL(dev))
- hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
- INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
- else if (IS_IVYBRIDGE(dev))
- hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
- INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
-
- hw->enable_fbc_wm =
- !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
-}
-
-/**
- * intel_update_watermarks - update FIFO watermark values based on current modes
- *
- * Calculate watermark values for the various WM regs based on current mode
- * and plane configuration.
- *
- * There are several cases to deal with here:
- * - normal (i.e. non-self-refresh)
- * - self-refresh (SR) mode
- * - lines are large relative to FIFO size (buffer can hold up to 2)
- * - lines are small relative to FIFO size (buffer can hold more than 2
- * lines), so need to account for TLB latency
- *
- * The normal calculation is:
- * watermark = dotclock * bytes per pixel * latency
- * where latency is platform & configuration dependent (we assume pessimal
- * values here).
- *
- * The SR calculation is:
- * watermark = (trunc(latency/line time)+1) * surface width *
- * bytes per pixel
- * where
- * line time = htotal / dotclock
- * surface width = hdisplay for normal plane and 64 for cursor
- * and latency is assumed to be high, as above.
- *
- * The final value programmed to the register should always be rounded up,
- * and include an extra 2 entries to account for clock crossings.
- *
- * We don't use the sprite, so we can ignore that. And on Crestline we have
- * to set the non-SR watermarks to 8.
- */
-void intel_update_watermarks(struct drm_crtc *crtc)
-{
- struct drm_i915_private *dev_priv = crtc->dev->dev_private;
-
- if (dev_priv->display.update_wm)
- dev_priv->display.update_wm(crtc);
-}
-
-void intel_update_sprite_watermarks(struct drm_plane *plane,
- struct drm_crtc *crtc,
- uint32_t sprite_width,
- uint32_t sprite_height,
- int pixel_size,
- bool enabled, bool scaled)
-{
- struct drm_i915_private *dev_priv = plane->dev->dev_private;
-
- if (dev_priv->display.update_sprite_wm)
- dev_priv->display.update_sprite_wm(plane, crtc,
- sprite_width, sprite_height,
- pixel_size, enabled, scaled);
-}
-
static struct drm_i915_gem_object *
intel_alloc_context_page(struct drm_device *dev)
{
@@ -6454,33 +3058,9 @@ void intel_init_pm(struct drm_device *dev)
intel_fbc_init(dev_priv);
- /* For cxsr */
- if (IS_PINEVIEW(dev))
- i915_pineview_get_mem_freq(dev);
- else if (IS_GEN5(dev))
- i915_ironlake_get_mem_freq(dev);
-
- /* For FIFO watermark updates */
if (INTEL_INFO(dev)->gen >= 9) {
- skl_setup_wm_latency(dev);
-
dev_priv->display.init_clock_gating = skl_init_clock_gating;
- dev_priv->display.update_wm = skl_update_wm;
- dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
} else if (HAS_PCH_SPLIT(dev)) {
- ilk_setup_wm_latency(dev);
-
- if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
- dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
- (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
- dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
- dev_priv->display.update_wm = ilk_update_wm;
- dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
- } else {
- DRM_DEBUG_KMS("Failed to read display plane latency. "
- "Disable CxSR\n");
- }
-
if (IS_GEN5(dev))
dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
else if (IS_GEN6(dev))
@@ -6492,53 +3072,23 @@ void intel_init_pm(struct drm_device *dev)
else if (INTEL_INFO(dev)->gen == 8)
dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
} else if (IS_CHERRYVIEW(dev)) {
- dev_priv->display.update_wm = cherryview_update_wm;
- dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
dev_priv->display.init_clock_gating =
cherryview_init_clock_gating;
} else if (IS_VALLEYVIEW(dev)) {
- dev_priv->display.update_wm = valleyview_update_wm;
- dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
dev_priv->display.init_clock_gating =
valleyview_init_clock_gating;
} else if (IS_PINEVIEW(dev)) {
- if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
- dev_priv->is_ddr3,
- dev_priv->fsb_freq,
- dev_priv->mem_freq)) {
- DRM_INFO("failed to find known CxSR latency "
- "(found ddr%s fsb freq %d, mem freq %d), "
- "disabling CxSR\n",
- (dev_priv->is_ddr3 == 1) ? "3" : "2",
- dev_priv->fsb_freq, dev_priv->mem_freq);
- /* Disable CxSR and never update its watermark again */
- intel_set_memory_cxsr(dev_priv, false);
- dev_priv->display.update_wm = NULL;
- } else
- dev_priv->display.update_wm = pineview_update_wm;
dev_priv->display.init_clock_gating = gen3_init_clock_gating;
} else if (IS_G4X(dev)) {
- dev_priv->display.update_wm = g4x_update_wm;
dev_priv->display.init_clock_gating = g4x_init_clock_gating;
} else if (IS_GEN4(dev)) {
- dev_priv->display.update_wm = i965_update_wm;
if (IS_CRESTLINE(dev))
dev_priv->display.init_clock_gating = crestline_init_clock_gating;
else if (IS_BROADWATER(dev))
dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
} else if (IS_GEN3(dev)) {
- dev_priv->display.update_wm = i9xx_update_wm;
- dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
dev_priv->display.init_clock_gating = gen3_init_clock_gating;
} else if (IS_GEN2(dev)) {
- if (INTEL_INFO(dev)->num_pipes == 1) {
- dev_priv->display.update_wm = i845_update_wm;
- dev_priv->display.get_fifo_size = i845_get_fifo_size;
- } else {
- dev_priv->display.update_wm = i9xx_update_wm;
- dev_priv->display.get_fifo_size = i830_get_fifo_size;
- }
-
if (IS_I85X(dev) || IS_I865G(dev))
dev_priv->display.init_clock_gating = i85x_init_clock_gating;
else
diff --git a/drivers/gpu/drm/i915/intel_wm.c b/drivers/gpu/drm/i915/intel_wm.c
new file mode 100644
index 0000000..d67f5255
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_wm.c
@@ -0,0 +1,3525 @@
+/*
+ * Copyright © 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+/**
+ * DOC: display watermarks
+ *
+ * Display watermarks are used to control the display engine memory reuquest
+ * behavior. The watermark values must be properly calculated and programmed
+ * in order to achieve optimum power and performance. Incorrectly programmed
+ * watermark values can result in screen corruption.
+ */
+
+#include "intel_drv.h"
+
+/* Pineview has different values for various configs */
+static const struct intel_watermark_params pineview_display_wm = {
+ .fifo_size = PINEVIEW_DISPLAY_FIFO,
+ .max_wm = PINEVIEW_MAX_WM,
+ .default_wm = PINEVIEW_DFT_WM,
+ .guard_size = PINEVIEW_GUARD_WM,
+ .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params pineview_display_hplloff_wm = {
+ .fifo_size = PINEVIEW_DISPLAY_FIFO,
+ .max_wm = PINEVIEW_MAX_WM,
+ .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
+ .guard_size = PINEVIEW_GUARD_WM,
+ .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params pineview_cursor_wm = {
+ .fifo_size = PINEVIEW_CURSOR_FIFO,
+ .max_wm = PINEVIEW_CURSOR_MAX_WM,
+ .default_wm = PINEVIEW_CURSOR_DFT_WM,
+ .guard_size = PINEVIEW_CURSOR_GUARD_WM,
+ .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
+ .fifo_size = PINEVIEW_CURSOR_FIFO,
+ .max_wm = PINEVIEW_CURSOR_MAX_WM,
+ .default_wm = PINEVIEW_CURSOR_DFT_WM,
+ .guard_size = PINEVIEW_CURSOR_GUARD_WM,
+ .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params g4x_wm_info = {
+ .fifo_size = G4X_FIFO_SIZE,
+ .max_wm = G4X_MAX_WM,
+ .default_wm = G4X_MAX_WM,
+ .guard_size = 2,
+ .cacheline_size = G4X_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params g4x_cursor_wm_info = {
+ .fifo_size = I965_CURSOR_FIFO,
+ .max_wm = I965_CURSOR_MAX_WM,
+ .default_wm = I965_CURSOR_DFT_WM,
+ .guard_size = 2,
+ .cacheline_size = G4X_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params valleyview_wm_info = {
+ .fifo_size = VALLEYVIEW_FIFO_SIZE,
+ .max_wm = VALLEYVIEW_MAX_WM,
+ .default_wm = VALLEYVIEW_MAX_WM,
+ .guard_size = 2,
+ .cacheline_size = G4X_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params valleyview_cursor_wm_info = {
+ .fifo_size = I965_CURSOR_FIFO,
+ .max_wm = VALLEYVIEW_CURSOR_MAX_WM,
+ .default_wm = I965_CURSOR_DFT_WM,
+ .guard_size = 2,
+ .cacheline_size = G4X_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params i965_cursor_wm_info = {
+ .fifo_size = I965_CURSOR_FIFO,
+ .max_wm = I965_CURSOR_MAX_WM,
+ .default_wm = I965_CURSOR_DFT_WM,
+ .guard_size = 2,
+ .cacheline_size = I915_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params i945_wm_info = {
+ .fifo_size = I945_FIFO_SIZE,
+ .max_wm = I915_MAX_WM,
+ .default_wm = 1,
+ .guard_size = 2,
+ .cacheline_size = I915_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params i915_wm_info = {
+ .fifo_size = I915_FIFO_SIZE,
+ .max_wm = I915_MAX_WM,
+ .default_wm = 1,
+ .guard_size = 2,
+ .cacheline_size = I915_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params i830_a_wm_info = {
+ .fifo_size = I855GM_FIFO_SIZE,
+ .max_wm = I915_MAX_WM,
+ .default_wm = 1,
+ .guard_size = 2,
+ .cacheline_size = I830_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params i830_bc_wm_info = {
+ .fifo_size = I855GM_FIFO_SIZE,
+ .max_wm = I915_MAX_WM/2,
+ .default_wm = 1,
+ .guard_size = 2,
+ .cacheline_size = I830_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params i845_wm_info = {
+ .fifo_size = I830_FIFO_SIZE,
+ .max_wm = I915_MAX_WM,
+ .default_wm = 1,
+ .guard_size = 2,
+ .cacheline_size = I830_FIFO_LINE_SIZE,
+};
+
+static void i915_pineview_get_mem_freq(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ u32 tmp;
+
+ tmp = I915_READ(CLKCFG);
+
+ switch (tmp & CLKCFG_FSB_MASK) {
+ case CLKCFG_FSB_533:
+ dev_priv->fsb_freq = 533; /* 133*4 */
+ break;
+ case CLKCFG_FSB_800:
+ dev_priv->fsb_freq = 800; /* 200*4 */
+ break;
+ case CLKCFG_FSB_667:
+ dev_priv->fsb_freq = 667; /* 167*4 */
+ break;
+ case CLKCFG_FSB_400:
+ dev_priv->fsb_freq = 400; /* 100*4 */
+ break;
+ }
+
+ switch (tmp & CLKCFG_MEM_MASK) {
+ case CLKCFG_MEM_533:
+ dev_priv->mem_freq = 533;
+ break;
+ case CLKCFG_MEM_667:
+ dev_priv->mem_freq = 667;
+ break;
+ case CLKCFG_MEM_800:
+ dev_priv->mem_freq = 800;
+ break;
+ }
+
+ /* detect pineview DDR3 setting */
+ tmp = I915_READ(CSHRDDR3CTL);
+ dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
+}
+
+static void i915_ironlake_get_mem_freq(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ u16 ddrpll, csipll;
+
+ ddrpll = I915_READ16(DDRMPLL1);
+ csipll = I915_READ16(CSIPLL0);
+
+ switch (ddrpll & 0xff) {
+ case 0xc:
+ dev_priv->mem_freq = 800;
+ break;
+ case 0x10:
+ dev_priv->mem_freq = 1066;
+ break;
+ case 0x14:
+ dev_priv->mem_freq = 1333;
+ break;
+ case 0x18:
+ dev_priv->mem_freq = 1600;
+ break;
+ default:
+ DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
+ ddrpll & 0xff);
+ dev_priv->mem_freq = 0;
+ break;
+ }
+
+ dev_priv->ips.r_t = dev_priv->mem_freq;
+
+ switch (csipll & 0x3ff) {
+ case 0x00c:
+ dev_priv->fsb_freq = 3200;
+ break;
+ case 0x00e:
+ dev_priv->fsb_freq = 3733;
+ break;
+ case 0x010:
+ dev_priv->fsb_freq = 4266;
+ break;
+ case 0x012:
+ dev_priv->fsb_freq = 4800;
+ break;
+ case 0x014:
+ dev_priv->fsb_freq = 5333;
+ break;
+ case 0x016:
+ dev_priv->fsb_freq = 5866;
+ break;
+ case 0x018:
+ dev_priv->fsb_freq = 6400;
+ break;
+ default:
+ DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
+ csipll & 0x3ff);
+ dev_priv->fsb_freq = 0;
+ break;
+ }
+
+ if (dev_priv->fsb_freq == 3200) {
+ dev_priv->ips.c_m = 0;
+ } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
+ dev_priv->ips.c_m = 1;
+ } else {
+ dev_priv->ips.c_m = 2;
+ }
+}
+
+static const struct cxsr_latency cxsr_latency_table[] = {
+ {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
+ {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
+ {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
+ {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
+ {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
+
+ {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
+ {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
+ {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
+ {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
+ {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
+
+ {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
+ {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
+ {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
+ {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
+ {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
+
+ {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
+ {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
+ {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
+ {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
+ {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
+
+ {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
+ {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
+ {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
+ {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
+ {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
+
+ {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
+ {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
+ {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
+ {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
+ {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
+};
+
+static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
+ int is_ddr3,
+ int fsb,
+ int mem)
+{
+ const struct cxsr_latency *latency;
+ int i;
+
+ if (fsb == 0 || mem == 0)
+ return NULL;
+
+ for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
+ latency = &cxsr_latency_table[i];
+ if (is_desktop == latency->is_desktop &&
+ is_ddr3 == latency->is_ddr3 &&
+ fsb == latency->fsb_freq && mem == latency->mem_freq)
+ return latency;
+ }
+
+ DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
+
+ return NULL;
+}
+
+/**
+ * intel_set_memory_cxsr - enable or disable memory self refresh
+ * @dev_priv: i915 private structure
+ * @enable: true = enable self refresh, false = disable self refresh
+ */
+void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
+{
+ struct drm_device *dev = dev_priv->dev;
+ u32 val;
+
+ if (IS_VALLEYVIEW(dev)) {
+ I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
+ } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) {
+ I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
+ } else if (IS_PINEVIEW(dev)) {
+ val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
+ val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
+ I915_WRITE(DSPFW3, val);
+ } else if (IS_I945G(dev) || IS_I945GM(dev)) {
+ val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
+ _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
+ I915_WRITE(FW_BLC_SELF, val);
+ } else if (IS_I915GM(dev)) {
+ val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
+ _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
+ I915_WRITE(INSTPM, val);
+ } else {
+ return;
+ }
+
+ DRM_DEBUG_KMS("memory self-refresh is %s\n",
+ enable ? "enabled" : "disabled");
+}
+
+/*
+ * Latency for FIFO fetches is dependent on several factors:
+ * - memory configuration (speed, channels)
+ * - chipset
+ * - current MCH state
+ * It can be fairly high in some situations, so here we assume a fairly
+ * pessimal value. It's a tradeoff between extra memory fetches (if we
+ * set this value too high, the FIFO will fetch frequently to stay full)
+ * and power consumption (set it too low to save power and we might see
+ * FIFO underruns and display "flicker").
+ *
+ * A value of 5us seems to be a good balance; safe for very low end
+ * platforms but not overly aggressive on lower latency configs.
+ */
+static const int pessimal_latency_ns = 5000;
+
+static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ uint32_t dsparb = I915_READ(DSPARB);
+ int size;
+
+ size = dsparb & 0x7f;
+ if (plane)
+ size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
+
+ DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
+ plane ? "B" : "A", size);
+
+ return size;
+}
+
+static int i830_get_fifo_size(struct drm_device *dev, int plane)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ uint32_t dsparb = I915_READ(DSPARB);
+ int size;
+
+ size = dsparb & 0x1ff;
+ if (plane)
+ size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
+ size >>= 1; /* Convert to cachelines */
+
+ DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
+ plane ? "B" : "A", size);
+
+ return size;
+}
+
+static int i845_get_fifo_size(struct drm_device *dev, int plane)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ uint32_t dsparb = I915_READ(DSPARB);
+ int size;
+
+ size = dsparb & 0x7f;
+ size >>= 2; /* Convert to cachelines */
+
+ DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
+ plane ? "B" : "A",
+ size);
+
+ return size;
+}
+
+/**
+ * intel_calculate_wm - calculate watermark level
+ * @clock_in_khz: pixel clock
+ * @wm: chip FIFO params
+ * @pixel_size: display pixel size
+ * @latency_ns: memory latency for the platform
+ *
+ * Calculate the watermark level (the level at which the display plane will
+ * start fetching from memory again). Each chip has a different display
+ * FIFO size and allocation, so the caller needs to figure that out and pass
+ * in the correct intel_watermark_params structure.
+ *
+ * As the pixel clock runs, the FIFO will be drained at a rate that depends
+ * on the pixel size. When it reaches the watermark level, it'll start
+ * fetching FIFO line sized based chunks from memory until the FIFO fills
+ * past the watermark point. If the FIFO drains completely, a FIFO underrun
+ * will occur, and a display engine hang could result.
+ */
+static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
+ const struct intel_watermark_params *wm,
+ int fifo_size,
+ int pixel_size,
+ unsigned long latency_ns)
+{
+ long entries_required, wm_size;
+
+ /*
+ * Note: we need to make sure we don't overflow for various clock &
+ * latency values.
+ * clocks go from a few thousand to several hundred thousand.
+ * latency is usually a few thousand
+ */
+ entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
+ 1000;
+ entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
+
+ DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
+
+ wm_size = fifo_size - (entries_required + wm->guard_size);
+
+ DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
+
+ /* Don't promote wm_size to unsigned... */
+ if (wm_size > (long)wm->max_wm)
+ wm_size = wm->max_wm;
+ if (wm_size <= 0)
+ wm_size = wm->default_wm;
+
+ /*
+ * Bspec seems to indicate that the value shouldn't be lower than
+ * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
+ * Lets go for 8 which is the burst size since certain platforms
+ * already use a hardcoded 8 (which is what the spec says should be
+ * done).
+ */
+ if (wm_size <= 8)
+ wm_size = 8;
+
+ return wm_size;
+}
+
+static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
+{
+ struct drm_crtc *crtc, *enabled = NULL;
+
+ for_each_crtc(dev, crtc) {
+ if (intel_crtc_active(crtc)) {
+ if (enabled)
+ return NULL;
+ enabled = crtc;
+ }
+ }
+
+ return enabled;
+}
+
+static void pineview_update_wm(struct drm_crtc *unused_crtc)
+{
+ struct drm_device *dev = unused_crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_crtc *crtc;
+ const struct cxsr_latency *latency;
+ u32 reg;
+ unsigned long wm;
+
+ latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
+ dev_priv->fsb_freq, dev_priv->mem_freq);
+ if (!latency) {
+ DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
+ intel_set_memory_cxsr(dev_priv, false);
+ return;
+ }
+
+ crtc = single_enabled_crtc(dev);
+ if (crtc) {
+ const struct drm_display_mode *adjusted_mode;
+ int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
+ int clock;
+
+ adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
+ clock = adjusted_mode->crtc_clock;
+
+ /* Display SR */
+ wm = intel_calculate_wm(clock, &pineview_display_wm,
+ pineview_display_wm.fifo_size,
+ pixel_size, latency->display_sr);
+ reg = I915_READ(DSPFW1);
+ reg &= ~DSPFW_SR_MASK;
+ reg |= wm << DSPFW_SR_SHIFT;
+ I915_WRITE(DSPFW1, reg);
+ DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
+
+ /* cursor SR */
+ wm = intel_calculate_wm(clock, &pineview_cursor_wm,
+ pineview_display_wm.fifo_size,
+ pixel_size, latency->cursor_sr);
+ reg = I915_READ(DSPFW3);
+ reg &= ~DSPFW_CURSOR_SR_MASK;
+ reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
+ I915_WRITE(DSPFW3, reg);
+
+ /* Display HPLL off SR */
+ wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
+ pineview_display_hplloff_wm.fifo_size,
+ pixel_size, latency->display_hpll_disable);
+ reg = I915_READ(DSPFW3);
+ reg &= ~DSPFW_HPLL_SR_MASK;
+ reg |= wm & DSPFW_HPLL_SR_MASK;
+ I915_WRITE(DSPFW3, reg);
+
+ /* cursor HPLL off SR */
+ wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
+ pineview_display_hplloff_wm.fifo_size,
+ pixel_size, latency->cursor_hpll_disable);
+ reg = I915_READ(DSPFW3);
+ reg &= ~DSPFW_HPLL_CURSOR_MASK;
+ reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
+ I915_WRITE(DSPFW3, reg);
+ DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
+
+ intel_set_memory_cxsr(dev_priv, true);
+ } else {
+ intel_set_memory_cxsr(dev_priv, false);
+ }
+}
+
+static bool g4x_compute_wm0(struct drm_device *dev,
+ int plane,
+ const struct intel_watermark_params *display,
+ int display_latency_ns,
+ const struct intel_watermark_params *cursor,
+ int cursor_latency_ns,
+ int *plane_wm,
+ int *cursor_wm)
+{
+ struct drm_crtc *crtc;
+ const struct drm_display_mode *adjusted_mode;
+ int htotal, hdisplay, clock, pixel_size;
+ int line_time_us, line_count;
+ int entries, tlb_miss;
+
+ crtc = intel_get_crtc_for_plane(dev, plane);
+ if (!intel_crtc_active(crtc)) {
+ *cursor_wm = cursor->guard_size;
+ *plane_wm = display->guard_size;
+ return false;
+ }
+
+ adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
+ clock = adjusted_mode->crtc_clock;
+ htotal = adjusted_mode->crtc_htotal;
+ hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
+ pixel_size = crtc->primary->fb->bits_per_pixel / 8;
+
+ /* Use the small buffer method to calculate plane watermark */
+ entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
+ tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
+ if (tlb_miss > 0)
+ entries += tlb_miss;
+ entries = DIV_ROUND_UP(entries, display->cacheline_size);
+ *plane_wm = entries + display->guard_size;
+ if (*plane_wm > (int)display->max_wm)
+ *plane_wm = display->max_wm;
+
+ /* Use the large buffer method to calculate cursor watermark */
+ line_time_us = max(htotal * 1000 / clock, 1);
+ line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
+ entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size;
+ tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
+ if (tlb_miss > 0)
+ entries += tlb_miss;
+ entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
+ *cursor_wm = entries + cursor->guard_size;
+ if (*cursor_wm > (int)cursor->max_wm)
+ *cursor_wm = (int)cursor->max_wm;
+
+ return true;
+}
+
+/*
+ * Check the wm result.
+ *
+ * If any calculated watermark values is larger than the maximum value that
+ * can be programmed into the associated watermark register, that watermark
+ * must be disabled.
+ */
+static bool g4x_check_srwm(struct drm_device *dev,
+ int display_wm, int cursor_wm,
+ const struct intel_watermark_params *display,
+ const struct intel_watermark_params *cursor)
+{
+ DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
+ display_wm, cursor_wm);
+
+ if (display_wm > display->max_wm) {
+ DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
+ display_wm, display->max_wm);
+ return false;
+ }
+
+ if (cursor_wm > cursor->max_wm) {
+ DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
+ cursor_wm, cursor->max_wm);
+ return false;
+ }
+
+ if (!(display_wm || cursor_wm)) {
+ DRM_DEBUG_KMS("SR latency is 0, disabling\n");
+ return false;
+ }
+
+ return true;
+}
+
+static bool g4x_compute_srwm(struct drm_device *dev,
+ int plane,
+ int latency_ns,
+ const struct intel_watermark_params *display,
+ const struct intel_watermark_params *cursor,
+ int *display_wm, int *cursor_wm)
+{
+ struct drm_crtc *crtc;
+ const struct drm_display_mode *adjusted_mode;
+ int hdisplay, htotal, pixel_size, clock;
+ unsigned long line_time_us;
+ int line_count, line_size;
+ int small, large;
+ int entries;
+
+ if (!latency_ns) {
+ *display_wm = *cursor_wm = 0;
+ return false;
+ }
+
+ crtc = intel_get_crtc_for_plane(dev, plane);
+ adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
+ clock = adjusted_mode->crtc_clock;
+ htotal = adjusted_mode->crtc_htotal;
+ hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
+ pixel_size = crtc->primary->fb->bits_per_pixel / 8;
+
+ line_time_us = max(htotal * 1000 / clock, 1);
+ line_count = (latency_ns / line_time_us + 1000) / 1000;
+ line_size = hdisplay * pixel_size;
+
+ /* Use the minimum of the small and large buffer method for primary */
+ small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
+ large = line_count * line_size;
+
+ entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
+ *display_wm = entries + display->guard_size;
+
+ /* calculate the self-refresh watermark for display cursor */
+ entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width;
+ entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
+ *cursor_wm = entries + cursor->guard_size;
+
+ return g4x_check_srwm(dev,
+ *display_wm, *cursor_wm,
+ display, cursor);
+}
+
+static bool vlv_compute_drain_latency(struct drm_crtc *crtc,
+ int pixel_size,
+ int *prec_mult,
+ int *drain_latency)
+{
+ struct drm_device *dev = crtc->dev;
+ int entries;
+ int clock = to_intel_crtc(crtc)->config->base.adjusted_mode.crtc_clock;
+
+ if (WARN(clock == 0, "Pixel clock is zero!\n"))
+ return false;
+
+ if (WARN(pixel_size == 0, "Pixel size is zero!\n"))
+ return false;
+
+ entries = DIV_ROUND_UP(clock, 1000) * pixel_size;
+ if (IS_CHERRYVIEW(dev))
+ *prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_32 :
+ DRAIN_LATENCY_PRECISION_16;
+ else
+ *prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_64 :
+ DRAIN_LATENCY_PRECISION_32;
+ *drain_latency = (64 * (*prec_mult) * 4) / entries;
+
+ if (*drain_latency > DRAIN_LATENCY_MASK)
+ *drain_latency = DRAIN_LATENCY_MASK;
+
+ return true;
+}
+
+/*
+ * Update drain latency registers of memory arbiter
+ *
+ * Valleyview SoC has a new memory arbiter and needs drain latency registers
+ * to be programmed. Each plane has a drain latency multiplier and a drain
+ * latency value.
+ */
+
+static void vlv_update_drain_latency(struct drm_crtc *crtc)
+{
+ struct drm_device *dev = crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ int pixel_size;
+ int drain_latency;
+ enum pipe pipe = intel_crtc->pipe;
+ int plane_prec, prec_mult, plane_dl;
+ const int high_precision = IS_CHERRYVIEW(dev) ?
+ DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_64;
+
+ plane_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_PLANE_PRECISION_HIGH |
+ DRAIN_LATENCY_MASK | DDL_CURSOR_PRECISION_HIGH |
+ (DRAIN_LATENCY_MASK << DDL_CURSOR_SHIFT));
+
+ if (!intel_crtc_active(crtc)) {
+ I915_WRITE(VLV_DDL(pipe), plane_dl);
+ return;
+ }
+
+ /* Primary plane Drain Latency */
+ pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */
+ if (vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
+ plane_prec = (prec_mult == high_precision) ?
+ DDL_PLANE_PRECISION_HIGH :
+ DDL_PLANE_PRECISION_LOW;
+ plane_dl |= plane_prec | drain_latency;
+ }
+
+ /* Cursor Drain Latency
+ * BPP is always 4 for cursor
+ */
+ pixel_size = 4;
+
+ /* Program cursor DL only if it is enabled */
+ if (intel_crtc->cursor_base &&
+ vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
+ plane_prec = (prec_mult == high_precision) ?
+ DDL_CURSOR_PRECISION_HIGH :
+ DDL_CURSOR_PRECISION_LOW;
+ plane_dl |= plane_prec | (drain_latency << DDL_CURSOR_SHIFT);
+ }
+
+ I915_WRITE(VLV_DDL(pipe), plane_dl);
+}
+
+#define single_plane_enabled(mask) is_power_of_2(mask)
+
+static void valleyview_update_wm(struct drm_crtc *crtc)
+{
+ struct drm_device *dev = crtc->dev;
+ static const int sr_latency_ns = 12000;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
+ int plane_sr, cursor_sr;
+ int ignore_plane_sr, ignore_cursor_sr;
+ unsigned int enabled = 0;
+ bool cxsr_enabled;
+
+ vlv_update_drain_latency(crtc);
+
+ if (g4x_compute_wm0(dev, PIPE_A,
+ &valleyview_wm_info, pessimal_latency_ns,
+ &valleyview_cursor_wm_info, pessimal_latency_ns,
+ &planea_wm, &cursora_wm))
+ enabled |= 1 << PIPE_A;
+
+ if (g4x_compute_wm0(dev, PIPE_B,
+ &valleyview_wm_info, pessimal_latency_ns,
+ &valleyview_cursor_wm_info, pessimal_latency_ns,
+ &planeb_wm, &cursorb_wm))
+ enabled |= 1 << PIPE_B;
+
+ if (single_plane_enabled(enabled) &&
+ g4x_compute_srwm(dev, ffs(enabled) - 1,
+ sr_latency_ns,
+ &valleyview_wm_info,
+ &valleyview_cursor_wm_info,
+ &plane_sr, &ignore_cursor_sr) &&
+ g4x_compute_srwm(dev, ffs(enabled) - 1,
+ 2*sr_latency_ns,
+ &valleyview_wm_info,
+ &valleyview_cursor_wm_info,
+ &ignore_plane_sr, &cursor_sr)) {
+ cxsr_enabled = true;
+ } else {
+ cxsr_enabled = false;
+ intel_set_memory_cxsr(dev_priv, false);
+ plane_sr = cursor_sr = 0;
+ }
+
+ DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
+ "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
+ planea_wm, cursora_wm,
+ planeb_wm, cursorb_wm,
+ plane_sr, cursor_sr);
+
+ I915_WRITE(DSPFW1,
+ (plane_sr << DSPFW_SR_SHIFT) |
+ (cursorb_wm << DSPFW_CURSORB_SHIFT) |
+ (planeb_wm << DSPFW_PLANEB_SHIFT) |
+ (planea_wm << DSPFW_PLANEA_SHIFT));
+ I915_WRITE(DSPFW2,
+ (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
+ (cursora_wm << DSPFW_CURSORA_SHIFT));
+ I915_WRITE(DSPFW3,
+ (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
+ (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
+
+ if (cxsr_enabled)
+ intel_set_memory_cxsr(dev_priv, true);
+}
+
+static void cherryview_update_wm(struct drm_crtc *crtc)
+{
+ struct drm_device *dev = crtc->dev;
+ static const int sr_latency_ns = 12000;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ int planea_wm, planeb_wm, planec_wm;
+ int cursora_wm, cursorb_wm, cursorc_wm;
+ int plane_sr, cursor_sr;
+ int ignore_plane_sr, ignore_cursor_sr;
+ unsigned int enabled = 0;
+ bool cxsr_enabled;
+
+ vlv_update_drain_latency(crtc);
+
+ if (g4x_compute_wm0(dev, PIPE_A,
+ &valleyview_wm_info, pessimal_latency_ns,
+ &valleyview_cursor_wm_info, pessimal_latency_ns,
+ &planea_wm, &cursora_wm))
+ enabled |= 1 << PIPE_A;
+
+ if (g4x_compute_wm0(dev, PIPE_B,
+ &valleyview_wm_info, pessimal_latency_ns,
+ &valleyview_cursor_wm_info, pessimal_latency_ns,
+ &planeb_wm, &cursorb_wm))
+ enabled |= 1 << PIPE_B;
+
+ if (g4x_compute_wm0(dev, PIPE_C,
+ &valleyview_wm_info, pessimal_latency_ns,
+ &valleyview_cursor_wm_info, pessimal_latency_ns,
+ &planec_wm, &cursorc_wm))
+ enabled |= 1 << PIPE_C;
+
+ if (single_plane_enabled(enabled) &&
+ g4x_compute_srwm(dev, ffs(enabled) - 1,
+ sr_latency_ns,
+ &valleyview_wm_info,
+ &valleyview_cursor_wm_info,
+ &plane_sr, &ignore_cursor_sr) &&
+ g4x_compute_srwm(dev, ffs(enabled) - 1,
+ 2*sr_latency_ns,
+ &valleyview_wm_info,
+ &valleyview_cursor_wm_info,
+ &ignore_plane_sr, &cursor_sr)) {
+ cxsr_enabled = true;
+ } else {
+ cxsr_enabled = false;
+ intel_set_memory_cxsr(dev_priv, false);
+ plane_sr = cursor_sr = 0;
+ }
+
+ DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
+ "B: plane=%d, cursor=%d, C: plane=%d, cursor=%d, "
+ "SR: plane=%d, cursor=%d\n",
+ planea_wm, cursora_wm,
+ planeb_wm, cursorb_wm,
+ planec_wm, cursorc_wm,
+ plane_sr, cursor_sr);
+
+ I915_WRITE(DSPFW1,
+ (plane_sr << DSPFW_SR_SHIFT) |
+ (cursorb_wm << DSPFW_CURSORB_SHIFT) |
+ (planeb_wm << DSPFW_PLANEB_SHIFT) |
+ (planea_wm << DSPFW_PLANEA_SHIFT));
+ I915_WRITE(DSPFW2,
+ (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
+ (cursora_wm << DSPFW_CURSORA_SHIFT));
+ I915_WRITE(DSPFW3,
+ (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
+ (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
+ I915_WRITE(DSPFW9_CHV,
+ (I915_READ(DSPFW9_CHV) & ~(DSPFW_PLANEC_MASK |
+ DSPFW_CURSORC_MASK)) |
+ (planec_wm << DSPFW_PLANEC_SHIFT) |
+ (cursorc_wm << DSPFW_CURSORC_SHIFT));
+
+ if (cxsr_enabled)
+ intel_set_memory_cxsr(dev_priv, true);
+}
+
+static void valleyview_update_sprite_wm(struct drm_plane *plane,
+ struct drm_crtc *crtc,
+ uint32_t sprite_width,
+ uint32_t sprite_height,
+ int pixel_size,
+ bool enabled, bool scaled)
+{
+ struct drm_device *dev = crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ int pipe = to_intel_plane(plane)->pipe;
+ int sprite = to_intel_plane(plane)->plane;
+ int drain_latency;
+ int plane_prec;
+ int sprite_dl;
+ int prec_mult;
+ const int high_precision = IS_CHERRYVIEW(dev) ?
+ DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_64;
+
+ sprite_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_SPRITE_PRECISION_HIGH(sprite) |
+ (DRAIN_LATENCY_MASK << DDL_SPRITE_SHIFT(sprite)));
+
+ if (enabled && vlv_compute_drain_latency(crtc, pixel_size, &prec_mult,
+ &drain_latency)) {
+ plane_prec = (prec_mult == high_precision) ?
+ DDL_SPRITE_PRECISION_HIGH(sprite) :
+ DDL_SPRITE_PRECISION_LOW(sprite);
+ sprite_dl |= plane_prec |
+ (drain_latency << DDL_SPRITE_SHIFT(sprite));
+ }
+
+ I915_WRITE(VLV_DDL(pipe), sprite_dl);
+}
+
+static void g4x_update_wm(struct drm_crtc *crtc)
+{
+ struct drm_device *dev = crtc->dev;
+ static const int sr_latency_ns = 12000;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
+ int plane_sr, cursor_sr;
+ unsigned int enabled = 0;
+ bool cxsr_enabled;
+
+ if (g4x_compute_wm0(dev, PIPE_A,
+ &g4x_wm_info, pessimal_latency_ns,
+ &g4x_cursor_wm_info, pessimal_latency_ns,
+ &planea_wm, &cursora_wm))
+ enabled |= 1 << PIPE_A;
+
+ if (g4x_compute_wm0(dev, PIPE_B,
+ &g4x_wm_info, pessimal_latency_ns,
+ &g4x_cursor_wm_info, pessimal_latency_ns,
+ &planeb_wm, &cursorb_wm))
+ enabled |= 1 << PIPE_B;
+
+ if (single_plane_enabled(enabled) &&
+ g4x_compute_srwm(dev, ffs(enabled) - 1,
+ sr_latency_ns,
+ &g4x_wm_info,
+ &g4x_cursor_wm_info,
+ &plane_sr, &cursor_sr)) {
+ cxsr_enabled = true;
+ } else {
+ cxsr_enabled = false;
+ intel_set_memory_cxsr(dev_priv, false);
+ plane_sr = cursor_sr = 0;
+ }
+
+ DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
+ "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
+ planea_wm, cursora_wm,
+ planeb_wm, cursorb_wm,
+ plane_sr, cursor_sr);
+
+ I915_WRITE(DSPFW1,
+ (plane_sr << DSPFW_SR_SHIFT) |
+ (cursorb_wm << DSPFW_CURSORB_SHIFT) |
+ (planeb_wm << DSPFW_PLANEB_SHIFT) |
+ (planea_wm << DSPFW_PLANEA_SHIFT));
+ I915_WRITE(DSPFW2,
+ (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
+ (cursora_wm << DSPFW_CURSORA_SHIFT));
+ /* HPLL off in SR has some issues on G4x... disable it */
+ I915_WRITE(DSPFW3,
+ (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
+ (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
+
+ if (cxsr_enabled)
+ intel_set_memory_cxsr(dev_priv, true);
+}
+
+static void i965_update_wm(struct drm_crtc *unused_crtc)
+{
+ struct drm_device *dev = unused_crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_crtc *crtc;
+ int srwm = 1;
+ int cursor_sr = 16;
+ bool cxsr_enabled;
+
+ /* Calc sr entries for one plane configs */
+ crtc = single_enabled_crtc(dev);
+ if (crtc) {
+ /* self-refresh has much higher latency */
+ static const int sr_latency_ns = 12000;
+ const struct drm_display_mode *adjusted_mode =
+ &to_intel_crtc(crtc)->config->base.adjusted_mode;
+ int clock = adjusted_mode->crtc_clock;
+ int htotal = adjusted_mode->crtc_htotal;
+ int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
+ int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
+ unsigned long line_time_us;
+ int entries;
+
+ line_time_us = max(htotal * 1000 / clock, 1);
+
+ /* Use ns/us then divide to preserve precision */
+ entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
+ pixel_size * hdisplay;
+ entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
+ srwm = I965_FIFO_SIZE - entries;
+ if (srwm < 0)
+ srwm = 1;
+ srwm &= 0x1ff;
+ DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
+ entries, srwm);
+
+ entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
+ pixel_size * to_intel_crtc(crtc)->cursor_width;
+ entries = DIV_ROUND_UP(entries,
+ i965_cursor_wm_info.cacheline_size);
+ cursor_sr = i965_cursor_wm_info.fifo_size -
+ (entries + i965_cursor_wm_info.guard_size);
+
+ if (cursor_sr > i965_cursor_wm_info.max_wm)
+ cursor_sr = i965_cursor_wm_info.max_wm;
+
+ DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
+ "cursor %d\n", srwm, cursor_sr);
+
+ cxsr_enabled = true;
+ } else {
+ cxsr_enabled = false;
+ /* Turn off self refresh if both pipes are enabled */
+ intel_set_memory_cxsr(dev_priv, false);
+ }
+
+ DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
+ srwm);
+
+ /* 965 has limitations... */
+ I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
+ (8 << DSPFW_CURSORB_SHIFT) |
+ (8 << DSPFW_PLANEB_SHIFT) |
+ (8 << DSPFW_PLANEA_SHIFT));
+ I915_WRITE(DSPFW2, (8 << DSPFW_CURSORA_SHIFT) |
+ (8 << DSPFW_PLANEC_SHIFT_OLD));
+ /* update cursor SR watermark */
+ I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
+
+ if (cxsr_enabled)
+ intel_set_memory_cxsr(dev_priv, true);
+}
+
+static void i9xx_update_wm(struct drm_crtc *unused_crtc)
+{
+ struct drm_device *dev = unused_crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ const struct intel_watermark_params *wm_info;
+ uint32_t fwater_lo;
+ uint32_t fwater_hi;
+ int cwm, srwm = 1;
+ int fifo_size;
+ int planea_wm, planeb_wm;
+ struct drm_crtc *crtc, *enabled = NULL;
+
+ if (IS_I945GM(dev))
+ wm_info = &i945_wm_info;
+ else if (!IS_GEN2(dev))
+ wm_info = &i915_wm_info;
+ else
+ wm_info = &i830_a_wm_info;
+
+ fifo_size = dev_priv->display.get_fifo_size(dev, 0);
+ crtc = intel_get_crtc_for_plane(dev, 0);
+ if (intel_crtc_active(crtc)) {
+ const struct drm_display_mode *adjusted_mode;
+ int cpp = crtc->primary->fb->bits_per_pixel / 8;
+ if (IS_GEN2(dev))
+ cpp = 4;
+
+ adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
+ planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
+ wm_info, fifo_size, cpp,
+ pessimal_latency_ns);
+ enabled = crtc;
+ } else {
+ planea_wm = fifo_size - wm_info->guard_size;
+ if (planea_wm > (long)wm_info->max_wm)
+ planea_wm = wm_info->max_wm;
+ }
+
+ if (IS_GEN2(dev))
+ wm_info = &i830_bc_wm_info;
+
+ fifo_size = dev_priv->display.get_fifo_size(dev, 1);
+ crtc = intel_get_crtc_for_plane(dev, 1);
+ if (intel_crtc_active(crtc)) {
+ const struct drm_display_mode *adjusted_mode;
+ int cpp = crtc->primary->fb->bits_per_pixel / 8;
+ if (IS_GEN2(dev))
+ cpp = 4;
+
+ adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
+ planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
+ wm_info, fifo_size, cpp,
+ pessimal_latency_ns);
+ if (enabled == NULL)
+ enabled = crtc;
+ else
+ enabled = NULL;
+ } else {
+ planeb_wm = fifo_size - wm_info->guard_size;
+ if (planeb_wm > (long)wm_info->max_wm)
+ planeb_wm = wm_info->max_wm;
+ }
+
+ DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
+
+ if (IS_I915GM(dev) && enabled) {
+ struct drm_i915_gem_object *obj;
+
+ obj = intel_fb_obj(enabled->primary->fb);
+
+ /* self-refresh seems busted with untiled */
+ if (obj->tiling_mode == I915_TILING_NONE)
+ enabled = NULL;
+ }
+
+ /*
+ * Overlay gets an aggressive default since video jitter is bad.
+ */
+ cwm = 2;
+
+ /* Play safe and disable self-refresh before adjusting watermarks. */
+ intel_set_memory_cxsr(dev_priv, false);
+
+ /* Calc sr entries for one plane configs */
+ if (HAS_FW_BLC(dev) && enabled) {
+ /* self-refresh has much higher latency */
+ static const int sr_latency_ns = 6000;
+ const struct drm_display_mode *adjusted_mode =
+ &to_intel_crtc(enabled)->config->base.adjusted_mode;
+ int clock = adjusted_mode->crtc_clock;
+ int htotal = adjusted_mode->crtc_htotal;
+ int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
+ int pixel_size = enabled->primary->fb->bits_per_pixel / 8;
+ unsigned long line_time_us;
+ int entries;
+
+ line_time_us = max(htotal * 1000 / clock, 1);
+
+ /* Use ns/us then divide to preserve precision */
+ entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
+ pixel_size * hdisplay;
+ entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
+ DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
+ srwm = wm_info->fifo_size - entries;
+ if (srwm < 0)
+ srwm = 1;
+
+ if (IS_I945G(dev) || IS_I945GM(dev))
+ I915_WRITE(FW_BLC_SELF,
+ FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
+ else if (IS_I915GM(dev))
+ I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
+ }
+
+ DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
+ planea_wm, planeb_wm, cwm, srwm);
+
+ fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
+ fwater_hi = (cwm & 0x1f);
+
+ /* Set request length to 8 cachelines per fetch */
+ fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
+ fwater_hi = fwater_hi | (1 << 8);
+
+ I915_WRITE(FW_BLC, fwater_lo);
+ I915_WRITE(FW_BLC2, fwater_hi);
+
+ if (enabled)
+ intel_set_memory_cxsr(dev_priv, true);
+}
+
+static void i845_update_wm(struct drm_crtc *unused_crtc)
+{
+ struct drm_device *dev = unused_crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_crtc *crtc;
+ const struct drm_display_mode *adjusted_mode;
+ uint32_t fwater_lo;
+ int planea_wm;
+
+ crtc = single_enabled_crtc(dev);
+ if (crtc == NULL)
+ return;
+
+ adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
+ planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
+ &i845_wm_info,
+ dev_priv->display.get_fifo_size(dev, 0),
+ 4, pessimal_latency_ns);
+ fwater_lo = I915_READ(FW_BLC) & ~0xfff;
+ fwater_lo |= (3<<8) | planea_wm;
+
+ DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
+
+ I915_WRITE(FW_BLC, fwater_lo);
+}
+
+static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
+ struct drm_crtc *crtc)
+{
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ uint32_t pixel_rate;
+
+ pixel_rate = intel_crtc->config->base.adjusted_mode.crtc_clock;
+
+ /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
+ * adjust the pixel_rate here. */
+
+ if (intel_crtc->config->pch_pfit.enabled) {
+ uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
+ uint32_t pfit_size = intel_crtc->config->pch_pfit.size;
+
+ pipe_w = intel_crtc->config->pipe_src_w;
+ pipe_h = intel_crtc->config->pipe_src_h;
+ pfit_w = (pfit_size >> 16) & 0xFFFF;
+ pfit_h = pfit_size & 0xFFFF;
+ if (pipe_w < pfit_w)
+ pipe_w = pfit_w;
+ if (pipe_h < pfit_h)
+ pipe_h = pfit_h;
+
+ pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
+ pfit_w * pfit_h);
+ }
+
+ return pixel_rate;
+}
+
+/* latency must be in 0.1us units. */
+static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
+ uint32_t latency)
+{
+ uint64_t ret;
+
+ if (WARN(latency == 0, "Latency value missing\n"))
+ return UINT_MAX;
+
+ ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
+ ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
+
+ return ret;
+}
+
+/* latency must be in 0.1us units. */
+static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
+ uint32_t horiz_pixels, uint8_t bytes_per_pixel,
+ uint32_t latency)
+{
+ uint32_t ret;
+
+ if (WARN(latency == 0, "Latency value missing\n"))
+ return UINT_MAX;
+
+ ret = (latency * pixel_rate) / (pipe_htotal * 10000);
+ ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
+ ret = DIV_ROUND_UP(ret, 64) + 2;
+ return ret;
+}
+
+static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
+ uint8_t bytes_per_pixel)
+{
+ return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
+}
+
+struct skl_pipe_wm_parameters {
+ bool active;
+ uint32_t pipe_htotal;
+ uint32_t pixel_rate; /* in KHz */
+ struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
+ struct intel_plane_wm_parameters cursor;
+};
+
+struct ilk_pipe_wm_parameters {
+ bool active;
+ uint32_t pipe_htotal;
+ uint32_t pixel_rate;
+ struct intel_plane_wm_parameters pri;
+ struct intel_plane_wm_parameters spr;
+ struct intel_plane_wm_parameters cur;
+};
+
+struct ilk_wm_maximums {
+ uint16_t pri;
+ uint16_t spr;
+ uint16_t cur;
+ uint16_t fbc;
+};
+
+/* used in computing the new watermarks state */
+struct intel_wm_config {
+ unsigned int num_pipes_active;
+ bool sprites_enabled;
+ bool sprites_scaled;
+};
+
+/*
+ * For both WM_PIPE and WM_LP.
+ * mem_value must be in 0.1us units.
+ */
+static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
+ uint32_t mem_value,
+ bool is_lp)
+{
+ uint32_t method1, method2;
+
+ if (!params->active || !params->pri.enabled)
+ return 0;
+
+ method1 = ilk_wm_method1(params->pixel_rate,
+ params->pri.bytes_per_pixel,
+ mem_value);
+
+ if (!is_lp)
+ return method1;
+
+ method2 = ilk_wm_method2(params->pixel_rate,
+ params->pipe_htotal,
+ params->pri.horiz_pixels,
+ params->pri.bytes_per_pixel,
+ mem_value);
+
+ return min(method1, method2);
+}
+
+/*
+ * For both WM_PIPE and WM_LP.
+ * mem_value must be in 0.1us units.
+ */
+static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
+ uint32_t mem_value)
+{
+ uint32_t method1, method2;
+
+ if (!params->active || !params->spr.enabled)
+ return 0;
+
+ method1 = ilk_wm_method1(params->pixel_rate,
+ params->spr.bytes_per_pixel,
+ mem_value);
+ method2 = ilk_wm_method2(params->pixel_rate,
+ params->pipe_htotal,
+ params->spr.horiz_pixels,
+ params->spr.bytes_per_pixel,
+ mem_value);
+ return min(method1, method2);
+}
+
+/*
+ * For both WM_PIPE and WM_LP.
+ * mem_value must be in 0.1us units.
+ */
+static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
+ uint32_t mem_value)
+{
+ if (!params->active || !params->cur.enabled)
+ return 0;
+
+ return ilk_wm_method2(params->pixel_rate,
+ params->pipe_htotal,
+ params->cur.horiz_pixels,
+ params->cur.bytes_per_pixel,
+ mem_value);
+}
+
+/* Only for WM_LP. */
+static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
+ uint32_t pri_val)
+{
+ if (!params->active || !params->pri.enabled)
+ return 0;
+
+ return ilk_wm_fbc(pri_val,
+ params->pri.horiz_pixels,
+ params->pri.bytes_per_pixel);
+}
+
+static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
+{
+ if (INTEL_INFO(dev)->gen >= 8)
+ return 3072;
+ else if (INTEL_INFO(dev)->gen >= 7)
+ return 768;
+ else
+ return 512;
+}
+
+static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
+ int level, bool is_sprite)
+{
+ if (INTEL_INFO(dev)->gen >= 8)
+ /* BDW primary/sprite plane watermarks */
+ return level == 0 ? 255 : 2047;
+ else if (INTEL_INFO(dev)->gen >= 7)
+ /* IVB/HSW primary/sprite plane watermarks */
+ return level == 0 ? 127 : 1023;
+ else if (!is_sprite)
+ /* ILK/SNB primary plane watermarks */
+ return level == 0 ? 127 : 511;
+ else
+ /* ILK/SNB sprite plane watermarks */
+ return level == 0 ? 63 : 255;
+}
+
+static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
+ int level)
+{
+ if (INTEL_INFO(dev)->gen >= 7)
+ return level == 0 ? 63 : 255;
+ else
+ return level == 0 ? 31 : 63;
+}
+
+static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
+{
+ if (INTEL_INFO(dev)->gen >= 8)
+ return 31;
+ else
+ return 15;
+}
+
+/* Calculate the maximum primary/sprite plane watermark */
+static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
+ int level,
+ const struct intel_wm_config *config,
+ enum intel_ddb_partitioning ddb_partitioning,
+ bool is_sprite)
+{
+ unsigned int fifo_size = ilk_display_fifo_size(dev);
+
+ /* if sprites aren't enabled, sprites get nothing */
+ if (is_sprite && !config->sprites_enabled)
+ return 0;
+
+ /* HSW allows LP1+ watermarks even with multiple pipes */
+ if (level == 0 || config->num_pipes_active > 1) {
+ fifo_size /= INTEL_INFO(dev)->num_pipes;
+
+ /*
+ * For some reason the non self refresh
+ * FIFO size is only half of the self
+ * refresh FIFO size on ILK/SNB.
+ */
+ if (INTEL_INFO(dev)->gen <= 6)
+ fifo_size /= 2;
+ }
+
+ if (config->sprites_enabled) {
+ /* level 0 is always calculated with 1:1 split */
+ if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
+ if (is_sprite)
+ fifo_size *= 5;
+ fifo_size /= 6;
+ } else {
+ fifo_size /= 2;
+ }
+ }
+
+ /* clamp to max that the registers can hold */
+ return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
+}
+
+/* Calculate the maximum cursor plane watermark */
+static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
+ int level,
+ const struct intel_wm_config *config)
+{
+ /* HSW LP1+ watermarks w/ multiple pipes */
+ if (level > 0 && config->num_pipes_active > 1)
+ return 64;
+
+ /* otherwise just report max that registers can hold */
+ return ilk_cursor_wm_reg_max(dev, level);
+}
+
+static void ilk_compute_wm_maximums(const struct drm_device *dev,
+ int level,
+ const struct intel_wm_config *config,
+ enum intel_ddb_partitioning ddb_partitioning,
+ struct ilk_wm_maximums *max)
+{
+ max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
+ max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
+ max->cur = ilk_cursor_wm_max(dev, level, config);
+ max->fbc = ilk_fbc_wm_reg_max(dev);
+}
+
+static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
+ int level,
+ struct ilk_wm_maximums *max)
+{
+ max->pri = ilk_plane_wm_reg_max(dev, level, false);
+ max->spr = ilk_plane_wm_reg_max(dev, level, true);
+ max->cur = ilk_cursor_wm_reg_max(dev, level);
+ max->fbc = ilk_fbc_wm_reg_max(dev);
+}
+
+static bool ilk_validate_wm_level(int level,
+ const struct ilk_wm_maximums *max,
+ struct intel_wm_level *result)
+{
+ bool ret;
+
+ /* already determined to be invalid? */
+ if (!result->enable)
+ return false;
+
+ result->enable = result->pri_val <= max->pri &&
+ result->spr_val <= max->spr &&
+ result->cur_val <= max->cur;
+
+ ret = result->enable;
+
+ /*
+ * HACK until we can pre-compute everything,
+ * and thus fail gracefully if LP0 watermarks
+ * are exceeded...
+ */
+ if (level == 0 && !result->enable) {
+ if (result->pri_val > max->pri)
+ DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
+ level, result->pri_val, max->pri);
+ if (result->spr_val > max->spr)
+ DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
+ level, result->spr_val, max->spr);
+ if (result->cur_val > max->cur)
+ DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
+ level, result->cur_val, max->cur);
+
+ result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
+ result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
+ result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
+ result->enable = true;
+ }
+
+ return ret;
+}
+
+static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
+ int level,
+ const struct ilk_pipe_wm_parameters *p,
+ struct intel_wm_level *result)
+{
+ uint16_t pri_latency = dev_priv->wm.pri_latency[level];
+ uint16_t spr_latency = dev_priv->wm.spr_latency[level];
+ uint16_t cur_latency = dev_priv->wm.cur_latency[level];
+
+ /* WM1+ latency values stored in 0.5us units */
+ if (level > 0) {
+ pri_latency *= 5;
+ spr_latency *= 5;
+ cur_latency *= 5;
+ }
+
+ result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
+ result->spr_val = ilk_compute_spr_wm(p, spr_latency);
+ result->cur_val = ilk_compute_cur_wm(p, cur_latency);
+ result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
+ result->enable = true;
+}
+
+static uint32_t
+hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ struct drm_display_mode *mode = &intel_crtc->config->base.adjusted_mode;
+ u32 linetime, ips_linetime;
+
+ if (!intel_crtc_active(crtc))
+ return 0;
+
+ /* The WM are computed with base on how long it takes to fill a single
+ * row at the given clock rate, multiplied by 8.
+ * */
+ linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
+ mode->crtc_clock);
+ ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
+ intel_ddi_get_cdclk_freq(dev_priv));
+
+ return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
+ PIPE_WM_LINETIME_TIME(linetime);
+}
+
+static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ if (IS_GEN9(dev)) {
+ uint32_t val;
+ int ret, i;
+ int level, max_level = ilk_wm_max_level(dev);
+
+ /* read the first set of memory latencies[0:3] */
+ val = 0; /* data0 to be programmed to 0 for first set */
+ mutex_lock(&dev_priv->rps.hw_lock);
+ ret = sandybridge_pcode_read(dev_priv,
+ GEN9_PCODE_READ_MEM_LATENCY,
+ &val);
+ mutex_unlock(&dev_priv->rps.hw_lock);
+
+ if (ret) {
+ DRM_ERROR("SKL Mailbox read error = %d\n", ret);
+ return;
+ }
+
+ wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
+ wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
+ GEN9_MEM_LATENCY_LEVEL_MASK;
+ wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
+ GEN9_MEM_LATENCY_LEVEL_MASK;
+ wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
+ GEN9_MEM_LATENCY_LEVEL_MASK;
+
+ /* read the second set of memory latencies[4:7] */
+ val = 1; /* data0 to be programmed to 1 for second set */
+ mutex_lock(&dev_priv->rps.hw_lock);
+ ret = sandybridge_pcode_read(dev_priv,
+ GEN9_PCODE_READ_MEM_LATENCY,
+ &val);
+ mutex_unlock(&dev_priv->rps.hw_lock);
+ if (ret) {
+ DRM_ERROR("SKL Mailbox read error = %d\n", ret);
+ return;
+ }
+
+ wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
+ wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
+ GEN9_MEM_LATENCY_LEVEL_MASK;
+ wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
+ GEN9_MEM_LATENCY_LEVEL_MASK;
+ wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
+ GEN9_MEM_LATENCY_LEVEL_MASK;
+
+ /*
+ * WaWmMemoryReadLatency:skl
+ *
+ * punit doesn't take into account the read latency so we need
+ * to add 2us to the various latency levels we retrieve from
+ * the punit.
+ * - W0 is a bit special in that it's the only level that
+ * can't be disabled if we want to have display working, so
+ * we always add 2us there.
+ * - For levels >=1, punit returns 0us latency when they are
+ * disabled, so we respect that and don't add 2us then
+ *
+ * Additionally, if a level n (n > 1) has a 0us latency, all
+ * levels m (m >= n) need to be disabled. We make sure to
+ * sanitize the values out of the punit to satisfy this
+ * requirement.
+ */
+ wm[0] += 2;
+ for (level = 1; level <= max_level; level++)
+ if (wm[level] != 0)
+ wm[level] += 2;
+ else {
+ for (i = level + 1; i <= max_level; i++)
+ wm[i] = 0;
+
+ break;
+ }
+ } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+ uint64_t sskpd = I915_READ64(MCH_SSKPD);
+
+ wm[0] = (sskpd >> 56) & 0xFF;
+ if (wm[0] == 0)
+ wm[0] = sskpd & 0xF;
+ wm[1] = (sskpd >> 4) & 0xFF;
+ wm[2] = (sskpd >> 12) & 0xFF;
+ wm[3] = (sskpd >> 20) & 0x1FF;
+ wm[4] = (sskpd >> 32) & 0x1FF;
+ } else if (INTEL_INFO(dev)->gen >= 6) {
+ uint32_t sskpd = I915_READ(MCH_SSKPD);
+
+ wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
+ wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
+ wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
+ wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
+ } else if (INTEL_INFO(dev)->gen >= 5) {
+ uint32_t mltr = I915_READ(MLTR_ILK);
+
+ /* ILK primary LP0 latency is 700 ns */
+ wm[0] = 7;
+ wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
+ wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
+ }
+}
+
+static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
+{
+ /* ILK sprite LP0 latency is 1300 ns */
+ if (INTEL_INFO(dev)->gen == 5)
+ wm[0] = 13;
+}
+
+static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
+{
+ /* ILK cursor LP0 latency is 1300 ns */
+ if (INTEL_INFO(dev)->gen == 5)
+ wm[0] = 13;
+
+ /* WaDoubleCursorLP3Latency:ivb */
+ if (IS_IVYBRIDGE(dev))
+ wm[3] *= 2;
+}
+
+/**
+ * ilk_wm_max_level - Returns maximum watermark level for current platform
+ * @dev: drm device
+ *
+ * Different Intel platforms have multiple watermark levels, with higher levels
+ * corresponding to deeper hardware sleep states.
+ */
+int ilk_wm_max_level(const struct drm_device *dev)
+{
+ /* how many WM levels are we expecting */
+ if (IS_GEN9(dev))
+ return 7;
+ else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+ return 4;
+ else if (INTEL_INFO(dev)->gen >= 6)
+ return 3;
+ else
+ return 2;
+}
+
+static void intel_print_wm_latency(struct drm_device *dev,
+ const char *name,
+ const uint16_t wm[8])
+{
+ int level, max_level = ilk_wm_max_level(dev);
+
+ for (level = 0; level <= max_level; level++) {
+ unsigned int latency = wm[level];
+
+ if (latency == 0) {
+ DRM_ERROR("%s WM%d latency not provided\n",
+ name, level);
+ continue;
+ }
+
+ /*
+ * - latencies are in us on gen9.
+ * - before then, WM1+ latency values are in 0.5us units
+ */
+ if (IS_GEN9(dev))
+ latency *= 10;
+ else if (level > 0)
+ latency *= 5;
+
+ DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
+ name, level, wm[level],
+ latency / 10, latency % 10);
+ }
+}
+
+static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
+ uint16_t wm[5], uint16_t min)
+{
+ int level, max_level = ilk_wm_max_level(dev_priv->dev);
+
+ if (wm[0] >= min)
+ return false;
+
+ wm[0] = max(wm[0], min);
+ for (level = 1; level <= max_level; level++)
+ wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
+
+ return true;
+}
+
+static void snb_wm_latency_quirk(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ bool changed;
+
+ /*
+ * The BIOS provided WM memory latency values are often
+ * inadequate for high resolution displays. Adjust them.
+ */
+ changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
+ ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
+ ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
+
+ if (!changed)
+ return;
+
+ DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
+ intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
+ intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
+ intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
+}
+
+static void ilk_setup_wm_latency(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
+
+ memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
+ sizeof(dev_priv->wm.pri_latency));
+ memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
+ sizeof(dev_priv->wm.pri_latency));
+
+ intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
+ intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
+
+ intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
+ intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
+ intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
+
+ if (IS_GEN6(dev))
+ snb_wm_latency_quirk(dev);
+}
+
+static void skl_setup_wm_latency(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
+ intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
+}
+
+static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
+ struct ilk_pipe_wm_parameters *p)
+{
+ struct drm_device *dev = crtc->dev;
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ enum pipe pipe = intel_crtc->pipe;
+ struct drm_plane *plane;
+
+ if (!intel_crtc_active(crtc))
+ return;
+
+ p->active = true;
+ p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
+ p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
+ p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8;
+ p->cur.bytes_per_pixel = 4;
+ p->pri.horiz_pixels = intel_crtc->config->pipe_src_w;
+ p->cur.horiz_pixels = intel_crtc->cursor_width;
+ /* TODO: for now, assume primary and cursor planes are always enabled. */
+ p->pri.enabled = true;
+ p->cur.enabled = true;
+
+ drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
+ struct intel_plane *intel_plane = to_intel_plane(plane);
+
+ if (intel_plane->pipe == pipe) {
+ p->spr = intel_plane->wm;
+ break;
+ }
+ }
+}
+
+static void ilk_compute_wm_config(struct drm_device *dev,
+ struct intel_wm_config *config)
+{
+ struct intel_crtc *intel_crtc;
+
+ /* Compute the currently _active_ config */
+ for_each_intel_crtc(dev, intel_crtc) {
+ const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
+
+ if (!wm->pipe_enabled)
+ continue;
+
+ config->sprites_enabled |= wm->sprites_enabled;
+ config->sprites_scaled |= wm->sprites_scaled;
+ config->num_pipes_active++;
+ }
+}
+
+/* Compute new watermarks for the pipe */
+static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
+ const struct ilk_pipe_wm_parameters *params,
+ struct intel_pipe_wm *pipe_wm)
+{
+ struct drm_device *dev = crtc->dev;
+ const struct drm_i915_private *dev_priv = dev->dev_private;
+ int level, max_level = ilk_wm_max_level(dev);
+ /* LP0 watermark maximums depend on this pipe alone */
+ struct intel_wm_config config = {
+ .num_pipes_active = 1,
+ .sprites_enabled = params->spr.enabled,
+ .sprites_scaled = params->spr.scaled,
+ };
+ struct ilk_wm_maximums max;
+
+ pipe_wm->pipe_enabled = params->active;
+ pipe_wm->sprites_enabled = params->spr.enabled;
+ pipe_wm->sprites_scaled = params->spr.scaled;
+
+ /* ILK/SNB: LP2+ watermarks only w/o sprites */
+ if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
+ max_level = 1;
+
+ /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
+ if (params->spr.scaled)
+ max_level = 0;
+
+ ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]);
+
+ if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+ pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
+
+ /* LP0 watermarks always use 1/2 DDB partitioning */
+ ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
+
+ /* At least LP0 must be valid */
+ if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
+ return false;
+
+ ilk_compute_wm_reg_maximums(dev, 1, &max);
+
+ for (level = 1; level <= max_level; level++) {
+ struct intel_wm_level wm = {};
+
+ ilk_compute_wm_level(dev_priv, level, params, &wm);
+
+ /*
+ * Disable any watermark level that exceeds the
+ * register maximums since such watermarks are
+ * always invalid.
+ */
+ if (!ilk_validate_wm_level(level, &max, &wm))
+ break;
+
+ pipe_wm->wm[level] = wm;
+ }
+
+ return true;
+}
+
+/*
+ * Merge the watermarks from all active pipes for a specific level.
+ */
+static void ilk_merge_wm_level(struct drm_device *dev,
+ int level,
+ struct intel_wm_level *ret_wm)
+{
+ const struct intel_crtc *intel_crtc;
+
+ ret_wm->enable = true;
+
+ for_each_intel_crtc(dev, intel_crtc) {
+ const struct intel_pipe_wm *active = &intel_crtc->wm.active;
+ const struct intel_wm_level *wm = &active->wm[level];
+
+ if (!active->pipe_enabled)
+ continue;
+
+ /*
+ * The watermark values may have been used in the past,
+ * so we must maintain them in the registers for some
+ * time even if the level is now disabled.
+ */
+ if (!wm->enable)
+ ret_wm->enable = false;
+
+ ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
+ ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
+ ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
+ ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
+ }
+}
+
+/*
+ * Merge all low power watermarks for all active pipes.
+ */
+static void ilk_wm_merge(struct drm_device *dev,
+ const struct intel_wm_config *config,
+ const struct ilk_wm_maximums *max,
+ struct intel_pipe_wm *merged)
+{
+ int level, max_level = ilk_wm_max_level(dev);
+ int last_enabled_level = max_level;
+
+ /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
+ if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
+ config->num_pipes_active > 1)
+ return;
+
+ /* ILK: FBC WM must be disabled always */
+ merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
+
+ /* merge each WM1+ level */
+ for (level = 1; level <= max_level; level++) {
+ struct intel_wm_level *wm = &merged->wm[level];
+
+ ilk_merge_wm_level(dev, level, wm);
+
+ if (level > last_enabled_level)
+ wm->enable = false;
+ else if (!ilk_validate_wm_level(level, max, wm))
+ /* make sure all following levels get disabled */
+ last_enabled_level = level - 1;
+
+ /*
+ * The spec says it is preferred to disable
+ * FBC WMs instead of disabling a WM level.
+ */
+ if (wm->fbc_val > max->fbc) {
+ if (wm->enable)
+ merged->fbc_wm_enabled = false;
+ wm->fbc_val = 0;
+ }
+ }
+
+ /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
+ /*
+ * FIXME this is racy. FBC might get enabled later.
+ * What we should check here is whether FBC can be
+ * enabled sometime later.
+ */
+ if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
+ for (level = 2; level <= max_level; level++) {
+ struct intel_wm_level *wm = &merged->wm[level];
+
+ wm->enable = false;
+ }
+ }
+}
+
+static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
+{
+ /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
+ return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
+}
+
+/* The value we need to program into the WM_LPx latency field */
+static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+ return 2 * level;
+ else
+ return dev_priv->wm.pri_latency[level];
+}
+
+static void ilk_compute_wm_results(struct drm_device *dev,
+ const struct intel_pipe_wm *merged,
+ enum intel_ddb_partitioning partitioning,
+ struct ilk_wm_values *results)
+{
+ struct intel_crtc *intel_crtc;
+ int level, wm_lp;
+
+ results->enable_fbc_wm = merged->fbc_wm_enabled;
+ results->partitioning = partitioning;
+
+ /* LP1+ register values */
+ for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
+ const struct intel_wm_level *r;
+
+ level = ilk_wm_lp_to_level(wm_lp, merged);
+
+ r = &merged->wm[level];
+
+ /*
+ * Maintain the watermark values even if the level is
+ * disabled. Doing otherwise could cause underruns.
+ */
+ results->wm_lp[wm_lp - 1] =
+ (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
+ (r->pri_val << WM1_LP_SR_SHIFT) |
+ r->cur_val;
+
+ if (r->enable)
+ results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
+
+ if (INTEL_INFO(dev)->gen >= 8)
+ results->wm_lp[wm_lp - 1] |=
+ r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
+ else
+ results->wm_lp[wm_lp - 1] |=
+ r->fbc_val << WM1_LP_FBC_SHIFT;
+
+ /*
+ * Always set WM1S_LP_EN when spr_val != 0, even if the
+ * level is disabled. Doing otherwise could cause underruns.
+ */
+ if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
+ WARN_ON(wm_lp != 1);
+ results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
+ } else
+ results->wm_lp_spr[wm_lp - 1] = r->spr_val;
+ }
+
+ /* LP0 register values */
+ for_each_intel_crtc(dev, intel_crtc) {
+ enum pipe pipe = intel_crtc->pipe;
+ const struct intel_wm_level *r =
+ &intel_crtc->wm.active.wm[0];
+
+ if (WARN_ON(!r->enable))
+ continue;
+
+ results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
+
+ results->wm_pipe[pipe] =
+ (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
+ (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
+ r->cur_val;
+ }
+}
+
+/* Find the result with the highest level enabled. Check for enable_fbc_wm in
+ * case both are at the same level. Prefer r1 in case they're the same. */
+static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
+ struct intel_pipe_wm *r1,
+ struct intel_pipe_wm *r2)
+{
+ int level, max_level = ilk_wm_max_level(dev);
+ int level1 = 0, level2 = 0;
+
+ for (level = 1; level <= max_level; level++) {
+ if (r1->wm[level].enable)
+ level1 = level;
+ if (r2->wm[level].enable)
+ level2 = level;
+ }
+
+ if (level1 == level2) {
+ if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
+ return r2;
+ else
+ return r1;
+ } else if (level1 > level2) {
+ return r1;
+ } else {
+ return r2;
+ }
+}
+
+/* dirty bits used to track which watermarks need changes */
+#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
+#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
+#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
+#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
+#define WM_DIRTY_FBC (1 << 24)
+#define WM_DIRTY_DDB (1 << 25)
+
+static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
+ const struct ilk_wm_values *old,
+ const struct ilk_wm_values *new)
+{
+ unsigned int dirty = 0;
+ enum pipe pipe;
+ int wm_lp;
+
+ for_each_pipe(dev_priv, pipe) {
+ if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
+ dirty |= WM_DIRTY_LINETIME(pipe);
+ /* Must disable LP1+ watermarks too */
+ dirty |= WM_DIRTY_LP_ALL;
+ }
+
+ if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
+ dirty |= WM_DIRTY_PIPE(pipe);
+ /* Must disable LP1+ watermarks too */
+ dirty |= WM_DIRTY_LP_ALL;
+ }
+ }
+
+ if (old->enable_fbc_wm != new->enable_fbc_wm) {
+ dirty |= WM_DIRTY_FBC;
+ /* Must disable LP1+ watermarks too */
+ dirty |= WM_DIRTY_LP_ALL;
+ }
+
+ if (old->partitioning != new->partitioning) {
+ dirty |= WM_DIRTY_DDB;
+ /* Must disable LP1+ watermarks too */
+ dirty |= WM_DIRTY_LP_ALL;
+ }
+
+ /* LP1+ watermarks already deemed dirty, no need to continue */
+ if (dirty & WM_DIRTY_LP_ALL)
+ return dirty;
+
+ /* Find the lowest numbered LP1+ watermark in need of an update... */
+ for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
+ if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
+ old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
+ break;
+ }
+
+ /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
+ for (; wm_lp <= 3; wm_lp++)
+ dirty |= WM_DIRTY_LP(wm_lp);
+
+ return dirty;
+}
+
+static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
+ unsigned int dirty)
+{
+ struct ilk_wm_values *previous = &dev_priv->wm.hw;
+ bool changed = false;
+
+ if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
+ previous->wm_lp[2] &= ~WM1_LP_SR_EN;
+ I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
+ changed = true;
+ }
+ if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
+ previous->wm_lp[1] &= ~WM1_LP_SR_EN;
+ I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
+ changed = true;
+ }
+ if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
+ previous->wm_lp[0] &= ~WM1_LP_SR_EN;
+ I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
+ changed = true;
+ }
+
+ /*
+ * Don't touch WM1S_LP_EN here.
+ * Doing so could cause underruns.
+ */
+
+ return changed;
+}
+
+/*
+ * The spec says we shouldn't write when we don't need, because every write
+ * causes WMs to be re-evaluated, expending some power.
+ */
+static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
+ struct ilk_wm_values *results)
+{
+ struct drm_device *dev = dev_priv->dev;
+ struct ilk_wm_values *previous = &dev_priv->wm.hw;
+ unsigned int dirty;
+ uint32_t val;
+
+ dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
+ if (!dirty)
+ return;
+
+ _ilk_disable_lp_wm(dev_priv, dirty);
+
+ if (dirty & WM_DIRTY_PIPE(PIPE_A))
+ I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
+ if (dirty & WM_DIRTY_PIPE(PIPE_B))
+ I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
+ if (dirty & WM_DIRTY_PIPE(PIPE_C))
+ I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
+
+ if (dirty & WM_DIRTY_LINETIME(PIPE_A))
+ I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
+ if (dirty & WM_DIRTY_LINETIME(PIPE_B))
+ I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
+ if (dirty & WM_DIRTY_LINETIME(PIPE_C))
+ I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
+
+ if (dirty & WM_DIRTY_DDB) {
+ if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+ val = I915_READ(WM_MISC);
+ if (results->partitioning == INTEL_DDB_PART_1_2)
+ val &= ~WM_MISC_DATA_PARTITION_5_6;
+ else
+ val |= WM_MISC_DATA_PARTITION_5_6;
+ I915_WRITE(WM_MISC, val);
+ } else {
+ val = I915_READ(DISP_ARB_CTL2);
+ if (results->partitioning == INTEL_DDB_PART_1_2)
+ val &= ~DISP_DATA_PARTITION_5_6;
+ else
+ val |= DISP_DATA_PARTITION_5_6;
+ I915_WRITE(DISP_ARB_CTL2, val);
+ }
+ }
+
+ if (dirty & WM_DIRTY_FBC) {
+ val = I915_READ(DISP_ARB_CTL);
+ if (results->enable_fbc_wm)
+ val &= ~DISP_FBC_WM_DIS;
+ else
+ val |= DISP_FBC_WM_DIS;
+ I915_WRITE(DISP_ARB_CTL, val);
+ }
+
+ if (dirty & WM_DIRTY_LP(1) &&
+ previous->wm_lp_spr[0] != results->wm_lp_spr[0])
+ I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
+
+ if (INTEL_INFO(dev)->gen >= 7) {
+ if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
+ I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
+ if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
+ I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
+ }
+
+ if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
+ I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
+ if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
+ I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
+ if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
+ I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
+
+ dev_priv->wm.hw = *results;
+}
+
+static bool ilk_disable_lp_wm(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
+}
+
+/*
+ * On gen9, we need to allocate Display Data Buffer (DDB) portions to the
+ * different active planes.
+ */
+
+#define SKL_DDB_SIZE 896 /* in blocks */
+
+static void
+skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
+ struct drm_crtc *for_crtc,
+ const struct intel_wm_config *config,
+ const struct skl_pipe_wm_parameters *params,
+ struct skl_ddb_entry *alloc /* out */)
+{
+ struct drm_crtc *crtc;
+ unsigned int pipe_size, ddb_size;
+ int nth_active_pipe;
+
+ if (!params->active) {
+ alloc->start = 0;
+ alloc->end = 0;
+ return;
+ }
+
+ ddb_size = SKL_DDB_SIZE;
+
+ ddb_size -= 4; /* 4 blocks for bypass path allocation */
+
+ nth_active_pipe = 0;
+ for_each_crtc(dev, crtc) {
+ if (!intel_crtc_active(crtc))
+ continue;
+
+ if (crtc == for_crtc)
+ break;
+
+ nth_active_pipe++;
+ }
+
+ pipe_size = ddb_size / config->num_pipes_active;
+ alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
+ alloc->end = alloc->start + pipe_size;
+}
+
+static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
+{
+ if (config->num_pipes_active == 1)
+ return 32;
+
+ return 8;
+}
+
+static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
+{
+ entry->start = reg & 0x3ff;
+ entry->end = (reg >> 16) & 0x3ff;
+ if (entry->end)
+ entry->end += 1;
+}
+
+/**
+ * skl_ddb_get_hw_state - Read current HW DDB information
+ * @dev_priv: i915 private structure
+ * @ddb: Structure to read current hardware values into
+ *
+ * The DDB allocation contains (stard,end) values for the display buffer of
+ * each plane.
+ */
+void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
+ struct skl_ddb_allocation *ddb /* out */)
+{
+ struct drm_device *dev = dev_priv->dev;
+ enum pipe pipe;
+ int plane;
+ u32 val;
+
+ for_each_pipe(dev_priv, pipe) {
+ for_each_plane(pipe, plane) {
+ val = I915_READ(PLANE_BUF_CFG(pipe, plane));
+ skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
+ val);
+ }
+
+ val = I915_READ(CUR_BUF_CFG(pipe));
+ skl_ddb_entry_init_from_hw(&ddb->cursor[pipe], val);
+ }
+}
+
+static unsigned int
+skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p)
+{
+ return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
+}
+
+/*
+ * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
+ * a 8192x4096 at 32bpp framebuffer:
+ * 3 * 4096 * 8192 * 4 < 2^32
+ */
+static unsigned int
+skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
+ const struct skl_pipe_wm_parameters *params)
+{
+ unsigned int total_data_rate = 0;
+ int plane;
+
+ for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
+ const struct intel_plane_wm_parameters *p;
+
+ p = ¶ms->plane[plane];
+ if (!p->enabled)
+ continue;
+
+ total_data_rate += skl_plane_relative_data_rate(p);
+ }
+
+ return total_data_rate;
+}
+
+static void
+skl_allocate_pipe_ddb(struct drm_crtc *crtc,
+ const struct intel_wm_config *config,
+ const struct skl_pipe_wm_parameters *params,
+ struct skl_ddb_allocation *ddb /* out */)
+{
+ struct drm_device *dev = crtc->dev;
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ enum pipe pipe = intel_crtc->pipe;
+ struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
+ uint16_t alloc_size, start, cursor_blocks;
+ unsigned int total_data_rate;
+ int plane;
+
+ skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
+ alloc_size = skl_ddb_entry_size(alloc);
+ if (alloc_size == 0) {
+ memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
+ memset(&ddb->cursor[pipe], 0, sizeof(ddb->cursor[pipe]));
+ return;
+ }
+
+ cursor_blocks = skl_cursor_allocation(config);
+ ddb->cursor[pipe].start = alloc->end - cursor_blocks;
+ ddb->cursor[pipe].end = alloc->end;
+
+ alloc_size -= cursor_blocks;
+ alloc->end -= cursor_blocks;
+
+ /*
+ * Each active plane get a portion of the remaining space, in
+ * proportion to the amount of data they need to fetch from memory.
+ *
+ * FIXME: we may not allocate every single block here.
+ */
+ total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
+
+ start = alloc->start;
+ for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
+ const struct intel_plane_wm_parameters *p;
+ unsigned int data_rate;
+ uint16_t plane_blocks;
+
+ p = ¶ms->plane[plane];
+ if (!p->enabled)
+ continue;
+
+ data_rate = skl_plane_relative_data_rate(p);
+
+ /*
+ * promote the expression to 64 bits to avoid overflowing, the
+ * result is < available as data_rate / total_data_rate < 1
+ */
+ plane_blocks = div_u64((uint64_t)alloc_size * data_rate,
+ total_data_rate);
+
+ ddb->plane[pipe][plane].start = start;
+ ddb->plane[pipe][plane].end = start + plane_blocks;
+
+ start += plane_blocks;
+ }
+
+}
+
+static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
+{
+ /* TODO: Take into account the scalers once we support them */
+ return config->base.adjusted_mode.crtc_clock;
+}
+
+/*
+ * The max latency should be 257 (max the punit can code is 255 and we add 2us
+ * for the read latency) and bytes_per_pixel should always be <= 8, so that
+ * should allow pixel_rate up to ~2 GHz which seems sufficient since max
+ * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
+*/
+static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
+ uint32_t latency)
+{
+ uint32_t wm_intermediate_val, ret;
+
+ if (latency == 0)
+ return UINT_MAX;
+
+ wm_intermediate_val = latency * pixel_rate * bytes_per_pixel;
+ ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
+
+ return ret;
+}
+
+static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
+ uint32_t horiz_pixels, uint8_t bytes_per_pixel,
+ uint32_t latency)
+{
+ uint32_t ret, plane_bytes_per_line, wm_intermediate_val;
+
+ if (latency == 0)
+ return UINT_MAX;
+
+ plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
+ wm_intermediate_val = latency * pixel_rate;
+ ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
+ plane_bytes_per_line;
+
+ return ret;
+}
+
+static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
+ const struct intel_crtc *intel_crtc)
+{
+ struct drm_device *dev = intel_crtc->base.dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
+ enum pipe pipe = intel_crtc->pipe;
+
+ if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
+ sizeof(new_ddb->plane[pipe])))
+ return true;
+
+ if (memcmp(&new_ddb->cursor[pipe], &cur_ddb->cursor[pipe],
+ sizeof(new_ddb->cursor[pipe])))
+ return true;
+
+ return false;
+}
+
+static void skl_compute_wm_global_parameters(struct drm_device *dev,
+ struct intel_wm_config *config)
+{
+ struct drm_crtc *crtc;
+ struct drm_plane *plane;
+
+ list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+ config->num_pipes_active += intel_crtc_active(crtc);
+
+ /* FIXME: I don't think we need those two global parameters on SKL */
+ list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
+ struct intel_plane *intel_plane = to_intel_plane(plane);
+
+ config->sprites_enabled |= intel_plane->wm.enabled;
+ config->sprites_scaled |= intel_plane->wm.scaled;
+ }
+}
+
+static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
+ struct skl_pipe_wm_parameters *p)
+{
+ struct drm_device *dev = crtc->dev;
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ enum pipe pipe = intel_crtc->pipe;
+ struct drm_plane *plane;
+ int i = 1; /* Index for sprite planes start */
+
+ p->active = intel_crtc_active(crtc);
+ if (p->active) {
+ p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
+ p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config);
+
+ /*
+ * For now, assume primary and cursor planes are always enabled.
+ */
+ p->plane[0].enabled = true;
+ p->plane[0].bytes_per_pixel =
+ crtc->primary->fb->bits_per_pixel / 8;
+ p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w;
+ p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h;
+
+ p->cursor.enabled = true;
+ p->cursor.bytes_per_pixel = 4;
+ p->cursor.horiz_pixels = intel_crtc->cursor_width ?
+ intel_crtc->cursor_width : 64;
+ }
+
+ list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
+ struct intel_plane *intel_plane = to_intel_plane(plane);
+
+ if (intel_plane->pipe == pipe &&
+ plane->type == DRM_PLANE_TYPE_OVERLAY)
+ p->plane[i++] = intel_plane->wm;
+ }
+}
+
+static bool skl_compute_plane_wm(struct skl_pipe_wm_parameters *p,
+ struct intel_plane_wm_parameters *p_params,
+ uint16_t ddb_allocation,
+ uint32_t mem_value,
+ uint16_t *out_blocks, /* out */
+ uint8_t *out_lines /* out */)
+{
+ uint32_t method1, method2, plane_bytes_per_line, res_blocks, res_lines;
+ uint32_t result_bytes;
+
+ if (mem_value == 0 || !p->active || !p_params->enabled)
+ return false;
+
+ method1 = skl_wm_method1(p->pixel_rate,
+ p_params->bytes_per_pixel,
+ mem_value);
+ method2 = skl_wm_method2(p->pixel_rate,
+ p->pipe_htotal,
+ p_params->horiz_pixels,
+ p_params->bytes_per_pixel,
+ mem_value);
+
+ plane_bytes_per_line = p_params->horiz_pixels *
+ p_params->bytes_per_pixel;
+
+ /* For now xtile and linear */
+ if (((ddb_allocation * 512) / plane_bytes_per_line) >= 1)
+ result_bytes = min(method1, method2);
+ else
+ result_bytes = method1;
+
+ res_blocks = DIV_ROUND_UP(result_bytes, 512) + 1;
+ res_lines = DIV_ROUND_UP(result_bytes, plane_bytes_per_line);
+
+ if (res_blocks > ddb_allocation || res_lines > 31)
+ return false;
+
+ *out_blocks = res_blocks;
+ *out_lines = res_lines;
+
+ return true;
+}
+
+static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
+ struct skl_ddb_allocation *ddb,
+ struct skl_pipe_wm_parameters *p,
+ enum pipe pipe,
+ int level,
+ int num_planes,
+ struct skl_wm_level *result)
+{
+ uint16_t latency = dev_priv->wm.skl_latency[level];
+ uint16_t ddb_blocks;
+ int i;
+
+ for (i = 0; i < num_planes; i++) {
+ ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
+
+ result->plane_en[i] = skl_compute_plane_wm(p, &p->plane[i],
+ ddb_blocks,
+ latency,
+ &result->plane_res_b[i],
+ &result->plane_res_l[i]);
+ }
+
+ ddb_blocks = skl_ddb_entry_size(&ddb->cursor[pipe]);
+ result->cursor_en = skl_compute_plane_wm(p, &p->cursor, ddb_blocks,
+ latency, &result->cursor_res_b,
+ &result->cursor_res_l);
+}
+
+static uint32_t
+skl_compute_linetime_wm(struct drm_crtc *crtc, struct skl_pipe_wm_parameters *p)
+{
+ if (!intel_crtc_active(crtc))
+ return 0;
+
+ return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
+
+}
+
+static void skl_compute_transition_wm(struct drm_crtc *crtc,
+ struct skl_pipe_wm_parameters *params,
+ struct skl_wm_level *trans_wm /* out */)
+{
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ int i;
+
+ if (!params->active)
+ return;
+
+ /* Until we know more, just disable transition WMs */
+ for (i = 0; i < intel_num_planes(intel_crtc); i++)
+ trans_wm->plane_en[i] = false;
+ trans_wm->cursor_en = false;
+}
+
+static void skl_compute_pipe_wm(struct drm_crtc *crtc,
+ struct skl_ddb_allocation *ddb,
+ struct skl_pipe_wm_parameters *params,
+ struct skl_pipe_wm *pipe_wm)
+{
+ struct drm_device *dev = crtc->dev;
+ const struct drm_i915_private *dev_priv = dev->dev_private;
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ int level, max_level = ilk_wm_max_level(dev);
+
+ for (level = 0; level <= max_level; level++) {
+ skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
+ level, intel_num_planes(intel_crtc),
+ &pipe_wm->wm[level]);
+ }
+ pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
+
+ skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
+}
+
+static void skl_compute_wm_results(struct drm_device *dev,
+ struct skl_pipe_wm_parameters *p,
+ struct skl_pipe_wm *p_wm,
+ struct skl_wm_values *r,
+ struct intel_crtc *intel_crtc)
+{
+ int level, max_level = ilk_wm_max_level(dev);
+ enum pipe pipe = intel_crtc->pipe;
+ uint32_t temp;
+ int i;
+
+ for (level = 0; level <= max_level; level++) {
+ for (i = 0; i < intel_num_planes(intel_crtc); i++) {
+ temp = 0;
+
+ temp |= p_wm->wm[level].plane_res_l[i] <<
+ PLANE_WM_LINES_SHIFT;
+ temp |= p_wm->wm[level].plane_res_b[i];
+ if (p_wm->wm[level].plane_en[i])
+ temp |= PLANE_WM_EN;
+
+ r->plane[pipe][i][level] = temp;
+ }
+
+ temp = 0;
+
+ temp |= p_wm->wm[level].cursor_res_l << PLANE_WM_LINES_SHIFT;
+ temp |= p_wm->wm[level].cursor_res_b;
+
+ if (p_wm->wm[level].cursor_en)
+ temp |= PLANE_WM_EN;
+
+ r->cursor[pipe][level] = temp;
+
+ }
+
+ /* transition WMs */
+ for (i = 0; i < intel_num_planes(intel_crtc); i++) {
+ temp = 0;
+ temp |= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
+ temp |= p_wm->trans_wm.plane_res_b[i];
+ if (p_wm->trans_wm.plane_en[i])
+ temp |= PLANE_WM_EN;
+
+ r->plane_trans[pipe][i] = temp;
+ }
+
+ temp = 0;
+ temp |= p_wm->trans_wm.cursor_res_l << PLANE_WM_LINES_SHIFT;
+ temp |= p_wm->trans_wm.cursor_res_b;
+ if (p_wm->trans_wm.cursor_en)
+ temp |= PLANE_WM_EN;
+
+ r->cursor_trans[pipe] = temp;
+
+ r->wm_linetime[pipe] = p_wm->linetime;
+}
+
+static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
+ const struct skl_ddb_entry *entry)
+{
+ if (entry->end)
+ I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
+ else
+ I915_WRITE(reg, 0);
+}
+
+static void skl_write_wm_values(struct drm_i915_private *dev_priv,
+ const struct skl_wm_values *new)
+{
+ struct drm_device *dev = dev_priv->dev;
+ struct intel_crtc *crtc;
+
+ list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
+ int i, level, max_level = ilk_wm_max_level(dev);
+ enum pipe pipe = crtc->pipe;
+
+ if (!new->dirty[pipe])
+ continue;
+
+ I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
+
+ for (level = 0; level <= max_level; level++) {
+ for (i = 0; i < intel_num_planes(crtc); i++)
+ I915_WRITE(PLANE_WM(pipe, i, level),
+ new->plane[pipe][i][level]);
+ I915_WRITE(CUR_WM(pipe, level),
+ new->cursor[pipe][level]);
+ }
+ for (i = 0; i < intel_num_planes(crtc); i++)
+ I915_WRITE(PLANE_WM_TRANS(pipe, i),
+ new->plane_trans[pipe][i]);
+ I915_WRITE(CUR_WM_TRANS(pipe), new->cursor_trans[pipe]);
+
+ for (i = 0; i < intel_num_planes(crtc); i++)
+ skl_ddb_entry_write(dev_priv,
+ PLANE_BUF_CFG(pipe, i),
+ &new->ddb.plane[pipe][i]);
+
+ skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
+ &new->ddb.cursor[pipe]);
+ }
+}
+
+/*
+ * When setting up a new DDB allocation arrangement, we need to correctly
+ * sequence the times at which the new allocations for the pipes are taken into
+ * account or we'll have pipes fetching from space previously allocated to
+ * another pipe.
+ *
+ * Roughly the sequence looks like:
+ * 1. re-allocate the pipe(s) with the allocation being reduced and not
+ * overlapping with a previous light-up pipe (another way to put it is:
+ * pipes with their new allocation strickly included into their old ones).
+ * 2. re-allocate the other pipes that get their allocation reduced
+ * 3. allocate the pipes having their allocation increased
+ *
+ * Steps 1. and 2. are here to take care of the following case:
+ * - Initially DDB looks like this:
+ * | B | C |
+ * - enable pipe A.
+ * - pipe B has a reduced DDB allocation that overlaps with the old pipe C
+ * allocation
+ * | A | B | C |
+ *
+ * We need to sequence the re-allocation: C, B, A (and not B, C, A).
+ */
+
+static void
+skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
+{
+ struct drm_device *dev = dev_priv->dev;
+ int plane;
+
+ DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
+
+ for_each_plane(pipe, plane) {
+ I915_WRITE(PLANE_SURF(pipe, plane),
+ I915_READ(PLANE_SURF(pipe, plane)));
+ }
+ I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
+}
+
+static bool
+skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
+ const struct skl_ddb_allocation *new,
+ enum pipe pipe)
+{
+ uint16_t old_size, new_size;
+
+ old_size = skl_ddb_entry_size(&old->pipe[pipe]);
+ new_size = skl_ddb_entry_size(&new->pipe[pipe]);
+
+ return old_size != new_size &&
+ new->pipe[pipe].start >= old->pipe[pipe].start &&
+ new->pipe[pipe].end <= old->pipe[pipe].end;
+}
+
+static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
+ struct skl_wm_values *new_values)
+{
+ struct drm_device *dev = dev_priv->dev;
+ struct skl_ddb_allocation *cur_ddb, *new_ddb;
+ bool reallocated[I915_MAX_PIPES] = {false, false, false};
+ struct intel_crtc *crtc;
+ enum pipe pipe;
+
+ new_ddb = &new_values->ddb;
+ cur_ddb = &dev_priv->wm.skl_hw.ddb;
+
+ /*
+ * First pass: flush the pipes with the new allocation contained into
+ * the old space.
+ *
+ * We'll wait for the vblank on those pipes to ensure we can safely
+ * re-allocate the freed space without this pipe fetching from it.
+ */
+ for_each_intel_crtc(dev, crtc) {
+ if (!crtc->active)
+ continue;
+
+ pipe = crtc->pipe;
+
+ if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
+ continue;
+
+ skl_wm_flush_pipe(dev_priv, pipe, 1);
+ intel_wait_for_vblank(dev, pipe);
+
+ reallocated[pipe] = true;
+ }
+
+
+ /*
+ * Second pass: flush the pipes that are having their allocation
+ * reduced, but overlapping with a previous allocation.
+ *
+ * Here as well we need to wait for the vblank to make sure the freed
+ * space is not used anymore.
+ */
+ for_each_intel_crtc(dev, crtc) {
+ if (!crtc->active)
+ continue;
+
+ pipe = crtc->pipe;
+
+ if (reallocated[pipe])
+ continue;
+
+ if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
+ skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
+ skl_wm_flush_pipe(dev_priv, pipe, 2);
+ intel_wait_for_vblank(dev, pipe);
+ reallocated[pipe] = true;
+ }
+ }
+
+ /*
+ * Third pass: flush the pipes that got more space allocated.
+ *
+ * We don't need to actively wait for the update here, next vblank
+ * will just get more DDB space with the correct WM values.
+ */
+ for_each_intel_crtc(dev, crtc) {
+ if (!crtc->active)
+ continue;
+
+ pipe = crtc->pipe;
+
+ /*
+ * At this point, only the pipes more space than before are
+ * left to re-allocate.
+ */
+ if (reallocated[pipe])
+ continue;
+
+ skl_wm_flush_pipe(dev_priv, pipe, 3);
+ }
+}
+
+static bool skl_update_pipe_wm(struct drm_crtc *crtc,
+ struct skl_pipe_wm_parameters *params,
+ struct intel_wm_config *config,
+ struct skl_ddb_allocation *ddb, /* out */
+ struct skl_pipe_wm *pipe_wm /* out */)
+{
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+
+ skl_compute_wm_pipe_parameters(crtc, params);
+ skl_allocate_pipe_ddb(crtc, config, params, ddb);
+ skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
+
+ if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
+ return false;
+
+ intel_crtc->wm.skl_active = *pipe_wm;
+ return true;
+}
+
+static void skl_update_other_pipe_wm(struct drm_device *dev,
+ struct drm_crtc *crtc,
+ struct intel_wm_config *config,
+ struct skl_wm_values *r)
+{
+ struct intel_crtc *intel_crtc;
+ struct intel_crtc *this_crtc = to_intel_crtc(crtc);
+
+ /*
+ * If the WM update hasn't changed the allocation for this_crtc (the
+ * crtc we are currently computing the new WM values for), other
+ * enabled crtcs will keep the same allocation and we don't need to
+ * recompute anything for them.
+ */
+ if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
+ return;
+
+ /*
+ * Otherwise, because of this_crtc being freshly enabled/disabled, the
+ * other active pipes need new DDB allocation and WM values.
+ */
+ list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
+ base.head) {
+ struct skl_pipe_wm_parameters params = {};
+ struct skl_pipe_wm pipe_wm = {};
+ bool wm_changed;
+
+ if (this_crtc->pipe == intel_crtc->pipe)
+ continue;
+
+ if (!intel_crtc->active)
+ continue;
+
+ wm_changed = skl_update_pipe_wm(&intel_crtc->base,
+ ¶ms, config,
+ &r->ddb, &pipe_wm);
+
+ /*
+ * If we end up re-computing the other pipe WM values, it's
+ * because it was really needed, so we expect the WM values to
+ * be different.
+ */
+ WARN_ON(!wm_changed);
+
+ skl_compute_wm_results(dev, ¶ms, &pipe_wm, r, intel_crtc);
+ r->dirty[intel_crtc->pipe] = true;
+ }
+}
+
+static void skl_update_wm(struct drm_crtc *crtc)
+{
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ struct drm_device *dev = crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct skl_pipe_wm_parameters params = {};
+ struct skl_wm_values *results = &dev_priv->wm.skl_results;
+ struct skl_pipe_wm pipe_wm = {};
+ struct intel_wm_config config = {};
+
+ memset(results, 0, sizeof(*results));
+
+ skl_compute_wm_global_parameters(dev, &config);
+
+ if (!skl_update_pipe_wm(crtc, ¶ms, &config,
+ &results->ddb, &pipe_wm))
+ return;
+
+ skl_compute_wm_results(dev, ¶ms, &pipe_wm, results, intel_crtc);
+ results->dirty[intel_crtc->pipe] = true;
+
+ skl_update_other_pipe_wm(dev, crtc, &config, results);
+ skl_write_wm_values(dev_priv, results);
+ skl_flush_wm_values(dev_priv, results);
+
+ /* store the new configuration */
+ dev_priv->wm.skl_hw = *results;
+}
+
+static void
+skl_update_sprite_wm(struct drm_plane *plane, struct drm_crtc *crtc,
+ uint32_t sprite_width, uint32_t sprite_height,
+ int pixel_size, bool enabled, bool scaled)
+{
+ struct intel_plane *intel_plane = to_intel_plane(plane);
+
+ intel_plane->wm.enabled = enabled;
+ intel_plane->wm.scaled = scaled;
+ intel_plane->wm.horiz_pixels = sprite_width;
+ intel_plane->wm.vert_pixels = sprite_height;
+ intel_plane->wm.bytes_per_pixel = pixel_size;
+
+ skl_update_wm(crtc);
+}
+
+static void ilk_update_wm(struct drm_crtc *crtc)
+{
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ struct drm_device *dev = crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct ilk_wm_maximums max;
+ struct ilk_pipe_wm_parameters params = {};
+ struct ilk_wm_values results = {};
+ enum intel_ddb_partitioning partitioning;
+ struct intel_pipe_wm pipe_wm = {};
+ struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
+ struct intel_wm_config config = {};
+
+ ilk_compute_wm_parameters(crtc, ¶ms);
+
+ intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm);
+
+ if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
+ return;
+
+ intel_crtc->wm.active = pipe_wm;
+
+ ilk_compute_wm_config(dev, &config);
+
+ ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
+ ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
+
+ /* 5/6 split only in single pipe config on IVB+ */
+ if (INTEL_INFO(dev)->gen >= 7 &&
+ config.num_pipes_active == 1 && config.sprites_enabled) {
+ ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
+ ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
+
+ best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
+ } else {
+ best_lp_wm = &lp_wm_1_2;
+ }
+
+ partitioning = (best_lp_wm == &lp_wm_1_2) ?
+ INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
+
+ ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
+
+ ilk_write_wm_values(dev_priv, &results);
+}
+
+static void
+ilk_update_sprite_wm(struct drm_plane *plane,
+ struct drm_crtc *crtc,
+ uint32_t sprite_width, uint32_t sprite_height,
+ int pixel_size, bool enabled, bool scaled)
+{
+ struct drm_device *dev = plane->dev;
+ struct intel_plane *intel_plane = to_intel_plane(plane);
+
+ intel_plane->wm.enabled = enabled;
+ intel_plane->wm.scaled = scaled;
+ intel_plane->wm.horiz_pixels = sprite_width;
+ intel_plane->wm.vert_pixels = sprite_width;
+ intel_plane->wm.bytes_per_pixel = pixel_size;
+
+ /*
+ * IVB workaround: must disable low power watermarks for at least
+ * one frame before enabling scaling. LP watermarks can be re-enabled
+ * when scaling is disabled.
+ *
+ * WaCxSRDisabledForSpriteScaling:ivb
+ */
+ if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
+ intel_wait_for_vblank(dev, intel_plane->pipe);
+
+ ilk_update_wm(crtc);
+}
+
+static void skl_pipe_wm_active_state(uint32_t val,
+ struct skl_pipe_wm *active,
+ bool is_transwm,
+ bool is_cursor,
+ int i,
+ int level)
+{
+ bool is_enabled = (val & PLANE_WM_EN) != 0;
+
+ if (!is_transwm) {
+ if (!is_cursor) {
+ active->wm[level].plane_en[i] = is_enabled;
+ active->wm[level].plane_res_b[i] =
+ val & PLANE_WM_BLOCKS_MASK;
+ active->wm[level].plane_res_l[i] =
+ (val >> PLANE_WM_LINES_SHIFT) &
+ PLANE_WM_LINES_MASK;
+ } else {
+ active->wm[level].cursor_en = is_enabled;
+ active->wm[level].cursor_res_b =
+ val & PLANE_WM_BLOCKS_MASK;
+ active->wm[level].cursor_res_l =
+ (val >> PLANE_WM_LINES_SHIFT) &
+ PLANE_WM_LINES_MASK;
+ }
+ } else {
+ if (!is_cursor) {
+ active->trans_wm.plane_en[i] = is_enabled;
+ active->trans_wm.plane_res_b[i] =
+ val & PLANE_WM_BLOCKS_MASK;
+ active->trans_wm.plane_res_l[i] =
+ (val >> PLANE_WM_LINES_SHIFT) &
+ PLANE_WM_LINES_MASK;
+ } else {
+ active->trans_wm.cursor_en = is_enabled;
+ active->trans_wm.cursor_res_b =
+ val & PLANE_WM_BLOCKS_MASK;
+ active->trans_wm.cursor_res_l =
+ (val >> PLANE_WM_LINES_SHIFT) &
+ PLANE_WM_LINES_MASK;
+ }
+ }
+}
+
+static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
+{
+ struct drm_device *dev = crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
+ enum pipe pipe = intel_crtc->pipe;
+ int level, i, max_level;
+ uint32_t temp;
+
+ max_level = ilk_wm_max_level(dev);
+
+ hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
+
+ for (level = 0; level <= max_level; level++) {
+ for (i = 0; i < intel_num_planes(intel_crtc); i++)
+ hw->plane[pipe][i][level] =
+ I915_READ(PLANE_WM(pipe, i, level));
+ hw->cursor[pipe][level] = I915_READ(CUR_WM(pipe, level));
+ }
+
+ for (i = 0; i < intel_num_planes(intel_crtc); i++)
+ hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
+ hw->cursor_trans[pipe] = I915_READ(CUR_WM_TRANS(pipe));
+
+ if (!intel_crtc_active(crtc))
+ return;
+
+ hw->dirty[pipe] = true;
+
+ active->linetime = hw->wm_linetime[pipe];
+
+ for (level = 0; level <= max_level; level++) {
+ for (i = 0; i < intel_num_planes(intel_crtc); i++) {
+ temp = hw->plane[pipe][i][level];
+ skl_pipe_wm_active_state(temp, active, false,
+ false, i, level);
+ }
+ temp = hw->cursor[pipe][level];
+ skl_pipe_wm_active_state(temp, active, false, true, i, level);
+ }
+
+ for (i = 0; i < intel_num_planes(intel_crtc); i++) {
+ temp = hw->plane_trans[pipe][i];
+ skl_pipe_wm_active_state(temp, active, true, false, i, 0);
+ }
+
+ temp = hw->cursor_trans[pipe];
+ skl_pipe_wm_active_state(temp, active, true, true, i, 0);
+}
+
+void skl_wm_get_hw_state(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
+ struct drm_crtc *crtc;
+
+ skl_ddb_get_hw_state(dev_priv, ddb);
+ list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+ skl_pipe_wm_get_hw_state(crtc);
+}
+
+static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
+{
+ struct drm_device *dev = crtc->dev;
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct ilk_wm_values *hw = &dev_priv->wm.hw;
+ struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+ struct intel_pipe_wm *active = &intel_crtc->wm.active;
+ enum pipe pipe = intel_crtc->pipe;
+ static const unsigned int wm0_pipe_reg[] = {
+ [PIPE_A] = WM0_PIPEA_ILK,
+ [PIPE_B] = WM0_PIPEB_ILK,
+ [PIPE_C] = WM0_PIPEC_IVB,
+ };
+
+ hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
+ if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+ hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
+
+ active->pipe_enabled = intel_crtc_active(crtc);
+
+ if (active->pipe_enabled) {
+ u32 tmp = hw->wm_pipe[pipe];
+
+ /*
+ * For active pipes LP0 watermark is marked as
+ * enabled, and LP1+ watermaks as disabled since
+ * we can't really reverse compute them in case
+ * multiple pipes are active.
+ */
+ active->wm[0].enable = true;
+ active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
+ active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
+ active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
+ active->linetime = hw->wm_linetime[pipe];
+ } else {
+ int level, max_level = ilk_wm_max_level(dev);
+
+ /*
+ * For inactive pipes, all watermark levels
+ * should be marked as enabled but zeroed,
+ * which is what we'd compute them to.
+ */
+ for (level = 0; level <= max_level; level++)
+ active->wm[level].enable = true;
+ }
+}
+
+/**
+ * ilk_wm_get_hw_state - Fetch current hardware watermark values
+ * @dev: DRM device
+ *
+ * Reads the currently programmed hardware watermark values.
+ */
+void ilk_wm_get_hw_state(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+ struct ilk_wm_values *hw = &dev_priv->wm.hw;
+ struct drm_crtc *crtc;
+
+ for_each_crtc(dev, crtc)
+ ilk_pipe_wm_get_hw_state(crtc);
+
+ hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
+ hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
+ hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
+
+ hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
+ if (INTEL_INFO(dev)->gen >= 7) {
+ hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
+ hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
+ }
+
+ if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+ hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
+ INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
+ else if (IS_IVYBRIDGE(dev))
+ hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
+ INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
+
+ hw->enable_fbc_wm =
+ !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
+}
+
+/**
+ * intel_update_watermarks - update FIFO watermark values based on current modes
+ *
+ * Calculate watermark values for the various WM regs based on current mode
+ * and plane configuration.
+ *
+ * There are several cases to deal with here:
+ * - normal (i.e. non-self-refresh)
+ * - self-refresh (SR) mode
+ * - lines are large relative to FIFO size (buffer can hold up to 2)
+ * - lines are small relative to FIFO size (buffer can hold more than 2
+ * lines), so need to account for TLB latency
+ *
+ * The normal calculation is:
+ * watermark = dotclock * bytes per pixel * latency
+ * where latency is platform & configuration dependent (we assume pessimal
+ * values here).
+ *
+ * The SR calculation is:
+ * watermark = (trunc(latency/line time)+1) * surface width *
+ * bytes per pixel
+ * where
+ * line time = htotal / dotclock
+ * surface width = hdisplay for normal plane and 64 for cursor
+ * and latency is assumed to be high, as above.
+ *
+ * The final value programmed to the register should always be rounded up,
+ * and include an extra 2 entries to account for clock crossings.
+ *
+ * We don't use the sprite, so we can ignore that. And on Crestline we have
+ * to set the non-SR watermarks to 8.
+ */
+void intel_update_watermarks(struct drm_crtc *crtc)
+{
+ struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+
+ if (dev_priv->display.update_wm)
+ dev_priv->display.update_wm(crtc);
+}
+
+/**
+ * intel_update_sprite_watermarks - update sprite watermark values
+ * @plane: Sprite plane to update watermark values for
+ * @crtc: Plane's CRTC
+ * @sprite_width: Width of sprite plane
+ * @sprite_height: Height of sprite plane
+ * @pixel_size: Bytes per pixel for current sprite pixel format
+ * @enabled: Is the sprite plane enabled?
+ * @scaled: Is the sprite plane scaled?
+ */
+void intel_update_sprite_watermarks(struct drm_plane *plane,
+ struct drm_crtc *crtc,
+ uint32_t sprite_width,
+ uint32_t sprite_height,
+ int pixel_size,
+ bool enabled, bool scaled)
+{
+ struct drm_i915_private *dev_priv = plane->dev->dev_private;
+
+ if (dev_priv->display.update_sprite_wm)
+ dev_priv->display.update_sprite_wm(plane, crtc,
+ sprite_width, sprite_height,
+ pixel_size, enabled, scaled);
+}
+
+/**
+ * intel_init_wm - initialize watermark handling for current platform
+ * @dev: drm device
+ */
+void intel_init_wm(struct drm_device *dev)
+{
+ struct drm_i915_private *dev_priv = dev->dev_private;
+
+ /* For cxsr */
+ if (IS_PINEVIEW(dev))
+ i915_pineview_get_mem_freq(dev);
+ else if (IS_GEN5(dev))
+ i915_ironlake_get_mem_freq(dev);
+
+ /* For FIFO watermark updates */
+ if (INTEL_INFO(dev)->gen >= 9) {
+ skl_setup_wm_latency(dev);
+
+ dev_priv->display.update_wm = skl_update_wm;
+ dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
+ } else if (HAS_PCH_SPLIT(dev)) {
+ ilk_setup_wm_latency(dev);
+
+ if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
+ dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
+ (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
+ dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
+ dev_priv->display.update_wm = ilk_update_wm;
+ dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
+ } else {
+ DRM_DEBUG_KMS("Failed to read display plane latency. "
+ "Disable CxSR\n");
+ }
+ } else if (IS_CHERRYVIEW(dev)) {
+ dev_priv->display.update_wm = cherryview_update_wm;
+ dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
+ } else if (IS_VALLEYVIEW(dev)) {
+ dev_priv->display.update_wm = valleyview_update_wm;
+ dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
+ } else if (IS_PINEVIEW(dev)) {
+ if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
+ dev_priv->is_ddr3,
+ dev_priv->fsb_freq,
+ dev_priv->mem_freq)) {
+ DRM_INFO("failed to find known CxSR latency "
+ "(found ddr%s fsb freq %d, mem freq %d), "
+ "disabling CxSR\n",
+ (dev_priv->is_ddr3 == 1) ? "3" : "2",
+ dev_priv->fsb_freq, dev_priv->mem_freq);
+ /* Disable CxSR and never update its watermark again */
+ intel_set_memory_cxsr(dev_priv, false);
+ dev_priv->display.update_wm = NULL;
+ } else
+ dev_priv->display.update_wm = pineview_update_wm;
+ } else if (IS_G4X(dev)) {
+ dev_priv->display.update_wm = g4x_update_wm;
+ } else if (IS_GEN4(dev)) {
+ dev_priv->display.update_wm = i965_update_wm;
+ } else if (IS_GEN3(dev)) {
+ dev_priv->display.update_wm = i9xx_update_wm;
+ dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
+ } else if (IS_GEN2(dev)) {
+ if (INTEL_INFO(dev)->num_pipes == 1) {
+ dev_priv->display.update_wm = i845_update_wm;
+ dev_priv->display.get_fifo_size = i845_get_fifo_size;
+ } else {
+ dev_priv->display.update_wm = i9xx_update_wm;
+ dev_priv->display.get_fifo_size = i830_get_fifo_size;
+ }
+ } else {
+ DRM_ERROR("unexpected fall-through in intel_init_wm\n");
+ }
+}
--
1.8.5.1
More information about the Intel-gfx
mailing list