diff options
Diffstat (limited to 'drivers/gpu/drm/i915/intel_pm.c')
| -rw-r--r-- | drivers/gpu/drm/i915/intel_pm.c | 616 |
1 files changed, 371 insertions, 245 deletions
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 989b94df6..68a1f4cc2 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -54,10 +54,38 @@ #define INTEL_RC6p_ENABLE (1<<1) #define INTEL_RC6pp_ENABLE (1<<2) +static void gen9_init_clock_gating(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl */ + I915_WRITE(CHICKEN_PAR1_1, + I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP); + + I915_WRITE(GEN8_CONFIG0, + I915_READ(GEN8_CONFIG0) | GEN9_DEFAULT_FIXES); + + /* WaEnableChickenDCPR:skl,bxt,kbl */ + I915_WRITE(GEN8_CHICKEN_DCPR_1, + I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM); + + /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl */ + /* WaFbcWakeMemOn:skl,bxt,kbl */ + I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | + DISP_FBC_WM_DIS | + DISP_FBC_MEMORY_WAKE); + + /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl */ + I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + ILK_DPFC_DISABLE_DUMMY0); +} + static void bxt_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + gen9_init_clock_gating(dev); + /* WaDisableSDEUnitClockGating:bxt */ I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); @@ -487,20 +515,6 @@ static const struct intel_watermark_params g4x_cursor_wm_info = { .guard_size = 2, .cacheline_size = G4X_FIFO_LINE_SIZE, }; -static const struct intel_watermark_params valleyview_wm_info = { - .fifo_size = VALLEYVIEW_FIFO_SIZE, - .max_wm = VALLEYVIEW_MAX_WM, - .default_wm = VALLEYVIEW_MAX_WM, - .guard_size = 2, - .cacheline_size = G4X_FIFO_LINE_SIZE, -}; -static const struct intel_watermark_params valleyview_cursor_wm_info = { - .fifo_size = I965_CURSOR_FIFO, - .max_wm = VALLEYVIEW_CURSOR_MAX_WM, - .default_wm = I965_CURSOR_DFT_WM, - .guard_size = 2, - .cacheline_size = G4X_FIFO_LINE_SIZE, -}; static const struct intel_watermark_params i965_cursor_wm_info = { .fifo_size = I965_CURSOR_FIFO, .max_wm = I965_CURSOR_MAX_WM, @@ -2010,11 +2024,18 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, cur_latency *= 5; } - result->pri_val = ilk_compute_pri_wm(cstate, pristate, - pri_latency, level); - result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency); - result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency); - result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val); + if (pristate) { + result->pri_val = ilk_compute_pri_wm(cstate, pristate, + pri_latency, level); + result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val); + } + + if (sprstate) + result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency); + + if (curstate) + result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency); + result->enable = true; } @@ -2278,96 +2299,167 @@ static void skl_setup_wm_latency(struct drm_device *dev) intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency); } +static bool ilk_validate_pipe_wm(struct drm_device *dev, + struct intel_pipe_wm *pipe_wm) +{ + /* LP0 watermark maximums depend on this pipe alone */ + const struct intel_wm_config config = { + .num_pipes_active = 1, + .sprites_enabled = pipe_wm->sprites_enabled, + .sprites_scaled = pipe_wm->sprites_scaled, + }; + struct ilk_wm_maximums max; + + /* LP0 watermarks always use 1/2 DDB partitioning */ + ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max); + + /* At least LP0 must be valid */ + if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) { + DRM_DEBUG_KMS("LP0 watermark invalid\n"); + return false; + } + + return true; +} + /* Compute new watermarks for the pipe */ -static int ilk_compute_pipe_wm(struct intel_crtc *intel_crtc, - struct drm_atomic_state *state) +static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate) { + struct drm_atomic_state *state = cstate->base.state; + struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); struct intel_pipe_wm *pipe_wm; - struct drm_device *dev = intel_crtc->base.dev; + struct drm_device *dev = state->dev; const struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc_state *cstate = NULL; struct intel_plane *intel_plane; - struct drm_plane_state *ps; struct intel_plane_state *pristate = NULL; struct intel_plane_state *sprstate = NULL; struct intel_plane_state *curstate = NULL; - int level, max_level = ilk_wm_max_level(dev); - /* LP0 watermark maximums depend on this pipe alone */ - struct intel_wm_config config = { - .num_pipes_active = 1, - }; + int level, max_level = ilk_wm_max_level(dev), usable_level; struct ilk_wm_maximums max; - cstate = intel_atomic_get_crtc_state(state, intel_crtc); - if (IS_ERR(cstate)) - return PTR_ERR(cstate); - pipe_wm = &cstate->wm.optimal.ilk; - memset(pipe_wm, 0, sizeof(*pipe_wm)); for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { - ps = drm_atomic_get_plane_state(state, - &intel_plane->base); - if (IS_ERR(ps)) - return PTR_ERR(ps); + struct intel_plane_state *ps; + + ps = intel_atomic_get_existing_plane_state(state, + intel_plane); + if (!ps) + continue; if (intel_plane->base.type == DRM_PLANE_TYPE_PRIMARY) - pristate = to_intel_plane_state(ps); + pristate = ps; else if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY) - sprstate = to_intel_plane_state(ps); + sprstate = ps; else if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR) - curstate = to_intel_plane_state(ps); + curstate = ps; } - config.sprites_enabled = sprstate->visible; - config.sprites_scaled = sprstate->visible && - (drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 || - drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16); - pipe_wm->pipe_enabled = cstate->base.active; - pipe_wm->sprites_enabled = config.sprites_enabled; - pipe_wm->sprites_scaled = config.sprites_scaled; + if (sprstate) { + pipe_wm->sprites_enabled = sprstate->visible; + pipe_wm->sprites_scaled = sprstate->visible && + (drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 || + drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16); + } + + usable_level = max_level; /* ILK/SNB: LP2+ watermarks only w/o sprites */ - if (INTEL_INFO(dev)->gen <= 6 && sprstate->visible) - max_level = 1; + if (INTEL_INFO(dev)->gen <= 6 && pipe_wm->sprites_enabled) + usable_level = 1; /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */ - if (config.sprites_scaled) - max_level = 0; + if (pipe_wm->sprites_scaled) + usable_level = 0; ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate, - pristate, sprstate, curstate, &pipe_wm->wm[0]); + pristate, sprstate, curstate, &pipe_wm->raw_wm[0]); + + memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm)); + pipe_wm->wm[0] = pipe_wm->raw_wm[0]; if (IS_HASWELL(dev) || IS_BROADWELL(dev)) pipe_wm->linetime = hsw_compute_linetime_wm(dev, cstate); - /* LP0 watermarks always use 1/2 DDB partitioning */ - ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max); - - /* At least LP0 must be valid */ - if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) + if (!ilk_validate_pipe_wm(dev, pipe_wm)) return -EINVAL; ilk_compute_wm_reg_maximums(dev, 1, &max); for (level = 1; level <= max_level; level++) { - struct intel_wm_level wm = {}; + struct intel_wm_level *wm = &pipe_wm->raw_wm[level]; ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate, - pristate, sprstate, curstate, &wm); + pristate, sprstate, curstate, wm); /* * Disable any watermark level that exceeds the * register maximums since such watermarks are * always invalid. */ - if (!ilk_validate_wm_level(level, &max, &wm)) - break; + if (level > usable_level) + continue; + + if (ilk_validate_wm_level(level, &max, wm)) + pipe_wm->wm[level] = *wm; + else + usable_level = level; + } + + return 0; +} + +/* + * Build a set of 'intermediate' watermark values that satisfy both the old + * state and the new state. These can be programmed to the hardware + * immediately. + */ +static int ilk_compute_intermediate_wm(struct drm_device *dev, + struct intel_crtc *intel_crtc, + struct intel_crtc_state *newstate) +{ + struct intel_pipe_wm *a = &newstate->wm.intermediate; + struct intel_pipe_wm *b = &intel_crtc->wm.active.ilk; + int level, max_level = ilk_wm_max_level(dev); + + /* + * Start with the final, target watermarks, then combine with the + * currently active watermarks to get values that are safe both before + * and after the vblank. + */ + *a = newstate->wm.optimal.ilk; + a->pipe_enabled |= b->pipe_enabled; + a->sprites_enabled |= b->sprites_enabled; + a->sprites_scaled |= b->sprites_scaled; - pipe_wm->wm[level] = wm; + for (level = 0; level <= max_level; level++) { + struct intel_wm_level *a_wm = &a->wm[level]; + const struct intel_wm_level *b_wm = &b->wm[level]; + + a_wm->enable &= b_wm->enable; + a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val); + a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val); + a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val); + a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val); } + /* + * We need to make sure that these merged watermark values are + * actually a valid configuration themselves. If they're not, + * there's no safe way to transition from the old state to + * the new state, so we need to fail the atomic transaction. + */ + if (!ilk_validate_pipe_wm(dev, a)) + return -EINVAL; + + /* + * If our intermediate WM are identical to the final WM, then we can + * omit the post-vblank programming; only update if it's different. + */ + if (memcmp(a, &newstate->wm.optimal.ilk, sizeof(*a)) == 0) + newstate->wm.need_postvbl_update = false; + return 0; } @@ -2383,9 +2475,7 @@ static void ilk_merge_wm_level(struct drm_device *dev, ret_wm->enable = true; for_each_intel_crtc(dev, intel_crtc) { - const struct intel_crtc_state *cstate = - to_intel_crtc_state(intel_crtc->base.state); - const struct intel_pipe_wm *active = &cstate->wm.optimal.ilk; + const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk; const struct intel_wm_level *wm = &active->wm[level]; if (!active->pipe_enabled) @@ -2421,7 +2511,7 @@ static void ilk_wm_merge(struct drm_device *dev, /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */ if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) && config->num_pipes_active > 1) - return; + last_enabled_level = 0; /* ILK: FBC WM must be disabled always */ merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6; @@ -2533,15 +2623,14 @@ static void ilk_compute_wm_results(struct drm_device *dev, /* LP0 register values */ for_each_intel_crtc(dev, intel_crtc) { - const struct intel_crtc_state *cstate = - to_intel_crtc_state(intel_crtc->base.state); enum pipe pipe = intel_crtc->pipe; - const struct intel_wm_level *r = &cstate->wm.optimal.ilk.wm[0]; + const struct intel_wm_level *r = + &intel_crtc->wm.active.ilk.wm[0]; if (WARN_ON(!r->enable)) continue; - results->wm_linetime[pipe] = cstate->wm.optimal.ilk.linetime; + results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime; results->wm_pipe[pipe] = (r->pri_val << WM0_PIPE_PLANE_SHIFT) | @@ -2748,7 +2837,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv, dev_priv->wm.hw = *results; } -static bool ilk_disable_lp_wm(struct drm_device *dev) +bool ilk_disable_lp_wm(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -3764,11 +3853,9 @@ static void ilk_compute_wm_config(struct drm_device *dev, } } -static void ilk_program_watermarks(struct intel_crtc_state *cstate) +static void ilk_program_watermarks(struct drm_i915_private *dev_priv) { - struct drm_crtc *crtc = cstate->base.crtc; - struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_device *dev = dev_priv->dev; struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; struct ilk_wm_maximums max; struct intel_wm_config config = {}; @@ -3799,28 +3886,28 @@ static void ilk_program_watermarks(struct intel_crtc_state *cstate) ilk_write_wm_values(dev_priv, &results); } -static void ilk_update_wm(struct drm_crtc *crtc) +static void ilk_initial_watermarks(struct intel_crtc_state *cstate) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); - - WARN_ON(cstate->base.active != intel_crtc->active); + struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev); + struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); - /* - * IVB workaround: must disable low power watermarks for at least - * one frame before enabling scaling. LP watermarks can be re-enabled - * when scaling is disabled. - * - * WaCxSRDisabledForSpriteScaling:ivb - */ - if (cstate->disable_lp_wm) { - ilk_disable_lp_wm(crtc->dev); - intel_wait_for_vblank(crtc->dev, intel_crtc->pipe); - } + mutex_lock(&dev_priv->wm.wm_mutex); + intel_crtc->wm.active.ilk = cstate->wm.intermediate; + ilk_program_watermarks(dev_priv); + mutex_unlock(&dev_priv->wm.wm_mutex); +} - intel_crtc->wm.active.ilk = cstate->wm.optimal.ilk; +static void ilk_optimize_watermarks(struct intel_crtc_state *cstate) +{ + struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev); + struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); - ilk_program_watermarks(cstate); + mutex_lock(&dev_priv->wm.wm_mutex); + if (cstate->wm.need_postvbl_update) { + intel_crtc->wm.active.ilk = cstate->wm.optimal.ilk; + ilk_program_watermarks(dev_priv); + } + mutex_unlock(&dev_priv->wm.wm_mutex); } static void skl_pipe_wm_active_state(uint32_t val, @@ -4354,7 +4441,7 @@ static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val) * the hw runs at the minimal clock before selecting the desired * frequency, if the down threshold expires in that window we will not * receive a down interrupt. */ - if (IS_GEN9(dev_priv->dev)) { + if (IS_GEN9(dev_priv)) { limits = (dev_priv->rps.max_freq_softlimit) << 23; if (val <= dev_priv->rps.min_freq_softlimit) limits |= (dev_priv->rps.min_freq_softlimit) << 14; @@ -4639,7 +4726,7 @@ void intel_set_rps(struct drm_device *dev, u8 val) gen6_set_rps(dev, val); } -static void gen9_disable_rps(struct drm_device *dev) +static void gen9_disable_rc6(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -4647,12 +4734,20 @@ static void gen9_disable_rps(struct drm_device *dev) I915_WRITE(GEN9_PG_ENABLE, 0); } +static void gen9_disable_rps(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + I915_WRITE(GEN6_RP_CONTROL, 0); +} + static void gen6_disable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; I915_WRITE(GEN6_RC_CONTROL, 0); I915_WRITE(GEN6_RPNSWREQ, 1 << 31); + I915_WRITE(GEN6_RP_CONTROL, 0); } static void cherryview_disable_rps(struct drm_device *dev) @@ -4696,7 +4791,8 @@ static void intel_print_rc6_info(struct drm_device *dev, u32 mode) static bool bxt_check_bios_rc6_setup(const struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); + struct i915_ggtt *ggtt = &dev_priv->ggtt; bool enable_rc6 = true; unsigned long rc6_ctx_base; @@ -4710,9 +4806,9 @@ static bool bxt_check_bios_rc6_setup(const struct drm_device *dev) * for this check. */ rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK; - if (!((rc6_ctx_base >= dev_priv->gtt.stolen_reserved_base) && - (rc6_ctx_base + PAGE_SIZE <= dev_priv->gtt.stolen_reserved_base + - dev_priv->gtt.stolen_reserved_size))) { + if (!((rc6_ctx_base >= ggtt->stolen_reserved_base) && + (rc6_ctx_base + PAGE_SIZE <= ggtt->stolen_reserved_base + + ggtt->stolen_reserved_size))) { DRM_DEBUG_KMS("RC6 Base address not as expected.\n"); enable_rc6 = false; } @@ -4855,6 +4951,16 @@ static void gen9_enable_rps(struct drm_device *dev) /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */ if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + /* + * BIOS could leave the Hw Turbo enabled, so need to explicitly + * clear out the Control register just to avoid inconsitency + * with debugfs interface, which will show Turbo as enabled + * only and that is not expected by the User after adding the + * WaGsvDisableTurbo. Apart from this there is no problem even + * if the Turbo is left enabled in the Control register, as the + * Up/Down interrupts would remain masked. + */ + gen9_disable_rps(dev); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); return; } @@ -4873,7 +4979,7 @@ static void gen9_enable_rps(struct drm_device *dev) * Up/Down EI & threshold registers, as well as the RP_CONTROL, * RP_INTERRUPT_LIMITS & RPNSWREQ registers */ dev_priv->rps.power = HIGH_POWER; /* force a reset */ - gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit); + gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); } @@ -4881,9 +4987,8 @@ static void gen9_enable_rps(struct drm_device *dev) static void gen9_enable_rc6(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; uint32_t rc6_mask = 0; - int unused; /* 1a: Software RC state - RC0 */ I915_WRITE(GEN6_RC_STATE, 0); @@ -4904,8 +5009,8 @@ static void gen9_enable_rc6(struct drm_device *dev) I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16); I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ - for_each_ring(ring, dev_priv, unused) - I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); + for_each_engine(engine, dev_priv) + I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); if (HAS_GUC_UCODE(dev)) I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA); @@ -4951,9 +5056,8 @@ static void gen9_enable_rc6(struct drm_device *dev) static void gen8_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; uint32_t rc6_mask = 0; - int unused; /* 1a: Software RC state - RC0 */ I915_WRITE(GEN6_RC_STATE, 0); @@ -4972,8 +5076,8 @@ static void gen8_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ - for_each_ring(ring, dev_priv, unused) - I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); + for_each_engine(engine, dev_priv) + I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); I915_WRITE(GEN6_RC_SLEEP, 0); if (IS_BROADWELL(dev)) I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */ @@ -5033,11 +5137,11 @@ static void gen8_enable_rps(struct drm_device *dev) static void gen6_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; u32 rc6vids, pcu_mbox = 0, rc6_mask = 0; u32 gtfifodbg; int rc6_mode; - int i, ret; + int ret; WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); @@ -5050,7 +5154,8 @@ static void gen6_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_STATE, 0); /* Clear the DBG now so we don't confuse earlier errors */ - if ((gtfifodbg = I915_READ(GTFIFODBG))) { + gtfifodbg = I915_READ(GTFIFODBG); + if (gtfifodbg) { DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg); I915_WRITE(GTFIFODBG, gtfifodbg); } @@ -5069,8 +5174,8 @@ static void gen6_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); - for_each_ring(ring, dev_priv, i) - I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); + for_each_engine(engine, dev_priv) + I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); I915_WRITE(GEN6_RC_SLEEP, 0); I915_WRITE(GEN6_RC1e_THRESHOLD, 1000); @@ -5355,9 +5460,9 @@ static void cherryview_check_pctx(struct drm_i915_private *dev_priv) static void cherryview_setup_pctx(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); + struct i915_ggtt *ggtt = &dev_priv->ggtt; unsigned long pctx_paddr, paddr; - struct i915_gtt *gtt = &dev_priv->gtt; u32 pcbr; int pctx_size = 32*1024; @@ -5365,7 +5470,7 @@ static void cherryview_setup_pctx(struct drm_device *dev) if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) { DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n"); paddr = (dev_priv->mm.stolen_base + - (gtt->stolen_size - pctx_size)); + (ggtt->stolen_size - pctx_size)); pctx_paddr = (paddr & (~4095)); I915_WRITE(VLV_PCBR, pctx_paddr); @@ -5433,6 +5538,17 @@ static void valleyview_cleanup_pctx(struct drm_device *dev) dev_priv->vlv_pctx = NULL; } +static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv) +{ + dev_priv->rps.gpll_ref_freq = + vlv_get_cck_clock(dev_priv, "GPLL ref", + CCK_GPLL_CLOCK_CONTROL, + dev_priv->czclk_freq); + + DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", + dev_priv->rps.gpll_ref_freq); +} + static void valleyview_init_gt_powersave(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -5440,6 +5556,8 @@ static void valleyview_init_gt_powersave(struct drm_device *dev) valleyview_setup_pctx(dev); + vlv_init_gpll_ref_freq(dev_priv); + mutex_lock(&dev_priv->rps.hw_lock); val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS); @@ -5497,6 +5615,8 @@ static void cherryview_init_gt_powersave(struct drm_device *dev) cherryview_setup_pctx(dev); + vlv_init_gpll_ref_freq(dev_priv); + mutex_lock(&dev_priv->rps.hw_lock); mutex_lock(&dev_priv->sb_lock); @@ -5561,13 +5681,13 @@ static void valleyview_cleanup_gt_powersave(struct drm_device *dev) static void cherryview_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; u32 gtfifodbg, val, rc6_mode = 0, pcbr; - int i; WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); - gtfifodbg = I915_READ(GTFIFODBG); + gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV | + GT_FIFO_FREE_ENTRIES_CHV); if (gtfifodbg) { DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n", gtfifodbg); @@ -5588,8 +5708,8 @@ static void cherryview_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ - for_each_ring(ring, dev_priv, i) - I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); + for_each_engine(engine, dev_priv) + I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); I915_WRITE(GEN6_RC_SLEEP, 0); /* TO threshold set to 500 us ( 0x186 * 1.28 us) */ @@ -5648,10 +5768,10 @@ static void cherryview_enable_rps(struct drm_device *dev) dev_priv->rps.cur_freq); DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n", - intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), - dev_priv->rps.efficient_freq); + intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq), + dev_priv->rps.idle_freq); - valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); + valleyview_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); } @@ -5659,15 +5779,15 @@ static void cherryview_enable_rps(struct drm_device *dev) static void valleyview_enable_rps(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; u32 gtfifodbg, val, rc6_mode = 0; - int i; WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); valleyview_check_pctx(dev_priv); - if ((gtfifodbg = I915_READ(GTFIFODBG))) { + gtfifodbg = I915_READ(GTFIFODBG); + if (gtfifodbg) { DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n", gtfifodbg); I915_WRITE(GTFIFODBG, gtfifodbg); @@ -5699,8 +5819,8 @@ static void valleyview_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); - for_each_ring(ring, dev_priv, i) - I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10); + for_each_engine(engine, dev_priv) + I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); I915_WRITE(GEN6_RC6_THRESHOLD, 0x557); @@ -5738,10 +5858,10 @@ static void valleyview_enable_rps(struct drm_device *dev) dev_priv->rps.cur_freq); DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n", - intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq), - dev_priv->rps.efficient_freq); + intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq), + dev_priv->rps.idle_freq); - valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq); + valleyview_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); } @@ -6076,17 +6196,16 @@ EXPORT_SYMBOL_GPL(i915_gpu_lower); bool i915_gpu_busy(void) { struct drm_i915_private *dev_priv; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; bool ret = false; - int i; spin_lock_irq(&mchdev_lock); if (!i915_mch_dev) goto out_unlock; dev_priv = i915_mch_dev; - for_each_ring(ring, dev_priv, i) - ret |= !list_empty(&ring->request_list); + for_each_engine(engine, dev_priv) + ret |= !list_empty(&engine->request_list); out_unlock: spin_unlock_irq(&mchdev_lock); @@ -6306,9 +6425,10 @@ void intel_disable_gt_powersave(struct drm_device *dev) intel_suspend_gt_powersave(dev); mutex_lock(&dev_priv->rps.hw_lock); - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_INFO(dev)->gen >= 9) { + gen9_disable_rc6(dev); gen9_disable_rps(dev); - else if (IS_CHERRYVIEW(dev)) + } else if (IS_CHERRYVIEW(dev)) cherryview_disable_rps(dev); else if (IS_VALLEYVIEW(dev)) valleyview_disable_rps(dev); @@ -6715,6 +6835,38 @@ static void lpt_suspend_hw(struct drm_device *dev) } } +static void kabylake_init_clock_gating(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + gen9_init_clock_gating(dev); + + /* WaDisableSDEUnitClockGating:kbl */ + if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) + I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | + GEN8_SDEUNIT_CLOCK_GATE_DISABLE); + + /* WaDisableGamClockGating:kbl */ + if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) + I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) | + GEN6_GAMUNIT_CLOCK_GATE_DISABLE); + + /* WaFbcNukeOnHostModify:kbl */ + I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + ILK_DPFC_NUKE_ON_ANY_MODIFICATION); +} + +static void skylake_init_clock_gating(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + gen9_init_clock_gating(dev); + + /* WaFbcNukeOnHostModify:skl */ + I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | + ILK_DPFC_NUKE_ON_ANY_MODIFICATION); +} + static void broadwell_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -6929,23 +7081,10 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) gen6_check_mch_setup(dev); } -static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv) -{ - I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE); - - /* - * Disable trickle feed and enable pnd deadline calculation - */ - I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); - I915_WRITE(CBR1_VLV, 0); -} - static void valleyview_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - vlv_init_display_clock_gating(dev_priv); - /* WaDisableEarlyCull:vlv */ I915_WRITE(_3D_CHICKEN3, _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); @@ -7028,8 +7167,6 @@ static void cherryview_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - vlv_init_display_clock_gating(dev_priv); - /* WaVSRefCountFullforceMissDisable:chv */ /* WaDSRefCountFullforceMissDisable:chv */ I915_WRITE(GEN7_FF_THREAD_MODE, @@ -7169,8 +7306,7 @@ void intel_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - if (dev_priv->display.init_clock_gating) - dev_priv->display.init_clock_gating(dev); + dev_priv->display.init_clock_gating(dev); } void intel_suspend_hw(struct drm_device *dev) @@ -7179,6 +7315,60 @@ void intel_suspend_hw(struct drm_device *dev) lpt_suspend_hw(dev); } +static void nop_init_clock_gating(struct drm_device *dev) +{ + DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n"); +} + +/** + * intel_init_clock_gating_hooks - setup the clock gating hooks + * @dev_priv: device private + * + * Setup the hooks that configure which clocks of a given platform can be + * gated and also apply various GT and display specific workarounds for these + * platforms. Note that some GT specific workarounds are applied separately + * when GPU contexts or batchbuffers start their execution. + */ +void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) +{ + if (IS_SKYLAKE(dev_priv)) + dev_priv->display.init_clock_gating = skylake_init_clock_gating; + else if (IS_KABYLAKE(dev_priv)) + dev_priv->display.init_clock_gating = kabylake_init_clock_gating; + else if (IS_BROXTON(dev_priv)) + dev_priv->display.init_clock_gating = bxt_init_clock_gating; + else if (IS_BROADWELL(dev_priv)) + dev_priv->display.init_clock_gating = broadwell_init_clock_gating; + else if (IS_CHERRYVIEW(dev_priv)) + dev_priv->display.init_clock_gating = cherryview_init_clock_gating; + else if (IS_HASWELL(dev_priv)) + dev_priv->display.init_clock_gating = haswell_init_clock_gating; + else if (IS_IVYBRIDGE(dev_priv)) + dev_priv->display.init_clock_gating = ivybridge_init_clock_gating; + else if (IS_VALLEYVIEW(dev_priv)) + dev_priv->display.init_clock_gating = valleyview_init_clock_gating; + else if (IS_GEN6(dev_priv)) + dev_priv->display.init_clock_gating = gen6_init_clock_gating; + else if (IS_GEN5(dev_priv)) + dev_priv->display.init_clock_gating = ironlake_init_clock_gating; + else if (IS_G4X(dev_priv)) + dev_priv->display.init_clock_gating = g4x_init_clock_gating; + else if (IS_CRESTLINE(dev_priv)) + dev_priv->display.init_clock_gating = crestline_init_clock_gating; + else if (IS_BROADWATER(dev_priv)) + dev_priv->display.init_clock_gating = broadwater_init_clock_gating; + else if (IS_GEN3(dev_priv)) + dev_priv->display.init_clock_gating = gen3_init_clock_gating; + else if (IS_I85X(dev_priv) || IS_I865G(dev_priv)) + dev_priv->display.init_clock_gating = i85x_init_clock_gating; + else if (IS_GEN2(dev_priv)) + dev_priv->display.init_clock_gating = i830_init_clock_gating; + else { + MISSING_CASE(INTEL_DEVID(dev_priv)); + dev_priv->display.init_clock_gating = nop_init_clock_gating; + } +} + /* Set up chip specific power management-related functions */ void intel_init_pm(struct drm_device *dev) { @@ -7195,10 +7385,6 @@ void intel_init_pm(struct drm_device *dev) /* For FIFO watermark updates */ if (INTEL_INFO(dev)->gen >= 9) { skl_setup_wm_latency(dev); - - if (IS_BROXTON(dev)) - dev_priv->display.init_clock_gating = - bxt_init_clock_gating; dev_priv->display.update_wm = skl_update_wm; } else if (HAS_PCH_SPLIT(dev)) { ilk_setup_wm_latency(dev); @@ -7207,36 +7393,23 @@ void intel_init_pm(struct drm_device *dev) dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] && dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { - dev_priv->display.update_wm = ilk_update_wm; dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm; - dev_priv->display.program_watermarks = ilk_program_watermarks; + dev_priv->display.compute_intermediate_wm = + ilk_compute_intermediate_wm; + dev_priv->display.initial_watermarks = + ilk_initial_watermarks; + dev_priv->display.optimize_watermarks = + ilk_optimize_watermarks; } else { DRM_DEBUG_KMS("Failed to read display plane latency. " "Disable CxSR\n"); } - - if (IS_GEN5(dev)) - dev_priv->display.init_clock_gating = ironlake_init_clock_gating; - else if (IS_GEN6(dev)) - dev_priv->display.init_clock_gating = gen6_init_clock_gating; - else if (IS_IVYBRIDGE(dev)) - dev_priv->display.init_clock_gating = ivybridge_init_clock_gating; - else if (IS_HASWELL(dev)) - dev_priv->display.init_clock_gating = haswell_init_clock_gating; - else if (INTEL_INFO(dev)->gen == 8) - dev_priv->display.init_clock_gating = broadwell_init_clock_gating; } else if (IS_CHERRYVIEW(dev)) { vlv_setup_wm_latency(dev); - dev_priv->display.update_wm = vlv_update_wm; - dev_priv->display.init_clock_gating = - cherryview_init_clock_gating; } else if (IS_VALLEYVIEW(dev)) { vlv_setup_wm_latency(dev); - dev_priv->display.update_wm = vlv_update_wm; - dev_priv->display.init_clock_gating = - valleyview_init_clock_gating; } else if (IS_PINEVIEW(dev)) { if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3, @@ -7252,20 +7425,13 @@ void intel_init_pm(struct drm_device *dev) dev_priv->display.update_wm = NULL; } else dev_priv->display.update_wm = pineview_update_wm; - dev_priv->display.init_clock_gating = gen3_init_clock_gating; } else if (IS_G4X(dev)) { dev_priv->display.update_wm = g4x_update_wm; - dev_priv->display.init_clock_gating = g4x_init_clock_gating; } else if (IS_GEN4(dev)) { dev_priv->display.update_wm = i965_update_wm; - if (IS_CRESTLINE(dev)) - dev_priv->display.init_clock_gating = crestline_init_clock_gating; - else if (IS_BROADWATER(dev)) - dev_priv->display.init_clock_gating = broadwater_init_clock_gating; } else if (IS_GEN3(dev)) { dev_priv->display.update_wm = i9xx_update_wm; dev_priv->display.get_fifo_size = i9xx_get_fifo_size; - dev_priv->display.init_clock_gating = gen3_init_clock_gating; } else if (IS_GEN2(dev)) { if (INTEL_INFO(dev)->num_pipes == 1) { dev_priv->display.update_wm = i845_update_wm; @@ -7274,11 +7440,6 @@ void intel_init_pm(struct drm_device *dev) dev_priv->display.update_wm = i9xx_update_wm; dev_priv->display.get_fifo_size = i830_get_fifo_size; } - - if (IS_I85X(dev) || IS_I865G(dev)) - dev_priv->display.init_clock_gating = i85x_init_clock_gating; - else - dev_priv->display.init_clock_gating = i830_init_clock_gating; } else { DRM_ERROR("unexpected fall-through in intel_init_pm\n"); } @@ -7332,78 +7493,43 @@ int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val return 0; } -static int vlv_gpu_freq_div(unsigned int czclk_freq) -{ - switch (czclk_freq) { - case 200: - return 10; - case 267: - return 12; - case 320: - case 333: - return 16; - case 400: - return 20; - default: - return -1; - } -} - static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val) { - int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); - - div = vlv_gpu_freq_div(czclk_freq); - if (div < 0) - return div; - - return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div); + /* + * N = val - 0xb7 + * Slow = Fast = GPLL ref * N + */ + return DIV_ROUND_CLOSEST(dev_priv->rps.gpll_ref_freq * (val - 0xb7), 1000); } static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val) { - int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); - - mul = vlv_gpu_freq_div(czclk_freq); - if (mul < 0) - return mul; - - return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6; + return DIV_ROUND_CLOSEST(1000 * val, dev_priv->rps.gpll_ref_freq) + 0xb7; } static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val) { - int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); - - div = vlv_gpu_freq_div(czclk_freq); - if (div < 0) - return div; - div /= 2; - - return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2; + /* + * N = val / 2 + * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 + */ + return DIV_ROUND_CLOSEST(dev_priv->rps.gpll_ref_freq * val, 2 * 2 * 1000); } static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val) { - int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000); - - mul = vlv_gpu_freq_div(czclk_freq); - if (mul < 0) - return mul; - mul /= 2; - /* CHV needs even values */ - return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2; + return DIV_ROUND_CLOSEST(2 * 1000 * val, dev_priv->rps.gpll_ref_freq) * 2; } int intel_gpu_freq(struct drm_i915_private *dev_priv, int val) { - if (IS_GEN9(dev_priv->dev)) + if (IS_GEN9(dev_priv)) return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, GEN9_FREQ_SCALER); - else if (IS_CHERRYVIEW(dev_priv->dev)) + else if (IS_CHERRYVIEW(dev_priv)) return chv_gpu_freq(dev_priv, val); - else if (IS_VALLEYVIEW(dev_priv->dev)) + else if (IS_VALLEYVIEW(dev_priv)) return byt_gpu_freq(dev_priv, val); else return val * GT_FREQUENCY_MULTIPLIER; @@ -7411,12 +7537,12 @@ int intel_gpu_freq(struct drm_i915_private *dev_priv, int val) int intel_freq_opcode(struct drm_i915_private *dev_priv, int val) { - if (IS_GEN9(dev_priv->dev)) + if (IS_GEN9(dev_priv)) return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, GT_FREQUENCY_MULTIPLIER); - else if (IS_CHERRYVIEW(dev_priv->dev)) + else if (IS_CHERRYVIEW(dev_priv)) return chv_freq_opcode(dev_priv, val); - else if (IS_VALLEYVIEW(dev_priv->dev)) + else if (IS_VALLEYVIEW(dev_priv)) return byt_freq_opcode(dev_priv, val); else return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); @@ -7433,7 +7559,7 @@ static void __intel_rps_boost_work(struct work_struct *work) struct drm_i915_gem_request *req = boost->req; if (!i915_gem_request_completed(req, true)) - gen6_rps_boost(to_i915(req->ring->dev), NULL, + gen6_rps_boost(to_i915(req->engine->dev), NULL, req->emitted_jiffies); i915_gem_request_unreference__unlocked(req); |