diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
94 files changed, 14040 insertions, 8735 deletions
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 29a32b119..7769e4691 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -57,6 +57,28 @@ config DRM_I915_USERPTR If in doubt, say "Y". +config DRM_I915_GVT + bool "Enable Intel GVT-g graphics virtualization host support" + depends on DRM_I915 + default n + help + Choose this option if you want to enable Intel GVT-g graphics + virtualization technology host support with integrated graphics. + With GVT-g, it's possible to have one integrated graphics + device shared by multiple VMs under different hypervisors. + + Note that at least one hypervisor like Xen or KVM is required for + this driver to work, and it only supports newer device from + Broadwell+. For further information and setup guide, you can + visit: http://01.org/igvt-g. + + Now it's just a stub to support the modifications of i915 for + GVT device model. It requires at least one MPT modules for Xen/KVM + and other components of GVT device model to work. Use it under + you own risk. + + If in doubt, say "N". + menu "drm/i915 Debugging" depends on DRM_I915 depends on EXPERT diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug index 8f4041033..cee87bfd1 100644 --- a/drivers/gpu/drm/i915/Kconfig.debug +++ b/drivers/gpu/drm/i915/Kconfig.debug @@ -18,6 +18,9 @@ config DRM_I915_WERROR config DRM_I915_DEBUG bool "Enable additional driver debugging" depends on DRM_I915 + select PREEMPT_COUNT + select X86_MSR # used by igt/pm_rpm + select DRM_VGEM # used by igt/prime_vgem (dmabuf interop checks) default n help Choose this option to turn on extra driver debugging that may affect diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 0b88ba0f3..684fc1cd0 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -10,9 +10,11 @@ subdir-ccflags-$(CONFIG_DRM_I915_WERROR) := -Werror i915-y := i915_drv.o \ i915_irq.o \ i915_params.o \ + i915_pci.o \ i915_suspend.o \ i915_sysfs.o \ intel_csr.o \ + intel_device_info.o \ intel_pm.o \ intel_runtime_pm.o @@ -37,6 +39,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_userptr.o \ i915_gpu_error.o \ i915_trace_points.o \ + intel_breadcrumbs.o \ intel_lrc.o \ intel_mocs.o \ intel_ringbuffer.o \ @@ -59,6 +62,7 @@ i915-y += intel_audio.o \ intel_bios.o \ intel_color.o \ intel_display.o \ + intel_dpio_phy.o \ intel_dpll_mgr.o \ intel_fbc.o \ intel_fifo_underrun.o \ @@ -81,10 +85,12 @@ i915-y += dvo_ch7017.o \ dvo_tfp410.o \ intel_crt.o \ intel_ddi.o \ + intel_dp_aux_backlight.o \ intel_dp_link_training.o \ intel_dp_mst.o \ intel_dp.o \ intel_dsi.o \ + intel_dsi_dcs_backlight.o \ intel_dsi_panel_vbt.o \ intel_dsi_pll.o \ intel_dvo.o \ @@ -98,8 +104,10 @@ i915-y += dvo_ch7017.o \ # virtual gpu code i915-y += i915_vgpu.o -# legacy horrors -i915-y += i915_dma.o +ifeq ($(CONFIG_DRM_I915_GVT),y) +i915-y += intel_gvt.o +include $(src)/gvt/Makefile +endif obj-$(CONFIG_DRM_I915) += i915.o diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile new file mode 100644 index 000000000..d0f21a6ad --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/Makefile @@ -0,0 +1,5 @@ +GVT_DIR := gvt +GVT_SOURCE := gvt.o + +ccflags-y += -I$(src) -I$(src)/$(GVT_DIR) -Wall +i915-y += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE)) diff --git a/drivers/gpu/drm/i915/gvt/debug.h b/drivers/gpu/drm/i915/gvt/debug.h new file mode 100644 index 000000000..7ef412be6 --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/debug.h @@ -0,0 +1,34 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __GVT_DEBUG_H__ +#define __GVT_DEBUG_H__ + +#define gvt_dbg_core(fmt, args...) \ + DRM_DEBUG_DRIVER("gvt: core: "fmt, ##args) + +/* + * Other GVT debug stuff will be introduced in the GVT device model patches. + */ + +#endif diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c new file mode 100644 index 000000000..927f4579f --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -0,0 +1,145 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/types.h> +#include <xen/xen.h> + +#include "i915_drv.h" + +struct intel_gvt_host intel_gvt_host; + +static const char * const supported_hypervisors[] = { + [INTEL_GVT_HYPERVISOR_XEN] = "XEN", + [INTEL_GVT_HYPERVISOR_KVM] = "KVM", +}; + +/** + * intel_gvt_init_host - Load MPT modules and detect if we're running in host + * @gvt: intel gvt device + * + * This function is called at the driver loading stage. If failed to find a + * loadable MPT module or detect currently we're running in a VM, then GVT-g + * will be disabled + * + * Returns: + * Zero on success, negative error code if failed. + * + */ +int intel_gvt_init_host(void) +{ + if (intel_gvt_host.initialized) + return 0; + + /* Xen DOM U */ + if (xen_domain() && !xen_initial_domain()) + return -ENODEV; + + /* Try to load MPT modules for hypervisors */ + if (xen_initial_domain()) { + /* In Xen dom0 */ + intel_gvt_host.mpt = try_then_request_module( + symbol_get(xengt_mpt), "xengt"); + intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_XEN; + } else { + /* not in Xen. Try KVMGT */ + intel_gvt_host.mpt = try_then_request_module( + symbol_get(kvmgt_mpt), "kvm"); + intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_KVM; + } + + /* Fail to load MPT modules - bail out */ + if (!intel_gvt_host.mpt) + return -EINVAL; + + /* Try to detect if we're running in host instead of VM. */ + if (!intel_gvt_hypervisor_detect_host()) + return -ENODEV; + + gvt_dbg_core("Running with hypervisor %s in host mode\n", + supported_hypervisors[intel_gvt_host.hypervisor_type]); + + intel_gvt_host.initialized = true; + return 0; +} + +static void init_device_info(struct intel_gvt *gvt) +{ + if (IS_BROADWELL(gvt->dev_priv)) + gvt->device_info.max_support_vgpus = 8; + /* This function will grow large in GVT device model patches. */ +} + +/** + * intel_gvt_clean_device - clean a GVT device + * @gvt: intel gvt device + * + * This function is called at the driver unloading stage, to free the + * resources owned by a GVT device. + * + */ +void intel_gvt_clean_device(struct drm_i915_private *dev_priv) +{ + struct intel_gvt *gvt = &dev_priv->gvt; + + if (WARN_ON(!gvt->initialized)) + return; + + /* Other de-initialization of GVT components will be introduced. */ + + gvt->initialized = false; +} + +/** + * intel_gvt_init_device - initialize a GVT device + * @dev_priv: drm i915 private data + * + * This function is called at the initialization stage, to initialize + * necessary GVT components. + * + * Returns: + * Zero on success, negative error code if failed. + * + */ +int intel_gvt_init_device(struct drm_i915_private *dev_priv) +{ + struct intel_gvt *gvt = &dev_priv->gvt; + /* + * Cannot initialize GVT device without intel_gvt_host gets + * initialized first. + */ + if (WARN_ON(!intel_gvt_host.initialized)) + return -EINVAL; + + if (WARN_ON(gvt->initialized)) + return -EEXIST; + + gvt_dbg_core("init gvt device\n"); + + init_device_info(gvt); + /* + * Other initialization of GVT components will be introduce here. + */ + gvt_dbg_core("gvt device creation is done\n"); + gvt->initialized = true; + return 0; +} diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h new file mode 100644 index 000000000..fb619a6e5 --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -0,0 +1,69 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _GVT_H_ +#define _GVT_H_ + +#include "debug.h" +#include "hypercall.h" + +#define GVT_MAX_VGPU 8 + +enum { + INTEL_GVT_HYPERVISOR_XEN = 0, + INTEL_GVT_HYPERVISOR_KVM, +}; + +struct intel_gvt_host { + bool initialized; + int hypervisor_type; + struct intel_gvt_mpt *mpt; +}; + +extern struct intel_gvt_host intel_gvt_host; + +/* Describe per-platform limitations. */ +struct intel_gvt_device_info { + u32 max_support_vgpus; + /* This data structure will grow bigger in GVT device model patches */ +}; + +struct intel_vgpu { + struct intel_gvt *gvt; + int id; + unsigned long handle; /* vGPU handle used by hypervisor MPT modules */ +}; + +struct intel_gvt { + struct mutex lock; + bool initialized; + + struct drm_i915_private *dev_priv; + struct idr vgpu_idr; /* vGPU IDR pool */ + + struct intel_gvt_device_info device_info; +}; + +#include "mpt.h" + +#endif diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h new file mode 100644 index 000000000..254df8bf1 --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/hypercall.h @@ -0,0 +1,38 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _GVT_HYPERCALL_H_ +#define _GVT_HYPERCALL_H_ + +/* + * Specific GVT-g MPT modules function collections. Currently GVT-g supports + * both Xen and KVM by providing dedicated hypervisor-related MPT modules. + */ +struct intel_gvt_mpt { + int (*detect_host)(void); +}; + +extern struct intel_gvt_mpt xengt_mpt; +extern struct intel_gvt_mpt kvmgt_mpt; + +#endif /* _GVT_HYPERCALL_H_ */ diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h new file mode 100644 index 000000000..03601e3ff --- /dev/null +++ b/drivers/gpu/drm/i915/gvt/mpt.h @@ -0,0 +1,49 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _GVT_MPT_H_ +#define _GVT_MPT_H_ + +/** + * DOC: Hypervisor Service APIs for GVT-g Core Logic + * + * This is the glue layer between specific hypervisor MPT modules and GVT-g core + * logic. Each kind of hypervisor MPT module provides a collection of function + * callbacks and will be attached to GVT host when the driver is loading. + * GVT-g core logic will call these APIs to request specific services from + * hypervisor. + */ + +/** + * intel_gvt_hypervisor_detect_host - check if GVT-g is running within + * hypervisor host/privilged domain + * + * Returns: + * Zero on success, -ENODEV if current kernel is running inside a VM + */ +static inline int intel_gvt_hypervisor_detect_host(void) +{ + return intel_gvt_host.mpt->detect_host(); +} + +#endif /* _GVT_MPT_H_ */ diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index a337f33be..b0fd6a7b0 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -215,7 +215,8 @@ static const struct drm_i915_cmd_descriptor hsw_render_cmds[] = { CMD( MI_RS_CONTEXT, SMI, F, 1, S ), CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, M ), CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ), - CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, R ), + CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W, + .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ), CMD( MI_RS_STORE_DATA_IMM, SMI, !F, 0xFF, S ), CMD( MI_LOAD_URB_MEM, SMI, !F, 0xFF, S ), CMD( MI_STORE_URB_MEM, SMI, !F, 0xFF, S ), @@ -736,7 +737,7 @@ static void fini_hash_table(struct intel_engine_cs *engine) /** * i915_cmd_parser_init_ring() - set cmd parser related fields for a ringbuffer - * @ring: the ringbuffer to initialize + * @engine: the engine to initialize * * Optionally initializes fields related to batch buffer command parsing in the * struct intel_engine_cs based on whether the platform requires software @@ -750,12 +751,12 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine) int cmd_table_count; int ret; - if (!IS_GEN7(engine->dev)) + if (!IS_GEN7(engine->i915)) return 0; switch (engine->id) { case RCS: - if (IS_HASWELL(engine->dev)) { + if (IS_HASWELL(engine->i915)) { cmd_tables = hsw_render_ring_cmds; cmd_table_count = ARRAY_SIZE(hsw_render_ring_cmds); @@ -764,7 +765,7 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine) cmd_table_count = ARRAY_SIZE(gen7_render_cmds); } - if (IS_HASWELL(engine->dev)) { + if (IS_HASWELL(engine->i915)) { engine->reg_tables = hsw_render_reg_tables; engine->reg_table_count = ARRAY_SIZE(hsw_render_reg_tables); } else { @@ -780,7 +781,7 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine) engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask; break; case BCS: - if (IS_HASWELL(engine->dev)) { + if (IS_HASWELL(engine->i915)) { cmd_tables = hsw_blt_ring_cmds; cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmds); } else { @@ -788,7 +789,7 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine) cmd_table_count = ARRAY_SIZE(gen7_blt_cmds); } - if (IS_HASWELL(engine->dev)) { + if (IS_HASWELL(engine->i915)) { engine->reg_tables = hsw_blt_reg_tables; engine->reg_table_count = ARRAY_SIZE(hsw_blt_reg_tables); } else { @@ -829,7 +830,7 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine) /** * i915_cmd_parser_fini_ring() - clean up cmd parser related fields - * @ring: the ringbuffer to clean up + * @engine: the engine to clean up * * Releases any resources related to command parsing that may have been * initialized for the specified ring. @@ -1023,7 +1024,7 @@ unpin_src: /** * i915_needs_cmd_parser() - should a given ring use software command parsing? - * @ring: the ring in question + * @engine: the engine in question * * Only certain platforms require software batch buffer command parsing, and * only when enabled via module parameter. @@ -1035,7 +1036,7 @@ bool i915_needs_cmd_parser(struct intel_engine_cs *engine) if (!engine->needs_cmd_parser) return false; - if (!USES_PPGTT(engine->dev)) + if (!USES_PPGTT(engine->i915)) return false; return (i915.enable_cmd_parser == 1); @@ -1098,6 +1099,11 @@ static bool check_cmd(const struct intel_engine_cs *engine, return false; } + if (desc->cmd.value == MI_LOAD_REGISTER_REG) { + DRM_DEBUG_DRIVER("CMD: Rejected LRR to OACONTROL\n"); + return false; + } + if (desc->cmd.value == MI_LOAD_REGISTER_IMM(1)) *oacontrol_set = (cmd[offset + 1] != 0); } @@ -1113,6 +1119,12 @@ static bool check_cmd(const struct intel_engine_cs *engine, return false; } + if (desc->cmd.value == MI_LOAD_REGISTER_REG) { + DRM_DEBUG_DRIVER("CMD: Rejected LRR to masked register 0x%08X\n", + reg_addr); + return false; + } + if (desc->cmd.value == MI_LOAD_REGISTER_IMM(1) && (offset + 2 > length || (cmd[offset + 1] & reg->mask) != reg->value)) { @@ -1164,7 +1176,7 @@ static bool check_cmd(const struct intel_engine_cs *engine, /** * i915_parse_cmds() - parse a submitted batch buffer for privilege violations - * @ring: the ring on which the batch is to execute + * @engine: the engine on which the batch is to execute * @batch_obj: the batch buffer in question * @shadow_batch_obj: copy of the batch buffer in question * @batch_start_offset: byte offset in the batch at which execution starts @@ -1269,14 +1281,28 @@ int i915_parse_cmds(struct intel_engine_cs *engine, /** * i915_cmd_parser_get_version() - get the cmd parser version number + * @dev_priv: i915 device private * * The cmd parser maintains a simple increasing integer version number suitable * for passing to userspace clients to determine what operations are permitted. * * Return: the current version number of the cmd parser */ -int i915_cmd_parser_get_version(void) +int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv) { + struct intel_engine_cs *engine; + bool active = false; + + /* If the command parser is not enabled, report 0 - unsupported */ + for_each_engine(engine, dev_priv) { + if (i915_needs_cmd_parser(engine)) { + active = true; + break; + } + } + if (!active) + return 0; + /* * Command parser version history * @@ -1288,6 +1314,7 @@ int i915_cmd_parser_get_version(void) * 4. L3 atomic chicken bits of HSW_SCRATCH1 and HSW_ROW_CHICKEN3. * 5. GPGPU dispatch compute indirect registers. * 6. TIMESTAMP register and Haswell CS GPR registers + * 7. Allow MI_LOAD_REGISTER_REG between whitelisted registers. */ - return 6; + return 7; } diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 2a6e12956..844fea795 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -89,17 +89,17 @@ static int i915_capabilities(struct seq_file *m, void *data) return 0; } -static const char get_active_flag(struct drm_i915_gem_object *obj) +static char get_active_flag(struct drm_i915_gem_object *obj) { return obj->active ? '*' : ' '; } -static const char get_pin_flag(struct drm_i915_gem_object *obj) +static char get_pin_flag(struct drm_i915_gem_object *obj) { return obj->pin_display ? 'p' : ' '; } -static const char get_tiling_flag(struct drm_i915_gem_object *obj) +static char get_tiling_flag(struct drm_i915_gem_object *obj) { switch (obj->tiling_mode) { default: @@ -109,12 +109,12 @@ static const char get_tiling_flag(struct drm_i915_gem_object *obj) } } -static inline const char get_global_flag(struct drm_i915_gem_object *obj) +static char get_global_flag(struct drm_i915_gem_object *obj) { return i915_gem_obj_to_ggtt(obj) ? 'g' : ' '; } -static inline const char get_pin_mapped_flag(struct drm_i915_gem_object *obj) +static char get_pin_mapped_flag(struct drm_i915_gem_object *obj) { return obj->mapping ? 'M' : ' '; } @@ -199,13 +199,6 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) seq_printf(m, " (frontbuffer: 0x%03x)", obj->frontbuffer_bits); } -static void describe_ctx(struct seq_file *m, struct intel_context *ctx) -{ - seq_putc(m, ctx->legacy_hw_ctx.initialized ? 'I' : 'i'); - seq_putc(m, ctx->remap_slice ? 'R' : 'r'); - seq_putc(m, ' '); -} - static int i915_gem_object_list_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; @@ -272,7 +265,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; u64 total_obj_size, total_gtt_size; LIST_HEAD(stolen); @@ -424,6 +417,42 @@ static void print_batch_pool_stats(struct seq_file *m, print_file_stats(m, "[k]batch pool", stats); } +static int per_file_ctx_stats(int id, void *ptr, void *data) +{ + struct i915_gem_context *ctx = ptr; + int n; + + for (n = 0; n < ARRAY_SIZE(ctx->engine); n++) { + if (ctx->engine[n].state) + per_file_stats(0, ctx->engine[n].state, data); + if (ctx->engine[n].ringbuf) + per_file_stats(0, ctx->engine[n].ringbuf->obj, data); + } + + return 0; +} + +static void print_context_stats(struct seq_file *m, + struct drm_i915_private *dev_priv) +{ + struct file_stats stats; + struct drm_file *file; + + memset(&stats, 0, sizeof(stats)); + + mutex_lock(&dev_priv->drm.struct_mutex); + if (dev_priv->kernel_context) + per_file_ctx_stats(0, dev_priv->kernel_context, &stats); + + list_for_each_entry(file, &dev_priv->drm.filelist, lhead) { + struct drm_i915_file_private *fpriv = file->driver_priv; + idr_for_each(&fpriv->context_idr, per_file_ctx_stats, &stats); + } + mutex_unlock(&dev_priv->drm.struct_mutex); + + print_file_stats(m, "[k]contexts", stats); +} + #define count_vmas(list, member) do { \ list_for_each_entry(vma, list, member) { \ size += i915_gem_obj_total_ggtt_size(vma->obj); \ @@ -528,10 +557,10 @@ static int i915_gem_object_info(struct seq_file *m, void* data) seq_putc(m, '\n'); print_batch_pool_stats(m, dev_priv); - mutex_unlock(&dev->struct_mutex); mutex_lock(&dev->filelist_mutex); + print_context_stats(m, dev_priv); list_for_each_entry_reverse(file, &dev->filelist, lhead) { struct file_stats stats; struct task_struct *task; @@ -562,7 +591,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data) struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; uintptr_t list = (uintptr_t) node->info_ent->data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; u64 total_obj_size, total_gtt_size; int count, ret; @@ -596,7 +625,7 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc; int ret; @@ -607,18 +636,20 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) for_each_intel_crtc(dev, crtc) { const char pipe = pipe_name(crtc->pipe); const char plane = plane_name(crtc->plane); - struct intel_unpin_work *work; + struct intel_flip_work *work; spin_lock_irq(&dev->event_lock); - work = crtc->unpin_work; + work = crtc->flip_work; if (work == NULL) { seq_printf(m, "No flip due on pipe %c (plane %c)\n", pipe, plane); } else { + u32 pending; u32 addr; - if (atomic_read(&work->pending) < INTEL_FLIP_COMPLETE) { - seq_printf(m, "Flip queued on pipe %c (plane %c)\n", + pending = atomic_read(&work->pending); + if (pending) { + seq_printf(m, "Flip ioctl preparing on pipe %c (plane %c)\n", pipe, plane); } else { seq_printf(m, "Flip pending (waiting for vsync) on pipe %c (plane %c)\n", @@ -631,18 +662,14 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) engine->name, i915_gem_request_get_seqno(work->flip_queued_req), dev_priv->next_seqno, - engine->get_seqno(engine), - i915_gem_request_completed(work->flip_queued_req, true)); + intel_engine_get_seqno(engine), + i915_gem_request_completed(work->flip_queued_req)); } else seq_printf(m, "Flip not associated with any ring\n"); seq_printf(m, "Flip queued on frame %d, (was ready on frame %d), now %d\n", work->flip_queued_vblank, work->flip_ready_vblank, - drm_crtc_vblank_count(&crtc->base)); - if (work->enable_stall_check) - seq_puts(m, "Stall check enabled, "); - else - seq_puts(m, "Stall check waiting for page flip ioctl, "); + intel_crtc_get_vblank_counter(crtc)); seq_printf(m, "%d prepares\n", atomic_read(&work->pending)); if (INTEL_INFO(dev)->gen >= 4) @@ -668,7 +695,7 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; struct intel_engine_cs *engine; int total = 0; @@ -713,7 +740,7 @@ static int i915_gem_request_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; struct drm_i915_gem_request *req; int ret, any; @@ -761,17 +788,29 @@ static int i915_gem_request_info(struct seq_file *m, void *data) static void i915_ring_seqno_info(struct seq_file *m, struct intel_engine_cs *engine) { + struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct rb_node *rb; + seq_printf(m, "Current sequence (%s): %x\n", - engine->name, engine->get_seqno(engine)); - seq_printf(m, "Current user interrupts (%s): %x\n", - engine->name, READ_ONCE(engine->user_interrupts)); + engine->name, intel_engine_get_seqno(engine)); + seq_printf(m, "Current user interrupts (%s): %lx\n", + engine->name, READ_ONCE(engine->breadcrumbs.irq_wakeups)); + + spin_lock(&b->lock); + for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { + struct intel_wait *w = container_of(rb, typeof(*w), node); + + seq_printf(m, "Waiting (%s): %s [%d] on %x\n", + engine->name, w->tsk->comm, w->tsk->pid, w->seqno); + } + spin_unlock(&b->lock); } static int i915_gem_seqno_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; int ret; @@ -794,7 +833,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; int ret, i, pipe; @@ -985,7 +1024,7 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i, ret; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -1013,7 +1052,7 @@ static int i915_hws_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; const u32 *hws; int i; @@ -1124,7 +1163,7 @@ static int i915_next_seqno_get(void *data, u64 *val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -1161,7 +1200,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret = 0; intel_runtime_pm_get(dev_priv); @@ -1281,6 +1320,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused) } seq_printf(m, "PM IER=0x%08x IMR=0x%08x ISR=0x%08x IIR=0x%08x, MASK=0x%08x\n", pm_ier, pm_imr, pm_isr, pm_iir, pm_mask); + seq_printf(m, "pm_intr_keep: 0x%08x\n", dev_priv->rps.pm_intr_keep); seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); seq_printf(m, "Render p-state ratio: %d\n", (gt_perf_status & (IS_GEN9(dev) ? 0x1ff00 : 0xff00)) >> 8); @@ -1363,7 +1403,7 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; u64 acthd[I915_NUM_ENGINES]; u32 seqno[I915_NUM_ENGINES]; @@ -1380,10 +1420,10 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) for_each_engine_id(engine, dev_priv, id) { acthd[id] = intel_ring_get_active_head(engine); - seqno[id] = engine->get_seqno(engine); + seqno[id] = intel_engine_get_seqno(engine); } - i915_get_extra_instdone(dev, instdone); + i915_get_extra_instdone(dev_priv, instdone); intel_runtime_pm_put(dev_priv); @@ -1400,9 +1440,11 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) engine->hangcheck.seqno, seqno[id], engine->last_submitted_seqno); - seq_printf(m, "\tuser interrupts = %x [current %x]\n", + seq_printf(m, "\twaiters? %d\n", + intel_engine_has_waiter(engine)); + seq_printf(m, "\tuser interrupts = %lx [current %lx]\n", engine->hangcheck.user_interrupts, - READ_ONCE(engine->user_interrupts)); + READ_ONCE(engine->breadcrumbs.irq_wakeups)); seq_printf(m, "\tACTHD = 0x%08llx [current 0x%08llx]\n", (long long)engine->hangcheck.acthd, (long long)acthd[id]); @@ -1432,7 +1474,7 @@ static int ironlake_drpc_info(struct seq_file *m) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 rgvmodectl, rstdbyctl; u16 crstandvid; int ret; @@ -1500,7 +1542,7 @@ static int i915_forcewake_domains(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_uncore_forcewake_domain *fw_domain; spin_lock_irq(&dev_priv->uncore.lock); @@ -1518,7 +1560,7 @@ static int vlv_drpc_info(struct seq_file *m) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 rpmodectl1, rcctl1, pw_status; intel_runtime_pm_get(dev_priv); @@ -1558,7 +1600,7 @@ static int gen6_drpc_info(struct seq_file *m) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 rpmodectl1, gt_core_status, rcctl1, rc6vids = 0; unsigned forcewake_count; int count = 0, ret; @@ -1670,7 +1712,7 @@ static int i915_frontbuffer_tracking(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); seq_printf(m, "FB tracking busy bits: 0x%08x\n", dev_priv->fb_tracking.busy_bits); @@ -1685,7 +1727,7 @@ static int i915_fbc_status(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!HAS_FBC(dev)) { seq_puts(m, "FBC unsupported on this chipset\n"); @@ -1715,7 +1757,7 @@ static int i915_fbc_status(struct seq_file *m, void *unused) static int i915_fbc_fc_get(void *data, u64 *val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev)) return -ENODEV; @@ -1728,7 +1770,7 @@ static int i915_fbc_fc_get(void *data, u64 *val) static int i915_fbc_fc_set(void *data, u64 val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 reg; if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev)) @@ -1755,7 +1797,7 @@ static int i915_ips_status(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!HAS_IPS(dev)) { seq_puts(m, "not supported\n"); @@ -1785,7 +1827,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); bool sr_enabled = false; intel_runtime_pm_get(dev_priv); @@ -1814,7 +1856,7 @@ static int i915_emon_status(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned long temp, chipset, gfx; int ret; @@ -1842,7 +1884,7 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret = 0; int gpu_freq, ia_freq; unsigned int max_gpu_freq, min_gpu_freq; @@ -1897,7 +1939,7 @@ static int i915_opregion(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_opregion *opregion = &dev_priv->opregion; int ret; @@ -1918,7 +1960,7 @@ static int i915_vbt(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_opregion *opregion = &dev_priv->opregion; if (opregion->vbt) @@ -1940,19 +1982,19 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data) return ret; #ifdef CONFIG_DRM_FBDEV_EMULATION - if (to_i915(dev)->fbdev) { - fbdev_fb = to_intel_framebuffer(to_i915(dev)->fbdev->helper.fb); - - seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ", - fbdev_fb->base.width, - fbdev_fb->base.height, - fbdev_fb->base.depth, - fbdev_fb->base.bits_per_pixel, - fbdev_fb->base.modifier[0], - drm_framebuffer_read_refcount(&fbdev_fb->base)); - describe_obj(m, fbdev_fb->obj); - seq_putc(m, '\n'); - } + if (to_i915(dev)->fbdev) { + fbdev_fb = to_intel_framebuffer(to_i915(dev)->fbdev->helper.fb); + + seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ", + fbdev_fb->base.width, + fbdev_fb->base.height, + fbdev_fb->base.depth, + fbdev_fb->base.bits_per_pixel, + fbdev_fb->base.modifier[0], + drm_framebuffer_read_refcount(&fbdev_fb->base)); + describe_obj(m, fbdev_fb->obj); + seq_putc(m, '\n'); + } #endif mutex_lock(&dev->mode_config.fb_lock); @@ -1989,10 +2031,9 @@ static int i915_context_status(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; - struct intel_context *ctx; - enum intel_engine_id id; + struct i915_gem_context *ctx; int ret; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -2000,32 +2041,36 @@ static int i915_context_status(struct seq_file *m, void *unused) return ret; list_for_each_entry(ctx, &dev_priv->context_list, link) { - if (!i915.enable_execlists && - ctx->legacy_hw_ctx.rcs_state == NULL) - continue; - - seq_puts(m, "HW context "); - describe_ctx(m, ctx); - if (ctx == dev_priv->kernel_context) - seq_printf(m, "(kernel context) "); + seq_printf(m, "HW context %u ", ctx->hw_id); + if (IS_ERR(ctx->file_priv)) { + seq_puts(m, "(deleted) "); + } else if (ctx->file_priv) { + struct pid *pid = ctx->file_priv->file->pid; + struct task_struct *task; - if (i915.enable_execlists) { - seq_putc(m, '\n'); - for_each_engine_id(engine, dev_priv, id) { - struct drm_i915_gem_object *ctx_obj = - ctx->engine[id].state; - struct intel_ringbuffer *ringbuf = - ctx->engine[id].ringbuf; - - seq_printf(m, "%s: ", engine->name); - if (ctx_obj) - describe_obj(m, ctx_obj); - if (ringbuf) - describe_ctx_ringbuf(m, ringbuf); - seq_putc(m, '\n'); + task = get_pid_task(pid, PIDTYPE_PID); + if (task) { + seq_printf(m, "(%s [%d]) ", + task->comm, task->pid); + put_task_struct(task); } } else { - describe_obj(m, ctx->legacy_hw_ctx.rcs_state); + seq_puts(m, "(kernel) "); + } + + seq_putc(m, ctx->remap_slice ? 'R' : 'r'); + seq_putc(m, '\n'); + + for_each_engine(engine, dev_priv) { + struct intel_context *ce = &ctx->engine[engine->id]; + + seq_printf(m, "%s: ", engine->name); + seq_putc(m, ce->initialised ? 'I' : 'i'); + if (ce->state) + describe_obj(m, ce->state); + if (ce->ringbuf) + describe_ctx_ringbuf(m, ce->ringbuf); + seq_putc(m, '\n'); } seq_putc(m, '\n'); @@ -2037,24 +2082,22 @@ static int i915_context_status(struct seq_file *m, void *unused) } static void i915_dump_lrc_obj(struct seq_file *m, - struct intel_context *ctx, + struct i915_gem_context *ctx, struct intel_engine_cs *engine) { + struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state; struct page *page; uint32_t *reg_state; int j; - struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state; unsigned long ggtt_offset = 0; + seq_printf(m, "CONTEXT: %s %u\n", engine->name, ctx->hw_id); + if (ctx_obj == NULL) { - seq_printf(m, "Context on %s with no gem object\n", - engine->name); + seq_puts(m, "\tNot allocated\n"); return; } - seq_printf(m, "CONTEXT: %s %u\n", engine->name, - intel_execlists_ctx_id(ctx, engine)); - if (!i915_gem_obj_ggtt_bound(ctx_obj)) seq_puts(m, "\tNot bound in GGTT\n"); else @@ -2085,9 +2128,9 @@ static int i915_dump_lrc(struct seq_file *m, void *unused) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; - struct intel_context *ctx; + struct i915_gem_context *ctx; int ret; if (!i915.enable_execlists) { @@ -2100,10 +2143,8 @@ static int i915_dump_lrc(struct seq_file *m, void *unused) return ret; list_for_each_entry(ctx, &dev_priv->context_list, link) - if (ctx != dev_priv->kernel_context) { - for_each_engine(engine, dev_priv) - i915_dump_lrc_obj(m, ctx, engine); - } + for_each_engine(engine, dev_priv) + i915_dump_lrc_obj(m, ctx, engine); mutex_unlock(&dev->struct_mutex); @@ -2114,7 +2155,7 @@ static int i915_execlists(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *)m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; u32 status_pointer; u8 read_pointer; @@ -2174,8 +2215,8 @@ static int i915_execlists(struct seq_file *m, void *data) seq_printf(m, "\t%d requests in queue\n", count); if (head_req) { - seq_printf(m, "\tHead request id: %u\n", - intel_execlists_ctx_id(head_req->ctx, engine)); + seq_printf(m, "\tHead request context: %u\n", + head_req->ctx->hw_id); seq_printf(m, "\tHead request tail: %u\n", head_req->tail); } @@ -2217,7 +2258,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -2269,7 +2310,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data) static int per_file_ctx(int id, void *ptr, void *data) { - struct intel_context *ctx = ptr; + struct i915_gem_context *ctx = ptr; struct seq_file *m = data; struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; @@ -2290,7 +2331,7 @@ static int per_file_ctx(int id, void *ptr, void *data) static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; int i; @@ -2311,15 +2352,15 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev) static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; - if (INTEL_INFO(dev)->gen == 6) + if (IS_GEN6(dev_priv)) seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(GFX_MODE)); for_each_engine(engine, dev_priv) { seq_printf(m, "%s\n", engine->name); - if (INTEL_INFO(dev)->gen == 7) + if (IS_GEN7(dev_priv)) seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(RING_MODE_GEN7(engine))); seq_printf(m, "PP_DIR_BASE: 0x%08x\n", @@ -2345,7 +2386,7 @@ static int i915_ppgtt_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_file *file; int ret = mutex_lock_interruptible(&dev->struct_mutex); @@ -2388,7 +2429,7 @@ static int count_irq_waiters(struct drm_i915_private *i915) int count = 0; for_each_engine(engine, i915) - count += engine->irq_refcount; + count += intel_engine_has_waiter(engine); return count; } @@ -2397,11 +2438,12 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_file *file; seq_printf(m, "RPS enabled? %d\n", dev_priv->rps.enabled); - seq_printf(m, "GPU busy? %d\n", dev_priv->mm.busy); + seq_printf(m, "GPU busy? %s [%x]\n", + yesno(dev_priv->gt.awake), dev_priv->gt.active_engines); seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv)); seq_printf(m, "Frequency requested %d; min hard:%d, soft:%d; max soft:%d, hard:%d\n", intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq), @@ -2442,7 +2484,7 @@ static int i915_llc(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); const bool edram = INTEL_GEN(dev_priv) > 8; seq_printf(m, "LLC: %s\n", yesno(HAS_LLC(dev))); @@ -2455,7 +2497,7 @@ static int i915_llc(struct seq_file *m, void *data) static int i915_guc_load_status_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; - struct drm_i915_private *dev_priv = node->minor->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(node->minor->dev); struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; u32 tmp, i; @@ -2510,15 +2552,16 @@ static void i915_guc_client_info(struct seq_file *m, seq_printf(m, "\tWQ size %d, offset: 0x%x, tail %d\n", client->wq_size, client->wq_offset, client->wq_tail); + seq_printf(m, "\tWork queue full: %u\n", client->no_wq_space); seq_printf(m, "\tFailed to queue: %u\n", client->q_fail); seq_printf(m, "\tFailed doorbell: %u\n", client->b_fail); seq_printf(m, "\tLast submission result: %d\n", client->retcode); for_each_engine(engine, dev_priv) { seq_printf(m, "\tSubmissions: %llu %s\n", - client->submissions[engine->guc_id], + client->submissions[engine->id], engine->name); - tot += client->submissions[engine->guc_id]; + tot += client->submissions[engine->id]; } seq_printf(m, "\tTotal: %llu\n", tot); } @@ -2527,7 +2570,7 @@ static int i915_guc_info(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_guc guc; struct i915_guc_client client = {}; struct intel_engine_cs *engine; @@ -2546,6 +2589,10 @@ static int i915_guc_info(struct seq_file *m, void *data) mutex_unlock(&dev->struct_mutex); + seq_printf(m, "Doorbell map:\n"); + seq_printf(m, "\t%*pb\n", GUC_MAX_DOORBELLS, guc.doorbell_bitmap); + seq_printf(m, "Doorbell next cacheline: 0x%x\n\n", guc.db_cacheline); + seq_printf(m, "GuC total action count: %llu\n", guc.action_count); seq_printf(m, "GuC action failure count: %u\n", guc.action_fail); seq_printf(m, "GuC last action command: 0x%x\n", guc.action_cmd); @@ -2555,9 +2602,9 @@ static int i915_guc_info(struct seq_file *m, void *data) seq_printf(m, "\nGuC submissions:\n"); for_each_engine(engine, dev_priv) { seq_printf(m, "\t%-24s: %10llu, last seqno 0x%08x\n", - engine->name, guc.submissions[engine->guc_id], - guc.last_seqno[engine->guc_id]); - total += guc.submissions[engine->guc_id]; + engine->name, guc.submissions[engine->id], + guc.last_seqno[engine->id]); + total += guc.submissions[engine->id]; } seq_printf(m, "\t%s: %llu\n", "Total", total); @@ -2573,7 +2620,7 @@ static int i915_guc_log_dump(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *log_obj = dev_priv->guc.log_obj; u32 *log; int i = 0, pg; @@ -2601,7 +2648,7 @@ static int i915_edp_psr_status(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 psrperf = 0; u32 stat[3]; enum pipe pipe; @@ -2669,7 +2716,6 @@ static int i915_sink_crc(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct intel_encoder *encoder; struct intel_connector *connector; struct intel_dp *intel_dp = NULL; int ret; @@ -2677,18 +2723,19 @@ static int i915_sink_crc(struct seq_file *m, void *data) drm_modeset_lock_all(dev); for_each_intel_connector(dev, connector) { + struct drm_crtc *crtc; - if (connector->base.dpms != DRM_MODE_DPMS_ON) + if (!connector->base.state->best_encoder) continue; - if (!connector->base.encoder) + crtc = connector->base.state->crtc; + if (!crtc->state->active) continue; - encoder = to_intel_encoder(connector->base.encoder); - if (encoder->type != INTEL_OUTPUT_EDP) + if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP) continue; - intel_dp = enc_to_intel_dp(&encoder->base); + intel_dp = enc_to_intel_dp(connector->base.state->best_encoder); ret = intel_dp_sink_crc(intel_dp, crc); if (ret) @@ -2709,7 +2756,7 @@ static int i915_energy_uJ(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u64 power; u32 units; @@ -2735,12 +2782,12 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!HAS_RUNTIME_PM(dev_priv)) seq_puts(m, "Runtime power management not supported\n"); - seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->mm.busy)); + seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->gt.awake)); seq_printf(m, "IRQs disabled: %s\n", yesno(!intel_irqs_enabled(dev_priv))); #ifdef CONFIG_PM @@ -2750,8 +2797,8 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused) seq_printf(m, "Device Power Management (CONFIG_PM) disabled\n"); #endif seq_printf(m, "PCI device power state: %s [%d]\n", - pci_power_name(dev_priv->dev->pdev->current_state), - dev_priv->dev->pdev->current_state); + pci_power_name(dev_priv->drm.pdev->current_state), + dev_priv->drm.pdev->current_state); return 0; } @@ -2760,7 +2807,7 @@ static int i915_power_domain_info(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct i915_power_domains *power_domains = &dev_priv->power_domains; int i; @@ -2795,7 +2842,7 @@ static int i915_dmc_info(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_csr *csr; if (!HAS_CSR(dev)) { @@ -2918,7 +2965,7 @@ static void intel_dp_info(struct seq_file *m, seq_printf(m, "\tDPCD rev: %x\n", intel_dp->dpcd[DP_DPCD_REV]); seq_printf(m, "\taudio support: %s\n", yesno(intel_dp->has_audio)); - if (intel_encoder->type == INTEL_OUTPUT_EDP) + if (intel_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP) intel_panel_info(m, &intel_connector->panel); } @@ -2957,14 +3004,26 @@ static void intel_connector_info(struct seq_file *m, seq_printf(m, "\tCEA rev: %d\n", connector->display_info.cea_rev); } - if (intel_encoder) { - if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT || - intel_encoder->type == INTEL_OUTPUT_EDP) - intel_dp_info(m, intel_connector); - else if (intel_encoder->type == INTEL_OUTPUT_HDMI) - intel_hdmi_info(m, intel_connector); - else if (intel_encoder->type == INTEL_OUTPUT_LVDS) + + if (!intel_encoder || intel_encoder->type == INTEL_OUTPUT_DP_MST) + return; + + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_DisplayPort: + case DRM_MODE_CONNECTOR_eDP: + intel_dp_info(m, intel_connector); + break; + case DRM_MODE_CONNECTOR_LVDS: + if (intel_encoder->type == INTEL_OUTPUT_LVDS) intel_lvds_info(m, intel_connector); + break; + case DRM_MODE_CONNECTOR_HDMIA: + if (intel_encoder->type == INTEL_OUTPUT_HDMI || + intel_encoder->type == INTEL_OUTPUT_UNKNOWN) + intel_hdmi_info(m, intel_connector); + break; + default: + break; } seq_printf(m, "\tmodes:\n"); @@ -2974,7 +3033,7 @@ static void intel_connector_info(struct seq_file *m, static bool cursor_active(struct drm_device *dev, int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 state; if (IS_845G(dev) || IS_I865G(dev)) @@ -2987,7 +3046,7 @@ static bool cursor_active(struct drm_device *dev, int pipe) static bool cursor_position(struct drm_device *dev, int pipe, int *x, int *y) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 pos; pos = I915_READ(CURPOS(pipe)); @@ -3108,7 +3167,7 @@ static int i915_display_info(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc; struct drm_connector *connector; @@ -3163,13 +3222,13 @@ static int i915_semaphore_status(struct seq_file *m, void *unused) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; int num_rings = hweight32(INTEL_INFO(dev)->ring_mask); enum intel_engine_id id; int j, ret; - if (!i915_semaphore_is_enabled(dev)) { + if (!i915_semaphore_is_enabled(dev_priv)) { seq_puts(m, "Semaphores are disabled\n"); return 0; } @@ -3236,7 +3295,7 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i; drm_modeset_lock_all(dev); @@ -3266,7 +3325,7 @@ static int i915_wa_registers(struct seq_file *m, void *unused) struct intel_engine_cs *engine; struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct i915_workarounds *workarounds = &dev_priv->workarounds; enum intel_engine_id id; @@ -3304,7 +3363,7 @@ static int i915_ddb_info(struct seq_file *m, void *unused) { struct drm_info_node *node = m->private; struct drm_device *dev = node->minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct skl_ddb_allocation *ddb; struct skl_ddb_entry *entry; enum pipe pipe; @@ -3342,31 +3401,16 @@ static int i915_ddb_info(struct seq_file *m, void *unused) static void drrs_status_per_crtc(struct seq_file *m, struct drm_device *dev, struct intel_crtc *intel_crtc) { - struct intel_encoder *intel_encoder; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct i915_drrs *drrs = &dev_priv->drrs; int vrefresh = 0; + struct drm_connector *connector; - for_each_encoder_on_crtc(dev, &intel_crtc->base, intel_encoder) { - /* Encoder connected on this CRTC */ - switch (intel_encoder->type) { - case INTEL_OUTPUT_EDP: - seq_puts(m, "eDP:\n"); - break; - case INTEL_OUTPUT_DSI: - seq_puts(m, "DSI:\n"); - break; - case INTEL_OUTPUT_HDMI: - seq_puts(m, "HDMI:\n"); - break; - case INTEL_OUTPUT_DISPLAYPORT: - seq_puts(m, "DP:\n"); - break; - default: - seq_printf(m, "Other encoder (id=%d).\n", - intel_encoder->type); - return; - } + drm_for_each_connector(connector, dev) { + if (connector->state->crtc != &intel_crtc->base) + continue; + + seq_printf(m, "%s:\n", connector->name); } if (dev_priv->vbt.drrs_type == STATIC_DRRS_SUPPORT) @@ -3429,18 +3473,16 @@ static int i915_drrs_status(struct seq_file *m, void *unused) struct intel_crtc *intel_crtc; int active_crtc_cnt = 0; + drm_modeset_lock_all(dev); for_each_intel_crtc(dev, intel_crtc) { - drm_modeset_lock(&intel_crtc->base.mutex, NULL); - if (intel_crtc->base.state->active) { active_crtc_cnt++; seq_printf(m, "\nCRTC %d: ", active_crtc_cnt); drrs_status_per_crtc(m, dev, intel_crtc); } - - drm_modeset_unlock(&intel_crtc->base.mutex); } + drm_modeset_unlock_all(dev); if (!active_crtc_cnt) seq_puts(m, "No active crtc found\n"); @@ -3458,17 +3500,23 @@ static int i915_dp_mst_info(struct seq_file *m, void *unused) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; - struct drm_encoder *encoder; struct intel_encoder *intel_encoder; struct intel_digital_port *intel_dig_port; + struct drm_connector *connector; + drm_modeset_lock_all(dev); - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { - intel_encoder = to_intel_encoder(encoder); - if (intel_encoder->type != INTEL_OUTPUT_DISPLAYPORT) + drm_for_each_connector(connector, dev) { + if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort) continue; - intel_dig_port = enc_to_dig_port(encoder); + + intel_encoder = intel_attached_encoder(connector); + if (!intel_encoder || intel_encoder->type == INTEL_OUTPUT_DP_MST) + continue; + + intel_dig_port = enc_to_dig_port(&intel_encoder->base); if (!intel_dig_port->dp.can_mst) continue; + seq_printf(m, "MST Source Port %c\n", port_name(intel_dig_port->port)); drm_dp_mst_dump_topology(m, &intel_dig_port->dp.mst_mgr); @@ -3480,7 +3528,7 @@ static int i915_dp_mst_info(struct seq_file *m, void *unused) static int i915_pipe_crc_open(struct inode *inode, struct file *filep) { struct pipe_crc_info *info = inode->i_private; - struct drm_i915_private *dev_priv = info->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(info->dev); struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe]; if (info->pipe >= INTEL_INFO(info->dev)->num_pipes) @@ -3504,7 +3552,7 @@ static int i915_pipe_crc_open(struct inode *inode, struct file *filep) static int i915_pipe_crc_release(struct inode *inode, struct file *filep) { struct pipe_crc_info *info = inode->i_private; - struct drm_i915_private *dev_priv = info->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(info->dev); struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe]; spin_lock_irq(&pipe_crc->lock); @@ -3532,7 +3580,7 @@ i915_pipe_crc_read(struct file *filep, char __user *user_buf, size_t count, { struct pipe_crc_info *info = filep->private_data; struct drm_device *dev = info->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe]; char buf[PIPE_CRC_BUFFER_LEN]; int n_entries; @@ -3665,7 +3713,7 @@ static const char *pipe_crc_source_name(enum intel_pipe_crc_source source) static int display_crc_ctl_show(struct seq_file *m, void *data) { struct drm_device *dev = m->private; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i; for (i = 0; i < I915_MAX_PIPES; i++) @@ -3726,7 +3774,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_device *dev, enum pipe pipe, case INTEL_OUTPUT_TVOUT: *source = INTEL_PIPE_CRC_SOURCE_TV; break; - case INTEL_OUTPUT_DISPLAYPORT: + case INTEL_OUTPUT_DP: case INTEL_OUTPUT_EDP: dig_port = enc_to_dig_port(&encoder->base); switch (dig_port->port) { @@ -3759,7 +3807,7 @@ static int vlv_pipe_crc_ctl_reg(struct drm_device *dev, enum intel_pipe_crc_source *source, uint32_t *val) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); bool need_stable_symbols = false; if (*source == INTEL_PIPE_CRC_SOURCE_AUTO) { @@ -3830,7 +3878,7 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_device *dev, enum intel_pipe_crc_source *source, uint32_t *val) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); bool need_stable_symbols = false; if (*source == INTEL_PIPE_CRC_SOURCE_AUTO) { @@ -3904,7 +3952,7 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_device *dev, static void vlv_undo_pipe_scramble_reset(struct drm_device *dev, enum pipe pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t tmp = I915_READ(PORT_DFT2_G4X); switch (pipe) { @@ -3929,7 +3977,7 @@ static void vlv_undo_pipe_scramble_reset(struct drm_device *dev, static void g4x_undo_pipe_scramble_reset(struct drm_device *dev, enum pipe pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t tmp = I915_READ(PORT_DFT2_G4X); if (pipe == PIPE_A) @@ -3972,7 +4020,7 @@ static int ilk_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source, static void hsw_trans_edp_pipe_A_crc_wa(struct drm_device *dev, bool enable) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_A]); struct intel_crtc_state *pipe_config; @@ -4040,7 +4088,7 @@ static int ivb_pipe_crc_ctl_reg(struct drm_device *dev, static int pipe_crc_set_source(struct drm_device *dev, enum pipe pipe, enum intel_pipe_crc_source source) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; struct intel_crtc *crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); @@ -4547,7 +4595,7 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8]) static int pri_wm_latency_show(struct seq_file *m, void *data) { struct drm_device *dev = m->private; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); const uint16_t *latencies; if (INTEL_INFO(dev)->gen >= 9) @@ -4563,7 +4611,7 @@ static int pri_wm_latency_show(struct seq_file *m, void *data) static int spr_wm_latency_show(struct seq_file *m, void *data) { struct drm_device *dev = m->private; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); const uint16_t *latencies; if (INTEL_INFO(dev)->gen >= 9) @@ -4579,7 +4627,7 @@ static int spr_wm_latency_show(struct seq_file *m, void *data) static int cur_wm_latency_show(struct seq_file *m, void *data) { struct drm_device *dev = m->private; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); const uint16_t *latencies; if (INTEL_INFO(dev)->gen >= 9) @@ -4670,7 +4718,7 @@ static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf, { struct seq_file *m = file->private_data; struct drm_device *dev = m->private; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint16_t *latencies; if (INTEL_INFO(dev)->gen >= 9) @@ -4686,7 +4734,7 @@ static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf, { struct seq_file *m = file->private_data; struct drm_device *dev = m->private; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint16_t *latencies; if (INTEL_INFO(dev)->gen >= 9) @@ -4702,7 +4750,7 @@ static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf, { struct seq_file *m = file->private_data; struct drm_device *dev = m->private; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint16_t *latencies; if (INTEL_INFO(dev)->gen >= 9) @@ -4744,7 +4792,7 @@ static int i915_wedged_get(void *data, u64 *val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); *val = i915_terminally_wedged(&dev_priv->gpu_error); @@ -4755,7 +4803,7 @@ static int i915_wedged_set(void *data, u64 val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* * There is no safeguard against this debugfs entry colliding @@ -4770,7 +4818,7 @@ i915_wedged_set(void *data, u64 val) intel_runtime_pm_get(dev_priv); - i915_handle_error(dev, val, + i915_handle_error(dev_priv, val, "Manually setting wedged to %llu", val); intel_runtime_pm_put(dev_priv); @@ -4783,44 +4831,10 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_wedged_fops, "%llu\n"); static int -i915_ring_stop_get(void *data, u64 *val) -{ - struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; - - *val = dev_priv->gpu_error.stop_rings; - - return 0; -} - -static int -i915_ring_stop_set(void *data, u64 val) -{ - struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; - int ret; - - DRM_DEBUG_DRIVER("Stopping rings 0x%08llx\n", val); - - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; - - dev_priv->gpu_error.stop_rings = val; - mutex_unlock(&dev->struct_mutex); - - return 0; -} - -DEFINE_SIMPLE_ATTRIBUTE(i915_ring_stop_fops, - i915_ring_stop_get, i915_ring_stop_set, - "0x%08llx\n"); - -static int i915_ring_missed_irq_get(void *data, u64 *val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); *val = dev_priv->gpu_error.missed_irq_rings; return 0; @@ -4830,7 +4844,7 @@ static int i915_ring_missed_irq_set(void *data, u64 val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; /* Lock against concurrent debugfs callers */ @@ -4851,7 +4865,7 @@ static int i915_ring_test_irq_get(void *data, u64 *val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); *val = dev_priv->gpu_error.test_irq_rings; @@ -4862,18 +4876,11 @@ static int i915_ring_test_irq_set(void *data, u64 val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; - int ret; + struct drm_i915_private *dev_priv = to_i915(dev); + val &= INTEL_INFO(dev_priv)->ring_mask; DRM_DEBUG_DRIVER("Masking interrupts on rings 0x%08llx\n", val); - - /* Lock against concurrent debugfs callers */ - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; - dev_priv->gpu_error.test_irq_rings = val; - mutex_unlock(&dev->struct_mutex); return 0; } @@ -4902,7 +4909,7 @@ static int i915_drop_caches_set(void *data, u64 val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; DRM_DEBUG("Dropping caches: 0x%08llx\n", val); @@ -4914,13 +4921,13 @@ i915_drop_caches_set(void *data, u64 val) return ret; if (val & DROP_ACTIVE) { - ret = i915_gpu_idle(dev); + ret = i915_gem_wait_for_idle(dev_priv); if (ret) goto unlock; } if (val & (DROP_RETIRE | DROP_ACTIVE)) - i915_gem_retire_requests(dev); + i915_gem_retire_requests(dev_priv); if (val & DROP_BOUND) i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_BOUND); @@ -4942,7 +4949,7 @@ static int i915_max_freq_get(void *data, u64 *val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; if (INTEL_INFO(dev)->gen < 6) @@ -4964,7 +4971,7 @@ static int i915_max_freq_set(void *data, u64 val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 hw_max, hw_min; int ret; @@ -4994,7 +5001,7 @@ i915_max_freq_set(void *data, u64 val) dev_priv->rps.max_freq_softlimit = val; - intel_set_rps(dev, val); + intel_set_rps(dev_priv, val); mutex_unlock(&dev_priv->rps.hw_lock); @@ -5009,7 +5016,7 @@ static int i915_min_freq_get(void *data, u64 *val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; if (INTEL_INFO(dev)->gen < 6) @@ -5031,7 +5038,7 @@ static int i915_min_freq_set(void *data, u64 val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 hw_max, hw_min; int ret; @@ -5061,7 +5068,7 @@ i915_min_freq_set(void *data, u64 val) dev_priv->rps.min_freq_softlimit = val; - intel_set_rps(dev, val); + intel_set_rps(dev_priv, val); mutex_unlock(&dev_priv->rps.hw_lock); @@ -5076,7 +5083,7 @@ static int i915_cache_sharing_get(void *data, u64 *val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 snpcr; int ret; @@ -5091,7 +5098,7 @@ i915_cache_sharing_get(void *data, u64 *val) snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev_priv->dev->struct_mutex); + mutex_unlock(&dev_priv->drm.struct_mutex); *val = (snpcr & GEN6_MBC_SNPCR_MASK) >> GEN6_MBC_SNPCR_SHIFT; @@ -5102,7 +5109,7 @@ static int i915_cache_sharing_set(void *data, u64 val) { struct drm_device *dev = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 snpcr; if (!(IS_GEN6(dev) || IS_GEN7(dev))) @@ -5139,7 +5146,7 @@ struct sseu_dev_status { static void cherryview_sseu_device_status(struct drm_device *dev, struct sseu_dev_status *stat) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ss_max = 2; int ss; u32 sig1[ss_max], sig2[ss_max]; @@ -5171,7 +5178,7 @@ static void cherryview_sseu_device_status(struct drm_device *dev, static void gen9_sseu_device_status(struct drm_device *dev, struct sseu_dev_status *stat) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int s_max = 3, ss_max = 4; int s, ss; u32 s_reg[s_max], eu_reg[2*s_max], eu_mask[2]; @@ -5236,7 +5243,7 @@ static void gen9_sseu_device_status(struct drm_device *dev, static void broadwell_sseu_device_status(struct drm_device *dev, struct sseu_dev_status *stat) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int s; u32 slice_info = I915_READ(GEN8_GT_SLICE_INFO); @@ -5278,6 +5285,10 @@ static int i915_sseu_status(struct seq_file *m, void *unused) INTEL_INFO(dev)->eu_total); seq_printf(m, " Available EU Per Subslice: %u\n", INTEL_INFO(dev)->eu_per_subslice); + seq_printf(m, " Has Pooled EU: %s\n", yesno(HAS_POOLED_EU(dev))); + if (HAS_POOLED_EU(dev)) + seq_printf(m, " Min EU in pool: %u\n", + INTEL_INFO(dev)->min_eu_in_pool); seq_printf(m, " Has Slice Power Gating: %s\n", yesno(INTEL_INFO(dev)->has_slice_pg)); seq_printf(m, " Has Subslice Power Gating: %s\n", @@ -5311,7 +5322,7 @@ static int i915_sseu_status(struct seq_file *m, void *unused) static int i915_forcewake_open(struct inode *inode, struct file *file) { struct drm_device *dev = inode->i_private; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (INTEL_INFO(dev)->gen < 6) return 0; @@ -5325,7 +5336,7 @@ static int i915_forcewake_open(struct inode *inode, struct file *file) static int i915_forcewake_release(struct inode *inode, struct file *file) { struct drm_device *dev = inode->i_private; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (INTEL_INFO(dev)->gen < 6) return 0; @@ -5441,7 +5452,6 @@ static const struct i915_debugfs_files { {"i915_max_freq", &i915_max_freq_fops}, {"i915_min_freq", &i915_min_freq_fops}, {"i915_cache_sharing", &i915_cache_sharing_fops}, - {"i915_ring_stop", &i915_ring_stop_fops}, {"i915_ring_missed_irq", &i915_ring_missed_irq_fops}, {"i915_ring_test_irq", &i915_ring_test_irq_fops}, {"i915_gem_drop_caches", &i915_drop_caches_fops}, @@ -5459,7 +5469,7 @@ static const struct i915_debugfs_files { void intel_display_crc_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; for_each_pipe(dev_priv, pipe) { @@ -5471,8 +5481,9 @@ void intel_display_crc_init(struct drm_device *dev) } } -int i915_debugfs_init(struct drm_minor *minor) +int i915_debugfs_register(struct drm_i915_private *dev_priv) { + struct drm_minor *minor = dev_priv->drm.primary; int ret, i; ret = i915_forcewake_create(minor->debugfs_root, minor); @@ -5498,8 +5509,9 @@ int i915_debugfs_init(struct drm_minor *minor) minor->debugfs_root, minor); } -void i915_debugfs_cleanup(struct drm_minor *minor) +void i915_debugfs_unregister(struct drm_i915_private *dev_priv) { + struct drm_minor *minor = dev_priv->drm.primary; int i; drm_debugfs_remove_files(i915_debugfs_list, diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index fd3553beb..5de36d8dc 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -27,400 +27,92 @@ * */ -#include <linux/device.h> #include <linux/acpi.h> -#include <drm/drmP.h> -#include <drm/i915_drm.h> -#include "i915_drv.h" -#include "i915_trace.h" -#include "intel_drv.h" - -#include <linux/apple-gmux.h> -#include <linux/console.h> +#include <linux/device.h> +#include <linux/oom.h> #include <linux/module.h> +#include <linux/pci.h> +#include <linux/pm.h> #include <linux/pm_runtime.h> +#include <linux/pnp.h> +#include <linux/slab.h> #include <linux/vgaarb.h> #include <linux/vga_switcheroo.h> -#include <drm/drm_crtc_helper.h> - -static struct drm_driver driver; - -#define GEN_DEFAULT_PIPEOFFSETS \ - .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \ - PIPE_C_OFFSET, PIPE_EDP_OFFSET }, \ - .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \ - TRANSCODER_C_OFFSET, TRANSCODER_EDP_OFFSET }, \ - .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET } - -#define GEN_CHV_PIPEOFFSETS \ - .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \ - CHV_PIPE_C_OFFSET }, \ - .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \ - CHV_TRANSCODER_C_OFFSET, }, \ - .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET, \ - CHV_PALETTE_C_OFFSET } - -#define CURSOR_OFFSETS \ - .cursor_offsets = { CURSOR_A_OFFSET, CURSOR_B_OFFSET, CHV_CURSOR_C_OFFSET } - -#define IVB_CURSOR_OFFSETS \ - .cursor_offsets = { CURSOR_A_OFFSET, IVB_CURSOR_B_OFFSET, IVB_CURSOR_C_OFFSET } - -#define BDW_COLORS \ - .color = { .degamma_lut_size = 512, .gamma_lut_size = 512 } -#define CHV_COLORS \ - .color = { .degamma_lut_size = 65, .gamma_lut_size = 257 } - -static const struct intel_device_info intel_i830_info = { - .gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .num_pipes = 2, - .has_overlay = 1, .overlay_needs_physical = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_845g_info = { - .gen = 2, .num_pipes = 1, - .has_overlay = 1, .overlay_needs_physical = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_i85x_info = { - .gen = 2, .is_i85x = 1, .is_mobile = 1, .num_pipes = 2, - .cursor_needs_physical = 1, - .has_overlay = 1, .overlay_needs_physical = 1, - .has_fbc = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_i865g_info = { - .gen = 2, .num_pipes = 1, - .has_overlay = 1, .overlay_needs_physical = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_i915g_info = { - .gen = 3, .is_i915g = 1, .cursor_needs_physical = 1, .num_pipes = 2, - .has_overlay = 1, .overlay_needs_physical = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; -static const struct intel_device_info intel_i915gm_info = { - .gen = 3, .is_mobile = 1, .num_pipes = 2, - .cursor_needs_physical = 1, - .has_overlay = 1, .overlay_needs_physical = 1, - .supports_tv = 1, - .has_fbc = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; -static const struct intel_device_info intel_i945g_info = { - .gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .num_pipes = 2, - .has_overlay = 1, .overlay_needs_physical = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; -static const struct intel_device_info intel_i945gm_info = { - .gen = 3, .is_i945gm = 1, .is_mobile = 1, .num_pipes = 2, - .has_hotplug = 1, .cursor_needs_physical = 1, - .has_overlay = 1, .overlay_needs_physical = 1, - .supports_tv = 1, - .has_fbc = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_i965g_info = { - .gen = 4, .is_broadwater = 1, .num_pipes = 2, - .has_hotplug = 1, - .has_overlay = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_i965gm_info = { - .gen = 4, .is_crestline = 1, .num_pipes = 2, - .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1, - .has_overlay = 1, - .supports_tv = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_g33_info = { - .gen = 3, .is_g33 = 1, .num_pipes = 2, - .need_gfx_hws = 1, .has_hotplug = 1, - .has_overlay = 1, - .ring_mask = RENDER_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_g45_info = { - .gen = 4, .is_g4x = 1, .need_gfx_hws = 1, .num_pipes = 2, - .has_pipe_cxsr = 1, .has_hotplug = 1, - .ring_mask = RENDER_RING | BSD_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_gm45_info = { - .gen = 4, .is_g4x = 1, .num_pipes = 2, - .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1, - .has_pipe_cxsr = 1, .has_hotplug = 1, - .supports_tv = 1, - .ring_mask = RENDER_RING | BSD_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_pineview_info = { - .gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .num_pipes = 2, - .need_gfx_hws = 1, .has_hotplug = 1, - .has_overlay = 1, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_ironlake_d_info = { - .gen = 5, .num_pipes = 2, - .need_gfx_hws = 1, .has_hotplug = 1, - .ring_mask = RENDER_RING | BSD_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; +#include <linux/vt.h> +#include <acpi/video.h> -static const struct intel_device_info intel_ironlake_m_info = { - .gen = 5, .is_mobile = 1, .num_pipes = 2, - .need_gfx_hws = 1, .has_hotplug = 1, - .has_fbc = 1, - .ring_mask = RENDER_RING | BSD_RING, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_sandybridge_d_info = { - .gen = 6, .num_pipes = 2, - .need_gfx_hws = 1, .has_hotplug = 1, - .has_fbc = 1, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING, - .has_llc = 1, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -static const struct intel_device_info intel_sandybridge_m_info = { - .gen = 6, .is_mobile = 1, .num_pipes = 2, - .need_gfx_hws = 1, .has_hotplug = 1, - .has_fbc = 1, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING, - .has_llc = 1, - GEN_DEFAULT_PIPEOFFSETS, - CURSOR_OFFSETS, -}; - -#define GEN7_FEATURES \ - .gen = 7, .num_pipes = 3, \ - .need_gfx_hws = 1, .has_hotplug = 1, \ - .has_fbc = 1, \ - .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \ - .has_llc = 1, \ - GEN_DEFAULT_PIPEOFFSETS, \ - IVB_CURSOR_OFFSETS - -static const struct intel_device_info intel_ivybridge_d_info = { - GEN7_FEATURES, - .is_ivybridge = 1, -}; - -static const struct intel_device_info intel_ivybridge_m_info = { - GEN7_FEATURES, - .is_ivybridge = 1, - .is_mobile = 1, -}; - -static const struct intel_device_info intel_ivybridge_q_info = { - GEN7_FEATURES, - .is_ivybridge = 1, - .num_pipes = 0, /* legal, last one wins */ -}; - -#define VLV_FEATURES \ - .gen = 7, .num_pipes = 2, \ - .need_gfx_hws = 1, .has_hotplug = 1, \ - .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \ - .display_mmio_offset = VLV_DISPLAY_BASE, \ - GEN_DEFAULT_PIPEOFFSETS, \ - CURSOR_OFFSETS - -static const struct intel_device_info intel_valleyview_m_info = { - VLV_FEATURES, - .is_valleyview = 1, - .is_mobile = 1, -}; +#include <drm/drmP.h> +#include <drm/drm_crtc_helper.h> +#include <drm/i915_drm.h> -static const struct intel_device_info intel_valleyview_d_info = { - VLV_FEATURES, - .is_valleyview = 1, -}; +#include "i915_drv.h" +#include "i915_trace.h" +#include "i915_vgpu.h" +#include "intel_drv.h" -#define HSW_FEATURES \ - GEN7_FEATURES, \ - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \ - .has_ddi = 1, \ - .has_fpga_dbg = 1 +static struct drm_driver driver; -static const struct intel_device_info intel_haswell_d_info = { - HSW_FEATURES, - .is_haswell = 1, -}; +static unsigned int i915_load_fail_count; -static const struct intel_device_info intel_haswell_m_info = { - HSW_FEATURES, - .is_haswell = 1, - .is_mobile = 1, -}; +bool __i915_inject_load_failure(const char *func, int line) +{ + if (i915_load_fail_count >= i915.inject_load_failure) + return false; -#define BDW_FEATURES \ - HSW_FEATURES, \ - BDW_COLORS + if (++i915_load_fail_count == i915.inject_load_failure) { + DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n", + i915.inject_load_failure, func, line); + return true; + } -static const struct intel_device_info intel_broadwell_d_info = { - BDW_FEATURES, - .gen = 8, -}; + return false; +} -static const struct intel_device_info intel_broadwell_m_info = { - BDW_FEATURES, - .gen = 8, .is_mobile = 1, -}; +#define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI" +#define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \ + "providing the dmesg log by booting with drm.debug=0xf" -static const struct intel_device_info intel_broadwell_gt3d_info = { - BDW_FEATURES, - .gen = 8, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, -}; +void +__i915_printk(struct drm_i915_private *dev_priv, const char *level, + const char *fmt, ...) +{ + static bool shown_bug_once; + struct device *dev = dev_priv->drm.dev; + bool is_error = level[1] <= KERN_ERR[1]; + bool is_debug = level[1] == KERN_DEBUG[1]; + struct va_format vaf; + va_list args; + + if (is_debug && !(drm_debug & DRM_UT_DRIVER)) + return; -static const struct intel_device_info intel_broadwell_gt3m_info = { - BDW_FEATURES, - .gen = 8, .is_mobile = 1, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, -}; + va_start(args, fmt); -static const struct intel_device_info intel_cherryview_info = { - .gen = 8, .num_pipes = 3, - .need_gfx_hws = 1, .has_hotplug = 1, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, - .is_cherryview = 1, - .display_mmio_offset = VLV_DISPLAY_BASE, - GEN_CHV_PIPEOFFSETS, - CURSOR_OFFSETS, - CHV_COLORS, -}; + vaf.fmt = fmt; + vaf.va = &args; -static const struct intel_device_info intel_skylake_info = { - BDW_FEATURES, - .is_skylake = 1, - .gen = 9, -}; + dev_printk(level, dev, "[" DRM_NAME ":%ps] %pV", + __builtin_return_address(0), &vaf); -static const struct intel_device_info intel_skylake_gt3_info = { - BDW_FEATURES, - .is_skylake = 1, - .gen = 9, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, -}; + if (is_error && !shown_bug_once) { + dev_notice(dev, "%s", FDO_BUG_MSG); + shown_bug_once = true; + } -static const struct intel_device_info intel_broxton_info = { - .is_preliminary = 1, - .is_broxton = 1, - .gen = 9, - .need_gfx_hws = 1, .has_hotplug = 1, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, - .num_pipes = 3, - .has_ddi = 1, - .has_fpga_dbg = 1, - .has_fbc = 1, - GEN_DEFAULT_PIPEOFFSETS, - IVB_CURSOR_OFFSETS, - BDW_COLORS, -}; + va_end(args); +} -static const struct intel_device_info intel_kabylake_info = { - BDW_FEATURES, - .is_kabylake = 1, - .gen = 9, -}; +static bool i915_error_injected(struct drm_i915_private *dev_priv) +{ + return i915.inject_load_failure && + i915_load_fail_count == i915.inject_load_failure; +} -static const struct intel_device_info intel_kabylake_gt3_info = { - BDW_FEATURES, - .is_kabylake = 1, - .gen = 9, - .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, -}; +#define i915_load_error(dev_priv, fmt, ...) \ + __i915_printk(dev_priv, \ + i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \ + fmt, ##__VA_ARGS__) -/* - * Make sure any device matches here are from most specific to most - * general. For example, since the Quanta match is based on the subsystem - * and subvendor IDs, we need it to come before the more general IVB - * PCI ID matches, otherwise we'll use the wrong info struct above. - */ -static const struct pci_device_id pciidlist[] = { - INTEL_I830_IDS(&intel_i830_info), - INTEL_I845G_IDS(&intel_845g_info), - INTEL_I85X_IDS(&intel_i85x_info), - INTEL_I865G_IDS(&intel_i865g_info), - INTEL_I915G_IDS(&intel_i915g_info), - INTEL_I915GM_IDS(&intel_i915gm_info), - INTEL_I945G_IDS(&intel_i945g_info), - INTEL_I945GM_IDS(&intel_i945gm_info), - INTEL_I965G_IDS(&intel_i965g_info), - INTEL_G33_IDS(&intel_g33_info), - INTEL_I965GM_IDS(&intel_i965gm_info), - INTEL_GM45_IDS(&intel_gm45_info), - INTEL_G45_IDS(&intel_g45_info), - INTEL_PINEVIEW_IDS(&intel_pineview_info), - INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info), - INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info), - INTEL_SNB_D_IDS(&intel_sandybridge_d_info), - INTEL_SNB_M_IDS(&intel_sandybridge_m_info), - INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */ - INTEL_IVB_M_IDS(&intel_ivybridge_m_info), - INTEL_IVB_D_IDS(&intel_ivybridge_d_info), - INTEL_HSW_D_IDS(&intel_haswell_d_info), - INTEL_HSW_M_IDS(&intel_haswell_m_info), - INTEL_VLV_M_IDS(&intel_valleyview_m_info), - INTEL_VLV_D_IDS(&intel_valleyview_d_info), - INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info), - INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info), - INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info), - INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info), - INTEL_CHV_IDS(&intel_cherryview_info), - INTEL_SKL_GT1_IDS(&intel_skylake_info), - INTEL_SKL_GT2_IDS(&intel_skylake_info), - INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info), - INTEL_SKL_GT4_IDS(&intel_skylake_gt3_info), - INTEL_BXT_IDS(&intel_broxton_info), - INTEL_KBL_GT1_IDS(&intel_kabylake_info), - INTEL_KBL_GT2_IDS(&intel_kabylake_info), - INTEL_KBL_GT3_IDS(&intel_kabylake_gt3_info), - INTEL_KBL_GT4_IDS(&intel_kabylake_gt3_info), - {0, 0, 0} -}; - -MODULE_DEVICE_TABLE(pci, pciidlist); static enum intel_pch intel_virt_detect_pch(struct drm_device *dev) { @@ -450,9 +142,9 @@ static enum intel_pch intel_virt_detect_pch(struct drm_device *dev) return ret; } -void intel_detect_pch(struct drm_device *dev) +static void intel_detect_pch(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct pci_dev *pch = NULL; /* In all current cases, num_pipes is equivalent to the PCH_NOP setting @@ -519,8 +211,10 @@ void intel_detect_pch(struct drm_device *dev) } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) || (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) || ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) && - pch->subsystem_vendor == 0x1af4 && - pch->subsystem_device == 0x1100)) { + pch->subsystem_vendor == + PCI_SUBVENDOR_ID_REDHAT_QUMRANET && + pch->subsystem_device == + PCI_SUBDEVICE_ID_QEMU)) { dev_priv->pch_type = intel_virt_detect_pch(dev); } else continue; @@ -534,9 +228,9 @@ void intel_detect_pch(struct drm_device *dev) pci_dev_put(pch); } -bool i915_semaphore_is_enabled(struct drm_device *dev) +bool i915_semaphore_is_enabled(struct drm_i915_private *dev_priv) { - if (INTEL_INFO(dev)->gen < 6) + if (INTEL_GEN(dev_priv) < 6) return false; if (i915.semaphores >= 0) @@ -546,22 +240,1177 @@ bool i915_semaphore_is_enabled(struct drm_device *dev) if (i915.enable_execlists) return false; - /* Until we get further testing... */ - if (IS_GEN8(dev)) - return false; - #ifdef CONFIG_INTEL_IOMMU /* Enable semaphores on SNB when IO remapping is off */ - if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) + if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) return false; #endif return true; } +static int i915_getparam(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + drm_i915_getparam_t *param = data; + int value; + + switch (param->param) { + case I915_PARAM_IRQ_ACTIVE: + case I915_PARAM_ALLOW_BATCHBUFFER: + case I915_PARAM_LAST_DISPATCH: + /* Reject all old ums/dri params. */ + return -ENODEV; + case I915_PARAM_CHIPSET_ID: + value = dev->pdev->device; + break; + case I915_PARAM_REVISION: + value = dev->pdev->revision; + break; + case I915_PARAM_HAS_GEM: + value = 1; + break; + case I915_PARAM_NUM_FENCES_AVAIL: + value = dev_priv->num_fence_regs; + break; + case I915_PARAM_HAS_OVERLAY: + value = dev_priv->overlay ? 1 : 0; + break; + case I915_PARAM_HAS_PAGEFLIPPING: + value = 1; + break; + case I915_PARAM_HAS_EXECBUF2: + /* depends on GEM */ + value = 1; + break; + case I915_PARAM_HAS_BSD: + value = intel_engine_initialized(&dev_priv->engine[VCS]); + break; + case I915_PARAM_HAS_BLT: + value = intel_engine_initialized(&dev_priv->engine[BCS]); + break; + case I915_PARAM_HAS_VEBOX: + value = intel_engine_initialized(&dev_priv->engine[VECS]); + break; + case I915_PARAM_HAS_BSD2: + value = intel_engine_initialized(&dev_priv->engine[VCS2]); + break; + case I915_PARAM_HAS_RELAXED_FENCING: + value = 1; + break; + case I915_PARAM_HAS_COHERENT_RINGS: + value = 1; + break; + case I915_PARAM_HAS_EXEC_CONSTANTS: + value = INTEL_INFO(dev)->gen >= 4; + break; + case I915_PARAM_HAS_RELAXED_DELTA: + value = 1; + break; + case I915_PARAM_HAS_GEN7_SOL_RESET: + value = 1; + break; + case I915_PARAM_HAS_LLC: + value = HAS_LLC(dev); + break; + case I915_PARAM_HAS_WT: + value = HAS_WT(dev); + break; + case I915_PARAM_HAS_ALIASING_PPGTT: + value = USES_PPGTT(dev); + break; + case I915_PARAM_HAS_WAIT_TIMEOUT: + value = 1; + break; + case I915_PARAM_HAS_SEMAPHORES: + value = i915_semaphore_is_enabled(dev_priv); + break; + case I915_PARAM_HAS_PRIME_VMAP_FLUSH: + value = 1; + break; + case I915_PARAM_HAS_SECURE_BATCHES: + value = capable(CAP_SYS_ADMIN); + break; + case I915_PARAM_HAS_PINNED_BATCHES: + value = 1; + break; + case I915_PARAM_HAS_EXEC_NO_RELOC: + value = 1; + break; + case I915_PARAM_HAS_EXEC_HANDLE_LUT: + value = 1; + break; + case I915_PARAM_CMD_PARSER_VERSION: + value = i915_cmd_parser_get_version(dev_priv); + break; + case I915_PARAM_HAS_COHERENT_PHYS_GTT: + value = 1; + break; + case I915_PARAM_MMAP_VERSION: + value = 1; + break; + case I915_PARAM_SUBSLICE_TOTAL: + value = INTEL_INFO(dev)->subslice_total; + if (!value) + return -ENODEV; + break; + case I915_PARAM_EU_TOTAL: + value = INTEL_INFO(dev)->eu_total; + if (!value) + return -ENODEV; + break; + case I915_PARAM_HAS_GPU_RESET: + value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv); + break; + case I915_PARAM_HAS_RESOURCE_STREAMER: + value = HAS_RESOURCE_STREAMER(dev); + break; + case I915_PARAM_HAS_EXEC_SOFTPIN: + value = 1; + break; + case I915_PARAM_HAS_POOLED_EU: + value = HAS_POOLED_EU(dev); + break; + case I915_PARAM_MIN_EU_IN_POOL: + value = INTEL_INFO(dev)->min_eu_in_pool; + break; + default: + DRM_DEBUG("Unknown parameter %d\n", param->param); + return -EINVAL; + } + + if (put_user(value, param->value)) + return -EFAULT; + + return 0; +} + +static int i915_get_bridge_dev(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + + dev_priv->bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0)); + if (!dev_priv->bridge_dev) { + DRM_ERROR("bridge device not found\n"); + return -1; + } + return 0; +} + +/* Allocate space for the MCH regs if needed, return nonzero on error */ +static int +intel_alloc_mchbar_resource(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; + u32 temp_lo, temp_hi = 0; + u64 mchbar_addr; + int ret; + + if (INTEL_INFO(dev)->gen >= 4) + pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi); + pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo); + mchbar_addr = ((u64)temp_hi << 32) | temp_lo; + + /* If ACPI doesn't have it, assume we need to allocate it ourselves */ +#ifdef CONFIG_PNP + if (mchbar_addr && + pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE)) + return 0; +#endif + + /* Get some space for it */ + dev_priv->mch_res.name = "i915 MCHBAR"; + dev_priv->mch_res.flags = IORESOURCE_MEM; + ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus, + &dev_priv->mch_res, + MCHBAR_SIZE, MCHBAR_SIZE, + PCIBIOS_MIN_MEM, + 0, pcibios_align_resource, + dev_priv->bridge_dev); + if (ret) { + DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret); + dev_priv->mch_res.start = 0; + return ret; + } + + if (INTEL_INFO(dev)->gen >= 4) + pci_write_config_dword(dev_priv->bridge_dev, reg + 4, + upper_32_bits(dev_priv->mch_res.start)); + + pci_write_config_dword(dev_priv->bridge_dev, reg, + lower_32_bits(dev_priv->mch_res.start)); + return 0; +} + +/* Setup MCHBAR if possible, return true if we should disable it again */ +static void +intel_setup_mchbar(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; + u32 temp; + bool enabled; + + if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) + return; + + dev_priv->mchbar_need_disable = false; + + if (IS_I915G(dev) || IS_I915GM(dev)) { + pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp); + enabled = !!(temp & DEVEN_MCHBAR_EN); + } else { + pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); + enabled = temp & 1; + } + + /* If it's already enabled, don't have to do anything */ + if (enabled) + return; + + if (intel_alloc_mchbar_resource(dev)) + return; + + dev_priv->mchbar_need_disable = true; + + /* Space is allocated or reserved, so enable it. */ + if (IS_I915G(dev) || IS_I915GM(dev)) { + pci_write_config_dword(dev_priv->bridge_dev, DEVEN, + temp | DEVEN_MCHBAR_EN); + } else { + pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp); + pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1); + } +} + +static void +intel_teardown_mchbar(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915; + + if (dev_priv->mchbar_need_disable) { + if (IS_I915G(dev) || IS_I915GM(dev)) { + u32 deven_val; + + pci_read_config_dword(dev_priv->bridge_dev, DEVEN, + &deven_val); + deven_val &= ~DEVEN_MCHBAR_EN; + pci_write_config_dword(dev_priv->bridge_dev, DEVEN, + deven_val); + } else { + u32 mchbar_val; + + pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, + &mchbar_val); + mchbar_val &= ~1; + pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, + mchbar_val); + } + } + + if (dev_priv->mch_res.start) + release_resource(&dev_priv->mch_res); +} + +/* true = enable decode, false = disable decoder */ +static unsigned int i915_vga_set_decode(void *cookie, bool state) +{ + struct drm_device *dev = cookie; + + intel_modeset_vga_set_state(dev, state); + if (state) + return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | + VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; + else + return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; +} + +static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) +{ + struct drm_device *dev = pci_get_drvdata(pdev); + pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; + + if (state == VGA_SWITCHEROO_ON) { + pr_info("switched on\n"); + dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; + /* i915 resume handler doesn't set to D0 */ + pci_set_power_state(dev->pdev, PCI_D0); + i915_resume_switcheroo(dev); + dev->switch_power_state = DRM_SWITCH_POWER_ON; + } else { + pr_info("switched off\n"); + dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; + i915_suspend_switcheroo(dev, pmm); + dev->switch_power_state = DRM_SWITCH_POWER_OFF; + } +} + +static bool i915_switcheroo_can_switch(struct pci_dev *pdev) +{ + struct drm_device *dev = pci_get_drvdata(pdev); + + /* + * FIXME: open_count is protected by drm_global_mutex but that would lead to + * locking inversion with the driver load path. And the access here is + * completely racy anyway. So don't bother with locking for now. + */ + return dev->open_count == 0; +} + +static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { + .set_gpu_state = i915_switcheroo_set_state, + .reprobe = NULL, + .can_switch = i915_switcheroo_can_switch, +}; + +static void i915_gem_fini(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + + /* + * Neither the BIOS, ourselves or any other kernel + * expects the system to be in execlists mode on startup, + * so we need to reset the GPU back to legacy mode. And the only + * known way to disable logical contexts is through a GPU reset. + * + * So in order to leave the system in a known default configuration, + * always reset the GPU upon unload. Afterwards we then clean up the + * GEM state tracking, flushing off the requests and leaving the + * system in a known idle state. + * + * Note that is of the upmost importance that the GPU is idle and + * all stray writes are flushed *before* we dismantle the backing + * storage for the pinned objects. + * + * However, since we are uncertain that reseting the GPU on older + * machines is a good idea, we don't - just in case it leaves the + * machine in an unusable condition. + */ + if (HAS_HW_CONTEXTS(dev)) { + int reset = intel_gpu_reset(dev_priv, ALL_ENGINES); + WARN_ON(reset && reset != -ENODEV); + } + + mutex_lock(&dev->struct_mutex); + i915_gem_reset(dev); + i915_gem_cleanup_engines(dev); + i915_gem_context_fini(dev); + mutex_unlock(&dev->struct_mutex); + + WARN_ON(!list_empty(&to_i915(dev)->context_list)); +} + +static int i915_load_modeset_init(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + int ret; + + if (i915_inject_load_failure()) + return -ENODEV; + + ret = intel_bios_init(dev_priv); + if (ret) + DRM_INFO("failed to find VBIOS tables\n"); + + /* If we have > 1 VGA cards, then we need to arbitrate access + * to the common VGA resources. + * + * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA), + * then we do not take part in VGA arbitration and the + * vga_client_register() fails with -ENODEV. + */ + ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode); + if (ret && ret != -ENODEV) + goto out; + + intel_register_dsm_handler(); + + ret = vga_switcheroo_register_client(dev->pdev, &i915_switcheroo_ops, false); + if (ret) + goto cleanup_vga_client; + + /* must happen before intel_power_domains_init_hw() on VLV/CHV */ + intel_update_rawclk(dev_priv); + + intel_power_domains_init_hw(dev_priv, false); + + intel_csr_ucode_init(dev_priv); + + ret = intel_irq_install(dev_priv); + if (ret) + goto cleanup_csr; + + intel_setup_gmbus(dev); + + /* Important: The output setup functions called by modeset_init need + * working irqs for e.g. gmbus and dp aux transfers. */ + intel_modeset_init(dev); + + intel_guc_init(dev); + + ret = i915_gem_init(dev); + if (ret) + goto cleanup_irq; + + intel_modeset_gem_init(dev); + + if (INTEL_INFO(dev)->num_pipes == 0) + return 0; + + ret = intel_fbdev_init(dev); + if (ret) + goto cleanup_gem; + + /* Only enable hotplug handling once the fbdev is fully set up. */ + intel_hpd_init(dev_priv); + + drm_kms_helper_poll_init(dev); + + return 0; + +cleanup_gem: + i915_gem_fini(dev); +cleanup_irq: + intel_guc_fini(dev); + drm_irq_uninstall(dev); + intel_teardown_gmbus(dev); +cleanup_csr: + intel_csr_ucode_fini(dev_priv); + intel_power_domains_fini(dev_priv); + vga_switcheroo_unregister_client(dev->pdev); +cleanup_vga_client: + vga_client_register(dev->pdev, NULL, NULL, NULL); +out: + return ret; +} + +#if IS_ENABLED(CONFIG_FB) +static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) +{ + struct apertures_struct *ap; + struct pci_dev *pdev = dev_priv->drm.pdev; + struct i915_ggtt *ggtt = &dev_priv->ggtt; + bool primary; + int ret; + + ap = alloc_apertures(1); + if (!ap) + return -ENOMEM; + + ap->ranges[0].base = ggtt->mappable_base; + ap->ranges[0].size = ggtt->mappable_end; + + primary = + pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW; + + ret = remove_conflicting_framebuffers(ap, "inteldrmfb", primary); + + kfree(ap); + + return ret; +} +#else +static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv) +{ + return 0; +} +#endif + +#if !defined(CONFIG_VGA_CONSOLE) +static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) +{ + return 0; +} +#elif !defined(CONFIG_DUMMY_CONSOLE) +static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) +{ + return -ENODEV; +} +#else +static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv) +{ + int ret = 0; + + DRM_INFO("Replacing VGA console driver\n"); + + console_lock(); + if (con_is_bound(&vga_con)) + ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1); + if (ret == 0) { + ret = do_unregister_con_driver(&vga_con); + + /* Ignore "already unregistered". */ + if (ret == -ENODEV) + ret = 0; + } + console_unlock(); + + return ret; +} +#endif + +static void intel_init_dpio(struct drm_i915_private *dev_priv) +{ + /* + * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C), + * CHV x1 PHY (DP/HDMI D) + * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C) + */ + if (IS_CHERRYVIEW(dev_priv)) { + DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2; + DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO; + } else if (IS_VALLEYVIEW(dev_priv)) { + DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO; + } +} + +static int i915_workqueues_init(struct drm_i915_private *dev_priv) +{ + /* + * The i915 workqueue is primarily used for batched retirement of + * requests (and thus managing bo) once the task has been completed + * by the GPU. i915_gem_retire_requests() is called directly when we + * need high-priority retirement, such as waiting for an explicit + * bo. + * + * It is also used for periodic low-priority events, such as + * idle-timers and recording error state. + * + * All tasks on the workqueue are expected to acquire the dev mutex + * so there is no point in running more than one instance of the + * workqueue at any time. Use an ordered one. + */ + dev_priv->wq = alloc_ordered_workqueue("i915", 0); + if (dev_priv->wq == NULL) + goto out_err; + + dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0); + if (dev_priv->hotplug.dp_wq == NULL) + goto out_free_wq; + + return 0; + +out_free_wq: + destroy_workqueue(dev_priv->wq); +out_err: + DRM_ERROR("Failed to allocate workqueues.\n"); + + return -ENOMEM; +} + +static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv) +{ + destroy_workqueue(dev_priv->hotplug.dp_wq); + destroy_workqueue(dev_priv->wq); +} + +/** + * i915_driver_init_early - setup state not requiring device access + * @dev_priv: device private + * + * Initialize everything that is a "SW-only" state, that is state not + * requiring accessing the device or exposing the driver via kernel internal + * or userspace interfaces. Example steps belonging here: lock initialization, + * system memory allocation, setting up device specific attributes and + * function hooks not requiring accessing the device. + */ +static int i915_driver_init_early(struct drm_i915_private *dev_priv, + const struct pci_device_id *ent) +{ + const struct intel_device_info *match_info = + (struct intel_device_info *)ent->driver_data; + struct intel_device_info *device_info; + int ret = 0; + + if (i915_inject_load_failure()) + return -ENODEV; + + /* Setup the write-once "constant" device info */ + device_info = mkwrite_device_info(dev_priv); + memcpy(device_info, match_info, sizeof(*device_info)); + device_info->device_id = dev_priv->drm.pdev->device; + + BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE); + device_info->gen_mask = BIT(device_info->gen - 1); + + spin_lock_init(&dev_priv->irq_lock); + spin_lock_init(&dev_priv->gpu_error.lock); + mutex_init(&dev_priv->backlight_lock); + spin_lock_init(&dev_priv->uncore.lock); + spin_lock_init(&dev_priv->mm.object_stat_lock); + spin_lock_init(&dev_priv->mmio_flip_lock); + mutex_init(&dev_priv->sb_lock); + mutex_init(&dev_priv->modeset_restore_lock); + mutex_init(&dev_priv->av_mutex); + mutex_init(&dev_priv->wm.wm_mutex); + mutex_init(&dev_priv->pps_mutex); + + ret = i915_workqueues_init(dev_priv); + if (ret < 0) + return ret; + + ret = intel_gvt_init(dev_priv); + if (ret < 0) + goto err_workqueues; + + /* This must be called before any calls to HAS_PCH_* */ + intel_detect_pch(&dev_priv->drm); + + intel_pm_setup(&dev_priv->drm); + intel_init_dpio(dev_priv); + intel_power_domains_init(dev_priv); + intel_irq_init(dev_priv); + intel_init_display_hooks(dev_priv); + intel_init_clock_gating_hooks(dev_priv); + intel_init_audio_hooks(dev_priv); + i915_gem_load_init(&dev_priv->drm); + + intel_display_crc_init(&dev_priv->drm); + + intel_device_info_dump(dev_priv); + + /* Not all pre-production machines fall into this category, only the + * very first ones. Almost everything should work, except for maybe + * suspend/resume. And we don't implement workarounds that affect only + * pre-production machines. */ + if (IS_HSW_EARLY_SDV(dev_priv)) + DRM_INFO("This is an early pre-production Haswell machine. " + "It may not be fully functional.\n"); + + return 0; + +err_workqueues: + i915_workqueues_cleanup(dev_priv); + return ret; +} + +/** + * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early() + * @dev_priv: device private + */ +static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv) +{ + i915_gem_load_cleanup(&dev_priv->drm); + i915_workqueues_cleanup(dev_priv); +} + +static int i915_mmio_setup(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + int mmio_bar; + int mmio_size; + + mmio_bar = IS_GEN2(dev) ? 1 : 0; + /* + * Before gen4, the registers and the GTT are behind different BARs. + * However, from gen4 onwards, the registers and the GTT are shared + * in the same BAR, so we want to restrict this ioremap from + * clobbering the GTT which we want ioremap_wc instead. Fortunately, + * the register BAR remains the same size for all the earlier + * generations up to Ironlake. + */ + if (INTEL_INFO(dev)->gen < 5) + mmio_size = 512 * 1024; + else + mmio_size = 2 * 1024 * 1024; + dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size); + if (dev_priv->regs == NULL) { + DRM_ERROR("failed to map registers\n"); + + return -EIO; + } + + /* Try to make sure MCHBAR is enabled before poking at it */ + intel_setup_mchbar(dev); + + return 0; +} + +static void i915_mmio_cleanup(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + + intel_teardown_mchbar(dev); + pci_iounmap(dev->pdev, dev_priv->regs); +} + +/** + * i915_driver_init_mmio - setup device MMIO + * @dev_priv: device private + * + * Setup minimal device state necessary for MMIO accesses later in the + * initialization sequence. The setup here should avoid any other device-wide + * side effects or exposing the driver via kernel internal or user space + * interfaces. + */ +static int i915_driver_init_mmio(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = &dev_priv->drm; + int ret; + + if (i915_inject_load_failure()) + return -ENODEV; + + if (i915_get_bridge_dev(dev)) + return -EIO; + + ret = i915_mmio_setup(dev); + if (ret < 0) + goto put_bridge; + + intel_uncore_init(dev_priv); + + return 0; + +put_bridge: + pci_dev_put(dev_priv->bridge_dev); + + return ret; +} + +/** + * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio() + * @dev_priv: device private + */ +static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = &dev_priv->drm; + + intel_uncore_fini(dev_priv); + i915_mmio_cleanup(dev); + pci_dev_put(dev_priv->bridge_dev); +} + +static void intel_sanitize_options(struct drm_i915_private *dev_priv) +{ + i915.enable_execlists = + intel_sanitize_enable_execlists(dev_priv, + i915.enable_execlists); + + /* + * i915.enable_ppgtt is read-only, so do an early pass to validate the + * user's requested state against the hardware/driver capabilities. We + * do this now so that we can print out any log messages once rather + * than every time we check intel_enable_ppgtt(). + */ + i915.enable_ppgtt = + intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt); + DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); +} + +/** + * i915_driver_init_hw - setup state requiring device access + * @dev_priv: device private + * + * Setup state that requires accessing the device, but doesn't require + * exposing the driver via kernel internal or userspace interfaces. + */ +static int i915_driver_init_hw(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = &dev_priv->drm; + struct i915_ggtt *ggtt = &dev_priv->ggtt; + uint32_t aperture_size; + int ret; + + if (i915_inject_load_failure()) + return -ENODEV; + + intel_device_info_runtime_init(dev_priv); + + intel_sanitize_options(dev_priv); + + ret = i915_ggtt_init_hw(dev); + if (ret) + return ret; + + ret = i915_ggtt_enable_hw(dev); + if (ret) { + DRM_ERROR("failed to enable GGTT\n"); + goto out_ggtt; + } + + /* WARNING: Apparently we must kick fbdev drivers before vgacon, + * otherwise the vga fbdev driver falls over. */ + ret = i915_kick_out_firmware_fb(dev_priv); + if (ret) { + DRM_ERROR("failed to remove conflicting framebuffer drivers\n"); + goto out_ggtt; + } + + ret = i915_kick_out_vgacon(dev_priv); + if (ret) { + DRM_ERROR("failed to remove conflicting VGA console\n"); + goto out_ggtt; + } + + pci_set_master(dev->pdev); + + /* overlay on gen2 is broken and can't address above 1G */ + if (IS_GEN2(dev)) { + ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30)); + if (ret) { + DRM_ERROR("failed to set DMA mask\n"); + + goto out_ggtt; + } + } + + + /* 965GM sometimes incorrectly writes to hardware status page (HWS) + * using 32bit addressing, overwriting memory if HWS is located + * above 4GB. + * + * The documentation also mentions an issue with undefined + * behaviour if any general state is accessed within a page above 4GB, + * which also needs to be handled carefully. + */ + if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) { + ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32)); + + if (ret) { + DRM_ERROR("failed to set DMA mask\n"); + + goto out_ggtt; + } + } + + aperture_size = ggtt->mappable_end; + + ggtt->mappable = + io_mapping_create_wc(ggtt->mappable_base, + aperture_size); + if (!ggtt->mappable) { + ret = -EIO; + goto out_ggtt; + } + + ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, + aperture_size); + + pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, + PM_QOS_DEFAULT_VALUE); + + intel_uncore_sanitize(dev_priv); + + intel_opregion_setup(dev_priv); + + i915_gem_load_init_fences(dev_priv); + + /* On the 945G/GM, the chipset reports the MSI capability on the + * integrated graphics even though the support isn't actually there + * according to the published specs. It doesn't appear to function + * correctly in testing on 945G. + * This may be a side effect of MSI having been made available for PEG + * and the registers being closely associated. + * + * According to chipset errata, on the 965GM, MSI interrupts may + * be lost or delayed, but we use them anyways to avoid + * stuck interrupts on some machines. + */ + if (!IS_I945G(dev) && !IS_I945GM(dev)) { + if (pci_enable_msi(dev->pdev) < 0) + DRM_DEBUG_DRIVER("can't enable MSI"); + } + + return 0; + +out_ggtt: + i915_ggtt_cleanup_hw(dev); + + return ret; +} + +/** + * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw() + * @dev_priv: device private + */ +static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = &dev_priv->drm; + struct i915_ggtt *ggtt = &dev_priv->ggtt; + + if (dev->pdev->msi_enabled) + pci_disable_msi(dev->pdev); + + pm_qos_remove_request(&dev_priv->pm_qos); + arch_phys_wc_del(ggtt->mtrr); + io_mapping_free(ggtt->mappable); + i915_ggtt_cleanup_hw(dev); +} + +/** + * i915_driver_register - register the driver with the rest of the system + * @dev_priv: device private + * + * Perform any steps necessary to make the driver available via kernel + * internal or userspace interfaces. + */ +static void i915_driver_register(struct drm_i915_private *dev_priv) +{ + struct drm_device *dev = &dev_priv->drm; + + i915_gem_shrinker_init(dev_priv); + + /* + * Notify a valid surface after modesetting, + * when running inside a VM. + */ + if (intel_vgpu_active(dev_priv)) + I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY); + + /* Reveal our presence to userspace */ + if (drm_dev_register(dev, 0) == 0) { + i915_debugfs_register(dev_priv); + i915_setup_sysfs(dev); + } else + DRM_ERROR("Failed to register driver for userspace access!\n"); + + if (INTEL_INFO(dev_priv)->num_pipes) { + /* Must be done after probing outputs */ + intel_opregion_register(dev_priv); + acpi_video_register(); + } + + if (IS_GEN5(dev_priv)) + intel_gpu_ips_init(dev_priv); + + i915_audio_component_init(dev_priv); + + /* + * Some ports require correctly set-up hpd registers for detection to + * work properly (leading to ghost connected connector status), e.g. VGA + * on gm45. Hence we can only set up the initial fbdev config after hpd + * irqs are fully enabled. We do it last so that the async config + * cannot run before the connectors are registered. + */ + intel_fbdev_initial_config_async(dev); +} + +/** + * i915_driver_unregister - cleanup the registration done in i915_driver_regiser() + * @dev_priv: device private + */ +static void i915_driver_unregister(struct drm_i915_private *dev_priv) +{ + i915_audio_component_cleanup(dev_priv); + + intel_gpu_ips_teardown(); + acpi_video_unregister(); + intel_opregion_unregister(dev_priv); + + i915_teardown_sysfs(&dev_priv->drm); + i915_debugfs_unregister(dev_priv); + drm_dev_unregister(&dev_priv->drm); + + i915_gem_shrinker_cleanup(dev_priv); +} + +/** + * i915_driver_load - setup chip and create an initial config + * @dev: DRM device + * @flags: startup flags + * + * The driver load routine has to do several things: + * - drive output discovery via intel_modeset_init() + * - initialize the memory manager + * - allocate initial config memory + * - setup the DRM framebuffer with the allocated memory + */ +int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct drm_i915_private *dev_priv; + int ret; + + if (i915.nuclear_pageflip) + driver.driver_features |= DRIVER_ATOMIC; + + ret = -ENOMEM; + dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL); + if (dev_priv) + ret = drm_dev_init(&dev_priv->drm, &driver, &pdev->dev); + if (ret) { + dev_printk(KERN_ERR, &pdev->dev, + "[" DRM_NAME ":%s] allocation failed\n", __func__); + kfree(dev_priv); + return ret; + } + + dev_priv->drm.pdev = pdev; + dev_priv->drm.dev_private = dev_priv; + + ret = pci_enable_device(pdev); + if (ret) + goto out_free_priv; + + pci_set_drvdata(pdev, &dev_priv->drm); + + ret = i915_driver_init_early(dev_priv, ent); + if (ret < 0) + goto out_pci_disable; + + intel_runtime_pm_get(dev_priv); + + ret = i915_driver_init_mmio(dev_priv); + if (ret < 0) + goto out_runtime_pm_put; + + ret = i915_driver_init_hw(dev_priv); + if (ret < 0) + goto out_cleanup_mmio; + + /* + * TODO: move the vblank init and parts of modeset init steps into one + * of the i915_driver_init_/i915_driver_register functions according + * to the role/effect of the given init step. + */ + if (INTEL_INFO(dev_priv)->num_pipes) { + ret = drm_vblank_init(&dev_priv->drm, + INTEL_INFO(dev_priv)->num_pipes); + if (ret) + goto out_cleanup_hw; + } + + ret = i915_load_modeset_init(&dev_priv->drm); + if (ret < 0) + goto out_cleanup_vblank; + + i915_driver_register(dev_priv); + + intel_runtime_pm_enable(dev_priv); + + /* Everything is in place, we can now relax! */ + DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n", + driver.name, driver.major, driver.minor, driver.patchlevel, + driver.date, pci_name(pdev), dev_priv->drm.primary->index); + + intel_runtime_pm_put(dev_priv); + + return 0; + +out_cleanup_vblank: + drm_vblank_cleanup(&dev_priv->drm); +out_cleanup_hw: + i915_driver_cleanup_hw(dev_priv); +out_cleanup_mmio: + i915_driver_cleanup_mmio(dev_priv); +out_runtime_pm_put: + intel_runtime_pm_put(dev_priv); + i915_driver_cleanup_early(dev_priv); +out_pci_disable: + pci_disable_device(pdev); +out_free_priv: + i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret); + drm_dev_unref(&dev_priv->drm); + return ret; +} + +void i915_driver_unload(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + + intel_fbdev_fini(dev); + + if (i915_gem_suspend(dev)) + DRM_ERROR("failed to idle hardware; continuing to unload!\n"); + + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); + + i915_driver_unregister(dev_priv); + + drm_vblank_cleanup(dev); + + intel_modeset_cleanup(dev); + + /* + * free the memory space allocated for the child device + * config parsed from VBT + */ + if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) { + kfree(dev_priv->vbt.child_dev); + dev_priv->vbt.child_dev = NULL; + dev_priv->vbt.child_dev_num = 0; + } + kfree(dev_priv->vbt.sdvo_lvds_vbt_mode); + dev_priv->vbt.sdvo_lvds_vbt_mode = NULL; + kfree(dev_priv->vbt.lfp_lvds_vbt_mode); + dev_priv->vbt.lfp_lvds_vbt_mode = NULL; + + vga_switcheroo_unregister_client(dev->pdev); + vga_client_register(dev->pdev, NULL, NULL, NULL); + + intel_csr_ucode_fini(dev_priv); + + /* Free error state after interrupts are fully disabled. */ + cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); + i915_destroy_error_state(dev); + + /* Flush any outstanding unpin_work. */ + flush_workqueue(dev_priv->wq); + + intel_guc_fini(dev); + i915_gem_fini(dev); + intel_fbc_cleanup_cfb(dev_priv); + + intel_power_domains_fini(dev_priv); + + i915_driver_cleanup_hw(dev_priv); + i915_driver_cleanup_mmio(dev_priv); + + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); + + i915_driver_cleanup_early(dev_priv); +} + +static int i915_driver_open(struct drm_device *dev, struct drm_file *file) +{ + int ret; + + ret = i915_gem_open(dev, file); + if (ret) + return ret; + + return 0; +} + +/** + * i915_driver_lastclose - clean up after all DRM clients have exited + * @dev: DRM device + * + * Take care of cleaning up after all DRM clients have exited. In the + * mode setting case, we want to restore the kernel's initial mode (just + * in case the last client left us in a bad state). + * + * Additionally, in the non-mode setting case, we'll tear down the GTT + * and DMA structures, since the kernel won't be using them, and clea + * up any GEM state. + */ +static void i915_driver_lastclose(struct drm_device *dev) +{ + intel_fbdev_restore_mode(dev); + vga_switcheroo_process_delayed_switch(); +} + +static void i915_driver_preclose(struct drm_device *dev, struct drm_file *file) +{ + mutex_lock(&dev->struct_mutex); + i915_gem_context_close(dev, file); + i915_gem_release(dev, file); + mutex_unlock(&dev->struct_mutex); +} + +static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file) +{ + struct drm_i915_file_private *file_priv = file->driver_priv; + + kfree(file_priv); +} + static void intel_suspend_encoders(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct intel_encoder *encoder; drm_modeset_lock_all(dev); @@ -586,7 +1435,7 @@ static bool suspend_to_idle(struct drm_i915_private *dev_priv) static int i915_drm_suspend(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); pci_power_t opregion_target_state; int error; @@ -614,7 +1463,7 @@ static int i915_drm_suspend(struct drm_device *dev) intel_guc_suspend(dev); - intel_suspend_gt_powersave(dev); + intel_suspend_gt_powersave(dev_priv); intel_display_suspend(dev); @@ -632,10 +1481,10 @@ static int i915_drm_suspend(struct drm_device *dev) i915_save_state(dev); opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold; - intel_opregion_notify_adapter(dev, opregion_target_state); + intel_opregion_notify_adapter(dev_priv, opregion_target_state); - intel_uncore_forcewake_reset(dev, false); - intel_opregion_fini(dev); + intel_uncore_forcewake_reset(dev_priv, false); + intel_opregion_unregister(dev_priv); intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true); @@ -653,7 +1502,7 @@ out: static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation) { - struct drm_i915_private *dev_priv = drm_dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(drm_dev); bool fw_csr; int ret; @@ -715,7 +1564,7 @@ int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state) { int error; - if (!dev || !dev->dev_private) { + if (!dev) { DRM_ERROR("dev: %p\n", dev); DRM_ERROR("DRM not initialized, aborting suspend.\n"); return -ENODEV; @@ -737,7 +1586,7 @@ int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state) static int i915_drm_resume(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; disable_rpm_wakeref_asserts(dev_priv); @@ -753,7 +1602,7 @@ static int i915_drm_resume(struct drm_device *dev) mutex_unlock(&dev->struct_mutex); i915_restore_state(dev); - intel_opregion_setup(dev); + intel_opregion_setup(dev_priv); intel_init_pch_refclk(dev); drm_mode_config_reset(dev); @@ -771,7 +1620,7 @@ static int i915_drm_resume(struct drm_device *dev) mutex_lock(&dev->struct_mutex); if (i915_gem_init_hw(dev)) { DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n"); - atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter); + atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter); } mutex_unlock(&dev->struct_mutex); @@ -781,7 +1630,7 @@ static int i915_drm_resume(struct drm_device *dev) spin_lock_irq(&dev_priv->irq_lock); if (dev_priv->display.hpd_irq_setup) - dev_priv->display.hpd_irq_setup(dev); + dev_priv->display.hpd_irq_setup(dev_priv); spin_unlock_irq(&dev_priv->irq_lock); intel_dp_mst_resume(dev); @@ -798,7 +1647,7 @@ static int i915_drm_resume(struct drm_device *dev) /* Config may have changed between suspend and resume */ drm_helper_hpd_irq_event(dev); - intel_opregion_init(dev); + intel_opregion_register(dev_priv); intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false); @@ -806,7 +1655,7 @@ static int i915_drm_resume(struct drm_device *dev) dev_priv->modeset_restore = MODESET_DONE; mutex_unlock(&dev_priv->modeset_restore_lock); - intel_opregion_notify_adapter(dev, PCI_D0); + intel_opregion_notify_adapter(dev_priv, PCI_D0); drm_kms_helper_poll_enable(dev); @@ -817,7 +1666,7 @@ static int i915_drm_resume(struct drm_device *dev) static int i915_drm_resume_early(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; /* @@ -874,9 +1723,9 @@ static int i915_drm_resume_early(struct drm_device *dev) DRM_ERROR("Resume prepare failed: %d, continuing anyway\n", ret); - intel_uncore_early_sanitize(dev, true); + intel_uncore_early_sanitize(dev_priv, true); - if (IS_BROXTON(dev)) { + if (IS_BROXTON(dev_priv)) { if (!dev_priv->suspended_to_idle) gen9_sanitize_dc_state(dev_priv); bxt_disable_dc9(dev_priv); @@ -884,7 +1733,7 @@ static int i915_drm_resume_early(struct drm_device *dev) hsw_disable_pc8(dev_priv); } - intel_uncore_sanitize(dev); + intel_uncore_sanitize(dev_priv); if (IS_BROXTON(dev_priv) || !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload)) @@ -927,14 +1776,14 @@ int i915_resume_switcheroo(struct drm_device *dev) * - re-init interrupt state * - re-init display */ -int i915_reset(struct drm_device *dev) +int i915_reset(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_device *dev = &dev_priv->drm; struct i915_gpu_error *error = &dev_priv->gpu_error; unsigned reset_counter; int ret; - intel_reset_gt_powersave(dev); + intel_reset_gt_powersave(dev_priv); mutex_lock(&dev->struct_mutex); @@ -948,24 +1797,11 @@ int i915_reset(struct drm_device *dev) goto error; } - i915_gem_reset(dev); - - ret = intel_gpu_reset(dev, ALL_ENGINES); - - /* Also reset the gpu hangman. */ - if (error->stop_rings != 0) { - DRM_INFO("Simulated gpu hang, resetting stop_rings\n"); - error->stop_rings = 0; - if (ret == -ENODEV) { - DRM_INFO("Reset not implemented, but ignoring " - "error for simulated gpu hangs\n"); - ret = 0; - } - } + pr_notice("drm/i915: Resetting chip after gpu hang\n"); - if (i915_stop_ring_allow_warn(dev_priv)) - pr_notice("drm/i915: Resetting chip after gpu hang\n"); + i915_gem_reset(dev); + ret = intel_gpu_reset(dev_priv, ALL_ENGINES); if (ret) { if (ret != -ENODEV) DRM_ERROR("Failed to reset chip: %i\n", ret); @@ -1005,7 +1841,7 @@ int i915_reset(struct drm_device *dev) * of re-init after reset. */ if (INTEL_INFO(dev)->gen > 5) - intel_enable_gt_powersave(dev); + intel_enable_gt_powersave(dev_priv); return 0; @@ -1015,51 +1851,12 @@ error: return ret; } -static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) -{ - struct intel_device_info *intel_info = - (struct intel_device_info *) ent->driver_data; - - if (IS_PRELIMINARY_HW(intel_info) && !i915.preliminary_hw_support) { - DRM_INFO("This hardware requires preliminary hardware support.\n" - "See CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT, and/or modparam preliminary_hw_support\n"); - return -ENODEV; - } - - /* Only bind to function 0 of the device. Early generations - * used function 1 as a placeholder for multi-head. This causes - * us confusion instead, especially on the systems where both - * functions have the same PCI-ID! - */ - if (PCI_FUNC(pdev->devfn)) - return -ENODEV; - - /* - * apple-gmux is needed on dual GPU MacBook Pro - * to probe the panel if we're the inactive GPU. - */ - if (IS_ENABLED(CONFIG_VGA_ARB) && IS_ENABLED(CONFIG_VGA_SWITCHEROO) && - apple_gmux_present() && pdev != vga_default_device() && - !vga_switcheroo_handler_flags()) - return -EPROBE_DEFER; - - return drm_get_pci_dev(pdev, ent, &driver); -} - -static void -i915_pci_remove(struct pci_dev *pdev) -{ - struct drm_device *dev = pci_get_drvdata(pdev); - - drm_put_dev(dev); -} - static int i915_pm_suspend(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); struct drm_device *drm_dev = pci_get_drvdata(pdev); - if (!drm_dev || !drm_dev->dev_private) { + if (!drm_dev) { dev_err(dev, "DRM not initialized, aborting suspend.\n"); return -ENODEV; } @@ -1072,7 +1869,7 @@ static int i915_pm_suspend(struct device *dev) static int i915_pm_suspend_late(struct device *dev) { - struct drm_device *drm_dev = dev_to_i915(dev)->dev; + struct drm_device *drm_dev = &dev_to_i915(dev)->drm; /* * We have a suspend ordering issue with the snd-hda driver also @@ -1091,7 +1888,7 @@ static int i915_pm_suspend_late(struct device *dev) static int i915_pm_poweroff_late(struct device *dev) { - struct drm_device *drm_dev = dev_to_i915(dev)->dev; + struct drm_device *drm_dev = &dev_to_i915(dev)->drm; if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; @@ -1101,7 +1898,7 @@ static int i915_pm_poweroff_late(struct device *dev) static int i915_pm_resume_early(struct device *dev) { - struct drm_device *drm_dev = dev_to_i915(dev)->dev; + struct drm_device *drm_dev = &dev_to_i915(dev)->drm; if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; @@ -1111,7 +1908,7 @@ static int i915_pm_resume_early(struct device *dev) static int i915_pm_resume(struct device *dev) { - struct drm_device *drm_dev = dev_to_i915(dev)->dev; + struct drm_device *drm_dev = &dev_to_i915(dev)->drm; if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; @@ -1119,6 +1916,49 @@ static int i915_pm_resume(struct device *dev) return i915_drm_resume(drm_dev); } +/* freeze: before creating the hibernation_image */ +static int i915_pm_freeze(struct device *dev) +{ + return i915_pm_suspend(dev); +} + +static int i915_pm_freeze_late(struct device *dev) +{ + int ret; + + ret = i915_pm_suspend_late(dev); + if (ret) + return ret; + + ret = i915_gem_freeze_late(dev_to_i915(dev)); + if (ret) + return ret; + + return 0; +} + +/* thaw: called after creating the hibernation image, but before turning off. */ +static int i915_pm_thaw_early(struct device *dev) +{ + return i915_pm_resume_early(dev); +} + +static int i915_pm_thaw(struct device *dev) +{ + return i915_pm_resume(dev); +} + +/* restore: called after loading the hibernation image. */ +static int i915_pm_restore_early(struct device *dev) +{ + return i915_pm_resume_early(dev); +} + +static int i915_pm_restore(struct device *dev) +{ + return i915_pm_resume(dev); +} + /* * Save all Gunit registers that may be lost after a D3 and a subsequent * S0i[R123] transition. The list of registers needing a save/restore is @@ -1318,8 +2158,6 @@ int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) u32 val; int err; -#define COND (I915_READ(VLV_GTLC_SURVIVABILITY_REG) & VLV_GFX_CLK_STATUS_BIT) - val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); val &= ~VLV_GFX_CLK_FORCE_ON_BIT; if (force_on) @@ -1329,13 +2167,16 @@ int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) if (!force_on) return 0; - err = wait_for(COND, 20); + err = intel_wait_for_register(dev_priv, + VLV_GTLC_SURVIVABILITY_REG, + VLV_GFX_CLK_STATUS_BIT, + VLV_GFX_CLK_STATUS_BIT, + 20); if (err) DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n", I915_READ(VLV_GTLC_SURVIVABILITY_REG)); return err; -#undef COND } static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow) @@ -1350,13 +2191,15 @@ static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow) I915_WRITE(VLV_GTLC_WAKE_CTRL, val); POSTING_READ(VLV_GTLC_WAKE_CTRL); -#define COND (!!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEACK) == \ - allow) - err = wait_for(COND, 1); + err = intel_wait_for_register(dev_priv, + VLV_GTLC_PW_STATUS, + VLV_GTLC_ALLOWWAKEACK, + allow, + 1); if (err) DRM_ERROR("timeout disabling GT waking\n"); + return err; -#undef COND } static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, @@ -1368,8 +2211,7 @@ static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; val = wait_for_on ? mask : 0; -#define COND ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val) - if (COND) + if ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val) return 0; DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n", @@ -1380,13 +2222,14 @@ static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, * RC6 transitioning can be delayed up to 2 msec (see * valleyview_enable_rps), use 3 msec for safety. */ - err = wait_for(COND, 3); + err = intel_wait_for_register(dev_priv, + VLV_GTLC_PW_STATUS, mask, val, + 3); if (err) DRM_ERROR("timeout waiting for GT wells to go %s\n", onoff(wait_for_on)); return err; -#undef COND } static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv) @@ -1443,7 +2286,7 @@ err1: static int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; int err; int ret; @@ -1479,10 +2322,10 @@ static int intel_runtime_suspend(struct device *device) { struct pci_dev *pdev = to_pci_dev(device); struct drm_device *dev = pci_get_drvdata(pdev); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; - if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6(dev)))) + if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6()))) return -ENODEV; if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev))) @@ -1517,11 +2360,8 @@ static int intel_runtime_suspend(struct device *device) i915_gem_release_all_mmaps(dev_priv); mutex_unlock(&dev->struct_mutex); - cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); - intel_guc_suspend(dev); - intel_suspend_gt_powersave(dev); intel_runtime_pm_disable_interrupts(dev_priv); ret = 0; @@ -1543,7 +2383,7 @@ static int intel_runtime_suspend(struct device *device) return ret; } - intel_uncore_forcewake_reset(dev, false); + intel_uncore_forcewake_reset(dev_priv, false); enable_rpm_wakeref_asserts(dev_priv); WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count)); @@ -1557,14 +2397,14 @@ static int intel_runtime_suspend(struct device *device) * FIXME: We really should find a document that references the arguments * used below! */ - if (IS_BROADWELL(dev)) { + if (IS_BROADWELL(dev_priv)) { /* * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop * being detected, and the call we do at intel_runtime_resume() * won't be able to restore them. Since PCI_D3hot matches the * actual specification and appears to be working, use it. */ - intel_opregion_notify_adapter(dev, PCI_D3hot); + intel_opregion_notify_adapter(dev_priv, PCI_D3hot); } else { /* * current versions of firmware which depend on this opregion @@ -1573,7 +2413,7 @@ static int intel_runtime_suspend(struct device *device) * to distinguish it from notifications that might be sent via * the suspend path. */ - intel_opregion_notify_adapter(dev, PCI_D1); + intel_opregion_notify_adapter(dev_priv, PCI_D1); } assert_forcewakes_inactive(dev_priv); @@ -1589,7 +2429,7 @@ static int intel_runtime_resume(struct device *device) { struct pci_dev *pdev = to_pci_dev(device); struct drm_device *dev = pci_get_drvdata(pdev); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret = 0; if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev))) @@ -1600,7 +2440,7 @@ static int intel_runtime_resume(struct device *device) WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count)); disable_rpm_wakeref_asserts(dev_priv); - intel_opregion_notify_adapter(dev, PCI_D0); + intel_opregion_notify_adapter(dev_priv, PCI_D0); dev_priv->pm.suspended = false; if (intel_uncore_unclaimed_mmio(dev_priv)) DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n"); @@ -1627,7 +2467,7 @@ static int intel_runtime_resume(struct device *device) * we can do is to hope that things will still work (and disable RPM). */ i915_gem_init_swizzling(dev); - gen6_update_ring_freq(dev); + gen6_update_ring_freq(dev_priv); intel_runtime_pm_enable_interrupts(dev_priv); @@ -1639,8 +2479,6 @@ static int intel_runtime_resume(struct device *device) if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) intel_hpd_init(dev_priv); - intel_enable_gt_powersave(dev); - enable_rpm_wakeref_asserts(dev_priv); if (ret) @@ -1651,7 +2489,7 @@ static int intel_runtime_resume(struct device *device) return ret; } -static const struct dev_pm_ops i915_pm_ops = { +const struct dev_pm_ops i915_pm_ops = { /* * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND, * PMSG_RESUME] @@ -1676,14 +2514,14 @@ static const struct dev_pm_ops i915_pm_ops = { * @restore, @restore_early : called after rebooting and restoring the * hibernation image [PMSG_RESTORE] */ - .freeze = i915_pm_suspend, - .freeze_late = i915_pm_suspend_late, - .thaw_early = i915_pm_resume_early, - .thaw = i915_pm_resume, + .freeze = i915_pm_freeze, + .freeze_late = i915_pm_freeze_late, + .thaw_early = i915_pm_thaw_early, + .thaw = i915_pm_thaw, .poweroff = i915_pm_suspend, .poweroff_late = i915_pm_poweroff_late, - .restore_early = i915_pm_resume_early, - .restore = i915_pm_resume, + .restore_early = i915_pm_restore_early, + .restore = i915_pm_restore, /* S0ix (via runtime suspend) event handlers */ .runtime_suspend = intel_runtime_suspend, @@ -1710,6 +2548,68 @@ static const struct file_operations i915_driver_fops = { .llseek = noop_llseek, }; +static int +i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + return -ENODEV; +} + +static const struct drm_ioctl_desc i915_ioctls[] = { + DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE, drm_noop, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH), + DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY), + DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0), + DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW), + DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW), + DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW), +}; + static struct drm_driver driver = { /* Don't use MTRRs here; the Xserver or userspace app should * deal with them for Intel hardware. @@ -1717,18 +2617,12 @@ static struct drm_driver driver = { .driver_features = DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER | DRIVER_MODESET, - .load = i915_driver_load, - .unload = i915_driver_unload, .open = i915_driver_open, .lastclose = i915_driver_lastclose, .preclose = i915_driver_preclose, .postclose = i915_driver_postclose, .set_busid = drm_pci_set_busid, -#if defined(CONFIG_DEBUG_FS) - .debugfs_init = i915_debugfs_init, - .debugfs_cleanup = i915_debugfs_cleanup, -#endif .gem_free_object = i915_gem_free_object, .gem_vm_ops = &i915_gem_vm_ops, @@ -1741,6 +2635,7 @@ static struct drm_driver driver = { .dumb_map_offset = i915_gem_mmap_gtt, .dumb_destroy = drm_gem_dumb_destroy, .ioctls = i915_ioctls, + .num_ioctls = ARRAY_SIZE(i915_ioctls), .fops = &i915_driver_fops, .name = DRIVER_NAME, .desc = DRIVER_DESC, @@ -1749,56 +2644,3 @@ static struct drm_driver driver = { .minor = DRIVER_MINOR, .patchlevel = DRIVER_PATCHLEVEL, }; - -static struct pci_driver i915_pci_driver = { - .name = DRIVER_NAME, - .id_table = pciidlist, - .probe = i915_pci_probe, - .remove = i915_pci_remove, - .driver.pm = &i915_pm_ops, -}; - -static int __init i915_init(void) -{ - driver.num_ioctls = i915_max_ioctl; - - /* - * Enable KMS by default, unless explicitly overriden by - * either the i915.modeset prarameter or by the - * vga_text_mode_force boot option. - */ - - if (i915.modeset == 0) - driver.driver_features &= ~DRIVER_MODESET; - - if (vgacon_text_force() && i915.modeset == -1) - driver.driver_features &= ~DRIVER_MODESET; - - if (!(driver.driver_features & DRIVER_MODESET)) { - /* Silently fail loading to not upset userspace. */ - DRM_DEBUG_DRIVER("KMS and UMS disabled.\n"); - return 0; - } - - if (i915.nuclear_pageflip) - driver.driver_features |= DRIVER_ATOMIC; - - return drm_pci_init(&driver, &i915_pci_driver); -} - -static void __exit i915_exit(void) -{ - if (!(driver.driver_features & DRIVER_MODESET)) - return; /* Never loaded a driver. */ - - drm_pci_exit(&driver, &i915_pci_driver); -} - -module_init(i915_init); -module_exit(i915_exit); - -MODULE_AUTHOR("Tungsten Graphics, Inc."); -MODULE_AUTHOR("Intel Corporation"); - -MODULE_DESCRIPTION(DRIVER_DESC); -MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 74cb400c8..f68c78918 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -47,6 +47,7 @@ #include <drm/intel-gtt.h> #include <drm/drm_legacy.h> /* for struct drm_dma_handle */ #include <drm/drm_gem.h> +#include <drm/drm_auth.h> #include "i915_params.h" #include "i915_reg.h" @@ -61,12 +62,14 @@ #include "i915_gem_gtt.h" #include "i915_gem_render_state.h" +#include "intel_gvt.h" + /* General customization: */ #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20160425" +#define DRIVER_DATE "20160711" #undef WARN_ON /* Many gcc seem to no see through this and fall over :( */ @@ -320,21 +323,36 @@ struct i915_hotplug { for_each_if ((__ports_mask) & (1 << (__port))) #define for_each_crtc(dev, crtc) \ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) + list_for_each_entry(crtc, &(dev)->mode_config.crtc_list, head) #define for_each_intel_plane(dev, intel_plane) \ list_for_each_entry(intel_plane, \ - &dev->mode_config.plane_list, \ + &(dev)->mode_config.plane_list, \ base.head) +#define for_each_intel_plane_mask(dev, intel_plane, plane_mask) \ + list_for_each_entry(intel_plane, \ + &(dev)->mode_config.plane_list, \ + base.head) \ + for_each_if ((plane_mask) & \ + (1 << drm_plane_index(&intel_plane->base))) + #define for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) \ list_for_each_entry(intel_plane, \ &(dev)->mode_config.plane_list, \ base.head) \ for_each_if ((intel_plane)->pipe == (intel_crtc)->pipe) -#define for_each_intel_crtc(dev, intel_crtc) \ - list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) +#define for_each_intel_crtc(dev, intel_crtc) \ + list_for_each_entry(intel_crtc, \ + &(dev)->mode_config.crtc_list, \ + base.head) + +#define for_each_intel_crtc_mask(dev, intel_crtc, crtc_mask) \ + list_for_each_entry(intel_crtc, \ + &(dev)->mode_config.crtc_list, \ + base.head) \ + for_each_if ((crtc_mask) & (1 << drm_crtc_index(&intel_crtc->base))) #define for_each_intel_encoder(dev, intel_encoder) \ list_for_each_entry(intel_encoder, \ @@ -343,7 +361,7 @@ struct i915_hotplug { #define for_each_intel_connector(dev, intel_connector) \ list_for_each_entry(intel_connector, \ - &dev->mode_config.connector_list, \ + &(dev)->mode_config.connector_list, \ base.head) #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \ @@ -465,6 +483,7 @@ struct drm_i915_error_state { struct timeval time; char error_msg[128]; + bool simulated; int iommu; u32 reset_count; u32 suspend_count; @@ -496,6 +515,7 @@ struct drm_i915_error_state { bool valid; /* Software tracked state */ bool waiting; + int num_waiters; int hangcheck_score; enum intel_ring_hangcheck_action hangcheck_action; int num_requests; @@ -541,6 +561,12 @@ struct drm_i915_error_state { u32 tail; } *requests; + struct drm_i915_error_waiter { + char comm[TASK_COMM_LEN]; + pid_t pid; + u32 seqno; + } *waiters; + struct { u32 gfx_mode; union { @@ -591,6 +617,7 @@ struct drm_i915_display_funcs { struct intel_crtc_state *newstate); void (*initial_watermarks)(struct intel_crtc_state *cstate); void (*optimize_watermarks)(struct intel_crtc_state *cstate); + int (*compute_global_watermarks)(struct drm_atomic_state *state); void (*update_wm)(struct drm_crtc *crtc); int (*modeset_calc_cdclk)(struct drm_atomic_state *state); void (*modeset_commit_cdclk)(struct drm_atomic_state *state); @@ -615,7 +642,7 @@ struct drm_i915_display_funcs { struct drm_i915_gem_object *obj, struct drm_i915_gem_request *req, uint32_t flags); - void (*hpd_irq_setup)(struct drm_device *dev); + void (*hpd_irq_setup)(struct drm_i915_private *dev_priv); /* clock updates for mode set */ /* cursor updates */ /* render clock increase/decrease */ @@ -738,6 +765,7 @@ struct intel_csr { func(is_valleyview) sep \ func(is_cherryview) sep \ func(is_haswell) sep \ + func(is_broadwell) sep \ func(is_skylake) sep \ func(is_broxton) sep \ func(is_kabylake) sep \ @@ -752,7 +780,8 @@ struct intel_csr { func(has_llc) sep \ func(has_snoop) sep \ func(has_ddi) sep \ - func(has_fpga_dbg) + func(has_fpga_dbg) sep \ + func(has_pooled_eu) #define DEFINE_FLAG(name) u8 name:1 #define SEP_SEMICOLON ; @@ -760,9 +789,10 @@ struct intel_csr { struct intel_device_info { u32 display_mmio_offset; u16 device_id; - u8 num_pipes:3; + u8 num_pipes; u8 num_sprites[I915_MAX_PIPES]; u8 gen; + u16 gen_mask; u8 ring_mask; /* Rings supported by the HW */ DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG, SEP_SEMICOLON); /* Register offsets for the various display pipes and transcoders */ @@ -777,6 +807,7 @@ struct intel_device_info { u8 subslice_per_slice; u8 eu_total; u8 eu_per_subslice; + u8 min_eu_in_pool; /* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */ u8 subslice_7eu[3]; u8 has_slice_pg:1; @@ -824,9 +855,8 @@ struct i915_ctx_hang_stats { /* This must match up with the value previously used for execbuf2.rsvd1. */ #define DEFAULT_CONTEXT_HANDLE 0 -#define CONTEXT_NO_ZEROMAP (1<<0) /** - * struct intel_context - as the name implies, represents a context. + * struct i915_gem_context - as the name implies, represents a context. * @ref: reference count. * @user_handle: userspace tracking identity for this context. * @remap_slice: l3 row remapping information. @@ -844,33 +874,41 @@ struct i915_ctx_hang_stats { * Contexts are memory images used by the hardware to store copies of their * internal state. */ -struct intel_context { +struct i915_gem_context { struct kref ref; - int user_handle; - uint8_t remap_slice; struct drm_i915_private *i915; - int flags; struct drm_i915_file_private *file_priv; - struct i915_ctx_hang_stats hang_stats; struct i915_hw_ppgtt *ppgtt; - /* Legacy ring buffer submission */ - struct { - struct drm_i915_gem_object *rcs_state; - bool initialized; - } legacy_hw_ctx; + struct i915_ctx_hang_stats hang_stats; - /* Execlists */ - struct { + unsigned long flags; +#define CONTEXT_NO_ZEROMAP BIT(0) +#define CONTEXT_NO_ERROR_CAPTURE BIT(1) + + /* Unique identifier for this context, used by the hw for tracking */ + unsigned int hw_id; + u32 user_handle; + + u32 ggtt_alignment; + + struct intel_context { struct drm_i915_gem_object *state; struct intel_ringbuffer *ringbuf; - int pin_count; struct i915_vma *lrc_vma; - u64 lrc_desc; uint32_t *lrc_reg_state; + u64 lrc_desc; + int pin_count; + bool initialised; } engine[I915_NUM_ENGINES]; + u32 ring_size; + u32 desc_template; + struct atomic_notifier_head status_notifier; + bool execlists_force_single_submission; struct list_head link; + + u8 remap_slice; }; enum fb_op_origin { @@ -1119,6 +1157,8 @@ struct intel_gen6_power_mgmt { bool interrupts_enabled; u32 pm_iir; + u32 pm_intr_keep; + /* Frequencies are stored in potentially platform dependent multiples. * In other words, *_freq needs to be multiplied by X to be interesting. * Soft limits are those which are used for the dynamic reclocking done @@ -1286,37 +1326,11 @@ struct i915_gem_mm { struct list_head fence_list; /** - * We leave the user IRQ off as much as possible, - * but this means that requests will finish and never - * be retired once the system goes idle. Set a timer to - * fire periodically while the ring is running. When it - * fires, go retire requests. - */ - struct delayed_work retire_work; - - /** - * When we detect an idle GPU, we want to turn on - * powersaving features. So once we see that there - * are no more requests outstanding and no more - * arrive within a small period of time, we fire - * off the idle_work. - */ - struct delayed_work idle_work; - - /** * Are we in a non-interruptible section of code like * modesetting? */ bool interruptible; - /** - * Is the GPU currently considered idle, or busy executing userspace - * requests? Whilst idle, we attempt to power down the hardware and - * display clocks. In order to reduce the effect on performance, there - * is a slight delay before we do so. - */ - bool busy; - /* the indicator for dispatch video commands on two BSD rings */ unsigned int bsd_ring_dispatch_index; @@ -1353,7 +1367,6 @@ struct i915_gpu_error { /* Hang gpu twice in this window and your context gets banned */ #define DRM_I915_CTX_BAN_PERIOD DIV_ROUND_UP(8*DRM_I915_HANGCHECK_PERIOD, 1000) - struct workqueue_struct *hangcheck_wq; struct delayed_work hangcheck_work; /* For reset and error_state handling. */ @@ -1390,20 +1403,19 @@ struct i915_gpu_error { #define I915_WEDGED (1 << 31) /** + * Waitqueue to signal when a hang is detected. Used to for waiters + * to release the struct_mutex for the reset to procede. + */ + wait_queue_head_t wait_queue; + + /** * Waitqueue to signal when the reset has completed. Used by clients * that wait for dev_priv->mm.wedged to settle. */ wait_queue_head_t reset_queue; - /* Userspace knobs for gpu hang simulation; - * combines both a ring mask, and extra flags - */ - u32 stop_rings; -#define I915_STOP_RING_ALLOW_BAN (1 << 31) -#define I915_STOP_RING_ALLOW_WARN (1 << 30) - /* For missed irq/seqno simulation. */ - unsigned int test_irq_rings; + unsigned long test_irq_rings; }; enum modeset_restore { @@ -1492,6 +1504,7 @@ struct intel_vbt_data { bool present; bool active_low_pwm; u8 min_brightness; /* min_brightness/255 of max */ + enum intel_backlight_type type; } backlight; /* MIPI DSI */ @@ -1584,7 +1597,7 @@ struct skl_ddb_allocation { }; struct skl_wm_values { - bool dirty[I915_MAX_PIPES]; + unsigned dirty_pipes; struct skl_ddb_allocation ddb; uint32_t wm_linetime[I915_MAX_PIPES]; uint32_t plane[I915_MAX_PIPES][I915_MAX_PLANES][8]; @@ -1701,7 +1714,7 @@ struct i915_execbuffer_params { uint64_t batch_obj_vm_offset; struct intel_engine_cs *engine; struct drm_i915_gem_object *batch_obj; - struct intel_context *ctx; + struct i915_gem_context *ctx; struct drm_i915_gem_request *request; }; @@ -1713,7 +1726,8 @@ struct intel_wm_config { }; struct drm_i915_private { - struct drm_device *dev; + struct drm_device drm; + struct kmem_cache *objects; struct kmem_cache *vmas; struct kmem_cache *requests; @@ -1728,6 +1742,8 @@ struct drm_i915_private { struct i915_virtual_gpu vgpu; + struct intel_gvt gvt; + struct intel_guc guc; struct intel_csr csr; @@ -1751,6 +1767,7 @@ struct drm_i915_private { wait_queue_head_t gmbus_wait_queue; struct pci_dev *bridge_dev; + struct i915_gem_context *kernel_context; struct intel_engine_cs engine[I915_NUM_ENGINES]; struct drm_i915_gem_object *semaphore_obj; uint32_t last_seqno, next_seqno; @@ -1806,13 +1823,17 @@ struct drm_i915_private { int num_fence_regs; /* 8 on pre-965, 16 otherwise */ unsigned int fsb_freq, mem_freq, is_ddr3; - unsigned int skl_boot_cdclk; + unsigned int skl_preferred_vco_freq; unsigned int cdclk_freq, max_cdclk_freq, atomic_cdclk_freq; unsigned int max_dotclk_freq; unsigned int rawclk_freq; unsigned int hpll_freq; unsigned int czclk_freq; + struct { + unsigned int vco, ref; + } cdclk_pll; + /** * wq - Driver workqueue for GEM. * @@ -1834,6 +1855,7 @@ struct drm_i915_private { enum modeset_restore modeset_restore; struct mutex modeset_restore_lock; struct drm_atomic_state *modeset_restore_state; + struct drm_modeset_acquire_ctx reset_ctx; struct list_head vm_list; /* Global list of all address spaces */ struct i915_ggtt ggtt; /* VM representing the global address space */ @@ -1842,6 +1864,13 @@ struct drm_i915_private { DECLARE_HASHTABLE(mm_structs, 7); struct mutex mm_lock; + /* The hw wants to have a stable context identifier for the lifetime + * of the context (for OA, PASID, faults, etc). This is limited + * in execlists to 21 bits. + */ + struct ida context_hw_ida; +#define MAX_CONTEXT_HW_ID (1<<21) /* exclusive */ + /* Kernel Modesetting */ struct drm_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; @@ -1935,7 +1964,12 @@ struct drm_i915_private { struct i915_suspend_saved_registers regfile; struct vlv_s0ix_state vlv_s0ix_state; - bool skl_sagv_enabled; + enum { + I915_SKL_SAGV_UNKNOWN = 0, + I915_SKL_SAGV_DISABLED, + I915_SKL_SAGV_ENABLED, + I915_SKL_SAGV_NOT_CONTROLLED + } skl_sagv_status; struct { /* @@ -1956,9 +1990,6 @@ struct drm_i915_private { */ uint16_t skl_latency[8]; - /* Committed wm config */ - struct intel_wm_config config; - /* * The skl_wm_values structure is a bit too big for stack * allocation, so we keep the staging struct where we store @@ -1981,6 +2012,13 @@ struct drm_i915_private { * cstate->wm.need_postvbl_update. */ struct mutex wm_mutex; + + /* + * Set during HW readout of watermarks/DDB. Some platforms + * need to know when we're still using BIOS-provided values + * (which we don't fully trust). + */ + bool distrust_bios_wm; } wm; struct i915_runtime_pm pm; @@ -1993,9 +2031,35 @@ struct drm_i915_private { int (*init_engines)(struct drm_device *dev); void (*cleanup_engine)(struct intel_engine_cs *engine); void (*stop_engine)(struct intel_engine_cs *engine); - } gt; - struct intel_context *kernel_context; + /** + * Is the GPU currently considered idle, or busy executing + * userspace requests? Whilst idle, we allow runtime power + * management to power down the hardware and display clocks. + * In order to reduce the effect on performance, there + * is a slight delay before we do so. + */ + unsigned int active_engines; + bool awake; + + /** + * We leave the user IRQ off as much as possible, + * but this means that requests will finish and never + * be retired once the system goes idle. Set a timer to + * fire periodically while the ring is running. When it + * fires, go retire requests. + */ + struct delayed_work retire_work; + + /** + * When we detect an idle GPU, we want to turn on + * powersaving features. So once we see that there + * are no more requests outstanding and no more + * arrive within a small period of time, we fire + * off the idle_work. + */ + struct delayed_work idle_work; + } gt; /* perform PHY state sanity checks? */ bool chv_phy_assert[2]; @@ -2010,7 +2074,7 @@ struct drm_i915_private { static inline struct drm_i915_private *to_i915(const struct drm_device *dev) { - return dev->dev_private; + return container_of(dev, struct drm_i915_private, drm); } static inline struct drm_i915_private *dev_to_i915(struct device *dev) @@ -2181,6 +2245,7 @@ struct drm_i915_gem_object { unsigned int frontbuffer_bits:INTEL_FRONTBUFFER_BITS; + unsigned int has_wc_mmap; unsigned int pin_display; struct sg_table *pages; @@ -2233,9 +2298,81 @@ struct drm_i915_gem_object { }; #define to_intel_bo(x) container_of(x, struct drm_i915_gem_object, base) -void i915_gem_track_fb(struct drm_i915_gem_object *old, - struct drm_i915_gem_object *new, - unsigned frontbuffer_bits); +static inline bool +i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) +{ + return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE; +} + +/* + * Optimised SGL iterator for GEM objects + */ +static __always_inline struct sgt_iter { + struct scatterlist *sgp; + union { + unsigned long pfn; + dma_addr_t dma; + }; + unsigned int curr; + unsigned int max; +} __sgt_iter(struct scatterlist *sgl, bool dma) { + struct sgt_iter s = { .sgp = sgl }; + + if (s.sgp) { + s.max = s.curr = s.sgp->offset; + s.max += s.sgp->length; + if (dma) + s.dma = sg_dma_address(s.sgp); + else + s.pfn = page_to_pfn(sg_page(s.sgp)); + } + + return s; +} + +/** + * __sg_next - return the next scatterlist entry in a list + * @sg: The current sg entry + * + * Description: + * If the entry is the last, return NULL; otherwise, step to the next + * element in the array (@sg@+1). If that's a chain pointer, follow it; + * otherwise just return the pointer to the current element. + **/ +static inline struct scatterlist *__sg_next(struct scatterlist *sg) +{ +#ifdef CONFIG_DEBUG_SG + BUG_ON(sg->sg_magic != SG_MAGIC); +#endif + return sg_is_last(sg) ? NULL : + likely(!sg_is_chain(++sg)) ? sg : + sg_chain_ptr(sg); +} + +/** + * for_each_sgt_dma - iterate over the DMA addresses of the given sg_table + * @__dmap: DMA address (output) + * @__iter: 'struct sgt_iter' (iterator state, internal) + * @__sgt: sg_table to iterate over (input) + */ +#define for_each_sgt_dma(__dmap, __iter, __sgt) \ + for ((__iter) = __sgt_iter((__sgt)->sgl, true); \ + ((__dmap) = (__iter).dma + (__iter).curr); \ + (((__iter).curr += PAGE_SIZE) < (__iter).max) || \ + ((__iter) = __sgt_iter(__sg_next((__iter).sgp), true), 0)) + +/** + * for_each_sgt_page - iterate over the pages of the given sg_table + * @__pp: page pointer (output) + * @__iter: 'struct sgt_iter' (iterator state, internal) + * @__sgt: sg_table to iterate over (input) + */ +#define for_each_sgt_page(__pp, __iter, __sgt) \ + for ((__iter) = __sgt_iter((__sgt)->sgl, false); \ + ((__pp) = (__iter).pfn == 0 ? NULL : \ + pfn_to_page((__iter).pfn + ((__iter).curr >> PAGE_SHIFT))); \ + (((__iter).curr += PAGE_SIZE) < (__iter).max) || \ + ((__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0)) /** * Request queue structure. @@ -2257,7 +2394,7 @@ struct drm_i915_gem_request { /** On Which ring this request was generated */ struct drm_i915_private *i915; struct intel_engine_cs *engine; - unsigned reset_counter; + struct intel_signal_node signaling; /** GEM sequence number associated with the previous request, * when the HWS breadcrumb is equal to this the GPU is processing @@ -2284,6 +2421,9 @@ struct drm_i915_gem_request { /** Position in the ringbuffer of the end of the whole request */ u32 tail; + /** Preallocate space in the ringbuffer for the emitting the request */ + u32 reserved_space; + /** * Context and ring buffer related to this request * Contexts are refcounted, so when this request is associated with a @@ -2294,9 +2434,20 @@ struct drm_i915_gem_request { * i915_gem_request_free() will then decrement the refcount on the * context. */ - struct intel_context *ctx; + struct i915_gem_context *ctx; struct intel_ringbuffer *ringbuf; + /** + * Context related to the previous request. + * As the contexts are accessed by the hardware until the switch is + * completed to a new context, the hardware may still be writing + * to the context object after the breadcrumb is visible. We must + * not unpin/unbind/prune that object whilst still active and so + * we keep the previous context pinned until the following (this) + * request is retired. + */ + struct i915_gem_context *previous_context; + /** Batch buffer related to this request if any (used for error state dump only) */ struct drm_i915_gem_object *batch_obj; @@ -2333,11 +2484,13 @@ struct drm_i915_gem_request { /** Execlists no. of times this request has been sent to the ELSP */ int elsp_submitted; + /** Execlists context hardware id. */ + unsigned ctx_hw_id; }; struct drm_i915_gem_request * __must_check i915_gem_request_alloc(struct intel_engine_cs *engine, - struct intel_context *ctx); + struct i915_gem_context *ctx); void i915_gem_request_free(struct kref *req_ref); int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, struct drm_file *file); @@ -2365,23 +2518,9 @@ i915_gem_request_reference(struct drm_i915_gem_request *req) static inline void i915_gem_request_unreference(struct drm_i915_gem_request *req) { - WARN_ON(!mutex_is_locked(&req->engine->dev->struct_mutex)); kref_put(&req->ref, i915_gem_request_free); } -static inline void -i915_gem_request_unreference__unlocked(struct drm_i915_gem_request *req) -{ - struct drm_device *dev; - - if (!req) - return; - - dev = req->engine->dev; - if (kref_put_mutex(&req->ref, i915_gem_request_free, &dev->struct_mutex)) - mutex_unlock(&dev->struct_mutex); -} - static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst, struct drm_i915_gem_request *src) { @@ -2509,9 +2648,29 @@ struct drm_i915_cmd_table { #define INTEL_INFO(p) (&__I915__(p)->info) #define INTEL_GEN(p) (INTEL_INFO(p)->gen) #define INTEL_DEVID(p) (INTEL_INFO(p)->device_id) -#define INTEL_REVID(p) (__I915__(p)->dev->pdev->revision) #define REVID_FOREVER 0xff +#define INTEL_REVID(p) (__I915__(p)->drm.pdev->revision) + +#define GEN_FOREVER (0) +/* + * Returns true if Gen is in inclusive range [Start, End]. + * + * Use GEN_FOREVER for unbound start and or end. + */ +#define IS_GEN(p, s, e) ({ \ + unsigned int __s = (s), __e = (e); \ + BUILD_BUG_ON(!__builtin_constant_p(s)); \ + BUILD_BUG_ON(!__builtin_constant_p(e)); \ + if ((__s) != GEN_FOREVER) \ + __s = (s) - 1; \ + if ((__e) == GEN_FOREVER) \ + __e = BITS_PER_LONG - 1; \ + else \ + __e = (e) - 1; \ + !!(INTEL_INFO(p)->gen_mask & GENMASK((__e), (__s))); \ +}) + /* * Return true if revision is in range [since,until] inclusive. * @@ -2544,7 +2703,7 @@ struct drm_i915_cmd_table { #define IS_VALLEYVIEW(dev) (INTEL_INFO(dev)->is_valleyview) #define IS_CHERRYVIEW(dev) (INTEL_INFO(dev)->is_cherryview) #define IS_HASWELL(dev) (INTEL_INFO(dev)->is_haswell) -#define IS_BROADWELL(dev) (!INTEL_INFO(dev)->is_cherryview && IS_GEN8(dev)) +#define IS_BROADWELL(dev) (INTEL_INFO(dev)->is_broadwell) #define IS_SKYLAKE(dev) (INTEL_INFO(dev)->is_skylake) #define IS_BROXTON(dev) (INTEL_INFO(dev)->is_broxton) #define IS_KABYLAKE(dev) (INTEL_INFO(dev)->is_kabylake) @@ -2623,29 +2782,34 @@ struct drm_i915_cmd_table { * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular * chips, etc.). */ -#define IS_GEN2(dev) (INTEL_INFO(dev)->gen == 2) -#define IS_GEN3(dev) (INTEL_INFO(dev)->gen == 3) -#define IS_GEN4(dev) (INTEL_INFO(dev)->gen == 4) -#define IS_GEN5(dev) (INTEL_INFO(dev)->gen == 5) -#define IS_GEN6(dev) (INTEL_INFO(dev)->gen == 6) -#define IS_GEN7(dev) (INTEL_INFO(dev)->gen == 7) -#define IS_GEN8(dev) (INTEL_INFO(dev)->gen == 8) -#define IS_GEN9(dev) (INTEL_INFO(dev)->gen == 9) - -#define RENDER_RING (1<<RCS) -#define BSD_RING (1<<VCS) -#define BLT_RING (1<<BCS) -#define VEBOX_RING (1<<VECS) -#define BSD2_RING (1<<VCS2) -#define ALL_ENGINES (~0) - -#define HAS_BSD(dev) (INTEL_INFO(dev)->ring_mask & BSD_RING) -#define HAS_BSD2(dev) (INTEL_INFO(dev)->ring_mask & BSD2_RING) -#define HAS_BLT(dev) (INTEL_INFO(dev)->ring_mask & BLT_RING) -#define HAS_VEBOX(dev) (INTEL_INFO(dev)->ring_mask & VEBOX_RING) +#define IS_GEN2(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(1))) +#define IS_GEN3(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(2))) +#define IS_GEN4(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(3))) +#define IS_GEN5(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(4))) +#define IS_GEN6(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(5))) +#define IS_GEN7(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(6))) +#define IS_GEN8(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(7))) +#define IS_GEN9(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(8))) + +#define ENGINE_MASK(id) BIT(id) +#define RENDER_RING ENGINE_MASK(RCS) +#define BSD_RING ENGINE_MASK(VCS) +#define BLT_RING ENGINE_MASK(BCS) +#define VEBOX_RING ENGINE_MASK(VECS) +#define BSD2_RING ENGINE_MASK(VCS2) +#define ALL_ENGINES (~0) + +#define HAS_ENGINE(dev_priv, id) \ + (!!(INTEL_INFO(dev_priv)->ring_mask & ENGINE_MASK(id))) + +#define HAS_BSD(dev_priv) HAS_ENGINE(dev_priv, VCS) +#define HAS_BSD2(dev_priv) HAS_ENGINE(dev_priv, VCS2) +#define HAS_BLT(dev_priv) HAS_ENGINE(dev_priv, BCS) +#define HAS_VEBOX(dev_priv) HAS_ENGINE(dev_priv, VECS) + #define HAS_LLC(dev) (INTEL_INFO(dev)->has_llc) #define HAS_SNOOP(dev) (INTEL_INFO(dev)->has_snoop) -#define HAS_EDRAM(dev) (__I915__(dev)->edram_cap & EDRAM_ENABLED) +#define HAS_EDRAM(dev) (!!(__I915__(dev)->edram_cap & EDRAM_ENABLED)) #define HAS_WT(dev) ((IS_HASWELL(dev) || IS_BROADWELL(dev)) && \ HAS_EDRAM(dev)) #define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws) @@ -2663,9 +2827,10 @@ struct drm_i915_cmd_table { #define HAS_BROKEN_CS_TLB(dev) (IS_I830(dev) || IS_845G(dev)) /* WaRsDisableCoarsePowerGating:skl,bxt */ -#define NEEDS_WaRsDisableCoarsePowerGating(dev) (IS_BXT_REVID(dev, 0, BXT_REVID_A1) || \ - IS_SKL_GT3(dev) || \ - IS_SKL_GT4(dev)) +#define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ + (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1) || \ + IS_SKL_GT3(dev_priv) || \ + IS_SKL_GT4(dev_priv)) /* * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts @@ -2703,12 +2868,18 @@ struct drm_i915_cmd_table { IS_CHERRYVIEW(dev) || IS_SKYLAKE(dev) || \ IS_KABYLAKE(dev) || IS_BROXTON(dev)) #define HAS_RC6(dev) (INTEL_INFO(dev)->gen >= 6) -#define HAS_RC6p(dev) (INTEL_INFO(dev)->gen == 6 || IS_IVYBRIDGE(dev)) +#define HAS_RC6p(dev) (IS_GEN6(dev) || IS_IVYBRIDGE(dev)) #define HAS_CSR(dev) (IS_GEN9(dev)) -#define HAS_GUC_UCODE(dev) (IS_GEN9(dev) && !IS_KABYLAKE(dev)) -#define HAS_GUC_SCHED(dev) (IS_GEN9(dev) && !IS_KABYLAKE(dev)) +/* + * For now, anything with a GuC requires uCode loading, and then supports + * command submission once loaded. But these are logically independent + * properties, so we have separate macros to test them. + */ +#define HAS_GUC(dev) (IS_GEN9(dev)) +#define HAS_GUC_UCODE(dev) (HAS_GUC(dev)) +#define HAS_GUC_SCHED(dev) (HAS_GUC(dev)) #define HAS_RESOURCE_STREAMER(dev) (IS_HASWELL(dev) || \ INTEL_INFO(dev)->gen >= 8) @@ -2717,6 +2888,8 @@ struct drm_i915_cmd_table { !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && \ !IS_BROXTON(dev)) +#define HAS_POOLED_EU(dev) (INTEL_INFO(dev)->has_pooled_eu) + #define INTEL_PCH_DEVICE_ID_MASK 0xff00 #define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 #define INTEL_PCH_CPT_DEVICE_ID_TYPE 0x1c00 @@ -2753,13 +2926,22 @@ struct drm_i915_cmd_table { #include "i915_trace.h" -extern const struct drm_ioctl_desc i915_ioctls[]; -extern int i915_max_ioctl; +static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) +{ +#ifdef CONFIG_INTEL_IOMMU + if (INTEL_GEN(dev_priv) >= 6 && intel_iommu_gfx_mapped) + return true; +#endif + return false; +} extern int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state); extern int i915_resume_switcheroo(struct drm_device *dev); -/* i915_dma.c */ +int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv, + int enable_ppgtt); + +/* i915_drv.c */ void __printf(3, 4) __i915_printk(struct drm_i915_private *dev_priv, const char *level, const char *fmt, ...); @@ -2767,21 +2949,13 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level, #define i915_report_error(dev_priv, fmt, ...) \ __i915_printk(dev_priv, KERN_ERR, fmt, ##__VA_ARGS__) -extern int i915_driver_load(struct drm_device *, unsigned long flags); -extern int i915_driver_unload(struct drm_device *); -extern int i915_driver_open(struct drm_device *dev, struct drm_file *file); -extern void i915_driver_lastclose(struct drm_device * dev); -extern void i915_driver_preclose(struct drm_device *dev, - struct drm_file *file); -extern void i915_driver_postclose(struct drm_device *dev, - struct drm_file *file); #ifdef CONFIG_COMPAT extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); #endif -extern int intel_gpu_reset(struct drm_device *dev, u32 engine_mask); -extern bool intel_has_gpu_reset(struct drm_device *dev); -extern int i915_reset(struct drm_device *dev); +extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask); +extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv); +extern int i915_reset(struct drm_i915_private *dev_priv); extern int intel_guc_reset(struct drm_i915_private *dev_priv); extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine); extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); @@ -2791,7 +2965,8 @@ extern void i915_update_gfx_val(struct drm_i915_private *dev_priv); int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on); /* intel_hotplug.c */ -void intel_hpd_irq_handler(struct drm_device *dev, u32 pin_mask, u32 long_mask); +void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, + u32 pin_mask, u32 long_mask); void intel_hpd_init(struct drm_i915_private *dev_priv); void intel_hpd_init_work(struct drm_i915_private *dev_priv); void intel_hpd_cancel_work(struct drm_i915_private *dev_priv); @@ -2800,23 +2975,41 @@ bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin); void intel_hpd_enable(struct drm_i915_private *dev_priv, enum hpd_pin pin); /* i915_irq.c */ -void i915_queue_hangcheck(struct drm_device *dev); +static inline void i915_queue_hangcheck(struct drm_i915_private *dev_priv) +{ + unsigned long delay; + + if (unlikely(!i915.enable_hangcheck)) + return; + + /* Don't continually defer the hangcheck so that it is always run at + * least once after work has been scheduled on any ring. Otherwise, + * we will ignore a hung ring if a second ring is kept busy. + */ + + delay = round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES); + queue_delayed_work(system_long_wq, + &dev_priv->gpu_error.hangcheck_work, delay); +} + __printf(3, 4) -void i915_handle_error(struct drm_device *dev, u32 engine_mask, +void i915_handle_error(struct drm_i915_private *dev_priv, + u32 engine_mask, const char *fmt, ...); extern void intel_irq_init(struct drm_i915_private *dev_priv); int intel_irq_install(struct drm_i915_private *dev_priv); void intel_irq_uninstall(struct drm_i915_private *dev_priv); -extern void intel_uncore_sanitize(struct drm_device *dev); -extern void intel_uncore_early_sanitize(struct drm_device *dev, +extern void intel_uncore_sanitize(struct drm_i915_private *dev_priv); +extern void intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, bool restore_forcewake); -extern void intel_uncore_init(struct drm_device *dev); +extern void intel_uncore_init(struct drm_i915_private *dev_priv); extern bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv); extern bool intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv); -extern void intel_uncore_fini(struct drm_device *dev); -extern void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore); +extern void intel_uncore_fini(struct drm_i915_private *dev_priv); +extern void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv, + bool restore); const char *intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id); void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv, enum forcewake_domains domains); @@ -2832,9 +3025,26 @@ void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv, u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv); void assert_forcewakes_inactive(struct drm_i915_private *dev_priv); -static inline bool intel_vgpu_active(struct drm_device *dev) + +int intel_wait_for_register(struct drm_i915_private *dev_priv, + i915_reg_t reg, + const u32 mask, + const u32 value, + const unsigned long timeout_ms); +int intel_wait_for_register_fw(struct drm_i915_private *dev_priv, + i915_reg_t reg, + const u32 mask, + const u32 value, + const unsigned long timeout_ms); + +static inline bool intel_gvt_active(struct drm_i915_private *dev_priv) +{ + return dev_priv->gvt.initialized; +} + +static inline bool intel_vgpu_active(struct drm_i915_private *dev_priv) { - return to_i915(dev)->vgpu.active; + return dev_priv->vgpu.active; } void @@ -2891,7 +3101,6 @@ ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits) ibx_display_interrupt_update(dev_priv, bits, 0); } - /* i915_gem.c */ int i915_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); @@ -2930,7 +3139,7 @@ int i915_gem_set_tiling(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_get_tiling(struct drm_device *dev, void *data, struct drm_file *file_priv); -int i915_gem_init_userptr(struct drm_device *dev); +void i915_gem_init_userptr(struct drm_i915_private *dev_priv); int i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, @@ -2940,11 +3149,13 @@ int i915_gem_wait_ioctl(struct drm_device *dev, void *data, void i915_gem_load_init(struct drm_device *dev); void i915_gem_load_cleanup(struct drm_device *dev); void i915_gem_load_init_fences(struct drm_i915_private *dev_priv); +int i915_gem_freeze_late(struct drm_i915_private *dev_priv); + void *i915_gem_object_alloc(struct drm_device *dev); void i915_gem_object_free(struct drm_i915_gem_object *obj); void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops); -struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, +struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev, size_t size); struct drm_i915_gem_object *i915_gem_object_create_from_data( struct drm_device *dev, const void *data, size_t size); @@ -2999,6 +3210,23 @@ static inline int __sg_page_count(struct scatterlist *sg) struct page * i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n); +static inline dma_addr_t +i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n) +{ + if (n < obj->get_page.last) { + obj->get_page.sg = obj->pages->sgl; + obj->get_page.last = 0; + } + + while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) { + obj->get_page.last += __sg_page_count(obj->get_page.sg++); + if (unlikely(sg_is_chain(obj->get_page.sg))) + obj->get_page.sg = sg_chain_ptr(obj->get_page.sg); + } + + return sg_dma_address(obj->get_page.sg) + ((n - obj->get_page.last) << PAGE_SHIFT); +} + static inline struct page * i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n) { @@ -3075,6 +3303,11 @@ int i915_gem_dumb_create(struct drm_file *file_priv, struct drm_mode_create_dumb *args); int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev, uint32_t handle, uint64_t *offset); + +void i915_gem_track_fb(struct drm_i915_gem_object *old, + struct drm_i915_gem_object *new, + unsigned frontbuffer_bits); + /** * Returns true if seq1 is later than seq2. */ @@ -3084,31 +3317,34 @@ i915_seqno_passed(uint32_t seq1, uint32_t seq2) return (int32_t)(seq1 - seq2) >= 0; } -static inline bool i915_gem_request_started(struct drm_i915_gem_request *req, - bool lazy_coherency) +static inline bool i915_gem_request_started(const struct drm_i915_gem_request *req) { - if (!lazy_coherency && req->engine->irq_seqno_barrier) - req->engine->irq_seqno_barrier(req->engine); - return i915_seqno_passed(req->engine->get_seqno(req->engine), + return i915_seqno_passed(intel_engine_get_seqno(req->engine), req->previous_seqno); } -static inline bool i915_gem_request_completed(struct drm_i915_gem_request *req, - bool lazy_coherency) +static inline bool i915_gem_request_completed(const struct drm_i915_gem_request *req) { - if (!lazy_coherency && req->engine->irq_seqno_barrier) - req->engine->irq_seqno_barrier(req->engine); - return i915_seqno_passed(req->engine->get_seqno(req->engine), + return i915_seqno_passed(intel_engine_get_seqno(req->engine), req->seqno); } -int __must_check i915_gem_get_seqno(struct drm_device *dev, u32 *seqno); +bool __i915_spin_request(const struct drm_i915_gem_request *request, + int state, unsigned long timeout_us); +static inline bool i915_spin_request(const struct drm_i915_gem_request *request, + int state, unsigned long timeout_us) +{ + return (i915_gem_request_started(request) && + __i915_spin_request(request, state, timeout_us)); +} + +int __must_check i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno); int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno); struct drm_i915_gem_request * i915_gem_find_active_request(struct intel_engine_cs *engine); -bool i915_gem_retire_requests(struct drm_device *dev); +void i915_gem_retire_requests(struct drm_i915_private *dev_priv); void i915_gem_retire_requests_ring(struct intel_engine_cs *engine); static inline u32 i915_reset_counter(struct i915_gpu_error *error) @@ -3151,27 +3387,14 @@ static inline u32 i915_reset_count(struct i915_gpu_error *error) return ((i915_reset_counter(error) & ~I915_WEDGED) + 1) / 2; } -static inline bool i915_stop_ring_allow_ban(struct drm_i915_private *dev_priv) -{ - return dev_priv->gpu_error.stop_rings == 0 || - dev_priv->gpu_error.stop_rings & I915_STOP_RING_ALLOW_BAN; -} - -static inline bool i915_stop_ring_allow_warn(struct drm_i915_private *dev_priv) -{ - return dev_priv->gpu_error.stop_rings == 0 || - dev_priv->gpu_error.stop_rings & I915_STOP_RING_ALLOW_WARN; -} - void i915_gem_reset(struct drm_device *dev); bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force); int __must_check i915_gem_init(struct drm_device *dev); int i915_gem_init_engines(struct drm_device *dev); int __must_check i915_gem_init_hw(struct drm_device *dev); -int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice); void i915_gem_init_swizzling(struct drm_device *dev); void i915_gem_cleanup_engines(struct drm_device *dev); -int __must_check i915_gpu_idle(struct drm_device *dev); +int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv); int __must_check i915_gem_suspend(struct drm_device *dev); void __i915_add_request(struct drm_i915_gem_request *req, struct drm_i915_gem_object *batch_obj, @@ -3236,8 +3459,6 @@ bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o, bool i915_gem_obj_bound(struct drm_i915_gem_object *o, struct i915_address_space *vm); -unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o, - struct i915_address_space *vm); struct i915_vma * i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, struct i915_address_space *vm); @@ -3272,14 +3493,8 @@ static inline bool i915_gem_obj_ggtt_bound(struct drm_i915_gem_object *obj) return i915_gem_obj_ggtt_bound_view(obj, &i915_ggtt_view_normal); } -static inline unsigned long -i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *dev_priv = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; - - return i915_gem_obj_size(obj, &ggtt->base); -} +unsigned long +i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj); static inline int __must_check i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj, @@ -3293,12 +3508,6 @@ i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj, alignment, flags | PIN_GLOBAL); } -static inline int -i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj) -{ - return i915_vma_unbind(i915_gem_obj_to_ggtt(obj)); -} - void i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj, const struct i915_ggtt_view *view); static inline void @@ -3322,28 +3531,44 @@ void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj); /* i915_gem_context.c */ int __must_check i915_gem_context_init(struct drm_device *dev); +void i915_gem_context_lost(struct drm_i915_private *dev_priv); void i915_gem_context_fini(struct drm_device *dev); void i915_gem_context_reset(struct drm_device *dev); int i915_gem_context_open(struct drm_device *dev, struct drm_file *file); -int i915_gem_context_enable(struct drm_i915_gem_request *req); void i915_gem_context_close(struct drm_device *dev, struct drm_file *file); int i915_switch_context(struct drm_i915_gem_request *req); -struct intel_context * -i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id); void i915_gem_context_free(struct kref *ctx_ref); struct drm_i915_gem_object * i915_gem_alloc_context_obj(struct drm_device *dev, size_t size); -static inline void i915_gem_context_reference(struct intel_context *ctx) +struct i915_gem_context * +i915_gem_context_create_gvt(struct drm_device *dev); + +static inline struct i915_gem_context * +i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id) +{ + struct i915_gem_context *ctx; + + lockdep_assert_held(&file_priv->dev_priv->drm.struct_mutex); + + ctx = idr_find(&file_priv->context_idr, id); + if (!ctx) + return ERR_PTR(-ENOENT); + + return ctx; +} + +static inline void i915_gem_context_reference(struct i915_gem_context *ctx) { kref_get(&ctx->ref); } -static inline void i915_gem_context_unreference(struct intel_context *ctx) +static inline void i915_gem_context_unreference(struct i915_gem_context *ctx) { + lockdep_assert_held(&ctx->i915->drm.struct_mutex); kref_put(&ctx->ref, i915_gem_context_free); } -static inline bool i915_gem_context_is_default(const struct intel_context *c) +static inline bool i915_gem_context_is_default(const struct i915_gem_context *c) { return c->user_handle == DEFAULT_CONTEXT_HANDLE; } @@ -3356,6 +3581,8 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); +int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); /* i915_gem_evict.c */ int __must_check i915_gem_evict_something(struct drm_device *dev, @@ -3370,9 +3597,10 @@ int __must_check i915_gem_evict_for_vma(struct i915_vma *target); int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle); /* belongs in i915_gem_gtt.h */ -static inline void i915_gem_chipset_flush(struct drm_device *dev) +static inline void i915_gem_chipset_flush(struct drm_i915_private *dev_priv) { - if (INTEL_INFO(dev)->gen < 6) + wmb(); + if (INTEL_GEN(dev_priv) < 6) intel_gtt_chipset_flush(); } @@ -3413,7 +3641,7 @@ void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv); /* i915_gem_tiling.c */ static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && obj->tiling_mode != I915_TILING_NONE; @@ -3427,12 +3655,14 @@ int i915_verify_lists(struct drm_device *dev); #endif /* i915_debugfs.c */ -int i915_debugfs_init(struct drm_minor *minor); -void i915_debugfs_cleanup(struct drm_minor *minor); #ifdef CONFIG_DEBUG_FS +int i915_debugfs_register(struct drm_i915_private *dev_priv); +void i915_debugfs_unregister(struct drm_i915_private *dev_priv); int i915_debugfs_connector_add(struct drm_connector *connector); void intel_display_crc_init(struct drm_device *dev); #else +static inline int i915_debugfs_register(struct drm_i915_private *dev_priv) {return 0;} +static inline void i915_debugfs_unregister(struct drm_i915_private *dev_priv) {} static inline int i915_debugfs_connector_add(struct drm_connector *connector) { return 0; } static inline void intel_display_crc_init(struct drm_device *dev) {} @@ -3451,18 +3681,19 @@ static inline void i915_error_state_buf_release( { kfree(eb->buf); } -void i915_capture_error_state(struct drm_device *dev, u32 engine_mask, +void i915_capture_error_state(struct drm_i915_private *dev_priv, + u32 engine_mask, const char *error_msg); void i915_error_state_get(struct drm_device *dev, struct i915_error_state_file_priv *error_priv); void i915_error_state_put(struct i915_error_state_file_priv *error_priv); void i915_destroy_error_state(struct drm_device *dev); -void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone); +void i915_get_extra_instdone(struct drm_i915_private *dev_priv, uint32_t *instdone); const char *i915_cache_level_str(struct drm_i915_private *i915, int type); /* i915_cmd_parser.c */ -int i915_cmd_parser_get_version(void); +int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv); int i915_cmd_parser_init_ring(struct intel_engine_cs *engine); void i915_cmd_parser_fini_ring(struct intel_engine_cs *engine); bool i915_needs_cmd_parser(struct intel_engine_cs *engine); @@ -3511,31 +3742,33 @@ bool intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv, /* intel_opregion.c */ #ifdef CONFIG_ACPI -extern int intel_opregion_setup(struct drm_device *dev); -extern void intel_opregion_init(struct drm_device *dev); -extern void intel_opregion_fini(struct drm_device *dev); -extern void intel_opregion_asle_intr(struct drm_device *dev); +extern int intel_opregion_setup(struct drm_i915_private *dev_priv); +extern void intel_opregion_register(struct drm_i915_private *dev_priv); +extern void intel_opregion_unregister(struct drm_i915_private *dev_priv); +extern void intel_opregion_asle_intr(struct drm_i915_private *dev_priv); extern int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, bool enable); -extern int intel_opregion_notify_adapter(struct drm_device *dev, +extern int intel_opregion_notify_adapter(struct drm_i915_private *dev_priv, pci_power_t state); -extern int intel_opregion_get_panel_type(struct drm_device *dev); +extern int intel_opregion_get_panel_type(struct drm_i915_private *dev_priv); #else -static inline int intel_opregion_setup(struct drm_device *dev) { return 0; } -static inline void intel_opregion_init(struct drm_device *dev) { return; } -static inline void intel_opregion_fini(struct drm_device *dev) { return; } -static inline void intel_opregion_asle_intr(struct drm_device *dev) { return; } +static inline int intel_opregion_setup(struct drm_i915_private *dev) { return 0; } +static inline void intel_opregion_register(struct drm_i915_private *dev_priv) { } +static inline void intel_opregion_unregister(struct drm_i915_private *dev_priv) { } +static inline void intel_opregion_asle_intr(struct drm_i915_private *dev_priv) +{ +} static inline int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, bool enable) { return 0; } static inline int -intel_opregion_notify_adapter(struct drm_device *dev, pci_power_t state) +intel_opregion_notify_adapter(struct drm_i915_private *dev, pci_power_t state) { return 0; } -static inline int intel_opregion_get_panel_type(struct drm_device *dev) +static inline int intel_opregion_get_panel_type(struct drm_i915_private *dev) { return -ENODEV; } @@ -3550,36 +3783,45 @@ static inline void intel_register_dsm_handler(void) { return; } static inline void intel_unregister_dsm_handler(void) { return; } #endif /* CONFIG_ACPI */ +/* intel_device_info.c */ +static inline struct intel_device_info * +mkwrite_device_info(struct drm_i915_private *dev_priv) +{ + return (struct intel_device_info *)&dev_priv->info; +} + +void intel_device_info_runtime_init(struct drm_i915_private *dev_priv); +void intel_device_info_dump(struct drm_i915_private *dev_priv); + /* modesetting */ extern void intel_modeset_init_hw(struct drm_device *dev); extern void intel_modeset_init(struct drm_device *dev); extern void intel_modeset_gem_init(struct drm_device *dev); extern void intel_modeset_cleanup(struct drm_device *dev); -extern void intel_connector_unregister(struct intel_connector *); +extern int intel_connector_register(struct drm_connector *); +extern void intel_connector_unregister(struct drm_connector *); extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state); extern void intel_display_resume(struct drm_device *dev); extern void i915_redisable_vga(struct drm_device *dev); extern void i915_redisable_vga_power_on(struct drm_device *dev); -extern bool ironlake_set_drps(struct drm_device *dev, u8 val); +extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val); extern void intel_init_pch_refclk(struct drm_device *dev); -extern void intel_set_rps(struct drm_device *dev, u8 val); +extern void intel_set_rps(struct drm_i915_private *dev_priv, u8 val); extern void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable); -extern void intel_detect_pch(struct drm_device *dev); -extern int intel_enable_rc6(const struct drm_device *dev); -extern bool i915_semaphore_is_enabled(struct drm_device *dev); +extern bool i915_semaphore_is_enabled(struct drm_i915_private *dev_priv); int i915_reg_read_ioctl(struct drm_device *dev, void *data, struct drm_file *file); -int i915_get_reset_stats_ioctl(struct drm_device *dev, void *data, - struct drm_file *file); /* overlay */ -extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev); +extern struct intel_overlay_error_state * +intel_overlay_capture_error_state(struct drm_i915_private *dev_priv); extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e, struct intel_overlay_error_state *error); -extern struct intel_display_error_state *intel_display_capture_error_state(struct drm_device *dev); +extern struct intel_display_error_state * +intel_display_capture_error_state(struct drm_i915_private *dev_priv); extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, struct drm_device *dev, struct intel_display_error_state *error); @@ -3608,6 +3850,24 @@ void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value, u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg); void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); +/* intel_dpio_phy.c */ +void chv_set_phy_signal_level(struct intel_encoder *encoder, + u32 deemph_reg_value, u32 margin_reg_value, + bool uniq_trans_scale); +void chv_data_lane_soft_reset(struct intel_encoder *encoder, + bool reset); +void chv_phy_pre_pll_enable(struct intel_encoder *encoder); +void chv_phy_pre_encoder_enable(struct intel_encoder *encoder); +void chv_phy_release_cl2_override(struct intel_encoder *encoder); +void chv_phy_post_pll_disable(struct intel_encoder *encoder); + +void vlv_set_phy_signal_level(struct intel_encoder *encoder, + u32 demph_reg_value, u32 preemph_reg_value, + u32 uniqtranscale_reg_value, u32 tx3_demph); +void vlv_phy_pre_pll_enable(struct intel_encoder *encoder); +void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder); +void vlv_phy_reset_lanes(struct intel_encoder *encoder); + int intel_gpu_freq(struct drm_i915_private *dev_priv, int val); int intel_freq_opcode(struct drm_i915_private *dev_priv, int val); @@ -3681,6 +3941,7 @@ __raw_write(64, q) */ #define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__)) #define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__)) +#define I915_WRITE64_FW(reg__, val__) __raw_i915_write64(dev_priv, (reg__), (val__)) #define POSTING_READ_FW(reg__) (void)I915_READ_FW(reg__) /* "Broadcast RGB" property */ @@ -3744,12 +4005,80 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) schedule_timeout_uninterruptible(remaining_jiffies); } } - -static inline void i915_trace_irq_get(struct intel_engine_cs *engine, - struct drm_i915_gem_request *req) +static inline bool __i915_request_irq_complete(struct drm_i915_gem_request *req) { - if (engine->trace_irq_req == NULL && engine->irq_get(engine)) - i915_gem_request_assign(&engine->trace_irq_req, req); + struct intel_engine_cs *engine = req->engine; + + /* Before we do the heavier coherent read of the seqno, + * check the value (hopefully) in the CPU cacheline. + */ + if (i915_gem_request_completed(req)) + return true; + + /* Ensure our read of the seqno is coherent so that we + * do not "miss an interrupt" (i.e. if this is the last + * request and the seqno write from the GPU is not visible + * by the time the interrupt fires, we will see that the + * request is incomplete and go back to sleep awaiting + * another interrupt that will never come.) + * + * Strictly, we only need to do this once after an interrupt, + * but it is easier and safer to do it every time the waiter + * is woken. + */ + if (engine->irq_seqno_barrier && + READ_ONCE(engine->breadcrumbs.irq_seqno_bh) == current && + cmpxchg_relaxed(&engine->breadcrumbs.irq_posted, 1, 0)) { + struct task_struct *tsk; + + /* The ordering of irq_posted versus applying the barrier + * is crucial. The clearing of the current irq_posted must + * be visible before we perform the barrier operation, + * such that if a subsequent interrupt arrives, irq_posted + * is reasserted and our task rewoken (which causes us to + * do another __i915_request_irq_complete() immediately + * and reapply the barrier). Conversely, if the clear + * occurs after the barrier, then an interrupt that arrived + * whilst we waited on the barrier would not trigger a + * barrier on the next pass, and the read may not see the + * seqno update. + */ + engine->irq_seqno_barrier(engine); + + /* If we consume the irq, but we are no longer the bottom-half, + * the real bottom-half may not have serialised their own + * seqno check with the irq-barrier (i.e. may have inspected + * the seqno before we believe it coherent since they see + * irq_posted == false but we are still running). + */ + rcu_read_lock(); + tsk = READ_ONCE(engine->breadcrumbs.irq_seqno_bh); + if (tsk && tsk != current) + /* Note that if the bottom-half is changed as we + * are sending the wake-up, the new bottom-half will + * be woken by whomever made the change. We only have + * to worry about when we steal the irq-posted for + * ourself. + */ + wake_up_process(tsk); + rcu_read_unlock(); + + if (i915_gem_request_completed(req)) + return true; + } + + /* We need to check whether any gpu reset happened in between + * the request being submitted and now. If a reset has occurred, + * the seqno will have been advance past ours and our request + * is complete. If we are in the process of handling a reset, + * the request is effectively complete as the rendering will + * be discarded, but we need to return in order to drop the + * struct_mutex. + */ + if (i915_reset_in_progress(&req->i915->gpu_error)) + return true; + + return false; } #endif diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index aad26851c..a77ce9983 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -54,12 +54,33 @@ static bool cpu_cache_is_coherent(struct drm_device *dev, static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) { + if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) + return false; + if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) return true; return obj->pin_display; } +static int +insert_mappable_node(struct drm_i915_private *i915, + struct drm_mm_node *node, u32 size) +{ + memset(node, 0, sizeof(*node)); + return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node, + size, 0, 0, 0, + i915->ggtt.mappable_end, + DRM_MM_SEARCH_DEFAULT, + DRM_MM_CREATE_DEFAULT); +} + +static void +remove_mappable_node(struct drm_mm_node *node) +{ + drm_mm_remove_node(node); +} + /* some bookkeeping */ static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) @@ -107,7 +128,7 @@ i915_gem_wait_for_error(struct i915_gpu_error *error) int i915_mutex_lock_interruptible(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; ret = i915_gem_wait_for_error(&dev_priv->gpu_error); @@ -151,7 +172,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) { - struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; + struct address_space *mapping = obj->base.filp->f_mapping; char *vaddr = obj->phys_handle->vaddr; struct sg_table *st; struct scatterlist *sg; @@ -177,7 +198,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) vaddr += PAGE_SIZE; } - i915_gem_chipset_flush(obj->base.dev); + i915_gem_chipset_flush(to_i915(obj->base.dev)); st = kmalloc(sizeof(*st), GFP_KERNEL); if (st == NULL) @@ -218,7 +239,7 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) obj->dirty = 0; if (obj->dirty) { - struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; + struct address_space *mapping = obj->base.filp->f_mapping; char *vaddr = obj->phys_handle->vaddr; int i; @@ -347,7 +368,7 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, } drm_clflush_virt_range(vaddr, args->size); - i915_gem_chipset_flush(dev); + i915_gem_chipset_flush(to_i915(dev)); out: intel_fb_obj_flush(obj, false, ORIGIN_CPU); @@ -356,13 +377,13 @@ out: void *i915_gem_object_alloc(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL); } void i915_gem_object_free(struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); kmem_cache_free(dev_priv->objects, obj); } @@ -381,9 +402,9 @@ i915_gem_create(struct drm_file *file, return -EINVAL; /* Allocate the new object */ - obj = i915_gem_alloc_object(dev, size); - if (obj == NULL) - return -ENOMEM; + obj = i915_gem_object_create(dev, size); + if (IS_ERR(obj)) + return PTR_ERR(obj); ret = drm_gem_handle_create(file, &obj->base, &handle); /* drop reference from allocate - handle holds it now */ @@ -409,6 +430,9 @@ i915_gem_dumb_create(struct drm_file *file, /** * Creates a new mm object and returns a handle to it. + * @dev: drm device pointer + * @data: ioctl data blob + * @file: drm file pointer */ int i915_gem_create_ioctl(struct drm_device *dev, void *data, @@ -484,7 +508,7 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, *needs_clflush = 0; - if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0)) + if (WARN_ON(!i915_gem_object_has_struct_page(obj))) return -EINVAL; if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { @@ -585,6 +609,142 @@ shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, return ret ? - EFAULT : 0; } +static inline unsigned long +slow_user_access(struct io_mapping *mapping, + uint64_t page_base, int page_offset, + char __user *user_data, + unsigned long length, bool pwrite) +{ + void __iomem *ioaddr; + void *vaddr; + uint64_t unwritten; + + ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE); + /* We can use the cpu mem copy function because this is X86. */ + vaddr = (void __force *)ioaddr + page_offset; + if (pwrite) + unwritten = __copy_from_user(vaddr, user_data, length); + else + unwritten = __copy_to_user(user_data, vaddr, length); + + io_mapping_unmap(ioaddr); + return unwritten; +} + +static int +i915_gem_gtt_pread(struct drm_device *dev, + struct drm_i915_gem_object *obj, uint64_t size, + uint64_t data_offset, uint64_t data_ptr) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + struct i915_ggtt *ggtt = &dev_priv->ggtt; + struct drm_mm_node node; + char __user *user_data; + uint64_t remain; + uint64_t offset; + int ret; + + ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE); + if (ret) { + ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE); + if (ret) + goto out; + + ret = i915_gem_object_get_pages(obj); + if (ret) { + remove_mappable_node(&node); + goto out; + } + + i915_gem_object_pin_pages(obj); + } else { + node.start = i915_gem_obj_ggtt_offset(obj); + node.allocated = false; + ret = i915_gem_object_put_fence(obj); + if (ret) + goto out_unpin; + } + + ret = i915_gem_object_set_to_gtt_domain(obj, false); + if (ret) + goto out_unpin; + + user_data = u64_to_user_ptr(data_ptr); + remain = size; + offset = data_offset; + + mutex_unlock(&dev->struct_mutex); + if (likely(!i915.prefault_disable)) { + ret = fault_in_multipages_writeable(user_data, remain); + if (ret) { + mutex_lock(&dev->struct_mutex); + goto out_unpin; + } + } + + while (remain > 0) { + /* Operation in this page + * + * page_base = page offset within aperture + * page_offset = offset within page + * page_length = bytes to copy for this page + */ + u32 page_base = node.start; + unsigned page_offset = offset_in_page(offset); + unsigned page_length = PAGE_SIZE - page_offset; + page_length = remain < page_length ? remain : page_length; + if (node.allocated) { + wmb(); + ggtt->base.insert_page(&ggtt->base, + i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), + node.start, + I915_CACHE_NONE, 0); + wmb(); + } else { + page_base += offset & PAGE_MASK; + } + /* This is a slow read/write as it tries to read from + * and write to user memory which may result into page + * faults, and so we cannot perform this under struct_mutex. + */ + if (slow_user_access(ggtt->mappable, page_base, + page_offset, user_data, + page_length, false)) { + ret = -EFAULT; + break; + } + + remain -= page_length; + user_data += page_length; + offset += page_length; + } + + mutex_lock(&dev->struct_mutex); + if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) { + /* The user has modified the object whilst we tried + * reading from it, and we now have no idea what domain + * the pages should be in. As we have just been touching + * them directly, flush everything back to the GTT + * domain. + */ + ret = i915_gem_object_set_to_gtt_domain(obj, false); + } + +out_unpin: + if (node.allocated) { + wmb(); + ggtt->base.clear_range(&ggtt->base, + node.start, node.size, + true); + i915_gem_object_unpin_pages(obj); + remove_mappable_node(&node); + } else { + i915_gem_object_ggtt_unpin(obj); + } +out: + return ret; +} + static int i915_gem_shmem_pread(struct drm_device *dev, struct drm_i915_gem_object *obj, @@ -600,6 +760,9 @@ i915_gem_shmem_pread(struct drm_device *dev, int needs_clflush = 0; struct sg_page_iter sg_iter; + if (!i915_gem_object_has_struct_page(obj)) + return -ENODEV; + user_data = u64_to_user_ptr(args->data_ptr); remain = args->size; @@ -672,6 +835,9 @@ out: /** * Reads data from the object referenced by handle. + * @dev: drm device pointer + * @data: ioctl data blob + * @file: drm file pointer * * On error, the contents of *data are undefined. */ @@ -708,18 +874,18 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, goto out; } - /* prime objects have no backing filp to GEM pread/pwrite - * pages from. - */ - if (!obj->base.filp) { - ret = -EINVAL; - goto out; - } - trace_i915_gem_object_pread(obj, args->offset, args->size); ret = i915_gem_shmem_pread(dev, obj, args, file); + /* pread for non shmem backed objects */ + if (ret == -EFAULT || ret == -ENODEV) { + intel_runtime_pm_get(to_i915(dev)); + ret = i915_gem_gtt_pread(dev, obj, args->size, + args->offset, args->data_ptr); + intel_runtime_pm_put(to_i915(dev)); + } + out: drm_gem_object_unreference(&obj->base); unlock: @@ -753,60 +919,99 @@ fast_user_write(struct io_mapping *mapping, /** * This is the fast pwrite path, where we copy the data directly from the * user into the GTT, uncached. + * @dev: drm device pointer + * @obj: i915 gem object + * @args: pwrite arguments structure + * @file: drm file pointer */ static int -i915_gem_gtt_pwrite_fast(struct drm_device *dev, +i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, struct drm_file *file) { - struct drm_i915_private *dev_priv = to_i915(dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; - ssize_t remain; - loff_t offset, page_base; + struct i915_ggtt *ggtt = &i915->ggtt; + struct drm_device *dev = obj->base.dev; + struct drm_mm_node node; + uint64_t remain, offset; char __user *user_data; - int page_offset, page_length, ret; + int ret; + bool hit_slow_path = false; + + if (obj->tiling_mode != I915_TILING_NONE) + return -EFAULT; ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK); - if (ret) - goto out; + if (ret) { + ret = insert_mappable_node(i915, &node, PAGE_SIZE); + if (ret) + goto out; + + ret = i915_gem_object_get_pages(obj); + if (ret) { + remove_mappable_node(&node); + goto out; + } + + i915_gem_object_pin_pages(obj); + } else { + node.start = i915_gem_obj_ggtt_offset(obj); + node.allocated = false; + ret = i915_gem_object_put_fence(obj); + if (ret) + goto out_unpin; + } ret = i915_gem_object_set_to_gtt_domain(obj, true); if (ret) goto out_unpin; - ret = i915_gem_object_put_fence(obj); - if (ret) - goto out_unpin; + intel_fb_obj_invalidate(obj, ORIGIN_GTT); + obj->dirty = true; user_data = u64_to_user_ptr(args->data_ptr); + offset = args->offset; remain = args->size; - - offset = i915_gem_obj_ggtt_offset(obj) + args->offset; - - intel_fb_obj_invalidate(obj, ORIGIN_GTT); - - while (remain > 0) { + while (remain) { /* Operation in this page * * page_base = page offset within aperture * page_offset = offset within page * page_length = bytes to copy for this page */ - page_base = offset & PAGE_MASK; - page_offset = offset_in_page(offset); - page_length = remain; - if ((page_offset + remain) > PAGE_SIZE) - page_length = PAGE_SIZE - page_offset; - + u32 page_base = node.start; + unsigned page_offset = offset_in_page(offset); + unsigned page_length = PAGE_SIZE - page_offset; + page_length = remain < page_length ? remain : page_length; + if (node.allocated) { + wmb(); /* flush the write before we modify the GGTT */ + ggtt->base.insert_page(&ggtt->base, + i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), + node.start, I915_CACHE_NONE, 0); + wmb(); /* flush modifications to the GGTT (insert_page) */ + } else { + page_base += offset & PAGE_MASK; + } /* If we get a fault while copying data, then (presumably) our * source page isn't available. Return the error and we'll * retry in the slow path. + * If the object is non-shmem backed, we retry again with the + * path that handles page fault. */ if (fast_user_write(ggtt->mappable, page_base, page_offset, user_data, page_length)) { - ret = -EFAULT; - goto out_flush; + hit_slow_path = true; + mutex_unlock(&dev->struct_mutex); + if (slow_user_access(ggtt->mappable, + page_base, + page_offset, user_data, + page_length, true)) { + ret = -EFAULT; + mutex_lock(&dev->struct_mutex); + goto out_flush; + } + + mutex_lock(&dev->struct_mutex); } remain -= page_length; @@ -815,9 +1020,31 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, } out_flush: + if (hit_slow_path) { + if (ret == 0 && + (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) { + /* The user has modified the object whilst we tried + * reading from it, and we now have no idea what domain + * the pages should be in. As we have just been touching + * them directly, flush everything back to the GTT + * domain. + */ + ret = i915_gem_object_set_to_gtt_domain(obj, false); + } + } + intel_fb_obj_flush(obj, false, ORIGIN_GTT); out_unpin: - i915_gem_object_ggtt_unpin(obj); + if (node.allocated) { + wmb(); + ggtt->base.clear_range(&ggtt->base, + node.start, node.size, + true); + i915_gem_object_unpin_pages(obj); + remove_mappable_node(&node); + } else { + i915_gem_object_ggtt_unpin(obj); + } out: return ret; } @@ -1006,7 +1233,7 @@ out: } if (needs_clflush_after) - i915_gem_chipset_flush(dev); + i915_gem_chipset_flush(to_i915(dev)); else obj->cache_dirty = true; @@ -1016,6 +1243,9 @@ out: /** * Writes data to the object referenced by handle. + * @dev: drm device + * @data: ioctl data blob + * @file: drm file * * On error, the contents of the buffer that were to be modified are undefined. */ @@ -1023,7 +1253,7 @@ int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_pwrite *args = data; struct drm_i915_gem_object *obj; int ret; @@ -1062,14 +1292,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, goto out; } - /* prime objects have no backing filp to GEM pread/pwrite - * pages from. - */ - if (!obj->base.filp) { - ret = -EINVAL; - goto out; - } - trace_i915_gem_object_pwrite(obj, args->offset, args->size); ret = -EFAULT; @@ -1079,10 +1301,9 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, * pread/pwrite currently are reading and writing from the CPU * perspective, requiring manual detiling by the client. */ - if (obj->tiling_mode == I915_TILING_NONE && - obj->base.write_domain != I915_GEM_DOMAIN_CPU && + if (!i915_gem_object_has_struct_page(obj) || cpu_write_needs_clflush(obj)) { - ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); + ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file); /* Note that the gtt paths might fail with non-page-backed user * pointers (e.g. gtt mappings when moving data between * textures). Fallback to the shmem path in that case. */ @@ -1091,8 +1312,10 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, if (ret == -EFAULT || ret == -ENOSPC) { if (obj->phys_handle) ret = i915_gem_phys_pwrite(obj, args, file); - else + else if (i915_gem_object_has_struct_page(obj)) ret = i915_gem_shmem_pwrite(dev, obj, args, file); + else + ret = -ENODEV; } out: @@ -1123,17 +1346,6 @@ i915_gem_check_wedge(unsigned reset_counter, bool interruptible) return 0; } -static void fake_irq(unsigned long data) -{ - wake_up_process((struct task_struct *)data); -} - -static bool missed_irq(struct drm_i915_private *dev_priv, - struct intel_engine_cs *engine) -{ - return test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings); -} - static unsigned long local_clock_us(unsigned *cpu) { unsigned long t; @@ -1166,9 +1378,9 @@ static bool busywait_stop(unsigned long timeout, unsigned cpu) return this_cpu != cpu; } -static int __i915_spin_request(struct drm_i915_gem_request *req, int state) +bool __i915_spin_request(const struct drm_i915_gem_request *req, + int state, unsigned long timeout_us) { - unsigned long timeout; unsigned cpu; /* When waiting for high frequency requests, e.g. during synchronous @@ -1181,31 +1393,21 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state) * takes to sleep on a request, on the order of a microsecond. */ - if (req->engine->irq_refcount) - return -EBUSY; - - /* Only spin if we know the GPU is processing this request */ - if (!i915_gem_request_started(req, true)) - return -EAGAIN; - - timeout = local_clock_us(&cpu) + 5; - while (!need_resched()) { - if (i915_gem_request_completed(req, true)) - return 0; + timeout_us += local_clock_us(&cpu); + do { + if (i915_gem_request_completed(req)) + return true; if (signal_pending_state(state, current)) break; - if (busywait_stop(timeout, cpu)) + if (busywait_stop(timeout_us, cpu)) break; cpu_relax_lowlatency(); - } + } while (!need_resched()); - if (i915_gem_request_completed(req, false)) - return 0; - - return -EAGAIN; + return false; } /** @@ -1213,6 +1415,7 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state) * @req: duh! * @interruptible: do an interruptible wait (normally yes) * @timeout: in - how long to wait (NULL forever); out - how much time remaining + * @rps: RPS client * * Note: It is of utmost importance that the passed in seqno and reset_counter * values have been read by the caller in an smp safe manner. Where read-side @@ -1229,26 +1432,22 @@ int __i915_wait_request(struct drm_i915_gem_request *req, s64 *timeout, struct intel_rps_client *rps) { - struct intel_engine_cs *engine = i915_gem_request_get_engine(req); - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - const bool irq_test_in_progress = - ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_engine_flag(engine); int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; - DEFINE_WAIT(wait); - unsigned long timeout_expire; + DEFINE_WAIT(reset); + struct intel_wait wait; + unsigned long timeout_remain; s64 before = 0; /* Only to silence a compiler warning. */ - int ret; + int ret = 0; - WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled"); + might_sleep(); if (list_empty(&req->list)) return 0; - if (i915_gem_request_completed(req, true)) + if (i915_gem_request_completed(req)) return 0; - timeout_expire = 0; + timeout_remain = MAX_SCHEDULE_TIMEOUT; if (timeout) { if (WARN_ON(*timeout < 0)) return -EINVAL; @@ -1256,7 +1455,7 @@ int __i915_wait_request(struct drm_i915_gem_request *req, if (*timeout == 0) return -ETIME; - timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout); + timeout_remain = nsecs_to_jiffies_timeout(*timeout); /* * Record current time in case interrupted by signal, or wedged. @@ -1264,75 +1463,76 @@ int __i915_wait_request(struct drm_i915_gem_request *req, before = ktime_get_raw_ns(); } - if (INTEL_INFO(dev_priv)->gen >= 6) - gen6_rps_boost(dev_priv, rps, req->emitted_jiffies); - trace_i915_gem_request_wait_begin(req); - /* Optimistic spin for the next jiffie before touching IRQs */ - ret = __i915_spin_request(req, state); - if (ret == 0) - goto out; - - if (!irq_test_in_progress && WARN_ON(!engine->irq_get(engine))) { - ret = -ENODEV; - goto out; - } + /* This client is about to stall waiting for the GPU. In many cases + * this is undesirable and limits the throughput of the system, as + * many clients cannot continue processing user input/output whilst + * blocked. RPS autotuning may take tens of milliseconds to respond + * to the GPU load and thus incurs additional latency for the client. + * We can circumvent that by promoting the GPU frequency to maximum + * before we wait. This makes the GPU throttle up much more quickly + * (good for benchmarks and user experience, e.g. window animations), + * but at a cost of spending more power processing the workload + * (bad for battery). Not all clients even want their results + * immediately and for them we should just let the GPU select its own + * frequency to maximise efficiency. To prevent a single client from + * forcing the clocks too high for the whole system, we only allow + * each client to waitboost once in a busy period. + */ + if (INTEL_INFO(req->i915)->gen >= 6) + gen6_rps_boost(req->i915, rps, req->emitted_jiffies); - for (;;) { - struct timer_list timer; + /* Optimistic spin for the next ~jiffie before touching IRQs */ + if (i915_spin_request(req, state, 5)) + goto complete; - prepare_to_wait(&engine->irq_queue, &wait, state); + set_current_state(state); + add_wait_queue(&req->i915->gpu_error.wait_queue, &reset); - /* We need to check whether any gpu reset happened in between - * the request being submitted and now. If a reset has occurred, - * the request is effectively complete (we either are in the - * process of or have discarded the rendering and completely - * reset the GPU. The results of the request are lost and we - * are free to continue on with the original operation. + intel_wait_init(&wait, req->seqno); + if (intel_engine_add_wait(req->engine, &wait)) + /* In order to check that we haven't missed the interrupt + * as we enabled it, we need to kick ourselves to do a + * coherent check on the seqno before we sleep. */ - if (req->reset_counter != i915_reset_counter(&dev_priv->gpu_error)) { - ret = 0; - break; - } - - if (i915_gem_request_completed(req, false)) { - ret = 0; - break; - } + goto wakeup; + for (;;) { if (signal_pending_state(state, current)) { ret = -ERESTARTSYS; break; } - if (timeout && time_after_eq(jiffies, timeout_expire)) { + timeout_remain = io_schedule_timeout(timeout_remain); + if (timeout_remain == 0) { ret = -ETIME; break; } - timer.function = NULL; - if (timeout || missed_irq(dev_priv, engine)) { - unsigned long expire; + if (intel_wait_complete(&wait)) + break; - setup_timer_on_stack(&timer, fake_irq, (unsigned long)current); - expire = missed_irq(dev_priv, engine) ? jiffies + 1 : timeout_expire; - mod_timer(&timer, expire); - } + set_current_state(state); - io_schedule(); +wakeup: + /* Carefully check if the request is complete, giving time + * for the seqno to be visible following the interrupt. + * We also have to check in case we are kicked by the GPU + * reset in order to drop the struct_mutex. + */ + if (__i915_request_irq_complete(req)) + break; - if (timer.function) { - del_singleshot_timer_sync(&timer); - destroy_timer_on_stack(&timer); - } + /* Only spin if we know the GPU is processing this request */ + if (i915_spin_request(req, state, 2)) + break; } - if (!irq_test_in_progress) - engine->irq_put(engine); + remove_wait_queue(&req->i915->gpu_error.wait_queue, &reset); - finish_wait(&engine->irq_queue, &wait); - -out: + intel_engine_remove_wait(req->engine, &wait); + __set_current_state(TASK_RUNNING); +complete: trace_i915_gem_request_wait_end(req); if (timeout) { @@ -1351,6 +1551,22 @@ out: *timeout = 0; } + if (rps && req->seqno == req->engine->last_submitted_seqno) { + /* The GPU is now idle and this client has stalled. + * Since no other client has submitted a request in the + * meantime, assume that this client is the only one + * supplying work to the GPU but is unable to keep that + * work supplied because it is waiting. Since the GPU is + * then never kept fully busy, RPS autoclocking will + * keep the clocks relatively low, causing further delays. + * Compensate by giving the synchronous client credit for + * a waitboost next time. + */ + spin_lock(&req->i915->rps.client_lock); + list_del_init(&rps->link); + spin_unlock(&req->i915->rps.client_lock); + } + return ret; } @@ -1413,6 +1629,13 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) list_del_init(&request->list); i915_gem_request_remove_from_client(request); + if (request->previous_context) { + if (i915.enable_execlists) + intel_lr_context_unpin(request->previous_context, + request->engine); + } + + i915_gem_context_unreference(request->ctx); i915_gem_request_unreference(request); } @@ -1422,7 +1645,7 @@ __i915_gem_request_retire__upto(struct drm_i915_gem_request *req) struct intel_engine_cs *engine = req->engine; struct drm_i915_gem_request *tmp; - lockdep_assert_held(&engine->dev->struct_mutex); + lockdep_assert_held(&engine->i915->drm.struct_mutex); if (list_empty(&req->list)) return; @@ -1440,6 +1663,7 @@ __i915_gem_request_retire__upto(struct drm_i915_gem_request *req) /** * Waits for a request to be signaled, and cleans up the * request and object lists appropriately for that event. + * @req: request to wait on */ int i915_wait_request(struct drm_i915_gem_request *req) @@ -1450,14 +1674,14 @@ i915_wait_request(struct drm_i915_gem_request *req) interruptible = dev_priv->mm.interruptible; - BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex)); + BUG_ON(!mutex_is_locked(&dev_priv->drm.struct_mutex)); ret = __i915_wait_request(req, interruptible, NULL, NULL); if (ret) return ret; /* If the GPU hung, we want to keep the requests to find the guilty. */ - if (req->reset_counter == i915_reset_counter(&dev_priv->gpu_error)) + if (!i915_reset_in_progress(&dev_priv->gpu_error)) __i915_gem_request_retire__upto(req); return 0; @@ -1466,6 +1690,8 @@ i915_wait_request(struct drm_i915_gem_request *req) /** * Ensures that all rendering to the object has completed and the object is * safe to unbind from the GTT or access from the CPU. + * @obj: i915 gem object + * @readonly: waiting for read access or write */ int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, @@ -1516,7 +1742,7 @@ i915_gem_object_retire_request(struct drm_i915_gem_object *obj, else if (obj->last_write_req == req) i915_gem_object_retire__write(obj); - if (req->reset_counter == i915_reset_counter(&req->i915->gpu_error)) + if (!i915_reset_in_progress(&req->i915->gpu_error)) __i915_gem_request_retire__upto(req); } @@ -1529,7 +1755,7 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, bool readonly) { struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_request *requests[I915_NUM_ENGINES]; int ret, i, n = 0; @@ -1580,9 +1806,19 @@ static struct intel_rps_client *to_rps_client(struct drm_file *file) return &fpriv->rps; } +static enum fb_op_origin +write_origin(struct drm_i915_gem_object *obj, unsigned domain) +{ + return domain == I915_GEM_DOMAIN_GTT && !obj->has_wc_mmap ? + ORIGIN_GTT : ORIGIN_CPU; +} + /** * Called when user space prepares to use an object with the CPU, either * through the mmap ioctl's mapping or a GTT mapping. + * @dev: drm device + * @data: ioctl data blob + * @file: drm file */ int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, @@ -1633,9 +1869,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); if (write_domain != 0) - intel_fb_obj_invalidate(obj, - write_domain == I915_GEM_DOMAIN_GTT ? - ORIGIN_GTT : ORIGIN_CPU); + intel_fb_obj_invalidate(obj, write_origin(obj, write_domain)); unref: drm_gem_object_unreference(&obj->base); @@ -1646,6 +1880,9 @@ unlock: /** * Called when user space has done writes to this buffer + * @dev: drm device + * @data: ioctl data blob + * @file: drm file */ int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, @@ -1676,8 +1913,11 @@ unlock: } /** - * Maps the contents of an object, returning the address it is mapped - * into. + * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address + * it is mapped to. + * @dev: drm device + * @data: ioctl data blob + * @file: drm file * * While the mapping holds a reference on the contents of the object, it doesn't * imply a ref on the object itself. @@ -1736,6 +1976,9 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, else addr = -ENOMEM; up_write(&mm->mmap_sem); + + /* This may race, but that's ok, it only gets set */ + WRITE_ONCE(to_intel_bo(obj)->has_wc_mmap, true); } drm_gem_object_unreference_unlocked(obj); if (IS_ERR((void *)addr)) @@ -1982,7 +2225,7 @@ i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) return size; /* Previous chips need a power-of-two fence region when tiling */ - if (INTEL_INFO(dev)->gen == 3) + if (IS_GEN3(dev)) gtt_size = 1024*1024; else gtt_size = 512*1024; @@ -1995,7 +2238,10 @@ i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) /** * i915_gem_get_gtt_alignment - return required GTT alignment for an object - * @obj: object to check + * @dev: drm device + * @size: object size + * @tiling_mode: tiling mode + * @fenced: is fenced alignemned required or not * * Return the required GTT alignment for an object, taking into account * potential fence register mapping. @@ -2021,7 +2267,7 @@ i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); int ret; dev_priv->mm.shrinker_no_lock_stealing = true; @@ -2155,14 +2401,15 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj) if (obj->base.filp == NULL) return; - mapping = file_inode(obj->base.filp)->i_mapping, + mapping = obj->base.filp->f_mapping, invalidate_mapping_pages(mapping, 0, (loff_t)-1); } static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) { - struct sg_page_iter sg_iter; + struct sgt_iter sgt_iter; + struct page *page; int ret; BUG_ON(obj->madv == __I915_MADV_PURGED); @@ -2184,9 +2431,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) if (obj->madv == I915_MADV_DONTNEED) obj->dirty = 0; - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { - struct page *page = sg_page_iter_page(&sg_iter); - + for_each_sgt_page(page, sgt_iter, obj->pages) { if (obj->dirty) set_page_dirty(page); @@ -2238,12 +2483,12 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj) static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); int page_count, i; struct address_space *mapping; struct sg_table *st; struct scatterlist *sg; - struct sg_page_iter sg_iter; + struct sgt_iter sgt_iter; struct page *page; unsigned long last_pfn = 0; /* suppress gcc warning */ int ret; @@ -2271,7 +2516,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) * * Fail silently without starting the shrinker */ - mapping = file_inode(obj->base.filp)->i_mapping; + mapping = obj->base.filp->f_mapping; gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM)); gfp |= __GFP_NORETRY | __GFP_NOWARN; sg = st->sgl; @@ -2340,8 +2585,8 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) err_pages: sg_mark_end(sg); - for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) - put_page(sg_page_iter_page(&sg_iter)); + for_each_sgt_page(page, sgt_iter, st) + put_page(page); sg_free_table(st); kfree(st); @@ -2369,7 +2614,7 @@ err_pages: int i915_gem_object_get_pages(struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); const struct drm_i915_gem_object_ops *ops = obj->ops; int ret; @@ -2395,6 +2640,44 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj) return 0; } +/* The 'mapping' part of i915_gem_object_pin_map() below */ +static void *i915_gem_object_map(const struct drm_i915_gem_object *obj) +{ + unsigned long n_pages = obj->base.size >> PAGE_SHIFT; + struct sg_table *sgt = obj->pages; + struct sgt_iter sgt_iter; + struct page *page; + struct page *stack_pages[32]; + struct page **pages = stack_pages; + unsigned long i = 0; + void *addr; + + /* A single page can always be kmapped */ + if (n_pages == 1) + return kmap(sg_page(sgt->sgl)); + + if (n_pages > ARRAY_SIZE(stack_pages)) { + /* Too big for stack -- allocate temporary array instead */ + pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY); + if (!pages) + return NULL; + } + + for_each_sgt_page(page, sgt_iter, sgt) + pages[i++] = page; + + /* Check that we have the expected number of pages */ + GEM_BUG_ON(i != n_pages); + + addr = vmap(pages, n_pages, 0, PAGE_KERNEL); + + if (pages != stack_pages) + drm_free_large(pages); + + return addr; +} + +/* get, pin, and map the pages of the object into kernel space */ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj) { int ret; @@ -2407,29 +2690,9 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj) i915_gem_object_pin_pages(obj); - if (obj->mapping == NULL) { - struct page **pages; - - pages = NULL; - if (obj->base.size == PAGE_SIZE) - obj->mapping = kmap(sg_page(obj->pages->sgl)); - else - pages = drm_malloc_gfp(obj->base.size >> PAGE_SHIFT, - sizeof(*pages), - GFP_TEMPORARY); - if (pages != NULL) { - struct sg_page_iter sg_iter; - int n; - - n = 0; - for_each_sg_page(obj->pages->sgl, &sg_iter, - obj->pages->nents, 0) - pages[n++] = sg_page_iter_page(&sg_iter); - - obj->mapping = vmap(pages, n, 0, PAGE_KERNEL); - drm_free_large(pages); - } - if (obj->mapping == NULL) { + if (!obj->mapping) { + obj->mapping = i915_gem_object_map(obj); + if (!obj->mapping) { i915_gem_object_unpin_pages(obj); return ERR_PTR(-ENOMEM); } @@ -2502,9 +2765,8 @@ i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring) } static int -i915_gem_init_seqno(struct drm_device *dev, u32 seqno) +i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; int ret; @@ -2514,7 +2776,14 @@ i915_gem_init_seqno(struct drm_device *dev, u32 seqno) if (ret) return ret; } - i915_gem_retire_requests(dev); + i915_gem_retire_requests(dev_priv); + + /* If the seqno wraps around, we need to clear the breadcrumb rbtree */ + if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) { + while (intel_kick_waiters(dev_priv) || + intel_kick_signalers(dev_priv)) + yield(); + } /* Finally reset hw state */ for_each_engine(engine, dev_priv) @@ -2525,7 +2794,7 @@ i915_gem_init_seqno(struct drm_device *dev, u32 seqno) int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; if (seqno == 0) @@ -2534,7 +2803,7 @@ int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) /* HWS page needs to be set less than what we * will inject to ring */ - ret = i915_gem_init_seqno(dev, seqno - 1); + ret = i915_gem_init_seqno(dev_priv, seqno - 1); if (ret) return ret; @@ -2550,13 +2819,11 @@ int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) } int -i915_gem_get_seqno(struct drm_device *dev, u32 *seqno) +i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno) { - struct drm_i915_private *dev_priv = dev->dev_private; - /* reserve 0 for non-seqno */ if (dev_priv->next_seqno == 0) { - int ret = i915_gem_init_seqno(dev, 0); + int ret = i915_gem_init_seqno(dev_priv, 0); if (ret) return ret; @@ -2567,6 +2834,26 @@ i915_gem_get_seqno(struct drm_device *dev, u32 *seqno) return 0; } +static void i915_gem_mark_busy(const struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + + dev_priv->gt.active_engines |= intel_engine_flag(engine); + if (dev_priv->gt.awake) + return; + + intel_runtime_pm_get_noresume(dev_priv); + dev_priv->gt.awake = true; + + i915_update_gfx_val(dev_priv); + if (INTEL_GEN(dev_priv) >= 6) + gen6_rps_busy(dev_priv); + + queue_delayed_work(dev_priv->wq, + &dev_priv->gt.retire_work, + round_jiffies_up_relative(HZ)); +} + /* * NB: This function is not allowed to fail. Doing so would mean the the * request is not being tracked for completion but the work itself is @@ -2577,16 +2864,15 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) { struct intel_engine_cs *engine; - struct drm_i915_private *dev_priv; struct intel_ringbuffer *ringbuf; u32 request_start; + u32 reserved_tail; int ret; if (WARN_ON(request == NULL)) return; engine = request->engine; - dev_priv = request->i915; ringbuf = request->ringbuf; /* @@ -2594,9 +2880,10 @@ void __i915_add_request(struct drm_i915_gem_request *request, * should already have been reserved in the ring buffer. Let the ring * know that it is time to use that space up. */ - intel_ring_reserved_space_use(ringbuf); - request_start = intel_ring_get_tail(ringbuf); + reserved_tail = request->reserved_space; + request->reserved_space = 0; + /* * Emit any outstanding flushes - execbuf can fail to emit the flush * after having emitted the batchbuffer command. Hence we need to fix @@ -2651,56 +2938,42 @@ void __i915_add_request(struct drm_i915_gem_request *request, } /* Not allowed to fail! */ WARN(ret, "emit|add_request failed: %d!\n", ret); - - i915_queue_hangcheck(engine->dev); - - queue_delayed_work(dev_priv->wq, - &dev_priv->mm.retire_work, - round_jiffies_up_relative(HZ)); - intel_mark_busy(dev_priv->dev); - /* Sanity check that the reserved size was large enough. */ - intel_ring_reserved_space_end(ringbuf); + ret = intel_ring_get_tail(ringbuf) - request_start; + if (ret < 0) + ret += ringbuf->size; + WARN_ONCE(ret > reserved_tail, + "Not enough space reserved (%d bytes) " + "for adding the request (%d bytes)\n", + reserved_tail, ret); + + i915_gem_mark_busy(engine); } -static bool i915_context_is_banned(struct drm_i915_private *dev_priv, - const struct intel_context *ctx) +static bool i915_context_is_banned(const struct i915_gem_context *ctx) { unsigned long elapsed; - elapsed = get_seconds() - ctx->hang_stats.guilty_ts; - if (ctx->hang_stats.banned) return true; + elapsed = get_seconds() - ctx->hang_stats.guilty_ts; if (ctx->hang_stats.ban_period_seconds && elapsed <= ctx->hang_stats.ban_period_seconds) { - if (!i915_gem_context_is_default(ctx)) { - DRM_DEBUG("context hanging too fast, banning!\n"); - return true; - } else if (i915_stop_ring_allow_ban(dev_priv)) { - if (i915_stop_ring_allow_warn(dev_priv)) - DRM_ERROR("gpu hanging too fast, banning!\n"); - return true; - } + DRM_DEBUG("context hanging too fast, banning!\n"); + return true; } return false; } -static void i915_set_reset_status(struct drm_i915_private *dev_priv, - struct intel_context *ctx, +static void i915_set_reset_status(struct i915_gem_context *ctx, const bool guilty) { - struct i915_ctx_hang_stats *hs; - - if (WARN_ON(!ctx)) - return; - - hs = &ctx->hang_stats; + struct i915_ctx_hang_stats *hs = &ctx->hang_stats; if (guilty) { - hs->banned = i915_context_is_banned(dev_priv, ctx); + hs->banned = i915_context_is_banned(ctx); hs->batch_active++; hs->guilty_ts = get_seconds(); } else { @@ -2712,27 +2985,15 @@ void i915_gem_request_free(struct kref *req_ref) { struct drm_i915_gem_request *req = container_of(req_ref, typeof(*req), ref); - struct intel_context *ctx = req->ctx; - - if (req->file_priv) - i915_gem_request_remove_from_client(req); - - if (ctx) { - if (i915.enable_execlists && ctx != req->i915->kernel_context) - intel_lr_context_unpin(ctx, req->engine); - - i915_gem_context_unreference(ctx); - } - kmem_cache_free(req->i915->requests, req); } static inline int __i915_gem_request_alloc(struct intel_engine_cs *engine, - struct intel_context *ctx, + struct i915_gem_context *ctx, struct drm_i915_gem_request **req_out) { - struct drm_i915_private *dev_priv = to_i915(engine->dev); + struct drm_i915_private *dev_priv = engine->i915; unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error); struct drm_i915_gem_request *req; int ret; @@ -2754,26 +3015,16 @@ __i915_gem_request_alloc(struct intel_engine_cs *engine, if (req == NULL) return -ENOMEM; - ret = i915_gem_get_seqno(engine->dev, &req->seqno); + ret = i915_gem_get_seqno(engine->i915, &req->seqno); if (ret) goto err; kref_init(&req->ref); req->i915 = dev_priv; req->engine = engine; - req->reset_counter = reset_counter; req->ctx = ctx; i915_gem_context_reference(req->ctx); - if (i915.enable_execlists) - ret = intel_logical_ring_alloc_request_extras(req); - else - ret = intel_ring_alloc_request_extras(req); - if (ret) { - i915_gem_context_unreference(req->ctx); - goto err; - } - /* * Reserve space in the ring buffer for all the commands required to * eventually emit this request. This is to guarantee that the @@ -2781,24 +3032,20 @@ __i915_gem_request_alloc(struct intel_engine_cs *engine, * to be redone if the request is not actually submitted straight * away, e.g. because a GPU scheduler has deferred it. */ + req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST; + if (i915.enable_execlists) - ret = intel_logical_ring_reserve_space(req); + ret = intel_logical_ring_alloc_request_extras(req); else - ret = intel_ring_reserve_space(req); - if (ret) { - /* - * At this point, the request is fully allocated even if not - * fully prepared. Thus it can be cleaned up using the proper - * free code. - */ - intel_ring_reserved_space_cancel(req->ringbuf); - i915_gem_request_unreference(req); - return ret; - } + ret = intel_ring_alloc_request_extras(req); + if (ret) + goto err_ctx; *req_out = req; return 0; +err_ctx: + i915_gem_context_unreference(ctx); err: kmem_cache_free(dev_priv->requests, req); return ret; @@ -2818,13 +3065,13 @@ err: */ struct drm_i915_gem_request * i915_gem_request_alloc(struct intel_engine_cs *engine, - struct intel_context *ctx) + struct i915_gem_context *ctx) { struct drm_i915_gem_request *req; int err; if (ctx == NULL) - ctx = to_i915(engine->dev)->kernel_context; + ctx = engine->i915->kernel_context; err = __i915_gem_request_alloc(engine, ctx, &req); return err ? ERR_PTR(err) : req; } @@ -2834,8 +3081,16 @@ i915_gem_find_active_request(struct intel_engine_cs *engine) { struct drm_i915_gem_request *request; + /* We are called by the error capture and reset at a random + * point in time. In particular, note that neither is crucially + * ordered with an interrupt. After a hang, the GPU is dead and we + * assume that no more writes can happen (we waited long enough for + * all writes that were in transaction to be flushed) - adding an + * extra delay for a recent interrupt is pointless. Hence, we do + * not need an engine->irq_seqno_barrier() before the seqno reads. + */ list_for_each_entry(request, &engine->request_list, list) { - if (i915_gem_request_completed(request, false)) + if (i915_gem_request_completed(request)) continue; return request; @@ -2844,27 +3099,23 @@ i915_gem_find_active_request(struct intel_engine_cs *engine) return NULL; } -static void i915_gem_reset_engine_status(struct drm_i915_private *dev_priv, - struct intel_engine_cs *engine) +static void i915_gem_reset_engine_status(struct intel_engine_cs *engine) { struct drm_i915_gem_request *request; bool ring_hung; request = i915_gem_find_active_request(engine); - if (request == NULL) return; ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG; - i915_set_reset_status(dev_priv, request->ctx, ring_hung); - + i915_set_reset_status(request->ctx, ring_hung); list_for_each_entry_continue(request, &engine->request_list, list) - i915_set_reset_status(dev_priv, request->ctx, false); + i915_set_reset_status(request->ctx, false); } -static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv, - struct intel_engine_cs *engine) +static void i915_gem_reset_engine_cleanup(struct intel_engine_cs *engine) { struct intel_ringbuffer *buffer; @@ -2888,13 +3139,7 @@ static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv, /* Ensure irq handler finishes or is cancelled. */ tasklet_kill(&engine->irq_tasklet); - spin_lock_bh(&engine->execlist_lock); - /* list_splice_tail_init checks for empty lists */ - list_splice_tail_init(&engine->execlist_queue, - &engine->execlist_retired_req_list); - spin_unlock_bh(&engine->execlist_lock); - - intel_execlists_retire_requests(engine); + intel_execlists_cancel_requests(engine); } /* @@ -2927,11 +3172,13 @@ static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv, } intel_ring_init_seqno(engine, engine->last_submitted_seqno); + + engine->i915->gt.active_engines &= ~intel_engine_flag(engine); } void i915_gem_reset(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; /* @@ -2940,10 +3187,11 @@ void i915_gem_reset(struct drm_device *dev) * their reference to the objects, the inspection must be done first. */ for_each_engine(engine, dev_priv) - i915_gem_reset_engine_status(dev_priv, engine); + i915_gem_reset_engine_status(engine); for_each_engine(engine, dev_priv) - i915_gem_reset_engine_cleanup(dev_priv, engine); + i915_gem_reset_engine_cleanup(engine); + mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0); i915_gem_context_reset(dev); @@ -2954,6 +3202,7 @@ void i915_gem_reset(struct drm_device *dev) /** * This function clears the request list as sequence numbers are passed. + * @engine: engine to retire requests on */ void i915_gem_retire_requests_ring(struct intel_engine_cs *engine) @@ -2972,7 +3221,7 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *engine) struct drm_i915_gem_request, list); - if (!i915_gem_request_completed(request, true)) + if (!i915_gem_request_completed(request)) break; i915_gem_request_retire(request); @@ -2995,58 +3244,52 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *engine) i915_gem_object_retire__read(obj, engine->id); } - if (unlikely(engine->trace_irq_req && - i915_gem_request_completed(engine->trace_irq_req, true))) { - engine->irq_put(engine); - i915_gem_request_assign(&engine->trace_irq_req, NULL); - } - WARN_ON(i915_verify_lists(engine->dev)); } -bool -i915_gem_retire_requests(struct drm_device *dev) +void i915_gem_retire_requests(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; - bool idle = true; + + lockdep_assert_held(&dev_priv->drm.struct_mutex); + + if (dev_priv->gt.active_engines == 0) + return; + + GEM_BUG_ON(!dev_priv->gt.awake); for_each_engine(engine, dev_priv) { i915_gem_retire_requests_ring(engine); - idle &= list_empty(&engine->request_list); - if (i915.enable_execlists) { - spin_lock_bh(&engine->execlist_lock); - idle &= list_empty(&engine->execlist_queue); - spin_unlock_bh(&engine->execlist_lock); - - intel_execlists_retire_requests(engine); - } + if (list_empty(&engine->request_list)) + dev_priv->gt.active_engines &= ~intel_engine_flag(engine); } - if (idle) - mod_delayed_work(dev_priv->wq, - &dev_priv->mm.idle_work, + if (dev_priv->gt.active_engines == 0) + queue_delayed_work(dev_priv->wq, + &dev_priv->gt.idle_work, msecs_to_jiffies(100)); - - return idle; } static void i915_gem_retire_work_handler(struct work_struct *work) { struct drm_i915_private *dev_priv = - container_of(work, typeof(*dev_priv), mm.retire_work.work); - struct drm_device *dev = dev_priv->dev; - bool idle; + container_of(work, typeof(*dev_priv), gt.retire_work.work); + struct drm_device *dev = &dev_priv->drm; /* Come back later if the device is busy... */ - idle = false; if (mutex_trylock(&dev->struct_mutex)) { - idle = i915_gem_retire_requests(dev); + i915_gem_retire_requests(dev_priv); mutex_unlock(&dev->struct_mutex); } - if (!idle) - queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, + + /* Keep the retire handler running until we are finally idle. + * We do not need to do this test under locking as in the worst-case + * we queue the retire worker once too often. + */ + if (READ_ONCE(dev_priv->gt.awake)) + queue_delayed_work(dev_priv->wq, + &dev_priv->gt.retire_work, round_jiffies_up_relative(HZ)); } @@ -3054,25 +3297,55 @@ static void i915_gem_idle_work_handler(struct work_struct *work) { struct drm_i915_private *dev_priv = - container_of(work, typeof(*dev_priv), mm.idle_work.work); - struct drm_device *dev = dev_priv->dev; + container_of(work, typeof(*dev_priv), gt.idle_work.work); + struct drm_device *dev = &dev_priv->drm; struct intel_engine_cs *engine; + unsigned int stuck_engines; + bool rearm_hangcheck; + + if (!READ_ONCE(dev_priv->gt.awake)) + return; + + if (READ_ONCE(dev_priv->gt.active_engines)) + return; + + rearm_hangcheck = + cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); + + if (!mutex_trylock(&dev->struct_mutex)) { + /* Currently busy, come back later */ + mod_delayed_work(dev_priv->wq, + &dev_priv->gt.idle_work, + msecs_to_jiffies(50)); + goto out_rearm; + } + + if (dev_priv->gt.active_engines) + goto out_unlock; for_each_engine(engine, dev_priv) - if (!list_empty(&engine->request_list)) - return; + i915_gem_batch_pool_fini(&engine->batch_pool); - /* we probably should sync with hangcheck here, using cancel_work_sync. - * Also locking seems to be fubar here, engine->request_list is protected - * by dev->struct_mutex. */ + GEM_BUG_ON(!dev_priv->gt.awake); + dev_priv->gt.awake = false; + rearm_hangcheck = false; - intel_mark_idle(dev); + stuck_engines = intel_kick_waiters(dev_priv); + if (unlikely(stuck_engines)) { + DRM_DEBUG_DRIVER("kicked stuck waiters...missed irq\n"); + dev_priv->gpu_error.missed_irq_rings |= stuck_engines; + } - if (mutex_trylock(&dev->struct_mutex)) { - for_each_engine(engine, dev_priv) - i915_gem_batch_pool_fini(&engine->batch_pool); + if (INTEL_GEN(dev_priv) >= 6) + gen6_rps_idle(dev_priv); + intel_runtime_pm_put(dev_priv); +out_unlock: + mutex_unlock(&dev->struct_mutex); - mutex_unlock(&dev->struct_mutex); +out_rearm: + if (rearm_hangcheck) { + GEM_BUG_ON(!dev_priv->gt.awake); + i915_queue_hangcheck(dev_priv); } } @@ -3080,6 +3353,7 @@ i915_gem_idle_work_handler(struct work_struct *work) * Ensures that an object will eventually get non-busy by flushing any required * write domains, emitting any outstanding lazy request and retiring and * completed requests. + * @obj: object to flush */ static int i915_gem_object_flush_active(struct drm_i915_gem_object *obj) @@ -3096,14 +3370,8 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj) if (req == NULL) continue; - if (list_empty(&req->list)) - goto retire; - - if (i915_gem_request_completed(req, true)) { - __i915_gem_request_retire__upto(req); -retire: + if (i915_gem_request_completed(req)) i915_gem_object_retire__read(obj, i); - } } return 0; @@ -3111,7 +3379,9 @@ retire: /** * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT - * @DRM_IOCTL_ARGS: standard ioctl arguments + * @dev: drm device pointer + * @data: ioctl data blob + * @file: drm file pointer * * Returns 0 if successful, else an error is returned with the remaining time in * the timeout parameter. @@ -3185,7 +3455,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) ret = __i915_wait_request(req[i], true, args->timeout_ns > 0 ? &args->timeout_ns : NULL, to_rps_client(file)); - i915_gem_request_unreference__unlocked(req[i]); + i915_gem_request_unreference(req[i]); } return ret; @@ -3208,10 +3478,10 @@ __i915_gem_object_sync(struct drm_i915_gem_object *obj, if (to == from) return 0; - if (i915_gem_request_completed(from_req, true)) + if (i915_gem_request_completed(from_req)) return 0; - if (!i915_semaphore_is_enabled(obj->base.dev)) { + if (!i915_semaphore_is_enabled(to_i915(obj->base.dev))) { struct drm_i915_private *i915 = to_i915(obj->base.dev); ret = __i915_wait_request(from_req, i915->mm.interruptible, @@ -3345,10 +3615,21 @@ static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) old_write_domain); } +static void __i915_vma_iounmap(struct i915_vma *vma) +{ + GEM_BUG_ON(vma->pin_count); + + if (vma->iomap == NULL) + return; + + io_mapping_unmap(vma->iomap); + vma->iomap = NULL; +} + static int __i915_vma_unbind(struct i915_vma *vma, bool wait) { struct drm_i915_gem_object *obj = vma->obj; - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); int ret; if (list_empty(&vma->obj_link)) @@ -3377,6 +3658,8 @@ static int __i915_vma_unbind(struct i915_vma *vma, bool wait) ret = i915_gem_object_put_fence(obj); if (ret) return ret; + + __i915_vma_iounmap(vma); } trace_i915_vma_unbind(vma); @@ -3422,26 +3705,16 @@ int __i915_vma_unbind_no_wait(struct i915_vma *vma) return __i915_vma_unbind(vma, false); } -int i915_gpu_idle(struct drm_device *dev) +int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; int ret; - /* Flush everything onto the inactive list. */ - for_each_engine(engine, dev_priv) { - if (!i915.enable_execlists) { - struct drm_i915_gem_request *req; + lockdep_assert_held(&dev_priv->drm.struct_mutex); - req = i915_gem_request_alloc(engine, NULL); - if (IS_ERR(req)) - return PTR_ERR(req); - - ret = i915_switch_context(req); - i915_add_request_no_flush(req); - if (ret) - return ret; - } + for_each_engine(engine, dev_priv) { + if (engine->last_context == NULL) + continue; ret = intel_engine_idle(engine); if (ret) @@ -3488,6 +3761,11 @@ static bool i915_gem_valid_gtt_space(struct i915_vma *vma, /** * Finds free space in the GTT aperture and binds the object or a view of it * there. + * @obj: object to bind + * @vm: address space to bind into + * @ggtt_view: global gtt view if applicable + * @alignment: requested alignment + * @flags: mask of PIN_* flags to use */ static struct i915_vma * i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, @@ -3731,7 +4009,7 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) return; if (i915_gem_clflush_object(obj, obj->pin_display)) - i915_gem_chipset_flush(obj->base.dev); + i915_gem_chipset_flush(to_i915(obj->base.dev)); old_write_domain = obj->base.write_domain; obj->base.write_domain = 0; @@ -3745,6 +4023,8 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) /** * Moves a single object to the GTT read, and possibly write domain. + * @obj: object to act on + * @write: ask for write access or read only * * This function returns when the move is complete, including waiting on * flushes to occur. @@ -3816,6 +4096,8 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) /** * Changes the cache-level of an object across all VMA. + * @obj: object to act on + * @cache_level: new cache level to set for the object * * After this function returns, the object will be in the new cache-level * across all GTT and the contents of the backing storage will be coherent, @@ -3925,11 +4207,9 @@ out: * object is now coherent at its new cache level (with respect * to the access domain). */ - if (obj->cache_dirty && - obj->base.write_domain != I915_GEM_DOMAIN_CPU && - cpu_write_needs_clflush(obj)) { + if (obj->cache_dirty && cpu_write_needs_clflush(obj)) { if (i915_gem_clflush_object(obj, true)) - i915_gem_chipset_flush(obj->base.dev); + i915_gem_chipset_flush(to_i915(obj->base.dev)); } return 0; @@ -3967,7 +4247,7 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_caching *args = data; struct drm_i915_gem_object *obj; enum i915_cache_level level; @@ -4097,6 +4377,8 @@ i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj, /** * Moves a single object to the CPU read, and possibly write domain. + * @obj: object to act on + * @write: requesting write or read-only access * * This function returns when the move is complete, including waiting on * flushes to occur. @@ -4159,7 +4441,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) static int i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_file_private *file_priv = file->driver_priv; unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; struct drm_i915_gem_request *request, *target = NULL; @@ -4195,10 +4477,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) return 0; ret = __i915_wait_request(target, true, NULL, NULL); - if (ret == 0) - queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); - - i915_gem_request_unreference__unlocked(target); + i915_gem_request_unreference(target); return ret; } @@ -4256,7 +4535,7 @@ i915_gem_object_do_pin(struct drm_i915_gem_object *obj, uint32_t alignment, uint64_t flags) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); struct i915_vma *vma; unsigned bound; int ret; @@ -4420,7 +4699,7 @@ int i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_madvise *args = data; struct drm_i915_gem_object *obj; int ret; @@ -4490,7 +4769,7 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, obj->fence_reg = I915_FENCE_REG_NONE; obj->madv = I915_MADV_WILLNEED; - i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); + i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size); } static const struct drm_i915_gem_object_ops i915_gem_object_ops = { @@ -4499,21 +4778,21 @@ static const struct drm_i915_gem_object_ops i915_gem_object_ops = { .put_pages = i915_gem_object_put_pages_gtt, }; -struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, +struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev, size_t size) { struct drm_i915_gem_object *obj; struct address_space *mapping; gfp_t mask; + int ret; obj = i915_gem_object_alloc(dev); if (obj == NULL) - return NULL; + return ERR_PTR(-ENOMEM); - if (drm_gem_object_init(dev, &obj->base, size) != 0) { - i915_gem_object_free(obj); - return NULL; - } + ret = drm_gem_object_init(dev, &obj->base, size); + if (ret) + goto fail; mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) { @@ -4522,7 +4801,7 @@ struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, mask |= __GFP_DMA32; } - mapping = file_inode(obj->base.filp)->i_mapping; + mapping = obj->base.filp->f_mapping; mapping_set_gfp_mask(mapping, mask); i915_gem_object_init(obj, &i915_gem_object_ops); @@ -4550,6 +4829,11 @@ struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, trace_i915_gem_object_create(obj); return obj; + +fail: + i915_gem_object_free(obj); + + return ERR_PTR(ret); } static bool discard_backing_storage(struct drm_i915_gem_object *obj) @@ -4580,7 +4864,7 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) { struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct i915_vma *vma, *next; intel_runtime_pm_get(dev_priv); @@ -4655,16 +4939,12 @@ struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj, const struct i915_ggtt_view *view) { - struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; struct i915_vma *vma; - BUG_ON(!view); + GEM_BUG_ON(!view); list_for_each_entry(vma, &obj->vma_list, obj_link) - if (vma->vm == &ggtt->base && - i915_ggtt_view_equal(&vma->ggtt_view, view)) + if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view)) return vma; return NULL; } @@ -4688,7 +4968,7 @@ void i915_gem_vma_destroy(struct i915_vma *vma) static void i915_gem_stop_engines(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; for_each_engine(engine, dev_priv) @@ -4698,27 +4978,28 @@ i915_gem_stop_engines(struct drm_device *dev) int i915_gem_suspend(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret = 0; mutex_lock(&dev->struct_mutex); - ret = i915_gpu_idle(dev); + ret = i915_gem_wait_for_idle(dev_priv); if (ret) goto err; - i915_gem_retire_requests(dev); + i915_gem_retire_requests(dev_priv); i915_gem_stop_engines(dev); + i915_gem_context_lost(dev_priv); mutex_unlock(&dev->struct_mutex); cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); - cancel_delayed_work_sync(&dev_priv->mm.retire_work); - flush_delayed_work(&dev_priv->mm.idle_work); + cancel_delayed_work_sync(&dev_priv->gt.retire_work); + flush_delayed_work(&dev_priv->gt.idle_work); /* Assert that we sucessfully flushed all the work and * reset the GPU back to its idle, low power state. */ - WARN_ON(dev_priv->mm.busy); + WARN_ON(dev_priv->gt.awake); return 0; @@ -4727,40 +5008,9 @@ err: return ret; } -int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice) -{ - struct intel_engine_cs *engine = req->engine; - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - u32 *remap_info = dev_priv->l3_parity.remap_info[slice]; - int i, ret; - - if (!HAS_L3_DPF(dev) || !remap_info) - return 0; - - ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3); - if (ret) - return ret; - - /* - * Note: We do not worry about the concurrent register cacheline hang - * here because no other code should access these registers other than - * at initialization time. - */ - for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) { - intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit_reg(engine, GEN7_L3LOG(slice, i)); - intel_ring_emit(engine, remap_info[i]); - } - - intel_ring_advance(engine); - - return ret; -} - void i915_gem_init_swizzling(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (INTEL_INFO(dev)->gen < 5 || dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) @@ -4785,7 +5035,7 @@ void i915_gem_init_swizzling(struct drm_device *dev) static void init_unused_ring(struct drm_device *dev, u32 base) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(RING_CTL(base), 0); I915_WRITE(RING_HEAD(base), 0); @@ -4812,7 +5062,7 @@ static void init_unused_rings(struct drm_device *dev) int i915_gem_init_engines(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; ret = intel_init_render_ring_buffer(dev); @@ -4860,9 +5110,9 @@ cleanup_render_ring: int i915_gem_init_hw(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; - int ret, j; + int ret; /* Double layer security blanket, see i915_gem_init() */ intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); @@ -4914,59 +5164,10 @@ i915_gem_init_hw(struct drm_device *dev) intel_mocs_init_l3cc_table(dev); /* We can't enable contexts until all firmware is loaded */ - if (HAS_GUC_UCODE(dev)) { - ret = intel_guc_ucode_load(dev); - if (ret) { - DRM_ERROR("Failed to initialize GuC, error %d\n", ret); - ret = -EIO; - goto out; - } - } - - /* - * Increment the next seqno by 0x100 so we have a visible break - * on re-initialisation - */ - ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100); + ret = intel_guc_setup(dev); if (ret) goto out; - /* Now it is safe to go back round and do everything else: */ - for_each_engine(engine, dev_priv) { - struct drm_i915_gem_request *req; - - req = i915_gem_request_alloc(engine, NULL); - if (IS_ERR(req)) { - ret = PTR_ERR(req); - break; - } - - if (engine->id == RCS) { - for (j = 0; j < NUM_L3_SLICES(dev); j++) { - ret = i915_gem_l3_remap(req, j); - if (ret) - goto err_request; - } - } - - ret = i915_ppgtt_init_ring(req); - if (ret) - goto err_request; - - ret = i915_gem_context_enable(req); - if (ret) - goto err_request; - -err_request: - i915_add_request_no_flush(req); - if (ret) { - DRM_ERROR("Failed to enable %s, error=%d\n", - engine->name, ret); - i915_gem_cleanup_engines(dev); - break; - } - } - out: intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); return ret; @@ -4974,12 +5175,9 @@ out: int i915_gem_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; - i915.enable_execlists = intel_sanitize_enable_execlists(dev, - i915.enable_execlists); - mutex_lock(&dev->struct_mutex); if (!i915.enable_execlists) { @@ -5002,10 +5200,7 @@ int i915_gem_init(struct drm_device *dev) */ intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); - ret = i915_gem_init_userptr(dev); - if (ret) - goto out_unlock; - + i915_gem_init_userptr(dev_priv); i915_gem_init_ggtt(dev); ret = i915_gem_context_init(dev); @@ -5037,19 +5232,11 @@ out_unlock: void i915_gem_cleanup_engines(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; for_each_engine(engine, dev_priv) dev_priv->gt.cleanup_engine(engine); - - if (i915.enable_execlists) - /* - * Neither the BIOS, ourselves or any other kernel - * expects the system to be in execlists mode on startup, - * so we need to reset the GPU back to legacy mode. - */ - intel_gpu_reset(dev, ALL_ENGINES); } static void @@ -5062,7 +5249,7 @@ init_engine_lists(struct intel_engine_cs *engine) void i915_gem_load_init_fences(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) @@ -5073,7 +5260,7 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv) else dev_priv->num_fence_regs = 8; - if (intel_vgpu_active(dev)) + if (intel_vgpu_active(dev_priv)) dev_priv->num_fence_regs = I915_READ(vgtif_reg(avail_rs.fence_num)); @@ -5086,7 +5273,7 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv) void i915_gem_load_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i; dev_priv->objects = @@ -5114,22 +5301,15 @@ i915_gem_load_init(struct drm_device *dev) init_engine_lists(&dev_priv->engine[i]); for (i = 0; i < I915_MAX_NUM_FENCES; i++) INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); - INIT_DELAYED_WORK(&dev_priv->mm.retire_work, + INIT_DELAYED_WORK(&dev_priv->gt.retire_work, i915_gem_retire_work_handler); - INIT_DELAYED_WORK(&dev_priv->mm.idle_work, + INIT_DELAYED_WORK(&dev_priv->gt.idle_work, i915_gem_idle_work_handler); + init_waitqueue_head(&dev_priv->gpu_error.wait_queue); init_waitqueue_head(&dev_priv->gpu_error.reset_queue); dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; - /* - * Set initial sequence number for requests. - * Using this number allows the wraparound to happen early, - * catching any obvious problems. - */ - dev_priv->next_seqno = ((u32)~0 - 0x1100); - dev_priv->last_seqno = ((u32)~0 - 0x1101); - INIT_LIST_HEAD(&dev_priv->mm.fence_list); init_waitqueue_head(&dev_priv->pending_flip_queue); @@ -5148,6 +5328,34 @@ void i915_gem_load_cleanup(struct drm_device *dev) kmem_cache_destroy(dev_priv->objects); } +int i915_gem_freeze_late(struct drm_i915_private *dev_priv) +{ + struct drm_i915_gem_object *obj; + + /* Called just before we write the hibernation image. + * + * We need to update the domain tracking to reflect that the CPU + * will be accessing all the pages to create and restore from the + * hibernation, and so upon restoration those pages will be in the + * CPU domain. + * + * To make sure the hibernation image contains the latest state, + * we update that state just before writing out the image. + */ + + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + obj->base.read_domains = I915_GEM_DOMAIN_CPU; + obj->base.write_domain = I915_GEM_DOMAIN_CPU; + } + + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + obj->base.read_domains = I915_GEM_DOMAIN_CPU; + obj->base.write_domain = I915_GEM_DOMAIN_CPU; + } + + return 0; +} + void i915_gem_release(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; @@ -5187,7 +5395,7 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file) return -ENOMEM; file->driver_priv = file_priv; - file_priv->dev_priv = dev->dev_private; + file_priv->dev_priv = to_i915(dev); file_priv->file = file; INIT_LIST_HEAD(&file_priv->rps.link); @@ -5233,7 +5441,7 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, u64 i915_gem_obj_offset(struct drm_i915_gem_object *o, struct i915_address_space *vm) { - struct drm_i915_private *dev_priv = o->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(o->base.dev); struct i915_vma *vma; WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); @@ -5254,13 +5462,10 @@ u64 i915_gem_obj_offset(struct drm_i915_gem_object *o, u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o, const struct i915_ggtt_view *view) { - struct drm_i915_private *dev_priv = to_i915(o->base.dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; struct i915_vma *vma; list_for_each_entry(vma, &o->vma_list, obj_link) - if (vma->vm == &ggtt->base && - i915_ggtt_view_equal(&vma->ggtt_view, view)) + if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view)) return vma->node.start; WARN(1, "global vma for this object not found. (view=%u)\n", view->type); @@ -5286,12 +5491,10 @@ bool i915_gem_obj_bound(struct drm_i915_gem_object *o, bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o, const struct i915_ggtt_view *view) { - struct drm_i915_private *dev_priv = to_i915(o->base.dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; struct i915_vma *vma; list_for_each_entry(vma, &o->vma_list, obj_link) - if (vma->vm == &ggtt->base && + if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view) && drm_mm_node_allocated(&vma->node)) return true; @@ -5310,23 +5513,18 @@ bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o) return false; } -unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o, - struct i915_address_space *vm) +unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o) { - struct drm_i915_private *dev_priv = o->base.dev->dev_private; struct i915_vma *vma; - WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); - - BUG_ON(list_empty(&o->vma_list)); + GEM_BUG_ON(list_empty(&o->vma_list)); list_for_each_entry(vma, &o->vma_list, obj_link) { if (vma->is_ggtt && - vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) - continue; - if (vma->vm == vm) + vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) return vma->node.size; } + return 0; } @@ -5347,7 +5545,7 @@ i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n) struct page *page; /* Only default objects have per-page dirty tracking */ - if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0)) + if (WARN_ON(!i915_gem_object_has_struct_page(obj))) return NULL; page = i915_gem_object_get_page(obj, n); @@ -5365,8 +5563,8 @@ i915_gem_object_create_from_data(struct drm_device *dev, size_t bytes; int ret; - obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE)); - if (IS_ERR_OR_NULL(obj)) + obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE)); + if (IS_ERR(obj)) return obj; ret = i915_gem_object_set_to_cpu_domain(obj, true); diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c index 7bf2f3f29..3752d5daa 100644 --- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c +++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c @@ -134,9 +134,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, if (obj == NULL) { int ret; - obj = i915_gem_alloc_object(pool->dev, size); - if (obj == NULL) - return ERR_PTR(-ENOMEM); + obj = i915_gem_object_create(pool->dev, size); + if (IS_ERR(obj)) + return obj; ret = i915_gem_object_get_pages(obj); if (ret) diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index e5acc3916..3c97f0e7a 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -90,6 +90,8 @@ #include "i915_drv.h" #include "i915_trace.h" +#define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 + /* This is a HW constraint. The value below is the largest known requirement * I've seen in a spec to date, and that was a workaround for a non-shipping * part. It should be safe to decrease this, but it's more future proof as is. @@ -97,28 +99,27 @@ #define GEN6_CONTEXT_ALIGN (64<<10) #define GEN7_CONTEXT_ALIGN 4096 -static size_t get_context_alignment(struct drm_device *dev) +static size_t get_context_alignment(struct drm_i915_private *dev_priv) { - if (IS_GEN6(dev)) + if (IS_GEN6(dev_priv)) return GEN6_CONTEXT_ALIGN; return GEN7_CONTEXT_ALIGN; } -static int get_context_size(struct drm_device *dev) +static int get_context_size(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; int ret; u32 reg; - switch (INTEL_INFO(dev)->gen) { + switch (INTEL_GEN(dev_priv)) { case 6: reg = I915_READ(CXT_SIZE); ret = GEN6_CXT_TOTAL_SIZE(reg) * 64; break; case 7: reg = I915_READ(GEN7_CXT_SIZE); - if (IS_HASWELL(dev)) + if (IS_HASWELL(dev_priv)) ret = HSW_CXT_TOTAL_SIZE; else ret = GEN7_CXT_TOTAL_SIZE(reg) * 64; @@ -133,7 +134,7 @@ static int get_context_size(struct drm_device *dev) return ret; } -static void i915_gem_context_clean(struct intel_context *ctx) +static void i915_gem_context_clean(struct i915_gem_context *ctx) { struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; struct i915_vma *vma, *next; @@ -150,13 +151,12 @@ static void i915_gem_context_clean(struct intel_context *ctx) void i915_gem_context_free(struct kref *ctx_ref) { - struct intel_context *ctx = container_of(ctx_ref, typeof(*ctx), ref); + struct i915_gem_context *ctx = container_of(ctx_ref, typeof(*ctx), ref); + int i; + lockdep_assert_held(&ctx->i915->drm.struct_mutex); trace_i915_context_free(ctx); - if (i915.enable_execlists) - intel_lr_context_free(ctx); - /* * This context is going away and we need to remove all VMAs still * around. This is to handle imported shared objects for which @@ -166,9 +166,22 @@ void i915_gem_context_free(struct kref *ctx_ref) i915_ppgtt_put(ctx->ppgtt); - if (ctx->legacy_hw_ctx.rcs_state) - drm_gem_object_unreference(&ctx->legacy_hw_ctx.rcs_state->base); + for (i = 0; i < I915_NUM_ENGINES; i++) { + struct intel_context *ce = &ctx->engine[i]; + + if (!ce->state) + continue; + + WARN_ON(ce->pin_count); + if (ce->ringbuf) + intel_ringbuffer_free(ce->ringbuf); + + drm_gem_object_unreference(&ce->state->base); + } + list_del(&ctx->link); + + ida_simple_remove(&ctx->i915->context_hw_ida, ctx->hw_id); kfree(ctx); } @@ -178,9 +191,11 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size) struct drm_i915_gem_object *obj; int ret; - obj = i915_gem_alloc_object(dev, size); - if (obj == NULL) - return ERR_PTR(-ENOMEM); + lockdep_assert_held(&dev->struct_mutex); + + obj = i915_gem_object_create(dev, size); + if (IS_ERR(obj)) + return obj; /* * Try to make the context utilize L3 as well as LLC. @@ -209,22 +224,52 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size) return obj; } -static struct intel_context * +static int assign_hw_id(struct drm_i915_private *dev_priv, unsigned *out) +{ + int ret; + + ret = ida_simple_get(&dev_priv->context_hw_ida, + 0, MAX_CONTEXT_HW_ID, GFP_KERNEL); + if (ret < 0) { + /* Contexts are only released when no longer active. + * Flush any pending retires to hopefully release some + * stale contexts and try again. + */ + i915_gem_retire_requests(dev_priv); + ret = ida_simple_get(&dev_priv->context_hw_ida, + 0, MAX_CONTEXT_HW_ID, GFP_KERNEL); + if (ret < 0) + return ret; + } + + *out = ret; + return 0; +} + +static struct i915_gem_context * __create_hw_context(struct drm_device *dev, struct drm_i915_file_private *file_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_context *ctx; + struct drm_i915_private *dev_priv = to_i915(dev); + struct i915_gem_context *ctx; int ret; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (ctx == NULL) return ERR_PTR(-ENOMEM); + ret = assign_hw_id(dev_priv, &ctx->hw_id); + if (ret) { + kfree(ctx); + return ERR_PTR(ret); + } + kref_init(&ctx->ref); list_add_tail(&ctx->link, &dev_priv->context_list); ctx->i915 = dev_priv; + ctx->ggtt_alignment = get_context_alignment(dev_priv); + if (dev_priv->hw_context_size) { struct drm_i915_gem_object *obj = i915_gem_alloc_context_obj(dev, dev_priv->hw_context_size); @@ -232,7 +277,7 @@ __create_hw_context(struct drm_device *dev, ret = PTR_ERR(obj); goto err_out; } - ctx->legacy_hw_ctx.rcs_state = obj; + ctx->engine[RCS].state = obj; } /* Default context will never have a file_priv */ @@ -249,9 +294,13 @@ __create_hw_context(struct drm_device *dev, /* NB: Mark all slices as needing a remap so that when the context first * loads it will restore whatever remap state already exists. If there * is no remap info, it will be a NOP. */ - ctx->remap_slice = (1 << NUM_L3_SLICES(dev)) - 1; + ctx->remap_slice = ALL_L3_SLICES(dev_priv); ctx->hang_stats.ban_period_seconds = DRM_I915_CTX_BAN_PERIOD; + ctx->ring_size = 4 * PAGE_SIZE; + ctx->desc_template = GEN8_CTX_ADDRESSING_MODE(dev_priv) << + GEN8_CTX_ADDRESSING_MODE_SHIFT; + ATOMIC_INIT_NOTIFIER_HEAD(&ctx->status_notifier); return ctx; @@ -265,44 +314,27 @@ err_out: * context state of the GPU for applications that don't utilize HW contexts, as * well as an idle case. */ -static struct intel_context * +static struct i915_gem_context * i915_gem_create_context(struct drm_device *dev, struct drm_i915_file_private *file_priv) { - const bool is_global_default_ctx = file_priv == NULL; - struct intel_context *ctx; - int ret = 0; + struct i915_gem_context *ctx; - BUG_ON(!mutex_is_locked(&dev->struct_mutex)); + lockdep_assert_held(&dev->struct_mutex); ctx = __create_hw_context(dev, file_priv); if (IS_ERR(ctx)) return ctx; - if (is_global_default_ctx && ctx->legacy_hw_ctx.rcs_state) { - /* We may need to do things with the shrinker which - * require us to immediately switch back to the default - * context. This can cause a problem as pinning the - * default context also requires GTT space which may not - * be available. To avoid this we always pin the default - * context. - */ - ret = i915_gem_obj_ggtt_pin(ctx->legacy_hw_ctx.rcs_state, - get_context_alignment(dev), 0); - if (ret) { - DRM_DEBUG_DRIVER("Couldn't pin %d\n", ret); - goto err_destroy; - } - } - if (USES_FULL_PPGTT(dev)) { struct i915_hw_ppgtt *ppgtt = i915_ppgtt_create(dev, file_priv); - if (IS_ERR_OR_NULL(ppgtt)) { + if (IS_ERR(ppgtt)) { DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n", PTR_ERR(ppgtt)); - ret = PTR_ERR(ppgtt); - goto err_unpin; + idr_remove(&file_priv->context_idr, ctx->user_handle); + i915_gem_context_unreference(ctx); + return ERR_CAST(ppgtt); } ctx->ppgtt = ppgtt; @@ -311,76 +343,102 @@ i915_gem_create_context(struct drm_device *dev, trace_i915_context_create(ctx); return ctx; +} -err_unpin: - if (is_global_default_ctx && ctx->legacy_hw_ctx.rcs_state) - i915_gem_object_ggtt_unpin(ctx->legacy_hw_ctx.rcs_state); -err_destroy: - idr_remove(&file_priv->context_idr, ctx->user_handle); - i915_gem_context_unreference(ctx); - return ERR_PTR(ret); +/** + * i915_gem_context_create_gvt - create a GVT GEM context + * @dev: drm device * + * + * This function is used to create a GVT specific GEM context. + * + * Returns: + * pointer to i915_gem_context on success, error pointer if failed + * + */ +struct i915_gem_context * +i915_gem_context_create_gvt(struct drm_device *dev) +{ + struct i915_gem_context *ctx; + int ret; + + if (!IS_ENABLED(CONFIG_DRM_I915_GVT)) + return ERR_PTR(-ENODEV); + + ret = i915_mutex_lock_interruptible(dev); + if (ret) + return ERR_PTR(ret); + + ctx = i915_gem_create_context(dev, NULL); + if (IS_ERR(ctx)) + goto out; + + ctx->execlists_force_single_submission = true; + ctx->ring_size = 512 * PAGE_SIZE; /* Max ring buffer size */ +out: + mutex_unlock(&dev->struct_mutex); + return ctx; } -static void i915_gem_context_unpin(struct intel_context *ctx, +static void i915_gem_context_unpin(struct i915_gem_context *ctx, struct intel_engine_cs *engine) { if (i915.enable_execlists) { intel_lr_context_unpin(ctx, engine); } else { - if (engine->id == RCS && ctx->legacy_hw_ctx.rcs_state) - i915_gem_object_ggtt_unpin(ctx->legacy_hw_ctx.rcs_state); + struct intel_context *ce = &ctx->engine[engine->id]; + + if (ce->state) + i915_gem_object_ggtt_unpin(ce->state); + i915_gem_context_unreference(ctx); } } void i915_gem_context_reset(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - int i; + struct drm_i915_private *dev_priv = to_i915(dev); + + lockdep_assert_held(&dev->struct_mutex); if (i915.enable_execlists) { - struct intel_context *ctx; + struct i915_gem_context *ctx; list_for_each_entry(ctx, &dev_priv->context_list, link) intel_lr_context_reset(dev_priv, ctx); } - for (i = 0; i < I915_NUM_ENGINES; i++) { - struct intel_engine_cs *engine = &dev_priv->engine[i]; - - if (engine->last_context) { - i915_gem_context_unpin(engine->last_context, engine); - engine->last_context = NULL; - } - } - - /* Force the GPU state to be reinitialised on enabling */ - dev_priv->kernel_context->legacy_hw_ctx.initialized = false; + i915_gem_context_lost(dev_priv); } int i915_gem_context_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_context *ctx; + struct drm_i915_private *dev_priv = to_i915(dev); + struct i915_gem_context *ctx; /* Init should only be called once per module load. Eventually the * restriction on the context_disabled check can be loosened. */ if (WARN_ON(dev_priv->kernel_context)) return 0; - if (intel_vgpu_active(dev) && HAS_LOGICAL_RING_CONTEXTS(dev)) { + if (intel_vgpu_active(dev_priv) && + HAS_LOGICAL_RING_CONTEXTS(dev_priv)) { if (!i915.enable_execlists) { DRM_INFO("Only EXECLIST mode is supported in vgpu.\n"); return -EINVAL; } } + /* Using the simple ida interface, the max is limited by sizeof(int) */ + BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX); + ida_init(&dev_priv->context_hw_ida); + if (i915.enable_execlists) { /* NB: intentionally left blank. We will allocate our own * backing objects as we need them, thank you very much */ dev_priv->hw_context_size = 0; - } else if (HAS_HW_CONTEXTS(dev)) { - dev_priv->hw_context_size = round_up(get_context_size(dev), 4096); + } else if (HAS_HW_CONTEXTS(dev_priv)) { + dev_priv->hw_context_size = + round_up(get_context_size(dev_priv), 4096); if (dev_priv->hw_context_size > (1<<20)) { DRM_DEBUG_DRIVER("Disabling HW Contexts; invalid size %d\n", dev_priv->hw_context_size); @@ -403,67 +461,60 @@ int i915_gem_context_init(struct drm_device *dev) return 0; } -void i915_gem_context_fini(struct drm_device *dev) +void i915_gem_context_lost(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_context *dctx = dev_priv->kernel_context; - int i; - - if (dctx->legacy_hw_ctx.rcs_state) { - /* The only known way to stop the gpu from accessing the hw context is - * to reset it. Do this as the very last operation to avoid confusing - * other code, leading to spurious errors. */ - intel_gpu_reset(dev, ALL_ENGINES); + struct intel_engine_cs *engine; - /* When default context is created and switched to, base object refcount - * will be 2 (+1 from object creation and +1 from do_switch()). - * i915_gem_context_fini() will be called after gpu_idle() has switched - * to default context. So we need to unreference the base object once - * to offset the do_switch part, so that i915_gem_context_unreference() - * can then free the base object correctly. */ - WARN_ON(!dev_priv->engine[RCS].last_context); - - i915_gem_object_ggtt_unpin(dctx->legacy_hw_ctx.rcs_state); - } - - for (i = I915_NUM_ENGINES; --i >= 0;) { - struct intel_engine_cs *engine = &dev_priv->engine[i]; + lockdep_assert_held(&dev_priv->drm.struct_mutex); + for_each_engine(engine, dev_priv) { if (engine->last_context) { i915_gem_context_unpin(engine->last_context, engine); engine->last_context = NULL; } } - i915_gem_context_unreference(dctx); - dev_priv->kernel_context = NULL; -} + /* Force the GPU state to be restored on enabling */ + if (!i915.enable_execlists) { + struct i915_gem_context *ctx; -int i915_gem_context_enable(struct drm_i915_gem_request *req) -{ - struct intel_engine_cs *engine = req->engine; - int ret; + list_for_each_entry(ctx, &dev_priv->context_list, link) { + if (!i915_gem_context_is_default(ctx)) + continue; - if (i915.enable_execlists) { - if (engine->init_context == NULL) - return 0; + for_each_engine(engine, dev_priv) + ctx->engine[engine->id].initialised = false; - ret = engine->init_context(req); - } else - ret = i915_switch_context(req); + ctx->remap_slice = ALL_L3_SLICES(dev_priv); + } - if (ret) { - DRM_ERROR("ring init context: %d\n", ret); - return ret; + for_each_engine(engine, dev_priv) { + struct intel_context *kce = + &dev_priv->kernel_context->engine[engine->id]; + + kce->initialised = true; + } } +} - return 0; +void i915_gem_context_fini(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + struct i915_gem_context *dctx = dev_priv->kernel_context; + + lockdep_assert_held(&dev->struct_mutex); + + i915_gem_context_unreference(dctx); + dev_priv->kernel_context = NULL; + + ida_destroy(&dev_priv->context_hw_ida); } static int context_idr_cleanup(int id, void *p, void *data) { - struct intel_context *ctx = p; + struct i915_gem_context *ctx = p; + ctx->file_priv = ERR_PTR(-EBADF); i915_gem_context_unreference(ctx); return 0; } @@ -471,7 +522,7 @@ static int context_idr_cleanup(int id, void *p, void *data) int i915_gem_context_open(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; - struct intel_context *ctx; + struct i915_gem_context *ctx; idr_init(&file_priv->context_idr); @@ -491,31 +542,22 @@ void i915_gem_context_close(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; + lockdep_assert_held(&dev->struct_mutex); + idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL); idr_destroy(&file_priv->context_idr); } -struct intel_context * -i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id) -{ - struct intel_context *ctx; - - ctx = (struct intel_context *)idr_find(&file_priv->context_idr, id); - if (!ctx) - return ERR_PTR(-ENOENT); - - return ctx; -} - static inline int mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags) { + struct drm_i915_private *dev_priv = req->i915; struct intel_engine_cs *engine = req->engine; u32 flags = hw_flags | MI_MM_SPACE_GTT; const int num_rings = /* Use an extended w/a on ivb+ if signalling from other rings */ - i915_semaphore_is_enabled(engine->dev) ? - hweight32(INTEL_INFO(engine->dev)->ring_mask) - 1 : + i915_semaphore_is_enabled(dev_priv) ? + hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1 : 0; int len, ret; @@ -524,21 +566,21 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags) * explicitly, so we rely on the value at ring init, stored in * itlb_before_ctx_switch. */ - if (IS_GEN6(engine->dev)) { + if (IS_GEN6(dev_priv)) { ret = engine->flush(req, I915_GEM_GPU_DOMAINS, 0); if (ret) return ret; } /* These flags are for resource streamer on HSW+ */ - if (IS_HASWELL(engine->dev) || INTEL_INFO(engine->dev)->gen >= 8) + if (IS_HASWELL(dev_priv) || INTEL_GEN(dev_priv) >= 8) flags |= (HSW_MI_RS_SAVE_STATE_EN | HSW_MI_RS_RESTORE_STATE_EN); - else if (INTEL_INFO(engine->dev)->gen < 8) + else if (INTEL_GEN(dev_priv) < 8) flags |= (MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN); len = 4; - if (INTEL_INFO(engine->dev)->gen >= 7) + if (INTEL_GEN(dev_priv) >= 7) len += 2 + (num_rings ? 4*num_rings + 6 : 0); ret = intel_ring_begin(req, len); @@ -546,14 +588,14 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags) return ret; /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ - if (INTEL_INFO(engine->dev)->gen >= 7) { + if (INTEL_GEN(dev_priv) >= 7) { intel_ring_emit(engine, MI_ARB_ON_OFF | MI_ARB_DISABLE); if (num_rings) { struct intel_engine_cs *signaller; intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(num_rings)); - for_each_engine(signaller, to_i915(engine->dev)) { + for_each_engine(signaller, dev_priv) { if (signaller == engine) continue; @@ -568,7 +610,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags) intel_ring_emit(engine, MI_NOOP); intel_ring_emit(engine, MI_SET_CONTEXT); intel_ring_emit(engine, - i915_gem_obj_ggtt_offset(req->ctx->legacy_hw_ctx.rcs_state) | + i915_gem_obj_ggtt_offset(req->ctx->engine[RCS].state) | flags); /* * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP @@ -576,14 +618,14 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags) */ intel_ring_emit(engine, MI_NOOP); - if (INTEL_INFO(engine->dev)->gen >= 7) { + if (INTEL_GEN(dev_priv) >= 7) { if (num_rings) { struct intel_engine_cs *signaller; i915_reg_t last_reg = {}; /* keep gcc quiet */ intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(num_rings)); - for_each_engine(signaller, to_i915(engine->dev)) { + for_each_engine(signaller, dev_priv) { if (signaller == engine) continue; @@ -609,45 +651,83 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags) return ret; } -static inline bool skip_rcs_switch(struct intel_engine_cs *engine, - struct intel_context *to) +static int remap_l3(struct drm_i915_gem_request *req, int slice) +{ + u32 *remap_info = req->i915->l3_parity.remap_info[slice]; + struct intel_engine_cs *engine = req->engine; + int i, ret; + + if (!remap_info) + return 0; + + ret = intel_ring_begin(req, GEN7_L3LOG_SIZE/4 * 2 + 2); + if (ret) + return ret; + + /* + * Note: We do not worry about the concurrent register cacheline hang + * here because no other code should access these registers other than + * at initialization time. + */ + intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4)); + for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) { + intel_ring_emit_reg(engine, GEN7_L3LOG(slice, i)); + intel_ring_emit(engine, remap_info[i]); + } + intel_ring_emit(engine, MI_NOOP); + intel_ring_advance(engine); + + return 0; +} + +static inline bool skip_rcs_switch(struct i915_hw_ppgtt *ppgtt, + struct intel_engine_cs *engine, + struct i915_gem_context *to) { if (to->remap_slice) return false; - if (!to->legacy_hw_ctx.initialized) + if (!to->engine[RCS].initialised) return false; - if (to->ppgtt && - !(intel_engine_flag(engine) & to->ppgtt->pd_dirty_rings)) + if (ppgtt && (intel_engine_flag(engine) & ppgtt->pd_dirty_rings)) return false; return to == engine->last_context; } static bool -needs_pd_load_pre(struct intel_engine_cs *engine, struct intel_context *to) +needs_pd_load_pre(struct i915_hw_ppgtt *ppgtt, + struct intel_engine_cs *engine, + struct i915_gem_context *to) { - if (!to->ppgtt) + if (!ppgtt) return false; + /* Always load the ppgtt on first use */ + if (!engine->last_context) + return true; + + /* Same context without new entries, skip */ if (engine->last_context == to && - !(intel_engine_flag(engine) & to->ppgtt->pd_dirty_rings)) + !(intel_engine_flag(engine) & ppgtt->pd_dirty_rings)) return false; if (engine->id != RCS) return true; - if (INTEL_INFO(engine->dev)->gen < 8) + if (INTEL_GEN(engine->i915) < 8) return true; return false; } static bool -needs_pd_load_post(struct intel_context *to, u32 hw_flags) +needs_pd_load_post(struct i915_hw_ppgtt *ppgtt, + struct i915_gem_context *to, + u32 hw_flags) { - if (!to->ppgtt) + if (!ppgtt) return false; if (!IS_GEN8(to->i915)) @@ -661,18 +741,19 @@ needs_pd_load_post(struct intel_context *to, u32 hw_flags) static int do_rcs_switch(struct drm_i915_gem_request *req) { - struct intel_context *to = req->ctx; + struct i915_gem_context *to = req->ctx; struct intel_engine_cs *engine = req->engine; - struct intel_context *from; + struct i915_hw_ppgtt *ppgtt = to->ppgtt ?: req->i915->mm.aliasing_ppgtt; + struct i915_gem_context *from; u32 hw_flags; int ret, i; - if (skip_rcs_switch(engine, to)) + if (skip_rcs_switch(ppgtt, engine, to)) return 0; /* Trying to pin first makes error handling easier. */ - ret = i915_gem_obj_ggtt_pin(to->legacy_hw_ctx.rcs_state, - get_context_alignment(engine->dev), + ret = i915_gem_obj_ggtt_pin(to->engine[RCS].state, + to->ggtt_alignment, 0); if (ret) return ret; @@ -694,37 +775,32 @@ static int do_rcs_switch(struct drm_i915_gem_request *req) * * XXX: We need a real interface to do this instead of trickery. */ - ret = i915_gem_object_set_to_gtt_domain(to->legacy_hw_ctx.rcs_state, false); + ret = i915_gem_object_set_to_gtt_domain(to->engine[RCS].state, false); if (ret) goto unpin_out; - if (needs_pd_load_pre(engine, to)) { + if (needs_pd_load_pre(ppgtt, engine, to)) { /* Older GENs and non render rings still want the load first, * "PP_DCLV followed by PP_DIR_BASE register through Load * Register Immediate commands in Ring Buffer before submitting * a context."*/ trace_switch_mm(engine, to); - ret = to->ppgtt->switch_mm(to->ppgtt, req); + ret = ppgtt->switch_mm(ppgtt, req); if (ret) goto unpin_out; } - if (!to->legacy_hw_ctx.initialized || i915_gem_context_is_default(to)) + if (!to->engine[RCS].initialised || i915_gem_context_is_default(to)) /* NB: If we inhibit the restore, the context is not allowed to * die because future work may end up depending on valid address * space. This means we must enforce that a page table load * occur when this occurs. */ hw_flags = MI_RESTORE_INHIBIT; - else if (to->ppgtt && - intel_engine_flag(engine) & to->ppgtt->pd_dirty_rings) + else if (ppgtt && intel_engine_flag(engine) & ppgtt->pd_dirty_rings) hw_flags = MI_FORCE_RESTORE; else hw_flags = 0; - /* We should never emit switch_mm more than once */ - WARN_ON(needs_pd_load_pre(engine, to) && - needs_pd_load_post(to, hw_flags)); - if (to != from || (hw_flags & MI_FORCE_RESTORE)) { ret = mi_set_context(req, hw_flags); if (ret) @@ -738,8 +814,8 @@ static int do_rcs_switch(struct drm_i915_gem_request *req) * MI_SET_CONTEXT instead of when the next seqno has completed. */ if (from != NULL) { - from->legacy_hw_ctx.rcs_state->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION; - i915_vma_move_to_active(i915_gem_obj_to_ggtt(from->legacy_hw_ctx.rcs_state), req); + from->engine[RCS].state->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION; + i915_vma_move_to_active(i915_gem_obj_to_ggtt(from->engine[RCS].state), req); /* As long as MI_SET_CONTEXT is serializing, ie. it flushes the * whole damn pipeline, we don't need to explicitly mark the * object dirty. The only exception is that the context must be @@ -747,10 +823,10 @@ static int do_rcs_switch(struct drm_i915_gem_request *req) * able to defer doing this until we know the object would be * swapped, but there is no way to do that yet. */ - from->legacy_hw_ctx.rcs_state->dirty = 1; + from->engine[RCS].state->dirty = 1; /* obj is kept alive until the next request by its active ref */ - i915_gem_object_ggtt_unpin(from->legacy_hw_ctx.rcs_state); + i915_gem_object_ggtt_unpin(from->engine[RCS].state); i915_gem_context_unreference(from); } i915_gem_context_reference(to); @@ -759,9 +835,9 @@ static int do_rcs_switch(struct drm_i915_gem_request *req) /* GEN8 does *not* require an explicit reload if the PDPs have been * setup, and we do not wish to move them. */ - if (needs_pd_load_post(to, hw_flags)) { + if (needs_pd_load_post(ppgtt, to, hw_flags)) { trace_switch_mm(engine, to); - ret = to->ppgtt->switch_mm(to->ppgtt, req); + ret = ppgtt->switch_mm(ppgtt, req); /* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail * here. Still, let the user know something dangerous has @@ -771,33 +847,33 @@ static int do_rcs_switch(struct drm_i915_gem_request *req) return ret; } - if (to->ppgtt) - to->ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine); + if (ppgtt) + ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine); for (i = 0; i < MAX_L3_SLICES; i++) { if (!(to->remap_slice & (1<<i))) continue; - ret = i915_gem_l3_remap(req, i); + ret = remap_l3(req, i); if (ret) return ret; to->remap_slice &= ~(1<<i); } - if (!to->legacy_hw_ctx.initialized) { + if (!to->engine[RCS].initialised) { if (engine->init_context) { ret = engine->init_context(req); if (ret) return ret; } - to->legacy_hw_ctx.initialized = true; + to->engine[RCS].initialised = true; } return 0; unpin_out: - i915_gem_object_ggtt_unpin(to->legacy_hw_ctx.rcs_state); + i915_gem_object_ggtt_unpin(to->engine[RCS].state); return ret; } @@ -817,25 +893,24 @@ unpin_out: int i915_switch_context(struct drm_i915_gem_request *req) { struct intel_engine_cs *engine = req->engine; - struct drm_i915_private *dev_priv = req->i915; WARN_ON(i915.enable_execlists); - WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex)); + lockdep_assert_held(&req->i915->drm.struct_mutex); - if (engine->id != RCS || - req->ctx->legacy_hw_ctx.rcs_state == NULL) { - struct intel_context *to = req->ctx; + if (!req->ctx->engine[engine->id].state) { + struct i915_gem_context *to = req->ctx; + struct i915_hw_ppgtt *ppgtt = + to->ppgtt ?: req->i915->mm.aliasing_ppgtt; - if (needs_pd_load_pre(engine, to)) { + if (needs_pd_load_pre(ppgtt, engine, to)) { int ret; trace_switch_mm(engine, to); - ret = to->ppgtt->switch_mm(to->ppgtt, req); + ret = ppgtt->switch_mm(ppgtt, req); if (ret) return ret; - /* Doing a PD load always reloads the page dirs */ - to->ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine); + ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine); } if (to != engine->last_context) { @@ -861,7 +936,7 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_context_create *args = data; struct drm_i915_file_private *file_priv = file->driver_priv; - struct intel_context *ctx; + struct i915_gem_context *ctx; int ret; if (!contexts_enabled(dev)) @@ -890,7 +965,7 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_context_destroy *args = data; struct drm_i915_file_private *file_priv = file->driver_priv; - struct intel_context *ctx; + struct i915_gem_context *ctx; int ret; if (args->pad != 0) @@ -903,13 +978,13 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, if (ret) return ret; - ctx = i915_gem_context_get(file_priv, args->ctx_id); + ctx = i915_gem_context_lookup(file_priv, args->ctx_id); if (IS_ERR(ctx)) { mutex_unlock(&dev->struct_mutex); return PTR_ERR(ctx); } - idr_remove(&ctx->file_priv->context_idr, ctx->user_handle); + idr_remove(&file_priv->context_idr, ctx->user_handle); i915_gem_context_unreference(ctx); mutex_unlock(&dev->struct_mutex); @@ -922,14 +997,14 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, { struct drm_i915_file_private *file_priv = file->driver_priv; struct drm_i915_gem_context_param *args = data; - struct intel_context *ctx; + struct i915_gem_context *ctx; int ret; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; - ctx = i915_gem_context_get(file_priv, args->ctx_id); + ctx = i915_gem_context_lookup(file_priv, args->ctx_id); if (IS_ERR(ctx)) { mutex_unlock(&dev->struct_mutex); return PTR_ERR(ctx); @@ -951,6 +1026,9 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, else args->value = to_i915(dev)->ggtt.base.total; break; + case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: + args->value = !!(ctx->flags & CONTEXT_NO_ERROR_CAPTURE); + break; default: ret = -EINVAL; break; @@ -965,14 +1043,14 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, { struct drm_i915_file_private *file_priv = file->driver_priv; struct drm_i915_gem_context_param *args = data; - struct intel_context *ctx; + struct i915_gem_context *ctx; int ret; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; - ctx = i915_gem_context_get(file_priv, args->ctx_id); + ctx = i915_gem_context_lookup(file_priv, args->ctx_id); if (IS_ERR(ctx)) { mutex_unlock(&dev->struct_mutex); return PTR_ERR(ctx); @@ -996,6 +1074,16 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, ctx->flags |= args->value ? CONTEXT_NO_ZEROMAP : 0; } break; + case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: + if (args->size) { + ret = -EINVAL; + } else { + if (args->value) + ctx->flags |= CONTEXT_NO_ERROR_CAPTURE; + else + ctx->flags &= ~CONTEXT_NO_ERROR_CAPTURE; + } + break; default: ret = -EINVAL; break; @@ -1004,3 +1092,42 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, return ret; } + +int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, + void *data, struct drm_file *file) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_reset_stats *args = data; + struct i915_ctx_hang_stats *hs; + struct i915_gem_context *ctx; + int ret; + + if (args->flags || args->pad) + return -EINVAL; + + if (args->ctx_id == DEFAULT_CONTEXT_HANDLE && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = i915_mutex_lock_interruptible(dev); + if (ret) + return ret; + + ctx = i915_gem_context_lookup(file->driver_priv, args->ctx_id); + if (IS_ERR(ctx)) { + mutex_unlock(&dev->struct_mutex); + return PTR_ERR(ctx); + } + hs = &ctx->hang_stats; + + if (capable(CAP_SYS_ADMIN)) + args->reset_count = i915_reset_count(&dev_priv->gpu_error); + else + args->reset_count = 0; + + args->batch_active = hs->batch_active; + args->batch_pending = hs->batch_pending; + + mutex_unlock(&dev->struct_mutex); + + return 0; +} diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.h b/drivers/gpu/drm/i915/i915_gem_dmabuf.h new file mode 100644 index 000000000..91315557e --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.h @@ -0,0 +1,45 @@ +/* + * Copyright 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _I915_GEM_DMABUF_H_ +#define _I915_GEM_DMABUF_H_ + +#include <linux/dma-buf.h> + +static inline struct reservation_object * +i915_gem_object_get_dmabuf_resv(struct drm_i915_gem_object *obj) +{ + struct dma_buf *dma_buf; + + if (obj->base.dma_buf) + dma_buf = obj->base.dma_buf; + else if (obj->base.import_attach) + dma_buf = obj->base.import_attach->dmabuf; + else + return NULL; + + return dma_buf->resv; +} + +#endif diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index ea1f8d1bd..3c1280ec7 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -33,6 +33,37 @@ #include "intel_drv.h" #include "i915_trace.h" +static int switch_to_pinned_context(struct drm_i915_private *dev_priv) +{ + struct intel_engine_cs *engine; + + if (i915.enable_execlists) + return 0; + + for_each_engine(engine, dev_priv) { + struct drm_i915_gem_request *req; + int ret; + + if (engine->last_context == NULL) + continue; + + if (engine->last_context == dev_priv->kernel_context) + continue; + + req = i915_gem_request_alloc(engine, dev_priv->kernel_context); + if (IS_ERR(req)) + return PTR_ERR(req); + + ret = i915_switch_context(req); + i915_add_request_no_flush(req); + if (ret) + return ret; + } + + return 0; +} + + static bool mark_free(struct i915_vma *vma, struct list_head *unwind) { @@ -150,11 +181,19 @@ none: /* Only idle the GPU and repeat the search once */ if (pass++ == 0) { - ret = i915_gpu_idle(dev); + struct drm_i915_private *dev_priv = to_i915(dev); + + if (i915_is_ggtt(vm)) { + ret = switch_to_pinned_context(dev_priv); + if (ret) + return ret; + } + + ret = i915_gem_wait_for_idle(dev_priv); if (ret) return ret; - i915_gem_retire_requests(dev); + i915_gem_retire_requests(dev_priv); goto search_again; } @@ -261,11 +300,19 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle) trace_i915_gem_evict_vm(vm); if (do_idle) { - ret = i915_gpu_idle(vm->dev); + struct drm_i915_private *dev_priv = to_i915(vm->dev); + + if (i915_is_ggtt(vm)) { + ret = switch_to_pinned_context(dev_priv); + if (ret) + return ret; + } + + ret = i915_gem_wait_for_idle(dev_priv); if (ret) return ret; - i915_gem_retire_requests(vm->dev); + i915_gem_retire_requests(dev_priv); WARN_ON(!list_empty(&vm->active_list)); } diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index 33df74d98..b35e5b647 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -714,7 +714,7 @@ eb_vma_misplaced(struct i915_vma *vma) static int i915_gem_execbuffer_reserve(struct intel_engine_cs *engine, struct list_head *vmas, - struct intel_context *ctx, + struct i915_gem_context *ctx, bool *need_relocs) { struct drm_i915_gem_object *obj; @@ -722,7 +722,7 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *engine, struct i915_address_space *vm; struct list_head ordered_vmas; struct list_head pinned_vmas; - bool has_fenced_gpu_access = INTEL_INFO(engine->dev)->gen < 4; + bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4; int retry; i915_gem_retire_requests_ring(engine); @@ -826,7 +826,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev, struct intel_engine_cs *engine, struct eb_vmas *eb, struct drm_i915_gem_exec_object2 *exec, - struct intel_context *ctx) + struct i915_gem_context *ctx) { struct drm_i915_gem_relocation_entry *reloc; struct i915_address_space *vm; @@ -943,8 +943,6 @@ i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req, { const unsigned other_rings = ~intel_engine_flag(req->engine); struct i915_vma *vma; - uint32_t flush_domains = 0; - bool flush_chipset = false; int ret; list_for_each_entry(vma, vmas, exec_list) { @@ -957,16 +955,11 @@ i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req, } if (obj->base.write_domain & I915_GEM_DOMAIN_CPU) - flush_chipset |= i915_gem_clflush_object(obj, false); - - flush_domains |= obj->base.write_domain; + i915_gem_clflush_object(obj, false); } - if (flush_chipset) - i915_gem_chipset_flush(req->engine->dev); - - if (flush_domains & I915_GEM_DOMAIN_GTT) - wmb(); + /* Unconditionally flush any chipset caches (for streaming writes). */ + i915_gem_chipset_flush(req->engine->i915); /* Unconditionally invalidate gpu caches and ensure that we do flush * any residual writes from the previous batch. @@ -1063,17 +1056,17 @@ validate_exec_list(struct drm_device *dev, return 0; } -static struct intel_context * +static struct i915_gem_context * i915_gem_validate_context(struct drm_device *dev, struct drm_file *file, struct intel_engine_cs *engine, const u32 ctx_id) { - struct intel_context *ctx = NULL; + struct i915_gem_context *ctx = NULL; struct i915_ctx_hang_stats *hs; if (engine->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE) return ERR_PTR(-EINVAL); - ctx = i915_gem_context_get(file->driver_priv, ctx_id); + ctx = i915_gem_context_lookup(file->driver_priv, ctx_id); if (IS_ERR(ctx)) return ctx; @@ -1083,14 +1076,6 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file, return ERR_PTR(-EIO); } - if (i915.enable_execlists && !ctx->engine[engine->id].state) { - int ret = intel_lr_context_deferred_alloc(ctx, engine); - if (ret) { - DRM_DEBUG("Could not create LRC %u: %d\n", ctx_id, ret); - return ERR_PTR(ret); - } - } - return ctx; } @@ -1125,7 +1110,7 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas, if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) { i915_gem_request_assign(&obj->last_fenced_req, req); if (entry->flags & __EXEC_OBJECT_HAS_FENCE) { - struct drm_i915_private *dev_priv = to_i915(engine->dev); + struct drm_i915_private *dev_priv = engine->i915; list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list, &dev_priv->mm.fence_list); } @@ -1150,7 +1135,7 @@ i915_reset_gen7_sol_offsets(struct drm_device *dev, struct drm_i915_gem_request *req) { struct intel_engine_cs *engine = req->engine; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret, i; if (!IS_GEN7(dev) || engine != &dev_priv->engine[RCS]) { @@ -1233,7 +1218,7 @@ i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params, { struct drm_device *dev = params->dev; struct intel_engine_cs *engine = params->engine; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u64 exec_start, exec_len; int instp_mode; u32 instp_mask; @@ -1336,10 +1321,10 @@ gen8_dispatch_bsd_ring(struct drm_i915_private *dev_priv, struct drm_file *file) /* Check whether the file_priv has already selected one ring. */ if ((int)file_priv->bsd_ring < 0) { /* If not, use the ping-pong mechanism to select one. */ - mutex_lock(&dev_priv->dev->struct_mutex); + mutex_lock(&dev_priv->drm.struct_mutex); file_priv->bsd_ring = dev_priv->mm.bsd_ring_dispatch_index; dev_priv->mm.bsd_ring_dispatch_index ^= 1; - mutex_unlock(&dev_priv->dev->struct_mutex); + mutex_unlock(&dev_priv->drm.struct_mutex); } return file_priv->bsd_ring; @@ -1436,7 +1421,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, struct drm_i915_gem_object *batch_obj; struct drm_i915_gem_exec_object2 shadow_exec_entry; struct intel_engine_cs *engine; - struct intel_context *ctx; + struct i915_gem_context *ctx; struct i915_address_space *vm; struct i915_execbuffer_params params_master; /* XXX: will be removed later */ struct i915_execbuffer_params *params = ¶ms_master; @@ -1454,7 +1439,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, dispatch_flags = 0; if (args->flags & I915_EXEC_SECURE) { - if (!file->is_master || !capable(CAP_SYS_ADMIN)) + if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN)) return -EPERM; dispatch_flags |= I915_DISPATCH_SECURE; @@ -1485,6 +1470,12 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, dispatch_flags |= I915_DISPATCH_RS; } + /* Take a local wakeref for preparing to dispatch the execbuf as + * we expect to access the hardware fairly frequently in the + * process. Upon first dispatch, we acquire another prolonged + * wakeref that we hold until the GPU has been idle for at least + * 100ms. + */ intel_runtime_pm_get(dev_priv); ret = i915_mutex_lock_interruptible(dev); @@ -1561,7 +1552,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data, batch_obj, args->batch_start_offset, args->batch_len, - file->is_master); + drm_is_current_master(file)); if (IS_ERR(parsed_batch_obj)) { ret = PTR_ERR(parsed_batch_obj); goto err; diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence.c index a2b938ec0..251d7a95a 100644 --- a/drivers/gpu/drm/i915/i915_gem_fence.c +++ b/drivers/gpu/drm/i915/i915_gem_fence.c @@ -58,7 +58,7 @@ static void i965_write_fence_reg(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t fence_reg_lo, fence_reg_hi; int fence_pitch_shift; @@ -117,7 +117,7 @@ static void i965_write_fence_reg(struct drm_device *dev, int reg, static void i915_write_fence_reg(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val; if (obj) { @@ -156,7 +156,7 @@ static void i915_write_fence_reg(struct drm_device *dev, int reg, static void i830_write_fence_reg(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t val; if (obj) { @@ -193,7 +193,7 @@ inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj) static void i915_gem_write_fence(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* Ensure that all CPU reads are completed before installing a fence * and all writes before removing the fence. @@ -229,7 +229,7 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, struct drm_i915_fence_reg *fence, bool enable) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); int reg = fence_number(dev_priv, fence); i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); @@ -286,7 +286,7 @@ i915_gem_object_wait_fence(struct drm_i915_gem_object *obj) int i915_gem_object_put_fence(struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); struct drm_i915_fence_reg *fence; int ret; @@ -311,7 +311,7 @@ i915_gem_object_put_fence(struct drm_i915_gem_object *obj) static struct drm_i915_fence_reg * i915_find_fence_reg(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_fence_reg *reg, *avail; int i; @@ -367,7 +367,7 @@ int i915_gem_object_get_fence(struct drm_i915_gem_object *obj) { struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); bool enable = obj->tiling_mode != I915_TILING_NONE; struct drm_i915_fence_reg *reg; int ret; @@ -433,7 +433,7 @@ bool i915_gem_object_pin_fence(struct drm_i915_gem_object *obj) { if (obj->fence_reg != I915_FENCE_REG_NONE) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj); WARN_ON(!ggtt_vma || @@ -457,7 +457,7 @@ void i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj) { if (obj->fence_reg != I915_FENCE_REG_NONE) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0); dev_priv->fence_regs[obj->fence_reg].pin_count--; } @@ -472,7 +472,7 @@ i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj) */ void i915_gem_restore_fences(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i; for (i = 0; i < dev_priv->num_fence_regs; i++) { @@ -549,7 +549,7 @@ void i915_gem_restore_fences(struct drm_device *dev) void i915_gem_detect_bit_6_swizzle(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; @@ -745,15 +745,15 @@ i915_gem_swizzle_page(struct page *page) void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) { - struct sg_page_iter sg_iter; + struct sgt_iter sgt_iter; + struct page *page; int i; if (obj->bit_17 == NULL) return; i = 0; - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { - struct page *page = sg_page_iter_page(&sg_iter); + for_each_sgt_page(page, sgt_iter, obj->pages) { char new_bit_17 = page_to_phys(page) >> 17; if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) { @@ -775,7 +775,8 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) { - struct sg_page_iter sg_iter; + struct sgt_iter sgt_iter; + struct page *page; int page_count = obj->base.size >> PAGE_SHIFT; int i; @@ -790,8 +791,9 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) } i = 0; - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { - if (page_to_phys(sg_page_iter_page(&sg_iter)) & (1 << 17)) + + for_each_sgt_page(page, sgt_iter, obj->pages) { + if (page_to_phys(page) & (1 << 17)) __set_bit(i, obj->bit_17); else __clear_bit(i, obj->bit_17); diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index e856f7906..f38ceffd8 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -93,6 +93,13 @@ * */ +static inline struct i915_ggtt * +i915_vm_to_ggtt(struct i915_address_space *vm) +{ + GEM_BUG_ON(!i915_is_ggtt(vm)); + return container_of(vm, struct i915_ggtt, base); +} + static int i915_get_ggtt_vma_pages(struct i915_vma *vma); @@ -103,25 +110,32 @@ const struct i915_ggtt_view i915_ggtt_view_rotated = { .type = I915_GGTT_VIEW_ROTATED, }; -static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) +int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv, + int enable_ppgtt) { bool has_aliasing_ppgtt; bool has_full_ppgtt; bool has_full_48bit_ppgtt; - has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6; - has_full_ppgtt = INTEL_INFO(dev)->gen >= 7; - has_full_48bit_ppgtt = IS_BROADWELL(dev) || INTEL_INFO(dev)->gen >= 9; + has_aliasing_ppgtt = INTEL_GEN(dev_priv) >= 6; + has_full_ppgtt = INTEL_GEN(dev_priv) >= 7; + has_full_48bit_ppgtt = + IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9; + + if (intel_vgpu_active(dev_priv)) { + /* emulation is too hard */ + has_full_ppgtt = false; + has_full_48bit_ppgtt = false; + } - if (intel_vgpu_active(dev)) - has_full_ppgtt = false; /* emulation is too hard */ + if (!has_aliasing_ppgtt) + return 0; /* * We don't allow disabling PPGTT for gen9+ as it's a requirement for * execlists, the sole mechanism available to submit work. */ - if (INTEL_INFO(dev)->gen < 9 && - (enable_ppgtt == 0 || !has_aliasing_ppgtt)) + if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9) return 0; if (enable_ppgtt == 1) @@ -135,19 +149,19 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) #ifdef CONFIG_INTEL_IOMMU /* Disable ppgtt on SNB if VT-d is on. */ - if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) { + if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) { DRM_INFO("Disabling PPGTT because VT-d is on\n"); return 0; } #endif /* Early VLV doesn't have this */ - if (IS_VALLEYVIEW(dev) && dev->pdev->revision < 0xb) { + if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) { DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n"); return 0; } - if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists) + if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists && has_full_ppgtt) return has_full_48bit_ppgtt ? 3 : 2; else return has_aliasing_ppgtt ? 1 : 0; @@ -866,6 +880,7 @@ static void gen8_free_page_tables(struct drm_device *dev, static int gen8_init_scratch(struct i915_address_space *vm) { struct drm_device *dev = vm->dev; + int ret; vm->scratch_page = alloc_scratch_page(dev); if (IS_ERR(vm->scratch_page)) @@ -873,24 +888,21 @@ static int gen8_init_scratch(struct i915_address_space *vm) vm->scratch_pt = alloc_pt(dev); if (IS_ERR(vm->scratch_pt)) { - free_scratch_page(dev, vm->scratch_page); - return PTR_ERR(vm->scratch_pt); + ret = PTR_ERR(vm->scratch_pt); + goto free_scratch_page; } vm->scratch_pd = alloc_pd(dev); if (IS_ERR(vm->scratch_pd)) { - free_pt(dev, vm->scratch_pt); - free_scratch_page(dev, vm->scratch_page); - return PTR_ERR(vm->scratch_pd); + ret = PTR_ERR(vm->scratch_pd); + goto free_pt; } if (USES_FULL_48BIT_PPGTT(dev)) { vm->scratch_pdp = alloc_pdp(dev); if (IS_ERR(vm->scratch_pdp)) { - free_pd(dev, vm->scratch_pd); - free_pt(dev, vm->scratch_pt); - free_scratch_page(dev, vm->scratch_page); - return PTR_ERR(vm->scratch_pdp); + ret = PTR_ERR(vm->scratch_pdp); + goto free_pd; } } @@ -900,6 +912,15 @@ static int gen8_init_scratch(struct i915_address_space *vm) gen8_initialize_pdp(vm, vm->scratch_pdp); return 0; + +free_pd: + free_pd(dev, vm->scratch_pd); +free_pt: + free_pt(dev, vm->scratch_pt); +free_scratch_page: + free_scratch_page(dev, vm->scratch_page); + + return ret; } static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) @@ -978,7 +999,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm) { struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - if (intel_vgpu_active(vm->dev)) + if (intel_vgpu_active(to_i915(vm->dev))) gen8_ppgtt_notify_vgt(ppgtt, false); if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) @@ -1529,14 +1550,14 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) 0, 0, GEN8_PML4E_SHIFT); - if (intel_vgpu_active(ppgtt->base.dev)) { + if (intel_vgpu_active(to_i915(ppgtt->base.dev))) { ret = gen8_preallocate_top_level_pdps(ppgtt); if (ret) goto free_scratch; } } - if (intel_vgpu_active(ppgtt->base.dev)) + if (intel_vgpu_active(to_i915(ppgtt->base.dev))) gen8_ppgtt_notify_vgt(ppgtt, true); return 0; @@ -1552,13 +1573,13 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) struct i915_page_table *unused; gen6_pte_t scratch_pte; uint32_t pd_entry; - uint32_t pte, pde, temp; + uint32_t pte, pde; uint32_t start = ppgtt->base.start, length = ppgtt->base.total; scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), I915_CACHE_LLC, true, 0); - gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) { + gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) { u32 expected; gen6_pte_t *pt_vaddr; const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]); @@ -1622,9 +1643,9 @@ static void gen6_write_page_range(struct drm_i915_private *dev_priv, { struct i915_ggtt *ggtt = &dev_priv->ggtt; struct i915_page_table *pt; - uint32_t pde, temp; + uint32_t pde; - gen6_for_each_pde(pt, pd, start, length, temp, pde) + gen6_for_each_pde(pt, pd, start, length, pde) gen6_write_pde(pd, pde, pt); /* Make sure write is complete before other code can use this page @@ -1665,17 +1686,6 @@ static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, return 0; } -static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt, - struct drm_i915_gem_request *req) -{ - struct intel_engine_cs *engine = req->engine; - struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); - - I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G); - I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt)); - return 0; -} - static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_request *req) { @@ -1713,21 +1723,16 @@ static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_request *req) { struct intel_engine_cs *engine = req->engine; - struct drm_device *dev = ppgtt->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - + struct drm_i915_private *dev_priv = req->i915; I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G); I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt)); - - POSTING_READ(RING_PP_DIR_DCLV(engine)); - return 0; } static void gen8_ppgtt_enable(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; for_each_engine(engine, dev_priv) { @@ -1739,7 +1744,7 @@ static void gen8_ppgtt_enable(struct drm_device *dev) static void gen7_ppgtt_enable(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine; uint32_t ecochk, ecobits; @@ -1764,7 +1769,7 @@ static void gen7_ppgtt_enable(struct drm_device *dev) static void gen6_ppgtt_enable(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t ecochk, gab_ctl, ecobits; ecobits = I915_READ(GAC_ECO_BITS); @@ -1821,20 +1826,19 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, enum i915_cache_level cache_level, u32 flags) { struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - gen6_pte_t *pt_vaddr; unsigned first_entry = start >> PAGE_SHIFT; unsigned act_pt = first_entry / GEN6_PTES; unsigned act_pte = first_entry % GEN6_PTES; - struct sg_page_iter sg_iter; + gen6_pte_t *pt_vaddr = NULL; + struct sgt_iter sgt_iter; + dma_addr_t addr; - pt_vaddr = NULL; - for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { + for_each_sgt_dma(addr, sgt_iter, pages) { if (pt_vaddr == NULL) pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); pt_vaddr[act_pte] = - vm->pte_encode(sg_page_iter_dma_address(&sg_iter), - cache_level, true, flags); + vm->pte_encode(addr, cache_level, true, flags); if (++act_pte == GEN6_PTES) { kunmap_px(ppgtt, pt_vaddr); @@ -1843,6 +1847,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, act_pte = 0; } } + if (pt_vaddr) kunmap_px(ppgtt, pt_vaddr); } @@ -1857,7 +1862,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); struct i915_page_table *pt; uint32_t start, length, start_save, length_save; - uint32_t pde, temp; + uint32_t pde; int ret; if (WARN_ON(start_in + length_in > ppgtt->base.total)) @@ -1873,7 +1878,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, * need allocation. The second stage marks use ptes within the page * tables. */ - gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { + gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) { if (pt != vm->scratch_pt) { WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES)); continue; @@ -1898,7 +1903,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, start = start_save; length = length_save; - gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { + gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) { DECLARE_BITMAP(tmp_bitmap, GEN6_PTES); bitmap_zero(tmp_bitmap, GEN6_PTES); @@ -1967,15 +1972,16 @@ static void gen6_free_scratch(struct i915_address_space *vm) static void gen6_ppgtt_cleanup(struct i915_address_space *vm) { struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); + struct i915_page_directory *pd = &ppgtt->pd; + struct drm_device *dev = vm->dev; struct i915_page_table *pt; uint32_t pde; drm_mm_remove_node(&ppgtt->node); - gen6_for_all_pdes(pt, ppgtt, pde) { + gen6_for_all_pdes(pt, pd, pde) if (pt != vm->scratch_pt) - free_pt(ppgtt->base.dev, pt); - } + free_pt(dev, pt); gen6_free_scratch(vm); } @@ -2041,9 +2047,9 @@ static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, uint64_t start, uint64_t length) { struct i915_page_table *unused; - uint32_t pde, temp; + uint32_t pde; - gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) + gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt; } @@ -2055,18 +2061,15 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) int ret; ppgtt->base.pte_encode = ggtt->base.pte_encode; - if (IS_GEN6(dev)) { + if (intel_vgpu_active(dev_priv) || IS_GEN6(dev)) ppgtt->switch_mm = gen6_mm_switch; - } else if (IS_HASWELL(dev)) { + else if (IS_HASWELL(dev)) ppgtt->switch_mm = hsw_mm_switch; - } else if (IS_GEN7(dev)) { + else if (IS_GEN7(dev)) ppgtt->switch_mm = gen7_mm_switch; - } else + else BUG(); - if (intel_vgpu_active(dev)) - ppgtt->switch_mm = vgpu_mm_switch; - ret = gen6_ppgtt_alloc(ppgtt); if (ret) return ret; @@ -2115,7 +2118,7 @@ static void i915_address_space_init(struct i915_address_space *vm, struct drm_i915_private *dev_priv) { drm_mm_init(&vm->mm, vm->start, vm->total); - vm->dev = dev_priv->dev; + vm->dev = &dev_priv->drm; INIT_LIST_HEAD(&vm->active_list); INIT_LIST_HEAD(&vm->inactive_list); list_add_tail(&vm->global_link, &dev_priv->vm_list); @@ -2123,7 +2126,7 @@ static void i915_address_space_init(struct i915_address_space *vm, static void gtt_write_workarounds(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* This function is for gtt related workarounds. This function is * called on driver load and after a GPU reset, so you can place @@ -2140,9 +2143,9 @@ static void gtt_write_workarounds(struct drm_device *dev) I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT); } -int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) +static int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret = 0; ret = __hw_ppgtt_init(dev, ppgtt); @@ -2179,20 +2182,6 @@ int i915_ppgtt_init_hw(struct drm_device *dev) return 0; } -int i915_ppgtt_init_ring(struct drm_i915_gem_request *req) -{ - struct drm_i915_private *dev_priv = req->i915; - struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; - - if (i915.enable_execlists) - return 0; - - if (!ppgtt) - return 0; - - return ppgtt->switch_mm(ppgtt, req); -} - struct i915_hw_ppgtt * i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv) { @@ -2257,8 +2246,8 @@ static bool do_idling(struct drm_i915_private *dev_priv) if (unlikely(ggtt->do_idle_maps)) { dev_priv->mm.interruptible = false; - if (i915_gpu_idle(dev_priv->dev)) { - DRM_ERROR("Couldn't idle GPU\n"); + if (i915_gem_wait_for_idle(dev_priv)) { + DRM_ERROR("Failed to wait for idle; VT'd may hang.\n"); /* Wait a bit, in hopes it avoids the hang */ udelay(10); } @@ -2275,12 +2264,11 @@ static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) dev_priv->mm.interruptible = interruptible; } -void i915_check_and_clear_faults(struct drm_device *dev) +void i915_check_and_clear_faults(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; - if (INTEL_INFO(dev)->gen < 6) + if (INTEL_INFO(dev_priv)->gen < 6) return; for_each_engine(engine, dev_priv) { @@ -2324,7 +2312,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev) if (INTEL_INFO(dev)->gen < 6) return; - i915_check_and_clear_faults(dev); + i915_check_and_clear_faults(dev_priv); ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total, true); @@ -2352,29 +2340,49 @@ static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) #endif } +static void gen8_ggtt_insert_page(struct i915_address_space *vm, + dma_addr_t addr, + uint64_t offset, + enum i915_cache_level level, + u32 unused) +{ + struct drm_i915_private *dev_priv = to_i915(vm->dev); + gen8_pte_t __iomem *pte = + (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + + (offset >> PAGE_SHIFT); + int rpm_atomic_seq; + + rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); + + gen8_set_pte(pte, gen8_pte_encode(addr, level, true)); + + I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); + POSTING_READ(GFX_FLSH_CNTL_GEN6); + + assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); +} + static void gen8_ggtt_insert_entries(struct i915_address_space *vm, struct sg_table *st, uint64_t start, enum i915_cache_level level, u32 unused) { struct drm_i915_private *dev_priv = to_i915(vm->dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; - unsigned first_entry = start >> PAGE_SHIFT; - gen8_pte_t __iomem *gtt_entries = - (gen8_pte_t __iomem *)ggtt->gsm + first_entry; - int i = 0; - struct sg_page_iter sg_iter; - dma_addr_t addr = 0; /* shut up gcc */ + struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); + struct sgt_iter sgt_iter; + gen8_pte_t __iomem *gtt_entries; + gen8_pte_t gtt_entry; + dma_addr_t addr; int rpm_atomic_seq; + int i = 0; rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); - for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { - addr = sg_dma_address(sg_iter.sg) + - (sg_iter.sg_pgoffset << PAGE_SHIFT); - gen8_set_pte(>t_entries[i], - gen8_pte_encode(addr, level, true)); - i++; + gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT); + + for_each_sgt_dma(addr, sgt_iter, st) { + gtt_entry = gen8_pte_encode(addr, level, true); + gen8_set_pte(>t_entries[i++], gtt_entry); } /* @@ -2385,8 +2393,7 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, * hardware should work, we must keep this posting read for paranoia. */ if (i != 0) - WARN_ON(readq(>t_entries[i-1]) - != gen8_pte_encode(addr, level, true)); + WARN_ON(readq(>t_entries[i-1]) != gtt_entry); /* This next bit makes the above posting read even more important. We * want to flush the TLBs only after we're certain all the PTE updates @@ -2424,6 +2431,28 @@ static void gen8_ggtt_insert_entries__BKL(struct i915_address_space *vm, stop_machine(gen8_ggtt_insert_entries__cb, &arg, NULL); } +static void gen6_ggtt_insert_page(struct i915_address_space *vm, + dma_addr_t addr, + uint64_t offset, + enum i915_cache_level level, + u32 flags) +{ + struct drm_i915_private *dev_priv = to_i915(vm->dev); + gen6_pte_t __iomem *pte = + (gen6_pte_t __iomem *)dev_priv->ggtt.gsm + + (offset >> PAGE_SHIFT); + int rpm_atomic_seq; + + rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); + + iowrite32(vm->pte_encode(addr, level, true, flags), pte); + + I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); + POSTING_READ(GFX_FLSH_CNTL_GEN6); + + assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); +} + /* * Binds an object into the global gtt with the specified cache level. The object * will be accessible to the GPU via commands whose operands reference offsets @@ -2436,21 +2465,21 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, enum i915_cache_level level, u32 flags) { struct drm_i915_private *dev_priv = to_i915(vm->dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; - unsigned first_entry = start >> PAGE_SHIFT; - gen6_pte_t __iomem *gtt_entries = - (gen6_pte_t __iomem *)ggtt->gsm + first_entry; - int i = 0; - struct sg_page_iter sg_iter; - dma_addr_t addr = 0; + struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); + struct sgt_iter sgt_iter; + gen6_pte_t __iomem *gtt_entries; + gen6_pte_t gtt_entry; + dma_addr_t addr; int rpm_atomic_seq; + int i = 0; rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); - for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { - addr = sg_page_iter_dma_address(&sg_iter); - iowrite32(vm->pte_encode(addr, level, true, flags), >t_entries[i]); - i++; + gtt_entries = (gen6_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT); + + for_each_sgt_dma(addr, sgt_iter, st) { + gtt_entry = vm->pte_encode(addr, level, true, flags); + iowrite32(gtt_entry, >t_entries[i++]); } /* XXX: This serves as a posting read to make sure that the PTE has @@ -2459,10 +2488,8 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, * of NUMA access patterns. Therefore, even with the way we assume * hardware should work, we must keep this posting read for paranoia. */ - if (i != 0) { - unsigned long gtt = readl(>t_entries[i-1]); - WARN_ON(gtt != vm->pte_encode(addr, level, true, flags)); - } + if (i != 0) + WARN_ON(readl(>t_entries[i-1]) != gtt_entry); /* This next bit makes the above posting read even more important. We * want to flush the TLBs only after we're certain all the PTE updates @@ -2474,13 +2501,20 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } +static void nop_clear_range(struct i915_address_space *vm, + uint64_t start, + uint64_t length, + bool use_scratch) +{ +} + static void gen8_ggtt_clear_range(struct i915_address_space *vm, uint64_t start, uint64_t length, bool use_scratch) { struct drm_i915_private *dev_priv = to_i915(vm->dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; + struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); unsigned first_entry = start >> PAGE_SHIFT; unsigned num_entries = length >> PAGE_SHIFT; gen8_pte_t scratch_pte, __iomem *gtt_base = @@ -2512,7 +2546,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, bool use_scratch) { struct drm_i915_private *dev_priv = to_i915(vm->dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; + struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); unsigned first_entry = start >> PAGE_SHIFT; unsigned num_entries = length >> PAGE_SHIFT; gen6_pte_t scratch_pte, __iomem *gtt_base = @@ -2538,12 +2572,30 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } +static void i915_ggtt_insert_page(struct i915_address_space *vm, + dma_addr_t addr, + uint64_t offset, + enum i915_cache_level cache_level, + u32 unused) +{ + struct drm_i915_private *dev_priv = to_i915(vm->dev); + unsigned int flags = (cache_level == I915_CACHE_NONE) ? + AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; + int rpm_atomic_seq; + + rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); + + intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags); + + assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); +} + static void i915_ggtt_insert_entries(struct i915_address_space *vm, struct sg_table *pages, uint64_t start, enum i915_cache_level cache_level, u32 unused) { - struct drm_i915_private *dev_priv = vm->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(vm->dev); unsigned int flags = (cache_level == I915_CACHE_NONE) ? AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; int rpm_atomic_seq; @@ -2561,7 +2613,7 @@ static void i915_ggtt_clear_range(struct i915_address_space *vm, uint64_t length, bool unused) { - struct drm_i915_private *dev_priv = vm->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(vm->dev); unsigned first_entry = start >> PAGE_SHIFT; unsigned num_entries = length >> PAGE_SHIFT; int rpm_atomic_seq; @@ -2642,7 +2694,7 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, static void ggtt_unbind_vma(struct i915_vma *vma) { struct drm_device *dev = vma->vm->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj = vma->obj; const uint64_t size = min_t(uint64_t, obj->base.size, @@ -2668,7 +2720,7 @@ static void ggtt_unbind_vma(struct i915_vma *vma) void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) { struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); bool interruptible; interruptible = do_idling(dev_priv); @@ -2727,11 +2779,9 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev, i915_address_space_init(&ggtt->base, dev_priv); ggtt->base.total += PAGE_SIZE; - if (intel_vgpu_active(dev)) { - ret = intel_vgt_balloon(dev); - if (ret) - return ret; - } + ret = intel_vgt_balloon(dev_priv); + if (ret) + return ret; if (!HAS_LLC(dev)) ggtt->base.mm.color_adjust = i915_gtt_color_adjust; @@ -2832,8 +2882,7 @@ void i915_ggtt_cleanup_hw(struct drm_device *dev) i915_gem_cleanup_stolen(dev); if (drm_mm_initialized(&ggtt->base.mm)) { - if (intel_vgpu_active(dev)) - intel_vgt_deballoon(); + intel_vgt_deballoon(dev_priv); drm_mm_takedown(&ggtt->base.mm); list_del(&ggtt->base.global_link); @@ -3070,13 +3119,16 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) ret = ggtt_probe_common(dev, ggtt->size); - ggtt->base.clear_range = gen8_ggtt_clear_range; - if (IS_CHERRYVIEW(dev_priv)) - ggtt->base.insert_entries = gen8_ggtt_insert_entries__BKL; - else - ggtt->base.insert_entries = gen8_ggtt_insert_entries; ggtt->base.bind_vma = ggtt_bind_vma; ggtt->base.unbind_vma = ggtt_unbind_vma; + ggtt->base.insert_page = gen8_ggtt_insert_page; + ggtt->base.clear_range = nop_clear_range; + if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv)) + ggtt->base.clear_range = gen8_ggtt_clear_range; + + ggtt->base.insert_entries = gen8_ggtt_insert_entries; + if (IS_CHERRYVIEW(dev_priv)) + ggtt->base.insert_entries = gen8_ggtt_insert_entries__BKL; return ret; } @@ -3109,6 +3161,7 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt) ret = ggtt_probe_common(dev, ggtt->size); ggtt->base.clear_range = gen6_ggtt_clear_range; + ggtt->base.insert_page = gen6_ggtt_insert_page; ggtt->base.insert_entries = gen6_ggtt_insert_entries; ggtt->base.bind_vma = ggtt_bind_vma; ggtt->base.unbind_vma = ggtt_unbind_vma; @@ -3130,7 +3183,7 @@ static int i915_gmch_probe(struct i915_ggtt *ggtt) struct drm_i915_private *dev_priv = to_i915(dev); int ret; - ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); + ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL); if (!ret) { DRM_ERROR("failed to set up gmch\n"); return -EIO; @@ -3139,7 +3192,8 @@ static int i915_gmch_probe(struct i915_ggtt *ggtt) intel_gtt_get(&ggtt->base.total, &ggtt->stolen_size, &ggtt->mappable_base, &ggtt->mappable_end); - ggtt->do_idle_maps = needs_idle_maps(dev_priv->dev); + ggtt->do_idle_maps = needs_idle_maps(&dev_priv->drm); + ggtt->base.insert_page = i915_ggtt_insert_page; ggtt->base.insert_entries = i915_ggtt_insert_entries; ggtt->base.clear_range = i915_ggtt_clear_range; ggtt->base.bind_vma = ggtt_bind_vma; @@ -3220,14 +3274,6 @@ int i915_ggtt_init_hw(struct drm_device *dev) if (intel_iommu_gfx_mapped) DRM_INFO("VT-d active for gfx access\n"); #endif - /* - * i915.enable_ppgtt is read-only, so do an early pass to validate the - * user's requested state against the hardware/driver capabilities. We - * do this now so that we can print out any log messages once rather - * than every time we check intel_enable_ppgtt(). - */ - i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt); - DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); return 0; @@ -3251,9 +3297,8 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev) struct i915_ggtt *ggtt = &dev_priv->ggtt; struct drm_i915_gem_object *obj; struct i915_vma *vma; - bool flush; - i915_check_and_clear_faults(dev); + i915_check_and_clear_faults(dev_priv); /* First fill our portion of the GTT with scratch pages */ ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total, @@ -3261,19 +3306,16 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev) /* Cache flush objects bound into GGTT and rebind them. */ list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { - flush = false; list_for_each_entry(vma, &obj->vma_list, obj_link) { if (vma->vm != &ggtt->base) continue; WARN_ON(i915_vma_bind(vma, obj->cache_level, PIN_UPDATE)); - - flush = true; } - if (flush) - i915_gem_clflush_object(obj, obj->pin_display); + if (obj->pin_display) + WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false)); } if (INTEL_INFO(dev)->gen >= 8) { @@ -3399,9 +3441,11 @@ static struct sg_table * intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info, struct drm_i915_gem_object *obj) { + const size_t n_pages = obj->base.size / PAGE_SIZE; unsigned int size_pages = rot_info->plane[0].width * rot_info->plane[0].height; unsigned int size_pages_uv; - struct sg_page_iter sg_iter; + struct sgt_iter sgt_iter; + dma_addr_t dma_addr; unsigned long i; dma_addr_t *page_addr_list; struct sg_table *st; @@ -3410,7 +3454,7 @@ intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info, int ret = -ENOMEM; /* Allocate a temporary list of source pages for random access. */ - page_addr_list = drm_malloc_gfp(obj->base.size / PAGE_SIZE, + page_addr_list = drm_malloc_gfp(n_pages, sizeof(dma_addr_t), GFP_TEMPORARY); if (!page_addr_list) @@ -3433,11 +3477,10 @@ intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info, /* Populate source page list from the object. */ i = 0; - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { - page_addr_list[i] = sg_page_iter_dma_address(&sg_iter); - i++; - } + for_each_sgt_dma(dma_addr, sgt_iter, obj->pages) + page_addr_list[i++] = dma_addr; + GEM_BUG_ON(i != n_pages); st->nents = 0; sg = st->sgl; @@ -3635,3 +3678,29 @@ i915_ggtt_view_size(struct drm_i915_gem_object *obj, return obj->base.size; } } + +void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) +{ + void __iomem *ptr; + + lockdep_assert_held(&vma->vm->dev->struct_mutex); + if (WARN_ON(!vma->obj->map_and_fenceable)) + return ERR_PTR(-ENODEV); + + GEM_BUG_ON(!vma->is_ggtt); + GEM_BUG_ON((vma->bound & GLOBAL_BIND) == 0); + + ptr = vma->iomap; + if (ptr == NULL) { + ptr = io_mapping_map_wc(i915_vm_to_ggtt(vma->vm)->mappable, + vma->node.start, + vma->node.size); + if (ptr == NULL) + return ERR_PTR(-ENOMEM); + + vma->iomap = ptr; + } + + vma->pin_count++; + return ptr; +} diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index 0008543d5..aa5f31d1c 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -34,6 +34,8 @@ #ifndef __I915_GEM_GTT_H__ #define __I915_GEM_GTT_H__ +#include <linux/io-mapping.h> + struct drm_i915_file_private; typedef uint32_t gen6_pte_t; @@ -175,6 +177,7 @@ struct i915_vma { struct drm_mm_node node; struct drm_i915_gem_object *obj; struct i915_address_space *vm; + void __iomem *iomap; /** Flags and address space this VMA is bound to */ #define GLOBAL_BIND (1<<0) @@ -316,6 +319,11 @@ struct i915_address_space { uint64_t start, uint64_t length, bool use_scratch); + void (*insert_page)(struct i915_address_space *vm, + dma_addr_t addr, + uint64_t offset, + enum i915_cache_level cache_level, + u32 flags); void (*insert_entries)(struct i915_address_space *vm, struct sg_table *st, uint64_t start, @@ -382,27 +390,27 @@ struct i915_hw_ppgtt { void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m); }; -/* For each pde iterates over every pde between from start until start + length. - * If start, and start+length are not perfectly divisible, the macro will round - * down, and up as needed. The macro modifies pde, start, and length. Dev is - * only used to differentiate shift values. Temp is temp. On gen6/7, start = 0, - * and length = 2G effectively iterates over every PDE in the system. - * - * XXX: temp is not actually needed, but it saves doing the ALIGN operation. +/* + * gen6_for_each_pde() iterates over every pde from start until start+length. + * If start and start+length are not perfectly divisible, the macro will round + * down and up as needed. Start=0 and length=2G effectively iterates over + * every PDE in the system. The macro modifies ALL its parameters except 'pd', + * so each of the other parameters should preferably be a simple variable, or + * at most an lvalue with no side-effects! */ -#define gen6_for_each_pde(pt, pd, start, length, temp, iter) \ - for (iter = gen6_pde_index(start); \ - length > 0 && iter < I915_PDES ? \ - (pt = (pd)->page_table[iter]), 1 : 0; \ - iter++, \ - temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT) - start, \ - temp = min_t(unsigned, temp, length), \ - start += temp, length -= temp) - -#define gen6_for_all_pdes(pt, ppgtt, iter) \ - for (iter = 0; \ - pt = ppgtt->pd.page_table[iter], iter < I915_PDES; \ - iter++) +#define gen6_for_each_pde(pt, pd, start, length, iter) \ + for (iter = gen6_pde_index(start); \ + length > 0 && iter < I915_PDES && \ + (pt = (pd)->page_table[iter], true); \ + ({ u32 temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT); \ + temp = min(temp - start, length); \ + start += temp, length -= temp; }), ++iter) + +#define gen6_for_all_pdes(pt, pd, iter) \ + for (iter = 0; \ + iter < I915_PDES && \ + (pt = (pd)->page_table[iter], true); \ + ++iter) static inline uint32_t i915_pte_index(uint64_t address, uint32_t pde_shift) { @@ -518,9 +526,7 @@ int i915_ggtt_enable_hw(struct drm_device *dev); void i915_gem_init_ggtt(struct drm_device *dev); void i915_ggtt_cleanup_hw(struct drm_device *dev); -int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt); int i915_ppgtt_init_hw(struct drm_device *dev); -int i915_ppgtt_init_ring(struct drm_i915_gem_request *req); void i915_ppgtt_release(struct kref *kref); struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv); @@ -535,7 +541,7 @@ static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt) kref_put(&ppgtt->ref, i915_ppgtt_release); } -void i915_check_and_clear_faults(struct drm_device *dev); +void i915_check_and_clear_faults(struct drm_i915_private *dev_priv); void i915_gem_suspend_gtt_mappings(struct drm_device *dev); void i915_gem_restore_gtt_mappings(struct drm_device *dev); @@ -560,4 +566,36 @@ size_t i915_ggtt_view_size(struct drm_i915_gem_object *obj, const struct i915_ggtt_view *view); +/** + * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture + * @vma: VMA to iomap + * + * The passed in VMA has to be pinned in the global GTT mappable region. + * An extra pinning of the VMA is acquired for the return iomapping, + * the caller must call i915_vma_unpin_iomap to relinquish the pinning + * after the iomapping is no longer required. + * + * Callers must hold the struct_mutex. + * + * Returns a valid iomapped pointer or ERR_PTR. + */ +void __iomem *i915_vma_pin_iomap(struct i915_vma *vma); + +/** + * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap + * @vma: VMA to unpin + * + * Unpins the previously iomapped VMA from i915_vma_pin_iomap(). + * + * Callers must hold the struct_mutex. This function is only valid to be + * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap(). + */ +static inline void i915_vma_unpin_iomap(struct i915_vma *vma) +{ + lockdep_assert_held(&vma->vm->dev->struct_mutex); + GEM_BUG_ON(vma->pin_count == 0); + GEM_BUG_ON(vma->iomap == NULL); + vma->pin_count--; +} + #endif diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c index 71611bf21..f75bbd67a 100644 --- a/drivers/gpu/drm/i915/i915_gem_render_state.c +++ b/drivers/gpu/drm/i915/i915_gem_render_state.c @@ -29,7 +29,7 @@ #include "intel_renderstate.h" static const struct intel_renderstate_rodata * -render_state_get_rodata(struct drm_device *dev, const int gen) +render_state_get_rodata(const int gen) { switch (gen) { case 6: @@ -45,21 +45,22 @@ render_state_get_rodata(struct drm_device *dev, const int gen) return NULL; } -static int render_state_init(struct render_state *so, struct drm_device *dev) +static int render_state_init(struct render_state *so, + struct drm_i915_private *dev_priv) { int ret; - so->gen = INTEL_INFO(dev)->gen; - so->rodata = render_state_get_rodata(dev, so->gen); + so->gen = INTEL_GEN(dev_priv); + so->rodata = render_state_get_rodata(so->gen); if (so->rodata == NULL) return 0; if (so->rodata->batch_items * 4 > 4096) return -EINVAL; - so->obj = i915_gem_alloc_object(dev, 4096); - if (so->obj == NULL) - return -ENOMEM; + so->obj = i915_gem_object_create(&dev_priv->drm, 4096); + if (IS_ERR(so->obj)) + return PTR_ERR(so->obj); ret = i915_gem_obj_ggtt_pin(so->obj, 4096, 0); if (ret) @@ -93,6 +94,7 @@ free_gem: static int render_state_setup(struct render_state *so) { + struct drm_device *dev = so->obj->base.dev; const struct intel_renderstate_rodata *rodata = so->rodata; unsigned int i = 0, reloc_index = 0; struct page *page; @@ -134,6 +136,33 @@ static int render_state_setup(struct render_state *so) so->aux_batch_offset = i * sizeof(u32); + if (HAS_POOLED_EU(dev)) { + /* + * We always program 3x6 pool config but depending upon which + * subslice is disabled HW drops down to appropriate config + * shown below. + * + * In the below table 2x6 config always refers to + * fused-down version, native 2x6 is not available and can + * be ignored + * + * SNo subslices config eu pool configuration + * ----------------------------------------------------------- + * 1 3 subslices enabled (3x6) - 0x00777000 (9+9) + * 2 ss0 disabled (2x6) - 0x00777000 (3+9) + * 3 ss1 disabled (2x6) - 0x00770000 (6+6) + * 4 ss2 disabled (2x6) - 0x00007000 (9+3) + */ + u32 eu_pool_config = 0x00777000; + + OUT_BATCH(d, i, GEN9_MEDIA_POOL_STATE); + OUT_BATCH(d, i, GEN9_MEDIA_POOL_ENABLE); + OUT_BATCH(d, i, eu_pool_config); + OUT_BATCH(d, i, 0); + OUT_BATCH(d, i, 0); + OUT_BATCH(d, i, 0); + } + OUT_BATCH(d, i, MI_BATCH_BUFFER_END); so->aux_batch_size = (i * sizeof(u32)) - so->aux_batch_offset; @@ -177,7 +206,7 @@ int i915_gem_render_state_prepare(struct intel_engine_cs *engine, if (WARN_ON(engine->id != RCS)) return -ENOENT; - ret = render_state_init(so, engine->dev); + ret = render_state_init(so, engine->i915); if (ret) return ret; diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c index 66571466e..6f10b4214 100644 --- a/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -131,7 +131,16 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, unsigned long count = 0; trace_i915_gem_shrink(dev_priv, target, flags); - i915_gem_retire_requests(dev_priv->dev); + i915_gem_retire_requests(dev_priv); + + /* + * Unbinding of objects will require HW access; Let us not wake the + * device just to recover a little memory. If absolutely necessary, + * we will force the wake during oom-notifier. + */ + if ((flags & I915_SHRINK_BOUND) && + !intel_runtime_pm_get_if_in_use(dev_priv)) + flags &= ~I915_SHRINK_BOUND; /* * As we may completely rewrite the (un)bound list whilst unbinding @@ -197,7 +206,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, list_splice(&still_in_list, phase->list); } - i915_gem_retire_requests(dev_priv->dev); + if (flags & I915_SHRINK_BOUND) + intel_runtime_pm_put(dev_priv); + + i915_gem_retire_requests(dev_priv); return count; } @@ -245,7 +257,7 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) { struct drm_i915_private *dev_priv = container_of(shrinker, struct drm_i915_private, mm.shrinker); - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct drm_i915_gem_object *obj; unsigned long count; bool unlock; @@ -253,6 +265,8 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) if (!i915_gem_shrinker_lock(dev, &unlock)) return 0; + i915_gem_retire_requests(dev_priv); + count = 0; list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) if (can_release_pages(obj)) @@ -274,7 +288,7 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) { struct drm_i915_private *dev_priv = container_of(shrinker, struct drm_i915_private, mm.shrinker); - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; unsigned long freed; bool unlock; @@ -309,7 +323,7 @@ i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv, { unsigned long timeout = msecs_to_jiffies(timeout_ms) + 1; - while (!i915_gem_shrinker_lock(dev_priv->dev, &slu->unlock)) { + while (!i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock)) { schedule_timeout_killable(1); if (fatal_signal_pending(current)) return false; @@ -330,7 +344,7 @@ i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv, { dev_priv->mm.interruptible = slu->was_interruptible; if (slu->unlock) - mutex_unlock(&dev_priv->dev->struct_mutex); + mutex_unlock(&dev_priv->drm.struct_mutex); } static int @@ -345,7 +359,9 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000)) return NOTIFY_DONE; + intel_runtime_pm_get(dev_priv); freed_pages = i915_gem_shrink_all(dev_priv); + intel_runtime_pm_put(dev_priv); /* Because we may be allocating inside our own driver, we cannot * assert that there are no objects with pinned pages that are not @@ -386,17 +402,35 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr struct drm_i915_private *dev_priv = container_of(nb, struct drm_i915_private, mm.vmap_notifier); struct shrinker_lock_uninterruptible slu; - unsigned long freed_pages; + struct i915_vma *vma, *next; + unsigned long freed_pages = 0; + int ret; if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000)) return NOTIFY_DONE; - freed_pages = i915_gem_shrink(dev_priv, -1UL, - I915_SHRINK_BOUND | - I915_SHRINK_UNBOUND | - I915_SHRINK_ACTIVE | - I915_SHRINK_VMAPS); + /* Force everything onto the inactive lists */ + ret = i915_gem_wait_for_idle(dev_priv); + if (ret) + goto out; + + intel_runtime_pm_get(dev_priv); + freed_pages += i915_gem_shrink(dev_priv, -1UL, + I915_SHRINK_BOUND | + I915_SHRINK_UNBOUND | + I915_SHRINK_ACTIVE | + I915_SHRINK_VMAPS); + intel_runtime_pm_put(dev_priv); + + /* We also want to clear any cached iomaps as they wrap vmap */ + list_for_each_entry_safe(vma, next, + &dev_priv->ggtt.base.inactive_list, vm_link) { + unsigned long count = vma->node.size >> PAGE_SHIFT; + if (vma->iomap && i915_vma_unbind(vma) == 0) + freed_pages += count; + } +out: i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu); *(unsigned long *)ptr += freed_pages; diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index 44004e3f0..66be299a1 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -111,9 +111,9 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) if (INTEL_INFO(dev)->gen >= 3) { u32 bsm; - pci_read_config_dword(dev->pdev, BSM, &bsm); + pci_read_config_dword(dev->pdev, INTEL_BSM, &bsm); - base = bsm & BSM_MASK; + base = bsm & INTEL_BSM_MASK; } else if (IS_I865G(dev)) { u16 toud = 0; @@ -270,7 +270,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) void i915_gem_cleanup_stolen(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!drm_mm_initialized(&dev_priv->mm.stolen)) return; @@ -550,7 +550,7 @@ static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj) static void i915_gem_object_release_stolen(struct drm_i915_gem_object *obj) { - struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(obj->base.dev); if (obj->stolen) { i915_gem_stolen_remove_node(dev_priv, obj->stolen); @@ -601,7 +601,7 @@ cleanup: struct drm_i915_gem_object * i915_gem_object_create_stolen(struct drm_device *dev, u32 size) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; struct drm_mm_node *stolen; int ret; diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index b9bdb3403..803019973 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -125,7 +125,7 @@ i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode) if (INTEL_INFO(obj->base.dev)->gen >= 4) return true; - if (INTEL_INFO(obj->base.dev)->gen == 3) { + if (IS_GEN3(obj->base.dev)) { if (i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) return false; } else { @@ -162,7 +162,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_set_tiling *args = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; int ret = 0; @@ -229,7 +229,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, */ if (obj->map_and_fenceable && !i915_gem_object_fence_ok(obj, args->tiling_mode)) - ret = i915_gem_object_ggtt_unbind(obj); + ret = i915_vma_unbind(i915_gem_obj_to_ggtt(obj)); if (ret == 0) { if (obj->pages && @@ -294,7 +294,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_get_tiling *args = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; obj = to_intel_bo(drm_gem_object_lookup(file, args->handle)); diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c index 32d9726e3..2314c8832 100644 --- a/drivers/gpu/drm/i915/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c @@ -706,7 +706,8 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) static void i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) { - struct sg_page_iter sg_iter; + struct sgt_iter sgt_iter; + struct page *page; BUG_ON(obj->userptr.work != NULL); __i915_gem_userptr_set_active(obj, false); @@ -716,9 +717,7 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) i915_gem_gtt_finish_object(obj); - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { - struct page *page = sg_page_iter_page(&sg_iter); - + for_each_sgt_page(page, sgt_iter, obj->pages) { if (obj->dirty) set_page_dirty(page); @@ -855,11 +854,8 @@ i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file return 0; } -int -i915_gem_init_userptr(struct drm_device *dev) +void i915_gem_init_userptr(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); mutex_init(&dev_priv->mm_lock); hash_init(dev_priv->mm_structs); - return 0; } diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 89725c9ef..9d73d2216 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -332,7 +332,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, const struct i915_error_state_file_priv *error_priv) { struct drm_device *dev = error_priv->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_error_state *error = error_priv->error; struct drm_i915_error_object *obj; int i, j, offset, elt; @@ -411,7 +411,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, err_printf(m, "DONE_REG: 0x%08x\n", error->done_reg); } - if (INTEL_INFO(dev)->gen == 7) + if (IS_GEN7(dev)) err_printf(m, "ERR_INT: 0x%08x\n", error->err_int); for (i = 0; i < ARRAY_SIZE(error->ring); i++) @@ -463,6 +463,18 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, } } + if (error->ring[i].num_waiters) { + err_printf(m, "%s --- %d waiters\n", + dev_priv->engine[i].name, + error->ring[i].num_waiters); + for (j = 0; j < error->ring[i].num_waiters; j++) { + err_printf(m, " seqno 0x%08x for %s [%d]\n", + error->ring[i].waiters[j].seqno, + error->ring[i].waiters[j].comm, + error->ring[i].waiters[j].pid); + } + } + if ((obj = error->ring[i].ringbuffer)) { err_printf(m, "%s --- ringbuffer = 0x%08x\n", dev_priv->engine[i].name, @@ -488,7 +500,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, hws_page[elt+1], hws_page[elt+2], hws_page[elt+3]); - offset += 16; + offset += 16; } } @@ -605,8 +617,9 @@ static void i915_error_state_free(struct kref *error_ref) i915_error_object_free(error->ring[i].ringbuffer); i915_error_object_free(error->ring[i].hws_page); i915_error_object_free(error->ring[i].ctx); - kfree(error->ring[i].requests); i915_error_object_free(error->ring[i].wa_ctx); + kfree(error->ring[i].requests); + kfree(error->ring[i].waiters); } i915_error_object_free(error->semaphore_obj); @@ -824,19 +837,18 @@ static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv, return error_code; } -static void i915_gem_record_fences(struct drm_device *dev, +static void i915_gem_record_fences(struct drm_i915_private *dev_priv, struct drm_i915_error_state *error) { - struct drm_i915_private *dev_priv = dev->dev_private; int i; - if (IS_GEN3(dev) || IS_GEN2(dev)) { + if (IS_GEN3(dev_priv) || IS_GEN2(dev_priv)) { for (i = 0; i < dev_priv->num_fence_regs; i++) error->fence[i] = I915_READ(FENCE_REG(i)); - } else if (IS_GEN5(dev) || IS_GEN4(dev)) { + } else if (IS_GEN5(dev_priv) || IS_GEN4(dev_priv)) { for (i = 0; i < dev_priv->num_fence_regs; i++) error->fence[i] = I915_READ64(FENCE_REG_965_LO(i)); - } else if (INTEL_INFO(dev)->gen >= 6) { + } else if (INTEL_GEN(dev_priv) >= 6) { for (i = 0; i < dev_priv->num_fence_regs; i++) error->fence[i] = I915_READ64(FENCE_REG_GEN6_LO(i)); } @@ -851,7 +863,7 @@ static void gen8_record_semaphore_state(struct drm_i915_private *dev_priv, struct intel_engine_cs *to; enum intel_engine_id id; - if (!i915_semaphore_is_enabled(dev_priv->dev)) + if (!i915_semaphore_is_enabled(dev_priv)) return; if (!error->semaphore_obj) @@ -893,31 +905,71 @@ static void gen6_record_semaphore_state(struct drm_i915_private *dev_priv, } } -static void i915_record_ring_state(struct drm_device *dev, +static void engine_record_waiters(struct intel_engine_cs *engine, + struct drm_i915_error_ring *ering) +{ + struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct drm_i915_error_waiter *waiter; + struct rb_node *rb; + int count; + + ering->num_waiters = 0; + ering->waiters = NULL; + + spin_lock(&b->lock); + count = 0; + for (rb = rb_first(&b->waiters); rb != NULL; rb = rb_next(rb)) + count++; + spin_unlock(&b->lock); + + waiter = NULL; + if (count) + waiter = kmalloc_array(count, + sizeof(struct drm_i915_error_waiter), + GFP_ATOMIC); + if (!waiter) + return; + + ering->waiters = waiter; + + spin_lock(&b->lock); + for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { + struct intel_wait *w = container_of(rb, typeof(*w), node); + + strcpy(waiter->comm, w->tsk->comm); + waiter->pid = w->tsk->pid; + waiter->seqno = w->seqno; + waiter++; + + if (++ering->num_waiters == count) + break; + } + spin_unlock(&b->lock); +} + +static void i915_record_ring_state(struct drm_i915_private *dev_priv, struct drm_i915_error_state *error, struct intel_engine_cs *engine, struct drm_i915_error_ring *ering) { - struct drm_i915_private *dev_priv = dev->dev_private; - - if (INTEL_INFO(dev)->gen >= 6) { + if (INTEL_GEN(dev_priv) >= 6) { ering->rc_psmi = I915_READ(RING_PSMI_CTL(engine->mmio_base)); ering->fault_reg = I915_READ(RING_FAULT_REG(engine)); - if (INTEL_INFO(dev)->gen >= 8) + if (INTEL_GEN(dev_priv) >= 8) gen8_record_semaphore_state(dev_priv, error, engine, ering); else gen6_record_semaphore_state(dev_priv, engine, ering); } - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { ering->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base)); ering->ipeir = I915_READ(RING_IPEIR(engine->mmio_base)); ering->ipehr = I915_READ(RING_IPEHR(engine->mmio_base)); ering->instdone = I915_READ(RING_INSTDONE(engine->mmio_base)); ering->instps = I915_READ(RING_INSTPS(engine->mmio_base)); ering->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base)); - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) { ering->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(engine->mmio_base)) << 32; ering->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(engine->mmio_base)) << 32; } @@ -929,20 +981,20 @@ static void i915_record_ring_state(struct drm_device *dev, ering->instdone = I915_READ(GEN2_INSTDONE); } - ering->waiting = waitqueue_active(&engine->irq_queue); + ering->waiting = intel_engine_has_waiter(engine); ering->instpm = I915_READ(RING_INSTPM(engine->mmio_base)); ering->acthd = intel_ring_get_active_head(engine); - ering->seqno = engine->get_seqno(engine); + ering->seqno = intel_engine_get_seqno(engine); ering->last_seqno = engine->last_submitted_seqno; ering->start = I915_READ_START(engine); ering->head = I915_READ_HEAD(engine); ering->tail = I915_READ_TAIL(engine); ering->ctl = I915_READ_CTL(engine); - if (I915_NEED_GFX_HWS(dev)) { + if (I915_NEED_GFX_HWS(dev_priv)) { i915_reg_t mmio; - if (IS_GEN7(dev)) { + if (IS_GEN7(dev_priv)) { switch (engine->id) { default: case RCS: @@ -958,7 +1010,7 @@ static void i915_record_ring_state(struct drm_device *dev, mmio = VEBOX_HWS_PGA_GEN7; break; } - } else if (IS_GEN6(engine->dev)) { + } else if (IS_GEN6(engine->i915)) { mmio = RING_HWS_PGA_GEN6(engine->mmio_base); } else { /* XXX: gen8 returns to sanity */ @@ -971,18 +1023,18 @@ static void i915_record_ring_state(struct drm_device *dev, ering->hangcheck_score = engine->hangcheck.score; ering->hangcheck_action = engine->hangcheck.action; - if (USES_PPGTT(dev)) { + if (USES_PPGTT(dev_priv)) { int i; ering->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine)); - if (IS_GEN6(dev)) + if (IS_GEN6(dev_priv)) ering->vm_info.pp_dir_base = I915_READ(RING_PP_DIR_BASE_READ(engine)); - else if (IS_GEN7(dev)) + else if (IS_GEN7(dev_priv)) ering->vm_info.pp_dir_base = I915_READ(RING_PP_DIR_BASE(engine)); - else if (INTEL_INFO(dev)->gen >= 8) + else if (INTEL_GEN(dev_priv) >= 8) for (i = 0; i < 4; i++) { ering->vm_info.pdp[i] = I915_READ(GEN8_RING_PDP_UDW(engine, i)); @@ -998,7 +1050,7 @@ static void i915_gem_record_active_context(struct intel_engine_cs *engine, struct drm_i915_error_state *error, struct drm_i915_error_ring *ering) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; struct drm_i915_gem_object *obj; /* Currently render ring is the only HW context user */ @@ -1016,34 +1068,33 @@ static void i915_gem_record_active_context(struct intel_engine_cs *engine, } } -static void i915_gem_record_rings(struct drm_device *dev, +static void i915_gem_record_rings(struct drm_i915_private *dev_priv, struct drm_i915_error_state *error) { - struct drm_i915_private *dev_priv = to_i915(dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; struct drm_i915_gem_request *request; int i, count; for (i = 0; i < I915_NUM_ENGINES; i++) { struct intel_engine_cs *engine = &dev_priv->engine[i]; - struct intel_ringbuffer *rbuf; error->ring[i].pid = -1; - if (engine->dev == NULL) + if (!intel_engine_initialized(engine)) continue; error->ring[i].valid = true; - i915_record_ring_state(dev, error, engine, &error->ring[i]); + i915_record_ring_state(dev_priv, error, engine, &error->ring[i]); + engine_record_waiters(engine, &error->ring[i]); request = i915_gem_find_active_request(engine); if (request) { struct i915_address_space *vm; + struct intel_ringbuffer *rb; - vm = request->ctx && request->ctx->ppgtt ? - &request->ctx->ppgtt->base : - &ggtt->base; + vm = request->ctx->ppgtt ? + &request->ctx->ppgtt->base : &ggtt->base; /* We need to copy these to an anonymous buffer * as the simplest method to avoid being overwritten @@ -1070,26 +1121,17 @@ static void i915_gem_record_rings(struct drm_device *dev, } rcu_read_unlock(); } - } - if (i915.enable_execlists) { - /* TODO: This is only a small fix to keep basic error - * capture working, but we need to add more information - * for it to be useful (e.g. dump the context being - * executed). - */ - if (request) - rbuf = request->ctx->engine[engine->id].ringbuf; - else - rbuf = dev_priv->kernel_context->engine[engine->id].ringbuf; - } else - rbuf = engine->buffer; - - error->ring[i].cpu_ring_head = rbuf->head; - error->ring[i].cpu_ring_tail = rbuf->tail; + error->simulated |= + request->ctx->flags & CONTEXT_NO_ERROR_CAPTURE; - error->ring[i].ringbuffer = - i915_error_ggtt_object_create(dev_priv, rbuf->obj); + rb = request->ringbuf; + error->ring[i].cpu_ring_head = rb->head; + error->ring[i].cpu_ring_tail = rb->tail; + error->ring[i].ringbuffer = + i915_error_ggtt_object_create(dev_priv, + rb->obj); + } error->ring[i].hws_page = i915_error_ggtt_object_create(dev_priv, @@ -1234,7 +1276,7 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv, static void i915_capture_reg_state(struct drm_i915_private *dev_priv, struct drm_i915_error_state *error) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; int i; /* General organization @@ -1301,15 +1343,14 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv, error->eir = I915_READ(EIR); error->pgtbl_er = I915_READ(PGTBL_ER); - i915_get_extra_instdone(dev, error->extra_instdone); + i915_get_extra_instdone(dev_priv, error->extra_instdone); } -static void i915_error_capture_msg(struct drm_device *dev, +static void i915_error_capture_msg(struct drm_i915_private *dev_priv, struct drm_i915_error_state *error, u32 engine_mask, const char *error_msg) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 ecode; int ring_id = -1, len; @@ -1317,7 +1358,7 @@ static void i915_error_capture_msg(struct drm_device *dev, len = scnprintf(error->error_msg, sizeof(error->error_msg), "GPU HANG: ecode %d:%d:0x%08x", - INTEL_INFO(dev)->gen, ring_id, ecode); + INTEL_GEN(dev_priv), ring_id, ecode); if (ring_id != -1 && error->ring[ring_id].pid != -1) len += scnprintf(error->error_msg + len, @@ -1352,14 +1393,17 @@ static void i915_capture_gen_state(struct drm_i915_private *dev_priv, * out a structure which becomes available in debugfs for user level tools * to pick up. */ -void i915_capture_error_state(struct drm_device *dev, u32 engine_mask, +void i915_capture_error_state(struct drm_i915_private *dev_priv, + u32 engine_mask, const char *error_msg) { static bool warned; - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_error_state *error; unsigned long flags; + if (READ_ONCE(dev_priv->gpu_error.first_error)) + return; + /* Account for pipe specific data like PIPE*STAT */ error = kzalloc(sizeof(*error), GFP_ATOMIC); if (!error) { @@ -1372,23 +1416,25 @@ void i915_capture_error_state(struct drm_device *dev, u32 engine_mask, i915_capture_gen_state(dev_priv, error); i915_capture_reg_state(dev_priv, error); i915_gem_capture_buffers(dev_priv, error); - i915_gem_record_fences(dev, error); - i915_gem_record_rings(dev, error); + i915_gem_record_fences(dev_priv, error); + i915_gem_record_rings(dev_priv, error); do_gettimeofday(&error->time); - error->overlay = intel_overlay_capture_error_state(dev); - error->display = intel_display_capture_error_state(dev); + error->overlay = intel_overlay_capture_error_state(dev_priv); + error->display = intel_display_capture_error_state(dev_priv); - i915_error_capture_msg(dev, error, engine_mask, error_msg); + i915_error_capture_msg(dev_priv, error, engine_mask, error_msg); DRM_INFO("%s\n", error->error_msg); - spin_lock_irqsave(&dev_priv->gpu_error.lock, flags); - if (dev_priv->gpu_error.first_error == NULL) { - dev_priv->gpu_error.first_error = error; - error = NULL; + if (!error->simulated) { + spin_lock_irqsave(&dev_priv->gpu_error.lock, flags); + if (!dev_priv->gpu_error.first_error) { + dev_priv->gpu_error.first_error = error; + error = NULL; + } + spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags); } - spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags); if (error) { i915_error_state_free(&error->ref); @@ -1400,7 +1446,8 @@ void i915_capture_error_state(struct drm_device *dev, u32 engine_mask, DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n"); DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n"); DRM_INFO("The gpu crash dump is required to analyze gpu hangs, so please always attach it.\n"); - DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n", dev->primary->index); + DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n", + dev_priv->drm.primary->index); warned = true; } } @@ -1408,7 +1455,7 @@ void i915_capture_error_state(struct drm_device *dev, u32 engine_mask, void i915_error_state_get(struct drm_device *dev, struct i915_error_state_file_priv *error_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); spin_lock_irq(&dev_priv->gpu_error.lock); error_priv->error = dev_priv->gpu_error.first_error; @@ -1426,7 +1473,7 @@ void i915_error_state_put(struct i915_error_state_file_priv *error_priv) void i915_destroy_error_state(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_error_state *error; spin_lock_irq(&dev_priv->gpu_error.lock); @@ -1450,17 +1497,17 @@ const char *i915_cache_level_str(struct drm_i915_private *i915, int type) } /* NB: please notice the memset */ -void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone) +void i915_get_extra_instdone(struct drm_i915_private *dev_priv, + uint32_t *instdone) { - struct drm_i915_private *dev_priv = dev->dev_private; memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG); - if (IS_GEN2(dev) || IS_GEN3(dev)) + if (IS_GEN2(dev_priv) || IS_GEN3(dev_priv)) instdone[0] = I915_READ(GEN2_INSTDONE); - else if (IS_GEN4(dev) || IS_GEN5(dev) || IS_GEN6(dev)) { + else if (IS_GEN4(dev_priv) || IS_GEN5(dev_priv) || IS_GEN6(dev_priv)) { instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE)); instdone[1] = I915_READ(GEN4_INSTDONE1); - } else if (INTEL_INFO(dev)->gen >= 7) { + } else if (INTEL_GEN(dev_priv) >= 7) { instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE)); instdone[1] = I915_READ(GEN7_SC_INSTDONE); instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE); diff --git a/drivers/gpu/drm/i915/i915_guc_reg.h b/drivers/gpu/drm/i915/i915_guc_reg.h index 80786d9f9..cf5a65be4 100644 --- a/drivers/gpu/drm/i915/i915_guc_reg.h +++ b/drivers/gpu/drm/i915/i915_guc_reg.h @@ -67,11 +67,11 @@ #define GUC_WOPCM_OFFSET_VALUE 0x80000 /* 512KB */ #define GUC_MAX_IDLE_COUNT _MMIO(0xC3E4) +/* Defines WOPCM space available to GuC firmware */ #define GUC_WOPCM_SIZE _MMIO(0xc050) -#define GUC_WOPCM_SIZE_VALUE (0x80 << 12) /* 512KB */ - /* GuC addresses below GUC_WOPCM_TOP don't map through the GTT */ -#define GUC_WOPCM_TOP (GUC_WOPCM_SIZE_VALUE) +#define GUC_WOPCM_TOP (0x80 << 12) /* 512KB */ +#define BXT_GUC_WOPCM_RC6_RESERVED (0x10 << 12) /* 64KB */ #define GEN8_GT_PM_CONFIG _MMIO(0x138140) #define GEN9LP_GT_PM_CONFIG _MMIO(0x138140) diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c index d40c13fb6..2112e029d 100644 --- a/drivers/gpu/drm/i915/i915_guc_submission.c +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -97,8 +97,14 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len) I915_WRITE(HOST2GUC_INTERRUPT, HOST2GUC_TRIGGER); - /* No HOST2GUC command should take longer than 10ms */ - ret = wait_for_atomic(host2guc_action_response(dev_priv, &status), 10); + /* + * Fast commands should complete in less than 10us, so sample quickly + * up to that length of time, then switch to a slower sleep-wait loop. + * No HOST2GUC command should ever take longer than 10ms. + */ + ret = wait_for_us(host2guc_action_response(dev_priv, &status), 10); + if (ret) + ret = wait_for(host2guc_action_response(dev_priv, &status), 10); if (status != GUC2HOST_STATUS_SUCCESS) { /* * Either the GuC explicitly returned an error (which @@ -153,13 +159,11 @@ static int host2guc_sample_forcewake(struct intel_guc *guc, struct i915_guc_client *client) { struct drm_i915_private *dev_priv = guc_to_i915(guc); - struct drm_device *dev = dev_priv->dev; u32 data[2]; data[0] = HOST2GUC_ACTION_SAMPLE_FORCEWAKE; /* WaRsDisableCoarsePowerGating:skl,bxt */ - if (!intel_enable_rc6(dev) || - NEEDS_WaRsDisableCoarsePowerGating(dev)) + if (!intel_enable_rc6() || NEEDS_WaRsDisableCoarsePowerGating(dev_priv)) data[1] = 0; else /* bit 0 and 1 are for Render and Media domain separately */ @@ -175,94 +179,88 @@ static int host2guc_sample_forcewake(struct intel_guc *guc, * client object which contains the page being used for the doorbell */ -static void guc_init_doorbell(struct intel_guc *guc, - struct i915_guc_client *client) +static int guc_update_doorbell_id(struct intel_guc *guc, + struct i915_guc_client *client, + u16 new_id) { + struct sg_table *sg = guc->ctx_pool_obj->pages; + void *doorbell_bitmap = guc->doorbell_bitmap; struct guc_doorbell_info *doorbell; + struct guc_context_desc desc; + size_t len; doorbell = client->client_base + client->doorbell_offset; - doorbell->db_status = GUC_DOORBELL_ENABLED; - doorbell->cookie = 0; -} - -static int guc_ring_doorbell(struct i915_guc_client *gc) -{ - struct guc_process_desc *desc; - union guc_doorbell_qw db_cmp, db_exc, db_ret; - union guc_doorbell_qw *db; - int attempt = 2, ret = -EAGAIN; - - desc = gc->client_base + gc->proc_desc_offset; - - /* Update the tail so it is visible to GuC */ - desc->tail = gc->wq_tail; - - /* current cookie */ - db_cmp.db_status = GUC_DOORBELL_ENABLED; - db_cmp.cookie = gc->cookie; - - /* cookie to be updated */ - db_exc.db_status = GUC_DOORBELL_ENABLED; - db_exc.cookie = gc->cookie + 1; - if (db_exc.cookie == 0) - db_exc.cookie = 1; - - /* pointer of current doorbell cacheline */ - db = gc->client_base + gc->doorbell_offset; - - while (attempt--) { - /* lets ring the doorbell */ - db_ret.value_qw = atomic64_cmpxchg((atomic64_t *)db, - db_cmp.value_qw, db_exc.value_qw); - - /* if the exchange was successfully executed */ - if (db_ret.value_qw == db_cmp.value_qw) { - /* db was successfully rung */ - gc->cookie = db_exc.cookie; - ret = 0; - break; - } + if (client->doorbell_id != GUC_INVALID_DOORBELL_ID && + test_bit(client->doorbell_id, doorbell_bitmap)) { + /* Deactivate the old doorbell */ + doorbell->db_status = GUC_DOORBELL_DISABLED; + (void)host2guc_release_doorbell(guc, client); + __clear_bit(client->doorbell_id, doorbell_bitmap); + } - /* XXX: doorbell was lost and need to acquire it again */ - if (db_ret.db_status == GUC_DOORBELL_DISABLED) - break; + /* Update the GuC's idea of the doorbell ID */ + len = sg_pcopy_to_buffer(sg->sgl, sg->nents, &desc, sizeof(desc), + sizeof(desc) * client->ctx_index); + if (len != sizeof(desc)) + return -EFAULT; + desc.db_id = new_id; + len = sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc), + sizeof(desc) * client->ctx_index); + if (len != sizeof(desc)) + return -EFAULT; - DRM_ERROR("Cookie mismatch. Expected %d, returned %d\n", - db_cmp.cookie, db_ret.cookie); + client->doorbell_id = new_id; + if (new_id == GUC_INVALID_DOORBELL_ID) + return 0; - /* update the cookie to newly read cookie from GuC */ - db_cmp.cookie = db_ret.cookie; - db_exc.cookie = db_ret.cookie + 1; - if (db_exc.cookie == 0) - db_exc.cookie = 1; - } + /* Activate the new doorbell */ + __set_bit(new_id, doorbell_bitmap); + doorbell->cookie = 0; + doorbell->db_status = GUC_DOORBELL_ENABLED; + return host2guc_allocate_doorbell(guc, client); +} - return ret; +static int guc_init_doorbell(struct intel_guc *guc, + struct i915_guc_client *client, + uint16_t db_id) +{ + return guc_update_doorbell_id(guc, client, db_id); } static void guc_disable_doorbell(struct intel_guc *guc, struct i915_guc_client *client) { - struct drm_i915_private *dev_priv = guc_to_i915(guc); - struct guc_doorbell_info *doorbell; - i915_reg_t drbreg = GEN8_DRBREGL(client->doorbell_id); - int value; - - doorbell = client->client_base + client->doorbell_offset; + (void)guc_update_doorbell_id(guc, client, GUC_INVALID_DOORBELL_ID); - doorbell->db_status = GUC_DOORBELL_DISABLED; + /* XXX: wait for any interrupts */ + /* XXX: wait for workqueue to drain */ +} - I915_WRITE(drbreg, I915_READ(drbreg) & ~GEN8_DRB_VALID); +static uint16_t +select_doorbell_register(struct intel_guc *guc, uint32_t priority) +{ + /* + * The bitmap tracks which doorbell registers are currently in use. + * It is split into two halves; the first half is used for normal + * priority contexts, the second half for high-priority ones. + * Note that logically higher priorities are numerically less than + * normal ones, so the test below means "is it high-priority?" + */ + const bool hi_pri = (priority <= GUC_CTX_PRIORITY_HIGH); + const uint16_t half = GUC_MAX_DOORBELLS / 2; + const uint16_t start = hi_pri ? half : 0; + const uint16_t end = start + half; + uint16_t id; - value = I915_READ(drbreg); - WARN_ON((value & GEN8_DRB_VALID) != 0); + id = find_next_zero_bit(guc->doorbell_bitmap, end, start); + if (id == end) + id = GUC_INVALID_DOORBELL_ID; - I915_WRITE(GEN8_DRBREGU(client->doorbell_id), 0); - I915_WRITE(drbreg, 0); + DRM_DEBUG_DRIVER("assigned %s priority doorbell id 0x%x\n", + hi_pri ? "high" : "normal", id); - /* XXX: wait for any interrupts */ - /* XXX: wait for workqueue to drain */ + return id; } /* @@ -289,37 +287,6 @@ static uint32_t select_doorbell_cacheline(struct intel_guc *guc) return offset; } -static uint16_t assign_doorbell(struct intel_guc *guc, uint32_t priority) -{ - /* - * The bitmap is split into two halves; the first half is used for - * normal priority contexts, the second half for high-priority ones. - * Note that logically higher priorities are numerically less than - * normal ones, so the test below means "is it high-priority?" - */ - const bool hi_pri = (priority <= GUC_CTX_PRIORITY_HIGH); - const uint16_t half = GUC_MAX_DOORBELLS / 2; - const uint16_t start = hi_pri ? half : 0; - const uint16_t end = start + half; - uint16_t id; - - id = find_next_zero_bit(guc->doorbell_bitmap, end, start); - if (id == end) - id = GUC_INVALID_DOORBELL_ID; - else - bitmap_set(guc->doorbell_bitmap, id, 1); - - DRM_DEBUG_DRIVER("assigned %s priority doorbell id 0x%x\n", - hi_pri ? "high" : "normal", id); - - return id; -} - -static void release_doorbell(struct intel_guc *guc, uint16_t id) -{ - bitmap_clear(guc->doorbell_bitmap, id, 1); -} - /* * Initialise the process descriptor shared with the GuC firmware. */ @@ -361,10 +328,9 @@ static void guc_init_ctx_desc(struct intel_guc *guc, struct drm_i915_gem_object *client_obj = client->client_obj; struct drm_i915_private *dev_priv = guc_to_i915(guc); struct intel_engine_cs *engine; - struct intel_context *ctx = client->owner; + struct i915_gem_context *ctx = client->owner; struct guc_context_desc desc; struct sg_table *sg; - enum intel_engine_id id; u32 gfx_addr; memset(&desc, 0, sizeof(desc)); @@ -374,10 +340,10 @@ static void guc_init_ctx_desc(struct intel_guc *guc, desc.priority = client->priority; desc.db_id = client->doorbell_id; - for_each_engine_id(engine, dev_priv, id) { + for_each_engine(engine, dev_priv) { + struct intel_context *ce = &ctx->engine[engine->id]; struct guc_execlist_context *lrc = &desc.lrc[engine->guc_id]; struct drm_i915_gem_object *obj; - uint64_t ctx_desc; /* TODO: We have a design issue to be solved here. Only when we * receive the first batch, we know which engine is used by the @@ -386,20 +352,18 @@ static void guc_init_ctx_desc(struct intel_guc *guc, * for now who owns a GuC client. But for future owner of GuC * client, need to make sure lrc is pinned prior to enter here. */ - obj = ctx->engine[id].state; - if (!obj) + if (!ce->state) break; /* XXX: continue? */ - ctx_desc = intel_lr_context_descriptor(ctx, engine); - lrc->context_desc = (u32)ctx_desc; + lrc->context_desc = lower_32_bits(ce->lrc_desc); /* The state page is after PPHWSP */ - gfx_addr = i915_gem_obj_ggtt_offset(obj); + gfx_addr = i915_gem_obj_ggtt_offset(ce->state); lrc->ring_lcra = gfx_addr + LRC_STATE_PN * PAGE_SIZE; lrc->context_id = (client->ctx_index << GUC_ELC_CTXID_OFFSET) | (engine->guc_id << GUC_ELC_ENGINE_OFFSET); - obj = ctx->engine[id].ringbuf->obj; + obj = ce->ringbuf->obj; gfx_addr = i915_gem_obj_ggtt_offset(obj); lrc->ring_begin = gfx_addr; @@ -427,7 +391,7 @@ static void guc_init_ctx_desc(struct intel_guc *guc, desc.wq_size = client->wq_size; /* - * XXX: Take LRCs from an existing intel_context if this is not an + * XXX: Take LRCs from an existing context if this is not an * IsKMDCreatedContext client */ desc.desc_private = (uintptr_t)client; @@ -451,47 +415,64 @@ static void guc_fini_ctx_desc(struct intel_guc *guc, sizeof(desc) * client->ctx_index); } -int i915_guc_wq_check_space(struct i915_guc_client *gc) +/** + * i915_guc_wq_check_space() - check that the GuC can accept a request + * @request: request associated with the commands + * + * Return: 0 if space is available + * -EAGAIN if space is not currently available + * + * This function must be called (and must return 0) before a request + * is submitted to the GuC via i915_guc_submit() below. Once a result + * of 0 has been returned, it remains valid until (but only until) + * the next call to submit(). + * + * This precheck allows the caller to determine in advance that space + * will be available for the next submission before committing resources + * to it, and helps avoid late failures with complicated recovery paths. + */ +int i915_guc_wq_check_space(struct drm_i915_gem_request *request) { + const size_t wqi_size = sizeof(struct guc_wq_item); + struct i915_guc_client *gc = request->i915->guc.execbuf_client; struct guc_process_desc *desc; - u32 size = sizeof(struct guc_wq_item); - int ret = -ETIMEDOUT, timeout_counter = 200; + u32 freespace; - if (!gc) - return 0; + GEM_BUG_ON(gc == NULL); desc = gc->client_base + gc->proc_desc_offset; - while (timeout_counter-- > 0) { - if (CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size) >= size) { - ret = 0; - break; - } + freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size); + if (likely(freespace >= wqi_size)) + return 0; - if (timeout_counter) - usleep_range(1000, 2000); - }; + gc->no_wq_space += 1; - return ret; + return -EAGAIN; } -static int guc_add_workqueue_item(struct i915_guc_client *gc, - struct drm_i915_gem_request *rq) +static void guc_add_workqueue_item(struct i915_guc_client *gc, + struct drm_i915_gem_request *rq) { + /* wqi_len is in DWords, and does not include the one-word header */ + const size_t wqi_size = sizeof(struct guc_wq_item); + const u32 wqi_len = wqi_size/sizeof(u32) - 1; struct guc_process_desc *desc; struct guc_wq_item *wqi; void *base; - u32 tail, wq_len, wq_off, space; + u32 freespace, tail, wq_off, wq_page; desc = gc->client_base + gc->proc_desc_offset; - space = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size); - if (WARN_ON(space < sizeof(struct guc_wq_item))) - return -ENOSPC; /* shouldn't happen */ - /* postincrement WQ tail for next time */ - wq_off = gc->wq_tail; - gc->wq_tail += sizeof(struct guc_wq_item); - gc->wq_tail &= gc->wq_size - 1; + /* Free space is guaranteed, see i915_guc_wq_check_space() above */ + freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size); + GEM_BUG_ON(freespace < wqi_size); + + /* The GuC firmware wants the tail index in QWords, not bytes */ + tail = rq->tail; + GEM_BUG_ON(tail & 7); + tail >>= 3; + GEM_BUG_ON(tail > WQ_RING_TAIL_MAX); /* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we * should not have the case where structure wqi is across page, neither @@ -500,19 +481,23 @@ static int guc_add_workqueue_item(struct i915_guc_client *gc, * XXX: if not the case, we need save data to a temp wqi and copy it to * workqueue buffer dw by dw. */ - WARN_ON(sizeof(struct guc_wq_item) != 16); - WARN_ON(wq_off & 3); + BUILD_BUG_ON(wqi_size != 16); + + /* postincrement WQ tail for next time */ + wq_off = gc->wq_tail; + gc->wq_tail += wqi_size; + gc->wq_tail &= gc->wq_size - 1; + GEM_BUG_ON(wq_off & (wqi_size - 1)); - /* wq starts from the page after doorbell / process_desc */ - base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, - (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT)); + /* WQ starts from the page after doorbell / process_desc */ + wq_page = (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT; wq_off &= PAGE_SIZE - 1; + base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, wq_page)); wqi = (struct guc_wq_item *)((char *)base + wq_off); - /* len does not include the header */ - wq_len = sizeof(struct guc_wq_item) / sizeof(u32) - 1; + /* Now fill in the 4-word work queue item */ wqi->header = WQ_TYPE_INORDER | - (wq_len << WQ_LEN_SHIFT) | + (wqi_len << WQ_LEN_SHIFT) | (rq->engine->guc_id << WQ_TARGET_SHIFT) | WQ_NO_WCFLUSH_WAIT; @@ -520,48 +505,105 @@ static int guc_add_workqueue_item(struct i915_guc_client *gc, wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, rq->engine); - /* The GuC firmware wants the tail index in QWords, not bytes */ - tail = rq->ringbuf->tail >> 3; wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT; - wqi->fence_id = 0; /*XXX: what fence to be here */ + wqi->fence_id = rq->seqno; kunmap_atomic(base); +} - return 0; +static int guc_ring_doorbell(struct i915_guc_client *gc) +{ + struct guc_process_desc *desc; + union guc_doorbell_qw db_cmp, db_exc, db_ret; + union guc_doorbell_qw *db; + int attempt = 2, ret = -EAGAIN; + + desc = gc->client_base + gc->proc_desc_offset; + + /* Update the tail so it is visible to GuC */ + desc->tail = gc->wq_tail; + + /* current cookie */ + db_cmp.db_status = GUC_DOORBELL_ENABLED; + db_cmp.cookie = gc->cookie; + + /* cookie to be updated */ + db_exc.db_status = GUC_DOORBELL_ENABLED; + db_exc.cookie = gc->cookie + 1; + if (db_exc.cookie == 0) + db_exc.cookie = 1; + + /* pointer of current doorbell cacheline */ + db = gc->client_base + gc->doorbell_offset; + + while (attempt--) { + /* lets ring the doorbell */ + db_ret.value_qw = atomic64_cmpxchg((atomic64_t *)db, + db_cmp.value_qw, db_exc.value_qw); + + /* if the exchange was successfully executed */ + if (db_ret.value_qw == db_cmp.value_qw) { + /* db was successfully rung */ + gc->cookie = db_exc.cookie; + ret = 0; + break; + } + + /* XXX: doorbell was lost and need to acquire it again */ + if (db_ret.db_status == GUC_DOORBELL_DISABLED) + break; + + DRM_ERROR("Cookie mismatch. Expected %d, returned %d\n", + db_cmp.cookie, db_ret.cookie); + + /* update the cookie to newly read cookie from GuC */ + db_cmp.cookie = db_ret.cookie; + db_exc.cookie = db_ret.cookie + 1; + if (db_exc.cookie == 0) + db_exc.cookie = 1; + } + + return ret; } /** * i915_guc_submit() - Submit commands through GuC - * @client: the guc client where commands will go through * @rq: request associated with the commands * - * Return: 0 if succeed + * Return: 0 on success, otherwise an errno. + * (Note: nonzero really shouldn't happen!) + * + * The caller must have already called i915_guc_wq_check_space() above + * with a result of 0 (success) since the last request submission. This + * guarantees that there is space in the work queue for the new request, + * so enqueuing the item cannot fail. + * + * Bad Things Will Happen if the caller violates this protocol e.g. calls + * submit() when check() says there's no space, or calls submit() multiple + * times with no intervening check(). + * + * The only error here arises if the doorbell hardware isn't functioning + * as expected, which really shouln't happen. */ -int i915_guc_submit(struct i915_guc_client *client, - struct drm_i915_gem_request *rq) +int i915_guc_submit(struct drm_i915_gem_request *rq) { - struct intel_guc *guc = client->guc; - unsigned int engine_id = rq->engine->guc_id; - int q_ret, b_ret; + unsigned int engine_id = rq->engine->id; + struct intel_guc *guc = &rq->i915->guc; + struct i915_guc_client *client = guc->execbuf_client; + int b_ret; - q_ret = guc_add_workqueue_item(client, rq); - if (q_ret == 0) - b_ret = guc_ring_doorbell(client); + guc_add_workqueue_item(client, rq); + b_ret = guc_ring_doorbell(client); client->submissions[engine_id] += 1; - if (q_ret) { - client->q_fail += 1; - client->retcode = q_ret; - } else if (b_ret) { + client->retcode = b_ret; + if (b_ret) client->b_fail += 1; - client->retcode = q_ret = b_ret; - } else { - client->retcode = 0; - } + guc->submissions[engine_id] += 1; guc->last_seqno[engine_id] = rq->seqno; - return q_ret; + return b_ret; } /* @@ -572,7 +614,7 @@ int i915_guc_submit(struct i915_guc_client *client, /** * gem_allocate_guc_obj() - Allocate gem object for GuC usage - * @dev: drm device + * @dev_priv: driver private data structure * @size: size of object * * This is a wrapper to create a gem obj. In order to use it inside GuC, the @@ -581,14 +623,13 @@ int i915_guc_submit(struct i915_guc_client *client, * * Return: A drm_i915_gem_object if successful, otherwise NULL. */ -static struct drm_i915_gem_object *gem_allocate_guc_obj(struct drm_device *dev, - u32 size) +static struct drm_i915_gem_object * +gem_allocate_guc_obj(struct drm_i915_private *dev_priv, u32 size) { - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; - obj = i915_gem_alloc_object(dev, size); - if (!obj) + obj = i915_gem_object_create(&dev_priv->drm, size); + if (IS_ERR(obj)) return NULL; if (i915_gem_object_get_pages(obj)) { @@ -623,10 +664,10 @@ static void gem_release_guc_obj(struct drm_i915_gem_object *obj) drm_gem_object_unreference(&obj->base); } -static void guc_client_free(struct drm_device *dev, - struct i915_guc_client *client) +static void +guc_client_free(struct drm_i915_private *dev_priv, + struct i915_guc_client *client) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_guc *guc = &dev_priv->guc; if (!client) @@ -639,17 +680,10 @@ static void guc_client_free(struct drm_device *dev, if (client->client_base) { /* - * If we got as far as setting up a doorbell, make sure - * we shut it down before unmapping & deallocating the - * memory. So first disable the doorbell, then tell the - * GuC that we've finished with it, finally deallocate - * it in our bitmap + * If we got as far as setting up a doorbell, make sure we + * shut it down before unmapping & deallocating the memory. */ - if (client->doorbell_id != GUC_INVALID_DOORBELL_ID) { - guc_disable_doorbell(guc, client); - host2guc_release_doorbell(guc, client); - release_doorbell(guc, client->doorbell_id); - } + guc_disable_doorbell(guc, client); kunmap(kmap_to_page(client->client_base)); } @@ -664,9 +698,51 @@ static void guc_client_free(struct drm_device *dev, kfree(client); } +/* + * Borrow the first client to set up & tear down every doorbell + * in turn, to ensure that all doorbell h/w is (re)initialised. + */ +static void guc_init_doorbell_hw(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + struct i915_guc_client *client = guc->execbuf_client; + uint16_t db_id, i; + int err; + + db_id = client->doorbell_id; + + for (i = 0; i < GUC_MAX_DOORBELLS; ++i) { + i915_reg_t drbreg = GEN8_DRBREGL(i); + u32 value = I915_READ(drbreg); + + err = guc_update_doorbell_id(guc, client, i); + + /* Report update failure or unexpectedly active doorbell */ + if (err || (i != db_id && (value & GUC_DOORBELL_ENABLED))) + DRM_DEBUG_DRIVER("Doorbell %d (reg 0x%x) was 0x%x, err %d\n", + i, drbreg.reg, value, err); + } + + /* Restore to original value */ + err = guc_update_doorbell_id(guc, client, db_id); + if (err) + DRM_ERROR("Failed to restore doorbell to %d, err %d\n", + db_id, err); + + for (i = 0; i < GUC_MAX_DOORBELLS; ++i) { + i915_reg_t drbreg = GEN8_DRBREGL(i); + u32 value = I915_READ(drbreg); + + if (i != db_id && (value & GUC_DOORBELL_ENABLED)) + DRM_DEBUG_DRIVER("Doorbell %d (reg 0x%x) finally 0x%x\n", + i, drbreg.reg, value); + + } +} + /** * guc_client_alloc() - Allocate an i915_guc_client - * @dev: drm device + * @dev_priv: driver private data structure * @priority: four levels priority _CRITICAL, _HIGH, _NORMAL and _LOW * The kernel client to replace ExecList submission is created with * NORMAL priority. Priority of a client for scheduler can be HIGH, @@ -676,14 +752,15 @@ static void guc_client_free(struct drm_device *dev, * * Return: An i915_guc_client object if success, else NULL. */ -static struct i915_guc_client *guc_client_alloc(struct drm_device *dev, - uint32_t priority, - struct intel_context *ctx) +static struct i915_guc_client * +guc_client_alloc(struct drm_i915_private *dev_priv, + uint32_t priority, + struct i915_gem_context *ctx) { struct i915_guc_client *client; - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_guc *guc = &dev_priv->guc; struct drm_i915_gem_object *obj; + uint16_t db_id; client = kzalloc(sizeof(*client), GFP_KERNEL); if (!client) @@ -702,7 +779,7 @@ static struct i915_guc_client *guc_client_alloc(struct drm_device *dev, } /* The first page is doorbell/proc_desc. Two followed pages are wq. */ - obj = gem_allocate_guc_obj(dev, GUC_DB_SIZE + GUC_WQ_SIZE); + obj = gem_allocate_guc_obj(dev_priv, GUC_DB_SIZE + GUC_WQ_SIZE); if (!obj) goto err; @@ -712,6 +789,11 @@ static struct i915_guc_client *guc_client_alloc(struct drm_device *dev, client->wq_offset = GUC_DB_SIZE; client->wq_size = GUC_WQ_SIZE; + db_id = select_doorbell_register(guc, client->priority); + if (db_id == GUC_INVALID_DOORBELL_ID) + /* XXX: evict a doorbell instead? */ + goto err; + client->doorbell_offset = select_doorbell_cacheline(guc); /* @@ -724,29 +806,22 @@ static struct i915_guc_client *guc_client_alloc(struct drm_device *dev, else client->proc_desc_offset = (GUC_DB_SIZE / 2); - client->doorbell_id = assign_doorbell(guc, client->priority); - if (client->doorbell_id == GUC_INVALID_DOORBELL_ID) - /* XXX: evict a doorbell instead */ - goto err; - guc_init_proc_desc(guc, client); guc_init_ctx_desc(guc, client); - guc_init_doorbell(guc, client); - - /* XXX: Any cache flushes needed? General domain mgmt calls? */ - - if (host2guc_allocate_doorbell(guc, client)) + if (guc_init_doorbell(guc, client, db_id)) goto err; - DRM_DEBUG_DRIVER("new priority %u client %p: ctx_index %u db_id %u\n", - priority, client, client->ctx_index, client->doorbell_id); + DRM_DEBUG_DRIVER("new priority %u client %p: ctx_index %u\n", + priority, client, client->ctx_index); + DRM_DEBUG_DRIVER("doorbell id %u, cacheline offset 0x%x\n", + client->doorbell_id, client->doorbell_offset); return client; err: DRM_ERROR("FAILED to create priority %u GuC client!\n", priority); - guc_client_free(dev, client); + guc_client_free(dev_priv, client); return NULL; } @@ -771,7 +846,7 @@ static void guc_create_log(struct intel_guc *guc) obj = guc->log_obj; if (!obj) { - obj = gem_allocate_guc_obj(dev_priv->dev, size); + obj = gem_allocate_guc_obj(dev_priv, size); if (!obj) { /* logging will be off */ i915.guc_log_level = -1; @@ -831,7 +906,7 @@ static void guc_create_ads(struct intel_guc *guc) obj = guc->ads_obj; if (!obj) { - obj = gem_allocate_guc_obj(dev_priv->dev, PAGE_ALIGN(size)); + obj = gem_allocate_guc_obj(dev_priv, PAGE_ALIGN(size)); if (!obj) return; @@ -885,66 +960,65 @@ static void guc_create_ads(struct intel_guc *guc) * Set up the memory resources to be shared with the GuC. At this point, * we require just one object that can be mapped through the GGTT. */ -int i915_guc_submission_init(struct drm_device *dev) +int i915_guc_submission_init(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; const size_t ctxsize = sizeof(struct guc_context_desc); const size_t poolsize = GUC_MAX_GPU_CONTEXTS * ctxsize; const size_t gemsize = round_up(poolsize, PAGE_SIZE); struct intel_guc *guc = &dev_priv->guc; + /* Wipe bitmap & delete client in case of reinitialisation */ + bitmap_clear(guc->doorbell_bitmap, 0, GUC_MAX_DOORBELLS); + i915_guc_submission_disable(dev_priv); + if (!i915.enable_guc_submission) return 0; /* not enabled */ if (guc->ctx_pool_obj) return 0; /* already allocated */ - guc->ctx_pool_obj = gem_allocate_guc_obj(dev_priv->dev, gemsize); + guc->ctx_pool_obj = gem_allocate_guc_obj(dev_priv, gemsize); if (!guc->ctx_pool_obj) return -ENOMEM; ida_init(&guc->ctx_ids); - guc_create_log(guc); - guc_create_ads(guc); return 0; } -int i915_guc_submission_enable(struct drm_device *dev) +int i915_guc_submission_enable(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_guc *guc = &dev_priv->guc; - struct intel_context *ctx = dev_priv->kernel_context; struct i915_guc_client *client; /* client for execbuf submission */ - client = guc_client_alloc(dev, GUC_CTX_PRIORITY_KMD_NORMAL, ctx); + client = guc_client_alloc(dev_priv, + GUC_CTX_PRIORITY_KMD_NORMAL, + dev_priv->kernel_context); if (!client) { DRM_ERROR("Failed to create execbuf guc_client\n"); return -ENOMEM; } guc->execbuf_client = client; - host2guc_sample_forcewake(guc, client); + guc_init_doorbell_hw(guc); return 0; } -void i915_guc_submission_disable(struct drm_device *dev) +void i915_guc_submission_disable(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_guc *guc = &dev_priv->guc; - guc_client_free(dev, guc->execbuf_client); + guc_client_free(dev_priv, guc->execbuf_client); guc->execbuf_client = NULL; } -void i915_guc_submission_fini(struct drm_device *dev) +void i915_guc_submission_fini(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_guc *guc = &dev_priv->guc; gem_release_guc_obj(dev_priv->guc.ads_obj); @@ -965,12 +1039,12 @@ void i915_guc_submission_fini(struct drm_device *dev) */ int intel_guc_suspend(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_guc *guc = &dev_priv->guc; - struct intel_context *ctx; + struct i915_gem_context *ctx; u32 data[3]; - if (!i915.enable_guc_submission) + if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS) return 0; ctx = dev_priv->kernel_context; @@ -991,12 +1065,12 @@ int intel_guc_suspend(struct drm_device *dev) */ int intel_guc_resume(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_guc *guc = &dev_priv->guc; - struct intel_context *ctx; + struct i915_gem_context *ctx; u32 data[3]; - if (!i915.enable_guc_submission) + if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS) return 0; ctx = dev_priv->kernel_context; diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index aab47f7bb..1c2aec392 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -259,12 +259,12 @@ static void ilk_update_gt_irq(struct drm_i915_private *dev_priv, dev_priv->gt_irq_mask &= ~interrupt_mask; dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask); I915_WRITE(GTIMR, dev_priv->gt_irq_mask); - POSTING_READ(GTIMR); } void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask) { ilk_update_gt_irq(dev_priv, mask, mask); + POSTING_READ_FW(GTIMR); } void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask) @@ -336,9 +336,8 @@ void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) __gen6_disable_pm_irq(dev_priv, mask); } -void gen6_reset_rps_interrupts(struct drm_device *dev) +void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; i915_reg_t reg = gen6_pm_iir(dev_priv); spin_lock_irq(&dev_priv->irq_lock); @@ -349,14 +348,11 @@ void gen6_reset_rps_interrupts(struct drm_device *dev) spin_unlock_irq(&dev_priv->irq_lock); } -void gen6_enable_rps_interrupts(struct drm_device *dev) +void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - spin_lock_irq(&dev_priv->irq_lock); - - WARN_ON(dev_priv->rps.pm_iir); - WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events); + WARN_ON_ONCE(dev_priv->rps.pm_iir); + WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events); dev_priv->rps.interrupts_enabled = true; I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) | dev_priv->pm_rps_events); @@ -367,32 +363,13 @@ void gen6_enable_rps_interrupts(struct drm_device *dev) u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask) { - /* - * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer - * if GEN6_PM_UP_EI_EXPIRED is masked. - * - * TODO: verify if this can be reproduced on VLV,CHV. - */ - if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv)) - mask &= ~GEN6_PM_RP_UP_EI_EXPIRED; - - if (INTEL_INFO(dev_priv)->gen >= 8) - mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP; - - return mask; + return (mask & ~dev_priv->rps.pm_intr_keep); } -void gen6_disable_rps_interrupts(struct drm_device *dev) +void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - spin_lock_irq(&dev_priv->irq_lock); dev_priv->rps.interrupts_enabled = false; - spin_unlock_irq(&dev_priv->irq_lock); - - cancel_work_sync(&dev_priv->rps.work); - - spin_lock_irq(&dev_priv->irq_lock); I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0)); @@ -401,8 +378,15 @@ void gen6_disable_rps_interrupts(struct drm_device *dev) ~dev_priv->pm_rps_events); spin_unlock_irq(&dev_priv->irq_lock); + synchronize_irq(dev_priv->drm.irq); - synchronize_irq(dev->irq); + /* Now that we will not be generating any more work, flush any + * outsanding tasks. As we are called on the RPS idle path, + * we will reset the GPU to minimum frequencies, so the current + * state of the worker can be discarded. + */ + cancel_work_sync(&dev_priv->rps.work); + gen6_reset_rps_interrupts(dev_priv); } /** @@ -582,7 +566,7 @@ i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, u32 enable_mask; if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev, + enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm, status_mask); else enable_mask = status_mask << 16; @@ -596,7 +580,7 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, u32 enable_mask; if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev, + enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm, status_mask); else enable_mask = status_mask << 16; @@ -605,19 +589,17 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe, /** * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion - * @dev: drm device + * @dev_priv: i915 device private */ -static void i915_enable_asle_pipestat(struct drm_device *dev) +static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - if (!dev_priv->opregion.asle || !IS_MOBILE(dev)) + if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv)) return; spin_lock_irq(&dev_priv->irq_lock); i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS); - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) i915_enable_pipestat(dev_priv, PIPE_A, PIPE_LEGACY_BLC_EVENT_STATUS); @@ -685,7 +667,7 @@ static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe) */ static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t high_frame, low_frame; u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; struct intel_crtc *intel_crtc = @@ -732,7 +714,7 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe) static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); return I915_READ(PIPE_FRMCOUNT_G4X(pipe)); } @@ -741,7 +723,7 @@ static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe) static int __intel_get_crtc_scanline(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); const struct drm_display_mode *mode = &crtc->base.hwmode; enum pipe pipe = crtc->pipe; int position, vtotal; @@ -750,7 +732,7 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc) if (mode->flags & DRM_MODE_FLAG_INTERLACE) vtotal /= 2; - if (IS_GEN2(dev)) + if (IS_GEN2(dev_priv)) position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2; else position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3; @@ -767,7 +749,7 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc) * problem. We may need to extend this to include other platforms, * but so far testing only shows the problem on HSW. */ - if (HAS_DDI(dev) && !position) { + if (HAS_DDI(dev_priv) && !position) { int i, temp; for (i = 0; i < 100; i++) { @@ -793,7 +775,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, ktime_t *stime, ktime_t *etime, const struct drm_display_mode *mode) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int position; @@ -835,7 +817,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, if (stime) *stime = ktime_get(); - if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) { + if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) { /* No obvious pixelcount register. Only query vertical * scanout position from Display scan line register. */ @@ -897,7 +879,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, else position += vtotal - vbl_end; - if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) { + if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) { *vpos = position; *hpos = 0; } else { @@ -914,7 +896,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, int intel_get_crtc_scanline(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); unsigned long irqflags; int position; @@ -955,9 +937,8 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, &crtc->hwmode); } -static void ironlake_rps_change_irq_handler(struct drm_device *dev) +static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 busy_up, busy_down, max_avg, min_avg; u8 new_delay; @@ -986,7 +967,7 @@ static void ironlake_rps_change_irq_handler(struct drm_device *dev) new_delay = dev_priv->ips.min_delay; } - if (ironlake_set_drps(dev, new_delay)) + if (ironlake_set_drps(dev_priv, new_delay)) dev_priv->ips.cur_delay = new_delay; spin_unlock(&mchdev_lock); @@ -996,13 +977,11 @@ static void ironlake_rps_change_irq_handler(struct drm_device *dev) static void notify_ring(struct intel_engine_cs *engine) { - if (!intel_engine_initialized(engine)) - return; - - trace_i915_gem_request_notify(engine); - engine->user_interrupts++; - - wake_up_all(&engine->irq_queue); + smp_store_mb(engine->breadcrumbs.irq_posted, true); + if (intel_engine_wakeup(engine)) { + trace_i915_gem_request_notify(engine); + engine->breadcrumbs.irq_wakeups++; + } } static void vlv_c0_read(struct drm_i915_private *dev_priv, @@ -1083,7 +1062,7 @@ static bool any_waiters(struct drm_i915_private *dev_priv) struct intel_engine_cs *engine; for_each_engine(engine, dev_priv) - if (engine->irq_refcount) + if (intel_engine_has_waiter(engine)) return true; return false; @@ -1104,13 +1083,6 @@ static void gen6_pm_rps_work(struct work_struct *work) return; } - /* - * The RPS work is synced during runtime suspend, we don't require a - * wakeref. TODO: instead of disabling the asserts make sure that we - * always hold an RPM reference while the work is running. - */ - DISABLE_RPM_WAKEREF_ASSERTS(dev_priv); - pm_iir = dev_priv->rps.pm_iir; dev_priv->rps.pm_iir = 0; /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */ @@ -1123,7 +1095,7 @@ static void gen6_pm_rps_work(struct work_struct *work) WARN_ON(pm_iir & ~dev_priv->pm_rps_events); if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost) - goto out; + return; mutex_lock(&dev_priv->rps.hw_lock); @@ -1175,11 +1147,9 @@ static void gen6_pm_rps_work(struct work_struct *work) new_delay += adj; new_delay = clamp_t(int, new_delay, min, max); - intel_set_rps(dev_priv->dev, new_delay); + intel_set_rps(dev_priv, new_delay); mutex_unlock(&dev_priv->rps.hw_lock); -out: - ENABLE_RPM_WAKEREF_ASSERTS(dev_priv); } @@ -1205,7 +1175,7 @@ static void ivybridge_parity_work(struct work_struct *work) * In order to prevent a get/put style interface, acquire struct mutex * any time we access those registers. */ - mutex_lock(&dev_priv->dev->struct_mutex); + mutex_lock(&dev_priv->drm.struct_mutex); /* If we've screwed up tracking, just let the interrupt fire again */ if (WARN_ON(!dev_priv->l3_parity.which_slice)) @@ -1241,7 +1211,7 @@ static void ivybridge_parity_work(struct work_struct *work) parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice); parity_event[5] = NULL; - kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj, + kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj, KOBJ_CHANGE, parity_event); DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n", @@ -1261,7 +1231,7 @@ out: gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv)); spin_unlock_irq(&dev_priv->irq_lock); - mutex_unlock(&dev_priv->dev->struct_mutex); + mutex_unlock(&dev_priv->drm.struct_mutex); } static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv, @@ -1287,8 +1257,7 @@ static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir) { - if (gt_iir & - (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT)) + if (gt_iir & GT_RENDER_USER_INTERRUPT) notify_ring(&dev_priv->engine[RCS]); if (gt_iir & ILK_BSD_USER_INTERRUPT) notify_ring(&dev_priv->engine[VCS]); @@ -1297,9 +1266,7 @@ static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv, static void snb_gt_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir) { - - if (gt_iir & - (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT)) + if (gt_iir & GT_RENDER_USER_INTERRUPT) notify_ring(&dev_priv->engine[RCS]); if (gt_iir & GT_BSD_USER_INTERRUPT) notify_ring(&dev_priv->engine[VCS]); @@ -1506,27 +1473,23 @@ static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask, } -static void gmbus_irq_handler(struct drm_device *dev) +static void gmbus_irq_handler(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - wake_up_all(&dev_priv->gmbus_wait_queue); } -static void dp_aux_irq_handler(struct drm_device *dev) +static void dp_aux_irq_handler(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - wake_up_all(&dev_priv->gmbus_wait_queue); } #if defined(CONFIG_DEBUG_FS) -static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe, +static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, + enum pipe pipe, uint32_t crc0, uint32_t crc1, uint32_t crc2, uint32_t crc3, uint32_t crc4) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; struct intel_pipe_crc_entry *entry; int head, tail; @@ -1550,7 +1513,8 @@ static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe, entry = &pipe_crc->entries[head]; - entry->frame = dev->driver->get_vblank_counter(dev, pipe); + entry->frame = dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm, + pipe); entry->crc[0] = crc0; entry->crc[1] = crc1; entry->crc[2] = crc2; @@ -1566,27 +1530,26 @@ static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe, } #else static inline void -display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe, +display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, + enum pipe pipe, uint32_t crc0, uint32_t crc1, uint32_t crc2, uint32_t crc3, uint32_t crc4) {} #endif -static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe) +static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, + enum pipe pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; - - display_pipe_crc_irq_handler(dev, pipe, + display_pipe_crc_irq_handler(dev_priv, pipe, I915_READ(PIPE_CRC_RES_1_IVB(pipe)), 0, 0, 0, 0); } -static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe) +static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, + enum pipe pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; - - display_pipe_crc_irq_handler(dev, pipe, + display_pipe_crc_irq_handler(dev_priv, pipe, I915_READ(PIPE_CRC_RES_1_IVB(pipe)), I915_READ(PIPE_CRC_RES_2_IVB(pipe)), I915_READ(PIPE_CRC_RES_3_IVB(pipe)), @@ -1594,22 +1557,22 @@ static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe) I915_READ(PIPE_CRC_RES_5_IVB(pipe))); } -static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe) +static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, + enum pipe pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; uint32_t res1, res2; - if (INTEL_INFO(dev)->gen >= 3) + if (INTEL_GEN(dev_priv) >= 3) res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe)); else res1 = 0; - if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) + if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe)); else res2 = 0; - display_pipe_crc_irq_handler(dev, pipe, + display_pipe_crc_irq_handler(dev_priv, pipe, I915_READ(PIPE_CRC_RES_RED(pipe)), I915_READ(PIPE_CRC_RES_GREEN(pipe)), I915_READ(PIPE_CRC_RES_BLUE(pipe)), @@ -1626,7 +1589,7 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events); if (dev_priv->rps.interrupts_enabled) { dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events; - queue_work(dev_priv->wq, &dev_priv->rps.work); + schedule_work(&dev_priv->rps.work); } spin_unlock(&dev_priv->irq_lock); } @@ -1643,18 +1606,21 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) } } -static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe) +static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv, + enum pipe pipe) { - if (!drm_handle_vblank(dev, pipe)) - return false; + bool ret; - return true; + ret = drm_handle_vblank(&dev_priv->drm, pipe); + if (ret) + intel_finish_page_flip_mmio(dev_priv, pipe); + + return ret; } -static void valleyview_pipestat_irq_ack(struct drm_device *dev, u32 iir, - u32 pipe_stats[I915_MAX_PIPES]) +static void valleyview_pipestat_irq_ack(struct drm_i915_private *dev_priv, + u32 iir, u32 pipe_stats[I915_MAX_PIPES]) { - struct drm_i915_private *dev_priv = dev->dev_private; int pipe; spin_lock(&dev_priv->irq_lock); @@ -1710,31 +1676,28 @@ static void valleyview_pipestat_irq_ack(struct drm_device *dev, u32 iir, spin_unlock(&dev_priv->irq_lock); } -static void valleyview_pipestat_irq_handler(struct drm_device *dev, +static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv, u32 pipe_stats[I915_MAX_PIPES]) { - struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; for_each_pipe(dev_priv, pipe) { if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS && - intel_pipe_handle_vblank(dev, pipe)) - intel_check_page_flip(dev, pipe); + intel_pipe_handle_vblank(dev_priv, pipe)) + intel_check_page_flip(dev_priv, pipe); - if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) { - intel_prepare_page_flip(dev, pipe); - intel_finish_page_flip(dev, pipe); - } + if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) + intel_finish_page_flip_cs(dev_priv, pipe); if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) - i9xx_pipe_crc_irq_handler(dev, pipe); + i9xx_pipe_crc_irq_handler(dev_priv, pipe); if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); } if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS) - gmbus_irq_handler(dev); + gmbus_irq_handler(dev_priv); } static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv) @@ -1747,12 +1710,13 @@ static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv) return hotplug_status; } -static void i9xx_hpd_irq_handler(struct drm_device *dev, +static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 hotplug_status) { u32 pin_mask = 0, long_mask = 0; - if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) { + if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || + IS_CHERRYVIEW(dev_priv)) { u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X; if (hotplug_trigger) { @@ -1760,11 +1724,11 @@ static void i9xx_hpd_irq_handler(struct drm_device *dev, hotplug_trigger, hpd_status_g4x, i9xx_port_hotplug_long_detect); - intel_hpd_irq_handler(dev, pin_mask, long_mask); + intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); } if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X) - dp_aux_irq_handler(dev); + dp_aux_irq_handler(dev_priv); } else { u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; @@ -1772,7 +1736,7 @@ static void i9xx_hpd_irq_handler(struct drm_device *dev, intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger, hotplug_trigger, hpd_status_i915, i9xx_port_hotplug_long_detect); - intel_hpd_irq_handler(dev, pin_mask, long_mask); + intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); } } } @@ -1780,7 +1744,7 @@ static void i9xx_hpd_irq_handler(struct drm_device *dev, static irqreturn_t valleyview_irq_handler(int irq, void *arg) { struct drm_device *dev = arg; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); irqreturn_t ret = IRQ_NONE; if (!intel_irqs_enabled(dev_priv)) @@ -1831,7 +1795,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg) /* Call regardless, as some status bits might not be * signalled in iir */ - valleyview_pipestat_irq_ack(dev, iir, pipe_stats); + valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats); /* * VLV_IIR is single buffered, and reflects the level @@ -1850,9 +1814,9 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg) gen6_rps_irq_handler(dev_priv, pm_iir); if (hotplug_status) - i9xx_hpd_irq_handler(dev, hotplug_status); + i9xx_hpd_irq_handler(dev_priv, hotplug_status); - valleyview_pipestat_irq_handler(dev, pipe_stats); + valleyview_pipestat_irq_handler(dev_priv, pipe_stats); } while (0); enable_rpm_wakeref_asserts(dev_priv); @@ -1863,7 +1827,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg) static irqreturn_t cherryview_irq_handler(int irq, void *arg) { struct drm_device *dev = arg; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); irqreturn_t ret = IRQ_NONE; if (!intel_irqs_enabled(dev_priv)) @@ -1911,7 +1875,7 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg) /* Call regardless, as some status bits might not be * signalled in iir */ - valleyview_pipestat_irq_ack(dev, iir, pipe_stats); + valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats); /* * VLV_IIR is single buffered, and reflects the level @@ -1927,9 +1891,9 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg) gen8_gt_irq_handler(dev_priv, gt_iir); if (hotplug_status) - i9xx_hpd_irq_handler(dev, hotplug_status); + i9xx_hpd_irq_handler(dev_priv, hotplug_status); - valleyview_pipestat_irq_handler(dev, pipe_stats); + valleyview_pipestat_irq_handler(dev_priv, pipe_stats); } while (0); enable_rpm_wakeref_asserts(dev_priv); @@ -1937,10 +1901,10 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg) return ret; } -static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger, +static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv, + u32 hotplug_trigger, const u32 hpd[HPD_NUM_PINS]) { - struct drm_i915_private *dev_priv = to_i915(dev); u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; /* @@ -1966,16 +1930,15 @@ static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger, dig_hotplug_reg, hpd, pch_port_hotplug_long_detect); - intel_hpd_irq_handler(dev, pin_mask, long_mask); + intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); } -static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir) +static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) { - struct drm_i915_private *dev_priv = dev->dev_private; int pipe; u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK; - ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx); + ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx); if (pch_iir & SDE_AUDIO_POWER_MASK) { int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >> @@ -1985,10 +1948,10 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir) } if (pch_iir & SDE_AUX_MASK) - dp_aux_irq_handler(dev); + dp_aux_irq_handler(dev_priv); if (pch_iir & SDE_GMBUS) - gmbus_irq_handler(dev); + gmbus_irq_handler(dev_priv); if (pch_iir & SDE_AUDIO_HDCP_MASK) DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n"); @@ -2018,9 +1981,8 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir) intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B); } -static void ivb_err_int_handler(struct drm_device *dev) +static void ivb_err_int_handler(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 err_int = I915_READ(GEN7_ERR_INT); enum pipe pipe; @@ -2032,19 +1994,18 @@ static void ivb_err_int_handler(struct drm_device *dev) intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) { - if (IS_IVYBRIDGE(dev)) - ivb_pipe_crc_irq_handler(dev, pipe); + if (IS_IVYBRIDGE(dev_priv)) + ivb_pipe_crc_irq_handler(dev_priv, pipe); else - hsw_pipe_crc_irq_handler(dev, pipe); + hsw_pipe_crc_irq_handler(dev_priv, pipe); } } I915_WRITE(GEN7_ERR_INT, err_int); } -static void cpt_serr_int_handler(struct drm_device *dev) +static void cpt_serr_int_handler(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 serr_int = I915_READ(SERR_INT); if (serr_int & SERR_INT_POISON) @@ -2062,13 +2023,12 @@ static void cpt_serr_int_handler(struct drm_device *dev) I915_WRITE(SERR_INT, serr_int); } -static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir) +static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) { - struct drm_i915_private *dev_priv = dev->dev_private; int pipe; u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT; - ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt); + ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt); if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) { int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >> @@ -2078,10 +2038,10 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir) } if (pch_iir & SDE_AUX_MASK_CPT) - dp_aux_irq_handler(dev); + dp_aux_irq_handler(dev_priv); if (pch_iir & SDE_GMBUS_CPT) - gmbus_irq_handler(dev); + gmbus_irq_handler(dev_priv); if (pch_iir & SDE_AUDIO_CP_REQ_CPT) DRM_DEBUG_DRIVER("Audio CP request interrupt\n"); @@ -2096,12 +2056,11 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir) I915_READ(FDI_RX_IIR(pipe))); if (pch_iir & SDE_ERROR_CPT) - cpt_serr_int_handler(dev); + cpt_serr_int_handler(dev_priv); } -static void spt_irq_handler(struct drm_device *dev, u32 pch_iir) +static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT & ~SDE_PORTE_HOTPLUG_SPT; u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT; @@ -2130,16 +2089,16 @@ static void spt_irq_handler(struct drm_device *dev, u32 pch_iir) } if (pin_mask) - intel_hpd_irq_handler(dev, pin_mask, long_mask); + intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); if (pch_iir & SDE_GMBUS_CPT) - gmbus_irq_handler(dev); + gmbus_irq_handler(dev_priv); } -static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger, +static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv, + u32 hotplug_trigger, const u32 hpd[HPD_NUM_PINS]) { - struct drm_i915_private *dev_priv = to_i915(dev); u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL); @@ -2149,97 +2108,93 @@ static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger, dig_hotplug_reg, hpd, ilk_port_hotplug_long_detect); - intel_hpd_irq_handler(dev, pin_mask, long_mask); + intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); } -static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir) +static void ilk_display_irq_handler(struct drm_i915_private *dev_priv, + u32 de_iir) { - struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe; u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG; if (hotplug_trigger) - ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk); + ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk); if (de_iir & DE_AUX_CHANNEL_A) - dp_aux_irq_handler(dev); + dp_aux_irq_handler(dev_priv); if (de_iir & DE_GSE) - intel_opregion_asle_intr(dev); + intel_opregion_asle_intr(dev_priv); if (de_iir & DE_POISON) DRM_ERROR("Poison interrupt\n"); for_each_pipe(dev_priv, pipe) { if (de_iir & DE_PIPE_VBLANK(pipe) && - intel_pipe_handle_vblank(dev, pipe)) - intel_check_page_flip(dev, pipe); + intel_pipe_handle_vblank(dev_priv, pipe)) + intel_check_page_flip(dev_priv, pipe); if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe)) intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); if (de_iir & DE_PIPE_CRC_DONE(pipe)) - i9xx_pipe_crc_irq_handler(dev, pipe); + i9xx_pipe_crc_irq_handler(dev_priv, pipe); /* plane/pipes map 1:1 on ilk+ */ - if (de_iir & DE_PLANE_FLIP_DONE(pipe)) { - intel_prepare_page_flip(dev, pipe); - intel_finish_page_flip_plane(dev, pipe); - } + if (de_iir & DE_PLANE_FLIP_DONE(pipe)) + intel_finish_page_flip_cs(dev_priv, pipe); } /* check event from PCH */ if (de_iir & DE_PCH_EVENT) { u32 pch_iir = I915_READ(SDEIIR); - if (HAS_PCH_CPT(dev)) - cpt_irq_handler(dev, pch_iir); + if (HAS_PCH_CPT(dev_priv)) + cpt_irq_handler(dev_priv, pch_iir); else - ibx_irq_handler(dev, pch_iir); + ibx_irq_handler(dev_priv, pch_iir); /* should clear PCH hotplug event before clear CPU irq */ I915_WRITE(SDEIIR, pch_iir); } - if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT) - ironlake_rps_change_irq_handler(dev); + if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT) + ironlake_rps_change_irq_handler(dev_priv); } -static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir) +static void ivb_display_irq_handler(struct drm_i915_private *dev_priv, + u32 de_iir) { - struct drm_i915_private *dev_priv = dev->dev_private; enum pipe pipe; u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB; if (hotplug_trigger) - ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb); + ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb); if (de_iir & DE_ERR_INT_IVB) - ivb_err_int_handler(dev); + ivb_err_int_handler(dev_priv); if (de_iir & DE_AUX_CHANNEL_A_IVB) - dp_aux_irq_handler(dev); + dp_aux_irq_handler(dev_priv); if (de_iir & DE_GSE_IVB) - intel_opregion_asle_intr(dev); + intel_opregion_asle_intr(dev_priv); for_each_pipe(dev_priv, pipe) { if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) && - intel_pipe_handle_vblank(dev, pipe)) - intel_check_page_flip(dev, pipe); + intel_pipe_handle_vblank(dev_priv, pipe)) + intel_check_page_flip(dev_priv, pipe); /* plane/pipes map 1:1 on ilk+ */ - if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) { - intel_prepare_page_flip(dev, pipe); - intel_finish_page_flip_plane(dev, pipe); - } + if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) + intel_finish_page_flip_cs(dev_priv, pipe); } /* check event from PCH */ - if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) { + if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) { u32 pch_iir = I915_READ(SDEIIR); - cpt_irq_handler(dev, pch_iir); + cpt_irq_handler(dev_priv, pch_iir); /* clear PCH hotplug event before clear CPU irq */ I915_WRITE(SDEIIR, pch_iir); @@ -2257,7 +2212,7 @@ static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir) static irqreturn_t ironlake_irq_handler(int irq, void *arg) { struct drm_device *dev = arg; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 de_iir, gt_iir, de_ier, sde_ier = 0; irqreturn_t ret = IRQ_NONE; @@ -2277,7 +2232,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg) * able to process them after we restore SDEIER (as soon as we restore * it, we'll get an interrupt if SDEIIR still has something to process * due to its back queue). */ - if (!HAS_PCH_NOP(dev)) { + if (!HAS_PCH_NOP(dev_priv)) { sde_ier = I915_READ(SDEIER); I915_WRITE(SDEIER, 0); POSTING_READ(SDEIER); @@ -2289,7 +2244,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg) if (gt_iir) { I915_WRITE(GTIIR, gt_iir); ret = IRQ_HANDLED; - if (INTEL_INFO(dev)->gen >= 6) + if (INTEL_GEN(dev_priv) >= 6) snb_gt_irq_handler(dev_priv, gt_iir); else ilk_gt_irq_handler(dev_priv, gt_iir); @@ -2299,13 +2254,13 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg) if (de_iir) { I915_WRITE(DEIIR, de_iir); ret = IRQ_HANDLED; - if (INTEL_INFO(dev)->gen >= 7) - ivb_display_irq_handler(dev, de_iir); + if (INTEL_GEN(dev_priv) >= 7) + ivb_display_irq_handler(dev_priv, de_iir); else - ilk_display_irq_handler(dev, de_iir); + ilk_display_irq_handler(dev_priv, de_iir); } - if (INTEL_INFO(dev)->gen >= 6) { + if (INTEL_GEN(dev_priv) >= 6) { u32 pm_iir = I915_READ(GEN6_PMIIR); if (pm_iir) { I915_WRITE(GEN6_PMIIR, pm_iir); @@ -2316,7 +2271,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg) I915_WRITE(DEIER, de_ier); POSTING_READ(DEIER); - if (!HAS_PCH_NOP(dev)) { + if (!HAS_PCH_NOP(dev_priv)) { I915_WRITE(SDEIER, sde_ier); POSTING_READ(SDEIER); } @@ -2327,10 +2282,10 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg) return ret; } -static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger, +static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv, + u32 hotplug_trigger, const u32 hpd[HPD_NUM_PINS]) { - struct drm_i915_private *dev_priv = to_i915(dev); u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); @@ -2340,13 +2295,12 @@ static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger, dig_hotplug_reg, hpd, bxt_port_hotplug_long_detect); - intel_hpd_irq_handler(dev, pin_mask, long_mask); + intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); } static irqreturn_t gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) { - struct drm_device *dev = dev_priv->dev; irqreturn_t ret = IRQ_NONE; u32 iir; enum pipe pipe; @@ -2357,7 +2311,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) I915_WRITE(GEN8_DE_MISC_IIR, iir); ret = IRQ_HANDLED; if (iir & GEN8_DE_MISC_GSE) - intel_opregion_asle_intr(dev); + intel_opregion_asle_intr(dev_priv); else DRM_ERROR("Unexpected DE Misc interrupt\n"); } @@ -2381,26 +2335,28 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) GEN9_AUX_CHANNEL_D; if (iir & tmp_mask) { - dp_aux_irq_handler(dev); + dp_aux_irq_handler(dev_priv); found = true; } if (IS_BROXTON(dev_priv)) { tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK; if (tmp_mask) { - bxt_hpd_irq_handler(dev, tmp_mask, hpd_bxt); + bxt_hpd_irq_handler(dev_priv, tmp_mask, + hpd_bxt); found = true; } } else if (IS_BROADWELL(dev_priv)) { tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG; if (tmp_mask) { - ilk_hpd_irq_handler(dev, tmp_mask, hpd_bdw); + ilk_hpd_irq_handler(dev_priv, + tmp_mask, hpd_bdw); found = true; } } - if (IS_BROXTON(dev) && (iir & BXT_DE_PORT_GMBUS)) { - gmbus_irq_handler(dev); + if (IS_BROXTON(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) { + gmbus_irq_handler(dev_priv); found = true; } @@ -2427,8 +2383,8 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir); if (iir & GEN8_PIPE_VBLANK && - intel_pipe_handle_vblank(dev, pipe)) - intel_check_page_flip(dev, pipe); + intel_pipe_handle_vblank(dev_priv, pipe)) + intel_check_page_flip(dev_priv, pipe); flip_done = iir; if (INTEL_INFO(dev_priv)->gen >= 9) @@ -2436,13 +2392,11 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) else flip_done &= GEN8_PIPE_PRIMARY_FLIP_DONE; - if (flip_done) { - intel_prepare_page_flip(dev, pipe); - intel_finish_page_flip_plane(dev, pipe); - } + if (flip_done) + intel_finish_page_flip_cs(dev_priv, pipe); if (iir & GEN8_PIPE_CDCLK_CRC_DONE) - hsw_pipe_crc_irq_handler(dev, pipe); + hsw_pipe_crc_irq_handler(dev_priv, pipe); if (iir & GEN8_PIPE_FIFO_UNDERRUN) intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); @@ -2459,7 +2413,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) fault_errors); } - if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) && + if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) && master_ctl & GEN8_DE_PCH_IRQ) { /* * FIXME(BDW): Assume for now that the new interrupt handling @@ -2472,9 +2426,9 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) ret = IRQ_HANDLED; if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv)) - spt_irq_handler(dev, iir); + spt_irq_handler(dev_priv, iir); else - cpt_irq_handler(dev, iir); + cpt_irq_handler(dev_priv, iir); } else { /* * Like on previous PCH there seems to be something @@ -2490,7 +2444,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) static irqreturn_t gen8_irq_handler(int irq, void *arg) { struct drm_device *dev = arg; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 master_ctl; u32 gt_iir[4] = {}; irqreturn_t ret; @@ -2521,11 +2475,8 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) return ret; } -static void i915_error_wake_up(struct drm_i915_private *dev_priv, - bool reset_completed) +static void i915_error_wake_up(struct drm_i915_private *dev_priv) { - struct intel_engine_cs *engine; - /* * Notify all waiters for GPU completion events that reset state has * been changed, and that they need to restart their wait after @@ -2534,36 +2485,28 @@ static void i915_error_wake_up(struct drm_i915_private *dev_priv, */ /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */ - for_each_engine(engine, dev_priv) - wake_up_all(&engine->irq_queue); + wake_up_all(&dev_priv->gpu_error.wait_queue); /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */ wake_up_all(&dev_priv->pending_flip_queue); - - /* - * Signal tasks blocked in i915_gem_wait_for_error that the pending - * reset state is cleared. - */ - if (reset_completed) - wake_up_all(&dev_priv->gpu_error.reset_queue); } /** * i915_reset_and_wakeup - do process context error handling work - * @dev: drm device + * @dev_priv: i915 device private * * Fire an error uevent so userspace can see that a hang or error * was detected. */ -static void i915_reset_and_wakeup(struct drm_device *dev) +static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); + struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj; char *error_event[] = { I915_ERROR_UEVENT "=1", NULL }; char *reset_event[] = { I915_RESET_UEVENT "=1", NULL }; char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL }; int ret; - kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event); + kobject_uevent_env(kobj, KOBJ_CHANGE, error_event); /* * Note that there's only one work item which does gpu resets, so we @@ -2577,8 +2520,7 @@ static void i915_reset_and_wakeup(struct drm_device *dev) */ if (i915_reset_in_progress(&dev_priv->gpu_error)) { DRM_DEBUG_DRIVER("resetting chip\n"); - kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, - reset_event); + kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event); /* * In most cases it's guaranteed that we get here with an RPM @@ -2589,7 +2531,7 @@ static void i915_reset_and_wakeup(struct drm_device *dev) */ intel_runtime_pm_get(dev_priv); - intel_prepare_reset(dev); + intel_prepare_reset(dev_priv); /* * All state reset _must_ be completed before we update the @@ -2597,27 +2539,26 @@ static void i915_reset_and_wakeup(struct drm_device *dev) * pending state and not properly drop locks, resulting in * deadlocks with the reset work. */ - ret = i915_reset(dev); + ret = i915_reset(dev_priv); - intel_finish_reset(dev); + intel_finish_reset(dev_priv); intel_runtime_pm_put(dev_priv); if (ret == 0) - kobject_uevent_env(&dev->primary->kdev->kobj, + kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event); /* * Note: The wake_up also serves as a memory barrier so that * waiters see the update value of the reset counter atomic_t. */ - i915_error_wake_up(dev_priv, true); + wake_up_all(&dev_priv->gpu_error.reset_queue); } } -static void i915_report_and_clear_eir(struct drm_device *dev) +static void i915_report_and_clear_eir(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; uint32_t instdone[I915_NUM_INSTDONE_REG]; u32 eir = I915_READ(EIR); int pipe, i; @@ -2627,9 +2568,9 @@ static void i915_report_and_clear_eir(struct drm_device *dev) pr_err("render error detected, EIR: 0x%08x\n", eir); - i915_get_extra_instdone(dev, instdone); + i915_get_extra_instdone(dev_priv, instdone); - if (IS_G4X(dev)) { + if (IS_G4X(dev_priv)) { if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) { u32 ipeir = I915_READ(IPEIR_I965); @@ -2651,7 +2592,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev) } } - if (!IS_GEN2(dev)) { + if (!IS_GEN2(dev_priv)) { if (eir & I915_ERROR_PAGE_TABLE) { u32 pgtbl_err = I915_READ(PGTBL_ER); pr_err("page table error\n"); @@ -2673,7 +2614,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev) pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM)); for (i = 0; i < ARRAY_SIZE(instdone); i++) pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]); - if (INTEL_INFO(dev)->gen < 4) { + if (INTEL_GEN(dev_priv) < 4) { u32 ipeir = I915_READ(IPEIR); pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR)); @@ -2709,18 +2650,19 @@ static void i915_report_and_clear_eir(struct drm_device *dev) /** * i915_handle_error - handle a gpu error - * @dev: drm device + * @dev_priv: i915 device private * @engine_mask: mask representing engines that are hung * Do some basic checking of register state at error time and * dump it to the syslog. Also call i915_capture_error_state() to make * sure we get a record and make it available in debugfs. Fire a uevent * so userspace knows something bad happened (should trigger collection * of a ring dump etc.). + * @fmt: Error message format string */ -void i915_handle_error(struct drm_device *dev, u32 engine_mask, +void i915_handle_error(struct drm_i915_private *dev_priv, + u32 engine_mask, const char *fmt, ...) { - struct drm_i915_private *dev_priv = dev->dev_private; va_list args; char error_msg[80]; @@ -2728,8 +2670,8 @@ void i915_handle_error(struct drm_device *dev, u32 engine_mask, vscnprintf(error_msg, sizeof(error_msg), fmt, args); va_end(args); - i915_capture_error_state(dev, engine_mask, error_msg); - i915_report_and_clear_eir(dev); + i915_capture_error_state(dev_priv, engine_mask, error_msg); + i915_report_and_clear_eir(dev_priv); if (engine_mask) { atomic_or(I915_RESET_IN_PROGRESS_FLAG, @@ -2748,10 +2690,10 @@ void i915_handle_error(struct drm_device *dev, u32 engine_mask, * ensure that the waiters see the updated value of the reset * counter atomic_t. */ - i915_error_wake_up(dev_priv, false); + i915_error_wake_up(dev_priv); } - i915_reset_and_wakeup(dev); + i915_reset_and_wakeup(dev_priv); } /* Called from drm generic code, passed 'crtc' which @@ -2759,7 +2701,7 @@ void i915_handle_error(struct drm_device *dev, u32 engine_mask, */ static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned long irqflags; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); @@ -2776,7 +2718,7 @@ static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe) static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned long irqflags; uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe); @@ -2790,7 +2732,7 @@ static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe) static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned long irqflags; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); @@ -2803,7 +2745,7 @@ static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe) static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned long irqflags; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); @@ -2818,7 +2760,7 @@ static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe) */ static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned long irqflags; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); @@ -2830,7 +2772,7 @@ static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe) static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned long irqflags; uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe); @@ -2842,7 +2784,7 @@ static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe) static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned long irqflags; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); @@ -2853,7 +2795,7 @@ static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe) static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned long irqflags; spin_lock_irqsave(&dev_priv->irq_lock, irqflags); @@ -2869,9 +2811,9 @@ ring_idle(struct intel_engine_cs *engine, u32 seqno) } static bool -ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr) +ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr) { - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(engine->i915) >= 8) { return (ipehr >> 23) == 0x1c; } else { ipehr &= ~MI_SEMAPHORE_SYNC_MASK; @@ -2884,10 +2826,10 @@ static struct intel_engine_cs * semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr, u64 offset) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; struct intel_engine_cs *signaller; - if (INTEL_INFO(dev_priv)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) { for_each_engine(signaller, dev_priv) { if (engine == signaller) continue; @@ -2916,7 +2858,7 @@ semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr, static struct intel_engine_cs * semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; u32 cmd, ipehr, head; u64 offset = 0; int i, backwards; @@ -2942,7 +2884,7 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) return NULL; ipehr = I915_READ(RING_IPEHR(engine->mmio_base)); - if (!ipehr_is_semaphore_wait(engine->dev, ipehr)) + if (!ipehr_is_semaphore_wait(engine, ipehr)) return NULL; /* @@ -2954,7 +2896,7 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) * ringbuffer itself. */ head = I915_READ_HEAD(engine) & HEAD_ADDR; - backwards = (INTEL_INFO(engine->dev)->gen >= 8) ? 5 : 4; + backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4; for (i = backwards; i; --i) { /* @@ -2976,7 +2918,7 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) return NULL; *seqno = ioread32(engine->buffer->virtual_start + head + 4) + 1; - if (INTEL_INFO(engine->dev)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) { offset = ioread32(engine->buffer->virtual_start + head + 12); offset <<= 32; offset = ioread32(engine->buffer->virtual_start + head + 8); @@ -2986,7 +2928,7 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) static int semaphore_passed(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; struct intel_engine_cs *signaller; u32 seqno; @@ -3000,7 +2942,7 @@ static int semaphore_passed(struct intel_engine_cs *engine) if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES) return -1; - if (i915_seqno_passed(signaller->get_seqno(signaller), seqno)) + if (i915_seqno_passed(intel_engine_get_seqno(signaller), seqno)) return 1; /* cursory check for an unkickable deadlock */ @@ -3028,7 +2970,7 @@ static bool subunits_stuck(struct intel_engine_cs *engine) if (engine->id != RCS) return true; - i915_get_extra_instdone(engine->dev, instdone); + i915_get_extra_instdone(engine->i915, instdone); /* There might be unstable subunit states even when * actual head is not moving. Filter out the unstable ones by @@ -3069,8 +3011,7 @@ head_stuck(struct intel_engine_cs *engine, u64 acthd) static enum intel_ring_hangcheck_action ring_stuck(struct intel_engine_cs *engine, u64 acthd) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; enum intel_ring_hangcheck_action ha; u32 tmp; @@ -3078,7 +3019,7 @@ ring_stuck(struct intel_engine_cs *engine, u64 acthd) if (ha != HANGCHECK_HUNG) return ha; - if (IS_GEN2(dev)) + if (IS_GEN2(dev_priv)) return HANGCHECK_HUNG; /* Is the chip hanging on a WAIT_FOR_EVENT? @@ -3088,19 +3029,19 @@ ring_stuck(struct intel_engine_cs *engine, u64 acthd) */ tmp = I915_READ_CTL(engine); if (tmp & RING_WAIT) { - i915_handle_error(dev, 0, + i915_handle_error(dev_priv, 0, "Kicking stuck wait on %s", engine->name); I915_WRITE_CTL(engine, tmp); return HANGCHECK_KICK; } - if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) { + if (INTEL_GEN(dev_priv) >= 6 && tmp & RING_WAIT_SEMAPHORE) { switch (semaphore_passed(engine)) { default: return HANGCHECK_HUNG; case 1: - i915_handle_error(dev, 0, + i915_handle_error(dev_priv, 0, "Kicking stuck semaphore on %s", engine->name); I915_WRITE_CTL(engine, tmp); @@ -3113,23 +3054,21 @@ ring_stuck(struct intel_engine_cs *engine, u64 acthd) return HANGCHECK_HUNG; } -static unsigned kick_waiters(struct intel_engine_cs *engine) +static unsigned long kick_waiters(struct intel_engine_cs *engine) { - struct drm_i915_private *i915 = to_i915(engine->dev); - unsigned user_interrupts = READ_ONCE(engine->user_interrupts); + struct drm_i915_private *i915 = engine->i915; + unsigned long irq_count = READ_ONCE(engine->breadcrumbs.irq_wakeups); - if (engine->hangcheck.user_interrupts == user_interrupts && + if (engine->hangcheck.user_interrupts == irq_count && !test_and_set_bit(engine->id, &i915->gpu_error.missed_irq_rings)) { - if (!(i915->gpu_error.test_irq_rings & intel_engine_flag(engine))) + if (!test_bit(engine->id, &i915->gpu_error.test_irq_rings)) DRM_ERROR("Hangcheck timer elapsed... %s idle\n", engine->name); - else - DRM_INFO("Fake missed irq on %s\n", - engine->name); - wake_up_all(&engine->irq_queue); + + intel_engine_enable_fake_irq(engine); } - return user_interrupts; + return irq_count; } /* * This is called when the chip hasn't reported back with completed @@ -3144,11 +3083,9 @@ static void i915_hangcheck_elapsed(struct work_struct *work) struct drm_i915_private *dev_priv = container_of(work, typeof(*dev_priv), gpu_error.hangcheck_work.work); - struct drm_device *dev = dev_priv->dev; struct intel_engine_cs *engine; - enum intel_engine_id id; - int busy_count = 0, rings_hung = 0; - bool stuck[I915_NUM_ENGINES] = { 0 }; + unsigned int hung = 0, stuck = 0; + int busy_count = 0; #define BUSY 1 #define KICK 5 #define HUNG 20 @@ -3157,12 +3094,8 @@ static void i915_hangcheck_elapsed(struct work_struct *work) if (!i915.enable_hangcheck) return; - /* - * The hangcheck work is synced during runtime suspend, we don't - * require a wakeref. TODO: instead of disabling the asserts make - * sure that we hold a reference when this work is running. - */ - DISABLE_RPM_WAKEREF_ASSERTS(dev_priv); + if (!READ_ONCE(dev_priv->gt.awake)) + return; /* As enabling the GPU requires fairly extensive mmio access, * periodically arm the mmio checker to see if we are triggering @@ -3170,11 +3103,11 @@ static void i915_hangcheck_elapsed(struct work_struct *work) */ intel_uncore_arm_unclaimed_mmio_detection(dev_priv); - for_each_engine_id(engine, dev_priv, id) { + for_each_engine(engine, dev_priv) { + bool busy = intel_engine_has_waiter(engine); u64 acthd; u32 seqno; unsigned user_interrupts; - bool busy = true; semaphore_clear_deadlocks(dev_priv); @@ -3189,7 +3122,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work) engine->irq_seqno_barrier(engine); acthd = intel_ring_get_active_head(engine); - seqno = engine->get_seqno(engine); + seqno = intel_engine_get_seqno(engine); /* Reset stuck interrupts between batch advances */ user_interrupts = 0; @@ -3197,12 +3130,11 @@ static void i915_hangcheck_elapsed(struct work_struct *work) if (engine->hangcheck.seqno == seqno) { if (ring_idle(engine, seqno)) { engine->hangcheck.action = HANGCHECK_IDLE; - if (waitqueue_active(&engine->irq_queue)) { + if (busy) { /* Safeguard against driver failure */ user_interrupts = kick_waiters(engine); engine->hangcheck.score += BUSY; - } else - busy = false; + } } else { /* We always increment the hangcheck score * if the ring is busy and still processing @@ -3234,10 +3166,15 @@ static void i915_hangcheck_elapsed(struct work_struct *work) break; case HANGCHECK_HUNG: engine->hangcheck.score += HUNG; - stuck[id] = true; break; } } + + if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { + hung |= intel_engine_flag(engine); + if (engine->hangcheck.action != HANGCHECK_HUNG) + stuck |= intel_engine_flag(engine); + } } else { engine->hangcheck.action = HANGCHECK_ACTIVE; @@ -3262,48 +3199,33 @@ static void i915_hangcheck_elapsed(struct work_struct *work) busy_count += busy; } - for_each_engine_id(engine, dev_priv, id) { - if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { - DRM_INFO("%s on %s\n", - stuck[id] ? "stuck" : "no progress", - engine->name); - rings_hung |= intel_engine_flag(engine); - } - } + if (hung) { + char msg[80]; + int len; - if (rings_hung) { - i915_handle_error(dev, rings_hung, "Engine(s) hung"); - goto out; + /* If some rings hung but others were still busy, only + * blame the hanging rings in the synopsis. + */ + if (stuck != hung) + hung &= ~stuck; + len = scnprintf(msg, sizeof(msg), + "%s on ", stuck == hung ? "No progress" : "Hang"); + for_each_engine_masked(engine, dev_priv, hung) + len += scnprintf(msg + len, sizeof(msg) - len, + "%s, ", engine->name); + msg[len-2] = '\0'; + + return i915_handle_error(dev_priv, hung, msg); } + /* Reset timer in case GPU hangs without another request being added */ if (busy_count) - /* Reset timer case chip hangs without another request - * being added */ - i915_queue_hangcheck(dev); - -out: - ENABLE_RPM_WAKEREF_ASSERTS(dev_priv); -} - -void i915_queue_hangcheck(struct drm_device *dev) -{ - struct i915_gpu_error *e = &to_i915(dev)->gpu_error; - - if (!i915.enable_hangcheck) - return; - - /* Don't continually defer the hangcheck so that it is always run at - * least once after work has been scheduled on any ring. Otherwise, - * we will ignore a hung ring if a second ring is kept busy. - */ - - queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work, - round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES)); + i915_queue_hangcheck(dev_priv); } static void ibx_irq_reset(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (HAS_PCH_NOP(dev)) return; @@ -3324,7 +3246,7 @@ static void ibx_irq_reset(struct drm_device *dev) */ static void ibx_irq_pre_postinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (HAS_PCH_NOP(dev)) return; @@ -3336,7 +3258,7 @@ static void ibx_irq_pre_postinstall(struct drm_device *dev) static void gen5_gt_irq_reset(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); GEN5_IRQ_RESET(GT); if (INTEL_INFO(dev)->gen >= 6) @@ -3396,7 +3318,7 @@ static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv) */ static void ironlake_irq_reset(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(HWSTAM, 0xffffffff); @@ -3411,7 +3333,7 @@ static void ironlake_irq_reset(struct drm_device *dev) static void valleyview_irq_preinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(VLV_MASTER_IER, 0); POSTING_READ(VLV_MASTER_IER); @@ -3434,7 +3356,7 @@ static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv) static void gen8_irq_reset(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe; I915_WRITE(GEN8_MASTER_IRQ, 0); @@ -3480,12 +3402,12 @@ void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv, spin_unlock_irq(&dev_priv->irq_lock); /* make sure we're done processing display irqs */ - synchronize_irq(dev_priv->dev->irq); + synchronize_irq(dev_priv->drm.irq); } static void cherryview_irq_preinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(GEN8_MASTER_IRQ, 0); POSTING_READ(GEN8_MASTER_IRQ); @@ -3500,31 +3422,29 @@ static void cherryview_irq_preinstall(struct drm_device *dev) spin_unlock_irq(&dev_priv->irq_lock); } -static u32 intel_hpd_enabled_irqs(struct drm_device *dev, +static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv, const u32 hpd[HPD_NUM_PINS]) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *encoder; u32 enabled_irqs = 0; - for_each_intel_encoder(dev, encoder) + for_each_intel_encoder(&dev_priv->drm, encoder) if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED) enabled_irqs |= hpd[encoder->hpd_pin]; return enabled_irqs; } -static void ibx_hpd_irq_setup(struct drm_device *dev) +static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 hotplug_irqs, hotplug, enabled_irqs; - if (HAS_PCH_IBX(dev)) { + if (HAS_PCH_IBX(dev_priv)) { hotplug_irqs = SDE_HOTPLUG_MASK; - enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx); } else { hotplug_irqs = SDE_HOTPLUG_MASK_CPT; - enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt); } ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); @@ -3543,18 +3463,17 @@ static void ibx_hpd_irq_setup(struct drm_device *dev) * When CPU and PCH are on the same package, port A * HPD must be enabled in both north and south. */ - if (HAS_PCH_LPT_LP(dev)) + if (HAS_PCH_LPT_LP(dev_priv)) hotplug |= PORTA_HOTPLUG_ENABLE; I915_WRITE(PCH_PORT_HOTPLUG, hotplug); } -static void spt_hpd_irq_setup(struct drm_device *dev) +static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 hotplug_irqs, hotplug, enabled_irqs; hotplug_irqs = SDE_HOTPLUG_MASK_SPT; - enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt); ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); @@ -3569,24 +3488,23 @@ static void spt_hpd_irq_setup(struct drm_device *dev) I915_WRITE(PCH_PORT_HOTPLUG2, hotplug); } -static void ilk_hpd_irq_setup(struct drm_device *dev) +static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 hotplug_irqs, hotplug, enabled_irqs; - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) { hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG; - enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw); bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); - } else if (INTEL_INFO(dev)->gen >= 7) { + } else if (INTEL_GEN(dev_priv) >= 7) { hotplug_irqs = DE_DP_A_HOTPLUG_IVB; - enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb); ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs); } else { hotplug_irqs = DE_DP_A_HOTPLUG; - enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk); ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs); } @@ -3601,15 +3519,14 @@ static void ilk_hpd_irq_setup(struct drm_device *dev) hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms; I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug); - ibx_hpd_irq_setup(dev); + ibx_hpd_irq_setup(dev_priv); } -static void bxt_hpd_irq_setup(struct drm_device *dev) +static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 hotplug_irqs, hotplug, enabled_irqs; - enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt); hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK; bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); @@ -3642,7 +3559,7 @@ static void bxt_hpd_irq_setup(struct drm_device *dev) static void ibx_irq_postinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 mask; if (HAS_PCH_NOP(dev)) @@ -3659,7 +3576,7 @@ static void ibx_irq_postinstall(struct drm_device *dev) static void gen5_gt_irq_postinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 pm_irqs, gt_irqs; pm_irqs = gt_irqs = 0; @@ -3673,8 +3590,7 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev) gt_irqs |= GT_RENDER_USER_INTERRUPT; if (IS_GEN5(dev)) { - gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT | - ILK_BSD_USER_INTERRUPT; + gt_irqs |= ILK_BSD_USER_INTERRUPT; } else { gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT; } @@ -3696,7 +3612,7 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev) static int ironlake_irq_postinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 display_mask, extra_mask; if (INTEL_INFO(dev)->gen >= 7) { @@ -3775,7 +3691,7 @@ void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv) static int valleyview_irq_postinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); gen5_gt_irq_postinstall(dev); @@ -3827,6 +3743,7 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) uint32_t de_pipe_enables; u32 de_port_masked = GEN8_AUX_CHANNEL_A; u32 de_port_enables; + u32 de_misc_masked = GEN8_DE_MISC_GSE; enum pipe pipe; if (INTEL_INFO(dev_priv)->gen >= 9) { @@ -3862,11 +3779,12 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) de_pipe_enables); GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables); + GEN5_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked); } static int gen8_irq_postinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (HAS_PCH_SPLIT(dev)) ibx_irq_pre_postinstall(dev); @@ -3885,7 +3803,7 @@ static int gen8_irq_postinstall(struct drm_device *dev) static int cherryview_irq_postinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); gen8_gt_irq_postinstall(dev_priv); @@ -3902,7 +3820,7 @@ static int cherryview_irq_postinstall(struct drm_device *dev) static void gen8_irq_uninstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!dev_priv) return; @@ -3912,7 +3830,7 @@ static void gen8_irq_uninstall(struct drm_device *dev) static void valleyview_irq_uninstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!dev_priv) return; @@ -3932,7 +3850,7 @@ static void valleyview_irq_uninstall(struct drm_device *dev) static void cherryview_irq_uninstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!dev_priv) return; @@ -3952,7 +3870,7 @@ static void cherryview_irq_uninstall(struct drm_device *dev) static void ironlake_irq_uninstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!dev_priv) return; @@ -3962,7 +3880,7 @@ static void ironlake_irq_uninstall(struct drm_device *dev) static void i8xx_irq_preinstall(struct drm_device * dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe; for_each_pipe(dev_priv, pipe) @@ -3974,7 +3892,7 @@ static void i8xx_irq_preinstall(struct drm_device * dev) static int i8xx_irq_postinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE16(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); @@ -4006,13 +3924,12 @@ static int i8xx_irq_postinstall(struct drm_device *dev) /* * Returns true when a page flip has completed. */ -static bool i8xx_handle_vblank(struct drm_device *dev, +static bool i8xx_handle_vblank(struct drm_i915_private *dev_priv, int plane, int pipe, u32 iir) { - struct drm_i915_private *dev_priv = dev->dev_private; u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane); - if (!intel_pipe_handle_vblank(dev, pipe)) + if (!intel_pipe_handle_vblank(dev_priv, pipe)) return false; if ((iir & flip_pending) == 0) @@ -4027,19 +3944,18 @@ static bool i8xx_handle_vblank(struct drm_device *dev, if (I915_READ16(ISR) & flip_pending) goto check_page_flip; - intel_prepare_page_flip(dev, plane); - intel_finish_page_flip(dev, pipe); + intel_finish_page_flip_cs(dev_priv, pipe); return true; check_page_flip: - intel_check_page_flip(dev, pipe); + intel_check_page_flip(dev_priv, pipe); return false; } static irqreturn_t i8xx_irq_handler(int irq, void *arg) { struct drm_device *dev = arg; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u16 iir, new_iir; u32 pipe_stats[2]; int pipe; @@ -4089,15 +4005,15 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) for_each_pipe(dev_priv, pipe) { int plane = pipe; - if (HAS_FBC(dev)) + if (HAS_FBC(dev_priv)) plane = !plane; if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS && - i8xx_handle_vblank(dev, plane, pipe, iir)) + i8xx_handle_vblank(dev_priv, plane, pipe, iir)) flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane); if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) - i9xx_pipe_crc_irq_handler(dev, pipe); + i9xx_pipe_crc_irq_handler(dev_priv, pipe); if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) intel_cpu_fifo_underrun_irq_handler(dev_priv, @@ -4116,7 +4032,7 @@ out: static void i8xx_irq_uninstall(struct drm_device * dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe; for_each_pipe(dev_priv, pipe) { @@ -4131,7 +4047,7 @@ static void i8xx_irq_uninstall(struct drm_device * dev) static void i915_irq_preinstall(struct drm_device * dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe; if (I915_HAS_HOTPLUG(dev)) { @@ -4149,7 +4065,7 @@ static void i915_irq_preinstall(struct drm_device * dev) static int i915_irq_postinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 enable_mask; I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH)); @@ -4182,7 +4098,7 @@ static int i915_irq_postinstall(struct drm_device *dev) I915_WRITE(IER, enable_mask); POSTING_READ(IER); - i915_enable_asle_pipestat(dev); + i915_enable_asle_pipestat(dev_priv); /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked check happy. */ @@ -4197,13 +4113,12 @@ static int i915_irq_postinstall(struct drm_device *dev) /* * Returns true when a page flip has completed. */ -static bool i915_handle_vblank(struct drm_device *dev, +static bool i915_handle_vblank(struct drm_i915_private *dev_priv, int plane, int pipe, u32 iir) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane); - if (!intel_pipe_handle_vblank(dev, pipe)) + if (!intel_pipe_handle_vblank(dev_priv, pipe)) return false; if ((iir & flip_pending) == 0) @@ -4218,19 +4133,18 @@ static bool i915_handle_vblank(struct drm_device *dev, if (I915_READ(ISR) & flip_pending) goto check_page_flip; - intel_prepare_page_flip(dev, plane); - intel_finish_page_flip(dev, pipe); + intel_finish_page_flip_cs(dev_priv, pipe); return true; check_page_flip: - intel_check_page_flip(dev, pipe); + intel_check_page_flip(dev_priv, pipe); return false; } static irqreturn_t i915_irq_handler(int irq, void *arg) { struct drm_device *dev = arg; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 iir, new_iir, pipe_stats[I915_MAX_PIPES]; u32 flip_mask = I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT | @@ -4273,11 +4187,11 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) break; /* Consume port. Then clear IIR or we'll miss events */ - if (I915_HAS_HOTPLUG(dev) && + if (I915_HAS_HOTPLUG(dev_priv) && iir & I915_DISPLAY_PORT_INTERRUPT) { u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv); if (hotplug_status) - i9xx_hpd_irq_handler(dev, hotplug_status); + i9xx_hpd_irq_handler(dev_priv, hotplug_status); } I915_WRITE(IIR, iir & ~flip_mask); @@ -4288,18 +4202,18 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) for_each_pipe(dev_priv, pipe) { int plane = pipe; - if (HAS_FBC(dev)) + if (HAS_FBC(dev_priv)) plane = !plane; if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS && - i915_handle_vblank(dev, plane, pipe, iir)) + i915_handle_vblank(dev_priv, plane, pipe, iir)) flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane); if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) blc_event = true; if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) - i9xx_pipe_crc_irq_handler(dev, pipe); + i9xx_pipe_crc_irq_handler(dev_priv, pipe); if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) intel_cpu_fifo_underrun_irq_handler(dev_priv, @@ -4307,7 +4221,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) } if (blc_event || (iir & I915_ASLE_INTERRUPT)) - intel_opregion_asle_intr(dev); + intel_opregion_asle_intr(dev_priv); /* With MSI, interrupts are only generated when iir * transitions from zero to nonzero. If another bit got @@ -4335,7 +4249,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) static void i915_irq_uninstall(struct drm_device * dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe; if (I915_HAS_HOTPLUG(dev)) { @@ -4357,7 +4271,7 @@ static void i915_irq_uninstall(struct drm_device * dev) static void i965_irq_preinstall(struct drm_device * dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe; i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); @@ -4373,7 +4287,7 @@ static void i965_irq_preinstall(struct drm_device * dev) static int i965_irq_postinstall(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 enable_mask; u32 error_mask; @@ -4391,7 +4305,7 @@ static int i965_irq_postinstall(struct drm_device *dev) I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT); enable_mask |= I915_USER_INTERRUPT; - if (IS_G4X(dev)) + if (IS_G4X(dev_priv)) enable_mask |= I915_BSD_USER_INTERRUPT; /* Interrupt setup is already guaranteed to be single-threaded, this is @@ -4406,7 +4320,7 @@ static int i965_irq_postinstall(struct drm_device *dev) * Enable some error detection, note the instruction error mask * bit is reserved, so we leave it masked. */ - if (IS_G4X(dev)) { + if (IS_G4X(dev_priv)) { error_mask = ~(GM45_ERROR_PAGE_TABLE | GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV | @@ -4424,26 +4338,25 @@ static int i965_irq_postinstall(struct drm_device *dev) i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0); POSTING_READ(PORT_HOTPLUG_EN); - i915_enable_asle_pipestat(dev); + i915_enable_asle_pipestat(dev_priv); return 0; } -static void i915_hpd_irq_setup(struct drm_device *dev) +static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 hotplug_en; assert_spin_locked(&dev_priv->irq_lock); /* Note HDMI and DP share hotplug bits */ /* enable bits are the same for all generations */ - hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915); + hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915); /* Programming the CRT detection parameters tends to generate a spurious hotplug event about three seconds later. So just do it once. */ - if (IS_G4X(dev)) + if (IS_G4X(dev_priv)) hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64; hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50; @@ -4458,7 +4371,7 @@ static void i915_hpd_irq_setup(struct drm_device *dev) static irqreturn_t i965_irq_handler(int irq, void *arg) { struct drm_device *dev = arg; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 iir, new_iir; u32 pipe_stats[I915_MAX_PIPES]; int ret = IRQ_NONE, pipe; @@ -4510,7 +4423,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) if (iir & I915_DISPLAY_PORT_INTERRUPT) { u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv); if (hotplug_status) - i9xx_hpd_irq_handler(dev, hotplug_status); + i9xx_hpd_irq_handler(dev_priv, hotplug_status); } I915_WRITE(IIR, iir & ~flip_mask); @@ -4523,24 +4436,24 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) for_each_pipe(dev_priv, pipe) { if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS && - i915_handle_vblank(dev, pipe, pipe, iir)) + i915_handle_vblank(dev_priv, pipe, pipe, iir)) flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe); if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) blc_event = true; if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) - i9xx_pipe_crc_irq_handler(dev, pipe); + i9xx_pipe_crc_irq_handler(dev_priv, pipe); if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS) intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); } if (blc_event || (iir & I915_ASLE_INTERRUPT)) - intel_opregion_asle_intr(dev); + intel_opregion_asle_intr(dev_priv); if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS) - gmbus_irq_handler(dev); + gmbus_irq_handler(dev_priv); /* With MSI, interrupts are only generated when iir * transitions from zero to nonzero. If another bit got @@ -4567,7 +4480,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) static void i965_irq_uninstall(struct drm_device * dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe; if (!dev_priv) @@ -4597,7 +4510,7 @@ static void i965_irq_uninstall(struct drm_device * dev) */ void intel_irq_init(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; intel_hpd_init_work(dev_priv); @@ -4611,6 +4524,20 @@ void intel_irq_init(struct drm_i915_private *dev_priv) else dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS; + dev_priv->rps.pm_intr_keep = 0; + + /* + * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer + * if GEN6_PM_UP_EI_EXPIRED is masked. + * + * TODO: verify if this can be reproduced on VLV,CHV. + */ + if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv)) + dev_priv->rps.pm_intr_keep |= GEN6_PM_RP_UP_EI_EXPIRED; + + if (INTEL_INFO(dev_priv)->gen >= 8) + dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_NON_DISP; + INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work, i915_hangcheck_elapsed); @@ -4674,12 +4601,12 @@ void intel_irq_init(struct drm_i915_private *dev_priv) dev->driver->disable_vblank = ironlake_disable_vblank; dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup; } else { - if (INTEL_INFO(dev_priv)->gen == 2) { + if (IS_GEN2(dev_priv)) { dev->driver->irq_preinstall = i8xx_irq_preinstall; dev->driver->irq_postinstall = i8xx_irq_postinstall; dev->driver->irq_handler = i8xx_irq_handler; dev->driver->irq_uninstall = i8xx_irq_uninstall; - } else if (INTEL_INFO(dev_priv)->gen == 3) { + } else if (IS_GEN3(dev_priv)) { dev->driver->irq_preinstall = i915_irq_preinstall; dev->driver->irq_postinstall = i915_irq_postinstall; dev->driver->irq_uninstall = i915_irq_uninstall; @@ -4717,7 +4644,7 @@ int intel_irq_install(struct drm_i915_private *dev_priv) */ dev_priv->pm.irqs_enabled = true; - return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq); + return drm_irq_install(&dev_priv->drm, dev_priv->drm.pdev->irq); } /** @@ -4729,7 +4656,7 @@ int intel_irq_install(struct drm_i915_private *dev_priv) */ void intel_irq_uninstall(struct drm_i915_private *dev_priv) { - drm_irq_uninstall(dev_priv->dev); + drm_irq_uninstall(&dev_priv->drm); intel_hpd_cancel_work(dev_priv); dev_priv->pm.irqs_enabled = false; } @@ -4743,9 +4670,9 @@ void intel_irq_uninstall(struct drm_i915_private *dev_priv) */ void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv) { - dev_priv->dev->driver->irq_uninstall(dev_priv->dev); + dev_priv->drm.driver->irq_uninstall(&dev_priv->drm); dev_priv->pm.irqs_enabled = false; - synchronize_irq(dev_priv->dev->irq); + synchronize_irq(dev_priv->drm.irq); } /** @@ -4758,6 +4685,6 @@ void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv) void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv) { dev_priv->pm.irqs_enabled = true; - dev_priv->dev->driver->irq_preinstall(dev_priv->dev); - dev_priv->dev->driver->irq_postinstall(dev_priv->dev); + dev_priv->drm.driver->irq_preinstall(&dev_priv->drm); + dev_priv->drm.driver->irq_postinstall(&dev_priv->drm); } diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c index 1779f02e6..b6e404c91 100644 --- a/drivers/gpu/drm/i915/i915_params.c +++ b/drivers/gpu/drm/i915/i915_params.c @@ -54,10 +54,13 @@ struct i915_params i915 __read_mostly = { .verbose_state_checks = 1, .nuclear_pageflip = 0, .edp_vswing = 0, - .enable_guc_submission = false, + .enable_guc_loading = 0, + .enable_guc_submission = 0, .guc_log_level = -1, .enable_dp_mst = true, .inject_load_failure = 0, + .enable_dpcd_backlight = false, + .enable_gvt = false, }; module_param_named(modeset, i915.modeset, int, 0400); @@ -197,8 +200,15 @@ MODULE_PARM_DESC(edp_vswing, "(0=use value from vbt [default], 1=low power swing(200mV)," "2=default swing(400mV))"); -module_param_named_unsafe(enable_guc_submission, i915.enable_guc_submission, bool, 0400); -MODULE_PARM_DESC(enable_guc_submission, "Enable GuC submission (default:false)"); +module_param_named_unsafe(enable_guc_loading, i915.enable_guc_loading, int, 0400); +MODULE_PARM_DESC(enable_guc_loading, + "Enable GuC firmware loading " + "(-1=auto, 0=never [default], 1=if available, 2=required)"); + +module_param_named_unsafe(enable_guc_submission, i915.enable_guc_submission, int, 0400); +MODULE_PARM_DESC(enable_guc_submission, + "Enable GuC submission " + "(-1=auto, 0=never [default], 1=if available, 2=required)"); module_param_named(guc_log_level, i915.guc_log_level, int, 0400); MODULE_PARM_DESC(guc_log_level, @@ -210,3 +220,10 @@ MODULE_PARM_DESC(enable_dp_mst, module_param_named_unsafe(inject_load_failure, i915.inject_load_failure, uint, 0400); MODULE_PARM_DESC(inject_load_failure, "Force an error after a number of failure check points (0:disabled (default), N:force failure at the Nth failure check point)"); +module_param_named(enable_dpcd_backlight, i915.enable_dpcd_backlight, bool, 0600); +MODULE_PARM_DESC(enable_dpcd_backlight, + "Enable support for DPCD backlight control (default:false)"); + +module_param_named(enable_gvt, i915.enable_gvt, bool, 0400); +MODULE_PARM_DESC(enable_gvt, + "Enable support for Intel GVT-g graphics virtualization host support(default:false)"); diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h index 02bc27804..0ad020b4a 100644 --- a/drivers/gpu/drm/i915/i915_params.h +++ b/drivers/gpu/drm/i915/i915_params.h @@ -45,6 +45,8 @@ struct i915_params { int enable_ips; int invert_brightness; int enable_cmd_parser; + int enable_guc_loading; + int enable_guc_submission; int guc_log_level; int use_mmio_flip; int mmio_debug; @@ -57,10 +59,11 @@ struct i915_params { bool load_detect_test; bool reset; bool disable_display; - bool enable_guc_submission; bool verbose_state_checks; bool nuclear_pageflip; bool enable_dp_mst; + bool enable_dpcd_backlight; + bool enable_gvt; }; extern struct i915_params i915 __read_mostly; diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c new file mode 100644 index 000000000..949c01686 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -0,0 +1,503 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include <linux/console.h> +#include <linux/vgaarb.h> +#include <linux/vga_switcheroo.h> + +#include "i915_drv.h" + +#define GEN_DEFAULT_PIPEOFFSETS \ + .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \ + PIPE_C_OFFSET, PIPE_EDP_OFFSET }, \ + .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \ + TRANSCODER_C_OFFSET, TRANSCODER_EDP_OFFSET }, \ + .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET } + +#define GEN_CHV_PIPEOFFSETS \ + .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \ + CHV_PIPE_C_OFFSET }, \ + .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \ + CHV_TRANSCODER_C_OFFSET, }, \ + .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET, \ + CHV_PALETTE_C_OFFSET } + +#define CURSOR_OFFSETS \ + .cursor_offsets = { CURSOR_A_OFFSET, CURSOR_B_OFFSET, CHV_CURSOR_C_OFFSET } + +#define IVB_CURSOR_OFFSETS \ + .cursor_offsets = { CURSOR_A_OFFSET, IVB_CURSOR_B_OFFSET, IVB_CURSOR_C_OFFSET } + +#define BDW_COLORS \ + .color = { .degamma_lut_size = 512, .gamma_lut_size = 512 } +#define CHV_COLORS \ + .color = { .degamma_lut_size = 65, .gamma_lut_size = 257 } + +static const struct intel_device_info intel_i830_info = { + .gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .num_pipes = 2, + .has_overlay = 1, .overlay_needs_physical = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_845g_info = { + .gen = 2, .num_pipes = 1, + .has_overlay = 1, .overlay_needs_physical = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_i85x_info = { + .gen = 2, .is_i85x = 1, .is_mobile = 1, .num_pipes = 2, + .cursor_needs_physical = 1, + .has_overlay = 1, .overlay_needs_physical = 1, + .has_fbc = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_i865g_info = { + .gen = 2, .num_pipes = 1, + .has_overlay = 1, .overlay_needs_physical = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_i915g_info = { + .gen = 3, .is_i915g = 1, .cursor_needs_physical = 1, .num_pipes = 2, + .has_overlay = 1, .overlay_needs_physical = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; +static const struct intel_device_info intel_i915gm_info = { + .gen = 3, .is_mobile = 1, .num_pipes = 2, + .cursor_needs_physical = 1, + .has_overlay = 1, .overlay_needs_physical = 1, + .supports_tv = 1, + .has_fbc = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; +static const struct intel_device_info intel_i945g_info = { + .gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .num_pipes = 2, + .has_overlay = 1, .overlay_needs_physical = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; +static const struct intel_device_info intel_i945gm_info = { + .gen = 3, .is_i945gm = 1, .is_mobile = 1, .num_pipes = 2, + .has_hotplug = 1, .cursor_needs_physical = 1, + .has_overlay = 1, .overlay_needs_physical = 1, + .supports_tv = 1, + .has_fbc = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_i965g_info = { + .gen = 4, .is_broadwater = 1, .num_pipes = 2, + .has_hotplug = 1, + .has_overlay = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_i965gm_info = { + .gen = 4, .is_crestline = 1, .num_pipes = 2, + .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1, + .has_overlay = 1, + .supports_tv = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_g33_info = { + .gen = 3, .is_g33 = 1, .num_pipes = 2, + .need_gfx_hws = 1, .has_hotplug = 1, + .has_overlay = 1, + .ring_mask = RENDER_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_g45_info = { + .gen = 4, .is_g4x = 1, .need_gfx_hws = 1, .num_pipes = 2, + .has_pipe_cxsr = 1, .has_hotplug = 1, + .ring_mask = RENDER_RING | BSD_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_gm45_info = { + .gen = 4, .is_g4x = 1, .num_pipes = 2, + .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1, + .has_pipe_cxsr = 1, .has_hotplug = 1, + .supports_tv = 1, + .ring_mask = RENDER_RING | BSD_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_pineview_info = { + .gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .num_pipes = 2, + .need_gfx_hws = 1, .has_hotplug = 1, + .has_overlay = 1, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_ironlake_d_info = { + .gen = 5, .num_pipes = 2, + .need_gfx_hws = 1, .has_hotplug = 1, + .ring_mask = RENDER_RING | BSD_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_ironlake_m_info = { + .gen = 5, .is_mobile = 1, .num_pipes = 2, + .need_gfx_hws = 1, .has_hotplug = 1, + .has_fbc = 1, + .ring_mask = RENDER_RING | BSD_RING, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_sandybridge_d_info = { + .gen = 6, .num_pipes = 2, + .need_gfx_hws = 1, .has_hotplug = 1, + .has_fbc = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING, + .has_llc = 1, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +static const struct intel_device_info intel_sandybridge_m_info = { + .gen = 6, .is_mobile = 1, .num_pipes = 2, + .need_gfx_hws = 1, .has_hotplug = 1, + .has_fbc = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING, + .has_llc = 1, + GEN_DEFAULT_PIPEOFFSETS, + CURSOR_OFFSETS, +}; + +#define GEN7_FEATURES \ + .gen = 7, .num_pipes = 3, \ + .need_gfx_hws = 1, .has_hotplug = 1, \ + .has_fbc = 1, \ + .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \ + .has_llc = 1, \ + GEN_DEFAULT_PIPEOFFSETS, \ + IVB_CURSOR_OFFSETS + +static const struct intel_device_info intel_ivybridge_d_info = { + GEN7_FEATURES, + .is_ivybridge = 1, +}; + +static const struct intel_device_info intel_ivybridge_m_info = { + GEN7_FEATURES, + .is_ivybridge = 1, + .is_mobile = 1, +}; + +static const struct intel_device_info intel_ivybridge_q_info = { + GEN7_FEATURES, + .is_ivybridge = 1, + .num_pipes = 0, /* legal, last one wins */ +}; + +#define VLV_FEATURES \ + .gen = 7, .num_pipes = 2, \ + .need_gfx_hws = 1, .has_hotplug = 1, \ + .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \ + .display_mmio_offset = VLV_DISPLAY_BASE, \ + GEN_DEFAULT_PIPEOFFSETS, \ + CURSOR_OFFSETS + +static const struct intel_device_info intel_valleyview_m_info = { + VLV_FEATURES, + .is_valleyview = 1, + .is_mobile = 1, +}; + +static const struct intel_device_info intel_valleyview_d_info = { + VLV_FEATURES, + .is_valleyview = 1, +}; + +#define HSW_FEATURES \ + GEN7_FEATURES, \ + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \ + .has_ddi = 1, \ + .has_fpga_dbg = 1 + +static const struct intel_device_info intel_haswell_d_info = { + HSW_FEATURES, + .is_haswell = 1, +}; + +static const struct intel_device_info intel_haswell_m_info = { + HSW_FEATURES, + .is_haswell = 1, + .is_mobile = 1, +}; + +#define BDW_FEATURES \ + HSW_FEATURES, \ + BDW_COLORS + +static const struct intel_device_info intel_broadwell_d_info = { + BDW_FEATURES, + .gen = 8, + .is_broadwell = 1, +}; + +static const struct intel_device_info intel_broadwell_m_info = { + BDW_FEATURES, + .gen = 8, .is_mobile = 1, + .is_broadwell = 1, +}; + +static const struct intel_device_info intel_broadwell_gt3d_info = { + BDW_FEATURES, + .gen = 8, + .is_broadwell = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, +}; + +static const struct intel_device_info intel_broadwell_gt3m_info = { + BDW_FEATURES, + .gen = 8, .is_mobile = 1, + .is_broadwell = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, +}; + +static const struct intel_device_info intel_cherryview_info = { + .gen = 8, .num_pipes = 3, + .need_gfx_hws = 1, .has_hotplug = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, + .is_cherryview = 1, + .display_mmio_offset = VLV_DISPLAY_BASE, + GEN_CHV_PIPEOFFSETS, + CURSOR_OFFSETS, + CHV_COLORS, +}; + +static const struct intel_device_info intel_skylake_info = { + BDW_FEATURES, + .is_skylake = 1, + .gen = 9, +}; + +static const struct intel_device_info intel_skylake_gt3_info = { + BDW_FEATURES, + .is_skylake = 1, + .gen = 9, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, +}; + +static const struct intel_device_info intel_broxton_info = { + .is_broxton = 1, + .gen = 9, + .need_gfx_hws = 1, .has_hotplug = 1, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, + .num_pipes = 3, + .has_ddi = 1, + .has_fpga_dbg = 1, + .has_fbc = 1, + .has_pooled_eu = 0, + GEN_DEFAULT_PIPEOFFSETS, + IVB_CURSOR_OFFSETS, + BDW_COLORS, +}; + +static const struct intel_device_info intel_kabylake_info = { + BDW_FEATURES, + .is_kabylake = 1, + .gen = 9, +}; + +static const struct intel_device_info intel_kabylake_gt3_info = { + BDW_FEATURES, + .is_kabylake = 1, + .gen = 9, + .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING, +}; + +/* + * Make sure any device matches here are from most specific to most + * general. For example, since the Quanta match is based on the subsystem + * and subvendor IDs, we need it to come before the more general IVB + * PCI ID matches, otherwise we'll use the wrong info struct above. + */ +static const struct pci_device_id pciidlist[] = { + INTEL_I830_IDS(&intel_i830_info), + INTEL_I845G_IDS(&intel_845g_info), + INTEL_I85X_IDS(&intel_i85x_info), + INTEL_I865G_IDS(&intel_i865g_info), + INTEL_I915G_IDS(&intel_i915g_info), + INTEL_I915GM_IDS(&intel_i915gm_info), + INTEL_I945G_IDS(&intel_i945g_info), + INTEL_I945GM_IDS(&intel_i945gm_info), + INTEL_I965G_IDS(&intel_i965g_info), + INTEL_G33_IDS(&intel_g33_info), + INTEL_I965GM_IDS(&intel_i965gm_info), + INTEL_GM45_IDS(&intel_gm45_info), + INTEL_G45_IDS(&intel_g45_info), + INTEL_PINEVIEW_IDS(&intel_pineview_info), + INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info), + INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info), + INTEL_SNB_D_IDS(&intel_sandybridge_d_info), + INTEL_SNB_M_IDS(&intel_sandybridge_m_info), + INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */ + INTEL_IVB_M_IDS(&intel_ivybridge_m_info), + INTEL_IVB_D_IDS(&intel_ivybridge_d_info), + INTEL_HSW_D_IDS(&intel_haswell_d_info), + INTEL_HSW_M_IDS(&intel_haswell_m_info), + INTEL_VLV_M_IDS(&intel_valleyview_m_info), + INTEL_VLV_D_IDS(&intel_valleyview_d_info), + INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info), + INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info), + INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info), + INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info), + INTEL_CHV_IDS(&intel_cherryview_info), + INTEL_SKL_GT1_IDS(&intel_skylake_info), + INTEL_SKL_GT2_IDS(&intel_skylake_info), + INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info), + INTEL_SKL_GT4_IDS(&intel_skylake_gt3_info), + INTEL_BXT_IDS(&intel_broxton_info), + INTEL_KBL_GT1_IDS(&intel_kabylake_info), + INTEL_KBL_GT2_IDS(&intel_kabylake_info), + INTEL_KBL_GT3_IDS(&intel_kabylake_gt3_info), + INTEL_KBL_GT4_IDS(&intel_kabylake_gt3_info), + {0, 0, 0} +}; +MODULE_DEVICE_TABLE(pci, pciidlist); + +extern int i915_driver_load(struct pci_dev *pdev, + const struct pci_device_id *ent); + +static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct intel_device_info *intel_info = + (struct intel_device_info *) ent->driver_data; + + if (IS_PRELIMINARY_HW(intel_info) && !i915.preliminary_hw_support) { + DRM_INFO("This hardware requires preliminary hardware support.\n" + "See CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT, and/or modparam preliminary_hw_support\n"); + return -ENODEV; + } + + /* Only bind to function 0 of the device. Early generations + * used function 1 as a placeholder for multi-head. This causes + * us confusion instead, especially on the systems where both + * functions have the same PCI-ID! + */ + if (PCI_FUNC(pdev->devfn)) + return -ENODEV; + + /* + * apple-gmux is needed on dual GPU MacBook Pro + * to probe the panel if we're the inactive GPU. + */ + if (vga_switcheroo_client_probe_defer(pdev)) + return -EPROBE_DEFER; + + return i915_driver_load(pdev, ent); +} + +extern void i915_driver_unload(struct drm_device *dev); + +static void i915_pci_remove(struct pci_dev *pdev) +{ + struct drm_device *dev = pci_get_drvdata(pdev); + + i915_driver_unload(dev); + drm_dev_unref(dev); +} + +extern const struct dev_pm_ops i915_pm_ops; + +static struct pci_driver i915_pci_driver = { + .name = DRIVER_NAME, + .id_table = pciidlist, + .probe = i915_pci_probe, + .remove = i915_pci_remove, + .driver.pm = &i915_pm_ops, +}; + +static int __init i915_init(void) +{ + bool use_kms = true; + + /* + * Enable KMS by default, unless explicitly overriden by + * either the i915.modeset prarameter or by the + * vga_text_mode_force boot option. + */ + + if (i915.modeset == 0) + use_kms = false; + + if (vgacon_text_force() && i915.modeset == -1) + use_kms = false; + + if (!use_kms) { + /* Silently fail loading to not upset userspace. */ + DRM_DEBUG_DRIVER("KMS disabled.\n"); + return 0; + } + + return pci_register_driver(&i915_pci_driver); +} + +static void __exit i915_exit(void) +{ + if (!i915_pci_driver.driver.owner) + return; + + pci_unregister_driver(&i915_pci_driver); +} + +module_init(i915_init); +module_exit(i915_exit); + +MODULE_AUTHOR("Tungsten Graphics, Inc."); +MODULE_AUTHOR("Intel Corporation"); + +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/i915/i915_pvinfo.h b/drivers/gpu/drm/i915/i915_pvinfo.h new file mode 100644 index 000000000..c0cb2974c --- /dev/null +++ b/drivers/gpu/drm/i915/i915_pvinfo.h @@ -0,0 +1,113 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _I915_PVINFO_H_ +#define _I915_PVINFO_H_ + +/* The MMIO offset of the shared info between guest and host emulator */ +#define VGT_PVINFO_PAGE 0x78000 +#define VGT_PVINFO_SIZE 0x1000 + +/* + * The following structure pages are defined in GEN MMIO space + * for virtualization. (One page for now) + */ +#define VGT_MAGIC 0x4776544776544776ULL /* 'vGTvGTvG' */ +#define VGT_VERSION_MAJOR 1 +#define VGT_VERSION_MINOR 0 + +#define INTEL_VGT_IF_VERSION_ENCODE(major, minor) ((major) << 16 | (minor)) +#define INTEL_VGT_IF_VERSION \ + INTEL_VGT_IF_VERSION_ENCODE(VGT_VERSION_MAJOR, VGT_VERSION_MINOR) + +/* + * notifications from guest to vgpu device model + */ +enum vgt_g2v_type { + VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE = 2, + VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY, + VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE, + VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY, + VGT_G2V_EXECLIST_CONTEXT_CREATE, + VGT_G2V_EXECLIST_CONTEXT_DESTROY, + VGT_G2V_MAX, +}; + +struct vgt_if { + u64 magic; /* VGT_MAGIC */ + uint16_t version_major; + uint16_t version_minor; + u32 vgt_id; /* ID of vGT instance */ + u32 rsv1[12]; /* pad to offset 0x40 */ + /* + * Data structure to describe the balooning info of resources. + * Each VM can only have one portion of continuous area for now. + * (May support scattered resource in future) + * (starting from offset 0x40) + */ + struct { + /* Aperture register balooning */ + struct { + u32 base; + u32 size; + } mappable_gmadr; /* aperture */ + /* GMADR register balooning */ + struct { + u32 base; + u32 size; + } nonmappable_gmadr; /* non aperture */ + /* allowed fence registers */ + u32 fence_num; + u32 rsv2[3]; + } avail_rs; /* available/assigned resource */ + u32 rsv3[0x200 - 24]; /* pad to half page */ + /* + * The bottom half page is for response from Gfx driver to hypervisor. + */ + u32 rsv4; + u32 display_ready; /* ready for display owner switch */ + + u32 rsv5[4]; + + u32 g2v_notify; + u32 rsv6[7]; + + struct { + u32 lo; + u32 hi; + } pdp[4]; + + u32 execlist_context_descriptor_lo; + u32 execlist_context_descriptor_hi; + + u32 rsv7[0x200 - 24]; /* pad to one page */ +} __packed; + +#define vgtif_reg(x) \ + _MMIO((VGT_PVINFO_PAGE + offsetof(struct vgt_if, x))) + +/* vGPU display status to be used by the host side */ +#define VGT_DRV_DISPLAY_NOT_READY 0 +#define VGT_DRV_DISPLAY_READY 1 /* ready for display switch */ + +#endif /* _I915_PVINFO_H_ */ diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 3c123d430..bf2cad3f9 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -445,6 +445,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) */ #define GFX_INSTR(opcode, flags) ((0x3 << 29) | ((opcode) << 24) | (flags)) +#define GEN9_MEDIA_POOL_STATE ((0x3 << 29) | (0x2 << 27) | (0x5 << 16) | 4) +#define GEN9_MEDIA_POOL_ENABLE (1 << 31) #define GFX_OP_RASTER_RULES ((0x3<<29)|(0x7<<24)) #define GFX_OP_SCISSOR ((0x3<<29)|(0x1c<<24)|(0x10<<19)) #define SC_UPDATE_SCISSOR (0x1<<1) @@ -716,6 +718,9 @@ enum skl_disp_power_wells { /* Not actual bit groups. Used as IDs for lookup_power_well() */ SKL_DISP_PW_ALWAYS_ON, SKL_DISP_PW_DC_OFF, + + BXT_DPIO_CMN_A, + BXT_DPIO_CMN_BC, }; #define SKL_POWER_WELL_STATE(pw) (1 << ((pw) * 2)) @@ -889,7 +894,7 @@ enum skl_disp_power_wells { * PLLs can be routed to any transcoder A/B/C. * * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is - * digital port D (CHV) or port A (BXT). + * digital port D (CHV) or port A (BXT). :: * * * Dual channel PHY (VLV/CHV/BXT) @@ -1276,6 +1281,15 @@ enum skl_disp_power_wells { #define BXT_P_CR_GT_DISP_PWRON _MMIO(0x138090) #define GT_DISPLAY_POWER_ON(phy) (1 << (phy)) +#define _BXT_PHY_CTL_DDI_A 0x64C00 +#define _BXT_PHY_CTL_DDI_B 0x64C10 +#define _BXT_PHY_CTL_DDI_C 0x64C20 +#define BXT_PHY_CMNLANE_POWERDOWN_ACK (1 << 10) +#define BXT_PHY_LANE_POWERDOWN_ACK (1 << 9) +#define BXT_PHY_LANE_ENABLED (1 << 8) +#define BXT_PHY_CTL(port) _MMIO_PORT(port, _BXT_PHY_CTL_DDI_A, \ + _BXT_PHY_CTL_DDI_B) + #define _PHY_CTL_FAMILY_EDP 0x64C80 #define _PHY_CTL_FAMILY_DDI 0x64C90 #define COMMON_RESET_DIS (1 << 31) @@ -2175,6 +2189,9 @@ enum skl_disp_power_wells { #define FBC_LL_SIZE (1536) +#define FBC_LLC_READ_CTRL _MMIO(0x9044) +#define FBC_LLC_FULLY_OPEN (1<<30) + /* Framebuffer compression for GM45+ */ #define DPFC_CB_BASE _MMIO(0x3200) #define DPFC_CONTROL _MMIO(0x3208) @@ -2465,6 +2482,8 @@ enum skl_disp_power_wells { #define DPLL_MD_VGA_UDI_MULTIPLIER_MASK 0x0000003f #define DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT 0 +#define RAWCLK_FREQ_VLV _MMIO(VLV_DISPLAY_BASE + 0x6024) + #define _FPA0 0x6040 #define _FPA1 0x6044 #define _FPB0 0x6048 @@ -3036,6 +3055,18 @@ enum skl_disp_power_wells { /* Same as Haswell, but 72064 bytes now. */ #define GEN8_CXT_TOTAL_SIZE (18 * PAGE_SIZE) +enum { + INTEL_ADVANCED_CONTEXT = 0, + INTEL_LEGACY_32B_CONTEXT, + INTEL_ADVANCED_AD_CONTEXT, + INTEL_LEGACY_64B_CONTEXT +}; + +#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3 +#define GEN8_CTX_ADDRESSING_MODE(dev_priv) (USES_FULL_48BIT_PPGTT(dev_priv) ?\ + INTEL_LEGACY_64B_CONTEXT : \ + INTEL_LEGACY_32B_CONTEXT) + #define CHV_CLK_CTL1 _MMIO(0x101100) #define VLV_CLK_CTL2 _MMIO(0x101104) #define CLK_CTL2_CZCOUNT_30NS_SHIFT 28 @@ -6049,6 +6080,9 @@ enum skl_disp_power_wells { #define FORCE_ARB_IDLE_PLANES (1 << 14) #define SKL_EDP_PSR_FIX_RDWRAP (1 << 3) +#define CHICKEN_PAR2_1 _MMIO(0x42090) +#define KVM_CONFIG_CHANGE_NOTIFICATION_SELECT (1 << 14) + #define _CHICKEN_PIPESL_1_A 0x420b0 #define _CHICKEN_PIPESL_1_B 0x420b4 #define HSW_FBCQ_DIS (1 << 22) @@ -6088,6 +6122,7 @@ enum skl_disp_power_wells { #define FF_SLICE_CS_CHICKEN2 _MMIO(0x20e4) #define GEN9_TSG_BARRIER_ACK_DISABLE (1<<8) +#define GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE (1<<10) #define GEN9_CS_DEBUG_MODE1 _MMIO(0x20ec) #define GEN9_CTX_PREEMPT_REG _MMIO(0x2248) @@ -6112,7 +6147,14 @@ enum skl_disp_power_wells { #define VLV_B0_WA_L3SQCREG1_VALUE 0x00D30000 #define GEN8_L3SQCREG1 _MMIO(0xB100) -#define BDW_WA_L3SQCREG1_DEFAULT 0x784000 +/* + * Note that on CHV the following has an off-by-one error wrt. to BSpec. + * Using the formula in BSpec leads to a hang, while the formula here works + * fine and matches the formulas for all other platforms. A BSpec change + * request has been filed to clarify this. + */ +#define L3_GENERAL_PRIO_CREDITS(x) (((x) >> 1) << 19) +#define L3_HIGH_PRIO_CREDITS(x) (((x) >> 1) << 14) #define GEN7_L3CNTLREG1 _MMIO(0xB01C) #define GEN7_WA_FOR_GEN7_L3_CONTROL 0x3C47FF8C @@ -7032,7 +7074,8 @@ enum skl_disp_power_wells { #define GEN6_RPDEUC _MMIO(0xA084) #define GEN6_RPDEUCSW _MMIO(0xA088) #define GEN6_RC_STATE _MMIO(0xA094) -#define RC6_STATE (1 << 18) +#define RC_SW_TARGET_STATE_SHIFT 16 +#define RC_SW_TARGET_STATE_MASK (7 << RC_SW_TARGET_STATE_SHIFT) #define GEN6_RC1_WAKE_RATE_LIMIT _MMIO(0xA098) #define GEN6_RC6_WAKE_RATE_LIMIT _MMIO(0xA09C) #define GEN6_RC6pp_WAKE_RATE_LIMIT _MMIO(0xA0A0) @@ -7046,13 +7089,17 @@ enum skl_disp_power_wells { #define VLV_RCEDATA _MMIO(0xA0BC) #define GEN6_RC6pp_THRESHOLD _MMIO(0xA0C0) #define GEN6_PMINTRMSK _MMIO(0xA168) -#define GEN8_PMINTR_REDIRECT_TO_NON_DISP (1<<31) +#define GEN8_PMINTR_REDIRECT_TO_NON_DISP (1<<31) +#define GEN8_MISC_CTRL0 _MMIO(0xA180) #define VLV_PWRDWNUPCTL _MMIO(0xA294) #define GEN9_MEDIA_PG_IDLE_HYSTERESIS _MMIO(0xA0C4) #define GEN9_RENDER_PG_IDLE_HYSTERESIS _MMIO(0xA0C8) #define GEN9_PG_ENABLE _MMIO(0xA210) #define GEN9_RENDER_PG_ENABLE (1<<0) #define GEN9_MEDIA_PG_ENABLE (1<<1) +#define GEN8_PUSHBUS_CONTROL _MMIO(0xA248) +#define GEN8_PUSHBUS_ENABLE _MMIO(0xA250) +#define GEN8_PUSHBUS_SHIFT _MMIO(0xA25C) #define VLV_CHICKEN_3 _MMIO(VLV_DISPLAY_BASE + 0x7040C) #define PIXEL_OVERLAP_CNT_MASK (3 << 30) @@ -7098,6 +7145,15 @@ enum skl_disp_power_wells { #define GEN6_PCODE_MAILBOX _MMIO(0x138124) #define GEN6_PCODE_READY (1<<31) +#define GEN6_PCODE_ERROR_MASK 0xFF +#define GEN6_PCODE_SUCCESS 0x0 +#define GEN6_PCODE_ILLEGAL_CMD 0x1 +#define GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x2 +#define GEN6_PCODE_TIMEOUT 0x3 +#define GEN6_PCODE_UNIMPLEMENTED_CMD 0xFF +#define GEN7_PCODE_TIMEOUT 0x2 +#define GEN7_PCODE_ILLEGAL_DATA 0x3 +#define GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x10 #define GEN6_PCODE_WRITE_RC6VIDS 0x4 #define GEN6_PCODE_READ_RC6VIDS 0x5 #define GEN6_ENCODE_RC6_VID(mv) (((mv) - 245) / 5) @@ -7121,9 +7177,8 @@ enum skl_disp_power_wells { #define HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL 0x1A #define GEN9_PCODE_SAGV_CONTROL 0x21 #define GEN9_SAGV_DISABLE 0x0 -#define GEN9_SAGV_LOW_FREQ 0x1 -#define GEN9_SAGV_HIGH_FREQ 0x2 -#define GEN9_SAGV_DYNAMIC_FREQ 0x3 +#define GEN9_SAGV_IS_DISABLED 0x1 +#define GEN9_SAGV_ENABLE 0x3 #define GEN6_PCODE_DATA _MMIO(0x138128) #define GEN6_PCODE_FREQ_IA_RATIO_SHIFT 8 #define GEN6_PCODE_FREQ_RING_RATIO_SHIFT 16 @@ -7547,8 +7602,6 @@ enum skl_disp_power_wells { #define CDCLK_FREQ _MMIO(0x46200) -#define CDCLK_FREQ _MMIO(0x46200) - #define _TRANSA_MSA_MISC 0x60410 #define _TRANSB_MSA_MISC 0x61410 #define _TRANSC_MSA_MISC 0x62410 @@ -7589,14 +7642,15 @@ enum skl_disp_power_wells { #define CDCLK_FREQ_540 (1<<26) #define CDCLK_FREQ_337_308 (2<<26) #define CDCLK_FREQ_675_617 (3<<26) -#define CDCLK_FREQ_DECIMAL_MASK (0x7ff) - #define BXT_CDCLK_CD2X_DIV_SEL_MASK (3<<22) #define BXT_CDCLK_CD2X_DIV_SEL_1 (0<<22) #define BXT_CDCLK_CD2X_DIV_SEL_1_5 (1<<22) #define BXT_CDCLK_CD2X_DIV_SEL_2 (2<<22) #define BXT_CDCLK_CD2X_DIV_SEL_4 (3<<22) +#define BXT_CDCLK_CD2X_PIPE(pipe) ((pipe)<<20) +#define BXT_CDCLK_CD2X_PIPE_NONE BXT_CDCLK_CD2X_PIPE(3) #define BXT_CDCLK_SSA_PRECHARGE_ENABLE (1<<16) +#define CDCLK_FREQ_DECIMAL_MASK (0x7ff) /* LCPLL_CTL */ #define LCPLL1_CTL _MMIO(0x46010) @@ -8172,6 +8226,8 @@ enum skl_disp_power_wells { #define _MIPIA_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb05c) #define _MIPIC_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb85c) #define MIPI_EOT_DISABLE(port) _MMIO_MIPI(port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE) +#define BXT_DEFEATURE_DPI_FIFO_CTR (1 << 9) +#define BXT_DPHY_DEFEATURE_EN (1 << 8) #define LP_RX_TIMEOUT_ERROR_RECOVERY_DISABLE (1 << 7) #define HS_RX_TIMEOUT_ERROR_RECOVERY_DISABLE (1 << 6) #define LOW_CONTENTION_RECOVERY_DISABLE (1 << 5) diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c index 34e061a9e..5cfe4c771 100644 --- a/drivers/gpu/drm/i915/i915_suspend.c +++ b/drivers/gpu/drm/i915/i915_suspend.c @@ -31,7 +31,7 @@ static void i915_save_display(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* Display arbitration control */ if (INTEL_INFO(dev)->gen <= 4) @@ -63,7 +63,7 @@ static void i915_save_display(struct drm_device *dev) static void i915_restore_display(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 mask = 0xffffffff; /* Display arbitration */ @@ -103,7 +103,7 @@ static void i915_restore_display(struct drm_device *dev) int i915_save_state(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i; mutex_lock(&dev->struct_mutex); @@ -148,7 +148,7 @@ int i915_save_state(struct drm_device *dev) int i915_restore_state(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i; mutex_lock(&dev->struct_mutex); diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index 2d576b7ff..d61829e54 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -38,12 +38,12 @@ static u32 calc_residency(struct drm_device *dev, i915_reg_t reg) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u64 raw_time; /* 32b value may overflow during fixed point math */ u64 units = 128ULL, div = 100000ULL; u32 ret; - if (!intel_enable_rc6(dev)) + if (!intel_enable_rc6()) return 0; intel_runtime_pm_get(dev_priv); @@ -70,8 +70,7 @@ static u32 calc_residency(struct drm_device *dev, static ssize_t show_rc6_mask(struct device *kdev, struct device_attribute *attr, char *buf) { - struct drm_minor *dminor = dev_to_drm_minor(kdev); - return snprintf(buf, PAGE_SIZE, "%x\n", intel_enable_rc6(dminor->dev)); + return snprintf(buf, PAGE_SIZE, "%x\n", intel_enable_rc6()); } static ssize_t @@ -167,7 +166,7 @@ i915_l3_read(struct file *filp, struct kobject *kobj, struct device *dev = kobj_to_dev(kobj); struct drm_minor *dminor = dev_to_drm_minor(dev); struct drm_device *drm_dev = dminor->dev; - struct drm_i915_private *dev_priv = drm_dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(drm_dev); int slice = (int)(uintptr_t)attr->private; int ret; @@ -203,8 +202,8 @@ i915_l3_write(struct file *filp, struct kobject *kobj, struct device *dev = kobj_to_dev(kobj); struct drm_minor *dminor = dev_to_drm_minor(dev); struct drm_device *drm_dev = dminor->dev; - struct drm_i915_private *dev_priv = drm_dev->dev_private; - struct intel_context *ctx; + struct drm_i915_private *dev_priv = to_i915(drm_dev); + struct i915_gem_context *ctx; u32 *temp = NULL; /* Just here to make handling failures easy */ int slice = (int)(uintptr_t)attr->private; int ret; @@ -228,13 +227,6 @@ i915_l3_write(struct file *filp, struct kobject *kobj, } } - ret = i915_gpu_idle(drm_dev); - if (ret) { - kfree(temp); - mutex_unlock(&drm_dev->struct_mutex); - return ret; - } - /* TODO: Ideally we really want a GPU reset here to make sure errors * aren't propagated. Since I cannot find a stable way to reset the GPU * at this point it is left as a TODO. @@ -276,7 +268,7 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev, { struct drm_minor *minor = dev_to_drm_minor(kdev); struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; flush_delayed_work(&dev_priv->rps.delayed_resume_work); @@ -310,7 +302,7 @@ static ssize_t gt_cur_freq_mhz_show(struct device *kdev, { struct drm_minor *minor = dev_to_drm_minor(kdev); struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; flush_delayed_work(&dev_priv->rps.delayed_resume_work); @@ -331,7 +323,7 @@ static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev, { struct drm_minor *minor = dev_to_drm_minor(kdev); struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); return snprintf(buf, PAGE_SIZE, "%d\n", @@ -342,7 +334,7 @@ static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute { struct drm_minor *minor = dev_to_drm_minor(kdev); struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; flush_delayed_work(&dev_priv->rps.delayed_resume_work); @@ -360,7 +352,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, { struct drm_minor *minor = dev_to_drm_minor(kdev); struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val; ssize_t ret; @@ -397,7 +389,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev, /* We still need *_set_rps to process the new max_delay and * update the interrupt limits and PMINTRMSK even though * frequency request may be unchanged. */ - intel_set_rps(dev, val); + intel_set_rps(dev_priv, val); mutex_unlock(&dev_priv->rps.hw_lock); @@ -410,7 +402,7 @@ static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute { struct drm_minor *minor = dev_to_drm_minor(kdev); struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; flush_delayed_work(&dev_priv->rps.delayed_resume_work); @@ -428,7 +420,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, { struct drm_minor *minor = dev_to_drm_minor(kdev); struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val; ssize_t ret; @@ -461,7 +453,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev, /* We still need *_set_rps to process the new min_delay and * update the interrupt limits and PMINTRMSK even though * frequency request may be unchanged. */ - intel_set_rps(dev, val); + intel_set_rps(dev_priv, val); mutex_unlock(&dev_priv->rps.hw_lock); @@ -488,7 +480,7 @@ static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr { struct drm_minor *minor = dev_to_drm_minor(kdev); struct drm_device *dev = minor->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val; if (attr == &dev_attr_gt_RP0_freq_mhz) diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index dc0def210..534154e05 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -118,7 +118,7 @@ TRACE_EVENT(i915_gem_shrink, ), TP_fast_assign( - __entry->dev = i915->dev->primary->index; + __entry->dev = i915->drm.primary->index; __entry->target = target; __entry->flags = flags; ), @@ -462,7 +462,7 @@ TRACE_EVENT(i915_gem_ring_sync_to, ), TP_fast_assign( - __entry->dev = from->dev->primary->index; + __entry->dev = from->i915->drm.primary->index; __entry->sync_from = from->id; __entry->sync_to = to_req->engine->id; __entry->seqno = i915_gem_request_get_seqno(req); @@ -486,13 +486,11 @@ TRACE_EVENT(i915_gem_ring_dispatch, ), TP_fast_assign( - struct intel_engine_cs *engine = - i915_gem_request_get_engine(req); - __entry->dev = engine->dev->primary->index; - __entry->ring = engine->id; - __entry->seqno = i915_gem_request_get_seqno(req); + __entry->dev = req->i915->drm.primary->index; + __entry->ring = req->engine->id; + __entry->seqno = req->seqno; __entry->flags = flags; - i915_trace_irq_get(engine, req); + intel_engine_enable_signaling(req); ), TP_printk("dev=%u, ring=%u, seqno=%u, flags=%x", @@ -511,7 +509,7 @@ TRACE_EVENT(i915_gem_ring_flush, ), TP_fast_assign( - __entry->dev = req->engine->dev->primary->index; + __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; __entry->invalidate = invalidate; __entry->flush = flush; @@ -533,11 +531,9 @@ DECLARE_EVENT_CLASS(i915_gem_request, ), TP_fast_assign( - struct intel_engine_cs *engine = - i915_gem_request_get_engine(req); - __entry->dev = engine->dev->primary->index; - __entry->ring = engine->id; - __entry->seqno = i915_gem_request_get_seqno(req); + __entry->dev = req->i915->drm.primary->index; + __entry->ring = req->engine->id; + __entry->seqno = req->seqno; ), TP_printk("dev=%u, ring=%u, seqno=%u", @@ -560,9 +556,9 @@ TRACE_EVENT(i915_gem_request_notify, ), TP_fast_assign( - __entry->dev = engine->dev->primary->index; + __entry->dev = engine->i915->drm.primary->index; __entry->ring = engine->id; - __entry->seqno = engine->get_seqno(engine); + __entry->seqno = intel_engine_get_seqno(engine); ), TP_printk("dev=%u, ring=%u, seqno=%u", @@ -597,13 +593,11 @@ TRACE_EVENT(i915_gem_request_wait_begin, * less desirable. */ TP_fast_assign( - struct intel_engine_cs *engine = - i915_gem_request_get_engine(req); - __entry->dev = engine->dev->primary->index; - __entry->ring = engine->id; - __entry->seqno = i915_gem_request_get_seqno(req); + __entry->dev = req->i915->drm.primary->index; + __entry->ring = req->engine->id; + __entry->seqno = req->seqno; __entry->blocking = - mutex_is_locked(&engine->dev->struct_mutex); + mutex_is_locked(&req->i915->drm.struct_mutex); ), TP_printk("dev=%u, ring=%u, seqno=%u, blocking=%s", @@ -740,19 +734,19 @@ DEFINE_EVENT(i915_ppgtt, i915_ppgtt_release, * the context. */ DECLARE_EVENT_CLASS(i915_context, - TP_PROTO(struct intel_context *ctx), + TP_PROTO(struct i915_gem_context *ctx), TP_ARGS(ctx), TP_STRUCT__entry( __field(u32, dev) - __field(struct intel_context *, ctx) + __field(struct i915_gem_context *, ctx) __field(struct i915_address_space *, vm) ), TP_fast_assign( __entry->ctx = ctx; __entry->vm = ctx->ppgtt ? &ctx->ppgtt->base : NULL; - __entry->dev = ctx->i915->dev->primary->index; + __entry->dev = ctx->i915->drm.primary->index; ), TP_printk("dev=%u, ctx=%p, ctx_vm=%p", @@ -760,12 +754,12 @@ DECLARE_EVENT_CLASS(i915_context, ) DEFINE_EVENT(i915_context, i915_context_create, - TP_PROTO(struct intel_context *ctx), + TP_PROTO(struct i915_gem_context *ctx), TP_ARGS(ctx) ); DEFINE_EVENT(i915_context, i915_context_free, - TP_PROTO(struct intel_context *ctx), + TP_PROTO(struct i915_gem_context *ctx), TP_ARGS(ctx) ); @@ -777,13 +771,13 @@ DEFINE_EVENT(i915_context, i915_context_free, * called only if full ppgtt is enabled. */ TRACE_EVENT(switch_mm, - TP_PROTO(struct intel_engine_cs *engine, struct intel_context *to), + TP_PROTO(struct intel_engine_cs *engine, struct i915_gem_context *to), TP_ARGS(engine, to), TP_STRUCT__entry( __field(u32, ring) - __field(struct intel_context *, to) + __field(struct i915_gem_context *, to) __field(struct i915_address_space *, vm) __field(u32, dev) ), @@ -792,7 +786,7 @@ TRACE_EVENT(switch_mm, __entry->ring = engine->id; __entry->to = to; __entry->vm = to->ppgtt? &to->ppgtt->base : NULL; - __entry->dev = engine->dev->primary->index; + __entry->dev = engine->i915->drm.primary->index; ), TP_printk("dev=%u, ring=%u, ctx=%p, ctx_vm=%p", diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c index d02efb8ca..b81cfb3b2 100644 --- a/drivers/gpu/drm/i915/i915_vgpu.c +++ b/drivers/gpu/drm/i915/i915_vgpu.c @@ -53,22 +53,18 @@ /** * i915_check_vgpu - detect virtual GPU - * @dev: drm device * + * @dev_priv: i915 device private * * This function is called at the initialization stage, to detect whether * running on a vGPU. */ -void i915_check_vgpu(struct drm_device *dev) +void i915_check_vgpu(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint64_t magic; uint32_t version; BUILD_BUG_ON(sizeof(struct vgt_if) != VGT_PVINFO_SIZE); - if (!IS_HASWELL(dev)) - return; - magic = __raw_i915_read64(dev_priv, vgtif_reg(magic)); if (magic != VGT_MAGIC) return; @@ -102,10 +98,13 @@ static struct _balloon_info_ bl_info; * This function is called to deallocate the ballooned-out graphic memory, when * driver is unloaded or when ballooning fails. */ -void intel_vgt_deballoon(void) +void intel_vgt_deballoon(struct drm_i915_private *dev_priv) { int i; + if (!intel_vgpu_active(dev_priv)) + return; + DRM_DEBUG("VGT deballoon.\n"); for (i = 0; i < 4; i++) { @@ -151,36 +150,35 @@ static int vgt_balloon_space(struct drm_mm *mm, * of its graphic space being zero. Yet there are some portions ballooned out( * the shadow part, which are marked as reserved by drm allocator). From the * host point of view, the graphic address space is partitioned by multiple - * vGPUs in different VMs. + * vGPUs in different VMs. :: * * vGPU1 view Host view * 0 ------> +-----------+ +-----------+ - * ^ |///////////| | vGPU3 | - * | |///////////| +-----------+ - * | |///////////| | vGPU2 | + * ^ |###########| | vGPU3 | + * | |###########| +-----------+ + * | |###########| | vGPU2 | * | +-----------+ +-----------+ * mappable GM | available | ==> | vGPU1 | * | +-----------+ +-----------+ - * | |///////////| | | - * v |///////////| | Host | + * | |###########| | | + * v |###########| | Host | * +=======+===========+ +===========+ - * ^ |///////////| | vGPU3 | - * | |///////////| +-----------+ - * | |///////////| | vGPU2 | + * ^ |###########| | vGPU3 | + * | |###########| +-----------+ + * | |###########| | vGPU2 | * | +-----------+ +-----------+ * unmappable GM | available | ==> | vGPU1 | * | +-----------+ +-----------+ - * | |///////////| | | - * | |///////////| | Host | - * v |///////////| | | + * | |###########| | | + * | |###########| | Host | + * v |###########| | | * total GM size ------> +-----------+ +-----------+ * * Returns: * zero on success, non-zero if configuration invalid or ballooning failed */ -int intel_vgt_balloon(struct drm_device *dev) +int intel_vgt_balloon(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; unsigned long ggtt_end = ggtt->base.start + ggtt->base.total; @@ -188,6 +186,9 @@ int intel_vgt_balloon(struct drm_device *dev) unsigned long unmappable_base, unmappable_size, unmappable_end; int ret; + if (!intel_vgpu_active(dev_priv)) + return 0; + mappable_base = I915_READ(vgtif_reg(avail_rs.mappable_gmadr.base)); mappable_size = I915_READ(vgtif_reg(avail_rs.mappable_gmadr.size)); unmappable_base = I915_READ(vgtif_reg(avail_rs.nonmappable_gmadr.base)); @@ -259,6 +260,6 @@ int intel_vgt_balloon(struct drm_device *dev) err: DRM_ERROR("VGT balloon fail\n"); - intel_vgt_deballoon(); + intel_vgt_deballoon(dev_priv); return ret; } diff --git a/drivers/gpu/drm/i915/i915_vgpu.h b/drivers/gpu/drm/i915/i915_vgpu.h index 3c83b47b5..3c3b2d24e 100644 --- a/drivers/gpu/drm/i915/i915_vgpu.h +++ b/drivers/gpu/drm/i915/i915_vgpu.h @@ -24,94 +24,10 @@ #ifndef _I915_VGPU_H_ #define _I915_VGPU_H_ -/* The MMIO offset of the shared info between guest and host emulator */ -#define VGT_PVINFO_PAGE 0x78000 -#define VGT_PVINFO_SIZE 0x1000 +#include "i915_pvinfo.h" -/* - * The following structure pages are defined in GEN MMIO space - * for virtualization. (One page for now) - */ -#define VGT_MAGIC 0x4776544776544776ULL /* 'vGTvGTvG' */ -#define VGT_VERSION_MAJOR 1 -#define VGT_VERSION_MINOR 0 - -#define INTEL_VGT_IF_VERSION_ENCODE(major, minor) ((major) << 16 | (minor)) -#define INTEL_VGT_IF_VERSION \ - INTEL_VGT_IF_VERSION_ENCODE(VGT_VERSION_MAJOR, VGT_VERSION_MINOR) - -/* - * notifications from guest to vgpu device model - */ -enum vgt_g2v_type { - VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE = 2, - VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY, - VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE, - VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY, - VGT_G2V_EXECLIST_CONTEXT_CREATE, - VGT_G2V_EXECLIST_CONTEXT_DESTROY, - VGT_G2V_MAX, -}; - -struct vgt_if { - uint64_t magic; /* VGT_MAGIC */ - uint16_t version_major; - uint16_t version_minor; - uint32_t vgt_id; /* ID of vGT instance */ - uint32_t rsv1[12]; /* pad to offset 0x40 */ - /* - * Data structure to describe the balooning info of resources. - * Each VM can only have one portion of continuous area for now. - * (May support scattered resource in future) - * (starting from offset 0x40) - */ - struct { - /* Aperture register balooning */ - struct { - uint32_t base; - uint32_t size; - } mappable_gmadr; /* aperture */ - /* GMADR register balooning */ - struct { - uint32_t base; - uint32_t size; - } nonmappable_gmadr; /* non aperture */ - /* allowed fence registers */ - uint32_t fence_num; - uint32_t rsv2[3]; - } avail_rs; /* available/assigned resource */ - uint32_t rsv3[0x200 - 24]; /* pad to half page */ - /* - * The bottom half page is for response from Gfx driver to hypervisor. - */ - uint32_t rsv4; - uint32_t display_ready; /* ready for display owner switch */ - - uint32_t rsv5[4]; - - uint32_t g2v_notify; - uint32_t rsv6[7]; - - struct { - uint32_t lo; - uint32_t hi; - } pdp[4]; - - uint32_t execlist_context_descriptor_lo; - uint32_t execlist_context_descriptor_hi; - - uint32_t rsv7[0x200 - 24]; /* pad to one page */ -} __packed; - -#define vgtif_reg(x) \ - _MMIO((VGT_PVINFO_PAGE + (long)&((struct vgt_if *)NULL)->x)) - -/* vGPU display status to be used by the host side */ -#define VGT_DRV_DISPLAY_NOT_READY 0 -#define VGT_DRV_DISPLAY_READY 1 /* ready for display switch */ - -extern void i915_check_vgpu(struct drm_device *dev); -extern int intel_vgt_balloon(struct drm_device *dev); -extern void intel_vgt_deballoon(void); +void i915_check_vgpu(struct drm_i915_private *dev_priv); +int intel_vgt_balloon(struct drm_i915_private *dev_priv); +void intel_vgt_deballoon(struct drm_i915_private *dev_priv); #endif /* _I915_VGPU_H_ */ diff --git a/drivers/gpu/drm/i915/intel_atomic.c b/drivers/gpu/drm/i915/intel_atomic.c index 50ff90aea..c5a166752 100644 --- a/drivers/gpu/drm/i915/intel_atomic.c +++ b/drivers/gpu/drm/i915/intel_atomic.c @@ -191,7 +191,7 @@ int intel_atomic_setup_scalers(struct drm_device *dev, /* plane scaler case: assign as a plane scaler */ /* find the plane that set the bit as scaler_user */ - plane = drm_state->planes[i]; + plane = drm_state->planes[i].ptr; /* * to enable/disable hq mode, add planes that are using scaler @@ -223,7 +223,8 @@ int intel_atomic_setup_scalers(struct drm_device *dev, continue; } - plane_state = to_intel_plane_state(drm_state->plane_states[i]); + plane_state = intel_atomic_get_existing_plane_state(drm_state, + intel_plane); scaler_id = &plane_state->scaler_id; } diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c index 74eca43b2..d32f586f9 100644 --- a/drivers/gpu/drm/i915/intel_audio.c +++ b/drivers/gpu/drm/i915/intel_audio.c @@ -154,7 +154,7 @@ static bool audio_rate_need_prog(struct intel_crtc *crtc, { if (((mode->clock == TMDS_297M) || (mode->clock == TMDS_296M)) && - intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI)) + intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI)) return true; else return false; @@ -165,7 +165,7 @@ static bool intel_eld_uptodate(struct drm_connector *connector, i915_reg_t reg_elda, uint32_t bits_elda, i915_reg_t reg_edid) { - struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(connector->dev); uint8_t *eld = connector->eld; uint32_t tmp; int i; @@ -189,7 +189,7 @@ static bool intel_eld_uptodate(struct drm_connector *connector, static void g4x_audio_codec_disable(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); uint32_t eldv, tmp; DRM_DEBUG_KMS("Disable audio codec\n"); @@ -210,7 +210,7 @@ static void g4x_audio_codec_enable(struct drm_connector *connector, struct intel_encoder *encoder, const struct drm_display_mode *adjusted_mode) { - struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(connector->dev); uint8_t *eld = connector->eld; uint32_t eldv; uint32_t tmp; @@ -247,7 +247,7 @@ static void g4x_audio_codec_enable(struct drm_connector *connector, static void hsw_audio_codec_disable(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); enum pipe pipe = intel_crtc->pipe; uint32_t tmp; @@ -262,7 +262,7 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder) tmp |= AUD_CONFIG_N_PROG_ENABLE; tmp &= ~AUD_CONFIG_UPPER_N_MASK; tmp &= ~AUD_CONFIG_LOWER_N_MASK; - if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT)) + if (intel_crtc_has_dp_encoder(intel_crtc->config)) tmp |= AUD_CONFIG_N_VALUE_INDEX; I915_WRITE(HSW_AUD_CFG(pipe), tmp); @@ -279,7 +279,7 @@ static void hsw_audio_codec_enable(struct drm_connector *connector, struct intel_encoder *encoder, const struct drm_display_mode *adjusted_mode) { - struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(connector->dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); enum pipe pipe = intel_crtc->pipe; struct i915_audio_component *acomp = dev_priv->audio_component; @@ -328,7 +328,7 @@ static void hsw_audio_codec_enable(struct drm_connector *connector, tmp = I915_READ(HSW_AUD_CFG(pipe)); tmp &= ~AUD_CONFIG_N_VALUE_INDEX; tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK; - if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT)) + if (intel_crtc_has_dp_encoder(intel_crtc->config)) tmp |= AUD_CONFIG_N_VALUE_INDEX; else tmp |= audio_config_hdmi_pixel_clock(adjusted_mode); @@ -357,7 +357,7 @@ static void hsw_audio_codec_enable(struct drm_connector *connector, static void ilk_audio_codec_disable(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base); @@ -389,7 +389,7 @@ static void ilk_audio_codec_disable(struct intel_encoder *encoder) tmp |= AUD_CONFIG_N_PROG_ENABLE; tmp &= ~AUD_CONFIG_UPPER_N_MASK; tmp &= ~AUD_CONFIG_LOWER_N_MASK; - if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT)) + if (intel_crtc_has_dp_encoder(intel_crtc->config)) tmp |= AUD_CONFIG_N_VALUE_INDEX; I915_WRITE(aud_config, tmp); @@ -405,7 +405,7 @@ static void ilk_audio_codec_enable(struct drm_connector *connector, struct intel_encoder *encoder, const struct drm_display_mode *adjusted_mode) { - struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(connector->dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base); @@ -475,7 +475,7 @@ static void ilk_audio_codec_enable(struct drm_connector *connector, tmp &= ~AUD_CONFIG_N_VALUE_INDEX; tmp &= ~AUD_CONFIG_N_PROG_ENABLE; tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK; - if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT)) + if (intel_crtc_has_dp_encoder(intel_crtc->config)) tmp |= AUD_CONFIG_N_VALUE_INDEX; else tmp |= audio_config_hdmi_pixel_clock(adjusted_mode); @@ -496,7 +496,7 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder) const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; struct drm_connector *connector; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct i915_audio_component *acomp = dev_priv->audio_component; struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); enum port port = intel_dig_port->port; @@ -513,7 +513,7 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder) /* ELD Conn_Type */ connector->eld[5] &= ~(3 << 2); - if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) + if (intel_crtc_has_dp_encoder(crtc->config)) connector->eld[5] |= (1 << 2); connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2; @@ -543,7 +543,7 @@ void intel_audio_codec_disable(struct intel_encoder *intel_encoder) { struct drm_encoder *encoder = &intel_encoder->base; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct i915_audio_component *acomp = dev_priv->audio_component; struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); enum port port = intel_dig_port->port; @@ -625,17 +625,11 @@ static void i915_audio_component_codec_wake_override(struct device *dev, static int i915_audio_component_get_cdclk_freq(struct device *dev) { struct drm_i915_private *dev_priv = dev_to_i915(dev); - int ret; if (WARN_ON_ONCE(!HAS_DDI(dev_priv))) return -ENODEV; - intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); - ret = dev_priv->display.get_display_clock_speed(dev_priv->dev); - - intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO); - - return ret; + return dev_priv->cdclk_freq; } static int i915_audio_component_sync_audio_rate(struct device *dev, @@ -761,14 +755,14 @@ static int i915_audio_component_bind(struct device *i915_dev, if (WARN_ON(acomp->ops || acomp->dev)) return -EEXIST; - drm_modeset_lock_all(dev_priv->dev); + drm_modeset_lock_all(&dev_priv->drm); acomp->ops = &i915_audio_component_ops; acomp->dev = i915_dev; BUILD_BUG_ON(MAX_PORTS != I915_MAX_PORTS); for (i = 0; i < ARRAY_SIZE(acomp->aud_sample_rate); i++) acomp->aud_sample_rate[i] = 0; dev_priv->audio_component = acomp; - drm_modeset_unlock_all(dev_priv->dev); + drm_modeset_unlock_all(&dev_priv->drm); return 0; } @@ -779,11 +773,11 @@ static void i915_audio_component_unbind(struct device *i915_dev, struct i915_audio_component *acomp = data; struct drm_i915_private *dev_priv = dev_to_i915(i915_dev); - drm_modeset_lock_all(dev_priv->dev); + drm_modeset_lock_all(&dev_priv->drm); acomp->ops = NULL; acomp->dev = NULL; dev_priv->audio_component = NULL; - drm_modeset_unlock_all(dev_priv->dev); + drm_modeset_unlock_all(&dev_priv->drm); } static const struct component_ops i915_audio_component_bind_ops = { @@ -811,7 +805,7 @@ void i915_audio_component_init(struct drm_i915_private *dev_priv) { int ret; - ret = component_add(dev_priv->dev->dev, &i915_audio_component_bind_ops); + ret = component_add(dev_priv->drm.dev, &i915_audio_component_bind_ops); if (ret < 0) { DRM_ERROR("failed to add audio component (%d)\n", ret); /* continue with reduced functionality */ @@ -833,6 +827,6 @@ void i915_audio_component_cleanup(struct drm_i915_private *dev_priv) if (!dev_priv->audio_component_registered) return; - component_del(dev_priv->dev->dev, &i915_audio_component_bind_ops); + component_del(dev_priv->drm.dev, &i915_audio_component_bind_ops); dev_priv->audio_component_registered = false; } diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c index b9022fa05..c6e69e4cf 100644 --- a/drivers/gpu/drm/i915/intel_bios.c +++ b/drivers/gpu/drm/i915/intel_bios.c @@ -218,7 +218,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv, dev_priv->vbt.lvds_dither = lvds_options->pixel_dither; - ret = intel_opregion_get_panel_type(dev_priv->dev); + ret = intel_opregion_get_panel_type(dev_priv); if (ret >= 0) { WARN_ON(ret > 0xf); panel_type = ret; @@ -323,6 +323,15 @@ parse_lfp_backlight(struct drm_i915_private *dev_priv, return; } + dev_priv->vbt.backlight.type = INTEL_BACKLIGHT_DISPLAY_DDI; + if (bdb->version >= 191 && + get_blocksize(backlight_data) >= sizeof(*backlight_data)) { + const struct bdb_lfp_backlight_control_method *method; + + method = &backlight_data->backlight_control[panel_type]; + dev_priv->vbt.backlight.type = method->type; + } + dev_priv->vbt.backlight.pwm_freq_hz = entry->pwm_freq_hz; dev_priv->vbt.backlight.active_low_pwm = entry->active_low_pwm; dev_priv->vbt.backlight.min_brightness = entry->min_brightness; @@ -768,6 +777,16 @@ parse_mipi_config(struct drm_i915_private *dev_priv, return; } + /* + * These fields are introduced from the VBT version 197 onwards, + * so making sure that these bits are set zero in the previous + * versions. + */ + if (dev_priv->vbt.dsi.config->dual_link && bdb->version < 197) { + dev_priv->vbt.dsi.config->dl_dcs_cabc_ports = 0; + dev_priv->vbt.dsi.config->dl_dcs_backlight_ports = 0; + } + /* We have mandatory mipi config blocks. Initialize as generic panel */ dev_priv->vbt.dsi.panel_id = MIPI_DSI_GENERIC_PANEL_ID; } @@ -1407,7 +1426,7 @@ static const struct vbt_header *find_vbt(void __iomem *bios, size_t size) int intel_bios_init(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev_priv->dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; const struct vbt_header *vbt = dev_priv->opregion.vbt; const struct bdb_header *bdb; u8 __iomem *bios = NULL; diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h index ab0ea315e..8405b5a36 100644 --- a/drivers/gpu/drm/i915/intel_bios.h +++ b/drivers/gpu/drm/i915/intel_bios.h @@ -30,6 +30,14 @@ #ifndef _INTEL_BIOS_H_ #define _INTEL_BIOS_H_ +enum intel_backlight_type { + INTEL_BACKLIGHT_PMIC, + INTEL_BACKLIGHT_LPSS, + INTEL_BACKLIGHT_DISPLAY_DDI, + INTEL_BACKLIGHT_DSI_DCS, + INTEL_BACKLIGHT_PANEL_DRIVER_INTERFACE, +}; + struct edp_power_seq { u16 t1_t3; u16 t8; @@ -113,7 +121,13 @@ struct mipi_config { u16 dual_link:2; u16 lane_cnt:2; u16 pixel_overlap:3; - u16 rsvd3:9; + u16 rgb_flip:1; +#define DL_DCS_PORT_A 0x00 +#define DL_DCS_PORT_C 0x01 +#define DL_DCS_PORT_A_AND_C 0x02 + u16 dl_dcs_cabc_ports:2; + u16 dl_dcs_backlight_ports:2; + u16 rsvd3:4; u16 rsvd4; diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c new file mode 100644 index 000000000..b074f3d6d --- /dev/null +++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c @@ -0,0 +1,595 @@ +/* + * Copyright © 2015 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include <linux/kthread.h> + +#include "i915_drv.h" + +static void intel_breadcrumbs_fake_irq(unsigned long data) +{ + struct intel_engine_cs *engine = (struct intel_engine_cs *)data; + + /* + * The timer persists in case we cannot enable interrupts, + * or if we have previously seen seqno/interrupt incoherency + * ("missed interrupt" syndrome). Here the worker will wake up + * every jiffie in order to kick the oldest waiter to do the + * coherent seqno check. + */ + rcu_read_lock(); + if (intel_engine_wakeup(engine)) + mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1); + rcu_read_unlock(); +} + +static void irq_enable(struct intel_engine_cs *engine) +{ + /* Enabling the IRQ may miss the generation of the interrupt, but + * we still need to force the barrier before reading the seqno, + * just in case. + */ + engine->breadcrumbs.irq_posted = true; + + spin_lock_irq(&engine->i915->irq_lock); + engine->irq_enable(engine); + spin_unlock_irq(&engine->i915->irq_lock); +} + +static void irq_disable(struct intel_engine_cs *engine) +{ + spin_lock_irq(&engine->i915->irq_lock); + engine->irq_disable(engine); + spin_unlock_irq(&engine->i915->irq_lock); + + engine->breadcrumbs.irq_posted = false; +} + +static void __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b) +{ + struct intel_engine_cs *engine = + container_of(b, struct intel_engine_cs, breadcrumbs); + struct drm_i915_private *i915 = engine->i915; + + assert_spin_locked(&b->lock); + if (b->rpm_wakelock) + return; + + /* Since we are waiting on a request, the GPU should be busy + * and should have its own rpm reference. For completeness, + * record an rpm reference for ourselves to cover the + * interrupt we unmask. + */ + intel_runtime_pm_get_noresume(i915); + b->rpm_wakelock = true; + + /* No interrupts? Kick the waiter every jiffie! */ + if (intel_irqs_enabled(i915)) { + if (!test_bit(engine->id, &i915->gpu_error.test_irq_rings)) + irq_enable(engine); + b->irq_enabled = true; + } + + if (!b->irq_enabled || + test_bit(engine->id, &i915->gpu_error.missed_irq_rings)) + mod_timer(&b->fake_irq, jiffies + 1); + + /* Ensure that even if the GPU hangs, we get woken up. + * + * However, note that if no one is waiting, we never notice + * a gpu hang. Eventually, we will have to wait for a resource + * held by the GPU and so trigger a hangcheck. In the most + * pathological case, this will be upon memory starvation! + */ + i915_queue_hangcheck(i915); +} + +static void __intel_breadcrumbs_disable_irq(struct intel_breadcrumbs *b) +{ + struct intel_engine_cs *engine = + container_of(b, struct intel_engine_cs, breadcrumbs); + + assert_spin_locked(&b->lock); + if (!b->rpm_wakelock) + return; + + if (b->irq_enabled) { + irq_disable(engine); + b->irq_enabled = false; + } + + intel_runtime_pm_put(engine->i915); + b->rpm_wakelock = false; +} + +static inline struct intel_wait *to_wait(struct rb_node *node) +{ + return container_of(node, struct intel_wait, node); +} + +static inline void __intel_breadcrumbs_finish(struct intel_breadcrumbs *b, + struct intel_wait *wait) +{ + assert_spin_locked(&b->lock); + + /* This request is completed, so remove it from the tree, mark it as + * complete, and *then* wake up the associated task. + */ + rb_erase(&wait->node, &b->waiters); + RB_CLEAR_NODE(&wait->node); + + wake_up_process(wait->tsk); /* implicit smp_wmb() */ +} + +static bool __intel_engine_add_wait(struct intel_engine_cs *engine, + struct intel_wait *wait) +{ + struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct rb_node **p, *parent, *completed; + bool first; + u32 seqno; + + /* Insert the request into the retirement ordered list + * of waiters by walking the rbtree. If we are the oldest + * seqno in the tree (the first to be retired), then + * set ourselves as the bottom-half. + * + * As we descend the tree, prune completed branches since we hold the + * spinlock we know that the first_waiter must be delayed and can + * reduce some of the sequential wake up latency if we take action + * ourselves and wake up the completed tasks in parallel. Also, by + * removing stale elements in the tree, we may be able to reduce the + * ping-pong between the old bottom-half and ourselves as first-waiter. + */ + first = true; + parent = NULL; + completed = NULL; + seqno = intel_engine_get_seqno(engine); + + /* If the request completed before we managed to grab the spinlock, + * return now before adding ourselves to the rbtree. We let the + * current bottom-half handle any pending wakeups and instead + * try and get out of the way quickly. + */ + if (i915_seqno_passed(seqno, wait->seqno)) { + RB_CLEAR_NODE(&wait->node); + return first; + } + + p = &b->waiters.rb_node; + while (*p) { + parent = *p; + if (wait->seqno == to_wait(parent)->seqno) { + /* We have multiple waiters on the same seqno, select + * the highest priority task (that with the smallest + * task->prio) to serve as the bottom-half for this + * group. + */ + if (wait->tsk->prio > to_wait(parent)->tsk->prio) { + p = &parent->rb_right; + first = false; + } else { + p = &parent->rb_left; + } + } else if (i915_seqno_passed(wait->seqno, + to_wait(parent)->seqno)) { + p = &parent->rb_right; + if (i915_seqno_passed(seqno, to_wait(parent)->seqno)) + completed = parent; + else + first = false; + } else { + p = &parent->rb_left; + } + } + rb_link_node(&wait->node, parent, p); + rb_insert_color(&wait->node, &b->waiters); + GEM_BUG_ON(!first && !b->irq_seqno_bh); + + if (completed) { + struct rb_node *next = rb_next(completed); + + GEM_BUG_ON(!next && !first); + if (next && next != &wait->node) { + GEM_BUG_ON(first); + b->first_wait = to_wait(next); + smp_store_mb(b->irq_seqno_bh, b->first_wait->tsk); + /* As there is a delay between reading the current + * seqno, processing the completed tasks and selecting + * the next waiter, we may have missed the interrupt + * and so need for the next bottom-half to wakeup. + * + * Also as we enable the IRQ, we may miss the + * interrupt for that seqno, so we have to wake up + * the next bottom-half in order to do a coherent check + * in case the seqno passed. + */ + __intel_breadcrumbs_enable_irq(b); + if (READ_ONCE(b->irq_posted)) + wake_up_process(to_wait(next)->tsk); + } + + do { + struct intel_wait *crumb = to_wait(completed); + completed = rb_prev(completed); + __intel_breadcrumbs_finish(b, crumb); + } while (completed); + } + + if (first) { + GEM_BUG_ON(rb_first(&b->waiters) != &wait->node); + b->first_wait = wait; + smp_store_mb(b->irq_seqno_bh, wait->tsk); + /* After assigning ourselves as the new bottom-half, we must + * perform a cursory check to prevent a missed interrupt. + * Either we miss the interrupt whilst programming the hardware, + * or if there was a previous waiter (for a later seqno) they + * may be woken instead of us (due to the inherent race + * in the unlocked read of b->irq_seqno_bh in the irq handler) + * and so we miss the wake up. + */ + __intel_breadcrumbs_enable_irq(b); + } + GEM_BUG_ON(!b->irq_seqno_bh); + GEM_BUG_ON(!b->first_wait); + GEM_BUG_ON(rb_first(&b->waiters) != &b->first_wait->node); + + return first; +} + +bool intel_engine_add_wait(struct intel_engine_cs *engine, + struct intel_wait *wait) +{ + struct intel_breadcrumbs *b = &engine->breadcrumbs; + bool first; + + spin_lock(&b->lock); + first = __intel_engine_add_wait(engine, wait); + spin_unlock(&b->lock); + + return first; +} + +void intel_engine_enable_fake_irq(struct intel_engine_cs *engine) +{ + mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1); +} + +static inline bool chain_wakeup(struct rb_node *rb, int priority) +{ + return rb && to_wait(rb)->tsk->prio <= priority; +} + +static inline int wakeup_priority(struct intel_breadcrumbs *b, + struct task_struct *tsk) +{ + if (tsk == b->signaler) + return INT_MIN; + else + return tsk->prio; +} + +void intel_engine_remove_wait(struct intel_engine_cs *engine, + struct intel_wait *wait) +{ + struct intel_breadcrumbs *b = &engine->breadcrumbs; + + /* Quick check to see if this waiter was already decoupled from + * the tree by the bottom-half to avoid contention on the spinlock + * by the herd. + */ + if (RB_EMPTY_NODE(&wait->node)) + return; + + spin_lock(&b->lock); + + if (RB_EMPTY_NODE(&wait->node)) + goto out_unlock; + + if (b->first_wait == wait) { + const int priority = wakeup_priority(b, wait->tsk); + struct rb_node *next; + + GEM_BUG_ON(b->irq_seqno_bh != wait->tsk); + + /* We are the current bottom-half. Find the next candidate, + * the first waiter in the queue on the remaining oldest + * request. As multiple seqnos may complete in the time it + * takes us to wake up and find the next waiter, we have to + * wake up that waiter for it to perform its own coherent + * completion check. + */ + next = rb_next(&wait->node); + if (chain_wakeup(next, priority)) { + /* If the next waiter is already complete, + * wake it up and continue onto the next waiter. So + * if have a small herd, they will wake up in parallel + * rather than sequentially, which should reduce + * the overall latency in waking all the completed + * clients. + * + * However, waking up a chain adds extra latency to + * the first_waiter. This is undesirable if that + * waiter is a high priority task. + */ + u32 seqno = intel_engine_get_seqno(engine); + + while (i915_seqno_passed(seqno, to_wait(next)->seqno)) { + struct rb_node *n = rb_next(next); + + __intel_breadcrumbs_finish(b, to_wait(next)); + next = n; + if (!chain_wakeup(next, priority)) + break; + } + } + + if (next) { + /* In our haste, we may have completed the first waiter + * before we enabled the interrupt. Do so now as we + * have a second waiter for a future seqno. Afterwards, + * we have to wake up that waiter in case we missed + * the interrupt, or if we have to handle an + * exception rather than a seqno completion. + */ + b->first_wait = to_wait(next); + smp_store_mb(b->irq_seqno_bh, b->first_wait->tsk); + if (b->first_wait->seqno != wait->seqno) + __intel_breadcrumbs_enable_irq(b); + wake_up_process(b->irq_seqno_bh); + } else { + b->first_wait = NULL; + WRITE_ONCE(b->irq_seqno_bh, NULL); + __intel_breadcrumbs_disable_irq(b); + } + } else { + GEM_BUG_ON(rb_first(&b->waiters) == &wait->node); + } + + GEM_BUG_ON(RB_EMPTY_NODE(&wait->node)); + rb_erase(&wait->node, &b->waiters); + +out_unlock: + GEM_BUG_ON(b->first_wait == wait); + GEM_BUG_ON(rb_first(&b->waiters) != + (b->first_wait ? &b->first_wait->node : NULL)); + GEM_BUG_ON(!b->irq_seqno_bh ^ RB_EMPTY_ROOT(&b->waiters)); + spin_unlock(&b->lock); +} + +static bool signal_complete(struct drm_i915_gem_request *request) +{ + if (!request) + return false; + + /* If another process served as the bottom-half it may have already + * signalled that this wait is already completed. + */ + if (intel_wait_complete(&request->signaling.wait)) + return true; + + /* Carefully check if the request is complete, giving time for the + * seqno to be visible or if the GPU hung. + */ + if (__i915_request_irq_complete(request)) + return true; + + return false; +} + +static struct drm_i915_gem_request *to_signaler(struct rb_node *rb) +{ + return container_of(rb, struct drm_i915_gem_request, signaling.node); +} + +static void signaler_set_rtpriority(void) +{ + struct sched_param param = { .sched_priority = 1 }; + + sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m); +} + +static int intel_breadcrumbs_signaler(void *arg) +{ + struct intel_engine_cs *engine = arg; + struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct drm_i915_gem_request *request; + + /* Install ourselves with high priority to reduce signalling latency */ + signaler_set_rtpriority(); + + do { + set_current_state(TASK_INTERRUPTIBLE); + + /* We are either woken up by the interrupt bottom-half, + * or by a client adding a new signaller. In both cases, + * the GPU seqno may have advanced beyond our oldest signal. + * If it has, propagate the signal, remove the waiter and + * check again with the next oldest signal. Otherwise we + * need to wait for a new interrupt from the GPU or for + * a new client. + */ + request = READ_ONCE(b->first_signal); + if (signal_complete(request)) { + /* Wake up all other completed waiters and select the + * next bottom-half for the next user interrupt. + */ + intel_engine_remove_wait(engine, + &request->signaling.wait); + + /* Find the next oldest signal. Note that as we have + * not been holding the lock, another client may + * have installed an even older signal than the one + * we just completed - so double check we are still + * the oldest before picking the next one. + */ + spin_lock(&b->lock); + if (request == b->first_signal) { + struct rb_node *rb = + rb_next(&request->signaling.node); + b->first_signal = rb ? to_signaler(rb) : NULL; + } + rb_erase(&request->signaling.node, &b->signals); + spin_unlock(&b->lock); + + i915_gem_request_unreference(request); + } else { + if (kthread_should_stop()) + break; + + schedule(); + } + } while (1); + __set_current_state(TASK_RUNNING); + + return 0; +} + +void intel_engine_enable_signaling(struct drm_i915_gem_request *request) +{ + struct intel_engine_cs *engine = request->engine; + struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct rb_node *parent, **p; + bool first, wakeup; + + if (unlikely(READ_ONCE(request->signaling.wait.tsk))) + return; + + spin_lock(&b->lock); + if (unlikely(request->signaling.wait.tsk)) { + wakeup = false; + goto unlock; + } + + request->signaling.wait.tsk = b->signaler; + request->signaling.wait.seqno = request->seqno; + i915_gem_request_reference(request); + + /* First add ourselves into the list of waiters, but register our + * bottom-half as the signaller thread. As per usual, only the oldest + * waiter (not just signaller) is tasked as the bottom-half waking + * up all completed waiters after the user interrupt. + * + * If we are the oldest waiter, enable the irq (after which we + * must double check that the seqno did not complete). + */ + wakeup = __intel_engine_add_wait(engine, &request->signaling.wait); + + /* Now insert ourselves into the retirement ordered list of signals + * on this engine. We track the oldest seqno as that will be the + * first signal to complete. + */ + parent = NULL; + first = true; + p = &b->signals.rb_node; + while (*p) { + parent = *p; + if (i915_seqno_passed(request->seqno, + to_signaler(parent)->seqno)) { + p = &parent->rb_right; + first = false; + } else { + p = &parent->rb_left; + } + } + rb_link_node(&request->signaling.node, parent, p); + rb_insert_color(&request->signaling.node, &b->signals); + if (first) + smp_store_mb(b->first_signal, request); + +unlock: + spin_unlock(&b->lock); + + if (wakeup) + wake_up_process(b->signaler); +} + +int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) +{ + struct intel_breadcrumbs *b = &engine->breadcrumbs; + struct task_struct *tsk; + + spin_lock_init(&b->lock); + setup_timer(&b->fake_irq, + intel_breadcrumbs_fake_irq, + (unsigned long)engine); + + /* Spawn a thread to provide a common bottom-half for all signals. + * As this is an asynchronous interface we cannot steal the current + * task for handling the bottom-half to the user interrupt, therefore + * we create a thread to do the coherent seqno dance after the + * interrupt and then signal the waitqueue (via the dma-buf/fence). + */ + tsk = kthread_run(intel_breadcrumbs_signaler, engine, + "i915/signal:%d", engine->id); + if (IS_ERR(tsk)) + return PTR_ERR(tsk); + + b->signaler = tsk; + + return 0; +} + +void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) +{ + struct intel_breadcrumbs *b = &engine->breadcrumbs; + + if (!IS_ERR_OR_NULL(b->signaler)) + kthread_stop(b->signaler); + + del_timer_sync(&b->fake_irq); +} + +unsigned int intel_kick_waiters(struct drm_i915_private *i915) +{ + struct intel_engine_cs *engine; + unsigned int mask = 0; + + /* To avoid the task_struct disappearing beneath us as we wake up + * the process, we must first inspect the task_struct->state under the + * RCU lock, i.e. as we call wake_up_process() we must be holding the + * rcu_read_lock(). + */ + rcu_read_lock(); + for_each_engine(engine, i915) + if (unlikely(intel_engine_wakeup(engine))) + mask |= intel_engine_flag(engine); + rcu_read_unlock(); + + return mask; +} + +unsigned int intel_kick_signalers(struct drm_i915_private *i915) +{ + struct intel_engine_cs *engine; + unsigned int mask = 0; + + for_each_engine(engine, i915) { + if (unlikely(READ_ONCE(engine->breadcrumbs.first_signal))) { + wake_up_process(engine->breadcrumbs.signaler); + mask |= intel_engine_flag(engine); + } + } + + return mask; +} diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c index 1b3f97449..bc0fef3d3 100644 --- a/drivers/gpu/drm/i915/intel_color.c +++ b/drivers/gpu/drm/i915/intel_color.c @@ -96,7 +96,7 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state) { struct drm_crtc *crtc = crtc_state->crtc; struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int i, pipe = intel_crtc->pipe; uint16_t coeffs[9] = { 0, }; @@ -207,7 +207,7 @@ static void cherryview_load_csc_matrix(struct drm_crtc_state *state) { struct drm_crtc *crtc = state->crtc; struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = to_intel_crtc(crtc)->pipe; uint32_t mode; @@ -255,7 +255,7 @@ static void cherryview_load_csc_matrix(struct drm_crtc_state *state) void intel_color_set_csc(struct drm_crtc_state *crtc_state) { struct drm_device *dev = crtc_state->crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (dev_priv->display.load_csc_matrix) dev_priv->display.load_csc_matrix(crtc_state); @@ -266,13 +266,13 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc, struct drm_property_blob *blob) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum pipe pipe = intel_crtc->pipe; int i; if (HAS_GMCH_DISPLAY(dev)) { - if (intel_crtc->config->has_dsi_encoder) + if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI)) assert_dsi_pll_enabled(dev_priv); else assert_pll_enabled(dev_priv, pipe); @@ -313,7 +313,7 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state) { struct drm_crtc *crtc = crtc_state->crtc; struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_crtc_state *intel_crtc_state = to_intel_crtc_state(crtc_state); @@ -343,7 +343,7 @@ static void broadwell_load_luts(struct drm_crtc_state *state) { struct drm_crtc *crtc = state->crtc; struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc_state *intel_state = to_intel_crtc_state(state); enum pipe pipe = to_intel_crtc(crtc)->pipe; uint32_t i, lut_size = INTEL_INFO(dev)->color.degamma_lut_size; @@ -426,7 +426,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state) { struct drm_crtc *crtc = state->crtc; struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = to_intel_crtc(crtc)->pipe; struct drm_color_lut *lut; uint32_t i, lut_size; @@ -485,7 +485,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state) void intel_color_load_luts(struct drm_crtc_state *crtc_state) { struct drm_device *dev = crtc_state->crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); dev_priv->display.load_luts(crtc_state); } @@ -526,7 +526,7 @@ int intel_color_check(struct drm_crtc *crtc, void intel_color_init(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); drm_mode_crtc_set_gamma_size(crtc, 256); @@ -547,7 +547,8 @@ void intel_color_init(struct drm_crtc *crtc) /* Enable color management support when we have degamma & gamma LUTs. */ if (INTEL_INFO(dev)->color.degamma_lut_size != 0 && INTEL_INFO(dev)->color.gamma_lut_size != 0) - drm_helper_crtc_enable_color_mgmt(crtc, + drm_crtc_enable_color_mgmt(crtc, INTEL_INFO(dev)->color.degamma_lut_size, + true, INTEL_INFO(dev)->color.gamma_lut_size); } diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c index a3f87d668..827b6ef4e 100644 --- a/drivers/gpu/drm/i915/intel_crt.c +++ b/drivers/gpu/drm/i915/intel_crt.c @@ -67,7 +67,7 @@ static bool intel_crt_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crt *crt = intel_encoder_to_crt(encoder); enum intel_display_power_domain power_domain; u32 tmp; @@ -98,7 +98,7 @@ out: static unsigned int intel_crt_get_flags(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crt *crt = intel_encoder_to_crt(encoder); u32 tmp, flags = 0; @@ -146,7 +146,7 @@ static void hsw_crt_get_config(struct intel_encoder *encoder, static void intel_crt_set_dpms(struct intel_encoder *encoder, int mode) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crt *crt = intel_encoder_to_crt(encoder); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; @@ -281,7 +281,7 @@ static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector) { struct drm_device *dev = connector->dev; struct intel_crt *crt = intel_attached_crt(connector); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 adpa; bool ret; @@ -301,8 +301,10 @@ static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector) I915_WRITE(crt->adpa_reg, adpa); - if (wait_for((I915_READ(crt->adpa_reg) & ADPA_CRT_HOTPLUG_FORCE_TRIGGER) == 0, - 1000)) + if (intel_wait_for_register(dev_priv, + crt->adpa_reg, + ADPA_CRT_HOTPLUG_FORCE_TRIGGER, 0, + 1000)) DRM_DEBUG_KMS("timed out waiting for FORCE_TRIGGER"); if (turn_off_dac) { @@ -326,7 +328,7 @@ static bool valleyview_crt_detect_hotplug(struct drm_connector *connector) { struct drm_device *dev = connector->dev; struct intel_crt *crt = intel_attached_crt(connector); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); bool reenable_hpd; u32 adpa; bool ret; @@ -353,8 +355,10 @@ static bool valleyview_crt_detect_hotplug(struct drm_connector *connector) I915_WRITE(crt->adpa_reg, adpa); - if (wait_for((I915_READ(crt->adpa_reg) & ADPA_CRT_HOTPLUG_FORCE_TRIGGER) == 0, - 1000)) { + if (intel_wait_for_register(dev_priv, + crt->adpa_reg, + ADPA_CRT_HOTPLUG_FORCE_TRIGGER, 0, + 1000)) { DRM_DEBUG_KMS("timed out waiting for FORCE_TRIGGER"); I915_WRITE(crt->adpa_reg, save_adpa); } @@ -385,7 +389,7 @@ static bool valleyview_crt_detect_hotplug(struct drm_connector *connector) static bool intel_crt_detect_hotplug(struct drm_connector *connector) { struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 stat; bool ret = false; int i, tries = 0; @@ -412,9 +416,9 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector) CRT_HOTPLUG_FORCE_DETECT, CRT_HOTPLUG_FORCE_DETECT); /* wait for FORCE_DETECT to go off */ - if (wait_for((I915_READ(PORT_HOTPLUG_EN) & - CRT_HOTPLUG_FORCE_DETECT) == 0, - 1000)) + if (intel_wait_for_register(dev_priv, PORT_HOTPLUG_EN, + CRT_HOTPLUG_FORCE_DETECT, 0, + 1000)) DRM_DEBUG_KMS("timed out waiting for FORCE_DETECT to go off"); } @@ -467,7 +471,7 @@ static int intel_crt_ddc_get_modes(struct drm_connector *connector, static bool intel_crt_detect_ddc(struct drm_connector *connector) { struct intel_crt *crt = intel_attached_crt(connector); - struct drm_i915_private *dev_priv = crt->base.base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crt->base.base.dev); struct edid *edid; struct i2c_adapter *i2c; @@ -503,7 +507,7 @@ static enum drm_connector_status intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe) { struct drm_device *dev = crt->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t save_bclrpat; uint32_t save_vtotal; uint32_t vtotal, vactive; @@ -618,7 +622,7 @@ static enum drm_connector_status intel_crt_detect(struct drm_connector *connector, bool force) { struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crt *crt = intel_attached_crt(connector); struct intel_encoder *intel_encoder = &crt->base; enum intel_display_power_domain power_domain; @@ -699,7 +703,7 @@ static void intel_crt_destroy(struct drm_connector *connector) static int intel_crt_get_modes(struct drm_connector *connector) { struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crt *crt = intel_attached_crt(connector); struct intel_encoder *intel_encoder = &crt->base; enum intel_display_power_domain power_domain; @@ -734,7 +738,7 @@ static int intel_crt_set_property(struct drm_connector *connector, void intel_crt_reset(struct drm_encoder *encoder) { struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crt *crt = intel_encoder_to_crt(to_intel_encoder(encoder)); if (INTEL_INFO(dev)->gen >= 5) { @@ -760,6 +764,8 @@ static const struct drm_connector_funcs intel_crt_connector_funcs = { .dpms = drm_atomic_helper_connector_dpms, .detect = intel_crt_detect, .fill_modes = drm_helper_probe_single_connector_modes, + .late_register = intel_connector_register, + .early_unregister = intel_connector_unregister, .destroy = intel_crt_destroy, .set_property = intel_crt_set_property, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, @@ -770,7 +776,6 @@ static const struct drm_connector_funcs intel_crt_connector_funcs = { static const struct drm_connector_helper_funcs intel_crt_connector_helper_funcs = { .mode_valid = intel_crt_mode_valid, .get_modes = intel_crt_get_modes, - .best_encoder = intel_best_encoder, }; static const struct drm_encoder_funcs intel_crt_enc_funcs = { @@ -809,7 +814,7 @@ void intel_crt_init(struct drm_device *dev) struct drm_connector *connector; struct intel_crt *crt; struct intel_connector *intel_connector; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t adpa_reg; u32 adpa; @@ -857,7 +862,7 @@ void intel_crt_init(struct drm_device *dev) &intel_crt_connector_funcs, DRM_MODE_CONNECTOR_VGA); drm_encoder_init(dev, &crt->base.base, &intel_crt_enc_funcs, - DRM_MODE_ENCODER_DAC, NULL); + DRM_MODE_ENCODER_DAC, "CRT"); intel_connector_attach_encoder(intel_connector, &crt->base); @@ -894,12 +899,9 @@ void intel_crt_init(struct drm_device *dev) crt->base.get_hw_state = intel_crt_get_hw_state; } intel_connector->get_hw_state = intel_connector_get_hw_state; - intel_connector->unregister = intel_connector_unregister; drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs); - drm_connector_register(connector); - if (!I915_HAS_HOTPLUG(dev)) intel_connector->polled = DRM_CONNECTOR_POLL_CONNECT; diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c index ecadf8709..40e9c3c3c 100644 --- a/drivers/gpu/drm/i915/intel_csr.c +++ b/drivers/gpu/drm/i915/intel_csr.c @@ -47,7 +47,7 @@ #define I915_CSR_SKL "/*(DEBLOBBED)*/" /*(DEBLOBBED)*/ -#define SKL_CSR_VERSION_REQUIRED CSR_VERSION(1, 23) +#define SKL_CSR_VERSION_REQUIRED CSR_VERSION(1, 26) #define I915_CSR_BXT "/*(DEBLOBBED)*/" /*(DEBLOBBED)*/ @@ -286,7 +286,7 @@ static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv, uint32_t dmc_offset = CSR_DEFAULT_FW_OFFSET, readcount = 0, nbytes; uint32_t i; uint32_t *dmc_payload; - uint32_t required_min_version; + uint32_t required_version; if (!fw) return NULL; @@ -303,24 +303,23 @@ static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv, csr->version = css_header->version; if (IS_KABYLAKE(dev_priv)) { - required_min_version = KBL_CSR_VERSION_REQUIRED; + required_version = KBL_CSR_VERSION_REQUIRED; } else if (IS_SKYLAKE(dev_priv)) { - required_min_version = SKL_CSR_VERSION_REQUIRED; + required_version = SKL_CSR_VERSION_REQUIRED; } else if (IS_BROXTON(dev_priv)) { - required_min_version = BXT_CSR_VERSION_REQUIRED; + required_version = BXT_CSR_VERSION_REQUIRED; } else { MISSING_CASE(INTEL_REVID(dev_priv)); - required_min_version = 0; + required_version = 0; } - if (csr->version < required_min_version) { - DRM_INFO("Refusing to load old DMC firmware v%u.%u," - " please upgrade to v%u.%u or later" - " [" FIRMWARE_URL "].\n", + if (csr->version != required_version) { + DRM_INFO("Refusing to load DMC firmware v%u.%u," + " please use v%u.%u [" FIRMWARE_URL "].\n", CSR_VERSION_MAJOR(csr->version), CSR_VERSION_MINOR(csr->version), - CSR_VERSION_MAJOR(required_min_version), - CSR_VERSION_MINOR(required_min_version)); + CSR_VERSION_MAJOR(required_version), + CSR_VERSION_MINOR(required_version)); return NULL; } @@ -413,7 +412,7 @@ static void csr_load_work_fn(struct work_struct *work) csr = &dev_priv->csr; ret = reject_firmware(&fw, dev_priv->csr.fw_path, - &dev_priv->dev->pdev->dev); + &dev_priv->drm.pdev->dev); if (fw) dev_priv->csr.dmc_payload = parse_csr_fw(dev_priv, fw); @@ -427,7 +426,7 @@ static void csr_load_work_fn(struct work_struct *work) CSR_VERSION_MAJOR(csr->version), CSR_VERSION_MINOR(csr->version)); } else { - dev_notice(dev_priv->dev->dev, + dev_notice(dev_priv->drm.dev, "Failed to load DMC firmware" " [" FIRMWARE_URL "]," " disabling runtime power management.\n"); diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c index 12c4f4356..1a7efac65 100644 --- a/drivers/gpu/drm/i915/intel_ddi.c +++ b/drivers/gpu/drm/i915/intel_ddi.c @@ -318,7 +318,7 @@ static void ddi_get_encoder_port(struct intel_encoder *intel_encoder, default: WARN(1, "Invalid DDI encoder type %d\n", intel_encoder->type); /* fallthrough and treat as unknown */ - case INTEL_OUTPUT_DISPLAYPORT: + case INTEL_OUTPUT_DP: case INTEL_OUTPUT_EDP: case INTEL_OUTPUT_HDMI: case INTEL_OUTPUT_UNKNOWN: @@ -512,7 +512,7 @@ void intel_prepare_ddi_buffer(struct intel_encoder *encoder) ddi_translations = ddi_translations_edp; size = n_edp_entries; break; - case INTEL_OUTPUT_DISPLAYPORT: + case INTEL_OUTPUT_DP: case INTEL_OUTPUT_HDMI: ddi_translations = ddi_translations_dp; size = n_dp_entries; @@ -568,7 +568,7 @@ static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv, void hsw_fdi_link_train(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; u32 temp, i, rx_ctl_val; @@ -859,7 +859,7 @@ static void ddi_dotclock_get(struct intel_crtc_state *pipe_config) if (pipe_config->has_pch_encoder) dotclock = intel_dotclock_calculate(pipe_config->port_clock, &pipe_config->fdi_m_n); - else if (pipe_config->has_dp_encoder) + else if (intel_crtc_has_dp_encoder(pipe_config)) dotclock = intel_dotclock_calculate(pipe_config->port_clock, &pipe_config->dp_m_n); else if (pipe_config->has_hdmi_sink && pipe_config->pipe_bpp == 36) @@ -876,7 +876,7 @@ static void ddi_dotclock_get(struct intel_crtc_state *pipe_config) static void skl_ddi_clock_get(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); int link_clock = 0; uint32_t dpll_ctl1, dpll; @@ -924,7 +924,7 @@ static void skl_ddi_clock_get(struct intel_encoder *encoder, static void hsw_ddi_clock_get(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); int link_clock = 0; u32 val, pll; @@ -973,7 +973,7 @@ static int bxt_calc_pll_link(struct drm_i915_private *dev_priv, { struct intel_shared_dpll *pll; struct intel_dpll_hw_state *state; - intel_clock_t clock; + struct dpll clock; /* For DDI ports we always use a shared PLL. */ if (WARN_ON(dpll == DPLL_ID_PRIVATE)) @@ -996,7 +996,7 @@ static int bxt_calc_pll_link(struct drm_i915_private *dev_priv, static void bxt_ddi_clock_get(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum port port = intel_ddi_get_encoder_port(encoder); uint32_t dpll = port; @@ -1086,14 +1086,14 @@ bool intel_ddi_pll_select(struct intel_crtc *intel_crtc, void intel_ddi_set_pipe_settings(struct drm_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc); enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; int type = intel_encoder->type; uint32_t temp; - if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP || type == INTEL_OUTPUT_DP_MST) { + if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP || type == INTEL_OUTPUT_DP_MST) { WARN_ON(transcoder_is_dsi(cpu_transcoder)); temp = TRANS_MSA_SYNC_CLK; @@ -1121,7 +1121,7 @@ void intel_ddi_set_vc_payload_alloc(struct drm_crtc *crtc, bool state) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; uint32_t temp; temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder)); @@ -1138,7 +1138,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc) struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc); struct drm_encoder *encoder = &intel_encoder->base; struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = intel_crtc->pipe; enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; enum port port = intel_ddi_get_encoder_port(intel_encoder); @@ -1207,7 +1207,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc) temp |= TRANS_DDI_MODE_SELECT_FDI; temp |= (intel_crtc->config->fdi_lanes - 1) << 1; - } else if (type == INTEL_OUTPUT_DISPLAYPORT || + } else if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); @@ -1248,7 +1248,7 @@ void intel_ddi_disable_transcoder_func(struct drm_i915_private *dev_priv, bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector) { struct drm_device *dev = intel_connector->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *intel_encoder = intel_connector->encoder; int type = intel_connector->base.connector_type; enum port port = intel_ddi_get_encoder_port(intel_encoder); @@ -1310,7 +1310,7 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_ddi_get_encoder_port(encoder); enum intel_display_power_domain power_domain; u32 tmp; @@ -1367,6 +1367,14 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder, DRM_DEBUG_KMS("No pipe for ddi port %c found\n", port_name(port)); out: + if (ret && IS_BROXTON(dev_priv)) { + tmp = I915_READ(BXT_PHY_CTL(port)); + if ((tmp & (BXT_PHY_LANE_POWERDOWN_ACK | + BXT_PHY_LANE_ENABLED)) != BXT_PHY_LANE_ENABLED) + DRM_ERROR("Port %c enabled but PHY powered down? " + "(PHY_CTL %08x)\n", port_name(port), tmp); + } + intel_display_power_put(dev_priv, power_domain); return ret; @@ -1376,7 +1384,7 @@ void intel_ddi_enable_pipe_clock(struct intel_crtc *intel_crtc) { struct drm_crtc *crtc = &intel_crtc->base; struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc); enum port port = intel_ddi_get_encoder_port(intel_encoder); enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; @@ -1388,7 +1396,7 @@ void intel_ddi_enable_pipe_clock(struct intel_crtc *intel_crtc) void intel_ddi_disable_pipe_clock(struct intel_crtc *intel_crtc) { - struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; if (cpu_transcoder != TRANSCODER_EDP) @@ -1425,7 +1433,7 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder, u32 level) dp_iboost = dev_priv->vbt.ddi_port_info[port].dp_boost_level; hdmi_iboost = dev_priv->vbt.ddi_port_info[port].hdmi_boost_level; - if (type == INTEL_OUTPUT_DISPLAYPORT) { + if (type == INTEL_OUTPUT_DP) { if (dp_iboost) { iboost = dp_iboost; } else { @@ -1477,7 +1485,7 @@ static void bxt_ddi_vswing_sequence(struct drm_i915_private *dev_priv, if (type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.low_vswing) { n_entries = ARRAY_SIZE(bxt_ddi_translations_edp); ddi_translations = bxt_ddi_translations_edp; - } else if (type == INTEL_OUTPUT_DISPLAYPORT + } else if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP) { n_entries = ARRAY_SIZE(bxt_ddi_translations_dp); ddi_translations = bxt_ddi_translations_dp; @@ -1651,7 +1659,7 @@ static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder) intel_ddi_clk_select(intel_encoder, crtc->config); - if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) { + if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); intel_dp_set_link_params(intel_dp, crtc->config); @@ -1679,7 +1687,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder) { struct drm_encoder *encoder = &intel_encoder->base; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_ddi_get_encoder_port(intel_encoder); int type = intel_encoder->type; uint32_t val; @@ -1700,7 +1708,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder) if (wait) intel_wait_ddi_buf_idle(dev_priv, port); - if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) { + if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF); intel_edp_panel_vdd_on(intel_dp); @@ -1726,7 +1734,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder) struct drm_crtc *crtc = encoder->crtc; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_ddi_get_encoder_port(intel_encoder); int type = intel_encoder->type; @@ -1765,7 +1773,7 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder) struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int type = intel_encoder->type; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (intel_crtc->config->has_audio) { intel_audio_codec_disable(intel_encoder); @@ -1781,9 +1789,11 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder) } } -static bool broxton_phy_is_enabled(struct drm_i915_private *dev_priv, - enum dpio_phy phy) +bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, + enum dpio_phy phy) { + enum port port; + if (!(I915_READ(BXT_P_CR_GT_DISP_PWRON) & GT_DISPLAY_POWER_ON(phy))) return false; @@ -1809,38 +1819,51 @@ static bool broxton_phy_is_enabled(struct drm_i915_private *dev_priv, return false; } + for_each_port_masked(port, + phy == DPIO_PHY0 ? BIT(PORT_B) | BIT(PORT_C) : + BIT(PORT_A)) { + u32 tmp = I915_READ(BXT_PHY_CTL(port)); + + if (tmp & BXT_PHY_CMNLANE_POWERDOWN_ACK) { + DRM_DEBUG_DRIVER("DDI PHY %d powered, but common lane " + "for port %c powered down " + "(PHY_CTL %08x)\n", + phy, port_name(port), tmp); + + return false; + } + } + return true; } -static u32 broxton_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy) +static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy) { u32 val = I915_READ(BXT_PORT_REF_DW6(phy)); return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT; } -static void broxton_phy_wait_grc_done(struct drm_i915_private *dev_priv, - enum dpio_phy phy) +static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv, + enum dpio_phy phy) { - if (wait_for(I915_READ(BXT_PORT_REF_DW3(phy)) & GRC_DONE, 10)) + if (intel_wait_for_register(dev_priv, + BXT_PORT_REF_DW3(phy), + GRC_DONE, GRC_DONE, + 10)) DRM_ERROR("timeout waiting for PHY%d GRC\n", phy); } -static bool broxton_phy_verify_state(struct drm_i915_private *dev_priv, - enum dpio_phy phy); - -static void broxton_phy_init(struct drm_i915_private *dev_priv, - enum dpio_phy phy) +void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy) { - enum port port; - u32 ports, val; + u32 val; - if (broxton_phy_is_enabled(dev_priv, phy)) { + if (bxt_ddi_phy_is_enabled(dev_priv, phy)) { /* Still read out the GRC value for state verification */ if (phy == DPIO_PHY0) - dev_priv->bxt_phy_grc = broxton_get_grc(dev_priv, phy); + dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, phy); - if (broxton_phy_verify_state(dev_priv, phy)) { + if (bxt_ddi_phy_verify_state(dev_priv, phy)) { DRM_DEBUG_DRIVER("DDI PHY %d already enabled, " "won't reprogram it\n", phy); @@ -1849,8 +1872,6 @@ static void broxton_phy_init(struct drm_i915_private *dev_priv, DRM_DEBUG_DRIVER("DDI PHY %d enabled with invalid state, " "force reprogramming it\n", phy); - } else { - DRM_DEBUG_DRIVER("DDI PHY %d not enabled, enabling it\n", phy); } val = I915_READ(BXT_P_CR_GT_DISP_PWRON); @@ -1870,28 +1891,6 @@ static void broxton_phy_init(struct drm_i915_private *dev_priv, DRM_ERROR("timeout during PHY%d power on\n", phy); } - if (phy == DPIO_PHY0) - ports = BIT(PORT_B) | BIT(PORT_C); - else - ports = BIT(PORT_A); - - for_each_port_masked(port, ports) { - int lane; - - for (lane = 0; lane < 4; lane++) { - val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane)); - /* - * Note that on CHV this flag is called UPAR, but has - * the same function. - */ - val &= ~LATENCY_OPTIM; - if (lane != 1) - val |= LATENCY_OPTIM; - - I915_WRITE(BXT_PORT_TX_DW14_LN(port, lane), val); - } - } - /* Program PLL Rcomp code offset */ val = I915_READ(BXT_PORT_CL1CM_DW9(phy)); val &= ~IREF0RC_OFFSET_MASK; @@ -1938,10 +1937,7 @@ static void broxton_phy_init(struct drm_i915_private *dev_priv, * the corresponding calibrated value from PHY1, and disable * the automatic calibration on PHY0. */ - broxton_phy_wait_grc_done(dev_priv, DPIO_PHY1); - - val = dev_priv->bxt_phy_grc = broxton_get_grc(dev_priv, - DPIO_PHY1); + val = dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, DPIO_PHY1); grc_code = val << GRC_CODE_FAST_SHIFT | val << GRC_CODE_SLOW_SHIFT | val; @@ -1951,31 +1947,16 @@ static void broxton_phy_init(struct drm_i915_private *dev_priv, val |= GRC_DIS | GRC_RDY_OVRD; I915_WRITE(BXT_PORT_REF_DW8(DPIO_PHY0), val); } - /* - * During PHY1 init delay waiting for GRC calibration to finish, since - * it can happen in parallel with the subsequent PHY0 init. - */ val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); val |= COMMON_RESET_DIS; I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); -} - -void broxton_ddi_phy_init(struct drm_i915_private *dev_priv) -{ - /* Enable PHY1 first since it provides Rcomp for PHY0 */ - broxton_phy_init(dev_priv, DPIO_PHY1); - broxton_phy_init(dev_priv, DPIO_PHY0); - /* - * If BIOS enabled only PHY0 and not PHY1, we skipped waiting for the - * PHY1 GRC calibration to finish, so wait for it here. - */ - broxton_phy_wait_grc_done(dev_priv, DPIO_PHY1); + if (phy == DPIO_PHY1) + bxt_phy_wait_grc_done(dev_priv, DPIO_PHY1); } -static void broxton_phy_uninit(struct drm_i915_private *dev_priv, - enum dpio_phy phy) +void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy) { uint32_t val; @@ -1988,12 +1969,6 @@ static void broxton_phy_uninit(struct drm_i915_private *dev_priv, I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); } -void broxton_ddi_phy_uninit(struct drm_i915_private *dev_priv) -{ - broxton_phy_uninit(dev_priv, DPIO_PHY1); - broxton_phy_uninit(dev_priv, DPIO_PHY0); -} - static bool __printf(6, 7) __phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy, i915_reg_t reg, u32 mask, u32 expected, @@ -2021,11 +1996,9 @@ __phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy, return false; } -static bool broxton_phy_verify_state(struct drm_i915_private *dev_priv, - enum dpio_phy phy) +bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, + enum dpio_phy phy) { - enum port port; - u32 ports; uint32_t mask; bool ok; @@ -2033,27 +2006,11 @@ static bool broxton_phy_verify_state(struct drm_i915_private *dev_priv, __phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt, \ ## __VA_ARGS__) - /* We expect the PHY to be always enabled */ - if (!broxton_phy_is_enabled(dev_priv, phy)) + if (!bxt_ddi_phy_is_enabled(dev_priv, phy)) return false; ok = true; - if (phy == DPIO_PHY0) - ports = BIT(PORT_B) | BIT(PORT_C); - else - ports = BIT(PORT_A); - - for_each_port_masked(port, ports) { - int lane; - - for (lane = 0; lane < 4; lane++) - ok &= _CHK(BXT_PORT_TX_DW14_LN(port, lane), - LATENCY_OPTIM, - lane != 1 ? LATENCY_OPTIM : 0, - "BXT_PORT_TX_DW14_LN(%d, %d)", port, lane); - } - /* PLL Rcomp code offset */ ok &= _CHK(BXT_PORT_CL1CM_DW9(phy), IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT, @@ -2097,11 +2054,65 @@ static bool broxton_phy_verify_state(struct drm_i915_private *dev_priv, #undef _CHK } -void broxton_ddi_phy_verify_state(struct drm_i915_private *dev_priv) +static uint8_t +bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder, + struct intel_crtc_state *pipe_config) +{ + switch (pipe_config->lane_count) { + case 1: + return 0; + case 2: + return BIT(2) | BIT(0); + case 4: + return BIT(3) | BIT(2) | BIT(0); + default: + MISSING_CASE(pipe_config->lane_count); + + return 0; + } +} + +static void bxt_ddi_pre_pll_enable(struct intel_encoder *encoder) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); + enum port port = dport->port; + struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); + int lane; + + for (lane = 0; lane < 4; lane++) { + u32 val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane)); + + /* + * Note that on CHV this flag is called UPAR, but has + * the same function. + */ + val &= ~LATENCY_OPTIM; + if (intel_crtc->config->lane_lat_optim_mask & BIT(lane)) + val |= LATENCY_OPTIM; + + I915_WRITE(BXT_PORT_TX_DW14_LN(port, lane), val); + } +} + +static uint8_t +bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder) { - if (!broxton_phy_verify_state(dev_priv, DPIO_PHY0) || - !broxton_phy_verify_state(dev_priv, DPIO_PHY1)) - i915_report_error(dev_priv, "DDI PHY state mismatch\n"); + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); + enum port port = dport->port; + int lane; + uint8_t mask; + + mask = 0; + for (lane = 0; lane < 4; lane++) { + u32 val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane)); + + if (val & LATENCY_OPTIM) + mask |= BIT(lane); + } + + return mask; } void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp) @@ -2152,7 +2163,7 @@ void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp) void intel_ddi_fdi_disable(struct drm_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc); uint32_t val; @@ -2185,7 +2196,7 @@ void intel_ddi_fdi_disable(struct drm_crtc *crtc) void intel_ddi_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; struct intel_hdmi *intel_hdmi; @@ -2239,7 +2250,6 @@ void intel_ddi_get_config(struct intel_encoder *encoder, break; case TRANS_DDI_MODE_SELECT_DP_SST: case TRANS_DDI_MODE_SELECT_DP_MST: - pipe_config->has_dp_encoder = true; pipe_config->lane_count = ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1; intel_dp_get_m_n(intel_crtc, pipe_config); @@ -2275,13 +2285,19 @@ void intel_ddi_get_config(struct intel_encoder *encoder, } intel_ddi_clock_get(encoder, pipe_config); + + if (IS_BROXTON(dev_priv)) + pipe_config->lane_lat_optim_mask = + bxt_ddi_phy_get_lane_lat_optim_mask(encoder); } static bool intel_ddi_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); int type = encoder->type; int port = intel_ddi_get_encoder_port(encoder); + int ret; WARN(type == INTEL_OUTPUT_UNKNOWN, "compute_config() on unknown output!\n"); @@ -2289,9 +2305,17 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder, pipe_config->cpu_transcoder = TRANSCODER_EDP; if (type == INTEL_OUTPUT_HDMI) - return intel_hdmi_compute_config(encoder, pipe_config); + ret = intel_hdmi_compute_config(encoder, pipe_config); else - return intel_dp_compute_config(encoder, pipe_config); + ret = intel_dp_compute_config(encoder, pipe_config); + + if (IS_BROXTON(dev_priv) && ret) + pipe_config->lane_lat_optim_mask = + bxt_ddi_phy_calc_lane_lat_optim_mask(encoder, + pipe_config); + + return ret; + } static const struct drm_encoder_funcs intel_ddi_funcs = { @@ -2336,7 +2360,7 @@ intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port) void intel_ddi_init(struct drm_device *dev, enum port port) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_digital_port *intel_dig_port; struct intel_encoder *intel_encoder; struct drm_encoder *encoder; @@ -2386,10 +2410,12 @@ void intel_ddi_init(struct drm_device *dev, enum port port) encoder = &intel_encoder->base; drm_encoder_init(dev, encoder, &intel_ddi_funcs, - DRM_MODE_ENCODER_TMDS, NULL); + DRM_MODE_ENCODER_TMDS, "DDI %c", port_name(port)); intel_encoder->compute_config = intel_ddi_compute_config; intel_encoder->enable = intel_enable_ddi; + if (IS_BROXTON(dev_priv)) + intel_encoder->pre_pll_enable = bxt_ddi_pre_pll_enable; intel_encoder->pre_enable = intel_ddi_pre_enable; intel_encoder->disable = intel_disable_ddi; intel_encoder->post_disable = intel_ddi_post_disable; diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c new file mode 100644 index 000000000..cba137f9a --- /dev/null +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -0,0 +1,388 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include "i915_drv.h" + +void intel_device_info_dump(struct drm_i915_private *dev_priv) +{ + const struct intel_device_info *info = &dev_priv->info; + +#define PRINT_S(name) "%s" +#define SEP_EMPTY +#define PRINT_FLAG(name) info->name ? #name "," : "" +#define SEP_COMMA , + DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x rev=0x%02x flags=" + DEV_INFO_FOR_EACH_FLAG(PRINT_S, SEP_EMPTY), + info->gen, + dev_priv->drm.pdev->device, + dev_priv->drm.pdev->revision, + DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_COMMA)); +#undef PRINT_S +#undef SEP_EMPTY +#undef PRINT_FLAG +#undef SEP_COMMA +} + +static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv) +{ + struct intel_device_info *info = mkwrite_device_info(dev_priv); + u32 fuse, eu_dis; + + fuse = I915_READ(CHV_FUSE_GT); + + info->slice_total = 1; + + if (!(fuse & CHV_FGT_DISABLE_SS0)) { + info->subslice_per_slice++; + eu_dis = fuse & (CHV_FGT_EU_DIS_SS0_R0_MASK | + CHV_FGT_EU_DIS_SS0_R1_MASK); + info->eu_total += 8 - hweight32(eu_dis); + } + + if (!(fuse & CHV_FGT_DISABLE_SS1)) { + info->subslice_per_slice++; + eu_dis = fuse & (CHV_FGT_EU_DIS_SS1_R0_MASK | + CHV_FGT_EU_DIS_SS1_R1_MASK); + info->eu_total += 8 - hweight32(eu_dis); + } + + info->subslice_total = info->subslice_per_slice; + /* + * CHV expected to always have a uniform distribution of EU + * across subslices. + */ + info->eu_per_subslice = info->subslice_total ? + info->eu_total / info->subslice_total : + 0; + /* + * CHV supports subslice power gating on devices with more than + * one subslice, and supports EU power gating on devices with + * more than one EU pair per subslice. + */ + info->has_slice_pg = 0; + info->has_subslice_pg = (info->subslice_total > 1); + info->has_eu_pg = (info->eu_per_subslice > 2); +} + +static void gen9_sseu_info_init(struct drm_i915_private *dev_priv) +{ + struct intel_device_info *info = mkwrite_device_info(dev_priv); + int s_max = 3, ss_max = 4, eu_max = 8; + int s, ss; + u32 fuse2, s_enable, ss_disable, eu_disable; + u8 eu_mask = 0xff; + + fuse2 = I915_READ(GEN8_FUSE2); + s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT; + ss_disable = (fuse2 & GEN9_F2_SS_DIS_MASK) >> GEN9_F2_SS_DIS_SHIFT; + + info->slice_total = hweight32(s_enable); + /* + * The subslice disable field is global, i.e. it applies + * to each of the enabled slices. + */ + info->subslice_per_slice = ss_max - hweight32(ss_disable); + info->subslice_total = info->slice_total * info->subslice_per_slice; + + /* + * Iterate through enabled slices and subslices to + * count the total enabled EU. + */ + for (s = 0; s < s_max; s++) { + if (!(s_enable & BIT(s))) + /* skip disabled slice */ + continue; + + eu_disable = I915_READ(GEN9_EU_DISABLE(s)); + for (ss = 0; ss < ss_max; ss++) { + int eu_per_ss; + + if (ss_disable & BIT(ss)) + /* skip disabled subslice */ + continue; + + eu_per_ss = eu_max - hweight8((eu_disable >> (ss*8)) & + eu_mask); + + /* + * Record which subslice(s) has(have) 7 EUs. we + * can tune the hash used to spread work among + * subslices if they are unbalanced. + */ + if (eu_per_ss == 7) + info->subslice_7eu[s] |= BIT(ss); + + info->eu_total += eu_per_ss; + } + } + + /* + * SKL is expected to always have a uniform distribution + * of EU across subslices with the exception that any one + * EU in any one subslice may be fused off for die + * recovery. BXT is expected to be perfectly uniform in EU + * distribution. + */ + info->eu_per_subslice = info->subslice_total ? + DIV_ROUND_UP(info->eu_total, + info->subslice_total) : 0; + /* + * SKL supports slice power gating on devices with more than + * one slice, and supports EU power gating on devices with + * more than one EU pair per subslice. BXT supports subslice + * power gating on devices with more than one subslice, and + * supports EU power gating on devices with more than one EU + * pair per subslice. + */ + info->has_slice_pg = + (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) && + info->slice_total > 1; + info->has_subslice_pg = + IS_BROXTON(dev_priv) && info->subslice_total > 1; + info->has_eu_pg = info->eu_per_subslice > 2; + + if (IS_BROXTON(dev_priv)) { +#define IS_SS_DISABLED(_ss_disable, ss) (_ss_disable & BIT(ss)) + /* + * There is a HW issue in 2x6 fused down parts that requires + * Pooled EU to be enabled as a WA. The pool configuration + * changes depending upon which subslice is fused down. This + * doesn't affect if the device has all 3 subslices enabled. + */ + /* WaEnablePooledEuFor2x6:bxt */ + info->has_pooled_eu = ((info->subslice_per_slice == 3) || + (info->subslice_per_slice == 2 && + INTEL_REVID(dev_priv) < BXT_REVID_C0)); + + info->min_eu_in_pool = 0; + if (info->has_pooled_eu) { + if (IS_SS_DISABLED(ss_disable, 0) || + IS_SS_DISABLED(ss_disable, 2)) + info->min_eu_in_pool = 3; + else if (IS_SS_DISABLED(ss_disable, 1)) + info->min_eu_in_pool = 6; + else + info->min_eu_in_pool = 9; + } +#undef IS_SS_DISABLED + } +} + +static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv) +{ + struct intel_device_info *info = mkwrite_device_info(dev_priv); + const int s_max = 3, ss_max = 3, eu_max = 8; + int s, ss; + u32 fuse2, eu_disable[s_max], s_enable, ss_disable; + + fuse2 = I915_READ(GEN8_FUSE2); + s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT; + ss_disable = (fuse2 & GEN8_F2_SS_DIS_MASK) >> GEN8_F2_SS_DIS_SHIFT; + + eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK; + eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) | + ((I915_READ(GEN8_EU_DISABLE1) & GEN8_EU_DIS1_S1_MASK) << + (32 - GEN8_EU_DIS0_S1_SHIFT)); + eu_disable[2] = (I915_READ(GEN8_EU_DISABLE1) >> GEN8_EU_DIS1_S2_SHIFT) | + ((I915_READ(GEN8_EU_DISABLE2) & GEN8_EU_DIS2_S2_MASK) << + (32 - GEN8_EU_DIS1_S2_SHIFT)); + + info->slice_total = hweight32(s_enable); + + /* + * The subslice disable field is global, i.e. it applies + * to each of the enabled slices. + */ + info->subslice_per_slice = ss_max - hweight32(ss_disable); + info->subslice_total = info->slice_total * info->subslice_per_slice; + + /* + * Iterate through enabled slices and subslices to + * count the total enabled EU. + */ + for (s = 0; s < s_max; s++) { + if (!(s_enable & (0x1 << s))) + /* skip disabled slice */ + continue; + + for (ss = 0; ss < ss_max; ss++) { + u32 n_disabled; + + if (ss_disable & (0x1 << ss)) + /* skip disabled subslice */ + continue; + + n_disabled = hweight8(eu_disable[s] >> (ss * eu_max)); + + /* + * Record which subslices have 7 EUs. + */ + if (eu_max - n_disabled == 7) + info->subslice_7eu[s] |= 1 << ss; + + info->eu_total += eu_max - n_disabled; + } + } + + /* + * BDW is expected to always have a uniform distribution of EU across + * subslices with the exception that any one EU in any one subslice may + * be fused off for die recovery. + */ + info->eu_per_subslice = info->subslice_total ? + DIV_ROUND_UP(info->eu_total, info->subslice_total) : 0; + + /* + * BDW supports slice power gating on devices with more than + * one slice. + */ + info->has_slice_pg = (info->slice_total > 1); + info->has_subslice_pg = 0; + info->has_eu_pg = 0; +} + +/* + * Determine various intel_device_info fields at runtime. + * + * Use it when either: + * - it's judged too laborious to fill n static structures with the limit + * when a simple if statement does the job, + * - run-time checks (eg read fuse/strap registers) are needed. + * + * This function needs to be called: + * - after the MMIO has been setup as we are reading registers, + * - after the PCH has been detected, + * - before the first usage of the fields it can tweak. + */ +void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) +{ + struct intel_device_info *info = mkwrite_device_info(dev_priv); + enum pipe pipe; + + /* + * Skylake and Broxton currently don't expose the topmost plane as its + * use is exclusive with the legacy cursor and we only want to expose + * one of those, not both. Until we can safely expose the topmost plane + * as a DRM_PLANE_TYPE_CURSOR with all the features exposed/supported, + * we don't expose the topmost plane at all to prevent ABI breakage + * down the line. + */ + if (IS_BROXTON(dev_priv)) { + info->num_sprites[PIPE_A] = 2; + info->num_sprites[PIPE_B] = 2; + info->num_sprites[PIPE_C] = 1; + } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + for_each_pipe(dev_priv, pipe) + info->num_sprites[pipe] = 2; + else + for_each_pipe(dev_priv, pipe) + info->num_sprites[pipe] = 1; + + if (i915.disable_display) { + DRM_INFO("Display disabled (module parameter)\n"); + info->num_pipes = 0; + } else if (info->num_pipes > 0 && + (IS_GEN7(dev_priv) || IS_GEN8(dev_priv)) && + HAS_PCH_SPLIT(dev_priv)) { + u32 fuse_strap = I915_READ(FUSE_STRAP); + u32 sfuse_strap = I915_READ(SFUSE_STRAP); + + /* + * SFUSE_STRAP is supposed to have a bit signalling the display + * is fused off. Unfortunately it seems that, at least in + * certain cases, fused off display means that PCH display + * reads don't land anywhere. In that case, we read 0s. + * + * On CPT/PPT, we can detect this case as SFUSE_STRAP_FUSE_LOCK + * should be set when taking over after the firmware. + */ + if (fuse_strap & ILK_INTERNAL_DISPLAY_DISABLE || + sfuse_strap & SFUSE_STRAP_DISPLAY_DISABLED || + (dev_priv->pch_type == PCH_CPT && + !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) { + DRM_INFO("Display fused off, disabling\n"); + info->num_pipes = 0; + } else if (fuse_strap & IVB_PIPE_C_DISABLE) { + DRM_INFO("PipeC fused off\n"); + info->num_pipes -= 1; + } + } else if (info->num_pipes > 0 && IS_GEN9(dev_priv)) { + u32 dfsm = I915_READ(SKL_DFSM); + u8 disabled_mask = 0; + bool invalid; + int num_bits; + + if (dfsm & SKL_DFSM_PIPE_A_DISABLE) + disabled_mask |= BIT(PIPE_A); + if (dfsm & SKL_DFSM_PIPE_B_DISABLE) + disabled_mask |= BIT(PIPE_B); + if (dfsm & SKL_DFSM_PIPE_C_DISABLE) + disabled_mask |= BIT(PIPE_C); + + num_bits = hweight8(disabled_mask); + + switch (disabled_mask) { + case BIT(PIPE_A): + case BIT(PIPE_B): + case BIT(PIPE_A) | BIT(PIPE_B): + case BIT(PIPE_A) | BIT(PIPE_C): + invalid = true; + break; + default: + invalid = false; + } + + if (num_bits > info->num_pipes || invalid) + DRM_ERROR("invalid pipe fuse configuration: 0x%x\n", + disabled_mask); + else + info->num_pipes -= num_bits; + } + + /* Initialize slice/subslice/EU info */ + if (IS_CHERRYVIEW(dev_priv)) + cherryview_sseu_info_init(dev_priv); + else if (IS_BROADWELL(dev_priv)) + broadwell_sseu_info_init(dev_priv); + else if (INTEL_INFO(dev_priv)->gen >= 9) + gen9_sseu_info_init(dev_priv); + + info->has_snoop = !info->has_llc; + + /* Snooping is broken on BXT A stepping. */ + if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) + info->has_snoop = false; + + DRM_DEBUG_DRIVER("slice total: %u\n", info->slice_total); + DRM_DEBUG_DRIVER("subslice total: %u\n", info->subslice_total); + DRM_DEBUG_DRIVER("subslice per slice: %u\n", info->subslice_per_slice); + DRM_DEBUG_DRIVER("EU total: %u\n", info->eu_total); + DRM_DEBUG_DRIVER("EU per subslice: %u\n", info->eu_per_subslice); + DRM_DEBUG_DRIVER("has slice power gating: %s\n", + info->has_slice_pg ? "y" : "n"); + DRM_DEBUG_DRIVER("has subslice power gating: %s\n", + info->has_subslice_pg ? "y" : "n"); + DRM_DEBUG_DRIVER("has EU power gating: %s\n", + info->has_eu_pg ? "y" : "n"); +} diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 32893195d..175595fc3 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -36,6 +36,7 @@ #include "intel_drv.h" #include <drm/i915_drm.h> #include "i915_drv.h" +#include "i915_gem_dmabuf.h" #include "intel_dsi.h" #include "i915_trace.h" #include <drm/drm_atomic.h> @@ -46,7 +47,11 @@ #include <drm/drm_rect.h> #include <linux/dma_remapping.h> #include <linux/reservation.h> -#include <linux/dma-buf.h> + +static bool is_mmio_work(struct intel_flip_work *work) +{ + return work->mmio_work.func; +} /* Primary plane formats for gen <= 3 */ static const uint32_t i8xx_primary_formats[] = { @@ -117,20 +122,18 @@ static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force); static void ironlake_pfit_enable(struct intel_crtc *crtc); static void intel_modeset_setup_hw_state(struct drm_device *dev); static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc); +static int ilk_max_pixel_rate(struct drm_atomic_state *state); +static int bxt_calc_cdclk(int max_pixclk); -typedef struct { - int min, max; -} intel_range_t; - -typedef struct { - int dot_limit; - int p2_slow, p2_fast; -} intel_p2_t; - -typedef struct intel_limit intel_limit_t; struct intel_limit { - intel_range_t dot, vco, n, m, m1, m2, p, p1; - intel_p2_t p2; + struct { + int min, max; + } dot, vco, n, m, m1, m2, p, p1; + + struct { + int dot_limit; + int p2_slow, p2_fast; + } p2; }; /* returns HPLL frequency in kHz */ @@ -185,6 +188,7 @@ intel_pch_rawclk(struct drm_i915_private *dev_priv) static int intel_vlv_hrawclk(struct drm_i915_private *dev_priv) { + /* RAWCLK_FREQ_VLV register updated from power well code */ return vlv_get_cck_clock_hpll(dev_priv, "hrawclk", CCK_DISPLAY_REF_CLOCK_CONTROL); } @@ -218,7 +222,7 @@ intel_g4x_hrawclk(struct drm_i915_private *dev_priv) } } -static void intel_update_rawclk(struct drm_i915_private *dev_priv) +void intel_update_rawclk(struct drm_i915_private *dev_priv) { if (HAS_PCH_SPLIT(dev_priv)) dev_priv->rawclk_freq = intel_pch_rawclk(dev_priv); @@ -255,7 +259,7 @@ intel_fdi_link_freq(struct drm_i915_private *dev_priv, return 270000; } -static const intel_limit_t intel_limits_i8xx_dac = { +static const struct intel_limit intel_limits_i8xx_dac = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 908000, .max = 1512000 }, .n = { .min = 2, .max = 16 }, @@ -268,7 +272,7 @@ static const intel_limit_t intel_limits_i8xx_dac = { .p2_slow = 4, .p2_fast = 2 }, }; -static const intel_limit_t intel_limits_i8xx_dvo = { +static const struct intel_limit intel_limits_i8xx_dvo = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 908000, .max = 1512000 }, .n = { .min = 2, .max = 16 }, @@ -281,7 +285,7 @@ static const intel_limit_t intel_limits_i8xx_dvo = { .p2_slow = 4, .p2_fast = 4 }, }; -static const intel_limit_t intel_limits_i8xx_lvds = { +static const struct intel_limit intel_limits_i8xx_lvds = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 908000, .max = 1512000 }, .n = { .min = 2, .max = 16 }, @@ -294,7 +298,7 @@ static const intel_limit_t intel_limits_i8xx_lvds = { .p2_slow = 14, .p2_fast = 7 }, }; -static const intel_limit_t intel_limits_i9xx_sdvo = { +static const struct intel_limit intel_limits_i9xx_sdvo = { .dot = { .min = 20000, .max = 400000 }, .vco = { .min = 1400000, .max = 2800000 }, .n = { .min = 1, .max = 6 }, @@ -307,7 +311,7 @@ static const intel_limit_t intel_limits_i9xx_sdvo = { .p2_slow = 10, .p2_fast = 5 }, }; -static const intel_limit_t intel_limits_i9xx_lvds = { +static const struct intel_limit intel_limits_i9xx_lvds = { .dot = { .min = 20000, .max = 400000 }, .vco = { .min = 1400000, .max = 2800000 }, .n = { .min = 1, .max = 6 }, @@ -321,7 +325,7 @@ static const intel_limit_t intel_limits_i9xx_lvds = { }; -static const intel_limit_t intel_limits_g4x_sdvo = { +static const struct intel_limit intel_limits_g4x_sdvo = { .dot = { .min = 25000, .max = 270000 }, .vco = { .min = 1750000, .max = 3500000}, .n = { .min = 1, .max = 4 }, @@ -336,7 +340,7 @@ static const intel_limit_t intel_limits_g4x_sdvo = { }, }; -static const intel_limit_t intel_limits_g4x_hdmi = { +static const struct intel_limit intel_limits_g4x_hdmi = { .dot = { .min = 22000, .max = 400000 }, .vco = { .min = 1750000, .max = 3500000}, .n = { .min = 1, .max = 4 }, @@ -349,7 +353,7 @@ static const intel_limit_t intel_limits_g4x_hdmi = { .p2_slow = 10, .p2_fast = 5 }, }; -static const intel_limit_t intel_limits_g4x_single_channel_lvds = { +static const struct intel_limit intel_limits_g4x_single_channel_lvds = { .dot = { .min = 20000, .max = 115000 }, .vco = { .min = 1750000, .max = 3500000 }, .n = { .min = 1, .max = 3 }, @@ -363,7 +367,7 @@ static const intel_limit_t intel_limits_g4x_single_channel_lvds = { }, }; -static const intel_limit_t intel_limits_g4x_dual_channel_lvds = { +static const struct intel_limit intel_limits_g4x_dual_channel_lvds = { .dot = { .min = 80000, .max = 224000 }, .vco = { .min = 1750000, .max = 3500000 }, .n = { .min = 1, .max = 3 }, @@ -377,7 +381,7 @@ static const intel_limit_t intel_limits_g4x_dual_channel_lvds = { }, }; -static const intel_limit_t intel_limits_pineview_sdvo = { +static const struct intel_limit intel_limits_pineview_sdvo = { .dot = { .min = 20000, .max = 400000}, .vco = { .min = 1700000, .max = 3500000 }, /* Pineview's Ncounter is a ring counter */ @@ -392,7 +396,7 @@ static const intel_limit_t intel_limits_pineview_sdvo = { .p2_slow = 10, .p2_fast = 5 }, }; -static const intel_limit_t intel_limits_pineview_lvds = { +static const struct intel_limit intel_limits_pineview_lvds = { .dot = { .min = 20000, .max = 400000 }, .vco = { .min = 1700000, .max = 3500000 }, .n = { .min = 3, .max = 6 }, @@ -410,7 +414,7 @@ static const intel_limit_t intel_limits_pineview_lvds = { * We calculate clock using (register_value + 2) for N/M1/M2, so here * the range value for them is (actual_value - 2). */ -static const intel_limit_t intel_limits_ironlake_dac = { +static const struct intel_limit intel_limits_ironlake_dac = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000 }, .n = { .min = 1, .max = 5 }, @@ -423,7 +427,7 @@ static const intel_limit_t intel_limits_ironlake_dac = { .p2_slow = 10, .p2_fast = 5 }, }; -static const intel_limit_t intel_limits_ironlake_single_lvds = { +static const struct intel_limit intel_limits_ironlake_single_lvds = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000 }, .n = { .min = 1, .max = 3 }, @@ -436,7 +440,7 @@ static const intel_limit_t intel_limits_ironlake_single_lvds = { .p2_slow = 14, .p2_fast = 14 }, }; -static const intel_limit_t intel_limits_ironlake_dual_lvds = { +static const struct intel_limit intel_limits_ironlake_dual_lvds = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000 }, .n = { .min = 1, .max = 3 }, @@ -450,7 +454,7 @@ static const intel_limit_t intel_limits_ironlake_dual_lvds = { }; /* LVDS 100mhz refclk limits. */ -static const intel_limit_t intel_limits_ironlake_single_lvds_100m = { +static const struct intel_limit intel_limits_ironlake_single_lvds_100m = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000 }, .n = { .min = 1, .max = 2 }, @@ -463,7 +467,7 @@ static const intel_limit_t intel_limits_ironlake_single_lvds_100m = { .p2_slow = 14, .p2_fast = 14 }, }; -static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = { +static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = { .dot = { .min = 25000, .max = 350000 }, .vco = { .min = 1760000, .max = 3510000 }, .n = { .min = 1, .max = 3 }, @@ -476,7 +480,7 @@ static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = { .p2_slow = 7, .p2_fast = 7 }, }; -static const intel_limit_t intel_limits_vlv = { +static const struct intel_limit intel_limits_vlv = { /* * These are the data rate limits (measured in fast clocks) * since those are the strictest limits we have. The fast @@ -492,7 +496,7 @@ static const intel_limit_t intel_limits_vlv = { .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */ }; -static const intel_limit_t intel_limits_chv = { +static const struct intel_limit intel_limits_chv = { /* * These are the data rate limits (measured in fast clocks) * since those are the strictest limits we have. The fast @@ -508,7 +512,7 @@ static const intel_limit_t intel_limits_chv = { .p2 = { .p2_slow = 1, .p2_fast = 14 }, }; -static const intel_limit_t intel_limits_bxt = { +static const struct intel_limit intel_limits_bxt = { /* FIXME: find real dot limits */ .dot = { .min = 0, .max = INT_MAX }, .vco = { .min = 4800000, .max = 6700000 }, @@ -526,52 +530,6 @@ needs_modeset(struct drm_crtc_state *state) return drm_atomic_crtc_needs_modeset(state); } -/** - * Returns whether any output on the specified pipe is of the specified type - */ -bool intel_pipe_has_type(struct intel_crtc *crtc, enum intel_output_type type) -{ - struct drm_device *dev = crtc->base.dev; - struct intel_encoder *encoder; - - for_each_encoder_on_crtc(dev, &crtc->base, encoder) - if (encoder->type == type) - return true; - - return false; -} - -/** - * Returns whether any output on the specified pipe will have the specified - * type after a staged modeset is complete, i.e., the same as - * intel_pipe_has_type() but looking at encoder->new_crtc instead of - * encoder->crtc. - */ -static bool intel_pipe_will_have_type(const struct intel_crtc_state *crtc_state, - int type) -{ - struct drm_atomic_state *state = crtc_state->base.state; - struct drm_connector *connector; - struct drm_connector_state *connector_state; - struct intel_encoder *encoder; - int i, num_connectors = 0; - - for_each_connector_in_state(state, connector, connector_state, i) { - if (connector_state->crtc != crtc_state->base.crtc) - continue; - - num_connectors++; - - encoder = to_intel_encoder(connector_state->best_encoder); - if (encoder->type == type) - return true; - } - - WARN_ON(num_connectors == 0); - - return false; -} - /* * Platform specific helpers to calculate the port PLL loopback- (clock.m), * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast @@ -581,7 +539,7 @@ static bool intel_pipe_will_have_type(const struct intel_crtc_state *crtc_state, * divided-down version of it. */ /* m1 is reserved as 0 in Pineview, n is a ring counter */ -static int pnv_calc_dpll_params(int refclk, intel_clock_t *clock) +static int pnv_calc_dpll_params(int refclk, struct dpll *clock) { clock->m = clock->m2 + 2; clock->p = clock->p1 * clock->p2; @@ -598,7 +556,7 @@ static uint32_t i9xx_dpll_compute_m(struct dpll *dpll) return 5 * (dpll->m1 + 2) + (dpll->m2 + 2); } -static int i9xx_calc_dpll_params(int refclk, intel_clock_t *clock) +static int i9xx_calc_dpll_params(int refclk, struct dpll *clock) { clock->m = i9xx_dpll_compute_m(clock); clock->p = clock->p1 * clock->p2; @@ -610,7 +568,7 @@ static int i9xx_calc_dpll_params(int refclk, intel_clock_t *clock) return clock->dot; } -static int vlv_calc_dpll_params(int refclk, intel_clock_t *clock) +static int vlv_calc_dpll_params(int refclk, struct dpll *clock) { clock->m = clock->m1 * clock->m2; clock->p = clock->p1 * clock->p2; @@ -622,7 +580,7 @@ static int vlv_calc_dpll_params(int refclk, intel_clock_t *clock) return clock->dot / 5; } -int chv_calc_dpll_params(int refclk, intel_clock_t *clock) +int chv_calc_dpll_params(int refclk, struct dpll *clock) { clock->m = clock->m1 * clock->m2; clock->p = clock->p1 * clock->p2; @@ -642,8 +600,8 @@ int chv_calc_dpll_params(int refclk, intel_clock_t *clock) */ static bool intel_PLL_is_valid(struct drm_device *dev, - const intel_limit_t *limit, - const intel_clock_t *clock) + const struct intel_limit *limit, + const struct dpll *clock) { if (clock->n < limit->n.min || limit->n.max < clock->n) INTELPllInvalid("n out of range\n"); @@ -678,13 +636,13 @@ static bool intel_PLL_is_valid(struct drm_device *dev, } static int -i9xx_select_p2_div(const intel_limit_t *limit, +i9xx_select_p2_div(const struct intel_limit *limit, const struct intel_crtc_state *crtc_state, int target) { struct drm_device *dev = crtc_state->base.crtc->dev; - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) { + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { /* * For LVDS just rely on its current settings for dual-channel. * We haven't figured out how to reliably set up different @@ -713,13 +671,13 @@ i9xx_select_p2_div(const intel_limit_t *limit, * divider from @match_clock used for LVDS downclocking. */ static bool -i9xx_find_best_dpll(const intel_limit_t *limit, +i9xx_find_best_dpll(const struct intel_limit *limit, struct intel_crtc_state *crtc_state, - int target, int refclk, intel_clock_t *match_clock, - intel_clock_t *best_clock) + int target, int refclk, struct dpll *match_clock, + struct dpll *best_clock) { struct drm_device *dev = crtc_state->base.crtc->dev; - intel_clock_t clock; + struct dpll clock; int err = target; memset(best_clock, 0, sizeof(*best_clock)); @@ -770,13 +728,13 @@ i9xx_find_best_dpll(const intel_limit_t *limit, * divider from @match_clock used for LVDS downclocking. */ static bool -pnv_find_best_dpll(const intel_limit_t *limit, +pnv_find_best_dpll(const struct intel_limit *limit, struct intel_crtc_state *crtc_state, - int target, int refclk, intel_clock_t *match_clock, - intel_clock_t *best_clock) + int target, int refclk, struct dpll *match_clock, + struct dpll *best_clock) { struct drm_device *dev = crtc_state->base.crtc->dev; - intel_clock_t clock; + struct dpll clock; int err = target; memset(best_clock, 0, sizeof(*best_clock)); @@ -825,13 +783,13 @@ pnv_find_best_dpll(const intel_limit_t *limit, * divider from @match_clock used for LVDS downclocking. */ static bool -g4x_find_best_dpll(const intel_limit_t *limit, +g4x_find_best_dpll(const struct intel_limit *limit, struct intel_crtc_state *crtc_state, - int target, int refclk, intel_clock_t *match_clock, - intel_clock_t *best_clock) + int target, int refclk, struct dpll *match_clock, + struct dpll *best_clock) { struct drm_device *dev = crtc_state->base.crtc->dev; - intel_clock_t clock; + struct dpll clock; int max_n; bool found = false; /* approximately equals target * 0.00585 */ @@ -877,8 +835,8 @@ g4x_find_best_dpll(const intel_limit_t *limit, * best configuration and error found so far. Return the calculated error. */ static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq, - const intel_clock_t *calculated_clock, - const intel_clock_t *best_clock, + const struct dpll *calculated_clock, + const struct dpll *best_clock, unsigned int best_error_ppm, unsigned int *error_ppm) { @@ -918,14 +876,14 @@ static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq, * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. */ static bool -vlv_find_best_dpll(const intel_limit_t *limit, +vlv_find_best_dpll(const struct intel_limit *limit, struct intel_crtc_state *crtc_state, - int target, int refclk, intel_clock_t *match_clock, - intel_clock_t *best_clock) + int target, int refclk, struct dpll *match_clock, + struct dpll *best_clock) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_device *dev = crtc->base.dev; - intel_clock_t clock; + struct dpll clock; unsigned int bestppm = 1000000; /* min update 19.2 MHz */ int max_n = min(limit->n.max, refclk / 19200); @@ -977,15 +935,15 @@ vlv_find_best_dpll(const intel_limit_t *limit, * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. */ static bool -chv_find_best_dpll(const intel_limit_t *limit, +chv_find_best_dpll(const struct intel_limit *limit, struct intel_crtc_state *crtc_state, - int target, int refclk, intel_clock_t *match_clock, - intel_clock_t *best_clock) + int target, int refclk, struct dpll *match_clock, + struct dpll *best_clock) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_device *dev = crtc->base.dev; unsigned int best_error_ppm; - intel_clock_t clock; + struct dpll clock; uint64_t m2; int found = false; @@ -1035,10 +993,10 @@ chv_find_best_dpll(const intel_limit_t *limit, } bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock, - intel_clock_t *best_clock) + struct dpll *best_clock) { int refclk = 100000; - const intel_limit_t *limit = &intel_limits_bxt; + const struct intel_limit *limit = &intel_limits_bxt; return chv_find_best_dpll(limit, crtc_state, target_clock, refclk, NULL, best_clock); @@ -1076,7 +1034,7 @@ enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t reg = PIPEDSL(pipe); u32 line1, line2; u32 line_mask; @@ -1112,7 +1070,7 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) static void intel_wait_for_pipe_off(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; enum pipe pipe = crtc->pipe; @@ -1120,8 +1078,9 @@ static void intel_wait_for_pipe_off(struct intel_crtc *crtc) i915_reg_t reg = PIPECONF(cpu_transcoder); /* Wait for the Pipe State to go off */ - if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0, - 100)) + if (intel_wait_for_register(dev_priv, + reg, I965_PIPECONF_ACTIVE, 0, + 100)) WARN(1, "pipe_off wait timed out\n"); } else { /* Wait for the display line to settle */ @@ -1203,7 +1162,7 @@ static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv, u32 val; /* ILK FDI PLL is always enabled */ - if (INTEL_INFO(dev_priv)->gen == 5) + if (IS_GEN5(dev_priv)) return; /* On Haswell, DDI ports are responsible for the FDI PLL setup */ @@ -1230,7 +1189,7 @@ void assert_fdi_rx_pll(struct drm_i915_private *dev_priv, void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; i915_reg_t pp_reg; u32 val; enum pipe panel_pipe = PIPE_A; @@ -1272,7 +1231,7 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv, static void assert_cursor(struct drm_i915_private *dev_priv, enum pipe pipe, bool state) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; bool cur_state; if (IS_845G(dev) || IS_I865G(dev)) @@ -1334,7 +1293,7 @@ static void assert_plane(struct drm_i915_private *dev_priv, static void assert_planes_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; int i; /* Primary planes are fixed to pipes on gen4+ */ @@ -1360,7 +1319,7 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv, static void assert_sprites_disabled(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; int sprite; if (INTEL_INFO(dev)->gen >= 9) { @@ -1540,7 +1499,11 @@ static void _vlv_enable_pll(struct intel_crtc *crtc, POSTING_READ(DPLL(pipe)); udelay(150); - if (wait_for(((I915_READ(DPLL(pipe)) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1)) + if (intel_wait_for_register(dev_priv, + DPLL(pipe), + DPLL_LOCK_VLV, + DPLL_LOCK_VLV, + 1)) DRM_ERROR("DPLL %d failed to lock\n", pipe); } @@ -1589,7 +1552,9 @@ static void _chv_enable_pll(struct intel_crtc *crtc, I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll); /* Check PLL is locked */ - if (wait_for(((I915_READ(DPLL(pipe)) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1)) + if (intel_wait_for_register(dev_priv, + DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV, + 1)) DRM_ERROR("PLL %d failed to lock\n", pipe); } @@ -1635,9 +1600,10 @@ static int intel_num_dvo_pipes(struct drm_device *dev) struct intel_crtc *crtc; int count = 0; - for_each_intel_crtc(dev, crtc) + for_each_intel_crtc(dev, crtc) { count += crtc->base.state->active && - intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO); + intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO); + } return count; } @@ -1645,7 +1611,7 @@ static int intel_num_dvo_pipes(struct drm_device *dev) static void i9xx_enable_pll(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t reg = DPLL(crtc->pipe); u32 dpll = crtc->config->dpll_hw_state.dpll; @@ -1717,12 +1683,12 @@ static void i9xx_enable_pll(struct intel_crtc *crtc) static void i9xx_disable_pll(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = crtc->pipe; /* Disable DVO 2x clock on both PLLs if necessary */ if (IS_I830(dev) && - intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO) && + intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO) && !intel_num_dvo_pipes(dev)) { I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE); @@ -1809,7 +1775,9 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv, BUG(); } - if (wait_for((I915_READ(dpll_reg) & port_mask) == expected_mask, 1000)) + if (intel_wait_for_register(dev_priv, + dpll_reg, port_mask, expected_mask, + 1000)) WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n", port_name(dport->port), I915_READ(dpll_reg) & port_mask, expected_mask); } @@ -1817,7 +1785,7 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv, static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); i915_reg_t reg; @@ -1850,7 +1818,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, * here for both 8bpc and 12bpc. */ val &= ~PIPECONF_BPC_MASK; - if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_HDMI)) + if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_HDMI)) val |= PIPECONF_8BPC; else val |= pipeconf_val & PIPECONF_BPC_MASK; @@ -1859,7 +1827,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, val &= ~TRANS_INTERLACE_MASK; if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) if (HAS_PCH_IBX(dev_priv) && - intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO)) + intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO)) val |= TRANS_LEGACY_INTERLACED_ILK; else val |= TRANS_INTERLACED; @@ -1867,7 +1835,9 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, val |= TRANS_PROGRESSIVE; I915_WRITE(reg, val | TRANS_ENABLE); - if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100)) + if (intel_wait_for_register(dev_priv, + reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE, + 100)) DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe)); } @@ -1895,14 +1865,18 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv, val |= TRANS_PROGRESSIVE; I915_WRITE(LPT_TRANSCONF, val); - if (wait_for(I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE, 100)) + if (intel_wait_for_register(dev_priv, + LPT_TRANSCONF, + TRANS_STATE_ENABLE, + TRANS_STATE_ENABLE, + 100)) DRM_ERROR("Failed to enable PCH transcoder\n"); } static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; i915_reg_t reg; uint32_t val; @@ -1918,7 +1892,9 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv, val &= ~TRANS_ENABLE; I915_WRITE(reg, val); /* wait for PCH transcoder off, transcoder state */ - if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50)) + if (intel_wait_for_register(dev_priv, + reg, TRANS_STATE_ENABLE, 0, + 50)) DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe)); if (HAS_PCH_CPT(dev)) { @@ -1938,7 +1914,9 @@ static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv) val &= ~TRANS_ENABLE; I915_WRITE(LPT_TRANSCONF, val); /* wait for PCH transcoder off, transcoder state */ - if (wait_for((I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE) == 0, 50)) + if (intel_wait_for_register(dev_priv, + LPT_TRANSCONF, TRANS_STATE_ENABLE, 0, + 50)) DRM_ERROR("Failed to disable PCH transcoder\n"); /* Workaround: clear timing override bit. */ @@ -1957,7 +1935,7 @@ static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv) static void intel_enable_pipe(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = crtc->pipe; enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; enum pipe pch_transcoder; @@ -1981,7 +1959,7 @@ static void intel_enable_pipe(struct intel_crtc *crtc) * need the check. */ if (HAS_GMCH_DISPLAY(dev_priv)) - if (crtc->config->has_dsi_encoder) + if (intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DSI)) assert_dsi_pll_enabled(dev_priv); else assert_pll_enabled(dev_priv, pipe); @@ -2030,7 +2008,7 @@ static void intel_enable_pipe(struct intel_crtc *crtc) */ static void intel_disable_pipe(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; enum pipe pipe = crtc->pipe; i915_reg_t reg; @@ -2068,15 +2046,6 @@ static void intel_disable_pipe(struct intel_crtc *crtc) intel_wait_for_pipe_off(crtc); } -static bool need_vtd_wa(struct drm_device *dev) -{ -#ifdef CONFIG_INTEL_IOMMU - if (INTEL_INFO(dev)->gen >= 6 && intel_iommu_gfx_mapped) - return true; -#endif - return false; -} - static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv) { return IS_GEN2(dev_priv) ? 2048 : 4096; @@ -2241,7 +2210,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation) { struct drm_device *dev = fb->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj = intel_fb_obj(fb); struct i915_ggtt_view view; u32 alignment; @@ -2258,7 +2227,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, * we should always have valid PTE following the scanout preventing * the VT-d warning. */ - if (need_vtd_wa(dev) && alignment < 256 * 1024) + if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024) alignment = 256 * 1024; /* @@ -2309,7 +2278,7 @@ err_pm: return ret; } -static void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation) +void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation) { struct drm_i915_gem_object *obj = intel_fb_obj(fb); struct i915_ggtt_view view; @@ -2543,7 +2512,7 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc, struct intel_initial_plane_config *plane_config) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *c; struct intel_crtc *i; struct drm_i915_gem_object *obj; @@ -2639,7 +2608,7 @@ static void i9xx_update_primary_plane(struct drm_plane *primary, const struct intel_plane_state *plane_state) { struct drm_device *dev = primary->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_framebuffer *fb = plane_state->base.fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); @@ -2752,7 +2721,7 @@ static void i9xx_disable_primary_plane(struct drm_plane *primary, struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int plane = intel_crtc->plane; @@ -2769,7 +2738,7 @@ static void ironlake_update_primary_plane(struct drm_plane *primary, const struct intel_plane_state *plane_state) { struct drm_device *dev = primary->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_framebuffer *fb = plane_state->base.fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); @@ -2897,7 +2866,7 @@ u32 intel_plane_obj_offset(struct intel_plane *intel_plane, static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0); I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0); @@ -3007,7 +2976,7 @@ static void skylake_update_primary_plane(struct drm_plane *plane, const struct intel_plane_state *plane_state) { struct drm_device *dev = plane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc); struct drm_framebuffer *fb = plane_state->base.fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); @@ -3091,7 +3060,7 @@ static void skylake_disable_primary_plane(struct drm_plane *primary, struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = to_intel_crtc(crtc)->pipe; I915_WRITE(PLANE_CTL(pipe, 0), 0); @@ -3110,17 +3079,12 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb, return -ENODEV; } -static void intel_complete_page_flips(struct drm_device *dev) +static void intel_complete_page_flips(struct drm_i915_private *dev_priv) { - struct drm_crtc *crtc; - - for_each_crtc(dev, crtc) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum plane plane = intel_crtc->plane; + struct intel_crtc *crtc; - intel_prepare_page_flip(dev, plane); - intel_finish_page_flip_plane(dev, plane); - } + for_each_intel_crtc(&dev_priv->drm, crtc) + intel_finish_page_flip_cs(dev_priv, crtc->pipe); } static void intel_update_primary_planes(struct drm_device *dev) @@ -3129,55 +3093,125 @@ static void intel_update_primary_planes(struct drm_device *dev) for_each_crtc(dev, crtc) { struct intel_plane *plane = to_intel_plane(crtc->primary); - struct intel_plane_state *plane_state; - - drm_modeset_lock_crtc(crtc, &plane->base); - plane_state = to_intel_plane_state(plane->base.state); + struct intel_plane_state *plane_state = + to_intel_plane_state(plane->base.state); if (plane_state->visible) plane->update_plane(&plane->base, to_intel_crtc_state(crtc->state), plane_state); + } +} + +static int +__intel_display_resume(struct drm_device *dev, + struct drm_atomic_state *state) +{ + struct drm_crtc_state *crtc_state; + struct drm_crtc *crtc; + int i, ret; + + intel_modeset_setup_hw_state(dev); + i915_redisable_vga(dev); + + if (!state) + return 0; - drm_modeset_unlock_crtc(crtc); + for_each_crtc_in_state(state, crtc, crtc_state, i) { + /* + * Force recalculation even if we restore + * current state. With fast modeset this may not result + * in a modeset when the state is compatible. + */ + crtc_state->mode_changed = true; } + + /* ignore any reset values/BIOS leftovers in the WM registers */ + to_intel_atomic_state(state)->skip_intermediate_wm = true; + + ret = drm_atomic_commit(state); + + WARN_ON(ret == -EDEADLK); + return ret; } -void intel_prepare_reset(struct drm_device *dev) +void intel_prepare_reset(struct drm_i915_private *dev_priv) { + struct drm_device *dev = &dev_priv->drm; + struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx; + struct drm_atomic_state *state; + int ret; + /* no reset support for gen2 */ - if (IS_GEN2(dev)) + if (IS_GEN2(dev_priv)) return; - /* reset doesn't touch the display */ - if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) + /* + * Need mode_config.mutex so that we don't + * trample ongoing ->detect() and whatnot. + */ + mutex_lock(&dev->mode_config.mutex); + drm_modeset_acquire_init(ctx, 0); + while (1) { + ret = drm_modeset_lock_all_ctx(dev, ctx); + if (ret != -EDEADLK) + break; + + drm_modeset_backoff(ctx); + } + + /* reset doesn't touch the display, but flips might get nuked anyway, */ + if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) return; - drm_modeset_lock_all(dev); /* * Disabling the crtcs gracefully seems nicer. Also the * g33 docs say we should at least disable all the planes. */ - intel_display_suspend(dev); + state = drm_atomic_helper_duplicate_state(dev, ctx); + if (IS_ERR(state)) { + ret = PTR_ERR(state); + state = NULL; + DRM_ERROR("Duplicating state failed with %i\n", ret); + goto err; + } + + ret = drm_atomic_helper_disable_all(dev, ctx); + if (ret) { + DRM_ERROR("Suspending crtc's failed with %i\n", ret); + goto err; + } + + dev_priv->modeset_restore_state = state; + state->acquire_ctx = ctx; + return; + +err: + drm_atomic_state_free(state); } -void intel_finish_reset(struct drm_device *dev) +void intel_finish_reset(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_device *dev = &dev_priv->drm; + struct drm_modeset_acquire_ctx *ctx = &dev_priv->reset_ctx; + struct drm_atomic_state *state = dev_priv->modeset_restore_state; + int ret; /* * Flips in the rings will be nuked by the reset, * so complete all pending flips so that user space * will get its events and not get stuck. */ - intel_complete_page_flips(dev); + intel_complete_page_flips(dev_priv); /* no reset support for gen2 */ - if (IS_GEN2(dev)) + if (IS_GEN2(dev_priv)) return; + dev_priv->modeset_restore_state = NULL; + /* reset doesn't touch the display */ - if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) { + if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) { /* * Flips in the rings have been nuked by the reset, * so update the base address of all primary @@ -3188,28 +3222,31 @@ void intel_finish_reset(struct drm_device *dev) * CS-based flips (which might get lost in gpu resets) any more. */ intel_update_primary_planes(dev); - return; - } - - /* - * The display has been reset as well, - * so need a full re-initialization. - */ - intel_runtime_pm_disable_interrupts(dev_priv); - intel_runtime_pm_enable_interrupts(dev_priv); + } else { + /* + * The display has been reset as well, + * so need a full re-initialization. + */ + intel_runtime_pm_disable_interrupts(dev_priv); + intel_runtime_pm_enable_interrupts(dev_priv); - intel_modeset_init_hw(dev); + intel_modeset_init_hw(dev); - spin_lock_irq(&dev_priv->irq_lock); - if (dev_priv->display.hpd_irq_setup) - dev_priv->display.hpd_irq_setup(dev); - spin_unlock_irq(&dev_priv->irq_lock); + spin_lock_irq(&dev_priv->irq_lock); + if (dev_priv->display.hpd_irq_setup) + dev_priv->display.hpd_irq_setup(dev_priv); + spin_unlock_irq(&dev_priv->irq_lock); - intel_display_resume(dev); + ret = __intel_display_resume(dev, state); + if (ret) + DRM_ERROR("Restoring old state failed with %i\n", ret); - intel_hpd_init(dev_priv); + intel_hpd_init(dev_priv); + } - drm_modeset_unlock_all(dev); + drm_modeset_drop_locks(ctx); + drm_modeset_acquire_fini(ctx); + mutex_unlock(&dev->mode_config.mutex); } static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc) @@ -3224,7 +3261,7 @@ static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc) return false; spin_lock_irq(&dev->event_lock); - pending = to_intel_crtc(crtc)->unpin_work != NULL; + pending = to_intel_crtc(crtc)->flip_work != NULL; spin_unlock_irq(&dev->event_lock); return pending; @@ -3234,7 +3271,7 @@ static void intel_update_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *old_crtc_state) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc->base.state); @@ -3275,7 +3312,7 @@ static void intel_update_pipe_config(struct intel_crtc *crtc, static void intel_fdi_normal_train(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; i915_reg_t reg; @@ -3318,7 +3355,7 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc) static void ironlake_fdi_link_train(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; i915_reg_t reg; @@ -3419,7 +3456,7 @@ static const int snb_b_fdi_train_param[] = { static void gen6_fdi_link_train(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; i915_reg_t reg; @@ -3552,7 +3589,7 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc) static void ivb_manual_fdi_link_train(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; i915_reg_t reg; @@ -3671,7 +3708,7 @@ train_done: static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = intel_crtc->pipe; i915_reg_t reg; u32 temp; @@ -3708,7 +3745,7 @@ static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc) static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = intel_crtc->pipe; i915_reg_t reg; u32 temp; @@ -3738,7 +3775,7 @@ static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc) static void ironlake_fdi_disable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; i915_reg_t reg; @@ -3803,7 +3840,7 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev) if (atomic_read(&crtc->unpin_work_count) == 0) continue; - if (crtc->unpin_work) + if (crtc->flip_work) intel_wait_for_vblank(dev, crtc->pipe); return true; @@ -3815,11 +3852,9 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev) static void page_flip_completed(struct intel_crtc *intel_crtc) { struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); - struct intel_unpin_work *work = intel_crtc->unpin_work; + struct intel_flip_work *work = intel_crtc->flip_work; - /* ensure that the unpin work is consistent wrt ->pending. */ - smp_rmb(); - intel_crtc->unpin_work = NULL; + intel_crtc->flip_work = NULL; if (work->event) drm_crtc_send_vblank_event(&intel_crtc->base, work->event); @@ -3827,7 +3862,7 @@ static void page_flip_completed(struct intel_crtc *intel_crtc) drm_crtc_vblank_put(&intel_crtc->base); wake_up_all(&dev_priv->pending_flip_queue); - queue_work(dev_priv->wq, &work->work); + queue_work(dev_priv->wq, &work->unpin_work); trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj); @@ -3836,7 +3871,7 @@ static void page_flip_completed(struct intel_crtc *intel_crtc) static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); long ret; WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue)); @@ -3851,9 +3886,11 @@ static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) if (ret == 0) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_flip_work *work; spin_lock_irq(&dev->event_lock); - if (intel_crtc->unpin_work) { + work = intel_crtc->flip_work; + if (work && !is_mmio_work(work)) { WARN_ONCE(1, "Removing stuck page flip\n"); page_flip_completed(intel_crtc); } @@ -3997,7 +4034,7 @@ static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc, enum pipe pch_transcoder) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder), @@ -4019,7 +4056,7 @@ static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc, static void cpt_set_fdi_bc_bifurcation(struct drm_device *dev, bool enable) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t temp; temp = I915_READ(SOUTH_CHICKEN1); @@ -4069,7 +4106,7 @@ intel_trans_dp_port_sel(struct drm_crtc *crtc) struct intel_encoder *encoder; for_each_encoder_on_crtc(dev, crtc, encoder) { - if (encoder->type == INTEL_OUTPUT_DISPLAYPORT || + if (encoder->type == INTEL_OUTPUT_DP || encoder->type == INTEL_OUTPUT_EDP) return enc_to_dig_port(&encoder->base)->port; } @@ -4088,7 +4125,7 @@ intel_trans_dp_port_sel(struct drm_crtc *crtc) static void ironlake_pch_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 temp; @@ -4138,7 +4175,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc) intel_fdi_normal_train(crtc); /* For PCH DP, enable TRANS_DP_CTL */ - if (HAS_PCH_CPT(dev) && intel_crtc->config->has_dp_encoder) { + if (HAS_PCH_CPT(dev) && intel_crtc_has_dp_encoder(intel_crtc->config)) { const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5; @@ -4178,7 +4215,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc) static void lpt_pch_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; @@ -4194,7 +4231,7 @@ static void lpt_pch_enable(struct drm_crtc *crtc) static void cpt_verify_modeset(struct drm_device *dev, int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t dslreg = PIPEDSL(pipe); u32 temp; @@ -4281,8 +4318,9 @@ int skl_update_scaler_crtc(struct intel_crtc_state *state) struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc); const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode; - DRM_DEBUG_KMS("Updating scaler for [CRTC:%i] scaler_user index %u.%u\n", - intel_crtc->base.base.id, intel_crtc->pipe, SKL_CRTC_INDEX); + DRM_DEBUG_KMS("Updating scaler for [CRTC:%d:%s] scaler_user index %u.%u\n", + intel_crtc->base.base.id, intel_crtc->base.name, + intel_crtc->pipe, SKL_CRTC_INDEX); return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX, &state->scaler_state.scaler_id, BIT(DRM_ROTATE_0), @@ -4312,9 +4350,9 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, bool force_detach = !fb || !plane_state->visible; - DRM_DEBUG_KMS("Updating scaler for [PLANE:%d] scaler_user index %u.%u\n", - intel_plane->base.base.id, intel_crtc->pipe, - drm_plane_index(&intel_plane->base)); + DRM_DEBUG_KMS("Updating scaler for [PLANE:%d:%s] scaler_user index %u.%u\n", + intel_plane->base.base.id, intel_plane->base.name, + intel_crtc->pipe, drm_plane_index(&intel_plane->base)); ret = skl_update_scaler(crtc_state, force_detach, drm_plane_index(&intel_plane->base), @@ -4330,8 +4368,9 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, /* check colorkey */ if (plane_state->ckey.flags != I915_SET_COLORKEY_NONE) { - DRM_DEBUG_KMS("[PLANE:%d] scaling with color key not allowed", - intel_plane->base.base.id); + DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed", + intel_plane->base.base.id, + intel_plane->base.name); return -EINVAL; } @@ -4350,8 +4389,9 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, case DRM_FORMAT_VYUY: break; default: - DRM_DEBUG_KMS("[PLANE:%d] FB:%d unsupported scaling format 0x%x\n", - intel_plane->base.base.id, fb->base.id, fb->pixel_format); + DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n", + intel_plane->base.base.id, intel_plane->base.name, + fb->base.id, fb->pixel_format); return -EINVAL; } @@ -4369,7 +4409,7 @@ static void skylake_scaler_disable(struct intel_crtc *crtc) static void skylake_pfit_enable(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = crtc->pipe; struct intel_crtc_scaler_state *scaler_state = &crtc->config->scaler_state; @@ -4397,7 +4437,7 @@ static void skylake_pfit_enable(struct intel_crtc *crtc) static void ironlake_pfit_enable(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = crtc->pipe; if (crtc->config->pch_pfit.enabled) { @@ -4418,7 +4458,7 @@ static void ironlake_pfit_enable(struct intel_crtc *crtc) void hsw_enable_ips(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!crtc->config->ips_enabled) return; @@ -4446,7 +4486,9 @@ void hsw_enable_ips(struct intel_crtc *crtc) * and don't wait for vblanks until the end of crtc_enable, then * the HW state readout code will complain that the expected * IPS_CTL value is not the one we read. */ - if (wait_for(I915_READ_NOTRACE(IPS_CTL) & IPS_ENABLE, 50)) + if (intel_wait_for_register(dev_priv, + IPS_CTL, IPS_ENABLE, IPS_ENABLE, + 50)) DRM_ERROR("Timed out waiting for IPS enable\n"); } } @@ -4454,7 +4496,7 @@ void hsw_enable_ips(struct intel_crtc *crtc) void hsw_disable_ips(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!crtc->config->ips_enabled) return; @@ -4465,7 +4507,9 @@ void hsw_disable_ips(struct intel_crtc *crtc) WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0)); mutex_unlock(&dev_priv->rps.hw_lock); /* wait for pcode to finish disabling IPS, which may take up to 42ms */ - if (wait_for((I915_READ(IPS_CTL) & IPS_ENABLE) == 0, 42)) + if (intel_wait_for_register(dev_priv, + IPS_CTL, IPS_ENABLE, 0, + 42)) DRM_ERROR("Timed out waiting for IPS disable\n"); } else { I915_WRITE(IPS_CTL, 0); @@ -4480,7 +4524,7 @@ static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc) { if (intel_crtc->overlay) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); mutex_lock(&dev->struct_mutex); dev_priv->mm.interruptible = false; @@ -4508,7 +4552,7 @@ static void intel_post_enable_primary(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; @@ -4540,7 +4584,7 @@ static void intel_pre_disable_primary(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; @@ -4567,7 +4611,7 @@ static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; @@ -4626,7 +4670,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) { struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc); struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc->base.state); struct drm_atomic_state *old_state = old_crtc_state->base.state; @@ -4641,14 +4685,14 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) struct intel_plane_state *old_primary_state = to_intel_plane_state(old_pri_state); - intel_fbc_pre_update(crtc); + intel_fbc_pre_update(crtc, pipe_config, primary_state); if (old_primary_state->visible && (modeset || !primary_state->visible)) intel_pre_disable_primary(&crtc->base); } - if (pipe_config->disable_cxsr) { + if (pipe_config->disable_cxsr && HAS_GMCH_DISPLAY(dev)) { crtc->wm.cxsr_allowed = false; /* @@ -4729,7 +4773,7 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask static void ironlake_crtc_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; @@ -4757,7 +4801,7 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) if (intel_crtc->config->has_pch_encoder) intel_prepare_shared_dpll(intel_crtc); - if (intel_crtc->config->has_dp_encoder) + if (intel_crtc_has_dp_encoder(intel_crtc->config)) intel_dp_set_m_n(intel_crtc, M1_N1); intel_set_pipe_timings(intel_crtc); @@ -4826,7 +4870,7 @@ static bool hsw_crtc_supports_ips(struct intel_crtc *crtc) static void haswell_crtc_enable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe, hsw_workaround_pipe; @@ -4841,13 +4885,17 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A, false); + for_each_encoder_on_crtc(dev, crtc, encoder) + if (encoder->pre_pll_enable) + encoder->pre_pll_enable(encoder); + if (intel_crtc->config->shared_dpll) intel_enable_shared_dpll(intel_crtc); - if (intel_crtc->config->has_dp_encoder) + if (intel_crtc_has_dp_encoder(intel_crtc->config)) intel_dp_set_m_n(intel_crtc, M1_N1); - if (!intel_crtc->config->has_dsi_encoder) + if (!transcoder_is_dsi(cpu_transcoder)) intel_set_pipe_timings(intel_crtc); intel_set_pipe_src_size(intel_crtc); @@ -4863,7 +4911,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) &intel_crtc->config->fdi_m_n, NULL); } - if (!intel_crtc->config->has_dsi_encoder) + if (!transcoder_is_dsi(cpu_transcoder)) haswell_set_pipeconf(crtc); haswell_set_pipemisc(crtc); @@ -4885,7 +4933,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) if (intel_crtc->config->has_pch_encoder) dev_priv->display.fdi_link_train(crtc); - if (!intel_crtc->config->has_dsi_encoder) + if (!transcoder_is_dsi(cpu_transcoder)) intel_ddi_enable_pipe_clock(intel_crtc); if (INTEL_INFO(dev)->gen >= 9) @@ -4900,7 +4948,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) intel_color_load_luts(&pipe_config->base); intel_ddi_set_pipe_settings(crtc); - if (!intel_crtc->config->has_dsi_encoder) + if (!transcoder_is_dsi(cpu_transcoder)) intel_ddi_enable_transcoder_func(crtc); if (dev_priv->display.initial_watermarks != NULL) @@ -4909,7 +4957,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) intel_update_watermarks(crtc); /* XXX: Do the pipe assertions at the right place for BXT DSI. */ - if (!intel_crtc->config->has_dsi_encoder) + if (!transcoder_is_dsi(cpu_transcoder)) intel_enable_pipe(intel_crtc); if (intel_crtc->config->has_pch_encoder) @@ -4946,7 +4994,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc) static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = crtc->pipe; /* To avoid upsetting the power well on haswell only disable the pfit if @@ -4961,7 +5009,7 @@ static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force) static void ironlake_crtc_disable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; @@ -5024,7 +5072,7 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc) static void haswell_crtc_disable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; @@ -5042,13 +5090,13 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) assert_vblank_disabled(crtc); /* XXX: Do the pipe assertions at the right place for BXT DSI. */ - if (!intel_crtc->config->has_dsi_encoder) + if (!transcoder_is_dsi(cpu_transcoder)) intel_disable_pipe(intel_crtc); if (intel_crtc->config->dp_encoder_is_mst) intel_ddi_set_vc_payload_alloc(crtc, false); - if (!intel_crtc->config->has_dsi_encoder) + if (!transcoder_is_dsi(cpu_transcoder)) intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder); if (INTEL_INFO(dev)->gen >= 9) @@ -5056,7 +5104,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) else ironlake_pfit_disable(intel_crtc, false); - if (!intel_crtc->config->has_dsi_encoder) + if (!transcoder_is_dsi(cpu_transcoder)) intel_ddi_disable_pipe_clock(intel_crtc); for_each_encoder_on_crtc(dev, crtc, encoder) @@ -5076,7 +5124,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc) static void i9xx_pfit_enable(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc_state *pipe_config = crtc->config; if (!pipe_config->gmch_pfit.control) @@ -5146,7 +5194,7 @@ intel_display_port_power_domain(struct intel_encoder *intel_encoder) case INTEL_OUTPUT_UNKNOWN: /* Only DDI platforms should ever use this output type */ WARN_ON_ONCE(!HAS_DDI(dev)); - case INTEL_OUTPUT_DISPLAYPORT: + case INTEL_OUTPUT_DP: case INTEL_OUTPUT_HDMI: case INTEL_OUTPUT_EDP: intel_dig_port = enc_to_dig_port(&intel_encoder->base); @@ -5180,7 +5228,7 @@ intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder) * run the DP detection too. */ WARN_ON_ONCE(!HAS_DDI(dev)); - case INTEL_OUTPUT_DISPLAYPORT: + case INTEL_OUTPUT_DP: case INTEL_OUTPUT_EDP: intel_dig_port = enc_to_dig_port(&intel_encoder->base); return port_to_aux_power_domain(intel_dig_port->port); @@ -5228,7 +5276,7 @@ static unsigned long modeset_get_crtc_power_domains(struct drm_crtc *crtc, struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum intel_display_power_domain domain; unsigned long domains, new_domains, old_domains; @@ -5269,21 +5317,34 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv) return max_cdclk_freq*90/100; } +static int skl_calc_cdclk(int max_pixclk, int vco); + static void intel_update_max_cdclk(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK; + int max_cdclk, vco; + vco = dev_priv->skl_preferred_vco_freq; + WARN_ON(vco != 8100000 && vco != 8640000); + + /* + * Use the lower (vco 8640) cdclk values as a + * first guess. skl_calc_cdclk() will correct it + * if the preferred vco is 8100 instead. + */ if (limit == SKL_DFSM_CDCLK_LIMIT_675) - dev_priv->max_cdclk_freq = 675000; + max_cdclk = 617143; else if (limit == SKL_DFSM_CDCLK_LIMIT_540) - dev_priv->max_cdclk_freq = 540000; + max_cdclk = 540000; else if (limit == SKL_DFSM_CDCLK_LIMIT_450) - dev_priv->max_cdclk_freq = 450000; + max_cdclk = 432000; else - dev_priv->max_cdclk_freq = 337500; + max_cdclk = 308571; + + dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco); } else if (IS_BROXTON(dev)) { dev_priv->max_cdclk_freq = 624000; } else if (IS_BROADWELL(dev)) { @@ -5321,267 +5382,322 @@ static void intel_update_max_cdclk(struct drm_device *dev) static void intel_update_cdclk(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev); - DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz\n", - dev_priv->cdclk_freq); + + if (INTEL_GEN(dev_priv) >= 9) + DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz, VCO: %d kHz, ref: %d kHz\n", + dev_priv->cdclk_freq, dev_priv->cdclk_pll.vco, + dev_priv->cdclk_pll.ref); + else + DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz\n", + dev_priv->cdclk_freq); /* - * Program the gmbus_freq based on the cdclk frequency. - * BSpec erroneously claims we should aim for 4MHz, but - * in fact 1MHz is the correct frequency. + * 9:0 CMBUS [sic] CDCLK frequency (cdfreq): + * Programmng [sic] note: bit[9:2] should be programmed to the number + * of cdclk that generates 4MHz reference clock freq which is used to + * generate GMBus clock. This will vary with the cdclk freq. */ - if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) { - /* - * Program the gmbus_freq based on the cdclk frequency. - * BSpec erroneously claims we should aim for 4MHz, but - * in fact 1MHz is the correct frequency. - */ + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) I915_WRITE(GMBUSFREQ_VLV, DIV_ROUND_UP(dev_priv->cdclk_freq, 1000)); - } +} - if (dev_priv->max_cdclk_freq == 0) - intel_update_max_cdclk(dev); +/* convert from kHz to .1 fixpoint MHz with -1MHz offset */ +static int skl_cdclk_decimal(int cdclk) +{ + return DIV_ROUND_CLOSEST(cdclk - 1000, 500); } -static void broxton_set_cdclk(struct drm_i915_private *dev_priv, int frequency) +static int bxt_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk) { - uint32_t divider; - uint32_t ratio; - uint32_t current_freq; - int ret; + int ratio; + + if (cdclk == dev_priv->cdclk_pll.ref) + return 0; - /* frequency = 19.2MHz * ratio / 2 / div{1,1.5,2,4} */ - switch (frequency) { + switch (cdclk) { + default: + MISSING_CASE(cdclk); case 144000: + case 288000: + case 384000: + case 576000: + ratio = 60; + break; + case 624000: + ratio = 65; + break; + } + + return dev_priv->cdclk_pll.ref * ratio; +} + +static void bxt_de_pll_disable(struct drm_i915_private *dev_priv) +{ + I915_WRITE(BXT_DE_PLL_ENABLE, 0); + + /* Timeout 200us */ + if (intel_wait_for_register(dev_priv, + BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 0, + 1)) + DRM_ERROR("timeout waiting for DE PLL unlock\n"); + + dev_priv->cdclk_pll.vco = 0; +} + +static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco) +{ + int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk_pll.ref); + u32 val; + + val = I915_READ(BXT_DE_PLL_CTL); + val &= ~BXT_DE_PLL_RATIO_MASK; + val |= BXT_DE_PLL_RATIO(ratio); + I915_WRITE(BXT_DE_PLL_CTL, val); + + I915_WRITE(BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE); + + /* Timeout 200us */ + if (intel_wait_for_register(dev_priv, + BXT_DE_PLL_ENABLE, + BXT_DE_PLL_LOCK, + BXT_DE_PLL_LOCK, + 1)) + DRM_ERROR("timeout waiting for DE PLL lock\n"); + + dev_priv->cdclk_pll.vco = vco; +} + +static void bxt_set_cdclk(struct drm_i915_private *dev_priv, int cdclk) +{ + u32 val, divider; + int vco, ret; + + vco = bxt_de_pll_vco(dev_priv, cdclk); + + DRM_DEBUG_DRIVER("Changing CDCLK to %d kHz (VCO %d kHz)\n", cdclk, vco); + + /* cdclk = vco / 2 / div{1,1.5,2,4} */ + switch (DIV_ROUND_CLOSEST(vco, cdclk)) { + case 8: divider = BXT_CDCLK_CD2X_DIV_SEL_4; - ratio = BXT_DE_PLL_RATIO(60); break; - case 288000: + case 4: divider = BXT_CDCLK_CD2X_DIV_SEL_2; - ratio = BXT_DE_PLL_RATIO(60); break; - case 384000: + case 3: divider = BXT_CDCLK_CD2X_DIV_SEL_1_5; - ratio = BXT_DE_PLL_RATIO(60); break; - case 576000: - divider = BXT_CDCLK_CD2X_DIV_SEL_1; - ratio = BXT_DE_PLL_RATIO(60); - break; - case 624000: + case 2: divider = BXT_CDCLK_CD2X_DIV_SEL_1; - ratio = BXT_DE_PLL_RATIO(65); - break; - case 19200: - /* - * Bypass frequency with DE PLL disabled. Init ratio, divider - * to suppress GCC warning. - */ - ratio = 0; - divider = 0; break; default: - DRM_ERROR("unsupported CDCLK freq %d", frequency); + WARN_ON(cdclk != dev_priv->cdclk_pll.ref); + WARN_ON(vco != 0); - return; + divider = BXT_CDCLK_CD2X_DIV_SEL_1; + break; } - mutex_lock(&dev_priv->rps.hw_lock); /* Inform power controller of upcoming frequency change */ + mutex_lock(&dev_priv->rps.hw_lock); ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, 0x80000000); mutex_unlock(&dev_priv->rps.hw_lock); if (ret) { DRM_ERROR("PCode CDCLK freq change notify failed (err %d, freq %d)\n", - ret, frequency); + ret, cdclk); return; } - current_freq = I915_READ(CDCLK_CTL) & CDCLK_FREQ_DECIMAL_MASK; - /* convert from .1 fixpoint MHz with -1MHz offset to kHz */ - current_freq = current_freq * 500 + 1000; + if (dev_priv->cdclk_pll.vco != 0 && + dev_priv->cdclk_pll.vco != vco) + bxt_de_pll_disable(dev_priv); - /* - * DE PLL has to be disabled when - * - setting to 19.2MHz (bypass, PLL isn't used) - * - before setting to 624MHz (PLL needs toggling) - * - before setting to any frequency from 624MHz (PLL needs toggling) - */ - if (frequency == 19200 || frequency == 624000 || - current_freq == 624000) { - I915_WRITE(BXT_DE_PLL_ENABLE, ~BXT_DE_PLL_PLL_ENABLE); - /* Timeout 200us */ - if (wait_for(!(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK), - 1)) - DRM_ERROR("timout waiting for DE PLL unlock\n"); - } - - if (frequency != 19200) { - uint32_t val; - - val = I915_READ(BXT_DE_PLL_CTL); - val &= ~BXT_DE_PLL_RATIO_MASK; - val |= ratio; - I915_WRITE(BXT_DE_PLL_CTL, val); - - I915_WRITE(BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE); - /* Timeout 200us */ - if (wait_for(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK, 1)) - DRM_ERROR("timeout waiting for DE PLL lock\n"); - - val = I915_READ(CDCLK_CTL); - val &= ~BXT_CDCLK_CD2X_DIV_SEL_MASK; - val |= divider; - /* - * Disable SSA Precharge when CD clock frequency < 500 MHz, - * enable otherwise. - */ - val &= ~BXT_CDCLK_SSA_PRECHARGE_ENABLE; - if (frequency >= 500000) - val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE; + if (dev_priv->cdclk_pll.vco != vco) + bxt_de_pll_enable(dev_priv, vco); - val &= ~CDCLK_FREQ_DECIMAL_MASK; - /* convert from kHz to .1 fixpoint MHz with -1MHz offset */ - val |= (frequency - 1000) / 500; - I915_WRITE(CDCLK_CTL, val); - } + val = divider | skl_cdclk_decimal(cdclk); + /* + * FIXME if only the cd2x divider needs changing, it could be done + * without shutting off the pipe (if only one pipe is active). + */ + val |= BXT_CDCLK_CD2X_PIPE_NONE; + /* + * Disable SSA Precharge when CD clock frequency < 500 MHz, + * enable otherwise. + */ + if (cdclk >= 500000) + val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE; + I915_WRITE(CDCLK_CTL, val); mutex_lock(&dev_priv->rps.hw_lock); ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, - DIV_ROUND_UP(frequency, 25000)); + DIV_ROUND_UP(cdclk, 25000)); mutex_unlock(&dev_priv->rps.hw_lock); if (ret) { DRM_ERROR("PCode CDCLK freq set failed, (err %d, freq %d)\n", - ret, frequency); + ret, cdclk); return; } - intel_update_cdclk(dev_priv->dev); + intel_update_cdclk(&dev_priv->drm); } -static bool broxton_cdclk_is_enabled(struct drm_i915_private *dev_priv) +static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv) { - if (!(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_PLL_ENABLE)) - return false; + u32 cdctl, expected; - /* TODO: Check for a valid CDCLK rate */ + intel_update_cdclk(&dev_priv->drm); - if (!(I915_READ(DBUF_CTL) & DBUF_POWER_REQUEST)) { - DRM_DEBUG_DRIVER("CDCLK enabled, but DBUF power not requested\n"); + if (dev_priv->cdclk_pll.vco == 0 || + dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref) + goto sanitize; - return false; - } + /* DPLL okay; verify the cdclock + * + * Some BIOS versions leave an incorrect decimal frequency value and + * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4, + * so sanitize this register. + */ + cdctl = I915_READ(CDCLK_CTL); + /* + * Let's ignore the pipe field, since BIOS could have configured the + * dividers both synching to an active pipe, or asynchronously + * (PIPE_NONE). + */ + cdctl &= ~BXT_CDCLK_CD2X_PIPE_NONE; - if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE)) { - DRM_DEBUG_DRIVER("CDCLK enabled, but DBUF power hasn't settled\n"); + expected = (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) | + skl_cdclk_decimal(dev_priv->cdclk_freq); + /* + * Disable SSA Precharge when CD clock frequency < 500 MHz, + * enable otherwise. + */ + if (dev_priv->cdclk_freq >= 500000) + expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE; - return false; - } + if (cdctl == expected) + /* All well; nothing to sanitize */ + return; - return true; -} +sanitize: + DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n"); -bool broxton_cdclk_verify_state(struct drm_i915_private *dev_priv) -{ - return broxton_cdclk_is_enabled(dev_priv); + /* force cdclk programming */ + dev_priv->cdclk_freq = 0; + + /* force full PLL disable + enable */ + dev_priv->cdclk_pll.vco = -1; } -void broxton_init_cdclk(struct drm_i915_private *dev_priv) +void bxt_init_cdclk(struct drm_i915_private *dev_priv) { - /* check if cd clock is enabled */ - if (broxton_cdclk_is_enabled(dev_priv)) { - DRM_DEBUG_KMS("CDCLK already enabled, won't reprogram it\n"); - return; - } + bxt_sanitize_cdclk(dev_priv); - DRM_DEBUG_KMS("CDCLK not enabled, enabling it\n"); + if (dev_priv->cdclk_freq != 0 && dev_priv->cdclk_pll.vco != 0) + return; /* * FIXME: * - The initial CDCLK needs to be read from VBT. * Need to make this change after VBT has changes for BXT. - * - check if setting the max (or any) cdclk freq is really necessary - * here, it belongs to modeset time */ - broxton_set_cdclk(dev_priv, 624000); - - I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST); - POSTING_READ(DBUF_CTL); + bxt_set_cdclk(dev_priv, bxt_calc_cdclk(0)); +} - udelay(10); +void bxt_uninit_cdclk(struct drm_i915_private *dev_priv) +{ + bxt_set_cdclk(dev_priv, dev_priv->cdclk_pll.ref); +} - if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE)) - DRM_ERROR("DBuf power enable timeout!\n"); +static int skl_calc_cdclk(int max_pixclk, int vco) +{ + if (vco == 8640000) { + if (max_pixclk > 540000) + return 617143; + else if (max_pixclk > 432000) + return 540000; + else if (max_pixclk > 308571) + return 432000; + else + return 308571; + } else { + if (max_pixclk > 540000) + return 675000; + else if (max_pixclk > 450000) + return 540000; + else if (max_pixclk > 337500) + return 450000; + else + return 337500; + } } -void broxton_uninit_cdclk(struct drm_i915_private *dev_priv) +static void +skl_dpll0_update(struct drm_i915_private *dev_priv) { - I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST); - POSTING_READ(DBUF_CTL); + u32 val; - udelay(10); + dev_priv->cdclk_pll.ref = 24000; + dev_priv->cdclk_pll.vco = 0; - if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE) - DRM_ERROR("DBuf power disable timeout!\n"); + val = I915_READ(LCPLL1_CTL); + if ((val & LCPLL_PLL_ENABLE) == 0) + return; - /* Set minimum (bypass) frequency, in effect turning off the DE PLL */ - broxton_set_cdclk(dev_priv, 19200); -} + if (WARN_ON((val & LCPLL_PLL_LOCK) == 0)) + return; -static const struct skl_cdclk_entry { - unsigned int freq; - unsigned int vco; -} skl_cdclk_frequencies[] = { - { .freq = 308570, .vco = 8640 }, - { .freq = 337500, .vco = 8100 }, - { .freq = 432000, .vco = 8640 }, - { .freq = 450000, .vco = 8100 }, - { .freq = 540000, .vco = 8100 }, - { .freq = 617140, .vco = 8640 }, - { .freq = 675000, .vco = 8100 }, -}; + val = I915_READ(DPLL_CTRL1); -static unsigned int skl_cdclk_decimal(unsigned int freq) -{ - return (freq - 1000) / 500; + if (WARN_ON((val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | + DPLL_CTRL1_SSC(SKL_DPLL0) | + DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) != + DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) + return; + + switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) { + case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0): + case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0): + case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0): + case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0): + dev_priv->cdclk_pll.vco = 8100000; + break; + case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0): + case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0): + dev_priv->cdclk_pll.vco = 8640000; + break; + default: + MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)); + break; + } } -static unsigned int skl_cdclk_get_vco(unsigned int freq) +void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco) { - unsigned int i; + bool changed = dev_priv->skl_preferred_vco_freq != vco; - for (i = 0; i < ARRAY_SIZE(skl_cdclk_frequencies); i++) { - const struct skl_cdclk_entry *e = &skl_cdclk_frequencies[i]; - - if (e->freq == freq) - return e->vco; - } + dev_priv->skl_preferred_vco_freq = vco; - return 8100; + if (changed) + intel_update_max_cdclk(&dev_priv->drm); } static void -skl_dpll0_enable(struct drm_i915_private *dev_priv, unsigned int required_vco) +skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco) { - unsigned int min_freq; + int min_cdclk = skl_calc_cdclk(0, vco); u32 val; - /* select the minimum CDCLK before enabling DPLL 0 */ - val = I915_READ(CDCLK_CTL); - val &= ~CDCLK_FREQ_SEL_MASK | ~CDCLK_FREQ_DECIMAL_MASK; - val |= CDCLK_FREQ_337_308; - - if (required_vco == 8640) - min_freq = 308570; - else - min_freq = 337500; - - val = CDCLK_FREQ_337_308 | skl_cdclk_decimal(min_freq); + WARN_ON(vco != 8100000 && vco != 8640000); + /* select the minimum CDCLK before enabling DPLL 0 */ + val = CDCLK_FREQ_337_308 | skl_cdclk_decimal(min_cdclk); I915_WRITE(CDCLK_CTL, val); POSTING_READ(CDCLK_CTL); @@ -5592,14 +5708,14 @@ skl_dpll0_enable(struct drm_i915_private *dev_priv, unsigned int required_vco) * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640. * The modeset code is responsible for the selection of the exact link * rate later on, with the constraint of choosing a frequency that - * works with required_vco. + * works with vco. */ val = I915_READ(DPLL_CTRL1); val &= ~(DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | DPLL_CTRL1_SSC(SKL_DPLL0) | DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)); val |= DPLL_CTRL1_OVERRIDE(SKL_DPLL0); - if (required_vco == 8640) + if (vco == 8640000) val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0); else @@ -5611,8 +5727,27 @@ skl_dpll0_enable(struct drm_i915_private *dev_priv, unsigned int required_vco) I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) | LCPLL_PLL_ENABLE); - if (wait_for(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK, 5)) + if (intel_wait_for_register(dev_priv, + LCPLL1_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK, + 5)) DRM_ERROR("DPLL0 not locked\n"); + + dev_priv->cdclk_pll.vco = vco; + + /* We'll want to keep using the current vco from now on. */ + skl_set_preferred_cdclk_vco(dev_priv, vco); +} + +static void +skl_dpll0_disable(struct drm_i915_private *dev_priv) +{ + I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE); + if (intel_wait_for_register(dev_priv, + LCPLL1_CTL, LCPLL_PLL_LOCK, 0, + 1)) + DRM_ERROR("Couldn't disable DPLL0\n"); + + dev_priv->cdclk_pll.vco = 0; } static bool skl_cdclk_pcu_ready(struct drm_i915_private *dev_priv) @@ -5631,23 +5766,17 @@ static bool skl_cdclk_pcu_ready(struct drm_i915_private *dev_priv) static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv) { - unsigned int i; - - for (i = 0; i < 15; i++) { - if (skl_cdclk_pcu_ready(dev_priv)) - return true; - udelay(10); - } - - return false; + return _wait_for(skl_cdclk_pcu_ready(dev_priv), 3000, 10) == 0; } -static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq) +static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; u32 freq_select, pcu_ack; - DRM_DEBUG_DRIVER("Changing CDCLK to %dKHz\n", freq); + WARN_ON((cdclk == 24000) != (vco == 0)); + + DRM_DEBUG_DRIVER("Changing CDCLK to %d kHz (VCO %d kHz)\n", cdclk, vco); if (!skl_cdclk_wait_for_pcu_ready(dev_priv)) { DRM_ERROR("failed to inform PCU about cdclk change\n"); @@ -5655,7 +5784,7 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq) } /* set CDCLK_CTL */ - switch(freq) { + switch (cdclk) { case 450000: case 432000: freq_select = CDCLK_FREQ_450_432; @@ -5665,20 +5794,27 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq) freq_select = CDCLK_FREQ_540; pcu_ack = 2; break; - case 308570: + case 308571: case 337500: default: freq_select = CDCLK_FREQ_337_308; pcu_ack = 0; break; - case 617140: + case 617143: case 675000: freq_select = CDCLK_FREQ_675_617; pcu_ack = 3; break; } - I915_WRITE(CDCLK_CTL, freq_select | skl_cdclk_decimal(freq)); + if (dev_priv->cdclk_pll.vco != 0 && + dev_priv->cdclk_pll.vco != vco) + skl_dpll0_disable(dev_priv); + + if (dev_priv->cdclk_pll.vco != vco) + skl_dpll0_enable(dev_priv, vco); + + I915_WRITE(CDCLK_CTL, freq_select | skl_cdclk_decimal(cdclk)); POSTING_READ(CDCLK_CTL); /* inform PCU of the change */ @@ -5689,52 +5825,41 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq) intel_update_cdclk(dev); } +static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv); + void skl_uninit_cdclk(struct drm_i915_private *dev_priv) { - /* disable DBUF power */ - I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST); - POSTING_READ(DBUF_CTL); - - udelay(10); - - if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE) - DRM_ERROR("DBuf power disable timeout\n"); - - /* disable DPLL0 */ - I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE); - if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1)) - DRM_ERROR("Couldn't disable DPLL0\n"); + skl_set_cdclk(dev_priv, dev_priv->cdclk_pll.ref, 0); } void skl_init_cdclk(struct drm_i915_private *dev_priv) { - unsigned int required_vco; - - /* DPLL0 not enabled (happens on early BIOS versions) */ - if (!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE)) { - /* enable DPLL0 */ - required_vco = skl_cdclk_get_vco(dev_priv->skl_boot_cdclk); - skl_dpll0_enable(dev_priv, required_vco); - } + int cdclk, vco; - /* set CDCLK to the frequency the BIOS chose */ - skl_set_cdclk(dev_priv, dev_priv->skl_boot_cdclk); + skl_sanitize_cdclk(dev_priv); - /* enable DBUF power */ - I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST); - POSTING_READ(DBUF_CTL); + if (dev_priv->cdclk_freq != 0 && dev_priv->cdclk_pll.vco != 0) { + /* + * Use the current vco as our initial + * guess as to what the preferred vco is. + */ + if (dev_priv->skl_preferred_vco_freq == 0) + skl_set_preferred_cdclk_vco(dev_priv, + dev_priv->cdclk_pll.vco); + return; + } - udelay(10); + vco = dev_priv->skl_preferred_vco_freq; + if (vco == 0) + vco = 8100000; + cdclk = skl_calc_cdclk(0, vco); - if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE)) - DRM_ERROR("DBuf power enable timeout\n"); + skl_set_cdclk(dev_priv, cdclk, vco); } -int skl_sanitize_cdclk(struct drm_i915_private *dev_priv) +static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv) { - uint32_t lcpll1 = I915_READ(LCPLL1_CTL); - uint32_t cdctl = I915_READ(CDCLK_CTL); - int freq = dev_priv->skl_boot_cdclk; + uint32_t cdctl, expected; /* * check if the pre-os intialized the display @@ -5744,8 +5869,10 @@ int skl_sanitize_cdclk(struct drm_i915_private *dev_priv) if ((I915_READ(SWF_ILK(0x18)) & 0x00FFFFFF) == 0) goto sanitize; + intel_update_cdclk(&dev_priv->drm); /* Is PLL enabled and locked ? */ - if (!((lcpll1 & LCPLL_PLL_ENABLE) && (lcpll1 & LCPLL_PLL_LOCK))) + if (dev_priv->cdclk_pll.vco == 0 || + dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref) goto sanitize; /* DPLL okay; verify the cdclock @@ -5754,25 +5881,26 @@ int skl_sanitize_cdclk(struct drm_i915_private *dev_priv) * decimal part is programmed wrong from BIOS where pre-os does not * enable display. Verify the same as well. */ - if (cdctl == ((cdctl & CDCLK_FREQ_SEL_MASK) | skl_cdclk_decimal(freq))) + cdctl = I915_READ(CDCLK_CTL); + expected = (cdctl & CDCLK_FREQ_SEL_MASK) | + skl_cdclk_decimal(dev_priv->cdclk_freq); + if (cdctl == expected) /* All well; nothing to sanitize */ - return false; + return; + sanitize: - /* - * As of now initialize with max cdclk till - * we get dynamic cdclk support - * */ - dev_priv->skl_boot_cdclk = dev_priv->max_cdclk_freq; - skl_init_cdclk(dev_priv); + DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n"); - /* we did have to sanitize */ - return true; + /* force cdclk programming */ + dev_priv->cdclk_freq = 0; + /* force full PLL disable + enable */ + dev_priv->cdclk_pll.vco = -1; } /* Adjust CDclk dividers to allow high res or save power if possible */ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val, cmd; WARN_ON(dev_priv->display.get_display_clock_speed(dev) @@ -5837,7 +5965,7 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) static void cherryview_set_cdclk(struct drm_device *dev, int cdclk) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val, cmd; WARN_ON(dev_priv->display.get_display_clock_speed(dev) @@ -5906,21 +6034,15 @@ static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv, return 200000; } -static int broxton_calc_cdclk(struct drm_i915_private *dev_priv, - int max_pixclk) +static int bxt_calc_cdclk(int max_pixclk) { - /* - * FIXME: - * - remove the guardband, it's not needed on BXT - * - set 19.2MHz bypass frequency if there are no active pipes - */ - if (max_pixclk > 576000*9/10) + if (max_pixclk > 576000) return 624000; - else if (max_pixclk > 384000*9/10) + else if (max_pixclk > 384000) return 576000; - else if (max_pixclk > 288000*9/10) + else if (max_pixclk > 288000) return 384000; - else if (max_pixclk > 144000*9/10) + else if (max_pixclk > 144000) return 288000; else return 144000; @@ -5931,7 +6053,7 @@ static int intel_mode_max_pixclk(struct drm_device *dev, struct drm_atomic_state *state) { struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc; struct drm_crtc_state *crtc_state; unsigned max_pixclk = 0, i; @@ -5958,14 +6080,11 @@ static int intel_mode_max_pixclk(struct drm_device *dev, static int valleyview_modeset_calc_cdclk(struct drm_atomic_state *state) { struct drm_device *dev = state->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int max_pixclk = intel_mode_max_pixclk(dev, state); struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - if (max_pixclk < 0) - return max_pixclk; - intel_state->cdclk = intel_state->dev_cdclk = valleyview_calc_cdclk(dev_priv, max_pixclk); @@ -5975,22 +6094,17 @@ static int valleyview_modeset_calc_cdclk(struct drm_atomic_state *state) return 0; } -static int broxton_modeset_calc_cdclk(struct drm_atomic_state *state) +static int bxt_modeset_calc_cdclk(struct drm_atomic_state *state) { - struct drm_device *dev = state->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - int max_pixclk = intel_mode_max_pixclk(dev, state); + int max_pixclk = ilk_max_pixel_rate(state); struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - if (max_pixclk < 0) - return max_pixclk; - intel_state->cdclk = intel_state->dev_cdclk = - broxton_calc_cdclk(dev_priv, max_pixclk); + bxt_calc_cdclk(max_pixclk); if (!intel_state->active_crtcs) - intel_state->dev_cdclk = broxton_calc_cdclk(dev_priv, 0); + intel_state->dev_cdclk = bxt_calc_cdclk(0); return 0; } @@ -6034,7 +6148,7 @@ static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv) static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state) { struct drm_device *dev = old_state->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_atomic_state *old_intel_state = to_intel_atomic_state(old_state); unsigned req_cdclk = old_intel_state->dev_cdclk; @@ -6073,14 +6187,14 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc) if (WARN_ON(intel_crtc->active)) return; - if (intel_crtc->config->has_dp_encoder) + if (intel_crtc_has_dp_encoder(intel_crtc->config)) intel_dp_set_m_n(intel_crtc, M1_N1); intel_set_pipe_timings(intel_crtc); intel_set_pipe_src_size(intel_crtc); if (IS_CHERRYVIEW(dev) && pipe == PIPE_B) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY); I915_WRITE(CHV_CANVAS(pipe), 0); @@ -6125,7 +6239,7 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc) static void i9xx_set_pll_dividers(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(FP0(crtc->pipe), crtc->config->dpll_hw_state.fp0); I915_WRITE(FP1(crtc->pipe), crtc->config->dpll_hw_state.fp1); @@ -6146,7 +6260,7 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc) i9xx_set_pll_dividers(intel_crtc); - if (intel_crtc->config->has_dp_encoder) + if (intel_crtc_has_dp_encoder(intel_crtc->config)) intel_dp_set_m_n(intel_crtc, M1_N1); intel_set_pipe_timings(intel_crtc); @@ -6182,7 +6296,7 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc) static void i9xx_pfit_disable(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!crtc->config->gmch_pfit.control) return; @@ -6197,7 +6311,7 @@ static void i9xx_pfit_disable(struct intel_crtc *crtc) static void i9xx_crtc_disable(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_encoder *encoder; int pipe = intel_crtc->pipe; @@ -6223,7 +6337,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc) if (encoder->post_disable) encoder->post_disable(encoder); - if (!intel_crtc->config->has_dsi_encoder) { + if (!intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI)) { if (IS_CHERRYVIEW(dev)) chv_disable_pll(dev_priv, pipe); else if (IS_VALLEYVIEW(dev)) @@ -6252,7 +6366,7 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc) return; if (to_intel_plane_state(crtc->primary->state)->visible) { - WARN_ON(intel_crtc->unpin_work); + WARN_ON(intel_crtc->flip_work); intel_pre_disable_primary_noatomic(crtc); @@ -6262,8 +6376,8 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc) dev_priv->display.crtc_disable(crtc); - DRM_DEBUG_KMS("[CRTC:%d] hw state adjusted, was enabled, now disabled\n", - crtc->base.id); + DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n", + crtc->base.id, crtc->name); WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0); crtc->state->active = false; @@ -6541,7 +6655,7 @@ static void hsw_compute_ips_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); pipe_config->ips_enabled = i915.enable_ips && hsw_crtc_supports_ips(crtc) && @@ -6561,12 +6675,12 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; + int clock_limit = dev_priv->max_dotclk_freq; - /* FIXME should check pixel clock limits on all platforms */ if (INTEL_INFO(dev)->gen < 4) { - int clock_limit = dev_priv->max_cdclk_freq * 9 / 10; + clock_limit = dev_priv->max_cdclk_freq * 9 / 10; /* * Enable double wide mode when the dot clock @@ -6574,16 +6688,16 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, */ if (intel_crtc_supports_double_wide(crtc) && adjusted_mode->crtc_clock > clock_limit) { - clock_limit *= 2; + clock_limit = dev_priv->max_dotclk_freq; pipe_config->double_wide = true; } + } - if (adjusted_mode->crtc_clock > clock_limit) { - DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n", - adjusted_mode->crtc_clock, clock_limit, - yesno(pipe_config->double_wide)); - return -EINVAL; - } + if (adjusted_mode->crtc_clock > clock_limit) { + DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n", + adjusted_mode->crtc_clock, clock_limit, + yesno(pipe_config->double_wide)); + return -EINVAL; } /* @@ -6592,7 +6706,7 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, * - LVDS dual channel mode * - Double wide pipe */ - if ((intel_pipe_will_have_type(pipe_config, INTEL_OUTPUT_LVDS) && + if ((intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) && intel_is_dual_link_lvds(dev)) || pipe_config->double_wide) pipe_config->pipe_src_w &= ~1; @@ -6615,81 +6729,103 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, static int skylake_get_display_clock_speed(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); - uint32_t lcpll1 = I915_READ(LCPLL1_CTL); - uint32_t cdctl = I915_READ(CDCLK_CTL); - uint32_t linkrate; + uint32_t cdctl; - if (!(lcpll1 & LCPLL_PLL_ENABLE)) - return 24000; /* 24MHz is the cd freq with NSSC ref */ + skl_dpll0_update(dev_priv); - if ((cdctl & CDCLK_FREQ_SEL_MASK) == CDCLK_FREQ_540) - return 540000; + if (dev_priv->cdclk_pll.vco == 0) + return dev_priv->cdclk_pll.ref; - linkrate = (I915_READ(DPLL_CTRL1) & - DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) >> 1; + cdctl = I915_READ(CDCLK_CTL); - if (linkrate == DPLL_CTRL1_LINK_RATE_2160 || - linkrate == DPLL_CTRL1_LINK_RATE_1080) { - /* vco 8640 */ + if (dev_priv->cdclk_pll.vco == 8640000) { switch (cdctl & CDCLK_FREQ_SEL_MASK) { case CDCLK_FREQ_450_432: return 432000; case CDCLK_FREQ_337_308: - return 308570; + return 308571; + case CDCLK_FREQ_540: + return 540000; case CDCLK_FREQ_675_617: - return 617140; + return 617143; default: - WARN(1, "Unknown cd freq selection\n"); + MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK); } } else { - /* vco 8100 */ switch (cdctl & CDCLK_FREQ_SEL_MASK) { case CDCLK_FREQ_450_432: return 450000; case CDCLK_FREQ_337_308: return 337500; + case CDCLK_FREQ_540: + return 540000; case CDCLK_FREQ_675_617: return 675000; default: - WARN(1, "Unknown cd freq selection\n"); + MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK); } } - /* error case, do as if DPLL0 isn't enabled */ - return 24000; + return dev_priv->cdclk_pll.ref; +} + +static void bxt_de_pll_update(struct drm_i915_private *dev_priv) +{ + u32 val; + + dev_priv->cdclk_pll.ref = 19200; + dev_priv->cdclk_pll.vco = 0; + + val = I915_READ(BXT_DE_PLL_ENABLE); + if ((val & BXT_DE_PLL_PLL_ENABLE) == 0) + return; + + if (WARN_ON((val & BXT_DE_PLL_LOCK) == 0)) + return; + + val = I915_READ(BXT_DE_PLL_CTL); + dev_priv->cdclk_pll.vco = (val & BXT_DE_PLL_RATIO_MASK) * + dev_priv->cdclk_pll.ref; } static int broxton_get_display_clock_speed(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); - uint32_t cdctl = I915_READ(CDCLK_CTL); - uint32_t pll_ratio = I915_READ(BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK; - uint32_t pll_enab = I915_READ(BXT_DE_PLL_ENABLE); - int cdclk; + u32 divider; + int div, vco; - if (!(pll_enab & BXT_DE_PLL_PLL_ENABLE)) - return 19200; + bxt_de_pll_update(dev_priv); + + vco = dev_priv->cdclk_pll.vco; + if (vco == 0) + return dev_priv->cdclk_pll.ref; - cdclk = 19200 * pll_ratio / 2; + divider = I915_READ(CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK; - switch (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) { + switch (divider) { case BXT_CDCLK_CD2X_DIV_SEL_1: - return cdclk; /* 576MHz or 624MHz */ + div = 2; + break; case BXT_CDCLK_CD2X_DIV_SEL_1_5: - return cdclk * 2 / 3; /* 384MHz */ + div = 3; + break; case BXT_CDCLK_CD2X_DIV_SEL_2: - return cdclk / 2; /* 288MHz */ + div = 4; + break; case BXT_CDCLK_CD2X_DIV_SEL_4: - return cdclk / 4; /* 144MHz */ + div = 8; + break; + default: + MISSING_CASE(divider); + return dev_priv->cdclk_pll.ref; } - /* error case, do as if DE PLL isn't enabled */ - return 19200; + return DIV_ROUND_CLOSEST(vco, div); } static int broadwell_get_display_clock_speed(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t lcpll = I915_READ(LCPLL_CTL); uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK; @@ -6709,7 +6845,7 @@ static int broadwell_get_display_clock_speed(struct drm_device *dev) static int haswell_get_display_clock_speed(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t lcpll = I915_READ(LCPLL_CTL); uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK; @@ -6843,7 +6979,7 @@ static int i830_get_display_clock_speed(struct drm_device *dev) static unsigned int intel_hpll_vco(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); static const unsigned int blb_vco[8] = { [0] = 3200000, [1] = 4000000, @@ -7063,7 +7199,7 @@ static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll) static void i9xx_update_pll_dividers(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, - intel_clock_t *reduced_clock) + struct dpll *reduced_clock) { struct drm_device *dev = crtc->base.dev; u32 fp, fp2 = 0; @@ -7081,7 +7217,7 @@ static void i9xx_update_pll_dividers(struct intel_crtc *crtc, crtc_state->dpll_hw_state.fp0 = fp; crtc->lowfreq_avail = false; - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) && + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && reduced_clock) { crtc_state->dpll_hw_state.fp1 = fp2; crtc->lowfreq_avail = true; @@ -7123,7 +7259,7 @@ static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc, struct intel_link_m_n *m_n) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = crtc->pipe; I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m); @@ -7137,7 +7273,7 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc, struct intel_link_m_n *m2_n2) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = crtc->pipe; enum transcoder transcoder = crtc->config->cpu_transcoder; @@ -7200,7 +7336,7 @@ static void vlv_compute_dpll(struct intel_crtc *crtc, pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; /* DPLL not used with DSI, but still need the rest set up */ - if (!pipe_config->has_dsi_encoder) + if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI)) pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV; @@ -7217,7 +7353,7 @@ static void chv_compute_dpll(struct intel_crtc *crtc, pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV; /* DPLL not used with DSI, but still need the rest set up */ - if (!pipe_config->has_dsi_encoder) + if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI)) pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE; pipe_config->dpll_hw_state.dpll_md = @@ -7228,7 +7364,7 @@ static void vlv_prepare_pll(struct intel_crtc *crtc, const struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = crtc->pipe; u32 mdiv; u32 bestn, bestm1, bestm2, bestp1, bestp2; @@ -7287,15 +7423,15 @@ static void vlv_prepare_pll(struct intel_crtc *crtc, /* Set HBR and RBR LPF coefficients */ if (pipe_config->port_clock == 162000 || - intel_pipe_has_type(crtc, INTEL_OUTPUT_ANALOG) || - intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI)) + intel_crtc_has_type(crtc->config, INTEL_OUTPUT_ANALOG) || + intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI)) vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe), 0x009f0003); else vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe), 0x00d0000f); - if (pipe_config->has_dp_encoder) { + if (intel_crtc_has_dp_encoder(pipe_config)) { /* Use SSC source */ if (pipe == PIPE_A) vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe), @@ -7315,8 +7451,7 @@ static void vlv_prepare_pll(struct intel_crtc *crtc, coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe)); coreclk = (coreclk & 0x0000ff00) | 0x01c00000; - if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) || - intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP)) + if (intel_crtc_has_dp_encoder(crtc->config)) coreclk |= 0x01000000; vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk); @@ -7328,7 +7463,7 @@ static void chv_prepare_pll(struct intel_crtc *crtc, const struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = crtc->pipe; enum dpio_channel port = vlv_pipe_to_channel(pipe); u32 loopfilter, tribuf_calcntr; @@ -7487,22 +7622,18 @@ void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe) static void i9xx_compute_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, - intel_clock_t *reduced_clock) + struct dpll *reduced_clock) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 dpll; - bool is_sdvo; struct dpll *clock = &crtc_state->dpll; i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock); - is_sdvo = intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO) || - intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI); - dpll = DPLL_VGA_MODE_DIS; - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) dpll |= DPLLB_MODE_LVDS; else dpll |= DPLLB_MODE_DAC_SERIAL; @@ -7512,10 +7643,11 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, << SDVO_MULTIPLIER_SHIFT_HIRES; } - if (is_sdvo) + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) || + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) dpll |= DPLL_SDVO_HIGH_SPEED; - if (crtc_state->has_dp_encoder) + if (intel_crtc_has_dp_encoder(crtc_state)) dpll |= DPLL_SDVO_HIGH_SPEED; /* compute bitmask from p1 value */ @@ -7545,7 +7677,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, if (crtc_state->sdvo_tv_clock) dpll |= PLL_REF_INPUT_TVCLKINBC; - else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) && + else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && intel_panel_use_ssc(dev_priv)) dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN; else @@ -7563,10 +7695,10 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, static void i8xx_compute_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, - intel_clock_t *reduced_clock) + struct dpll *reduced_clock) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 dpll; struct dpll *clock = &crtc_state->dpll; @@ -7574,7 +7706,7 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc, dpll = DPLL_VGA_MODE_DIS; - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) { + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; } else { if (clock->p1 == 2) @@ -7585,10 +7717,10 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc, dpll |= PLL_P2_DIVIDE_BY_4; } - if (!IS_I830(dev) && intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO)) + if (!IS_I830(dev) && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) dpll |= DPLL_DVO_2X_MODE; - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) && + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && intel_panel_use_ssc(dev_priv)) dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN; else @@ -7601,7 +7733,7 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc, static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = intel_crtc->pipe; enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; @@ -7618,7 +7750,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) crtc_vtotal -= 1; crtc_vblank_end -= 1; - if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO)) + if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO)) vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2; else vsyncshift = adjusted_mode->crtc_hsync_start - @@ -7663,7 +7795,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc) static void intel_set_pipe_src_size(struct intel_crtc *intel_crtc) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = intel_crtc->pipe; /* pipesrc controls the size that is scaled from, which should @@ -7678,7 +7810,7 @@ static void intel_get_pipe_timings(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; uint32_t tmp; @@ -7713,7 +7845,7 @@ static void intel_get_pipe_src_size(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 tmp; tmp = I915_READ(PIPESRC(crtc->pipe)); @@ -7751,7 +7883,7 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode, static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t pipeconf; pipeconf = 0; @@ -7797,7 +7929,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc) if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) { if (INTEL_INFO(dev)->gen < 4 || - intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO)) + intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO)) pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION; else pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT; @@ -7816,21 +7948,21 @@ static int i8xx_crtc_compute_clock(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - const intel_limit_t *limit; + struct drm_i915_private *dev_priv = to_i915(dev); + const struct intel_limit *limit; int refclk = 48000; memset(&crtc_state->dpll_hw_state, 0, sizeof(crtc_state->dpll_hw_state)); - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) { + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if (intel_panel_use_ssc(dev_priv)) { refclk = dev_priv->vbt.lvds_ssc_freq; DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk); } limit = &intel_limits_i8xx_lvds; - } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO)) { + } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) { limit = &intel_limits_i8xx_dvo; } else { limit = &intel_limits_i8xx_dac; @@ -7852,14 +7984,14 @@ static int g4x_crtc_compute_clock(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - const intel_limit_t *limit; + struct drm_i915_private *dev_priv = to_i915(dev); + const struct intel_limit *limit; int refclk = 96000; memset(&crtc_state->dpll_hw_state, 0, sizeof(crtc_state->dpll_hw_state)); - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) { + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if (intel_panel_use_ssc(dev_priv)) { refclk = dev_priv->vbt.lvds_ssc_freq; DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk); @@ -7869,10 +8001,10 @@ static int g4x_crtc_compute_clock(struct intel_crtc *crtc, limit = &intel_limits_g4x_dual_channel_lvds; else limit = &intel_limits_g4x_single_channel_lvds; - } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI) || - intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_ANALOG)) { + } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) || + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) { limit = &intel_limits_g4x_hdmi; - } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO)) { + } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) { limit = &intel_limits_g4x_sdvo; } else { /* The option is for other outputs */ @@ -7895,14 +8027,14 @@ static int pnv_crtc_compute_clock(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - const intel_limit_t *limit; + struct drm_i915_private *dev_priv = to_i915(dev); + const struct intel_limit *limit; int refclk = 96000; memset(&crtc_state->dpll_hw_state, 0, sizeof(crtc_state->dpll_hw_state)); - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) { + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if (intel_panel_use_ssc(dev_priv)) { refclk = dev_priv->vbt.lvds_ssc_freq; DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk); @@ -7929,14 +8061,14 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - const intel_limit_t *limit; + struct drm_i915_private *dev_priv = to_i915(dev); + const struct intel_limit *limit; int refclk = 96000; memset(&crtc_state->dpll_hw_state, 0, sizeof(crtc_state->dpll_hw_state)); - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) { + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if (intel_panel_use_ssc(dev_priv)) { refclk = dev_priv->vbt.lvds_ssc_freq; DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk); @@ -7963,7 +8095,7 @@ static int chv_crtc_compute_clock(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state) { int refclk = 100000; - const intel_limit_t *limit = &intel_limits_chv; + const struct intel_limit *limit = &intel_limits_chv; memset(&crtc_state->dpll_hw_state, 0, sizeof(crtc_state->dpll_hw_state)); @@ -7984,7 +8116,7 @@ static int vlv_crtc_compute_clock(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state) { int refclk = 100000; - const intel_limit_t *limit = &intel_limits_vlv; + const struct intel_limit *limit = &intel_limits_vlv; memset(&crtc_state->dpll_hw_state, 0, sizeof(crtc_state->dpll_hw_state)); @@ -8005,7 +8137,7 @@ static void i9xx_get_pfit_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t tmp; if (INTEL_INFO(dev)->gen <= 3 && (IS_I830(dev) || !IS_MOBILE(dev))) @@ -8032,9 +8164,9 @@ static void vlv_crtc_clock_get(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = pipe_config->cpu_transcoder; - intel_clock_t clock; + struct dpll clock; u32 mdiv; int refclk = 100000; @@ -8060,7 +8192,7 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc, struct intel_initial_plane_config *plane_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val, base, offset; int pipe = crtc->pipe, plane = crtc->plane; int fourcc, pixel_format; @@ -8128,10 +8260,10 @@ static void chv_crtc_clock_get(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = pipe_config->cpu_transcoder; enum dpio_channel port = vlv_pipe_to_channel(pipe); - intel_clock_t clock; + struct dpll clock; u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3; int refclk = 100000; @@ -8162,7 +8294,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; uint32_t tmp; bool ret; @@ -8273,7 +8405,7 @@ out: static void ironlake_init_pch_refclk(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *encoder; int i; u32 val, final; @@ -8544,7 +8676,7 @@ static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv) static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread, bool with_fdi) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t reg, tmp; if (WARN(with_fdi && !with_spread, "FDI requires downspread\n")) @@ -8583,7 +8715,7 @@ static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread, /* Sequence to disable CLKOUT_DP */ static void lpt_disable_clkout_dp(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t reg, tmp; mutex_lock(&dev_priv->sb_lock); @@ -8704,7 +8836,7 @@ void intel_init_pch_refclk(struct drm_device *dev) static void ironlake_set_pipeconf(struct drm_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; uint32_t val; @@ -8746,7 +8878,7 @@ static void ironlake_set_pipeconf(struct drm_crtc *crtc) static void haswell_set_pipeconf(struct drm_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; u32 val = 0; @@ -8765,7 +8897,7 @@ static void haswell_set_pipeconf(struct drm_crtc *crtc) static void haswell_set_pipemisc(struct drm_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); if (IS_BROADWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 9) { @@ -8814,41 +8946,17 @@ static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor) static void ironlake_compute_dpll(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state, - intel_clock_t *reduced_clock) + struct dpll *reduced_clock) { struct drm_crtc *crtc = &intel_crtc->base; struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_atomic_state *state = crtc_state->base.state; - struct drm_connector *connector; - struct drm_connector_state *connector_state; - struct intel_encoder *encoder; + struct drm_i915_private *dev_priv = to_i915(dev); u32 dpll, fp, fp2; - int factor, i; - bool is_lvds = false, is_sdvo = false; - - for_each_connector_in_state(state, connector, connector_state, i) { - if (connector_state->crtc != crtc_state->base.crtc) - continue; - - encoder = to_intel_encoder(connector_state->best_encoder); - - switch (encoder->type) { - case INTEL_OUTPUT_LVDS: - is_lvds = true; - break; - case INTEL_OUTPUT_SDVO: - case INTEL_OUTPUT_HDMI: - is_sdvo = true; - break; - default: - break; - } - } + int factor; /* Enable autotuning of the PLL clock (if permissible) */ factor = 21; - if (is_lvds) { + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if ((intel_panel_use_ssc(dev_priv) && dev_priv->vbt.lvds_ssc_freq == 100000) || (HAS_PCH_IBX(dev) && intel_is_dual_link_lvds(dev))) @@ -8872,7 +8980,7 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc, dpll = 0; - if (is_lvds) + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) dpll |= DPLLB_MODE_LVDS; else dpll |= DPLLB_MODE_DAC_SERIAL; @@ -8880,9 +8988,11 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc, dpll |= (crtc_state->pixel_multiplier - 1) << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT; - if (is_sdvo) + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) || + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) dpll |= DPLL_SDVO_HIGH_SPEED; - if (crtc_state->has_dp_encoder) + + if (intel_crtc_has_dp_encoder(crtc_state)) dpll |= DPLL_SDVO_HIGH_SPEED; /* compute bitmask from p1 value */ @@ -8905,7 +9015,8 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc, break; } - if (is_lvds && intel_panel_use_ssc(dev_priv)) + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && + intel_panel_use_ssc(dev_priv)) dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN; else dpll |= PLL_REF_INPUT_DREFCLK; @@ -8921,11 +9032,11 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - intel_clock_t reduced_clock; + struct drm_i915_private *dev_priv = to_i915(dev); + struct dpll reduced_clock; bool has_reduced_clock = false; struct intel_shared_dpll *pll; - const intel_limit_t *limit; + const struct intel_limit *limit; int refclk = 120000; memset(&crtc_state->dpll_hw_state, 0, @@ -8937,7 +9048,7 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc, if (!crtc_state->has_pch_encoder) return 0; - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) { + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if (intel_panel_use_ssc(dev_priv)) { DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", dev_priv->vbt.lvds_ssc_freq); @@ -8976,7 +9087,7 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc, return -EINVAL; } - if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) && + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) && has_reduced_clock) crtc->lowfreq_avail = true; @@ -8987,7 +9098,7 @@ static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc, struct intel_link_m_n *m_n) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = crtc->pipe; m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe)); @@ -9005,7 +9116,7 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc, struct intel_link_m_n *m2_n2) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = crtc->pipe; if (INTEL_INFO(dev)->gen >= 5) { @@ -9063,7 +9174,7 @@ static void skylake_get_pfit_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state; uint32_t ps_ctrl = 0; int id = -1; @@ -9094,7 +9205,7 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc, struct intel_initial_plane_config *plane_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val, base, offset, stride_mult, tiling; int pipe = crtc->pipe; int fourcc, pixel_format; @@ -9177,7 +9288,7 @@ static void ironlake_get_pfit_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t tmp; tmp = I915_READ(PF_CTL(crtc->pipe)); @@ -9202,7 +9313,7 @@ ironlake_get_initial_plane_config(struct intel_crtc *crtc, struct intel_initial_plane_config *plane_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val, base, offset; int pipe = crtc->pipe; int fourcc, pixel_format; @@ -9270,7 +9381,7 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; uint32_t tmp; bool ret; @@ -9320,6 +9431,10 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc, ironlake_get_fdi_m_n_config(crtc, pipe_config); if (HAS_PCH_IBX(dev_priv)) { + /* + * The pipe->pch transcoder and pch transcoder->pll + * mapping is fixed. + */ pll_id = (enum intel_dpll_id) crtc->pipe; } else { tmp = I915_READ(PCH_DPLL_SEL); @@ -9361,7 +9476,7 @@ out: static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct intel_crtc *crtc; for_each_intel_crtc(dev, crtc) @@ -9395,7 +9510,7 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv) static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; if (IS_HASWELL(dev)) return I915_READ(D_COMP_HSW); @@ -9405,7 +9520,7 @@ static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv) static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; if (IS_HASWELL(dev)) { mutex_lock(&dev_priv->rps.hw_lock); @@ -9451,7 +9566,7 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv, I915_WRITE(LCPLL_CTL, val); POSTING_READ(LCPLL_CTL); - if (wait_for((I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK) == 0, 1)) + if (intel_wait_for_register(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 0, 1)) DRM_ERROR("LCPLL still locked\n"); val = hsw_read_dcomp(dev_priv); @@ -9506,7 +9621,9 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) val &= ~LCPLL_PLL_DISABLE; I915_WRITE(LCPLL_CTL, val); - if (wait_for(I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK, 5)) + if (intel_wait_for_register(dev_priv, + LCPLL_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK, + 5)) DRM_ERROR("LCPLL not locked yet\n"); if (val & LCPLL_CD_SOURCE_FCLK) { @@ -9520,7 +9637,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) } intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); - intel_update_cdclk(dev_priv->dev); + intel_update_cdclk(&dev_priv->drm); } /* @@ -9548,7 +9665,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) */ void hsw_enable_pc8(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; uint32_t val; DRM_DEBUG_KMS("Enabling package C8+\n"); @@ -9565,7 +9682,7 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv) void hsw_disable_pc8(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; uint32_t val; DRM_DEBUG_KMS("Disabling package C8+\n"); @@ -9580,21 +9697,21 @@ void hsw_disable_pc8(struct drm_i915_private *dev_priv) } } -static void broxton_modeset_commit_cdclk(struct drm_atomic_state *old_state) +static void bxt_modeset_commit_cdclk(struct drm_atomic_state *old_state) { struct drm_device *dev = old_state->dev; struct intel_atomic_state *old_intel_state = to_intel_atomic_state(old_state); unsigned int req_cdclk = old_intel_state->dev_cdclk; - broxton_set_cdclk(to_i915(dev), req_cdclk); + bxt_set_cdclk(to_i915(dev), req_cdclk); } /* compute the max rate for new configuration */ static int ilk_max_pixel_rate(struct drm_atomic_state *state) { struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct drm_i915_private *dev_priv = state->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(state->dev); struct drm_crtc *crtc; struct drm_crtc_state *cstate; struct intel_crtc_state *crtc_state; @@ -9630,7 +9747,7 @@ static int ilk_max_pixel_rate(struct drm_atomic_state *state) static void broadwell_set_cdclk(struct drm_device *dev, int cdclk) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t val, data; int ret; @@ -9700,8 +9817,6 @@ static void broadwell_set_cdclk(struct drm_device *dev, int cdclk) I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1); - I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1); - intel_update_cdclk(dev); WARN(cdclk != dev_priv->cdclk_freq, @@ -9709,6 +9824,18 @@ static void broadwell_set_cdclk(struct drm_device *dev, int cdclk) cdclk, dev_priv->cdclk_freq); } +static int broadwell_calc_cdclk(int max_pixclk) +{ + if (max_pixclk > 540000) + return 675000; + else if (max_pixclk > 450000) + return 540000; + else if (max_pixclk > 337500) + return 450000; + else + return 337500; +} + static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(state->dev); @@ -9720,14 +9847,7 @@ static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state) * FIXME should also account for plane ratio * once 64bpp pixel formats are supported. */ - if (max_pixclk > 540000) - cdclk = 675000; - else if (max_pixclk > 450000) - cdclk = 540000; - else if (max_pixclk > 337500) - cdclk = 450000; - else - cdclk = 337500; + cdclk = broadwell_calc_cdclk(max_pixclk); if (cdclk > dev_priv->max_cdclk_freq) { DRM_DEBUG_KMS("requested cdclk (%d kHz) exceeds max (%d kHz)\n", @@ -9737,7 +9857,7 @@ static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state) intel_state->cdclk = intel_state->dev_cdclk = cdclk; if (!intel_state->active_crtcs) - intel_state->dev_cdclk = 337500; + intel_state->dev_cdclk = broadwell_calc_cdclk(0); return 0; } @@ -9752,13 +9872,51 @@ static void broadwell_modeset_commit_cdclk(struct drm_atomic_state *old_state) broadwell_set_cdclk(dev, req_cdclk); } +static int skl_modeset_calc_cdclk(struct drm_atomic_state *state) +{ + struct intel_atomic_state *intel_state = to_intel_atomic_state(state); + struct drm_i915_private *dev_priv = to_i915(state->dev); + const int max_pixclk = ilk_max_pixel_rate(state); + int vco = intel_state->cdclk_pll_vco; + int cdclk; + + /* + * FIXME should also account for plane ratio + * once 64bpp pixel formats are supported. + */ + cdclk = skl_calc_cdclk(max_pixclk, vco); + + /* + * FIXME move the cdclk caclulation to + * compute_config() so we can fail gracegully. + */ + if (cdclk > dev_priv->max_cdclk_freq) { + DRM_ERROR("requested cdclk (%d kHz) exceeds max (%d kHz)\n", + cdclk, dev_priv->max_cdclk_freq); + cdclk = dev_priv->max_cdclk_freq; + } + + intel_state->cdclk = intel_state->dev_cdclk = cdclk; + if (!intel_state->active_crtcs) + intel_state->dev_cdclk = skl_calc_cdclk(0, vco); + + return 0; +} + +static void skl_modeset_commit_cdclk(struct drm_atomic_state *old_state) +{ + struct drm_i915_private *dev_priv = to_i915(old_state->dev); + struct intel_atomic_state *intel_state = to_intel_atomic_state(old_state); + unsigned int req_cdclk = intel_state->dev_cdclk; + unsigned int req_vco = intel_state->cdclk_pll_vco; + + skl_set_cdclk(dev_priv, req_cdclk, req_vco); +} + static int haswell_crtc_compute_clock(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state) { - struct intel_encoder *intel_encoder = - intel_ddi_get_crtc_new_encoder(crtc_state); - - if (intel_encoder->type != INTEL_OUTPUT_DSI) { + if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) { if (!intel_ddi_pll_select(crtc, crtc_state)) return -EINVAL; } @@ -9868,10 +10026,14 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc, unsigned long *power_domain_mask) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; u32 tmp; + /* + * The pipe->transcoder mapping is fixed with the exception of the eDP + * transcoder handled below. + */ pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; /* @@ -9915,14 +10077,12 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc, unsigned long *power_domain_mask) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; enum port port; enum transcoder cpu_transcoder; u32 tmp; - pipe_config->has_dsi_encoder = false; - for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) { if (port == PORT_A) cpu_transcoder = TRANSCODER_DSI_A; @@ -9954,18 +10114,17 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc, continue; pipe_config->cpu_transcoder = cpu_transcoder; - pipe_config->has_dsi_encoder = true; break; } - return pipe_config->has_dsi_encoder; + return transcoder_is_dsi(pipe_config->cpu_transcoder); } static void haswell_get_ddi_port_state(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_shared_dpll *pll; enum port port; uint32_t tmp; @@ -10008,7 +10167,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; unsigned long power_domain_mask; bool active; @@ -10022,18 +10181,16 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_mask); - if (IS_BROXTON(dev_priv)) { - bxt_get_dsi_transcoder_state(crtc, pipe_config, - &power_domain_mask); - WARN_ON(active && pipe_config->has_dsi_encoder); - if (pipe_config->has_dsi_encoder) - active = true; + if (IS_BROXTON(dev_priv) && + bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_mask)) { + WARN_ON(active); + active = true; } if (!active) goto out; - if (!pipe_config->has_dsi_encoder) { + if (!transcoder_is_dsi(pipe_config->cpu_transcoder)) { haswell_get_ddi_port_state(crtc, pipe_config); intel_get_pipe_timings(crtc, pipe_config); } @@ -10084,7 +10241,7 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base, const struct intel_plane_state *plane_state) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t cntl = 0, size = 0; @@ -10147,7 +10304,7 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base, const struct intel_plane_state *plane_state) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; uint32_t cntl = 0; @@ -10195,7 +10352,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc, const struct intel_plane_state *plane_state) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; u32 base = intel_crtc->cursor_addr; @@ -10339,10 +10496,10 @@ intel_framebuffer_create_for_mode(struct drm_device *dev, struct drm_i915_gem_object *obj; struct drm_mode_fb_cmd2 mode_cmd = { 0 }; - obj = i915_gem_alloc_object(dev, + obj = i915_gem_object_create(dev, intel_framebuffer_size_for_mode(mode, bpp)); - if (obj == NULL) - return ERR_PTR(-ENOMEM); + if (IS_ERR(obj)) + return ERR_CAST(obj); mode_cmd.width = mode->hdisplay; mode_cmd.height = mode->vdisplay; @@ -10362,7 +10519,7 @@ mode_fits_in_fbdev(struct drm_device *dev, struct drm_display_mode *mode) { #ifdef CONFIG_DRM_FBDEV_EMULATION - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; struct drm_framebuffer *fb; @@ -10632,7 +10789,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector, static int i9xx_pll_refclk(struct drm_device *dev, const struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 dpll = pipe_config->dpll_hw_state.dpll; if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN) @@ -10650,11 +10807,11 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe = pipe_config->cpu_transcoder; u32 dpll = pipe_config->dpll_hw_state.dpll; u32 fp; - intel_clock_t clock; + struct dpll clock; int port_clock; int refclk = i9xx_pll_refclk(dev, pipe_config); @@ -10776,7 +10933,7 @@ static void ironlake_pch_clock_get(struct intel_crtc *crtc, struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev, struct drm_crtc *crtc) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; struct drm_display_mode *mode; @@ -10828,48 +10985,20 @@ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev, return mode; } -void intel_mark_busy(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - if (dev_priv->mm.busy) - return; - - intel_runtime_pm_get(dev_priv); - i915_update_gfx_val(dev_priv); - if (INTEL_INFO(dev)->gen >= 6) - gen6_rps_busy(dev_priv); - dev_priv->mm.busy = true; -} - -void intel_mark_idle(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - - if (!dev_priv->mm.busy) - return; - - dev_priv->mm.busy = false; - - if (INTEL_INFO(dev)->gen >= 6) - gen6_rps_idle(dev->dev_private); - - intel_runtime_pm_put(dev_priv); -} - static void intel_crtc_destroy(struct drm_crtc *crtc) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_device *dev = crtc->dev; - struct intel_unpin_work *work; + struct intel_flip_work *work; spin_lock_irq(&dev->event_lock); - work = intel_crtc->unpin_work; - intel_crtc->unpin_work = NULL; + work = intel_crtc->flip_work; + intel_crtc->flip_work = NULL; spin_unlock_irq(&dev->event_lock); if (work) { - cancel_work_sync(&work->work); + cancel_work_sync(&work->mmio_work); + cancel_work_sync(&work->unpin_work); kfree(work); } @@ -10880,12 +11009,15 @@ static void intel_crtc_destroy(struct drm_crtc *crtc) static void intel_unpin_work_fn(struct work_struct *__work) { - struct intel_unpin_work *work = - container_of(__work, struct intel_unpin_work, work); + struct intel_flip_work *work = + container_of(__work, struct intel_flip_work, unpin_work); struct intel_crtc *crtc = to_intel_crtc(work->crtc); struct drm_device *dev = crtc->base.dev; struct drm_plane *primary = crtc->base.primary; + if (is_mmio_work(work)) + flush_work(&work->mmio_work); + mutex_lock(&dev->struct_mutex); intel_unpin_fb_obj(work->old_fb, primary->state->rotation); drm_gem_object_unreference(&work->pending_flip_obj->base); @@ -10904,63 +11036,17 @@ static void intel_unpin_work_fn(struct work_struct *__work) kfree(work); } -static void do_intel_finish_page_flip(struct drm_device *dev, - struct drm_crtc *crtc) -{ - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_unpin_work *work; - unsigned long flags; - - /* Ignore early vblank irqs */ - if (intel_crtc == NULL) - return; - - /* - * This is called both by irq handlers and the reset code (to complete - * lost pageflips) so needs the full irqsave spinlocks. - */ - spin_lock_irqsave(&dev->event_lock, flags); - work = intel_crtc->unpin_work; - - /* Ensure we don't miss a work->pending update ... */ - smp_rmb(); - - if (work == NULL || atomic_read(&work->pending) < INTEL_FLIP_COMPLETE) { - spin_unlock_irqrestore(&dev->event_lock, flags); - return; - } - - page_flip_completed(intel_crtc); - - spin_unlock_irqrestore(&dev->event_lock, flags); -} - -void intel_finish_page_flip(struct drm_device *dev, int pipe) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - - do_intel_finish_page_flip(dev, crtc); -} - -void intel_finish_page_flip_plane(struct drm_device *dev, int plane) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane]; - - do_intel_finish_page_flip(dev, crtc); -} - /* Is 'a' after or equal to 'b'? */ static bool g4x_flip_count_after_eq(u32 a, u32 b) { return !((a - b) & 0x80000000); } -static bool page_flip_finished(struct intel_crtc *crtc) +static bool __pageflip_finished_cs(struct intel_crtc *crtc, + struct intel_flip_work *work) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned reset_counter; reset_counter = i915_reset_counter(&dev_priv->gpu_error); @@ -10999,40 +11085,103 @@ static bool page_flip_finished(struct intel_crtc *crtc) * anyway, we don't really care. */ return (I915_READ(DSPSURFLIVE(crtc->plane)) & ~0xfff) == - crtc->unpin_work->gtt_offset && + crtc->flip_work->gtt_offset && g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_G4X(crtc->pipe)), - crtc->unpin_work->flip_count); + crtc->flip_work->flip_count); } -void intel_prepare_page_flip(struct drm_device *dev, int plane) +static bool +__pageflip_finished_mmio(struct intel_crtc *crtc, + struct intel_flip_work *work) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = - to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]); + /* + * MMIO work completes when vblank is different from + * flip_queued_vblank. + * + * Reset counter value doesn't matter, this is handled by + * i915_wait_request finishing early, so no need to handle + * reset here. + */ + return intel_crtc_get_vblank_counter(crtc) != work->flip_queued_vblank; +} + + +static bool pageflip_finished(struct intel_crtc *crtc, + struct intel_flip_work *work) +{ + if (!atomic_read(&work->pending)) + return false; + + smp_rmb(); + + if (is_mmio_work(work)) + return __pageflip_finished_mmio(crtc, work); + else + return __pageflip_finished_cs(crtc, work); +} + +void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) +{ + struct drm_device *dev = &dev_priv->drm; + struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_flip_work *work; unsigned long flags; + /* Ignore early vblank irqs */ + if (!crtc) + return; /* * This is called both by irq handlers and the reset code (to complete * lost pageflips) so needs the full irqsave spinlocks. - * - * NB: An MMIO update of the plane base pointer will also - * generate a page-flip completion irq, i.e. every modeset - * is also accompanied by a spurious intel_prepare_page_flip(). */ spin_lock_irqsave(&dev->event_lock, flags); - if (intel_crtc->unpin_work && page_flip_finished(intel_crtc)) - atomic_inc_not_zero(&intel_crtc->unpin_work->pending); + work = intel_crtc->flip_work; + + if (work != NULL && + !is_mmio_work(work) && + pageflip_finished(intel_crtc, work)) + page_flip_completed(intel_crtc); + + spin_unlock_irqrestore(&dev->event_lock, flags); +} + +void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe) +{ + struct drm_device *dev = &dev_priv->drm; + struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_flip_work *work; + unsigned long flags; + + /* Ignore early vblank irqs */ + if (!crtc) + return; + + /* + * This is called both by irq handlers and the reset code (to complete + * lost pageflips) so needs the full irqsave spinlocks. + */ + spin_lock_irqsave(&dev->event_lock, flags); + work = intel_crtc->flip_work; + + if (work != NULL && + is_mmio_work(work) && + pageflip_finished(intel_crtc, work)) + page_flip_completed(intel_crtc); + spin_unlock_irqrestore(&dev->event_lock, flags); } -static inline void intel_mark_page_flip_active(struct intel_unpin_work *work) +static inline void intel_mark_page_flip_active(struct intel_crtc *crtc, + struct intel_flip_work *work) { + work->flip_queued_vblank = intel_crtc_get_vblank_counter(crtc); + /* Ensure that the work item is consistent when activating it ... */ - smp_wmb(); - atomic_set(&work->pending, INTEL_FLIP_PENDING); - /* and that it is marked active as soon as the irq could fire. */ - smp_wmb(); + smp_mb__before_atomic(); + atomic_set(&work->pending, 1); } static int intel_gen2_queue_flip(struct drm_device *dev, @@ -11063,10 +11212,9 @@ static int intel_gen2_queue_flip(struct drm_device *dev, intel_ring_emit(engine, MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(engine, fb->pitches[0]); - intel_ring_emit(engine, intel_crtc->unpin_work->gtt_offset); + intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset); intel_ring_emit(engine, 0); /* aux display base address, unused */ - intel_mark_page_flip_active(intel_crtc->unpin_work); return 0; } @@ -11095,10 +11243,9 @@ static int intel_gen3_queue_flip(struct drm_device *dev, intel_ring_emit(engine, MI_DISPLAY_FLIP_I915 | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(engine, fb->pitches[0]); - intel_ring_emit(engine, intel_crtc->unpin_work->gtt_offset); + intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset); intel_ring_emit(engine, MI_NOOP); - intel_mark_page_flip_active(intel_crtc->unpin_work); return 0; } @@ -11110,7 +11257,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev, uint32_t flags) { struct intel_engine_cs *engine = req->engine; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t pf, pipesrc; int ret; @@ -11126,7 +11273,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev, intel_ring_emit(engine, MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(engine, fb->pitches[0]); - intel_ring_emit(engine, intel_crtc->unpin_work->gtt_offset | + intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset | obj->tiling_mode); /* XXX Enabling the panel-fitter across page-flip is so far @@ -11137,7 +11284,6 @@ static int intel_gen4_queue_flip(struct drm_device *dev, pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff; intel_ring_emit(engine, pf | pipesrc); - intel_mark_page_flip_active(intel_crtc->unpin_work); return 0; } @@ -11149,7 +11295,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev, uint32_t flags) { struct intel_engine_cs *engine = req->engine; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); uint32_t pf, pipesrc; int ret; @@ -11161,7 +11307,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev, intel_ring_emit(engine, MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); intel_ring_emit(engine, fb->pitches[0] | obj->tiling_mode); - intel_ring_emit(engine, intel_crtc->unpin_work->gtt_offset); + intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset); /* Contrary to the suggestions in the documentation, * "Enable Panel Fitter" does not seem to be required when page @@ -11173,7 +11319,6 @@ static int intel_gen6_queue_flip(struct drm_device *dev, pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff; intel_ring_emit(engine, pf | pipesrc); - intel_mark_page_flip_active(intel_crtc->unpin_work); return 0; } @@ -11265,16 +11410,17 @@ static int intel_gen7_queue_flip(struct drm_device *dev, intel_ring_emit(engine, MI_DISPLAY_FLIP_I915 | plane_bit); intel_ring_emit(engine, (fb->pitches[0] | obj->tiling_mode)); - intel_ring_emit(engine, intel_crtc->unpin_work->gtt_offset); + intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset); intel_ring_emit(engine, (MI_NOOP)); - intel_mark_page_flip_active(intel_crtc->unpin_work); return 0; } static bool use_mmio_flip(struct intel_engine_cs *engine, struct drm_i915_gem_object *obj) { + struct reservation_object *resv; + /* * This is not being used for older platforms, because * non-availability of flip done interrupt forces us to use @@ -11286,7 +11432,7 @@ static bool use_mmio_flip(struct intel_engine_cs *engine, if (engine == NULL) return true; - if (INTEL_INFO(engine->dev)->gen < 5) + if (INTEL_GEN(engine->i915) < 5) return false; if (i915.use_mmio_flip < 0) @@ -11295,20 +11441,20 @@ static bool use_mmio_flip(struct intel_engine_cs *engine, return true; else if (i915.enable_execlists) return true; - else if (obj->base.dma_buf && - !reservation_object_test_signaled_rcu(obj->base.dma_buf->resv, - false)) + + resv = i915_gem_object_get_dmabuf_resv(obj); + if (resv && !reservation_object_test_signaled_rcu(resv, false)) return true; - else - return engine != i915_gem_request_get_engine(obj->last_write_req); + + return engine != i915_gem_request_get_engine(obj->last_write_req); } static void skl_do_mmio_flip(struct intel_crtc *intel_crtc, unsigned int rotation, - struct intel_unpin_work *work) + struct intel_flip_work *work) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_framebuffer *fb = intel_crtc->base.primary->fb; const enum pipe pipe = intel_crtc->pipe; u32 ctl, stride, tile_height; @@ -11357,10 +11503,10 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc, } static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc, - struct intel_unpin_work *work) + struct intel_flip_work *work) { struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_framebuffer *intel_fb = to_intel_framebuffer(intel_crtc->base.primary->fb); struct drm_i915_gem_object *obj = intel_fb->obj; @@ -11380,78 +11526,37 @@ static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc, POSTING_READ(DSPSURF(intel_crtc->plane)); } -/* - * XXX: This is the temporary way to update the plane registers until we get - * around to using the usual plane update functions for MMIO flips - */ -static void intel_do_mmio_flip(struct intel_mmio_flip *mmio_flip) -{ - struct intel_crtc *crtc = mmio_flip->crtc; - struct intel_unpin_work *work; - - spin_lock_irq(&crtc->base.dev->event_lock); - work = crtc->unpin_work; - spin_unlock_irq(&crtc->base.dev->event_lock); - if (work == NULL) - return; - - intel_mark_page_flip_active(work); - - intel_pipe_update_start(crtc); - - if (INTEL_INFO(mmio_flip->i915)->gen >= 9) - skl_do_mmio_flip(crtc, mmio_flip->rotation, work); - else - /* use_mmio_flip() retricts MMIO flips to ilk+ */ - ilk_do_mmio_flip(crtc, work); - - intel_pipe_update_end(crtc); -} - -static void intel_mmio_flip_work_func(struct work_struct *work) +static void intel_mmio_flip_work_func(struct work_struct *w) { - struct intel_mmio_flip *mmio_flip = - container_of(work, struct intel_mmio_flip, work); + struct intel_flip_work *work = + container_of(w, struct intel_flip_work, mmio_work); + struct intel_crtc *crtc = to_intel_crtc(work->crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_framebuffer *intel_fb = - to_intel_framebuffer(mmio_flip->crtc->base.primary->fb); + to_intel_framebuffer(crtc->base.primary->fb); struct drm_i915_gem_object *obj = intel_fb->obj; + struct reservation_object *resv; - if (mmio_flip->req) { - WARN_ON(__i915_wait_request(mmio_flip->req, + if (work->flip_queued_req) + WARN_ON(__i915_wait_request(work->flip_queued_req, false, NULL, - &mmio_flip->i915->rps.mmioflips)); - i915_gem_request_unreference__unlocked(mmio_flip->req); - } + &dev_priv->rps.mmioflips)); /* For framebuffer backed by dmabuf, wait for fence */ - if (obj->base.dma_buf) - WARN_ON(reservation_object_wait_timeout_rcu(obj->base.dma_buf->resv, - false, false, + resv = i915_gem_object_get_dmabuf_resv(obj); + if (resv) + WARN_ON(reservation_object_wait_timeout_rcu(resv, false, false, MAX_SCHEDULE_TIMEOUT) < 0); - intel_do_mmio_flip(mmio_flip); - kfree(mmio_flip); -} - -static int intel_queue_mmio_flip(struct drm_device *dev, - struct drm_crtc *crtc, - struct drm_i915_gem_object *obj) -{ - struct intel_mmio_flip *mmio_flip; - - mmio_flip = kmalloc(sizeof(*mmio_flip), GFP_KERNEL); - if (mmio_flip == NULL) - return -ENOMEM; - - mmio_flip->i915 = to_i915(dev); - mmio_flip->req = i915_gem_request_reference(obj->last_write_req); - mmio_flip->crtc = to_intel_crtc(crtc); - mmio_flip->rotation = crtc->primary->state->rotation; + intel_pipe_update_start(crtc); - INIT_WORK(&mmio_flip->work, intel_mmio_flip_work_func); - schedule_work(&mmio_flip->work); + if (INTEL_GEN(dev_priv) >= 9) + skl_do_mmio_flip(crtc, work->rotation, work); + else + /* use_mmio_flip() retricts MMIO flips to ilk+ */ + ilk_do_mmio_flip(crtc, work); - return 0; + intel_pipe_update_end(crtc, work); } static int intel_default_queue_flip(struct drm_device *dev, @@ -11464,37 +11569,32 @@ static int intel_default_queue_flip(struct drm_device *dev, return -ENODEV; } -static bool __intel_pageflip_stall_check(struct drm_device *dev, - struct drm_crtc *crtc) +static bool __pageflip_stall_check_cs(struct drm_i915_private *dev_priv, + struct intel_crtc *intel_crtc, + struct intel_flip_work *work) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_unpin_work *work = intel_crtc->unpin_work; - u32 addr; - - if (atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE) - return true; + u32 addr, vblank; - if (atomic_read(&work->pending) < INTEL_FLIP_PENDING) + if (!atomic_read(&work->pending)) return false; - if (!work->enable_stall_check) - return false; + smp_rmb(); + vblank = intel_crtc_get_vblank_counter(intel_crtc); if (work->flip_ready_vblank == 0) { if (work->flip_queued_req && - !i915_gem_request_completed(work->flip_queued_req, true)) + !i915_gem_request_completed(work->flip_queued_req)) return false; - work->flip_ready_vblank = drm_crtc_vblank_count(crtc); + work->flip_ready_vblank = vblank; } - if (drm_crtc_vblank_count(crtc) - work->flip_ready_vblank < 3) + if (vblank - work->flip_ready_vblank < 3) return false; /* Potential stall - if we see that the flip has happened, * assume a missed interrupt. */ - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) addr = I915_HI_DISPBASE(I915_READ(DSPSURF(intel_crtc->plane))); else addr = I915_READ(DSPADDR(intel_crtc->plane)); @@ -11506,12 +11606,12 @@ static bool __intel_pageflip_stall_check(struct drm_device *dev, return addr == work->gtt_offset; } -void intel_check_page_flip(struct drm_device *dev, int pipe) +void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_device *dev = &dev_priv->drm; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_unpin_work *work; + struct intel_flip_work *work; WARN_ON(!in_interrupt()); @@ -11519,16 +11619,20 @@ void intel_check_page_flip(struct drm_device *dev, int pipe) return; spin_lock(&dev->event_lock); - work = intel_crtc->unpin_work; - if (work != NULL && __intel_pageflip_stall_check(dev, crtc)) { - WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n", - work->flip_queued_vblank, drm_vblank_count(dev, pipe)); + work = intel_crtc->flip_work; + + if (work != NULL && !is_mmio_work(work) && + __pageflip_stall_check_cs(dev_priv, intel_crtc, work)) { + WARN_ONCE(1, + "Kicking stuck page flip: queued at %d, now %d\n", + work->flip_queued_vblank, intel_crtc_get_vblank_counter(intel_crtc)); page_flip_completed(intel_crtc); work = NULL; } - if (work != NULL && - drm_vblank_count(dev, pipe) - work->flip_queued_vblank > 1) - intel_queue_rps_boost_for_request(dev, work->flip_queued_req); + + if (work != NULL && !is_mmio_work(work) && + intel_crtc_get_vblank_counter(intel_crtc) - work->flip_queued_vblank > 1) + intel_queue_rps_boost_for_request(work->flip_queued_req); spin_unlock(&dev->event_lock); } @@ -11538,13 +11642,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, uint32_t page_flip_flags) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_framebuffer *old_fb = crtc->primary->fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_plane *primary = crtc->primary; enum pipe pipe = intel_crtc->pipe; - struct intel_unpin_work *work; + struct intel_flip_work *work; struct intel_engine_cs *engine; bool mmio_flip; struct drm_i915_gem_request *request = NULL; @@ -11581,19 +11685,19 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, work->event = event; work->crtc = crtc; work->old_fb = old_fb; - INIT_WORK(&work->work, intel_unpin_work_fn); + INIT_WORK(&work->unpin_work, intel_unpin_work_fn); ret = drm_crtc_vblank_get(crtc); if (ret) goto free_work; - /* We borrow the event spin lock for protecting unpin_work */ + /* We borrow the event spin lock for protecting flip_work */ spin_lock_irq(&dev->event_lock); - if (intel_crtc->unpin_work) { + if (intel_crtc->flip_work) { /* Before declaring the flip queue wedged, check if * the hardware completed the operation behind our backs. */ - if (__intel_pageflip_stall_check(dev, crtc)) { + if (pageflip_finished(intel_crtc, intel_crtc->flip_work)) { DRM_DEBUG_DRIVER("flip queue: previous flip completed, continuing\n"); page_flip_completed(intel_crtc); } else { @@ -11605,7 +11709,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, return -EBUSY; } } - intel_crtc->unpin_work = work; + intel_crtc->flip_work = work; spin_unlock_irq(&dev->event_lock); if (atomic_read(&intel_crtc->unpin_work_count) >= 2) @@ -11617,7 +11721,9 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, crtc->primary->fb = fb; update_state_fb(crtc->primary); - intel_fbc_pre_update(intel_crtc); + + intel_fbc_pre_update(intel_crtc, intel_crtc->config, + to_intel_plane_state(primary->state)); work->pending_flip_obj = obj; @@ -11660,6 +11766,11 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, */ if (!mmio_flip) { ret = i915_gem_object_sync(obj, engine, &request); + if (!ret && !request) { + request = i915_gem_request_alloc(engine, NULL); + ret = PTR_ERR_OR_ZERO(request); + } + if (ret) goto cleanup_pending; } @@ -11671,38 +11782,28 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary), obj, 0); work->gtt_offset += intel_crtc->dspaddr_offset; + work->rotation = crtc->primary->state->rotation; if (mmio_flip) { - ret = intel_queue_mmio_flip(dev, crtc, obj); - if (ret) - goto cleanup_unpin; + INIT_WORK(&work->mmio_work, intel_mmio_flip_work_func); i915_gem_request_assign(&work->flip_queued_req, obj->last_write_req); - } else { - if (!request) { - request = i915_gem_request_alloc(engine, NULL); - if (IS_ERR(request)) { - ret = PTR_ERR(request); - goto cleanup_unpin; - } - } + schedule_work(&work->mmio_work); + } else { + i915_gem_request_assign(&work->flip_queued_req, request); ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, request, page_flip_flags); if (ret) goto cleanup_unpin; - i915_gem_request_assign(&work->flip_queued_req, request); - } + intel_mark_page_flip_active(intel_crtc, work); - if (request) i915_add_request_no_flush(request); + } - work->flip_queued_vblank = drm_crtc_vblank_count(crtc); - work->enable_stall_check = true; - - i915_gem_track_fb(intel_fb_obj(work->old_fb), obj, + i915_gem_track_fb(intel_fb_obj(old_fb), obj, to_intel_plane(primary)->frontbuffer_bit); mutex_unlock(&dev->struct_mutex); @@ -11728,7 +11829,7 @@ cleanup: drm_framebuffer_unreference(work->old_fb); spin_lock_irq(&dev->event_lock); - intel_crtc->unpin_work = NULL; + intel_crtc->flip_work = NULL; spin_unlock_irq(&dev->event_lock); drm_crtc_vblank_put(crtc); @@ -11830,15 +11931,14 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state, struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane_state *old_plane_state = to_intel_plane_state(plane->state); - int idx = intel_crtc->base.base.id, ret; bool mode_changed = needs_modeset(crtc_state); bool was_crtc_enabled = crtc->state->active; bool is_crtc_enabled = crtc_state->active; bool turn_off, turn_on, visible, was_visible; struct drm_framebuffer *fb = plane_state->fb; + int ret; - if (crtc_state && INTEL_INFO(dev)->gen >= 9 && - plane->type != DRM_PLANE_TYPE_CURSOR) { + if (INTEL_GEN(dev) >= 9 && plane->type != DRM_PLANE_TYPE_CURSOR) { ret = skl_update_scaler_plane( to_intel_crtc_state(crtc_state), to_intel_plane_state(plane_state)); @@ -11856,6 +11956,11 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state, * Visibility is calculated as if the crtc was on, but * after scaler setup everything depends on it being off * when the crtc isn't active. + * + * FIXME this is wrong for watermarks. Watermarks should also + * be computed as if the pipe would be active. Perhaps move + * per-plane wm computation to the .check_plane() hook, and + * only combine the results from all planes in the current place? */ if (!is_crtc_enabled) to_intel_plane_state(plane_state)->visible = visible = false; @@ -11869,11 +11974,15 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state, turn_off = was_visible && (!visible || mode_changed); turn_on = visible && (!was_visible || mode_changed); - DRM_DEBUG_ATOMIC("[CRTC:%i] has [PLANE:%i] with fb %i\n", idx, - plane->base.id, fb ? fb->base.id : -1); + DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n", + intel_crtc->base.base.id, + intel_crtc->base.name, + plane->base.id, plane->name, + fb ? fb->base.id : -1); - DRM_DEBUG_ATOMIC("[PLANE:%i] visible %i -> %i, off %i, on %i, ms %i\n", - plane->base.id, was_visible, visible, + DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n", + plane->base.id, plane->name, + was_visible, visible, turn_off, turn_on, mode_changed); if (turn_on) { @@ -11946,31 +12055,11 @@ static bool check_single_encoder_cloning(struct drm_atomic_state *state, return true; } -static bool check_encoder_cloning(struct drm_atomic_state *state, - struct intel_crtc *crtc) -{ - struct intel_encoder *encoder; - struct drm_connector *connector; - struct drm_connector_state *connector_state; - int i; - - for_each_connector_in_state(state, connector, connector_state, i) { - if (connector_state->crtc != &crtc->base) - continue; - - encoder = to_intel_encoder(connector_state->best_encoder); - if (!check_single_encoder_cloning(state, crtc, encoder)) - return false; - } - - return true; -} - static int intel_crtc_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc_state); @@ -11978,11 +12067,6 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc, int ret; bool mode_changed = needs_modeset(crtc_state); - if (mode_changed && !check_encoder_cloning(state, intel_crtc)) { - DRM_DEBUG_KMS("rejecting invalid cloning configuration\n"); - return -EINVAL; - } - if (mode_changed && !crtc_state->active) pipe_config->update_wm_post = true; @@ -12035,7 +12119,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc, } } else if (dev_priv->display.compute_intermediate_wm) { if (HAS_PCH_SPLIT(dev_priv) && INTEL_GEN(dev_priv) < 9) - pipe_config->wm.intermediate = pipe_config->wm.optimal.ilk; + pipe_config->wm.ilk.intermediate = pipe_config->wm.ilk.optimal; } if (INTEL_INFO(dev)->gen >= 9) { @@ -12160,7 +12244,8 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, struct intel_plane_state *state; struct drm_framebuffer *fb; - DRM_DEBUG_KMS("[CRTC:%d]%s config %p for pipe %c\n", crtc->base.base.id, + DRM_DEBUG_KMS("[CRTC:%d:%s]%s config %p for pipe %c\n", + crtc->base.base.id, crtc->base.name, context, pipe_config, pipe_name(crtc->pipe)); DRM_DEBUG_KMS("cpu_transcoder: %s\n", transcoder_name(pipe_config->cpu_transcoder)); @@ -12173,14 +12258,14 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n, pipe_config->fdi_m_n.tu); DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n", - pipe_config->has_dp_encoder, + intel_crtc_has_dp_encoder(pipe_config), pipe_config->lane_count, pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n, pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n, pipe_config->dp_m_n.tu); DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n", - pipe_config->has_dp_encoder, + intel_crtc_has_dp_encoder(pipe_config), pipe_config->lane_count, pipe_config->dp_m2_n2.gmch_m, pipe_config->dp_m2_n2.gmch_n, @@ -12261,29 +12346,24 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc, state = to_intel_plane_state(plane->state); fb = state->base.fb; if (!fb) { - DRM_DEBUG_KMS("%s PLANE:%d plane: %u.%u idx: %d " - "disabled, scaler_id = %d\n", - plane->type == DRM_PLANE_TYPE_CURSOR ? "CURSOR" : "STANDARD", - plane->base.id, intel_plane->pipe, - (crtc->base.primary == plane) ? 0 : intel_plane->plane + 1, - drm_plane_index(plane), state->scaler_id); + DRM_DEBUG_KMS("[PLANE:%d:%s] disabled, scaler_id = %d\n", + plane->base.id, plane->name, state->scaler_id); continue; } - DRM_DEBUG_KMS("%s PLANE:%d plane: %u.%u idx: %d enabled", - plane->type == DRM_PLANE_TYPE_CURSOR ? "CURSOR" : "STANDARD", - plane->base.id, intel_plane->pipe, - crtc->base.primary == plane ? 0 : intel_plane->plane + 1, - drm_plane_index(plane)); - DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = 0x%x", - fb->base.id, fb->width, fb->height, fb->pixel_format); - DRM_DEBUG_KMS("\tscaler:%d src (%u, %u) %ux%u dst (%u, %u) %ux%u\n", - state->scaler_id, - state->src.x1 >> 16, state->src.y1 >> 16, - drm_rect_width(&state->src) >> 16, - drm_rect_height(&state->src) >> 16, - state->dst.x1, state->dst.y1, - drm_rect_width(&state->dst), drm_rect_height(&state->dst)); + DRM_DEBUG_KMS("[PLANE:%d:%s] enabled", + plane->base.id, plane->name); + DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = %s", + fb->base.id, fb->width, fb->height, + drm_get_format_name(fb->pixel_format)); + DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n", + state->scaler_id, + state->src.x1 >> 16, state->src.y1 >> 16, + drm_rect_width(&state->src) >> 16, + drm_rect_height(&state->src) >> 16, + state->dst.x1, state->dst.y1, + drm_rect_width(&state->dst), + drm_rect_height(&state->dst)); } } @@ -12318,7 +12398,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state) case INTEL_OUTPUT_UNKNOWN: if (WARN_ON(!HAS_DDI(dev))) break; - case INTEL_OUTPUT_DISPLAYPORT: + case INTEL_OUTPUT_DP: case INTEL_OUTPUT_HDMI: case INTEL_OUTPUT_EDP: port_mask = 1 << enc_to_dig_port(&encoder->base)->port; @@ -12415,6 +12495,24 @@ intel_modeset_pipe_config(struct drm_crtc *crtc, &pipe_config->pipe_src_w, &pipe_config->pipe_src_h); + for_each_connector_in_state(state, connector, connector_state, i) { + if (connector_state->crtc != crtc) + continue; + + encoder = to_intel_encoder(connector_state->best_encoder); + + if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) { + DRM_DEBUG_KMS("rejecting invalid cloning configuration\n"); + goto fail; + } + + /* + * Determine output_types before calling the .compute_config() + * hooks so that the hooks can use this information safely. + */ + pipe_config->output_types |= 1 << encoder->type; + } + encoder_retry: /* Ensure the port clock defaults are reset when retrying. */ pipe_config->port_clock = 0; @@ -12700,8 +12798,8 @@ intel_pipe_config_compare(struct drm_device *dev, PIPE_CONF_CHECK_I(fdi_lanes); PIPE_CONF_CHECK_M_N(fdi_m_n); - PIPE_CONF_CHECK_I(has_dp_encoder); PIPE_CONF_CHECK_I(lane_count); + PIPE_CONF_CHECK_X(lane_lat_optim_mask); if (INTEL_INFO(dev)->gen < 8) { PIPE_CONF_CHECK_M_N(dp_m_n); @@ -12711,7 +12809,7 @@ intel_pipe_config_compare(struct drm_device *dev, } else PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2); - PIPE_CONF_CHECK_I(has_dsi_encoder); + PIPE_CONF_CHECK_X(output_types); PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay); PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal); @@ -12830,7 +12928,7 @@ static void verify_wm_state(struct drm_crtc *crtc, struct drm_crtc_state *new_state) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct skl_ddb_allocation hw_ddb, *sw_ddb; struct skl_ddb_entry *hw_entry, *sw_entry; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); @@ -12936,7 +13034,7 @@ verify_crtc_state(struct drm_crtc *crtc, struct drm_crtc_state *new_crtc_state) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *encoder; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_crtc_state *pipe_config, *sw_config; @@ -12950,7 +13048,7 @@ verify_crtc_state(struct drm_crtc *crtc, pipe_config->base.crtc = crtc; pipe_config->base.state = old_state; - DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id); + DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name); active = dev_priv->display.get_pipe_config(intel_crtc, pipe_config); @@ -12979,8 +13077,10 @@ verify_crtc_state(struct drm_crtc *crtc, "Encoder connected to wrong pipe %c\n", pipe_name(pipe)); - if (active) + if (active) { + pipe_config->output_types |= 1 << encoder->type; encoder->get_config(encoder, pipe_config); + } } if (!new_crtc_state->active) @@ -13059,7 +13159,7 @@ verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc, struct drm_crtc_state *old_crtc_state, struct drm_crtc_state *new_crtc_state) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc_state *old_state = to_intel_crtc_state(old_crtc_state); struct intel_crtc_state *new_state = to_intel_crtc_state(new_crtc_state); @@ -13098,7 +13198,7 @@ intel_modeset_verify_crtc(struct drm_crtc *crtc, static void verify_disabled_dpll_state(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i; for (i = 0; i < dev_priv->num_shared_dpll; i++) @@ -13145,7 +13245,7 @@ static void update_scanline_offset(struct intel_crtc *crtc) crtc->scanline_offset = vtotal - 1; } else if (HAS_DDI(dev) && - intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI)) { + intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI)) { crtc->scanline_offset = 2; } else crtc->scanline_offset = 1; @@ -13280,7 +13380,7 @@ static int intel_modeset_all_pipes(struct drm_atomic_state *state) static int intel_modeset_checks(struct drm_atomic_state *state) { struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct drm_i915_private *dev_priv = state->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(state->dev); struct drm_crtc *crtc; struct drm_crtc_state *crtc_state; int ret = 0, i; @@ -13298,6 +13398,9 @@ static int intel_modeset_checks(struct drm_atomic_state *state) intel_state->active_crtcs |= 1 << i; else intel_state->active_crtcs &= ~(1 << i); + + if (crtc_state->active != crtc->state->active) + intel_state->active_pipe_changes |= drm_crtc_mask(crtc); } /* @@ -13308,9 +13411,17 @@ static int intel_modeset_checks(struct drm_atomic_state *state) * adjusted_mode bits in the crtc directly. */ if (dev_priv->display.modeset_calc_cdclk) { + if (!intel_state->cdclk_pll_vco) + intel_state->cdclk_pll_vco = dev_priv->cdclk_pll.vco; + if (!intel_state->cdclk_pll_vco) + intel_state->cdclk_pll_vco = dev_priv->skl_preferred_vco_freq; + ret = dev_priv->display.modeset_calc_cdclk(state); + if (ret < 0) + return ret; - if (!ret && intel_state->dev_cdclk != dev_priv->cdclk_freq) + if (intel_state->dev_cdclk != dev_priv->cdclk_freq || + intel_state->cdclk_pll_vco != dev_priv->cdclk_pll.vco) ret = intel_modeset_all_pipes(state); if (ret < 0) @@ -13334,38 +13445,16 @@ static int intel_modeset_checks(struct drm_atomic_state *state) * phase. The code here should be run after the per-crtc and per-plane 'check' * handlers to ensure that all derived state has been updated. */ -static void calc_watermark_data(struct drm_atomic_state *state) +static int calc_watermark_data(struct drm_atomic_state *state) { struct drm_device *dev = state->dev; - struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct drm_crtc *crtc; - struct drm_crtc_state *cstate; - struct drm_plane *plane; - struct drm_plane_state *pstate; - - /* - * Calculate watermark configuration details now that derived - * plane/crtc state is all properly updated. - */ - drm_for_each_crtc(crtc, dev) { - cstate = drm_atomic_get_existing_crtc_state(state, crtc) ?: - crtc->state; - - if (cstate->active) - intel_state->wm_config.num_pipes_active++; - } - drm_for_each_legacy_plane(plane, dev) { - pstate = drm_atomic_get_existing_plane_state(state, plane) ?: - plane->state; + struct drm_i915_private *dev_priv = to_i915(dev); - if (!to_intel_plane_state(pstate)->visible) - continue; + /* Is there platform-specific watermark information to calculate? */ + if (dev_priv->display.compute_global_watermarks) + return dev_priv->display.compute_global_watermarks(state); - intel_state->wm_config.sprites_enabled = true; - if (pstate->crtc_w != pstate->src_w >> 16 || - pstate->crtc_h != pstate->src_h >> 16) - intel_state->wm_config.sprites_scaled = true; - } + return 0; } /** @@ -13395,14 +13484,13 @@ static int intel_atomic_check(struct drm_device *dev, if (crtc_state->mode.private_flags != crtc->state->mode.private_flags) crtc_state->mode_changed = true; - if (!crtc_state->enable) { - if (needs_modeset(crtc_state)) - any_ms = true; + if (!needs_modeset(crtc_state)) continue; - } - if (!needs_modeset(crtc_state)) + if (!crtc_state->enable) { + any_ms = true; continue; + } /* FIXME: For only active_changed we shouldn't need to do any * state recomputation at all. */ @@ -13412,8 +13500,11 @@ static int intel_atomic_check(struct drm_device *dev, return ret; ret = intel_modeset_pipe_config(crtc, pipe_config); - if (ret) + if (ret) { + intel_dump_pipe_config(to_intel_crtc(crtc), + pipe_config, "[failed]"); return ret; + } if (i915.fastboot && intel_pipe_config_compare(dev, @@ -13423,13 +13514,12 @@ static int intel_atomic_check(struct drm_device *dev, to_intel_crtc_state(crtc_state)->update_pipe = true; } - if (needs_modeset(crtc_state)) { + if (needs_modeset(crtc_state)) any_ms = true; - ret = drm_atomic_add_affected_planes(state, crtc); - if (ret) - return ret; - } + ret = drm_atomic_add_affected_planes(state, crtc); + if (ret) + return ret; intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config, needs_modeset(crtc_state) ? @@ -13449,27 +13539,20 @@ static int intel_atomic_check(struct drm_device *dev, return ret; intel_fbc_choose_crtc(dev_priv, state); - calc_watermark_data(state); - - return 0; + return calc_watermark_data(state); } static int intel_atomic_prepare_commit(struct drm_device *dev, struct drm_atomic_state *state, bool nonblock) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_plane_state *plane_state; struct drm_crtc_state *crtc_state; struct drm_plane *plane; struct drm_crtc *crtc; int i, ret; - if (nonblock) { - DRM_DEBUG_KMS("i915 does not yet support nonblocking commit\n"); - return -EINVAL; - } - for_each_crtc_in_state(state, crtc, crtc_state, i) { if (state->legacy_cursor_update) continue; @@ -13513,6 +13596,16 @@ static int intel_atomic_prepare_commit(struct drm_device *dev, return ret; } +u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + + if (!dev->max_vblank_count) + return drm_accurate_vblank_count(&crtc->base); + + return dev->driver->get_vblank_counter(dev, crtc->pipe); +} + static void intel_atomic_wait_for_vblanks(struct drm_device *dev, struct drm_i915_private *dev_priv, unsigned crtc_mask) @@ -13578,45 +13671,36 @@ static bool needs_vblank_wait(struct intel_crtc_state *crtc_state) return false; } -/** - * intel_atomic_commit - commit validated state object - * @dev: DRM device - * @state: the top-level driver state object - * @nonblock: nonblocking commit - * - * This function commits a top-level state object that has been validated - * with drm_atomic_helper_check(). - * - * FIXME: Atomic modeset support for i915 is not yet complete. At the moment - * we can only handle plane-related operations and do not yet support - * nonblocking commit. - * - * RETURNS - * Zero for success or -errno. - */ -static int intel_atomic_commit(struct drm_device *dev, - struct drm_atomic_state *state, - bool nonblock) +static void intel_atomic_commit_tail(struct drm_atomic_state *state) { + struct drm_device *dev = state->dev; struct intel_atomic_state *intel_state = to_intel_atomic_state(state); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc_state *old_crtc_state; struct drm_crtc *crtc; struct intel_crtc_state *intel_cstate; - int ret = 0, i; + struct drm_plane *plane; + struct drm_plane_state *plane_state; bool hw_check = intel_state->modeset; unsigned long put_domains[I915_MAX_PIPES] = {}; unsigned crtc_vblank_mask = 0; + int i, ret; - ret = intel_atomic_prepare_commit(dev, state, nonblock); - if (ret) { - DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret); - return ret; + for_each_plane_in_state(state, plane, plane_state, i) { + struct intel_plane_state *intel_plane_state = + to_intel_plane_state(plane_state); + + if (!intel_plane_state->wait_req) + continue; + + ret = __i915_wait_request(intel_plane_state->wait_req, + true, NULL, NULL); + /* EIO should be eaten, and we can't get interrupted in the + * worker, and blocking commits have waited already. */ + WARN_ON(ret); } - drm_atomic_helper_swap_state(dev, state); - dev_priv->wm.config = intel_state->wm_config; - intel_shared_dpll_commit(state); + drm_atomic_helper_wait_for_dependencies(state); if (intel_state->modeset) { memcpy(dev_priv->min_pixclk, intel_state->min_pixclk, @@ -13671,9 +13755,17 @@ static int intel_atomic_commit(struct drm_device *dev, drm_atomic_helper_update_legacy_modeset_state(state->dev, state); if (dev_priv->display.modeset_commit_cdclk && - intel_state->dev_cdclk != dev_priv->cdclk_freq) + (intel_state->dev_cdclk != dev_priv->cdclk_freq || + intel_state->cdclk_pll_vco != dev_priv->cdclk_pll.vco)) dev_priv->display.modeset_commit_cdclk(state); + /* + * SKL workaround: bspec recommends we disable the SAGV when we + * have more then one pipe enabled + */ + if (IS_SKYLAKE(dev_priv) && !skl_can_enable_sagv(state)) + skl_disable_sagv(dev_priv); + intel_modeset_verify_disabled(dev); } @@ -13683,30 +13775,44 @@ static int intel_atomic_commit(struct drm_device *dev, bool modeset = needs_modeset(crtc->state); struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc->state); - bool update_pipe = !modeset && pipe_config->update_pipe; if (modeset && crtc->state->active) { update_scanline_offset(to_intel_crtc(crtc)); dev_priv->display.crtc_enable(crtc); } + /* Complete events for now disable pipes here. */ + if (modeset && !crtc->state->active && crtc->state->event) { + spin_lock_irq(&dev->event_lock); + drm_crtc_send_vblank_event(crtc, crtc->state->event); + spin_unlock_irq(&dev->event_lock); + + crtc->state->event = NULL; + } + if (!modeset) intel_pre_plane_update(to_intel_crtc_state(old_crtc_state)); if (crtc->state->active && drm_atomic_get_existing_plane_state(state, crtc->primary)) - intel_fbc_enable(intel_crtc); + intel_fbc_enable(intel_crtc, pipe_config, to_intel_plane_state(crtc->primary->state)); - if (crtc->state->active && - (crtc->state->planes_changed || update_pipe)) + if (crtc->state->active) drm_atomic_helper_commit_planes_on_crtc(old_crtc_state); if (pipe_config->base.active && needs_vblank_wait(pipe_config)) crtc_vblank_mask |= 1 << i; } - /* FIXME: add subpixel order */ - + /* FIXME: We should call drm_atomic_helper_commit_hw_done() here + * already, but still need the state for the delayed optimization. To + * fix this: + * - wrap the optimization/post_plane_update stuff into a per-crtc work. + * - schedule that vblank worker _before_ calling hw_done + * - at the start of commit_tail, cancel it _synchrously + * - switch over to the vblank wait helper in the core after that since + * we don't need out special handling any more. + */ if (!state->legacy_cursor_update) intel_atomic_wait_for_vblanks(dev, dev_priv, crtc_vblank_mask); @@ -13733,6 +13839,12 @@ static int intel_atomic_commit(struct drm_device *dev, intel_modeset_verify_crtc(crtc, old_crtc_state, crtc->state); } + if (IS_SKYLAKE(dev_priv) && intel_state->modeset && + skl_can_enable_sagv(state)) + skl_enable_sagv(dev_priv); + + drm_atomic_helper_commit_hw_done(state); + if (intel_state->modeset) intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET); @@ -13740,6 +13852,8 @@ static int intel_atomic_commit(struct drm_device *dev, drm_atomic_helper_cleanup_planes(dev, state); mutex_unlock(&dev->struct_mutex); + drm_atomic_helper_commit_cleanup_done(state); + drm_atomic_state_free(state); /* As one of the primary mmio accessors, KMS has a high likelihood @@ -13754,6 +13868,86 @@ static int intel_atomic_commit(struct drm_device *dev, * can happen also when the device is completely off. */ intel_uncore_arm_unclaimed_mmio_detection(dev_priv); +} + +static void intel_atomic_commit_work(struct work_struct *work) +{ + struct drm_atomic_state *state = container_of(work, + struct drm_atomic_state, + commit_work); + intel_atomic_commit_tail(state); +} + +static void intel_atomic_track_fbs(struct drm_atomic_state *state) +{ + struct drm_plane_state *old_plane_state; + struct drm_plane *plane; + struct drm_i915_gem_object *obj, *old_obj; + struct intel_plane *intel_plane; + int i; + + mutex_lock(&state->dev->struct_mutex); + for_each_plane_in_state(state, plane, old_plane_state, i) { + obj = intel_fb_obj(plane->state->fb); + old_obj = intel_fb_obj(old_plane_state->fb); + intel_plane = to_intel_plane(plane); + + i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit); + } + mutex_unlock(&state->dev->struct_mutex); +} + +/** + * intel_atomic_commit - commit validated state object + * @dev: DRM device + * @state: the top-level driver state object + * @nonblock: nonblocking commit + * + * This function commits a top-level state object that has been validated + * with drm_atomic_helper_check(). + * + * FIXME: Atomic modeset support for i915 is not yet complete. At the moment + * nonblocking commits are only safe for pure plane updates. Everything else + * should work though. + * + * RETURNS + * Zero for success or -errno. + */ +static int intel_atomic_commit(struct drm_device *dev, + struct drm_atomic_state *state, + bool nonblock) +{ + struct intel_atomic_state *intel_state = to_intel_atomic_state(state); + struct drm_i915_private *dev_priv = to_i915(dev); + int ret = 0; + + if (intel_state->modeset && nonblock) { + DRM_DEBUG_KMS("nonblocking commit for modeset not yet implemented.\n"); + return -EINVAL; + } + + ret = drm_atomic_helper_setup_commit(state, nonblock); + if (ret) + return ret; + + INIT_WORK(&state->commit_work, intel_atomic_commit_work); + + ret = intel_atomic_prepare_commit(dev, state, nonblock); + if (ret) { + DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret); + return ret; + } + + drm_atomic_helper_swap_state(state, true); + dev_priv->wm.distrust_bios_wm = false; + dev_priv->wm.skl_results = intel_state->wm_results; + intel_shared_dpll_commit(state); + intel_atomic_track_fbs(state); + + if (nonblock) + queue_work(system_unbound_wq, &state->commit_work); + else + intel_atomic_commit_tail(state); return 0; } @@ -13767,8 +13961,8 @@ void intel_crtc_restore_mode(struct drm_crtc *crtc) state = drm_atomic_state_alloc(dev); if (!state) { - DRM_DEBUG_KMS("[CRTC:%d] crtc restore failed, out of memory", - crtc->base.id); + DRM_DEBUG_KMS("[CRTC:%d:%s] crtc restore failed, out of memory", + crtc->base.id, crtc->name); return; } @@ -13798,8 +13992,50 @@ out: #undef for_each_intel_crtc_masked +/* + * FIXME: Remove this once i915 is fully DRIVER_ATOMIC by calling + * drm_atomic_helper_legacy_gamma_set() directly. + */ +static int intel_atomic_legacy_gamma_set(struct drm_crtc *crtc, + u16 *red, u16 *green, u16 *blue, + uint32_t size) +{ + struct drm_device *dev = crtc->dev; + struct drm_mode_config *config = &dev->mode_config; + struct drm_crtc_state *state; + int ret; + + ret = drm_atomic_helper_legacy_gamma_set(crtc, red, green, blue, size); + if (ret) + return ret; + + /* + * Make sure we update the legacy properties so this works when + * atomic is not enabled. + */ + + state = crtc->state; + + drm_object_property_set_value(&crtc->base, + config->degamma_lut_property, + (state->degamma_lut) ? + state->degamma_lut->base.id : 0); + + drm_object_property_set_value(&crtc->base, + config->ctm_property, + (state->ctm) ? + state->ctm->base.id : 0); + + drm_object_property_set_value(&crtc->base, + config->gamma_lut_property, + (state->gamma_lut) ? + state->gamma_lut->base.id : 0); + + return 0; +} + static const struct drm_crtc_funcs intel_crtc_funcs = { - .gamma_set = drm_atomic_helper_legacy_gamma_set, + .gamma_set = intel_atomic_legacy_gamma_set, .set_config = drm_atomic_helper_set_config, .set_property = drm_atomic_helper_crtc_set_property, .destroy = intel_crtc_destroy, @@ -13828,9 +14064,9 @@ intel_prepare_plane_fb(struct drm_plane *plane, { struct drm_device *dev = plane->dev; struct drm_framebuffer *fb = new_state->fb; - struct intel_plane *intel_plane = to_intel_plane(plane); struct drm_i915_gem_object *obj = intel_fb_obj(fb); struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb); + struct reservation_object *resv; int ret = 0; if (!obj && !old_obj) @@ -13860,12 +14096,15 @@ intel_prepare_plane_fb(struct drm_plane *plane, } } + if (!obj) + return 0; + /* For framebuffer backed by dmabuf, wait for fence */ - if (obj && obj->base.dma_buf) { + resv = i915_gem_object_get_dmabuf_resv(obj); + if (resv) { long lret; - lret = reservation_object_wait_timeout_rcu(obj->base.dma_buf->resv, - false, true, + lret = reservation_object_wait_timeout_rcu(resv, false, true, MAX_SCHEDULE_TIMEOUT); if (lret == -ERESTARTSYS) return lret; @@ -13873,9 +14112,7 @@ intel_prepare_plane_fb(struct drm_plane *plane, WARN(lret < 0, "waiting returns %li\n", lret); } - if (!obj) { - ret = 0; - } else if (plane->type == DRM_PLANE_TYPE_CURSOR && + if (plane->type == DRM_PLANE_TYPE_CURSOR && INTEL_INFO(dev)->cursor_needs_physical) { int align = IS_I830(dev) ? 16 * 1024 : 256; ret = i915_gem_object_attach_phys(obj, align); @@ -13886,15 +14123,11 @@ intel_prepare_plane_fb(struct drm_plane *plane, } if (ret == 0) { - if (obj) { - struct intel_plane_state *plane_state = - to_intel_plane_state(new_state); - - i915_gem_request_assign(&plane_state->wait_req, - obj->last_write_req); - } + struct intel_plane_state *plane_state = + to_intel_plane_state(new_state); - i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit); + i915_gem_request_assign(&plane_state->wait_req, + obj->last_write_req); } return ret; @@ -13914,7 +14147,6 @@ intel_cleanup_plane_fb(struct drm_plane *plane, const struct drm_plane_state *old_state) { struct drm_device *dev = plane->dev; - struct intel_plane *intel_plane = to_intel_plane(plane); struct intel_plane_state *old_intel_state; struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb); struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb); @@ -13928,11 +14160,6 @@ intel_cleanup_plane_fb(struct drm_plane *plane, !INTEL_INFO(dev)->cursor_needs_physical)) intel_unpin_fb_obj(old_state->fb, old_state->rotation); - /* prepare_fb aborted? */ - if ((old_obj && (old_obj->frontbuffer_bits & intel_plane->frontbuffer_bit)) || - (obj && !(obj->frontbuffer_bits & intel_plane->frontbuffer_bit))) - i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit); - i915_gem_request_assign(&old_intel_state->wait_req, NULL); } @@ -13940,15 +14167,11 @@ int skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state) { int max_scale; - struct drm_device *dev; - struct drm_i915_private *dev_priv; int crtc_clock, cdclk; if (!intel_crtc || !crtc_state->base.enable) return DRM_PLANE_HELPER_NO_SCALING; - dev = intel_crtc->base.dev; - dev_priv = dev->dev_private; crtc_clock = crtc_state->base.adjusted_mode.crtc_clock; cdclk = to_intel_atomic_state(crtc_state->base.state)->cdclk; @@ -13988,6 +14211,7 @@ intel_check_primary_plane(struct drm_plane *plane, return drm_plane_helper_check_update(plane, crtc, fb, &state->src, &state->dst, &state->clip, + state->base.rotation, min_scale, max_scale, can_position, true, &state->visible); @@ -14024,7 +14248,7 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc, { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - intel_pipe_update_end(intel_crtc); + intel_pipe_update_end(intel_crtc, NULL); } /** @@ -14036,9 +14260,11 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc, */ void intel_plane_destroy(struct drm_plane *plane) { - struct intel_plane *intel_plane = to_intel_plane(plane); + if (!plane) + return; + drm_plane_cleanup(plane); - kfree(intel_plane); + kfree(to_intel_plane(plane)); } const struct drm_plane_funcs intel_plane_funcs = { @@ -14110,10 +14336,24 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, primary->disable_plane = i9xx_disable_primary_plane; } - ret = drm_universal_plane_init(dev, &primary->base, 0, - &intel_plane_funcs, - intel_primary_formats, num_formats, - DRM_PLANE_TYPE_PRIMARY, NULL); + if (INTEL_INFO(dev)->gen >= 9) + ret = drm_universal_plane_init(dev, &primary->base, 0, + &intel_plane_funcs, + intel_primary_formats, num_formats, + DRM_PLANE_TYPE_PRIMARY, + "plane 1%c", pipe_name(pipe)); + else if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) + ret = drm_universal_plane_init(dev, &primary->base, 0, + &intel_plane_funcs, + intel_primary_formats, num_formats, + DRM_PLANE_TYPE_PRIMARY, + "primary %c", pipe_name(pipe)); + else + ret = drm_universal_plane_init(dev, &primary->base, 0, + &intel_plane_funcs, + intel_primary_formats, num_formats, + DRM_PLANE_TYPE_PRIMARY, + "plane %c", plane_name(primary->plane)); if (ret) goto fail; @@ -14163,6 +14403,7 @@ intel_check_cursor_plane(struct drm_plane *plane, ret = drm_plane_helper_check_update(plane, crtc, fb, &state->src, &state->dst, &state->clip, + state->base.rotation, DRM_PLANE_HELPER_NO_SCALING, DRM_PLANE_HELPER_NO_SCALING, true, true, &state->visible); @@ -14271,7 +14512,8 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev, &intel_plane_funcs, intel_cursor_formats, ARRAY_SIZE(intel_cursor_formats), - DRM_PLANE_TYPE_CURSOR, NULL); + DRM_PLANE_TYPE_CURSOR, + "cursor %c", pipe_name(pipe)); if (ret) goto fail; @@ -14319,7 +14561,7 @@ static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_cr static void intel_crtc_init(struct drm_device *dev, int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc; struct intel_crtc_state *crtc_state = NULL; struct drm_plane *primary = NULL; @@ -14356,7 +14598,8 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) goto fail; ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary, - cursor, &intel_crtc_funcs, NULL); + cursor, &intel_crtc_funcs, + "pipe %c", pipe_name(pipe)); if (ret) goto fail; @@ -14390,10 +14633,8 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) return; fail: - if (primary) - drm_plane_cleanup(primary); - if (cursor) - drm_plane_cleanup(cursor); + intel_plane_destroy(primary); + intel_plane_destroy(cursor); kfree(crtc_state); kfree(intel_crtc); } @@ -14419,11 +14660,8 @@ int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data, struct intel_crtc *crtc; drmmode_crtc = drm_crtc_find(dev, pipe_from_crtc_id->crtc_id); - - if (!drmmode_crtc) { - DRM_ERROR("no such CRTC id\n"); + if (!drmmode_crtc) return -ENOENT; - } crtc = to_intel_crtc(drmmode_crtc); pipe_from_crtc_id->pipe = crtc->pipe; @@ -14450,7 +14688,7 @@ static int intel_encoder_clones(struct intel_encoder *encoder) static bool has_edp_a(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!IS_MOBILE(dev)) return false; @@ -14466,7 +14704,7 @@ static bool has_edp_a(struct drm_device *dev) static bool intel_crt_present(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (INTEL_INFO(dev)->gen >= 9) return false; @@ -14492,10 +14730,15 @@ static bool intel_crt_present(struct drm_device *dev) static void intel_setup_outputs(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *encoder; bool dpd_is_edp = false; + /* + * intel_edp_init_connector() depends on this completing first, to + * prevent the registeration of both eDP and LVDS and the incorrect + * sharing of the PPS. + */ intel_lvds_init(dev); if (intel_crt_present(dev)) @@ -15080,12 +15323,13 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv) dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train; } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { dev_priv->display.fdi_link_train = hsw_fdi_link_train; - if (IS_BROADWELL(dev_priv)) { - dev_priv->display.modeset_commit_cdclk = - broadwell_modeset_commit_cdclk; - dev_priv->display.modeset_calc_cdclk = - broadwell_modeset_calc_cdclk; - } + } + + if (IS_BROADWELL(dev_priv)) { + dev_priv->display.modeset_commit_cdclk = + broadwell_modeset_commit_cdclk; + dev_priv->display.modeset_calc_cdclk = + broadwell_modeset_calc_cdclk; } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { dev_priv->display.modeset_commit_cdclk = valleyview_modeset_commit_cdclk; @@ -15093,9 +15337,14 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv) valleyview_modeset_calc_cdclk; } else if (IS_BROXTON(dev_priv)) { dev_priv->display.modeset_commit_cdclk = - broxton_modeset_commit_cdclk; + bxt_modeset_commit_cdclk; + dev_priv->display.modeset_calc_cdclk = + bxt_modeset_calc_cdclk; + } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + dev_priv->display.modeset_commit_cdclk = + skl_modeset_commit_cdclk; dev_priv->display.modeset_calc_cdclk = - broxton_modeset_calc_cdclk; + skl_modeset_calc_cdclk; } switch (INTEL_INFO(dev_priv)->gen) { @@ -15134,7 +15383,7 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv) */ static void quirk_pipea_force(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); dev_priv->quirks |= QUIRK_PIPEA_FORCE; DRM_INFO("applying pipe a force quirk\n"); @@ -15142,7 +15391,7 @@ static void quirk_pipea_force(struct drm_device *dev) static void quirk_pipeb_force(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); dev_priv->quirks |= QUIRK_PIPEB_FORCE; DRM_INFO("applying pipe b force quirk\n"); @@ -15153,7 +15402,7 @@ static void quirk_pipeb_force(struct drm_device *dev) */ static void quirk_ssc_force_disable(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE; DRM_INFO("applying lvds SSC disable quirk\n"); } @@ -15164,7 +15413,7 @@ static void quirk_ssc_force_disable(struct drm_device *dev) */ static void quirk_invert_brightness(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS; DRM_INFO("applying inverted panel brightness quirk\n"); } @@ -15172,7 +15421,7 @@ static void quirk_invert_brightness(struct drm_device *dev) /* Some VBT's incorrectly indicate no backlight is present */ static void quirk_backlight_present(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); dev_priv->quirks |= QUIRK_BACKLIGHT_PRESENT; DRM_INFO("applying backlight present quirk\n"); } @@ -15298,7 +15547,7 @@ static void intel_init_quirks(struct drm_device *dev) /* Disable the VGA plane that we never use */ static void i915_disable_vga(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u8 sr1; i915_reg_t vga_reg = i915_vgacntrl_reg(dev); @@ -15316,14 +15565,14 @@ static void i915_disable_vga(struct drm_device *dev) void intel_modeset_init_hw(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); intel_update_cdclk(dev); dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq; intel_init_clock_gating(dev); - intel_enable_gt_powersave(dev); + intel_enable_gt_powersave(dev_priv); } /* @@ -15393,7 +15642,6 @@ retry: } /* Write calculated watermark values back */ - to_i915(dev)->wm.config = to_intel_atomic_state(state)->wm_config; for_each_crtc_in_state(state, crtc, cstate, i) { struct intel_crtc_state *cs = to_intel_crtc_state(cstate); @@ -15491,11 +15739,13 @@ void intel_modeset_init(struct drm_device *dev) } intel_update_czclk(dev_priv); - intel_update_rawclk(dev_priv); intel_update_cdclk(dev); intel_shared_dpll_init(dev); + if (dev_priv->max_cdclk_freq == 0) + intel_update_max_cdclk(dev); + /* Just disable it once at startup */ i915_disable_vga(dev); intel_setup_outputs(dev); @@ -15563,7 +15813,7 @@ static bool intel_check_plane_mapping(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val; if (INTEL_INFO(dev)->num_pipes == 1) @@ -15603,7 +15853,7 @@ static bool intel_encoder_has_connectors(struct intel_encoder *encoder) static void intel_sanitize_crtc(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; /* Clear any frame start delays used for debugging left by the BIOS */ @@ -15636,8 +15886,8 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc) if (INTEL_INFO(dev)->gen < 4 && !intel_check_plane_mapping(crtc)) { bool plane; - DRM_DEBUG_KMS("[CRTC:%d] wrong plane connection detected!\n", - crtc->base.base.id); + DRM_DEBUG_KMS("[CRTC:%d:%s] wrong plane connection detected!\n", + crtc->base.base.id, crtc->base.name); /* Pipe has the wrong plane attached and the plane is active. * Temporarily change the plane mapping and disable everything @@ -15728,7 +15978,7 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) void i915_redisable_vga_power_on(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t vga_reg = i915_vgacntrl_reg(dev); if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) { @@ -15739,7 +15989,7 @@ void i915_redisable_vga_power_on(struct drm_device *dev) void i915_redisable_vga(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* This function can be called both from intel_modeset_setup_hw_state or * at a very early point in our resume sequence, where the power well @@ -15779,7 +16029,7 @@ static void readout_plane_state(struct intel_crtc *crtc) static void intel_modeset_readout_hw_state(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; struct intel_crtc *crtc; struct intel_encoder *encoder; @@ -15805,26 +16055,24 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) if (crtc_state->base.active) { dev_priv->active_crtcs |= 1 << crtc->pipe; - if (IS_BROADWELL(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) pixclk = ilk_pipe_pixel_rate(crtc_state); - - /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ - if (crtc_state->ips_enabled) - pixclk = DIV_ROUND_UP(pixclk * 100, 95); - } else if (IS_VALLEYVIEW(dev_priv) || - IS_CHERRYVIEW(dev_priv) || - IS_BROXTON(dev_priv)) + else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) pixclk = crtc_state->base.adjusted_mode.crtc_clock; else WARN_ON(dev_priv->display.modeset_calc_cdclk); + + /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ + if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled) + pixclk = DIV_ROUND_UP(pixclk * 100, 95); } dev_priv->min_pixclk[crtc->pipe] = pixclk; readout_plane_state(crtc); - DRM_DEBUG_KMS("[CRTC:%d] hw state readout: %s\n", - crtc->base.base.id, + DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n", + crtc->base.base.id, crtc->base.name, crtc->active ? "enabled" : "disabled"); } @@ -15850,6 +16098,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) if (encoder->get_hw_state(encoder, &pipe)) { crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); encoder->base.crtc = &crtc->base; + crtc->config->output_types |= 1 << encoder->type; encoder->get_config(encoder, crtc->config); } else { encoder->base.crtc = NULL; @@ -15934,7 +16183,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) static void intel_modeset_setup_hw_state(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; struct intel_crtc *crtc; struct intel_encoder *encoder; @@ -15993,9 +16242,10 @@ void intel_display_resume(struct drm_device *dev) struct drm_atomic_state *state = dev_priv->modeset_restore_state; struct drm_modeset_acquire_ctx ctx; int ret; - bool setup = false; dev_priv->modeset_restore_state = NULL; + if (state) + state->acquire_ctx = &ctx; /* * This is a cludge because with real atomic modeset mode_config.mutex @@ -16006,43 +16256,17 @@ void intel_display_resume(struct drm_device *dev) mutex_lock(&dev->mode_config.mutex); drm_modeset_acquire_init(&ctx, 0); -retry: - ret = drm_modeset_lock_all_ctx(dev, &ctx); - - if (ret == 0 && !setup) { - setup = true; - - intel_modeset_setup_hw_state(dev); - i915_redisable_vga(dev); - } - - if (ret == 0 && state) { - struct drm_crtc_state *crtc_state; - struct drm_crtc *crtc; - int i; - - state->acquire_ctx = &ctx; - - /* ignore any reset values/BIOS leftovers in the WM registers */ - to_intel_atomic_state(state)->skip_intermediate_wm = true; - - for_each_crtc_in_state(state, crtc, crtc_state, i) { - /* - * Force recalculation even if we restore - * current state. With fast modeset this may not result - * in a modeset when the state is compatible. - */ - crtc_state->mode_changed = true; - } - - ret = drm_atomic_commit(state); - } + while (1) { + ret = drm_modeset_lock_all_ctx(dev, &ctx); + if (ret != -EDEADLK) + break; - if (ret == -EDEADLK) { drm_modeset_backoff(&ctx); - goto retry; } + if (!ret) + ret = __intel_display_resume(dev, state); + drm_modeset_drop_locks(&ctx); drm_modeset_acquire_fini(&ctx); mutex_unlock(&dev->mode_config.mutex); @@ -16055,15 +16279,16 @@ retry: void intel_modeset_gem_init(struct drm_device *dev) { + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *c; struct drm_i915_gem_object *obj; int ret; - intel_init_gt_powersave(dev); + intel_init_gt_powersave(dev_priv); intel_modeset_init_hw(dev); - intel_setup_overlay(dev); + intel_setup_overlay(dev_priv); /* * Make sure any fbs we allocated at startup are properly @@ -16089,26 +16314,36 @@ void intel_modeset_gem_init(struct drm_device *dev) c->state->plane_mask &= ~(1 << drm_plane_index(c->primary)); } } +} - intel_backlight_register(dev); +int intel_connector_register(struct drm_connector *connector) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + int ret; + + ret = intel_backlight_device_register(intel_connector); + if (ret) + goto err; + + return 0; + +err: + return ret; } -void intel_connector_unregister(struct intel_connector *intel_connector) +void intel_connector_unregister(struct drm_connector *connector) { - struct drm_connector *connector = &intel_connector->base; + struct intel_connector *intel_connector = to_intel_connector(connector); + intel_backlight_device_unregister(intel_connector); intel_panel_destroy_backlight(connector); - drm_connector_unregister(connector); } void intel_modeset_cleanup(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_connector *connector; - - intel_disable_gt_powersave(dev); + struct drm_i915_private *dev_priv = to_i915(dev); - intel_backlight_unregister(dev); + intel_disable_gt_powersave(dev_priv); /* * Interrupts and polling as the first thing to avoid creating havoc. @@ -16130,27 +16365,15 @@ void intel_modeset_cleanup(struct drm_device *dev) /* flush any delayed tasks or pending work */ flush_scheduled_work(); - /* destroy the backlight and sysfs files before encoders/connectors */ - for_each_intel_connector(dev, connector) - connector->unregister(connector); - drm_mode_config_cleanup(dev); - intel_cleanup_overlay(dev); + intel_cleanup_overlay(dev_priv); - intel_cleanup_gt_powersave(dev); + intel_cleanup_gt_powersave(dev_priv); intel_teardown_gmbus(dev); } -/* - * Return which encoder is currently attached for connector. - */ -struct drm_encoder *intel_best_encoder(struct drm_connector *connector) -{ - return &intel_attached_encoder(connector)->base; -} - void intel_connector_attach_encoder(struct intel_connector *connector, struct intel_encoder *encoder) { @@ -16164,7 +16387,7 @@ void intel_connector_attach_encoder(struct intel_connector *connector, */ int intel_modeset_vga_set_state(struct drm_device *dev, bool state) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL; u16 gmch_ctrl; @@ -16234,9 +16457,8 @@ struct intel_display_error_state { }; struct intel_display_error_state * -intel_display_capture_error_state(struct drm_device *dev) +intel_display_capture_error_state(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_display_error_state *error; int transcoders[] = { TRANSCODER_A, @@ -16246,14 +16468,14 @@ intel_display_capture_error_state(struct drm_device *dev) }; int i; - if (INTEL_INFO(dev)->num_pipes == 0) + if (INTEL_INFO(dev_priv)->num_pipes == 0) return NULL; error = kzalloc(sizeof(*error), GFP_ATOMIC); if (error == NULL) return NULL; - if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER); for_each_pipe(dev_priv, i) { @@ -16269,25 +16491,25 @@ intel_display_capture_error_state(struct drm_device *dev) error->plane[i].control = I915_READ(DSPCNTR(i)); error->plane[i].stride = I915_READ(DSPSTRIDE(i)); - if (INTEL_INFO(dev)->gen <= 3) { + if (INTEL_GEN(dev_priv) <= 3) { error->plane[i].size = I915_READ(DSPSIZE(i)); error->plane[i].pos = I915_READ(DSPPOS(i)); } - if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) + if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv)) error->plane[i].addr = I915_READ(DSPADDR(i)); - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { error->plane[i].surface = I915_READ(DSPSURF(i)); error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i)); } error->pipe[i].source = I915_READ(PIPESRC(i)); - if (HAS_GMCH_DISPLAY(dev)) + if (HAS_GMCH_DISPLAY(dev_priv)) error->pipe[i].stat = I915_READ(PIPESTAT(i)); } /* Note: this does not include DSI transcoders. */ - error->num_transcoders = INTEL_INFO(dev)->num_pipes; + error->num_transcoders = INTEL_INFO(dev_priv)->num_pipes; if (HAS_DDI(dev_priv)) error->num_transcoders++; /* Account for eDP. */ @@ -16321,7 +16543,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m, struct drm_device *dev, struct intel_display_error_state *error) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i; if (!error) diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index 891107f92..21b04c3ed 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -131,11 +131,6 @@ static void vlv_steal_power_sequencer(struct drm_device *dev, enum pipe pipe); static void intel_dp_unset_edid(struct intel_dp *intel_dp); -static unsigned int intel_dp_unused_lane_mask(int lane_count) -{ - return ~((1 << lane_count) - 1) & 0xf; -} - static int intel_dp_max_link_bw(struct intel_dp *intel_dp) { @@ -267,7 +262,7 @@ static void pps_lock(struct intel_dp *intel_dp) struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct intel_encoder *encoder = &intel_dig_port->base; struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; /* @@ -285,7 +280,7 @@ static void pps_unlock(struct intel_dp *intel_dp) struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct intel_encoder *encoder = &intel_dig_port->base; struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; mutex_unlock(&dev_priv->pps_mutex); @@ -299,7 +294,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe = intel_dp->pps_pipe; bool pll_enabled, release_cl_override = false; enum dpio_phy phy = DPIO_PHY(pipe); @@ -373,7 +368,7 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *encoder; unsigned int pipes = (1 << PIPE_A) | (1 << PIPE_B); enum pipe pipe; @@ -431,6 +426,37 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp) return intel_dp->pps_pipe; } +static int +bxt_power_sequencer_idx(struct intel_dp *intel_dp) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = intel_dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + + lockdep_assert_held(&dev_priv->pps_mutex); + + /* We should never land here with regular DP ports */ + WARN_ON(!is_edp(intel_dp)); + + /* + * TODO: BXT has 2 PPS instances. The correct port->PPS instance + * mapping needs to be retrieved from VBT, for now just hard-code to + * use instance #0 always. + */ + if (!intel_dp->pps_reset) + return 0; + + intel_dp->pps_reset = false; + + /* + * Only the HW needs to be reprogrammed, the SW state is fixed and + * has been setup during connector init. + */ + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp); + + return 0; +} + typedef bool (*vlv_pipe_check)(struct drm_i915_private *dev_priv, enum pipe pipe); @@ -480,7 +506,7 @@ vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_dig_port->port; lockdep_assert_held(&dev_priv->pps_mutex); @@ -512,12 +538,13 @@ vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp) intel_dp_init_panel_power_sequencer_registers(dev, intel_dp); } -void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv) +void intel_power_sequencer_reset(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct intel_encoder *encoder; - if (WARN_ON(!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev))) + if (WARN_ON(!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && + !IS_BROXTON(dev))) return; /* @@ -537,34 +564,71 @@ void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv) continue; intel_dp = enc_to_intel_dp(&encoder->base); - intel_dp->pps_pipe = INVALID_PIPE; + if (IS_BROXTON(dev)) + intel_dp->pps_reset = true; + else + intel_dp->pps_pipe = INVALID_PIPE; + } +} + +struct pps_registers { + i915_reg_t pp_ctrl; + i915_reg_t pp_stat; + i915_reg_t pp_on; + i915_reg_t pp_off; + i915_reg_t pp_div; +}; + +static void intel_pps_get_registers(struct drm_i915_private *dev_priv, + struct intel_dp *intel_dp, + struct pps_registers *regs) +{ + memset(regs, 0, sizeof(*regs)); + + if (IS_BROXTON(dev_priv)) { + int idx = bxt_power_sequencer_idx(intel_dp); + + regs->pp_ctrl = BXT_PP_CONTROL(idx); + regs->pp_stat = BXT_PP_STATUS(idx); + regs->pp_on = BXT_PP_ON_DELAYS(idx); + regs->pp_off = BXT_PP_OFF_DELAYS(idx); + } else if (HAS_PCH_SPLIT(dev_priv)) { + regs->pp_ctrl = PCH_PP_CONTROL; + regs->pp_stat = PCH_PP_STATUS; + regs->pp_on = PCH_PP_ON_DELAYS; + regs->pp_off = PCH_PP_OFF_DELAYS; + regs->pp_div = PCH_PP_DIVISOR; + } else { + enum pipe pipe = vlv_power_sequencer_pipe(intel_dp); + + regs->pp_ctrl = VLV_PIPE_PP_CONTROL(pipe); + regs->pp_stat = VLV_PIPE_PP_STATUS(pipe); + regs->pp_on = VLV_PIPE_PP_ON_DELAYS(pipe); + regs->pp_off = VLV_PIPE_PP_OFF_DELAYS(pipe); + regs->pp_div = VLV_PIPE_PP_DIVISOR(pipe); } } static i915_reg_t _pp_ctrl_reg(struct intel_dp *intel_dp) { - struct drm_device *dev = intel_dp_to_dev(intel_dp); + struct pps_registers regs; - if (IS_BROXTON(dev)) - return BXT_PP_CONTROL(0); - else if (HAS_PCH_SPLIT(dev)) - return PCH_PP_CONTROL; - else - return VLV_PIPE_PP_CONTROL(vlv_power_sequencer_pipe(intel_dp)); + intel_pps_get_registers(to_i915(intel_dp_to_dev(intel_dp)), intel_dp, + ®s); + + return regs.pp_ctrl; } static i915_reg_t _pp_stat_reg(struct intel_dp *intel_dp) { - struct drm_device *dev = intel_dp_to_dev(intel_dp); + struct pps_registers regs; - if (IS_BROXTON(dev)) - return BXT_PP_STATUS(0); - else if (HAS_PCH_SPLIT(dev)) - return PCH_PP_STATUS; - else - return VLV_PIPE_PP_STATUS(vlv_power_sequencer_pipe(intel_dp)); + intel_pps_get_registers(to_i915(intel_dp_to_dev(intel_dp)), intel_dp, + ®s); + + return regs.pp_stat; } /* Reboot notifier handler to shutdown panel power to guarantee T12 timing @@ -575,7 +639,7 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code, struct intel_dp *intel_dp = container_of(this, typeof(* intel_dp), edp_notifier); struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!is_edp(intel_dp) || code != SYS_RESTART) return 0; @@ -606,7 +670,7 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code, static bool edp_have_panel_power(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); lockdep_assert_held(&dev_priv->pps_mutex); @@ -620,7 +684,7 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp) static bool edp_have_panel_vdd(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); lockdep_assert_held(&dev_priv->pps_mutex); @@ -635,7 +699,7 @@ static void intel_dp_check_edp(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (!is_edp(intel_dp)) return; @@ -653,7 +717,7 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg; uint32_t status; bool done; @@ -775,6 +839,7 @@ static uint32_t skl_get_aux_send_ctl(struct intel_dp *intel_dp, DP_AUX_CH_CTL_TIME_OUT_1600us | DP_AUX_CH_CTL_RECEIVE_ERROR | (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) | + DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(32) | DP_AUX_CH_CTL_SYNC_PULSE_SKL(32); } @@ -785,7 +850,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp, { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg; uint32_t aux_clock_divider; int i, ret, recv_bytes; @@ -1181,48 +1246,21 @@ static void intel_aux_reg_init(struct intel_dp *intel_dp) static void intel_dp_aux_fini(struct intel_dp *intel_dp) { - drm_dp_aux_unregister(&intel_dp->aux); kfree(intel_dp->aux.name); } -static int +static void intel_dp_aux_init(struct intel_dp *intel_dp, struct intel_connector *connector) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); enum port port = intel_dig_port->port; - int ret; intel_aux_reg_init(intel_dp); + drm_dp_aux_init(&intel_dp->aux); + /* Failure to allocate our preferred name is not critical */ intel_dp->aux.name = kasprintf(GFP_KERNEL, "DPDDC-%c", port_name(port)); - if (!intel_dp->aux.name) - return -ENOMEM; - - intel_dp->aux.dev = connector->base.kdev; intel_dp->aux.transfer = intel_dp_aux_transfer; - - DRM_DEBUG_KMS("registering %s bus for %s\n", - intel_dp->aux.name, - connector->base.kdev->kobj.name); - - ret = drm_dp_aux_register(&intel_dp->aux); - if (ret < 0) { - DRM_ERROR("drm_dp_aux_register() for %s failed (%d)\n", - intel_dp->aux.name, ret); - kfree(intel_dp->aux.name); - return ret; - } - - return 0; -} - -static void -intel_dp_connector_unregister(struct intel_connector *intel_connector) -{ - struct intel_dp *intel_dp = intel_attached_dp(&intel_connector->base); - - intel_dp_aux_fini(intel_dp); - intel_connector_unregister(intel_connector); } static int @@ -1435,7 +1473,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); enum port port = dp_to_dig_port(intel_dp)->port; @@ -1463,7 +1501,6 @@ intel_dp_compute_config(struct intel_encoder *encoder, if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev) && port != PORT_A) pipe_config->has_pch_encoder = true; - pipe_config->has_dp_encoder = true; pipe_config->has_drrs = false; pipe_config->has_audio = intel_dp->has_audio && port != PORT_A; @@ -1582,6 +1619,27 @@ found: &pipe_config->dp_m2_n2); } + /* + * DPLL0 VCO may need to be adjusted to get the correct + * clock for eDP. This will affect cdclk as well. + */ + if (is_edp(intel_dp) && + (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))) { + int vco; + + switch (pipe_config->port_clock / 2) { + case 108000: + case 216000: + vco = 8640000; + break; + default: + vco = 8100000; + break; + } + + to_intel_atomic_state(pipe_config->base.state)->cdclk_pll_vco = vco; + } + if (!HAS_DDI(dev)) intel_dp_set_clock(encoder, pipe_config); @@ -1598,7 +1656,7 @@ void intel_dp_set_link_params(struct intel_dp *intel_dp, static void intel_dp_prepare(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); enum port port = dp_to_dig_port(intel_dp)->port; struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); @@ -1686,16 +1744,21 @@ static void intel_dp_prepare(struct intel_encoder *encoder) #define IDLE_CYCLE_MASK (PP_ON | PP_SEQUENCE_MASK | PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK) #define IDLE_CYCLE_VALUE (0 | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_OFF_IDLE) +static void intel_pps_verify_state(struct drm_i915_private *dev_priv, + struct intel_dp *intel_dp); + static void wait_panel_status(struct intel_dp *intel_dp, u32 mask, u32 value) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t pp_stat_reg, pp_ctrl_reg; lockdep_assert_held(&dev_priv->pps_mutex); + intel_pps_verify_state(dev_priv, intel_dp); + pp_stat_reg = _pp_stat_reg(intel_dp); pp_ctrl_reg = _pp_ctrl_reg(intel_dp); @@ -1704,8 +1767,9 @@ static void wait_panel_status(struct intel_dp *intel_dp, I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg)); - if (_wait_for((I915_READ(pp_stat_reg) & mask) == value, - 5 * USEC_PER_SEC, 10 * USEC_PER_MSEC)) + if (intel_wait_for_register(dev_priv, + pp_stat_reg, mask, value, + 5000)) DRM_ERROR("Panel status timeout: status %08x control %08x\n", I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg)); @@ -1765,7 +1829,7 @@ static void edp_wait_backlight_off(struct intel_dp *intel_dp) static u32 ironlake_get_pp_control(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 control; lockdep_assert_held(&dev_priv->pps_mutex); @@ -1788,7 +1852,7 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp) struct drm_device *dev = intel_dp_to_dev(intel_dp); struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct intel_encoder *intel_encoder = &intel_dig_port->base; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; u32 pp; i915_reg_t pp_stat_reg, pp_ctrl_reg; @@ -1861,7 +1925,7 @@ void intel_edp_panel_vdd_on(struct intel_dp *intel_dp) static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct intel_encoder *intel_encoder = &intel_dig_port->base; @@ -1930,8 +1994,7 @@ static void edp_panel_vdd_schedule_off(struct intel_dp *intel_dp) */ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) { - struct drm_i915_private *dev_priv = - intel_dp_to_dev(intel_dp)->dev_private; + struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp)); lockdep_assert_held(&dev_priv->pps_mutex); @@ -1952,7 +2015,7 @@ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) static void edp_panel_on(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 pp; i915_reg_t pp_ctrl_reg; @@ -2013,7 +2076,7 @@ static void edp_panel_off(struct intel_dp *intel_dp) struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; u32 pp; i915_reg_t pp_ctrl_reg; @@ -2065,7 +2128,7 @@ static void _intel_edp_backlight_on(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 pp; i915_reg_t pp_ctrl_reg; @@ -2106,7 +2169,7 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp) static void _intel_edp_backlight_off(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 pp; i915_reg_t pp_ctrl_reg; @@ -2222,7 +2285,7 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp) * 2. Program DP PLL enable */ if (IS_GEN5(dev_priv)) - intel_wait_for_vblank_if_active(dev_priv->dev, !crtc->pipe); + intel_wait_for_vblank_if_active(&dev_priv->drm, !crtc->pipe); intel_dp->DP |= DP_PLL_ENABLE; @@ -2287,7 +2350,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); enum port port = dp_to_dig_port(intel_dp)->port; struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; u32 tmp; bool ret; @@ -2340,7 +2403,7 @@ static void intel_dp_get_config(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); u32 tmp, flags = 0; struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = dp_to_dig_port(intel_dp)->port; struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); @@ -2378,8 +2441,6 @@ static void intel_dp_get_config(struct intel_encoder *encoder, !IS_CHERRYVIEW(dev) && tmp & DP_COLOR_RANGE_16_235) pipe_config->limited_color_range = true; - pipe_config->has_dp_encoder = true; - pipe_config->lane_count = ((tmp & DP_PORT_WIDTH_MASK) >> DP_PORT_WIDTH_SHIFT) + 1; @@ -2460,55 +2521,11 @@ static void vlv_post_disable_dp(struct intel_encoder *encoder) intel_dp_link_down(intel_dp); } -static void chv_data_lane_soft_reset(struct intel_encoder *encoder, - bool reset) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base)); - struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - enum pipe pipe = crtc->pipe; - uint32_t val; - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); - if (reset) - val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - else - val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); - - if (crtc->config->lane_count > 2) { - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); - if (reset) - val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - else - val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); - } - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - if (reset) - val &= ~DPIO_PCS_CLK_SOFT_RESET; - else - val |= DPIO_PCS_CLK_SOFT_RESET; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); - - if (crtc->config->lane_count > 2) { - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - if (reset) - val &= ~DPIO_PCS_CLK_SOFT_RESET; - else - val |= DPIO_PCS_CLK_SOFT_RESET; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); - } -} - static void chv_post_disable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); intel_dp_link_down(intel_dp); @@ -2527,7 +2544,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp, { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_dig_port->port; if (HAS_DDI(dev)) { @@ -2607,7 +2624,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp, static void intel_dp_enable_port(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(dp_to_dig_port(intel_dp)->base.base.crtc); @@ -2636,7 +2653,7 @@ static void intel_enable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); uint32_t dp_reg = I915_READ(intel_dp->output_reg); enum pipe pipe = crtc->pipe; @@ -2709,7 +2726,7 @@ static void g4x_pre_enable_dp(struct intel_encoder *encoder) static void vlv_detach_power_sequencer(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_i915_private *dev_priv = intel_dig_port->base.base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); enum pipe pipe = intel_dp->pps_pipe; i915_reg_t pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe); @@ -2735,7 +2752,7 @@ static void vlv_detach_power_sequencer(struct intel_dp *intel_dp) static void vlv_steal_power_sequencer(struct drm_device *dev, enum pipe pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *encoder; lockdep_assert_held(&dev_priv->pps_mutex); @@ -2773,7 +2790,7 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp) struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct intel_encoder *encoder = &intel_dig_port->base; struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); lockdep_assert_held(&dev_priv->pps_mutex); @@ -2811,266 +2828,38 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp) static void vlv_pre_enable_dp(struct intel_encoder *encoder) { - struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); - struct intel_digital_port *dport = dp_to_dig_port(intel_dp); - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); - enum dpio_channel port = vlv_dport_to_channel(dport); - int pipe = intel_crtc->pipe; - u32 val; - - mutex_lock(&dev_priv->sb_lock); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(port)); - val = 0; - if (pipe) - val |= (1<<21); - else - val &= ~(1<<21); - val |= 0x001000c4; - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW8(port), val); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW14(port), 0x00760018); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW23(port), 0x00400888); - - mutex_unlock(&dev_priv->sb_lock); + vlv_phy_pre_encoder_enable(encoder); intel_enable_dp(encoder); } static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder) { - struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = - to_intel_crtc(encoder->base.crtc); - enum dpio_channel port = vlv_dport_to_channel(dport); - int pipe = intel_crtc->pipe; - intel_dp_prepare(encoder); - /* Program Tx lane resets to default */ - mutex_lock(&dev_priv->sb_lock); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), - DPIO_PCS_TX_LANE2_RESET | - DPIO_PCS_TX_LANE1_RESET); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), - DPIO_PCS_CLK_CRI_RXEB_EIOS_EN | - DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN | - (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) | - DPIO_PCS_CLK_SOFT_RESET); - - /* Fix up inter-pair skew failure */ - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW12(port), 0x00750f00); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW11(port), 0x00001500); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW14(port), 0x40400000); - mutex_unlock(&dev_priv->sb_lock); + vlv_phy_pre_pll_enable(encoder); } static void chv_pre_enable_dp(struct intel_encoder *encoder) { - struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); - struct intel_digital_port *dport = dp_to_dig_port(intel_dp); - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = - to_intel_crtc(encoder->base.crtc); - enum dpio_channel ch = vlv_dport_to_channel(dport); - int pipe = intel_crtc->pipe; - int data, i, stagger; - u32 val; - - mutex_lock(&dev_priv->sb_lock); - - /* allow hardware to manage TX FIFO reset source */ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch)); - val &= ~DPIO_LANEDESKEW_STRAP_OVRD; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); - - if (intel_crtc->config->lane_count > 2) { - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); - val &= ~DPIO_LANEDESKEW_STRAP_OVRD; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); - } - - /* Program Tx lane latency optimal setting*/ - for (i = 0; i < intel_crtc->config->lane_count; i++) { - /* Set the upar bit */ - if (intel_crtc->config->lane_count == 1) - data = 0x0; - else - data = (i == 1) ? 0x0 : 0x1; - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i), - data << DPIO_UPAR_SHIFT); - } - - /* Data lane stagger programming */ - if (intel_crtc->config->port_clock > 270000) - stagger = 0x18; - else if (intel_crtc->config->port_clock > 135000) - stagger = 0xd; - else if (intel_crtc->config->port_clock > 67500) - stagger = 0x7; - else if (intel_crtc->config->port_clock > 33750) - stagger = 0x4; - else - stagger = 0x2; - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch)); - val |= DPIO_TX2_STAGGER_MASK(0x1f); - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); - - if (intel_crtc->config->lane_count > 2) { - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); - val |= DPIO_TX2_STAGGER_MASK(0x1f); - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); - } - - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch), - DPIO_LANESTAGGER_STRAP(stagger) | - DPIO_LANESTAGGER_STRAP_OVRD | - DPIO_TX1_STAGGER_MASK(0x1f) | - DPIO_TX1_STAGGER_MULT(6) | - DPIO_TX2_STAGGER_MULT(0)); - - if (intel_crtc->config->lane_count > 2) { - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch), - DPIO_LANESTAGGER_STRAP(stagger) | - DPIO_LANESTAGGER_STRAP_OVRD | - DPIO_TX1_STAGGER_MASK(0x1f) | - DPIO_TX1_STAGGER_MULT(7) | - DPIO_TX2_STAGGER_MULT(5)); - } - - /* Deassert data lane reset */ - chv_data_lane_soft_reset(encoder, false); - - mutex_unlock(&dev_priv->sb_lock); + chv_phy_pre_encoder_enable(encoder); intel_enable_dp(encoder); /* Second common lane will stay alive on its own now */ - if (dport->release_cl2_override) { - chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false); - dport->release_cl2_override = false; - } + chv_phy_release_cl2_override(encoder); } static void chv_dp_pre_pll_enable(struct intel_encoder *encoder) { - struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = - to_intel_crtc(encoder->base.crtc); - enum dpio_channel ch = vlv_dport_to_channel(dport); - enum pipe pipe = intel_crtc->pipe; - unsigned int lane_mask = - intel_dp_unused_lane_mask(intel_crtc->config->lane_count); - u32 val; - intel_dp_prepare(encoder); - /* - * Must trick the second common lane into life. - * Otherwise we can't even access the PLL. - */ - if (ch == DPIO_CH0 && pipe == PIPE_B) - dport->release_cl2_override = - !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true); - - chv_phy_powergate_lanes(encoder, true, lane_mask); - - mutex_lock(&dev_priv->sb_lock); - - /* Assert data lane reset */ - chv_data_lane_soft_reset(encoder, true); - - /* program left/right clock distribution */ - if (pipe != PIPE_B) { - val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); - val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); - if (ch == DPIO_CH0) - val |= CHV_BUFLEFTENA1_FORCE; - if (ch == DPIO_CH1) - val |= CHV_BUFRIGHTENA1_FORCE; - vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); - } else { - val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); - val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); - if (ch == DPIO_CH0) - val |= CHV_BUFLEFTENA2_FORCE; - if (ch == DPIO_CH1) - val |= CHV_BUFRIGHTENA2_FORCE; - vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); - } - - /* program clock channel usage */ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(ch)); - val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; - if (pipe != PIPE_B) - val &= ~CHV_PCS_USEDCLKCHANNEL; - else - val |= CHV_PCS_USEDCLKCHANNEL; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val); - - if (intel_crtc->config->lane_count > 2) { - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch)); - val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; - if (pipe != PIPE_B) - val &= ~CHV_PCS_USEDCLKCHANNEL; - else - val |= CHV_PCS_USEDCLKCHANNEL; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val); - } - - /* - * This a a bit weird since generally CL - * matches the pipe, but here we need to - * pick the CL based on the port. - */ - val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW19(ch)); - if (pipe != PIPE_B) - val &= ~CHV_CMN_USEDCLKCHANNEL; - else - val |= CHV_CMN_USEDCLKCHANNEL; - vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW19(ch), val); - - mutex_unlock(&dev_priv->sb_lock); + chv_phy_pre_pll_enable(encoder); } static void chv_dp_post_pll_disable(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe; - u32 val; - - mutex_lock(&dev_priv->sb_lock); - - /* disable left/right clock distribution */ - if (pipe != PIPE_B) { - val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); - val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); - vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); - } else { - val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); - val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); - vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); - } - - mutex_unlock(&dev_priv->sb_lock); - - /* - * Leave the power down bit cleared for at least one - * lane so that chv_powergate_phy_ch() will power - * on something when the channel is otherwise unused. - * When the port is off and the override is removed - * the lanes power down anyway, so otherwise it doesn't - * really matter what the state of power down bits is - * after this. - */ - chv_phy_powergate_lanes(encoder, false, 0x0); + chv_phy_post_pll_disable(encoder); } /* @@ -3089,7 +2878,7 @@ uint8_t intel_dp_voltage_max(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = dp_to_dig_port(intel_dp)->port; if (IS_BROXTON(dev)) @@ -3178,16 +2967,10 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing) static uint32_t vlv_signal_levels(struct intel_dp *intel_dp) { - struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_digital_port *dport = dp_to_dig_port(intel_dp); - struct intel_crtc *intel_crtc = - to_intel_crtc(dport->base.base.crtc); + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; unsigned long demph_reg_value, preemph_reg_value, uniqtranscale_reg_value; uint8_t train_set = intel_dp->train_set[0]; - enum dpio_channel port = vlv_dport_to_channel(dport); - int pipe = intel_crtc->pipe; switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) { case DP_TRAIN_PRE_EMPH_LEVEL_0: @@ -3262,37 +3045,18 @@ static uint32_t vlv_signal_levels(struct intel_dp *intel_dp) return 0; } - mutex_lock(&dev_priv->sb_lock); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x00000000); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW4(port), demph_reg_value); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(port), - uniqtranscale_reg_value); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(port), 0x0C782040); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW11(port), 0x00030000); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), preemph_reg_value); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x80000000); - mutex_unlock(&dev_priv->sb_lock); + vlv_set_phy_signal_level(encoder, demph_reg_value, preemph_reg_value, + uniqtranscale_reg_value, 0); return 0; } -static bool chv_need_uniq_trans_scale(uint8_t train_set) -{ - return (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) == DP_TRAIN_PRE_EMPH_LEVEL_0 && - (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) == DP_TRAIN_VOLTAGE_SWING_LEVEL_3; -} - static uint32_t chv_signal_levels(struct intel_dp *intel_dp) { - struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_digital_port *dport = dp_to_dig_port(intel_dp); - struct intel_crtc *intel_crtc = to_intel_crtc(dport->base.base.crtc); - u32 deemph_reg_value, margin_reg_value, val; + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + u32 deemph_reg_value, margin_reg_value; + bool uniq_trans_scale = false; uint8_t train_set = intel_dp->train_set[0]; - enum dpio_channel ch = vlv_dport_to_channel(dport); - enum pipe pipe = intel_crtc->pipe; - int i; switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) { case DP_TRAIN_PRE_EMPH_LEVEL_0: @@ -3312,7 +3076,7 @@ static uint32_t chv_signal_levels(struct intel_dp *intel_dp) case DP_TRAIN_VOLTAGE_SWING_LEVEL_3: deemph_reg_value = 128; margin_reg_value = 154; - /* FIXME extra to set for 1200 */ + uniq_trans_scale = true; break; default: return 0; @@ -3364,88 +3128,8 @@ static uint32_t chv_signal_levels(struct intel_dp *intel_dp) return 0; } - mutex_lock(&dev_priv->sb_lock); - - /* Clear calc init */ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); - val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); - val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); - val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); - - if (intel_crtc->config->lane_count > 2) { - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); - val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); - val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); - val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); - } - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch)); - val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); - val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val); - - if (intel_crtc->config->lane_count > 2) { - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch)); - val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); - val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val); - } - - /* Program swing deemph */ - for (i = 0; i < intel_crtc->config->lane_count; i++) { - val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i)); - val &= ~DPIO_SWING_DEEMPH9P5_MASK; - val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT; - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val); - } - - /* Program swing margin */ - for (i = 0; i < intel_crtc->config->lane_count; i++) { - val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); - - val &= ~DPIO_SWING_MARGIN000_MASK; - val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT; - - /* - * Supposedly this value shouldn't matter when unique transition - * scale is disabled, but in fact it does matter. Let's just - * always program the same value and hope it's OK. - */ - val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT); - val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT; - - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); - } - - /* - * The document said it needs to set bit 27 for ch0 and bit 26 - * for ch1. Might be a typo in the doc. - * For now, for this unique transition scale selection, set bit - * 27 for ch0 and ch1. - */ - for (i = 0; i < intel_crtc->config->lane_count; i++) { - val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); - if (chv_need_uniq_trans_scale(train_set)) - val |= DPIO_TX_UNIQ_TRANS_SCALE_EN; - else - val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN; - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); - } - - /* Start swing calculation */ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); - val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); - - if (intel_crtc->config->lane_count > 2) { - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); - val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); - } - - mutex_unlock(&dev_priv->sb_lock); + chv_set_phy_signal_level(encoder, deemph_reg_value, + margin_reg_value, uniq_trans_scale); return 0; } @@ -3612,7 +3296,7 @@ void intel_dp_set_idle_link_train(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_dig_port->port; uint32_t val; @@ -3634,8 +3318,10 @@ void intel_dp_set_idle_link_train(struct intel_dp *intel_dp) if (port == PORT_A) return; - if (wait_for((I915_READ(DP_TP_STATUS(port)) & DP_TP_STATUS_IDLE_DONE), - 1)) + if (intel_wait_for_register(dev_priv,DP_TP_STATUS(port), + DP_TP_STATUS_IDLE_DONE, + DP_TP_STATUS_IDLE_DONE, + 1)) DRM_ERROR("Timed out waiting for DP idle patterns\n"); } @@ -3646,7 +3332,7 @@ intel_dp_link_down(struct intel_dp *intel_dp) struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc); enum port port = intel_dig_port->port; struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t DP = intel_dp->DP; if (WARN_ON(HAS_DDI(dev))) @@ -3698,7 +3384,7 @@ intel_dp_link_down(struct intel_dp *intel_dp) I915_WRITE(intel_dp->output_reg, DP); POSTING_READ(intel_dp->output_reg); - intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A); + intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } @@ -3713,8 +3399,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - uint8_t rev; + struct drm_i915_private *dev_priv = to_i915(dev); if (drm_dp_dpcd_read(&intel_dp->aux, 0x000, intel_dp->dpcd, sizeof(intel_dp->dpcd)) < 0) @@ -3771,6 +3456,15 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) DRM_DEBUG_KMS("PSR2 %s on sink", dev_priv->psr.psr2_support ? "supported" : "not supported"); } + + /* Read the eDP Display control capabilities registers */ + memset(intel_dp->edp_dpcd, 0, sizeof(intel_dp->edp_dpcd)); + if ((intel_dp->dpcd[DP_EDP_CONFIGURATION_CAP] & DP_DPCD_DISPLAY_CONTROL_CAPABLE) && + (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV, + intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd)) == + sizeof(intel_dp->edp_dpcd))) + DRM_DEBUG_KMS("EDP DPCD : %*ph\n", (int) sizeof(intel_dp->edp_dpcd), + intel_dp->edp_dpcd); } DRM_DEBUG_KMS("Display Port TPS3 support: source %s, sink %s\n", @@ -3778,10 +3472,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) yesno(drm_dp_tps3_supported(intel_dp->dpcd))); /* Intermediate frequency support */ - if (is_edp(intel_dp) && - (intel_dp->dpcd[DP_EDP_CONFIGURATION_CAP] & DP_DPCD_DISPLAY_CONTROL_CAPABLE) && - (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV, &rev, 1) == 1) && - (rev >= 0x03)) { /* eDp v1.4 or higher */ + if (is_edp(intel_dp) && (intel_dp->edp_dpcd[0] >= 0x03)) { /* eDp v1.4 or higher */ __le16 sink_rates[DP_MAX_SUPPORTED_RATES]; int i; @@ -4559,7 +4250,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector) } if (intel_encoder->type != INTEL_OUTPUT_EDP) - intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; + intel_encoder->type = INTEL_OUTPUT_DP; intel_dp_probe_oui(intel_dp); @@ -4635,7 +4326,7 @@ intel_dp_detect(struct drm_connector *connector, bool force) /* MST devices are disconnected from a monitor POV */ intel_dp_unset_edid(intel_dp); if (intel_encoder->type != INTEL_OUTPUT_EDP) - intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; + intel_encoder->type = INTEL_OUTPUT_DP; return connector_status_disconnected; } @@ -4674,7 +4365,7 @@ intel_dp_force(struct drm_connector *connector) intel_display_power_put(dev_priv, power_domain); if (intel_encoder->type != INTEL_OUTPUT_EDP) - intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; + intel_encoder->type = INTEL_OUTPUT_DP; } static int intel_dp_get_modes(struct drm_connector *connector) @@ -4723,7 +4414,7 @@ intel_dp_set_property(struct drm_connector *connector, struct drm_property *property, uint64_t val) { - struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(connector->dev); struct intel_connector *intel_connector = to_intel_connector(connector); struct intel_encoder *intel_encoder = intel_attached_encoder(connector); struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); @@ -4811,6 +4502,32 @@ done: return 0; } +static int +intel_dp_connector_register(struct drm_connector *connector) +{ + struct intel_dp *intel_dp = intel_attached_dp(connector); + int ret; + + ret = intel_connector_register(connector); + if (ret) + return ret; + + i915_debugfs_connector_add(connector); + + DRM_DEBUG_KMS("registering %s bus for %s\n", + intel_dp->aux.name, connector->kdev->kobj.name); + + intel_dp->aux.dev = connector->kdev; + return drm_dp_aux_register(&intel_dp->aux); +} + +static void +intel_dp_connector_unregister(struct drm_connector *connector) +{ + drm_dp_aux_unregister(&intel_attached_dp(connector)->aux); + intel_connector_unregister(connector); +} + static void intel_dp_connector_destroy(struct drm_connector *connector) { @@ -4851,6 +4568,9 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder) intel_dp->edp_notifier.notifier_call = NULL; } } + + intel_dp_aux_fini(intel_dp); + drm_encoder_cleanup(encoder); kfree(intel_dig_port); } @@ -4876,7 +4596,7 @@ static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; lockdep_assert_held(&dev_priv->pps_mutex); @@ -4929,6 +4649,8 @@ static const struct drm_connector_funcs intel_dp_connector_funcs = { .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_dp_set_property, .atomic_get_property = intel_connector_atomic_get_property, + .late_register = intel_dp_connector_register, + .early_unregister = intel_dp_connector_unregister, .destroy = intel_dp_connector_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, @@ -4937,7 +4659,6 @@ static const struct drm_connector_funcs intel_dp_connector_funcs = { static const struct drm_connector_helper_funcs intel_dp_connector_helper_funcs = { .get_modes = intel_dp_get_modes, .mode_valid = intel_dp_mode_valid, - .best_encoder = intel_best_encoder, }; static const struct drm_encoder_funcs intel_dp_enc_funcs = { @@ -4951,13 +4672,13 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) struct intel_dp *intel_dp = &intel_dig_port->dp; struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum intel_display_power_domain power_domain; enum irqreturn ret = IRQ_NONE; if (intel_dig_port->base.type != INTEL_OUTPUT_EDP && intel_dig_port->base.type != INTEL_OUTPUT_HDMI) - intel_dig_port->base.type = INTEL_OUTPUT_DISPLAYPORT; + intel_dig_port->base.type = INTEL_OUTPUT_DP; if (long_hpd && intel_dig_port->base.type == INTEL_OUTPUT_EDP) { /* @@ -5019,7 +4740,7 @@ put_power: /* check the VBT to see whether the eDP is on another port */ bool intel_dp_is_edp(struct drm_device *dev, enum port port) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* * eDP not supported on g4x. so bail out early just @@ -5061,82 +4782,93 @@ static void intel_dp_init_panel_power_timestamps(struct intel_dp *intel_dp) } static void -intel_dp_init_panel_power_sequencer(struct drm_device *dev, - struct intel_dp *intel_dp) +intel_pps_readout_hw_state(struct drm_i915_private *dev_priv, + struct intel_dp *intel_dp, struct edp_power_seq *seq) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct edp_power_seq cur, vbt, spec, - *final = &intel_dp->pps_delays; u32 pp_on, pp_off, pp_div = 0, pp_ctl = 0; - i915_reg_t pp_ctrl_reg, pp_on_reg, pp_off_reg, pp_div_reg; - - lockdep_assert_held(&dev_priv->pps_mutex); - - /* already initialized? */ - if (final->t11_t12 != 0) - return; - - if (IS_BROXTON(dev)) { - /* - * TODO: BXT has 2 sets of PPS registers. - * Correct Register for Broxton need to be identified - * using VBT. hardcoding for now - */ - pp_ctrl_reg = BXT_PP_CONTROL(0); - pp_on_reg = BXT_PP_ON_DELAYS(0); - pp_off_reg = BXT_PP_OFF_DELAYS(0); - } else if (HAS_PCH_SPLIT(dev)) { - pp_ctrl_reg = PCH_PP_CONTROL; - pp_on_reg = PCH_PP_ON_DELAYS; - pp_off_reg = PCH_PP_OFF_DELAYS; - pp_div_reg = PCH_PP_DIVISOR; - } else { - enum pipe pipe = vlv_power_sequencer_pipe(intel_dp); + struct pps_registers regs; - pp_ctrl_reg = VLV_PIPE_PP_CONTROL(pipe); - pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe); - pp_off_reg = VLV_PIPE_PP_OFF_DELAYS(pipe); - pp_div_reg = VLV_PIPE_PP_DIVISOR(pipe); - } + intel_pps_get_registers(dev_priv, intel_dp, ®s); /* Workaround: Need to write PP_CONTROL with the unlock key as * the very first thing. */ pp_ctl = ironlake_get_pp_control(intel_dp); - pp_on = I915_READ(pp_on_reg); - pp_off = I915_READ(pp_off_reg); - if (!IS_BROXTON(dev)) { - I915_WRITE(pp_ctrl_reg, pp_ctl); - pp_div = I915_READ(pp_div_reg); + pp_on = I915_READ(regs.pp_on); + pp_off = I915_READ(regs.pp_off); + if (!IS_BROXTON(dev_priv)) { + I915_WRITE(regs.pp_ctrl, pp_ctl); + pp_div = I915_READ(regs.pp_div); } /* Pull timing values out of registers */ - cur.t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >> - PANEL_POWER_UP_DELAY_SHIFT; + seq->t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >> + PANEL_POWER_UP_DELAY_SHIFT; - cur.t8 = (pp_on & PANEL_LIGHT_ON_DELAY_MASK) >> - PANEL_LIGHT_ON_DELAY_SHIFT; + seq->t8 = (pp_on & PANEL_LIGHT_ON_DELAY_MASK) >> + PANEL_LIGHT_ON_DELAY_SHIFT; - cur.t9 = (pp_off & PANEL_LIGHT_OFF_DELAY_MASK) >> - PANEL_LIGHT_OFF_DELAY_SHIFT; + seq->t9 = (pp_off & PANEL_LIGHT_OFF_DELAY_MASK) >> + PANEL_LIGHT_OFF_DELAY_SHIFT; - cur.t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >> - PANEL_POWER_DOWN_DELAY_SHIFT; + seq->t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >> + PANEL_POWER_DOWN_DELAY_SHIFT; - if (IS_BROXTON(dev)) { + if (IS_BROXTON(dev_priv)) { u16 tmp = (pp_ctl & BXT_POWER_CYCLE_DELAY_MASK) >> BXT_POWER_CYCLE_DELAY_SHIFT; if (tmp > 0) - cur.t11_t12 = (tmp - 1) * 1000; + seq->t11_t12 = (tmp - 1) * 1000; else - cur.t11_t12 = 0; + seq->t11_t12 = 0; } else { - cur.t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >> + seq->t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >> PANEL_POWER_CYCLE_DELAY_SHIFT) * 1000; } +} + +static void +intel_pps_dump_state(const char *state_name, const struct edp_power_seq *seq) +{ + DRM_DEBUG_KMS("%s t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", + state_name, + seq->t1_t3, seq->t8, seq->t9, seq->t10, seq->t11_t12); +} + +static void +intel_pps_verify_state(struct drm_i915_private *dev_priv, + struct intel_dp *intel_dp) +{ + struct edp_power_seq hw; + struct edp_power_seq *sw = &intel_dp->pps_delays; + + intel_pps_readout_hw_state(dev_priv, intel_dp, &hw); + + if (hw.t1_t3 != sw->t1_t3 || hw.t8 != sw->t8 || hw.t9 != sw->t9 || + hw.t10 != sw->t10 || hw.t11_t12 != sw->t11_t12) { + DRM_ERROR("PPS state mismatch\n"); + intel_pps_dump_state("sw", sw); + intel_pps_dump_state("hw", &hw); + } +} + +static void +intel_dp_init_panel_power_sequencer(struct drm_device *dev, + struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = to_i915(dev); + struct edp_power_seq cur, vbt, spec, + *final = &intel_dp->pps_delays; + + lockdep_assert_held(&dev_priv->pps_mutex); + + /* already initialized? */ + if (final->t11_t12 != 0) + return; - DRM_DEBUG_KMS("cur t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", - cur.t1_t3, cur.t8, cur.t9, cur.t10, cur.t11_t12); + intel_pps_readout_hw_state(dev_priv, intel_dp, &cur); + + intel_pps_dump_state("cur", &cur); vbt = dev_priv->vbt.edp.pps; @@ -5152,8 +4884,7 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev, * too. */ spec.t11_t12 = (510 + 100) * 10; - DRM_DEBUG_KMS("vbt t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", - vbt.t1_t3, vbt.t8, vbt.t9, vbt.t10, vbt.t11_t12); + intel_pps_dump_state("vbt", &vbt); /* Use the max of the register settings and vbt. If both are * unset, fall back to the spec limits. */ @@ -5181,59 +4912,41 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev, DRM_DEBUG_KMS("backlight on delay %d, off delay %d\n", intel_dp->backlight_on_delay, intel_dp->backlight_off_delay); + + /* + * We override the HW backlight delays to 1 because we do manual waits + * on them. For T8, even BSpec recommends doing it. For T9, if we + * don't do this, we'll end up waiting for the backlight off delay + * twice: once when we do the manual sleep, and once when we disable + * the panel and wait for the PP_STATUS bit to become zero. + */ + final->t8 = 1; + final->t9 = 1; } static void intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, struct intel_dp *intel_dp) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 pp_on, pp_off, pp_div, port_sel = 0; int div = dev_priv->rawclk_freq / 1000; - i915_reg_t pp_on_reg, pp_off_reg, pp_div_reg, pp_ctrl_reg; + struct pps_registers regs; enum port port = dp_to_dig_port(intel_dp)->port; const struct edp_power_seq *seq = &intel_dp->pps_delays; lockdep_assert_held(&dev_priv->pps_mutex); - if (IS_BROXTON(dev)) { - /* - * TODO: BXT has 2 sets of PPS registers. - * Correct Register for Broxton need to be identified - * using VBT. hardcoding for now - */ - pp_ctrl_reg = BXT_PP_CONTROL(0); - pp_on_reg = BXT_PP_ON_DELAYS(0); - pp_off_reg = BXT_PP_OFF_DELAYS(0); - - } else if (HAS_PCH_SPLIT(dev)) { - pp_on_reg = PCH_PP_ON_DELAYS; - pp_off_reg = PCH_PP_OFF_DELAYS; - pp_div_reg = PCH_PP_DIVISOR; - } else { - enum pipe pipe = vlv_power_sequencer_pipe(intel_dp); + intel_pps_get_registers(dev_priv, intel_dp, ®s); - pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe); - pp_off_reg = VLV_PIPE_PP_OFF_DELAYS(pipe); - pp_div_reg = VLV_PIPE_PP_DIVISOR(pipe); - } - - /* - * And finally store the new values in the power sequencer. The - * backlight delays are set to 1 because we do manual waits on them. For - * T8, even BSpec recommends doing it. For T9, if we don't do this, - * we'll end up waiting for the backlight off delay twice: once when we - * do the manual sleep, and once when we disable the panel and wait for - * the PP_STATUS bit to become zero. - */ pp_on = (seq->t1_t3 << PANEL_POWER_UP_DELAY_SHIFT) | - (1 << PANEL_LIGHT_ON_DELAY_SHIFT); - pp_off = (1 << PANEL_LIGHT_OFF_DELAY_SHIFT) | + (seq->t8 << PANEL_LIGHT_ON_DELAY_SHIFT); + pp_off = (seq->t9 << PANEL_LIGHT_OFF_DELAY_SHIFT) | (seq->t10 << PANEL_POWER_DOWN_DELAY_SHIFT); /* Compute the divisor for the pp clock, simply match the Bspec * formula. */ if (IS_BROXTON(dev)) { - pp_div = I915_READ(pp_ctrl_reg); + pp_div = I915_READ(regs.pp_ctrl); pp_div &= ~BXT_POWER_CYCLE_DELAY_MASK; pp_div |= (DIV_ROUND_UP((seq->t11_t12 + 1), 1000) << BXT_POWER_CYCLE_DELAY_SHIFT); @@ -5256,19 +4969,19 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, pp_on |= port_sel; - I915_WRITE(pp_on_reg, pp_on); - I915_WRITE(pp_off_reg, pp_off); + I915_WRITE(regs.pp_on, pp_on); + I915_WRITE(regs.pp_off, pp_off); if (IS_BROXTON(dev)) - I915_WRITE(pp_ctrl_reg, pp_div); + I915_WRITE(regs.pp_ctrl, pp_div); else - I915_WRITE(pp_div_reg, pp_div); + I915_WRITE(regs.pp_div, pp_div); DRM_DEBUG_KMS("panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n", - I915_READ(pp_on_reg), - I915_READ(pp_off_reg), + I915_READ(regs.pp_on), + I915_READ(regs.pp_off), IS_BROXTON(dev) ? - (I915_READ(pp_ctrl_reg) & BXT_POWER_CYCLE_DELAY_MASK) : - I915_READ(pp_div_reg)); + (I915_READ(regs.pp_ctrl) & BXT_POWER_CYCLE_DELAY_MASK) : + I915_READ(regs.pp_div)); } /** @@ -5285,7 +4998,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, */ static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *encoder; struct intel_digital_port *dig_port = NULL; struct intel_dp *intel_dp = dev_priv->drrs.dp; @@ -5384,7 +5097,7 @@ static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate) void intel_edp_drrs_enable(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_crtc *crtc = dig_port->base.base.crtc; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); @@ -5416,7 +5129,7 @@ unlock: void intel_edp_drrs_disable(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_crtc *crtc = dig_port->base.base.crtc; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); @@ -5431,9 +5144,9 @@ void intel_edp_drrs_disable(struct intel_dp *intel_dp) } if (dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) - intel_dp_set_drrs_state(dev_priv->dev, - intel_dp->attached_connector->panel. - fixed_mode->vrefresh); + intel_dp_set_drrs_state(&dev_priv->drm, + intel_dp->attached_connector->panel. + fixed_mode->vrefresh); dev_priv->drrs.dp = NULL; mutex_unlock(&dev_priv->drrs.mutex); @@ -5463,9 +5176,9 @@ static void intel_edp_drrs_downclock_work(struct work_struct *work) goto unlock; if (dev_priv->drrs.refresh_rate_type != DRRS_LOW_RR) - intel_dp_set_drrs_state(dev_priv->dev, - intel_dp->attached_connector->panel. - downclock_mode->vrefresh); + intel_dp_set_drrs_state(&dev_priv->drm, + intel_dp->attached_connector->panel. + downclock_mode->vrefresh); unlock: mutex_unlock(&dev_priv->drrs.mutex); @@ -5484,7 +5197,7 @@ unlock: void intel_edp_drrs_invalidate(struct drm_device *dev, unsigned frontbuffer_bits) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc; enum pipe pipe; @@ -5507,9 +5220,9 @@ void intel_edp_drrs_invalidate(struct drm_device *dev, /* invalidate means busy screen hence upclock */ if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) - intel_dp_set_drrs_state(dev_priv->dev, - dev_priv->drrs.dp->attached_connector->panel. - fixed_mode->vrefresh); + intel_dp_set_drrs_state(&dev_priv->drm, + dev_priv->drrs.dp->attached_connector->panel. + fixed_mode->vrefresh); mutex_unlock(&dev_priv->drrs.mutex); } @@ -5529,7 +5242,7 @@ void intel_edp_drrs_invalidate(struct drm_device *dev, void intel_edp_drrs_flush(struct drm_device *dev, unsigned frontbuffer_bits) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc; enum pipe pipe; @@ -5552,9 +5265,9 @@ void intel_edp_drrs_flush(struct drm_device *dev, /* flush means busy screen hence upclock */ if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR) - intel_dp_set_drrs_state(dev_priv->dev, - dev_priv->drrs.dp->attached_connector->panel. - fixed_mode->vrefresh); + intel_dp_set_drrs_state(&dev_priv->drm, + dev_priv->drrs.dp->attached_connector->panel. + fixed_mode->vrefresh); /* * flush also means no more activity hence schedule downclock, if all @@ -5589,14 +5302,14 @@ void intel_edp_drrs_flush(struct drm_device *dev, * * DRRS saves power by switching to low RR based on usage scenarios. * - * eDP DRRS:- - * The implementation is based on frontbuffer tracking implementation. - * When there is a disturbance on the screen triggered by user activity or a - * periodic system activity, DRRS is disabled (RR is changed to high RR). - * When there is no movement on screen, after a timeout of 1 second, a switch - * to low RR is made. - * For integration with frontbuffer tracking code, - * intel_edp_drrs_invalidate() and intel_edp_drrs_flush() are called. + * The implementation is based on frontbuffer tracking implementation. When + * there is a disturbance on the screen triggered by user activity or a periodic + * system activity, DRRS is disabled (RR is changed to high RR). When there is + * no movement on screen, after a timeout of 1 second, a switch to low RR is + * made. + * + * For integration with frontbuffer tracking code, intel_edp_drrs_invalidate() + * and intel_edp_drrs_flush() are called. * * DRRS can be further extended to support other internal panels and also * the scenario of video playback wherein RR is set based on the rate @@ -5622,7 +5335,7 @@ intel_dp_drrs_init(struct intel_connector *intel_connector, { struct drm_connector *connector = &intel_connector->base; struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_display_mode *downclock_mode = NULL; INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work); @@ -5660,7 +5373,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_device *dev = intel_encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_display_mode *fixed_mode = NULL; struct drm_display_mode *downclock_mode = NULL; bool has_dpcd; @@ -5671,8 +5384,32 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, if (!is_edp(intel_dp)) return true; + /* + * On IBX/CPT we may get here with LVDS already registered. Since the + * driver uses the only internal power sequencer available for both + * eDP and LVDS bail out early in this case to prevent interfering + * with an already powered-on LVDS power sequencer. + */ + if (intel_get_lvds_encoder(dev)) { + WARN_ON(!(HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))); + DRM_INFO("LVDS was detected, not registering eDP\n"); + + return false; + } + pps_lock(intel_dp); + + intel_dp_init_panel_power_timestamps(intel_dp); + + if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) { + vlv_initial_power_sequencer_setup(intel_dp); + } else { + intel_dp_init_panel_power_sequencer(dev, intel_dp); + intel_dp_init_panel_power_sequencer_registers(dev, intel_dp); + } + intel_edp_panel_vdd_sanitize(intel_dp); + pps_unlock(intel_dp); /* Cache DPCD and EDID for edp. */ @@ -5686,14 +5423,9 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, } else { /* if this fails, presume the device is a ghost */ DRM_INFO("failed to retrieve link info, disabling eDP\n"); - return false; + goto out_vdd_off; } - /* We now know it's not a ghost, init power sequence regs. */ - pps_lock(intel_dp); - intel_dp_init_panel_power_sequencer_registers(dev, intel_dp); - pps_unlock(intel_dp); - mutex_lock(&dev->mode_config.mutex); edid = drm_get_edid(connector, &intel_dp->aux.ddc); if (edid) { @@ -5761,6 +5493,18 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, intel_panel_setup_backlight(connector, pipe); return true; + +out_vdd_off: + cancel_delayed_work_sync(&intel_dp->panel_vdd_work); + /* + * vdd might still be enabled do to the delayed vdd off. + * Make sure vdd is actually turned off here. + */ + pps_lock(intel_dp); + edp_panel_vdd_off_sync(intel_dp); + pps_unlock(intel_dp); + + return false; } bool @@ -5771,9 +5515,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, struct intel_dp *intel_dp = &intel_dig_port->dp; struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_device *dev = intel_encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_dig_port->port; - int type, ret; + int type; if (WARN(intel_dig_port->max_lanes < 1, "Not enough lanes (%d) for DP on port %c\n", @@ -5832,17 +5576,17 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, connector->interlace_allowed = true; connector->doublescan_allowed = 0; + intel_dp_aux_init(intel_dp, intel_connector); + INIT_DELAYED_WORK(&intel_dp->panel_vdd_work, edp_panel_vdd_work); intel_connector_attach_encoder(intel_connector, intel_encoder); - drm_connector_register(connector); if (HAS_DDI(dev)) intel_connector->get_hw_state = intel_ddi_connector_get_hw_state; else intel_connector->get_hw_state = intel_connector_get_hw_state; - intel_connector->unregister = intel_dp_connector_unregister; /* Set up the hotplug pin. */ switch (port) { @@ -5867,22 +5611,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, BUG(); } - if (is_edp(intel_dp)) { - pps_lock(intel_dp); - intel_dp_init_panel_power_timestamps(intel_dp); - if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) - vlv_initial_power_sequencer_setup(intel_dp); - else - intel_dp_init_panel_power_sequencer(dev, intel_dp); - pps_unlock(intel_dp); - } - - ret = intel_dp_aux_init(intel_dp, intel_connector); - if (ret) - goto fail; - /* init MST on ports that can support it */ - if (HAS_DP_MST(dev) && + if (HAS_DP_MST(dev) && !is_edp(intel_dp) && (port == PORT_B || port == PORT_C || port == PORT_D)) intel_dp_mst_encoder_init(intel_dig_port, intel_connector->base.base.id); @@ -5904,22 +5634,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd); } - i915_debugfs_connector_add(connector); - return true; fail: - if (is_edp(intel_dp)) { - cancel_delayed_work_sync(&intel_dp->panel_vdd_work); - /* - * vdd might still be enabled do to the delayed vdd off. - * Make sure vdd is actually turned off here. - */ - pps_lock(intel_dp); - edp_panel_vdd_off_sync(intel_dp); - pps_unlock(intel_dp); - } - drm_connector_unregister(connector); drm_connector_cleanup(connector); return false; @@ -5929,7 +5646,7 @@ bool intel_dp_init(struct drm_device *dev, i915_reg_t output_reg, enum port port) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_digital_port *intel_dig_port; struct intel_encoder *intel_encoder; struct drm_encoder *encoder; @@ -5947,7 +5664,7 @@ bool intel_dp_init(struct drm_device *dev, encoder = &intel_encoder->base; if (drm_encoder_init(dev, &intel_encoder->base, &intel_dp_enc_funcs, - DRM_MODE_ENCODER_TMDS, NULL)) + DRM_MODE_ENCODER_TMDS, "DP %c", port_name(port))) goto err_encoder_init; intel_encoder->compute_config = intel_dp_compute_config; @@ -5977,7 +5694,7 @@ bool intel_dp_init(struct drm_device *dev, intel_dig_port->dp.output_reg = output_reg; intel_dig_port->max_lanes = 4; - intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; + intel_encoder->type = INTEL_OUTPUT_DP; if (IS_CHERRYVIEW(dev)) { if (port == PORT_D) intel_encoder->crtc_mask = 1 << 2; @@ -6007,43 +5724,35 @@ err_connector_alloc: void intel_dp_mst_suspend(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i; /* disable MST */ for (i = 0; i < I915_MAX_PORTS; i++) { struct intel_digital_port *intel_dig_port = dev_priv->hotplug.irq_port[i]; - if (!intel_dig_port) + + if (!intel_dig_port || !intel_dig_port->dp.can_mst) continue; - if (intel_dig_port->base.type == INTEL_OUTPUT_DISPLAYPORT) { - if (!intel_dig_port->dp.can_mst) - continue; - if (intel_dig_port->dp.is_mst) - drm_dp_mst_topology_mgr_suspend(&intel_dig_port->dp.mst_mgr); - } + if (intel_dig_port->dp.is_mst) + drm_dp_mst_topology_mgr_suspend(&intel_dig_port->dp.mst_mgr); } } void intel_dp_mst_resume(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int i; for (i = 0; i < I915_MAX_PORTS; i++) { struct intel_digital_port *intel_dig_port = dev_priv->hotplug.irq_port[i]; - if (!intel_dig_port) - continue; - if (intel_dig_port->base.type == INTEL_OUTPUT_DISPLAYPORT) { - int ret; + int ret; - if (!intel_dig_port->dp.can_mst) - continue; + if (!intel_dig_port || !intel_dig_port->dp.can_mst) + continue; - ret = drm_dp_mst_topology_mgr_resume(&intel_dig_port->dp.mst_mgr); - if (ret != 0) { - intel_dp_check_mst_status(&intel_dig_port->dp); - } - } + ret = drm_dp_mst_topology_mgr_resume(&intel_dig_port->dp.mst_mgr); + if (ret) + intel_dp_check_mst_status(&intel_dig_port->dp); } } diff --git a/drivers/gpu/drm/i915/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/intel_dp_aux_backlight.c new file mode 100644 index 000000000..6532e226d --- /dev/null +++ b/drivers/gpu/drm/i915/intel_dp_aux_backlight.c @@ -0,0 +1,172 @@ +/* + * Copyright © 2015 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include "intel_drv.h" + +static void set_aux_backlight_enable(struct intel_dp *intel_dp, bool enable) +{ + uint8_t reg_val = 0; + + if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_DISPLAY_CONTROL_REGISTER, + ®_val) < 0) { + DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n", + DP_EDP_DISPLAY_CONTROL_REGISTER); + return; + } + if (enable) + reg_val |= DP_EDP_BACKLIGHT_ENABLE; + else + reg_val &= ~(DP_EDP_BACKLIGHT_ENABLE); + + if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_DISPLAY_CONTROL_REGISTER, + reg_val) != 1) { + DRM_DEBUG_KMS("Failed to %s aux backlight\n", + enable ? "enable" : "disable"); + } +} + +/* + * Read the current backlight value from DPCD register(s) based + * on if 8-bit(MSB) or 16-bit(MSB and LSB) values are supported + */ +static uint32_t intel_dp_aux_get_backlight(struct intel_connector *connector) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); + uint8_t read_val[2] = { 0x0 }; + uint16_t level = 0; + + if (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_BACKLIGHT_BRIGHTNESS_MSB, + &read_val, sizeof(read_val)) < 0) { + DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n", + DP_EDP_BACKLIGHT_BRIGHTNESS_MSB); + return 0; + } + level = read_val[0]; + if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_BYTE_COUNT) + level = (read_val[0] << 8 | read_val[1]); + + return level; +} + +/* + * Sends the current backlight level over the aux channel, checking if its using + * 8-bit or 16 bit value (MSB and LSB) + */ +static void +intel_dp_aux_set_backlight(struct intel_connector *connector, u32 level) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); + uint8_t vals[2] = { 0x0 }; + + vals[0] = level; + + /* Write the MSB and/or LSB */ + if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_BYTE_COUNT) { + vals[0] = (level & 0xFF00) >> 8; + vals[1] = (level & 0xFF); + } + if (drm_dp_dpcd_write(&intel_dp->aux, DP_EDP_BACKLIGHT_BRIGHTNESS_MSB, + vals, sizeof(vals)) < 0) { + DRM_DEBUG_KMS("Failed to write aux backlight level\n"); + return; + } +} + +static void intel_dp_aux_enable_backlight(struct intel_connector *connector) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); + uint8_t dpcd_buf = 0; + + set_aux_backlight_enable(intel_dp, true); + + if ((drm_dp_dpcd_readb(&intel_dp->aux, + DP_EDP_BACKLIGHT_MODE_SET_REGISTER, &dpcd_buf) == 1) && + ((dpcd_buf & DP_EDP_BACKLIGHT_CONTROL_MODE_MASK) == + DP_EDP_BACKLIGHT_CONTROL_MODE_PRESET)) + drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_BACKLIGHT_MODE_SET_REGISTER, + (dpcd_buf | DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD)); +} + +static void intel_dp_aux_disable_backlight(struct intel_connector *connector) +{ + set_aux_backlight_enable(enc_to_intel_dp(&connector->encoder->base), false); +} + +static int intel_dp_aux_setup_backlight(struct intel_connector *connector, + enum pipe pipe) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); + struct intel_panel *panel = &connector->panel; + + intel_dp_aux_enable_backlight(connector); + + if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_BYTE_COUNT) + panel->backlight.max = 0xFFFF; + else + panel->backlight.max = 0xFF; + + panel->backlight.min = 0; + panel->backlight.level = intel_dp_aux_get_backlight(connector); + + panel->backlight.enabled = panel->backlight.level != 0; + + return 0; +} + +static bool +intel_dp_aux_display_control_capable(struct intel_connector *connector) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base); + + /* Check the eDP Display control capabilities registers to determine if + * the panel can support backlight control over the aux channel + */ + if (intel_dp->edp_dpcd[1] & DP_EDP_TCON_BACKLIGHT_ADJUSTMENT_CAP && + (intel_dp->edp_dpcd[1] & DP_EDP_BACKLIGHT_AUX_ENABLE_CAP) && + !((intel_dp->edp_dpcd[1] & DP_EDP_BACKLIGHT_PIN_ENABLE_CAP) || + (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_PWM_PIN_CAP))) { + DRM_DEBUG_KMS("AUX Backlight Control Supported!\n"); + return true; + } + return false; +} + +int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector) +{ + struct intel_panel *panel = &intel_connector->panel; + + if (!i915.enable_dpcd_backlight) + return -ENODEV; + + if (!intel_dp_aux_display_control_capable(intel_connector)) + return -ENODEV; + + panel->backlight.setup = intel_dp_aux_setup_backlight; + panel->backlight.enable = intel_dp_aux_enable_backlight; + panel->backlight.disable = intel_dp_aux_disable_backlight; + panel->backlight.set = intel_dp_aux_set_backlight; + panel->backlight.get = intel_dp_aux_get_backlight; + + return 0; +} diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c index 7a34090ce..68a005d72 100644 --- a/drivers/gpu/drm/i915/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/intel_dp_mst.c @@ -47,7 +47,6 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder, pipe_config->dp_encoder_is_mst = true; pipe_config->has_pch_encoder = false; - pipe_config->has_dp_encoder = true; bpp = 24; /* * for MST we always configure max link bw - the spec doesn't @@ -140,7 +139,7 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder) struct intel_digital_port *intel_dig_port = intel_mst->primary; struct intel_dp *intel_dp = &intel_dig_port->dp; struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_dig_port->port; int ret; uint32_t temp; @@ -207,14 +206,17 @@ static void intel_mst_enable_dp(struct intel_encoder *encoder) struct intel_digital_port *intel_dig_port = intel_mst->primary; struct intel_dp *intel_dp = &intel_dig_port->dp; struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_dig_port->port; int ret; DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links); - if (wait_for((I915_READ(DP_TP_STATUS(port)) & DP_TP_STATUS_ACT_SENT), - 1)) + if (intel_wait_for_register(dev_priv, + DP_TP_STATUS(port), + DP_TP_STATUS_ACT_SENT, + DP_TP_STATUS_ACT_SENT, + 1)) DRM_ERROR("Timed out waiting for ACT sent\n"); ret = drm_dp_check_act_status(&intel_dp->mst_mgr); @@ -239,12 +241,10 @@ static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder, struct intel_digital_port *intel_dig_port = intel_mst->primary; struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; u32 temp, flags = 0; - pipe_config->has_dp_encoder = true; - temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder)); if (temp & TRANS_DDI_PHSYNC) flags |= DRM_MODE_FLAG_PHSYNC; @@ -336,6 +336,8 @@ static const struct drm_connector_funcs intel_dp_mst_connector_funcs = { .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_dp_mst_set_property, .atomic_get_property = intel_connector_atomic_get_property, + .late_register = intel_connector_register, + .early_unregister = intel_connector_unregister, .destroy = intel_dp_mst_connector_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, @@ -455,7 +457,6 @@ static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topolo drm_connector_init(dev, connector, &intel_dp_mst_connector_funcs, DRM_MODE_CONNECTOR_DisplayPort); drm_connector_helper_add(connector, &intel_dp_mst_connector_helper_funcs); - intel_connector->unregister = intel_connector_unregister; intel_connector->get_hw_state = intel_dp_mst_get_hw_state; intel_connector->mst_port = intel_dp; intel_connector->port = port; @@ -477,9 +478,11 @@ static void intel_dp_register_mst_connector(struct drm_connector *connector) { struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_device *dev = connector->dev; + drm_modeset_lock_all(dev); intel_connector_add_to_fbdev(intel_connector); drm_modeset_unlock_all(dev); + drm_connector_register(&intel_connector->base); } @@ -489,7 +492,7 @@ static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr, struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_device *dev = connector->dev; - intel_connector->unregister(intel_connector); + drm_connector_unregister(connector); /* need to nuke the connector */ drm_modeset_lock_all(dev); @@ -534,7 +537,7 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum intel_mst->primary = intel_dig_port; drm_encoder_init(dev, &intel_encoder->base, &intel_dp_mst_enc_funcs, - DRM_MODE_ENCODER_DPMST, NULL); + DRM_MODE_ENCODER_DPMST, "DP-MST %c", pipe_name(pipe)); intel_encoder->type = INTEL_OUTPUT_DP_MST; intel_encoder->crtc_mask = 0x7; diff --git a/drivers/gpu/drm/i915/intel_dpio_phy.c b/drivers/gpu/drm/i915/intel_dpio_phy.c new file mode 100644 index 000000000..047f48748 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_dpio_phy.c @@ -0,0 +1,470 @@ +/* + * Copyright © 2014-2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "intel_drv.h" + +void chv_set_phy_signal_level(struct intel_encoder *encoder, + u32 deemph_reg_value, u32 margin_reg_value, + bool uniq_trans_scale) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct intel_crtc *intel_crtc = to_intel_crtc(dport->base.base.crtc); + enum dpio_channel ch = vlv_dport_to_channel(dport); + enum pipe pipe = intel_crtc->pipe; + u32 val; + int i; + + mutex_lock(&dev_priv->sb_lock); + + /* Clear calc init */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); + val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); + val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); + val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); + + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); + val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); + val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); + val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); + } + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch)); + val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); + val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val); + + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch)); + val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); + val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val); + } + + /* Program swing deemph */ + for (i = 0; i < intel_crtc->config->lane_count; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i)); + val &= ~DPIO_SWING_DEEMPH9P5_MASK; + val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val); + } + + /* Program swing margin */ + for (i = 0; i < intel_crtc->config->lane_count; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); + + val &= ~DPIO_SWING_MARGIN000_MASK; + val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT; + + /* + * Supposedly this value shouldn't matter when unique transition + * scale is disabled, but in fact it does matter. Let's just + * always program the same value and hope it's OK. + */ + val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT); + val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT; + + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); + } + + /* + * The document said it needs to set bit 27 for ch0 and bit 26 + * for ch1. Might be a typo in the doc. + * For now, for this unique transition scale selection, set bit + * 27 for ch0 and ch1. + */ + for (i = 0; i < intel_crtc->config->lane_count; i++) { + val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); + if (uniq_trans_scale) + val |= DPIO_TX_UNIQ_TRANS_SCALE_EN; + else + val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); + } + + /* Start swing calculation */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); + val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); + + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); + val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); + } + + mutex_unlock(&dev_priv->sb_lock); + +} + +void chv_data_lane_soft_reset(struct intel_encoder *encoder, + bool reset) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base)); + struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); + enum pipe pipe = crtc->pipe; + uint32_t val; + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); + if (reset) + val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + else + val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); + + if (crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); + if (reset) + val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); + else + val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); + } + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + if (reset) + val &= ~DPIO_PCS_CLK_SOFT_RESET; + else + val |= DPIO_PCS_CLK_SOFT_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); + + if (crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); + val |= CHV_PCS_REQ_SOFTRESET_EN; + if (reset) + val &= ~DPIO_PCS_CLK_SOFT_RESET; + else + val |= DPIO_PCS_CLK_SOFT_RESET; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); + } +} + +void chv_phy_pre_pll_enable(struct intel_encoder *encoder) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *intel_crtc = + to_intel_crtc(encoder->base.crtc); + enum dpio_channel ch = vlv_dport_to_channel(dport); + enum pipe pipe = intel_crtc->pipe; + unsigned int lane_mask = + intel_dp_unused_lane_mask(intel_crtc->config->lane_count); + u32 val; + + /* + * Must trick the second common lane into life. + * Otherwise we can't even access the PLL. + */ + if (ch == DPIO_CH0 && pipe == PIPE_B) + dport->release_cl2_override = + !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true); + + chv_phy_powergate_lanes(encoder, true, lane_mask); + + mutex_lock(&dev_priv->sb_lock); + + /* Assert data lane reset */ + chv_data_lane_soft_reset(encoder, true); + + /* program left/right clock distribution */ + if (pipe != PIPE_B) { + val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); + val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); + if (ch == DPIO_CH0) + val |= CHV_BUFLEFTENA1_FORCE; + if (ch == DPIO_CH1) + val |= CHV_BUFRIGHTENA1_FORCE; + vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); + } else { + val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); + val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); + if (ch == DPIO_CH0) + val |= CHV_BUFLEFTENA2_FORCE; + if (ch == DPIO_CH1) + val |= CHV_BUFRIGHTENA2_FORCE; + vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); + } + + /* program clock channel usage */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(ch)); + val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; + if (pipe != PIPE_B) + val &= ~CHV_PCS_USEDCLKCHANNEL; + else + val |= CHV_PCS_USEDCLKCHANNEL; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val); + + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch)); + val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; + if (pipe != PIPE_B) + val &= ~CHV_PCS_USEDCLKCHANNEL; + else + val |= CHV_PCS_USEDCLKCHANNEL; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val); + } + + /* + * This a a bit weird since generally CL + * matches the pipe, but here we need to + * pick the CL based on the port. + */ + val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW19(ch)); + if (pipe != PIPE_B) + val &= ~CHV_CMN_USEDCLKCHANNEL; + else + val |= CHV_CMN_USEDCLKCHANNEL; + vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW19(ch), val); + + mutex_unlock(&dev_priv->sb_lock); +} + +void chv_phy_pre_encoder_enable(struct intel_encoder *encoder) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + struct intel_digital_port *dport = dp_to_dig_port(intel_dp); + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *intel_crtc = + to_intel_crtc(encoder->base.crtc); + enum dpio_channel ch = vlv_dport_to_channel(dport); + int pipe = intel_crtc->pipe; + int data, i, stagger; + u32 val; + + mutex_lock(&dev_priv->sb_lock); + + /* allow hardware to manage TX FIFO reset source */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch)); + val &= ~DPIO_LANEDESKEW_STRAP_OVRD; + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); + + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); + val &= ~DPIO_LANEDESKEW_STRAP_OVRD; + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); + } + + /* Program Tx lane latency optimal setting*/ + for (i = 0; i < intel_crtc->config->lane_count; i++) { + /* Set the upar bit */ + if (intel_crtc->config->lane_count == 1) + data = 0x0; + else + data = (i == 1) ? 0x0 : 0x1; + vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i), + data << DPIO_UPAR_SHIFT); + } + + /* Data lane stagger programming */ + if (intel_crtc->config->port_clock > 270000) + stagger = 0x18; + else if (intel_crtc->config->port_clock > 135000) + stagger = 0xd; + else if (intel_crtc->config->port_clock > 67500) + stagger = 0x7; + else if (intel_crtc->config->port_clock > 33750) + stagger = 0x4; + else + stagger = 0x2; + + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch)); + val |= DPIO_TX2_STAGGER_MASK(0x1f); + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); + + if (intel_crtc->config->lane_count > 2) { + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); + val |= DPIO_TX2_STAGGER_MASK(0x1f); + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); + } + + vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch), + DPIO_LANESTAGGER_STRAP(stagger) | + DPIO_LANESTAGGER_STRAP_OVRD | + DPIO_TX1_STAGGER_MASK(0x1f) | + DPIO_TX1_STAGGER_MULT(6) | + DPIO_TX2_STAGGER_MULT(0)); + + if (intel_crtc->config->lane_count > 2) { + vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch), + DPIO_LANESTAGGER_STRAP(stagger) | + DPIO_LANESTAGGER_STRAP_OVRD | + DPIO_TX1_STAGGER_MASK(0x1f) | + DPIO_TX1_STAGGER_MULT(7) | + DPIO_TX2_STAGGER_MULT(5)); + } + + /* Deassert data lane reset */ + chv_data_lane_soft_reset(encoder, false); + + mutex_unlock(&dev_priv->sb_lock); +} + +void chv_phy_release_cl2_override(struct intel_encoder *encoder) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + + if (dport->release_cl2_override) { + chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false); + dport->release_cl2_override = false; + } +} + +void chv_phy_post_pll_disable(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe; + u32 val; + + mutex_lock(&dev_priv->sb_lock); + + /* disable left/right clock distribution */ + if (pipe != PIPE_B) { + val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); + val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); + vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); + } else { + val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); + val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); + vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); + } + + mutex_unlock(&dev_priv->sb_lock); + + /* + * Leave the power down bit cleared for at least one + * lane so that chv_powergate_phy_ch() will power + * on something when the channel is otherwise unused. + * When the port is off and the override is removed + * the lanes power down anyway, so otherwise it doesn't + * really matter what the state of power down bits is + * after this. + */ + chv_phy_powergate_lanes(encoder, false, 0x0); +} + +void vlv_set_phy_signal_level(struct intel_encoder *encoder, + u32 demph_reg_value, u32 preemph_reg_value, + u32 uniqtranscale_reg_value, u32 tx3_demph) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + enum dpio_channel port = vlv_dport_to_channel(dport); + int pipe = intel_crtc->pipe; + + mutex_lock(&dev_priv->sb_lock); + vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x00000000); + vlv_dpio_write(dev_priv, pipe, VLV_TX_DW4(port), demph_reg_value); + vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(port), + uniqtranscale_reg_value); + vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(port), 0x0C782040); + + if (tx3_demph) + vlv_dpio_write(dev_priv, pipe, VLV_TX3_DW4(port), tx3_demph); + + vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW11(port), 0x00030000); + vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), preemph_reg_value); + vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN); + mutex_unlock(&dev_priv->sb_lock); +} + +void vlv_phy_pre_pll_enable(struct intel_encoder *encoder) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *intel_crtc = + to_intel_crtc(encoder->base.crtc); + enum dpio_channel port = vlv_dport_to_channel(dport); + int pipe = intel_crtc->pipe; + + /* Program Tx lane resets to default */ + mutex_lock(&dev_priv->sb_lock); + vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), + DPIO_PCS_TX_LANE2_RESET | + DPIO_PCS_TX_LANE1_RESET); + vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), + DPIO_PCS_CLK_CRI_RXEB_EIOS_EN | + DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN | + (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) | + DPIO_PCS_CLK_SOFT_RESET); + + /* Fix up inter-pair skew failure */ + vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW12(port), 0x00750f00); + vlv_dpio_write(dev_priv, pipe, VLV_TX_DW11(port), 0x00001500); + vlv_dpio_write(dev_priv, pipe, VLV_TX_DW14(port), 0x40400000); + mutex_unlock(&dev_priv->sb_lock); +} + +void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder) +{ + struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); + struct intel_digital_port *dport = dp_to_dig_port(intel_dp); + struct drm_device *dev = encoder->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); + enum dpio_channel port = vlv_dport_to_channel(dport); + int pipe = intel_crtc->pipe; + u32 val; + + mutex_lock(&dev_priv->sb_lock); + + /* Enable clock channels for this port */ + val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(port)); + val = 0; + if (pipe) + val |= (1<<21); + else + val &= ~(1<<21); + val |= 0x001000c4; + vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW8(port), val); + + /* Program lane clock */ + vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW14(port), 0x00760018); + vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW23(port), 0x00400888); + + mutex_unlock(&dev_priv->sb_lock); +} + +void vlv_phy_reset_lanes(struct intel_encoder *encoder) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_crtc *intel_crtc = + to_intel_crtc(encoder->base.crtc); + enum dpio_channel port = vlv_dport_to_channel(dport); + int pipe = intel_crtc->pipe; + + mutex_lock(&dev_priv->sb_lock); + vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), 0x00000000); + vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), 0x00e00060); + mutex_unlock(&dev_priv->sb_lock); +} diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c index 58f60b278..5c1f2d235 100644 --- a/drivers/gpu/drm/i915/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c @@ -83,7 +83,7 @@ void assert_shared_dpll(struct drm_i915_private *dev_priv, void intel_prepare_shared_dpll(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_shared_dpll *pll = crtc->config->shared_dpll; if (WARN_ON(pll == NULL)) @@ -112,7 +112,7 @@ void intel_prepare_shared_dpll(struct intel_crtc *crtc) void intel_enable_shared_dpll(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_shared_dpll *pll = crtc->config->shared_dpll; unsigned crtc_mask = 1 << drm_crtc_index(&crtc->base); unsigned old_mask; @@ -151,7 +151,7 @@ out: void intel_disable_shared_dpll(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_shared_dpll *pll = crtc->config->shared_dpll; unsigned crtc_mask = 1 << drm_crtc_index(&crtc->base); @@ -191,7 +191,7 @@ intel_find_shared_dpll(struct intel_crtc *crtc, enum intel_dpll_id range_min, enum intel_dpll_id range_max) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_shared_dpll *pll; struct intel_shared_dpll_config *shared_dpll; enum intel_dpll_id i; @@ -208,8 +208,8 @@ intel_find_shared_dpll(struct intel_crtc *crtc, if (memcmp(&crtc_state->dpll_hw_state, &shared_dpll[i].hw_state, sizeof(crtc_state->dpll_hw_state)) == 0) { - DRM_DEBUG_KMS("CRTC:%d sharing existing %s (crtc mask 0x%08x, active %x)\n", - crtc->base.base.id, pll->name, + DRM_DEBUG_KMS("[CRTC:%d:%s] sharing existing %s (crtc mask 0x%08x, active %x)\n", + crtc->base.base.id, crtc->base.name, pll->name, shared_dpll[i].crtc_mask, pll->active_mask); return pll; @@ -220,8 +220,8 @@ intel_find_shared_dpll(struct intel_crtc *crtc, for (i = range_min; i <= range_max; i++) { pll = &dev_priv->shared_dplls[i]; if (shared_dpll[i].crtc_mask == 0) { - DRM_DEBUG_KMS("CRTC:%d allocated %s\n", - crtc->base.base.id, pll->name); + DRM_DEBUG_KMS("[CRTC:%d:%s] allocated %s\n", + crtc->base.base.id, crtc->base.name, pll->name); return pll; } } @@ -331,7 +331,7 @@ static void ibx_pch_dpll_enable(struct drm_i915_private *dev_priv, static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct intel_crtc *crtc; /* Make sure no transcoder isn't still depending on us. */ @@ -358,8 +358,8 @@ ibx_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, i = (enum intel_dpll_id) crtc->pipe; pll = &dev_priv->shared_dplls[i]; - DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n", - crtc->base.base.id, pll->name); + DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n", + crtc->base.base.id, crtc->base.name, pll->name); } else { pll = intel_find_shared_dpll(crtc, crtc_state, DPLL_ID_PCH_PLL_A, @@ -713,7 +713,7 @@ hsw_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, pll = intel_find_shared_dpll(crtc, crtc_state, DPLL_ID_WRPLL1, DPLL_ID_WRPLL2); - } else if (encoder->type == INTEL_OUTPUT_DISPLAYPORT || + } else if (encoder->type == INTEL_OUTPUT_DP || encoder->type == INTEL_OUTPUT_DP_MST || encoder->type == INTEL_OUTPUT_EDP) { enum intel_dpll_id pll_id; @@ -856,7 +856,11 @@ static void skl_ddi_pll_enable(struct drm_i915_private *dev_priv, I915_WRITE(regs[pll->id].ctl, I915_READ(regs[pll->id].ctl) | LCPLL_PLL_ENABLE); - if (wait_for(I915_READ(DPLL_STATUS) & DPLL_LOCK(pll->id), 5)) + if (intel_wait_for_register(dev_priv, + DPLL_STATUS, + DPLL_LOCK(pll->id), + DPLL_LOCK(pll->id), + 5)) DRM_ERROR("DPLL %d not locked\n", pll->id); } @@ -1222,7 +1226,7 @@ skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, DPLL_CFGCR2_KDIV(wrpll_params.kdiv) | DPLL_CFGCR2_PDIV(wrpll_params.pdiv) | wrpll_params.central_freq; - } else if (encoder->type == INTEL_OUTPUT_DISPLAYPORT || + } else if (encoder->type == INTEL_OUTPUT_DP || encoder->type == INTEL_OUTPUT_DP_MST || encoder->type == INTEL_OUTPUT_EDP) { switch (crtc_state->port_clock / 2) { @@ -1239,9 +1243,6 @@ skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, case 162000: ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, 0); break; - /* TBD: For DP link rates 2.16 GHz and 4.32 GHz, VCO is 8640 which - results in CDCLK change. Need to handle the change of CDCLK by - disabling pipes and re-enabling them */ case 108000: ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, 0); break; @@ -1511,7 +1512,7 @@ bxt_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, int clock = crtc_state->port_clock; if (encoder->type == INTEL_OUTPUT_HDMI) { - intel_clock_t best_clock; + struct dpll best_clock; /* Calculate HDMI div */ /* @@ -1533,7 +1534,7 @@ bxt_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, clk_div.m2_frac_en = clk_div.m2_frac != 0; vco = best_clock.vco; - } else if (encoder->type == INTEL_OUTPUT_DISPLAYPORT || + } else if (encoder->type == INTEL_OUTPUT_DP || encoder->type == INTEL_OUTPUT_EDP) { int i; @@ -1616,8 +1617,8 @@ bxt_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, i = (enum intel_dpll_id) intel_dig_port->port; pll = intel_get_shared_dpll_by_id(dev_priv, i); - DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n", - crtc->base.base.id, pll->name); + DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n", + crtc->base.base.id, crtc->base.name, pll->name); intel_reference_shared_dpll(pll, crtc_state); @@ -1635,19 +1636,11 @@ static const struct intel_shared_dpll_funcs bxt_ddi_pll_funcs = { static void intel_ddi_pll_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t val = I915_READ(LCPLL_CTL); - - if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { - int cdclk_freq; - - cdclk_freq = dev_priv->display.get_display_clock_speed(dev); - dev_priv->skl_boot_cdclk = cdclk_freq; - if (skl_sanitize_cdclk(dev_priv)) - DRM_DEBUG_KMS("Sanitized cdclk programmed by pre-os\n"); - if (!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE)) - DRM_ERROR("LCPLL1 is disabled\n"); - } else if (!IS_BROXTON(dev_priv)) { + struct drm_i915_private *dev_priv = to_i915(dev); + + if (INTEL_GEN(dev_priv) < 9) { + uint32_t val = I915_READ(LCPLL_CTL); + /* * The LCPLL register should be turned on by the BIOS. For now * let's just check its state and print errors in case @@ -1730,7 +1723,7 @@ static const struct intel_dpll_mgr bxt_pll_mgr = { void intel_shared_dpll_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); const struct intel_dpll_mgr *dpll_mgr = NULL; const struct dpll_info *dpll_info; int i; diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index 94144a70b..ff399b9a5 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -69,39 +69,63 @@ }) #define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 1000) -#define wait_for_us(COND, US) _wait_for((COND), (US), 1) /* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */ #if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT) -# define _WAIT_FOR_ATOMIC_CHECK WARN_ON_ONCE(!in_atomic()) +# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) WARN_ON_ONCE((ATOMIC) && !in_atomic()) #else -# define _WAIT_FOR_ATOMIC_CHECK do { } while (0) +# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) do { } while (0) #endif -#define _wait_for_atomic(COND, US) ({ \ - unsigned long end__; \ - int ret__ = 0; \ - _WAIT_FOR_ATOMIC_CHECK; \ +#define _wait_for_atomic(COND, US, ATOMIC) \ +({ \ + int cpu, ret, timeout = (US) * 1000; \ + u64 base; \ + _WAIT_FOR_ATOMIC_CHECK(ATOMIC); \ BUILD_BUG_ON((US) > 50000); \ - end__ = (local_clock() >> 10) + (US) + 1; \ - while (!(COND)) { \ - if (time_after((unsigned long)(local_clock() >> 10), end__)) { \ - /* Unlike the regular wait_for(), this atomic variant \ - * cannot be preempted (and we'll just ignore the issue\ - * of irq interruptions) and so we know that no time \ - * has passed since the last check of COND and can \ - * immediately report the timeout. \ - */ \ - ret__ = -ETIMEDOUT; \ + if (!(ATOMIC)) { \ + preempt_disable(); \ + cpu = smp_processor_id(); \ + } \ + base = local_clock(); \ + for (;;) { \ + u64 now = local_clock(); \ + if (!(ATOMIC)) \ + preempt_enable(); \ + if (COND) { \ + ret = 0; \ + break; \ + } \ + if (now - base >= timeout) { \ + ret = -ETIMEDOUT; \ break; \ } \ cpu_relax(); \ + if (!(ATOMIC)) { \ + preempt_disable(); \ + if (unlikely(cpu != smp_processor_id())) { \ + timeout -= now - base; \ + cpu = smp_processor_id(); \ + base = local_clock(); \ + } \ + } \ } \ + ret; \ +}) + +#define wait_for_us(COND, US) \ +({ \ + int ret__; \ + BUILD_BUG_ON(!__builtin_constant_p(US)); \ + if ((US) > 10) \ + ret__ = _wait_for((COND), (US), 10); \ + else \ + ret__ = _wait_for_atomic((COND), (US), 0); \ ret__; \ }) -#define wait_for_atomic(COND, MS) _wait_for_atomic((COND), (MS) * 1000) -#define wait_for_atomic_us(COND, US) _wait_for_atomic((COND), (US)) +#define wait_for_atomic(COND, MS) _wait_for_atomic((COND), (MS) * 1000, 1) +#define wait_for_atomic_us(COND, US) _wait_for_atomic((COND), (US), 1) #define KHz(x) (1000 * (x)) #define MHz(x) KHz(1000 * (x)) @@ -135,7 +159,7 @@ enum intel_output_type { INTEL_OUTPUT_LVDS = 4, INTEL_OUTPUT_TVOUT = 5, INTEL_OUTPUT_HDMI = 6, - INTEL_OUTPUT_DISPLAYPORT = 7, + INTEL_OUTPUT_DP = 7, INTEL_OUTPUT_EDP = 8, INTEL_OUTPUT_DSI = 9, INTEL_OUTPUT_UNKNOWN = 10, @@ -159,6 +183,7 @@ struct intel_framebuffer { struct intel_fbdev { struct drm_fb_helper helper; struct intel_framebuffer *fb; + async_cookie_t cookie; int preferred_bpp; }; @@ -242,14 +267,6 @@ struct intel_connector { * and active (i.e. dpms ON state). */ bool (*get_hw_state)(struct intel_connector *); - /* - * Removes all interfaces through which the connector is accessible - * - like sysfs, debugfs entries -, so that no new operations can be - * started on the connector. Also makes sure all currently pending - * operations finish before returing. - */ - void (*unregister)(struct intel_connector *); - /* Panel info for eDP and LVDS */ struct intel_panel panel; @@ -266,7 +283,7 @@ struct intel_connector { struct intel_dp *mst_port; }; -typedef struct dpll { +struct dpll { /* given values */ int n; int m1, m2; @@ -276,7 +293,7 @@ typedef struct dpll { int vco; int m; int p; -} intel_clock_t; +}; struct intel_atomic_state { struct drm_atomic_state base; @@ -291,17 +308,32 @@ struct intel_atomic_state { bool dpll_set, modeset; + /* + * Does this transaction change the pipes that are active? This mask + * tracks which CRTC's have changed their active state at the end of + * the transaction (not counting the temporary disable during modesets). + * This mask should only be non-zero when intel_state->modeset is true, + * but the converse is not necessarily true; simply changing a mode may + * not flip the final active status of any CRTC's + */ + unsigned int active_pipe_changes; + unsigned int active_crtcs; unsigned int min_pixclk[I915_MAX_PIPES]; + /* SKL/KBL Only */ + unsigned int cdclk_pll_vco; + struct intel_shared_dpll_config shared_dpll[I915_NUM_PLLS]; - struct intel_wm_config wm_config; /* * Current watermarks can't be trusted during hardware readout, so * don't bother calculating intermediate watermarks. */ bool skip_intermediate_wm; + + /* Gen9+ only */ + struct skl_wm_values wm_results; }; struct intel_plane_state { @@ -405,6 +437,48 @@ struct skl_pipe_wm { uint32_t linetime; }; +struct intel_crtc_wm_state { + union { + struct { + /* + * Intermediate watermarks; these can be + * programmed immediately since they satisfy + * both the current configuration we're + * switching away from and the new + * configuration we're switching to. + */ + struct intel_pipe_wm intermediate; + + /* + * Optimal watermarks, programmed post-vblank + * when this state is committed. + */ + struct intel_pipe_wm optimal; + } ilk; + + struct { + /* gen9+ only needs 1-step wm programming */ + struct skl_pipe_wm optimal; + + /* cached plane data rate */ + unsigned plane_data_rate[I915_MAX_PLANES]; + unsigned plane_y_data_rate[I915_MAX_PLANES]; + + /* minimum block allocation */ + uint16_t minimum_blocks[I915_MAX_PLANES]; + uint16_t minimum_y_blocks[I915_MAX_PLANES]; + } skl; + }; + + /* + * Platforms with two-step watermark programming will need to + * update watermark programming post-vblank to switch from the + * safe intermediate watermarks to the optimal final + * watermarks. + */ + bool need_postvbl_update; +}; + struct intel_crtc_state { struct drm_crtc_state base; @@ -448,12 +522,10 @@ struct intel_crtc_state { */ bool limited_color_range; - /* DP has a bunch of special case unfortunately, so mark the pipe - * accordingly. */ - bool has_dp_encoder; - - /* DSI has special cases */ - bool has_dsi_encoder; + /* Bitmask of encoder types (enum intel_output_type) + * driven by the pipe. + */ + unsigned int output_types; /* Whether we should send NULL infoframes. Required for audio. */ bool has_hdmi_sink; @@ -522,6 +594,12 @@ struct intel_crtc_state { uint8_t lane_count; + /* + * Used by platforms having DP/HDMI PHY with programmable lane + * latency optimization. + */ + uint8_t lane_lat_optim_mask; + /* Panel fitter controls for gen2-gen4 + VLV */ struct { u32 control; @@ -558,32 +636,7 @@ struct intel_crtc_state { /* IVB sprite scaling w/a (WaCxSRDisabledForSpriteScaling:ivb) */ bool disable_lp_wm; - struct { - /* - * Optimal watermarks, programmed post-vblank when this state - * is committed. - */ - union { - struct intel_pipe_wm ilk; - struct skl_pipe_wm skl; - } optimal; - - /* - * Intermediate watermarks; these can be programmed immediately - * since they satisfy both the current configuration we're - * switching away from and the new configuration we're switching - * to. - */ - struct intel_pipe_wm intermediate; - - /* - * Platforms with two-step watermark programming will need to - * update watermark programming post-vblank to switch from the - * safe intermediate watermarks to the optimal final - * watermarks. - */ - bool need_postvbl_update; - } wm; + struct intel_crtc_wm_state wm; /* Gamma mode programmed on the pipe */ uint32_t gamma_mode; @@ -598,14 +651,6 @@ struct vlv_wm_state { bool cxsr; }; -struct intel_mmio_flip { - struct work_struct work; - struct drm_i915_private *i915; - struct drm_i915_gem_request *req; - struct intel_crtc *crtc; - unsigned int rotation; -}; - struct intel_crtc { struct drm_crtc base; enum pipe pipe; @@ -620,7 +665,7 @@ struct intel_crtc { unsigned long enabled_power_domains; bool lowfreq_avail; struct intel_overlay *overlay; - struct intel_unpin_work *unpin_work; + struct intel_flip_work *flip_work; atomic_t unpin_work_count; @@ -815,6 +860,7 @@ struct intel_dp { uint8_t dpcd[DP_RECEIVER_CAP_SIZE]; uint8_t psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE]; uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS]; + uint8_t edp_dpcd[EDP_DISPLAY_CTL_CAP_SIZE]; /* sink rates as reported by DP_SUPPORTED_LINK_RATES */ uint8_t num_sink_rates; int sink_rates[DP_MAX_SUPPORTED_RATES]; @@ -838,6 +884,11 @@ struct intel_dp { * this port. Only relevant on VLV/CHV. */ enum pipe pps_pipe; + /* + * Set if the sequencer may be reset due to a power transition, + * requiring a reinitialization. Only relevant on BXT. + */ + bool pps_reset; struct edp_power_seq pps_delays; bool can_mst; /* this port supports mst */ @@ -934,33 +985,32 @@ vlv_pipe_to_channel(enum pipe pipe) static inline struct drm_crtc * intel_get_crtc_for_pipe(struct drm_device *dev, int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); return dev_priv->pipe_to_crtc_mapping[pipe]; } static inline struct drm_crtc * intel_get_crtc_for_plane(struct drm_device *dev, int plane) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); return dev_priv->plane_to_crtc_mapping[plane]; } -struct intel_unpin_work { - struct work_struct work; +struct intel_flip_work { + struct work_struct unpin_work; + struct work_struct mmio_work; + struct drm_crtc *crtc; struct drm_framebuffer *old_fb; struct drm_i915_gem_object *pending_flip_obj; struct drm_pending_vblank_event *event; atomic_t pending; -#define INTEL_FLIP_INACTIVE 0 -#define INTEL_FLIP_PENDING 1 -#define INTEL_FLIP_COMPLETE 2 u32 flip_count; u32 gtt_offset; struct drm_i915_gem_request *flip_queued_req; u32 flip_queued_vblank; u32 flip_ready_vblank; - bool enable_stall_check; + unsigned int rotation; }; struct intel_load_detect_pipe { @@ -1029,9 +1079,9 @@ void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask); void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask); -void gen6_reset_rps_interrupts(struct drm_device *dev); -void gen6_enable_rps_interrupts(struct drm_device *dev); -void gen6_disable_rps_interrupts(struct drm_device *dev); +void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv); +void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv); +void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv); u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask); void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv); void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv); @@ -1110,14 +1160,16 @@ void i915_audio_component_init(struct drm_i915_private *dev_priv); void i915_audio_component_cleanup(struct drm_i915_private *dev_priv); /* intel_display.c */ +void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco); +void intel_update_rawclk(struct drm_i915_private *dev_priv); int vlv_get_cck_clock(struct drm_i915_private *dev_priv, const char *name, u32 reg, int ref_freq); extern const struct drm_plane_funcs intel_plane_funcs; void intel_init_display_hooks(struct drm_i915_private *dev_priv); unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info); bool intel_has_pending_fb_unpin(struct drm_device *dev); -void intel_mark_busy(struct drm_device *dev); -void intel_mark_idle(struct drm_device *dev); +void intel_mark_busy(struct drm_i915_private *dev_priv); +void intel_mark_idle(struct drm_i915_private *dev_priv); void intel_crtc_restore_mode(struct drm_crtc *crtc); int intel_display_suspend(struct drm_device *dev); void intel_encoder_destroy(struct drm_encoder *encoder); @@ -1126,7 +1178,6 @@ struct intel_connector *intel_connector_alloc(void); bool intel_connector_get_hw_state(struct intel_connector *connector); void intel_connector_attach_encoder(struct intel_connector *connector, struct intel_encoder *encoder); -struct drm_encoder *intel_best_encoder(struct drm_connector *connector); struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev, struct drm_crtc *crtc); enum pipe intel_get_pipe_from_connector(struct intel_connector *connector); @@ -1134,7 +1185,20 @@ int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data, struct drm_file *file_priv); enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe); -bool intel_pipe_has_type(struct intel_crtc *crtc, enum intel_output_type type); +static inline bool +intel_crtc_has_type(const struct intel_crtc_state *crtc_state, + enum intel_output_type type) +{ + return crtc_state->output_types & (1 << type); +} +static inline bool +intel_crtc_has_dp_encoder(const struct intel_crtc_state *crtc_state) +{ + return crtc_state->output_types & + ((1 << INTEL_OUTPUT_DP) | + (1 << INTEL_OUTPUT_DP_MST) | + (1 << INTEL_OUTPUT_EDP)); +} static inline void intel_wait_for_vblank(struct drm_device *dev, int pipe) { @@ -1149,6 +1213,9 @@ intel_wait_for_vblank_if_active(struct drm_device *dev, int pipe) if (crtc->active) intel_wait_for_vblank(dev, pipe); } + +u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc); + int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp); void vlv_wait_port_ready(struct drm_i915_private *dev_priv, struct intel_digital_port *dport, @@ -1162,14 +1229,14 @@ void intel_release_load_detect_pipe(struct drm_connector *connector, struct drm_modeset_acquire_ctx *ctx); int intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation); +void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation); struct drm_framebuffer * __intel_framebuffer_create(struct drm_device *dev, struct drm_mode_fb_cmd2 *mode_cmd, struct drm_i915_gem_object *obj); -void intel_prepare_page_flip(struct drm_device *dev, int plane); -void intel_finish_page_flip(struct drm_device *dev, int pipe); -void intel_finish_page_flip_plane(struct drm_device *dev, int plane); -void intel_check_page_flip(struct drm_device *dev, int pipe); +void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe); +void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe); +void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe); int intel_prepare_plane_fb(struct drm_plane *plane, const struct drm_plane_state *new_state); void intel_cleanup_plane_fb(struct drm_plane *plane, @@ -1226,23 +1293,25 @@ u32 intel_compute_tile_offset(int *x, int *y, const struct drm_framebuffer *fb, int plane, unsigned int pitch, unsigned int rotation); -void intel_prepare_reset(struct drm_device *dev); -void intel_finish_reset(struct drm_device *dev); +void intel_prepare_reset(struct drm_i915_private *dev_priv); +void intel_finish_reset(struct drm_i915_private *dev_priv); void hsw_enable_pc8(struct drm_i915_private *dev_priv); void hsw_disable_pc8(struct drm_i915_private *dev_priv); -void broxton_init_cdclk(struct drm_i915_private *dev_priv); -void broxton_uninit_cdclk(struct drm_i915_private *dev_priv); -bool broxton_cdclk_verify_state(struct drm_i915_private *dev_priv); -void broxton_ddi_phy_init(struct drm_i915_private *dev_priv); -void broxton_ddi_phy_uninit(struct drm_i915_private *dev_priv); -void broxton_ddi_phy_verify_state(struct drm_i915_private *dev_priv); +void bxt_init_cdclk(struct drm_i915_private *dev_priv); +void bxt_uninit_cdclk(struct drm_i915_private *dev_priv); +void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy); +void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy); +bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, + enum dpio_phy phy); +bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, + enum dpio_phy phy); void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv); void bxt_enable_dc9(struct drm_i915_private *dev_priv); void bxt_disable_dc9(struct drm_i915_private *dev_priv); void gen9_enable_dc5(struct drm_i915_private *dev_priv); void skl_init_cdclk(struct drm_i915_private *dev_priv); -int skl_sanitize_cdclk(struct drm_i915_private *dev_priv); void skl_uninit_cdclk(struct drm_i915_private *dev_priv); +unsigned int skl_cdclk_get_vco(unsigned int freq); void skl_enable_dc6(struct drm_i915_private *dev_priv); void skl_disable_dc6(struct drm_i915_private *dev_priv); void intel_dp_get_m_n(struct intel_crtc *crtc, @@ -1250,8 +1319,8 @@ void intel_dp_get_m_n(struct intel_crtc *crtc, void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n); int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n); bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock, - intel_clock_t *best_clock); -int chv_calc_dpll_params(int refclk, intel_clock_t *pll_clock); + struct dpll *best_clock); +int chv_calc_dpll_params(int refclk, struct dpll *pll_clock); bool intel_crtc_active(struct drm_crtc *crtc); void hsw_enable_ips(struct intel_crtc *crtc); @@ -1310,7 +1379,7 @@ void intel_dp_mst_resume(struct drm_device *dev); int intel_dp_max_link_rate(struct intel_dp *intel_dp); int intel_dp_rate_select(struct intel_dp *intel_dp, int rate); void intel_dp_hot_plug(struct intel_encoder *intel_encoder); -void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv); +void intel_power_sequencer_reset(struct drm_i915_private *dev_priv); uint32_t intel_dp_pack_aux(const uint8_t *src, int src_bytes); void intel_plane_destroy(struct drm_plane *plane); void intel_edp_drrs_enable(struct intel_dp *intel_dp); @@ -1337,12 +1406,22 @@ bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp); bool intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE]); +static inline unsigned int intel_dp_unused_lane_mask(int lane_count) +{ + return ~((1 << lane_count) - 1) & 0xf; +} + +/* intel_dp_aux_backlight.c */ +int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector); + /* intel_dp_mst.c */ int intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_id); void intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port); /* intel_dsi.c */ void intel_dsi_init(struct drm_device *dev); +/* intel_dsi_dcs_backlight.c */ +int intel_dsi_dcs_init_backlight_funcs(struct intel_connector *intel_connector); /* intel_dvo.c */ void intel_dvo_init(struct drm_device *dev); @@ -1385,11 +1464,15 @@ static inline void intel_fbdev_restore_mode(struct drm_device *dev) void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv, struct drm_atomic_state *state); bool intel_fbc_is_active(struct drm_i915_private *dev_priv); -void intel_fbc_pre_update(struct intel_crtc *crtc); +void intel_fbc_pre_update(struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state); void intel_fbc_post_update(struct intel_crtc *crtc); void intel_fbc_init(struct drm_i915_private *dev_priv); void intel_fbc_init_pipe_state(struct drm_i915_private *dev_priv); -void intel_fbc_enable(struct intel_crtc *crtc); +void intel_fbc_enable(struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state); void intel_fbc_disable(struct intel_crtc *crtc); void intel_fbc_global_disable(struct drm_i915_private *dev_priv); void intel_fbc_invalidate(struct drm_i915_private *dev_priv, @@ -1411,6 +1494,7 @@ void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable); /* intel_lvds.c */ void intel_lvds_init(struct drm_device *dev); +struct intel_encoder *intel_get_lvds_encoder(struct drm_device *dev); bool intel_is_dual_link_lvds(struct drm_device *dev); @@ -1424,13 +1508,13 @@ void intel_attach_aspect_ratio_property(struct drm_connector *connector); /* intel_overlay.c */ -void intel_setup_overlay(struct drm_device *dev); -void intel_cleanup_overlay(struct drm_device *dev); +void intel_setup_overlay(struct drm_i915_private *dev_priv); +void intel_cleanup_overlay(struct drm_i915_private *dev_priv); int intel_overlay_switch_off(struct intel_overlay *overlay); -int intel_overlay_put_image(struct drm_device *dev, void *data, - struct drm_file *file_priv); -int intel_overlay_attrs(struct drm_device *dev, void *data, - struct drm_file *file_priv); +int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); +int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); void intel_overlay_reset(struct drm_i915_private *dev_priv); @@ -1449,7 +1533,8 @@ void intel_gmch_panel_fitting(struct intel_crtc *crtc, int fitting_mode); void intel_panel_set_backlight_acpi(struct intel_connector *connector, u32 level, u32 max); -int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe); +int intel_panel_setup_backlight(struct drm_connector *connector, + enum pipe pipe); void intel_panel_enable_backlight(struct intel_connector *connector); void intel_panel_disable_backlight(struct intel_connector *connector); void intel_panel_destroy_backlight(struct drm_connector *connector); @@ -1458,8 +1543,19 @@ extern struct drm_display_mode *intel_find_panel_downclock( struct drm_device *dev, struct drm_display_mode *fixed_mode, struct drm_connector *connector); -void intel_backlight_register(struct drm_device *dev); -void intel_backlight_unregister(struct drm_device *dev); + +#if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE) +int intel_backlight_device_register(struct intel_connector *connector); +void intel_backlight_device_unregister(struct intel_connector *connector); +#else /* CONFIG_BACKLIGHT_CLASS_DEVICE */ +static int intel_backlight_device_register(struct intel_connector *connector) +{ + return 0; +} +static inline void intel_backlight_device_unregister(struct intel_connector *connector) +{ +} +#endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */ /* intel_psr.c */ @@ -1601,29 +1697,35 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv); void intel_pm_setup(struct drm_device *dev); void intel_gpu_ips_init(struct drm_i915_private *dev_priv); void intel_gpu_ips_teardown(void); -void intel_init_gt_powersave(struct drm_device *dev); -void intel_cleanup_gt_powersave(struct drm_device *dev); -void intel_enable_gt_powersave(struct drm_device *dev); -void intel_disable_gt_powersave(struct drm_device *dev); -void intel_suspend_gt_powersave(struct drm_device *dev); -void intel_reset_gt_powersave(struct drm_device *dev); -void gen6_update_ring_freq(struct drm_device *dev); +void intel_init_gt_powersave(struct drm_i915_private *dev_priv); +void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv); +void intel_enable_gt_powersave(struct drm_i915_private *dev_priv); +void intel_disable_gt_powersave(struct drm_i915_private *dev_priv); +void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv); +void intel_reset_gt_powersave(struct drm_i915_private *dev_priv); +void gen6_update_ring_freq(struct drm_i915_private *dev_priv); void gen6_rps_busy(struct drm_i915_private *dev_priv); void gen6_rps_reset_ei(struct drm_i915_private *dev_priv); void gen6_rps_idle(struct drm_i915_private *dev_priv); void gen6_rps_boost(struct drm_i915_private *dev_priv, struct intel_rps_client *rps, unsigned long submitted); -void intel_queue_rps_boost_for_request(struct drm_device *dev, - struct drm_i915_gem_request *req); +void intel_queue_rps_boost_for_request(struct drm_i915_gem_request *req); void vlv_wm_get_hw_state(struct drm_device *dev); void ilk_wm_get_hw_state(struct drm_device *dev); void skl_wm_get_hw_state(struct drm_device *dev); void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, struct skl_ddb_allocation *ddb /* out */); +bool skl_can_enable_sagv(struct drm_atomic_state *state); +int skl_enable_sagv(struct drm_i915_private *dev_priv); +int skl_disable_sagv(struct drm_i915_private *dev_priv); uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config); bool ilk_disable_lp_wm(struct drm_device *dev); -int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6); +int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6); +static inline int intel_enable_rc6(void) +{ + return i915.enable_rc6; +} /* intel_sdvo.c */ bool intel_sdvo_init(struct drm_device *dev, @@ -1631,11 +1733,13 @@ bool intel_sdvo_init(struct drm_device *dev, /* intel_sprite.c */ +int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, + int usecs); int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane); int intel_sprite_set_colorkey(struct drm_device *dev, void *data, struct drm_file *file_priv); void intel_pipe_update_start(struct intel_crtc *crtc); -void intel_pipe_update_end(struct intel_crtc *crtc); +void intel_pipe_update_end(struct intel_crtc *crtc, struct intel_flip_work *work); /* intel_tv.c */ void intel_tv_init(struct drm_device *dev); diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c index 4756ef639..de8e9fb51 100644 --- a/drivers/gpu/drm/i915/intel_dsi.c +++ b/drivers/gpu/drm/i915/intel_dsi.c @@ -84,13 +84,15 @@ static void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi, enum port port) { struct drm_encoder *encoder = &intel_dsi->base.base; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 mask; mask = LP_CTRL_FIFO_EMPTY | HS_CTRL_FIFO_EMPTY | LP_DATA_FIFO_EMPTY | HS_DATA_FIFO_EMPTY; - if (wait_for((I915_READ(MIPI_GEN_FIFO_STAT(port)) & mask) == mask, 100)) + if (intel_wait_for_register(dev_priv, + MIPI_GEN_FIFO_STAT(port), mask, mask, + 100)) DRM_ERROR("DPI FIFOs are not empty\n"); } @@ -129,7 +131,7 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host, { struct intel_dsi_host *intel_dsi_host = to_intel_dsi_host(host); struct drm_device *dev = intel_dsi_host->intel_dsi->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_dsi_host->port; struct mipi_dsi_packet packet; ssize_t ret; @@ -158,8 +160,10 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host, /* note: this is never true for reads */ if (packet.payload_length) { - - if (wait_for((I915_READ(MIPI_GEN_FIFO_STAT(port)) & data_mask) == 0, 50)) + if (intel_wait_for_register(dev_priv, + MIPI_GEN_FIFO_STAT(port), + data_mask, 0, + 50)) DRM_ERROR("Timeout waiting for HS/LP DATA FIFO !full\n"); write_data(dev_priv, data_reg, packet.payload, @@ -170,7 +174,10 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host, I915_WRITE(MIPI_INTR_STAT(port), GEN_READ_DATA_AVAIL); } - if (wait_for((I915_READ(MIPI_GEN_FIFO_STAT(port)) & ctrl_mask) == 0, 50)) { + if (intel_wait_for_register(dev_priv, + MIPI_GEN_FIFO_STAT(port), + ctrl_mask, 0, + 50)) { DRM_ERROR("Timeout waiting for HS/LP CTRL FIFO !full\n"); } @@ -179,7 +186,10 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host, /* ->rx_len is set only for reads */ if (msg->rx_len) { data_mask = GEN_READ_DATA_AVAIL; - if (wait_for((I915_READ(MIPI_INTR_STAT(port)) & data_mask) == data_mask, 50)) + if (intel_wait_for_register(dev_priv, + MIPI_INTR_STAT(port), + data_mask, data_mask, + 50)) DRM_ERROR("Timeout waiting for read data.\n"); read_data(dev_priv, data_reg, msg->rx_buf, msg->rx_len); @@ -250,7 +260,7 @@ static int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd, bool hs, { struct drm_encoder *encoder = &intel_dsi->base.base; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 mask; /* XXX: pipe, hs */ @@ -269,7 +279,9 @@ static int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd, bool hs, I915_WRITE(MIPI_DPI_CONTROL(port), cmd); mask = SPL_PKT_SENT_INTERRUPT; - if (wait_for((I915_READ(MIPI_INTR_STAT(port)) & mask) == mask, 100)) + if (intel_wait_for_register(dev_priv, + MIPI_INTR_STAT(port), mask, mask, + 100)) DRM_ERROR("Video mode command 0x%08x send failed.\n", cmd); return 0; @@ -302,7 +314,7 @@ static inline bool is_cmd_mode(struct intel_dsi *intel_dsi) static bool intel_dsi_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi, base); struct intel_connector *intel_connector = intel_dsi->attached_connector; @@ -313,8 +325,6 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder, DRM_DEBUG_KMS("\n"); - pipe_config->has_dsi_encoder = true; - if (fixed_mode) { intel_fixed_panel_mode(fixed_mode, adjusted_mode); @@ -348,7 +358,7 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder, static void bxt_dsi_device_ready(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); enum port port; u32 val; @@ -387,7 +397,7 @@ static void bxt_dsi_device_ready(struct intel_encoder *encoder) static void vlv_dsi_device_ready(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); enum port port; u32 val; @@ -437,7 +447,7 @@ static void intel_dsi_device_ready(struct intel_encoder *encoder) static void intel_dsi_port_enable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); enum port port; @@ -478,7 +488,7 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder) static void intel_dsi_port_disable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); enum port port; @@ -497,7 +507,7 @@ static void intel_dsi_port_disable(struct intel_encoder *encoder) static void intel_dsi_enable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); enum port port; @@ -528,11 +538,10 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder); static void intel_dsi_pre_enable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); enum port port; - u32 tmp; DRM_DEBUG_KMS("\n"); @@ -551,11 +560,13 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder) msleep(intel_dsi->panel_on_delay); - if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) { + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + u32 val; + /* Disable DPOunit clock gating, can stall pipe */ - tmp = I915_READ(DSPCLK_GATE_D); - tmp |= DPOUNIT_CLOCK_GATE_DISABLE; - I915_WRITE(DSPCLK_GATE_D, tmp); + val = I915_READ(DSPCLK_GATE_D); + val |= DPOUNIT_CLOCK_GATE_DISABLE; + I915_WRITE(DSPCLK_GATE_D, val); } /* put device in ready state */ @@ -601,7 +612,7 @@ static void intel_dsi_pre_disable(struct intel_encoder *encoder) static void intel_dsi_disable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); enum port port; u32 temp; @@ -640,7 +651,7 @@ static void intel_dsi_disable(struct intel_encoder *encoder) static void intel_dsi_clear_device_ready(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); enum port port; @@ -666,8 +677,9 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder) /* Wait till Clock lanes are in LP-00 state for MIPI Port A * only. MIPI Port C has no similar bit for checking */ - if (wait_for(((I915_READ(port_ctrl) & AFE_LATCHOUT) - == 0x00000), 30)) + if (intel_wait_for_register(dev_priv, + port_ctrl, AFE_LATCHOUT, 0, + 30)) DRM_ERROR("DSI LP not going Low\n"); /* Disable MIPI PHY transparent latch */ @@ -684,7 +696,7 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder) static void intel_dsi_post_disable(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); DRM_DEBUG_KMS("\n"); @@ -693,7 +705,7 @@ static void intel_dsi_post_disable(struct intel_encoder *encoder) intel_dsi_clear_device_ready(encoder); - if (!IS_BROXTON(dev_priv)) { + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { u32 val; val = I915_READ(DSPCLK_GATE_D); @@ -719,7 +731,7 @@ static void intel_dsi_post_disable(struct intel_encoder *encoder) static bool intel_dsi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); struct drm_device *dev = encoder->base.dev; enum intel_display_power_domain power_domain; @@ -793,7 +805,7 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode; struct drm_display_mode *adjusted_mode_sw; @@ -953,8 +965,6 @@ static void intel_dsi_get_config(struct intel_encoder *encoder, u32 pclk; DRM_DEBUG_KMS("\n"); - pipe_config->has_dsi_encoder = true; - if (IS_BROXTON(dev)) bxt_dsi_get_pipe_config(encoder, pipe_config); @@ -1012,7 +1022,7 @@ static void set_dsi_timings(struct drm_encoder *encoder, const struct drm_display_mode *adjusted_mode) { struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); enum port port; unsigned int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); @@ -1098,7 +1108,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) { struct drm_encoder *encoder = &intel_encoder->base; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; @@ -1171,6 +1181,12 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder) if (intel_dsi->clock_stop) tmp |= CLOCKSTOP; + if (IS_BROXTON(dev_priv)) { + tmp |= BXT_DPHY_DEFEATURE_EN; + if (!is_cmd_mode(intel_dsi)) + tmp |= BXT_DEFEATURE_DPI_FIFO_CTR; + } + for_each_dsi_port(port, intel_dsi->ports) { I915_WRITE(MIPI_DSI_FUNC_PRG(port), val); @@ -1378,12 +1394,13 @@ static const struct drm_encoder_funcs intel_dsi_funcs = { static const struct drm_connector_helper_funcs intel_dsi_connector_helper_funcs = { .get_modes = intel_dsi_get_modes, .mode_valid = intel_dsi_mode_valid, - .best_encoder = intel_best_encoder, }; static const struct drm_connector_funcs intel_dsi_connector_funcs = { .dpms = drm_atomic_helper_connector_dpms, .detect = intel_dsi_detect, + .late_register = intel_connector_register, + .early_unregister = intel_connector_unregister, .destroy = intel_dsi_connector_destroy, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_dsi_set_property, @@ -1413,7 +1430,7 @@ void intel_dsi_init(struct drm_device *dev) struct intel_connector *intel_connector; struct drm_connector *connector; struct drm_display_mode *scan, *fixed_mode = NULL; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port; unsigned int i; @@ -1449,7 +1466,7 @@ void intel_dsi_init(struct drm_device *dev) connector = &intel_connector->base; drm_encoder_init(dev, encoder, &intel_dsi_funcs, DRM_MODE_ENCODER_DSI, - NULL); + "DSI %c", port_name(port)); intel_encoder->compute_config = intel_dsi_compute_config; intel_encoder->pre_enable = intel_dsi_pre_enable; @@ -1460,7 +1477,6 @@ void intel_dsi_init(struct drm_device *dev) intel_encoder->get_config = intel_dsi_get_config; intel_connector->get_hw_state = intel_connector_get_hw_state; - intel_connector->unregister = intel_connector_unregister; /* * On BYT/CHV, pipe A maps to MIPI DSI port A, pipe B maps to MIPI DSI @@ -1473,10 +1489,42 @@ void intel_dsi_init(struct drm_device *dev) else intel_encoder->crtc_mask = BIT(PIPE_B); - if (dev_priv->vbt.dsi.config->dual_link) + if (dev_priv->vbt.dsi.config->dual_link) { intel_dsi->ports = BIT(PORT_A) | BIT(PORT_C); - else + + switch (dev_priv->vbt.dsi.config->dl_dcs_backlight_ports) { + case DL_DCS_PORT_A: + intel_dsi->dcs_backlight_ports = BIT(PORT_A); + break; + case DL_DCS_PORT_C: + intel_dsi->dcs_backlight_ports = BIT(PORT_C); + break; + default: + case DL_DCS_PORT_A_AND_C: + intel_dsi->dcs_backlight_ports = BIT(PORT_A) | BIT(PORT_C); + break; + } + + switch (dev_priv->vbt.dsi.config->dl_dcs_cabc_ports) { + case DL_DCS_PORT_A: + intel_dsi->dcs_cabc_ports = BIT(PORT_A); + break; + case DL_DCS_PORT_C: + intel_dsi->dcs_cabc_ports = BIT(PORT_C); + break; + default: + case DL_DCS_PORT_A_AND_C: + intel_dsi->dcs_cabc_ports = BIT(PORT_A) | BIT(PORT_C); + break; + } + } else { intel_dsi->ports = BIT(port); + intel_dsi->dcs_backlight_ports = BIT(port); + intel_dsi->dcs_cabc_ports = BIT(port); + } + + if (!dev_priv->vbt.dsi.config->cabc_supported) + intel_dsi->dcs_cabc_ports = 0; /* Create a DSI host (and a device) for each port. */ for_each_dsi_port(port, intel_dsi->ports) { @@ -1549,13 +1597,10 @@ void intel_dsi_init(struct drm_device *dev) connector->display_info.height_mm = fixed_mode->height_mm; intel_panel_init(&intel_connector->panel, fixed_mode, NULL); + intel_panel_setup_backlight(connector, INVALID_PIPE); intel_dsi_add_properties(intel_connector); - drm_connector_register(connector); - - intel_panel_setup_backlight(connector, INVALID_PIPE); - return; err: diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h index 61a6957fc..5967ea6d6 100644 --- a/drivers/gpu/drm/i915/intel_dsi.h +++ b/drivers/gpu/drm/i915/intel_dsi.h @@ -78,6 +78,10 @@ struct intel_dsi { u8 escape_clk_div; u8 dual_link; + + u16 dcs_backlight_ports; + u16 dcs_cabc_ports; + u8 pixel_overlap; u32 port_bits; u32 bw_timer; diff --git a/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c b/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c new file mode 100644 index 000000000..ac7c6020c --- /dev/null +++ b/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c @@ -0,0 +1,179 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Author: Deepak M <m.deepak at intel.com> + */ + +#include "intel_drv.h" +#include "intel_dsi.h" +#include "i915_drv.h" +#include <video/mipi_display.h> +#include <drm/drm_mipi_dsi.h> + +#define CONTROL_DISPLAY_BCTRL (1 << 5) +#define CONTROL_DISPLAY_DD (1 << 3) +#define CONTROL_DISPLAY_BL (1 << 2) + +#define POWER_SAVE_OFF (0 << 0) +#define POWER_SAVE_LOW (1 << 0) +#define POWER_SAVE_MEDIUM (2 << 0) +#define POWER_SAVE_HIGH (3 << 0) +#define POWER_SAVE_OUTDOOR_MODE (4 << 0) + +#define PANEL_PWM_MAX_VALUE 0xFF + +static u32 dcs_get_backlight(struct intel_connector *connector) +{ + struct intel_encoder *encoder = connector->encoder; + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + struct mipi_dsi_device *dsi_device; + u8 data; + enum port port; + + /* FIXME: Need to take care of 16 bit brightness level */ + for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) { + dsi_device = intel_dsi->dsi_hosts[port]->device; + mipi_dsi_dcs_read(dsi_device, MIPI_DCS_GET_DISPLAY_BRIGHTNESS, + &data, sizeof(data)); + break; + } + + return data; +} + +static void dcs_set_backlight(struct intel_connector *connector, u32 level) +{ + struct intel_encoder *encoder = connector->encoder; + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + struct mipi_dsi_device *dsi_device; + u8 data = level; + enum port port; + + /* FIXME: Need to take care of 16 bit brightness level */ + for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) { + dsi_device = intel_dsi->dsi_hosts[port]->device; + mipi_dsi_dcs_write(dsi_device, MIPI_DCS_SET_DISPLAY_BRIGHTNESS, + &data, sizeof(data)); + } +} + +static void dcs_disable_backlight(struct intel_connector *connector) +{ + struct intel_encoder *encoder = connector->encoder; + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + struct mipi_dsi_device *dsi_device; + enum port port; + + dcs_set_backlight(connector, 0); + + for_each_dsi_port(port, intel_dsi->dcs_cabc_ports) { + u8 cabc = POWER_SAVE_OFF; + + dsi_device = intel_dsi->dsi_hosts[port]->device; + mipi_dsi_dcs_write(dsi_device, MIPI_DCS_WRITE_POWER_SAVE, + &cabc, sizeof(cabc)); + } + + for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) { + u8 ctrl = 0; + + dsi_device = intel_dsi->dsi_hosts[port]->device; + + mipi_dsi_dcs_read(dsi_device, MIPI_DCS_GET_CONTROL_DISPLAY, + &ctrl, sizeof(ctrl)); + + ctrl &= ~CONTROL_DISPLAY_BL; + ctrl &= ~CONTROL_DISPLAY_DD; + ctrl &= ~CONTROL_DISPLAY_BCTRL; + + mipi_dsi_dcs_write(dsi_device, MIPI_DCS_WRITE_CONTROL_DISPLAY, + &ctrl, sizeof(ctrl)); + } +} + +static void dcs_enable_backlight(struct intel_connector *connector) +{ + struct intel_encoder *encoder = connector->encoder; + struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); + struct intel_panel *panel = &connector->panel; + struct mipi_dsi_device *dsi_device; + enum port port; + + for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) { + u8 ctrl = 0; + + dsi_device = intel_dsi->dsi_hosts[port]->device; + + mipi_dsi_dcs_read(dsi_device, MIPI_DCS_GET_CONTROL_DISPLAY, + &ctrl, sizeof(ctrl)); + + ctrl |= CONTROL_DISPLAY_BL; + ctrl |= CONTROL_DISPLAY_DD; + ctrl |= CONTROL_DISPLAY_BCTRL; + + mipi_dsi_dcs_write(dsi_device, MIPI_DCS_WRITE_CONTROL_DISPLAY, + &ctrl, sizeof(ctrl)); + } + + for_each_dsi_port(port, intel_dsi->dcs_cabc_ports) { + u8 cabc = POWER_SAVE_MEDIUM; + + dsi_device = intel_dsi->dsi_hosts[port]->device; + mipi_dsi_dcs_write(dsi_device, MIPI_DCS_WRITE_POWER_SAVE, + &cabc, sizeof(cabc)); + } + + dcs_set_backlight(connector, panel->backlight.level); +} + +static int dcs_setup_backlight(struct intel_connector *connector, + enum pipe unused) +{ + struct intel_panel *panel = &connector->panel; + + panel->backlight.max = PANEL_PWM_MAX_VALUE; + panel->backlight.level = PANEL_PWM_MAX_VALUE; + + return 0; +} + +int intel_dsi_dcs_init_backlight_funcs(struct intel_connector *intel_connector) +{ + struct drm_device *dev = intel_connector->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_encoder *encoder = intel_connector->encoder; + struct intel_panel *panel = &intel_connector->panel; + + if (dev_priv->vbt.backlight.type != INTEL_BACKLIGHT_DSI_DCS) + return -ENODEV; + + if (WARN_ON(encoder->type != INTEL_OUTPUT_DSI)) + return -EINVAL; + + panel->backlight.setup = dcs_setup_backlight; + panel->backlight.enable = dcs_enable_backlight; + panel->backlight.disable = dcs_disable_backlight; + panel->backlight.set = dcs_set_backlight; + panel->backlight.get = dcs_get_backlight; + + return 0; +} diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c index e498f1c32..cd154ce6b 100644 --- a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c +++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c @@ -95,6 +95,24 @@ static struct gpio_map vlv_gpio_table[] = { { VLV_GPIO_NC_11_PANEL1_BKLTCTL }, }; +#define CHV_GPIO_IDX_START_N 0 +#define CHV_GPIO_IDX_START_E 73 +#define CHV_GPIO_IDX_START_SW 100 +#define CHV_GPIO_IDX_START_SE 198 + +#define CHV_VBT_MAX_PINS_PER_FMLY 15 + +#define CHV_GPIO_PAD_CFG0(f, i) (0x4400 + (f) * 0x400 + (i) * 8) +#define CHV_GPIO_GPIOEN (1 << 15) +#define CHV_GPIO_GPIOCFG_GPIO (0 << 8) +#define CHV_GPIO_GPIOCFG_GPO (1 << 8) +#define CHV_GPIO_GPIOCFG_GPI (2 << 8) +#define CHV_GPIO_GPIOCFG_HIZ (3 << 8) +#define CHV_GPIO_GPIOTXSTATE(state) ((!!(state)) << 1) + +#define CHV_GPIO_PAD_CFG1(f, i) (0x4400 + (f) * 0x400 + (i) * 8 + 4) +#define CHV_GPIO_CFGLOCK (1 << 31) + static inline enum port intel_dsi_seq_port_to_port(u8 port) { return port ? PORT_C : PORT_A; @@ -203,13 +221,14 @@ static void vlv_exec_gpio(struct drm_i915_private *dev_priv, map = &vlv_gpio_table[gpio_index]; if (dev_priv->vbt.dsi.seq_version >= 3) { - DRM_DEBUG_KMS("GPIO element v3 not supported\n"); - return; + /* XXX: this assumes vlv_gpio_table only has NC GPIOs. */ + port = IOSF_PORT_GPIO_NC; } else { if (gpio_source == 0) { port = IOSF_PORT_GPIO_NC; } else if (gpio_source == 1) { - port = IOSF_PORT_GPIO_SC; + DRM_DEBUG_KMS("SC gpio not supported\n"); + return; } else { DRM_DEBUG_KMS("unknown gpio source %u\n", gpio_source); return; @@ -231,10 +250,60 @@ static void vlv_exec_gpio(struct drm_i915_private *dev_priv, mutex_unlock(&dev_priv->sb_lock); } +static void chv_exec_gpio(struct drm_i915_private *dev_priv, + u8 gpio_source, u8 gpio_index, bool value) +{ + u16 cfg0, cfg1; + u16 family_num; + u8 port; + + if (dev_priv->vbt.dsi.seq_version >= 3) { + if (gpio_index >= CHV_GPIO_IDX_START_SE) { + /* XXX: it's unclear whether 255->57 is part of SE. */ + gpio_index -= CHV_GPIO_IDX_START_SE; + port = CHV_IOSF_PORT_GPIO_SE; + } else if (gpio_index >= CHV_GPIO_IDX_START_SW) { + gpio_index -= CHV_GPIO_IDX_START_SW; + port = CHV_IOSF_PORT_GPIO_SW; + } else if (gpio_index >= CHV_GPIO_IDX_START_E) { + gpio_index -= CHV_GPIO_IDX_START_E; + port = CHV_IOSF_PORT_GPIO_E; + } else { + port = CHV_IOSF_PORT_GPIO_N; + } + } else { + /* XXX: The spec is unclear about CHV GPIO on seq v2 */ + if (gpio_source != 0) { + DRM_DEBUG_KMS("unknown gpio source %u\n", gpio_source); + return; + } + + if (gpio_index >= CHV_GPIO_IDX_START_E) { + DRM_DEBUG_KMS("invalid gpio index %u for GPIO N\n", + gpio_index); + return; + } + + port = CHV_IOSF_PORT_GPIO_N; + } + + family_num = gpio_index / CHV_VBT_MAX_PINS_PER_FMLY; + gpio_index = gpio_index % CHV_VBT_MAX_PINS_PER_FMLY; + + cfg0 = CHV_GPIO_PAD_CFG0(family_num, gpio_index); + cfg1 = CHV_GPIO_PAD_CFG1(family_num, gpio_index); + + mutex_lock(&dev_priv->sb_lock); + vlv_iosf_sb_write(dev_priv, port, cfg1, 0); + vlv_iosf_sb_write(dev_priv, port, cfg0, + CHV_GPIO_GPIOCFG_GPO | CHV_GPIO_GPIOTXSTATE(value)); + mutex_unlock(&dev_priv->sb_lock); +} + static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data) { struct drm_device *dev = intel_dsi->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u8 gpio_source, gpio_index; bool value; @@ -254,6 +323,8 @@ static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data) if (IS_VALLEYVIEW(dev_priv)) vlv_exec_gpio(dev_priv, gpio_source, gpio_index, value); + else if (IS_CHERRYVIEW(dev_priv)) + chv_exec_gpio(dev_priv, gpio_source, gpio_index, value); else DRM_DEBUG_KMS("GPIO element not supported on this platform\n"); @@ -398,7 +469,7 @@ static int vbt_panel_get_modes(struct drm_panel *panel) struct vbt_panel *vbt_panel = to_vbt_panel(panel); struct intel_dsi *intel_dsi = vbt_panel->intel_dsi; struct drm_device *dev = intel_dsi->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_display_mode *mode; if (!panel->connector) @@ -426,7 +497,7 @@ static const struct drm_panel_funcs vbt_panel_funcs = { struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id) { struct drm_device *dev = intel_dsi->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct mipi_config *mipi_config = dev_priv->vbt.dsi.config; struct mipi_pps_data *pps = dev_priv->vbt.dsi.pps; struct drm_display_mode *mode = dev_priv->vbt.lfp_lvds_vbt_mode; @@ -578,14 +649,13 @@ struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id) ); /* - * Exit zero is unified val ths_zero and ths_exit + * Exit zero is unified val ths_zero and ths_exit * minimum value for ths_exit = 110ns * min (exit_zero_cnt * 2) = 110/UI * exit_zero_cnt = 55/UI */ - if (exit_zero_cnt < (55 * ui_den / ui_num)) - if ((55 * ui_den) % ui_num) - exit_zero_cnt += 1; + if (exit_zero_cnt < (55 * ui_den / ui_num) && (55 * ui_den) % ui_num) + exit_zero_cnt += 1; /* clk zero count */ clk_zero_cnt = DIV_ROUND_UP( diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/intel_dsi_pll.c index 1765e6e18..6ab58a01b 100644 --- a/drivers/gpu/drm/i915/intel_dsi_pll.c +++ b/drivers/gpu/drm/i915/intel_dsi_pll.c @@ -55,12 +55,10 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv, struct intel_crtc_state *config, int target_dsi_clk) { - unsigned int calc_m = 0, calc_p = 0; unsigned int m_min, m_max, p_min = 2, p_max = 6; unsigned int m, n, p; - int ref_clk; - int delta = target_dsi_clk; - u32 m_seed; + unsigned int calc_m, calc_p; + int delta, ref_clk; /* target_dsi_clk is expected in kHz */ if (target_dsi_clk < 300000 || target_dsi_clk > 1150000) { @@ -80,6 +78,10 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv, m_max = 92; } + calc_p = p_min; + calc_m = m_min; + delta = abs(target_dsi_clk - (m_min * ref_clk) / (p_min * n)); + for (m = m_min; m <= m_max && delta; m++) { for (p = p_min; p <= p_max && delta; p++) { /* @@ -97,11 +99,10 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv, } /* register has log2(N1), this works fine for powers of two */ - n = ffs(n) - 1; - m_seed = lfsr_converts[calc_m - 62]; config->dsi_pll.ctrl = 1 << (DSI_PLL_P1_POST_DIV_SHIFT + calc_p - 2); - config->dsi_pll.div = n << DSI_PLL_N1_DIV_SHIFT | - m_seed << DSI_PLL_M1_DIV_SHIFT; + config->dsi_pll.div = + (ffs(n) - 1) << DSI_PLL_N1_DIV_SHIFT | + (u32)lfsr_converts[calc_m - 62] << DSI_PLL_M1_DIV_SHIFT; return 0; } @@ -113,7 +114,7 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv, static int vlv_compute_dsi_pll(struct intel_encoder *encoder, struct intel_crtc_state *config) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); int ret; u32 dsi_clk; @@ -234,8 +235,11 @@ static void bxt_disable_dsi_pll(struct intel_encoder *encoder) * PLL lock should deassert within 200us. * Wait up to 1ms before timing out. */ - if (wait_for((I915_READ(BXT_DSI_PLL_ENABLE) - & BXT_DSI_PLL_LOCKED) == 0, 1)) + if (intel_wait_for_register(dev_priv, + BXT_DSI_PLL_ENABLE, + BXT_DSI_PLL_LOCKED, + 0, + 1)) DRM_ERROR("Timeout waiting for PLL lock deassertion\n"); } @@ -321,7 +325,7 @@ static u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp, u32 dsi_clk; u32 dsi_ratio; struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); /* Divide by zero */ if (!pipe_bpp) { @@ -356,7 +360,7 @@ u32 intel_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp, static void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port) { u32 temp; - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); temp = I915_READ(MIPI_CTRL(port)); @@ -370,7 +374,7 @@ static void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port) static void bxt_dsi_program_clocks(struct drm_device *dev, enum port port, const struct intel_crtc_state *config) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 tmp; u32 dsi_rate = 0; u32 pll_ratio = 0; @@ -465,7 +469,7 @@ static int bxt_compute_dsi_pll(struct intel_encoder *encoder, static void bxt_enable_dsi_pll(struct intel_encoder *encoder, const struct intel_crtc_state *config) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base); enum port port; u32 val; @@ -486,7 +490,11 @@ static void bxt_enable_dsi_pll(struct intel_encoder *encoder, I915_WRITE(BXT_DSI_PLL_ENABLE, val); /* Timeout and fail if PLL not locked */ - if (wait_for(I915_READ(BXT_DSI_PLL_ENABLE) & BXT_DSI_PLL_LOCKED, 1)) { + if (intel_wait_for_register(dev_priv, + BXT_DSI_PLL_ENABLE, + BXT_DSI_PLL_LOCKED, + BXT_DSI_PLL_LOCKED, + 1)) { DRM_ERROR("Timed out waiting for DSI PLL to lock\n"); return; } @@ -542,7 +550,7 @@ static void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port) { u32 tmp; struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* Clear old configurations */ tmp = I915_READ(BXT_MIPI_CLOCK_CTL); diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c index 286baec97..b9e5a63a7 100644 --- a/drivers/gpu/drm/i915/intel_dvo.c +++ b/drivers/gpu/drm/i915/intel_dvo.c @@ -122,7 +122,7 @@ static struct intel_dvo *intel_attached_dvo(struct drm_connector *connector) static bool intel_dvo_connector_get_hw_state(struct intel_connector *connector) { struct drm_device *dev = connector->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_dvo *intel_dvo = intel_attached_dvo(&connector->base); u32 tmp; @@ -138,7 +138,7 @@ static bool intel_dvo_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_dvo *intel_dvo = enc_to_dvo(encoder); u32 tmp; @@ -155,7 +155,7 @@ static bool intel_dvo_get_hw_state(struct intel_encoder *encoder, static void intel_dvo_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dvo *intel_dvo = enc_to_dvo(encoder); u32 tmp, flags = 0; @@ -176,7 +176,7 @@ static void intel_dvo_get_config(struct intel_encoder *encoder, static void intel_disable_dvo(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dvo *intel_dvo = enc_to_dvo(encoder); i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg; u32 temp = I915_READ(dvo_reg); @@ -188,7 +188,7 @@ static void intel_disable_dvo(struct intel_encoder *encoder) static void intel_enable_dvo(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dvo *intel_dvo = enc_to_dvo(encoder); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg; @@ -256,7 +256,7 @@ static bool intel_dvo_compute_config(struct intel_encoder *encoder, static void intel_dvo_pre_enable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; struct intel_dvo *intel_dvo = enc_to_dvo(encoder); @@ -305,7 +305,7 @@ intel_dvo_detect(struct drm_connector *connector, bool force) static int intel_dvo_get_modes(struct drm_connector *connector) { - struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(connector->dev); const struct drm_display_mode *fixed_mode = to_intel_connector(connector)->panel.fixed_mode; @@ -341,6 +341,8 @@ static void intel_dvo_destroy(struct drm_connector *connector) static const struct drm_connector_funcs intel_dvo_connector_funcs = { .dpms = drm_atomic_helper_connector_dpms, .detect = intel_dvo_detect, + .late_register = intel_connector_register, + .early_unregister = intel_connector_unregister, .destroy = intel_dvo_destroy, .fill_modes = drm_helper_probe_single_connector_modes, .atomic_get_property = intel_connector_atomic_get_property, @@ -351,7 +353,6 @@ static const struct drm_connector_funcs intel_dvo_connector_funcs = { static const struct drm_connector_helper_funcs intel_dvo_connector_helper_funcs = { .mode_valid = intel_dvo_mode_valid, .get_modes = intel_dvo_get_modes, - .best_encoder = intel_best_encoder, }; static void intel_dvo_enc_destroy(struct drm_encoder *encoder) @@ -378,7 +379,7 @@ static struct drm_display_mode * intel_dvo_get_current_mode(struct drm_connector *connector) { struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_dvo *intel_dvo = intel_attached_dvo(connector); uint32_t dvo_val = I915_READ(intel_dvo->dev.dvo_reg); struct drm_display_mode *mode = NULL; @@ -406,9 +407,21 @@ intel_dvo_get_current_mode(struct drm_connector *connector) return mode; } +static char intel_dvo_port_name(i915_reg_t dvo_reg) +{ + if (i915_mmio_reg_equal(dvo_reg, DVOA)) + return 'A'; + else if (i915_mmio_reg_equal(dvo_reg, DVOB)) + return 'B'; + else if (i915_mmio_reg_equal(dvo_reg, DVOC)) + return 'C'; + else + return '?'; +} + void intel_dvo_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *intel_encoder; struct intel_dvo *intel_dvo; struct intel_connector *intel_connector; @@ -428,8 +441,6 @@ void intel_dvo_init(struct drm_device *dev) intel_dvo->attached_connector = intel_connector; intel_encoder = &intel_dvo->base; - drm_encoder_init(dev, &intel_encoder->base, - &intel_dvo_enc_funcs, encoder_type, NULL); intel_encoder->disable = intel_disable_dvo; intel_encoder->enable = intel_enable_dvo; @@ -438,7 +449,6 @@ void intel_dvo_init(struct drm_device *dev) intel_encoder->compute_config = intel_dvo_compute_config; intel_encoder->pre_enable = intel_dvo_pre_enable; intel_connector->get_hw_state = intel_dvo_connector_get_hw_state; - intel_connector->unregister = intel_connector_unregister; /* Now, try to find a controller */ for (i = 0; i < ARRAY_SIZE(intel_dvo_devices); i++) { @@ -496,6 +506,10 @@ void intel_dvo_init(struct drm_device *dev) if (!dvoinit) continue; + drm_encoder_init(dev, &intel_encoder->base, + &intel_dvo_enc_funcs, encoder_type, + "DVO %c", intel_dvo_port_name(dvo->dvo_reg)); + intel_encoder->type = INTEL_OUTPUT_DVO; intel_encoder->crtc_mask = (1 << 0) | (1 << 1); switch (dvo->type) { @@ -537,11 +551,9 @@ void intel_dvo_init(struct drm_device *dev) intel_dvo->panel_wants_dither = true; } - drm_connector_register(connector); return; } - drm_encoder_cleanup(&intel_encoder->base); kfree(intel_dvo); kfree(intel_connector); } diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c index 647127f3a..3836a1c79 100644 --- a/drivers/gpu/drm/i915/intel_fbc.c +++ b/drivers/gpu/drm/i915/intel_fbc.c @@ -124,7 +124,9 @@ static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv) I915_WRITE(FBC_CONTROL, fbc_ctl); /* Wait for compressing bit to clear */ - if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) { + if (intel_wait_for_register(dev_priv, + FBC_STATUS, FBC_STAT_COMPRESSING, 0, + 10)) { DRM_DEBUG_KMS("FBC idle timed out\n"); return; } @@ -374,8 +376,9 @@ static void intel_fbc_hw_deactivate(struct drm_i915_private *dev_priv) * @dev_priv: i915 device instance * * This function is used to verify the current state of FBC. + * * FIXME: This should be tracked in the plane config eventually - * instead of queried at runtime for most callers. + * instead of queried at runtime for most callers. */ bool intel_fbc_is_active(struct drm_i915_private *dev_priv) { @@ -389,7 +392,7 @@ static void intel_fbc_work_fn(struct work_struct *__work) struct intel_fbc *fbc = &dev_priv->fbc; struct intel_fbc_work *work = &fbc->work; struct intel_crtc *crtc = fbc->crtc; - struct drm_vblank_crtc *vblank = &dev_priv->dev->vblank[crtc->pipe]; + struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[crtc->pipe]; if (drm_crtc_vblank_get(&crtc->base)) { DRM_ERROR("vblank not available for FBC on pipe %c\n", @@ -442,7 +445,7 @@ out: static void intel_fbc_schedule_activation(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; struct intel_fbc_work *work = &fbc->work; @@ -480,10 +483,10 @@ static void intel_fbc_deactivate(struct drm_i915_private *dev_priv) intel_fbc_hw_deactivate(dev_priv); } -static bool multiple_pipes_ok(struct intel_crtc *crtc) +static bool multiple_pipes_ok(struct intel_crtc *crtc, + struct intel_plane_state *plane_state) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; - struct drm_plane *primary = crtc->base.primary; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; enum pipe pipe = crtc->pipe; @@ -491,9 +494,7 @@ static bool multiple_pipes_ok(struct intel_crtc *crtc) if (!no_fbc_on_multiple_pipes(dev_priv)) return true; - WARN_ON(!drm_modeset_is_locked(&primary->mutex)); - - if (to_intel_plane_state(primary->state)->visible) + if (plane_state->visible) fbc->visible_pipes_mask |= (1 << pipe); else fbc->visible_pipes_mask &= ~(1 << pipe); @@ -554,7 +555,7 @@ again: static int intel_fbc_alloc_cfb(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; struct drm_mm_node *uninitialized_var(compressed_llb); int size, fb_cpp, ret; @@ -685,7 +686,7 @@ static bool pixel_format_is_valid(struct drm_i915_private *dev_priv, */ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; unsigned int effective_w, effective_h, max_w, max_h; @@ -708,21 +709,16 @@ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc) return effective_w <= max_w && effective_h <= max_h; } -static void intel_fbc_update_state_cache(struct intel_crtc *crtc) +static void intel_fbc_update_state_cache(struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; struct intel_fbc_state_cache *cache = &fbc->state_cache; - struct intel_crtc_state *crtc_state = - to_intel_crtc_state(crtc->base.state); - struct intel_plane_state *plane_state = - to_intel_plane_state(crtc->base.primary->state); struct drm_framebuffer *fb = plane_state->base.fb; struct drm_i915_gem_object *obj; - WARN_ON(!drm_modeset_is_locked(&crtc->base.mutex)); - WARN_ON(!drm_modeset_is_locked(&crtc->base.primary->mutex)); - cache->crtc.mode_flags = crtc_state->base.adjusted_mode.flags; if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) cache->crtc.hsw_bdw_pixel_rate = @@ -740,7 +736,7 @@ static void intel_fbc_update_state_cache(struct intel_crtc *crtc) /* FIXME: We lack the proper locking here, so only run this on the * platforms that need. */ - if (INTEL_INFO(dev_priv)->gen >= 5 && INTEL_INFO(dev_priv)->gen < 7) + if (IS_GEN(dev_priv, 5, 6)) cache->fb.ilk_ggtt_offset = i915_gem_obj_ggtt_offset(obj); cache->fb.pixel_format = fb->pixel_format; cache->fb.stride = fb->pitches[0]; @@ -750,7 +746,7 @@ static void intel_fbc_update_state_cache(struct intel_crtc *crtc) static bool intel_fbc_can_activate(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; struct intel_fbc_state_cache *cache = &fbc->state_cache; @@ -822,22 +818,16 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc) static bool intel_fbc_can_choose(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; - bool enable_by_default = IS_BROADWELL(dev_priv); - if (intel_vgpu_active(dev_priv->dev)) { + if (intel_vgpu_active(dev_priv)) { fbc->no_fbc_reason = "VGPU is active"; return false; } - if (i915.enable_fbc < 0 && !enable_by_default) { - fbc->no_fbc_reason = "disabled per chip default"; - return false; - } - if (!i915.enable_fbc) { - fbc->no_fbc_reason = "disabled per module param"; + fbc->no_fbc_reason = "disabled per module param or by default"; return false; } @@ -857,7 +847,7 @@ static bool intel_fbc_can_choose(struct intel_crtc *crtc) static void intel_fbc_get_reg_params(struct intel_crtc *crtc, struct intel_fbc_reg_params *params) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; struct intel_fbc_state_cache *cache = &fbc->state_cache; @@ -886,9 +876,11 @@ static bool intel_fbc_reg_params_equal(struct intel_fbc_reg_params *params1, return memcmp(params1, params2, sizeof(*params1)) == 0; } -void intel_fbc_pre_update(struct intel_crtc *crtc) +void intel_fbc_pre_update(struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; if (!fbc_supported(dev_priv)) @@ -896,7 +888,7 @@ void intel_fbc_pre_update(struct intel_crtc *crtc) mutex_lock(&fbc->lock); - if (!multiple_pipes_ok(crtc)) { + if (!multiple_pipes_ok(crtc, plane_state)) { fbc->no_fbc_reason = "more than one pipe active"; goto deactivate; } @@ -904,7 +896,7 @@ void intel_fbc_pre_update(struct intel_crtc *crtc) if (!fbc->enabled || fbc->crtc != crtc) goto unlock; - intel_fbc_update_state_cache(crtc); + intel_fbc_update_state_cache(crtc, crtc_state, plane_state); deactivate: intel_fbc_deactivate(dev_priv); @@ -914,7 +906,7 @@ unlock: static void __intel_fbc_post_update(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; struct intel_fbc_reg_params old_params; @@ -947,7 +939,7 @@ static void __intel_fbc_post_update(struct intel_crtc *crtc) void intel_fbc_post_update(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; if (!fbc_supported(dev_priv)) @@ -996,13 +988,13 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv, if (!fbc_supported(dev_priv)) return; - if (origin == ORIGIN_GTT || origin == ORIGIN_FLIP) - return; - mutex_lock(&fbc->lock); fbc->busy_bits &= ~frontbuffer_bits; + if (origin == ORIGIN_GTT || origin == ORIGIN_FLIP) + goto out; + if (!fbc->busy_bits && fbc->enabled && (frontbuffer_bits & intel_fbc_get_frontbuffer_bit(fbc))) { if (fbc->active) @@ -1011,6 +1003,7 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv, __intel_fbc_post_update(fbc->crtc); } +out: mutex_unlock(&fbc->lock); } @@ -1088,9 +1081,11 @@ out: * intel_fbc_enable multiple times for the same pipe without an * intel_fbc_disable in the middle, as long as it is deactivated. */ -void intel_fbc_enable(struct intel_crtc *crtc) +void intel_fbc_enable(struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; if (!fbc_supported(dev_priv)) @@ -1101,19 +1096,19 @@ void intel_fbc_enable(struct intel_crtc *crtc) if (fbc->enabled) { WARN_ON(fbc->crtc == NULL); if (fbc->crtc == crtc) { - WARN_ON(!crtc->config->enable_fbc); + WARN_ON(!crtc_state->enable_fbc); WARN_ON(fbc->active); } goto out; } - if (!crtc->config->enable_fbc) + if (!crtc_state->enable_fbc) goto out; WARN_ON(fbc->active); WARN_ON(fbc->crtc != NULL); - intel_fbc_update_state_cache(crtc); + intel_fbc_update_state_cache(crtc, crtc_state, plane_state); if (intel_fbc_alloc_cfb(crtc)) { fbc->no_fbc_reason = "not enough stolen memory"; goto out; @@ -1161,7 +1156,7 @@ static void __intel_fbc_disable(struct drm_i915_private *dev_priv) */ void intel_fbc_disable(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; if (!fbc_supported(dev_priv)) @@ -1215,12 +1210,49 @@ void intel_fbc_init_pipe_state(struct drm_i915_private *dev_priv) if (!no_fbc_on_multiple_pipes(dev_priv)) return; - for_each_intel_crtc(dev_priv->dev, crtc) + for_each_intel_crtc(&dev_priv->drm, crtc) if (intel_crtc_active(&crtc->base) && to_intel_plane_state(crtc->base.primary->state)->visible) dev_priv->fbc.visible_pipes_mask |= (1 << crtc->pipe); } +/* + * The DDX driver changes its behavior depending on the value it reads from + * i915.enable_fbc, so sanitize it by translating the default value into either + * 0 or 1 in order to allow it to know what's going on. + * + * Notice that this is done at driver initialization and we still allow user + * space to change the value during runtime without sanitizing it again. IGT + * relies on being able to change i915.enable_fbc at runtime. + */ +static int intel_sanitize_fbc_option(struct drm_i915_private *dev_priv) +{ + if (i915.enable_fbc >= 0) + return !!i915.enable_fbc; + + if (!HAS_FBC(dev_priv)) + return 0; + + if (IS_BROADWELL(dev_priv)) + return 1; + + return 0; +} + +static bool need_fbc_vtd_wa(struct drm_i915_private *dev_priv) +{ +#ifdef CONFIG_INTEL_IOMMU + /* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */ + if (intel_iommu_gfx_mapped && + (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))) { + DRM_INFO("Disabling framebuffer compression (FBC) to prevent screen flicker with VT-d enabled\n"); + return true; + } +#endif + + return false; +} + /** * intel_fbc_init - Initialize FBC * @dev_priv: the i915 device @@ -1238,6 +1270,12 @@ void intel_fbc_init(struct drm_i915_private *dev_priv) fbc->active = false; fbc->work.scheduled = false; + if (need_fbc_vtd_wa(dev_priv)) + mkwrite_device_info(dev_priv)->has_fbc = false; + + i915.enable_fbc = intel_sanitize_fbc_option(dev_priv); + DRM_DEBUG_KMS("Sanitized enable_fbc value: %d\n", i915.enable_fbc); + if (!HAS_FBC(dev_priv)) { fbc->no_fbc_reason = "unsupported by this chipset"; return; diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c index ab8d09a81..3e3632c18 100644 --- a/drivers/gpu/drm/i915/intel_fbdev.c +++ b/drivers/gpu/drm/i915/intel_fbdev.c @@ -150,10 +150,10 @@ static int intelfb_alloc(struct drm_fb_helper *helper, if (size * 2 < ggtt->stolen_usable_size) obj = i915_gem_object_create_stolen(dev, size); if (obj == NULL) - obj = i915_gem_alloc_object(dev, size); - if (!obj) { + obj = i915_gem_object_create(dev, size); + if (IS_ERR(obj)) { DRM_ERROR("failed to allocate framebuffer\n"); - ret = -ENOMEM; + ret = PTR_ERR(obj); goto out; } @@ -186,9 +186,11 @@ static int intelfb_create(struct drm_fb_helper *helper, struct i915_ggtt *ggtt = &dev_priv->ggtt; struct fb_info *info; struct drm_framebuffer *fb; + struct i915_vma *vma; struct drm_i915_gem_object *obj; - int size, ret; bool prealloc = false; + void *vaddr; + int ret; if (intel_fb && (sizes->fb_width > intel_fb->base.width || @@ -214,7 +216,6 @@ static int intelfb_create(struct drm_fb_helper *helper, } obj = intel_fb->obj; - size = obj->base.size; mutex_lock(&dev->struct_mutex); @@ -244,22 +245,23 @@ static int intelfb_create(struct drm_fb_helper *helper, info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT; info->fbops = &intelfb_ops; + vma = i915_gem_obj_to_ggtt(obj); + /* setup aperture base/size for vesafb takeover */ info->apertures->ranges[0].base = dev->mode_config.fb_base; info->apertures->ranges[0].size = ggtt->mappable_end; - info->fix.smem_start = dev->mode_config.fb_base + i915_gem_obj_ggtt_offset(obj); - info->fix.smem_len = size; + info->fix.smem_start = dev->mode_config.fb_base + vma->node.start; + info->fix.smem_len = vma->node.size; - info->screen_base = - ioremap_wc(ggtt->mappable_base + i915_gem_obj_ggtt_offset(obj), - size); - if (!info->screen_base) { + vaddr = i915_vma_pin_iomap(vma); + if (IS_ERR(vaddr)) { DRM_ERROR("Failed to remap framebuffer into virtual memory\n"); - ret = -ENOSPC; + ret = PTR_ERR(vaddr); goto out_destroy_fbi; } - info->screen_size = size; + info->screen_base = vaddr; + info->screen_size = vma->node.size; /* This driver doesn't need a VT switch to restore the mode on resume */ info->skip_vt_switch = true; @@ -287,7 +289,7 @@ static int intelfb_create(struct drm_fb_helper *helper, out_destroy_fbi: drm_fb_helper_release_fbi(helper); out_unpin: - i915_gem_object_ggtt_unpin(obj); + intel_unpin_fb_obj(&ifbdev->fb->base, BIT(DRM_ROTATE_0)); out_unlock: mutex_unlock(&dev->struct_mutex); return ret; @@ -360,23 +362,24 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper, bool *enabled, int width, int height) { struct drm_device *dev = fb_helper->dev; + unsigned long conn_configured, mask; + unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG); int i, j; bool *save_enabled; bool fallback = true; int num_connectors_enabled = 0; int num_connectors_detected = 0; - uint64_t conn_configured = 0, mask; int pass = 0; - save_enabled = kcalloc(fb_helper->connector_count, sizeof(bool), - GFP_KERNEL); + save_enabled = kcalloc(count, sizeof(bool), GFP_KERNEL); if (!save_enabled) return false; - memcpy(save_enabled, enabled, fb_helper->connector_count); - mask = (1 << fb_helper->connector_count) - 1; + memcpy(save_enabled, enabled, count); + mask = BIT(count) - 1; + conn_configured = 0; retry: - for (i = 0; i < fb_helper->connector_count; i++) { + for (i = 0; i < count; i++) { struct drm_fb_helper_connector *fb_conn; struct drm_connector *connector; struct drm_encoder *encoder; @@ -386,7 +389,7 @@ retry: fb_conn = fb_helper->connector_info[i]; connector = fb_conn->connector; - if (conn_configured & (1 << i)) + if (conn_configured & BIT(i)) continue; if (pass == 0 && !connector->has_tile) @@ -398,7 +401,7 @@ retry: if (!enabled[i]) { DRM_DEBUG_KMS("connector %s not enabled, skipping\n", connector->name); - conn_configured |= (1 << i); + conn_configured |= BIT(i); continue; } @@ -417,7 +420,7 @@ retry: DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n", connector->name); enabled[i] = false; - conn_configured |= (1 << i); + conn_configured |= BIT(i); continue; } @@ -430,14 +433,15 @@ retry: intel_crtc->lut_b[j] = j; } - new_crtc = intel_fb_helper_crtc(fb_helper, connector->state->crtc); + new_crtc = intel_fb_helper_crtc(fb_helper, + connector->state->crtc); /* * Make sure we're not trying to drive multiple connectors * with a single CRTC, since our cloning support may not * match the BIOS. */ - for (j = 0; j < fb_helper->connector_count; j++) { + for (j = 0; j < count; j++) { if (crtcs[j] == new_crtc) { DRM_DEBUG_KMS("fallback: cloned configuration\n"); goto bail; @@ -488,15 +492,15 @@ retry: } crtcs[i] = new_crtc; - DRM_DEBUG_KMS("connector %s on pipe %c [CRTC:%d]: %dx%d%s\n", + DRM_DEBUG_KMS("connector %s on [CRTC:%d:%s]: %dx%d%s\n", connector->name, - pipe_name(to_intel_crtc(connector->state->crtc)->pipe), connector->state->crtc->base.id, + connector->state->crtc->name, modes[i]->hdisplay, modes[i]->vdisplay, modes[i]->flags & DRM_MODE_FLAG_INTERLACE ? "i" :""); fallback = false; - conn_configured |= (1 << i); + conn_configured |= BIT(i); } if ((conn_configured & mask) != mask) { @@ -520,7 +524,7 @@ retry: if (fallback) { bail: DRM_DEBUG_KMS("Not using firmware configuration\n"); - memcpy(enabled, save_enabled, fb_helper->connector_count); + memcpy(enabled, save_enabled, count); kfree(save_enabled); return false; } @@ -536,8 +540,7 @@ static const struct drm_fb_helper_funcs intel_fb_helper_funcs = { .fb_probe = intelfb_create, }; -static void intel_fbdev_destroy(struct drm_device *dev, - struct intel_fbdev *ifbdev) +static void intel_fbdev_destroy(struct intel_fbdev *ifbdev) { /* We rely on the object-free to release the VMA pinning for * the info->screen_base mmaping. Leaking the VMA is simpler than @@ -550,9 +553,14 @@ static void intel_fbdev_destroy(struct drm_device *dev, drm_fb_helper_fini(&ifbdev->helper); if (ifbdev->fb) { - drm_framebuffer_unregister_private(&ifbdev->fb->base); + mutex_lock(&ifbdev->helper.dev->struct_mutex); + intel_unpin_fb_obj(&ifbdev->fb->base, BIT(DRM_ROTATE_0)); + mutex_unlock(&ifbdev->helper.dev->struct_mutex); + drm_framebuffer_remove(&ifbdev->fb->base); } + + kfree(ifbdev); } /* @@ -685,9 +693,9 @@ out: static void intel_fbdev_suspend_worker(struct work_struct *work) { - intel_fbdev_set_suspend(container_of(work, - struct drm_i915_private, - fbdev_suspend_work)->dev, + intel_fbdev_set_suspend(&container_of(work, + struct drm_i915_private, + fbdev_suspend_work)->drm, FBINFO_STATE_RUNNING, true); } @@ -695,7 +703,7 @@ static void intel_fbdev_suspend_worker(struct work_struct *work) int intel_fbdev_init(struct drm_device *dev) { struct intel_fbdev *ifbdev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int ret; if (WARN_ON(INTEL_INFO(dev)->num_pipes == 0)) @@ -717,8 +725,6 @@ int intel_fbdev_init(struct drm_device *dev) return ret; } - ifbdev->helper.atomic = true; - dev_priv->fbdev = ifbdev; INIT_WORK(&dev_priv->fbdev_suspend_work, intel_fbdev_suspend_worker); @@ -729,42 +735,54 @@ int intel_fbdev_init(struct drm_device *dev) static void intel_fbdev_initial_config(void *data, async_cookie_t cookie) { - struct drm_i915_private *dev_priv = data; - struct intel_fbdev *ifbdev = dev_priv->fbdev; + struct intel_fbdev *ifbdev = data; /* Due to peculiar init order wrt to hpd handling this is separate. */ if (drm_fb_helper_initial_config(&ifbdev->helper, ifbdev->preferred_bpp)) - intel_fbdev_fini(dev_priv->dev); + intel_fbdev_fini(ifbdev->helper.dev); } void intel_fbdev_initial_config_async(struct drm_device *dev) { - async_schedule(intel_fbdev_initial_config, to_i915(dev)); + struct intel_fbdev *ifbdev = to_i915(dev)->fbdev; + + ifbdev->cookie = async_schedule(intel_fbdev_initial_config, ifbdev); +} + +static void intel_fbdev_sync(struct intel_fbdev *ifbdev) +{ + if (!ifbdev->cookie) + return; + + /* Only serialises with all preceding async calls, hence +1 */ + async_synchronize_cookie(ifbdev->cookie + 1); + ifbdev->cookie = 0; } void intel_fbdev_fini(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - if (!dev_priv->fbdev) + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_fbdev *ifbdev = dev_priv->fbdev; + + if (!ifbdev) return; flush_work(&dev_priv->fbdev_suspend_work); - if (!current_is_async()) - async_synchronize_full(); - intel_fbdev_destroy(dev, dev_priv->fbdev); - kfree(dev_priv->fbdev); + intel_fbdev_sync(ifbdev); + + intel_fbdev_destroy(ifbdev); dev_priv->fbdev = NULL; } void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_fbdev *ifbdev = dev_priv->fbdev; struct fb_info *info; - if (!ifbdev) + if (!ifbdev || !ifbdev->fb) return; info = ifbdev->helper.fbdev; @@ -809,29 +827,28 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous void intel_fbdev_output_poll_changed(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; - if (dev_priv->fbdev) - drm_fb_helper_hotplug_event(&dev_priv->fbdev->helper); + struct intel_fbdev *ifbdev = to_i915(dev)->fbdev; + + if (ifbdev && ifbdev->fb) + drm_fb_helper_hotplug_event(&ifbdev->helper); } void intel_fbdev_restore_mode(struct drm_device *dev) { - int ret; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_fbdev *ifbdev = dev_priv->fbdev; - struct drm_fb_helper *fb_helper; + struct intel_fbdev *ifbdev = to_i915(dev)->fbdev; if (!ifbdev) return; - fb_helper = &ifbdev->helper; + intel_fbdev_sync(ifbdev); + if (!ifbdev->fb) + return; - ret = drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper); - if (ret) { + if (drm_fb_helper_restore_fbdev_mode_unlocked(&ifbdev->helper)) { DRM_DEBUG("failed to restore crtc mode\n"); } else { - mutex_lock(&fb_helper->dev->struct_mutex); + mutex_lock(&dev->struct_mutex); intel_fb_obj_invalidate(ifbdev->fb->obj, ORIGIN_GTT); - mutex_unlock(&fb_helper->dev->struct_mutex); + mutex_unlock(&dev->struct_mutex); } } diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c index 9be839a24..2aa744081 100644 --- a/drivers/gpu/drm/i915/intel_fifo_underrun.c +++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c @@ -50,7 +50,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc; enum pipe pipe; @@ -68,7 +68,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev) static bool cpt_can_enable_serr_int(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; struct intel_crtc *crtc; @@ -105,7 +105,7 @@ static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable, bool old) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t reg = PIPESTAT(pipe); u32 pipestat = I915_READ(reg) & 0xffff0000; @@ -123,7 +123,7 @@ static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev, static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN : DE_PIPEB_FIFO_UNDERRUN; @@ -154,7 +154,7 @@ static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable, bool old) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (enable) { I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe)); @@ -176,7 +176,7 @@ static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev, static void broadwell_set_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (enable) bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_FIFO_UNDERRUN); @@ -188,7 +188,7 @@ static void ibx_set_fifo_underrun_reporting(struct drm_device *dev, enum transcoder pch_transcoder, bool enable) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t bit = (pch_transcoder == TRANSCODER_A) ? SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER; @@ -220,7 +220,7 @@ static void cpt_set_fifo_underrun_reporting(struct drm_device *dev, enum transcoder pch_transcoder, bool enable, bool old) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (enable) { I915_WRITE(SERR_INT, @@ -244,7 +244,7 @@ static void cpt_set_fifo_underrun_reporting(struct drm_device *dev, static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); bool old; @@ -289,7 +289,7 @@ bool intel_set_cpu_fifo_underrun_reporting(struct drm_i915_private *dev_priv, bool ret; spin_lock_irqsave(&dev_priv->irq_lock, flags); - ret = __intel_set_cpu_fifo_underrun_reporting(dev_priv->dev, pipe, + ret = __intel_set_cpu_fifo_underrun_reporting(&dev_priv->drm, pipe, enable); spin_unlock_irqrestore(&dev_priv->irq_lock, flags); @@ -334,10 +334,12 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, intel_crtc->pch_fifo_underrun_disabled = !enable; if (HAS_PCH_IBX(dev_priv)) - ibx_set_fifo_underrun_reporting(dev_priv->dev, pch_transcoder, + ibx_set_fifo_underrun_reporting(&dev_priv->drm, + pch_transcoder, enable); else - cpt_set_fifo_underrun_reporting(dev_priv->dev, pch_transcoder, + cpt_set_fifo_underrun_reporting(&dev_priv->drm, + pch_transcoder, enable, old); spin_unlock_irqrestore(&dev_priv->irq_lock, flags); @@ -405,7 +407,7 @@ void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv) spin_lock_irq(&dev_priv->irq_lock); - for_each_intel_crtc(dev_priv->dev, crtc) { + for_each_intel_crtc(&dev_priv->drm, crtc) { if (crtc->cpu_fifo_underrun_disabled) continue; @@ -432,7 +434,7 @@ void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv) spin_lock_irq(&dev_priv->irq_lock); - for_each_intel_crtc(dev_priv->dev, crtc) { + for_each_intel_crtc(&dev_priv->drm, crtc) { if (crtc->pch_fifo_underrun_disabled) continue; diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h index 9d79c4c3e..3e3e74374 100644 --- a/drivers/gpu/drm/i915/intel_guc.h +++ b/drivers/gpu/drm/i915/intel_guc.h @@ -26,6 +26,7 @@ #include "intel_guc_fwif.h" #include "i915_guc_reg.h" +#include "intel_ringbuffer.h" struct drm_i915_gem_request; @@ -48,14 +49,23 @@ struct drm_i915_gem_request; * queue (a circular array of work items), again described in the process * descriptor. Work queue pages are mapped momentarily as required. * - * Finally, we also keep a few statistics here, including the number of - * submissions to each engine, and a record of the last submission failure - * (if any). + * We also keep a few statistics on failures. Ideally, these should all + * be zero! + * no_wq_space: times that the submission pre-check found no space was + * available in the work queue (note, the queue is shared, + * not per-engine). It is OK for this to be nonzero, but + * it should not be huge! + * q_fail: failed to enqueue a work item. This should never happen, + * because we check for space beforehand. + * b_fail: failed to ring the doorbell. This should never happen, unless + * somehow the hardware misbehaves, or maybe if the GuC firmware + * crashes? We probably need to reset the GPU to recover. + * retcode: errno from last guc_submit() */ struct i915_guc_client { struct drm_i915_gem_object *client_obj; void *client_base; /* first page (only) of above */ - struct intel_context *owner; + struct i915_gem_context *owner; struct intel_guc *guc; uint32_t priority; uint32_t ctx_index; @@ -71,12 +81,13 @@ struct i915_guc_client { uint32_t wq_tail; uint32_t unused; /* Was 'wq_head' */ - /* GuC submission statistics & status */ - uint64_t submissions[GUC_MAX_ENGINES_NUM]; - uint32_t q_fail; + uint32_t no_wq_space; + uint32_t q_fail; /* No longer used */ uint32_t b_fail; int retcode; - int spare; /* pad to 32 DWords */ + + /* Per-engine counts of GuC submissions */ + uint64_t submissions[I915_NUM_ENGINES]; }; enum intel_guc_fw_status { @@ -133,25 +144,24 @@ struct intel_guc { uint32_t action_fail; /* Total number of failures */ int32_t action_err; /* Last error code */ - uint64_t submissions[GUC_MAX_ENGINES_NUM]; - uint32_t last_seqno[GUC_MAX_ENGINES_NUM]; + uint64_t submissions[I915_NUM_ENGINES]; + uint32_t last_seqno[I915_NUM_ENGINES]; }; /* intel_guc_loader.c */ -extern void intel_guc_ucode_init(struct drm_device *dev); -extern int intel_guc_ucode_load(struct drm_device *dev); -extern void intel_guc_ucode_fini(struct drm_device *dev); +extern void intel_guc_init(struct drm_device *dev); +extern int intel_guc_setup(struct drm_device *dev); +extern void intel_guc_fini(struct drm_device *dev); extern const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status); extern int intel_guc_suspend(struct drm_device *dev); extern int intel_guc_resume(struct drm_device *dev); /* i915_guc_submission.c */ -int i915_guc_submission_init(struct drm_device *dev); -int i915_guc_submission_enable(struct drm_device *dev); -int i915_guc_submit(struct i915_guc_client *client, - struct drm_i915_gem_request *rq); -void i915_guc_submission_disable(struct drm_device *dev); -void i915_guc_submission_fini(struct drm_device *dev); -int i915_guc_wq_check_space(struct i915_guc_client *client); +int i915_guc_submission_init(struct drm_i915_private *dev_priv); +int i915_guc_submission_enable(struct drm_i915_private *dev_priv); +int i915_guc_wq_check_space(struct drm_i915_gem_request *rq); +int i915_guc_submit(struct drm_i915_gem_request *rq); +void i915_guc_submission_disable(struct drm_i915_private *dev_priv); +void i915_guc_submission_fini(struct drm_i915_private *dev_priv); #endif diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h index 2de57ffe5..944786d70 100644 --- a/drivers/gpu/drm/i915/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/intel_guc_fwif.h @@ -71,7 +71,8 @@ #define WQ_WORKLOAD_TOUCH (2 << WQ_WORKLOAD_SHIFT) #define WQ_RING_TAIL_SHIFT 20 -#define WQ_RING_TAIL_MASK (0x7FF << WQ_RING_TAIL_SHIFT) +#define WQ_RING_TAIL_MAX 0x7FF /* 2^11 QWords */ +#define WQ_RING_TAIL_MASK (WQ_RING_TAIL_MAX << WQ_RING_TAIL_SHIFT) #define GUC_DOORBELL_ENABLED 1 #define GUC_DOORBELL_DISABLED 0 diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c index 6021d5872..a00f2d654 100644 --- a/drivers/gpu/drm/i915/intel_guc_loader.c +++ b/drivers/gpu/drm/i915/intel_guc_loader.c @@ -62,6 +62,12 @@ #define I915_SKL_GUC_UCODE "/*(DEBLOBBED)*/" /*(DEBLOBBED)*/ +#define I915_BXT_GUC_UCODE "/*(DEBLOBBED)*/" +/*(DEBLOBBED)*/ + +#define I915_KBL_GUC_UCODE "/*(DEBLOBBED)*/" +/*(DEBLOBBED)*/ + /* User-friendly representation of an enum */ const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status) { @@ -84,7 +90,7 @@ static void direct_interrupts_to_host(struct drm_i915_private *dev_priv) struct intel_engine_cs *engine; int irqs; - /* tell all command streamers NOT to forward interrupts and vblank to GuC */ + /* tell all command streamers NOT to forward interrupts or vblank to GuC */ irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_NEVER); irqs |= _MASKED_BIT_DISABLE(GFX_INTERRUPT_STEERING); for_each_engine(engine, dev_priv) @@ -100,10 +106,10 @@ static void direct_interrupts_to_guc(struct drm_i915_private *dev_priv) { struct intel_engine_cs *engine; int irqs; + u32 tmp; - /* tell all command streamers to forward interrupts and vblank to GuC */ - irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_ALWAYS); - irqs |= _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING); + /* tell all command streamers to forward interrupts (but not vblank) to GuC */ + irqs = _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING); for_each_engine(engine, dev_priv) I915_WRITE(RING_MODE_GEN7(engine), irqs); @@ -114,6 +120,16 @@ static void direct_interrupts_to_guc(struct drm_i915_private *dev_priv) I915_WRITE(GUC_BCS_RCS_IER, ~irqs); I915_WRITE(GUC_VCS2_VCS1_IER, ~irqs); I915_WRITE(GUC_WD_VECS_IER, ~irqs); + + /* + * If GuC has routed PM interrupts to itself, don't keep it. + * and keep other interrupts those are unmasked by GuC. + */ + tmp = I915_READ(GEN6_PMINTRMSK); + if (tmp & GEN8_PMINTR_REDIRECT_TO_NON_DISP) { + dev_priv->rps.pm_intr_keep |= ~(tmp & ~GEN8_PMINTR_REDIRECT_TO_NON_DISP); + dev_priv->rps.pm_intr_keep &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP; + } } static u32 get_gttype(struct drm_i915_private *dev_priv) @@ -281,13 +297,24 @@ static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv) return ret; } +static u32 guc_wopcm_size(struct drm_i915_private *dev_priv) +{ + u32 wopcm_size = GUC_WOPCM_TOP; + + /* On BXT, the top of WOPCM is reserved for RC6 context */ + if (IS_BROXTON(dev_priv)) + wopcm_size -= BXT_GUC_WOPCM_RC6_RESERVED; + + return wopcm_size; +} + /* * Load the GuC firmware blob into the MinuteIA. */ static int guc_ucode_xfer(struct drm_i915_private *dev_priv) { struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; int ret; ret = i915_gem_object_set_to_gtt_domain(guc_fw->guc_fw_obj, false); @@ -308,7 +335,7 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv) intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); /* init WOPCM */ - I915_WRITE(GUC_WOPCM_SIZE, GUC_WOPCM_SIZE_VALUE); + I915_WRITE(GUC_WOPCM_SIZE, guc_wopcm_size(dev_priv)); I915_WRITE(DMA_GUC_WOPCM_OFFSET, GUC_WOPCM_OFFSET_VALUE); /* Enable MIA caching. GuC clock gating is disabled. */ @@ -372,66 +399,63 @@ static int i915_reset_guc(struct drm_i915_private *dev_priv) } /** - * intel_guc_ucode_load() - load GuC uCode into the device + * intel_guc_setup() - finish preparing the GuC for activity * @dev: drm device * * Called from gem_init_hw() during driver loading and also after a GPU reset. * + * The main action required here it to load the GuC uCode into the device. * The firmware image should have already been fetched into memory by the - * earlier call to intel_guc_ucode_init(), so here we need only check that - * is succeeded, and then transfer the image to the h/w. + * earlier call to intel_guc_init(), so here we need only check that worked, + * and then transfer the image to the h/w. * * Return: non-zero code on error */ -int intel_guc_ucode_load(struct drm_device *dev) +int intel_guc_setup(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; - int retries, err = 0; + const char *fw_path = guc_fw->guc_fw_path; + int retries, ret, err; - if (!i915.enable_guc_submission) - return 0; - - DRM_DEBUG_DRIVER("GuC fw status: fetch %s, load %s\n", + DRM_DEBUG_DRIVER("GuC fw status: path %s, fetch %s, load %s\n", + fw_path, intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status), intel_guc_fw_status_repr(guc_fw->guc_fw_load_status)); - direct_interrupts_to_host(dev_priv); - - if (guc_fw->guc_fw_fetch_status == GUC_FIRMWARE_NONE) - return 0; - - if (guc_fw->guc_fw_fetch_status == GUC_FIRMWARE_SUCCESS && - guc_fw->guc_fw_load_status == GUC_FIRMWARE_FAIL) - return -ENOEXEC; - - guc_fw->guc_fw_load_status = GUC_FIRMWARE_PENDING; - - DRM_DEBUG_DRIVER("GuC fw fetch status %s\n", - intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status)); - - switch (guc_fw->guc_fw_fetch_status) { - case GUC_FIRMWARE_FAIL: - /* something went wrong :( */ - err = -EIO; + /* Loading forbidden, or no firmware to load? */ + if (!i915.enable_guc_loading) { + err = 0; goto fail; - - case GUC_FIRMWARE_NONE: - case GUC_FIRMWARE_PENDING: - default: - /* "can't happen" */ - WARN_ONCE(1, "GuC fw %s invalid guc_fw_fetch_status %s [%d]\n", - guc_fw->guc_fw_path, - intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status), - guc_fw->guc_fw_fetch_status); + } else if (fw_path == NULL) { + /* Device is known to have no uCode (e.g. no GuC) */ err = -ENXIO; goto fail; + } else if (*fw_path == '\0') { + /* Device has a GuC but we don't know what f/w to load? */ + DRM_INFO("No GuC firmware known for this platform\n"); + err = -ENODEV; + goto fail; + } - case GUC_FIRMWARE_SUCCESS: - break; + /* Fetch failed, or already fetched but failed to load? */ + if (guc_fw->guc_fw_fetch_status != GUC_FIRMWARE_SUCCESS) { + err = -EIO; + goto fail; + } else if (guc_fw->guc_fw_load_status == GUC_FIRMWARE_FAIL) { + err = -ENOEXEC; + goto fail; } - err = i915_guc_submission_init(dev); + direct_interrupts_to_host(dev_priv); + + guc_fw->guc_fw_load_status = GUC_FIRMWARE_PENDING; + + DRM_DEBUG_DRIVER("GuC fw status: fetch %s, load %s\n", + intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status), + intel_guc_fw_status_repr(guc_fw->guc_fw_load_status)); + + err = i915_guc_submission_init(dev_priv); if (err) goto fail; @@ -448,7 +472,7 @@ int intel_guc_ucode_load(struct drm_device *dev) */ err = i915_reset_guc(dev_priv); if (err) { - DRM_ERROR("GuC reset failed, err %d\n", err); + DRM_ERROR("GuC reset failed: %d\n", err); goto fail; } @@ -459,8 +483,8 @@ int intel_guc_ucode_load(struct drm_device *dev) if (--retries == 0) goto fail; - DRM_INFO("GuC fw load failed, err %d; will reset and " - "retry %d more time(s)\n", err, retries); + DRM_INFO("GuC fw load failed: %d; will reset and " + "retry %d more time(s)\n", err, retries); } guc_fw->guc_fw_load_status = GUC_FIRMWARE_SUCCESS; @@ -470,10 +494,7 @@ int intel_guc_ucode_load(struct drm_device *dev) intel_guc_fw_status_repr(guc_fw->guc_fw_load_status)); if (i915.enable_guc_submission) { - /* The execbuf_client will be recreated. Release it first. */ - i915_guc_submission_disable(dev); - - err = i915_guc_submission_enable(dev); + err = i915_guc_submission_enable(dev_priv); if (err) goto fail; direct_interrupts_to_guc(dev_priv); @@ -482,15 +503,50 @@ int intel_guc_ucode_load(struct drm_device *dev) return 0; fail: - DRM_ERROR("GuC firmware load failed, err %d\n", err); if (guc_fw->guc_fw_load_status == GUC_FIRMWARE_PENDING) guc_fw->guc_fw_load_status = GUC_FIRMWARE_FAIL; direct_interrupts_to_host(dev_priv); - i915_guc_submission_disable(dev); - i915_guc_submission_fini(dev); + i915_guc_submission_disable(dev_priv); + i915_guc_submission_fini(dev_priv); + + /* + * We've failed to load the firmware :( + * + * Decide whether to disable GuC submission and fall back to + * execlist mode, and whether to hide the error by returning + * zero or to return -EIO, which the caller will treat as a + * nonfatal error (i.e. it doesn't prevent driver load, but + * marks the GPU as wedged until reset). + */ + if (i915.enable_guc_loading > 1) { + ret = -EIO; + } else if (i915.enable_guc_submission > 1) { + ret = -EIO; + } else { + ret = 0; + } + + if (err == 0 && !HAS_GUC_UCODE(dev)) + ; /* Don't mention the GuC! */ + else if (err == 0) + DRM_INFO("GuC firmware load skipped\n"); + else if (ret != -EIO) + DRM_INFO("GuC firmware load failed: %d\n", err); + else + DRM_ERROR("GuC firmware load failed: %d\n", err); + + if (i915.enable_guc_submission) { + if (fw_path == NULL) + DRM_INFO("GuC submission without firmware not supported\n"); + if (ret == 0) + DRM_INFO("Falling back from GuC submission to execlist mode\n"); + else + DRM_ERROR("GuC init failed: %d\n", ret); + } + i915.enable_guc_submission = 0; - return err; + return ret; } static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw) @@ -552,9 +608,7 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw) /* Header and uCode will be loaded to WOPCM. Size of the two. */ size = guc_fw->header_size + guc_fw->ucode_size; - - /* Top 32k of WOPCM is reserved (8K stack + 24k RC6 context). */ - if (size > GUC_WOPCM_SIZE_VALUE - 0x8000) { + if (size > guc_wopcm_size(to_i915(dev))) { DRM_ERROR("Firmware is too large to fit in WOPCM\n"); goto fail; } @@ -617,22 +671,25 @@ fail: } /** - * intel_guc_ucode_init() - define parameters and fetch firmware + * intel_guc_init() - define parameters and fetch firmware * @dev: drm device * * Called early during driver load, but after GEM is initialised. * * The firmware will be transferred to the GuC's memory later, - * when intel_guc_ucode_load() is called. + * when intel_guc_setup() is called. */ -void intel_guc_ucode_init(struct drm_device *dev) +void intel_guc_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; const char *fw_path; - if (!HAS_GUC_SCHED(dev)) - i915.enable_guc_submission = false; + /* A negative value means "use platform default" */ + if (i915.enable_guc_loading < 0) + i915.enable_guc_loading = HAS_GUC_UCODE(dev); + if (i915.enable_guc_submission < 0) + i915.enable_guc_submission = HAS_GUC_SCHED(dev); if (!HAS_GUC_UCODE(dev)) { fw_path = NULL; @@ -640,27 +697,30 @@ void intel_guc_ucode_init(struct drm_device *dev) fw_path = I915_SKL_GUC_UCODE; guc_fw->guc_fw_major_wanted = 6; guc_fw->guc_fw_minor_wanted = 1; + } else if (IS_BROXTON(dev)) { + fw_path = I915_BXT_GUC_UCODE; + guc_fw->guc_fw_major_wanted = 8; + guc_fw->guc_fw_minor_wanted = 7; + } else if (IS_KABYLAKE(dev)) { + fw_path = I915_KBL_GUC_UCODE; + guc_fw->guc_fw_major_wanted = 9; + guc_fw->guc_fw_minor_wanted = 14; } else { - i915.enable_guc_submission = false; fw_path = ""; /* unknown device */ } - if (!i915.enable_guc_submission) - return; - guc_fw->guc_dev = dev; guc_fw->guc_fw_path = fw_path; guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_NONE; guc_fw->guc_fw_load_status = GUC_FIRMWARE_NONE; + /* Early (and silent) return if GuC loading is disabled */ + if (!i915.enable_guc_loading) + return; if (fw_path == NULL) return; - - if (*fw_path == '\0') { - DRM_ERROR("No GuC firmware known for this platform\n"); - guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_FAIL; + if (*fw_path == '\0') return; - } guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_PENDING; DRM_DEBUG_DRIVER("GuC firmware pending, path %s\n", fw_path); @@ -669,18 +729,18 @@ void intel_guc_ucode_init(struct drm_device *dev) } /** - * intel_guc_ucode_fini() - clean up all allocated resources + * intel_guc_fini() - clean up all allocated resources * @dev: drm device */ -void intel_guc_ucode_fini(struct drm_device *dev) +void intel_guc_fini(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw; mutex_lock(&dev->struct_mutex); direct_interrupts_to_host(dev_priv); - i915_guc_submission_disable(dev); - i915_guc_submission_fini(dev); + i915_guc_submission_disable(dev_priv); + i915_guc_submission_fini(dev_priv); if (guc_fw->guc_fw_obj) drm_gem_object_unreference(&guc_fw->guc_fw_obj->base); diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c new file mode 100644 index 000000000..434f4d5c5 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_gvt.c @@ -0,0 +1,104 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "i915_drv.h" +#include "intel_gvt.h" + +/** + * DOC: Intel GVT-g host support + * + * Intel GVT-g is a graphics virtualization technology which shares the + * GPU among multiple virtual machines on a time-sharing basis. Each + * virtual machine is presented a virtual GPU (vGPU), which has equivalent + * features as the underlying physical GPU (pGPU), so i915 driver can run + * seamlessly in a virtual machine. This file provides the englightments + * of GVT and the necessary components used by GVT in i915 driver. + */ + +static bool is_supported_device(struct drm_i915_private *dev_priv) +{ + if (IS_BROADWELL(dev_priv)) + return true; + return false; +} + +/** + * intel_gvt_init - initialize GVT components + * @dev_priv: drm i915 private data + * + * This function is called at the initialization stage to create a GVT device. + * + * Returns: + * Zero on success, negative error code if failed. + * + */ +int intel_gvt_init(struct drm_i915_private *dev_priv) +{ + int ret; + + if (!i915.enable_gvt) { + DRM_DEBUG_DRIVER("GVT-g is disabled by kernel params\n"); + return 0; + } + + if (!is_supported_device(dev_priv)) { + DRM_DEBUG_DRIVER("Unsupported device. GVT-g is disabled\n"); + goto bail; + } + + /* + * We're not in host or fail to find a MPT module, disable GVT-g + */ + ret = intel_gvt_init_host(); + if (ret) { + DRM_DEBUG_DRIVER("Not in host or MPT modules not found\n"); + goto bail; + } + + ret = intel_gvt_init_device(dev_priv); + if (ret) { + DRM_DEBUG_DRIVER("Fail to init GVT device\n"); + goto bail; + } + + return 0; + +bail: + i915.enable_gvt = 0; + return 0; +} + +/** + * intel_gvt_cleanup - cleanup GVT components when i915 driver is unloading + * @dev_priv: drm i915 private * + * + * This function is called at the i915 driver unloading stage, to shutdown + * GVT components and release the related resources. + */ +void intel_gvt_cleanup(struct drm_i915_private *dev_priv) +{ + if (!intel_gvt_active(dev_priv)) + return; + + intel_gvt_clean_device(dev_priv); +} diff --git a/drivers/gpu/drm/i915/intel_gvt.h b/drivers/gpu/drm/i915/intel_gvt.h new file mode 100644 index 000000000..960211df7 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_gvt.h @@ -0,0 +1,45 @@ +/* + * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef _INTEL_GVT_H_ +#define _INTEL_GVT_H_ + +#include "gvt/gvt.h" + +#ifdef CONFIG_DRM_I915_GVT +int intel_gvt_init(struct drm_i915_private *dev_priv); +void intel_gvt_cleanup(struct drm_i915_private *dev_priv); +int intel_gvt_init_device(struct drm_i915_private *dev_priv); +void intel_gvt_clean_device(struct drm_i915_private *dev_priv); +int intel_gvt_init_host(void); +#else +static inline int intel_gvt_init(struct drm_i915_private *dev_priv) +{ + return 0; +} +static inline void intel_gvt_cleanup(struct drm_i915_private *dev_priv) +{ +} +#endif + +#endif /* _INTEL_GVT_H_ */ diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c index a8844702d..4df9f3849 100644 --- a/drivers/gpu/drm/i915/intel_hdmi.c +++ b/drivers/gpu/drm/i915/intel_hdmi.c @@ -47,7 +47,7 @@ static void assert_hdmi_port_disabled(struct intel_hdmi *intel_hdmi) { struct drm_device *dev = intel_hdmi_to_dev(intel_hdmi); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t enabled_bits; enabled_bits = HAS_DDI(dev) ? DDI_BUF_CTL_ENABLE : SDVO_ENABLE; @@ -138,7 +138,7 @@ static void g4x_write_infoframe(struct drm_encoder *encoder, { const uint32_t *data = frame; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 val = I915_READ(VIDEO_DIP_CTL); int i; @@ -192,7 +192,7 @@ static void ibx_write_infoframe(struct drm_encoder *encoder, { const uint32_t *data = frame; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); @@ -251,7 +251,7 @@ static void cpt_write_infoframe(struct drm_encoder *encoder, { const uint32_t *data = frame; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); @@ -308,7 +308,7 @@ static void vlv_write_infoframe(struct drm_encoder *encoder, { const uint32_t *data = frame; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); i915_reg_t reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe); u32 val = I915_READ(reg); @@ -366,7 +366,7 @@ static void hsw_write_infoframe(struct drm_encoder *encoder, { const uint32_t *data = frame; struct drm_device *dev = encoder->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder; i915_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder); @@ -508,7 +508,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder, bool enable, const struct drm_display_mode *adjusted_mode) { - struct drm_i915_private *dev_priv = encoder->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi; i915_reg_t reg = VIDEO_DIP_CTL; @@ -629,7 +629,7 @@ static bool gcp_default_phase_possible(int pipe_bpp, static bool intel_hdmi_set_gcp_infoframe(struct drm_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_crtc *crtc = to_intel_crtc(encoder->crtc); i915_reg_t reg; u32 val = 0; @@ -661,7 +661,7 @@ static void ibx_set_infoframes(struct drm_encoder *encoder, bool enable, const struct drm_display_mode *adjusted_mode) { - struct drm_i915_private *dev_priv = encoder->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi; @@ -713,7 +713,7 @@ static void cpt_set_infoframes(struct drm_encoder *encoder, bool enable, const struct drm_display_mode *adjusted_mode) { - struct drm_i915_private *dev_priv = encoder->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe); @@ -755,7 +755,7 @@ static void vlv_set_infoframes(struct drm_encoder *encoder, bool enable, const struct drm_display_mode *adjusted_mode) { - struct drm_i915_private *dev_priv = encoder->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); @@ -807,7 +807,7 @@ static void hsw_set_infoframes(struct drm_encoder *encoder, bool enable, const struct drm_display_mode *adjusted_mode) { - struct drm_i915_private *dev_priv = encoder->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); i915_reg_t reg = HSW_TVIDEO_DIP_CTL(intel_crtc->config->cpu_transcoder); @@ -855,7 +855,7 @@ void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable) static void intel_hdmi_prepare(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; @@ -894,7 +894,7 @@ static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); enum intel_display_power_domain power_domain; u32 tmp; @@ -931,7 +931,7 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder, { struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 tmp, flags = 0; int dotclock; @@ -988,7 +988,7 @@ static void intel_enable_hdmi_audio(struct intel_encoder *encoder) static void g4x_enable_hdmi(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); u32 temp; @@ -1009,7 +1009,7 @@ static void g4x_enable_hdmi(struct intel_encoder *encoder) static void ibx_enable_hdmi(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); u32 temp; @@ -1058,7 +1058,7 @@ static void ibx_enable_hdmi(struct intel_encoder *encoder) static void cpt_enable_hdmi(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); enum pipe pipe = crtc->pipe; @@ -1115,7 +1115,7 @@ static void vlv_enable_hdmi(struct intel_encoder *encoder) static void intel_disable_hdmi(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); u32 temp; @@ -1154,7 +1154,7 @@ static void intel_disable_hdmi(struct intel_encoder *encoder) I915_WRITE(intel_hdmi->hdmi_reg, temp); POSTING_READ(intel_hdmi->hdmi_reg); - intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A); + intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } @@ -1273,33 +1273,15 @@ intel_hdmi_mode_valid(struct drm_connector *connector, static bool hdmi_12bpc_possible(struct intel_crtc_state *crtc_state) { struct drm_device *dev = crtc_state->base.crtc->dev; - struct drm_atomic_state *state; - struct intel_encoder *encoder; - struct drm_connector *connector; - struct drm_connector_state *connector_state; - int count = 0, count_hdmi = 0; - int i; if (HAS_GMCH_DISPLAY(dev)) return false; - state = crtc_state->base.state; - - for_each_connector_in_state(state, connector, connector_state, i) { - if (connector_state->crtc != crtc_state->base.crtc) - continue; - - encoder = to_intel_encoder(connector_state->best_encoder); - - count_hdmi += encoder->type == INTEL_OUTPUT_HDMI; - count++; - } - /* * HDMI 12bpc affects the clocks, so it's only possible * when not cloning with other encoder types. */ - return count_hdmi > 0 && count_hdmi == count; + return crtc_state->output_types == 1 << INTEL_OUTPUT_HDMI; } bool intel_hdmi_compute_config(struct intel_encoder *encoder, @@ -1575,7 +1557,7 @@ intel_hdmi_set_property(struct drm_connector *connector, struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); struct intel_digital_port *intel_dig_port = hdmi_to_dig_port(intel_hdmi); - struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(connector->dev); int ret; ret = drm_object_property_set_value(&connector->base, property, val); @@ -1674,39 +1656,16 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder) struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); struct intel_hdmi *intel_hdmi = &dport->hdmi; struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; - enum dpio_channel port = vlv_dport_to_channel(dport); - int pipe = intel_crtc->pipe; - u32 val; - /* Enable clock channels for this port */ - mutex_lock(&dev_priv->sb_lock); - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(port)); - val = 0; - if (pipe) - val |= (1<<21); - else - val &= ~(1<<21); - val |= 0x001000c4; - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW8(port), val); + vlv_phy_pre_encoder_enable(encoder); /* HDMI 1.0V-2dB */ - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW4(port), 0x2b245f5f); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(port), 0x5578b83a); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(port), 0x0c782040); - vlv_dpio_write(dev_priv, pipe, VLV_TX3_DW4(port), 0x2b247878); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW11(port), 0x00030000); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), 0x00002000); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN); - - /* Program lane clock */ - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW14(port), 0x00760018); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW23(port), 0x00400888); - mutex_unlock(&dev_priv->sb_lock); + vlv_set_phy_signal_level(encoder, 0x2b245f5f, 0x00002000, 0x5578b83a, + 0x2b247878); intel_hdmi->set_infoframes(&encoder->base, intel_crtc->config->has_hdmi_sink, @@ -1719,213 +1678,33 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder) static void vlv_hdmi_pre_pll_enable(struct intel_encoder *encoder) { - struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = - to_intel_crtc(encoder->base.crtc); - enum dpio_channel port = vlv_dport_to_channel(dport); - int pipe = intel_crtc->pipe; - intel_hdmi_prepare(encoder); - /* Program Tx lane resets to default */ - mutex_lock(&dev_priv->sb_lock); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), - DPIO_PCS_TX_LANE2_RESET | - DPIO_PCS_TX_LANE1_RESET); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), - DPIO_PCS_CLK_CRI_RXEB_EIOS_EN | - DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN | - (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) | - DPIO_PCS_CLK_SOFT_RESET); - - /* Fix up inter-pair skew failure */ - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW12(port), 0x00750f00); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW11(port), 0x00001500); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW14(port), 0x40400000); - - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), 0x00002000); - vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN); - mutex_unlock(&dev_priv->sb_lock); -} - -static void chv_data_lane_soft_reset(struct intel_encoder *encoder, - bool reset) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base)); - struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); - enum pipe pipe = crtc->pipe; - uint32_t val; - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch)); - if (reset) - val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - else - val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val); - - if (crtc->config->lane_count > 2) { - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch)); - if (reset) - val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET); - else - val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val); - } - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - if (reset) - val &= ~DPIO_PCS_CLK_SOFT_RESET; - else - val |= DPIO_PCS_CLK_SOFT_RESET; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val); - - if (crtc->config->lane_count > 2) { - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch)); - val |= CHV_PCS_REQ_SOFTRESET_EN; - if (reset) - val &= ~DPIO_PCS_CLK_SOFT_RESET; - else - val |= DPIO_PCS_CLK_SOFT_RESET; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val); - } + vlv_phy_pre_pll_enable(encoder); } static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder) { - struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); - struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_crtc *intel_crtc = - to_intel_crtc(encoder->base.crtc); - enum dpio_channel ch = vlv_dport_to_channel(dport); - enum pipe pipe = intel_crtc->pipe; - u32 val; - intel_hdmi_prepare(encoder); - /* - * Must trick the second common lane into life. - * Otherwise we can't even access the PLL. - */ - if (ch == DPIO_CH0 && pipe == PIPE_B) - dport->release_cl2_override = - !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true); - - chv_phy_powergate_lanes(encoder, true, 0x0); - - mutex_lock(&dev_priv->sb_lock); - - /* Assert data lane reset */ - chv_data_lane_soft_reset(encoder, true); - - /* program left/right clock distribution */ - if (pipe != PIPE_B) { - val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); - val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); - if (ch == DPIO_CH0) - val |= CHV_BUFLEFTENA1_FORCE; - if (ch == DPIO_CH1) - val |= CHV_BUFRIGHTENA1_FORCE; - vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); - } else { - val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); - val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); - if (ch == DPIO_CH0) - val |= CHV_BUFLEFTENA2_FORCE; - if (ch == DPIO_CH1) - val |= CHV_BUFRIGHTENA2_FORCE; - vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); - } - - /* program clock channel usage */ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(ch)); - val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; - if (pipe != PIPE_B) - val &= ~CHV_PCS_USEDCLKCHANNEL; - else - val |= CHV_PCS_USEDCLKCHANNEL; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch)); - val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE; - if (pipe != PIPE_B) - val &= ~CHV_PCS_USEDCLKCHANNEL; - else - val |= CHV_PCS_USEDCLKCHANNEL; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val); - - /* - * This a a bit weird since generally CL - * matches the pipe, but here we need to - * pick the CL based on the port. - */ - val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW19(ch)); - if (pipe != PIPE_B) - val &= ~CHV_CMN_USEDCLKCHANNEL; - else - val |= CHV_CMN_USEDCLKCHANNEL; - vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW19(ch), val); - - mutex_unlock(&dev_priv->sb_lock); + chv_phy_pre_pll_enable(encoder); } static void chv_hdmi_post_pll_disable(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe; - u32 val; - - mutex_lock(&dev_priv->sb_lock); - - /* disable left/right clock distribution */ - if (pipe != PIPE_B) { - val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0); - val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK); - vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val); - } else { - val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1); - val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK); - vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val); - } - - mutex_unlock(&dev_priv->sb_lock); - - /* - * Leave the power down bit cleared for at least one - * lane so that chv_powergate_phy_ch() will power - * on something when the channel is otherwise unused. - * When the port is off and the override is removed - * the lanes power down anyway, so otherwise it doesn't - * really matter what the state of power down bits is - * after this. - */ - chv_phy_powergate_lanes(encoder, false, 0x0); + chv_phy_post_pll_disable(encoder); } static void vlv_hdmi_post_disable(struct intel_encoder *encoder) { - struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; - struct intel_crtc *intel_crtc = - to_intel_crtc(encoder->base.crtc); - enum dpio_channel port = vlv_dport_to_channel(dport); - int pipe = intel_crtc->pipe; - /* Reset lanes to avoid HDMI flicker (VLV w/a) */ - mutex_lock(&dev_priv->sb_lock); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), 0x00000000); - vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), 0x00e00060); - mutex_unlock(&dev_priv->sb_lock); + vlv_phy_reset_lanes(encoder); } static void chv_hdmi_post_disable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); mutex_lock(&dev_priv->sb_lock); @@ -1940,142 +1719,16 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder) struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); struct intel_hdmi *intel_hdmi = &dport->hdmi; struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode; - enum dpio_channel ch = vlv_dport_to_channel(dport); - int pipe = intel_crtc->pipe; - int data, i, stagger; - u32 val; - mutex_lock(&dev_priv->sb_lock); - - /* allow hardware to manage TX FIFO reset source */ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch)); - val &= ~DPIO_LANEDESKEW_STRAP_OVRD; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); - val &= ~DPIO_LANEDESKEW_STRAP_OVRD; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); - - /* Program Tx latency optimal setting */ - for (i = 0; i < 4; i++) { - /* Set the upar bit */ - data = (i == 1) ? 0x0 : 0x1; - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i), - data << DPIO_UPAR_SHIFT); - } - - /* Data lane stagger programming */ - if (intel_crtc->config->port_clock > 270000) - stagger = 0x18; - else if (intel_crtc->config->port_clock > 135000) - stagger = 0xd; - else if (intel_crtc->config->port_clock > 67500) - stagger = 0x7; - else if (intel_crtc->config->port_clock > 33750) - stagger = 0x4; - else - stagger = 0x2; - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch)); - val |= DPIO_TX2_STAGGER_MASK(0x1f); - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch)); - val |= DPIO_TX2_STAGGER_MASK(0x1f); - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val); - - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch), - DPIO_LANESTAGGER_STRAP(stagger) | - DPIO_LANESTAGGER_STRAP_OVRD | - DPIO_TX1_STAGGER_MASK(0x1f) | - DPIO_TX1_STAGGER_MULT(6) | - DPIO_TX2_STAGGER_MULT(0)); - - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch), - DPIO_LANESTAGGER_STRAP(stagger) | - DPIO_LANESTAGGER_STRAP_OVRD | - DPIO_TX1_STAGGER_MASK(0x1f) | - DPIO_TX1_STAGGER_MULT(7) | - DPIO_TX2_STAGGER_MULT(5)); - - /* Deassert data lane reset */ - chv_data_lane_soft_reset(encoder, false); - - /* Clear calc init */ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); - val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); - val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); - val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); - val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3); - val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK); - val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch)); - val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); - val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch)); - val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK); - val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val); + chv_phy_pre_encoder_enable(encoder); /* FIXME: Program the support xxx V-dB */ /* Use 800mV-0dB */ - for (i = 0; i < 4; i++) { - val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i)); - val &= ~DPIO_SWING_DEEMPH9P5_MASK; - val |= 128 << DPIO_SWING_DEEMPH9P5_SHIFT; - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val); - } - - for (i = 0; i < 4; i++) { - val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i)); - - val &= ~DPIO_SWING_MARGIN000_MASK; - val |= 102 << DPIO_SWING_MARGIN000_SHIFT; - - /* - * Supposedly this value shouldn't matter when unique transition - * scale is disabled, but in fact it does matter. Let's just - * always program the same value and hope it's OK. - */ - val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT); - val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT; - - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val); - } - - /* - * The document said it needs to set bit 27 for ch0 and bit 26 - * for ch1. Might be a typo in the doc. - * For now, for this unique transition scale selection, set bit - * 27 for ch0 and ch1. - */ - for (i = 0; i < 4; i++) { - val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i)); - val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN; - vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val); - } - - /* Start swing calculation */ - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch)); - val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; - vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val); - - val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch)); - val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3; - vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val); - - mutex_unlock(&dev_priv->sb_lock); + chv_set_phy_signal_level(encoder, 128, 102, false); intel_hdmi->set_infoframes(&encoder->base, intel_crtc->config->has_hdmi_sink, @@ -2086,10 +1739,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder) vlv_wait_port_ready(dev_priv, dport, 0x0); /* Second common lane will stay alive on its own now */ - if (dport->release_cl2_override) { - chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false); - dport->release_cl2_override = false; - } + chv_phy_release_cl2_override(encoder); } static void intel_hdmi_destroy(struct drm_connector *connector) @@ -2106,6 +1756,8 @@ static const struct drm_connector_funcs intel_hdmi_connector_funcs = { .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_hdmi_set_property, .atomic_get_property = intel_connector_atomic_get_property, + .late_register = intel_connector_register, + .early_unregister = intel_connector_unregister, .destroy = intel_hdmi_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, @@ -2114,7 +1766,6 @@ static const struct drm_connector_funcs intel_hdmi_connector_funcs = { static const struct drm_connector_helper_funcs intel_hdmi_connector_helper_funcs = { .get_modes = intel_hdmi_get_modes, .mode_valid = intel_hdmi_mode_valid, - .best_encoder = intel_best_encoder, }; static const struct drm_encoder_funcs intel_hdmi_enc_funcs = { @@ -2138,7 +1789,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi; struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_device *dev = intel_encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum port port = intel_dig_port->port; uint8_t alternate_ddc_pin; @@ -2242,12 +1893,10 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, intel_connector->get_hw_state = intel_ddi_connector_get_hw_state; else intel_connector->get_hw_state = intel_connector_get_hw_state; - intel_connector->unregister = intel_connector_unregister; intel_hdmi_add_properties(intel_hdmi, connector); intel_connector_attach_encoder(intel_connector, intel_encoder); - drm_connector_register(connector); intel_hdmi->attached_connector = intel_connector; /* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written @@ -2280,7 +1929,7 @@ void intel_hdmi_init(struct drm_device *dev, intel_encoder = &intel_dig_port->base; drm_encoder_init(dev, &intel_encoder->base, &intel_hdmi_enc_funcs, - DRM_MODE_ENCODER_TMDS, NULL); + DRM_MODE_ENCODER_TMDS, "HDMI %c", port_name(port)); intel_encoder->compute_config = intel_hdmi_compute_config; if (HAS_PCH_SPLIT(dev)) { diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c index 2c49458a9..f48957ea1 100644 --- a/drivers/gpu/drm/i915/intel_hotplug.c +++ b/drivers/gpu/drm/i915/intel_hotplug.c @@ -144,7 +144,7 @@ static bool intel_hpd_irq_storm_detect(struct drm_i915_private *dev_priv, static void intel_hpd_irq_storm_disable(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_connector *intel_connector; struct intel_encoder *intel_encoder; @@ -191,7 +191,7 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) struct drm_i915_private *dev_priv = container_of(work, typeof(*dev_priv), hotplug.reenable_work.work); - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct drm_mode_config *mode_config = &dev->mode_config; int i; @@ -220,7 +220,7 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) } } if (dev_priv->display.hpd_irq_setup) - dev_priv->display.hpd_irq_setup(dev); + dev_priv->display.hpd_irq_setup(dev_priv); spin_unlock_irq(&dev_priv->irq_lock); intel_runtime_pm_put(dev_priv); @@ -302,7 +302,7 @@ static void i915_hotplug_work_func(struct work_struct *work) { struct drm_i915_private *dev_priv = container_of(work, struct drm_i915_private, hotplug.hotplug_work); - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct drm_mode_config *mode_config = &dev->mode_config; struct intel_connector *intel_connector; struct intel_encoder *intel_encoder; @@ -346,7 +346,7 @@ static void i915_hotplug_work_func(struct work_struct *work) /** * intel_hpd_irq_handler - main hotplug irq handler - * @dev: drm device + * @dev_priv: drm_i915_private * @pin_mask: a mask of hpd pins that have triggered the irq * @long_mask: a mask of hpd pins that may be long hpd pulses * @@ -360,10 +360,9 @@ static void i915_hotplug_work_func(struct work_struct *work) * Here, we do hotplug irq storm detection and mitigation, and pass further * processing to appropriate bottom halves. */ -void intel_hpd_irq_handler(struct drm_device *dev, +void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 pin_mask, u32 long_mask) { - struct drm_i915_private *dev_priv = dev->dev_private; int i; enum port port; bool storm_detected = false; @@ -407,7 +406,7 @@ void intel_hpd_irq_handler(struct drm_device *dev, * hotplug bits itself. So only WARN about unexpected * interrupts on saner platforms. */ - WARN_ONCE(!HAS_GMCH_DISPLAY(dev), + WARN_ONCE(!HAS_GMCH_DISPLAY(dev_priv), "Received HPD interrupt on pin %d although disabled\n", i); continue; } @@ -427,7 +426,7 @@ void intel_hpd_irq_handler(struct drm_device *dev, } if (storm_detected) - dev_priv->display.hpd_irq_setup(dev); + dev_priv->display.hpd_irq_setup(dev_priv); spin_unlock(&dev_priv->irq_lock); /* @@ -474,7 +473,7 @@ void intel_hpd_init(struct drm_i915_private *dev_priv) */ spin_lock_irq(&dev_priv->irq_lock); if (dev_priv->display.hpd_irq_setup) - dev_priv->display.hpd_irq_setup(dev_priv->dev); + dev_priv->display.hpd_irq_setup(dev_priv); spin_unlock_irq(&dev_priv->irq_lock); } @@ -482,7 +481,7 @@ void i915_hpd_poll_init_work(struct work_struct *work) { struct drm_i915_private *dev_priv = container_of(work, struct drm_i915_private, hotplug.poll_init_work); - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct drm_mode_config *mode_config = &dev->mode_config; struct drm_connector *connector; bool enabled; diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c index 81de23098..1f266d7df 100644 --- a/drivers/gpu/drm/i915/intel_i2c.c +++ b/drivers/gpu/drm/i915/intel_i2c.c @@ -113,7 +113,7 @@ to_intel_gmbus(struct i2c_adapter *i2c) void intel_i2c_reset(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(GMBUS0, 0); I915_WRITE(GMBUS4, 0); @@ -138,7 +138,7 @@ static void intel_i2c_quirk_set(struct drm_i915_private *dev_priv, bool enable) static u32 get_reserved(struct intel_gmbus *bus) { struct drm_i915_private *dev_priv = bus->dev_priv; - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; u32 reserved = 0; /* On most chips, these bits must be preserved in software. */ @@ -212,7 +212,7 @@ intel_gpio_pre_xfer(struct i2c_adapter *adapter) adapter); struct drm_i915_private *dev_priv = bus->dev_priv; - intel_i2c_reset(dev_priv->dev); + intel_i2c_reset(&dev_priv->drm); intel_i2c_quirk_set(dev_priv, true); set_data(bus, 1); set_clock(bus, 1); @@ -298,15 +298,16 @@ gmbus_wait_idle(struct drm_i915_private *dev_priv) { int ret; -#define C ((I915_READ_NOTRACE(GMBUS2) & GMBUS_ACTIVE) == 0) - if (!HAS_GMBUS_IRQ(dev_priv)) - return wait_for(C, 10); + return intel_wait_for_register(dev_priv, + GMBUS2, GMBUS_ACTIVE, 0, + 10); /* Important: The hw handles only the first bit, so set only one! */ I915_WRITE(GMBUS4, GMBUS_IDLE_EN); - ret = wait_event_timeout(dev_priv->gmbus_wait_queue, C, + ret = wait_event_timeout(dev_priv->gmbus_wait_queue, + (I915_READ_NOTRACE(GMBUS2) & GMBUS_ACTIVE) == 0, msecs_to_jiffies_timeout(10)); I915_WRITE(GMBUS4, 0); @@ -315,7 +316,6 @@ gmbus_wait_idle(struct drm_i915_private *dev_priv) return 0; else return -ETIMEDOUT; -#undef C } static int @@ -632,7 +632,7 @@ static const struct i2c_algorithm gmbus_algorithm = { */ int intel_setup_gmbus(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_gmbus *bus; unsigned int pin; int ret; @@ -688,7 +688,7 @@ int intel_setup_gmbus(struct drm_device *dev) goto err; } - intel_i2c_reset(dev_priv->dev); + intel_i2c_reset(&dev_priv->drm); return 0; @@ -736,7 +736,7 @@ void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit) void intel_teardown_gmbus(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_gmbus *bus; unsigned int pin; diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index 7f2d8415e..414ddda43 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -208,31 +208,27 @@ } while (0) enum { - ADVANCED_CONTEXT = 0, - LEGACY_32B_CONTEXT, - ADVANCED_AD_CONTEXT, - LEGACY_64B_CONTEXT -}; -#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3 -#define GEN8_CTX_ADDRESSING_MODE(dev) (USES_FULL_48BIT_PPGTT(dev) ?\ - LEGACY_64B_CONTEXT :\ - LEGACY_32B_CONTEXT) -enum { FAULT_AND_HANG = 0, FAULT_AND_HALT, /* Debug only */ FAULT_AND_STREAM, FAULT_AND_CONTINUE /* Unsupported */ }; #define GEN8_CTX_ID_SHIFT 32 +#define GEN8_CTX_ID_WIDTH 21 #define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17 #define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x26 -static int intel_lr_context_pin(struct intel_context *ctx, +/* Typical size of the average request (2 pipecontrols and a MI_BB) */ +#define EXECLISTS_REQUEST_SIZE 64 /* bytes */ + +static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, + struct intel_engine_cs *engine); +static int intel_lr_context_pin(struct i915_gem_context *ctx, struct intel_engine_cs *engine); /** * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists - * @dev: DRM device. + * @dev_priv: i915 device private * @enable_execlists: value of i915.enable_execlists module parameter. * * Only certain platforms support Execlists (the prerequisites being @@ -240,23 +236,22 @@ static int intel_lr_context_pin(struct intel_context *ctx, * * Return: 1 if Execlists is supported and has to be enabled. */ -int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists) +int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists) { - WARN_ON(i915.enable_ppgtt == -1); - /* On platforms with execlist available, vGPU will only * support execlist mode, no ring buffer mode. */ - if (HAS_LOGICAL_RING_CONTEXTS(dev) && intel_vgpu_active(dev)) + if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && intel_vgpu_active(dev_priv)) return 1; - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_GEN(dev_priv) >= 9) return 1; if (enable_execlists == 0) return 0; - if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev) && + if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && + USES_PPGTT(dev_priv) && i915.use_mmio_flip >= 0) return 1; @@ -266,19 +261,17 @@ int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists static void logical_ring_init_platform_invariants(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; + struct drm_i915_private *dev_priv = engine->i915; - if (IS_GEN8(dev) || IS_GEN9(dev)) + if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv)) engine->idle_lite_restore_wa = ~0; - engine->disable_lite_restore_wa = (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || - IS_BXT_REVID(dev, 0, BXT_REVID_A1)) && + engine->disable_lite_restore_wa = (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) || + IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) && (engine->id == VCS || engine->id == VCS2); engine->ctx_desc_template = GEN8_CTX_VALID; - engine->ctx_desc_template |= GEN8_CTX_ADDRESSING_MODE(dev) << - GEN8_CTX_ADDRESSING_MODE_SHIFT; - if (IS_GEN8(dev)) + if (IS_GEN8(dev_priv)) engine->ctx_desc_template |= GEN8_CTX_L3LLC_COHERENT; engine->ctx_desc_template |= GEN8_CTX_PRIVILEGE; @@ -297,7 +290,7 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine) * descriptor for a pinned context * * @ctx: Context to work on - * @ring: Engine the descriptor will be used with + * @engine: Engine the descriptor will be used with * * The context descriptor encodes various attributes of a context, * including its GTT address and some flags. Because it's fairly @@ -305,62 +298,42 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine) * which remains valid until the context is unpinned. * * This is what a descriptor looks like, from LSB to MSB: - * bits 0-11: flags, GEN8_CTX_* (cached in ctx_desc_template) + * bits 0-11: flags, GEN8_CTX_* (cached in ctx_desc_template) * bits 12-31: LRCA, GTT address of (the HWSP of) this context - * bits 32-51: ctx ID, a globally unique tag (the LRCA again!) - * bits 52-63: reserved, may encode the engine ID (for GuC) + * bits 32-52: ctx ID, a globally unique tag + * bits 53-54: mbz, reserved for use by hardware + * bits 55-63: group ID, currently unused and set to 0 */ static void -intel_lr_context_descriptor_update(struct intel_context *ctx, +intel_lr_context_descriptor_update(struct i915_gem_context *ctx, struct intel_engine_cs *engine) { - uint64_t lrca, desc; + struct intel_context *ce = &ctx->engine[engine->id]; + u64 desc; - lrca = ctx->engine[engine->id].lrc_vma->node.start + - LRC_PPHWSP_PN * PAGE_SIZE; + BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH)); - desc = engine->ctx_desc_template; /* bits 0-11 */ - desc |= lrca; /* bits 12-31 */ - desc |= (lrca >> PAGE_SHIFT) << GEN8_CTX_ID_SHIFT; /* bits 32-51 */ + desc = ctx->desc_template; /* bits 3-4 */ + desc |= engine->ctx_desc_template; /* bits 0-11 */ + desc |= ce->lrc_vma->node.start + LRC_PPHWSP_PN * PAGE_SIZE; + /* bits 12-31 */ + desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT; /* bits 32-52 */ - ctx->engine[engine->id].lrc_desc = desc; + ce->lrc_desc = desc; } -uint64_t intel_lr_context_descriptor(struct intel_context *ctx, +uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx, struct intel_engine_cs *engine) { return ctx->engine[engine->id].lrc_desc; } -/** - * intel_execlists_ctx_id() - get the Execlists Context ID - * @ctx: Context to get the ID for - * @ring: Engine to get the ID for - * - * Do not confuse with ctx->id! Unfortunately we have a name overload - * here: the old context ID we pass to userspace as a handler so that - * they can refer to a context, and the new context ID we pass to the - * ELSP so that the GPU can inform us of the context status via - * interrupts. - * - * The context ID is a portion of the context descriptor, so we can - * just extract the required part from the cached descriptor. - * - * Return: 20-bits globally unique context ID. - */ -u32 intel_execlists_ctx_id(struct intel_context *ctx, - struct intel_engine_cs *engine) -{ - return intel_lr_context_descriptor(ctx, engine) >> GEN8_CTX_ID_SHIFT; -} - static void execlists_elsp_write(struct drm_i915_gem_request *rq0, struct drm_i915_gem_request *rq1) { struct intel_engine_cs *engine = rq0->engine; - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = rq0->i915; uint64_t desc[2]; if (rq1) { @@ -431,6 +404,20 @@ static void execlists_submit_requests(struct drm_i915_gem_request *rq0, spin_unlock_irq(&dev_priv->uncore.lock); } +static inline void execlists_context_status_change( + struct drm_i915_gem_request *rq, + unsigned long status) +{ + /* + * Only used when GVT-g is enabled now. When GVT-g is disabled, + * The compiler should eliminate this function as dead-code. + */ + if (!IS_ENABLED(CONFIG_DRM_I915_GVT)) + return; + + atomic_notifier_call_chain(&rq->ctx->status_notifier, status, rq); +} + static void execlists_context_unqueue(struct intel_engine_cs *engine) { struct drm_i915_gem_request *req0 = NULL, *req1 = NULL; @@ -442,7 +429,7 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine) * If irqs are not active generate a warning as batches that finish * without the irqs may get lost and a GPU Hang may occur. */ - WARN_ON(!intel_irqs_enabled(engine->dev->dev_private)); + WARN_ON(!intel_irqs_enabled(engine->i915)); /* Try to read in pairs */ list_for_each_entry_safe(cursor, tmp, &engine->execlist_queue, @@ -453,10 +440,24 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine) /* Same ctx: ignore first request, as second request * will update tail past first request's workload */ cursor->elsp_submitted = req0->elsp_submitted; - list_move_tail(&req0->execlist_link, - &engine->execlist_retired_req_list); + list_del(&req0->execlist_link); + i915_gem_request_unreference(req0); req0 = cursor; } else { + if (IS_ENABLED(CONFIG_DRM_I915_GVT)) { + /* + * req0 (after merged) ctx requires single + * submission, stop picking + */ + if (req0->ctx->execlists_force_single_submission) + break; + /* + * req0 ctx doesn't require single submission, + * but next req ctx requires, stop picking + */ + if (cursor->ctx->execlists_force_single_submission) + break; + } req1 = cursor; WARN_ON(req1->elsp_submitted); break; @@ -466,6 +467,12 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine) if (unlikely(!req0)) return; + execlists_context_status_change(req0, INTEL_CONTEXT_SCHEDULE_IN); + + if (req1) + execlists_context_status_change(req1, + INTEL_CONTEXT_SCHEDULE_IN); + if (req0->elsp_submitted & engine->idle_lite_restore_wa) { /* * WaIdleLiteRestore: make sure we never cause a lite restore @@ -486,7 +493,7 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine) } static unsigned int -execlists_check_remove_request(struct intel_engine_cs *engine, u32 request_id) +execlists_check_remove_request(struct intel_engine_cs *engine, u32 ctx_id) { struct drm_i915_gem_request *head_req; @@ -496,19 +503,18 @@ execlists_check_remove_request(struct intel_engine_cs *engine, u32 request_id) struct drm_i915_gem_request, execlist_link); - if (!head_req) - return 0; - - if (unlikely(intel_execlists_ctx_id(head_req->ctx, engine) != request_id)) - return 0; + if (WARN_ON(!head_req || (head_req->ctx_hw_id != ctx_id))) + return 0; WARN(head_req->elsp_submitted == 0, "Never submitted head request\n"); if (--head_req->elsp_submitted > 0) return 0; - list_move_tail(&head_req->execlist_link, - &engine->execlist_retired_req_list); + execlists_context_status_change(head_req, INTEL_CONTEXT_SCHEDULE_OUT); + + list_del(&head_req->execlist_link); + i915_gem_request_unreference(head_req); return 1; } @@ -517,7 +523,7 @@ static u32 get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer, u32 *context_id) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; u32 status; read_pointer %= GEN8_CSB_ENTRIES; @@ -535,7 +541,7 @@ get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer, /** * intel_lrc_irq_handler() - handle Context Switch interrupts - * @engine: Engine Command Streamer to handle. + * @data: tasklet handler passed in unsigned long * * Check the unread Context Status Buffers and manage the submission of new * contexts to the ELSP accordingly. @@ -543,7 +549,7 @@ get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer, static void intel_lrc_irq_handler(unsigned long data) { struct intel_engine_cs *engine = (struct intel_engine_cs *)data; - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; u32 status_pointer; unsigned int read_pointer, write_pointer; u32 csb[GEN8_CSB_ENTRIES][2]; @@ -612,11 +618,6 @@ static void execlists_context_queue(struct drm_i915_gem_request *request) struct drm_i915_gem_request *cursor; int num_elements = 0; - if (request->ctx != request->i915->kernel_context) - intel_lr_context_pin(request->ctx, engine); - - i915_gem_request_reference(request); - spin_lock_bh(&engine->execlist_lock); list_for_each_entry(cursor, &engine->execlist_queue, execlist_link) @@ -633,12 +634,14 @@ static void execlists_context_queue(struct drm_i915_gem_request *request) if (request->ctx == tail_req->ctx) { WARN(tail_req->elsp_submitted != 0, "More than 2 already-submitted reqs queued\n"); - list_move_tail(&tail_req->execlist_link, - &engine->execlist_retired_req_list); + list_del(&tail_req->execlist_link); + i915_gem_request_unreference(tail_req); } } + i915_gem_request_reference(request); list_add_tail(&request->execlist_link, &engine->execlist_queue); + request->ctx_hw_id = request->ctx->hw_id; if (num_elements == 0) execlists_context_unqueue(engine); @@ -698,9 +701,23 @@ static int execlists_move_to_gpu(struct drm_i915_gem_request *req, int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request) { - int ret = 0; + struct intel_engine_cs *engine = request->engine; + struct intel_context *ce = &request->ctx->engine[engine->id]; + int ret; - request->ringbuf = request->ctx->engine[request->engine->id].ringbuf; + /* Flush enough space to reduce the likelihood of waiting after + * we start building the request - in which case we will just + * have to repeat work. + */ + request->reserved_space += EXECLISTS_REQUEST_SIZE; + + if (!ce->state) { + ret = execlists_context_deferred_alloc(request->ctx, engine); + if (ret) + return ret; + } + + request->ringbuf = ce->ringbuf; if (i915.enable_guc_submission) { /* @@ -708,16 +725,39 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request * going any further, as the i915_add_request() call * later on mustn't fail ... */ - struct intel_guc *guc = &request->i915->guc; - - ret = i915_guc_wq_check_space(guc->execbuf_client); + ret = i915_guc_wq_check_space(request); if (ret) return ret; } - if (request->ctx != request->i915->kernel_context) - ret = intel_lr_context_pin(request->ctx, request->engine); + ret = intel_lr_context_pin(request->ctx, engine); + if (ret) + return ret; + + ret = intel_ring_begin(request, 0); + if (ret) + goto err_unpin; + + if (!ce->initialised) { + ret = engine->init_context(request); + if (ret) + goto err_unpin; + + ce->initialised = true; + } + + /* Note that after this point, we have committed to using + * this request as it is being used to both track the + * state of engine initialisation and liveness of the + * golden renderstate above. Think twice before you try + * to cancel/unwind this request now. + */ + request->reserved_space -= EXECLISTS_REQUEST_SIZE; + return 0; + +err_unpin: + intel_lr_context_unpin(request->ctx, engine); return ret; } @@ -734,7 +774,6 @@ static int intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request) { struct intel_ringbuffer *ringbuf = request->ringbuf; - struct drm_i915_private *dev_priv = request->i915; struct intel_engine_cs *engine = request->engine; intel_logical_ring_advance(ringbuf); @@ -750,54 +789,28 @@ intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request) intel_logical_ring_emit(ringbuf, MI_NOOP); intel_logical_ring_advance(ringbuf); - if (intel_engine_stopped(engine)) - return 0; - - if (engine->last_context != request->ctx) { - if (engine->last_context) - intel_lr_context_unpin(engine->last_context, engine); - if (request->ctx != request->i915->kernel_context) { - intel_lr_context_pin(request->ctx, engine); - engine->last_context = request->ctx; - } else { - engine->last_context = NULL; - } - } + /* We keep the previous context alive until we retire the following + * request. This ensures that any the context object is still pinned + * for any residual writes the HW makes into it on the context switch + * into the next object following the breadcrumb. Otherwise, we may + * retire the context too early. + */ + request->previous_context = engine->last_context; + engine->last_context = request->ctx; - if (dev_priv->guc.execbuf_client) - i915_guc_submit(dev_priv->guc.execbuf_client, request); + if (i915.enable_guc_submission) + i915_guc_submit(request); else execlists_context_queue(request); return 0; } -int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request) -{ - /* - * The first call merely notes the reserve request and is common for - * all back ends. The subsequent localised _begin() call actually - * ensures that the reservation is available. Without the begin, if - * the request creator immediately submitted the request without - * adding any commands to it then there might not actually be - * sufficient room for the submission commands. - */ - intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST); - - return intel_ring_begin(request, 0); -} - /** * execlists_submission() - submit a batchbuffer for execution, Execlists style - * @dev: DRM device. - * @file: DRM file. - * @ring: Engine Command Streamer to submit to. - * @ctx: Context to employ for this submission. + * @params: execbuffer call parameters. * @args: execbuffer call arguments. * @vmas: list of vmas. - * @batch_obj: the batchbuffer to submit. - * @exec_start: batchbuffer start virtual address pointer. - * @dispatch_flags: translated execbuffer call flags. * * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts * away the submission details of the execbuffer ioctl call. @@ -810,7 +823,7 @@ int intel_execlists_submission(struct i915_execbuffer_params *params, { struct drm_device *dev = params->dev; struct intel_engine_cs *engine = params->engine; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_ringbuffer *ringbuf = params->ctx->engine[engine->id].ringbuf; u64 exec_start; int instp_mode; @@ -881,28 +894,18 @@ int intel_execlists_submission(struct i915_execbuffer_params *params, return 0; } -void intel_execlists_retire_requests(struct intel_engine_cs *engine) +void intel_execlists_cancel_requests(struct intel_engine_cs *engine) { struct drm_i915_gem_request *req, *tmp; - struct list_head retired_list; + LIST_HEAD(cancel_list); - WARN_ON(!mutex_is_locked(&engine->dev->struct_mutex)); - if (list_empty(&engine->execlist_retired_req_list)) - return; + WARN_ON(!mutex_is_locked(&engine->i915->drm.struct_mutex)); - INIT_LIST_HEAD(&retired_list); spin_lock_bh(&engine->execlist_lock); - list_replace_init(&engine->execlist_retired_req_list, &retired_list); + list_replace_init(&engine->execlist_queue, &cancel_list); spin_unlock_bh(&engine->execlist_lock); - list_for_each_entry_safe(req, tmp, &retired_list, execlist_link) { - struct intel_context *ctx = req->ctx; - struct drm_i915_gem_object *ctx_obj = - ctx->engine[engine->id].state; - - if (ctx_obj && (ctx != req->i915->kernel_context)) - intel_lr_context_unpin(ctx, engine); - + list_for_each_entry_safe(req, tmp, &cancel_list, execlist_link) { list_del(&req->execlist_link); i915_gem_request_unreference(req); } @@ -910,7 +913,7 @@ void intel_execlists_retire_requests(struct intel_engine_cs *engine) void intel_logical_ring_stop(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; int ret; if (!intel_engine_initialized(engine)) @@ -923,7 +926,10 @@ void intel_logical_ring_stop(struct intel_engine_cs *engine) /* TODO: Is this correct with Execlists enabled? */ I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING)); - if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) { + if (intel_wait_for_register(dev_priv, + RING_MI_MODE(engine->mmio_base), + MODE_IDLE, MODE_IDLE, + 1000)) { DRM_ERROR("%s :timed out trying to stop ring\n", engine->name); return; } @@ -946,25 +952,26 @@ int logical_ring_flush_all_caches(struct drm_i915_gem_request *req) return 0; } -static int intel_lr_context_do_pin(struct intel_context *ctx, - struct intel_engine_cs *engine) +static int intel_lr_context_pin(struct i915_gem_context *ctx, + struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state; - struct intel_ringbuffer *ringbuf = ctx->engine[engine->id].ringbuf; + struct drm_i915_private *dev_priv = ctx->i915; + struct intel_context *ce = &ctx->engine[engine->id]; void *vaddr; u32 *lrc_reg_state; int ret; - WARN_ON(!mutex_is_locked(&engine->dev->struct_mutex)); + lockdep_assert_held(&ctx->i915->drm.struct_mutex); - ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, - PIN_OFFSET_BIAS | GUC_WOPCM_TOP); + if (ce->pin_count++) + return 0; + + ret = i915_gem_obj_ggtt_pin(ce->state, GEN8_LR_CONTEXT_ALIGN, + PIN_OFFSET_BIAS | GUC_WOPCM_TOP); if (ret) - return ret; + goto err; - vaddr = i915_gem_object_pin_map(ctx_obj); + vaddr = i915_gem_object_pin_map(ce->state); if (IS_ERR(vaddr)) { ret = PTR_ERR(vaddr); goto unpin_ctx_obj; @@ -972,65 +979,54 @@ static int intel_lr_context_do_pin(struct intel_context *ctx, lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE; - ret = intel_pin_and_map_ringbuffer_obj(engine->dev, ringbuf); + ret = intel_pin_and_map_ringbuffer_obj(dev_priv, ce->ringbuf); if (ret) goto unpin_map; - ctx->engine[engine->id].lrc_vma = i915_gem_obj_to_ggtt(ctx_obj); + i915_gem_context_reference(ctx); + ce->lrc_vma = i915_gem_obj_to_ggtt(ce->state); intel_lr_context_descriptor_update(ctx, engine); - lrc_reg_state[CTX_RING_BUFFER_START+1] = ringbuf->vma->node.start; - ctx->engine[engine->id].lrc_reg_state = lrc_reg_state; - ctx_obj->dirty = true; + + lrc_reg_state[CTX_RING_BUFFER_START+1] = ce->ringbuf->vma->node.start; + ce->lrc_reg_state = lrc_reg_state; + ce->state->dirty = true; /* Invalidate GuC TLB. */ if (i915.enable_guc_submission) I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); - return ret; + return 0; unpin_map: - i915_gem_object_unpin_map(ctx_obj); + i915_gem_object_unpin_map(ce->state); unpin_ctx_obj: - i915_gem_object_ggtt_unpin(ctx_obj); - + i915_gem_object_ggtt_unpin(ce->state); +err: + ce->pin_count = 0; return ret; } -static int intel_lr_context_pin(struct intel_context *ctx, - struct intel_engine_cs *engine) +void intel_lr_context_unpin(struct i915_gem_context *ctx, + struct intel_engine_cs *engine) { - int ret = 0; + struct intel_context *ce = &ctx->engine[engine->id]; - if (ctx->engine[engine->id].pin_count++ == 0) { - ret = intel_lr_context_do_pin(ctx, engine); - if (ret) - goto reset_pin_count; + lockdep_assert_held(&ctx->i915->drm.struct_mutex); + GEM_BUG_ON(ce->pin_count == 0); - i915_gem_context_reference(ctx); - } - return ret; + if (--ce->pin_count) + return; -reset_pin_count: - ctx->engine[engine->id].pin_count = 0; - return ret; -} + intel_unpin_ringbuffer_obj(ce->ringbuf); -void intel_lr_context_unpin(struct intel_context *ctx, - struct intel_engine_cs *engine) -{ - struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state; + i915_gem_object_unpin_map(ce->state); + i915_gem_object_ggtt_unpin(ce->state); - WARN_ON(!mutex_is_locked(&ctx->i915->dev->struct_mutex)); - if (--ctx->engine[engine->id].pin_count == 0) { - i915_gem_object_unpin_map(ctx_obj); - intel_unpin_ringbuffer_obj(ctx->engine[engine->id].ringbuf); - i915_gem_object_ggtt_unpin(ctx_obj); - ctx->engine[engine->id].lrc_vma = NULL; - ctx->engine[engine->id].lrc_desc = 0; - ctx->engine[engine->id].lrc_reg_state = NULL; + ce->lrc_vma = NULL; + ce->lrc_desc = 0; + ce->lrc_reg_state = NULL; - i915_gem_context_unreference(ctx); - } + i915_gem_context_unreference(ctx); } static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req) @@ -1038,9 +1034,7 @@ static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req) int ret, i; struct intel_engine_cs *engine = req->engine; struct intel_ringbuffer *ringbuf = req->ringbuf; - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_workarounds *w = &dev_priv->workarounds; + struct i915_workarounds *w = &req->i915->workarounds; if (w->count == 0) return 0; @@ -1103,7 +1097,7 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, uint32_t *const batch, uint32_t index) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES); /* @@ -1165,7 +1159,7 @@ static inline int wa_ctx_end(struct i915_wa_ctx_bb *wa_ctx, /** * gen8_init_indirectctx_bb() - initialize indirect ctx batch with WA * - * @ring: only applicable for RCS + * @engine: only applicable for RCS * @wa_ctx: structure representing wa_ctx * offset: specifies start of the batch, should be cache-aligned. This is updated * with the offset value received as input. @@ -1202,7 +1196,7 @@ static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine, wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE); /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */ - if (IS_BROADWELL(engine->dev)) { + if (IS_BROADWELL(engine->i915)) { int rc = gen8_emit_flush_coherentl3_wa(engine, batch, index); if (rc < 0) return rc; @@ -1239,7 +1233,7 @@ static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine, /** * gen8_init_perctx_bb() - initialize per ctx batch with WA * - * @ring: only applicable for RCS + * @engine: only applicable for RCS * @wa_ctx: structure representing wa_ctx * offset: specifies start of the batch, should be cache-aligned. * size: size of the batch in DWORDS but HW expects in terms of cachelines @@ -1274,13 +1268,12 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine, uint32_t *offset) { int ret; - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); /* WaDisableCtxRestoreArbitration:skl,bxt */ - if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || - IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) || + IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE); /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */ @@ -1305,6 +1298,31 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine, wa_ctx_emit(batch, index, 0); wa_ctx_emit(batch, index, 0); } + + /* WaMediaPoolStateCmdInWABB:bxt */ + if (HAS_POOLED_EU(engine->i915)) { + /* + * EU pool configuration is setup along with golden context + * during context initialization. This value depends on + * device type (2x6 or 3x6) and needs to be updated based + * on which subslice is disabled especially for 2x6 + * devices, however it is safe to load default + * configuration of 3x6 device instead of masking off + * corresponding bits because HW ignores bits of a disabled + * subslice and drops down to appropriate config. Please + * see render_state_setup() in i915_gem_render_state.c for + * possible configurations, to avoid duplication they are + * not shown here again. + */ + u32 eu_pool_config = 0x00777000; + wa_ctx_emit(batch, index, GEN9_MEDIA_POOL_STATE); + wa_ctx_emit(batch, index, GEN9_MEDIA_POOL_ENABLE); + wa_ctx_emit(batch, index, eu_pool_config); + wa_ctx_emit(batch, index, 0); + wa_ctx_emit(batch, index, 0); + wa_ctx_emit(batch, index, 0); + } + /* Pad to end of cacheline */ while (index % CACHELINE_DWORDS) wa_ctx_emit(batch, index, MI_NOOP); @@ -1317,12 +1335,11 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine, uint32_t *const batch, uint32_t *offset) { - struct drm_device *dev = engine->dev; uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS); /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */ - if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || - IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_B0) || + IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) { wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1)); wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0); wa_ctx_emit(batch, index, @@ -1331,7 +1348,7 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine, } /* WaClearTdlStateAckDirtyBits:bxt */ - if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) { + if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_B0)) { wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(4)); wa_ctx_emit_reg(batch, index, GEN8_STATE_ACK); @@ -1350,8 +1367,8 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine, } /* WaDisableCtxRestoreArbitration:skl,bxt */ - if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || - IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_D0) || + IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE); wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END); @@ -1363,11 +1380,13 @@ static int lrc_setup_wa_ctx_obj(struct intel_engine_cs *engine, u32 size) { int ret; - engine->wa_ctx.obj = i915_gem_alloc_object(engine->dev, - PAGE_ALIGN(size)); - if (!engine->wa_ctx.obj) { + engine->wa_ctx.obj = i915_gem_object_create(&engine->i915->drm, + PAGE_ALIGN(size)); + if (IS_ERR(engine->wa_ctx.obj)) { DRM_DEBUG_DRIVER("alloc LRC WA ctx backing obj failed.\n"); - return -ENOMEM; + ret = PTR_ERR(engine->wa_ctx.obj); + engine->wa_ctx.obj = NULL; + return ret; } ret = i915_gem_obj_ggtt_pin(engine->wa_ctx.obj, PAGE_SIZE, 0); @@ -1401,9 +1420,9 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine) WARN_ON(engine->id != RCS); /* update this when WA for higher Gen are added */ - if (INTEL_INFO(engine->dev)->gen > 9) { + if (INTEL_GEN(engine->i915) > 9) { DRM_ERROR("WA batch buffer is not initialized for Gen%d\n", - INTEL_INFO(engine->dev)->gen); + INTEL_GEN(engine->i915)); return 0; } @@ -1423,7 +1442,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine) batch = kmap_atomic(page); offset = 0; - if (INTEL_INFO(engine->dev)->gen == 8) { + if (IS_GEN8(engine->i915)) { ret = gen8_init_indirectctx_bb(engine, &wa_ctx->indirect_ctx, batch, @@ -1437,7 +1456,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine) &offset); if (ret) goto out; - } else if (INTEL_INFO(engine->dev)->gen == 9) { + } else if (IS_GEN9(engine->i915)) { ret = gen9_init_indirectctx_bb(engine, &wa_ctx->indirect_ctx, batch, @@ -1463,7 +1482,7 @@ out: static void lrc_init_hws(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; I915_WRITE(RING_HWS_PGA(engine->mmio_base), (u32)engine->status_page.gfx_addr); @@ -1472,8 +1491,7 @@ static void lrc_init_hws(struct intel_engine_cs *engine) static int gen8_init_common_ring(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; unsigned int next_context_status_buffer_hw; lrc_init_hws(engine); @@ -1520,8 +1538,7 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine) static int gen8_init_render_ring(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; int ret; ret = gen8_init_common_ring(engine); @@ -1598,7 +1615,7 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req, if (req->ctx->ppgtt && (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings)) { if (!USES_FULL_48BIT_PPGTT(req->i915) && - !intel_vgpu_active(req->i915->dev)) { + !intel_vgpu_active(req->i915)) { ret = intel_logical_ring_emit_pdps(req); if (ret) return ret; @@ -1624,38 +1641,18 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req, return 0; } -static bool gen8_logical_ring_get_irq(struct intel_engine_cs *engine) +static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; - - if (WARN_ON(!intel_irqs_enabled(dev_priv))) - return false; - - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (engine->irq_refcount++ == 0) { - I915_WRITE_IMR(engine, - ~(engine->irq_enable_mask | engine->irq_keep_mask)); - POSTING_READ(RING_IMR(engine->mmio_base)); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - - return true; + struct drm_i915_private *dev_priv = engine->i915; + I915_WRITE_IMR(engine, + ~(engine->irq_enable_mask | engine->irq_keep_mask)); + POSTING_READ_FW(RING_IMR(engine->mmio_base)); } -static void gen8_logical_ring_put_irq(struct intel_engine_cs *engine) +static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; - - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--engine->irq_refcount == 0) { - I915_WRITE_IMR(engine, ~engine->irq_keep_mask); - POSTING_READ(RING_IMR(engine->mmio_base)); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + struct drm_i915_private *dev_priv = engine->i915; + I915_WRITE_IMR(engine, ~engine->irq_keep_mask); } static int gen8_emit_flush(struct drm_i915_gem_request *request, @@ -1664,8 +1661,7 @@ static int gen8_emit_flush(struct drm_i915_gem_request *request, { struct intel_ringbuffer *ringbuf = request->ringbuf; struct intel_engine_cs *engine = ringbuf->engine; - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = request->i915; uint32_t cmd; int ret; @@ -1734,7 +1730,7 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request, * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL * pipe control. */ - if (IS_GEN9(engine->dev)) + if (IS_GEN9(request->i915)) vf_flush_wa = true; /* WaForGAMHang:kbl */ @@ -1793,16 +1789,6 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request, return 0; } -static u32 gen8_get_seqno(struct intel_engine_cs *engine) -{ - return intel_read_status_page(engine, I915_GEM_HWS_INDEX); -} - -static void gen8_set_seqno(struct intel_engine_cs *engine, u32 seqno) -{ - intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); -} - static void bxt_a_seqno_barrier(struct intel_engine_cs *engine) { /* @@ -1818,14 +1804,6 @@ static void bxt_a_seqno_barrier(struct intel_engine_cs *engine) intel_flush_status_page(engine, I915_GEM_HWS_INDEX); } -static void bxt_a_set_seqno(struct intel_engine_cs *engine, u32 seqno) -{ - intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); - - /* See bxt_a_get_seqno() explaining the reason for the clflush. */ - intel_flush_status_page(engine, I915_GEM_HWS_INDEX); -} - /* * Reserve space for 2 NOOPs at the end of each request to be * used as a workaround for not being allowed to do lite @@ -1833,11 +1811,6 @@ static void bxt_a_set_seqno(struct intel_engine_cs *engine, u32 seqno) */ #define WA_TAIL_DWORDS 2 -static inline u32 hws_seqno_address(struct intel_engine_cs *engine) -{ - return engine->status_page.gfx_addr + I915_GEM_HWS_INDEX_ADDR; -} - static int gen8_emit_request(struct drm_i915_gem_request *request) { struct intel_ringbuffer *ringbuf = request->ringbuf; @@ -1853,10 +1826,10 @@ static int gen8_emit_request(struct drm_i915_gem_request *request) intel_logical_ring_emit(ringbuf, (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW); intel_logical_ring_emit(ringbuf, - hws_seqno_address(request->engine) | + intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT); intel_logical_ring_emit(ringbuf, 0); - intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request)); + intel_logical_ring_emit(ringbuf, request->seqno); intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT); intel_logical_ring_emit(ringbuf, MI_NOOP); return intel_logical_ring_advance_and_submit(request); @@ -1883,7 +1856,8 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request) (PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL | PIPE_CONTROL_QW_WRITE)); - intel_logical_ring_emit(ringbuf, hws_seqno_address(request->engine)); + intel_logical_ring_emit(ringbuf, + intel_hws_seqno_address(request->engine)); intel_logical_ring_emit(ringbuf, 0); intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request)); /* We're thrashing one dword of HWS. */ @@ -1945,7 +1919,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req) /** * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer * - * @ring: Engine Command Streamer. + * @engine: Engine Command Streamer. * */ void intel_logical_ring_cleanup(struct intel_engine_cs *engine) @@ -1962,7 +1936,7 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine) if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->irq_tasklet.state))) tasklet_kill(&engine->irq_tasklet); - dev_priv = engine->dev->dev_private; + dev_priv = engine->i915; if (engine->buffer) { intel_logical_ring_stop(engine); @@ -1975,36 +1949,34 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine) i915_cmd_parser_fini_ring(engine); i915_gem_batch_pool_fini(&engine->batch_pool); + intel_engine_fini_breadcrumbs(engine); + if (engine->status_page.obj) { i915_gem_object_unpin_map(engine->status_page.obj); engine->status_page.obj = NULL; } + intel_lr_context_unpin(dev_priv->kernel_context, engine); engine->idle_lite_restore_wa = 0; engine->disable_lite_restore_wa = false; engine->ctx_desc_template = 0; lrc_destroy_wa_ctx_obj(engine); - engine->dev = NULL; + engine->i915 = NULL; } static void -logical_ring_default_vfuncs(struct drm_device *dev, - struct intel_engine_cs *engine) +logical_ring_default_vfuncs(struct intel_engine_cs *engine) { /* Default vfuncs which can be overriden by each engine. */ engine->init_hw = gen8_init_common_ring; engine->emit_request = gen8_emit_request; engine->emit_flush = gen8_emit_flush; - engine->irq_get = gen8_logical_ring_get_irq; - engine->irq_put = gen8_logical_ring_put_irq; + engine->irq_enable = gen8_logical_ring_enable_irq; + engine->irq_disable = gen8_logical_ring_disable_irq; engine->emit_bb_start = gen8_emit_bb_start; - engine->get_seqno = gen8_get_seqno; - engine->set_seqno = gen8_set_seqno; - if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) engine->irq_seqno_barrier = bxt_a_seqno_barrier; - engine->set_seqno = bxt_a_set_seqno; - } } static inline void @@ -2033,60 +2005,28 @@ lrc_setup_hws(struct intel_engine_cs *engine, } static int -logical_ring_init(struct drm_device *dev, struct intel_engine_cs *engine) +logical_ring_init(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_context *dctx = dev_priv->kernel_context; - enum forcewake_domains fw_domains; + struct i915_gem_context *dctx = engine->i915->kernel_context; int ret; - /* Intentionally left blank. */ - engine->buffer = NULL; - - engine->dev = dev; - INIT_LIST_HEAD(&engine->active_list); - INIT_LIST_HEAD(&engine->request_list); - i915_gem_batch_pool_init(dev, &engine->batch_pool); - init_waitqueue_head(&engine->irq_queue); - - INIT_LIST_HEAD(&engine->buffers); - INIT_LIST_HEAD(&engine->execlist_queue); - INIT_LIST_HEAD(&engine->execlist_retired_req_list); - spin_lock_init(&engine->execlist_lock); - - tasklet_init(&engine->irq_tasklet, - intel_lrc_irq_handler, (unsigned long)engine); - - logical_ring_init_platform_invariants(engine); - - fw_domains = intel_uncore_forcewake_for_reg(dev_priv, - RING_ELSP(engine), - FW_REG_WRITE); - - fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, - RING_CONTEXT_STATUS_PTR(engine), - FW_REG_READ | FW_REG_WRITE); - - fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, - RING_CONTEXT_STATUS_BUF_BASE(engine), - FW_REG_READ); - - engine->fw_domains = fw_domains; + ret = intel_engine_init_breadcrumbs(engine); + if (ret) + goto error; ret = i915_cmd_parser_init_ring(engine); if (ret) goto error; - ret = intel_lr_context_deferred_alloc(dctx, engine); + ret = execlists_context_deferred_alloc(dctx, engine); if (ret) goto error; /* As this is the default context, always pin it */ - ret = intel_lr_context_do_pin(dctx, engine); + ret = intel_lr_context_pin(dctx, engine); if (ret) { - DRM_ERROR( - "Failed to pin and map ringbuffer %s: %d\n", - engine->name, ret); + DRM_ERROR("Failed to pin context for %s: %d\n", + engine->name, ret); goto error; } @@ -2104,26 +2044,16 @@ error: return ret; } -static int logical_render_ring_init(struct drm_device *dev) +static int logical_render_ring_init(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *engine = &dev_priv->engine[RCS]; + struct drm_i915_private *dev_priv = engine->i915; int ret; - engine->name = "render ring"; - engine->id = RCS; - engine->exec_id = I915_EXEC_RENDER; - engine->guc_id = GUC_RENDER_ENGINE; - engine->mmio_base = RENDER_RING_BASE; - - logical_ring_default_irqs(engine, GEN8_RCS_IRQ_SHIFT); - if (HAS_L3_DPF(dev)) + if (HAS_L3_DPF(dev_priv)) engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; - logical_ring_default_vfuncs(dev, engine); - /* Override some for render ring. */ - if (INTEL_INFO(dev)->gen >= 9) + if (INTEL_GEN(dev_priv) >= 9) engine->init_hw = gen9_init_render_ring; else engine->init_hw = gen8_init_render_ring; @@ -2132,9 +2062,7 @@ static int logical_render_ring_init(struct drm_device *dev) engine->emit_flush = gen8_emit_flush_render; engine->emit_request = gen8_emit_request_render; - engine->dev = dev; - - ret = intel_init_pipe_control(engine); + ret = intel_init_pipe_control(engine, 4096); if (ret) return ret; @@ -2149,7 +2077,7 @@ static int logical_render_ring_init(struct drm_device *dev) ret); } - ret = logical_ring_init(dev, engine); + ret = logical_ring_init(engine); if (ret) { lrc_destroy_wa_ctx_obj(engine); } @@ -2157,133 +2085,164 @@ static int logical_render_ring_init(struct drm_device *dev) return ret; } -static int logical_bsd_ring_init(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *engine = &dev_priv->engine[VCS]; - - engine->name = "bsd ring"; - engine->id = VCS; - engine->exec_id = I915_EXEC_BSD; - engine->guc_id = GUC_VIDEO_ENGINE; - engine->mmio_base = GEN6_BSD_RING_BASE; - - logical_ring_default_irqs(engine, GEN8_VCS1_IRQ_SHIFT); - logical_ring_default_vfuncs(dev, engine); - - return logical_ring_init(dev, engine); -} +static const struct logical_ring_info { + const char *name; + unsigned exec_id; + unsigned guc_id; + u32 mmio_base; + unsigned irq_shift; + int (*init)(struct intel_engine_cs *engine); +} logical_rings[] = { + [RCS] = { + .name = "render ring", + .exec_id = I915_EXEC_RENDER, + .guc_id = GUC_RENDER_ENGINE, + .mmio_base = RENDER_RING_BASE, + .irq_shift = GEN8_RCS_IRQ_SHIFT, + .init = logical_render_ring_init, + }, + [BCS] = { + .name = "blitter ring", + .exec_id = I915_EXEC_BLT, + .guc_id = GUC_BLITTER_ENGINE, + .mmio_base = BLT_RING_BASE, + .irq_shift = GEN8_BCS_IRQ_SHIFT, + .init = logical_ring_init, + }, + [VCS] = { + .name = "bsd ring", + .exec_id = I915_EXEC_BSD, + .guc_id = GUC_VIDEO_ENGINE, + .mmio_base = GEN6_BSD_RING_BASE, + .irq_shift = GEN8_VCS1_IRQ_SHIFT, + .init = logical_ring_init, + }, + [VCS2] = { + .name = "bsd2 ring", + .exec_id = I915_EXEC_BSD, + .guc_id = GUC_VIDEO_ENGINE2, + .mmio_base = GEN8_BSD2_RING_BASE, + .irq_shift = GEN8_VCS2_IRQ_SHIFT, + .init = logical_ring_init, + }, + [VECS] = { + .name = "video enhancement ring", + .exec_id = I915_EXEC_VEBOX, + .guc_id = GUC_VIDEOENHANCE_ENGINE, + .mmio_base = VEBOX_RING_BASE, + .irq_shift = GEN8_VECS_IRQ_SHIFT, + .init = logical_ring_init, + }, +}; -static int logical_bsd2_ring_init(struct drm_device *dev) +static struct intel_engine_cs * +logical_ring_setup(struct drm_i915_private *dev_priv, enum intel_engine_id id) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *engine = &dev_priv->engine[VCS2]; + const struct logical_ring_info *info = &logical_rings[id]; + struct intel_engine_cs *engine = &dev_priv->engine[id]; + enum forcewake_domains fw_domains; - engine->name = "bsd2 ring"; - engine->id = VCS2; - engine->exec_id = I915_EXEC_BSD; - engine->guc_id = GUC_VIDEO_ENGINE2; - engine->mmio_base = GEN8_BSD2_RING_BASE; + engine->id = id; + engine->name = info->name; + engine->exec_id = info->exec_id; + engine->guc_id = info->guc_id; + engine->mmio_base = info->mmio_base; - logical_ring_default_irqs(engine, GEN8_VCS2_IRQ_SHIFT); - logical_ring_default_vfuncs(dev, engine); + engine->i915 = dev_priv; - return logical_ring_init(dev, engine); -} + /* Intentionally left blank. */ + engine->buffer = NULL; -static int logical_blt_ring_init(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *engine = &dev_priv->engine[BCS]; + fw_domains = intel_uncore_forcewake_for_reg(dev_priv, + RING_ELSP(engine), + FW_REG_WRITE); - engine->name = "blitter ring"; - engine->id = BCS; - engine->exec_id = I915_EXEC_BLT; - engine->guc_id = GUC_BLITTER_ENGINE; - engine->mmio_base = BLT_RING_BASE; + fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, + RING_CONTEXT_STATUS_PTR(engine), + FW_REG_READ | FW_REG_WRITE); - logical_ring_default_irqs(engine, GEN8_BCS_IRQ_SHIFT); - logical_ring_default_vfuncs(dev, engine); + fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, + RING_CONTEXT_STATUS_BUF_BASE(engine), + FW_REG_READ); - return logical_ring_init(dev, engine); -} + engine->fw_domains = fw_domains; -static int logical_vebox_ring_init(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *engine = &dev_priv->engine[VECS]; + INIT_LIST_HEAD(&engine->active_list); + INIT_LIST_HEAD(&engine->request_list); + INIT_LIST_HEAD(&engine->buffers); + INIT_LIST_HEAD(&engine->execlist_queue); + spin_lock_init(&engine->execlist_lock); - engine->name = "video enhancement ring"; - engine->id = VECS; - engine->exec_id = I915_EXEC_VEBOX; - engine->guc_id = GUC_VIDEOENHANCE_ENGINE; - engine->mmio_base = VEBOX_RING_BASE; + tasklet_init(&engine->irq_tasklet, + intel_lrc_irq_handler, (unsigned long)engine); - logical_ring_default_irqs(engine, GEN8_VECS_IRQ_SHIFT); - logical_ring_default_vfuncs(dev, engine); + logical_ring_init_platform_invariants(engine); + logical_ring_default_vfuncs(engine); + logical_ring_default_irqs(engine, info->irq_shift); + + intel_engine_init_hangcheck(engine); + i915_gem_batch_pool_init(&dev_priv->drm, &engine->batch_pool); - return logical_ring_init(dev, engine); + return engine; } /** * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers * @dev: DRM device. * - * This function inits the engines for an Execlists submission style (the equivalent in the - * legacy ringbuffer submission world would be i915_gem_init_engines). It does it only for - * those engines that are present in the hardware. + * This function inits the engines for an Execlists submission style (the + * equivalent in the legacy ringbuffer submission world would be + * i915_gem_init_engines). It does it only for those engines that are present in + * the hardware. * * Return: non-zero if the initialization failed. */ int intel_logical_rings_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); + unsigned int mask = 0; + unsigned int i; int ret; - ret = logical_render_ring_init(dev); - if (ret) - return ret; + WARN_ON(INTEL_INFO(dev_priv)->ring_mask & + GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES)); - if (HAS_BSD(dev)) { - ret = logical_bsd_ring_init(dev); - if (ret) - goto cleanup_render_ring; - } + for (i = 0; i < ARRAY_SIZE(logical_rings); i++) { + if (!HAS_ENGINE(dev_priv, i)) + continue; - if (HAS_BLT(dev)) { - ret = logical_blt_ring_init(dev); - if (ret) - goto cleanup_bsd_ring; - } + if (!logical_rings[i].init) + continue; - if (HAS_VEBOX(dev)) { - ret = logical_vebox_ring_init(dev); + ret = logical_rings[i].init(logical_ring_setup(dev_priv, i)); if (ret) - goto cleanup_blt_ring; + goto cleanup; + + mask |= ENGINE_MASK(i); } - if (HAS_BSD2(dev)) { - ret = logical_bsd2_ring_init(dev); - if (ret) - goto cleanup_vebox_ring; + /* + * Catch failures to update logical_rings table when the new engines + * are added to the driver by a warning and disabling the forgotten + * engines. + */ + if (WARN_ON(mask != INTEL_INFO(dev_priv)->ring_mask)) { + struct intel_device_info *info = + (struct intel_device_info *)&dev_priv->info; + info->ring_mask = mask; } return 0; -cleanup_vebox_ring: - intel_logical_ring_cleanup(&dev_priv->engine[VECS]); -cleanup_blt_ring: - intel_logical_ring_cleanup(&dev_priv->engine[BCS]); -cleanup_bsd_ring: - intel_logical_ring_cleanup(&dev_priv->engine[VCS]); -cleanup_render_ring: - intel_logical_ring_cleanup(&dev_priv->engine[RCS]); +cleanup: + for (i = 0; i < I915_NUM_ENGINES; i++) + intel_logical_ring_cleanup(&dev_priv->engine[i]); return ret; } static u32 -make_rpcs(struct drm_device *dev) +make_rpcs(struct drm_i915_private *dev_priv) { u32 rpcs = 0; @@ -2291,7 +2250,7 @@ make_rpcs(struct drm_device *dev) * No explicit RPCS request is needed to ensure full * slice/subslice/EU enablement prior to Gen9. */ - if (INTEL_INFO(dev)->gen < 9) + if (INTEL_GEN(dev_priv) < 9) return 0; /* @@ -2300,24 +2259,24 @@ make_rpcs(struct drm_device *dev) * must make an explicit request through RPCS for full * enablement. */ - if (INTEL_INFO(dev)->has_slice_pg) { + if (INTEL_INFO(dev_priv)->has_slice_pg) { rpcs |= GEN8_RPCS_S_CNT_ENABLE; - rpcs |= INTEL_INFO(dev)->slice_total << + rpcs |= INTEL_INFO(dev_priv)->slice_total << GEN8_RPCS_S_CNT_SHIFT; rpcs |= GEN8_RPCS_ENABLE; } - if (INTEL_INFO(dev)->has_subslice_pg) { + if (INTEL_INFO(dev_priv)->has_subslice_pg) { rpcs |= GEN8_RPCS_SS_CNT_ENABLE; - rpcs |= INTEL_INFO(dev)->subslice_per_slice << + rpcs |= INTEL_INFO(dev_priv)->subslice_per_slice << GEN8_RPCS_SS_CNT_SHIFT; rpcs |= GEN8_RPCS_ENABLE; } - if (INTEL_INFO(dev)->has_eu_pg) { - rpcs |= INTEL_INFO(dev)->eu_per_subslice << + if (INTEL_INFO(dev_priv)->has_eu_pg) { + rpcs |= INTEL_INFO(dev_priv)->eu_per_subslice << GEN8_RPCS_EU_MIN_SHIFT; - rpcs |= INTEL_INFO(dev)->eu_per_subslice << + rpcs |= INTEL_INFO(dev_priv)->eu_per_subslice << GEN8_RPCS_EU_MAX_SHIFT; rpcs |= GEN8_RPCS_ENABLE; } @@ -2329,9 +2288,9 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine) { u32 indirect_ctx_offset; - switch (INTEL_INFO(engine->dev)->gen) { + switch (INTEL_GEN(engine->i915)) { default: - MISSING_CASE(INTEL_INFO(engine->dev)->gen); + MISSING_CASE(INTEL_GEN(engine->i915)); /* fall through */ case 9: indirect_ctx_offset = @@ -2347,13 +2306,12 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine) } static int -populate_lr_context(struct intel_context *ctx, +populate_lr_context(struct i915_gem_context *ctx, struct drm_i915_gem_object *ctx_obj, struct intel_engine_cs *engine, struct intel_ringbuffer *ringbuf) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = ctx->i915; struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; void *vaddr; u32 *reg_state; @@ -2391,7 +2349,7 @@ populate_lr_context(struct intel_context *ctx, RING_CONTEXT_CONTROL(engine), _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | - (HAS_RESOURCE_STREAMER(dev) ? + (HAS_RESOURCE_STREAMER(dev_priv) ? CTX_CTRL_RS_CTX_ENABLE : 0))); ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(engine->mmio_base), 0); @@ -2480,7 +2438,7 @@ populate_lr_context(struct intel_context *ctx, if (engine->id == RCS) { reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1); ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, - make_rpcs(dev)); + make_rpcs(dev_priv)); } i915_gem_object_unpin_map(ctx_obj); @@ -2489,39 +2447,8 @@ populate_lr_context(struct intel_context *ctx, } /** - * intel_lr_context_free() - free the LRC specific bits of a context - * @ctx: the LR context to free. - * - * The real context freeing is done in i915_gem_context_free: this only - * takes care of the bits that are LRC related: the per-engine backing - * objects and the logical ringbuffer. - */ -void intel_lr_context_free(struct intel_context *ctx) -{ - int i; - - for (i = I915_NUM_ENGINES; --i >= 0; ) { - struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf; - struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state; - - if (!ctx_obj) - continue; - - if (ctx == ctx->i915->kernel_context) { - intel_unpin_ringbuffer_obj(ringbuf); - i915_gem_object_ggtt_unpin(ctx_obj); - i915_gem_object_unpin_map(ctx_obj); - } - - WARN_ON(ctx->engine[i].pin_count); - intel_ringbuffer_free(ringbuf); - drm_gem_object_unreference(&ctx_obj->base); - } -} - -/** * intel_lr_context_size() - return the size of the context for an engine - * @ring: which engine to find the context size for + * @engine: which engine to find the context size for * * Each engine may require a different amount of space for a context image, * so when allocating (or copying) an image, this function can be used to @@ -2537,11 +2464,11 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine) { int ret = 0; - WARN_ON(INTEL_INFO(engine->dev)->gen < 8); + WARN_ON(INTEL_GEN(engine->i915) < 8); switch (engine->id) { case RCS: - if (INTEL_INFO(engine->dev)->gen >= 9) + if (INTEL_GEN(engine->i915) >= 9) ret = GEN9_LR_CONTEXT_RENDER_SIZE; else ret = GEN8_LR_CONTEXT_RENDER_SIZE; @@ -2558,9 +2485,9 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine) } /** - * intel_lr_context_deferred_alloc() - create the LRC specific bits of a context + * execlists_context_deferred_alloc() - create the LRC specific bits of a context * @ctx: LR context to create. - * @ring: engine to be used with the context. + * @engine: engine to be used with the context. * * This function can be called more than once, with different engines, if we plan * to use the context with them. The context backing objects and the ringbuffers @@ -2570,31 +2497,29 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine) * * Return: non-zero on error. */ - -int intel_lr_context_deferred_alloc(struct intel_context *ctx, - struct intel_engine_cs *engine) +static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, + struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; struct drm_i915_gem_object *ctx_obj; + struct intel_context *ce = &ctx->engine[engine->id]; uint32_t context_size; struct intel_ringbuffer *ringbuf; int ret; - WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL); - WARN_ON(ctx->engine[engine->id].state); + WARN_ON(ce->state); context_size = round_up(intel_lr_context_size(engine), 4096); /* One extra page as the sharing data between driver and GuC */ context_size += PAGE_SIZE * LRC_PPHWSP_PN; - ctx_obj = i915_gem_alloc_object(dev, context_size); - if (!ctx_obj) { + ctx_obj = i915_gem_object_create(&ctx->i915->drm, context_size); + if (IS_ERR(ctx_obj)) { DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n"); - return -ENOMEM; + return PTR_ERR(ctx_obj); } - ringbuf = intel_engine_create_ringbuffer(engine, 4 * PAGE_SIZE); + ringbuf = intel_engine_create_ringbuffer(engine, ctx->ring_size); if (IS_ERR(ringbuf)) { ret = PTR_ERR(ringbuf); goto error_deref_obj; @@ -2606,48 +2531,29 @@ int intel_lr_context_deferred_alloc(struct intel_context *ctx, goto error_ringbuf; } - ctx->engine[engine->id].ringbuf = ringbuf; - ctx->engine[engine->id].state = ctx_obj; + ce->ringbuf = ringbuf; + ce->state = ctx_obj; + ce->initialised = engine->init_context == NULL; - if (ctx != ctx->i915->kernel_context && engine->init_context) { - struct drm_i915_gem_request *req; - - req = i915_gem_request_alloc(engine, ctx); - if (IS_ERR(req)) { - ret = PTR_ERR(req); - DRM_ERROR("ring create req: %d\n", ret); - goto error_ringbuf; - } - - ret = engine->init_context(req); - i915_add_request_no_flush(req); - if (ret) { - DRM_ERROR("ring init context: %d\n", - ret); - goto error_ringbuf; - } - } return 0; error_ringbuf: intel_ringbuffer_free(ringbuf); error_deref_obj: drm_gem_object_unreference(&ctx_obj->base); - ctx->engine[engine->id].ringbuf = NULL; - ctx->engine[engine->id].state = NULL; + ce->ringbuf = NULL; + ce->state = NULL; return ret; } void intel_lr_context_reset(struct drm_i915_private *dev_priv, - struct intel_context *ctx) + struct i915_gem_context *ctx) { struct intel_engine_cs *engine; for_each_engine(engine, dev_priv) { - struct drm_i915_gem_object *ctx_obj = - ctx->engine[engine->id].state; - struct intel_ringbuffer *ringbuf = - ctx->engine[engine->id].ringbuf; + struct intel_context *ce = &ctx->engine[engine->id]; + struct drm_i915_gem_object *ctx_obj = ce->state; void *vaddr; uint32_t *reg_state; @@ -2666,7 +2572,7 @@ void intel_lr_context_reset(struct drm_i915_private *dev_priv, i915_gem_object_unpin_map(ctx_obj); - ringbuf->head = 0; - ringbuf->tail = 0; + ce->ringbuf->head = 0; + ce->ringbuf->tail = 0; } } diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h index 60a7385bc..2b8255c19 100644 --- a/drivers/gpu/drm/i915/intel_lrc.h +++ b/drivers/gpu/drm/i915/intel_lrc.h @@ -57,6 +57,11 @@ #define GEN8_CSB_READ_PTR(csb_status) \ (((csb_status) & GEN8_CSB_READ_PTR_MASK) >> 8) +enum { + INTEL_CONTEXT_SCHEDULE_IN = 0, + INTEL_CONTEXT_SCHEDULE_OUT, +}; + /* Logical Rings */ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request); int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request); @@ -99,30 +104,27 @@ static inline void intel_logical_ring_emit_reg(struct intel_ringbuffer *ringbuf, #define LRC_PPHWSP_PN (LRC_GUCSHR_PN + 1) #define LRC_STATE_PN (LRC_PPHWSP_PN + 1) -void intel_lr_context_free(struct intel_context *ctx); +struct i915_gem_context; + uint32_t intel_lr_context_size(struct intel_engine_cs *engine); -int intel_lr_context_deferred_alloc(struct intel_context *ctx, - struct intel_engine_cs *engine); -void intel_lr_context_unpin(struct intel_context *ctx, +void intel_lr_context_unpin(struct i915_gem_context *ctx, struct intel_engine_cs *engine); struct drm_i915_private; void intel_lr_context_reset(struct drm_i915_private *dev_priv, - struct intel_context *ctx); -uint64_t intel_lr_context_descriptor(struct intel_context *ctx, + struct i915_gem_context *ctx); +uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx, struct intel_engine_cs *engine); -u32 intel_execlists_ctx_id(struct intel_context *ctx, - struct intel_engine_cs *engine); - /* Execlists */ -int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists); +int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, + int enable_execlists); struct i915_execbuffer_params; int intel_execlists_submission(struct i915_execbuffer_params *params, struct drm_i915_gem_execbuffer2 *args, struct list_head *vmas); -void intel_execlists_retire_requests(struct intel_engine_cs *engine); +void intel_execlists_cancel_requests(struct intel_engine_cs *engine); #endif /* _INTEL_LRC_H_ */ diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c index 96281e628..495504704 100644 --- a/drivers/gpu/drm/i915/intel_lvds.c +++ b/drivers/gpu/drm/i915/intel_lvds.c @@ -72,7 +72,7 @@ static bool intel_lvds_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); enum intel_display_power_domain power_domain; u32 tmp; @@ -106,7 +106,7 @@ static void intel_lvds_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); u32 tmp, flags = 0; @@ -140,7 +140,7 @@ static void intel_pre_enable_lvds(struct intel_encoder *encoder) { struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; int pipe = crtc->pipe; @@ -184,13 +184,13 @@ static void intel_pre_enable_lvds(struct intel_encoder *encoder) * panels behave in the two modes. For now, let's just maintain the * value we got from the BIOS. */ - temp &= ~LVDS_A3_POWER_MASK; - temp |= lvds_encoder->a3_power; + temp &= ~LVDS_A3_POWER_MASK; + temp |= lvds_encoder->a3_power; /* Set the dithering flag on LVDS as needed, note that there is no * special lvds dither control bit on pch-split platforms, dithering is * only controlled through the PIPECONF reg. */ - if (INTEL_INFO(dev)->gen == 4) { + if (IS_GEN4(dev_priv)) { /* Bspec wording suggests that LVDS port dithering only exists * for 18bpp panels. */ if (crtc->config->dither && crtc->config->pipe_bpp == 18) @@ -216,7 +216,7 @@ static void intel_enable_lvds(struct intel_encoder *encoder) struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); struct intel_connector *intel_connector = &lvds_encoder->attached_connector->base; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t ctl_reg, stat_reg; if (HAS_PCH_SPLIT(dev)) { @@ -231,7 +231,7 @@ static void intel_enable_lvds(struct intel_encoder *encoder) I915_WRITE(ctl_reg, I915_READ(ctl_reg) | POWER_TARGET_ON); POSTING_READ(lvds_encoder->reg); - if (wait_for((I915_READ(stat_reg) & PP_ON) != 0, 1000)) + if (intel_wait_for_register(dev_priv, stat_reg, PP_ON, PP_ON, 1000)) DRM_ERROR("timed out waiting for panel to power on\n"); intel_panel_enable_backlight(intel_connector); @@ -241,7 +241,7 @@ static void intel_disable_lvds(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base); - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t ctl_reg, stat_reg; if (HAS_PCH_SPLIT(dev)) { @@ -253,7 +253,7 @@ static void intel_disable_lvds(struct intel_encoder *encoder) } I915_WRITE(ctl_reg, I915_READ(ctl_reg) & ~POWER_TARGET_ON); - if (wait_for((I915_READ(stat_reg) & PP_ON) == 0, 1000)) + if (intel_wait_for_register(dev_priv, stat_reg, PP_ON, 0, 1000)) DRM_ERROR("timed out waiting for panel to power off\n"); I915_WRITE(lvds_encoder->reg, I915_READ(lvds_encoder->reg) & ~LVDS_PORT_EN); @@ -442,7 +442,7 @@ static int intel_lid_notify(struct notifier_block *nb, unsigned long val, container_of(nb, struct intel_lvds_connector, lid_notifier); struct drm_connector *connector = &lvds_connector->base.base; struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (dev->switch_power_state != DRM_SWITCH_POWER_ON) return NOTIFY_OK; @@ -547,7 +547,6 @@ static int intel_lvds_set_property(struct drm_connector *connector, static const struct drm_connector_helper_funcs intel_lvds_connector_helper_funcs = { .get_modes = intel_lvds_get_modes, .mode_valid = intel_lvds_mode_valid, - .best_encoder = intel_best_encoder, }; static const struct drm_connector_funcs intel_lvds_connector_funcs = { @@ -556,6 +555,8 @@ static const struct drm_connector_funcs intel_lvds_connector_funcs = { .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_lvds_set_property, .atomic_get_property = intel_connector_atomic_get_property, + .late_register = intel_connector_register, + .early_unregister = intel_connector_unregister, .destroy = intel_lvds_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, @@ -810,27 +811,29 @@ static const struct dmi_system_id intel_dual_link_lvds[] = { { } /* terminating entry */ }; -bool intel_is_dual_link_lvds(struct drm_device *dev) +struct intel_encoder *intel_get_lvds_encoder(struct drm_device *dev) { - struct intel_encoder *encoder; - struct intel_lvds_encoder *lvds_encoder; + struct intel_encoder *intel_encoder; - for_each_intel_encoder(dev, encoder) { - if (encoder->type == INTEL_OUTPUT_LVDS) { - lvds_encoder = to_lvds_encoder(&encoder->base); + for_each_intel_encoder(dev, intel_encoder) + if (intel_encoder->type == INTEL_OUTPUT_LVDS) + return intel_encoder; - return lvds_encoder->is_dual_link; - } - } + return NULL; +} - return false; +bool intel_is_dual_link_lvds(struct drm_device *dev) +{ + struct intel_encoder *encoder = intel_get_lvds_encoder(dev); + + return encoder && to_lvds_encoder(&encoder->base)->is_dual_link; } static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder) { struct drm_device *dev = lvds_encoder->base.base.dev; unsigned int val; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* use the module option value if specified */ if (i915.lvds_channel_mode > 0) @@ -880,7 +883,7 @@ static bool intel_lvds_supported(struct drm_device *dev) */ void intel_lvds_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_lvds_encoder *lvds_encoder; struct intel_encoder *intel_encoder; struct intel_lvds_connector *lvds_connector; @@ -978,7 +981,7 @@ void intel_lvds_init(struct drm_device *dev) DRM_MODE_CONNECTOR_LVDS); drm_encoder_init(dev, &intel_encoder->base, &intel_lvds_enc_funcs, - DRM_MODE_ENCODER_LVDS, NULL); + DRM_MODE_ENCODER_LVDS, "LVDS"); intel_encoder->enable = intel_enable_lvds; intel_encoder->pre_enable = intel_pre_enable_lvds; @@ -992,7 +995,6 @@ void intel_lvds_init(struct drm_device *dev) intel_encoder->get_hw_state = intel_lvds_get_hw_state; intel_encoder->get_config = intel_lvds_get_config; intel_connector->get_hw_state = intel_connector_get_hw_state; - intel_connector->unregister = intel_connector_unregister; intel_connector_attach_encoder(intel_connector, intel_encoder); intel_encoder->type = INTEL_OUTPUT_LVDS; @@ -1119,6 +1121,7 @@ out: mutex_unlock(&dev->mode_config.mutex); intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode); + intel_panel_setup_backlight(connector, INVALID_PIPE); lvds_encoder->is_dual_link = compute_is_dual_link_lvds(lvds_encoder); DRM_DEBUG_KMS("detected %s-link lvds configuration\n", @@ -1131,9 +1134,6 @@ out: DRM_DEBUG_KMS("lid notifier registration failed\n"); lvds_connector->lid_notifier.notifier_call = NULL; } - drm_connector_register(connector); - - intel_panel_setup_backlight(connector, INVALID_PIPE); return; diff --git a/drivers/gpu/drm/i915/intel_mocs.c b/drivers/gpu/drm/i915/intel_mocs.c index 6ba4bf7f2..927825f5b 100644 --- a/drivers/gpu/drm/i915/intel_mocs.c +++ b/drivers/gpu/drm/i915/intel_mocs.c @@ -66,9 +66,10 @@ struct drm_i915_mocs_table { #define L3_WB 3 /* Target cache */ -#define ELLC 0 -#define LLC 1 -#define LLC_ELLC 2 +#define LE_TC_PAGETABLE 0 +#define LE_TC_LLC 1 +#define LE_TC_LLC_ELLC 2 +#define LE_TC_LLC_ELLC_ALT 3 /* * MOCS tables @@ -96,34 +97,67 @@ struct drm_i915_mocs_table { * end. */ static const struct drm_i915_mocs_entry skylake_mocs_table[] = { - /* { 0x00000009, 0x0010 } */ - { (LE_CACHEABILITY(LE_UC) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(0) | - LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), - (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC)) }, - /* { 0x00000038, 0x0030 } */ - { (LE_CACHEABILITY(LE_PAGETABLE) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) | - LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), - (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) }, - /* { 0x0000003b, 0x0030 } */ - { (LE_CACHEABILITY(LE_WB) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) | - LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), - (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) } + { /* 0x00000009 */ + .control_value = LE_CACHEABILITY(LE_UC) | + LE_TGT_CACHE(LE_TC_LLC_ELLC) | + LE_LRUM(0) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | + LE_PFM(0) | LE_SCF(0), + + /* 0x0010 */ + .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC), + }, + { + /* 0x00000038 */ + .control_value = LE_CACHEABILITY(LE_PAGETABLE) | + LE_TGT_CACHE(LE_TC_LLC_ELLC) | + LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | + LE_PFM(0) | LE_SCF(0), + /* 0x0030 */ + .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB), + }, + { + /* 0x0000003b */ + .control_value = LE_CACHEABILITY(LE_WB) | + LE_TGT_CACHE(LE_TC_LLC_ELLC) | + LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | + LE_PFM(0) | LE_SCF(0), + /* 0x0030 */ + .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB), + }, }; /* NOTE: the LE_TGT_CACHE is not used on Broxton */ static const struct drm_i915_mocs_entry broxton_mocs_table[] = { - /* { 0x00000009, 0x0010 } */ - { (LE_CACHEABILITY(LE_UC) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(0) | - LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), - (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC)) }, - /* { 0x00000038, 0x0030 } */ - { (LE_CACHEABILITY(LE_PAGETABLE) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) | - LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), - (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) }, - /* { 0x0000003b, 0x0030 } */ - { (LE_CACHEABILITY(LE_WB) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) | - LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)), - (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) } + { + /* 0x00000009 */ + .control_value = LE_CACHEABILITY(LE_UC) | + LE_TGT_CACHE(LE_TC_LLC_ELLC) | + LE_LRUM(0) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | + LE_PFM(0) | LE_SCF(0), + + /* 0x0010 */ + .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC), + }, + { + /* 0x00000038 */ + .control_value = LE_CACHEABILITY(LE_PAGETABLE) | + LE_TGT_CACHE(LE_TC_LLC_ELLC) | + LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | + LE_PFM(0) | LE_SCF(0), + + /* 0x0030 */ + .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB), + }, + { + /* 0x00000039 */ + .control_value = LE_CACHEABILITY(LE_UC) | + LE_TGT_CACHE(LE_TC_LLC_ELLC) | + LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | + LE_PFM(0) | LE_SCF(0), + + /* 0x0030 */ + .l3cc_value = L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB), + }, }; /** @@ -156,6 +190,16 @@ static bool get_mocs_settings(struct drm_i915_private *dev_priv, "Platform that should have a MOCS table does not.\n"); } + /* WaDisableSkipCaching:skl,bxt,kbl */ + if (IS_GEN9(dev_priv)) { + int i; + + for (i = 0; i < table->size; i++) + if (WARN_ON(table->table[i].l3cc_value & + (L3_ESC(1) | L3_SCC(0x7)))) + return false; + } + return result; } @@ -189,7 +233,7 @@ static i915_reg_t mocs_register(enum intel_engine_id ring, int index) */ int intel_mocs_init_engine(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = to_i915(engine->dev); + struct drm_i915_private *dev_priv = engine->i915; struct drm_i915_mocs_table table; unsigned int index; diff --git a/drivers/gpu/drm/i915/intel_modes.c b/drivers/gpu/drm/i915/intel_modes.c index 38a4c8ce7..f2584d0a0 100644 --- a/drivers/gpu/drm/i915/intel_modes.c +++ b/drivers/gpu/drm/i915/intel_modes.c @@ -82,7 +82,7 @@ void intel_attach_force_audio_property(struct drm_connector *connector) { struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_property *prop; prop = dev_priv->force_audio_property; @@ -109,7 +109,7 @@ void intel_attach_broadcast_rgb_property(struct drm_connector *connector) { struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_property *prop; prop = dev_priv->broadcast_rgb_property; diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c index 72842aafd..7acbbbf97 100644 --- a/drivers/gpu/drm/i915/intel_opregion.c +++ b/drivers/gpu/drm/i915/intel_opregion.c @@ -232,18 +232,36 @@ struct opregion_asle_ext { #define SWSCI_SBCB_POST_VBE_PM SWSCI_FUNCTION_CODE(SWSCI_SBCB, 19) #define SWSCI_SBCB_ENABLE_DISABLE_AUDIO SWSCI_FUNCTION_CODE(SWSCI_SBCB, 21) -#define ACPI_OTHER_OUTPUT (0<<8) -#define ACPI_VGA_OUTPUT (1<<8) -#define ACPI_TV_OUTPUT (2<<8) -#define ACPI_DIGITAL_OUTPUT (3<<8) -#define ACPI_LVDS_OUTPUT (4<<8) +/* + * ACPI Specification, Revision 5.0, Appendix B.3.2 _DOD (Enumerate All Devices + * Attached to the Display Adapter). + */ +#define ACPI_DISPLAY_INDEX_SHIFT 0 +#define ACPI_DISPLAY_INDEX_MASK (0xf << 0) +#define ACPI_DISPLAY_PORT_ATTACHMENT_SHIFT 4 +#define ACPI_DISPLAY_PORT_ATTACHMENT_MASK (0xf << 4) +#define ACPI_DISPLAY_TYPE_SHIFT 8 +#define ACPI_DISPLAY_TYPE_MASK (0xf << 8) +#define ACPI_DISPLAY_TYPE_OTHER (0 << 8) +#define ACPI_DISPLAY_TYPE_VGA (1 << 8) +#define ACPI_DISPLAY_TYPE_TV (2 << 8) +#define ACPI_DISPLAY_TYPE_EXTERNAL_DIGITAL (3 << 8) +#define ACPI_DISPLAY_TYPE_INTERNAL_DIGITAL (4 << 8) +#define ACPI_VENDOR_SPECIFIC_SHIFT 12 +#define ACPI_VENDOR_SPECIFIC_MASK (0xf << 12) +#define ACPI_BIOS_CAN_DETECT (1 << 16) +#define ACPI_DEPENDS_ON_VGA (1 << 17) +#define ACPI_PIPE_ID_SHIFT 18 +#define ACPI_PIPE_ID_MASK (7 << 18) +#define ACPI_DEVICE_ID_SCHEME (1 << 31) #define MAX_DSLP 1500 -static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) +static int swsci(struct drm_i915_private *dev_priv, + u32 function, u32 parm, u32 *parm_out) { - struct drm_i915_private *dev_priv = dev->dev_private; struct opregion_swsci *swsci = dev_priv->opregion.swsci; + struct pci_dev *pdev = dev_priv->drm.pdev; u32 main_function, sub_function, scic; u16 swsci_val; u32 dslp; @@ -293,16 +311,16 @@ static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) swsci->scic = scic; /* Ensure SCI event is selected and event trigger is cleared. */ - pci_read_config_word(dev->pdev, SWSCI, &swsci_val); + pci_read_config_word(pdev, SWSCI, &swsci_val); if (!(swsci_val & SWSCI_SCISEL) || (swsci_val & SWSCI_GSSCIE)) { swsci_val |= SWSCI_SCISEL; swsci_val &= ~SWSCI_GSSCIE; - pci_write_config_word(dev->pdev, SWSCI, swsci_val); + pci_write_config_word(pdev, SWSCI, swsci_val); } /* Use event trigger to tell bios to check the mail. */ swsci_val |= SWSCI_GSSCIE; - pci_write_config_word(dev->pdev, SWSCI, swsci_val); + pci_write_config_word(pdev, SWSCI, swsci_val); /* Poll for the result. */ #define C (((scic = swsci->scic) & SWSCI_SCIC_INDICATOR) == 0) @@ -336,13 +354,13 @@ static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out) int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, bool enable) { - struct drm_device *dev = intel_encoder->base.dev; + struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev); u32 parm = 0; u32 type = 0; u32 port; /* don't care about old stuff for now */ - if (!HAS_DDI(dev)) + if (!HAS_DDI(dev_priv)) return 0; if (intel_encoder->type == INTEL_OUTPUT_DSI) @@ -365,7 +383,7 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, type = DISPLAY_TYPE_CRT; break; case INTEL_OUTPUT_UNKNOWN: - case INTEL_OUTPUT_DISPLAYPORT: + case INTEL_OUTPUT_DP: case INTEL_OUTPUT_HDMI: case INTEL_OUTPUT_DP_MST: type = DISPLAY_TYPE_EXTERNAL_FLAT_PANEL; @@ -382,7 +400,7 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, parm |= type << (16 + port * 3); - return swsci(dev, SWSCI_SBCB_DISPLAY_POWER_STATE, parm, NULL); + return swsci(dev_priv, SWSCI_SBCB_DISPLAY_POWER_STATE, parm, NULL); } static const struct { @@ -396,27 +414,28 @@ static const struct { { PCI_D3cold, 0x04 }, }; -int intel_opregion_notify_adapter(struct drm_device *dev, pci_power_t state) +int intel_opregion_notify_adapter(struct drm_i915_private *dev_priv, + pci_power_t state) { int i; - if (!HAS_DDI(dev)) + if (!HAS_DDI(dev_priv)) return 0; for (i = 0; i < ARRAY_SIZE(power_state_map); i++) { if (state == power_state_map[i].pci_power_state) - return swsci(dev, SWSCI_SBCB_ADAPTER_POWER_STATE, + return swsci(dev_priv, SWSCI_SBCB_ADAPTER_POWER_STATE, power_state_map[i].parm, NULL); } return -EINVAL; } -static u32 asle_set_backlight(struct drm_device *dev, u32 bclp) +static u32 asle_set_backlight(struct drm_i915_private *dev_priv, u32 bclp) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_connector *connector; struct opregion_asle *asle = dev_priv->opregion.asle; + struct drm_device *dev = &dev_priv->drm; DRM_DEBUG_DRIVER("bclp = 0x%08x\n", bclp); @@ -449,7 +468,7 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp) return 0; } -static u32 asle_set_als_illum(struct drm_device *dev, u32 alsi) +static u32 asle_set_als_illum(struct drm_i915_private *dev_priv, u32 alsi) { /* alsi is the current ALS reading in lux. 0 indicates below sensor range, 0xffff indicates above sensor range. 1-0xfffe are valid */ @@ -457,13 +476,13 @@ static u32 asle_set_als_illum(struct drm_device *dev, u32 alsi) return ASLC_ALS_ILLUM_FAILED; } -static u32 asle_set_pwm_freq(struct drm_device *dev, u32 pfmb) +static u32 asle_set_pwm_freq(struct drm_i915_private *dev_priv, u32 pfmb) { DRM_DEBUG_DRIVER("PWM freq is not supported\n"); return ASLC_PWM_FREQ_FAILED; } -static u32 asle_set_pfit(struct drm_device *dev, u32 pfit) +static u32 asle_set_pfit(struct drm_i915_private *dev_priv, u32 pfit) { /* Panel fitting is currently controlled by the X code, so this is a noop until modesetting support works fully */ @@ -471,13 +490,13 @@ static u32 asle_set_pfit(struct drm_device *dev, u32 pfit) return ASLC_PFIT_FAILED; } -static u32 asle_set_supported_rotation_angles(struct drm_device *dev, u32 srot) +static u32 asle_set_supported_rotation_angles(struct drm_i915_private *dev_priv, u32 srot) { DRM_DEBUG_DRIVER("SROT is not supported\n"); return ASLC_ROTATION_ANGLES_FAILED; } -static u32 asle_set_button_array(struct drm_device *dev, u32 iuer) +static u32 asle_set_button_array(struct drm_i915_private *dev_priv, u32 iuer) { if (!iuer) DRM_DEBUG_DRIVER("Button array event is not supported (nothing)\n"); @@ -495,7 +514,7 @@ static u32 asle_set_button_array(struct drm_device *dev, u32 iuer) return ASLC_BUTTON_ARRAY_FAILED; } -static u32 asle_set_convertible(struct drm_device *dev, u32 iuer) +static u32 asle_set_convertible(struct drm_i915_private *dev_priv, u32 iuer) { if (iuer & ASLE_IUER_CONVERTIBLE) DRM_DEBUG_DRIVER("Convertible is not supported (clamshell)\n"); @@ -505,7 +524,7 @@ static u32 asle_set_convertible(struct drm_device *dev, u32 iuer) return ASLC_CONVERTIBLE_FAILED; } -static u32 asle_set_docking(struct drm_device *dev, u32 iuer) +static u32 asle_set_docking(struct drm_i915_private *dev_priv, u32 iuer) { if (iuer & ASLE_IUER_DOCKING) DRM_DEBUG_DRIVER("Docking is not supported (docked)\n"); @@ -515,7 +534,7 @@ static u32 asle_set_docking(struct drm_device *dev, u32 iuer) return ASLC_DOCKING_FAILED; } -static u32 asle_isct_state(struct drm_device *dev) +static u32 asle_isct_state(struct drm_i915_private *dev_priv) { DRM_DEBUG_DRIVER("ISCT is not supported\n"); return ASLC_ISCT_STATE_FAILED; @@ -527,7 +546,6 @@ static void asle_work(struct work_struct *work) container_of(work, struct intel_opregion, asle_work); struct drm_i915_private *dev_priv = container_of(opregion, struct drm_i915_private, opregion); - struct drm_device *dev = dev_priv->dev; struct opregion_asle *asle = dev_priv->opregion.asle; u32 aslc_stat = 0; u32 aslc_req; @@ -544,40 +562,38 @@ static void asle_work(struct work_struct *work) } if (aslc_req & ASLC_SET_ALS_ILLUM) - aslc_stat |= asle_set_als_illum(dev, asle->alsi); + aslc_stat |= asle_set_als_illum(dev_priv, asle->alsi); if (aslc_req & ASLC_SET_BACKLIGHT) - aslc_stat |= asle_set_backlight(dev, asle->bclp); + aslc_stat |= asle_set_backlight(dev_priv, asle->bclp); if (aslc_req & ASLC_SET_PFIT) - aslc_stat |= asle_set_pfit(dev, asle->pfit); + aslc_stat |= asle_set_pfit(dev_priv, asle->pfit); if (aslc_req & ASLC_SET_PWM_FREQ) - aslc_stat |= asle_set_pwm_freq(dev, asle->pfmb); + aslc_stat |= asle_set_pwm_freq(dev_priv, asle->pfmb); if (aslc_req & ASLC_SUPPORTED_ROTATION_ANGLES) - aslc_stat |= asle_set_supported_rotation_angles(dev, + aslc_stat |= asle_set_supported_rotation_angles(dev_priv, asle->srot); if (aslc_req & ASLC_BUTTON_ARRAY) - aslc_stat |= asle_set_button_array(dev, asle->iuer); + aslc_stat |= asle_set_button_array(dev_priv, asle->iuer); if (aslc_req & ASLC_CONVERTIBLE_INDICATOR) - aslc_stat |= asle_set_convertible(dev, asle->iuer); + aslc_stat |= asle_set_convertible(dev_priv, asle->iuer); if (aslc_req & ASLC_DOCKING_INDICATOR) - aslc_stat |= asle_set_docking(dev, asle->iuer); + aslc_stat |= asle_set_docking(dev_priv, asle->iuer); if (aslc_req & ASLC_ISCT_STATE_CHANGE) - aslc_stat |= asle_isct_state(dev); + aslc_stat |= asle_isct_state(dev_priv); asle->aslc = aslc_stat; } -void intel_opregion_asle_intr(struct drm_device *dev) +void intel_opregion_asle_intr(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - if (dev_priv->opregion.asle) schedule_work(&dev_priv->opregion.asle_work); } @@ -658,10 +674,51 @@ static void set_did(struct intel_opregion *opregion, int i, u32 val) } } -static void intel_didl_outputs(struct drm_device *dev) +static u32 acpi_display_type(struct drm_connector *connector) +{ + u32 display_type; + + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_VGA: + case DRM_MODE_CONNECTOR_DVIA: + display_type = ACPI_DISPLAY_TYPE_VGA; + break; + case DRM_MODE_CONNECTOR_Composite: + case DRM_MODE_CONNECTOR_SVIDEO: + case DRM_MODE_CONNECTOR_Component: + case DRM_MODE_CONNECTOR_9PinDIN: + case DRM_MODE_CONNECTOR_TV: + display_type = ACPI_DISPLAY_TYPE_TV; + break; + case DRM_MODE_CONNECTOR_DVII: + case DRM_MODE_CONNECTOR_DVID: + case DRM_MODE_CONNECTOR_DisplayPort: + case DRM_MODE_CONNECTOR_HDMIA: + case DRM_MODE_CONNECTOR_HDMIB: + display_type = ACPI_DISPLAY_TYPE_EXTERNAL_DIGITAL; + break; + case DRM_MODE_CONNECTOR_LVDS: + case DRM_MODE_CONNECTOR_eDP: + case DRM_MODE_CONNECTOR_DSI: + display_type = ACPI_DISPLAY_TYPE_INTERNAL_DIGITAL; + break; + case DRM_MODE_CONNECTOR_Unknown: + case DRM_MODE_CONNECTOR_VIRTUAL: + display_type = ACPI_DISPLAY_TYPE_OTHER; + break; + default: + MISSING_CASE(connector->connector_type); + display_type = ACPI_DISPLAY_TYPE_OTHER; + break; + } + + return display_type; +} + +static void intel_didl_outputs(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_opregion *opregion = &dev_priv->opregion; + struct pci_dev *pdev = dev_priv->drm.pdev; struct drm_connector *connector; acpi_handle handle; struct acpi_device *acpi_dev, *acpi_cdev, *acpi_video_bus = NULL; @@ -670,7 +727,7 @@ static void intel_didl_outputs(struct drm_device *dev) u32 temp, max_outputs; int i = 0; - handle = ACPI_HANDLE(&dev->pdev->dev); + handle = ACPI_HANDLE(&pdev->dev); if (!handle || acpi_bus_get_device(handle, &acpi_dev)) return; @@ -725,45 +782,25 @@ end: blind_set: i = 0; - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { - int output_type = ACPI_OTHER_OUTPUT; + list_for_each_entry(connector, + &dev_priv->drm.mode_config.connector_list, head) { + int display_type = acpi_display_type(connector); + if (i >= max_outputs) { DRM_DEBUG_KMS("More than %u outputs in connector list\n", max_outputs); return; } - switch (connector->connector_type) { - case DRM_MODE_CONNECTOR_VGA: - case DRM_MODE_CONNECTOR_DVIA: - output_type = ACPI_VGA_OUTPUT; - break; - case DRM_MODE_CONNECTOR_Composite: - case DRM_MODE_CONNECTOR_SVIDEO: - case DRM_MODE_CONNECTOR_Component: - case DRM_MODE_CONNECTOR_9PinDIN: - output_type = ACPI_TV_OUTPUT; - break; - case DRM_MODE_CONNECTOR_DVII: - case DRM_MODE_CONNECTOR_DVID: - case DRM_MODE_CONNECTOR_DisplayPort: - case DRM_MODE_CONNECTOR_HDMIA: - case DRM_MODE_CONNECTOR_HDMIB: - output_type = ACPI_DIGITAL_OUTPUT; - break; - case DRM_MODE_CONNECTOR_LVDS: - output_type = ACPI_LVDS_OUTPUT; - break; - } + temp = get_did(opregion, i); - set_did(opregion, i, temp | (1 << 31) | output_type | i); + set_did(opregion, i, temp | (1 << 31) | display_type | i); i++; } goto end; } -static void intel_setup_cadls(struct drm_device *dev) +static void intel_setup_cadls(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_opregion *opregion = &dev_priv->opregion; int i = 0; u32 disp_id; @@ -780,17 +817,16 @@ static void intel_setup_cadls(struct drm_device *dev) } while (++i < 8 && disp_id != 0); } -void intel_opregion_init(struct drm_device *dev) +void intel_opregion_register(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_opregion *opregion = &dev_priv->opregion; if (!opregion->header) return; if (opregion->acpi) { - intel_didl_outputs(dev); - intel_setup_cadls(dev); + intel_didl_outputs(dev_priv); + intel_setup_cadls(dev_priv); /* Notify BIOS we are ready to handle ACPI video ext notifs. * Right now, all the events are handled by the ACPI video module. @@ -808,9 +844,8 @@ void intel_opregion_init(struct drm_device *dev) } } -void intel_opregion_fini(struct drm_device *dev) +void intel_opregion_unregister(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_opregion *opregion = &dev_priv->opregion; if (!opregion->header) @@ -842,9 +877,8 @@ void intel_opregion_fini(struct drm_device *dev) opregion->lid_state = NULL; } -static void swsci_setup(struct drm_device *dev) +static void swsci_setup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_opregion *opregion = &dev_priv->opregion; bool requested_callbacks = false; u32 tmp; @@ -854,7 +888,7 @@ static void swsci_setup(struct drm_device *dev) opregion->swsci_sbcb_sub_functions = 1; /* We use GBDA to ask for supported GBDA calls. */ - if (swsci(dev, SWSCI_GBDA_SUPPORTED_CALLS, 0, &tmp) == 0) { + if (swsci(dev_priv, SWSCI_GBDA_SUPPORTED_CALLS, 0, &tmp) == 0) { /* make the bits match the sub-function codes */ tmp <<= 1; opregion->swsci_gbda_sub_functions |= tmp; @@ -865,7 +899,7 @@ static void swsci_setup(struct drm_device *dev) * must not call interfaces that are not specifically requested by the * bios. */ - if (swsci(dev, SWSCI_GBDA_REQUESTED_CALLBACKS, 0, &tmp) == 0) { + if (swsci(dev_priv, SWSCI_GBDA_REQUESTED_CALLBACKS, 0, &tmp) == 0) { /* here, the bits already match sub-function codes */ opregion->swsci_sbcb_sub_functions |= tmp; requested_callbacks = true; @@ -876,7 +910,7 @@ static void swsci_setup(struct drm_device *dev) * the callback is _requested_. But we still can't call interfaces that * are not requested. */ - if (swsci(dev, SWSCI_SBCB_SUPPORTED_CALLBACKS, 0, &tmp) == 0) { + if (swsci(dev_priv, SWSCI_SBCB_SUPPORTED_CALLBACKS, 0, &tmp) == 0) { /* make the bits match the sub-function codes */ u32 low = tmp & 0x7ff; u32 high = tmp & ~0xfff; /* bit 11 is reserved */ @@ -918,10 +952,10 @@ static const struct dmi_system_id intel_no_opregion_vbt[] = { { } }; -int intel_opregion_setup(struct drm_device *dev) +int intel_opregion_setup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_opregion *opregion = &dev_priv->opregion; + struct pci_dev *pdev = dev_priv->drm.pdev; u32 asls, mboxes; char buf[sizeof(OPREGION_SIGNATURE)]; int err = 0; @@ -933,7 +967,7 @@ int intel_opregion_setup(struct drm_device *dev) BUILD_BUG_ON(sizeof(struct opregion_asle) != 0x100); BUILD_BUG_ON(sizeof(struct opregion_asle_ext) != 0x400); - pci_read_config_dword(dev->pdev, ASLS, &asls); + pci_read_config_dword(pdev, ASLS, &asls); DRM_DEBUG_DRIVER("graphic opregion physical addr: 0x%x\n", asls); if (asls == 0) { DRM_DEBUG_DRIVER("ACPI OpRegion not supported!\n"); @@ -965,7 +999,7 @@ int intel_opregion_setup(struct drm_device *dev) if (mboxes & MBOX_SWSCI) { DRM_DEBUG_DRIVER("SWSCI supported\n"); opregion->swsci = base + OPREGION_SWSCI_OFFSET; - swsci_setup(dev); + swsci_setup(dev_priv); } if (mboxes & MBOX_ASLE) { @@ -1031,12 +1065,12 @@ static const struct dmi_system_id intel_use_opregion_panel_type[] = { }; int -intel_opregion_get_panel_type(struct drm_device *dev) +intel_opregion_get_panel_type(struct drm_i915_private *dev_priv) { u32 panel_details; int ret; - ret = swsci(dev, SWSCI_GBDA_PANEL_DETAILS, 0x0, &panel_details); + ret = swsci(dev_priv, SWSCI_GBDA_PANEL_DETAILS, 0x0, &panel_details); if (ret) { DRM_DEBUG_KMS("Failed to get panel details from OpRegion (%d)\n", ret); @@ -1071,7 +1105,7 @@ intel_opregion_get_panel_type(struct drm_device *dev) * vswing instead. Low vswing results in some display flickers, so * let's simply ignore the OpRegion panel type on SKL for now. */ - if (IS_SKYLAKE(dev)) { + if (IS_SKYLAKE(dev_priv)) { DRM_DEBUG_KMS("Ignoring OpRegion panel type (%d)\n", ret - 1); return -ENODEV; } diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c index bd38e49f7..3212d8806 100644 --- a/drivers/gpu/drm/i915/intel_overlay.c +++ b/drivers/gpu/drm/i915/intel_overlay.c @@ -168,7 +168,7 @@ struct overlay_registers { }; struct intel_overlay { - struct drm_device *dev; + struct drm_i915_private *i915; struct intel_crtc *crtc; struct drm_i915_gem_object *vid_bo; struct drm_i915_gem_object *old_vid_bo; @@ -190,15 +190,15 @@ struct intel_overlay { static struct overlay_registers __iomem * intel_overlay_map_regs(struct intel_overlay *overlay) { - struct drm_i915_private *dev_priv = to_i915(overlay->dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; + struct drm_i915_private *dev_priv = overlay->i915; struct overlay_registers __iomem *regs; - if (OVERLAY_NEEDS_PHYSICAL(overlay->dev)) + if (OVERLAY_NEEDS_PHYSICAL(dev_priv)) regs = (struct overlay_registers __iomem *)overlay->reg_bo->phys_handle->vaddr; else - regs = io_mapping_map_wc(ggtt->mappable, - i915_gem_obj_ggtt_offset(overlay->reg_bo)); + regs = io_mapping_map_wc(dev_priv->ggtt.mappable, + overlay->flip_addr, + PAGE_SIZE); return regs; } @@ -206,7 +206,7 @@ intel_overlay_map_regs(struct intel_overlay *overlay) static void intel_overlay_unmap_regs(struct intel_overlay *overlay, struct overlay_registers __iomem *regs) { - if (!OVERLAY_NEEDS_PHYSICAL(overlay->dev)) + if (!OVERLAY_NEEDS_PHYSICAL(overlay->i915)) io_mapping_unmap(regs); } @@ -232,14 +232,13 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay, /* overlay needs to be disable in OCMD reg */ static int intel_overlay_on(struct intel_overlay *overlay) { - struct drm_device *dev = overlay->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = overlay->i915; struct intel_engine_cs *engine = &dev_priv->engine[RCS]; struct drm_i915_gem_request *req; int ret; WARN_ON(overlay->active); - WARN_ON(IS_I830(dev) && !(dev_priv->quirks & QUIRK_PIPEA_FORCE)); + WARN_ON(IS_I830(dev_priv) && !(dev_priv->quirks & QUIRK_PIPEA_FORCE)); req = i915_gem_request_alloc(engine, NULL); if (IS_ERR(req)) @@ -266,8 +265,7 @@ static int intel_overlay_on(struct intel_overlay *overlay) static int intel_overlay_continue(struct intel_overlay *overlay, bool load_polyphase_filter) { - struct drm_device *dev = overlay->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = overlay->i915; struct intel_engine_cs *engine = &dev_priv->engine[RCS]; struct drm_i915_gem_request *req; u32 flip_addr = overlay->flip_addr; @@ -335,8 +333,7 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay) /* overlay needs to be disabled in OCMD reg */ static int intel_overlay_off(struct intel_overlay *overlay) { - struct drm_device *dev = overlay->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = overlay->i915; struct intel_engine_cs *engine = &dev_priv->engine[RCS]; struct drm_i915_gem_request *req; u32 flip_addr = overlay->flip_addr; @@ -365,7 +362,7 @@ static int intel_overlay_off(struct intel_overlay *overlay) intel_ring_emit(engine, flip_addr); intel_ring_emit(engine, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); /* turn overlay off */ - if (IS_I830(dev)) { + if (IS_I830(dev_priv)) { /* Workaround: Don't disable the overlay fully, since otherwise * it dies on the next OVERLAY_ON cmd. */ intel_ring_emit(engine, MI_NOOP); @@ -408,12 +405,11 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay) */ static int intel_overlay_release_old_vid(struct intel_overlay *overlay) { - struct drm_device *dev = overlay->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = overlay->i915; struct intel_engine_cs *engine = &dev_priv->engine[RCS]; int ret; - WARN_ON(!mutex_is_locked(&dev->struct_mutex)); + lockdep_assert_held(&dev_priv->drm.struct_mutex); /* Only wait if there is actually an old frame to release to * guarantee forward progress. @@ -537,10 +533,10 @@ static int uv_vsubsampling(u32 format) } } -static u32 calc_swidthsw(struct drm_device *dev, u32 offset, u32 width) +static u32 calc_swidthsw(struct drm_i915_private *dev_priv, u32 offset, u32 width) { u32 mask, shift, ret; - if (IS_GEN2(dev)) { + if (IS_GEN2(dev_priv)) { mask = 0x1f; shift = 5; } else { @@ -548,7 +544,7 @@ static u32 calc_swidthsw(struct drm_device *dev, u32 offset, u32 width) shift = 6; } ret = ((offset + width + mask) >> shift) - (offset >> shift); - if (!IS_GEN2(dev)) + if (!IS_GEN2(dev_priv)) ret <<= 1; ret -= 1; return ret << 2; @@ -741,12 +737,12 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, int ret, tmp_width; struct overlay_registers __iomem *regs; bool scale_changed = false; - struct drm_device *dev = overlay->dev; + struct drm_i915_private *dev_priv = overlay->i915; u32 swidth, swidthsw, sheight, ostride; enum pipe pipe = overlay->crtc->pipe; - WARN_ON(!mutex_is_locked(&dev->struct_mutex)); - WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); + lockdep_assert_held(&dev_priv->drm.struct_mutex); + WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex)); ret = intel_overlay_release_old_vid(overlay); if (ret != 0) @@ -769,7 +765,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, goto out_unpin; } oconfig = OCONF_CC_OUT_8BIT; - if (IS_GEN4(overlay->dev)) + if (IS_GEN4(dev_priv)) oconfig |= OCONF_CSC_MODE_BT709; oconfig |= pipe == 0 ? OCONF_PIPE_A : OCONF_PIPE_B; @@ -796,7 +792,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, tmp_width = params->src_w; swidth = params->src_w; - swidthsw = calc_swidthsw(overlay->dev, params->offset_Y, tmp_width); + swidthsw = calc_swidthsw(dev_priv, params->offset_Y, tmp_width); sheight = params->src_h; iowrite32(i915_gem_obj_ggtt_offset(new_bo) + params->offset_Y, ®s->OBUF_0Y); ostride = params->stride_Y; @@ -806,9 +802,9 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, int uv_vscale = uv_vsubsampling(params->format); u32 tmp_U, tmp_V; swidth |= (params->src_w/uv_hscale) << 16; - tmp_U = calc_swidthsw(overlay->dev, params->offset_U, + tmp_U = calc_swidthsw(dev_priv, params->offset_U, params->src_w/uv_hscale); - tmp_V = calc_swidthsw(overlay->dev, params->offset_V, + tmp_V = calc_swidthsw(dev_priv, params->offset_V, params->src_w/uv_hscale); swidthsw |= max_t(u32, tmp_U, tmp_V) << 16; sheight |= (params->src_h/uv_vscale) << 16; @@ -840,7 +836,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, overlay->old_vid_bo = overlay->vid_bo; overlay->vid_bo = new_bo; - intel_frontbuffer_flip(dev, + intel_frontbuffer_flip(&dev_priv->drm, INTEL_FRONTBUFFER_OVERLAY(pipe)); return 0; @@ -852,12 +848,12 @@ out_unpin: int intel_overlay_switch_off(struct intel_overlay *overlay) { + struct drm_i915_private *dev_priv = overlay->i915; struct overlay_registers __iomem *regs; - struct drm_device *dev = overlay->dev; int ret; - WARN_ON(!mutex_is_locked(&dev->struct_mutex)); - WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); + lockdep_assert_held(&dev_priv->drm.struct_mutex); + WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex)); ret = intel_overlay_recover_from_interrupt(overlay); if (ret != 0) @@ -897,15 +893,14 @@ static int check_overlay_possible_on_crtc(struct intel_overlay *overlay, static void update_pfit_vscale_ratio(struct intel_overlay *overlay) { - struct drm_device *dev = overlay->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = overlay->i915; u32 pfit_control = I915_READ(PFIT_CONTROL); u32 ratio; /* XXX: This is not the same logic as in the xorg driver, but more in * line with the intel documentation for the i965 */ - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { /* on i965 use the PGM reg to read out the autoscaler values */ ratio = I915_READ(PFIT_PGM_RATIOS) >> PFIT_VERT_SCALE_SHIFT_965; } else { @@ -948,7 +943,7 @@ static int check_overlay_scaling(struct put_image_params *rec) return 0; } -static int check_overlay_src(struct drm_device *dev, +static int check_overlay_src(struct drm_i915_private *dev_priv, struct drm_intel_overlay_put_image *rec, struct drm_i915_gem_object *new_bo) { @@ -959,7 +954,7 @@ static int check_overlay_src(struct drm_device *dev, u32 tmp; /* check src dimensions */ - if (IS_845G(dev) || IS_I830(dev)) { + if (IS_845G(dev_priv) || IS_I830(dev_priv)) { if (rec->src_height > IMAGE_MAX_HEIGHT_LEGACY || rec->src_width > IMAGE_MAX_WIDTH_LEGACY) return -EINVAL; @@ -1011,14 +1006,14 @@ static int check_overlay_src(struct drm_device *dev, return -EINVAL; /* stride checking */ - if (IS_I830(dev) || IS_845G(dev)) + if (IS_I830(dev_priv) || IS_845G(dev_priv)) stride_mask = 255; else stride_mask = 63; if (rec->stride_Y & stride_mask || rec->stride_UV & stride_mask) return -EINVAL; - if (IS_GEN4(dev) && rec->stride_Y < 512) + if (IS_GEN4(dev_priv) && rec->stride_Y < 512) return -EINVAL; tmp = (rec->flags & I915_OVERLAY_TYPE_MASK) == I915_OVERLAY_YUV_PLANAR ? @@ -1063,13 +1058,13 @@ static int check_overlay_src(struct drm_device *dev, * Return the pipe currently connected to the panel fitter, * or -1 if the panel fitter is not present or not in use */ -static int intel_panel_fitter_pipe(struct drm_device *dev) +static int intel_panel_fitter_pipe(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 pfit_control; /* i830 doesn't have a panel fitter */ - if (INTEL_INFO(dev)->gen <= 3 && (IS_I830(dev) || !IS_MOBILE(dev))) + if (INTEL_GEN(dev_priv) <= 3 && + (IS_I830(dev_priv) || !IS_MOBILE(dev_priv))) return -1; pfit_control = I915_READ(PFIT_CONTROL); @@ -1079,18 +1074,18 @@ static int intel_panel_fitter_pipe(struct drm_device *dev) return -1; /* 965 can place panel fitter on either pipe */ - if (IS_GEN4(dev)) + if (IS_GEN4(dev_priv)) return (pfit_control >> 29) & 0x3; /* older chips can only use pipe 1 */ return 1; } -int intel_overlay_put_image(struct drm_device *dev, void *data, - struct drm_file *file_priv) +int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) { struct drm_intel_overlay_put_image *put_image_rec = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_overlay *overlay; struct drm_crtc *drmmode_crtc; struct intel_crtc *crtc; @@ -1162,7 +1157,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data, /* line too wide, i.e. one-line-mode */ if (mode->hdisplay > 1024 && - intel_panel_fitter_pipe(dev) == crtc->pipe) { + intel_panel_fitter_pipe(dev_priv) == crtc->pipe) { overlay->pfit_active = true; update_pfit_vscale_ratio(overlay); } else @@ -1196,7 +1191,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data, goto out_unlock; } - ret = check_overlay_src(dev, put_image_rec, new_bo); + ret = check_overlay_src(dev_priv, put_image_rec, new_bo); if (ret != 0) goto out_unlock; params->format = put_image_rec->flags & ~I915_OVERLAY_FLAGS_MASK; @@ -1284,11 +1279,11 @@ static int check_gamma(struct drm_intel_overlay_attrs *attrs) return 0; } -int intel_overlay_attrs(struct drm_device *dev, void *data, - struct drm_file *file_priv) +int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) { struct drm_intel_overlay_attrs *attrs = data; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_overlay *overlay; struct overlay_registers __iomem *regs; int ret; @@ -1309,7 +1304,7 @@ int intel_overlay_attrs(struct drm_device *dev, void *data, attrs->contrast = overlay->contrast; attrs->saturation = overlay->saturation; - if (!IS_GEN2(dev)) { + if (!IS_GEN2(dev_priv)) { attrs->gamma0 = I915_READ(OGAMC0); attrs->gamma1 = I915_READ(OGAMC1); attrs->gamma2 = I915_READ(OGAMC2); @@ -1341,7 +1336,7 @@ int intel_overlay_attrs(struct drm_device *dev, void *data, intel_overlay_unmap_regs(overlay, regs); if (attrs->flags & I915_OVERLAY_UPDATE_GAMMA) { - if (IS_GEN2(dev)) + if (IS_GEN2(dev_priv)) goto out_unlock; if (overlay->active) { @@ -1371,37 +1366,37 @@ out_unlock: return ret; } -void intel_setup_overlay(struct drm_device *dev) +void intel_setup_overlay(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_overlay *overlay; struct drm_i915_gem_object *reg_bo; struct overlay_registers __iomem *regs; int ret; - if (!HAS_OVERLAY(dev)) + if (!HAS_OVERLAY(dev_priv)) return; overlay = kzalloc(sizeof(*overlay), GFP_KERNEL); if (!overlay) return; - mutex_lock(&dev->struct_mutex); + mutex_lock(&dev_priv->drm.struct_mutex); if (WARN_ON(dev_priv->overlay)) goto out_free; - overlay->dev = dev; + overlay->i915 = dev_priv; reg_bo = NULL; - if (!OVERLAY_NEEDS_PHYSICAL(dev)) - reg_bo = i915_gem_object_create_stolen(dev, PAGE_SIZE); - if (reg_bo == NULL) - reg_bo = i915_gem_alloc_object(dev, PAGE_SIZE); + if (!OVERLAY_NEEDS_PHYSICAL(dev_priv)) + reg_bo = i915_gem_object_create_stolen(&dev_priv->drm, + PAGE_SIZE); if (reg_bo == NULL) + reg_bo = i915_gem_object_create(&dev_priv->drm, PAGE_SIZE); + if (IS_ERR(reg_bo)) goto out_free; overlay->reg_bo = reg_bo; - if (OVERLAY_NEEDS_PHYSICAL(dev)) { + if (OVERLAY_NEEDS_PHYSICAL(dev_priv)) { ret = i915_gem_object_attach_phys(reg_bo, PAGE_SIZE); if (ret) { DRM_ERROR("failed to attach phys overlay regs\n"); @@ -1441,25 +1436,23 @@ void intel_setup_overlay(struct drm_device *dev) intel_overlay_unmap_regs(overlay, regs); dev_priv->overlay = overlay; - mutex_unlock(&dev->struct_mutex); + mutex_unlock(&dev_priv->drm.struct_mutex); DRM_INFO("initialized overlay support\n"); return; out_unpin_bo: - if (!OVERLAY_NEEDS_PHYSICAL(dev)) + if (!OVERLAY_NEEDS_PHYSICAL(dev_priv)) i915_gem_object_ggtt_unpin(reg_bo); out_free_bo: drm_gem_object_unreference(®_bo->base); out_free: - mutex_unlock(&dev->struct_mutex); + mutex_unlock(&dev_priv->drm.struct_mutex); kfree(overlay); return; } -void intel_cleanup_overlay(struct drm_device *dev) +void intel_cleanup_overlay(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - if (!dev_priv->overlay) return; @@ -1482,18 +1475,17 @@ struct intel_overlay_error_state { static struct overlay_registers __iomem * intel_overlay_map_regs_atomic(struct intel_overlay *overlay) { - struct drm_i915_private *dev_priv = to_i915(overlay->dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; + struct drm_i915_private *dev_priv = overlay->i915; struct overlay_registers __iomem *regs; - if (OVERLAY_NEEDS_PHYSICAL(overlay->dev)) + if (OVERLAY_NEEDS_PHYSICAL(dev_priv)) /* Cast to make sparse happy, but it's wc memory anyway, so * equivalent to the wc io mapping on X86. */ regs = (struct overlay_registers __iomem *) overlay->reg_bo->phys_handle->vaddr; else - regs = io_mapping_map_atomic_wc(ggtt->mappable, - i915_gem_obj_ggtt_offset(overlay->reg_bo)); + regs = io_mapping_map_atomic_wc(dev_priv->ggtt.mappable, + overlay->flip_addr); return regs; } @@ -1501,15 +1493,13 @@ intel_overlay_map_regs_atomic(struct intel_overlay *overlay) static void intel_overlay_unmap_regs_atomic(struct intel_overlay *overlay, struct overlay_registers __iomem *regs) { - if (!OVERLAY_NEEDS_PHYSICAL(overlay->dev)) + if (!OVERLAY_NEEDS_PHYSICAL(overlay->i915)) io_mapping_unmap_atomic(regs); } - struct intel_overlay_error_state * -intel_overlay_capture_error_state(struct drm_device *dev) +intel_overlay_capture_error_state(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_overlay *overlay = dev_priv->overlay; struct intel_overlay_error_state *error; struct overlay_registers __iomem *regs; @@ -1523,10 +1513,7 @@ intel_overlay_capture_error_state(struct drm_device *dev) error->dovsta = I915_READ(DOVSTA); error->isr = I915_READ(ISR); - if (OVERLAY_NEEDS_PHYSICAL(overlay->dev)) - error->base = (__force long)overlay->reg_bo->phys_handle->vaddr; - else - error->base = i915_gem_obj_ggtt_offset(overlay->reg_bo); + error->base = overlay->flip_addr; regs = intel_overlay_map_regs_atomic(overlay); if (!regs) diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c index aba940998..96c65d77e 100644 --- a/drivers/gpu/drm/i915/intel_panel.c +++ b/drivers/gpu/drm/i915/intel_panel.c @@ -377,7 +377,7 @@ out: enum drm_connector_status intel_panel_detect(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* Assume that the BIOS does not lie through the OpRegion... */ if (!i915.panel_ignore_lid && dev_priv->opregion.lid_state) { @@ -504,7 +504,7 @@ static u32 i9xx_get_backlight(struct intel_connector *connector) if (panel->backlight.combination_mode) { u8 lbpc; - pci_read_config_byte(dev_priv->dev->pdev, LBPC, &lbpc); + pci_read_config_byte(dev_priv->drm.pdev, LBPC, &lbpc); val *= lbpc; } @@ -592,7 +592,7 @@ static void i9xx_set_backlight(struct intel_connector *connector, u32 level) lbpc = level * 0xfe / panel->backlight.max + 1; level /= lbpc; - pci_write_config_byte(dev_priv->dev->pdev, LBPC, lbpc); + pci_write_config_byte(dev_priv->drm.pdev, LBPC, lbpc); } if (IS_GEN4(dev_priv)) { @@ -822,7 +822,7 @@ void intel_panel_disable_backlight(struct intel_connector *connector) * backlight. This will leave the backlight on unnecessarily when * another client is not activated. */ - if (dev_priv->dev->switch_power_state == DRM_SWITCH_POWER_CHANGING) { + if (dev_priv->drm.switch_power_state == DRM_SWITCH_POWER_CHANGING) { DRM_DEBUG_DRIVER("Skipping backlight disable on vga switch\n"); return; } @@ -1142,7 +1142,7 @@ static int intel_backlight_device_get_brightness(struct backlight_device *bd) { struct intel_connector *connector = bl_get_data(bd); struct drm_device *dev = connector->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 hw_level; int ret; @@ -1163,7 +1163,7 @@ static const struct backlight_ops intel_backlight_device_ops = { .get_brightness = intel_backlight_device_get_brightness, }; -static int intel_backlight_device_register(struct intel_connector *connector) +int intel_backlight_device_register(struct intel_connector *connector) { struct intel_panel *panel = &connector->panel; struct backlight_properties props; @@ -1216,7 +1216,7 @@ static int intel_backlight_device_register(struct intel_connector *connector) return 0; } -static void intel_backlight_device_unregister(struct intel_connector *connector) +void intel_backlight_device_unregister(struct intel_connector *connector) { struct intel_panel *panel = &connector->panel; @@ -1225,14 +1225,6 @@ static void intel_backlight_device_unregister(struct intel_connector *connector) panel->backlight.device = NULL; } } -#else /* CONFIG_BACKLIGHT_CLASS_DEVICE */ -static int intel_backlight_device_register(struct intel_connector *connector) -{ - return 0; -} -static void intel_backlight_device_unregister(struct intel_connector *connector) -{ -} #endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */ /* @@ -1324,7 +1316,7 @@ static u32 i9xx_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) static u32 i965_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) { struct drm_device *dev = connector->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int clock; if (IS_G4X(dev_priv)) @@ -1724,6 +1716,14 @@ intel_panel_init_backlight_funcs(struct intel_panel *panel) container_of(panel, struct intel_connector, panel); struct drm_i915_private *dev_priv = to_i915(connector->base.dev); + if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP && + intel_dp_aux_init_backlight_funcs(connector) == 0) + return; + + if (connector->base.connector_type == DRM_MODE_CONNECTOR_DSI && + intel_dsi_dcs_init_backlight_funcs(connector) == 0) + return; + if (IS_BROXTON(dev_priv)) { panel->backlight.setup = bxt_setup_backlight; panel->backlight.enable = bxt_enable_backlight; @@ -1805,19 +1805,3 @@ void intel_panel_fini(struct intel_panel *panel) drm_mode_destroy(intel_connector->base.dev, panel->downclock_mode); } - -void intel_backlight_register(struct drm_device *dev) -{ - struct intel_connector *connector; - - for_each_intel_connector(dev, connector) - intel_backlight_device_register(connector); -} - -void intel_backlight_unregister(struct drm_device *dev) -{ - struct intel_connector *connector; - - for_each_intel_connector(dev, connector) - intel_backlight_device_unregister(connector); -} diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 64dd842ae..0f042d133 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -26,6 +26,7 @@ */ #include <linux/cpufreq.h> +#include <drm/drm_plane_helper.h> #include "i915_drv.h" #include "intel_drv.h" #include "../../../platform/x86/intel_ips.h" @@ -82,7 +83,7 @@ static void gen9_init_clock_gating(struct drm_device *dev) static void bxt_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); gen9_init_clock_gating(dev); @@ -108,7 +109,7 @@ static void bxt_init_clock_gating(struct drm_device *dev) static void i915_pineview_get_mem_freq(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 tmp; tmp = I915_READ(CLKCFG); @@ -147,7 +148,7 @@ static void i915_pineview_get_mem_freq(struct drm_device *dev) static void i915_ironlake_get_mem_freq(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u16 ddrpll, csipll; ddrpll = I915_READ16(DDRMPLL1); @@ -318,7 +319,7 @@ static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable) void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; u32 val; if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) { @@ -374,7 +375,7 @@ static const int pessimal_latency_ns = 5000; static int vlv_get_fifo_size(struct drm_device *dev, enum pipe pipe, int plane) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int sprite0_start, sprite1_start, size; switch (pipe) { @@ -425,7 +426,7 @@ static int vlv_get_fifo_size(struct drm_device *dev, static int i9xx_get_fifo_size(struct drm_device *dev, int plane) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dsparb = I915_READ(DSPARB); int size; @@ -441,7 +442,7 @@ static int i9xx_get_fifo_size(struct drm_device *dev, int plane) static int i830_get_fifo_size(struct drm_device *dev, int plane) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dsparb = I915_READ(DSPARB); int size; @@ -458,7 +459,7 @@ static int i830_get_fifo_size(struct drm_device *dev, int plane) static int i845_get_fifo_size(struct drm_device *dev, int plane) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dsparb = I915_READ(DSPARB); int size; @@ -636,7 +637,7 @@ static struct drm_crtc *single_enabled_crtc(struct drm_device *dev) static void pineview_update_wm(struct drm_crtc *unused_crtc) { struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc; const struct cxsr_latency *latency; u32 reg; @@ -933,7 +934,7 @@ static unsigned int vlv_wm_method2(unsigned int pixel_rate, static void vlv_setup_wm_latency(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* all latencies in usec */ dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; @@ -1324,7 +1325,7 @@ static void vlv_merge_wm(struct drm_device *dev, static void vlv_update_wm(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(crtc); enum pipe pipe = intel_crtc->pipe; struct vlv_wm_values wm = {}; @@ -1380,7 +1381,7 @@ static void g4x_update_wm(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; static const int sr_latency_ns = 12000; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int planea_wm, planeb_wm, cursora_wm, cursorb_wm; int plane_sr, cursor_sr; unsigned int enabled = 0; @@ -1437,7 +1438,7 @@ static void g4x_update_wm(struct drm_crtc *crtc) static void i965_update_wm(struct drm_crtc *unused_crtc) { struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc; int srwm = 1; int cursor_sr = 16; @@ -1511,7 +1512,7 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) static void i9xx_update_wm(struct drm_crtc *unused_crtc) { struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); const struct intel_watermark_params *wm_info; uint32_t fwater_lo; uint32_t fwater_hi; @@ -1641,7 +1642,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) static void i845_update_wm(struct drm_crtc *unused_crtc) { struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc; const struct drm_display_mode *adjusted_mode; uint32_t fwater_lo; @@ -2040,10 +2041,10 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, } static uint32_t -hsw_compute_linetime_wm(struct drm_device *dev, - struct intel_crtc_state *cstate) +hsw_compute_linetime_wm(const struct intel_crtc_state *cstate) { - struct drm_i915_private *dev_priv = dev->dev_private; + const struct intel_atomic_state *intel_state = + to_intel_atomic_state(cstate->base.state); const struct drm_display_mode *adjusted_mode = &cstate->base.adjusted_mode; u32 linetime, ips_linetime; @@ -2052,7 +2053,7 @@ hsw_compute_linetime_wm(struct drm_device *dev, return 0; if (WARN_ON(adjusted_mode->crtc_clock == 0)) return 0; - if (WARN_ON(dev_priv->cdclk_freq == 0)) + if (WARN_ON(intel_state->cdclk == 0)) return 0; /* The WM are computed with base on how long it takes to fill a single @@ -2061,7 +2062,7 @@ hsw_compute_linetime_wm(struct drm_device *dev, linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8, adjusted_mode->crtc_clock); ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8, - dev_priv->cdclk_freq); + intel_state->cdclk); return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) | PIPE_WM_LINETIME_TIME(linetime); @@ -2069,7 +2070,7 @@ hsw_compute_linetime_wm(struct drm_device *dev, static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8]) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (IS_GEN9(dev)) { uint32_t val; @@ -2174,14 +2175,14 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8]) static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5]) { /* ILK sprite LP0 latency is 1300 ns */ - if (INTEL_INFO(dev)->gen == 5) + if (IS_GEN5(dev)) wm[0] = 13; } static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5]) { /* ILK cursor LP0 latency is 1300 ns */ - if (INTEL_INFO(dev)->gen == 5) + if (IS_GEN5(dev)) wm[0] = 13; /* WaDoubleCursorLP3Latency:ivb */ @@ -2235,7 +2236,7 @@ static void intel_print_wm_latency(struct drm_device *dev, static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, uint16_t wm[5], uint16_t min) { - int level, max_level = ilk_wm_max_level(dev_priv->dev); + int level, max_level = ilk_wm_max_level(&dev_priv->drm); if (wm[0] >= min) return false; @@ -2249,7 +2250,7 @@ static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, static void snb_wm_latency_quirk(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); bool changed; /* @@ -2271,7 +2272,7 @@ static void snb_wm_latency_quirk(struct drm_device *dev) static void ilk_setup_wm_latency(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); intel_read_wm_latency(dev, dev_priv->wm.pri_latency); @@ -2293,7 +2294,7 @@ static void ilk_setup_wm_latency(struct drm_device *dev) static void skl_setup_wm_latency(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); intel_read_wm_latency(dev, dev_priv->wm.skl_latency); intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency); @@ -2329,7 +2330,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate) struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); struct intel_pipe_wm *pipe_wm; struct drm_device *dev = state->dev; - const struct drm_i915_private *dev_priv = dev->dev_private; + const struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane; struct intel_plane_state *pristate = NULL; struct intel_plane_state *sprstate = NULL; @@ -2337,7 +2338,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate) int level, max_level = ilk_wm_max_level(dev), usable_level; struct ilk_wm_maximums max; - pipe_wm = &cstate->wm.optimal.ilk; + pipe_wm = &cstate->wm.ilk.optimal; for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { struct intel_plane_state *ps; @@ -2380,7 +2381,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate) pipe_wm->wm[0] = pipe_wm->raw_wm[0]; if (IS_HASWELL(dev) || IS_BROADWELL(dev)) - pipe_wm->linetime = hsw_compute_linetime_wm(dev, cstate); + pipe_wm->linetime = hsw_compute_linetime_wm(cstate); if (!ilk_validate_pipe_wm(dev, pipe_wm)) return -EINVAL; @@ -2419,7 +2420,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev, struct intel_crtc *intel_crtc, struct intel_crtc_state *newstate) { - struct intel_pipe_wm *a = &newstate->wm.intermediate; + struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate; struct intel_pipe_wm *b = &intel_crtc->wm.active.ilk; int level, max_level = ilk_wm_max_level(dev); @@ -2428,7 +2429,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev, * currently active watermarks to get values that are safe both before * and after the vblank. */ - *a = newstate->wm.optimal.ilk; + *a = newstate->wm.ilk.optimal; a->pipe_enabled |= b->pipe_enabled; a->sprites_enabled |= b->sprites_enabled; a->sprites_scaled |= b->sprites_scaled; @@ -2457,7 +2458,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev, * If our intermediate WM are identical to the final WM, then we can * omit the post-vblank programming; only update if it's different. */ - if (memcmp(a, &newstate->wm.optimal.ilk, sizeof(*a)) == 0) + if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) == 0) newstate->wm.need_postvbl_update = false; return 0; @@ -2504,7 +2505,7 @@ static void ilk_wm_merge(struct drm_device *dev, const struct ilk_wm_maximums *max, struct intel_pipe_wm *merged) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int level, max_level = ilk_wm_max_level(dev); int last_enabled_level = max_level; @@ -2564,7 +2565,7 @@ static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm) /* The value we need to program into the WM_LPx latency field */ static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (IS_HASWELL(dev) || IS_BROADWELL(dev)) return 2 * level; @@ -2764,7 +2765,7 @@ static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, static void ilk_write_wm_values(struct drm_i915_private *dev_priv, struct ilk_wm_values *results) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct ilk_wm_values *previous = &dev_priv->wm.hw; unsigned int dirty; uint32_t val; @@ -2839,7 +2840,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv, bool ilk_disable_lp_wm(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL); } @@ -2851,6 +2852,7 @@ bool ilk_disable_lp_wm(struct drm_device *dev) #define SKL_DDB_SIZE 896 /* in blocks */ #define BXT_DDB_SIZE 512 +#define SKL_SAGV_BLOCK_TIME 30 /* µs */ /* * Return the index of a plane in the SKL DDB and wm result arrays. Primary @@ -2874,126 +2876,179 @@ skl_wm_plane_id(const struct intel_plane *plane) } } -static void -skl_sagv_get_hw_state(struct drm_i915_private *dev_priv) -{ - u32 temp; - int ret; - - if (IS_BROXTON(dev_priv)) - return; - - mutex_lock(&dev_priv->rps.hw_lock); - ret = sandybridge_pcode_read(dev_priv, GEN9_PCODE_SAGV_CONTROL, &temp); - mutex_unlock(&dev_priv->rps.hw_lock); - - if (!ret) { - dev_priv->skl_sagv_enabled = !!(temp & GEN9_SAGV_DYNAMIC_FREQ); - } else { - /* - * If for some reason we can't access the SAGV state, follow - * the bspec and assume it's enabled - */ - DRM_ERROR("Failed to get SAGV state, assuming enabled\n"); - dev_priv->skl_sagv_enabled = true; - } -} - /* * SAGV dynamically adjusts the system agent voltage and clock frequencies * depending on power and performance requirements. The display engine access - * to system memory is blocked during the adjustment time. Having this enabled - * in multi-pipe configurations can cause issues (such as underruns causing - * full system hangs), and the bspec also suggests that software disable it - * when more then one pipe is enabled. + * to system memory is blocked during the adjustment time. Because of the + * blocking time, having this enabled can cause full system hangs and/or pipe + * underruns if we don't meet all of the following requirements: + * + * - <= 1 pipe enabled + * - All planes can enable watermarks for latencies >= SAGV engine block time + * - We're not using an interlaced display configuration */ -static int +int skl_enable_sagv(struct drm_i915_private *dev_priv) { int ret; - if (IS_BROXTON(dev_priv)) - return 0; - if (dev_priv->skl_sagv_enabled) + if (dev_priv->skl_sagv_status == I915_SKL_SAGV_NOT_CONTROLLED || + dev_priv->skl_sagv_status == I915_SKL_SAGV_ENABLED) return 0; - mutex_lock(&dev_priv->rps.hw_lock); DRM_DEBUG_KMS("Enabling the SAGV\n"); + mutex_lock(&dev_priv->rps.hw_lock); ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL, - GEN9_SAGV_DYNAMIC_FREQ); - if (!ret) - dev_priv->skl_sagv_enabled = true; - else - DRM_ERROR("Failed to enable the SAGV\n"); + GEN9_SAGV_ENABLE); - /* We don't need to wait for SAGV when enabling */ + /* We don't need to wait for the SAGV when enabling */ mutex_unlock(&dev_priv->rps.hw_lock); - return ret; + + /* + * Some skl systems, pre-release machines in particular, + * don't actually have an SAGV. + */ + if (ret == -ENXIO) { + DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n"); + dev_priv->skl_sagv_status = I915_SKL_SAGV_NOT_CONTROLLED; + return 0; + } else if (ret < 0) { + DRM_ERROR("Failed to enable the SAGV\n"); + return ret; + } + + dev_priv->skl_sagv_status = I915_SKL_SAGV_ENABLED; + return 0; } static int +skl_do_sagv_disable(struct drm_i915_private *dev_priv) +{ + int ret; + uint32_t temp = GEN9_SAGV_DISABLE; + + ret = sandybridge_pcode_read(dev_priv, GEN9_PCODE_SAGV_CONTROL, + &temp); + if (ret) + return ret; + else + return temp & GEN9_SAGV_IS_DISABLED; +} + +int skl_disable_sagv(struct drm_i915_private *dev_priv) { - int ret = 0; - unsigned long timeout; - u32 temp; + int ret, result; - if (IS_BROXTON(dev_priv)) - return 0; - if (!dev_priv->skl_sagv_enabled) + if (dev_priv->skl_sagv_status == I915_SKL_SAGV_NOT_CONTROLLED || + dev_priv->skl_sagv_status == I915_SKL_SAGV_DISABLED) return 0; - mutex_lock(&dev_priv->rps.hw_lock); DRM_DEBUG_KMS("Disabling the SAGV\n"); + mutex_lock(&dev_priv->rps.hw_lock); /* bspec says to keep retrying for at least 1 ms */ - timeout = jiffies + msecs_to_jiffies(1); - do { - ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL, - GEN9_SAGV_DISABLE); - if (ret) { - DRM_ERROR("Failed to disable the SAGV\n"); - goto out; - } - - ret = sandybridge_pcode_read(dev_priv, GEN9_PCODE_SAGV_CONTROL, - &temp); - if (ret) { - DRM_ERROR("Failed to check the status of the SAGV\n"); - goto out; - } - } while (!(temp & 0x1) && jiffies < timeout); + ret = wait_for(result = skl_do_sagv_disable(dev_priv), 1); + mutex_unlock(&dev_priv->rps.hw_lock); - if (temp & 0x1) { - dev_priv->skl_sagv_enabled = false; - } else { - ret = -1; + if (ret == -ETIMEDOUT) { DRM_ERROR("Request to disable SAGV timed out\n"); + return -ETIMEDOUT; } -out: - mutex_unlock(&dev_priv->rps.hw_lock); - return ret; + /* + * Some skl systems, pre-release machines in particular, + * don't actually have an SAGV. + */ + if (result == -ENXIO) { + DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n"); + dev_priv->skl_sagv_status = I915_SKL_SAGV_NOT_CONTROLLED; + return 0; + } else if (result < 0) { + DRM_ERROR("Failed to disable the SAGV\n"); + return result; + } + + dev_priv->skl_sagv_status = I915_SKL_SAGV_DISABLED; + return 0; +} + +bool skl_can_enable_sagv(struct drm_atomic_state *state) +{ + struct drm_device *dev = state->dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_atomic_state *intel_state = to_intel_atomic_state(state); + struct drm_crtc *crtc; + enum pipe pipe; + int level, plane; + + /* + * SKL workaround: bspec recommends we disable the SAGV when we have + * more then one pipe enabled + * + * If there are no active CRTCs, no additional checks need be performed + */ + if (hweight32(intel_state->active_crtcs) == 0) + return true; + else if (hweight32(intel_state->active_crtcs) > 1) + return false; + + /* Since we're now guaranteed to only have one active CRTC... */ + pipe = ffs(intel_state->active_crtcs) - 1; + crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + + if (crtc->state->mode.flags & DRM_MODE_FLAG_INTERLACE) + return false; + + for_each_plane(dev_priv, pipe, plane) { + /* Skip this plane if it's not enabled */ + if (intel_state->wm_results.plane[pipe][plane][0] == 0) + continue; + + /* Find the highest enabled wm level for this plane */ + for (level = ilk_wm_max_level(dev); + intel_state->wm_results.plane[pipe][plane][level] == 0; --level) + { } + + /* + * If any of the planes on this pipe don't enable wm levels + * that incur memory latencies higher then 30µs we can't enable + * the SAGV + */ + if (dev_priv->wm.skl_latency[level] < SKL_SAGV_BLOCK_TIME) + return false; + } + + return true; } static void skl_ddb_get_pipe_allocation_limits(struct drm_device *dev, const struct intel_crtc_state *cstate, - const struct intel_wm_config *config, - struct skl_ddb_entry *alloc /* out */) + struct skl_ddb_entry *alloc, /* out */ + int *num_active /* out */) { + struct drm_atomic_state *state = cstate->base.state; + struct intel_atomic_state *intel_state = to_intel_atomic_state(state); + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *for_crtc = cstate->base.crtc; - struct drm_crtc *crtc; unsigned int pipe_size, ddb_size; int nth_active_pipe; + int pipe = to_intel_crtc(for_crtc)->pipe; - if (!cstate->base.active) { + if (WARN_ON(!state) || !cstate->base.active) { alloc->start = 0; alloc->end = 0; + *num_active = hweight32(dev_priv->active_crtcs); return; } + if (intel_state->active_pipe_changes) + *num_active = hweight32(intel_state->active_crtcs); + else + *num_active = hweight32(dev_priv->active_crtcs); + if (IS_BROXTON(dev)) ddb_size = BXT_DDB_SIZE; else @@ -3001,25 +3056,29 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev, ddb_size -= 4; /* 4 blocks for bypass path allocation */ - nth_active_pipe = 0; - for_each_crtc(dev, crtc) { - if (!to_intel_crtc(crtc)->active) - continue; - - if (crtc == for_crtc) - break; - - nth_active_pipe++; + /* + * If the state doesn't change the active CRTC's, then there's + * no need to recalculate; the existing pipe allocation limits + * should remain unchanged. Note that we're safe from racing + * commits since any racing commit that changes the active CRTC + * list would need to grab _all_ crtc locks, including the one + * we currently hold. + */ + if (!intel_state->active_pipe_changes) { + *alloc = dev_priv->wm.skl_hw.ddb.pipe[pipe]; + return; } - pipe_size = ddb_size / config->num_pipes_active; - alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active; + nth_active_pipe = hweight32(intel_state->active_crtcs & + (drm_crtc_mask(for_crtc) - 1)); + pipe_size = ddb_size / hweight32(intel_state->active_crtcs); + alloc->start = nth_active_pipe * ddb_size / *num_active; alloc->end = alloc->start + pipe_size; } -static unsigned int skl_cursor_allocation(const struct intel_wm_config *config) +static unsigned int skl_cursor_allocation(int num_active) { - if (config->num_pipes_active == 1) + if (num_active == 1) return 32; return 8; @@ -3063,6 +3122,46 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, } } +/* + * Determines the downscale amount of a plane for the purposes of watermark calculations. + * The bspec defines downscale amount as: + * + * """ + * Horizontal down scale amount = maximum[1, Horizontal source size / + * Horizontal destination size] + * Vertical down scale amount = maximum[1, Vertical source size / + * Vertical destination size] + * Total down scale amount = Horizontal down scale amount * + * Vertical down scale amount + * """ + * + * Return value is provided in 16.16 fixed point form to retain fractional part. + * Caller should take care of dividing & rounding off the value. + */ +static uint32_t +skl_plane_downscale_amount(const struct intel_plane_state *pstate) +{ + uint32_t downscale_h, downscale_w; + uint32_t src_w, src_h, dst_w, dst_h; + + if (WARN_ON(!pstate->visible)) + return DRM_PLANE_HELPER_NO_SCALING; + + /* n.b., src is 16.16 fixed point, dst is whole integer */ + src_w = drm_rect_width(&pstate->src); + src_h = drm_rect_height(&pstate->src); + dst_w = drm_rect_width(&pstate->dst); + dst_h = drm_rect_height(&pstate->dst); + if (intel_rotation_90_or_270(pstate->base.rotation)) + swap(dst_w, dst_h); + + downscale_h = max(src_h / dst_h, (uint32_t)DRM_PLANE_HELPER_NO_SCALING); + downscale_w = max(src_w / dst_w, (uint32_t)DRM_PLANE_HELPER_NO_SCALING); + + /* Provide result in 16.16 fixed point */ + return (uint64_t)downscale_w * downscale_h >> 16; +} + static unsigned int skl_plane_relative_data_rate(const struct intel_crtc_state *cstate, const struct drm_plane_state *pstate, @@ -3070,7 +3169,16 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate, { struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate); struct drm_framebuffer *fb = pstate->fb; + uint32_t down_scale_amount, data_rate; uint32_t width = 0, height = 0; + unsigned format = fb ? fb->pixel_format : DRM_FORMAT_XRGB8888; + + if (!intel_pstate->visible) + return 0; + if (pstate->plane->type == DRM_PLANE_TYPE_CURSOR) + return 0; + if (y && format != DRM_FORMAT_NV12) + return 0; width = drm_rect_width(&intel_pstate->src) >> 16; height = drm_rect_height(&intel_pstate->src) >> 16; @@ -3079,17 +3187,21 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate, swap(width, height); /* for planar format */ - if (fb->pixel_format == DRM_FORMAT_NV12) { + if (format == DRM_FORMAT_NV12) { if (y) /* y-plane data rate */ - return width * height * - drm_format_plane_cpp(fb->pixel_format, 0); + data_rate = width * height * + drm_format_plane_cpp(format, 0); else /* uv-plane data rate */ - return (width / 2) * (height / 2) * - drm_format_plane_cpp(fb->pixel_format, 1); + data_rate = (width / 2) * (height / 2) * + drm_format_plane_cpp(format, 1); + } else { + /* for packed formats */ + data_rate = width * height * drm_format_plane_cpp(format, 0); } - /* for packed formats */ - return width * height * drm_format_plane_cpp(fb->pixel_format, 0); + down_scale_amount = skl_plane_downscale_amount(intel_pstate); + + return (uint64_t)data_rate * down_scale_amount >> 16; } /* @@ -3098,86 +3210,186 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate, * 3 * 4096 * 8192 * 4 < 2^32 */ static unsigned int -skl_get_total_relative_data_rate(const struct intel_crtc_state *cstate) +skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate) { - struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); - struct drm_device *dev = intel_crtc->base.dev; + struct drm_crtc_state *cstate = &intel_cstate->base; + struct drm_atomic_state *state = cstate->state; + struct drm_crtc *crtc = cstate->crtc; + struct drm_device *dev = crtc->dev; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + const struct drm_plane *plane; const struct intel_plane *intel_plane; - unsigned int total_data_rate = 0; + struct drm_plane_state *pstate; + unsigned int rate, total_data_rate = 0; + int id; + int i; - for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { - const struct drm_plane_state *pstate = intel_plane->base.state; + if (WARN_ON(!state)) + return 0; - if (pstate->fb == NULL) - continue; + /* Calculate and cache data rate for each plane */ + for_each_plane_in_state(state, plane, pstate, i) { + id = skl_wm_plane_id(to_intel_plane(plane)); + intel_plane = to_intel_plane(plane); - if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR) + if (intel_plane->pipe != intel_crtc->pipe) continue; /* packed/uv */ - total_data_rate += skl_plane_relative_data_rate(cstate, - pstate, - 0); + rate = skl_plane_relative_data_rate(intel_cstate, + pstate, 0); + intel_cstate->wm.skl.plane_data_rate[id] = rate; - if (pstate->fb->pixel_format == DRM_FORMAT_NV12) - /* y-plane */ - total_data_rate += skl_plane_relative_data_rate(cstate, - pstate, - 1); + /* y-plane */ + rate = skl_plane_relative_data_rate(intel_cstate, + pstate, 1); + intel_cstate->wm.skl.plane_y_data_rate[id] = rate; + } + + /* Calculate CRTC's total data rate from cached values */ + for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { + int id = skl_wm_plane_id(intel_plane); + + /* packed/uv */ + total_data_rate += intel_cstate->wm.skl.plane_data_rate[id]; + total_data_rate += intel_cstate->wm.skl.plane_y_data_rate[id]; } return total_data_rate; } -static void +static uint16_t +skl_ddb_min_alloc(const struct drm_plane_state *pstate, + const int y) +{ + struct drm_framebuffer *fb = pstate->fb; + struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate); + uint32_t src_w, src_h; + uint32_t min_scanlines = 8; + uint8_t plane_bpp; + + if (WARN_ON(!fb)) + return 0; + + /* For packed formats, no y-plane, return 0 */ + if (y && fb->pixel_format != DRM_FORMAT_NV12) + return 0; + + /* For Non Y-tile return 8-blocks */ + if (fb->modifier[0] != I915_FORMAT_MOD_Y_TILED && + fb->modifier[0] != I915_FORMAT_MOD_Yf_TILED) + return 8; + + src_w = drm_rect_width(&intel_pstate->src) >> 16; + src_h = drm_rect_height(&intel_pstate->src) >> 16; + + if (intel_rotation_90_or_270(pstate->rotation)) + swap(src_w, src_h); + + /* Halve UV plane width and height for NV12 */ + if (fb->pixel_format == DRM_FORMAT_NV12 && !y) { + src_w /= 2; + src_h /= 2; + } + + if (fb->pixel_format == DRM_FORMAT_NV12 && !y) + plane_bpp = drm_format_plane_cpp(fb->pixel_format, 1); + else + plane_bpp = drm_format_plane_cpp(fb->pixel_format, 0); + + if (intel_rotation_90_or_270(pstate->rotation)) { + switch (plane_bpp) { + case 1: + min_scanlines = 32; + break; + case 2: + min_scanlines = 16; + break; + case 4: + min_scanlines = 8; + break; + case 8: + min_scanlines = 4; + break; + default: + WARN(1, "Unsupported pixel depth %u for rotation", + plane_bpp); + min_scanlines = 32; + } + } + + return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3; +} + +static int skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, struct skl_ddb_allocation *ddb /* out */) { + struct drm_atomic_state *state = cstate->base.state; struct drm_crtc *crtc = cstate->base.crtc; struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_wm_config *config = &dev_priv->wm.config; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_plane *intel_plane; + struct drm_plane *plane; + struct drm_plane_state *pstate; enum pipe pipe = intel_crtc->pipe; struct skl_ddb_entry *alloc = &ddb->pipe[pipe]; uint16_t alloc_size, start, cursor_blocks; - uint16_t minimum[I915_MAX_PLANES]; - uint16_t y_minimum[I915_MAX_PLANES]; + uint16_t *minimum = cstate->wm.skl.minimum_blocks; + uint16_t *y_minimum = cstate->wm.skl.minimum_y_blocks; unsigned int total_data_rate; + int num_active; + int id, i; - skl_ddb_get_pipe_allocation_limits(dev, cstate, config, alloc); + if (WARN_ON(!state)) + return 0; + + if (!cstate->base.active) { + ddb->pipe[pipe].start = ddb->pipe[pipe].end = 0; + memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); + memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe])); + return 0; + } + + skl_ddb_get_pipe_allocation_limits(dev, cstate, alloc, &num_active); alloc_size = skl_ddb_entry_size(alloc); if (alloc_size == 0) { memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); - memset(&ddb->plane[pipe][PLANE_CURSOR], 0, - sizeof(ddb->plane[pipe][PLANE_CURSOR])); - return; + return 0; } - cursor_blocks = skl_cursor_allocation(config); + cursor_blocks = skl_cursor_allocation(num_active); ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks; ddb->plane[pipe][PLANE_CURSOR].end = alloc->end; alloc_size -= cursor_blocks; - alloc->end -= cursor_blocks; /* 1. Allocate the mininum required blocks for each active plane */ - for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { - struct drm_plane *plane = &intel_plane->base; - struct drm_framebuffer *fb = plane->state->fb; - int id = skl_wm_plane_id(intel_plane); + for_each_plane_in_state(state, plane, pstate, i) { + intel_plane = to_intel_plane(plane); + id = skl_wm_plane_id(intel_plane); - if (!to_intel_plane_state(plane->state)->visible) + if (intel_plane->pipe != pipe) continue; - if (plane->type == DRM_PLANE_TYPE_CURSOR) + if (!to_intel_plane_state(pstate)->visible) { + minimum[id] = 0; + y_minimum[id] = 0; + continue; + } + if (plane->type == DRM_PLANE_TYPE_CURSOR) { + minimum[id] = 0; + y_minimum[id] = 0; continue; + } + + minimum[id] = skl_ddb_min_alloc(pstate, 0); + y_minimum[id] = skl_ddb_min_alloc(pstate, 1); + } - minimum[id] = 8; - alloc_size -= minimum[id]; - y_minimum[id] = (fb->pixel_format == DRM_FORMAT_NV12) ? 8 : 0; - alloc_size -= y_minimum[id]; + for (i = 0; i < PLANE_CURSOR; i++) { + alloc_size -= minimum[i]; + alloc_size -= y_minimum[i]; } /* @@ -3187,21 +3399,16 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, * FIXME: we may not allocate every single block here. */ total_data_rate = skl_get_total_relative_data_rate(cstate); + if (total_data_rate == 0) + return 0; start = alloc->start; for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { - struct drm_plane *plane = &intel_plane->base; - struct drm_plane_state *pstate = intel_plane->base.state; unsigned int data_rate, y_data_rate; uint16_t plane_blocks, y_plane_blocks = 0; int id = skl_wm_plane_id(intel_plane); - if (!to_intel_plane_state(pstate)->visible) - continue; - if (plane->type == DRM_PLANE_TYPE_CURSOR) - continue; - - data_rate = skl_plane_relative_data_rate(cstate, pstate, 0); + data_rate = cstate->wm.skl.plane_data_rate[id]; /* * allocation for (packed formats) or (uv-plane part of planar format): @@ -3212,30 +3419,32 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, plane_blocks += div_u64((uint64_t)alloc_size * data_rate, total_data_rate); - ddb->plane[pipe][id].start = start; - ddb->plane[pipe][id].end = start + plane_blocks; + /* Leave disabled planes at (0,0) */ + if (data_rate) { + ddb->plane[pipe][id].start = start; + ddb->plane[pipe][id].end = start + plane_blocks; + } start += plane_blocks; /* * allocation for y_plane part of planar format: */ - if (pstate->fb->pixel_format == DRM_FORMAT_NV12) { - y_data_rate = skl_plane_relative_data_rate(cstate, - pstate, - 1); - y_plane_blocks = y_minimum[id]; - y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate, - total_data_rate); + y_data_rate = cstate->wm.skl.plane_y_data_rate[id]; + y_plane_blocks = y_minimum[id]; + y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate, + total_data_rate); + + if (y_data_rate) { ddb->y_plane[pipe][id].start = start; ddb->y_plane[pipe][id].end = start + y_plane_blocks; - - start += y_plane_blocks; } + start += y_plane_blocks; } + return 0; } static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config) @@ -3281,6 +3490,8 @@ static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal, plane_bytes_per_line *= 4; plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512); plane_blocks_per_line /= 4; + } else if (tiling == DRM_FORMAT_MOD_NONE) { + plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512) + 1; } else { plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512); } @@ -3292,35 +3503,41 @@ static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal, return ret; } -static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb, - const struct intel_crtc *intel_crtc) +static uint32_t skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate, + struct intel_plane_state *pstate) { - struct drm_device *dev = intel_crtc->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; - const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb; + uint64_t adjusted_pixel_rate; + uint64_t downscale_amount; + uint64_t pixel_rate; + + /* Shouldn't reach here on disabled planes... */ + if (WARN_ON(!pstate->visible)) + return 0; /* - * If ddb allocation of pipes changed, it may require recalculation of - * watermarks + * Adjusted plane pixel rate is just the pipe's adjusted pixel rate + * with additional adjustments for plane-specific scaling. */ - if (memcmp(new_ddb->pipe, cur_ddb->pipe, sizeof(new_ddb->pipe))) - return true; + adjusted_pixel_rate = skl_pipe_pixel_rate(cstate); + downscale_amount = skl_plane_downscale_amount(pstate); - return false; + pixel_rate = adjusted_pixel_rate * downscale_amount >> 16; + WARN_ON(pixel_rate != clamp_t(uint32_t, pixel_rate, 0, ~0)); + + return pixel_rate; } -static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, - struct intel_crtc_state *cstate, - struct intel_plane *intel_plane, - uint16_t ddb_allocation, - int level, - uint16_t *out_blocks, /* out */ - uint8_t *out_lines /* out */) +static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv, + struct intel_crtc_state *cstate, + struct intel_plane_state *intel_pstate, + uint16_t ddb_allocation, + int level, + uint16_t *out_blocks, /* out */ + uint8_t *out_lines, /* out */ + bool *enabled /* out */) { - struct drm_plane *plane = &intel_plane->base; - struct drm_framebuffer *fb = plane->state->fb; - struct intel_plane_state *intel_pstate = - to_intel_plane_state(plane->state); + struct drm_plane_state *pstate = &intel_pstate->base; + struct drm_framebuffer *fb = pstate->fb; uint32_t latency = dev_priv->wm.skl_latency[level]; uint32_t method1, method2; uint32_t plane_bytes_per_line, plane_blocks_per_line; @@ -3328,20 +3545,24 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, uint32_t selected_result; uint8_t cpp; uint32_t width = 0, height = 0; + uint32_t plane_pixel_rate; - if (latency == 0 || !cstate->base.active || !intel_pstate->visible) - return false; + if (latency == 0 || !cstate->base.active || !intel_pstate->visible) { + *enabled = false; + return 0; + } width = drm_rect_width(&intel_pstate->src) >> 16; height = drm_rect_height(&intel_pstate->src) >> 16; - if (intel_rotation_90_or_270(plane->state->rotation)) + if (intel_rotation_90_or_270(pstate->rotation)) swap(width, height); cpp = drm_format_plane_cpp(fb->pixel_format, 0); - method1 = skl_wm_method1(skl_pipe_pixel_rate(cstate), - cpp, latency); - method2 = skl_wm_method2(skl_pipe_pixel_rate(cstate), + plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate, intel_pstate); + + method1 = skl_wm_method1(plane_pixel_rate, cpp, latency); + method2 = skl_wm_method2(plane_pixel_rate, cstate->base.adjusted_mode.crtc_htotal, width, cpp, @@ -3355,7 +3576,7 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) { uint32_t min_scanlines = 4; uint32_t y_tile_minimum; - if (intel_rotation_90_or_270(plane->state->rotation)) { + if (intel_rotation_90_or_270(pstate->rotation)) { int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ? drm_format_plane_cpp(fb->pixel_format, 1) : drm_format_plane_cpp(fb->pixel_format, 0); @@ -3391,40 +3612,100 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv, res_blocks++; } - if (res_blocks >= ddb_allocation || res_lines > 31) - return false; + if (res_blocks >= ddb_allocation || res_lines > 31) { + *enabled = false; + + /* + * If there are no valid level 0 watermarks, then we can't + * support this display configuration. + */ + if (level) { + return 0; + } else { + DRM_DEBUG_KMS("Requested display configuration exceeds system watermark limitations\n"); + DRM_DEBUG_KMS("Plane %d.%d: blocks required = %u/%u, lines required = %u/31\n", + to_intel_crtc(cstate->base.crtc)->pipe, + skl_wm_plane_id(to_intel_plane(pstate->plane)), + res_blocks, ddb_allocation, res_lines); + + return -EINVAL; + } + } *out_blocks = res_blocks; *out_lines = res_lines; + *enabled = true; - return true; + return 0; } -static void skl_compute_wm_level(const struct drm_i915_private *dev_priv, - struct skl_ddb_allocation *ddb, - struct intel_crtc_state *cstate, - int level, - struct skl_wm_level *result) +static int +skl_compute_wm_level(const struct drm_i915_private *dev_priv, + struct skl_ddb_allocation *ddb, + struct intel_crtc_state *cstate, + int level, + struct skl_wm_level *result) { - struct drm_device *dev = dev_priv->dev; + struct drm_atomic_state *state = cstate->base.state; struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); + struct drm_plane *plane; struct intel_plane *intel_plane; + struct intel_plane_state *intel_pstate; uint16_t ddb_blocks; enum pipe pipe = intel_crtc->pipe; + int ret; - for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { + /* + * We'll only calculate watermarks for planes that are actually + * enabled, so make sure all other planes are set as disabled. + */ + memset(result, 0, sizeof(*result)); + + for_each_intel_plane_mask(&dev_priv->drm, + intel_plane, + cstate->base.plane_mask) { int i = skl_wm_plane_id(intel_plane); + plane = &intel_plane->base; + intel_pstate = NULL; + if (state) + intel_pstate = + intel_atomic_get_existing_plane_state(state, + intel_plane); + + /* + * Note: If we start supporting multiple pending atomic commits + * against the same planes/CRTC's in the future, plane->state + * will no longer be the correct pre-state to use for the + * calculations here and we'll need to change where we get the + * 'unchanged' plane data from. + * + * For now this is fine because we only allow one queued commit + * against a CRTC. Even if the plane isn't modified by this + * transaction and we don't have a plane lock, we still have + * the CRTC's lock, so we know that no other transactions are + * racing with us to update it. + */ + if (!intel_pstate) + intel_pstate = to_intel_plane_state(plane->state); + + WARN_ON(!intel_pstate->base.fb); + ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]); - result->plane_en[i] = skl_compute_plane_wm(dev_priv, - cstate, - intel_plane, - ddb_blocks, - level, - &result->plane_res_b[i], - &result->plane_res_l[i]); + ret = skl_compute_plane_wm(dev_priv, + cstate, + intel_pstate, + ddb_blocks, + level, + &result->plane_res_b[i], + &result->plane_res_l[i], + &result->plane_en[i]); + if (ret) + return ret; } + + return 0; } static uint32_t @@ -3458,21 +3739,26 @@ static void skl_compute_transition_wm(struct intel_crtc_state *cstate, } } -static void skl_compute_pipe_wm(struct intel_crtc_state *cstate, - struct skl_ddb_allocation *ddb, - struct skl_pipe_wm *pipe_wm) +static int skl_build_pipe_wm(struct intel_crtc_state *cstate, + struct skl_ddb_allocation *ddb, + struct skl_pipe_wm *pipe_wm) { struct drm_device *dev = cstate->base.crtc->dev; - const struct drm_i915_private *dev_priv = dev->dev_private; + const struct drm_i915_private *dev_priv = to_i915(dev); int level, max_level = ilk_wm_max_level(dev); + int ret; for (level = 0; level <= max_level; level++) { - skl_compute_wm_level(dev_priv, ddb, cstate, - level, &pipe_wm->wm[level]); + ret = skl_compute_wm_level(dev_priv, ddb, cstate, + level, &pipe_wm->wm[level]); + if (ret) + return ret; } pipe_wm->linetime = skl_compute_linetime_wm(cstate); skl_compute_transition_wm(cstate, &pipe_wm->trans_wm); + + return 0; } static void skl_compute_wm_results(struct drm_device *dev, @@ -3545,19 +3831,16 @@ static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, static void skl_write_wm_values(struct drm_i915_private *dev_priv, const struct skl_wm_values *new) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct intel_crtc *crtc; - if (dev_priv->active_crtcs == 1) - skl_enable_sagv(dev_priv); - else - skl_disable_sagv(dev_priv); - for_each_intel_crtc(dev, crtc) { int i, level, max_level = ilk_wm_max_level(dev); enum pipe pipe = crtc->pipe; - if (!new->dirty[pipe]) + if ((new->dirty_pipes & drm_crtc_mask(&crtc->base)) == 0) + continue; + if (!crtc->active) continue; I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]); @@ -3644,7 +3927,7 @@ skl_ddb_allocation_included(const struct skl_ddb_allocation *old, static void skl_flush_wm_values(struct drm_i915_private *dev_priv, struct skl_wm_values *new_values) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct skl_ddb_allocation *cur_ddb, *new_ddb; bool reallocated[I915_MAX_PIPES] = {}; struct intel_crtc *crtc; @@ -3723,116 +4006,225 @@ static void skl_flush_wm_values(struct drm_i915_private *dev_priv, } } -static bool skl_update_pipe_wm(struct drm_crtc *crtc, - struct skl_ddb_allocation *ddb, /* out */ - struct skl_pipe_wm *pipe_wm /* out */) +static int skl_update_pipe_wm(struct drm_crtc_state *cstate, + struct skl_ddb_allocation *ddb, /* out */ + struct skl_pipe_wm *pipe_wm, /* out */ + bool *changed /* out */) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); + struct intel_crtc *intel_crtc = to_intel_crtc(cstate->crtc); + struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate); + int ret; - skl_allocate_pipe_ddb(cstate, ddb); - skl_compute_pipe_wm(cstate, ddb, pipe_wm); + ret = skl_build_pipe_wm(intel_cstate, ddb, pipe_wm); + if (ret) + return ret; if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm))) - return false; + *changed = false; + else + *changed = true; - intel_crtc->wm.active.skl = *pipe_wm; + return 0; +} - return true; +static uint32_t +pipes_modified(struct drm_atomic_state *state) +{ + struct drm_crtc *crtc; + struct drm_crtc_state *cstate; + uint32_t i, ret = 0; + + for_each_crtc_in_state(state, crtc, cstate, i) + ret |= drm_crtc_mask(crtc); + + return ret; } -static void skl_update_other_pipe_wm(struct drm_device *dev, - struct drm_crtc *crtc, - struct skl_wm_values *r) +static int +skl_compute_ddb(struct drm_atomic_state *state) { + struct drm_device *dev = state->dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_atomic_state *intel_state = to_intel_atomic_state(state); struct intel_crtc *intel_crtc; - struct intel_crtc *this_crtc = to_intel_crtc(crtc); + struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb; + uint32_t realloc_pipes = pipes_modified(state); + int ret; /* - * If the WM update hasn't changed the allocation for this_crtc (the - * crtc we are currently computing the new WM values for), other - * enabled crtcs will keep the same allocation and we don't need to - * recompute anything for them. + * If this is our first atomic update following hardware readout, + * we can't trust the DDB that the BIOS programmed for us. Let's + * pretend that all pipes switched active status so that we'll + * ensure a full DDB recompute. */ - if (!skl_ddb_allocation_changed(&r->ddb, this_crtc)) - return; + if (dev_priv->wm.distrust_bios_wm) { + ret = drm_modeset_lock(&dev->mode_config.connection_mutex, + state->acquire_ctx); + if (ret) + return ret; + + intel_state->active_pipe_changes = ~0; + + /* + * We usually only initialize intel_state->active_crtcs if we + * we're doing a modeset; make sure this field is always + * initialized during the sanitization process that happens + * on the first commit too. + */ + if (!intel_state->modeset) + intel_state->active_crtcs = dev_priv->active_crtcs; + } /* - * Otherwise, because of this_crtc being freshly enabled/disabled, the - * other active pipes need new DDB allocation and WM values. + * If the modeset changes which CRTC's are active, we need to + * recompute the DDB allocation for *all* active pipes, even + * those that weren't otherwise being modified in any way by this + * atomic commit. Due to the shrinking of the per-pipe allocations + * when new active CRTC's are added, it's possible for a pipe that + * we were already using and aren't changing at all here to suddenly + * become invalid if its DDB needs exceeds its new allocation. + * + * Note that if we wind up doing a full DDB recompute, we can't let + * any other display updates race with this transaction, so we need + * to grab the lock on *all* CRTC's. */ - for_each_intel_crtc(dev, intel_crtc) { - struct skl_pipe_wm pipe_wm = {}; - bool wm_changed; - - if (this_crtc->pipe == intel_crtc->pipe) - continue; + if (intel_state->active_pipe_changes) { + realloc_pipes = ~0; + intel_state->wm_results.dirty_pipes = ~0; + } - if (!intel_crtc->active) - continue; + for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) { + struct intel_crtc_state *cstate; - wm_changed = skl_update_pipe_wm(&intel_crtc->base, - &r->ddb, &pipe_wm); + cstate = intel_atomic_get_crtc_state(state, intel_crtc); + if (IS_ERR(cstate)) + return PTR_ERR(cstate); - /* - * If we end up re-computing the other pipe WM values, it's - * because it was really needed, so we expect the WM values to - * be different. - */ - WARN_ON(!wm_changed); + ret = skl_allocate_pipe_ddb(cstate, ddb); + if (ret) + return ret; - skl_compute_wm_results(dev, &pipe_wm, r, intel_crtc); - r->dirty[intel_crtc->pipe] = true; + ret = drm_atomic_add_affected_planes(state, &intel_crtc->base); + if (ret) + return ret; } + + return 0; } -static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe) +static void +skl_copy_wm_for_pipe(struct skl_wm_values *dst, + struct skl_wm_values *src, + enum pipe pipe) { - watermarks->wm_linetime[pipe] = 0; - memset(watermarks->plane[pipe], 0, - sizeof(uint32_t) * 8 * I915_MAX_PLANES); - memset(watermarks->plane_trans[pipe], - 0, sizeof(uint32_t) * I915_MAX_PLANES); - watermarks->plane_trans[pipe][PLANE_CURSOR] = 0; + dst->wm_linetime[pipe] = src->wm_linetime[pipe]; + memcpy(dst->plane[pipe], src->plane[pipe], + sizeof(dst->plane[pipe])); + memcpy(dst->plane_trans[pipe], src->plane_trans[pipe], + sizeof(dst->plane_trans[pipe])); + + dst->ddb.pipe[pipe] = src->ddb.pipe[pipe]; + memcpy(dst->ddb.y_plane[pipe], src->ddb.y_plane[pipe], + sizeof(dst->ddb.y_plane[pipe])); + memcpy(dst->ddb.plane[pipe], src->ddb.plane[pipe], + sizeof(dst->ddb.plane[pipe])); +} + +static int +skl_compute_wm(struct drm_atomic_state *state) +{ + struct drm_crtc *crtc; + struct drm_crtc_state *cstate; + struct intel_atomic_state *intel_state = to_intel_atomic_state(state); + struct skl_wm_values *results = &intel_state->wm_results; + struct skl_pipe_wm *pipe_wm; + bool changed = false; + int ret, i; - /* Clear ddb entries for pipe */ - memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry)); - memset(&watermarks->ddb.plane[pipe], 0, - sizeof(struct skl_ddb_entry) * I915_MAX_PLANES); - memset(&watermarks->ddb.y_plane[pipe], 0, - sizeof(struct skl_ddb_entry) * I915_MAX_PLANES); - memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0, - sizeof(struct skl_ddb_entry)); + /* + * If this transaction isn't actually touching any CRTC's, don't + * bother with watermark calculation. Note that if we pass this + * test, we're guaranteed to hold at least one CRTC state mutex, + * which means we can safely use values like dev_priv->active_crtcs + * since any racing commits that want to update them would need to + * hold _all_ CRTC state mutexes. + */ + for_each_crtc_in_state(state, crtc, cstate, i) + changed = true; + if (!changed) + return 0; + + /* Clear all dirty flags */ + results->dirty_pipes = 0; + ret = skl_compute_ddb(state); + if (ret) + return ret; + + /* + * Calculate WM's for all pipes that are part of this transaction. + * Note that the DDB allocation above may have added more CRTC's that + * weren't otherwise being modified (and set bits in dirty_pipes) if + * pipe allocations had to change. + * + * FIXME: Now that we're doing this in the atomic check phase, we + * should allow skl_update_pipe_wm() to return failure in cases where + * no suitable watermark values can be found. + */ + for_each_crtc_in_state(state, crtc, cstate, i) { + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc_state *intel_cstate = + to_intel_crtc_state(cstate); + + pipe_wm = &intel_cstate->wm.skl.optimal; + ret = skl_update_pipe_wm(cstate, &results->ddb, pipe_wm, + &changed); + if (ret) + return ret; + + if (changed) + results->dirty_pipes |= drm_crtc_mask(crtc); + + if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0) + /* This pipe's WM's did not change */ + continue; + + intel_cstate->update_wm_pre = true; + skl_compute_wm_results(crtc->dev, pipe_wm, results, intel_crtc); + } + + return 0; } static void skl_update_wm(struct drm_crtc *crtc) { struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct skl_wm_values *results = &dev_priv->wm.skl_results; + struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw; struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); - struct skl_pipe_wm *pipe_wm = &cstate->wm.optimal.skl; - - - /* Clear all dirty flags */ - memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES); - - skl_clear_wm(results, intel_crtc->pipe); + struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal; + int pipe; - if (!skl_update_pipe_wm(crtc, &results->ddb, pipe_wm)) + if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0) return; - skl_compute_wm_results(dev, pipe_wm, results, intel_crtc); - results->dirty[intel_crtc->pipe] = true; + intel_crtc->wm.active.skl = *pipe_wm; + + mutex_lock(&dev_priv->wm.wm_mutex); - skl_update_other_pipe_wm(dev, crtc, results); skl_write_wm_values(dev_priv, results); skl_flush_wm_values(dev_priv, results); - /* store the new configuration */ - dev_priv->wm.skl_hw = *results; + /* + * Store the new configuration (but only for the pipes that have + * changed; the other values weren't recomputed). + */ + for_each_pipe_masked(dev_priv, pipe, results->dirty_pipes) + skl_copy_wm_for_pipe(hw_vals, results, pipe); + + mutex_unlock(&dev_priv->wm.wm_mutex); } static void ilk_compute_wm_config(struct drm_device *dev, @@ -3855,7 +4247,7 @@ static void ilk_compute_wm_config(struct drm_device *dev, static void ilk_program_watermarks(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; struct ilk_wm_maximums max; struct intel_wm_config config = {}; @@ -3892,7 +4284,7 @@ static void ilk_initial_watermarks(struct intel_crtc_state *cstate) struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc); mutex_lock(&dev_priv->wm.wm_mutex); - intel_crtc->wm.active.ilk = cstate->wm.intermediate; + intel_crtc->wm.active.ilk = cstate->wm.ilk.intermediate; ilk_program_watermarks(dev_priv); mutex_unlock(&dev_priv->wm.wm_mutex); } @@ -3904,7 +4296,7 @@ static void ilk_optimize_watermarks(struct intel_crtc_state *cstate) mutex_lock(&dev_priv->wm.wm_mutex); if (cstate->wm.need_postvbl_update) { - intel_crtc->wm.active.ilk = cstate->wm.optimal.ilk; + intel_crtc->wm.active.ilk = cstate->wm.ilk.optimal; ilk_program_watermarks(dev_priv); } mutex_unlock(&dev_priv->wm.wm_mutex); @@ -3957,11 +4349,11 @@ static void skl_pipe_wm_active_state(uint32_t val, static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct skl_wm_values *hw = &dev_priv->wm.skl_hw; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); - struct skl_pipe_wm *active = &cstate->wm.optimal.skl; + struct skl_pipe_wm *active = &cstate->wm.skl.optimal; enum pipe pipe = intel_crtc->pipe; int level, i, max_level; uint32_t temp; @@ -3984,7 +4376,7 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc) if (!intel_crtc->active) return; - hw->dirty[pipe] = true; + hw->dirty_pipes |= drm_crtc_mask(crtc); active->linetime = hw->wm_linetime[pipe]; @@ -4011,7 +4403,7 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc) void skl_wm_get_hw_state(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb; struct drm_crtc *crtc; @@ -4019,17 +4411,23 @@ void skl_wm_get_hw_state(struct drm_device *dev) list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) skl_pipe_wm_get_hw_state(crtc); - skl_sagv_get_hw_state(dev_priv); + if (dev_priv->active_crtcs) { + /* Fully recompute DDB on first atomic commit */ + dev_priv->wm.distrust_bios_wm = true; + } else { + /* Easy/common case; just sanitize DDB now if everything off */ + memset(ddb, 0, sizeof(*ddb)); + } } static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct ilk_wm_values *hw = &dev_priv->wm.hw; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); - struct intel_pipe_wm *active = &cstate->wm.optimal.ilk; + struct intel_pipe_wm *active = &cstate->wm.ilk.optimal; enum pipe pipe = intel_crtc->pipe; static const i915_reg_t wm0_pipe_reg[] = { [PIPE_A] = WM0_PIPEA_ILK, @@ -4229,7 +4627,7 @@ void vlv_wm_get_hw_state(struct drm_device *dev) void ilk_wm_get_hw_state(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct ilk_wm_values *hw = &dev_priv->wm.hw; struct drm_crtc *crtc; @@ -4291,7 +4689,7 @@ void ilk_wm_get_hw_state(struct drm_device *dev) */ void intel_update_watermarks(struct drm_crtc *crtc) { - struct drm_i915_private *dev_priv = crtc->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(crtc->dev); if (dev_priv->display.update_wm) dev_priv->display.update_wm(crtc); @@ -4306,9 +4704,8 @@ DEFINE_SPINLOCK(mchdev_lock); * mchdev_lock. */ static struct drm_i915_private *i915_mch_dev; -bool ironlake_set_drps(struct drm_device *dev, u8 val) +bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val) { - struct drm_i915_private *dev_priv = dev->dev_private; u16 rgvswctl; assert_spin_locked(&mchdev_lock); @@ -4330,9 +4727,8 @@ bool ironlake_set_drps(struct drm_device *dev, u8 val) return true; } -static void ironlake_enable_drps(struct drm_device *dev) +static void ironlake_enable_drps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 rgvmodectl; u8 fmax, fmin, fstart, vstart; @@ -4389,7 +4785,7 @@ static void ironlake_enable_drps(struct drm_device *dev) DRM_ERROR("stuck trying to change perf mode\n"); mdelay(1); - ironlake_set_drps(dev, fstart); + ironlake_set_drps(dev_priv, fstart); dev_priv->ips.last_count1 = I915_READ(DMIEC) + I915_READ(DDREC) + I915_READ(CSIEC); @@ -4400,9 +4796,8 @@ static void ironlake_enable_drps(struct drm_device *dev) spin_unlock_irq(&mchdev_lock); } -static void ironlake_disable_drps(struct drm_device *dev) +static void ironlake_disable_drps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u16 rgvswctl; spin_lock_irq(&mchdev_lock); @@ -4417,7 +4812,7 @@ static void ironlake_disable_drps(struct drm_device *dev) I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT); /* Go back to the starting frequency */ - ironlake_set_drps(dev, dev_priv->ips.fstart); + ironlake_set_drps(dev_priv, dev_priv->ips.fstart); mdelay(1); rgvswctl |= MEMCTL_CMD_STS; I915_WRITE(MEMSWCTL, rgvswctl); @@ -4463,19 +4858,23 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) new_power = dev_priv->rps.power; switch (dev_priv->rps.power) { case LOW_POWER: - if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq) + if (val > dev_priv->rps.efficient_freq + 1 && + val > dev_priv->rps.cur_freq) new_power = BETWEEN; break; case BETWEEN: - if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq) + if (val <= dev_priv->rps.efficient_freq && + val < dev_priv->rps.cur_freq) new_power = LOW_POWER; - else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq) + else if (val >= dev_priv->rps.rp0_freq && + val > dev_priv->rps.cur_freq) new_power = HIGH_POWER; break; case HIGH_POWER: - if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq) + if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && + val < dev_priv->rps.cur_freq) new_power = BETWEEN; break; } @@ -4521,22 +4920,24 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val) } I915_WRITE(GEN6_RP_UP_EI, - GT_INTERVAL_FROM_US(dev_priv, ei_up)); + GT_INTERVAL_FROM_US(dev_priv, ei_up)); I915_WRITE(GEN6_RP_UP_THRESHOLD, - GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100))); + GT_INTERVAL_FROM_US(dev_priv, + ei_up * threshold_up / 100)); I915_WRITE(GEN6_RP_DOWN_EI, - GT_INTERVAL_FROM_US(dev_priv, ei_down)); + GT_INTERVAL_FROM_US(dev_priv, ei_down)); I915_WRITE(GEN6_RP_DOWN_THRESHOLD, - GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100))); + GT_INTERVAL_FROM_US(dev_priv, + ei_down * threshold_down / 100)); - I915_WRITE(GEN6_RP_CONTROL, - GEN6_RP_MEDIA_TURBO | - GEN6_RP_MEDIA_HW_NORMAL_MODE | - GEN6_RP_MEDIA_IS_GFX | - GEN6_RP_ENABLE | - GEN6_RP_UP_BUSY_AVG | - GEN6_RP_DOWN_IDLE_AVG); + I915_WRITE(GEN6_RP_CONTROL, + GEN6_RP_MEDIA_TURBO | + GEN6_RP_MEDIA_HW_NORMAL_MODE | + GEN6_RP_MEDIA_IS_GFX | + GEN6_RP_ENABLE | + GEN6_RP_UP_BUSY_AVG | + GEN6_RP_DOWN_IDLE_AVG); dev_priv->rps.power = new_power; dev_priv->rps.up_threshold = threshold_up; @@ -4561,12 +4962,10 @@ static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val) /* gen6_set_rps is called to update the frequency request, but should also be * called when the range (min_delay and max_delay) is modified so that we can * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */ -static void gen6_set_rps(struct drm_device *dev, u8 val) +static void gen6_set_rps(struct drm_i915_private *dev_priv, u8 val) { - struct drm_i915_private *dev_priv = dev->dev_private; - /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */ - if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) return; WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); @@ -4579,10 +4978,10 @@ static void gen6_set_rps(struct drm_device *dev, u8 val) if (val != dev_priv->rps.cur_freq) { gen6_set_rps_thresholds(dev_priv, val); - if (IS_GEN9(dev)) + if (IS_GEN9(dev_priv)) I915_WRITE(GEN6_RPNSWREQ, GEN9_FREQUENCY(val)); - else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) I915_WRITE(GEN6_RPNSWREQ, HSW_FREQUENCY(val)); else @@ -4604,15 +5003,13 @@ static void gen6_set_rps(struct drm_device *dev, u8 val) trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val)); } -static void valleyview_set_rps(struct drm_device *dev, u8 val) +static void valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val) { - struct drm_i915_private *dev_priv = dev->dev_private; - WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); WARN_ON(val > dev_priv->rps.max_freq); WARN_ON(val < dev_priv->rps.min_freq); - if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1), + if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1), "Odd GPU freq value\n")) val &= ~1; @@ -4645,7 +5042,7 @@ static void vlv_set_rps_idle(struct drm_i915_private *dev_priv) /* Wake up the media well, as that takes a lot less * power than the Render well. */ intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA); - valleyview_set_rps(dev_priv->dev, val); + valleyview_set_rps(dev_priv, val); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA); } @@ -4657,20 +5054,33 @@ void gen6_rps_busy(struct drm_i915_private *dev_priv) gen6_rps_reset_ei(dev_priv); I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq)); + + gen6_enable_rps_interrupts(dev_priv); + + /* Ensure we start at the user's desired frequency */ + intel_set_rps(dev_priv, + clamp(dev_priv->rps.cur_freq, + dev_priv->rps.min_freq_softlimit, + dev_priv->rps.max_freq_softlimit)); } mutex_unlock(&dev_priv->rps.hw_lock); } void gen6_rps_idle(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + /* Flush our bottom-half so that it does not race with us + * setting the idle frequency and so that it is bounded by + * our rpm wakeref. And then disable the interrupts to stop any + * futher RPS reclocking whilst we are asleep. + */ + gen6_disable_rps_interrupts(dev_priv); mutex_lock(&dev_priv->rps.hw_lock); if (dev_priv->rps.enabled) { - if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) vlv_set_rps_idle(dev_priv); else - gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); + gen6_set_rps(dev_priv, dev_priv->rps.idle_freq); dev_priv->rps.last_adj = 0; I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0)); @@ -4690,7 +5100,7 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv, /* This is intentionally racy! We peek at the state here, then * validate inside the RPS worker. */ - if (!(dev_priv->mm.busy && + if (!(dev_priv->gt.awake && dev_priv->rps.enabled && dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit)) return; @@ -4706,7 +5116,7 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv, spin_lock_irq(&dev_priv->irq_lock); if (dev_priv->rps.interrupts_enabled) { dev_priv->rps.client_boost = true; - queue_work(dev_priv->wq, &dev_priv->rps.work); + schedule_work(&dev_priv->rps.work); } spin_unlock_irq(&dev_priv->irq_lock); @@ -4719,49 +5129,39 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv, spin_unlock(&dev_priv->rps.client_lock); } -void intel_set_rps(struct drm_device *dev, u8 val) +void intel_set_rps(struct drm_i915_private *dev_priv, u8 val) { - if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) - valleyview_set_rps(dev, val); + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + valleyview_set_rps(dev_priv, val); else - gen6_set_rps(dev, val); + gen6_set_rps(dev_priv, val); } -static void gen9_disable_rc6(struct drm_device *dev) +static void gen9_disable_rc6(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - I915_WRITE(GEN6_RC_CONTROL, 0); I915_WRITE(GEN9_PG_ENABLE, 0); } -static void gen9_disable_rps(struct drm_device *dev) +static void gen9_disable_rps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - I915_WRITE(GEN6_RP_CONTROL, 0); } -static void gen6_disable_rps(struct drm_device *dev) +static void gen6_disable_rps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - I915_WRITE(GEN6_RC_CONTROL, 0); I915_WRITE(GEN6_RPNSWREQ, 1 << 31); I915_WRITE(GEN6_RP_CONTROL, 0); } -static void cherryview_disable_rps(struct drm_device *dev) +static void cherryview_disable_rps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - I915_WRITE(GEN6_RC_CONTROL, 0); } -static void valleyview_disable_rps(struct drm_device *dev) +static void valleyview_disable_rps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - /* we're doing forcewake before Disabling RC6, * This what the BIOS expects when going into suspend */ intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); @@ -4771,34 +5171,45 @@ static void valleyview_disable_rps(struct drm_device *dev) intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); } -static void intel_print_rc6_info(struct drm_device *dev, u32 mode) +static void intel_print_rc6_info(struct drm_i915_private *dev_priv, u32 mode) { - if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) { + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1))) mode = GEN6_RC_CTL_RC6_ENABLE; else mode = 0; } - if (HAS_RC6p(dev)) - DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n", - onoff(mode & GEN6_RC_CTL_RC6_ENABLE), - onoff(mode & GEN6_RC_CTL_RC6p_ENABLE), - onoff(mode & GEN6_RC_CTL_RC6pp_ENABLE)); + if (HAS_RC6p(dev_priv)) + DRM_DEBUG_DRIVER("Enabling RC6 states: " + "RC6 %s RC6p %s RC6pp %s\n", + onoff(mode & GEN6_RC_CTL_RC6_ENABLE), + onoff(mode & GEN6_RC_CTL_RC6p_ENABLE), + onoff(mode & GEN6_RC_CTL_RC6pp_ENABLE)); else - DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n", - onoff(mode & GEN6_RC_CTL_RC6_ENABLE)); + DRM_DEBUG_DRIVER("Enabling RC6 states: RC6 %s\n", + onoff(mode & GEN6_RC_CTL_RC6_ENABLE)); } -static bool bxt_check_bios_rc6_setup(const struct drm_device *dev) +static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; bool enable_rc6 = true; unsigned long rc6_ctx_base; + u32 rc_ctl; + int rc_sw_target; + + rc_ctl = I915_READ(GEN6_RC_CONTROL); + rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >> + RC_SW_TARGET_STATE_SHIFT; + DRM_DEBUG_DRIVER("BIOS enabled RC states: " + "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n", + onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE), + onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE), + rc_sw_target); if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) { - DRM_DEBUG_KMS("RC6 Base location not set properly.\n"); + DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n"); enable_rc6 = false; } @@ -4810,7 +5221,7 @@ static bool bxt_check_bios_rc6_setup(const struct drm_device *dev) if (!((rc6_ctx_base >= ggtt->stolen_reserved_base) && (rc6_ctx_base + PAGE_SIZE <= ggtt->stolen_reserved_base + ggtt->stolen_reserved_size))) { - DRM_DEBUG_KMS("RC6 Base address not as expected.\n"); + DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n"); enable_rc6 = false; } @@ -4818,31 +5229,40 @@ static bool bxt_check_bios_rc6_setup(const struct drm_device *dev) ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) && ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) && ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) { - DRM_DEBUG_KMS("Engine Idle wait time not set properly.\n"); + DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n"); enable_rc6 = false; } - if (!(I915_READ(GEN6_RC_CONTROL) & (GEN6_RC_CTL_RC6_ENABLE | - GEN6_RC_CTL_HW_ENABLE)) && - ((I915_READ(GEN6_RC_CONTROL) & GEN6_RC_CTL_HW_ENABLE) || - !(I915_READ(GEN6_RC_STATE) & RC6_STATE))) { - DRM_DEBUG_KMS("HW/SW RC6 is not enabled by BIOS.\n"); + if (!I915_READ(GEN8_PUSHBUS_CONTROL) || + !I915_READ(GEN8_PUSHBUS_ENABLE) || + !I915_READ(GEN8_PUSHBUS_SHIFT)) { + DRM_DEBUG_DRIVER("Pushbus not setup properly.\n"); + enable_rc6 = false; + } + + if (!I915_READ(GEN6_GFXPAUSE)) { + DRM_DEBUG_DRIVER("GFX pause not setup properly.\n"); + enable_rc6 = false; + } + + if (!I915_READ(GEN8_MISC_CTRL0)) { + DRM_DEBUG_DRIVER("GPM control not setup properly.\n"); enable_rc6 = false; } return enable_rc6; } -int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6) +int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6) { /* No RC6 before Ironlake and code is gone for ilk. */ - if (INTEL_INFO(dev)->gen < 6) + if (INTEL_INFO(dev_priv)->gen < 6) return 0; if (!enable_rc6) return 0; - if (IS_BROXTON(dev) && !bxt_check_bios_rc6_setup(dev)) { + if (IS_BROXTON(dev_priv) && !bxt_check_bios_rc6_setup(dev_priv)) { DRM_INFO("RC6 disabled by BIOS\n"); return 0; } @@ -4851,33 +5271,28 @@ int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6) if (enable_rc6 >= 0) { int mask; - if (HAS_RC6p(dev)) + if (HAS_RC6p(dev_priv)) mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE | INTEL_RC6pp_ENABLE; else mask = INTEL_RC6_ENABLE; if ((enable_rc6 & mask) != enable_rc6) - DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n", - enable_rc6 & mask, enable_rc6, mask); + DRM_DEBUG_DRIVER("Adjusting RC6 mask to %d " + "(requested %d, valid %d)\n", + enable_rc6 & mask, enable_rc6, mask); return enable_rc6 & mask; } - if (IS_IVYBRIDGE(dev)) + if (IS_IVYBRIDGE(dev_priv)) return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE); return INTEL_RC6_ENABLE; } -int intel_enable_rc6(const struct drm_device *dev) +static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv) { - return i915.enable_rc6; -} - -static void gen6_init_rps_frequencies(struct drm_device *dev) -{ - struct drm_i915_private *dev_priv = dev->dev_private; uint32_t rp_state_cap; u32 ddcc_status = 0; int ret; @@ -4885,7 +5300,7 @@ static void gen6_init_rps_frequencies(struct drm_device *dev) /* All of these values are in units of 50MHz */ dev_priv->rps.cur_freq = 0; /* static values from HW: RP0 > RP1 > RPn (min_freq) */ - if (IS_BROXTON(dev)) { + if (IS_BROXTON(dev_priv)) { rp_state_cap = I915_READ(BXT_RP_STATE_CAP); dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff; dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff; @@ -4901,8 +5316,8 @@ static void gen6_init_rps_frequencies(struct drm_device *dev) dev_priv->rps.max_freq = dev_priv->rps.rp0_freq; dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq; - if (IS_HASWELL(dev) || IS_BROADWELL(dev) || - IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) || + IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { ret = sandybridge_pcode_read(dev_priv, HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, &ddcc_status); @@ -4914,7 +5329,7 @@ static void gen6_init_rps_frequencies(struct drm_device *dev) dev_priv->rps.max_freq); } - if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { /* Store the frequency values in 16.66 MHZ units, which is the natural hardware unit for SKL */ dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER; @@ -4931,7 +5346,7 @@ static void gen6_init_rps_frequencies(struct drm_device *dev) dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq; if (dev_priv->rps.min_freq_softlimit == 0) { - if (IS_HASWELL(dev) || IS_BROADWELL(dev)) + if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) dev_priv->rps.min_freq_softlimit = max_t(int, dev_priv->rps.efficient_freq, intel_freq_opcode(dev_priv, 450)); @@ -4942,16 +5357,14 @@ static void gen6_init_rps_frequencies(struct drm_device *dev) } /* See the Gen9_GT_PM_Programming_Guide doc for the below */ -static void gen9_enable_rps(struct drm_device *dev) +static void gen9_enable_rps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); - gen6_init_rps_frequencies(dev); + gen6_init_rps_frequencies(dev_priv); /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */ - if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) { /* * BIOS could leave the Hw Turbo enabled, so need to explicitly * clear out the Control register just to avoid inconsitency @@ -4961,7 +5374,7 @@ static void gen9_enable_rps(struct drm_device *dev) * if the Turbo is left enabled in the Control register, as the * Up/Down interrupts would remain masked. */ - gen9_disable_rps(dev); + gen9_disable_rps(dev_priv); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); return; } @@ -4980,14 +5393,13 @@ static void gen9_enable_rps(struct drm_device *dev) * Up/Down EI & threshold registers, as well as the RP_CONTROL, * RP_INTERRUPT_LIMITS & RPNSWREQ registers */ dev_priv->rps.power = HIGH_POWER; /* force a reset */ - gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); + gen6_set_rps(dev_priv, dev_priv->rps.idle_freq); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); } -static void gen9_enable_rc6(struct drm_device *dev) +static void gen9_enable_rc6(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; uint32_t rc6_mask = 0; @@ -5004,7 +5416,7 @@ static void gen9_enable_rc6(struct drm_device *dev) /* 2b: Program RC6 thresholds.*/ /* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */ - if (IS_SKYLAKE(dev)) + if (IS_SKYLAKE(dev_priv)) I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16); else I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16); @@ -5013,7 +5425,7 @@ static void gen9_enable_rc6(struct drm_device *dev) for_each_engine(engine, dev_priv) I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); - if (HAS_GUC_UCODE(dev)) + if (HAS_GUC(dev_priv)) I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA); I915_WRITE(GEN6_RC_SLEEP, 0); @@ -5023,12 +5435,12 @@ static void gen9_enable_rc6(struct drm_device *dev) I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25); /* 3a: Enable RC6 */ - if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) + if (intel_enable_rc6() & INTEL_RC6_ENABLE) rc6_mask = GEN6_RC_CTL_RC6_ENABLE; DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE)); /* WaRsUseTimeoutMode */ - if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) || - IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) || + IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) { I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */ I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | GEN7_RC_CTL_TO_MODE | @@ -5044,19 +5456,17 @@ static void gen9_enable_rc6(struct drm_device *dev) * 3b: Enable Coarse Power Gating only when RC6 is enabled. * WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6. */ - if (NEEDS_WaRsDisableCoarsePowerGating(dev)) + if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv)) I915_WRITE(GEN9_PG_ENABLE, 0); else I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? (GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE) : 0); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); - } -static void gen8_enable_rps(struct drm_device *dev) +static void gen8_enable_rps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; uint32_t rc6_mask = 0; @@ -5071,7 +5481,7 @@ static void gen8_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_CONTROL, 0); /* Initialize rps frequencies */ - gen6_init_rps_frequencies(dev); + gen6_init_rps_frequencies(dev_priv); /* 2b: Program RC6 thresholds.*/ I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); @@ -5080,16 +5490,16 @@ static void gen8_enable_rps(struct drm_device *dev) for_each_engine(engine, dev_priv) I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10); I915_WRITE(GEN6_RC_SLEEP, 0); - if (IS_BROADWELL(dev)) + if (IS_BROADWELL(dev_priv)) I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */ else I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ /* 3: Enable RC6 */ - if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) + if (intel_enable_rc6() & INTEL_RC6_ENABLE) rc6_mask = GEN6_RC_CTL_RC6_ENABLE; - intel_print_rc6_info(dev, rc6_mask); - if (IS_BROADWELL(dev)) + intel_print_rc6_info(dev_priv, rc6_mask); + if (IS_BROADWELL(dev_priv)) I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE | GEN7_RC_CTL_TO_MODE | rc6_mask); @@ -5130,14 +5540,13 @@ static void gen8_enable_rps(struct drm_device *dev) /* 6: Ring frequency + overclocking (our driver does this later */ dev_priv->rps.power = HIGH_POWER; /* force a reset */ - gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); + gen6_set_rps(dev_priv, dev_priv->rps.idle_freq); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); } -static void gen6_enable_rps(struct drm_device *dev) +static void gen6_enable_rps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; u32 rc6vids, pcu_mbox = 0, rc6_mask = 0; u32 gtfifodbg; @@ -5164,7 +5573,7 @@ static void gen6_enable_rps(struct drm_device *dev) intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); /* Initialize rps frequencies */ - gen6_init_rps_frequencies(dev); + gen6_init_rps_frequencies(dev_priv); /* disable the counters and set deterministic thresholds */ I915_WRITE(GEN6_RC_CONTROL, 0); @@ -5180,7 +5589,7 @@ static void gen6_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC_SLEEP, 0); I915_WRITE(GEN6_RC1e_THRESHOLD, 1000); - if (IS_IVYBRIDGE(dev)) + if (IS_IVYBRIDGE(dev_priv)) I915_WRITE(GEN6_RC6_THRESHOLD, 125000); else I915_WRITE(GEN6_RC6_THRESHOLD, 50000); @@ -5188,12 +5597,12 @@ static void gen6_enable_rps(struct drm_device *dev) I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */ /* Check if we are enabling RC6 */ - rc6_mode = intel_enable_rc6(dev_priv->dev); + rc6_mode = intel_enable_rc6(); if (rc6_mode & INTEL_RC6_ENABLE) rc6_mask |= GEN6_RC_CTL_RC6_ENABLE; /* We don't use those on Haswell */ - if (!IS_HASWELL(dev)) { + if (!IS_HASWELL(dev_priv)) { if (rc6_mode & INTEL_RC6p_ENABLE) rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE; @@ -5201,7 +5610,7 @@ static void gen6_enable_rps(struct drm_device *dev) rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE; } - intel_print_rc6_info(dev, rc6_mask); + intel_print_rc6_info(dev_priv, rc6_mask); I915_WRITE(GEN6_RC_CONTROL, rc6_mask | @@ -5225,13 +5634,13 @@ static void gen6_enable_rps(struct drm_device *dev) } dev_priv->rps.power = HIGH_POWER; /* force a reset */ - gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); + gen6_set_rps(dev_priv, dev_priv->rps.idle_freq); rc6vids = 0; ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids); - if (IS_GEN6(dev) && ret) { + if (IS_GEN6(dev_priv) && ret) { DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n"); - } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) { + } else if (IS_GEN6(dev_priv) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) { DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n", GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450); rc6vids &= 0xffff00; @@ -5244,9 +5653,8 @@ static void gen6_enable_rps(struct drm_device *dev) intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); } -static void __gen6_update_ring_freq(struct drm_device *dev) +static void __gen6_update_ring_freq(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; int min_freq = 15; unsigned int gpu_freq; unsigned int max_ia_freq, min_ring_freq; @@ -5275,7 +5683,7 @@ static void __gen6_update_ring_freq(struct drm_device *dev) /* convert DDR frequency from units of 266.6MHz to bandwidth */ min_ring_freq = mult_frac(min_ring_freq, 8, 3); - if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { /* Convert GT frequency to 50 HZ units */ min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER; max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER; @@ -5293,16 +5701,16 @@ static void __gen6_update_ring_freq(struct drm_device *dev) int diff = max_gpu_freq - gpu_freq; unsigned int ia_freq = 0, ring_freq = 0; - if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) { + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { /* * ring_freq = 2 * GT. ring_freq is in 100MHz units * No floor required for ring frequency on SKL. */ ring_freq = gpu_freq; - } else if (INTEL_INFO(dev)->gen >= 8) { + } else if (INTEL_INFO(dev_priv)->gen >= 8) { /* max(2 * GT, DDR). NB: GT is 50MHz units */ ring_freq = max(min_ring_freq, gpu_freq); - } else if (IS_HASWELL(dev)) { + } else if (IS_HASWELL(dev_priv)) { ring_freq = mult_frac(gpu_freq, 5, 4); ring_freq = max(min_ring_freq, ring_freq); /* leave ia_freq as the default, chosen by cpufreq */ @@ -5329,26 +5737,23 @@ static void __gen6_update_ring_freq(struct drm_device *dev) } } -void gen6_update_ring_freq(struct drm_device *dev) +void gen6_update_ring_freq(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - if (!HAS_CORE_RING_FREQ(dev)) + if (!HAS_CORE_RING_FREQ(dev_priv)) return; mutex_lock(&dev_priv->rps.hw_lock); - __gen6_update_ring_freq(dev); + __gen6_update_ring_freq(dev_priv); mutex_unlock(&dev_priv->rps.hw_lock); } static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; u32 val, rp0; val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE); - switch (INTEL_INFO(dev)->eu_total) { + switch (INTEL_INFO(dev_priv)->eu_total) { case 8: /* (2 * 4) config */ rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT); @@ -5459,9 +5864,8 @@ static void cherryview_check_pctx(struct drm_i915_private *dev_priv) WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0); } -static void cherryview_setup_pctx(struct drm_device *dev) +static void cherryview_setup_pctx(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; unsigned long pctx_paddr, paddr; u32 pcbr; @@ -5480,15 +5884,14 @@ static void cherryview_setup_pctx(struct drm_device *dev) DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR)); } -static void valleyview_setup_pctx(struct drm_device *dev) +static void valleyview_setup_pctx(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *pctx; unsigned long pctx_paddr; u32 pcbr; int pctx_size = 24*1024; - mutex_lock(&dev->struct_mutex); + mutex_lock(&dev_priv->drm.struct_mutex); pcbr = I915_READ(VLV_PCBR); if (pcbr) { @@ -5496,7 +5899,7 @@ static void valleyview_setup_pctx(struct drm_device *dev) int pcbr_offset; pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base; - pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev, + pctx = i915_gem_object_create_stolen_for_preallocated(&dev_priv->drm, pcbr_offset, I915_GTT_OFFSET_NONE, pctx_size); @@ -5513,7 +5916,7 @@ static void valleyview_setup_pctx(struct drm_device *dev) * overlap with other ranges, such as the frame buffer, protected * memory, or any other relevant ranges. */ - pctx = i915_gem_object_create_stolen(dev, pctx_size); + pctx = i915_gem_object_create_stolen(&dev_priv->drm, pctx_size); if (!pctx) { DRM_DEBUG("not enough stolen space for PCTX, disabling\n"); goto out; @@ -5525,13 +5928,11 @@ static void valleyview_setup_pctx(struct drm_device *dev) out: DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR)); dev_priv->vlv_pctx = pctx; - mutex_unlock(&dev->struct_mutex); + mutex_unlock(&dev_priv->drm.struct_mutex); } -static void valleyview_cleanup_pctx(struct drm_device *dev) +static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - if (WARN_ON(!dev_priv->vlv_pctx)) return; @@ -5550,12 +5951,11 @@ static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv) dev_priv->rps.gpll_ref_freq); } -static void valleyview_init_gt_powersave(struct drm_device *dev) +static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 val; - valleyview_setup_pctx(dev); + valleyview_setup_pctx(dev_priv); vlv_init_gpll_ref_freq(dev_priv); @@ -5609,12 +6009,11 @@ static void valleyview_init_gt_powersave(struct drm_device *dev) mutex_unlock(&dev_priv->rps.hw_lock); } -static void cherryview_init_gt_powersave(struct drm_device *dev) +static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 val; - cherryview_setup_pctx(dev); + cherryview_setup_pctx(dev_priv); vlv_init_gpll_ref_freq(dev_priv); @@ -5674,14 +6073,13 @@ static void cherryview_init_gt_powersave(struct drm_device *dev) mutex_unlock(&dev_priv->rps.hw_lock); } -static void valleyview_cleanup_gt_powersave(struct drm_device *dev) +static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv) { - valleyview_cleanup_pctx(dev); + valleyview_cleanup_pctx(dev_priv); } -static void cherryview_enable_rps(struct drm_device *dev) +static void cherryview_enable_rps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; u32 gtfifodbg, val, rc6_mode = 0, pcbr; @@ -5726,8 +6124,8 @@ static void cherryview_enable_rps(struct drm_device *dev) pcbr = I915_READ(VLV_PCBR); /* 3: Enable RC6 */ - if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) && - (pcbr >> VLV_PCBR_ADDR_SHIFT)) + if ((intel_enable_rc6() & INTEL_RC6_ENABLE) && + (pcbr >> VLV_PCBR_ADDR_SHIFT)) rc6_mode = GEN7_RC_CTL_TO_MODE; I915_WRITE(GEN6_RC_CONTROL, rc6_mode); @@ -5772,14 +6170,13 @@ static void cherryview_enable_rps(struct drm_device *dev) intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq), dev_priv->rps.idle_freq); - valleyview_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); + valleyview_set_rps(dev_priv, dev_priv->rps.idle_freq); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); } -static void valleyview_enable_rps(struct drm_device *dev) +static void valleyview_enable_rps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; u32 gtfifodbg, val, rc6_mode = 0; @@ -5832,10 +6229,10 @@ static void valleyview_enable_rps(struct drm_device *dev) VLV_MEDIA_RC6_COUNT_EN | VLV_RENDER_RC6_COUNT_EN)); - if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE) + if (intel_enable_rc6() & INTEL_RC6_ENABLE) rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL; - intel_print_rc6_info(dev, rc6_mode); + intel_print_rc6_info(dev_priv, rc6_mode); I915_WRITE(GEN6_RC_CONTROL, rc6_mode); @@ -5862,7 +6259,7 @@ static void valleyview_enable_rps(struct drm_device *dev) intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq), dev_priv->rps.idle_freq); - valleyview_set_rps(dev_priv->dev, dev_priv->rps.idle_freq); + valleyview_set_rps(dev_priv, dev_priv->rps.idle_freq); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); } @@ -5952,10 +6349,9 @@ static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv) unsigned long i915_chipset_val(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; unsigned long val; - if (INTEL_INFO(dev)->gen != 5) + if (INTEL_INFO(dev_priv)->gen != 5) return 0; spin_lock_irq(&mchdev_lock); @@ -5995,11 +6391,10 @@ static int _pxvid_to_vd(u8 pxvid) static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) { - struct drm_device *dev = dev_priv->dev; const int vd = _pxvid_to_vd(pxvid); const int vm = vd - 1125; - if (INTEL_INFO(dev)->is_mobile) + if (INTEL_INFO(dev_priv)->is_mobile) return vm > 0 ? vm : 0; return vd; @@ -6040,9 +6435,7 @@ static void __i915_update_gfx_val(struct drm_i915_private *dev_priv) void i915_update_gfx_val(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; - - if (INTEL_INFO(dev)->gen != 5) + if (INTEL_INFO(dev_priv)->gen != 5) return; spin_lock_irq(&mchdev_lock); @@ -6091,10 +6484,9 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv) unsigned long i915_gfx_val(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; unsigned long val; - if (INTEL_INFO(dev)->gen != 5) + if (INTEL_INFO(dev_priv)->gen != 5) return 0; spin_lock_irq(&mchdev_lock); @@ -6235,7 +6627,7 @@ bool i915_gpu_turbo_disable(void) dev_priv->ips.max_delay = dev_priv->ips.fstart; - if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart)) + if (!ironlake_set_drps(dev_priv, dev_priv->ips.fstart)) ret = false; out_unlock: @@ -6283,9 +6675,8 @@ void intel_gpu_ips_teardown(void) spin_unlock_irq(&mchdev_lock); } -static void intel_init_emon(struct drm_device *dev) +static void intel_init_emon(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; u32 lcfuse; u8 pxw[16]; int i; @@ -6354,10 +6745,8 @@ static void intel_init_emon(struct drm_device *dev) dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); } -void intel_init_gt_powersave(struct drm_device *dev) +void intel_init_gt_powersave(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - /* * RPM depends on RC6 to save restore the GT HW context, so make RC6 a * requirement. @@ -6367,74 +6756,64 @@ void intel_init_gt_powersave(struct drm_device *dev) intel_runtime_pm_get(dev_priv); } - if (IS_CHERRYVIEW(dev)) - cherryview_init_gt_powersave(dev); - else if (IS_VALLEYVIEW(dev)) - valleyview_init_gt_powersave(dev); + if (IS_CHERRYVIEW(dev_priv)) + cherryview_init_gt_powersave(dev_priv); + else if (IS_VALLEYVIEW(dev_priv)) + valleyview_init_gt_powersave(dev_priv); } -void intel_cleanup_gt_powersave(struct drm_device *dev) +void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - if (IS_CHERRYVIEW(dev)) - return; - else if (IS_VALLEYVIEW(dev)) - valleyview_cleanup_gt_powersave(dev); + if (IS_VALLEYVIEW(dev_priv)) + valleyview_cleanup_gt_powersave(dev_priv); if (!i915.enable_rc6) intel_runtime_pm_put(dev_priv); } -static void gen6_suspend_rps(struct drm_device *dev) +static void gen6_suspend_rps(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - flush_delayed_work(&dev_priv->rps.delayed_resume_work); - gen6_disable_rps_interrupts(dev); + gen6_disable_rps_interrupts(dev_priv); } /** * intel_suspend_gt_powersave - suspend PM work and helper threads - * @dev: drm device + * @dev_priv: i915 device * * We don't want to disable RC6 or other features here, we just want * to make sure any work we've queued has finished and won't bother * us while we're suspended. */ -void intel_suspend_gt_powersave(struct drm_device *dev) +void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - if (INTEL_INFO(dev)->gen < 6) + if (INTEL_GEN(dev_priv) < 6) return; - gen6_suspend_rps(dev); + gen6_suspend_rps(dev_priv); /* Force GPU to min freq during suspend */ gen6_rps_idle(dev_priv); } -void intel_disable_gt_powersave(struct drm_device *dev) +void intel_disable_gt_powersave(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - if (IS_IRONLAKE_M(dev)) { - ironlake_disable_drps(dev); - } else if (INTEL_INFO(dev)->gen >= 6) { - intel_suspend_gt_powersave(dev); + if (IS_IRONLAKE_M(dev_priv)) { + ironlake_disable_drps(dev_priv); + } else if (INTEL_INFO(dev_priv)->gen >= 6) { + intel_suspend_gt_powersave(dev_priv); mutex_lock(&dev_priv->rps.hw_lock); - if (INTEL_INFO(dev)->gen >= 9) { - gen9_disable_rc6(dev); - gen9_disable_rps(dev); - } else if (IS_CHERRYVIEW(dev)) - cherryview_disable_rps(dev); - else if (IS_VALLEYVIEW(dev)) - valleyview_disable_rps(dev); + if (INTEL_INFO(dev_priv)->gen >= 9) { + gen9_disable_rc6(dev_priv); + gen9_disable_rps(dev_priv); + } else if (IS_CHERRYVIEW(dev_priv)) + cherryview_disable_rps(dev_priv); + else if (IS_VALLEYVIEW(dev_priv)) + valleyview_disable_rps(dev_priv); else - gen6_disable_rps(dev); + gen6_disable_rps(dev_priv); dev_priv->rps.enabled = false; mutex_unlock(&dev_priv->rps.hw_lock); @@ -6446,27 +6825,26 @@ static void intel_gen6_powersave_work(struct work_struct *work) struct drm_i915_private *dev_priv = container_of(work, struct drm_i915_private, rps.delayed_resume_work.work); - struct drm_device *dev = dev_priv->dev; mutex_lock(&dev_priv->rps.hw_lock); - gen6_reset_rps_interrupts(dev); + gen6_reset_rps_interrupts(dev_priv); - if (IS_CHERRYVIEW(dev)) { - cherryview_enable_rps(dev); - } else if (IS_VALLEYVIEW(dev)) { - valleyview_enable_rps(dev); - } else if (INTEL_INFO(dev)->gen >= 9) { - gen9_enable_rc6(dev); - gen9_enable_rps(dev); - if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) - __gen6_update_ring_freq(dev); - } else if (IS_BROADWELL(dev)) { - gen8_enable_rps(dev); - __gen6_update_ring_freq(dev); + if (IS_CHERRYVIEW(dev_priv)) { + cherryview_enable_rps(dev_priv); + } else if (IS_VALLEYVIEW(dev_priv)) { + valleyview_enable_rps(dev_priv); + } else if (INTEL_INFO(dev_priv)->gen >= 9) { + gen9_enable_rc6(dev_priv); + gen9_enable_rps(dev_priv); + if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) + __gen6_update_ring_freq(dev_priv); + } else if (IS_BROADWELL(dev_priv)) { + gen8_enable_rps(dev_priv); + __gen6_update_ring_freq(dev_priv); } else { - gen6_enable_rps(dev); - __gen6_update_ring_freq(dev); + gen6_enable_rps(dev_priv); + __gen6_update_ring_freq(dev_priv); } WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq); @@ -6477,27 +6855,25 @@ static void intel_gen6_powersave_work(struct work_struct *work) dev_priv->rps.enabled = true; - gen6_enable_rps_interrupts(dev); + gen6_enable_rps_interrupts(dev_priv); mutex_unlock(&dev_priv->rps.hw_lock); intel_runtime_pm_put(dev_priv); } -void intel_enable_gt_powersave(struct drm_device *dev) +void intel_enable_gt_powersave(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - /* Powersaving is controlled by the host when inside a VM */ - if (intel_vgpu_active(dev)) + if (intel_vgpu_active(dev_priv)) return; - if (IS_IRONLAKE_M(dev)) { - ironlake_enable_drps(dev); - mutex_lock(&dev->struct_mutex); - intel_init_emon(dev); - mutex_unlock(&dev->struct_mutex); - } else if (INTEL_INFO(dev)->gen >= 6) { + if (IS_IRONLAKE_M(dev_priv)) { + ironlake_enable_drps(dev_priv); + mutex_lock(&dev_priv->drm.struct_mutex); + intel_init_emon(dev_priv); + mutex_unlock(&dev_priv->drm.struct_mutex); + } else if (INTEL_INFO(dev_priv)->gen >= 6) { /* * PCU communication is slow and this doesn't need to be * done at any specific time, so do this out of our fast path @@ -6516,20 +6892,18 @@ void intel_enable_gt_powersave(struct drm_device *dev) } } -void intel_reset_gt_powersave(struct drm_device *dev) +void intel_reset_gt_powersave(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - if (INTEL_INFO(dev)->gen < 6) + if (INTEL_INFO(dev_priv)->gen < 6) return; - gen6_suspend_rps(dev); + gen6_suspend_rps(dev_priv); dev_priv->rps.enabled = false; } static void ibx_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* * On Ibex Peak and Cougar Point, we need to disable clock @@ -6541,7 +6915,7 @@ static void ibx_init_clock_gating(struct drm_device *dev) static void g4x_disable_trickle_feed(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; for_each_pipe(dev_priv, pipe) { @@ -6556,7 +6930,7 @@ static void g4x_disable_trickle_feed(struct drm_device *dev) static void ilk_init_lp_watermarks(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN); I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN); @@ -6570,7 +6944,7 @@ static void ilk_init_lp_watermarks(struct drm_device *dev) static void ironlake_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; /* @@ -6644,7 +7018,7 @@ static void ironlake_init_clock_gating(struct drm_device *dev) static void cpt_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); int pipe; uint32_t val; @@ -6681,7 +7055,7 @@ static void cpt_init_clock_gating(struct drm_device *dev) static void gen6_check_mch_setup(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t tmp; tmp = I915_READ(MCH_SSKPD); @@ -6692,7 +7066,7 @@ static void gen6_check_mch_setup(struct drm_device *dev) static void gen6_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); @@ -6807,7 +7181,7 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) static void lpt_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* * TODO: this bit should only be enabled when really needed, then @@ -6826,7 +7200,7 @@ static void lpt_init_clock_gating(struct drm_device *dev) static void lpt_suspend_hw(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (HAS_PCH_LPT_LP(dev)) { uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D); @@ -6836,6 +7210,29 @@ static void lpt_suspend_hw(struct drm_device *dev) } } +static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv, + int general_prio_credits, + int high_prio_credits) +{ + u32 misccpctl; + + /* WaTempDisableDOPClkGating:bdw */ + misccpctl = I915_READ(GEN7_MISCCPCTL); + I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); + + I915_WRITE(GEN8_L3SQCREG1, + L3_GENERAL_PRIO_CREDITS(general_prio_credits) | + L3_HIGH_PRIO_CREDITS(high_prio_credits)); + + /* + * Wait at least 100 clocks before re-enabling clock gating. + * See the definition of L3SQCREG1 in BSpec. + */ + POSTING_READ(GEN8_L3SQCREG1); + udelay(1); + I915_WRITE(GEN7_MISCCPCTL, misccpctl); +} + static void kabylake_init_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -6863,6 +7260,10 @@ static void skylake_init_clock_gating(struct drm_device *dev) gen9_init_clock_gating(dev); + /* WAC6entrylatency:skl */ + I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) | + FBC_LLC_FULLY_OPEN); + /* WaFbcNukeOnHostModify:skl */ I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) | ILK_DPFC_NUKE_ON_ANY_MODIFICATION); @@ -6870,9 +7271,8 @@ static void skylake_init_clock_gating(struct drm_device *dev) static void broadwell_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; - uint32_t misccpctl; ilk_init_lp_watermarks(dev); @@ -6903,20 +7303,8 @@ static void broadwell_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | GEN8_SDEUNIT_CLOCK_GATE_DISABLE); - /* - * WaProgramL3SqcReg1Default:bdw - * WaTempDisableDOPClkGating:bdw - */ - misccpctl = I915_READ(GEN7_MISCCPCTL); - I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); - I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT); - /* - * Wait at least 100 clocks before re-enabling clock gating. See - * the definition of L3SQCREG1 in BSpec. - */ - POSTING_READ(GEN8_L3SQCREG1); - udelay(1); - I915_WRITE(GEN7_MISCCPCTL, misccpctl); + /* WaProgramL3SqcReg1Default:bdw */ + gen8_set_l3sqc_credits(dev_priv, 30, 2); /* * WaGttCachingOffByDefault:bdw @@ -6925,12 +7313,16 @@ static void broadwell_init_clock_gating(struct drm_device *dev) */ I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL); + /* WaKVMNotificationOnConfigChange:bdw */ + I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1) + | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT); + lpt_init_clock_gating(dev); } static void haswell_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); ilk_init_lp_watermarks(dev); @@ -6986,7 +7378,7 @@ static void haswell_init_clock_gating(struct drm_device *dev) static void ivybridge_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t snpcr; ilk_init_lp_watermarks(dev); @@ -7084,7 +7476,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) static void valleyview_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* WaDisableEarlyCull:vlv */ I915_WRITE(_3D_CHICKEN3, @@ -7166,7 +7558,7 @@ static void valleyview_init_clock_gating(struct drm_device *dev) static void cherryview_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* WaVSRefCountFullforceMissDisable:chv */ /* WaDSRefCountFullforceMissDisable:chv */ @@ -7187,6 +7579,13 @@ static void cherryview_init_clock_gating(struct drm_device *dev) GEN8_SDEUNIT_CLOCK_GATE_DISABLE); /* + * WaProgramL3SqcReg1Default:chv + * See gfxspecs/Related Documents/Performance Guide/ + * LSQC Setting Recommendations. + */ + gen8_set_l3sqc_credits(dev_priv, 38, 2); + + /* * GTT cache may not work with big pages, so if those * are ever enabled GTT cache may need to be disabled. */ @@ -7195,7 +7594,7 @@ static void cherryview_init_clock_gating(struct drm_device *dev) static void g4x_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dspclk_gate; I915_WRITE(RENCLK_GATE_D1, 0); @@ -7222,7 +7621,7 @@ static void g4x_init_clock_gating(struct drm_device *dev) static void crestline_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE); I915_WRITE(RENCLK_GATE_D2, 0); @@ -7238,7 +7637,7 @@ static void crestline_init_clock_gating(struct drm_device *dev) static void broadwater_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE | I965_RCC_CLOCK_GATE_DISABLE | @@ -7255,7 +7654,7 @@ static void broadwater_init_clock_gating(struct drm_device *dev) static void gen3_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 dstate = I915_READ(D_STATE); dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING | @@ -7280,7 +7679,7 @@ static void gen3_init_clock_gating(struct drm_device *dev) static void i85x_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE); @@ -7294,7 +7693,7 @@ static void i85x_init_clock_gating(struct drm_device *dev) static void i830_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE); @@ -7305,7 +7704,7 @@ static void i830_init_clock_gating(struct drm_device *dev) void intel_init_clock_gating(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); dev_priv->display.init_clock_gating(dev); } @@ -7373,7 +7772,7 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) /* Set up chip specific power management-related functions */ void intel_init_pm(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); intel_fbc_init(dev_priv); @@ -7387,6 +7786,7 @@ void intel_init_pm(struct drm_device *dev) if (INTEL_INFO(dev)->gen >= 9) { skl_setup_wm_latency(dev); dev_priv->display.update_wm = skl_update_wm; + dev_priv->display.compute_global_watermarks = skl_compute_wm; } else if (HAS_PCH_SPLIT(dev)) { ilk_setup_wm_latency(dev); @@ -7446,50 +7846,133 @@ void intel_init_pm(struct drm_device *dev) } } +static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv) +{ + uint32_t flags = + I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK; + + switch (flags) { + case GEN6_PCODE_SUCCESS: + return 0; + case GEN6_PCODE_UNIMPLEMENTED_CMD: + case GEN6_PCODE_ILLEGAL_CMD: + return -ENXIO; + case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE: + case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE: + return -EOVERFLOW; + case GEN6_PCODE_TIMEOUT: + return -ETIMEDOUT; + default: + MISSING_CASE(flags) + return 0; + } +} + +static inline int gen7_check_mailbox_status(struct drm_i915_private *dev_priv) +{ + uint32_t flags = + I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK; + + switch (flags) { + case GEN6_PCODE_SUCCESS: + return 0; + case GEN6_PCODE_ILLEGAL_CMD: + return -ENXIO; + case GEN7_PCODE_TIMEOUT: + return -ETIMEDOUT; + case GEN7_PCODE_ILLEGAL_DATA: + return -EINVAL; + case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE: + return -EOVERFLOW; + default: + MISSING_CASE(flags); + return 0; + } +} + int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val) { + int status; + WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); - if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) { + /* GEN6_PCODE_* are outside of the forcewake domain, we can + * use te fw I915_READ variants to reduce the amount of work + * required when reading/writing. + */ + + if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) { DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n"); return -EAGAIN; } - I915_WRITE(GEN6_PCODE_DATA, *val); - I915_WRITE(GEN6_PCODE_DATA1, 0); - I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); + I915_WRITE_FW(GEN6_PCODE_DATA, *val); + I915_WRITE_FW(GEN6_PCODE_DATA1, 0); + I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); - if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, - 500)) { + if (intel_wait_for_register_fw(dev_priv, + GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0, + 500)) { DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox); return -ETIMEDOUT; } - *val = I915_READ(GEN6_PCODE_DATA); - I915_WRITE(GEN6_PCODE_DATA, 0); + *val = I915_READ_FW(GEN6_PCODE_DATA); + I915_WRITE_FW(GEN6_PCODE_DATA, 0); + + if (INTEL_GEN(dev_priv) > 6) + status = gen7_check_mailbox_status(dev_priv); + else + status = gen6_check_mailbox_status(dev_priv); + + if (status) { + DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed: %d\n", + status); + return status; + } return 0; } -int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val) +int sandybridge_pcode_write(struct drm_i915_private *dev_priv, + u32 mbox, u32 val) { + int status; + WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock)); - if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) { + /* GEN6_PCODE_* are outside of the forcewake domain, we can + * use te fw I915_READ variants to reduce the amount of work + * required when reading/writing. + */ + + if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) { DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n"); return -EAGAIN; } - I915_WRITE(GEN6_PCODE_DATA, val); - I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); + I915_WRITE_FW(GEN6_PCODE_DATA, val); + I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox); - if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, - 500)) { + if (intel_wait_for_register_fw(dev_priv, + GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0, + 500)) { DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox); return -ETIMEDOUT; } - I915_WRITE(GEN6_PCODE_DATA, 0); + I915_WRITE_FW(GEN6_PCODE_DATA, 0); + + if (INTEL_GEN(dev_priv) > 6) + status = gen7_check_mailbox_status(dev_priv); + else + status = gen6_check_mailbox_status(dev_priv); + + if (status) { + DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed: %d\n", + status); + return status; + } return 0; } @@ -7559,23 +8042,21 @@ static void __intel_rps_boost_work(struct work_struct *work) struct request_boost *boost = container_of(work, struct request_boost, work); struct drm_i915_gem_request *req = boost->req; - if (!i915_gem_request_completed(req, true)) - gen6_rps_boost(to_i915(req->engine->dev), NULL, - req->emitted_jiffies); + if (!i915_gem_request_completed(req)) + gen6_rps_boost(req->i915, NULL, req->emitted_jiffies); - i915_gem_request_unreference__unlocked(req); + i915_gem_request_unreference(req); kfree(boost); } -void intel_queue_rps_boost_for_request(struct drm_device *dev, - struct drm_i915_gem_request *req) +void intel_queue_rps_boost_for_request(struct drm_i915_gem_request *req) { struct request_boost *boost; - if (req == NULL || INTEL_INFO(dev)->gen < 6) + if (req == NULL || INTEL_GEN(req->i915) < 6) return; - if (i915_gem_request_completed(req, true)) + if (i915_gem_request_completed(req)) return; boost = kmalloc(sizeof(*boost), GFP_ATOMIC); @@ -7586,12 +8067,12 @@ void intel_queue_rps_boost_for_request(struct drm_device *dev, boost->req = req; INIT_WORK(&boost->work, __intel_rps_boost_work); - queue_work(to_i915(dev)->wq, &boost->work); + queue_work(req->i915->wq, &boost->work); } void intel_pm_setup(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); mutex_init(&dev_priv->rps.hw_lock); spin_lock_init(&dev_priv->rps.client_lock); diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c index a788d1e95..cf171b4b8 100644 --- a/drivers/gpu/drm/i915/intel_psr.c +++ b/drivers/gpu/drm/i915/intel_psr.c @@ -63,7 +63,7 @@ static bool is_edp_psr(struct intel_dp *intel_dp) static bool vlv_is_psr_active_on_pipe(struct drm_device *dev, int pipe) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t val; val = I915_READ(VLV_PSRSTAT(pipe)) & @@ -77,7 +77,7 @@ static void intel_psr_write_vsc(struct intel_dp *intel_dp, { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(dig_port->base.base.crtc); enum transcoder cpu_transcoder = crtc->config->cpu_transcoder; i915_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder); @@ -107,7 +107,7 @@ static void vlv_psr_setup_vsc(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc = intel_dig_port->base.base.crtc; enum pipe pipe = to_intel_crtc(crtc)->pipe; uint32_t val; @@ -173,10 +173,9 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t aux_clock_divider; i915_reg_t aux_ctl_reg; - int precharge = 0x3; static const uint8_t aux_msg[] = { [0] = DP_AUX_NATIVE_WRITE << 4, [1] = DP_SET_POWER >> 8, @@ -185,6 +184,7 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp) [4] = DP_SET_POWER_D0, }; enum port port = dig_port->port; + u32 aux_ctl; int i; BUILD_BUG_ON(sizeof(aux_msg) > 20); @@ -197,6 +197,13 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp) DP_SINK_DEVICE_AUX_FRAME_SYNC_CONF, DP_AUX_FRAME_SYNC_ENABLE); + if (dev_priv->psr.link_standby) + drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, + DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE); + else + drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, + DP_PSR_ENABLE); + aux_ctl_reg = psr_aux_ctl_reg(dev_priv, port); /* Setup AUX registers */ @@ -204,40 +211,16 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp) I915_WRITE(psr_aux_data_reg(dev_priv, port, i >> 2), intel_dp_pack_aux(&aux_msg[i], sizeof(aux_msg) - i)); - if (INTEL_INFO(dev)->gen >= 9) { - uint32_t val; - - val = I915_READ(aux_ctl_reg); - val &= ~DP_AUX_CH_CTL_TIME_OUT_MASK; - val |= DP_AUX_CH_CTL_TIME_OUT_1600us; - val &= ~DP_AUX_CH_CTL_MESSAGE_SIZE_MASK; - val |= (sizeof(aux_msg) << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT); - /* Use hardcoded data values for PSR, frame sync and GTC */ - val &= ~DP_AUX_CH_CTL_PSR_DATA_AUX_REG_SKL; - val &= ~DP_AUX_CH_CTL_FS_DATA_AUX_REG_SKL; - val &= ~DP_AUX_CH_CTL_GTC_DATA_AUX_REG_SKL; - I915_WRITE(aux_ctl_reg, val); - } else { - I915_WRITE(aux_ctl_reg, - DP_AUX_CH_CTL_TIME_OUT_400us | - (sizeof(aux_msg) << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) | - (precharge << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) | - (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT)); - } - - if (dev_priv->psr.link_standby) - drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, - DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE); - else - drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, - DP_PSR_ENABLE); + aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, 0, sizeof(aux_msg), + aux_clock_divider); + I915_WRITE(aux_ctl_reg, aux_ctl); } static void vlv_psr_enable_source(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc = dig_port->base.base.crtc; enum pipe pipe = to_intel_crtc(crtc)->pipe; @@ -252,7 +235,7 @@ static void vlv_psr_activate(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc = dig_port->base.base.crtc; enum pipe pipe = to_intel_crtc(crtc)->pipe; @@ -269,7 +252,7 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); uint32_t max_sleep_time = 0x1f; /* @@ -341,9 +324,12 @@ static bool intel_psr_match_conditions(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc = dig_port->base.base.crtc; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + const struct drm_display_mode *adjusted_mode = + &intel_crtc->config->base.adjusted_mode; + int psr_setup_time; lockdep_assert_held(&dev_priv->psr.lock); WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); @@ -382,11 +368,25 @@ static bool intel_psr_match_conditions(struct intel_dp *intel_dp) } if (IS_HASWELL(dev) && - intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) { + adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) { DRM_DEBUG_KMS("PSR condition failed: Interlaced is Enabled\n"); return false; } + psr_setup_time = drm_dp_psr_setup_time(intel_dp->psr_dpcd); + if (psr_setup_time < 0) { + DRM_DEBUG_KMS("PSR condition failed: Invalid PSR setup time (0x%02x)\n", + intel_dp->psr_dpcd[1]); + return false; + } + + if (intel_usecs_to_scanlines(adjusted_mode, psr_setup_time) > + adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vdisplay - 1) { + DRM_DEBUG_KMS("PSR condition failed: PSR setup time (%d us) too long\n", + psr_setup_time); + return false; + } + dev_priv->psr.source_ok = true; return true; } @@ -395,7 +395,7 @@ static void intel_psr_activate(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE); WARN_ON(dev_priv->psr.active); @@ -424,7 +424,7 @@ void intel_psr_enable(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc); if (!HAS_PSR(dev)) { @@ -511,15 +511,18 @@ static void vlv_psr_disable(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(intel_dig_port->base.base.crtc); uint32_t val; if (dev_priv->psr.active) { /* Put VLV PSR back to PSR_state 0 that is PSR Disabled. */ - if (wait_for((I915_READ(VLV_PSRSTAT(intel_crtc->pipe)) & - VLV_EDP_PSR_IN_TRANS) == 0, 1)) + if (intel_wait_for_register(dev_priv, + VLV_PSRSTAT(intel_crtc->pipe), + VLV_EDP_PSR_IN_TRANS, + 0, + 1)) WARN(1, "PSR transition took longer than expected\n"); val = I915_READ(VLV_PSRCTL(intel_crtc->pipe)); @@ -538,16 +541,18 @@ static void hsw_psr_disable(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); if (dev_priv->psr.active) { I915_WRITE(EDP_PSR_CTL, I915_READ(EDP_PSR_CTL) & ~EDP_PSR_ENABLE); /* Wait till PSR is idle */ - if (_wait_for((I915_READ(EDP_PSR_STATUS_CTL) & - EDP_PSR_STATUS_STATE_MASK) == 0, - 2 * USEC_PER_SEC, 10 * USEC_PER_MSEC)) + if (intel_wait_for_register(dev_priv, + EDP_PSR_STATUS_CTL, + EDP_PSR_STATUS_STATE_MASK, + 0, + 2000)) DRM_ERROR("Timed out waiting for PSR Idle State\n"); dev_priv->psr.active = false; @@ -566,7 +571,7 @@ void intel_psr_disable(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); mutex_lock(&dev_priv->psr.lock); if (!dev_priv->psr.enabled) { @@ -603,14 +608,20 @@ static void intel_psr_work(struct work_struct *work) * and be ready for re-enable. */ if (HAS_DDI(dev_priv)) { - if (wait_for((I915_READ(EDP_PSR_STATUS_CTL) & - EDP_PSR_STATUS_STATE_MASK) == 0, 50)) { + if (intel_wait_for_register(dev_priv, + EDP_PSR_STATUS_CTL, + EDP_PSR_STATUS_STATE_MASK, + 0, + 50)) { DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n"); return; } } else { - if (wait_for((I915_READ(VLV_PSRSTAT(pipe)) & - VLV_EDP_PSR_IN_TRANS) == 0, 1)) { + if (intel_wait_for_register(dev_priv, + VLV_PSRSTAT(pipe), + VLV_EDP_PSR_IN_TRANS, + 0, + 1)) { DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n"); return; } @@ -636,7 +647,7 @@ unlock: static void intel_psr_exit(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_dp *intel_dp = dev_priv->psr.enabled; struct drm_crtc *crtc = dp_to_dig_port(intel_dp)->base.base.crtc; enum pipe pipe = to_intel_crtc(crtc)->pipe; @@ -691,7 +702,7 @@ static void intel_psr_exit(struct drm_device *dev) void intel_psr_single_frame_update(struct drm_device *dev, unsigned frontbuffer_bits) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc; enum pipe pipe; u32 val; @@ -739,7 +750,7 @@ void intel_psr_single_frame_update(struct drm_device *dev, void intel_psr_invalidate(struct drm_device *dev, unsigned frontbuffer_bits) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc; enum pipe pipe; @@ -777,7 +788,7 @@ void intel_psr_invalidate(struct drm_device *dev, void intel_psr_flush(struct drm_device *dev, unsigned frontbuffer_bits, enum fb_op_origin origin) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_crtc *crtc; enum pipe pipe; @@ -813,7 +824,7 @@ void intel_psr_flush(struct drm_device *dev, */ void intel_psr_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); dev_priv->psr_mmio_base = IS_HASWELL(dev_priv) ? HSW_EDP_PSR_BASE : BDW_EDP_PSR_BASE; diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 9d778f3ab..1d3161bbe 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -34,6 +34,11 @@ #include "i915_trace.h" #include "intel_drv.h" +/* Rough estimate of the typical request size, performing a flush, + * set-context and then emitting the batch. + */ +#define LEGACY_REQUEST_SIZE 200 + int __intel_ring_space(int head, int tail, int size) { int space = head - tail; @@ -53,18 +58,10 @@ void intel_ring_update_space(struct intel_ringbuffer *ringbuf) ringbuf->tail, ringbuf->size); } -bool intel_engine_stopped(struct intel_engine_cs *engine) -{ - struct drm_i915_private *dev_priv = engine->dev->dev_private; - return dev_priv->gpu_error.stop_rings & intel_engine_flag(engine); -} - static void __intel_ring_advance(struct intel_engine_cs *engine) { struct intel_ringbuffer *ringbuf = engine->buffer; ringbuf->tail &= ringbuf->size - 1; - if (intel_engine_stopped(engine)) - return; engine->write_tail(engine, ringbuf->tail); } @@ -101,7 +98,6 @@ gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 flush_domains) { struct intel_engine_cs *engine = req->engine; - struct drm_device *dev = engine->dev; u32 cmd; int ret; @@ -140,7 +136,7 @@ gen4_render_ring_flush(struct drm_i915_gem_request *req, cmd |= MI_EXE_FLUSH; if (invalidate_domains & I915_GEM_DOMAIN_COMMAND && - (IS_G4X(dev) || IS_GEN5(dev))) + (IS_G4X(req->i915) || IS_GEN5(req->i915))) cmd |= MI_INVALIDATE_ISP; ret = intel_ring_begin(req, 2); @@ -426,19 +422,19 @@ gen8_render_ring_flush(struct drm_i915_gem_request *req, static void ring_write_tail(struct intel_engine_cs *engine, u32 value) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; I915_WRITE_TAIL(engine, value); } u64 intel_ring_get_active_head(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; u64 acthd; - if (INTEL_INFO(engine->dev)->gen >= 8) + if (INTEL_GEN(dev_priv) >= 8) acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base), RING_ACTHD_UDW(engine->mmio_base)); - else if (INTEL_INFO(engine->dev)->gen >= 4) + else if (INTEL_GEN(dev_priv) >= 4) acthd = I915_READ(RING_ACTHD(engine->mmio_base)); else acthd = I915_READ(ACTHD); @@ -448,25 +444,24 @@ u64 intel_ring_get_active_head(struct intel_engine_cs *engine) static void ring_setup_phys_status_page(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; u32 addr; addr = dev_priv->status_page_dmah->busaddr; - if (INTEL_INFO(engine->dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0; I915_WRITE(HWS_PGA, addr); } static void intel_ring_setup_status_page(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; i915_reg_t mmio; /* The ring status page addresses are no longer next to the rest of * the ring registers as of gen7. */ - if (IS_GEN7(dev)) { + if (IS_GEN7(dev_priv)) { switch (engine->id) { case RCS: mmio = RENDER_HWS_PGA_GEN7; @@ -486,7 +481,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine) mmio = VEBOX_HWS_PGA_GEN7; break; } - } else if (IS_GEN6(engine->dev)) { + } else if (IS_GEN6(dev_priv)) { mmio = RING_HWS_PGA_GEN6(engine->mmio_base); } else { /* XXX: gen8 returns to sanity */ @@ -503,7 +498,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine) * arises: do we still need this and if so how should we go about * invalidating the TLB? */ - if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) { + if (IS_GEN(dev_priv, 6, 7)) { i915_reg_t reg = RING_INSTPM(engine->mmio_base); /* ring should be idle before issuing a sync flush*/ @@ -512,8 +507,9 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine) I915_WRITE(reg, _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | INSTPM_SYNC_FLUSH)); - if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0, - 1000)) + if (intel_wait_for_register(dev_priv, + reg, INSTPM_SYNC_FLUSH, 0, + 1000)) DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", engine->name); } @@ -521,11 +517,15 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine) static bool stop_ring(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = to_i915(engine->dev); + struct drm_i915_private *dev_priv = engine->i915; - if (!IS_GEN2(engine->dev)) { + if (!IS_GEN2(dev_priv)) { I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING)); - if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) { + if (intel_wait_for_register(dev_priv, + RING_MI_MODE(engine->mmio_base), + MODE_IDLE, + MODE_IDLE, + 1000)) { DRM_ERROR("%s : timed out trying to stop ring\n", engine->name); /* Sometimes we observe that the idle flag is not @@ -541,7 +541,7 @@ static bool stop_ring(struct intel_engine_cs *engine) I915_WRITE_HEAD(engine, 0); engine->write_tail(engine, 0); - if (!IS_GEN2(engine->dev)) { + if (!IS_GEN2(dev_priv)) { (void)I915_READ_CTL(engine); I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING)); } @@ -556,8 +556,7 @@ void intel_engine_init_hangcheck(struct intel_engine_cs *engine) static int init_ring_common(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; struct intel_ringbuffer *ringbuf = engine->buffer; struct drm_i915_gem_object *obj = ringbuf->obj; int ret = 0; @@ -587,7 +586,7 @@ static int init_ring_common(struct intel_engine_cs *engine) } } - if (I915_NEED_GFX_HWS(dev)) + if (I915_NEED_GFX_HWS(dev_priv)) intel_ring_setup_status_page(engine); else ring_setup_phys_status_page(engine); @@ -641,59 +640,42 @@ out: return ret; } -void -intel_fini_pipe_control(struct intel_engine_cs *engine) +void intel_fini_pipe_control(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - if (engine->scratch.obj == NULL) return; - if (INTEL_INFO(dev)->gen >= 5) { - kunmap(sg_page(engine->scratch.obj->pages->sgl)); - i915_gem_object_ggtt_unpin(engine->scratch.obj); - } - + i915_gem_object_ggtt_unpin(engine->scratch.obj); drm_gem_object_unreference(&engine->scratch.obj->base); engine->scratch.obj = NULL; } -int -intel_init_pipe_control(struct intel_engine_cs *engine) +int intel_init_pipe_control(struct intel_engine_cs *engine, int size) { + struct drm_i915_gem_object *obj; int ret; WARN_ON(engine->scratch.obj); - engine->scratch.obj = i915_gem_alloc_object(engine->dev, 4096); - if (engine->scratch.obj == NULL) { - DRM_ERROR("Failed to allocate seqno page\n"); - ret = -ENOMEM; + obj = i915_gem_object_create_stolen(&engine->i915->drm, size); + if (!obj) + obj = i915_gem_object_create(&engine->i915->drm, size); + if (IS_ERR(obj)) { + DRM_ERROR("Failed to allocate scratch page\n"); + ret = PTR_ERR(obj); goto err; } - ret = i915_gem_object_set_cache_level(engine->scratch.obj, - I915_CACHE_LLC); - if (ret) - goto err_unref; - - ret = i915_gem_obj_ggtt_pin(engine->scratch.obj, 4096, 0); + ret = i915_gem_obj_ggtt_pin(obj, 4096, PIN_HIGH); if (ret) goto err_unref; - engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(engine->scratch.obj); - engine->scratch.cpu_page = kmap(sg_page(engine->scratch.obj->pages->sgl)); - if (engine->scratch.cpu_page == NULL) { - ret = -ENOMEM; - goto err_unpin; - } - + engine->scratch.obj = obj; + engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj); DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n", engine->name, engine->scratch.gtt_offset); return 0; -err_unpin: - i915_gem_object_ggtt_unpin(engine->scratch.obj); err_unref: drm_gem_object_unreference(&engine->scratch.obj->base); err: @@ -702,11 +684,9 @@ err: static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req) { - int ret, i; struct intel_engine_cs *engine = req->engine; - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_workarounds *w = &dev_priv->workarounds; + struct i915_workarounds *w = &req->i915->workarounds; + int ret, i; if (w->count == 0) return 0; @@ -795,7 +775,7 @@ static int wa_add(struct drm_i915_private *dev_priv, static int wa_ring_whitelist_reg(struct intel_engine_cs *engine, i915_reg_t reg) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; struct i915_workarounds *wa = &dev_priv->workarounds; const uint32_t index = wa->hw_whitelist_count[engine->id]; @@ -811,8 +791,7 @@ static int wa_ring_whitelist_reg(struct intel_engine_cs *engine, static int gen8_init_workarounds(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING); @@ -863,9 +842,8 @@ static int gen8_init_workarounds(struct intel_engine_cs *engine) static int bdw_init_workarounds(struct intel_engine_cs *engine) { + struct drm_i915_private *dev_priv = engine->i915; int ret; - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; ret = gen8_init_workarounds(engine); if (ret) @@ -885,16 +863,15 @@ static int bdw_init_workarounds(struct intel_engine_cs *engine) /* WaForceContextSaveRestoreNonCoherent:bdw */ HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */ - (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0)); + (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0)); return 0; } static int chv_init_workarounds(struct intel_engine_cs *engine) { + struct drm_i915_private *dev_priv = engine->i915; int ret; - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; ret = gen8_init_workarounds(engine); if (ret) @@ -911,8 +888,7 @@ static int chv_init_workarounds(struct intel_engine_cs *engine) static int gen9_init_workarounds(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; int ret; /* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl */ @@ -937,14 +913,14 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine) GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC); /* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */ - if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || - IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) || + IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5, GEN9_DG_MIRROR_FIX_ENABLE); /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */ - if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) || - IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) || + IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) { WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1, GEN9_RHWO_OPTIMIZATION_DISABLE); /* @@ -970,8 +946,8 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine) GEN9_CCS_TLB_PREFETCH_ENABLE); /* WaDisableMaskBasedCammingInRCC:skl,bxt */ - if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_C0) || - IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_C0) || + IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0, PIXEL_MASK_CAMMING_DISABLE); @@ -1035,8 +1011,7 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine) static int skl_tune_iz_hashing(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; u8 vals[3] = { 0, 0, 0 }; unsigned int i; @@ -1077,9 +1052,8 @@ static int skl_tune_iz_hashing(struct intel_engine_cs *engine) static int skl_init_workarounds(struct intel_engine_cs *engine) { + struct drm_i915_private *dev_priv = engine->i915; int ret; - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; ret = gen9_init_workarounds(engine); if (ret) @@ -1090,12 +1064,12 @@ static int skl_init_workarounds(struct intel_engine_cs *engine) * until D0 which is the default case so this is equivalent to * !WaDisablePerCtxtPreemptionGranularityControl:skl */ - if (IS_SKL_REVID(dev, SKL_REVID_E0, REVID_FOREVER)) { + if (IS_SKL_REVID(dev_priv, SKL_REVID_E0, REVID_FOREVER)) { I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1, _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL)); } - if (IS_SKL_REVID(dev, 0, SKL_REVID_D0)) { + if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0)) { /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */ I915_WRITE(FF_SLICE_CS_CHICKEN2, _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE)); @@ -1104,30 +1078,30 @@ static int skl_init_workarounds(struct intel_engine_cs *engine) /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes * involving this register should also be added to WA batch as required. */ - if (IS_SKL_REVID(dev, 0, SKL_REVID_E0)) + if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0)) /* WaDisableLSQCROPERFforOCL:skl */ I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) | GEN8_LQSC_RO_PERF_DIS); /* WaEnableGapsTsvCreditFix:skl */ - if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER)) { + if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, REVID_FOREVER)) { I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) | GEN9_GAPS_TSV_CREDIT_DISABLE)); } /* WaDisablePowerCompilerClockGating:skl */ - if (IS_SKL_REVID(dev, SKL_REVID_B0, SKL_REVID_B0)) + if (IS_SKL_REVID(dev_priv, SKL_REVID_B0, SKL_REVID_B0)) WA_SET_BIT_MASKED(HIZ_CHICKEN, BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE); /* WaBarrierPerformanceFixDisable:skl */ - if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_D0)) + if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_D0)) WA_SET_BIT_MASKED(HDC_CHICKEN0, HDC_FENCE_DEST_SLM_DISABLE | HDC_BARRIER_PERFORMANCE_DISABLE); /* WaDisableSbeCacheDispatchPortSharing:skl */ - if (IS_SKL_REVID(dev, 0, SKL_REVID_F0)) + if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0)) WA_SET_BIT_MASKED( GEN7_HALF_SLICE_CHICKEN1, GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); @@ -1150,9 +1124,8 @@ static int skl_init_workarounds(struct intel_engine_cs *engine) static int bxt_init_workarounds(struct intel_engine_cs *engine) { + struct drm_i915_private *dev_priv = engine->i915; int ret; - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; ret = gen9_init_workarounds(engine); if (ret) @@ -1160,11 +1133,11 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine) /* WaStoreMultiplePTEenable:bxt */ /* This is a requirement according to Hardware specification */ - if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) + if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF); /* WaSetClckGatingDisableMedia:bxt */ - if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) { I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) & ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE)); } @@ -1173,8 +1146,14 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine) WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); + /* WaDisablePooledEuLoadBalancingFix:bxt */ + if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) { + WA_SET_BIT_MASKED(FF_SLICE_CS_CHICKEN2, + GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE); + } + /* WaDisableSbeCacheDispatchPortSharing:bxt */ - if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) { + if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) { WA_SET_BIT_MASKED( GEN7_HALF_SLICE_CHICKEN1, GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); @@ -1184,7 +1163,7 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine) /* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */ /* WaDisableObjectLevelPreemtionForInstanceId:bxt */ /* WaDisableLSQCROPERFforOCL:bxt */ - if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) { + if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) { ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1); if (ret) return ret; @@ -1194,8 +1173,13 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine) return ret; } - /* WaInsertDummyPushConstPs:bxt */ - if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) + /* WaProgramL3SqcReg1DefaultForPerf:bxt */ + if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) + I915_WRITE(GEN8_L3SQCREG1, L3_GENERAL_PRIO_CREDITS(62) | + L3_HIGH_PRIO_CREDITS(2)); + + /* WaToEnableHwFixForPushConstHWBug:bxt */ + if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER)) WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); @@ -1209,7 +1193,7 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine) static int kbl_init_workarounds(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; int ret; ret = gen9_init_workarounds(engine); @@ -1238,8 +1222,8 @@ static int kbl_init_workarounds(struct intel_engine_cs *engine) I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) | GEN8_LQSC_RO_PERF_DIS); - /* WaInsertDummyPushConstPs:kbl */ - if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) + /* WaToEnableHwFixForPushConstHWBug:kbl */ + if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER)) WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); @@ -1265,24 +1249,23 @@ static int kbl_init_workarounds(struct intel_engine_cs *engine) int init_workarounds_ring(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; WARN_ON(engine->id != RCS); dev_priv->workarounds.count = 0; dev_priv->workarounds.hw_whitelist_count[RCS] = 0; - if (IS_BROADWELL(dev)) + if (IS_BROADWELL(dev_priv)) return bdw_init_workarounds(engine); - if (IS_CHERRYVIEW(dev)) + if (IS_CHERRYVIEW(dev_priv)) return chv_init_workarounds(engine); - if (IS_SKYLAKE(dev)) + if (IS_SKYLAKE(dev_priv)) return skl_init_workarounds(engine); - if (IS_BROXTON(dev)) + if (IS_BROXTON(dev_priv)) return bxt_init_workarounds(engine); if (IS_KABYLAKE(dev_priv)) @@ -1293,14 +1276,13 @@ int init_workarounds_ring(struct intel_engine_cs *engine) static int init_render_ring(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; int ret = init_ring_common(engine); if (ret) return ret; /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */ - if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7) + if (IS_GEN(dev_priv, 4, 6)) I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH)); /* We need to disable the AsyncFlip performance optimisations in order @@ -1309,22 +1291,22 @@ static int init_render_ring(struct intel_engine_cs *engine) * * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv */ - if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) + if (IS_GEN(dev_priv, 6, 7)) I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); /* Required for the hardware to program scanline values for waiting */ /* WaEnableFlushTlbInvalidationMode:snb */ - if (INTEL_INFO(dev)->gen == 6) + if (IS_GEN6(dev_priv)) I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT)); /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */ - if (IS_GEN7(dev)) + if (IS_GEN7(dev_priv)) I915_WRITE(GFX_MODE_GEN7, _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) | _MASKED_BIT_ENABLE(GFX_REPLAY_MODE)); - if (IS_GEN6(dev)) { + if (IS_GEN6(dev_priv)) { /* From the Sandybridge PRM, volume 1 part 3, page 24: * "If this bit is set, STCunit will have LRA as replacement * policy. [...] This bit must be reset. LRA replacement @@ -1334,19 +1316,18 @@ static int init_render_ring(struct intel_engine_cs *engine) _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); } - if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) + if (IS_GEN(dev_priv, 6, 7)) I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); - if (HAS_L3_DPF(dev)) - I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev)); + if (INTEL_INFO(dev_priv)->gen >= 6) + I915_WRITE_IMR(engine, ~engine->irq_keep_mask); return init_workarounds_ring(engine); } static void render_ring_cleanup(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; if (dev_priv->semaphore_obj) { i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj); @@ -1362,13 +1343,12 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req, { #define MBOX_UPDATE_DWORDS 8 struct intel_engine_cs *signaller = signaller_req->engine; - struct drm_device *dev = signaller->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = signaller_req->i915; struct intel_engine_cs *waiter; enum intel_engine_id id; int ret, num_rings; - num_rings = hweight32(INTEL_INFO(dev)->ring_mask); + num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask); num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS; #undef MBOX_UPDATE_DWORDS @@ -1377,19 +1357,17 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req, return ret; for_each_engine_id(waiter, dev_priv, id) { - u32 seqno; u64 gtt_offset = signaller->semaphore.signal_ggtt[id]; if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) continue; - seqno = i915_gem_request_get_seqno(signaller_req); intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6)); intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE | - PIPE_CONTROL_FLUSH_ENABLE); + PIPE_CONTROL_CS_STALL); intel_ring_emit(signaller, lower_32_bits(gtt_offset)); intel_ring_emit(signaller, upper_32_bits(gtt_offset)); - intel_ring_emit(signaller, seqno); + intel_ring_emit(signaller, signaller_req->seqno); intel_ring_emit(signaller, 0); intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL | MI_SEMAPHORE_TARGET(waiter->hw_id)); @@ -1404,13 +1382,12 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req, { #define MBOX_UPDATE_DWORDS 6 struct intel_engine_cs *signaller = signaller_req->engine; - struct drm_device *dev = signaller->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = signaller_req->i915; struct intel_engine_cs *waiter; enum intel_engine_id id; int ret, num_rings; - num_rings = hweight32(INTEL_INFO(dev)->ring_mask); + num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask); num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS; #undef MBOX_UPDATE_DWORDS @@ -1419,18 +1396,16 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req, return ret; for_each_engine_id(waiter, dev_priv, id) { - u32 seqno; u64 gtt_offset = signaller->semaphore.signal_ggtt[id]; if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) continue; - seqno = i915_gem_request_get_seqno(signaller_req); intel_ring_emit(signaller, (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW); intel_ring_emit(signaller, lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT); intel_ring_emit(signaller, upper_32_bits(gtt_offset)); - intel_ring_emit(signaller, seqno); + intel_ring_emit(signaller, signaller_req->seqno); intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL | MI_SEMAPHORE_TARGET(waiter->hw_id)); intel_ring_emit(signaller, 0); @@ -1443,14 +1418,13 @@ static int gen6_signal(struct drm_i915_gem_request *signaller_req, unsigned int num_dwords) { struct intel_engine_cs *signaller = signaller_req->engine; - struct drm_device *dev = signaller->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = signaller_req->i915; struct intel_engine_cs *useless; enum intel_engine_id id; int ret, num_rings; #define MBOX_UPDATE_DWORDS 3 - num_rings = hweight32(INTEL_INFO(dev)->ring_mask); + num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask); num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2); #undef MBOX_UPDATE_DWORDS @@ -1462,11 +1436,9 @@ static int gen6_signal(struct drm_i915_gem_request *signaller_req, i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[id]; if (i915_mmio_reg_valid(mbox_reg)) { - u32 seqno = i915_gem_request_get_seqno(signaller_req); - intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit_reg(signaller, mbox_reg); - intel_ring_emit(signaller, seqno); + intel_ring_emit(signaller, signaller_req->seqno); } } @@ -1502,17 +1474,45 @@ gen6_add_request(struct drm_i915_gem_request *req) intel_ring_emit(engine, MI_STORE_DWORD_INDEX); intel_ring_emit(engine, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); + intel_ring_emit(engine, req->seqno); + intel_ring_emit(engine, MI_USER_INTERRUPT); + __intel_ring_advance(engine); + + return 0; +} + +static int +gen8_render_add_request(struct drm_i915_gem_request *req) +{ + struct intel_engine_cs *engine = req->engine; + int ret; + + if (engine->semaphore.signal) + ret = engine->semaphore.signal(req, 8); + else + ret = intel_ring_begin(req, 8); + if (ret) + return ret; + + intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(6)); + intel_ring_emit(engine, (PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_CS_STALL | + PIPE_CONTROL_QW_WRITE)); + intel_ring_emit(engine, intel_hws_seqno_address(req->engine)); + intel_ring_emit(engine, 0); intel_ring_emit(engine, i915_gem_request_get_seqno(req)); + /* We're thrashing one dword of HWS. */ + intel_ring_emit(engine, 0); intel_ring_emit(engine, MI_USER_INTERRUPT); + intel_ring_emit(engine, MI_NOOP); __intel_ring_advance(engine); return 0; } -static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev, +static inline bool i915_gem_has_seqno_wrapped(struct drm_i915_private *dev_priv, u32 seqno) { - struct drm_i915_private *dev_priv = dev->dev_private; return dev_priv->last_seqno < seqno; } @@ -1530,7 +1530,9 @@ gen8_ring_sync(struct drm_i915_gem_request *waiter_req, u32 seqno) { struct intel_engine_cs *waiter = waiter_req->engine; - struct drm_i915_private *dev_priv = waiter->dev->dev_private; + struct drm_i915_private *dev_priv = waiter_req->i915; + u64 offset = GEN8_WAIT_OFFSET(waiter, signaller->id); + struct i915_hw_ppgtt *ppgtt; int ret; ret = intel_ring_begin(waiter_req, 4); @@ -1539,14 +1541,20 @@ gen8_ring_sync(struct drm_i915_gem_request *waiter_req, intel_ring_emit(waiter, MI_SEMAPHORE_WAIT | MI_SEMAPHORE_GLOBAL_GTT | - MI_SEMAPHORE_POLL | MI_SEMAPHORE_SAD_GTE_SDD); intel_ring_emit(waiter, seqno); - intel_ring_emit(waiter, - lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id))); - intel_ring_emit(waiter, - upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id))); + intel_ring_emit(waiter, lower_32_bits(offset)); + intel_ring_emit(waiter, upper_32_bits(offset)); intel_ring_advance(waiter); + + /* When the !RCS engines idle waiting upon a semaphore, they lose their + * pagetables and we must reload them before executing the batch. + * We do this on the i915_switch_context() following the wait and + * before the dispatch. + */ + ppgtt = waiter_req->ctx->ppgtt; + if (ppgtt && waiter_req->engine->id != RCS) + ppgtt->pd_dirty_rings |= intel_engine_flag(waiter_req->engine); return 0; } @@ -1575,7 +1583,7 @@ gen6_ring_sync(struct drm_i915_gem_request *waiter_req, return ret; /* If seqno wrap happened, omit the wait with no-ops */ - if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) { + if (likely(!i915_gem_has_seqno_wrapped(waiter_req->i915, seqno))) { intel_ring_emit(waiter, dw1 | wait_mbox); intel_ring_emit(waiter, seqno); intel_ring_emit(waiter, 0); @@ -1591,72 +1599,28 @@ gen6_ring_sync(struct drm_i915_gem_request *waiter_req, return 0; } -#define PIPE_CONTROL_FLUSH(ring__, addr__) \ -do { \ - intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | \ - PIPE_CONTROL_DEPTH_STALL); \ - intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \ - intel_ring_emit(ring__, 0); \ - intel_ring_emit(ring__, 0); \ -} while (0) - -static int -pc_render_add_request(struct drm_i915_gem_request *req) +static void +gen5_seqno_barrier(struct intel_engine_cs *ring) { - struct intel_engine_cs *engine = req->engine; - u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES; - int ret; - - /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently - * incoherent with writes to memory, i.e. completely fubar, - * so we need to use PIPE_NOTIFY instead. + /* MI_STORE are internally buffered by the GPU and not flushed + * either by MI_FLUSH or SyncFlush or any other combination of + * MI commands. + * + * "Only the submission of the store operation is guaranteed. + * The write result will be complete (coherent) some time later + * (this is practically a finite period but there is no guaranteed + * latency)." * - * However, we also need to workaround the qword write - * incoherence by flushing the 6 PIPE_NOTIFY buffers out to - * memory before requesting an interrupt. + * Empirically, we observe that we need a delay of at least 75us to + * be sure that the seqno write is visible by the CPU. */ - ret = intel_ring_begin(req, 32); - if (ret) - return ret; - - intel_ring_emit(engine, - GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | - PIPE_CONTROL_WRITE_FLUSH | - PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE); - intel_ring_emit(engine, - engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); - intel_ring_emit(engine, i915_gem_request_get_seqno(req)); - intel_ring_emit(engine, 0); - PIPE_CONTROL_FLUSH(engine, scratch_addr); - scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */ - PIPE_CONTROL_FLUSH(engine, scratch_addr); - scratch_addr += 2 * CACHELINE_BYTES; - PIPE_CONTROL_FLUSH(engine, scratch_addr); - scratch_addr += 2 * CACHELINE_BYTES; - PIPE_CONTROL_FLUSH(engine, scratch_addr); - scratch_addr += 2 * CACHELINE_BYTES; - PIPE_CONTROL_FLUSH(engine, scratch_addr); - scratch_addr += 2 * CACHELINE_BYTES; - PIPE_CONTROL_FLUSH(engine, scratch_addr); - - intel_ring_emit(engine, - GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE | - PIPE_CONTROL_WRITE_FLUSH | - PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | - PIPE_CONTROL_NOTIFY); - intel_ring_emit(engine, - engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT); - intel_ring_emit(engine, i915_gem_request_get_seqno(req)); - intel_ring_emit(engine, 0); - __intel_ring_advance(engine); - - return 0; + usleep_range(125, 250); } static void gen6_seqno_barrier(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; /* Workaround to force correct ordering between irq and seqno writes on * ivb (and maybe also on snb) by reading from a CS register (like @@ -1678,133 +1642,54 @@ gen6_seqno_barrier(struct intel_engine_cs *engine) spin_unlock_irq(&dev_priv->uncore.lock); } -static u32 -ring_get_seqno(struct intel_engine_cs *engine) -{ - return intel_read_status_page(engine, I915_GEM_HWS_INDEX); -} - static void -ring_set_seqno(struct intel_engine_cs *engine, u32 seqno) -{ - intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); -} - -static u32 -pc_render_get_seqno(struct intel_engine_cs *engine) +gen5_irq_enable(struct intel_engine_cs *engine) { - return engine->scratch.cpu_page[0]; + gen5_enable_gt_irq(engine->i915, engine->irq_enable_mask); } static void -pc_render_set_seqno(struct intel_engine_cs *engine, u32 seqno) -{ - engine->scratch.cpu_page[0] = seqno; -} - -static bool -gen5_ring_get_irq(struct intel_engine_cs *engine) +gen5_irq_disable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; - - if (WARN_ON(!intel_irqs_enabled(dev_priv))) - return false; - - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (engine->irq_refcount++ == 0) - gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask); - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - - return true; + gen5_disable_gt_irq(engine->i915, engine->irq_enable_mask); } static void -gen5_ring_put_irq(struct intel_engine_cs *engine) +i9xx_irq_enable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; + struct drm_i915_private *dev_priv = engine->i915; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--engine->irq_refcount == 0) - gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask); - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); -} - -static bool -i9xx_ring_get_irq(struct intel_engine_cs *engine) -{ - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; - - if (!intel_irqs_enabled(dev_priv)) - return false; - - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (engine->irq_refcount++ == 0) { - dev_priv->irq_mask &= ~engine->irq_enable_mask; - I915_WRITE(IMR, dev_priv->irq_mask); - POSTING_READ(IMR); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - - return true; + dev_priv->irq_mask &= ~engine->irq_enable_mask; + I915_WRITE(IMR, dev_priv->irq_mask); + POSTING_READ_FW(RING_IMR(engine->mmio_base)); } static void -i9xx_ring_put_irq(struct intel_engine_cs *engine) +i9xx_irq_disable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; + struct drm_i915_private *dev_priv = engine->i915; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--engine->irq_refcount == 0) { - dev_priv->irq_mask |= engine->irq_enable_mask; - I915_WRITE(IMR, dev_priv->irq_mask); - POSTING_READ(IMR); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + dev_priv->irq_mask |= engine->irq_enable_mask; + I915_WRITE(IMR, dev_priv->irq_mask); } -static bool -i8xx_ring_get_irq(struct intel_engine_cs *engine) +static void +i8xx_irq_enable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; - - if (!intel_irqs_enabled(dev_priv)) - return false; + struct drm_i915_private *dev_priv = engine->i915; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (engine->irq_refcount++ == 0) { - dev_priv->irq_mask &= ~engine->irq_enable_mask; - I915_WRITE16(IMR, dev_priv->irq_mask); - POSTING_READ16(IMR); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - - return true; + dev_priv->irq_mask &= ~engine->irq_enable_mask; + I915_WRITE16(IMR, dev_priv->irq_mask); + POSTING_READ16(RING_IMR(engine->mmio_base)); } static void -i8xx_ring_put_irq(struct intel_engine_cs *engine) +i8xx_irq_disable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; + struct drm_i915_private *dev_priv = engine->i915; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--engine->irq_refcount == 0) { - dev_priv->irq_mask |= engine->irq_enable_mask; - I915_WRITE16(IMR, dev_priv->irq_mask); - POSTING_READ16(IMR); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + dev_priv->irq_mask |= engine->irq_enable_mask; + I915_WRITE16(IMR, dev_priv->irq_mask); } static int @@ -1838,135 +1723,68 @@ i9xx_add_request(struct drm_i915_gem_request *req) intel_ring_emit(engine, MI_STORE_DWORD_INDEX); intel_ring_emit(engine, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(engine, i915_gem_request_get_seqno(req)); + intel_ring_emit(engine, req->seqno); intel_ring_emit(engine, MI_USER_INTERRUPT); __intel_ring_advance(engine); return 0; } -static bool -gen6_ring_get_irq(struct intel_engine_cs *engine) +static void +gen6_irq_enable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; - - if (WARN_ON(!intel_irqs_enabled(dev_priv))) - return false; - - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (engine->irq_refcount++ == 0) { - if (HAS_L3_DPF(dev) && engine->id == RCS) - I915_WRITE_IMR(engine, - ~(engine->irq_enable_mask | - GT_PARITY_ERROR(dev))); - else - I915_WRITE_IMR(engine, ~engine->irq_enable_mask); - gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + struct drm_i915_private *dev_priv = engine->i915; - return true; + I915_WRITE_IMR(engine, + ~(engine->irq_enable_mask | + engine->irq_keep_mask)); + gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask); } static void -gen6_ring_put_irq(struct intel_engine_cs *engine) +gen6_irq_disable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; + struct drm_i915_private *dev_priv = engine->i915; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--engine->irq_refcount == 0) { - if (HAS_L3_DPF(dev) && engine->id == RCS) - I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev)); - else - I915_WRITE_IMR(engine, ~0); - gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + I915_WRITE_IMR(engine, ~engine->irq_keep_mask); + gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask); } -static bool -hsw_vebox_get_irq(struct intel_engine_cs *engine) +static void +hsw_vebox_irq_enable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; - - if (WARN_ON(!intel_irqs_enabled(dev_priv))) - return false; + struct drm_i915_private *dev_priv = engine->i915; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (engine->irq_refcount++ == 0) { - I915_WRITE_IMR(engine, ~engine->irq_enable_mask); - gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - - return true; + I915_WRITE_IMR(engine, ~engine->irq_enable_mask); + gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask); } static void -hsw_vebox_put_irq(struct intel_engine_cs *engine) +hsw_vebox_irq_disable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; + struct drm_i915_private *dev_priv = engine->i915; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--engine->irq_refcount == 0) { - I915_WRITE_IMR(engine, ~0); - gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + I915_WRITE_IMR(engine, ~0); + gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask); } -static bool -gen8_ring_get_irq(struct intel_engine_cs *engine) +static void +gen8_irq_enable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; + struct drm_i915_private *dev_priv = engine->i915; - if (WARN_ON(!intel_irqs_enabled(dev_priv))) - return false; - - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (engine->irq_refcount++ == 0) { - if (HAS_L3_DPF(dev) && engine->id == RCS) { - I915_WRITE_IMR(engine, - ~(engine->irq_enable_mask | - GT_RENDER_L3_PARITY_ERROR_INTERRUPT)); - } else { - I915_WRITE_IMR(engine, ~engine->irq_enable_mask); - } - POSTING_READ(RING_IMR(engine->mmio_base)); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); - - return true; + I915_WRITE_IMR(engine, + ~(engine->irq_enable_mask | + engine->irq_keep_mask)); + POSTING_READ_FW(RING_IMR(engine->mmio_base)); } static void -gen8_ring_put_irq(struct intel_engine_cs *engine) +gen8_irq_disable(struct intel_engine_cs *engine) { - struct drm_device *dev = engine->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned long flags; + struct drm_i915_private *dev_priv = engine->i915; - spin_lock_irqsave(&dev_priv->irq_lock, flags); - if (--engine->irq_refcount == 0) { - if (HAS_L3_DPF(dev) && engine->id == RCS) { - I915_WRITE_IMR(engine, - ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT); - } else { - I915_WRITE_IMR(engine, ~0); - } - POSTING_READ(RING_IMR(engine->mmio_base)); - } - spin_unlock_irqrestore(&dev_priv->irq_lock, flags); + I915_WRITE_IMR(engine, ~engine->irq_keep_mask); } static int @@ -2080,12 +1898,12 @@ i915_dispatch_execbuffer(struct drm_i915_gem_request *req, static void cleanup_phys_status_page(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = to_i915(engine->dev); + struct drm_i915_private *dev_priv = engine->i915; if (!dev_priv->status_page_dmah) return; - drm_pci_free(engine->dev, dev_priv->status_page_dmah); + drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah); engine->status_page.page_addr = NULL; } @@ -2111,10 +1929,10 @@ static int init_status_page(struct intel_engine_cs *engine) unsigned flags; int ret; - obj = i915_gem_alloc_object(engine->dev, 4096); - if (obj == NULL) { + obj = i915_gem_object_create(&engine->i915->drm, 4096); + if (IS_ERR(obj)) { DRM_ERROR("Failed to allocate status page\n"); - return -ENOMEM; + return PTR_ERR(obj); } ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); @@ -2122,7 +1940,7 @@ static int init_status_page(struct intel_engine_cs *engine) goto err_unref; flags = 0; - if (!HAS_LLC(engine->dev)) + if (!HAS_LLC(engine->i915)) /* On g33, we cannot place HWS above 256MiB, so * restrict its pinning to the low mappable arena. * Though this restriction is not documented for @@ -2156,11 +1974,11 @@ err_unref: static int init_phys_status_page(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; if (!dev_priv->status_page_dmah) { dev_priv->status_page_dmah = - drm_pci_alloc(engine->dev, PAGE_SIZE, PAGE_SIZE); + drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE); if (!dev_priv->status_page_dmah) return -ENOMEM; } @@ -2173,20 +1991,22 @@ static int init_phys_status_page(struct intel_engine_cs *engine) void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf) { + GEM_BUG_ON(ringbuf->vma == NULL); + GEM_BUG_ON(ringbuf->virtual_start == NULL); + if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen) i915_gem_object_unpin_map(ringbuf->obj); else - iounmap(ringbuf->virtual_start); + i915_vma_unpin_iomap(ringbuf->vma); ringbuf->virtual_start = NULL; - ringbuf->vma = NULL; + i915_gem_object_ggtt_unpin(ringbuf->obj); + ringbuf->vma = NULL; } -int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev, +int intel_pin_and_map_ringbuffer_obj(struct drm_i915_private *dev_priv, struct intel_ringbuffer *ringbuf) { - struct drm_i915_private *dev_priv = to_i915(dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; struct drm_i915_gem_object *obj = ringbuf->obj; /* Ring wraparound at offset 0 sometimes hangs. No idea why. */ unsigned flags = PIN_OFFSET_BIAS | 4096; @@ -2220,10 +2040,9 @@ int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev, /* Access through the GTT requires the device to be awake. */ assert_rpm_wakelock_held(dev_priv); - addr = ioremap_wc(ggtt->mappable_base + - i915_gem_obj_ggtt_offset(obj), ringbuf->size); - if (addr == NULL) { - ret = -ENOMEM; + addr = i915_vma_pin_iomap(i915_gem_obj_to_ggtt(obj)); + if (IS_ERR(addr)) { + ret = PTR_ERR(addr); goto err_unpin; } } @@ -2252,9 +2071,9 @@ static int intel_alloc_ringbuffer_obj(struct drm_device *dev, if (!HAS_LLC(dev)) obj = i915_gem_object_create_stolen(dev, ringbuf->size); if (obj == NULL) - obj = i915_gem_alloc_object(dev, ringbuf->size); - if (obj == NULL) - return -ENOMEM; + obj = i915_gem_object_create(dev, ringbuf->size); + if (IS_ERR(obj)) + return PTR_ERR(obj); /* mark ring buffers as read-only from GPU side by default */ obj->gt_ro = 1; @@ -2286,13 +2105,13 @@ intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size) * of the buffer. */ ring->effective_size = size; - if (IS_I830(engine->dev) || IS_845G(engine->dev)) + if (IS_I830(engine->i915) || IS_845G(engine->i915)) ring->effective_size -= 2 * CACHELINE_BYTES; ring->last_retired_head = -1; intel_ring_update_space(ring); - ret = intel_alloc_ringbuffer_obj(engine->dev, ring); + ret = intel_alloc_ringbuffer_obj(&engine->i915->drm, ring); if (ret) { DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n", engine->name, ret); @@ -2312,15 +2131,67 @@ intel_ringbuffer_free(struct intel_ringbuffer *ring) kfree(ring); } +static int intel_ring_context_pin(struct i915_gem_context *ctx, + struct intel_engine_cs *engine) +{ + struct intel_context *ce = &ctx->engine[engine->id]; + int ret; + + lockdep_assert_held(&ctx->i915->drm.struct_mutex); + + if (ce->pin_count++) + return 0; + + if (ce->state) { + ret = i915_gem_obj_ggtt_pin(ce->state, ctx->ggtt_alignment, 0); + if (ret) + goto error; + } + + /* The kernel context is only used as a placeholder for flushing the + * active context. It is never used for submitting user rendering and + * as such never requires the golden render context, and so we can skip + * emitting it when we switch to the kernel context. This is required + * as during eviction we cannot allocate and pin the renderstate in + * order to initialise the context. + */ + if (ctx == ctx->i915->kernel_context) + ce->initialised = true; + + i915_gem_context_reference(ctx); + return 0; + +error: + ce->pin_count = 0; + return ret; +} + +static void intel_ring_context_unpin(struct i915_gem_context *ctx, + struct intel_engine_cs *engine) +{ + struct intel_context *ce = &ctx->engine[engine->id]; + + lockdep_assert_held(&ctx->i915->drm.struct_mutex); + + if (--ce->pin_count) + return; + + if (ce->state) + i915_gem_object_ggtt_unpin(ce->state); + + i915_gem_context_unreference(ctx); +} + static int intel_init_ring_buffer(struct drm_device *dev, struct intel_engine_cs *engine) { + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_ringbuffer *ringbuf; int ret; WARN_ON(engine->buffer); - engine->dev = dev; + engine->i915 = dev_priv; INIT_LIST_HEAD(&engine->active_list); INIT_LIST_HEAD(&engine->request_list); INIT_LIST_HEAD(&engine->execlist_queue); @@ -2329,7 +2200,20 @@ static int intel_init_ring_buffer(struct drm_device *dev, memset(engine->semaphore.sync_seqno, 0, sizeof(engine->semaphore.sync_seqno)); - init_waitqueue_head(&engine->irq_queue); + ret = intel_engine_init_breadcrumbs(engine); + if (ret) + goto error; + + /* We may need to do things with the shrinker which + * require us to immediately switch back to the default + * context. This can cause a problem as pinning the + * default context also requires GTT space which may not + * be available. To avoid this we always pin the default + * context. + */ + ret = intel_ring_context_pin(dev_priv->kernel_context, engine); + if (ret) + goto error; ringbuf = intel_engine_create_ringbuffer(engine, 32 * PAGE_SIZE); if (IS_ERR(ringbuf)) { @@ -2338,7 +2222,7 @@ static int intel_init_ring_buffer(struct drm_device *dev, } engine->buffer = ringbuf; - if (I915_NEED_GFX_HWS(dev)) { + if (I915_NEED_GFX_HWS(dev_priv)) { ret = init_status_page(engine); if (ret) goto error; @@ -2349,7 +2233,7 @@ static int intel_init_ring_buffer(struct drm_device *dev, goto error; } - ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf); + ret = intel_pin_and_map_ringbuffer_obj(dev_priv, ringbuf); if (ret) { DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n", engine->name, ret); @@ -2375,11 +2259,11 @@ void intel_cleanup_engine(struct intel_engine_cs *engine) if (!intel_engine_initialized(engine)) return; - dev_priv = to_i915(engine->dev); + dev_priv = engine->i915; if (engine->buffer) { intel_stop_engine(engine); - WARN_ON(!IS_GEN2(engine->dev) && (I915_READ_MODE(engine) & MODE_IDLE) == 0); + WARN_ON(!IS_GEN2(dev_priv) && (I915_READ_MODE(engine) & MODE_IDLE) == 0); intel_unpin_ringbuffer_obj(engine->buffer); intel_ringbuffer_free(engine->buffer); @@ -2389,7 +2273,7 @@ void intel_cleanup_engine(struct intel_engine_cs *engine) if (engine->cleanup) engine->cleanup(engine); - if (I915_NEED_GFX_HWS(engine->dev)) { + if (I915_NEED_GFX_HWS(dev_priv)) { cleanup_status_page(engine); } else { WARN_ON(engine->id != RCS); @@ -2398,7 +2282,11 @@ void intel_cleanup_engine(struct intel_engine_cs *engine) i915_cmd_parser_fini_ring(engine); i915_gem_batch_pool_fini(&engine->batch_pool); - engine->dev = NULL; + intel_engine_fini_breadcrumbs(engine); + + intel_ring_context_unpin(dev_priv->kernel_context, engine); + + engine->i915 = NULL; } int intel_engine_idle(struct intel_engine_cs *engine) @@ -2421,46 +2309,22 @@ int intel_engine_idle(struct intel_engine_cs *engine) int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request) { - request->ringbuf = request->engine->buffer; - return 0; -} + int ret; -int intel_ring_reserve_space(struct drm_i915_gem_request *request) -{ - /* - * The first call merely notes the reserve request and is common for - * all back ends. The subsequent localised _begin() call actually - * ensures that the reservation is available. Without the begin, if - * the request creator immediately submitted the request without - * adding any commands to it then there might not actually be - * sufficient room for the submission commands. + /* Flush enough space to reduce the likelihood of waiting after + * we start building the request - in which case we will just + * have to repeat work. */ - intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST); - - return intel_ring_begin(request, 0); -} - -void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size) -{ - GEM_BUG_ON(ringbuf->reserved_size); - ringbuf->reserved_size = size; -} + request->reserved_space += LEGACY_REQUEST_SIZE; -void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf) -{ - GEM_BUG_ON(!ringbuf->reserved_size); - ringbuf->reserved_size = 0; -} + request->ringbuf = request->engine->buffer; -void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf) -{ - GEM_BUG_ON(!ringbuf->reserved_size); - ringbuf->reserved_size = 0; -} + ret = intel_ring_begin(request, 0); + if (ret) + return ret; -void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf) -{ - GEM_BUG_ON(ringbuf->reserved_size); + request->reserved_space -= LEGACY_REQUEST_SIZE; + return 0; } static int wait_for_space(struct drm_i915_gem_request *req, int bytes) @@ -2482,7 +2346,7 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes) * * See also i915_gem_request_alloc() and i915_add_request(). */ - GEM_BUG_ON(!ringbuf->reserved_size); + GEM_BUG_ON(!req->reserved_space); list_for_each_entry(target, &engine->request_list, list) { unsigned space; @@ -2517,7 +2381,7 @@ int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords) int total_bytes, wait_bytes; bool need_wrap = false; - total_bytes = bytes + ringbuf->reserved_size; + total_bytes = bytes + req->reserved_space; if (unlikely(bytes > remain_usable)) { /* @@ -2533,7 +2397,7 @@ int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords) * and only need to effectively wait for the reserved * size space from the start of ringbuffer. */ - wait_bytes = remain_actual + ringbuf->reserved_size; + wait_bytes = remain_actual + req->reserved_space; } else { /* No wrapping required, just waiting. */ wait_bytes = total_bytes; @@ -2590,7 +2454,7 @@ int intel_ring_cacheline_align(struct drm_i915_gem_request *req) void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno) { - struct drm_i915_private *dev_priv = to_i915(engine->dev); + struct drm_i915_private *dev_priv = engine->i915; /* Our semaphore implementation is strictly monotonic (i.e. we proceed * so long as the semaphore value in the register/page is greater @@ -2600,7 +2464,7 @@ void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno) * the semaphore value, then when the seqno moves backwards all * future waits will complete instantly (causing rendering corruption). */ - if (INTEL_INFO(dev_priv)->gen == 6 || INTEL_INFO(dev_priv)->gen == 7) { + if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) { I915_WRITE(RING_SYNC_0(engine->mmio_base), 0); I915_WRITE(RING_SYNC_1(engine->mmio_base), 0); if (HAS_VEBOX(dev_priv)) @@ -2617,43 +2481,58 @@ void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno) memset(engine->semaphore.sync_seqno, 0, sizeof(engine->semaphore.sync_seqno)); - engine->set_seqno(engine, seqno); + intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); + if (engine->irq_seqno_barrier) + engine->irq_seqno_barrier(engine); engine->last_submitted_seqno = seqno; engine->hangcheck.seqno = seqno; + + /* After manually advancing the seqno, fake the interrupt in case + * there are any waiters for that seqno. + */ + rcu_read_lock(); + intel_engine_wakeup(engine); + rcu_read_unlock(); } static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine, u32 value) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; + + intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); /* Every tail move must follow the sequence below */ /* Disable notification that the ring is IDLE. The GT * will then assume that it is busy and bring it out of rc6. */ - I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, - _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); + I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL, + _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); /* Clear the context id. Here be magic! */ - I915_WRITE64(GEN6_BSD_RNCID, 0x0); + I915_WRITE64_FW(GEN6_BSD_RNCID, 0x0); /* Wait for the ring not to be idle, i.e. for it to wake up. */ - if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) & - GEN6_BSD_SLEEP_INDICATOR) == 0, - 50)) + if (intel_wait_for_register_fw(dev_priv, + GEN6_BSD_SLEEP_PSMI_CONTROL, + GEN6_BSD_SLEEP_INDICATOR, + 0, + 50)) DRM_ERROR("timed out waiting for the BSD ring to wake up\n"); /* Now that the ring is fully powered up, update the tail */ - I915_WRITE_TAIL(engine, value); - POSTING_READ(RING_TAIL(engine->mmio_base)); + I915_WRITE_FW(RING_TAIL(engine->mmio_base), value); + POSTING_READ_FW(RING_TAIL(engine->mmio_base)); /* Let the ring send IDLE messages to the GT again, * and so let it sleep to conserve power when idle. */ - I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, - _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); + I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL, + _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); + + intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); } static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, @@ -2668,7 +2547,7 @@ static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, return ret; cmd = MI_FLUSH_DW; - if (INTEL_INFO(engine->dev)->gen >= 8) + if (INTEL_GEN(req->i915) >= 8) cmd += 1; /* We always require a command barrier so that subsequent @@ -2690,7 +2569,7 @@ static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, intel_ring_emit(engine, cmd); intel_ring_emit(engine, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); - if (INTEL_INFO(engine->dev)->gen >= 8) { + if (INTEL_GEN(req->i915) >= 8) { intel_ring_emit(engine, 0); /* upper addr */ intel_ring_emit(engine, 0); /* value */ } else { @@ -2781,7 +2660,6 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 invalidate, u32 flush) { struct intel_engine_cs *engine = req->engine; - struct drm_device *dev = engine->dev; uint32_t cmd; int ret; @@ -2790,7 +2668,7 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req, return ret; cmd = MI_FLUSH_DW; - if (INTEL_INFO(dev)->gen >= 8) + if (INTEL_GEN(req->i915) >= 8) cmd += 1; /* We always require a command barrier so that subsequent @@ -2811,7 +2689,7 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req, intel_ring_emit(engine, cmd); intel_ring_emit(engine, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT); - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(req->i915) >= 8) { intel_ring_emit(engine, 0); /* upper addr */ intel_ring_emit(engine, 0); /* value */ } else { @@ -2823,11 +2701,159 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req, return 0; } +static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv, + struct intel_engine_cs *engine) +{ + struct drm_i915_gem_object *obj; + int ret, i; + + if (!i915_semaphore_is_enabled(dev_priv)) + return; + + if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore_obj) { + obj = i915_gem_object_create(&dev_priv->drm, 4096); + if (IS_ERR(obj)) { + DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n"); + i915.semaphores = 0; + } else { + i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); + ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK); + if (ret != 0) { + drm_gem_object_unreference(&obj->base); + DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n"); + i915.semaphores = 0; + } else { + dev_priv->semaphore_obj = obj; + } + } + } + + if (!i915_semaphore_is_enabled(dev_priv)) + return; + + if (INTEL_GEN(dev_priv) >= 8) { + u64 offset = i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj); + + engine->semaphore.sync_to = gen8_ring_sync; + engine->semaphore.signal = gen8_xcs_signal; + + for (i = 0; i < I915_NUM_ENGINES; i++) { + u64 ring_offset; + + if (i != engine->id) + ring_offset = offset + GEN8_SEMAPHORE_OFFSET(engine->id, i); + else + ring_offset = MI_SEMAPHORE_SYNC_INVALID; + + engine->semaphore.signal_ggtt[i] = ring_offset; + } + } else if (INTEL_GEN(dev_priv) >= 6) { + engine->semaphore.sync_to = gen6_ring_sync; + engine->semaphore.signal = gen6_signal; + + /* + * The current semaphore is only applied on pre-gen8 + * platform. And there is no VCS2 ring on the pre-gen8 + * platform. So the semaphore between RCS and VCS2 is + * initialized as INVALID. Gen8 will initialize the + * sema between VCS2 and RCS later. + */ + for (i = 0; i < I915_NUM_ENGINES; i++) { + static const struct { + u32 wait_mbox; + i915_reg_t mbox_reg; + } sem_data[I915_NUM_ENGINES][I915_NUM_ENGINES] = { + [RCS] = { + [VCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RV, .mbox_reg = GEN6_VRSYNC }, + [BCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RB, .mbox_reg = GEN6_BRSYNC }, + [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC }, + }, + [VCS] = { + [RCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VR, .mbox_reg = GEN6_RVSYNC }, + [BCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VB, .mbox_reg = GEN6_BVSYNC }, + [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC }, + }, + [BCS] = { + [RCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BR, .mbox_reg = GEN6_RBSYNC }, + [VCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BV, .mbox_reg = GEN6_VBSYNC }, + [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC }, + }, + [VECS] = { + [RCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC }, + [VCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC }, + [BCS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC }, + }, + }; + u32 wait_mbox; + i915_reg_t mbox_reg; + + if (i == engine->id || i == VCS2) { + wait_mbox = MI_SEMAPHORE_SYNC_INVALID; + mbox_reg = GEN6_NOSYNC; + } else { + wait_mbox = sem_data[engine->id][i].wait_mbox; + mbox_reg = sem_data[engine->id][i].mbox_reg; + } + + engine->semaphore.mbox.wait[i] = wait_mbox; + engine->semaphore.mbox.signal[i] = mbox_reg; + } + } +} + +static void intel_ring_init_irq(struct drm_i915_private *dev_priv, + struct intel_engine_cs *engine) +{ + if (INTEL_GEN(dev_priv) >= 8) { + engine->irq_enable = gen8_irq_enable; + engine->irq_disable = gen8_irq_disable; + engine->irq_seqno_barrier = gen6_seqno_barrier; + } else if (INTEL_GEN(dev_priv) >= 6) { + engine->irq_enable = gen6_irq_enable; + engine->irq_disable = gen6_irq_disable; + engine->irq_seqno_barrier = gen6_seqno_barrier; + } else if (INTEL_GEN(dev_priv) >= 5) { + engine->irq_enable = gen5_irq_enable; + engine->irq_disable = gen5_irq_disable; + engine->irq_seqno_barrier = gen5_seqno_barrier; + } else if (INTEL_GEN(dev_priv) >= 3) { + engine->irq_enable = i9xx_irq_enable; + engine->irq_disable = i9xx_irq_disable; + } else { + engine->irq_enable = i8xx_irq_enable; + engine->irq_disable = i8xx_irq_disable; + } +} + +static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv, + struct intel_engine_cs *engine) +{ + engine->init_hw = init_ring_common; + engine->write_tail = ring_write_tail; + + engine->add_request = i9xx_add_request; + if (INTEL_GEN(dev_priv) >= 6) + engine->add_request = gen6_add_request; + + if (INTEL_GEN(dev_priv) >= 8) + engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; + else if (INTEL_GEN(dev_priv) >= 6) + engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; + else if (INTEL_GEN(dev_priv) >= 4) + engine->dispatch_execbuffer = i965_dispatch_execbuffer; + else if (IS_I830(dev_priv) || IS_845G(dev_priv)) + engine->dispatch_execbuffer = i830_dispatch_execbuffer; + else + engine->dispatch_execbuffer = i915_dispatch_execbuffer; + + intel_ring_init_irq(dev_priv, engine); + intel_ring_init_semaphores(dev_priv, engine); +} + int intel_init_render_ring_buffer(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine = &dev_priv->engine[RCS]; - struct drm_i915_gem_object *obj; int ret; engine->name = "render ring"; @@ -2836,140 +2862,49 @@ int intel_init_render_ring_buffer(struct drm_device *dev) engine->hw_id = 0; engine->mmio_base = RENDER_RING_BASE; - if (INTEL_INFO(dev)->gen >= 8) { - if (i915_semaphore_is_enabled(dev)) { - obj = i915_gem_alloc_object(dev, 4096); - if (obj == NULL) { - DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n"); - i915.semaphores = 0; - } else { - i915_gem_object_set_cache_level(obj, I915_CACHE_LLC); - ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK); - if (ret != 0) { - drm_gem_object_unreference(&obj->base); - DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n"); - i915.semaphores = 0; - } else - dev_priv->semaphore_obj = obj; - } - } + intel_ring_default_vfuncs(dev_priv, engine); + + engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; + if (HAS_L3_DPF(dev_priv)) + engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT; + if (INTEL_GEN(dev_priv) >= 8) { engine->init_context = intel_rcs_ctx_init; - engine->add_request = gen6_add_request; + engine->add_request = gen8_render_add_request; engine->flush = gen8_render_ring_flush; - engine->irq_get = gen8_ring_get_irq; - engine->irq_put = gen8_ring_put_irq; - engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; - engine->irq_seqno_barrier = gen6_seqno_barrier; - engine->get_seqno = ring_get_seqno; - engine->set_seqno = ring_set_seqno; - if (i915_semaphore_is_enabled(dev)) { - WARN_ON(!dev_priv->semaphore_obj); - engine->semaphore.sync_to = gen8_ring_sync; + if (i915_semaphore_is_enabled(dev_priv)) engine->semaphore.signal = gen8_rcs_signal; - GEN8_RING_SEMAPHORE_INIT(engine); - } - } else if (INTEL_INFO(dev)->gen >= 6) { + } else if (INTEL_GEN(dev_priv) >= 6) { engine->init_context = intel_rcs_ctx_init; - engine->add_request = gen6_add_request; engine->flush = gen7_render_ring_flush; - if (INTEL_INFO(dev)->gen == 6) + if (IS_GEN6(dev_priv)) engine->flush = gen6_render_ring_flush; - engine->irq_get = gen6_ring_get_irq; - engine->irq_put = gen6_ring_put_irq; - engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; - engine->irq_seqno_barrier = gen6_seqno_barrier; - engine->get_seqno = ring_get_seqno; - engine->set_seqno = ring_set_seqno; - if (i915_semaphore_is_enabled(dev)) { - engine->semaphore.sync_to = gen6_ring_sync; - engine->semaphore.signal = gen6_signal; - /* - * The current semaphore is only applied on pre-gen8 - * platform. And there is no VCS2 ring on the pre-gen8 - * platform. So the semaphore between RCS and VCS2 is - * initialized as INVALID. Gen8 will initialize the - * sema between VCS2 and RCS later. - */ - engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID; - engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV; - engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB; - engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE; - engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; - engine->semaphore.mbox.signal[RCS] = GEN6_NOSYNC; - engine->semaphore.mbox.signal[VCS] = GEN6_VRSYNC; - engine->semaphore.mbox.signal[BCS] = GEN6_BRSYNC; - engine->semaphore.mbox.signal[VECS] = GEN6_VERSYNC; - engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; - } - } else if (IS_GEN5(dev)) { - engine->add_request = pc_render_add_request; + } else if (IS_GEN5(dev_priv)) { engine->flush = gen4_render_ring_flush; - engine->get_seqno = pc_render_get_seqno; - engine->set_seqno = pc_render_set_seqno; - engine->irq_get = gen5_ring_get_irq; - engine->irq_put = gen5_ring_put_irq; - engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT | - GT_RENDER_PIPECTL_NOTIFY_INTERRUPT; } else { - engine->add_request = i9xx_add_request; - if (INTEL_INFO(dev)->gen < 4) + if (INTEL_GEN(dev_priv) < 4) engine->flush = gen2_render_ring_flush; else engine->flush = gen4_render_ring_flush; - engine->get_seqno = ring_get_seqno; - engine->set_seqno = ring_set_seqno; - if (IS_GEN2(dev)) { - engine->irq_get = i8xx_ring_get_irq; - engine->irq_put = i8xx_ring_put_irq; - } else { - engine->irq_get = i9xx_ring_get_irq; - engine->irq_put = i9xx_ring_put_irq; - } engine->irq_enable_mask = I915_USER_INTERRUPT; } - engine->write_tail = ring_write_tail; - if (IS_HASWELL(dev)) + if (IS_HASWELL(dev_priv)) engine->dispatch_execbuffer = hsw_ring_dispatch_execbuffer; - else if (IS_GEN8(dev)) - engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; - else if (INTEL_INFO(dev)->gen >= 6) - engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; - else if (INTEL_INFO(dev)->gen >= 4) - engine->dispatch_execbuffer = i965_dispatch_execbuffer; - else if (IS_I830(dev) || IS_845G(dev)) - engine->dispatch_execbuffer = i830_dispatch_execbuffer; - else - engine->dispatch_execbuffer = i915_dispatch_execbuffer; + engine->init_hw = init_render_ring; engine->cleanup = render_ring_cleanup; - /* Workaround batchbuffer to combat CS tlb bug. */ - if (HAS_BROKEN_CS_TLB(dev)) { - obj = i915_gem_alloc_object(dev, I830_WA_SIZE); - if (obj == NULL) { - DRM_ERROR("Failed to allocate batch bo\n"); - return -ENOMEM; - } - - ret = i915_gem_obj_ggtt_pin(obj, 0, 0); - if (ret != 0) { - drm_gem_object_unreference(&obj->base); - DRM_ERROR("Failed to ping batch bo\n"); - return ret; - } - - engine->scratch.obj = obj; - engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj); - } - ret = intel_init_ring_buffer(dev, engine); if (ret) return ret; - if (INTEL_INFO(dev)->gen >= 5) { - ret = intel_init_pipe_control(engine); + if (INTEL_GEN(dev_priv) >= 6) { + ret = intel_init_pipe_control(engine, 4096); + if (ret) + return ret; + } else if (HAS_BROKEN_CS_TLB(dev_priv)) { + ret = intel_init_pipe_control(engine, I830_WA_SIZE); if (ret) return ret; } @@ -2979,7 +2914,7 @@ int intel_init_render_ring_buffer(struct drm_device *dev) int intel_init_bsd_ring_buffer(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine = &dev_priv->engine[VCS]; engine->name = "bsd ring"; @@ -2987,68 +2922,27 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev) engine->exec_id = I915_EXEC_BSD; engine->hw_id = 1; - engine->write_tail = ring_write_tail; - if (INTEL_INFO(dev)->gen >= 6) { + intel_ring_default_vfuncs(dev_priv, engine); + + if (INTEL_GEN(dev_priv) >= 6) { engine->mmio_base = GEN6_BSD_RING_BASE; /* gen6 bsd needs a special wa for tail updates */ - if (IS_GEN6(dev)) + if (IS_GEN6(dev_priv)) engine->write_tail = gen6_bsd_ring_write_tail; engine->flush = gen6_bsd_ring_flush; - engine->add_request = gen6_add_request; - engine->irq_seqno_barrier = gen6_seqno_barrier; - engine->get_seqno = ring_get_seqno; - engine->set_seqno = ring_set_seqno; - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT; - engine->irq_get = gen8_ring_get_irq; - engine->irq_put = gen8_ring_put_irq; - engine->dispatch_execbuffer = - gen8_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - engine->semaphore.sync_to = gen8_ring_sync; - engine->semaphore.signal = gen8_xcs_signal; - GEN8_RING_SEMAPHORE_INIT(engine); - } - } else { + else engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; - engine->irq_get = gen6_ring_get_irq; - engine->irq_put = gen6_ring_put_irq; - engine->dispatch_execbuffer = - gen6_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - engine->semaphore.sync_to = gen6_ring_sync; - engine->semaphore.signal = gen6_signal; - engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR; - engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID; - engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB; - engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE; - engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; - engine->semaphore.mbox.signal[RCS] = GEN6_RVSYNC; - engine->semaphore.mbox.signal[VCS] = GEN6_NOSYNC; - engine->semaphore.mbox.signal[BCS] = GEN6_BVSYNC; - engine->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC; - engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; - } - } } else { engine->mmio_base = BSD_RING_BASE; engine->flush = bsd_ring_flush; - engine->add_request = i9xx_add_request; - engine->get_seqno = ring_get_seqno; - engine->set_seqno = ring_set_seqno; - if (IS_GEN5(dev)) { + if (IS_GEN5(dev_priv)) engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; - engine->irq_get = gen5_ring_get_irq; - engine->irq_put = gen5_ring_put_irq; - } else { + else engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; - engine->irq_get = i9xx_ring_get_irq; - engine->irq_put = i9xx_ring_put_irq; - } - engine->dispatch_execbuffer = i965_dispatch_execbuffer; } - engine->init_hw = init_ring_common; return intel_init_ring_buffer(dev, engine); } @@ -3058,147 +2952,70 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev) */ int intel_init_bsd2_ring_buffer(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine = &dev_priv->engine[VCS2]; engine->name = "bsd2 ring"; engine->id = VCS2; engine->exec_id = I915_EXEC_BSD; engine->hw_id = 4; - - engine->write_tail = ring_write_tail; engine->mmio_base = GEN8_BSD2_RING_BASE; + + intel_ring_default_vfuncs(dev_priv, engine); + engine->flush = gen6_bsd_ring_flush; - engine->add_request = gen6_add_request; - engine->irq_seqno_barrier = gen6_seqno_barrier; - engine->get_seqno = ring_get_seqno; - engine->set_seqno = ring_set_seqno; engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT; - engine->irq_get = gen8_ring_get_irq; - engine->irq_put = gen8_ring_put_irq; - engine->dispatch_execbuffer = - gen8_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - engine->semaphore.sync_to = gen8_ring_sync; - engine->semaphore.signal = gen8_xcs_signal; - GEN8_RING_SEMAPHORE_INIT(engine); - } - engine->init_hw = init_ring_common; return intel_init_ring_buffer(dev, engine); } int intel_init_blt_ring_buffer(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine = &dev_priv->engine[BCS]; engine->name = "blitter ring"; engine->id = BCS; engine->exec_id = I915_EXEC_BLT; engine->hw_id = 2; - engine->mmio_base = BLT_RING_BASE; - engine->write_tail = ring_write_tail; + + intel_ring_default_vfuncs(dev_priv, engine); + engine->flush = gen6_ring_flush; - engine->add_request = gen6_add_request; - engine->irq_seqno_barrier = gen6_seqno_barrier; - engine->get_seqno = ring_get_seqno; - engine->set_seqno = ring_set_seqno; - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT; - engine->irq_get = gen8_ring_get_irq; - engine->irq_put = gen8_ring_put_irq; - engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - engine->semaphore.sync_to = gen8_ring_sync; - engine->semaphore.signal = gen8_xcs_signal; - GEN8_RING_SEMAPHORE_INIT(engine); - } - } else { + else engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; - engine->irq_get = gen6_ring_get_irq; - engine->irq_put = gen6_ring_put_irq; - engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - engine->semaphore.signal = gen6_signal; - engine->semaphore.sync_to = gen6_ring_sync; - /* - * The current semaphore is only applied on pre-gen8 - * platform. And there is no VCS2 ring on the pre-gen8 - * platform. So the semaphore between BCS and VCS2 is - * initialized as INVALID. Gen8 will initialize the - * sema between BCS and VCS2 later. - */ - engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR; - engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV; - engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID; - engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE; - engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; - engine->semaphore.mbox.signal[RCS] = GEN6_RBSYNC; - engine->semaphore.mbox.signal[VCS] = GEN6_VBSYNC; - engine->semaphore.mbox.signal[BCS] = GEN6_NOSYNC; - engine->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC; - engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; - } - } - engine->init_hw = init_ring_common; return intel_init_ring_buffer(dev, engine); } int intel_init_vebox_ring_buffer(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_engine_cs *engine = &dev_priv->engine[VECS]; engine->name = "video enhancement ring"; engine->id = VECS; engine->exec_id = I915_EXEC_VEBOX; engine->hw_id = 3; - engine->mmio_base = VEBOX_RING_BASE; - engine->write_tail = ring_write_tail; + + intel_ring_default_vfuncs(dev_priv, engine); + engine->flush = gen6_ring_flush; - engine->add_request = gen6_add_request; - engine->irq_seqno_barrier = gen6_seqno_barrier; - engine->get_seqno = ring_get_seqno; - engine->set_seqno = ring_set_seqno; - if (INTEL_INFO(dev)->gen >= 8) { + if (INTEL_GEN(dev_priv) >= 8) { engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT; - engine->irq_get = gen8_ring_get_irq; - engine->irq_put = gen8_ring_put_irq; - engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - engine->semaphore.sync_to = gen8_ring_sync; - engine->semaphore.signal = gen8_xcs_signal; - GEN8_RING_SEMAPHORE_INIT(engine); - } } else { engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; - engine->irq_get = hsw_vebox_get_irq; - engine->irq_put = hsw_vebox_put_irq; - engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; - if (i915_semaphore_is_enabled(dev)) { - engine->semaphore.sync_to = gen6_ring_sync; - engine->semaphore.signal = gen6_signal; - engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER; - engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV; - engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB; - engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID; - engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID; - engine->semaphore.mbox.signal[RCS] = GEN6_RVESYNC; - engine->semaphore.mbox.signal[VCS] = GEN6_VVESYNC; - engine->semaphore.mbox.signal[BCS] = GEN6_BVESYNC; - engine->semaphore.mbox.signal[VECS] = GEN6_NOSYNC; - engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC; - } + engine->irq_enable = hsw_vebox_irq_enable; + engine->irq_disable = hsw_vebox_irq_disable; } - engine->init_hw = init_ring_common; return intel_init_ring_buffer(dev, engine); } diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index ff126485d..12cb7ed90 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -62,18 +62,6 @@ struct intel_hw_status_page { (i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \ GEN8_SEMAPHORE_OFFSET(from, (__ring)->id)) -#define GEN8_RING_SEMAPHORE_INIT(e) do { \ - if (!dev_priv->semaphore_obj) { \ - break; \ - } \ - (e)->semaphore.signal_ggtt[RCS] = GEN8_SIGNAL_OFFSET((e), RCS); \ - (e)->semaphore.signal_ggtt[VCS] = GEN8_SIGNAL_OFFSET((e), VCS); \ - (e)->semaphore.signal_ggtt[BCS] = GEN8_SIGNAL_OFFSET((e), BCS); \ - (e)->semaphore.signal_ggtt[VECS] = GEN8_SIGNAL_OFFSET((e), VECS); \ - (e)->semaphore.signal_ggtt[VCS2] = GEN8_SIGNAL_OFFSET((e), VCS2); \ - (e)->semaphore.signal_ggtt[(e)->id] = MI_SEMAPHORE_SYNC_INVALID; \ - } while(0) - enum intel_ring_hangcheck_action { HANGCHECK_IDLE = 0, HANGCHECK_WAIT, @@ -86,8 +74,8 @@ enum intel_ring_hangcheck_action { struct intel_ring_hangcheck { u64 acthd; + unsigned long user_interrupts; u32 seqno; - unsigned user_interrupts; int score; enum intel_ring_hangcheck_action action; int deadlock; @@ -107,7 +95,6 @@ struct intel_ringbuffer { int space; int size; int effective_size; - int reserved_size; /** We track the position of the requests in the ring buffer, and * when each is retired we increment last_retired_head as the GPU @@ -120,7 +107,7 @@ struct intel_ringbuffer { u32 last_retired_head; }; -struct intel_context; +struct i915_gem_context; struct drm_i915_reg_table; /* @@ -142,7 +129,10 @@ struct i915_ctx_workarounds { struct drm_i915_gem_object *obj; }; -struct intel_engine_cs { +struct drm_i915_gem_request; + +struct intel_engine_cs { + struct drm_i915_private *i915; const char *name; enum intel_engine_id { RCS = 0, @@ -157,10 +147,42 @@ struct intel_engine_cs { unsigned int hw_id; unsigned int guc_id; /* XXX same as hw_id? */ u32 mmio_base; - struct drm_device *dev; struct intel_ringbuffer *buffer; struct list_head buffers; + /* Rather than have every client wait upon all user interrupts, + * with the herd waking after every interrupt and each doing the + * heavyweight seqno dance, we delegate the task (of being the + * bottom-half of the user interrupt) to the first client. After + * every interrupt, we wake up one client, who does the heavyweight + * coherent seqno read and either goes back to sleep (if incomplete), + * or wakes up all the completed clients in parallel, before then + * transferring the bottom-half status to the next client in the queue. + * + * Compared to walking the entire list of waiters in a single dedicated + * bottom-half, we reduce the latency of the first waiter by avoiding + * a context switch, but incur additional coherent seqno reads when + * following the chain of request breadcrumbs. Since it is most likely + * that we have a single client waiting on each seqno, then reducing + * the overhead of waking that client is much preferred. + */ + struct intel_breadcrumbs { + struct task_struct *irq_seqno_bh; /* bh for user interrupts */ + unsigned long irq_wakeups; + bool irq_posted; + + spinlock_t lock; /* protects the lists of requests */ + struct rb_root waiters; /* sorted by retirement, priority */ + struct rb_root signals; /* sorted by retirement */ + struct intel_wait *first_wait; /* oldest waiter by retirement */ + struct task_struct *signaler; /* used for fence signalling */ + struct drm_i915_gem_request *first_signal; + struct timer_list fake_irq; /* used after a missed interrupt */ + + bool irq_enabled : 1; + bool rpm_wakelock : 1; + } breadcrumbs; + /* * A pool of objects to use as shadow copies of client batch buffers * when the command parser is enabled. Prevents the client from @@ -171,11 +193,10 @@ struct intel_engine_cs { struct intel_hw_status_page status_page; struct i915_ctx_workarounds wa_ctx; - unsigned irq_refcount; /* protected by dev_priv->irq_lock */ - u32 irq_enable_mask; /* bitmask to enable ring interrupt */ - struct drm_i915_gem_request *trace_irq_req; - bool __must_check (*irq_get)(struct intel_engine_cs *ring); - void (*irq_put)(struct intel_engine_cs *ring); + u32 irq_keep_mask; /* always keep these interrupts */ + u32 irq_enable_mask; /* bitmask to enable ring interrupt */ + void (*irq_enable)(struct intel_engine_cs *ring); + void (*irq_disable)(struct intel_engine_cs *ring); int (*init_hw)(struct intel_engine_cs *ring); @@ -194,9 +215,6 @@ struct intel_engine_cs { * monotonic, even if not coherent. */ void (*irq_seqno_barrier)(struct intel_engine_cs *ring); - u32 (*get_seqno)(struct intel_engine_cs *ring); - void (*set_seqno)(struct intel_engine_cs *ring, - u32 seqno); int (*dispatch_execbuffer)(struct drm_i915_gem_request *req, u64 offset, u32 length, unsigned dispatch_flags); @@ -268,13 +286,11 @@ struct intel_engine_cs { struct tasklet_struct irq_tasklet; spinlock_t execlist_lock; /* used inside tasklet, use spin_lock_bh */ struct list_head execlist_queue; - struct list_head execlist_retired_req_list; unsigned int fw_domains; unsigned int next_context_status_buffer; unsigned int idle_lite_restore_wa; bool disable_lite_restore_wa; u32 ctx_desc_template; - u32 irq_keep_mask; /* bitmask for interrupts that should not be masked */ int (*emit_request)(struct drm_i915_gem_request *request); int (*emit_flush)(struct drm_i915_gem_request *request, u32 invalidate_domains, @@ -306,20 +322,16 @@ struct intel_engine_cs { * inspecting request list. */ u32 last_submitted_seqno; - unsigned user_interrupts; bool gpu_caches_dirty; - wait_queue_head_t irq_queue; - - struct intel_context *last_context; + struct i915_gem_context *last_context; struct intel_ring_hangcheck hangcheck; struct { struct drm_i915_gem_object *obj; u32 gtt_offset; - volatile u32 *cpu_page; } scratch; bool needs_cmd_parser; @@ -350,13 +362,13 @@ struct intel_engine_cs { }; static inline bool -intel_engine_initialized(struct intel_engine_cs *engine) +intel_engine_initialized(const struct intel_engine_cs *engine) { - return engine->dev != NULL; + return engine->i915 != NULL; } static inline unsigned -intel_engine_flag(struct intel_engine_cs *engine) +intel_engine_flag(const struct intel_engine_cs *engine) { return 1 << engine->id; } @@ -427,7 +439,7 @@ intel_write_status_page(struct intel_engine_cs *engine, struct intel_ringbuffer * intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size); -int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev, +int intel_pin_and_map_ringbuffer_obj(struct drm_i915_private *dev_priv, struct intel_ringbuffer *ringbuf); void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf); void intel_ringbuffer_free(struct intel_ringbuffer *ring); @@ -458,15 +470,14 @@ static inline void intel_ring_advance(struct intel_engine_cs *engine) } int __intel_ring_space(int head, int tail, int size); void intel_ring_update_space(struct intel_ringbuffer *ringbuf); -bool intel_engine_stopped(struct intel_engine_cs *engine); int __must_check intel_engine_idle(struct intel_engine_cs *engine); void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno); int intel_ring_flush_all_caches(struct drm_i915_gem_request *req); int intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req); +int intel_init_pipe_control(struct intel_engine_cs *engine, int size); void intel_fini_pipe_control(struct intel_engine_cs *engine); -int intel_init_pipe_control(struct intel_engine_cs *engine); int intel_init_render_ring_buffer(struct drm_device *dev); int intel_init_bsd_ring_buffer(struct drm_device *dev); @@ -475,6 +486,10 @@ int intel_init_blt_ring_buffer(struct drm_device *dev); int intel_init_vebox_ring_buffer(struct drm_device *dev); u64 intel_ring_get_active_head(struct intel_engine_cs *engine); +static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine) +{ + return intel_read_status_page(engine, I915_GEM_HWS_INDEX); +} int init_workarounds_ring(struct intel_engine_cs *engine); @@ -486,26 +501,73 @@ static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf) /* * Arbitrary size for largest possible 'add request' sequence. The code paths * are complex and variable. Empirical measurement shows that the worst case - * is ILK at 136 words. Reserving too much is better than reserving too little - * as that allows for corner cases that might have been missed. So the figure - * has been rounded up to 160 words. + * is BDW at 192 bytes (6 + 6 + 36 dwords), then ILK at 136 bytes. However, + * we need to allocate double the largest single packet within that emission + * to account for tail wraparound (so 6 + 6 + 72 dwords for BDW). */ -#define MIN_SPACE_FOR_ADD_REQUEST 160 +#define MIN_SPACE_FOR_ADD_REQUEST 336 -/* - * Reserve space in the ring to guarantee that the i915_add_request() call - * will always have sufficient room to do its stuff. The request creation - * code calls this automatically. - */ -void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size); -/* Cancel the reservation, e.g. because the request is being discarded. */ -void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf); -/* Use the reserved space - for use by i915_add_request() only. */ -void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf); -/* Finish with the reserved space - for use by i915_add_request() only. */ -void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf); - -/* Legacy ringbuffer specific portion of reservation code: */ -int intel_ring_reserve_space(struct drm_i915_gem_request *request); +static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine) +{ + return engine->status_page.gfx_addr + I915_GEM_HWS_INDEX_ADDR; +} + +/* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */ +struct intel_wait { + struct rb_node node; + struct task_struct *tsk; + u32 seqno; +}; + +struct intel_signal_node { + struct rb_node node; + struct intel_wait wait; +}; + +int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine); + +static inline void intel_wait_init(struct intel_wait *wait, u32 seqno) +{ + wait->tsk = current; + wait->seqno = seqno; +} + +static inline bool intel_wait_complete(const struct intel_wait *wait) +{ + return RB_EMPTY_NODE(&wait->node); +} + +bool intel_engine_add_wait(struct intel_engine_cs *engine, + struct intel_wait *wait); +void intel_engine_remove_wait(struct intel_engine_cs *engine, + struct intel_wait *wait); +void intel_engine_enable_signaling(struct drm_i915_gem_request *request); + +static inline bool intel_engine_has_waiter(struct intel_engine_cs *engine) +{ + return READ_ONCE(engine->breadcrumbs.irq_seqno_bh); +} + +static inline bool intel_engine_wakeup(struct intel_engine_cs *engine) +{ + bool wakeup = false; + struct task_struct *tsk = READ_ONCE(engine->breadcrumbs.irq_seqno_bh); + /* Note that for this not to dangerously chase a dangling pointer, + * the caller is responsible for ensure that the task remain valid for + * wake_up_process() i.e. that the RCU grace period cannot expire. + * + * Also note that tsk is likely to be in !TASK_RUNNING state so an + * early test for tsk->state != TASK_RUNNING before wake_up_process() + * is unlikely to be beneficial. + */ + if (tsk) + wakeup = wake_up_process(tsk); + return wakeup; +} + +void intel_engine_enable_fake_irq(struct intel_engine_cs *engine); +void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); +unsigned int intel_kick_waiters(struct drm_i915_private *i915); +unsigned int intel_kick_signalers(struct drm_i915_private *i915); #endif /* _INTEL_RINGBUFFER_H_ */ diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index 2592b39ff..1c603bbe5 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -65,6 +65,9 @@ bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv, int power_well_id); +static struct i915_power_well * +lookup_power_well(struct drm_i915_private *dev_priv, int power_well_id); + const char * intel_display_power_domain_str(enum intel_display_power_domain domain) { @@ -151,6 +154,23 @@ static void intel_power_well_disable(struct drm_i915_private *dev_priv, power_well->ops->disable(dev_priv, power_well); } +static void intel_power_well_get(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + if (!power_well->count++) + intel_power_well_enable(dev_priv, power_well); +} + +static void intel_power_well_put(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + WARN(!power_well->count, "Use count on power well %s is already zero", + power_well->name); + + if (!--power_well->count) + intel_power_well_disable(dev_priv, power_well); +} + /* * We should only use the power well if we explicitly asked the hardware to * enable it, so check if it's enabled and also check if we've requested it to @@ -267,7 +287,7 @@ void intel_display_set_init_power(struct drm_i915_private *dev_priv, */ static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; /* * After we re-enable the power well, if we touch VGA register 0x3d5 @@ -298,7 +318,7 @@ static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv) static void skl_power_well_post_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; /* * After we re-enable the power well, if we touch VGA register 0x3d5 @@ -345,8 +365,11 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv, if (!is_enabled) { DRM_DEBUG_KMS("Enabling power well\n"); - if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) & - HSW_PWR_WELL_STATE_ENABLED), 20)) + if (intel_wait_for_register(dev_priv, + HSW_PWR_WELL_DRIVER, + HSW_PWR_WELL_STATE_ENABLED, + HSW_PWR_WELL_STATE_ENABLED, + 20)) DRM_ERROR("Timeout enabling power well\n"); hsw_power_well_post_enable(dev_priv); } @@ -419,6 +442,16 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv, BIT(POWER_DOMAIN_MODESET) | \ BIT(POWER_DOMAIN_AUX_A) | \ BIT(POWER_DOMAIN_INIT)) +#define BXT_DPIO_CMN_A_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_A_LANES) | \ + BIT(POWER_DOMAIN_AUX_A) | \ + BIT(POWER_DOMAIN_INIT)) +#define BXT_DPIO_CMN_BC_POWER_DOMAINS ( \ + BIT(POWER_DOMAIN_PORT_DDI_B_LANES) | \ + BIT(POWER_DOMAIN_PORT_DDI_C_LANES) | \ + BIT(POWER_DOMAIN_AUX_B) | \ + BIT(POWER_DOMAIN_AUX_C) | \ + BIT(POWER_DOMAIN_INIT)) static void assert_can_enable_dc9(struct drm_i915_private *dev_priv) { @@ -548,6 +581,7 @@ void bxt_enable_dc9(struct drm_i915_private *dev_priv) DRM_DEBUG_KMS("Enabling DC9\n"); + intel_power_sequencer_reset(dev_priv); gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9); } @@ -669,8 +703,11 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv, switch (power_well->data) { case SKL_DISP_PW_1: - if (wait_for((I915_READ(SKL_FUSE_STATUS) & - SKL_FUSE_PG0_DIST_STATUS), 1)) { + if (intel_wait_for_register(dev_priv, + SKL_FUSE_STATUS, + SKL_FUSE_PG0_DIST_STATUS, + SKL_FUSE_PG0_DIST_STATUS, + 1)) { DRM_ERROR("PG0 not enabled\n"); return; } @@ -731,12 +768,18 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv, if (check_fuse_status) { if (power_well->data == SKL_DISP_PW_1) { - if (wait_for((I915_READ(SKL_FUSE_STATUS) & - SKL_FUSE_PG1_DIST_STATUS), 1)) + if (intel_wait_for_register(dev_priv, + SKL_FUSE_STATUS, + SKL_FUSE_PG1_DIST_STATUS, + SKL_FUSE_PG1_DIST_STATUS, + 1)) DRM_ERROR("PG1 distributing status timeout\n"); } else if (power_well->data == SKL_DISP_PW_2) { - if (wait_for((I915_READ(SKL_FUSE_STATUS) & - SKL_FUSE_PG2_DIST_STATUS), 1)) + if (intel_wait_for_register(dev_priv, + SKL_FUSE_STATUS, + SKL_FUSE_PG2_DIST_STATUS, + SKL_FUSE_PG2_DIST_STATUS, + 1)) DRM_ERROR("PG2 distributing status timeout\n"); } } @@ -800,21 +843,99 @@ static void skl_power_well_disable(struct drm_i915_private *dev_priv, skl_set_power_well(dev_priv, power_well, false); } +static enum dpio_phy bxt_power_well_to_phy(struct i915_power_well *power_well) +{ + enum skl_disp_power_wells power_well_id = power_well->data; + + return power_well_id == BXT_DPIO_CMN_A ? DPIO_PHY1 : DPIO_PHY0; +} + +static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + enum skl_disp_power_wells power_well_id = power_well->data; + struct i915_power_well *cmn_a_well; + + if (power_well_id == BXT_DPIO_CMN_BC) { + /* + * We need to copy the GRC calibration value from the eDP PHY, + * so make sure it's powered up. + */ + cmn_a_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A); + intel_power_well_get(dev_priv, cmn_a_well); + } + + bxt_ddi_phy_init(dev_priv, bxt_power_well_to_phy(power_well)); + + if (power_well_id == BXT_DPIO_CMN_BC) + intel_power_well_put(dev_priv, cmn_a_well); +} + +static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + bxt_ddi_phy_uninit(dev_priv, bxt_power_well_to_phy(power_well)); +} + +static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + return bxt_ddi_phy_is_enabled(dev_priv, + bxt_power_well_to_phy(power_well)); +} + +static void bxt_dpio_cmn_power_well_sync_hw(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + if (power_well->count > 0) + bxt_dpio_cmn_power_well_enable(dev_priv, power_well); + else + bxt_dpio_cmn_power_well_disable(dev_priv, power_well); +} + + +static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv) +{ + struct i915_power_well *power_well; + + power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A); + if (power_well->count > 0) + bxt_ddi_phy_verify_state(dev_priv, + bxt_power_well_to_phy(power_well)); + + power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC); + if (power_well->count > 0) + bxt_ddi_phy_verify_state(dev_priv, + bxt_power_well_to_phy(power_well)); +} + static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0; } +static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv) +{ + u32 tmp = I915_READ(DBUF_CTL); + + WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) != + (DBUF_POWER_STATE | DBUF_POWER_REQUEST), + "Unexpected DBuf power power state (0x%08x)\n", tmp); +} + static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); - if (IS_BROXTON(dev_priv)) { - broxton_cdclk_verify_state(dev_priv); - broxton_ddi_phy_verify_state(dev_priv); - } + WARN_ON(dev_priv->cdclk_freq != + dev_priv->display.get_display_clock_speed(&dev_priv->drm)); + + gen9_assert_dbuf_enabled(dev_priv); + + if (IS_BROXTON(dev_priv)) + bxt_verify_ddi_phy_power_wells(dev_priv); } static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv, @@ -948,6 +1069,11 @@ static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv) */ I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); I915_WRITE(CBR1_VLV, 0); + + WARN_ON(dev_priv->rawclk_freq == 0); + + I915_WRITE(RAWCLK_FREQ_VLV, + DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000)); } static void vlv_display_power_well_init(struct drm_i915_private *dev_priv) @@ -963,7 +1089,7 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv) * * CHV DPLL B/C have some issues if VGA mode is enabled. */ - for_each_pipe(dev_priv->dev, pipe) { + for_each_pipe(&dev_priv->drm, pipe) { u32 val = I915_READ(DPLL(pipe)); val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS; @@ -989,12 +1115,12 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv) intel_hpd_init(dev_priv); /* Re-enable the ADPA, if we have one */ - for_each_intel_encoder(dev_priv->dev, encoder) { + for_each_intel_encoder(&dev_priv->drm, encoder) { if (encoder->type == INTEL_OUTPUT_ANALOG) intel_crt_reset(&encoder->base); } - i915_redisable_vga_power_on(dev_priv->dev); + i915_redisable_vga_power_on(&dev_priv->drm); } static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv) @@ -1004,9 +1130,9 @@ static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv) spin_unlock_irq(&dev_priv->irq_lock); /* make sure we're done processing display irqs */ - synchronize_irq(dev_priv->dev->irq); + synchronize_irq(dev_priv->drm.irq); - vlv_power_sequencer_reset(dev_priv); + intel_power_sequencer_reset(dev_priv); intel_hpd_poll_init(dev_priv); } @@ -1101,7 +1227,6 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv) u32 phy_control = dev_priv->chv_phy_control; u32 phy_status = 0; u32 phy_status_mask = 0xffffffff; - u32 tmp; /* * The BIOS can leave the PHY is some weird state @@ -1189,10 +1314,14 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv) * The PHY may be busy with some initial calibration and whatnot, * so the power state can take a while to actually change. */ - if (wait_for((tmp = I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask) == phy_status, 10)) - WARN(phy_status != tmp, - "Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n", - tmp, phy_status, dev_priv->chv_phy_control); + if (intel_wait_for_register(dev_priv, + DISPLAY_PHY_STATUS, + phy_status_mask, + phy_status, + 10)) + DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n", + I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask, + phy_status, dev_priv->chv_phy_control); } #undef BITS_SET @@ -1220,7 +1349,11 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, vlv_set_power_well(dev_priv, power_well, true); /* Poll for phypwrgood signal */ - if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & PHY_POWERGOOD(phy), 1)) + if (intel_wait_for_register(dev_priv, + DISPLAY_PHY_STATUS, + PHY_POWERGOOD(phy), + PHY_POWERGOOD(phy), + 1)) DRM_ERROR("Display PHY %d is not power up\n", phy); mutex_lock(&dev_priv->sb_lock); @@ -1510,10 +1643,8 @@ __intel_display_power_get_domain(struct drm_i915_private *dev_priv, struct i915_power_well *power_well; int i; - for_each_power_well(i, power_well, BIT(domain), power_domains) { - if (!power_well->count++) - intel_power_well_enable(dev_priv, power_well); - } + for_each_power_well(i, power_well, BIT(domain), power_domains) + intel_power_well_get(dev_priv, power_well); power_domains->domain_use_count[domain]++; } @@ -1607,14 +1738,8 @@ void intel_display_power_put(struct drm_i915_private *dev_priv, intel_display_power_domain_str(domain)); power_domains->domain_use_count[domain]--; - for_each_power_well_rev(i, power_well, BIT(domain), power_domains) { - WARN(!power_well->count, - "Use count on power well %s is already zero", - power_well->name); - - if (!--power_well->count) - intel_power_well_disable(dev_priv, power_well); - } + for_each_power_well_rev(i, power_well, BIT(domain), power_domains) + intel_power_well_put(dev_priv, power_well); mutex_unlock(&power_domains->lock); @@ -1785,6 +1910,13 @@ static const struct i915_power_well_ops gen9_dc_off_power_well_ops = { .is_enabled = gen9_dc_off_power_well_enabled, }; +static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = { + .sync_hw = bxt_dpio_cmn_power_well_sync_hw, + .enable = bxt_dpio_cmn_power_well_enable, + .disable = bxt_dpio_cmn_power_well_disable, + .is_enabled = bxt_dpio_cmn_power_well_enabled, +}; + static struct i915_power_well hsw_power_wells[] = { { .name = "always-on", @@ -2021,6 +2153,18 @@ static struct i915_power_well bxt_power_wells[] = { .ops = &skl_power_well_ops, .data = SKL_DISP_PW_2, }, + { + .name = "dpio-common-a", + .domains = BXT_DPIO_CMN_A_POWER_DOMAINS, + .ops = &bxt_dpio_cmn_power_well_ops, + .data = BXT_DPIO_CMN_A, + }, + { + .name = "dpio-common-bc", + .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS, + .ops = &bxt_dpio_cmn_power_well_ops, + .data = BXT_DPIO_CMN_BC, + }, }; static int @@ -2140,7 +2284,7 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) */ void intel_power_domains_fini(struct drm_i915_private *dev_priv) { - struct device *device = &dev_priv->dev->pdev->dev; + struct device *device = &dev_priv->drm.pdev->dev; /* * The i915.ko module is still not prepared to be loaded when @@ -2180,6 +2324,28 @@ static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv) mutex_unlock(&power_domains->lock); } +static void gen9_dbuf_enable(struct drm_i915_private *dev_priv) +{ + I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST); + POSTING_READ(DBUF_CTL); + + udelay(10); + + if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE)) + DRM_ERROR("DBuf power enable timeout\n"); +} + +static void gen9_dbuf_disable(struct drm_i915_private *dev_priv) +{ + I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST); + POSTING_READ(DBUF_CTL); + + udelay(10); + + if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE) + DRM_ERROR("DBuf power disable timeout!\n"); +} + static void skl_display_core_init(struct drm_i915_private *dev_priv, bool resume) { @@ -2204,12 +2370,11 @@ static void skl_display_core_init(struct drm_i915_private *dev_priv, mutex_unlock(&power_domains->lock); - if (!resume) - return; - skl_init_cdclk(dev_priv); - if (dev_priv->csr.dmc_payload) + gen9_dbuf_enable(dev_priv); + + if (resume && dev_priv->csr.dmc_payload) intel_csr_load_program(dev_priv); } @@ -2220,6 +2385,8 @@ static void skl_display_core_uninit(struct drm_i915_private *dev_priv) gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); + gen9_dbuf_disable(dev_priv); + skl_uninit_cdclk(dev_priv); /* The spec doesn't call for removing the reset handshake flag */ @@ -2263,11 +2430,9 @@ void bxt_display_core_init(struct drm_i915_private *dev_priv, mutex_unlock(&power_domains->lock); - broxton_init_cdclk(dev_priv); - broxton_ddi_phy_init(dev_priv); + bxt_init_cdclk(dev_priv); - broxton_cdclk_verify_state(dev_priv); - broxton_ddi_phy_verify_state(dev_priv); + gen9_dbuf_enable(dev_priv); if (resume && dev_priv->csr.dmc_payload) intel_csr_load_program(dev_priv); @@ -2280,8 +2445,9 @@ void bxt_display_core_uninit(struct drm_i915_private *dev_priv) gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); - broxton_ddi_phy_uninit(dev_priv); - broxton_uninit_cdclk(dev_priv); + gen9_dbuf_disable(dev_priv); + + bxt_uninit_cdclk(dev_priv); /* The spec doesn't call for removing the reset handshake flag */ @@ -2412,13 +2578,14 @@ static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv) /** * intel_power_domains_init_hw - initialize hardware power domain state * @dev_priv: i915 device instance + * @resume: Called from resume code paths or not * * This function initializes the hardware power domain state and enables all * power domains using intel_display_set_init_power(). */ void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct i915_power_domains *power_domains = &dev_priv->power_domains; power_domains->initializing = true; @@ -2480,7 +2647,7 @@ void intel_power_domains_suspend(struct drm_i915_private *dev_priv) */ void intel_runtime_pm_get(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct device *device = &dev->pdev->dev; pm_runtime_get_sync(device); @@ -2501,7 +2668,7 @@ void intel_runtime_pm_get(struct drm_i915_private *dev_priv) */ bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct device *device = &dev->pdev->dev; if (IS_ENABLED(CONFIG_PM)) { @@ -2543,7 +2710,7 @@ bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv) */ void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct device *device = &dev->pdev->dev; assert_rpm_wakelock_held(dev_priv); @@ -2562,7 +2729,7 @@ void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv) */ void intel_runtime_pm_put(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct device *device = &dev->pdev->dev; assert_rpm_wakelock_held(dev_priv); @@ -2585,7 +2752,7 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv) */ void intel_runtime_pm_enable(struct drm_i915_private *dev_priv) { - struct drm_device *dev = dev_priv->dev; + struct drm_device *dev = &dev_priv->drm; struct device *device = &dev->pdev->dev; pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */ diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c index 2128fae56..e378f3536 100644 --- a/drivers/gpu/drm/i915/intel_sdvo.c +++ b/drivers/gpu/drm/i915/intel_sdvo.c @@ -240,7 +240,7 @@ intel_sdvo_create_enhance_property(struct intel_sdvo *intel_sdvo, static void intel_sdvo_write_sdvox(struct intel_sdvo *intel_sdvo, u32 val) { struct drm_device *dev = intel_sdvo->base.base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 bval = val, cval = val; int i; @@ -1195,7 +1195,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder, static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder) { struct drm_device *dev = intel_encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc = to_intel_crtc(intel_encoder->base.crtc); const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; struct drm_display_mode *mode = &crtc->config->base.mode; @@ -1330,7 +1330,7 @@ static bool intel_sdvo_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_sdvo *intel_sdvo = to_sdvo(encoder); u16 active_outputs = 0; u32 tmp; @@ -1353,7 +1353,7 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_sdvo *intel_sdvo = to_sdvo(encoder); struct intel_sdvo_dtd dtd; int encoder_pixel_multiplier = 0; @@ -1436,7 +1436,7 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder, static void intel_disable_sdvo(struct intel_encoder *encoder) { - struct drm_i915_private *dev_priv = encoder->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_sdvo *intel_sdvo = to_sdvo(encoder); struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc); u32 temp; @@ -1471,7 +1471,7 @@ static void intel_disable_sdvo(struct intel_encoder *encoder) temp &= ~SDVO_ENABLE; intel_sdvo_write_sdvox(intel_sdvo, temp); - intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A); + intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } @@ -1489,7 +1489,7 @@ static void pch_post_disable_sdvo(struct intel_encoder *encoder) static void intel_enable_sdvo(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_sdvo *intel_sdvo = to_sdvo(encoder); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); u32 temp; @@ -1633,7 +1633,7 @@ intel_sdvo_get_edid(struct drm_connector *connector) static struct edid * intel_sdvo_get_analog_edid(struct drm_connector *connector) { - struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(connector->dev); return drm_get_edid(connector, intel_gmbus_get_adapter(dev_priv, @@ -1916,7 +1916,7 @@ static void intel_sdvo_get_tv_modes(struct drm_connector *connector) static void intel_sdvo_get_lvds_modes(struct drm_connector *connector) { struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector); - struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(connector->dev); struct drm_display_mode *newmode; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", @@ -2001,7 +2001,7 @@ intel_sdvo_set_property(struct drm_connector *connector, { struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector); struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector); - struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(connector->dev); uint16_t temp_value; uint8_t cmd; int ret; @@ -2177,12 +2177,39 @@ done: #undef CHECK_PROPERTY } +static int +intel_sdvo_connector_register(struct drm_connector *connector) +{ + struct intel_sdvo *sdvo = intel_attached_sdvo(connector); + int ret; + + ret = intel_connector_register(connector); + if (ret) + return ret; + + return sysfs_create_link(&connector->kdev->kobj, + &sdvo->ddc.dev.kobj, + sdvo->ddc.dev.kobj.name); +} + +static void +intel_sdvo_connector_unregister(struct drm_connector *connector) +{ + struct intel_sdvo *sdvo = intel_attached_sdvo(connector); + + sysfs_remove_link(&connector->kdev->kobj, + sdvo->ddc.dev.kobj.name); + intel_connector_unregister(connector); +} + static const struct drm_connector_funcs intel_sdvo_connector_funcs = { .dpms = drm_atomic_helper_connector_dpms, .detect = intel_sdvo_detect, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_sdvo_set_property, .atomic_get_property = intel_connector_atomic_get_property, + .late_register = intel_sdvo_connector_register, + .early_unregister = intel_sdvo_connector_unregister, .destroy = intel_sdvo_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, @@ -2191,7 +2218,6 @@ static const struct drm_connector_funcs intel_sdvo_connector_funcs = { static const struct drm_connector_helper_funcs intel_sdvo_connector_helper_funcs = { .get_modes = intel_sdvo_get_modes, .mode_valid = intel_sdvo_mode_valid, - .best_encoder = intel_best_encoder, }; static void intel_sdvo_enc_destroy(struct drm_encoder *encoder) @@ -2312,7 +2338,7 @@ intel_sdvo_is_hdmi_connector(struct intel_sdvo *intel_sdvo, int device) static u8 intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct sdvo_device_mapping *my_mapping, *other_mapping; if (sdvo->port == PORT_B) { @@ -2346,20 +2372,6 @@ intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo) return 0x72; } -static void -intel_sdvo_connector_unregister(struct intel_connector *intel_connector) -{ - struct drm_connector *drm_connector; - struct intel_sdvo *sdvo_encoder; - - drm_connector = &intel_connector->base; - sdvo_encoder = intel_attached_sdvo(&intel_connector->base); - - sysfs_remove_link(&drm_connector->kdev->kobj, - sdvo_encoder->ddc.dev.kobj.name); - intel_connector_unregister(intel_connector); -} - static int intel_sdvo_connector_init(struct intel_sdvo_connector *connector, struct intel_sdvo *encoder) @@ -2382,27 +2394,10 @@ intel_sdvo_connector_init(struct intel_sdvo_connector *connector, connector->base.base.doublescan_allowed = 0; connector->base.base.display_info.subpixel_order = SubPixelHorizontalRGB; connector->base.get_hw_state = intel_sdvo_connector_get_hw_state; - connector->base.unregister = intel_sdvo_connector_unregister; intel_connector_attach_encoder(&connector->base, &encoder->base); - ret = drm_connector_register(drm_connector); - if (ret < 0) - goto err1; - - ret = sysfs_create_link(&drm_connector->kdev->kobj, - &encoder->ddc.dev.kobj, - encoder->ddc.dev.kobj.name); - if (ret < 0) - goto err2; return 0; - -err2: - drm_connector_unregister(drm_connector); -err1: - drm_connector_cleanup(drm_connector); - - return ret; } static void @@ -2529,7 +2524,6 @@ intel_sdvo_tv_init(struct intel_sdvo *intel_sdvo, int type) return true; err: - drm_connector_unregister(connector); intel_sdvo_destroy(connector); return false; } @@ -2608,7 +2602,6 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device) return true; err: - drm_connector_unregister(connector); intel_sdvo_destroy(connector); return false; } @@ -2959,7 +2952,7 @@ static void assert_sdvo_port_valid(const struct drm_i915_private *dev_priv, bool intel_sdvo_init(struct drm_device *dev, i915_reg_t sdvo_reg, enum port port) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_encoder *intel_encoder; struct intel_sdvo *intel_sdvo; int i; @@ -2981,7 +2974,7 @@ bool intel_sdvo_init(struct drm_device *dev, intel_encoder = &intel_sdvo->base; intel_encoder->type = INTEL_OUTPUT_SDVO; drm_encoder_init(dev, &intel_encoder->base, &intel_sdvo_enc_funcs, 0, - NULL); + "SDVO %c", port_name(port)); /* Read the regs to test if we can talk to the device */ for (i = 0; i < 0x40; i++) { diff --git a/drivers/gpu/drm/i915/intel_sideband.c b/drivers/gpu/drm/i915/intel_sideband.c index c3998188c..1a840bf92 100644 --- a/drivers/gpu/drm/i915/intel_sideband.c +++ b/drivers/gpu/drm/i915/intel_sideband.c @@ -51,7 +51,9 @@ static int vlv_sideband_rw(struct drm_i915_private *dev_priv, u32 devfn, WARN_ON(!mutex_is_locked(&dev_priv->sb_lock)); - if (wait_for((I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) == 0, 5)) { + if (intel_wait_for_register(dev_priv, + VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0, + 5)) { DRM_DEBUG_DRIVER("IOSF sideband idle wait (%s) timed out\n", is_read ? "read" : "write"); return -EAGAIN; @@ -62,7 +64,9 @@ static int vlv_sideband_rw(struct drm_i915_private *dev_priv, u32 devfn, I915_WRITE(VLV_IOSF_DATA, *val); I915_WRITE(VLV_IOSF_DOORBELL_REQ, cmd); - if (wait_for((I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) == 0, 5)) { + if (intel_wait_for_register(dev_priv, + VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0, + 5)) { DRM_DEBUG_DRIVER("IOSF sideband finish wait (%s) timed out\n", is_read ? "read" : "write"); return -ETIMEDOUT; @@ -202,8 +206,9 @@ u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg, u32 value = 0; WARN_ON(!mutex_is_locked(&dev_priv->sb_lock)); - if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_BUSY) == 0, - 100)) { + if (intel_wait_for_register(dev_priv, + SBI_CTL_STAT, SBI_BUSY, 0, + 100)) { DRM_ERROR("timeout waiting for SBI to become ready\n"); return 0; } @@ -216,8 +221,11 @@ u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg, value = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IORD; I915_WRITE(SBI_CTL_STAT, value | SBI_BUSY); - if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_BUSY | SBI_RESPONSE_FAIL)) == 0, - 100)) { + if (intel_wait_for_register(dev_priv, + SBI_CTL_STAT, + SBI_BUSY | SBI_RESPONSE_FAIL, + 0, + 100)) { DRM_ERROR("timeout waiting for SBI to complete read transaction\n"); return 0; } @@ -232,8 +240,9 @@ void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value, WARN_ON(!mutex_is_locked(&dev_priv->sb_lock)); - if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_BUSY) == 0, - 100)) { + if (intel_wait_for_register(dev_priv, + SBI_CTL_STAT, SBI_BUSY, 0, + 100)) { DRM_ERROR("timeout waiting for SBI to become ready\n"); return; } @@ -247,8 +256,11 @@ void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value, tmp = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IOWR; I915_WRITE(SBI_CTL_STAT, SBI_BUSY | tmp); - if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_BUSY | SBI_RESPONSE_FAIL)) == 0, - 100)) { + if (intel_wait_for_register(dev_priv, + SBI_CTL_STAT, + SBI_BUSY | SBI_RESPONSE_FAIL, + 0, + 100)) { DRM_ERROR("timeout waiting for SBI to complete write transaction\n"); return; } diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c index 0f3e2303e..7c08e4f29 100644 --- a/drivers/gpu/drm/i915/intel_sprite.c +++ b/drivers/gpu/drm/i915/intel_sprite.c @@ -53,8 +53,8 @@ format_is_yuv(uint32_t format) } } -static int usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, - int usecs) +int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, + int usecs) { /* paranoia */ if (!adjusted_mode->crtc_htotal) @@ -80,9 +80,7 @@ static int usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, */ void intel_pipe_update_start(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->base.dev; const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode; - enum pipe pipe = crtc->pipe; long timeout = msecs_to_jiffies_timeout(1); int scanline, min, max, vblank_start; wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base); @@ -93,7 +91,7 @@ void intel_pipe_update_start(struct intel_crtc *crtc) vblank_start = DIV_ROUND_UP(vblank_start, 2); /* FIXME needs to be calibrated sensibly */ - min = vblank_start - usecs_to_scanlines(adjusted_mode, 100); + min = vblank_start - intel_usecs_to_scanlines(adjusted_mode, 100); max = vblank_start - 1; local_irq_disable(); @@ -139,8 +137,7 @@ void intel_pipe_update_start(struct intel_crtc *crtc) crtc->debug.scanline_start = scanline; crtc->debug.start_vbl_time = ktime_get(); - crtc->debug.start_vbl_count = - dev->driver->get_vblank_counter(dev, pipe); + crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc); trace_i915_pipe_update_vblank_evaded(crtc); } @@ -154,16 +151,35 @@ void intel_pipe_update_start(struct intel_crtc *crtc) * re-enables interrupts and verifies the update was actually completed * before a vblank using the value of @start_vbl_count. */ -void intel_pipe_update_end(struct intel_crtc *crtc) +void intel_pipe_update_end(struct intel_crtc *crtc, struct intel_flip_work *work) { - struct drm_device *dev = crtc->base.dev; enum pipe pipe = crtc->pipe; int scanline_end = intel_get_crtc_scanline(crtc); - u32 end_vbl_count = dev->driver->get_vblank_counter(dev, pipe); + u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc); ktime_t end_vbl_time = ktime_get(); + if (work) { + work->flip_queued_vblank = end_vbl_count; + smp_mb__before_atomic(); + atomic_set(&work->pending, 1); + } + trace_i915_pipe_update_end(crtc, end_vbl_count, scanline_end); + /* We're still in the vblank-evade critical section, this can't race. + * Would be slightly nice to just grab the vblank count and arm the + * event outside of the critical section - the spinlock might spin for a + * while ... */ + if (crtc->base.state->event) { + WARN_ON(drm_crtc_vblank_get(&crtc->base) != 0); + + spin_lock(&crtc->base.dev->event_lock); + drm_crtc_arm_vblank_event(&crtc->base, crtc->base.state->event); + spin_unlock(&crtc->base.dev->event_lock); + + crtc->base.state->event = NULL; + } + local_irq_enable(); if (crtc->debug.start_vbl_count && @@ -183,7 +199,7 @@ skl_update_plane(struct drm_plane *drm_plane, const struct intel_plane_state *plane_state) { struct drm_device *dev = drm_plane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = to_intel_plane(drm_plane); struct drm_framebuffer *fb = plane_state->base.fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); @@ -203,8 +219,6 @@ skl_update_plane(struct drm_plane *drm_plane, uint32_t y = plane_state->src.y1 >> 16; uint32_t src_w = drm_rect_width(&plane_state->src) >> 16; uint32_t src_h = drm_rect_height(&plane_state->src) >> 16; - const struct intel_scaler *scaler = - &crtc_state->scaler_state.scalers[plane_state->scaler_id]; plane_ctl = PLANE_CTL_ENABLE | PLANE_CTL_PIPE_GAMMA_ENABLE | @@ -260,13 +274,16 @@ skl_update_plane(struct drm_plane *drm_plane, /* program plane scaler */ if (plane_state->scaler_id >= 0) { - uint32_t ps_ctrl = 0; int scaler_id = plane_state->scaler_id; + const struct intel_scaler *scaler; DRM_DEBUG_KMS("plane = %d PS_PLANE_SEL(plane) = 0x%x\n", plane, PS_PLANE_SEL(plane)); - ps_ctrl = PS_SCALER_EN | PS_PLANE_SEL(plane) | scaler->mode; - I915_WRITE(SKL_PS_CTRL(pipe, scaler_id), ps_ctrl); + + scaler = &crtc_state->scaler_state.scalers[scaler_id]; + + I915_WRITE(SKL_PS_CTRL(pipe, scaler_id), + PS_SCALER_EN | PS_PLANE_SEL(plane) | scaler->mode); I915_WRITE(SKL_PS_PWR_GATE(pipe, scaler_id), 0); I915_WRITE(SKL_PS_WIN_POS(pipe, scaler_id), (crtc_x << 16) | crtc_y); I915_WRITE(SKL_PS_WIN_SZ(pipe, scaler_id), @@ -286,7 +303,7 @@ static void skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc) { struct drm_device *dev = dplane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = to_intel_plane(dplane); const int pipe = intel_plane->pipe; const int plane = intel_plane->plane + 1; @@ -300,7 +317,7 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc) static void chv_update_csc(struct intel_plane *intel_plane, uint32_t format) { - struct drm_i915_private *dev_priv = intel_plane->base.dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev); int plane = intel_plane->plane; /* Seems RGB data bypasses the CSC always */ @@ -342,7 +359,7 @@ vlv_update_plane(struct drm_plane *dplane, const struct intel_plane_state *plane_state) { struct drm_device *dev = dplane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = to_intel_plane(dplane); struct drm_framebuffer *fb = plane_state->base.fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); @@ -468,7 +485,7 @@ static void vlv_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc) { struct drm_device *dev = dplane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = to_intel_plane(dplane); int pipe = intel_plane->pipe; int plane = intel_plane->plane; @@ -485,7 +502,7 @@ ivb_update_plane(struct drm_plane *plane, const struct intel_plane_state *plane_state) { struct drm_device *dev = plane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = to_intel_plane(plane); struct drm_framebuffer *fb = plane_state->base.fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); @@ -607,7 +624,7 @@ static void ivb_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc) { struct drm_device *dev = plane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = to_intel_plane(plane); int pipe = intel_plane->pipe; @@ -626,7 +643,7 @@ ilk_update_plane(struct drm_plane *plane, const struct intel_plane_state *plane_state) { struct drm_device *dev = plane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = to_intel_plane(plane); struct drm_framebuffer *fb = plane_state->base.fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); @@ -736,7 +753,7 @@ static void ilk_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc) { struct drm_device *dev = plane->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = to_intel_plane(plane); int pipe = intel_plane->pipe; @@ -1111,10 +1128,18 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) possible_crtcs = (1 << pipe); - ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs, - &intel_plane_funcs, - plane_formats, num_plane_formats, - DRM_PLANE_TYPE_OVERLAY, NULL); + if (INTEL_INFO(dev)->gen >= 9) + ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs, + &intel_plane_funcs, + plane_formats, num_plane_formats, + DRM_PLANE_TYPE_OVERLAY, + "plane %d%c", plane + 2, pipe_name(pipe)); + else + ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs, + &intel_plane_funcs, + plane_formats, num_plane_formats, + DRM_PLANE_TYPE_OVERLAY, + "sprite %c", sprite_name(pipe, plane)); if (ret) goto fail; diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c index 223129d3c..49136ad54 100644 --- a/drivers/gpu/drm/i915/intel_tv.c +++ b/drivers/gpu/drm/i915/intel_tv.c @@ -826,7 +826,7 @@ static bool intel_tv_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 tmp = I915_READ(TV_CTL); if (!(tmp & TV_ENC_ENABLE)) @@ -841,7 +841,7 @@ static void intel_enable_tv(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); /* Prevents vblank waits from timing out in intel_tv_detect_type() */ intel_wait_for_vblank(encoder->base.dev, @@ -854,7 +854,7 @@ static void intel_disable_tv(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); I915_WRITE(TV_CTL, I915_READ(TV_CTL) & ~TV_ENC_ENABLE); } @@ -1013,7 +1013,7 @@ static void set_color_conversion(struct drm_i915_private *dev_priv, static void intel_tv_pre_enable(struct intel_encoder *encoder) { struct drm_device *dev = encoder->base.dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); struct intel_tv *intel_tv = enc_to_tv(encoder); const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv); @@ -1173,7 +1173,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv, struct drm_crtc *crtc = connector->state->crtc; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct drm_device *dev = connector->dev; - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); u32 tv_ctl, save_tv_ctl; u32 tv_dac, save_tv_dac; int type; @@ -1501,6 +1501,8 @@ out: static const struct drm_connector_funcs intel_tv_connector_funcs = { .dpms = drm_atomic_helper_connector_dpms, .detect = intel_tv_detect, + .late_register = intel_connector_register, + .early_unregister = intel_connector_unregister, .destroy = intel_tv_destroy, .set_property = intel_tv_set_property, .atomic_get_property = intel_connector_atomic_get_property, @@ -1512,7 +1514,6 @@ static const struct drm_connector_funcs intel_tv_connector_funcs = { static const struct drm_connector_helper_funcs intel_tv_connector_helper_funcs = { .mode_valid = intel_tv_mode_valid, .get_modes = intel_tv_get_modes, - .best_encoder = intel_best_encoder, }; static const struct drm_encoder_funcs intel_tv_enc_funcs = { @@ -1522,7 +1523,7 @@ static const struct drm_encoder_funcs intel_tv_enc_funcs = { void intel_tv_init(struct drm_device *dev) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_connector *connector; struct intel_tv *intel_tv; struct intel_encoder *intel_encoder; @@ -1591,7 +1592,7 @@ intel_tv_init(struct drm_device *dev) DRM_MODE_CONNECTOR_SVIDEO); drm_encoder_init(dev, &intel_encoder->base, &intel_tv_enc_funcs, - DRM_MODE_ENCODER_TVDAC, NULL); + DRM_MODE_ENCODER_TVDAC, "TV"); intel_encoder->compute_config = intel_tv_compute_config; intel_encoder->get_config = intel_tv_get_config; @@ -1600,7 +1601,6 @@ intel_tv_init(struct drm_device *dev) intel_encoder->disable = intel_disable_tv; intel_encoder->get_hw_state = intel_tv_get_hw_state; intel_connector->get_hw_state = intel_connector_get_hw_state; - intel_connector->unregister = intel_connector_unregister; intel_connector_attach_encoder(intel_connector, intel_encoder); intel_encoder->type = INTEL_OUTPUT_TVOUT; @@ -1642,5 +1642,4 @@ intel_tv_init(struct drm_device *dev) drm_object_attach_property(&connector->base, dev->mode_config.tv_bottom_margin_property, intel_tv->margin[TV_MARGIN_BOTTOM]); - drm_connector_register(connector); } diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index 4f1dfe616..ff80a81b1 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -248,9 +248,9 @@ intel_uncore_fw_release_timer(struct hrtimer *timer) return HRTIMER_NORESTART; } -void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) +void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv, + bool restore) { - struct drm_i915_private *dev_priv = dev->dev_private; unsigned long irqflags; struct intel_uncore_forcewake_domain *domain; int retry_count = 100; @@ -304,7 +304,7 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) if (fw) dev_priv->uncore.funcs.force_wake_get(dev_priv, fw); - if (IS_GEN6(dev) || IS_GEN7(dev)) + if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) dev_priv->uncore.fifo_count = fifo_free_entries(dev_priv); } @@ -400,43 +400,42 @@ check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) return false; } -static void __intel_uncore_early_sanitize(struct drm_device *dev, +static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, bool restore_forcewake) { - struct drm_i915_private *dev_priv = dev->dev_private; - /* clear out unclaimed reg detection bit */ if (check_for_unclaimed_mmio(dev_priv)) DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n"); /* clear out old GT FIFO errors */ - if (IS_GEN6(dev) || IS_GEN7(dev)) + if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) __raw_i915_write32(dev_priv, GTFIFODBG, __raw_i915_read32(dev_priv, GTFIFODBG)); /* WaDisableShadowRegForCpd:chv */ - if (IS_CHERRYVIEW(dev)) { + if (IS_CHERRYVIEW(dev_priv)) { __raw_i915_write32(dev_priv, GTFIFOCTL, __raw_i915_read32(dev_priv, GTFIFOCTL) | GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL | GT_FIFO_CTL_RC6_POLICY_STALL); } - intel_uncore_forcewake_reset(dev, restore_forcewake); + intel_uncore_forcewake_reset(dev_priv, restore_forcewake); } -void intel_uncore_early_sanitize(struct drm_device *dev, bool restore_forcewake) +void intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, + bool restore_forcewake) { - __intel_uncore_early_sanitize(dev, restore_forcewake); - i915_check_and_clear_faults(dev); + __intel_uncore_early_sanitize(dev_priv, restore_forcewake); + i915_check_and_clear_faults(dev_priv); } -void intel_uncore_sanitize(struct drm_device *dev) +void intel_uncore_sanitize(struct drm_i915_private *dev_priv) { - i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6); + i915.enable_rc6 = sanitize_rc6_option(dev_priv, i915.enable_rc6); /* BIOS often leaves RC6 enabled, but disable it for hw init */ - intel_disable_gt_powersave(dev); + intel_disable_gt_powersave(dev_priv); } static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv, @@ -1233,14 +1232,12 @@ static void fw_domain_init(struct drm_i915_private *dev_priv, fw_domain_reset(d); } -static void intel_uncore_fw_domains_init(struct drm_device *dev) +static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - if (INTEL_INFO(dev_priv)->gen <= 5) return; - if (IS_GEN9(dev)) { + if (IS_GEN9(dev_priv)) { dev_priv->uncore.funcs.force_wake_get = fw_domains_get; dev_priv->uncore.funcs.force_wake_put = fw_domains_put; fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, @@ -1251,9 +1248,9 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev) FORCEWAKE_ACK_BLITTER_GEN9); fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA, FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9); - } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) { + } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { dev_priv->uncore.funcs.force_wake_get = fw_domains_get; - if (!IS_CHERRYVIEW(dev)) + if (!IS_CHERRYVIEW(dev_priv)) dev_priv->uncore.funcs.force_wake_put = fw_domains_put_with_fifo; else @@ -1262,17 +1259,17 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev) FORCEWAKE_VLV, FORCEWAKE_ACK_VLV); fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA, FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV); - } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { + } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { dev_priv->uncore.funcs.force_wake_get = fw_domains_get_with_thread_status; - if (IS_HASWELL(dev)) + if (IS_HASWELL(dev_priv)) dev_priv->uncore.funcs.force_wake_put = fw_domains_put_with_fifo; else dev_priv->uncore.funcs.force_wake_put = fw_domains_put; fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, FORCEWAKE_MT, FORCEWAKE_ACK_HSW); - } else if (IS_IVYBRIDGE(dev)) { + } else if (IS_IVYBRIDGE(dev_priv)) { u32 ecobus; /* IVB configs may use multi-threaded forcewake */ @@ -1302,11 +1299,11 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev) fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, FORCEWAKE_MT, FORCEWAKE_MT_ACK); - mutex_lock(&dev->struct_mutex); + spin_lock_irq(&dev_priv->uncore.lock); fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_ALL); ecobus = __raw_i915_read32(dev_priv, ECOBUS); fw_domains_put_with_fifo(dev_priv, FORCEWAKE_ALL); - mutex_unlock(&dev->struct_mutex); + spin_unlock_irq(&dev_priv->uncore.lock); if (!(ecobus & FORCEWAKE_MT_ENABLE)) { DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n"); @@ -1314,7 +1311,7 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev) fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER, FORCEWAKE, FORCEWAKE_ACK); } - } else if (IS_GEN6(dev)) { + } else if (IS_GEN6(dev_priv)) { dev_priv->uncore.funcs.force_wake_get = fw_domains_get_with_thread_status; dev_priv->uncore.funcs.force_wake_put = @@ -1327,26 +1324,24 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev) WARN_ON(dev_priv->uncore.fw_domains == 0); } -void intel_uncore_init(struct drm_device *dev) +void intel_uncore_init(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - i915_check_vgpu(dev); + i915_check_vgpu(dev_priv); intel_uncore_edram_detect(dev_priv); - intel_uncore_fw_domains_init(dev); - __intel_uncore_early_sanitize(dev, false); + intel_uncore_fw_domains_init(dev_priv); + __intel_uncore_early_sanitize(dev_priv, false); dev_priv->uncore.unclaimed_mmio_check = 1; - switch (INTEL_INFO(dev)->gen) { + switch (INTEL_INFO(dev_priv)->gen) { default: case 9: ASSIGN_WRITE_MMIO_VFUNCS(gen9); ASSIGN_READ_MMIO_VFUNCS(gen9); break; case 8: - if (IS_CHERRYVIEW(dev)) { + if (IS_CHERRYVIEW(dev_priv)) { ASSIGN_WRITE_MMIO_VFUNCS(chv); ASSIGN_READ_MMIO_VFUNCS(chv); @@ -1357,13 +1352,13 @@ void intel_uncore_init(struct drm_device *dev) break; case 7: case 6: - if (IS_HASWELL(dev)) { + if (IS_HASWELL(dev_priv)) { ASSIGN_WRITE_MMIO_VFUNCS(hsw); } else { ASSIGN_WRITE_MMIO_VFUNCS(gen6); } - if (IS_VALLEYVIEW(dev)) { + if (IS_VALLEYVIEW(dev_priv)) { ASSIGN_READ_MMIO_VFUNCS(vlv); } else { ASSIGN_READ_MMIO_VFUNCS(gen6); @@ -1381,24 +1376,24 @@ void intel_uncore_init(struct drm_device *dev) break; } - if (intel_vgpu_active(dev)) { + if (intel_vgpu_active(dev_priv)) { ASSIGN_WRITE_MMIO_VFUNCS(vgpu); ASSIGN_READ_MMIO_VFUNCS(vgpu); } - i915_check_and_clear_faults(dev); + i915_check_and_clear_faults(dev_priv); } #undef ASSIGN_WRITE_MMIO_VFUNCS #undef ASSIGN_READ_MMIO_VFUNCS -void intel_uncore_fini(struct drm_device *dev) +void intel_uncore_fini(struct drm_i915_private *dev_priv) { /* Paranoia: make sure we have disabled everything before we exit. */ - intel_uncore_sanitize(dev); - intel_uncore_forcewake_reset(dev, false); + intel_uncore_sanitize(dev_priv); + intel_uncore_forcewake_reset(dev_priv, false); } -#define GEN_RANGE(l, h) GENMASK(h, l) +#define GEN_RANGE(l, h) GENMASK((h) - 1, (l) - 1) static const struct register_whitelist { i915_reg_t offset_ldw, offset_udw; @@ -1414,7 +1409,7 @@ static const struct register_whitelist { int i915_reg_read_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_reg_read *reg = data; struct register_whitelist const *entry = whitelist; unsigned size; @@ -1423,7 +1418,7 @@ int i915_reg_read_ioctl(struct drm_device *dev, for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) { if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) && - (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask)) + (INTEL_INFO(dev)->gen_mask & entry->gen_bitmask)) break; } @@ -1467,83 +1462,47 @@ out: return ret; } -int i915_get_reset_stats_ioctl(struct drm_device *dev, - void *data, struct drm_file *file) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_reset_stats *args = data; - struct i915_ctx_hang_stats *hs; - struct intel_context *ctx; - int ret; - - if (args->flags || args->pad) - return -EINVAL; - - if (args->ctx_id == DEFAULT_CONTEXT_HANDLE && !capable(CAP_SYS_ADMIN)) - return -EPERM; - - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; - - ctx = i915_gem_context_get(file->driver_priv, args->ctx_id); - if (IS_ERR(ctx)) { - mutex_unlock(&dev->struct_mutex); - return PTR_ERR(ctx); - } - hs = &ctx->hang_stats; - - if (capable(CAP_SYS_ADMIN)) - args->reset_count = i915_reset_count(&dev_priv->gpu_error); - else - args->reset_count = 0; - - args->batch_active = hs->batch_active; - args->batch_pending = hs->batch_pending; - - mutex_unlock(&dev->struct_mutex); - - return 0; -} - -static int i915_reset_complete(struct drm_device *dev) +static int i915_reset_complete(struct pci_dev *pdev) { u8 gdrst; - pci_read_config_byte(dev->pdev, I915_GDRST, &gdrst); + pci_read_config_byte(pdev, I915_GDRST, &gdrst); return (gdrst & GRDOM_RESET_STATUS) == 0; } -static int i915_do_reset(struct drm_device *dev, unsigned engine_mask) +static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) { + struct pci_dev *pdev = dev_priv->drm.pdev; + /* assert reset for at least 20 usec */ - pci_write_config_byte(dev->pdev, I915_GDRST, GRDOM_RESET_ENABLE); + pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); udelay(20); - pci_write_config_byte(dev->pdev, I915_GDRST, 0); + pci_write_config_byte(pdev, I915_GDRST, 0); - return wait_for(i915_reset_complete(dev), 500); + return wait_for(i915_reset_complete(pdev), 500); } -static int g4x_reset_complete(struct drm_device *dev) +static int g4x_reset_complete(struct pci_dev *pdev) { u8 gdrst; - pci_read_config_byte(dev->pdev, I915_GDRST, &gdrst); + pci_read_config_byte(pdev, I915_GDRST, &gdrst); return (gdrst & GRDOM_RESET_ENABLE) == 0; } -static int g33_do_reset(struct drm_device *dev, unsigned engine_mask) +static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) { - pci_write_config_byte(dev->pdev, I915_GDRST, GRDOM_RESET_ENABLE); - return wait_for(g4x_reset_complete(dev), 500); + struct pci_dev *pdev = dev_priv->drm.pdev; + pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE); + return wait_for(g4x_reset_complete(pdev), 500); } -static int g4x_do_reset(struct drm_device *dev, unsigned engine_mask) +static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) { - struct drm_i915_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = dev_priv->drm.pdev; int ret; - pci_write_config_byte(dev->pdev, I915_GDRST, + pci_write_config_byte(pdev, I915_GDRST, GRDOM_RENDER | GRDOM_RESET_ENABLE); - ret = wait_for(g4x_reset_complete(dev), 500); + ret = wait_for(g4x_reset_complete(pdev), 500); if (ret) return ret; @@ -1551,9 +1510,9 @@ static int g4x_do_reset(struct drm_device *dev, unsigned engine_mask) I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE); POSTING_READ(VDECCLK_GATE_D); - pci_write_config_byte(dev->pdev, I915_GDRST, + pci_write_config_byte(pdev, I915_GDRST, GRDOM_MEDIA | GRDOM_RESET_ENABLE); - ret = wait_for(g4x_reset_complete(dev), 500); + ret = wait_for(g4x_reset_complete(pdev), 500); if (ret) return ret; @@ -1561,27 +1520,29 @@ static int g4x_do_reset(struct drm_device *dev, unsigned engine_mask) I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE); POSTING_READ(VDECCLK_GATE_D); - pci_write_config_byte(dev->pdev, I915_GDRST, 0); + pci_write_config_byte(pdev, I915_GDRST, 0); return 0; } -static int ironlake_do_reset(struct drm_device *dev, unsigned engine_mask) +static int ironlake_do_reset(struct drm_i915_private *dev_priv, + unsigned engine_mask) { - struct drm_i915_private *dev_priv = dev->dev_private; int ret; I915_WRITE(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE); - ret = wait_for((I915_READ(ILK_GDSR) & - ILK_GRDOM_RESET_ENABLE) == 0, 500); + ret = intel_wait_for_register(dev_priv, + ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0, + 500); if (ret) return ret; I915_WRITE(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE); - ret = wait_for((I915_READ(ILK_GDSR) & - ILK_GRDOM_RESET_ENABLE) == 0, 500); + ret = intel_wait_for_register(dev_priv, + ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0, + 500); if (ret) return ret; @@ -1594,25 +1555,21 @@ static int ironlake_do_reset(struct drm_device *dev, unsigned engine_mask) static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv, u32 hw_domain_mask) { - int ret; - /* GEN6_GDRST is not in the gt power well, no need to check * for fifo space for the write or forcewake the chip for * the read */ __raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask); -#define ACKED ((__raw_i915_read32(dev_priv, GEN6_GDRST) & hw_domain_mask) == 0) /* Spin waiting for the device to ack the reset requests */ - ret = wait_for(ACKED, 500); -#undef ACKED - - return ret; + return intel_wait_for_register_fw(dev_priv, + GEN6_GDRST, hw_domain_mask, 0, + 500); } /** * gen6_reset_engines - reset individual engines - * @dev: DRM device + * @dev_priv: i915 device * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset * * This function will reset the individual engines that are set in engine_mask. @@ -1623,9 +1580,9 @@ static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv, * * Returns 0 on success, nonzero on error. */ -static int gen6_reset_engines(struct drm_device *dev, unsigned engine_mask) +static int gen6_reset_engines(struct drm_i915_private *dev_priv, + unsigned engine_mask) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; const u32 hw_engine_mask[I915_NUM_ENGINES] = { [RCS] = GEN6_GRDOM_RENDER, @@ -1647,33 +1604,94 @@ static int gen6_reset_engines(struct drm_device *dev, unsigned engine_mask) ret = gen6_hw_domain_reset(dev_priv, hw_mask); - intel_uncore_forcewake_reset(dev, true); + intel_uncore_forcewake_reset(dev_priv, true); return ret; } -static int wait_for_register_fw(struct drm_i915_private *dev_priv, - i915_reg_t reg, - const u32 mask, - const u32 value, - const unsigned long timeout_ms) +/** + * intel_wait_for_register_fw - wait until register matches expected state + * @dev_priv: the i915 device + * @reg: the register to read + * @mask: mask to apply to register value + * @value: expected value + * @timeout_ms: timeout in millisecond + * + * This routine waits until the target register @reg contains the expected + * @value after applying the @mask, i.e. it waits until + * (I915_READ_FW(@reg) & @mask) == @value + * Otherwise, the wait will timeout after @timeout_ms milliseconds. + * + * Note that this routine assumes the caller holds forcewake asserted, it is + * not suitable for very long waits. See intel_wait_for_register() if you + * wish to wait without holding forcewake for the duration (i.e. you expect + * the wait to be slow). + * + * Returns 0 if the register matches the desired condition, or -ETIMEOUT. + */ +int intel_wait_for_register_fw(struct drm_i915_private *dev_priv, + i915_reg_t reg, + const u32 mask, + const u32 value, + const unsigned long timeout_ms) +{ +#define done ((I915_READ_FW(reg) & mask) == value) + int ret = wait_for_us(done, 2); + if (ret) + ret = wait_for(done, timeout_ms); + return ret; +#undef done +} + +/** + * intel_wait_for_register - wait until register matches expected state + * @dev_priv: the i915 device + * @reg: the register to read + * @mask: mask to apply to register value + * @value: expected value + * @timeout_ms: timeout in millisecond + * + * This routine waits until the target register @reg contains the expected + * @value after applying the @mask, i.e. it waits until + * (I915_READ(@reg) & @mask) == @value + * Otherwise, the wait will timeout after @timeout_ms milliseconds. + * + * Returns 0 if the register matches the desired condition, or -ETIMEOUT. + */ +int intel_wait_for_register(struct drm_i915_private *dev_priv, + i915_reg_t reg, + const u32 mask, + const u32 value, + const unsigned long timeout_ms) { - return wait_for((I915_READ_FW(reg) & mask) == value, timeout_ms); + + unsigned fw = + intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ); + int ret; + + intel_uncore_forcewake_get(dev_priv, fw); + ret = wait_for_us((I915_READ_FW(reg) & mask) == value, 2); + intel_uncore_forcewake_put(dev_priv, fw); + if (ret) + ret = wait_for((I915_READ_NOTRACE(reg) & mask) == value, + timeout_ms); + + return ret; } static int gen8_request_engine_reset(struct intel_engine_cs *engine) { + struct drm_i915_private *dev_priv = engine->i915; int ret; - struct drm_i915_private *dev_priv = engine->dev->dev_private; I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET)); - ret = wait_for_register_fw(dev_priv, - RING_RESET_CTL(engine->mmio_base), - RESET_CTL_READY_TO_RESET, - RESET_CTL_READY_TO_RESET, - 700); + ret = intel_wait_for_register_fw(dev_priv, + RING_RESET_CTL(engine->mmio_base), + RESET_CTL_READY_TO_RESET, + RESET_CTL_READY_TO_RESET, + 700); if (ret) DRM_ERROR("%s: reset request timeout\n", engine->name); @@ -1682,22 +1700,22 @@ static int gen8_request_engine_reset(struct intel_engine_cs *engine) static void gen8_unrequest_engine_reset(struct intel_engine_cs *engine) { - struct drm_i915_private *dev_priv = engine->dev->dev_private; + struct drm_i915_private *dev_priv = engine->i915; I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base), _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET)); } -static int gen8_reset_engines(struct drm_device *dev, unsigned engine_mask) +static int gen8_reset_engines(struct drm_i915_private *dev_priv, + unsigned engine_mask) { - struct drm_i915_private *dev_priv = dev->dev_private; struct intel_engine_cs *engine; for_each_engine_masked(engine, dev_priv, engine_mask) if (gen8_request_engine_reset(engine)) goto not_ready; - return gen6_reset_engines(dev, engine_mask); + return gen6_reset_engines(dev_priv, engine_mask); not_ready: for_each_engine_masked(engine, dev_priv, engine_mask) @@ -1706,35 +1724,35 @@ not_ready: return -EIO; } -static int (*intel_get_gpu_reset(struct drm_device *dev))(struct drm_device *, - unsigned engine_mask) +typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask); + +static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv) { if (!i915.reset) return NULL; - if (INTEL_INFO(dev)->gen >= 8) + if (INTEL_INFO(dev_priv)->gen >= 8) return gen8_reset_engines; - else if (INTEL_INFO(dev)->gen >= 6) + else if (INTEL_INFO(dev_priv)->gen >= 6) return gen6_reset_engines; - else if (IS_GEN5(dev)) + else if (IS_GEN5(dev_priv)) return ironlake_do_reset; - else if (IS_G4X(dev)) + else if (IS_G4X(dev_priv)) return g4x_do_reset; - else if (IS_G33(dev)) + else if (IS_G33(dev_priv)) return g33_do_reset; - else if (INTEL_INFO(dev)->gen >= 3) + else if (INTEL_INFO(dev_priv)->gen >= 3) return i915_do_reset; else return NULL; } -int intel_gpu_reset(struct drm_device *dev, unsigned engine_mask) +int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask) { - struct drm_i915_private *dev_priv = to_i915(dev); - int (*reset)(struct drm_device *, unsigned); + reset_func reset; int ret; - reset = intel_get_gpu_reset(dev); + reset = intel_get_gpu_reset(dev_priv); if (reset == NULL) return -ENODEV; @@ -1742,15 +1760,15 @@ int intel_gpu_reset(struct drm_device *dev, unsigned engine_mask) * request may be dropped and never completes (causing -EIO). */ intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); - ret = reset(dev, engine_mask); + ret = reset(dev_priv, engine_mask); intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); return ret; } -bool intel_has_gpu_reset(struct drm_device *dev) +bool intel_has_gpu_reset(struct drm_i915_private *dev_priv) { - return intel_get_gpu_reset(dev) != NULL; + return intel_get_gpu_reset(dev_priv) != NULL; } int intel_guc_reset(struct drm_i915_private *dev_priv) @@ -1758,7 +1776,7 @@ int intel_guc_reset(struct drm_i915_private *dev_priv) int ret; unsigned long irqflags; - if (!i915.enable_guc_submission) + if (!HAS_GUC(dev_priv)) return -EINVAL; intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); @@ -1802,10 +1820,10 @@ intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv, { enum forcewake_domains fw_domains; - if (intel_vgpu_active(dev_priv->dev)) + if (intel_vgpu_active(dev_priv)) return 0; - switch (INTEL_INFO(dev_priv)->gen) { + switch (INTEL_GEN(dev_priv)) { case 9: fw_domains = __gen9_reg_read_fw_domains(i915_mmio_reg_offset(reg)); break; @@ -1842,10 +1860,10 @@ intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv, { enum forcewake_domains fw_domains; - if (intel_vgpu_active(dev_priv->dev)) + if (intel_vgpu_active(dev_priv)) return 0; - switch (INTEL_INFO(dev_priv)->gen) { + switch (INTEL_GEN(dev_priv)) { case 9: fw_domains = __gen9_reg_write_fw_domains(i915_mmio_reg_offset(reg)); break; diff --git a/drivers/gpu/drm/i915/intel_vbt_defs.h b/drivers/gpu/drm/i915/intel_vbt_defs.h index 44fb0b35e..68db9621f 100644 --- a/drivers/gpu/drm/i915/intel_vbt_defs.h +++ b/drivers/gpu/drm/i915/intel_vbt_defs.h @@ -447,10 +447,16 @@ struct bdb_lfp_backlight_data_entry { u8 obsolete3; } __packed; +struct bdb_lfp_backlight_control_method { + u8 type:4; + u8 controller:4; +} __packed; + struct bdb_lfp_backlight_data { u8 entry_size; struct bdb_lfp_backlight_data_entry data[16]; u8 level[16]; + struct bdb_lfp_backlight_control_method backlight_control[16]; } __packed; struct aimdb_header { |