From d0b2f91bede3bd5e3d24dd6803e56eee959c1797 Mon Sep 17 00:00:00 2001 From: André Fabian Silva Delgado Date: Thu, 20 Oct 2016 00:10:27 -0300 Subject: Linux-libre 4.8.2-gnu --- drivers/Makefile | 8 +- drivers/acpi/Kconfig | 60 +- drivers/acpi/Makefile | 8 +- drivers/acpi/acpi_cmos_rtc.c | 2 +- drivers/acpi/acpi_configfs.c | 267 + drivers/acpi/acpi_lpat.c | 4 +- drivers/acpi/acpi_lpss.c | 5 +- drivers/acpi/acpi_video.c | 3 + drivers/acpi/apei/Makefile | 2 +- drivers/acpi/apei/apei-internal.h | 2 +- drivers/acpi/apei/bert.c | 150 + drivers/acpi/apei/einj.c | 57 +- drivers/acpi/apei/erst.c | 7 +- drivers/acpi/bus.c | 99 +- drivers/acpi/button.c | 153 +- drivers/acpi/dock.c | 7 +- drivers/acpi/dptf/Kconfig | 15 + drivers/acpi/dptf/Makefile | 4 + drivers/acpi/dptf/dptf_power.c | 128 + drivers/acpi/dptf/int340x_thermal.c | 53 + drivers/acpi/ec.c | 106 +- drivers/acpi/internal.h | 3 + drivers/acpi/nfit/Kconfig | 26 + drivers/acpi/nfit/Makefile | 3 + drivers/acpi/nfit/core.c | 2793 +++++ drivers/acpi/nfit/mce.c | 89 + drivers/acpi/nfit/nfit.h | 227 + drivers/acpi/numa.c | 226 +- drivers/acpi/osl.c | 5 +- drivers/acpi/pci_mcfg.c | 92 + drivers/acpi/pci_root.c | 35 + drivers/acpi/pci_slot.c | 43 +- drivers/acpi/pmic/intel_pmic.c | 84 +- drivers/acpi/pmic/intel_pmic.h | 4 +- drivers/acpi/pmic/intel_pmic_bxtwc.c | 420 + drivers/acpi/pmic/intel_pmic_crc.c | 5 +- drivers/acpi/pmic/intel_pmic_xpower.c | 7 +- drivers/acpi/processor_core.c | 26 +- drivers/acpi/processor_driver.c | 13 +- drivers/acpi/processor_idle.c | 542 +- drivers/acpi/scan.c | 155 +- drivers/acpi/sleep.c | 29 +- drivers/acpi/sysfs.c | 6 +- drivers/acpi/tables.c | 23 +- drivers/acpi/thermal.c | 3 +- drivers/acpi/video_detect.c | 8 + drivers/ata/Kconfig | 8 +- drivers/ata/Makefile | 2 +- drivers/ata/ahci.c | 2 +- drivers/ata/ahci_brcm.c | 406 + drivers/ata/libahci.c | 10 +- drivers/ata/libata-core.c | 18 +- drivers/ata/libata-eh.c | 8 +- drivers/ata/libata-scsi.c | 84 +- drivers/ata/libata-transport.c | 9 +- drivers/ata/pata_arasan_cf.c | 2 +- drivers/ata/pata_atiixp.c | 4 +- drivers/ata/pata_hpt366.c | 2 +- drivers/ata/pata_marvell.c | 2 +- drivers/ata/pata_ninja32.c | 2 +- drivers/ata/sata_dwc_460ex.c | 14 +- drivers/atm/horizon.c | 4 +- drivers/atm/nicstar.c | 3 +- drivers/base/firmware_class.c | 183 +- drivers/base/memory.c | 28 +- drivers/base/node.c | 85 +- drivers/base/power/clock_ops.c | 45 + drivers/base/power/domain.c | 301 +- drivers/base/power/opp/core.c | 31 +- drivers/base/power/runtime.c | 27 +- drivers/base/power/trace.c | 6 +- drivers/base/power/wakeup.c | 18 +- drivers/base/property.c | 28 + drivers/base/regmap/regcache-rbtree.c | 38 +- drivers/base/regmap/regcache.c | 5 +- drivers/base/regmap/regmap-i2c.c | 2 +- drivers/base/regmap/regmap-irq.c | 15 +- drivers/base/regmap/regmap.c | 37 +- drivers/base/topology.c | 13 + drivers/bcma/Kconfig | 11 +- drivers/bcma/driver_chipcommon_b.c | 9 +- drivers/block/brd.c | 31 +- drivers/block/cciss.c | 3 +- drivers/block/drbd/drbd_actlog.c | 62 +- drivers/block/drbd/drbd_bitmap.c | 92 +- drivers/block/drbd/drbd_debugfs.c | 17 +- drivers/block/drbd/drbd_int.h | 58 +- drivers/block/drbd/drbd_interval.h | 14 +- drivers/block/drbd/drbd_main.c | 135 +- drivers/block/drbd/drbd_nl.c | 282 +- drivers/block/drbd/drbd_proc.c | 30 +- drivers/block/drbd/drbd_protocol.h | 79 +- drivers/block/drbd/drbd_receiver.c | 571 +- drivers/block/drbd/drbd_req.c | 120 +- drivers/block/drbd/drbd_req.h | 5 +- drivers/block/drbd/drbd_state.c | 61 +- drivers/block/drbd/drbd_state.h | 2 +- drivers/block/drbd/drbd_strings.c | 8 +- drivers/block/drbd/drbd_worker.c | 118 +- drivers/block/floppy.c | 6 +- drivers/block/loop.c | 72 +- drivers/block/mg_disk.c | 9 +- drivers/block/mtip32xx/mtip32xx.c | 7 +- drivers/block/nbd.c | 16 +- drivers/block/null_blk.c | 2 +- drivers/block/osdblk.c | 2 +- drivers/block/pktcdvd.c | 6 +- drivers/block/ps3disk.c | 7 +- drivers/block/ps3vram.c | 3 +- drivers/block/rbd.c | 29 +- drivers/block/rsxx/dev.c | 4 +- drivers/block/rsxx/dma.c | 2 +- drivers/block/skd_main.c | 10 +- drivers/block/sunvdc.c | 3 +- drivers/block/umem.c | 10 +- drivers/block/virtio_blk.c | 52 +- drivers/block/xen-blkback/blkback.c | 27 +- drivers/block/xen-blkback/xenbus.c | 22 +- drivers/block/xen-blkfront.c | 212 +- drivers/block/zram/Kconfig | 15 +- drivers/block/zram/Makefile | 4 +- drivers/block/zram/zcomp.c | 150 +- drivers/block/zram/zcomp.h | 36 +- drivers/block/zram/zram_drv.c | 79 +- drivers/block/zram/zram_drv.h | 5 +- drivers/bluetooth/bpa10x.c | 2 + drivers/bluetooth/btmrvl_main.c | 2 +- drivers/bluetooth/btmrvl_sdio.c | 15 +- drivers/bluetooth/btsdio.c | 2 +- drivers/bluetooth/btusb.c | 16 +- drivers/bluetooth/btwilink.c | 4 +- drivers/bluetooth/hci_intel.c | 28 +- drivers/bluetooth/hci_ldisc.c | 2 +- drivers/bluetooth/hci_vhci.c | 6 +- drivers/bus/Kconfig | 13 + drivers/bus/Makefile | 1 + drivers/bus/arm-cci.c | 55 +- drivers/bus/arm-ccn.c | 142 +- drivers/bus/imx-weim.c | 5 +- drivers/bus/mvebu-mbus.c | 10 +- drivers/bus/tegra-aconnect.c | 112 + drivers/bus/uniphier-system-bus.c | 3 +- drivers/bus/vexpress-config.c | 1 + drivers/cdrom/cdrom.c | 28 +- drivers/char/Kconfig | 42 +- drivers/char/Makefile | 2 +- drivers/char/agp/intel-gtt.c | 8 + drivers/char/dsp56k.c | 2 +- drivers/char/hw_random/Kconfig | 18 +- drivers/char/hw_random/Makefile | 1 + drivers/char/hw_random/bcm2835-rng.c | 47 +- drivers/char/hw_random/exynos-rng.c | 4 +- drivers/char/hw_random/meson-rng.c | 131 + drivers/char/hw_random/omap-rng.c | 16 +- drivers/char/hw_random/stm32-rng.c | 10 +- drivers/char/ipmi/Kconfig | 12 - drivers/char/ipmi/ipmi_msghandler.c | 4 +- drivers/char/ipmi/ipmi_si_intf.c | 73 - drivers/char/ipmi/ipmi_ssif.c | 6 +- drivers/char/mem.c | 30 +- drivers/char/powernv-op-panel.c | 223 + drivers/char/random.c | 474 +- drivers/char/tpm/Kconfig | 30 + drivers/char/tpm/Makefile | 3 + drivers/char/tpm/st33zp24/Kconfig | 11 +- drivers/char/tpm/st33zp24/i2c.c | 70 +- drivers/char/tpm/st33zp24/spi.c | 184 +- drivers/char/tpm/st33zp24/st33zp24.c | 129 +- drivers/char/tpm/st33zp24/st33zp24.h | 14 +- drivers/char/tpm/tpm-chip.c | 299 +- drivers/char/tpm/tpm-dev.c | 17 +- drivers/char/tpm/tpm-interface.c | 183 +- drivers/char/tpm/tpm-sysfs.c | 80 +- drivers/char/tpm/tpm.h | 94 +- drivers/char/tpm/tpm2-cmd.c | 159 +- drivers/char/tpm/tpm_atmel.c | 63 +- drivers/char/tpm/tpm_atmel.h | 16 +- drivers/char/tpm/tpm_crb.c | 90 +- drivers/char/tpm/tpm_eventlog.c | 2 +- drivers/char/tpm/tpm_eventlog.h | 4 +- drivers/char/tpm/tpm_i2c_atmel.c | 45 +- drivers/char/tpm/tpm_i2c_infineon.c | 59 +- drivers/char/tpm/tpm_i2c_nuvoton.c | 131 +- drivers/char/tpm/tpm_ibmvtpm.c | 38 +- drivers/char/tpm/tpm_infineon.c | 22 +- drivers/char/tpm/tpm_nsc.c | 84 +- drivers/char/tpm/tpm_tis.c | 829 +- drivers/char/tpm/tpm_tis_core.c | 835 ++ drivers/char/tpm/tpm_tis_core.h | 156 + drivers/char/tpm/tpm_tis_spi.c | 272 + drivers/char/tpm/tpm_vtpm_proxy.c | 637 ++ drivers/char/tpm/xen-tpmfront.c | 36 +- drivers/char/virtio_console.c | 23 +- drivers/clk/Kconfig | 7 +- drivers/clk/Makefile | 21 +- drivers/clk/at91/clk-generated.c | 2 +- drivers/clk/bcm/clk-iproc-armpll.c | 11 +- drivers/clk/bcm/clk-iproc-asiu.c | 27 +- drivers/clk/bcm/clk-iproc-pll.c | 32 +- drivers/clk/clk-clps711x.c | 2 +- drivers/clk/clk-conf.c | 2 +- drivers/clk/clk-fixed-factor.c | 11 +- drivers/clk/clk-fixed-rate.c | 11 + drivers/clk/clk-highbank.c | 9 +- drivers/clk/clk-multiplier.c | 20 +- drivers/clk/clk-nomadik.c | 48 +- drivers/clk/clk-oxnas.c | 15 +- drivers/clk/clk-s2mps11.c | 21 +- drivers/clk/clk-stm32f4.c | 14 +- drivers/clk/clk-u300.c | 59 +- drivers/clk/clk-vt8500.c | 99 +- drivers/clk/clk.c | 347 +- drivers/clk/clkdev.c | 4 +- drivers/clk/hisilicon/clk-hi3519.c | 116 +- drivers/clk/hisilicon/clk-hi6220.c | 6 +- drivers/clk/hisilicon/clk.c | 89 +- drivers/clk/hisilicon/clk.h | 34 +- drivers/clk/hisilicon/clkdivider-hi6220.c | 2 + drivers/clk/hisilicon/reset.c | 19 +- drivers/clk/hisilicon/reset.h | 5 +- drivers/clk/imx/clk-imx6q.c | 14 +- drivers/clk/imx/clk-imx6sl.c | 14 +- drivers/clk/imx/clk-imx6sx.c | 14 +- drivers/clk/imx/clk-imx6ul.c | 18 +- drivers/clk/imx/clk-imx7d.c | 755 +- drivers/clk/imx/clk-pllv3.c | 28 +- drivers/clk/imx/clk-vf610.c | 12 +- drivers/clk/imx/clk.h | 90 +- drivers/clk/meson/Kconfig | 19 + drivers/clk/meson/Makefile | 5 +- drivers/clk/meson/clk-cpu.c | 73 +- drivers/clk/meson/clk-mpll.c | 94 + drivers/clk/meson/clk-pll.c | 104 +- drivers/clk/meson/clkc.h | 179 +- drivers/clk/meson/gxbb.c | 944 ++ drivers/clk/meson/gxbb.h | 271 + drivers/clk/meson/meson8b-clkc.c | 425 +- drivers/clk/nxp/clk-lpc32xx.c | 3 +- drivers/clk/qcom/gcc-msm8660.c | 28 + drivers/clk/qcom/gcc-msm8996.c | 16 - drivers/clk/renesas/Kconfig | 2 + drivers/clk/renesas/Makefile | 4 +- drivers/clk/renesas/r8a7795-cpg-mssr.c | 374 +- drivers/clk/renesas/r8a7796-cpg-mssr.c | 192 + drivers/clk/renesas/rcar-gen3-cpg.c | 359 + drivers/clk/renesas/rcar-gen3-cpg.h | 43 + drivers/clk/renesas/renesas-cpg-mssr.c | 6 + drivers/clk/renesas/renesas-cpg-mssr.h | 1 + drivers/clk/rockchip/clk-rk3228.c | 125 +- drivers/clk/rockchip/clk-rk3399.c | 20 +- drivers/clk/samsung/Kconfig | 9 + drivers/clk/samsung/Makefile | 2 +- drivers/clk/samsung/clk-cpu.c | 131 +- drivers/clk/samsung/clk-cpu.h | 4 +- drivers/clk/samsung/clk-exynos-audss.c | 12 +- drivers/clk/samsung/clk-exynos-clkout.c | 2 + drivers/clk/samsung/clk-exynos3250.c | 41 +- drivers/clk/samsung/clk-exynos4.c | 64 +- drivers/clk/samsung/clk-exynos4415.c | 30 +- drivers/clk/samsung/clk-exynos5250.c | 27 +- drivers/clk/samsung/clk-exynos5260.c | 114 +- drivers/clk/samsung/clk-exynos5410.c | 58 +- drivers/clk/samsung/clk-exynos5420.c | 40 +- drivers/clk/samsung/clk-exynos5433.c | 437 +- drivers/clk/samsung/clk-exynos5440.c | 12 +- drivers/clk/samsung/clk-exynos7.c | 119 +- drivers/clk/samsung/clk-pll.c | 122 +- drivers/clk/samsung/clk-s3c2410-dclk.c | 5 +- drivers/clk/samsung/clk-s3c2410.c | 2 - drivers/clk/samsung/clk-s3c2412.c | 2 - drivers/clk/samsung/clk-s3c2443.c | 2 - drivers/clk/samsung/clk-s3c64xx.c | 2 - drivers/clk/samsung/clk-s5pv210-audss.c | 29 +- drivers/clk/samsung/clk-s5pv210.c | 2 - drivers/clk/samsung/clk.c | 14 +- drivers/clk/samsung/clk.h | 22 +- drivers/clk/st/clk-flexgen.c | 4 +- drivers/clk/st/clkgen-fsyn.c | 10 +- drivers/clk/st/clkgen-pll.c | 27 +- drivers/clk/sunxi-ng/Kconfig | 65 + drivers/clk/sunxi-ng/Makefile | 20 + drivers/clk/sunxi-ng/ccu-sun8i-h3.c | 826 ++ drivers/clk/sunxi-ng/ccu-sun8i-h3.h | 62 + drivers/clk/sunxi-ng/ccu_common.c | 90 + drivers/clk/sunxi-ng/ccu_common.h | 85 + drivers/clk/sunxi-ng/ccu_div.c | 136 + drivers/clk/sunxi-ng/ccu_div.h | 133 + drivers/clk/sunxi-ng/ccu_frac.c | 110 + drivers/clk/sunxi-ng/ccu_frac.h | 53 + drivers/clk/sunxi-ng/ccu_gate.c | 82 + drivers/clk/sunxi-ng/ccu_gate.h | 52 + drivers/clk/sunxi-ng/ccu_mp.c | 158 + drivers/clk/sunxi-ng/ccu_mp.h | 77 + drivers/clk/sunxi-ng/ccu_mult.h | 15 + drivers/clk/sunxi-ng/ccu_mux.c | 187 + drivers/clk/sunxi-ng/ccu_mux.h | 91 + drivers/clk/sunxi-ng/ccu_nk.c | 147 + drivers/clk/sunxi-ng/ccu_nk.h | 71 + drivers/clk/sunxi-ng/ccu_nkm.c | 153 + drivers/clk/sunxi-ng/ccu_nkm.h | 68 + drivers/clk/sunxi-ng/ccu_nkmp.c | 167 + drivers/clk/sunxi-ng/ccu_nkmp.h | 71 + drivers/clk/sunxi-ng/ccu_nm.c | 114 + drivers/clk/sunxi-ng/ccu_nm.h | 91 + drivers/clk/sunxi-ng/ccu_phase.c | 126 + drivers/clk/sunxi-ng/ccu_phase.h | 50 + drivers/clk/sunxi-ng/ccu_reset.c | 55 + drivers/clk/sunxi-ng/ccu_reset.h | 40 + drivers/clk/sunxi/clk-a10-pll2.c | 4 +- drivers/clk/sunxi/clk-factors.c | 1 - drivers/clk/sunxi/clk-sun6i-apb0-gates.c | 9 +- drivers/clk/sunxi/clk-sun6i-apb0.c | 9 +- drivers/clk/sunxi/clk-sun6i-ar100.c | 21 +- drivers/clk/sunxi/clk-sun8i-apb0.c | 9 +- drivers/clk/sunxi/clk-sun8i-mbus.c | 2 +- drivers/clk/sunxi/clk-sun9i-mmc.c | 28 +- drivers/clk/tegra/clk-id.h | 1 - drivers/clk/tegra/clk-pll.c | 505 + drivers/clk/tegra/clk-tegra-periph.c | 25 +- drivers/clk/tegra/clk-tegra114.c | 160 +- drivers/clk/tegra/clk-tegra124.c | 156 +- drivers/clk/tegra/clk-tegra210.c | 295 +- drivers/clk/tegra/clk-tegra30.c | 113 +- drivers/clk/tegra/clk.h | 17 + drivers/clk/ti/clk-33xx.c | 3 + drivers/clk/ti/clk-43xx.c | 7 + drivers/clk/ux500/u8500_of_clk.c | 16 +- drivers/clk/ux500/u8540_clk.c | 16 +- drivers/clk/ux500/u9540_clk.c | 4 +- drivers/clocksource/Kconfig | 116 +- drivers/clocksource/Makefile | 23 +- drivers/clocksource/arm_arch_timer.c | 136 +- drivers/clocksource/arm_global_timer.c | 59 +- drivers/clocksource/armv7m_systick.c | 17 +- drivers/clocksource/asm9260_timer.c | 22 +- drivers/clocksource/bcm2835_timer.c | 38 +- drivers/clocksource/bcm_kona_timer.c | 28 +- drivers/clocksource/cadence_ttc_timer.c | 74 +- drivers/clocksource/clksrc-dbx500-prcmu.c | 4 +- drivers/clocksource/clksrc-probe.c | 14 +- drivers/clocksource/clksrc_st_lpc.c | 20 +- drivers/clocksource/clps711x-timer.c | 12 +- drivers/clocksource/dummy_timer.c | 36 +- drivers/clocksource/dw_apb_timer_of.c | 4 +- drivers/clocksource/exynos_mct.c | 78 +- drivers/clocksource/fsl_ftm_timer.c | 20 +- drivers/clocksource/h8300_timer16.c | 12 +- drivers/clocksource/h8300_timer8.c | 11 +- drivers/clocksource/h8300_tpu.c | 10 +- drivers/clocksource/meson6_timer.c | 19 +- drivers/clocksource/metag_generic.c | 33 +- drivers/clocksource/mips-gic-timer.c | 64 +- drivers/clocksource/moxart_timer.c | 39 +- drivers/clocksource/mps2-timer.c | 8 +- drivers/clocksource/mtk_timer.c | 8 +- drivers/clocksource/mxs_timer.c | 26 +- drivers/clocksource/nomadik-mtu.c | 43 +- drivers/clocksource/pxa_timer.c | 46 +- drivers/clocksource/qcom-timer.c | 64 +- drivers/clocksource/rockchip_timer.c | 53 +- drivers/clocksource/samsung_pwm_timer.c | 70 +- drivers/clocksource/sun4i_timer.c | 43 +- drivers/clocksource/tango_xtal.c | 10 +- drivers/clocksource/tegra20_timer.c | 24 +- drivers/clocksource/time-armada-370-xp.c | 134 +- drivers/clocksource/time-efm32.c | 17 +- drivers/clocksource/time-lpc32xx.c | 10 +- drivers/clocksource/time-orion.c | 50 +- drivers/clocksource/time-pistachio.c | 26 +- drivers/clocksource/timer-atlas7.c | 69 +- drivers/clocksource/timer-atmel-pit.c | 48 +- drivers/clocksource/timer-atmel-st.c | 42 +- drivers/clocksource/timer-digicolor.c | 16 +- drivers/clocksource/timer-imx-gpt.c | 51 +- drivers/clocksource/timer-integrator-ap.c | 57 +- drivers/clocksource/timer-keystone.c | 13 +- drivers/clocksource/timer-nps.c | 14 +- drivers/clocksource/timer-oxnas-rps.c | 297 + drivers/clocksource/timer-prima2.c | 42 +- drivers/clocksource/timer-sp804.c | 86 +- drivers/clocksource/timer-stm32.c | 8 +- drivers/clocksource/timer-sun5i.c | 33 +- drivers/clocksource/timer-ti-32k.c | 8 +- drivers/clocksource/timer-u300.c | 36 +- drivers/clocksource/versatile.c | 6 +- drivers/clocksource/vf_pit_timer.c | 25 +- drivers/clocksource/vt8500_timer.c | 24 +- drivers/clocksource/zevio-timer.c | 4 +- drivers/connector/cn_proc.c | 4 +- drivers/cpufreq/Kconfig | 14 +- drivers/cpufreq/acpi-cpufreq.c | 17 +- drivers/cpufreq/amd_freq_sensitivity.c | 10 +- drivers/cpufreq/cpufreq.c | 223 +- drivers/cpufreq/cpufreq_conservative.c | 88 +- drivers/cpufreq/cpufreq_governor.c | 73 +- drivers/cpufreq/cpufreq_governor.h | 24 +- drivers/cpufreq/cpufreq_ondemand.c | 38 +- drivers/cpufreq/cpufreq_ondemand.h | 1 - drivers/cpufreq/cpufreq_performance.c | 19 +- drivers/cpufreq/cpufreq_powersave.c | 19 +- drivers/cpufreq/cpufreq_stats.c | 157 +- drivers/cpufreq/cpufreq_userspace.c | 104 +- drivers/cpufreq/davinci-cpufreq.c | 22 +- drivers/cpufreq/freq_table.c | 106 +- drivers/cpufreq/intel_pstate.c | 111 +- drivers/cpufreq/mvebu-cpufreq.c | 2 +- drivers/cpufreq/powernv-cpufreq.c | 205 +- drivers/cpufreq/ppc_cbe_cpufreq_pmi.c | 3 +- drivers/cpufreq/s3c24xx-cpufreq.c | 33 +- drivers/cpufreq/s5pv210-cpufreq.c | 75 +- drivers/cpuidle/cpuidle-arm.c | 27 +- drivers/cpuidle/cpuidle-powernv.c | 101 +- drivers/crypto/Kconfig | 13 + drivers/crypto/bfin_crc.c | 5 - drivers/crypto/caam/Kconfig | 18 +- drivers/crypto/caam/Makefile | 4 +- drivers/crypto/caam/caamhash.c | 5 +- drivers/crypto/caam/caampkc.c | 607 + drivers/crypto/caam/caampkc.h | 70 + drivers/crypto/caam/compat.h | 3 + drivers/crypto/caam/ctrl.c | 125 +- drivers/crypto/caam/desc.h | 11 +- drivers/crypto/caam/desc_constr.h | 51 +- drivers/crypto/caam/jr.c | 22 +- drivers/crypto/caam/pdb.h | 188 +- drivers/crypto/caam/pkc_desc.c | 36 + drivers/crypto/caam/regs.h | 151 +- drivers/crypto/caam/sg_sw_sec4.h | 17 +- drivers/crypto/ccp/ccp-crypto-aes-xts.c | 43 +- drivers/crypto/ccp/ccp-crypto.h | 3 +- drivers/crypto/marvell/cesa.c | 143 +- drivers/crypto/marvell/cesa.h | 120 +- drivers/crypto/marvell/cipher.c | 170 +- drivers/crypto/marvell/hash.c | 162 +- drivers/crypto/marvell/tdma.c | 130 +- drivers/crypto/mxs-dcp.c | 47 +- drivers/crypto/omap-aes.c | 36 +- drivers/crypto/omap-des.c | 14 +- drivers/crypto/omap-sham.c | 47 +- drivers/crypto/picoxcell_crypto.c | 60 +- drivers/crypto/qat/Kconfig | 3 +- drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + drivers/crypto/qat/qat_common/Makefile | 11 - drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_aer.c | 49 +- drivers/crypto/qat/qat_common/adf_common_drv.h | 2 + drivers/crypto/qat/qat_common/adf_sriov.c | 2 +- drivers/crypto/qat/qat_common/adf_vf_isr.c | 2 +- drivers/crypto/qat/qat_common/qat_algs.c | 8 +- drivers/crypto/qat/qat_common/qat_asym_algs.c | 872 +- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 + drivers/crypto/qce/ablkcipher.c | 27 +- drivers/crypto/qce/cipher.h | 2 +- drivers/crypto/s5p-sss.c | 80 +- drivers/crypto/sahara.c | 112 +- drivers/crypto/talitos.c | 672 +- drivers/crypto/ux500/cryp/Makefile | 6 +- drivers/crypto/ux500/hash/Makefile | 2 +- drivers/crypto/vmx/.gitignore | 2 + drivers/crypto/vmx/Kconfig | 2 +- drivers/crypto/vmx/Makefile | 2 +- drivers/crypto/vmx/aes_xts.c | 190 + drivers/crypto/vmx/aesp8-ppc.h | 4 + drivers/crypto/vmx/aesp8-ppc.pl | 1863 +++- drivers/crypto/vmx/vmx.c | 8 +- drivers/dax/dax.c | 8 +- drivers/dax/pmem.c | 14 +- drivers/devfreq/Kconfig | 2 +- drivers/devfreq/devfreq-event.c | 12 +- drivers/devfreq/devfreq.c | 15 +- drivers/devfreq/event/Kconfig | 4 +- drivers/devfreq/event/exynos-ppmu.c | 3 +- drivers/devfreq/exynos-bus.c | 11 +- drivers/dma-buf/Kconfig | 15 +- drivers/dma-buf/Makefile | 2 +- drivers/dma-buf/dma-buf.c | 59 +- drivers/dma-buf/fence-array.c | 144 + drivers/dma-buf/fence.c | 8 +- drivers/dma-buf/sync_file.c | 2 +- drivers/dma/Kconfig | 32 +- drivers/dma/Makefile | 1 + drivers/dma/amba-pl08x.c | 10 +- drivers/dma/at_xdmac.c | 12 +- drivers/dma/bcm2835-dma.c | 7 +- drivers/dma/bestcomm/bestcomm.c | 2 - drivers/dma/coh901318.c | 32 +- drivers/dma/cppi41.c | 3 - drivers/dma/dma-axi-dmac.c | 8 +- drivers/dma/dma-jz4740.c | 14 + drivers/dma/dmatest.c | 43 +- drivers/dma/edma.c | 52 +- drivers/dma/fsl-edma.c | 49 +- drivers/dma/fsl_raid.c | 10 +- drivers/dma/fsldma.c | 2 - drivers/dma/hsu/hsu.c | 90 +- drivers/dma/hsu/pci.c | 11 +- drivers/dma/img-mdc-dma.c | 4 +- drivers/dma/imx-dma.c | 31 +- drivers/dma/imx-sdma.c | 32 +- drivers/dma/ioat/init.c | 2 +- drivers/dma/k3dma.c | 19 +- drivers/dma/mmp_pdma.c | 19 + drivers/dma/mmp_tdma.c | 9 +- drivers/dma/moxart-dma.c | 8 +- drivers/dma/mpc512x_dma.c | 1 + drivers/dma/mv_xor.c | 2 +- drivers/dma/mv_xor_v2.c | 878 ++ drivers/dma/nbpfaxi.c | 18 +- drivers/dma/omap-dma.c | 100 +- drivers/dma/pl330.c | 11 +- drivers/dma/ppc4xx/adma.c | 2 - drivers/dma/pxa_dma.c | 27 +- drivers/dma/qcom/bam_dma.c | 109 +- drivers/dma/qcom/hidma.c | 1 + drivers/dma/qcom/hidma_ll.c | 1 + drivers/dma/qcom/hidma_mgmt.c | 7 +- drivers/dma/s3c24xx-dma.c | 29 +- drivers/dma/sh/rcar-dmac.c | 41 +- drivers/dma/sh/shdmac.c | 9 +- drivers/dma/sh/sudmac.c | 9 +- drivers/dma/sirf-dma.c | 12 +- drivers/dma/ste_dma40.c | 6 +- drivers/dma/ste_dma40_ll.c | 2 +- drivers/dma/sun6i-dma.c | 2 +- drivers/dma/tegra20-apb-dma.c | 49 +- drivers/dma/ti-dma-crossbar.c | 2 +- drivers/dma/timb_dma.c | 8 +- drivers/dma/txx9dmac.c | 9 +- drivers/dma/xilinx/Makefile | 3 +- drivers/dma/xilinx/xilinx_dma.c | 2689 +++++ drivers/dma/xilinx/zynqmp_dma.c | 1151 ++ drivers/edac/Kconfig | 15 + drivers/edac/Makefile | 1 + drivers/edac/altera_edac.c | 492 +- drivers/edac/altera_edac.h | 17 +- drivers/edac/amd64_edac.c | 4 +- drivers/edac/skx_edac.c | 1121 ++ drivers/extcon/Makefile | 3 +- drivers/extcon/devres.c | 216 + drivers/extcon/extcon-adc-jack.c | 34 + drivers/extcon/extcon-usb-gpio.c | 32 +- drivers/extcon/extcon.c | 344 +- drivers/firmware/Kconfig | 12 +- drivers/firmware/Makefile | 1 + drivers/firmware/arm_scpi.c | 47 +- drivers/firmware/broadcom/bcm47xx_sprom.c | 2 +- drivers/firmware/dmi-id.c | 8 +- drivers/firmware/efi/arm-runtime.c | 5 + drivers/firmware/efi/efi-pstore.c | 13 + drivers/firmware/efi/efi.c | 180 +- drivers/firmware/efi/efibc.c | 4 +- drivers/firmware/efi/runtime-wrappers.c | 53 +- drivers/firmware/psci.c | 66 +- drivers/firmware/qcom_scm-32.c | 327 +- drivers/firmware/qcom_scm-64.c | 307 +- drivers/firmware/qcom_scm.c | 353 +- drivers/firmware/qcom_scm.h | 47 +- drivers/firmware/scpi_pm_domain.c | 163 + drivers/fpga/Kconfig | 1 + drivers/gpio/Kconfig | 35 +- drivers/gpio/Makefile | 2 + drivers/gpio/gpio-74x164.c | 9 +- drivers/gpio/gpio-clps711x.c | 10 +- drivers/gpio/gpio-dwapb.c | 1 + drivers/gpio/gpio-f7188x.c | 22 + drivers/gpio/gpio-intel-mid.c | 16 +- drivers/gpio/gpio-lynxpoint.c | 1 - drivers/gpio/gpio-max77620.c | 315 + drivers/gpio/gpio-mcp23s08.c | 2 +- drivers/gpio/gpio-menz127.c | 1 - drivers/gpio/gpio-merrifield.c | 444 + drivers/gpio/gpio-mmio.c | 51 +- drivers/gpio/gpio-palmas.c | 1 - drivers/gpio/gpio-pca953x.c | 21 +- drivers/gpio/gpio-pcf857x.c | 9 + drivers/gpio/gpio-rcar.c | 3 + drivers/gpio/gpio-rdc321x.c | 1 - drivers/gpio/gpio-sa1100.c | 2 +- drivers/gpio/gpio-sch311x.c | 1 - drivers/gpio/gpio-stmpe.c | 18 +- drivers/gpio/gpio-syscon.c | 4 +- drivers/gpio/gpio-tc3589x.c | 1 - drivers/gpio/gpio-tps65218.c | 7 + drivers/gpio/gpio-tps6586x.c | 1 - drivers/gpio/gpio-tps65910.c | 1 - drivers/gpio/gpio-viperboard.c | 1 - drivers/gpio/gpio-wm831x.c | 1 - drivers/gpio/gpio-wm8350.c | 1 - drivers/gpio/gpio-wm8994.c | 1 - drivers/gpio/gpio-xilinx.c | 48 + drivers/gpio/gpio-xlp.c | 52 +- drivers/gpio/gpiolib-acpi.c | 1 + drivers/gpio/gpiolib-of.c | 128 +- drivers/gpio/gpiolib.c | 511 +- drivers/gpu/drm/Makefile | 5 +- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 112 +- drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c | 23 - drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c | 54 +- drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c | 14 +- drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c | 3 + drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c | 57 +- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 41 +- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 271 +- drivers/gpu/drm/amd/amdgpu/amdgpu_display.c | 18 +- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 31 +- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 17 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 44 + drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 2 + drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 22 +- drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c | 12 + drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 80 +- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 264 +- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 3 + drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c | 179 +- drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 230 +- drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 90 +- drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h | 115 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 19 +- drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c | 99 +- drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h | 3 + drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c | 170 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 339 +- drivers/gpu/drm/amd/amdgpu/ci_dpm.c | 215 +- drivers/gpu/drm/amd/amdgpu/ci_dpm.h | 1 + drivers/gpu/drm/amd/amdgpu/cik.c | 25 +- drivers/gpu/drm/amd/amdgpu/cik_sdma.c | 50 +- drivers/gpu/drm/amd/amdgpu/cz_dpm.c | 3 +- drivers/gpu/drm/amd/amdgpu/dce_v10_0.c | 16 +- drivers/gpu/drm/amd/amdgpu/dce_v11_0.c | 21 +- drivers/gpu/drm/amd/amdgpu/dce_v8_0.c | 83 +- drivers/gpu/drm/amd/amdgpu/fiji_smc.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 137 +- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h | 7 - drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 480 +- drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h | 1 - drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 41 +- drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h | 7 - drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 54 +- drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h | 7 - drivers/gpu/drm/amd/amdgpu/iceland_dpm.c | 2 +- drivers/gpu/drm/amd/amdgpu/iceland_smc.c | 4 +- drivers/gpu/drm/amd/amdgpu/iceland_smum.h | 41 + drivers/gpu/drm/amd/amdgpu/kv_dpm.c | 8 +- drivers/gpu/drm/amd/amdgpu/ppsmc.h | 4 + drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 77 +- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 48 +- drivers/gpu/drm/amd/amdgpu/tonga_smc.c | 2 +- drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c | 77 +- drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c | 76 +- drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c | 182 +- drivers/gpu/drm/amd/amdgpu/vce_v2_0.c | 2 + drivers/gpu/drm/amd/amdgpu/vce_v3_0.c | 160 +- drivers/gpu/drm/amd/amdgpu/vi.c | 57 +- .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h | 4 +- drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c | 2 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 3 - drivers/gpu/drm/amd/amdkfd/kfd_process.c | 4 +- drivers/gpu/drm/amd/include/amd_pcie.h | 14 + drivers/gpu/drm/amd/include/amd_shared.h | 11 +- .../gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h | 1 + .../gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h | 3 + .../drm/amd/include/asic_reg/gca/gfx_8_0_sh_mask.h | 108 +- .../gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h | 2 + drivers/gpu/drm/amd/include/cgs_common.h | 19 +- drivers/gpu/drm/amd/powerplay/amd_powerplay.c | 153 +- .../drm/amd/powerplay/eventmgr/eventactionchains.c | 2 - .../gpu/drm/amd/powerplay/eventmgr/eventtasks.c | 3 +- .../drm/amd/powerplay/hwmgr/cz_clockpowergating.c | 17 +- drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c | 23 +- .../amd/powerplay/hwmgr/fiji_clockpowergating.c | 2 +- drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c | 404 +- drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h | 3 - .../gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c | 74 +- .../gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h | 15 + .../gpu/drm/amd/powerplay/hwmgr/functiontables.c | 9 +- .../gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c | 26 +- drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c | 10 +- .../powerplay/hwmgr/polaris10_clockpowergating.c | 20 +- .../gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c | 464 +- .../gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h | 6 +- .../drm/amd/powerplay/hwmgr/polaris10_powertune.c | 590 + .../drm/amd/powerplay/hwmgr/polaris10_powertune.h | 26 +- drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c | 2 +- drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c | 299 +- drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h | 1 + drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h | 165 +- .../gpu/drm/amd/powerplay/hwmgr/processpptables.c | 33 +- .../gpu/drm/amd/powerplay/hwmgr/processpptables.h | 17 +- drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c | 272 +- drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h | 3 - .../amd/powerplay/hwmgr/tonga_processpptables.c | 27 +- drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h | 7 + .../gpu/drm/amd/powerplay/inc/hardwaremanager.h | 1 + drivers/gpu/drm/amd/powerplay/inc/hwmgr.h | 10 +- drivers/gpu/drm/amd/powerplay/inc/smumgr.h | 29 + drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c | 5 +- .../gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c | 3 +- drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h | 4 +- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 190 +- drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 60 +- drivers/gpu/drm/amd/scheduler/sched_fence.c | 81 +- drivers/gpu/drm/arc/Kconfig | 1 - drivers/gpu/drm/arc/Makefile | 2 +- drivers/gpu/drm/arc/arcpgu.h | 2 +- drivers/gpu/drm/arc/arcpgu_crtc.c | 16 +- drivers/gpu/drm/arc/arcpgu_drv.c | 64 +- drivers/gpu/drm/arc/arcpgu_hdmi.c | 18 - drivers/gpu/drm/arc/arcpgu_sim.c | 128 + drivers/gpu/drm/arm/Kconfig | 17 +- drivers/gpu/drm/arm/Makefile | 2 + drivers/gpu/drm/arm/hdlcd_crtc.c | 19 - drivers/gpu/drm/arm/hdlcd_drv.c | 13 +- drivers/gpu/drm/arm/malidp_crtc.c | 216 + drivers/gpu/drm/arm/malidp_drv.c | 519 + drivers/gpu/drm/arm/malidp_drv.h | 54 + drivers/gpu/drm/arm/malidp_hw.c | 691 ++ drivers/gpu/drm/arm/malidp_hw.h | 241 + drivers/gpu/drm/arm/malidp_planes.c | 298 + drivers/gpu/drm/arm/malidp_regs.h | 172 + drivers/gpu/drm/armada/Kconfig | 4 - drivers/gpu/drm/armada/armada_crtc.c | 16 +- drivers/gpu/drm/armada/armada_drv.c | 3 +- drivers/gpu/drm/armada/armada_gem.c | 4 +- drivers/gpu/drm/armada/armada_overlay.c | 1 + drivers/gpu/drm/ast/Kconfig | 4 - drivers/gpu/drm/ast/ast_drv.c | 2 +- drivers/gpu/drm/ast/ast_fb.c | 3 - drivers/gpu/drm/ast/ast_main.c | 3 +- drivers/gpu/drm/ast/ast_mode.c | 10 +- drivers/gpu/drm/ast/ast_ttm.c | 13 +- drivers/gpu/drm/atmel-hlcdc/Kconfig | 1 - drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c | 8 +- drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c | 19 +- drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c | 12 - drivers/gpu/drm/bochs/Kconfig | 4 - drivers/gpu/drm/bochs/bochs_drv.c | 2 +- drivers/gpu/drm/bochs/bochs_mm.c | 15 +- drivers/gpu/drm/bridge/Kconfig | 19 + drivers/gpu/drm/bridge/Makefile | 3 + drivers/gpu/drm/bridge/adv7511/Kconfig | 15 + drivers/gpu/drm/bridge/adv7511/Makefile | 3 + drivers/gpu/drm/bridge/adv7511/adv7511.h | 392 + drivers/gpu/drm/bridge/adv7511/adv7511_drv.c | 1124 ++ drivers/gpu/drm/bridge/adv7511/adv7533.c | 265 + drivers/gpu/drm/bridge/analogix-anx78xx.c | 8 - drivers/gpu/drm/bridge/analogix/analogix_dp_core.c | 3 +- drivers/gpu/drm/bridge/analogix/analogix_dp_core.h | 8 +- drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c | 12 +- drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h | 5 +- drivers/gpu/drm/bridge/dw-hdmi.c | 30 +- drivers/gpu/drm/bridge/nxp-ptn3460.c | 8 - drivers/gpu/drm/bridge/parade-ps8622.c | 14 +- drivers/gpu/drm/bridge/sii902x.c | 467 + drivers/gpu/drm/bridge/tc358767.c | 1413 +++ drivers/gpu/drm/cirrus/Kconfig | 4 - drivers/gpu/drm/cirrus/cirrus_drv.c | 2 +- drivers/gpu/drm/cirrus/cirrus_main.c | 17 +- drivers/gpu/drm/cirrus/cirrus_mode.c | 11 +- drivers/gpu/drm/cirrus/cirrus_ttm.c | 13 +- drivers/gpu/drm/drm_atomic.c | 109 +- drivers/gpu/drm/drm_atomic_helper.c | 573 +- drivers/gpu/drm/drm_auth.c | 285 +- drivers/gpu/drm/drm_blend.c | 238 + drivers/gpu/drm/drm_bridge.c | 2 +- drivers/gpu/drm/drm_bufs.c | 8 +- drivers/gpu/drm/drm_crtc.c | 725 +- drivers/gpu/drm/drm_crtc_helper.c | 36 +- drivers/gpu/drm/drm_crtc_internal.h | 92 +- drivers/gpu/drm/drm_debugfs.c | 3 - drivers/gpu/drm/drm_dp_aux_dev.c | 3 +- drivers/gpu/drm/drm_dp_helper.c | 88 +- drivers/gpu/drm/drm_dp_mst_topology.c | 14 +- drivers/gpu/drm/drm_drv.c | 245 +- drivers/gpu/drm/drm_edid.c | 14 + drivers/gpu/drm/drm_edid_load.c | 2 +- drivers/gpu/drm/drm_fb_cma_helper.c | 43 +- drivers/gpu/drm/drm_fb_helper.c | 56 +- drivers/gpu/drm/drm_fops.c | 149 +- drivers/gpu/drm/drm_fourcc.c | 320 + drivers/gpu/drm/drm_gem.c | 6 +- drivers/gpu/drm/drm_info.c | 117 +- drivers/gpu/drm/drm_internal.h | 21 +- drivers/gpu/drm/drm_ioctl.c | 176 +- drivers/gpu/drm/drm_irq.c | 243 +- drivers/gpu/drm/drm_legacy.h | 8 +- drivers/gpu/drm/drm_lock.c | 240 +- drivers/gpu/drm/drm_memory.c | 2 +- drivers/gpu/drm/drm_mipi_dsi.c | 38 + drivers/gpu/drm/drm_mm.c | 4 +- drivers/gpu/drm/drm_modes.c | 4 +- drivers/gpu/drm/drm_modeset_lock.c | 13 +- drivers/gpu/drm/drm_pci.c | 51 - drivers/gpu/drm/drm_plane_helper.c | 38 +- drivers/gpu/drm/drm_platform.c | 18 - drivers/gpu/drm/drm_prime.c | 10 +- drivers/gpu/drm/drm_probe_helper.c | 21 +- drivers/gpu/drm/drm_scatter.c | 2 +- drivers/gpu/drm/drm_simple_kms_helper.c | 206 + drivers/gpu/drm/drm_sysfs.c | 71 - drivers/gpu/drm/drm_vm.c | 58 +- drivers/gpu/drm/drm_vma_manager.c | 3 + drivers/gpu/drm/etnaviv/etnaviv_drv.c | 5 +- drivers/gpu/drm/etnaviv/etnaviv_gem.c | 18 +- drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 64 +- drivers/gpu/drm/etnaviv/etnaviv_gpu.h | 2 +- drivers/gpu/drm/etnaviv/state_hi.xml.h | 7 + drivers/gpu/drm/exynos/Kconfig | 6 +- drivers/gpu/drm/exynos/exynos_dp.c | 4 +- drivers/gpu/drm/exynos/exynos_drm_dpi.c | 9 - drivers/gpu/drm/exynos/exynos_drm_drv.c | 18 +- drivers/gpu/drm/exynos/exynos_drm_drv.h | 4 - drivers/gpu/drm/exynos/exynos_drm_dsi.c | 9 - drivers/gpu/drm/exynos/exynos_drm_fb.c | 6 +- drivers/gpu/drm/exynos/exynos_drm_fbdev.c | 5 +- drivers/gpu/drm/exynos/exynos_drm_fimc.c | 29 +- drivers/gpu/drm/exynos/exynos_drm_g2d.c | 41 +- drivers/gpu/drm/exynos/exynos_drm_gem.c | 20 +- drivers/gpu/drm/exynos/exynos_drm_gem.h | 2 +- drivers/gpu/drm/exynos/exynos_drm_gsc.c | 35 +- drivers/gpu/drm/exynos/exynos_drm_iommu.c | 77 +- drivers/gpu/drm/exynos/exynos_drm_iommu.h | 91 + drivers/gpu/drm/exynos/exynos_drm_plane.c | 67 +- drivers/gpu/drm/exynos/exynos_drm_rotator.c | 26 +- drivers/gpu/drm/exynos/exynos_drm_vidi.c | 8 - drivers/gpu/drm/exynos/exynos_hdmi.c | 10 +- drivers/gpu/drm/exynos/exynos_mixer.c | 6 +- drivers/gpu/drm/fsl-dcu/Kconfig | 5 - drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c | 44 +- drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c | 49 +- drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h | 2 + drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_kms.c | 15 +- drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_output.h | 3 +- drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c | 16 + drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.h | 1 + drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c | 87 +- drivers/gpu/drm/fsl-dcu/fsl_tcon.c | 1 + drivers/gpu/drm/gma500/Kconfig | 4 - drivers/gpu/drm/gma500/cdv_intel_hdmi.c | 3 +- drivers/gpu/drm/gma500/cdv_intel_lvds.c | 9 +- drivers/gpu/drm/gma500/framebuffer.c | 9 - drivers/gpu/drm/gma500/gma_display.c | 11 +- drivers/gpu/drm/gma500/gma_display.h | 4 +- drivers/gpu/drm/gma500/psb_drv.c | 6 +- drivers/gpu/drm/gma500/psb_intel_display.c | 7 - drivers/gpu/drm/gma500/psb_intel_lvds.c | 9 +- drivers/gpu/drm/hisilicon/kirin/Kconfig | 1 + drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c | 34 +- drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c | 11 +- drivers/gpu/drm/i2c/Kconfig | 6 - drivers/gpu/drm/i2c/Makefile | 2 - drivers/gpu/drm/i2c/ch7006_drv.c | 9 +- drivers/gpu/drm/i915/Kconfig | 22 + drivers/gpu/drm/i915/Kconfig.debug | 3 + drivers/gpu/drm/i915/Makefile | 12 +- drivers/gpu/drm/i915/gvt/Makefile | 5 + drivers/gpu/drm/i915/gvt/debug.h | 34 + drivers/gpu/drm/i915/gvt/gvt.c | 145 + drivers/gpu/drm/i915/gvt/gvt.h | 69 + drivers/gpu/drm/i915/gvt/hypercall.h | 38 + drivers/gpu/drm/i915/gvt/mpt.h | 49 + drivers/gpu/drm/i915/i915_cmd_parser.c | 53 +- drivers/gpu/drm/i915/i915_debugfs.c | 570 +- drivers/gpu/drm/i915/i915_drv.c | 2122 ++-- drivers/gpu/drm/i915/i915_drv.h | 789 +- drivers/gpu/drm/i915/i915_gem.c | 1276 ++- drivers/gpu/drm/i915/i915_gem_batch_pool.c | 6 +- drivers/gpu/drm/i915/i915_gem_context.c | 515 +- drivers/gpu/drm/i915/i915_gem_dmabuf.h | 45 + drivers/gpu/drm/i915/i915_gem_evict.c | 55 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 55 +- drivers/gpu/drm/i915/i915_gem_fence.c | 38 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 395 +- drivers/gpu/drm/i915/i915_gem_gtt.h | 84 +- drivers/gpu/drm/i915/i915_gem_render_state.c | 45 +- drivers/gpu/drm/i915/i915_gem_shrinker.c | 58 +- drivers/gpu/drm/i915/i915_gem_stolen.c | 10 +- drivers/gpu/drm/i915/i915_gem_tiling.c | 8 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 12 +- drivers/gpu/drm/i915/i915_gpu_error.c | 203 +- drivers/gpu/drm/i915/i915_guc_reg.h | 6 +- drivers/gpu/drm/i915/i915_guc_submission.c | 538 +- drivers/gpu/drm/i915/i915_irq.c | 807 +- drivers/gpu/drm/i915/i915_params.c | 23 +- drivers/gpu/drm/i915/i915_params.h | 5 +- drivers/gpu/drm/i915/i915_pci.c | 503 + drivers/gpu/drm/i915/i915_pvinfo.h | 113 + drivers/gpu/drm/i915/i915_reg.h | 78 +- drivers/gpu/drm/i915/i915_suspend.c | 8 +- drivers/gpu/drm/i915/i915_sysfs.c | 40 +- drivers/gpu/drm/i915/i915_trace.h | 54 +- drivers/gpu/drm/i915/i915_vgpu.c | 45 +- drivers/gpu/drm/i915/i915_vgpu.h | 92 +- drivers/gpu/drm/i915/intel_atomic.c | 5 +- drivers/gpu/drm/i915/intel_audio.c | 50 +- drivers/gpu/drm/i915/intel_bios.c | 23 +- drivers/gpu/drm/i915/intel_bios.h | 16 +- drivers/gpu/drm/i915/intel_breadcrumbs.c | 595 + drivers/gpu/drm/i915/intel_color.c | 23 +- drivers/gpu/drm/i915/intel_crt.c | 50 +- drivers/gpu/drm/i915/intel_csr.c | 27 +- drivers/gpu/drm/i915/intel_ddi.c | 274 +- drivers/gpu/drm/i915/intel_device_info.c | 388 + drivers/gpu/drm/i915/intel_display.c | 2830 ++--- drivers/gpu/drm/i915/intel_dp.c | 1087 +- drivers/gpu/drm/i915/intel_dp_aux_backlight.c | 172 + drivers/gpu/drm/i915/intel_dp_mst.c | 25 +- drivers/gpu/drm/i915/intel_dpio_phy.c | 470 + drivers/gpu/drm/i915/intel_dpll_mgr.c | 63 +- drivers/gpu/drm/i915/intel_drv.h | 354 +- drivers/gpu/drm/i915/intel_dsi.c | 135 +- drivers/gpu/drm/i915/intel_dsi.h | 4 + drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c | 179 + drivers/gpu/drm/i915/intel_dsi_panel_vbt.c | 90 +- drivers/gpu/drm/i915/intel_dsi_pll.c | 42 +- drivers/gpu/drm/i915/intel_dvo.c | 42 +- drivers/gpu/drm/i915/intel_fbc.c | 136 +- drivers/gpu/drm/i915/intel_fbdev.c | 141 +- drivers/gpu/drm/i915/intel_fifo_underrun.c | 30 +- drivers/gpu/drm/i915/intel_guc.h | 50 +- drivers/gpu/drm/i915/intel_guc_fwif.h | 3 +- drivers/gpu/drm/i915/intel_guc_loader.c | 220 +- drivers/gpu/drm/i915/intel_gvt.c | 104 + drivers/gpu/drm/i915/intel_gvt.h | 45 + drivers/gpu/drm/i915/intel_hdmi.c | 429 +- drivers/gpu/drm/i915/intel_hotplug.c | 21 +- drivers/gpu/drm/i915/intel_i2c.c | 22 +- drivers/gpu/drm/i915/intel_lrc.c | 964 +- drivers/gpu/drm/i915/intel_lrc.h | 24 +- drivers/gpu/drm/i915/intel_lvds.c | 58 +- drivers/gpu/drm/i915/intel_mocs.c | 100 +- drivers/gpu/drm/i915/intel_modes.c | 4 +- drivers/gpu/drm/i915/intel_opregion.c | 210 +- drivers/gpu/drm/i915/intel_overlay.c | 151 +- drivers/gpu/drm/i915/intel_panel.c | 48 +- drivers/gpu/drm/i915/intel_pm.c | 1789 +-- drivers/gpu/drm/i915/intel_psr.c | 123 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 1317 +-- drivers/gpu/drm/i915/intel_ringbuffer.h | 176 +- drivers/gpu/drm/i915/intel_runtime_pm.c | 275 +- drivers/gpu/drm/i915/intel_sdvo.c | 87 +- drivers/gpu/drm/i915/intel_sideband.c | 32 +- drivers/gpu/drm/i915/intel_sprite.c | 81 +- drivers/gpu/drm/i915/intel_tv.c | 19 +- drivers/gpu/drm/i915/intel_uncore.c | 328 +- drivers/gpu/drm/i915/intel_vbt_defs.h | 6 + drivers/gpu/drm/imx/Kconfig | 1 - drivers/gpu/drm/imx/dw_hdmi-imx.c | 32 +- drivers/gpu/drm/imx/imx-drm-core.c | 145 +- drivers/gpu/drm/imx/imx-drm.h | 21 +- drivers/gpu/drm/imx/imx-ldb.c | 189 +- drivers/gpu/drm/imx/imx-tve.c | 97 +- drivers/gpu/drm/imx/ipuv3-crtc.c | 402 +- drivers/gpu/drm/imx/ipuv3-plane.c | 555 +- drivers/gpu/drm/imx/ipuv3-plane.h | 16 - drivers/gpu/drm/imx/parallel-display.c | 149 +- drivers/gpu/drm/mediatek/Kconfig | 12 +- drivers/gpu/drm/mediatek/Makefile | 7 + drivers/gpu/drm/mediatek/mtk_cec.c | 265 + drivers/gpu/drm/mediatek/mtk_cec.h | 26 + drivers/gpu/drm/mediatek/mtk_drm_drv.c | 13 +- drivers/gpu/drm/mediatek/mtk_drm_gem.c | 13 +- drivers/gpu/drm/mediatek/mtk_drm_gem.h | 2 +- drivers/gpu/drm/mediatek/mtk_drm_plane.c | 1 + drivers/gpu/drm/mediatek/mtk_dsi.c | 9 - drivers/gpu/drm/mediatek/mtk_hdmi.c | 1828 ++++ drivers/gpu/drm/mediatek/mtk_hdmi.h | 23 + drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c | 358 + drivers/gpu/drm/mediatek/mtk_hdmi_regs.h | 238 + drivers/gpu/drm/mediatek/mtk_mipi_tx.c | 2 +- drivers/gpu/drm/mediatek/mtk_mt8173_hdmi_phy.c | 515 + drivers/gpu/drm/mgag200/Kconfig | 4 - drivers/gpu/drm/mgag200/mgag200_drv.c | 2 +- drivers/gpu/drm/mgag200/mgag200_main.c | 4 +- drivers/gpu/drm/mgag200/mgag200_mode.c | 9 +- drivers/gpu/drm/mgag200/mgag200_ttm.c | 13 +- drivers/gpu/drm/msm/Kconfig | 1 + drivers/gpu/drm/msm/Makefile | 2 + drivers/gpu/drm/msm/adreno/adreno_gpu.c | 17 +- drivers/gpu/drm/msm/dsi/dsi.c | 2 +- drivers/gpu/drm/msm/dsi/dsi_cfg.c | 8 + drivers/gpu/drm/msm/dsi/dsi_cfg.h | 2 + drivers/gpu/drm/msm/dsi/dsi_host.c | 69 +- drivers/gpu/drm/msm/dsi/phy/dsi_phy.c | 32 +- drivers/gpu/drm/msm/dsi/phy/dsi_phy.h | 2 + drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c | 4 +- drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c | 4 + drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c | 2 + drivers/gpu/drm/msm/edp/edp_connector.c | 10 - drivers/gpu/drm/msm/hdmi/hdmi.c | 117 +- drivers/gpu/drm/msm/hdmi/hdmi.h | 14 + drivers/gpu/drm/msm/hdmi/hdmi_connector.c | 8 - drivers/gpu/drm/msm/hdmi/hdmi_hdcp.c | 2 +- drivers/gpu/drm/msm/mdp/mdp4/mdp4_dtv_encoder.c | 31 +- drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c | 44 +- drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h | 2 + drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c | 9 - drivers/gpu/drm/msm/mdp/mdp5/mdp5.xml.h | 203 +- drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c | 113 +- drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c | 14 +- drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c | 16 +- drivers/gpu/drm/msm/mdp/mdp5/mdp5_ctl.c | 26 +- drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c | 10 +- drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c | 125 +- drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c | 339 +- drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h | 16 +- drivers/gpu/drm/msm/mdp/mdp5/mdp5_mdss.c | 235 + drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c | 22 +- drivers/gpu/drm/msm/msm_atomic.c | 39 +- drivers/gpu/drm/msm/msm_drv.c | 283 +- drivers/gpu/drm/msm/msm_drv.h | 24 +- drivers/gpu/drm/msm/msm_fb.c | 12 +- drivers/gpu/drm/msm/msm_fbdev.c | 17 +- drivers/gpu/drm/msm/msm_gem.c | 139 +- drivers/gpu/drm/msm/msm_gem.h | 23 +- drivers/gpu/drm/msm/msm_gem_prime.c | 4 +- drivers/gpu/drm/msm/msm_gem_shrinker.c | 168 + drivers/gpu/drm/msm/msm_gem_submit.c | 27 +- drivers/gpu/drm/msm/msm_iommu.c | 6 +- drivers/gpu/drm/msm/msm_kms.h | 8 +- drivers/gpu/drm/msm/msm_perf.c | 7 +- drivers/gpu/drm/msm/msm_rd.c | 71 +- drivers/gpu/drm/msm/msm_ringbuffer.c | 6 +- drivers/gpu/drm/nouveau/Kconfig | 6 - drivers/gpu/drm/nouveau/dispnv04/crtc.c | 12 +- drivers/gpu/drm/nouveau/dispnv04/disp.c | 10 - drivers/gpu/drm/nouveau/dispnv04/tvnv17.c | 9 +- drivers/gpu/drm/nouveau/include/nvif/cl0080.h | 1 + drivers/gpu/drm/nouveau/include/nvif/class.h | 10 + drivers/gpu/drm/nouveau/include/nvkm/core/device.h | 17 +- drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h | 1 + drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h | 2 + drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h | 2 + drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h | 1 + drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h | 1 + drivers/gpu/drm/nouveau/include/nvkm/subdev/bios.h | 10 +- drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h | 4 + drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h | 1 + drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h | 14 +- drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h | 1 + .../gpu/drm/nouveau/include/nvkm/subdev/secboot.h | 3 +- drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h | 9 +- drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h | 3 + drivers/gpu/drm/nouveau/nouveau_abi16.c | 1 + drivers/gpu/drm/nouveau/nouveau_acpi.c | 116 +- drivers/gpu/drm/nouveau/nouveau_bo.c | 76 +- drivers/gpu/drm/nouveau/nouveau_chan.c | 3 +- drivers/gpu/drm/nouveau/nouveau_display.c | 27 +- drivers/gpu/drm/nouveau/nouveau_display.h | 3 +- drivers/gpu/drm/nouveau/nouveau_drm.c | 25 +- drivers/gpu/drm/nouveau/nouveau_fence.h | 3 +- drivers/gpu/drm/nouveau/nouveau_hwmon.c | 36 + drivers/gpu/drm/nouveau/nouveau_ttm.c | 1 + drivers/gpu/drm/nouveau/nouveau_usif.c | 1 - drivers/gpu/drm/nouveau/nv50_display.c | 11 +- drivers/gpu/drm/nouveau/nvkm/core/subdev.c | 6 +- drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild | 2 + drivers/gpu/drm/nouveau/nvkm/engine/ce/gp100.c | 102 + drivers/gpu/drm/nouveau/nvkm/engine/ce/gp104.c | 44 + drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/device/base.c | 72 + drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c | 11 +- drivers/gpu/drm/nouveau/nvkm/engine/device/user.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild | 9 + .../gpu/drm/nouveau/nvkm/engine/disp/basegp104.c | 38 + .../gpu/drm/nouveau/nvkm/engine/disp/channv50.h | 1 + .../gpu/drm/nouveau/nvkm/engine/disp/coregf119.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/disp/coregp100.c | 38 + .../gpu/drm/nouveau/nvkm/engine/disp/coregp104.c | 78 + .../gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/disp/dmacgp104.c | 66 + .../gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h | 10 + drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c | 9 +- drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/disp/gk110.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c | 55 + drivers/gpu/drm/nouveau/nvkm/engine/disp/gp104.c | 81 + drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c | 136 +- drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h | 2 + .../gpu/drm/nouveau/nvkm/engine/disp/ovlygk104.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/disp/ovlygp104.c | 38 + .../gpu/drm/nouveau/nvkm/engine/disp/rootgp100.c | 58 + .../gpu/drm/nouveau/nvkm/engine/disp/rootgp104.c | 58 + .../gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h | 2 + drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/changk104.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c | 3 + drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 9 +- drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 67 + .../gpu/drm/nouveau/nvkm/engine/fifo/gpfifogp100.c | 34 + drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild | 2 + drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 4 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h | 2 + drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c | 4 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c | 4 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c | 179 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 18 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 2 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c | 1 - drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c | 171 + drivers/gpu/drm/nouveau/nvkm/subdev/bios/base.c | 59 +- drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c | 8 + drivers/gpu/drm/nouveau/nvkm/subdev/bios/image.c | 7 +- drivers/gpu/drm/nouveau/nvkm/subdev/bios/pll.c | 20 +- drivers/gpu/drm/nouveau/nvkm/subdev/bios/pmu.c | 20 +- drivers/gpu/drm/nouveau/nvkm/subdev/bios/rammap.c | 6 +- drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c | 28 +- drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c | 8 +- drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.c | 394 +- drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.h | 96 +- drivers/gpu/drm/nouveau/nvkm/subdev/clk/gm20b.c | 896 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild | 3 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c | 6 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c | 19 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h | 2 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c | 19 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c | 69 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp104.c | 43 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h | 5 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c | 146 + .../gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c | 13 +- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c | 3 +- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c | 12 +- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c | 75 + drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h | 3 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c | 118 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c | 11 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c | 2 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c | 103 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c | 9 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c | 4 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h | 12 +- drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c | 14 +- drivers/gpu/drm/nouveau/nvkm/subdev/pci/gp100.c | 44 + drivers/gpu/drm/nouveau/nvkm/subdev/secboot/base.c | 28 +- .../gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c | 72 +- .../gpu/drm/nouveau/nvkm/subdev/secboot/gm20b.c | 54 +- drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h | 18 +- drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c | 28 +- drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c | 39 +- drivers/gpu/drm/nouveau/nvkm/subdev/volt/base.c | 14 +- drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk20a.c | 27 +- drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk20a.h | 11 +- drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c | 40 +- drivers/gpu/drm/omapdrm/Kconfig | 5 - drivers/gpu/drm/omapdrm/displays/Kconfig | 28 +- drivers/gpu/drm/omapdrm/displays/Makefile | 28 +- .../gpu/drm/omapdrm/displays/connector-analog-tv.c | 11 +- drivers/gpu/drm/omapdrm/displays/connector-dvi.c | 5 +- drivers/gpu/drm/omapdrm/displays/connector-hdmi.c | 4 +- drivers/gpu/drm/omapdrm/displays/encoder-opa362.c | 3 +- drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c | 3 +- .../gpu/drm/omapdrm/displays/encoder-tpd12s015.c | 3 +- drivers/gpu/drm/omapdrm/displays/panel-dpi.c | 26 +- drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c | 7 +- .../omapdrm/displays/panel-lgphilips-lb035q02.c | 22 +- .../drm/omapdrm/displays/panel-nec-nl8048hl11.c | 2 +- .../drm/omapdrm/displays/panel-sharp-ls037v7dw01.c | 4 +- .../drm/omapdrm/displays/panel-sony-acx565akm.c | 3 +- .../drm/omapdrm/displays/panel-tpo-td028ttec1.c | 3 +- .../drm/omapdrm/displays/panel-tpo-td043mtea1.c | 2 +- drivers/gpu/drm/omapdrm/dss/core.c | 5 +- drivers/gpu/drm/omapdrm/dss/dispc.c | 471 +- drivers/gpu/drm/omapdrm/dss/dispc.h | 5 + drivers/gpu/drm/omapdrm/dss/dispc_coefs.c | 2 +- drivers/gpu/drm/omapdrm/dss/display.c | 2 +- drivers/gpu/drm/omapdrm/dss/dpi.c | 136 +- drivers/gpu/drm/omapdrm/dss/dsi.c | 57 +- drivers/gpu/drm/omapdrm/dss/dss-of.c | 10 +- drivers/gpu/drm/omapdrm/dss/dss.c | 255 +- drivers/gpu/drm/omapdrm/dss/dss.h | 45 +- drivers/gpu/drm/omapdrm/dss/dss_features.c | 46 +- drivers/gpu/drm/omapdrm/dss/dss_features.h | 1 - drivers/gpu/drm/omapdrm/dss/hdmi.h | 6 +- drivers/gpu/drm/omapdrm/dss/hdmi4.c | 11 +- drivers/gpu/drm/omapdrm/dss/hdmi5.c | 11 +- drivers/gpu/drm/omapdrm/dss/hdmi_common.c | 2 +- drivers/gpu/drm/omapdrm/dss/hdmi_phy.c | 2 +- drivers/gpu/drm/omapdrm/dss/hdmi_pll.c | 78 +- drivers/gpu/drm/omapdrm/dss/hdmi_wp.c | 2 +- drivers/gpu/drm/omapdrm/dss/omapdss.h | 871 +- drivers/gpu/drm/omapdrm/dss/output.c | 3 +- drivers/gpu/drm/omapdrm/dss/pll.c | 129 +- drivers/gpu/drm/omapdrm/dss/rfbi.c | 2 +- drivers/gpu/drm/omapdrm/dss/sdi.c | 2 +- drivers/gpu/drm/omapdrm/dss/venc.c | 3 +- drivers/gpu/drm/omapdrm/dss/video-pll.c | 9 +- drivers/gpu/drm/omapdrm/omap_connector.c | 10 - drivers/gpu/drm/omapdrm/omap_crtc.c | 56 + drivers/gpu/drm/omapdrm/omap_drv.c | 16 +- drivers/gpu/drm/omapdrm/omap_drv.h | 14 - drivers/gpu/drm/omapdrm/omap_fb.c | 20 +- drivers/gpu/drm/omapdrm/omap_fbdev.c | 8 +- drivers/gpu/drm/omapdrm/omap_gem.c | 17 +- drivers/gpu/drm/panel/panel-simple.c | 166 +- drivers/gpu/drm/qxl/Kconfig | 5 - drivers/gpu/drm/qxl/qxl_cmd.c | 2 +- drivers/gpu/drm/qxl/qxl_display.c | 11 +- drivers/gpu/drm/qxl/qxl_draw.c | 3 +- drivers/gpu/drm/qxl/qxl_drv.c | 2 +- drivers/gpu/drm/qxl/qxl_drv.h | 1 - drivers/gpu/drm/qxl/qxl_fb.c | 8 +- drivers/gpu/drm/qxl/qxl_kms.c | 10 +- drivers/gpu/drm/qxl/qxl_release.c | 2 +- drivers/gpu/drm/qxl/qxl_ttm.c | 10 +- drivers/gpu/drm/radeon/atombios_crtc.c | 4 +- drivers/gpu/drm/radeon/ci_dpm.c | 5 +- drivers/gpu/drm/radeon/cik.c | 15 +- drivers/gpu/drm/radeon/evergreen.c | 2 +- drivers/gpu/drm/radeon/evergreen_cs.c | 7 + drivers/gpu/drm/radeon/evergreend.h | 1 + drivers/gpu/drm/radeon/radeon.h | 2 +- drivers/gpu/drm/radeon/radeon_acpi.c | 7 - drivers/gpu/drm/radeon/radeon_atpx_handler.c | 53 +- drivers/gpu/drm/radeon/radeon_device.c | 4 + drivers/gpu/drm/radeon/radeon_display.c | 24 +- drivers/gpu/drm/radeon/radeon_drv.c | 26 +- drivers/gpu/drm/radeon/radeon_kms.c | 5 +- drivers/gpu/drm/radeon/radeon_legacy_crtc.c | 4 +- drivers/gpu/drm/radeon/radeon_pm.c | 13 +- drivers/gpu/drm/radeon/radeon_ttm.c | 14 +- drivers/gpu/drm/radeon/si.c | 40 +- drivers/gpu/drm/radeon/si_dpm.c | 6 + drivers/gpu/drm/rcar-du/Kconfig | 1 - drivers/gpu/drm/rcar-du/Makefile | 4 +- drivers/gpu/drm/rcar-du/rcar_du_crtc.c | 4 +- drivers/gpu/drm/rcar-du/rcar_du_drv.c | 17 +- drivers/gpu/drm/rcar-du/rcar_du_drv.h | 1 - drivers/gpu/drm/rcar-du/rcar_du_encoder.c | 15 +- drivers/gpu/drm/rcar-du/rcar_du_encoder.h | 10 +- drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c | 69 +- drivers/gpu/drm/rcar-du/rcar_du_kms.c | 15 +- drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c | 1 - drivers/gpu/drm/rcar-du/rcar_du_plane.c | 29 +- drivers/gpu/drm/rcar-du/rcar_du_plane.h | 2 - drivers/gpu/drm/rcar-du/rcar_du_regs.h | 5 +- drivers/gpu/drm/rcar-du/rcar_du_vgacon.c | 3 - drivers/gpu/drm/rcar-du/rcar_du_vsp.c | 43 +- drivers/gpu/drm/rcar-du/rcar_du_vsp.h | 2 + drivers/gpu/drm/rockchip/Kconfig | 5 +- drivers/gpu/drm/rockchip/analogix_dp-rockchip.c | 189 +- drivers/gpu/drm/rockchip/dw-mipi-dsi.c | 9 - drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c | 1 - drivers/gpu/drm/rockchip/inno_hdmi.c | 9 - drivers/gpu/drm/rockchip/rockchip_drm_drv.c | 210 +- drivers/gpu/drm/rockchip/rockchip_drm_drv.h | 12 +- drivers/gpu/drm/rockchip/rockchip_drm_fb.c | 81 +- drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c | 5 +- drivers/gpu/drm/rockchip/rockchip_drm_gem.c | 19 +- drivers/gpu/drm/rockchip/rockchip_drm_gem.h | 2 +- drivers/gpu/drm/rockchip/rockchip_drm_vop.c | 77 +- drivers/gpu/drm/rockchip/rockchip_vop_reg.c | 7 +- drivers/gpu/drm/shmobile/Kconfig | 1 - drivers/gpu/drm/shmobile/shmob_drm_crtc.c | 4 +- drivers/gpu/drm/shmobile/shmob_drm_drv.c | 3 +- drivers/gpu/drm/sis/sis_mm.c | 2 +- drivers/gpu/drm/sti/Kconfig | 1 + drivers/gpu/drm/sti/sti_awg_utils.c | 4 +- drivers/gpu/drm/sti/sti_compositor.c | 26 +- drivers/gpu/drm/sti/sti_compositor.h | 3 + drivers/gpu/drm/sti/sti_crtc.c | 71 +- drivers/gpu/drm/sti/sti_cursor.c | 39 +- drivers/gpu/drm/sti/sti_drv.c | 148 +- drivers/gpu/drm/sti/sti_drv.h | 1 + drivers/gpu/drm/sti/sti_dvo.c | 43 +- drivers/gpu/drm/sti/sti_gdp.c | 46 +- drivers/gpu/drm/sti/sti_hda.c | 43 +- drivers/gpu/drm/sti/sti_hdmi.c | 350 +- drivers/gpu/drm/sti/sti_hdmi.h | 13 + drivers/gpu/drm/sti/sti_hqvdp.c | 40 +- drivers/gpu/drm/sti/sti_mixer.c | 21 +- drivers/gpu/drm/sti/sti_mixer.h | 2 + drivers/gpu/drm/sti/sti_plane.c | 112 +- drivers/gpu/drm/sti/sti_plane.h | 8 +- drivers/gpu/drm/sti/sti_tvout.c | 43 +- drivers/gpu/drm/sti/sti_vid.c | 12 +- drivers/gpu/drm/sti/sti_vid.h | 2 + drivers/gpu/drm/sti/sti_vtg.c | 3 +- drivers/gpu/drm/sun4i/sun4i_crtc.c | 12 + drivers/gpu/drm/sun4i/sun4i_drv.c | 13 +- drivers/gpu/drm/sun4i/sun4i_framebuffer.c | 3 +- drivers/gpu/drm/sun4i/sun4i_rgb.c | 10 - drivers/gpu/drm/sun4i/sun4i_tv.c | 9 - drivers/gpu/drm/tegra/dc.c | 176 +- drivers/gpu/drm/tegra/dpaux.c | 245 +- drivers/gpu/drm/tegra/drm.c | 4 +- drivers/gpu/drm/tegra/drm.h | 2 - drivers/gpu/drm/tegra/dsi.c | 277 +- drivers/gpu/drm/tegra/fb.c | 2 +- drivers/gpu/drm/tegra/hdmi.c | 508 +- drivers/gpu/drm/tegra/hdmi.h | 21 +- drivers/gpu/drm/tegra/output.c | 9 +- drivers/gpu/drm/tegra/rgb.c | 1 - drivers/gpu/drm/tegra/sor.c | 717 +- drivers/gpu/drm/tegra/sor.h | 3 + drivers/gpu/drm/tilcdc/Kconfig | 1 - drivers/gpu/drm/tilcdc/tilcdc_crtc.c | 2 +- drivers/gpu/drm/tilcdc/tilcdc_drv.c | 3 +- drivers/gpu/drm/ttm/ttm_bo.c | 168 +- drivers/gpu/drm/ttm/ttm_bo_util.c | 121 +- drivers/gpu/drm/ttm/ttm_bo_vm.c | 19 +- drivers/gpu/drm/ttm/ttm_tt.c | 12 +- drivers/gpu/drm/udl/Kconfig | 5 - drivers/gpu/drm/udl/udl_drv.c | 1 - drivers/gpu/drm/udl/udl_fb.c | 6 +- drivers/gpu/drm/udl/udl_modeset.c | 2 +- drivers/gpu/drm/vc4/vc4_bo.c | 4 +- drivers/gpu/drm/vc4/vc4_crtc.c | 183 +- drivers/gpu/drm/vc4/vc4_dpi.c | 23 +- drivers/gpu/drm/vc4/vc4_drv.c | 70 +- drivers/gpu/drm/vc4/vc4_drv.h | 12 +- drivers/gpu/drm/vc4/vc4_gem.c | 25 +- drivers/gpu/drm/vc4/vc4_hdmi.c | 22 +- drivers/gpu/drm/vc4/vc4_kms.c | 15 +- drivers/gpu/drm/vc4/vc4_plane.c | 13 +- drivers/gpu/drm/vc4/vc4_qpu_defines.h | 17 +- drivers/gpu/drm/vc4/vc4_regs.h | 22 +- drivers/gpu/drm/vc4/vc4_validate.c | 13 +- drivers/gpu/drm/vc4/vc4_validate_shaders.c | 455 +- drivers/gpu/drm/vgem/Makefile | 2 +- drivers/gpu/drm/vgem/vgem_drv.c | 291 +- drivers/gpu/drm/vgem/vgem_drv.h | 20 +- drivers/gpu/drm/vgem/vgem_fence.c | 283 + drivers/gpu/drm/via/via_mm.c | 2 +- drivers/gpu/drm/virtio/Kconfig | 4 - drivers/gpu/drm/virtio/virtgpu_display.c | 187 +- drivers/gpu/drm/virtio/virtgpu_drm_bus.c | 10 - drivers/gpu/drm/virtio/virtgpu_drv.c | 3 +- drivers/gpu/drm/virtio/virtgpu_drv.h | 3 +- drivers/gpu/drm/virtio/virtgpu_plane.c | 150 +- drivers/gpu/drm/virtio/virtgpu_ttm.c | 6 + drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c | 7 +- drivers/gpu/drm/vmwgfx/vmwgfx_drv.c | 8 +- drivers/gpu/drm/vmwgfx/vmwgfx_drv.h | 1 + drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c | 4 +- drivers/gpu/drm/vmwgfx/vmwgfx_fence.c | 2 +- drivers/gpu/drm/vmwgfx/vmwgfx_kms.c | 8 +- drivers/gpu/drm/vmwgfx/vmwgfx_kms.h | 4 +- drivers/gpu/host1x/cdma.c | 42 +- drivers/gpu/host1x/channel.c | 5 +- drivers/gpu/host1x/debug.c | 38 +- drivers/gpu/host1x/dev.c | 16 +- drivers/gpu/host1x/dev.h | 38 +- drivers/gpu/host1x/hw/cdma_hw.c | 23 +- drivers/gpu/host1x/hw/channel_hw.c | 5 +- drivers/gpu/host1x/hw/debug_hw.c | 36 +- drivers/gpu/host1x/hw/intr_hw.c | 30 +- drivers/gpu/host1x/hw/syncpt_hw.c | 10 +- drivers/gpu/host1x/intr.c | 16 +- drivers/gpu/host1x/intr.h | 4 +- drivers/gpu/host1x/job.c | 8 +- drivers/gpu/host1x/mipi.c | 63 +- drivers/gpu/host1x/syncpt.c | 58 +- drivers/gpu/host1x/syncpt.h | 8 +- drivers/gpu/ipu-v3/ipu-dc.c | 9 +- drivers/gpu/ipu-v3/ipu-di.c | 3 - drivers/gpu/ipu-v3/ipu-dmfc.c | 213 +- drivers/gpu/vga/vga_switcheroo.c | 62 +- drivers/hid/Kconfig | 31 +- drivers/hid/Makefile | 3 +- drivers/hid/hid-alps.c | 506 + drivers/hid/hid-apple.c | 2 + drivers/hid/hid-core.c | 10 +- drivers/hid/hid-ids.h | 10 + drivers/hid/hid-led.c | 523 + drivers/hid/i2c-hid/i2c-hid.c | 11 +- drivers/hsi/Makefile | 3 +- drivers/hsi/clients/cmt_speech.c | 4 +- drivers/hsi/clients/ssi_protocol.c | 110 +- drivers/hsi/controllers/omap_ssi.h | 18 +- drivers/hsi/controllers/omap_ssi_core.c | 46 +- drivers/hsi/controllers/omap_ssi_port.c | 226 +- drivers/hsi/hsi_core.c | 781 ++ drivers/hwmon/Kconfig | 42 +- drivers/hwmon/Makefile | 3 + drivers/hwmon/ad7314.c | 48 +- drivers/hwmon/ads7871.c | 65 +- drivers/hwmon/adt7411.c | 47 +- drivers/hwmon/dell-smm-hwmon.c | 41 +- drivers/hwmon/emc6w201.c | 2 +- drivers/hwmon/ftsteutates.c | 819 ++ drivers/hwmon/ina3221.c | 445 + drivers/hwmon/it87.c | 2 +- drivers/hwmon/jc42.c | 16 +- drivers/hwmon/jz4740-hwmon.c | 65 +- drivers/hwmon/lm75.c | 243 +- drivers/hwmon/lm90.c | 455 +- drivers/hwmon/sht3x.c | 775 ++ drivers/hwmon/tmp102.c | 249 +- drivers/hwmon/tmp401.c | 35 +- drivers/hwspinlock/qcom_hwspinlock.c | 1 + drivers/hwtracing/coresight/coresight-etm3x.c | 90 +- drivers/hwtracing/coresight/coresight-etm4x.c | 87 +- drivers/hwtracing/intel_th/core.c | 89 +- drivers/hwtracing/intel_th/gth.c | 18 +- drivers/hwtracing/intel_th/intel_th.h | 6 + drivers/hwtracing/intel_th/pci.c | 5 + drivers/hwtracing/stm/core.c | 52 + drivers/i2c/Kconfig | 4 +- drivers/i2c/busses/Kconfig | 11 +- drivers/i2c/busses/i2c-at91.c | 24 +- drivers/i2c/busses/i2c-bcm-iproc.c | 2 +- drivers/i2c/busses/i2c-bcm-kona.c | 4 +- drivers/i2c/busses/i2c-bcm2835.c | 3 +- drivers/i2c/busses/i2c-brcmstb.c | 5 +- drivers/i2c/busses/i2c-cadence.c | 2 +- drivers/i2c/busses/i2c-designware-core.c | 18 +- drivers/i2c/busses/i2c-designware-core.h | 1 + drivers/i2c/busses/i2c-designware-pcidrv.c | 143 +- drivers/i2c/busses/i2c-elektor.c | 14 +- drivers/i2c/busses/i2c-i801.c | 152 +- drivers/i2c/busses/i2c-jz4780.c | 4 - drivers/i2c/busses/i2c-meson.c | 6 +- drivers/i2c/busses/i2c-ocores.c | 14 +- drivers/i2c/busses/i2c-opal.c | 2 +- drivers/i2c/busses/i2c-pca-isa.c | 15 +- drivers/i2c/busses/i2c-qup.c | 155 +- drivers/i2c/busses/i2c-rcar.c | 2 +- drivers/i2c/busses/i2c-rk3x.c | 512 +- drivers/i2c/busses/i2c-robotfuzz-osif.c | 2 +- drivers/i2c/busses/i2c-sh_mobile.c | 2 +- drivers/i2c/busses/i2c-versatile.c | 46 +- drivers/i2c/busses/i2c-xlp9xx.c | 13 +- drivers/i2c/i2c-core.c | 340 +- drivers/i2c/i2c-dev.c | 7 +- drivers/i2c/i2c-smbus.c | 112 +- drivers/i2c/muxes/i2c-demux-pinctrl.c | 15 +- drivers/ide/cmd640.c | 2 +- drivers/ide/hpt366.c | 2 +- drivers/ide/ide-cd.c | 3 +- drivers/ide/ide-cd_ioctl.c | 3 - drivers/ide/ide-disk.c | 4 +- drivers/ide/ide-floppy.c | 2 +- drivers/ide/ide-gd.c | 3 +- drivers/ide/ide-tape.c | 6 +- drivers/ide/pmac.c | 1 + drivers/idle/intel_idle.c | 178 +- drivers/iio/Kconfig | 8 + drivers/iio/Makefile | 1 + drivers/iio/accel/Kconfig | 29 +- drivers/iio/accel/Makefile | 3 + drivers/iio/accel/bma180.c | 2 +- drivers/iio/accel/bma220_spi.c | 338 + drivers/iio/accel/bmc150-accel-core.c | 4 +- drivers/iio/accel/kxcjk-1013.c | 2 +- drivers/iio/accel/mma7455_core.c | 3 +- drivers/iio/accel/mma7660.c | 277 + drivers/iio/accel/mma8452.c | 223 +- drivers/iio/accel/mma9551.c | 2 +- drivers/iio/accel/mma9553.c | 2 +- drivers/iio/accel/st_accel.h | 1 + drivers/iio/accel/st_accel_core.c | 76 +- drivers/iio/accel/st_accel_i2c.c | 5 + drivers/iio/accel/st_accel_spi.c | 1 + drivers/iio/adc/Kconfig | 12 + drivers/iio/adc/Makefile | 1 + drivers/iio/adc/ad7266.c | 8 +- drivers/iio/adc/ad7291.c | 3 +- drivers/iio/adc/ad7298.c | 3 +- drivers/iio/adc/ad7476.c | 14 +- drivers/iio/adc/ad7791.c | 37 +- drivers/iio/adc/ad7793.c | 33 +- drivers/iio/adc/ad7887.c | 14 +- drivers/iio/adc/ad7923.c | 14 +- drivers/iio/adc/ad799x.c | 29 +- drivers/iio/adc/bcm_iproc_adc.c | 644 ++ drivers/iio/adc/cc10001_adc.c | 2 +- drivers/iio/adc/hi8435.c | 3 +- drivers/iio/adc/ina2xx-adc.c | 7 +- drivers/iio/adc/max1027.c | 1 + drivers/iio/adc/max1363.c | 67 +- drivers/iio/adc/mcp320x.c | 1 + drivers/iio/adc/mcp3422.c | 1 + drivers/iio/adc/mxs-lradc.c | 34 +- drivers/iio/adc/nau7802.c | 20 + drivers/iio/adc/ti-adc081c.c | 30 +- drivers/iio/adc/ti-adc0832.c | 1 + drivers/iio/adc/ti-adc128s052.c | 1 + drivers/iio/adc/ti-ads1015.c | 132 +- drivers/iio/adc/ti-ads8688.c | 1 + drivers/iio/adc/ti_am335x_adc.c | 22 +- drivers/iio/adc/vf610_adc.c | 3 +- drivers/iio/adc/xilinx-xadc-events.c | 4 +- drivers/iio/buffer/industrialio-buffer-dma.c | 4 +- drivers/iio/chemical/Kconfig | 8 +- drivers/iio/chemical/atlas-ph-sensor.c | 269 +- drivers/iio/common/st_sensors/st_sensors_buffer.c | 49 +- drivers/iio/common/st_sensors/st_sensors_core.c | 57 +- drivers/iio/common/st_sensors/st_sensors_i2c.c | 4 +- drivers/iio/common/st_sensors/st_sensors_trigger.c | 156 +- drivers/iio/dac/Kconfig | 9 +- drivers/iio/dac/ad5421.c | 6 +- drivers/iio/dac/ad5504.c | 2 +- drivers/iio/dac/ad5755.c | 188 +- drivers/iio/dac/stx104.c | 145 +- drivers/iio/dummy/Kconfig | 1 + drivers/iio/dummy/iio_simple_dummy.c | 102 +- drivers/iio/dummy/iio_simple_dummy_buffer.c | 3 +- drivers/iio/dummy/iio_simple_dummy_events.c | 2 +- drivers/iio/gyro/bmg160_core.c | 138 +- drivers/iio/gyro/st_gyro_core.c | 12 +- drivers/iio/health/afe4403.c | 299 +- drivers/iio/health/afe4404.c | 308 +- drivers/iio/health/afe440x.h | 48 +- drivers/iio/humidity/Kconfig | 2 + drivers/iio/humidity/am2315.c | 1 + drivers/iio/humidity/htu21.c | 1 + drivers/iio/iio_core.h | 3 + drivers/iio/imu/bmi160/bmi160_core.c | 30 +- drivers/iio/imu/inv_mpu6050/Kconfig | 8 +- drivers/iio/imu/inv_mpu6050/inv_mpu_core.c | 6 + drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c | 1 + drivers/iio/imu/inv_mpu6050/inv_mpu_iio.h | 2 + drivers/iio/imu/inv_mpu6050/inv_mpu_ring.c | 2 +- drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c | 1 + drivers/iio/industrialio-core.c | 180 +- drivers/iio/industrialio-event.c | 19 +- drivers/iio/industrialio-sw-device.c | 182 + drivers/iio/industrialio-trigger.c | 37 +- drivers/iio/light/Kconfig | 3 +- drivers/iio/light/acpi-als.c | 2 +- drivers/iio/light/adjd_s311.c | 2 +- drivers/iio/light/apds9300.c | 2 +- drivers/iio/light/apds9960.c | 4 +- drivers/iio/light/cm36651.c | 2 +- drivers/iio/light/gp2ap020a00f.c | 26 +- drivers/iio/light/isl29125.c | 23 +- drivers/iio/light/jsa1212.c | 3 - drivers/iio/light/lm3533-als.c | 2 +- drivers/iio/light/ltr501.c | 7 +- drivers/iio/light/max44000.c | 3 +- drivers/iio/light/opt3001.c | 4 +- drivers/iio/light/stk3310.c | 2 +- drivers/iio/light/tcs3414.c | 14 +- drivers/iio/light/tcs3472.c | 15 +- drivers/iio/light/tsl2563.c | 2 +- drivers/iio/light/us5182d.c | 2 +- drivers/iio/magnetometer/Kconfig | 2 + drivers/iio/magnetometer/ak8975.c | 154 +- drivers/iio/magnetometer/bmc150_magn_i2c.c | 3 + drivers/iio/magnetometer/bmc150_magn_spi.c | 3 + drivers/iio/magnetometer/hmc5843_core.c | 2 +- drivers/iio/magnetometer/mag3110.c | 2 +- drivers/iio/magnetometer/st_magn_core.c | 12 +- drivers/iio/potentiometer/Kconfig | 23 +- drivers/iio/potentiometer/Makefile | 1 + drivers/iio/potentiometer/max5487.c | 161 + drivers/iio/potentiometer/mcp4531.c | 159 +- drivers/iio/potentiometer/tpl0102.c | 4 - drivers/iio/pressure/Kconfig | 31 +- drivers/iio/pressure/Makefile | 3 + drivers/iio/pressure/bmp280-core.c | 1119 ++ drivers/iio/pressure/bmp280-i2c.c | 91 + drivers/iio/pressure/bmp280-regmap.c | 84 + drivers/iio/pressure/bmp280-spi.c | 125 + drivers/iio/pressure/bmp280.h | 112 + drivers/iio/pressure/hp206c.c | 1 + drivers/iio/pressure/mpl3115.c | 2 +- drivers/iio/pressure/ms5611_core.c | 3 +- drivers/iio/pressure/ms5637.c | 13 +- drivers/iio/pressure/st_pressure.h | 1 + drivers/iio/pressure/st_pressure_core.c | 252 +- drivers/iio/pressure/st_pressure_i2c.c | 4 + drivers/iio/pressure/st_pressure_spi.c | 1 + drivers/iio/proximity/as3935.c | 12 +- drivers/iio/proximity/pulsedlight-lidar-lite-v2.c | 16 +- drivers/iio/proximity/sx9500.c | 4 +- drivers/iio/temperature/tsys02d.c | 1 + drivers/iio/trigger/Kconfig | 12 + drivers/iio/trigger/Makefile | 1 + drivers/iio/trigger/iio-trig-loop.c | 143 + drivers/infiniband/Kconfig | 1 + drivers/infiniband/core/cma.c | 116 +- drivers/infiniband/core/device.c | 9 + drivers/infiniband/core/iwcm.c | 54 +- drivers/infiniband/core/iwcm.h | 2 +- drivers/infiniband/core/iwpm_util.c | 3 +- drivers/infiniband/core/multicast.c | 25 +- drivers/infiniband/core/netlink.c | 6 +- drivers/infiniband/core/sa_query.c | 41 + drivers/infiniband/core/sysfs.c | 15 +- drivers/infiniband/core/ucma.c | 18 +- drivers/infiniband/core/umem.c | 7 +- drivers/infiniband/core/uverbs.h | 13 + drivers/infiniband/core/uverbs_cmd.c | 535 +- drivers/infiniband/core/uverbs_main.c | 38 + drivers/infiniband/core/verbs.c | 163 +- drivers/infiniband/hw/cxgb3/iwch_cm.c | 4 +- drivers/infiniband/hw/cxgb3/iwch_provider.c | 27 +- drivers/infiniband/hw/cxgb4/cm.c | 193 +- drivers/infiniband/hw/cxgb4/cq.c | 52 +- drivers/infiniband/hw/cxgb4/device.c | 7 +- drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 25 +- drivers/infiniband/hw/cxgb4/mem.c | 127 +- drivers/infiniband/hw/cxgb4/provider.c | 31 +- drivers/infiniband/hw/cxgb4/qp.c | 40 +- drivers/infiniband/hw/cxgb4/t4.h | 5 + drivers/infiniband/hw/hfi1/Kconfig | 3 +- drivers/infiniband/hw/hfi1/Makefile | 2 +- drivers/infiniband/hw/hfi1/affinity.c | 572 +- drivers/infiniband/hw/hfi1/affinity.h | 38 +- drivers/infiniband/hw/hfi1/chip.c | 388 +- drivers/infiniband/hw/hfi1/chip.h | 6 +- drivers/infiniband/hw/hfi1/chip_registers.h | 4 + drivers/infiniband/hw/hfi1/debugfs.c | 132 +- drivers/infiniband/hw/hfi1/driver.c | 63 +- drivers/infiniband/hw/hfi1/file_ops.c | 97 +- drivers/infiniband/hw/hfi1/firmware.c | 125 + drivers/infiniband/hw/hfi1/hfi.h | 147 +- drivers/infiniband/hw/hfi1/init.c | 56 +- drivers/infiniband/hw/hfi1/mad.c | 86 +- drivers/infiniband/hw/hfi1/mad.h | 7 - drivers/infiniband/hw/hfi1/mmu_rb.c | 254 +- drivers/infiniband/hw/hfi1/mmu_rb.h | 37 +- drivers/infiniband/hw/hfi1/pcie.c | 68 +- drivers/infiniband/hw/hfi1/pio.c | 21 +- drivers/infiniband/hw/hfi1/pio_copy.c | 12 + drivers/infiniband/hw/hfi1/platform.c | 14 - drivers/infiniband/hw/hfi1/qp.c | 68 +- drivers/infiniband/hw/hfi1/qp.h | 4 +- drivers/infiniband/hw/hfi1/qsfp.c | 441 +- drivers/infiniband/hw/hfi1/qsfp.h | 6 + drivers/infiniband/hw/hfi1/rc.c | 90 +- drivers/infiniband/hw/hfi1/ruc.c | 57 +- drivers/infiniband/hw/hfi1/sysfs.c | 25 + drivers/infiniband/hw/hfi1/trace.h | 1333 +-- drivers/infiniband/hw/hfi1/trace_ctxts.h | 141 + drivers/infiniband/hw/hfi1/trace_dbg.h | 155 + drivers/infiniband/hw/hfi1/trace_ibhdrs.h | 209 + drivers/infiniband/hw/hfi1/trace_misc.h | 81 + drivers/infiniband/hw/hfi1/trace_rc.h | 123 + drivers/infiniband/hw/hfi1/trace_rx.h | 322 + drivers/infiniband/hw/hfi1/trace_tx.h | 642 ++ drivers/infiniband/hw/hfi1/uc.c | 61 +- drivers/infiniband/hw/hfi1/ud.c | 86 +- drivers/infiniband/hw/hfi1/user_exp_rcv.c | 124 +- drivers/infiniband/hw/hfi1/user_pages.c | 19 +- drivers/infiniband/hw/hfi1/user_sdma.c | 306 +- drivers/infiniband/hw/hfi1/user_sdma.h | 8 +- drivers/infiniband/hw/hfi1/verbs.c | 89 +- drivers/infiniband/hw/hfi1/verbs.h | 8 +- drivers/infiniband/hw/hfi1/verbs_txreq.h | 2 +- drivers/infiniband/hw/i40iw/i40iw.h | 4 +- drivers/infiniband/hw/i40iw/i40iw_cm.c | 30 +- drivers/infiniband/hw/i40iw/i40iw_d.h | 3 + drivers/infiniband/hw/i40iw/i40iw_hw.c | 1 + drivers/infiniband/hw/i40iw/i40iw_main.c | 12 +- drivers/infiniband/hw/i40iw/i40iw_puda.c | 4 + drivers/infiniband/hw/i40iw/i40iw_type.h | 2 +- drivers/infiniband/hw/i40iw/i40iw_uk.c | 29 +- drivers/infiniband/hw/i40iw/i40iw_user.h | 2 +- drivers/infiniband/hw/i40iw/i40iw_utils.c | 5 +- drivers/infiniband/hw/i40iw/i40iw_verbs.c | 80 +- drivers/infiniband/hw/mlx4/cq.c | 44 +- drivers/infiniband/hw/mlx4/mad.c | 23 + drivers/infiniband/hw/mlx4/main.c | 225 +- drivers/infiniband/hw/mlx4/mcg.c | 14 +- drivers/infiniband/hw/mlx4/mlx4_ib.h | 11 +- drivers/infiniband/hw/mlx4/qp.c | 48 +- drivers/infiniband/hw/mlx5/cq.c | 109 +- drivers/infiniband/hw/mlx5/main.c | 468 +- drivers/infiniband/hw/mlx5/mem.c | 6 +- drivers/infiniband/hw/mlx5/mlx5_ib.h | 75 + drivers/infiniband/hw/mlx5/mr.c | 4 + drivers/infiniband/hw/mlx5/qp.c | 704 +- drivers/infiniband/hw/mlx5/srq.c | 112 +- drivers/infiniband/hw/mlx5/user.h | 88 +- drivers/infiniband/hw/mthca/mthca_provider.c | 24 +- drivers/infiniband/hw/mthca/mthca_reset.c | 42 +- drivers/infiniband/hw/nes/nes_verbs.c | 33 +- drivers/infiniband/hw/ocrdma/ocrdma_hw.c | 14 +- drivers/infiniband/hw/ocrdma/ocrdma_main.c | 19 +- drivers/infiniband/hw/ocrdma/ocrdma_sli.h | 12 +- drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 4 +- drivers/infiniband/hw/qib/qib_fs.c | 26 +- drivers/infiniband/hw/qib/qib_qp.c | 43 + drivers/infiniband/hw/qib/qib_ud.c | 8 +- drivers/infiniband/hw/qib/qib_verbs.c | 2 + drivers/infiniband/hw/qib/qib_verbs.h | 2 + drivers/infiniband/hw/usnic/usnic_ib_main.c | 19 +- drivers/infiniband/hw/usnic/usnic_ib_sysfs.c | 17 - drivers/infiniband/sw/Makefile | 1 + drivers/infiniband/sw/rdmavt/mr.c | 126 +- drivers/infiniband/sw/rdmavt/mr.h | 2 + drivers/infiniband/sw/rdmavt/qp.c | 249 +- drivers/infiniband/sw/rdmavt/vt.c | 10 +- drivers/infiniband/sw/rxe/Kconfig | 24 + drivers/infiniband/sw/rxe/Makefile | 24 + drivers/infiniband/sw/rxe/rxe.c | 405 + drivers/infiniband/sw/rxe/rxe.h | 77 + drivers/infiniband/sw/rxe/rxe_av.c | 98 + drivers/infiniband/sw/rxe/rxe_comp.c | 747 ++ drivers/infiniband/sw/rxe/rxe_cq.c | 165 + drivers/infiniband/sw/rxe/rxe_dma.c | 166 + drivers/infiniband/sw/rxe/rxe_hdr.h | 952 ++ drivers/infiniband/sw/rxe/rxe_icrc.c | 96 + drivers/infiniband/sw/rxe/rxe_loc.h | 286 + drivers/infiniband/sw/rxe/rxe_mcast.c | 190 + drivers/infiniband/sw/rxe/rxe_mmap.c | 173 + drivers/infiniband/sw/rxe/rxe_mr.c | 643 ++ drivers/infiniband/sw/rxe/rxe_net.c | 703 ++ drivers/infiniband/sw/rxe/rxe_net.h | 56 + drivers/infiniband/sw/rxe/rxe_opcode.c | 961 ++ drivers/infiniband/sw/rxe/rxe_opcode.h | 129 + drivers/infiniband/sw/rxe/rxe_param.h | 172 + drivers/infiniband/sw/rxe/rxe_pool.c | 502 + drivers/infiniband/sw/rxe/rxe_pool.h | 163 + drivers/infiniband/sw/rxe/rxe_qp.c | 851 ++ drivers/infiniband/sw/rxe/rxe_queue.c | 217 + drivers/infiniband/sw/rxe/rxe_queue.h | 178 + drivers/infiniband/sw/rxe/rxe_recv.c | 420 + drivers/infiniband/sw/rxe/rxe_req.c | 757 ++ drivers/infiniband/sw/rxe/rxe_resp.c | 1381 +++ drivers/infiniband/sw/rxe/rxe_srq.c | 193 + drivers/infiniband/sw/rxe/rxe_sysfs.c | 157 + drivers/infiniband/sw/rxe/rxe_task.c | 154 + drivers/infiniband/sw/rxe/rxe_task.h | 95 + drivers/infiniband/sw/rxe/rxe_verbs.c | 1330 +++ drivers/infiniband/sw/rxe/rxe_verbs.h | 480 + drivers/infiniband/ulp/ipoib/ipoib.h | 1 + drivers/infiniband/ulp/ipoib/ipoib_cm.c | 16 + drivers/infiniband/ulp/ipoib/ipoib_ethtool.c | 6 +- drivers/infiniband/ulp/ipoib/ipoib_ib.c | 9 + drivers/infiniband/ulp/ipoib/ipoib_main.c | 5 +- drivers/infiniband/ulp/ipoib/ipoib_verbs.c | 10 +- drivers/infiniband/ulp/isert/ib_isert.c | 27 +- drivers/infiniband/ulp/isert/ib_isert.h | 3 +- drivers/infiniband/ulp/srpt/ib_srpt.c | 16 +- drivers/input/input-mt.c | 17 +- drivers/input/input.c | 7 +- drivers/input/joydev.c | 6 + drivers/input/joystick/xpad.c | 43 + drivers/input/keyboard/clps711x-keypad.c | 4 +- drivers/input/keyboard/cros_ec_keyb.c | 19 +- drivers/input/keyboard/tc3589x-keypad.c | 2 +- drivers/input/keyboard/tegra-kbc.c | 2 +- drivers/input/misc/Kconfig | 23 + drivers/input/misc/Makefile | 2 + drivers/input/misc/apanel.c | 2 +- drivers/input/misc/atmel_captouch.c | 290 + drivers/input/misc/hisi_powerkey.c | 142 + drivers/input/misc/regulator-haptic.c | 2 +- drivers/input/misc/rotary_encoder.c | 23 +- drivers/input/misc/xen-kbdfront.c | 8 +- drivers/input/mouse/elantech.c | 10 +- drivers/input/mouse/lifebook.c | 2 +- drivers/input/rmi4/rmi_bus.c | 5 +- drivers/input/rmi4/rmi_f01.c | 22 +- drivers/input/rmi4/rmi_f11.c | 9 +- drivers/input/rmi4/rmi_f12.c | 1 - drivers/input/rmi4/rmi_i2c.c | 46 + drivers/input/serio/ams_delta_serio.c | 2 +- drivers/input/tablet/Kconfig | 15 + drivers/input/tablet/Makefile | 1 + drivers/input/tablet/pegasus_notetaker.c | 450 + drivers/input/touchscreen/Kconfig | 53 +- drivers/input/touchscreen/Makefile | 4 + drivers/input/touchscreen/ad7879.c | 2 +- drivers/input/touchscreen/ads7846.c | 1 - drivers/input/touchscreen/chipone_icn8318.c | 61 +- drivers/input/touchscreen/cyttsp_core.c | 2 +- drivers/input/touchscreen/edt-ft5x06.c | 7 +- drivers/input/touchscreen/ili210x.c | 2 +- drivers/input/touchscreen/migor_ts.c | 6 +- drivers/input/touchscreen/of_touchscreen.c | 81 +- drivers/input/touchscreen/pixcir_i2c_ts.c | 53 +- drivers/input/touchscreen/raydium_i2c_ts.c | 1238 +++ drivers/input/touchscreen/silead.c | 566 + drivers/input/touchscreen/sis_i2c.c | 413 + drivers/input/touchscreen/sur40.c | 16 +- drivers/input/touchscreen/surface3_spi.c | 427 + drivers/input/touchscreen/ti_am335x_tsc.c | 2 +- drivers/input/touchscreen/tsc200x-core.c | 2 +- drivers/input/touchscreen/wacom_w8001.c | 10 +- drivers/iommu/Kconfig | 21 +- drivers/iommu/Makefile | 3 +- drivers/iommu/amd_iommu.c | 1014 +- drivers/iommu/amd_iommu_types.h | 1 - drivers/iommu/amd_iommu_v2.c | 5 +- drivers/iommu/arm-smmu-v3.c | 2 + drivers/iommu/arm-smmu.c | 28 +- drivers/iommu/dma-iommu.c | 19 +- drivers/iommu/dmar.c | 21 +- drivers/iommu/exynos-iommu.c | 106 +- drivers/iommu/intel-iommu.c | 16 +- drivers/iommu/intel-svm.c | 2 +- drivers/iommu/iommu.c | 32 +- drivers/iommu/msm_iommu.c | 870 +- drivers/iommu/msm_iommu.h | 73 +- drivers/iommu/mtk_iommu.c | 49 +- drivers/iommu/mtk_iommu.h | 77 + drivers/iommu/mtk_iommu_v1.c | 727 ++ drivers/iommu/of_iommu.c | 5 +- drivers/iommu/rockchip-iommu.c | 181 +- drivers/irqchip/Kconfig | 24 +- drivers/irqchip/Makefile | 2 + drivers/irqchip/exynos-combiner.c | 14 +- drivers/irqchip/irq-armada-370-xp.c | 46 +- drivers/irqchip/irq-aspeed-vic.c | 230 + drivers/irqchip/irq-bcm2835.c | 3 +- drivers/irqchip/irq-bcm2836.c | 40 +- drivers/irqchip/irq-bcm7120-l2.c | 10 +- drivers/irqchip/irq-brcmstb-l2.c | 4 +- drivers/irqchip/irq-clps711x.c | 2 +- drivers/irqchip/irq-gic-common.c | 4 +- drivers/irqchip/irq-gic-pm.c | 184 + drivers/irqchip/irq-gic-v2m.c | 1 + drivers/irqchip/irq-gic-v3-its.c | 411 +- drivers/irqchip/irq-gic-v3.c | 40 +- drivers/irqchip/irq-gic.c | 164 +- drivers/irqchip/irq-hip04.c | 25 +- drivers/irqchip/irq-mips-gic.c | 5 +- drivers/irqchip/irq-omap-intc.c | 2 + drivers/irqchip/irq-s3c24xx.c | 36 +- drivers/irqchip/irq-sirfsoc.c | 11 +- drivers/irqchip/irq-tegra.c | 4 +- drivers/irqchip/irq-vic.c | 5 +- drivers/isdn/hardware/eicon/divasmain.c | 12 +- drivers/isdn/hardware/eicon/platform.h | 6 +- drivers/leds/Kconfig | 14 + drivers/leds/Makefile | 1 + drivers/leds/led-triggers.c | 2 + drivers/leds/leds-gpio.c | 5 +- drivers/leds/leds-hp6xx.c | 2 +- drivers/leds/leds-is31fl32xx.c | 24 +- drivers/leds/leds-lp3952.c | 301 + drivers/leds/leds-pca9532.c | 75 +- drivers/leds/leds-powernv.c | 2 +- drivers/leds/trigger/Kconfig | 8 +- drivers/leds/trigger/Makefile | 2 +- drivers/leds/trigger/ledtrig-cpu.c | 32 +- drivers/leds/trigger/ledtrig-disk.c | 41 + drivers/lightnvm/Kconfig | 10 +- drivers/lightnvm/core.c | 242 +- drivers/lightnvm/gennvm.c | 385 +- drivers/lightnvm/gennvm.h | 10 +- drivers/lightnvm/rrpc.c | 155 +- drivers/lightnvm/rrpc.h | 13 +- drivers/lightnvm/sysblk.c | 8 +- drivers/macintosh/Kconfig | 13 +- drivers/macintosh/ams/ams-i2c.c | 1 - drivers/macintosh/smu.c | 9 +- drivers/macintosh/via-pmu-led.c | 4 +- drivers/macintosh/windfarm_pm112.c | 1 - drivers/macintosh/windfarm_pm72.c | 1 - drivers/macintosh/windfarm_rm31.c | 1 - drivers/mailbox/Kconfig | 10 + drivers/mailbox/Makefile | 2 + drivers/mailbox/bcm-pdc-mailbox.c | 1532 +++ drivers/mailbox/mailbox-test.c | 1 + drivers/mailbox/pl320-ipc.c | 46 +- drivers/md/Makefile | 3 +- drivers/md/bcache/btree.c | 4 +- drivers/md/bcache/closure.c | 2 +- drivers/md/bcache/closure.h | 3 +- drivers/md/bcache/debug.c | 6 +- drivers/md/bcache/io.c | 3 +- drivers/md/bcache/journal.c | 9 +- drivers/md/bcache/movinggc.c | 2 +- drivers/md/bcache/request.c | 34 +- drivers/md/bcache/super.c | 48 +- drivers/md/bcache/writeback.c | 4 +- drivers/md/bcache/writeback.h | 2 +- drivers/md/bitmap.c | 53 +- drivers/md/dm-bufio.c | 11 +- drivers/md/dm-builtin.c | 2 +- drivers/md/dm-cache-target.c | 18 +- drivers/md/dm-core.h | 149 + drivers/md/dm-crypt.c | 17 +- drivers/md/dm-era-target.c | 4 +- drivers/md/dm-flakey.c | 6 +- drivers/md/dm-io.c | 65 +- drivers/md/dm-ioctl.c | 31 +- drivers/md/dm-kcopyd.c | 13 +- drivers/md/dm-linear.c | 21 +- drivers/md/dm-log-writes.c | 19 +- drivers/md/dm-log.c | 14 +- drivers/md/dm-mpath.c | 374 +- drivers/md/dm-raid.c | 3100 +++++- drivers/md/dm-raid1.c | 32 +- drivers/md/dm-region-hash.c | 6 +- drivers/md/dm-rq.c | 988 ++ drivers/md/dm-rq.h | 64 + drivers/md/dm-snap-persistent.c | 24 +- drivers/md/dm-snap.c | 27 +- drivers/md/dm-stats.c | 11 +- drivers/md/dm-stripe.c | 34 +- drivers/md/dm-sysfs.c | 3 +- drivers/md/dm-table.c | 114 +- drivers/md/dm-target.c | 11 +- drivers/md/dm-thin-metadata.c | 30 + drivers/md/dm-thin-metadata.h | 3 + drivers/md/dm-thin.c | 124 +- drivers/md/dm-zero.c | 15 +- drivers/md/dm.c | 1540 +-- drivers/md/dm.h | 36 +- drivers/md/linear.c | 4 +- drivers/md/md-cluster.c | 12 +- drivers/md/md.c | 122 +- drivers/md/md.h | 15 +- drivers/md/multipath.c | 37 +- drivers/md/persistent-data/dm-btree.c | 9 +- drivers/md/raid0.c | 4 +- drivers/md/raid1.c | 173 +- drivers/md/raid10.c | 320 +- drivers/md/raid10.h | 3 +- drivers/md/raid5-cache.c | 79 +- drivers/md/raid5.c | 183 +- drivers/media/Kconfig | 3 + drivers/media/Makefile | 4 + drivers/media/cec-edid.c | 171 + drivers/media/common/v4l2-tpg/v4l2-tpg-core.c | 4 +- drivers/media/dvb-core/demux.h | 167 +- drivers/media/dvb-core/dmxdev.c | 2 +- drivers/media/dvb-core/dvb_ca_en50221.c | 39 +- drivers/media/dvb-core/dvb_demux.c | 17 +- drivers/media/dvb-core/dvb_demux.h | 4 +- drivers/media/dvb-core/dvb_frontend.c | 33 +- drivers/media/dvb-core/dvb_frontend.h | 53 +- drivers/media/dvb-core/dvb_math.h | 7 + drivers/media/dvb-core/dvb_net.c | 2 +- drivers/media/dvb-core/dvb_ringbuffer.h | 15 +- drivers/media/dvb-frontends/Kconfig | 15 + drivers/media/dvb-frontends/Makefile | 2 + drivers/media/dvb-frontends/af9033.c | 327 +- drivers/media/dvb-frontends/ascot2e.c | 2 +- drivers/media/dvb-frontends/cxd2841er.c | 1941 +++- drivers/media/dvb-frontends/cxd2841er.h | 24 +- drivers/media/dvb-frontends/cxd2841er_priv.h | 1 + drivers/media/dvb-frontends/dib0090.c | 6 + drivers/media/dvb-frontends/drx39xyj/drxj.c | 3 + drivers/media/dvb-frontends/ds3000.c | 9 + drivers/media/dvb-frontends/helene.c | 1042 ++ drivers/media/dvb-frontends/helene.h | 79 + drivers/media/dvb-frontends/horus3a.c | 26 +- drivers/media/dvb-frontends/m88ds3103.c | 144 +- drivers/media/dvb-frontends/m88ds3103_priv.h | 3 +- drivers/media/dvb-frontends/m88rs2000.c | 2 +- drivers/media/dvb-frontends/mb86a20s.c | 3 +- drivers/media/dvb-frontends/mn88472.c | 613 ++ drivers/media/dvb-frontends/mn88472.h | 45 +- drivers/media/dvb-frontends/mn88472_priv.h | 38 + drivers/media/dvb-frontends/mn88473.c | 7 +- drivers/media/dvb-frontends/rtl2830.c | 203 +- drivers/media/dvb-frontends/rtl2830_priv.h | 2 +- drivers/media/dvb-frontends/rtl2832.c | 26 +- drivers/media/dvb-frontends/rtl2832_priv.h | 1 + drivers/media/dvb-frontends/rtl2832_sdr.c | 2 +- drivers/media/dvb-frontends/si2168.c | 127 +- drivers/media/dvb-frontends/si2168_priv.h | 8 +- drivers/media/i2c/Kconfig | 24 + drivers/media/i2c/adv7180.c | 18 +- drivers/media/i2c/adv7511.c | 465 +- drivers/media/i2c/adv7604.c | 393 +- drivers/media/i2c/adv7842.c | 413 +- drivers/media/i2c/cs53l32a.c | 7 - drivers/media/i2c/cx25840/cx25840-core.c | 7 - drivers/media/i2c/msp3400-driver.c | 7 - drivers/media/i2c/mt9t001.c | 17 +- drivers/media/i2c/mt9v032.c | 279 +- drivers/media/i2c/saa7115.c | 7 - drivers/media/i2c/smiapp/smiapp-core.c | 4 +- drivers/media/i2c/tc358743.c | 15 +- drivers/media/i2c/tvaudio.c | 7 - drivers/media/i2c/wm8775.c | 7 - drivers/media/media-device.c | 47 +- drivers/media/media-devnode.c | 149 +- drivers/media/pci/bt8xx/dst_ca.c | 2 - drivers/media/pci/cobalt/cobalt-driver.c | 11 - drivers/media/pci/cobalt/cobalt-driver.h | 1 - drivers/media/pci/cobalt/cobalt-v4l2.c | 4 +- drivers/media/pci/cx18/cx18-alsa-mixer.c | 6 +- drivers/media/pci/cx18/cx18-driver.c | 2 +- drivers/media/pci/cx18/cx18-driver.h | 6 +- drivers/media/pci/cx18/cx18-ioctl.c | 2 +- drivers/media/pci/cx18/cx18-streams.c | 12 +- drivers/media/pci/cx18/cx18-vbi.c | 6 +- drivers/media/pci/cx23885/cx23885-417.c | 4 +- drivers/media/pci/cx23885/cx23885-cards.c | 59 +- drivers/media/pci/cx23885/cx23885-core.c | 10 +- drivers/media/pci/cx23885/cx23885-dvb.c | 104 +- drivers/media/pci/cx23885/cx23885-vbi.c | 3 +- drivers/media/pci/cx23885/cx23885-video.c | 5 +- drivers/media/pci/cx23885/cx23885.h | 2 +- drivers/media/pci/cx25821/cx25821-alsa.c | 2 +- drivers/media/pci/cx25821/cx25821-core.c | 10 +- drivers/media/pci/cx25821/cx25821-video.c | 5 +- drivers/media/pci/cx25821/cx25821.h | 1 - drivers/media/pci/cx88/cx88-alsa.c | 8 +- drivers/media/pci/cx88/cx88-blackbird.c | 4 +- drivers/media/pci/cx88/cx88-dvb.c | 4 +- drivers/media/pci/cx88/cx88-mpeg.c | 10 +- drivers/media/pci/cx88/cx88-vbi.c | 3 +- drivers/media/pci/cx88/cx88-video.c | 13 +- drivers/media/pci/cx88/cx88.h | 2 - drivers/media/pci/ddbridge/ddbridge-core.c | 3 +- drivers/media/pci/dt3155/dt3155.c | 15 +- drivers/media/pci/dt3155/dt3155.h | 2 - drivers/media/pci/ivtv/ivtv-alsa-mixer.c | 6 +- drivers/media/pci/netup_unidvb/Kconfig | 7 +- drivers/media/pci/netup_unidvb/netup_unidvb.h | 10 + drivers/media/pci/netup_unidvb/netup_unidvb_ci.c | 4 +- drivers/media/pci/netup_unidvb/netup_unidvb_core.c | 174 +- drivers/media/pci/saa7134/saa7134-core.c | 22 +- drivers/media/pci/saa7134/saa7134-dvb.c | 1 + drivers/media/pci/saa7134/saa7134-empress.c | 1 + drivers/media/pci/saa7134/saa7134-ts.c | 3 +- drivers/media/pci/saa7134/saa7134-vbi.c | 3 +- drivers/media/pci/saa7134/saa7134-video.c | 5 +- drivers/media/pci/saa7134/saa7134.h | 3 +- drivers/media/pci/saa7164/saa7164-encoder.c | 6 +- drivers/media/pci/saa7164/saa7164.h | 4 - drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c | 15 +- drivers/media/pci/solo6x10/solo6x10-v4l2.c | 44 +- drivers/media/pci/solo6x10/solo6x10.h | 2 - drivers/media/pci/sta2x11/sta2x11_vip.c | 20 +- drivers/media/pci/tw68/tw68-core.c | 15 +- drivers/media/pci/tw68/tw68-video.c | 4 +- drivers/media/pci/tw68/tw68.h | 1 - drivers/media/pci/tw686x/Kconfig | 2 + drivers/media/pci/tw686x/tw686x-audio.c | 92 +- drivers/media/pci/tw686x/tw686x-core.c | 56 +- drivers/media/pci/tw686x/tw686x-regs.h | 9 + drivers/media/pci/tw686x/tw686x-video.c | 595 +- drivers/media/pci/tw686x/tw686x.h | 42 +- drivers/media/pci/zoran/zr36016.c | 4 - drivers/media/platform/Kconfig | 45 + drivers/media/platform/Makefile | 7 + drivers/media/platform/am437x/am437x-vpfe.c | 14 +- drivers/media/platform/am437x/am437x-vpfe.h | 2 - drivers/media/platform/blackfin/bfin_capture.c | 17 +- drivers/media/platform/coda/coda-common.c | 20 +- drivers/media/platform/coda/coda.h | 1 - drivers/media/platform/davinci/ccdc_hw_device.h | 7 - drivers/media/platform/davinci/vpbe_display.c | 14 +- drivers/media/platform/davinci/vpif_capture.c | 15 +- drivers/media/platform/davinci/vpif_capture.h | 2 - drivers/media/platform/davinci/vpif_display.c | 15 +- drivers/media/platform/davinci/vpif_display.h | 2 - drivers/media/platform/exynos-gsc/gsc-core.c | 12 +- drivers/media/platform/exynos-gsc/gsc-core.h | 2 - drivers/media/platform/exynos-gsc/gsc-m2m.c | 8 +- drivers/media/platform/exynos4-is/fimc-capture.c | 9 +- drivers/media/platform/exynos4-is/fimc-core.c | 12 +- drivers/media/platform/exynos4-is/fimc-core.h | 3 - drivers/media/platform/exynos4-is/fimc-is.c | 15 +- drivers/media/platform/exynos4-is/fimc-is.h | 2 - drivers/media/platform/exynos4-is/fimc-isp-video.c | 11 +- drivers/media/platform/exynos4-is/fimc-isp.h | 2 - drivers/media/platform/exynos4-is/fimc-lite.c | 22 +- drivers/media/platform/exynos4-is/fimc-lite.h | 2 - drivers/media/platform/exynos4-is/fimc-m2m.c | 32 +- drivers/media/platform/exynos4-is/mipi-csis.c | 17 - drivers/media/platform/m2m-deinterlace.c | 17 +- drivers/media/platform/marvell-ccic/mcam-core.c | 28 +- drivers/media/platform/marvell-ccic/mcam-core.h | 2 - drivers/media/platform/mtk-vcodec/Makefile | 19 + drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h | 334 + drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c | 1302 +++ drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.h | 58 + .../media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c | 439 + .../media/platform/mtk-vcodec/mtk_vcodec_enc_pm.c | 137 + .../media/platform/mtk-vcodec/mtk_vcodec_enc_pm.h | 26 + .../media/platform/mtk-vcodec/mtk_vcodec_intr.c | 54 + .../media/platform/mtk-vcodec/mtk_vcodec_intr.h | 26 + .../media/platform/mtk-vcodec/mtk_vcodec_util.c | 94 + .../media/platform/mtk-vcodec/mtk_vcodec_util.h | 87 + .../media/platform/mtk-vcodec/venc/venc_h264_if.c | 679 ++ .../media/platform/mtk-vcodec/venc/venc_vp8_if.c | 484 + drivers/media/platform/mtk-vcodec/venc_drv_base.h | 62 + drivers/media/platform/mtk-vcodec/venc_drv_if.c | 113 + drivers/media/platform/mtk-vcodec/venc_drv_if.h | 163 + drivers/media/platform/mtk-vcodec/venc_ipi_msg.h | 210 + drivers/media/platform/mtk-vcodec/venc_vpu_if.c | 238 + drivers/media/platform/mtk-vcodec/venc_vpu_if.h | 61 + drivers/media/platform/mtk-vpu/Makefile | 3 + drivers/media/platform/mtk-vpu/mtk_vpu.c | 946 ++ drivers/media/platform/mtk-vpu/mtk_vpu.h | 162 + drivers/media/platform/mx2_emmaprp.c | 19 +- drivers/media/platform/omap/omap_vout.c | 111 +- drivers/media/platform/omap/omap_voutdef.h | 7 +- drivers/media/platform/omap/omap_voutlib.c | 2 +- drivers/media/platform/omap3isp/ispvideo.c | 14 +- drivers/media/platform/omap3isp/ispvideo.h | 1 - drivers/media/platform/rcar-fcp.c | 187 + drivers/media/platform/rcar-vin/Kconfig | 11 + drivers/media/platform/rcar-vin/Makefile | 3 + drivers/media/platform/rcar-vin/rcar-core.c | 334 + drivers/media/platform/rcar-vin/rcar-dma.c | 1187 ++ drivers/media/platform/rcar-vin/rcar-v4l2.c | 874 ++ drivers/media/platform/rcar-vin/rcar-vin.h | 163 + drivers/media/platform/rcar_jpu.c | 24 +- drivers/media/platform/s3c-camif/camif-capture.c | 5 +- drivers/media/platform/s3c-camif/camif-core.c | 11 +- drivers/media/platform/s3c-camif/camif-core.h | 2 - drivers/media/platform/s5p-g2d/g2d.c | 17 +- drivers/media/platform/s5p-g2d/g2d.h | 1 - drivers/media/platform/s5p-jpeg/jpeg-core.c | 21 +- drivers/media/platform/s5p-jpeg/jpeg-core.h | 2 - drivers/media/platform/s5p-mfc/s5p_mfc.c | 230 +- drivers/media/platform/s5p-mfc/s5p_mfc_common.h | 4 +- drivers/media/platform/s5p-mfc/s5p_mfc_dec.c | 28 +- drivers/media/platform/s5p-mfc/s5p_mfc_enc.c | 35 +- drivers/media/platform/s5p-mfc/s5p_mfc_iommu.h | 79 + drivers/media/platform/s5p-mfc/s5p_mfc_pm.c | 13 +- drivers/media/platform/s5p-tv/mixer.h | 2 - drivers/media/platform/s5p-tv/mixer_video.c | 19 +- drivers/media/platform/sh_veu.c | 19 +- drivers/media/platform/sh_vou.c | 16 +- drivers/media/platform/soc_camera/Kconfig | 4 +- drivers/media/platform/soc_camera/Makefile | 2 +- drivers/media/platform/soc_camera/atmel-isi.c | 15 +- drivers/media/platform/soc_camera/rcar_vin.c | 14 +- .../platform/soc_camera/sh_mobile_ceu_camera.c | 17 +- drivers/media/platform/sti/bdisp/bdisp-filter.h | 304 - drivers/media/platform/sti/bdisp/bdisp-hw.c | 331 +- drivers/media/platform/sti/bdisp/bdisp-v4l2.c | 18 +- drivers/media/platform/sti/bdisp/bdisp.h | 2 - .../media/platform/sti/c8sectpfe/c8sectpfe-core.c | 1 + drivers/media/platform/ti-vpe/cal.c | 17 +- drivers/media/platform/ti-vpe/vpe.c | 22 +- drivers/media/platform/via-camera.c | 2 +- drivers/media/platform/vim2m.c | 22 +- drivers/media/platform/vivid/Kconfig | 8 + drivers/media/platform/vivid/Makefile | 4 + drivers/media/platform/vivid/vivid-cec.c | 231 + drivers/media/platform/vivid/vivid-cec.h | 33 + drivers/media/platform/vivid/vivid-core.c | 118 +- drivers/media/platform/vivid/vivid-core.h | 27 + drivers/media/platform/vivid/vivid-kthread-cap.c | 13 + drivers/media/platform/vivid/vivid-sdr-cap.c | 4 +- drivers/media/platform/vivid/vivid-vbi-cap.c | 2 +- drivers/media/platform/vivid/vivid-vbi-out.c | 2 +- drivers/media/platform/vivid/vivid-vid-cap.c | 34 +- drivers/media/platform/vivid/vivid-vid-common.c | 7 + drivers/media/platform/vivid/vivid-vid-out.c | 7 +- drivers/media/platform/vsp1/Makefile | 3 +- drivers/media/platform/vsp1/vsp1.h | 11 +- drivers/media/platform/vsp1/vsp1_bru.c | 12 +- drivers/media/platform/vsp1/vsp1_clu.c | 292 + drivers/media/platform/vsp1/vsp1_clu.h | 48 + drivers/media/platform/vsp1/vsp1_dl.c | 72 +- drivers/media/platform/vsp1/vsp1_drm.c | 74 +- drivers/media/platform/vsp1/vsp1_drv.c | 191 +- drivers/media/platform/vsp1/vsp1_entity.c | 92 +- drivers/media/platform/vsp1/vsp1_entity.h | 17 +- drivers/media/platform/vsp1/vsp1_hsit.c | 14 +- drivers/media/platform/vsp1/vsp1_lif.c | 16 +- drivers/media/platform/vsp1/vsp1_lut.c | 101 +- drivers/media/platform/vsp1/vsp1_lut.h | 7 +- drivers/media/platform/vsp1/vsp1_pipe.c | 58 +- drivers/media/platform/vsp1/vsp1_pipe.h | 8 +- drivers/media/platform/vsp1/vsp1_regs.h | 24 +- drivers/media/platform/vsp1/vsp1_rpf.c | 38 +- drivers/media/platform/vsp1/vsp1_rwpf.c | 6 +- drivers/media/platform/vsp1/vsp1_rwpf.h | 14 +- drivers/media/platform/vsp1/vsp1_sru.c | 14 +- drivers/media/platform/vsp1/vsp1_uds.c | 16 +- drivers/media/platform/vsp1/vsp1_uds.h | 2 +- drivers/media/platform/vsp1/vsp1_video.c | 40 +- drivers/media/platform/vsp1/vsp1_video.h | 2 - drivers/media/platform/vsp1/vsp1_wpf.c | 161 +- drivers/media/platform/xilinx/xilinx-dma.c | 13 +- drivers/media/platform/xilinx/xilinx-dma.h | 2 - drivers/media/radio/radio-aztech.c | 1 - drivers/media/radio/radio-maxiradio.c | 1 + drivers/media/radio/wl128x/fmdrv_common.c | 2 +- drivers/media/rc/Kconfig | 2 +- drivers/media/rc/ene_ir.c | 2 +- drivers/media/rc/iguanair.c | 2 +- drivers/media/rc/ir-lirc-codec.c | 5 +- drivers/media/rc/ir-rx51.c | 229 +- drivers/media/rc/keymaps/Makefile | 2 + drivers/media/rc/keymaps/rc-cec.c | 182 + drivers/media/rc/keymaps/rc-dtt200u.c | 59 + drivers/media/rc/lirc_dev.c | 299 +- drivers/media/rc/mceusb.c | 8 +- drivers/media/rc/nuvoton-cir.c | 137 +- drivers/media/rc/nuvoton-cir.h | 25 - drivers/media/rc/rc-main.c | 11 +- drivers/media/rc/redrat3.c | 84 +- drivers/media/rc/winbond-cir.c | 4 + drivers/media/tuners/it913x.c | 1 + drivers/media/tuners/mt2063.c | 30 +- drivers/media/tuners/r820t.c | 29 +- drivers/media/tuners/si2157.c | 3 +- drivers/media/usb/airspy/airspy.c | 2 +- drivers/media/usb/au0828/au0828-core.c | 4 +- drivers/media/usb/au0828/au0828-vbi.c | 2 +- drivers/media/usb/au0828/au0828-video.c | 2 +- drivers/media/usb/cx231xx/cx231xx-417.c | 4 +- drivers/media/usb/dvb-usb-v2/Kconfig | 13 +- drivers/media/usb/dvb-usb-v2/af9035.c | 275 +- drivers/media/usb/dvb-usb-v2/af9035.h | 3 +- drivers/media/usb/dvb-usb-v2/rtl28xxu.c | 2 +- drivers/media/usb/dvb-usb/dtt200u.c | 74 +- drivers/media/usb/dvb-usb/dvb-usb-dvb.c | 2 - drivers/media/usb/dvb-usb/dw2102.c | 48 +- drivers/media/usb/em28xx/em28xx-dvb.c | 11 - drivers/media/usb/em28xx/em28xx-i2c.c | 5 +- drivers/media/usb/em28xx/em28xx-vbi.c | 2 +- drivers/media/usb/em28xx/em28xx-video.c | 2 +- drivers/media/usb/go7007/go7007-v4l2.c | 2 +- drivers/media/usb/gspca/cpia1.c | 2 +- drivers/media/usb/gspca/gspca.c | 29 +- drivers/media/usb/gspca/konica.c | 2 +- drivers/media/usb/gspca/m5602/m5602_bridge.h | 15 - drivers/media/usb/gspca/m5602/m5602_core.c | 15 + drivers/media/usb/gspca/m5602/m5602_mt9m111.c | 144 + drivers/media/usb/gspca/m5602/m5602_mt9m111.h | 144 - drivers/media/usb/gspca/m5602/m5602_ov7660.c | 153 + drivers/media/usb/gspca/m5602/m5602_ov7660.h | 153 - drivers/media/usb/gspca/m5602/m5602_ov9650.c | 152 + drivers/media/usb/gspca/m5602/m5602_ov9650.h | 150 - drivers/media/usb/gspca/m5602/m5602_po1030.c | 104 + drivers/media/usb/gspca/m5602/m5602_po1030.h | 104 - drivers/media/usb/gspca/m5602/m5602_s5k4aa.c | 199 + drivers/media/usb/gspca/m5602/m5602_s5k4aa.h | 197 - drivers/media/usb/gspca/m5602/m5602_s5k83a.c | 124 + drivers/media/usb/gspca/m5602/m5602_s5k83a.h | 124 - drivers/media/usb/gspca/ov534.c | 7 +- drivers/media/usb/gspca/sn9c20x.c | 14 +- drivers/media/usb/gspca/t613.c | 2 +- drivers/media/usb/gspca/topro.c | 6 +- drivers/media/usb/gspca/zc3xx.c | 13 +- drivers/media/usb/hackrf/hackrf.c | 2 +- drivers/media/usb/hdpvr/hdpvr-core.c | 10 +- drivers/media/usb/hdpvr/hdpvr-video.c | 6 +- drivers/media/usb/hdpvr/hdpvr.h | 2 - drivers/media/usb/msi2500/msi2500.c | 2 +- drivers/media/usb/pvrusb2/pvrusb2-hdw.c | 6 +- drivers/media/usb/pwc/pwc-if.c | 4 +- drivers/media/usb/s2255/s2255drv.c | 2 +- drivers/media/usb/stk1160/stk1160-v4l.c | 5 +- drivers/media/usb/usbtv/usbtv-audio.c | 28 +- drivers/media/usb/usbtv/usbtv-core.c | 40 +- drivers/media/usb/usbtv/usbtv-video.c | 61 +- drivers/media/usb/usbtv/usbtv.h | 22 +- drivers/media/usb/usbvision/usbvision-core.c | 5 - drivers/media/usb/usbvision/usbvision-video.c | 40 +- drivers/media/usb/uvc/uvc_driver.c | 2 +- drivers/media/usb/uvc/uvc_queue.c | 2 +- drivers/media/usb/uvc/uvc_v4l2.c | 19 +- drivers/media/usb/uvc/uvc_video.c | 1 + drivers/media/v4l2-core/v4l2-ctrls.c | 45 - drivers/media/v4l2-core/v4l2-dev.c | 34 - drivers/media/v4l2-core/v4l2-flash-led-class.c | 9 +- drivers/media/v4l2-core/v4l2-ioctl.c | 4 +- drivers/media/v4l2-core/v4l2-subdev.c | 10 - drivers/media/v4l2-core/videobuf2-core.c | 30 +- drivers/media/v4l2-core/videobuf2-dma-contig.c | 112 +- drivers/media/v4l2-core/videobuf2-dma-sg.c | 64 +- drivers/media/v4l2-core/videobuf2-v4l2.c | 47 +- drivers/media/v4l2-core/videobuf2-vmalloc.c | 9 +- drivers/memory/Kconfig | 13 +- drivers/memory/Makefile | 1 + drivers/memory/atmel-ebi.c | 766 ++ drivers/memory/atmel-sdramc.c | 11 +- drivers/memory/fsl_ifc.c | 4 +- drivers/memory/mtk-smi.c | 167 +- drivers/memory/omap-gpmc.c | 139 +- drivers/memory/samsung/exynos-srom.c | 41 +- drivers/memory/tegra/mc.c | 10 +- drivers/memory/tegra/tegra124-emc.c | 8 +- drivers/memstick/core/ms_block.c | 23 +- drivers/memstick/core/mspro_block.c | 6 +- drivers/mfd/Kconfig | 25 +- drivers/mfd/Makefile | 2 + drivers/mfd/ab8500-core.c | 4 - drivers/mfd/ab8500-sysctrl.c | 34 - drivers/mfd/altera-a10sr.c | 169 + drivers/mfd/arizona-core.c | 4 +- drivers/mfd/arizona-irq.c | 16 +- drivers/mfd/atmel-hlcdc.c | 5 +- drivers/mfd/axp20x.c | 17 + drivers/mfd/db8500-prcmu.c | 10 +- drivers/mfd/dm355evm_msp.c | 7 +- drivers/mfd/hi655x-pmic.c | 59 +- drivers/mfd/kempld-core.c | 16 + drivers/mfd/max14577.c | 2 +- drivers/mfd/max77620.c | 75 +- drivers/mfd/max77843.c | 24 +- drivers/mfd/max8925-i2c.c | 14 +- drivers/mfd/max8997.c | 30 +- drivers/mfd/max8998.c | 27 +- drivers/mfd/omap-usb-tll.c | 2 + drivers/mfd/qcom_rpm.c | 2 - drivers/mfd/rn5t618.c | 70 +- drivers/mfd/rtsx_usb.c | 10 +- drivers/mfd/si476x-i2c.c | 2 +- drivers/mfd/smsc-ece1099.c | 11 +- drivers/mfd/stmpe.c | 40 +- drivers/mfd/ti_am335x_tscadc.c | 135 +- drivers/mfd/tps6507x.c | 4 +- drivers/mfd/twl-core.c | 28 +- drivers/mfd/twl6040.c | 41 +- drivers/misc/Kconfig | 10 - drivers/misc/Makefile | 16 +- drivers/misc/cxl/Kconfig | 17 +- drivers/misc/cxl/Makefile | 2 +- drivers/misc/cxl/api.c | 168 +- drivers/misc/cxl/base.c | 104 +- drivers/misc/cxl/context.c | 5 +- drivers/misc/cxl/cxl.h | 87 +- drivers/misc/cxl/debugfs.c | 35 +- drivers/misc/cxl/file.c | 64 +- drivers/misc/cxl/flash.c | 4 +- drivers/misc/cxl/guest.c | 26 +- drivers/misc/cxl/irq.c | 32 +- drivers/misc/cxl/main.c | 5 + drivers/misc/cxl/native.c | 209 +- drivers/misc/cxl/pci.c | 492 +- drivers/misc/cxl/phb.c | 44 + drivers/misc/cxl/vphb.c | 90 +- drivers/misc/eeprom/at24.c | 498 +- drivers/misc/genwqe/card_base.c | 18 +- drivers/misc/lkdtm.h | 60 + drivers/misc/lkdtm_bugs.c | 148 + drivers/misc/lkdtm_core.c | 544 + drivers/misc/lkdtm_heap.c | 142 + drivers/misc/lkdtm_perms.c | 199 + drivers/misc/lkdtm_rodata.c | 10 + drivers/misc/lkdtm_usercopy.c | 322 + drivers/misc/mei/hbm.c | 137 +- drivers/misc/mei/mei_dev.h | 10 +- drivers/misc/mic/Kconfig | 4 + drivers/misc/mic/host/mic_boot.c | 20 +- drivers/misc/ti-st/st_core.c | 2 +- drivers/mmc/card/block.c | 42 +- drivers/mmc/card/queue.c | 11 +- drivers/mmc/card/queue.h | 8 +- drivers/mmc/core/bus.c | 3 +- drivers/mmc/core/core.c | 95 +- drivers/mmc/core/debugfs.c | 3 +- drivers/mmc/core/host.c | 8 + drivers/mmc/core/mmc.c | 253 +- drivers/mmc/core/mmc_ops.c | 27 +- drivers/mmc/core/quirks.c | 2 + drivers/mmc/core/sd.c | 19 + drivers/mmc/host/Kconfig | 22 +- drivers/mmc/host/Makefile | 2 +- drivers/mmc/host/dw_mmc-exynos.c | 2 +- drivers/mmc/host/dw_mmc-k3.c | 7 + drivers/mmc/host/dw_mmc-rockchip.c | 11 +- drivers/mmc/host/dw_mmc.c | 118 +- drivers/mmc/host/dw_mmc.h | 11 +- drivers/mmc/host/jz4740_mmc.c | 2 - drivers/mmc/host/mtk-sd.c | 70 +- drivers/mmc/host/mxcmmc.c | 2 +- drivers/mmc/host/omap.c | 18 +- drivers/mmc/host/omap_hsmmc.c | 16 +- drivers/mmc/host/rtsx_pci_sdmmc.c | 16 +- drivers/mmc/host/s3cmci.c | 2 +- drivers/mmc/host/s3cmci.h | 2 +- drivers/mmc/host/sdhci-acpi.c | 11 +- drivers/mmc/host/sdhci-bcm-kona.c | 8 +- drivers/mmc/host/sdhci-brcmstb.c | 143 + drivers/mmc/host/sdhci-cns3xxx.c | 2 +- drivers/mmc/host/sdhci-dove.c | 2 +- drivers/mmc/host/sdhci-esdhc-imx.c | 153 +- drivers/mmc/host/sdhci-iproc.c | 15 +- drivers/mmc/host/sdhci-msm.c | 117 +- drivers/mmc/host/sdhci-of-arasan.c | 332 +- drivers/mmc/host/sdhci-of-at91.c | 2 +- drivers/mmc/host/sdhci-of-esdhc.c | 16 +- drivers/mmc/host/sdhci-of-hlwd.c | 2 +- drivers/mmc/host/sdhci-pci-core.c | 83 +- drivers/mmc/host/sdhci-pci.h | 2 +- drivers/mmc/host/sdhci-pltfm.c | 13 +- drivers/mmc/host/sdhci-pltfm.h | 7 - drivers/mmc/host/sdhci-pxav2.c | 2 +- drivers/mmc/host/sdhci-pxav3.c | 9 +- drivers/mmc/host/sdhci-s3c.c | 9 +- drivers/mmc/host/sdhci-sirf.c | 4 +- drivers/mmc/host/sdhci-st.c | 2 +- drivers/mmc/host/sdhci-tegra.c | 51 +- drivers/mmc/host/sdhci.c | 745 +- drivers/mmc/host/sdhci.h | 30 +- drivers/mmc/host/sdhci_f_sdh30.c | 2 +- drivers/mmc/host/sh_mmcif.c | 53 +- drivers/mmc/host/sh_mobile_sdhi.c | 14 +- drivers/mmc/host/tmio_mmc.h | 2 +- drivers/mmc/host/tmio_mmc_pio.c | 2 +- drivers/mtd/bcm47xxpart.c | 2 +- drivers/mtd/chips/cfi_cmdset_0020.c | 2 +- drivers/mtd/devices/Kconfig | 18 +- drivers/mtd/devices/m25p80.c | 37 +- drivers/mtd/devices/powernv_flash.c | 2 +- drivers/mtd/maps/physmap_of.c | 2 +- drivers/mtd/mtd_blkdevs.c | 8 +- drivers/mtd/nand/Kconfig | 10 +- drivers/mtd/nand/Makefile | 1 + drivers/mtd/nand/brcmnand/brcmnand.c | 173 +- drivers/mtd/nand/davinci_nand.c | 3 + drivers/mtd/nand/jz4780_bch.c | 2 +- drivers/mtd/nand/jz4780_nand.c | 2 +- drivers/mtd/nand/mtk_ecc.c | 534 + drivers/mtd/nand/mtk_ecc.h | 50 + drivers/mtd/nand/mtk_nand.c | 1529 +++ drivers/mtd/nand/nand_ids.c | 1 + drivers/mtd/nand/omap2.c | 13 +- drivers/mtd/nand/sunxi_nand.c | 397 +- drivers/mtd/nand/xway_nand.c | 231 +- drivers/mtd/onenand/onenand_base.c | 4 +- drivers/mtd/spi-nor/Kconfig | 27 + drivers/mtd/spi-nor/Makefile | 3 + drivers/mtd/spi-nor/atmel-quadspi.c | 732 ++ drivers/mtd/spi-nor/cadence-quadspi.c | 1299 +++ drivers/mtd/spi-nor/fsl-quadspi.c | 29 +- drivers/mtd/spi-nor/hisi-sfc.c | 489 + drivers/mtd/spi-nor/mtk-quadspi.c | 43 +- drivers/mtd/spi-nor/nxp-spifi.c | 25 +- drivers/mtd/spi-nor/spi-nor.c | 125 +- drivers/mtd/ssfdc.c | 3 +- drivers/mtd/tests/nandbiterrs.c | 2 +- drivers/mtd/ubi/attach.c | 141 +- drivers/mtd/ubi/fastmap.c | 65 +- drivers/mtd/ubi/gluebi.c | 5 - drivers/mtd/ubi/io.c | 2 +- drivers/mtd/ubi/ubi.h | 46 +- drivers/mtd/ubi/wl.c | 41 +- drivers/net/bonding/bond_main.c | 26 +- drivers/net/caif/Kconfig | 2 +- drivers/net/caif/caif_hsi.c | 5 +- drivers/net/caif/caif_spi.c | 4 +- drivers/net/can/Kconfig | 11 +- drivers/net/can/Makefile | 2 +- drivers/net/can/dev.c | 167 +- drivers/net/can/rcar/Kconfig | 21 + drivers/net/can/rcar/Makefile | 6 + drivers/net/can/rcar/rcar_can.c | 929 ++ drivers/net/can/rcar/rcar_canfd.c | 1858 ++++ drivers/net/can/sja1000/tscan1.c | 12 +- drivers/net/can/slcan.c | 4 +- drivers/net/can/spi/mcp251x.c | 7 +- drivers/net/can/usb/gs_usb.c | 141 +- drivers/net/dsa/Kconfig | 12 +- drivers/net/dsa/Makefile | 4 +- drivers/net/dsa/b53/Kconfig | 33 + drivers/net/dsa/b53/Makefile | 6 + drivers/net/dsa/b53/b53_common.c | 1799 +++ drivers/net/dsa/b53/b53_mdio.c | 392 + drivers/net/dsa/b53/b53_mmap.c | 273 + drivers/net/dsa/b53/b53_priv.h | 388 + drivers/net/dsa/b53/b53_regs.h | 434 + drivers/net/dsa/b53/b53_spi.c | 331 + drivers/net/dsa/b53/b53_srab.c | 442 + drivers/net/dsa/bcm_sf2.c | 702 +- drivers/net/dsa/bcm_sf2.h | 16 + drivers/net/dsa/bcm_sf2_regs.h | 70 + drivers/net/dsa/mv88e6xxx/Kconfig | 7 + drivers/net/dsa/mv88e6xxx/Makefile | 1 + drivers/net/dsa/mv88e6xxx/chip.c | 4093 +++++++ drivers/net/dsa/mv88e6xxx/mv88e6xxx.h | 678 ++ drivers/net/ethernet/8390/ax88796.c | 43 +- drivers/net/ethernet/adi/bfin_mac.c | 48 +- drivers/net/ethernet/adi/bfin_mac.h | 1 - drivers/net/ethernet/agere/et131x.c | 60 +- drivers/net/ethernet/allwinner/sun4i-emac.c | 54 +- drivers/net/ethernet/altera/altera_tse.h | 1 - drivers/net/ethernet/altera/altera_tse_ethtool.c | 26 +- drivers/net/ethernet/altera/altera_tse_main.c | 17 +- drivers/net/ethernet/amd/au1000_eth.c | 55 +- drivers/net/ethernet/amd/au1000_eth.h | 1 - drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 2 +- drivers/net/ethernet/apm/xgene/Kconfig | 1 + .../net/ethernet/apm/xgene/xgene_enet_ethtool.c | 22 +- drivers/net/ethernet/apm/xgene/xgene_enet_hw.c | 257 +- drivers/net/ethernet/apm/xgene/xgene_enet_hw.h | 11 +- drivers/net/ethernet/apm/xgene/xgene_enet_main.c | 215 +- drivers/net/ethernet/apm/xgene/xgene_enet_main.h | 33 +- drivers/net/ethernet/apm/xgene/xgene_enet_sgmac.c | 239 +- drivers/net/ethernet/apm/xgene/xgene_enet_sgmac.h | 8 + drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.c | 66 +- drivers/net/ethernet/apm/xgene/xgene_enet_xgmac.h | 3 + drivers/net/ethernet/arc/emac.h | 1 - drivers/net/ethernet/arc/emac_main.c | 86 +- drivers/net/ethernet/atheros/alx/main.c | 14 +- drivers/net/ethernet/atheros/alx/reg.h | 1 + drivers/net/ethernet/aurora/nb8800.c | 73 +- drivers/net/ethernet/aurora/nb8800.h | 1 - drivers/net/ethernet/broadcom/Kconfig | 44 +- drivers/net/ethernet/broadcom/Makefile | 2 + drivers/net/ethernet/broadcom/bcm63xx_enet.c | 2 +- drivers/net/ethernet/broadcom/bcmsysport.c | 49 +- drivers/net/ethernet/broadcom/bcmsysport.h | 1 - drivers/net/ethernet/broadcom/bgmac-bcma-mdio.c | 266 + drivers/net/ethernet/broadcom/bgmac-bcma.c | 315 + drivers/net/ethernet/broadcom/bgmac-platform.c | 185 + drivers/net/ethernet/broadcom/bgmac.c | 808 +- drivers/net/ethernet/broadcom/bgmac.h | 117 +- drivers/net/ethernet/broadcom/bnx2.c | 12 +- drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c | 113 +- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 743 +- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 90 +- drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 265 +- drivers/net/ethernet/broadcom/bnxt/bnxt_fw_hdr.h | 1 + drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h | 87 +- drivers/net/ethernet/broadcom/bnxt/bnxt_nvm_defs.h | 1 + drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c | 3 + drivers/net/ethernet/broadcom/tg3.c | 26 +- drivers/net/ethernet/brocade/bna/bnad_debugfs.c | 6 +- drivers/net/ethernet/brocade/bna/bnad_ethtool.c | 57 +- drivers/net/ethernet/cadence/macb.c | 77 +- drivers/net/ethernet/cadence/macb.h | 1 - .../net/ethernet/cavium/liquidio/cn66xx_device.c | 61 +- .../net/ethernet/cavium/liquidio/cn66xx_device.h | 5 +- .../net/ethernet/cavium/liquidio/cn68xx_device.c | 13 +- .../net/ethernet/cavium/liquidio/cn68xx_device.h | 1 - drivers/net/ethernet/cavium/liquidio/cn68xx_regs.h | 1 - drivers/net/ethernet/cavium/liquidio/lio_ethtool.c | 1009 +- drivers/net/ethernet/cavium/liquidio/lio_main.c | 1421 ++- .../net/ethernet/cavium/liquidio/liquidio_common.h | 408 +- .../net/ethernet/cavium/liquidio/octeon_config.h | 16 +- .../net/ethernet/cavium/liquidio/octeon_console.c | 50 +- .../net/ethernet/cavium/liquidio/octeon_device.c | 262 +- .../net/ethernet/cavium/liquidio/octeon_device.h | 52 +- drivers/net/ethernet/cavium/liquidio/octeon_droq.c | 213 +- drivers/net/ethernet/cavium/liquidio/octeon_droq.h | 41 +- drivers/net/ethernet/cavium/liquidio/octeon_iq.h | 85 +- drivers/net/ethernet/cavium/liquidio/octeon_main.h | 25 +- .../net/ethernet/cavium/liquidio/octeon_mem_ops.c | 24 +- .../net/ethernet/cavium/liquidio/octeon_network.h | 252 +- drivers/net/ethernet/cavium/liquidio/octeon_nic.c | 67 +- drivers/net/ethernet/cavium/liquidio/octeon_nic.h | 154 +- .../net/ethernet/cavium/liquidio/request_manager.c | 313 +- .../ethernet/cavium/liquidio/response_manager.c | 30 +- drivers/net/ethernet/cavium/octeon/octeon_mgmt.c | 106 +- drivers/net/ethernet/cavium/thunder/nic.h | 1 + drivers/net/ethernet/cavium/thunder/nic_main.c | 11 +- drivers/net/ethernet/cavium/thunder/nicvf_main.c | 20 +- drivers/net/ethernet/cavium/thunder/nicvf_queues.c | 64 +- drivers/net/ethernet/chelsio/Kconfig | 16 +- drivers/net/ethernet/chelsio/Makefile | 1 + drivers/net/ethernet/chelsio/cxgb4/Makefile | 1 - drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 5 +- drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c | 375 +- drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 97 +- drivers/net/ethernet/chelsio/cxgb4/sge.c | 2 +- drivers/net/ethernet/chelsio/cxgb4/t4_hw.c | 9 +- drivers/net/ethernet/chelsio/cxgb4/t4_msg.h | 2 + drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 30 +- drivers/net/ethernet/chelsio/cxgb4vf/adapter.h | 8 + .../net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c | 286 +- drivers/net/ethernet/chelsio/cxgb4vf/sge.c | 2 +- drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h | 16 +- drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c | 67 +- drivers/net/ethernet/chelsio/libcxgb/Makefile | 3 + drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.c | 498 + drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.h | 334 + drivers/net/ethernet/cirrus/cs89x0.c | 12 +- drivers/net/ethernet/cisco/enic/enic_ethtool.c | 28 +- drivers/net/ethernet/cisco/enic/enic_main.c | 4 +- drivers/net/ethernet/davicom/dm9000.c | 8 +- drivers/net/ethernet/dec/tulip/de4x5.c | 2 +- drivers/net/ethernet/dnet.c | 48 +- drivers/net/ethernet/dnet.h | 1 - drivers/net/ethernet/emulex/benet/Kconfig | 8 - drivers/net/ethernet/emulex/benet/be.h | 58 +- drivers/net/ethernet/emulex/benet/be_cmds.c | 160 +- drivers/net/ethernet/emulex/benet/be_cmds.h | 16 +- drivers/net/ethernet/emulex/benet/be_ethtool.c | 66 +- drivers/net/ethernet/emulex/benet/be_main.c | 334 +- drivers/net/ethernet/emulex/benet/be_roce.c | 2 +- drivers/net/ethernet/emulex/benet/be_roce.h | 2 +- drivers/net/ethernet/ethoc.c | 40 +- drivers/net/ethernet/ezchip/nps_enet.c | 27 +- drivers/net/ethernet/faraday/ftgmac100.c | 271 +- drivers/net/ethernet/freescale/fec.h | 4 + drivers/net/ethernet/freescale/fec_main.c | 61 +- drivers/net/ethernet/freescale/gianfar.c | 22 +- drivers/net/ethernet/freescale/gianfar.h | 3 +- drivers/net/ethernet/hisilicon/Kconfig | 14 +- drivers/net/ethernet/hisilicon/Makefile | 1 + drivers/net/ethernet/hisilicon/hisi_femac.c | 1007 ++ drivers/net/ethernet/hisilicon/hix5hd2_gmac.c | 44 +- drivers/net/ethernet/hisilicon/hns/hnae.c | 19 +- drivers/net/ethernet/hisilicon/hns/hnae.h | 20 +- drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c | 60 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c | 8 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c | 291 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.h | 5 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c | 334 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.h | 45 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c | 302 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.h | 7 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c | 16 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c | 7 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.h | 2 +- drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h | 21 +- .../net/ethernet/hisilicon/hns/hns_dsaf_xgmac.c | 10 +- drivers/net/ethernet/hisilicon/hns/hns_enet.c | 160 +- drivers/net/ethernet/hisilicon/hns/hns_enet.h | 2 +- drivers/net/ethernet/hisilicon/hns/hns_ethtool.c | 63 +- drivers/net/ethernet/hisilicon/hns_mdio.c | 204 +- drivers/net/ethernet/ibm/emac/core.c | 34 +- drivers/net/ethernet/intel/Kconfig | 43 - drivers/net/ethernet/intel/e1000e/82571.c | 6 +- drivers/net/ethernet/intel/e1000e/e1000.h | 1 + drivers/net/ethernet/intel/e1000e/ich8lan.c | 3 +- drivers/net/ethernet/intel/e1000e/netdev.c | 71 +- drivers/net/ethernet/intel/fm10k/fm10k.h | 4 +- drivers/net/ethernet/intel/fm10k/fm10k_common.c | 6 +- drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c | 7 +- drivers/net/ethernet/intel/fm10k/fm10k_main.c | 19 +- drivers/net/ethernet/intel/fm10k/fm10k_mbx.h | 2 + drivers/net/ethernet/intel/fm10k/fm10k_netdev.c | 40 +- drivers/net/ethernet/intel/fm10k/fm10k_pci.c | 333 +- drivers/net/ethernet/intel/fm10k/fm10k_pf.c | 38 +- drivers/net/ethernet/intel/fm10k/fm10k_type.h | 2 + drivers/net/ethernet/intel/fm10k/fm10k_vf.c | 12 +- drivers/net/ethernet/intel/i40e/i40e.h | 15 +- drivers/net/ethernet/intel/i40e/i40e_client.c | 45 +- drivers/net/ethernet/intel/i40e/i40e_common.c | 61 +- drivers/net/ethernet/intel/i40e/i40e_debugfs.c | 16 +- drivers/net/ethernet/intel/i40e/i40e_devids.h | 1 - drivers/net/ethernet/intel/i40e/i40e_ethtool.c | 52 +- drivers/net/ethernet/intel/i40e/i40e_main.c | 998 +- drivers/net/ethernet/intel/i40e/i40e_prototype.h | 2 + drivers/net/ethernet/intel/i40e/i40e_txrx.c | 4 +- drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c | 6 + drivers/net/ethernet/intel/i40evf/i40e_common.c | 1 - drivers/net/ethernet/intel/i40evf/i40e_devids.h | 1 - drivers/net/ethernet/intel/i40evf/i40e_txrx.c | 3 +- drivers/net/ethernet/intel/i40evf/i40evf_main.c | 8 +- .../net/ethernet/intel/i40evf/i40evf_virtchnl.c | 8 + drivers/net/ethernet/intel/igb/igb.h | 7 +- drivers/net/ethernet/intel/igb/igb_main.c | 22 +- drivers/net/ethernet/intel/igb/igb_ptp.c | 92 +- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 2 - drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c | 3 - drivers/net/ethernet/intel/ixgbe/ixgbe_common.c | 89 +- drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 12 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 192 +- drivers/net/ethernet/intel/ixgbe/ixgbe_model.h | 4 + drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c | 1 + drivers/net/ethernet/intel/ixgbe/ixgbe_type.h | 1 + drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c | 4 +- drivers/net/ethernet/intel/ixgbevf/defines.h | 1 + drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 2 + drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c | 24 +- drivers/net/ethernet/intel/ixgbevf/vf.c | 96 +- drivers/net/ethernet/intel/ixgbevf/vf.h | 3 +- drivers/net/ethernet/lantiq_etop.c | 37 +- drivers/net/ethernet/marvell/mvneta.c | 1 + drivers/net/ethernet/marvell/mvpp2.c | 50 +- drivers/net/ethernet/marvell/pxa168_eth.c | 72 +- drivers/net/ethernet/mediatek/mtk_eth_soc.c | 308 +- drivers/net/ethernet/mediatek/mtk_eth_soc.h | 38 +- drivers/net/ethernet/mellanox/mlx4/Kconfig | 7 - drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c | 280 +- drivers/net/ethernet/mellanox/mlx4/en_ethtool.c | 34 +- drivers/net/ethernet/mellanox/mlx4/en_netdev.c | 190 +- drivers/net/ethernet/mellanox/mlx4/en_rx.c | 126 +- drivers/net/ethernet/mellanox/mlx4/en_tx.c | 282 +- drivers/net/ethernet/mellanox/mlx4/eq.c | 4 +- drivers/net/ethernet/mellanox/mlx4/fw.c | 41 + drivers/net/ethernet/mellanox/mlx4/fw.h | 1 + drivers/net/ethernet/mellanox/mlx4/intf.c | 5 +- drivers/net/ethernet/mellanox/mlx4/main.c | 6 +- drivers/net/ethernet/mellanox/mlx4/mcg.c | 8 +- drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 63 +- drivers/net/ethernet/mellanox/mlx4/mr.c | 2 +- drivers/net/ethernet/mellanox/mlx4/pd.c | 4 +- drivers/net/ethernet/mellanox/mlx4/port.c | 12 + .../net/ethernet/mellanox/mlx4/resource_tracker.c | 22 +- drivers/net/ethernet/mellanox/mlx5/core/Kconfig | 1 + drivers/net/ethernet/mellanox/mlx5/core/Makefile | 12 +- drivers/net/ethernet/mellanox/mlx5/core/en.h | 201 +- drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c | 82 +- .../net/ethernet/mellanox/mlx5/core/en_common.c | 161 + drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c | 30 +- .../net/ethernet/mellanox/mlx5/core/en_ethtool.c | 640 +- drivers/net/ethernet/mellanox/mlx5/core/en_fs.c | 101 +- .../ethernet/mellanox/mlx5/core/en_fs_ethtool.c | 586 + drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 951 +- drivers/net/ethernet/mellanox/mlx5/core/en_rep.c | 431 + drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 41 +- drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c | 335 + drivers/net/ethernet/mellanox/mlx5/core/en_stats.h | 31 +- drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 153 +- drivers/net/ethernet/mellanox/mlx5/core/en_tx.c | 117 +- drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c | 11 +- drivers/net/ethernet/mellanox/mlx5/core/eswitch.c | 207 +- drivers/net/ethernet/mellanox/mlx5/core/eswitch.h | 86 +- .../ethernet/mellanox/mlx5/core/eswitch_offloads.c | 646 ++ drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c | 67 + drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h | 12 + drivers/net/ethernet/mellanox/mlx5/core/fs_core.c | 307 +- drivers/net/ethernet/mellanox/mlx5/core/fs_core.h | 9 + .../net/ethernet/mellanox/mlx5/core/fs_counters.c | 150 +- drivers/net/ethernet/mellanox/mlx5/core/fw.c | 6 + drivers/net/ethernet/mellanox/mlx5/core/health.c | 7 +- drivers/net/ethernet/mellanox/mlx5/core/main.c | 36 +- drivers/net/ethernet/mellanox/mlx5/core/port.c | 48 +- drivers/net/ethernet/mellanox/mlx5/core/rl.c | 209 + drivers/net/ethernet/mellanox/mlx5/core/sriov.c | 5 +- drivers/net/ethernet/mellanox/mlx5/core/srq.c | 265 +- drivers/net/ethernet/mellanox/mlx5/core/transobj.c | 4 + drivers/net/ethernet/mellanox/mlx5/core/vport.c | 12 + drivers/net/ethernet/mellanox/mlxsw/Makefile | 3 +- drivers/net/ethernet/mellanox/mlxsw/cmd.h | 75 + drivers/net/ethernet/mellanox/mlxsw/core.c | 30 +- drivers/net/ethernet/mellanox/mlxsw/core.h | 17 +- drivers/net/ethernet/mellanox/mlxsw/pci.c | 78 +- drivers/net/ethernet/mellanox/mlxsw/port.h | 1 + drivers/net/ethernet/mellanox/mlxsw/reg.h | 1397 ++- drivers/net/ethernet/mellanox/mlxsw/spectrum.c | 2468 +++-- drivers/net/ethernet/mellanox/mlxsw/spectrum.h | 268 +- .../net/ethernet/mellanox/mlxsw/spectrum_buffers.c | 14 +- drivers/net/ethernet/mellanox/mlxsw/spectrum_dcb.c | 10 +- .../net/ethernet/mellanox/mlxsw/spectrum_kvdl.c | 91 + .../net/ethernet/mellanox/mlxsw/spectrum_router.c | 1863 ++++ .../ethernet/mellanox/mlxsw/spectrum_switchdev.c | 512 +- drivers/net/ethernet/mellanox/mlxsw/switchx2.c | 1 + drivers/net/ethernet/mellanox/mlxsw/trap.h | 9 + drivers/net/ethernet/neterion/s2io.c | 2 +- drivers/net/ethernet/netronome/nfp/nfp_net.h | 2 +- .../net/ethernet/netronome/nfp/nfp_net_common.c | 50 +- .../net/ethernet/netronome/nfp/nfp_net_ethtool.c | 2 +- .../net/ethernet/netronome/nfp/nfp_netvf_main.c | 11 +- drivers/net/ethernet/nxp/lpc_eth.c | 65 +- drivers/net/ethernet/pasemi/pasemi_mac.c | 28 +- drivers/net/ethernet/pasemi/pasemi_mac.h | 1 - drivers/net/ethernet/pasemi/pasemi_mac_ethtool.c | 30 +- drivers/net/ethernet/qlogic/Kconfig | 30 - drivers/net/ethernet/qlogic/qed/qed.h | 28 +- drivers/net/ethernet/qlogic/qed/qed_cxt.c | 1347 ++- drivers/net/ethernet/qlogic/qed/qed_cxt.h | 24 + drivers/net/ethernet/qlogic/qed/qed_dcbx.c | 1814 ++- drivers/net/ethernet/qlogic/qed/qed_dcbx.h | 28 + drivers/net/ethernet/qlogic/qed/qed_dev.c | 649 +- drivers/net/ethernet/qlogic/qed/qed_dev_api.h | 55 +- drivers/net/ethernet/qlogic/qed/qed_hsi.h | 10956 +++++++++++-------- drivers/net/ethernet/qlogic/qed/qed_hw.c | 55 +- drivers/net/ethernet/qlogic/qed/qed_hw.h | 12 +- .../net/ethernet/qlogic/qed/qed_init_fw_funcs.c | 184 +- drivers/net/ethernet/qlogic/qed/qed_init_ops.c | 9 +- drivers/net/ethernet/qlogic/qed/qed_int.c | 75 +- drivers/net/ethernet/qlogic/qed/qed_int.h | 3 + drivers/net/ethernet/qlogic/qed/qed_l2.c | 128 +- drivers/net/ethernet/qlogic/qed/qed_main.c | 68 +- drivers/net/ethernet/qlogic/qed/qed_mcp.c | 61 +- drivers/net/ethernet/qlogic/qed/qed_mcp.h | 3 + drivers/net/ethernet/qlogic/qed/qed_reg_addr.h | 43 +- drivers/net/ethernet/qlogic/qed/qed_sp.h | 26 + drivers/net/ethernet/qlogic/qed/qed_sp_commands.c | 26 +- drivers/net/ethernet/qlogic/qed/qed_spq.c | 40 +- drivers/net/ethernet/qlogic/qed/qed_sriov.c | 508 +- drivers/net/ethernet/qlogic/qed/qed_sriov.h | 9 +- drivers/net/ethernet/qlogic/qed/qed_vf.c | 99 +- drivers/net/ethernet/qlogic/qed/qed_vf.h | 13 +- drivers/net/ethernet/qlogic/qede/Makefile | 1 + drivers/net/ethernet/qlogic/qede/qede.h | 9 +- drivers/net/ethernet/qlogic/qede/qede_dcbnl.c | 348 + drivers/net/ethernet/qlogic/qede/qede_ethtool.c | 104 + drivers/net/ethernet/qlogic/qede/qede_main.c | 295 +- drivers/net/ethernet/qlogic/qlcnic/qlcnic.h | 6 +- .../net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c | 1 - .../net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c | 4 - drivers/net/ethernet/qlogic/qlcnic/qlcnic_dcb.h | 1 - drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c | 2 - drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c | 30 +- drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov.h | 9 +- .../ethernet/qlogic/qlcnic/qlcnic_sriov_common.c | 95 +- drivers/net/ethernet/qlogic/qlge/qlge_main.c | 1 - drivers/net/ethernet/rdc/r6040.c | 91 +- drivers/net/ethernet/realtek/8139cp.c | 2 +- drivers/net/ethernet/realtek/8139too.c | 12 +- drivers/net/ethernet/realtek/r8169.c | 37 +- drivers/net/ethernet/renesas/ravb_main.c | 10 +- drivers/net/ethernet/renesas/sh_eth.c | 9 +- drivers/net/ethernet/rocker/rocker_main.c | 3 +- drivers/net/ethernet/samsung/sxgbe/sxgbe_common.h | 1 - drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c | 31 +- drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c | 32 +- drivers/net/ethernet/sfc/ef10.c | 747 +- drivers/net/ethernet/sfc/ef10_sriov.c | 44 +- drivers/net/ethernet/sfc/ef10_sriov.h | 3 + drivers/net/ethernet/sfc/efx.c | 66 +- drivers/net/ethernet/sfc/efx.h | 9 + drivers/net/ethernet/sfc/mcdi_pcol.h | 1327 ++- drivers/net/ethernet/sfc/net_driver.h | 19 +- drivers/net/ethernet/sfc/nic.h | 5 + drivers/net/ethernet/smsc/smc91x.c | 13 +- drivers/net/ethernet/smsc/smc91x.h | 2 + drivers/net/ethernet/smsc/smsc911x.c | 287 +- drivers/net/ethernet/smsc/smsc9420.c | 60 +- drivers/net/ethernet/stmicro/stmmac/Kconfig | 14 +- drivers/net/ethernet/stmicro/stmmac/Makefile | 3 +- drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.c | 274 + drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.h | 36 + drivers/net/ethernet/stmicro/stmmac/common.h | 19 +- drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c | 165 +- .../net/ethernet/stmicro/stmmac/dwmac-socfpga.c | 147 +- drivers/net/ethernet/stmicro/stmmac/dwmac1000.h | 86 +- .../net/ethernet/stmicro/stmmac/dwmac1000_core.c | 149 +- drivers/net/ethernet/stmicro/stmmac/dwmac4.h | 43 +- drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c | 98 +- drivers/net/ethernet/stmicro/stmmac/stmmac.h | 1 - .../net/ethernet/stmicro/stmmac/stmmac_ethtool.c | 60 +- drivers/net/ethernet/stmicro/stmmac/stmmac_main.c | 51 +- drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.h | 159 + .../net/ethernet/stmicro/stmmac/stmmac_platform.c | 17 +- drivers/net/ethernet/synopsys/dwc_eth_qos.c | 128 +- drivers/net/ethernet/tehuti/tehuti.c | 2 +- drivers/net/ethernet/ti/Kconfig | 3 +- drivers/net/ethernet/ti/cpmac.c | 70 +- drivers/net/ethernet/ti/cpsw.c | 118 +- drivers/net/ethernet/ti/cpsw.h | 1 - drivers/net/ethernet/ti/davinci_cpdma.c | 261 +- drivers/net/ethernet/ti/davinci_cpdma.h | 3 +- drivers/net/ethernet/ti/davinci_emac.c | 189 +- drivers/net/ethernet/ti/davinci_mdio.c | 169 +- drivers/net/ethernet/ti/tlan.c | 1 - drivers/net/ethernet/tile/tilepro.c | 4 +- drivers/net/ethernet/toshiba/tc35815.c | 65 +- drivers/net/ethernet/tundra/tsi108_eth.c | 2 + drivers/net/ethernet/wiznet/w5100.c | 3 +- drivers/net/ethernet/xilinx/ll_temac.h | 1 - drivers/net/ethernet/xilinx/ll_temac_main.c | 47 +- drivers/net/ethernet/xilinx/xilinx_axienet.h | 2 - drivers/net/ethernet/xilinx/xilinx_axienet_main.c | 82 +- drivers/net/ethernet/xilinx/xilinx_emaclite.c | 8 +- drivers/net/ethernet/xircom/xirc2ps_cs.c | 4 +- drivers/net/ethernet/xscale/ixp4xx_eth.c | 46 +- drivers/net/fjes/fjes_main.c | 5 +- drivers/net/geneve.c | 61 +- drivers/net/gtp.c | 1 - drivers/net/hamradio/baycom_par.c | 6 +- drivers/net/hyperv/hyperv_net.h | 43 +- drivers/net/hyperv/netvsc.c | 130 +- drivers/net/hyperv/netvsc_drv.c | 123 +- drivers/net/hyperv/rndis_filter.c | 159 +- drivers/net/ieee802154/atusb.c | 6 +- drivers/net/ieee802154/fakelb.c | 8 + drivers/net/ieee802154/mrf24j40.c | 2 + drivers/net/ipvlan/ipvlan_core.c | 39 +- drivers/net/ipvlan/ipvlan_main.c | 22 +- drivers/net/loopback.c | 5 +- drivers/net/macsec.c | 176 +- drivers/net/macvlan.c | 61 +- drivers/net/macvtap.c | 178 +- drivers/net/phy/Kconfig | 37 + drivers/net/phy/Makefile | 7 +- drivers/net/phy/fixed_phy.c | 153 +- drivers/net/phy/intel-xway.c | 376 + drivers/net/phy/marvell.c | 346 +- drivers/net/phy/mdio-hisi-femac.c | 166 + drivers/net/phy/mdio-mux-bcm-iproc.c | 248 + drivers/net/phy/mdio-mux-gpio.c | 2 +- drivers/net/phy/mdio-mux-mmioreg.c | 2 +- drivers/net/phy/mdio-mux.c | 26 +- drivers/net/phy/mdio-xgene.c | 473 + drivers/net/phy/mdio-xgene.h | 143 + drivers/net/phy/micrel.c | 67 +- drivers/net/phy/swphy.c | 179 + drivers/net/phy/swphy.h | 9 + drivers/net/ppp/ppp_generic.c | 3 +- drivers/net/team/team.c | 21 +- drivers/net/team/team_mode_loadbalance.c | 15 + drivers/net/tun.c | 240 +- drivers/net/usb/ax88172a.c | 22 +- drivers/net/usb/cdc_ether.c | 51 + drivers/net/usb/kaweth.c | 10 +- drivers/net/usb/r8152.c | 367 +- drivers/net/usb/rndis_host.c | 6 +- drivers/net/usb/usbnet.c | 138 - drivers/net/virtio_net.c | 103 +- drivers/net/vmxnet3/Makefile | 4 +- drivers/net/vmxnet3/upt1_defs.h | 4 +- drivers/net/vmxnet3/vmxnet3_defs.h | 105 +- drivers/net/vmxnet3/vmxnet3_drv.c | 289 +- drivers/net/vmxnet3/vmxnet3_ethtool.c | 215 +- drivers/net/vmxnet3/vmxnet3_int.h | 54 +- drivers/net/vrf.c | 582 +- drivers/net/vxlan.c | 157 +- drivers/net/wan/Kconfig | 22 + drivers/net/wan/Makefile | 2 + drivers/net/wan/fsl_ucc_hdlc.c | 1178 ++ drivers/net/wan/fsl_ucc_hdlc.h | 147 + drivers/net/wan/slic_ds26522.c | 255 + drivers/net/wan/slic_ds26522.h | 134 + drivers/net/wireless/ath/ath10k/ahb.c | 11 +- drivers/net/wireless/ath/ath10k/core.c | 133 +- drivers/net/wireless/ath/ath10k/core.h | 34 +- drivers/net/wireless/ath/ath10k/debug.c | 63 +- drivers/net/wireless/ath/ath10k/debug.h | 11 +- drivers/net/wireless/ath/ath10k/debugfs_sta.c | 74 +- drivers/net/wireless/ath/ath10k/hif.h | 14 + drivers/net/wireless/ath/ath10k/htc.h | 1 - drivers/net/wireless/ath/ath10k/htt.h | 6 +- drivers/net/wireless/ath/ath10k/htt_rx.c | 54 +- drivers/net/wireless/ath/ath10k/htt_tx.c | 22 +- drivers/net/wireless/ath/ath10k/hw.c | 39 +- drivers/net/wireless/ath/ath10k/hw.h | 106 +- drivers/net/wireless/ath/ath10k/mac.c | 150 +- drivers/net/wireless/ath/ath10k/pci.c | 266 +- drivers/net/wireless/ath/ath10k/pci.h | 6 + drivers/net/wireless/ath/ath10k/rx_desc.h | 87 +- drivers/net/wireless/ath/ath10k/spectral.c | 4 +- drivers/net/wireless/ath/ath10k/targaddrs.h | 3 + drivers/net/wireless/ath/ath10k/txrx.c | 9 +- drivers/net/wireless/ath/ath10k/wmi.c | 73 +- drivers/net/wireless/ath/ath10k/wmi.h | 18 +- drivers/net/wireless/ath/ath5k/pcu.c | 2 +- drivers/net/wireless/ath/ath6kl/cfg80211.c | 25 +- drivers/net/wireless/ath/ath6kl/core.h | 2 +- drivers/net/wireless/ath/ath6kl/txrx.c | 9 +- drivers/net/wireless/ath/ath6kl/wmi.c | 3 +- drivers/net/wireless/ath/ath9k/ahb.c | 18 +- drivers/net/wireless/ath/ath9k/ar9002_phy.c | 32 +- drivers/net/wireless/ath/ath9k/ar9002_phy.h | 5 +- drivers/net/wireless/ath/ath9k/ar9003_calib.c | 128 +- drivers/net/wireless/ath/ath9k/ar9003_eeprom.c | 5 +- drivers/net/wireless/ath/ath9k/ar9003_phy.h | 25 +- drivers/net/wireless/ath/ath9k/ath9k.h | 7 +- drivers/net/wireless/ath/ath9k/beacon.c | 240 +- drivers/net/wireless/ath/ath9k/channel.c | 5 +- drivers/net/wireless/ath/ath9k/common-spectral.c | 6 +- drivers/net/wireless/ath/ath9k/common.h | 1 + drivers/net/wireless/ath/ath9k/dynack.c | 4 +- drivers/net/wireless/ath/ath9k/eeprom.c | 33 +- drivers/net/wireless/ath/ath9k/htc_drv_beacon.c | 2 +- drivers/net/wireless/ath/ath9k/htc_drv_init.c | 2 +- drivers/net/wireless/ath/ath9k/hw.c | 58 +- drivers/net/wireless/ath/ath9k/hw.h | 1 + drivers/net/wireless/ath/ath9k/init.c | 56 +- drivers/net/wireless/ath/ath9k/mac.h | 4 - drivers/net/wireless/ath/ath9k/main.c | 86 +- drivers/net/wireless/ath/ath9k/pci.c | 41 +- drivers/net/wireless/ath/ath9k/recv.c | 2 +- drivers/net/wireless/ath/ath9k/tx99.c | 3 +- drivers/net/wireless/ath/carl9170/Kconfig | 8 +- drivers/net/wireless/ath/dfs_pattern_detector.c | 2 +- drivers/net/wireless/ath/regd.c | 4 +- drivers/net/wireless/ath/wcn36xx/dxe.c | 31 +- drivers/net/wireless/ath/wcn36xx/dxe.h | 7 +- drivers/net/wireless/ath/wcn36xx/hal.h | 4 +- drivers/net/wireless/ath/wcn36xx/main.c | 67 +- drivers/net/wireless/ath/wcn36xx/smd.c | 44 +- drivers/net/wireless/ath/wcn36xx/smd.h | 4 +- drivers/net/wireless/ath/wcn36xx/wcn36xx.h | 10 +- drivers/net/wireless/ath/wil6210/cfg80211.c | 20 + drivers/net/wireless/ath/wil6210/debug.c | 46 +- drivers/net/wireless/ath/wil6210/main.c | 12 +- drivers/net/wireless/ath/wil6210/p2p.c | 12 +- drivers/net/wireless/ath/wil6210/pcie_bus.c | 68 +- drivers/net/wireless/ath/wil6210/pm.c | 25 +- drivers/net/wireless/ath/wil6210/txrx.c | 42 +- drivers/net/wireless/ath/wil6210/wil6210.h | 6 + drivers/net/wireless/ath/wil6210/wil_platform.h | 4 +- drivers/net/wireless/ath/wil6210/wmi.c | 8 +- drivers/net/wireless/atmel/at76c50x-usb.c | 5 +- drivers/net/wireless/broadcom/b43/Makefile | 2 +- drivers/net/wireless/broadcom/b43/leds.c | 8 +- drivers/net/wireless/broadcom/b43/main.c | 31 +- drivers/net/wireless/broadcom/b43/phy_a.h | 22 - drivers/net/wireless/broadcom/b43/phy_common.h | 3 - drivers/net/wireless/broadcom/b43/phy_g.c | 25 +- drivers/net/wireless/broadcom/b43/wa.c | 283 +- drivers/net/wireless/broadcom/b43/xmit.c | 30 +- .../wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c | 51 +- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 285 +- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 11 +- .../wireless/broadcom/brcm80211/brcmfmac/chip.c | 2 + .../wireless/broadcom/brcm80211/brcmfmac/core.c | 59 +- .../wireless/broadcom/brcm80211/brcmfmac/core.h | 5 +- .../wireless/broadcom/brcm80211/brcmfmac/fweh.c | 10 +- .../broadcom/brcm80211/brcmfmac/fwsignal.c | 2 +- .../net/wireless/broadcom/brcm80211/brcmfmac/p2p.c | 32 +- .../net/wireless/broadcom/brcm80211/brcmfmac/p2p.h | 2 +- .../wireless/broadcom/brcm80211/brcmfmac/pcie.c | 6 +- .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 12 +- .../wireless/broadcom/brcm80211/brcmfmac/sdio.h | 3 +- .../net/wireless/broadcom/brcm80211/brcmsmac/dma.c | 4 +- .../wireless/broadcom/brcm80211/brcmsmac/main.c | 4 +- .../broadcom/brcm80211/brcmsmac/phy/phy_n.c | 2 +- .../net/wireless/broadcom/brcm80211/brcmsmac/stf.c | 2 +- .../net/wireless/broadcom/brcm80211/brcmutil/d11.c | 18 +- .../broadcom/brcm80211/include/brcm_hw_ids.h | 2 + .../broadcom/brcm80211/include/brcmu_d11.h | 22 + drivers/net/wireless/cisco/airo.c | 4 +- drivers/net/wireless/intel/ipw2x00/ipw2200.c | 2 +- drivers/net/wireless/intel/iwlegacy/common.c | 6 +- drivers/net/wireless/intel/iwlwifi/Makefile | 2 +- drivers/net/wireless/intel/iwlwifi/dvm/lib.c | 17 - drivers/net/wireless/intel/iwlwifi/dvm/main.c | 3 + drivers/net/wireless/intel/iwlwifi/dvm/rxon.c | 5 - drivers/net/wireless/intel/iwlwifi/dvm/scan.c | 6 +- drivers/net/wireless/intel/iwlwifi/iwl-7000.c | 4 +- drivers/net/wireless/intel/iwlwifi/iwl-8000.c | 4 +- drivers/net/wireless/intel/iwlwifi/iwl-9000.c | 3 +- drivers/net/wireless/intel/iwlwifi/iwl-a000.c | 131 + drivers/net/wireless/intel/iwlwifi/iwl-config.h | 13 +- drivers/net/wireless/intel/iwlwifi/iwl-csr.h | 6 +- drivers/net/wireless/intel/iwlwifi/iwl-debug.h | 1 + .../net/wireless/intel/iwlwifi/iwl-devtrace-io.h | 35 + drivers/net/wireless/intel/iwlwifi/iwl-devtrace.h | 25 +- drivers/net/wireless/intel/iwlwifi/iwl-drv.c | 49 +- .../net/wireless/intel/iwlwifi/iwl-eeprom-parse.c | 4 +- .../net/wireless/intel/iwlwifi/iwl-eeprom-parse.h | 1 + drivers/net/wireless/intel/iwlwifi/iwl-fh.h | 99 +- .../net/wireless/intel/iwlwifi/iwl-fw-error-dump.h | 4 + drivers/net/wireless/intel/iwlwifi/iwl-fw-file.h | 13 +- drivers/net/wireless/intel/iwlwifi/iwl-fw.h | 17 +- drivers/net/wireless/intel/iwlwifi/iwl-io.c | 142 +- drivers/net/wireless/intel/iwlwifi/iwl-io.h | 3 + drivers/net/wireless/intel/iwlwifi/iwl-modparams.h | 10 +- drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c | 7 + drivers/net/wireless/intel/iwlwifi/iwl-prph.h | 1 + drivers/net/wireless/intel/iwlwifi/iwl-trans.h | 33 +- drivers/net/wireless/intel/iwlwifi/mvm/coex.c | 6 +- drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c | 2 + .../net/wireless/intel/iwlwifi/mvm/fw-api-coex.h | 222 - .../net/wireless/intel/iwlwifi/mvm/fw-api-mac.h | 3 + .../net/wireless/intel/iwlwifi/mvm/fw-api-power.h | 22 +- drivers/net/wireless/intel/iwlwifi/mvm/fw-api-rx.h | 56 +- .../net/wireless/intel/iwlwifi/mvm/fw-api-sta.h | 8 +- .../net/wireless/intel/iwlwifi/mvm/fw-api-stats.h | 24 +- drivers/net/wireless/intel/iwlwifi/mvm/fw-api-tx.h | 58 +- drivers/net/wireless/intel/iwlwifi/mvm/fw-api.h | 26 +- drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.c | 6 +- drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.h | 3 +- drivers/net/wireless/intel/iwlwifi/mvm/fw.c | 217 +- drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c | 108 +- drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c | 73 +- drivers/net/wireless/intel/iwlwifi/mvm/mvm.h | 49 +- drivers/net/wireless/intel/iwlwifi/mvm/nvm.c | 45 +- drivers/net/wireless/intel/iwlwifi/mvm/ops.c | 61 +- drivers/net/wireless/intel/iwlwifi/mvm/power.c | 2 +- drivers/net/wireless/intel/iwlwifi/mvm/rs.c | 94 +- drivers/net/wireless/intel/iwlwifi/mvm/rs.h | 3 +- drivers/net/wireless/intel/iwlwifi/mvm/rx.c | 44 +- drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c | 103 +- drivers/net/wireless/intel/iwlwifi/mvm/scan.c | 55 +- drivers/net/wireless/intel/iwlwifi/mvm/sf.c | 2 +- drivers/net/wireless/intel/iwlwifi/mvm/sta.c | 539 +- drivers/net/wireless/intel/iwlwifi/mvm/sta.h | 8 + drivers/net/wireless/intel/iwlwifi/mvm/tx.c | 106 +- drivers/net/wireless/intel/iwlwifi/mvm/utils.c | 186 +- drivers/net/wireless/intel/iwlwifi/pcie/drv.c | 17 +- drivers/net/wireless/intel/iwlwifi/pcie/internal.h | 49 +- drivers/net/wireless/intel/iwlwifi/pcie/rx.c | 129 +- drivers/net/wireless/intel/iwlwifi/pcie/trans.c | 154 +- drivers/net/wireless/intel/iwlwifi/pcie/tx.c | 188 +- drivers/net/wireless/intersil/orinoco/scan.c | 12 +- drivers/net/wireless/mac80211_hwsim.c | 225 +- drivers/net/wireless/marvell/libertas/cfg.c | 15 +- drivers/net/wireless/marvell/libertas/cmdresp.c | 4 + drivers/net/wireless/marvell/libertas/if_sdio.c | 3 +- drivers/net/wireless/marvell/libertas/if_spi.c | 4 +- drivers/net/wireless/marvell/libertas_tf/main.c | 9 +- drivers/net/wireless/marvell/mwifiex/11n_aggr.c | 5 +- drivers/net/wireless/marvell/mwifiex/cfg80211.c | 105 +- drivers/net/wireless/marvell/mwifiex/cmdevt.c | 40 +- drivers/net/wireless/marvell/mwifiex/fw.h | 7 +- drivers/net/wireless/marvell/mwifiex/init.c | 5 +- drivers/net/wireless/marvell/mwifiex/ioctl.h | 12 +- drivers/net/wireless/marvell/mwifiex/join.c | 14 +- drivers/net/wireless/marvell/mwifiex/main.c | 12 +- drivers/net/wireless/marvell/mwifiex/main.h | 6 +- drivers/net/wireless/marvell/mwifiex/pcie.c | 86 +- drivers/net/wireless/marvell/mwifiex/pcie.h | 4 +- drivers/net/wireless/marvell/mwifiex/scan.c | 48 +- drivers/net/wireless/marvell/mwifiex/sdio.c | 64 +- drivers/net/wireless/marvell/mwifiex/sta_cmd.c | 78 +- drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c | 52 +- drivers/net/wireless/marvell/mwifiex/sta_event.c | 18 +- drivers/net/wireless/marvell/mwifiex/sta_ioctl.c | 25 +- drivers/net/wireless/marvell/mwifiex/uap_cmd.c | 30 +- drivers/net/wireless/marvell/mwifiex/uap_txrx.c | 2 +- drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h | 18 +- .../net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192c.c | 11 +- .../net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c | 11 +- .../net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723a.c | 2 + .../net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c | 9 +- .../net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c | 315 +- .../net/wireless/realtek/rtl8xxxu/rtl8xxxu_regs.h | 14 +- .../realtek/rtlwifi/btcoexist/halbtcoutsrc.c | 2 +- .../realtek/rtlwifi/btcoexist/halbtcoutsrc.h | 4 +- drivers/net/wireless/realtek/rtlwifi/core.c | 2 +- drivers/net/wireless/realtek/rtlwifi/debug.c | 25 + drivers/net/wireless/realtek/rtlwifi/debug.h | 17 +- drivers/net/wireless/realtek/rtlwifi/efuse.c | 78 + drivers/net/wireless/realtek/rtlwifi/efuse.h | 2 + drivers/net/wireless/realtek/rtlwifi/ps.c | 25 +- drivers/net/wireless/realtek/rtlwifi/ps.h | 3 - drivers/net/wireless/realtek/rtlwifi/rc.c | 2 +- .../wireless/realtek/rtlwifi/rtl8188ee/Makefile | 2 +- .../net/wireless/realtek/rtlwifi/rtl8188ee/dm.c | 6 +- .../net/wireless/realtek/rtlwifi/rtl8188ee/hw.c | 74 +- .../net/wireless/realtek/rtlwifi/rtl8188ee/phy.c | 6 +- .../net/wireless/realtek/rtlwifi/rtl8188ee/rf.c | 2 +- .../net/wireless/realtek/rtlwifi/rtl8188ee/trx.c | 6 +- .../net/wireless/realtek/rtlwifi/rtl8188ee/trx.h | 4 +- .../wireless/realtek/rtlwifi/rtl8192c/dm_common.h | 2 +- .../wireless/realtek/rtlwifi/rtl8192c/phy_common.c | 10 +- .../wireless/realtek/rtlwifi/rtl8192c/phy_common.h | 2 +- .../net/wireless/realtek/rtlwifi/rtl8192ce/hw.c | 76 +- .../net/wireless/realtek/rtlwifi/rtl8192ce/phy.h | 2 +- .../net/wireless/realtek/rtlwifi/rtl8192ce/trx.c | 10 +- .../net/wireless/realtek/rtlwifi/rtl8192ce/trx.h | 4 +- .../net/wireless/realtek/rtlwifi/rtl8192cu/hw.c | 57 +- .../net/wireless/realtek/rtlwifi/rtl8192cu/mac.c | 6 +- .../net/wireless/realtek/rtlwifi/rtl8192cu/mac.h | 4 +- .../net/wireless/realtek/rtlwifi/rtl8192cu/rf.c | 2 +- .../net/wireless/realtek/rtlwifi/rtl8192de/hw.c | 73 +- .../net/wireless/realtek/rtlwifi/rtl8192de/phy.c | 2 +- .../net/wireless/realtek/rtlwifi/rtl8192de/phy.h | 2 +- .../net/wireless/realtek/rtlwifi/rtl8192de/rf.c | 2 +- .../net/wireless/realtek/rtlwifi/rtl8192de/trx.c | 10 +- .../net/wireless/realtek/rtlwifi/rtl8192de/trx.h | 4 +- .../net/wireless/realtek/rtlwifi/rtl8192ee/dm.c | 6 +- .../net/wireless/realtek/rtlwifi/rtl8192ee/hw.c | 71 +- .../net/wireless/realtek/rtlwifi/rtl8192ee/phy.c | 29 +- .../net/wireless/realtek/rtlwifi/rtl8192ee/phy.h | 2 +- .../net/wireless/realtek/rtlwifi/rtl8192ee/rf.c | 2 +- .../net/wireless/realtek/rtlwifi/rtl8192ee/trx.c | 6 +- .../net/wireless/realtek/rtlwifi/rtl8192ee/trx.h | 4 +- .../net/wireless/realtek/rtlwifi/rtl8192se/hw.c | 22 +- .../net/wireless/realtek/rtlwifi/rtl8192se/rf.c | 2 +- .../net/wireless/realtek/rtlwifi/rtl8192se/trx.c | 4 +- .../net/wireless/realtek/rtlwifi/rtl8723ae/dm.c | 6 +- .../wireless/realtek/rtlwifi/rtl8723ae/hal_btc.c | 4 +- .../net/wireless/realtek/rtlwifi/rtl8723ae/hw.c | 302 +- .../net/wireless/realtek/rtlwifi/rtl8723ae/phy.c | 6 +- .../net/wireless/realtek/rtlwifi/rtl8723ae/rf.c | 2 +- .../net/wireless/realtek/rtlwifi/rtl8723ae/trx.c | 4 +- .../net/wireless/realtek/rtlwifi/rtl8723ae/trx.h | 4 +- .../net/wireless/realtek/rtlwifi/rtl8723be/dm.c | 6 +- .../net/wireless/realtek/rtlwifi/rtl8723be/hw.c | 68 +- .../net/wireless/realtek/rtlwifi/rtl8723be/phy.c | 12 +- .../net/wireless/realtek/rtlwifi/rtl8723be/rf.c | 2 +- .../net/wireless/realtek/rtlwifi/rtl8723be/trx.c | 6 +- .../net/wireless/realtek/rtlwifi/rtl8723be/trx.h | 8 +- .../net/wireless/realtek/rtlwifi/rtl8821ae/dm.c | 12 +- .../net/wireless/realtek/rtlwifi/rtl8821ae/hw.c | 79 +- .../net/wireless/realtek/rtlwifi/rtl8821ae/phy.c | 62 +- .../net/wireless/realtek/rtlwifi/rtl8821ae/phy.h | 2 +- .../net/wireless/realtek/rtlwifi/rtl8821ae/rf.c | 2 +- .../net/wireless/realtek/rtlwifi/rtl8821ae/trx.c | 16 +- .../net/wireless/realtek/rtlwifi/rtl8821ae/trx.h | 10 +- drivers/net/wireless/realtek/rtlwifi/stats.c | 6 +- drivers/net/wireless/realtek/rtlwifi/stats.h | 4 +- drivers/net/wireless/realtek/rtlwifi/wifi.h | 68 +- drivers/net/wireless/rndis_wlan.c | 10 +- drivers/net/wireless/rsi/rsi_91x_mgmt.c | 2 +- drivers/net/wireless/st/cw1200/scan.c | 6 +- drivers/net/wireless/ti/wl1251/event.c | 6 +- drivers/net/wireless/ti/wl1251/main.c | 6 +- drivers/net/wireless/ti/wl18xx/event.c | 26 +- drivers/net/wireless/ti/wl18xx/event.h | 19 +- drivers/net/wireless/ti/wl18xx/main.c | 19 +- drivers/net/wireless/ti/wl18xx/tx.c | 22 +- drivers/net/wireless/ti/wl18xx/wl18xx.h | 8 +- drivers/net/wireless/ti/wlcore/acx.h | 1 + drivers/net/wireless/ti/wlcore/boot.c | 2 +- drivers/net/wireless/ti/wlcore/cmd.c | 20 +- drivers/net/wireless/ti/wlcore/main.c | 61 +- drivers/net/wireless/ti/wlcore/rx.c | 7 + drivers/net/wireless/ti/wlcore/scan.c | 5 +- drivers/net/wireless/ti/wlcore/sdio.c | 1 - drivers/net/wireless/ti/wlcore/spi.c | 124 +- drivers/net/wireless/ti/wlcore/wlcore_i.h | 14 + drivers/net/wireless/wl3501_cs.c | 31 +- drivers/net/xen-netback/xenbus.c | 46 +- drivers/nfc/Kconfig | 1 + drivers/nfc/fdp/fdp.c | 6 +- drivers/nfc/nfcsim.c | 643 +- drivers/nfc/nfcwilink.c | 4 +- drivers/nfc/pn533/usb.c | 9 +- drivers/nfc/port100.c | 82 +- drivers/nfc/trf7970a.c | 4 + drivers/ntb/hw/intel/ntb_hw_intel.c | 49 +- drivers/ntb/ntb_transport.c | 38 +- drivers/ntb/test/ntb_perf.c | 240 +- drivers/ntb/test/ntb_pingpong.c | 62 +- drivers/ntb/test/ntb_tool.c | 459 +- drivers/nvdimm/Kconfig | 2 +- drivers/nvdimm/blk.c | 14 +- drivers/nvdimm/btt.c | 22 +- drivers/nvdimm/btt_devs.c | 23 +- drivers/nvdimm/bus.c | 220 +- drivers/nvdimm/claim.c | 7 +- drivers/nvdimm/core.c | 259 +- drivers/nvdimm/dimm_devs.c | 5 +- drivers/nvdimm/e820.c | 1 + drivers/nvdimm/nd-core.h | 5 +- drivers/nvdimm/nd.h | 29 +- drivers/nvdimm/pmem.c | 101 +- drivers/nvdimm/pmem.h | 24 + drivers/nvdimm/region.c | 19 +- drivers/nvdimm/region_devs.c | 158 +- drivers/nvme/Kconfig | 1 + drivers/nvme/Makefile | 1 + drivers/nvme/host/Kconfig | 19 + drivers/nvme/host/Makefile | 6 + drivers/nvme/host/core.c | 333 +- drivers/nvme/host/fabrics.c | 961 ++ drivers/nvme/host/fabrics.h | 132 + drivers/nvme/host/lightnvm.c | 4 +- drivers/nvme/host/nvme.h | 52 +- drivers/nvme/host/pci.c | 110 +- drivers/nvme/host/rdma.c | 2033 ++++ drivers/nvme/target/Kconfig | 36 + drivers/nvme/target/Makefile | 9 + drivers/nvme/target/admin-cmd.c | 461 + drivers/nvme/target/configfs.c | 917 ++ drivers/nvme/target/core.c | 968 ++ drivers/nvme/target/discovery.c | 221 + drivers/nvme/target/fabrics-cmd.c | 240 + drivers/nvme/target/io-cmd.c | 215 + drivers/nvme/target/loop.c | 752 ++ drivers/nvme/target/nvmet.h | 332 + drivers/nvme/target/rdma.c | 1497 +++ drivers/nvmem/Kconfig | 4 +- drivers/nvmem/imx-ocotp.c | 17 +- drivers/nvmem/mtk-efuse.c | 47 +- drivers/nvmem/mxs-ocotp.c | 83 +- drivers/of/Kconfig | 1 + drivers/of/address.c | 49 +- drivers/of/base.c | 36 +- drivers/of/dynamic.c | 47 +- drivers/of/fdt.c | 38 +- drivers/of/fdt_address.c | 35 +- drivers/of/irq.c | 5 + drivers/of/of_mdio.c | 38 +- drivers/of/of_numa.c | 4 +- drivers/of/of_pci.c | 6 +- drivers/of/of_reserved_mem.c | 105 +- drivers/of/overlay.c | 43 +- drivers/of/platform.c | 55 +- drivers/of/resolver.c | 8 + drivers/of/unittest.c | 5 +- drivers/parisc/ccio-dma.c | 16 +- drivers/parisc/sba_iommu.c | 16 +- drivers/pci/Kconfig | 2 +- drivers/pci/Makefile | 3 + drivers/pci/bus.c | 31 +- drivers/pci/ecam.c | 8 +- drivers/pci/host-bridge.c | 1 + drivers/pci/host/Kconfig | 49 +- drivers/pci/host/Makefile | 2 + drivers/pci/host/pci-aardvark.c | 1001 ++ drivers/pci/host/pci-dra7xx.c | 4 +- drivers/pci/host/pci-host-common.c | 44 +- drivers/pci/host/pci-host-generic.c | 13 +- drivers/pci/host/pci-hyperv.c | 30 +- drivers/pci/host/pci-keystone.c | 10 +- drivers/pci/host/pci-layerscape.c | 10 +- drivers/pci/host/pci-mvebu.c | 28 +- drivers/pci/host/pci-rcar-gen2.c | 27 +- drivers/pci/host/pci-tegra.c | 99 +- drivers/pci/host/pci-thunder-ecam.c | 11 +- drivers/pci/host/pci-thunder-pem.c | 14 +- drivers/pci/host/pci-versatile.c | 29 +- drivers/pci/host/pci-xgene.c | 24 +- drivers/pci/host/pcie-altera.c | 83 +- drivers/pci/host/pcie-armada8k.c | 14 +- drivers/pci/host/pcie-artpec6.c | 280 + drivers/pci/host/pcie-designware-plat.c | 10 +- drivers/pci/host/pcie-designware.c | 34 +- drivers/pci/host/pcie-hisi.c | 13 +- drivers/pci/host/pcie-iproc.c | 4 + drivers/pci/host/pcie-rcar.c | 44 +- drivers/pci/host/pcie-xilinx-nwl.c | 20 +- drivers/pci/host/pcie-xilinx.c | 22 +- drivers/pci/hotplug/Kconfig | 13 + drivers/pci/hotplug/Makefile | 3 + drivers/pci/hotplug/acpiphp_glue.c | 6 + drivers/pci/hotplug/pciehp_hpc.c | 4 + drivers/pci/hotplug/pnv_php.c | 711 ++ drivers/pci/hotplug/rpaphp_slot.c | 17 +- drivers/pci/msi.c | 268 +- drivers/pci/pci-driver.c | 5 +- drivers/pci/pci-mid.c | 82 + drivers/pci/pci-sysfs.c | 5 + drivers/pci/pci.c | 285 +- drivers/pci/pci.h | 11 + drivers/pci/pcie/Kconfig | 5 +- drivers/pci/pcie/aspm.c | 2 +- drivers/pci/pcie/pcie-dpc.c | 19 +- drivers/pci/pcie/portdrv_core.c | 3 + drivers/pci/pcie/portdrv_pci.c | 52 +- drivers/pci/probe.c | 22 +- drivers/pci/proc.c | 9 +- drivers/pci/quirks.c | 14 +- drivers/pci/remove.c | 1 + drivers/pci/setup-bus.c | 73 + drivers/pci/xen-pcifront.c | 2 +- drivers/pcmcia/ds.c | 12 +- drivers/pcmcia/pxa2xx_base.c | 9 +- drivers/pcmcia/pxa2xx_base.h | 2 +- drivers/pcmcia/sa1111_badge4.c | 22 +- drivers/pcmcia/sa1111_generic.c | 22 +- drivers/pcmcia/sa1111_jornada720.c | 25 +- drivers/pcmcia/sa1111_lubbock.c | 32 +- drivers/pcmcia/sa1111_neponset.c | 26 +- drivers/pcmcia/sa11xx_base.c | 8 +- drivers/pcmcia/soc_common.c | 2 +- drivers/perf/arm_pmu.c | 88 +- drivers/phy/Kconfig | 19 + drivers/phy/Makefile | 3 +- drivers/phy/phy-bcm-ns2-pcie.c | 115 + drivers/phy/phy-brcm-sata.c | 83 +- drivers/phy/phy-core.c | 15 + drivers/phy/phy-da8xx-usb.c | 245 + drivers/phy/phy-qcom-ufs-qmp-14nm.c | 1 - drivers/phy/phy-qcom-ufs-qmp-20nm.c | 1 - drivers/phy/phy-rockchip-emmc.c | 241 +- drivers/phy/phy-rockchip-usb.c | 23 +- drivers/phy/phy-sun4i-usb.c | 113 +- drivers/phy/phy-sun9i-usb.c | 4 +- drivers/phy/phy-xgene.c | 4 +- drivers/pinctrl/Kconfig | 26 +- drivers/pinctrl/Makefile | 4 +- drivers/pinctrl/bcm/Kconfig | 15 + drivers/pinctrl/bcm/Makefile | 1 + drivers/pinctrl/bcm/pinctrl-iproc-gpio.c | 118 +- drivers/pinctrl/bcm/pinctrl-ns2-mux.c | 12 +- drivers/pinctrl/bcm/pinctrl-nsp-gpio.c | 6 +- drivers/pinctrl/bcm/pinctrl-nsp-mux.c | 642 ++ drivers/pinctrl/core.c | 25 +- drivers/pinctrl/core.h | 2 + drivers/pinctrl/devicetree.c | 7 +- drivers/pinctrl/freescale/pinctrl-imx.c | 29 +- drivers/pinctrl/freescale/pinctrl-imx1-core.c | 3 +- drivers/pinctrl/freescale/pinctrl-imx1.c | 9 +- drivers/pinctrl/freescale/pinctrl-imx21.c | 9 +- drivers/pinctrl/freescale/pinctrl-imx23.c | 17 +- drivers/pinctrl/freescale/pinctrl-imx25.c | 10 - drivers/pinctrl/freescale/pinctrl-imx27.c | 10 - drivers/pinctrl/freescale/pinctrl-imx28.c | 17 +- drivers/pinctrl/freescale/pinctrl-imx35.c | 10 - drivers/pinctrl/freescale/pinctrl-imx50.c | 9 - drivers/pinctrl/freescale/pinctrl-imx51.c | 10 - drivers/pinctrl/freescale/pinctrl-imx53.c | 10 - drivers/pinctrl/freescale/pinctrl-imx6dl.c | 14 +- drivers/pinctrl/freescale/pinctrl-imx6q.c | 10 - drivers/pinctrl/freescale/pinctrl-imx6sl.c | 15 +- drivers/pinctrl/freescale/pinctrl-imx6sx.c | 14 +- drivers/pinctrl/freescale/pinctrl-imx6ul.c | 14 +- drivers/pinctrl/freescale/pinctrl-imx7d.c | 14 +- drivers/pinctrl/freescale/pinctrl-mxs.c | 12 - drivers/pinctrl/freescale/pinctrl-mxs.h | 1 - drivers/pinctrl/freescale/pinctrl-vf610.c | 10 - drivers/pinctrl/intel/Kconfig | 11 + drivers/pinctrl/intel/Makefile | 1 + drivers/pinctrl/intel/pinctrl-baytrail.c | 25 +- drivers/pinctrl/intel/pinctrl-broxton.c | 43 +- drivers/pinctrl/intel/pinctrl-cherryview.c | 80 +- drivers/pinctrl/intel/pinctrl-intel.c | 52 +- drivers/pinctrl/intel/pinctrl-merrifield.c | 912 ++ drivers/pinctrl/mediatek/pinctrl-mtk-common.c | 4 +- drivers/pinctrl/meson/pinctrl-meson-gxbb.c | 163 +- drivers/pinctrl/mvebu/pinctrl-kirkwood.c | 85 +- drivers/pinctrl/nomadik/pinctrl-nomadik.c | 96 - drivers/pinctrl/pinconf-generic.c | 22 +- drivers/pinctrl/pinconf.c | 8 +- drivers/pinctrl/pinctrl-at91-pio4.c | 26 +- drivers/pinctrl/pinctrl-at91.c | 13 +- drivers/pinctrl/pinctrl-digicolor.c | 16 +- drivers/pinctrl/pinctrl-lpc18xx.c | 20 +- drivers/pinctrl/pinctrl-max77620.c | 673 ++ drivers/pinctrl/pinctrl-oxnas.c | 846 ++ drivers/pinctrl/pinctrl-pistachio.c | 9 +- drivers/pinctrl/pinctrl-rockchip.c | 4 +- drivers/pinctrl/pinctrl-st.c | 2 +- drivers/pinctrl/pinctrl-u300.c | 2 +- drivers/pinctrl/pinctrl-xway.c | 77 +- drivers/pinctrl/pinctrl-zynq.c | 13 +- drivers/pinctrl/pinmux.c | 16 +- drivers/pinctrl/qcom/Kconfig | 8 + drivers/pinctrl/qcom/Makefile | 1 + drivers/pinctrl/qcom/pinctrl-mdm9615.c | 483 + drivers/pinctrl/qcom/pinctrl-msm.c | 6 +- drivers/pinctrl/qcom/pinctrl-msm8660.c | 114 +- drivers/pinctrl/qcom/pinctrl-msm8x74.c | 43 +- drivers/pinctrl/qcom/pinctrl-ssbi-mpp.c | 1 + drivers/pinctrl/samsung/pinctrl-exynos5440.c | 1 + drivers/pinctrl/samsung/pinctrl-samsung.c | 1 + drivers/pinctrl/sh-pfc/core.c | 10 - drivers/pinctrl/sh-pfc/core.h | 58 - drivers/pinctrl/sh-pfc/gpio.c | 13 +- drivers/pinctrl/sh-pfc/pfc-r8a73a4.c | 1 - drivers/pinctrl/sh-pfc/pfc-r8a7740.c | 1 - drivers/pinctrl/sh-pfc/pfc-r8a7778.c | 2 +- drivers/pinctrl/sh-pfc/pfc-r8a7790.c | 59 +- drivers/pinctrl/sh-pfc/pfc-r8a7791.c | 1 - drivers/pinctrl/sh-pfc/pfc-r8a7795.c | 327 +- drivers/pinctrl/sh-pfc/pfc-sh7757.c | 1 - drivers/pinctrl/sh-pfc/pinctrl.c | 46 +- drivers/pinctrl/sh-pfc/sh_pfc.h | 55 +- drivers/pinctrl/sirf/pinctrl-atlas7.c | 4 +- drivers/pinctrl/stm32/Kconfig | 6 + drivers/pinctrl/stm32/Makefile | 1 + drivers/pinctrl/stm32/pinctrl-stm32.c | 31 +- drivers/pinctrl/stm32/pinctrl-stm32f746.c | 1681 +++ drivers/pinctrl/sunxi/pinctrl-sun8i-a23.c | 6 +- drivers/pinctrl/sunxi/pinctrl-sun8i-a33.c | 6 +- drivers/pinctrl/sunxi/pinctrl-sun8i-h3.c | 6 +- drivers/pinctrl/tegra/pinctrl-tegra.c | 8 +- drivers/pinctrl/tegra/pinctrl-tegra.h | 6 +- drivers/pinctrl/tegra/pinctrl-tegra114.c | 4 +- drivers/pinctrl/tegra/pinctrl-tegra124.c | 4 +- drivers/pinctrl/tegra/pinctrl-tegra20.c | 6 +- drivers/pinctrl/tegra/pinctrl-tegra210.c | 4 +- drivers/pinctrl/tegra/pinctrl-tegra30.c | 4 +- drivers/pinctrl/uniphier/Kconfig | 20 +- drivers/pinctrl/uniphier/Makefile | 2 + drivers/pinctrl/uniphier/pinctrl-uniphier-core.c | 298 +- drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c | 952 ++ drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c | 1050 ++ drivers/pinctrl/uniphier/pinctrl-uniphier-ld4.c | 494 +- drivers/pinctrl/uniphier/pinctrl-uniphier-ld6b.c | 609 +- drivers/pinctrl/uniphier/pinctrl-uniphier-pro4.c | 819 +- drivers/pinctrl/uniphier/pinctrl-uniphier-pro5.c | 653 +- drivers/pinctrl/uniphier/pinctrl-uniphier-pxs2.c | 603 +- drivers/pinctrl/uniphier/pinctrl-uniphier-sld8.c | 455 +- drivers/pinctrl/uniphier/pinctrl-uniphier.h | 59 +- drivers/platform/olpc/olpc-ec.c | 8 +- drivers/platform/x86/Kconfig | 15 + drivers/platform/x86/Makefile | 1 + drivers/platform/x86/apple-gmux.c | 55 +- drivers/platform/x86/asus-nb-wmi.c | 50 +- drivers/platform/x86/asus-wireless.c | 91 +- drivers/platform/x86/asus-wmi.c | 8 +- drivers/platform/x86/asus-wmi.h | 1 + drivers/platform/x86/dell-wmi.c | 293 +- drivers/platform/x86/fujitsu-laptop.c | 81 +- drivers/platform/x86/intel-hid.c | 5 +- drivers/platform/x86/intel-vbtn.c | 188 + drivers/platform/x86/intel_pmc_core.c | 54 +- drivers/platform/x86/intel_pmc_core.h | 3 +- drivers/platform/x86/intel_pmc_ipc.c | 10 +- drivers/platform/x86/intel_pmic_gpio.c | 8 +- drivers/platform/x86/intel_telemetry_debugfs.c | 5 +- drivers/platform/x86/intel_telemetry_pltdrv.c | 5 +- drivers/platform/x86/toshiba_acpi.c | 136 +- drivers/pnp/isapnp/proc.c | 2 +- drivers/pnp/pnpbios/core.c | 4 +- drivers/power/Kconfig | 2 + drivers/power/axp20x_usb_power.c | 92 +- drivers/power/axp288_charger.c | 77 +- drivers/power/bq27xxx_battery.c | 289 +- drivers/power/bq27xxx_battery_i2c.c | 2 +- drivers/power/max8903_charger.c | 239 +- drivers/power/power_supply_core.c | 6 +- drivers/power/power_supply_sysfs.c | 2 +- drivers/power/qcom_smbb.c | 21 + drivers/power/reset/Kconfig | 27 +- drivers/power/reset/Makefile | 3 + drivers/power/reset/brcm-kona-reset.c | 73 + drivers/power/reset/reboot-mode.c | 140 + drivers/power/reset/reboot-mode.h | 14 + drivers/power/reset/syscon-poweroff.c | 2 +- drivers/power/reset/syscon-reboot-mode.c | 99 + drivers/power/reset/vexpress-poweroff.c | 4 +- drivers/powercap/intel_rapl.c | 157 +- drivers/pwm/Kconfig | 26 +- drivers/pwm/Makefile | 3 + drivers/pwm/core.c | 27 + drivers/pwm/pwm-atmel.c | 30 +- drivers/pwm/pwm-bcm-iproc.c | 277 + drivers/pwm/pwm-clps711x.c | 2 +- drivers/pwm/pwm-cros-ec.c | 260 + drivers/pwm/pwm-lpc32xx.c | 7 + drivers/pwm/pwm-lpss-pci.c | 1 + drivers/pwm/pwm-lpss.c | 26 +- drivers/pwm/pwm-omap-dmtimer.c | 12 +- drivers/pwm/pwm-rockchip.c | 178 +- drivers/pwm/pwm-stmpe.c | 319 + drivers/pwm/pwm-tegra.c | 69 +- drivers/pwm/pwm-tiecap.c | 37 +- drivers/pwm/pwm-tiehrpwm.c | 38 +- drivers/pwm/pwm-tipwmss.c | 49 - drivers/pwm/sysfs.c | 17 + drivers/rapidio/Kconfig | 9 + drivers/rapidio/Makefile | 1 + drivers/rapidio/devices/rio_mport_cdev.c | 6 +- drivers/rapidio/devices/tsi721.c | 57 +- drivers/rapidio/devices/tsi721.h | 2 +- drivers/rapidio/devices/tsi721_dma.c | 27 +- drivers/rapidio/rio-scan.c | 74 +- drivers/rapidio/rio.c | 212 +- drivers/rapidio/rio.h | 2 +- drivers/rapidio/rio_cm.c | 2381 ++++ drivers/rapidio/switches/Kconfig | 6 + drivers/rapidio/switches/Makefile | 1 + drivers/rapidio/switches/idt_gen2.c | 7 +- drivers/rapidio/switches/idt_gen3.c | 382 + drivers/rapidio/switches/tsi57x.c | 26 +- drivers/regulator/Kconfig | 18 +- drivers/regulator/Makefile | 1 + drivers/regulator/ab8500-ext.c | 465 +- drivers/regulator/act8865-regulator.c | 12 +- drivers/regulator/axp20x-regulator.c | 147 +- drivers/regulator/core.c | 27 - drivers/regulator/da9052-regulator.c | 6 +- drivers/regulator/da9210-regulator.c | 21 +- drivers/regulator/da9211-regulator.c | 13 +- drivers/regulator/da9211-regulator.h | 3 +- drivers/regulator/fixed.c | 14 +- drivers/regulator/lp873x-regulator.c | 14 +- drivers/regulator/max14577-regulator.c | 4 +- drivers/regulator/max77693-regulator.c | 4 +- drivers/regulator/max8973-regulator.c | 16 +- drivers/regulator/mt6323-regulator.c | 425 + drivers/regulator/mt6397-regulator.c | 95 +- drivers/regulator/of_regulator.c | 3 + drivers/regulator/pfuze100-regulator.c | 15 +- drivers/regulator/pv88060-regulator.c | 3 - drivers/regulator/pv88080-regulator.c | 3 - drivers/regulator/pv88090-regulator.c | 3 - drivers/regulator/pwm-regulator.c | 196 +- drivers/regulator/qcom_smd-regulator.c | 32 +- drivers/regulator/qcom_spmi-regulator.c | 7 +- drivers/regulator/rn5t618-regulator.c | 40 +- drivers/regulator/tps65217-regulator.c | 69 +- drivers/regulator/tps65218-regulator.c | 76 +- drivers/regulator/twl-regulator.c | 2 +- drivers/remoteproc/Kconfig | 14 + drivers/remoteproc/Makefile | 2 + drivers/remoteproc/qcom_mdt_loader.c | 179 + drivers/remoteproc/qcom_mdt_loader.h | 13 + drivers/remoteproc/qcom_q6v5_pil.c | 907 ++ drivers/reset/Kconfig | 14 + drivers/reset/Makefile | 2 + drivers/reset/core.c | 37 + drivers/reset/hisilicon/hi6220_reset.c | 122 +- drivers/reset/reset-ath79.c | 3 +- drivers/reset/reset-meson.c | 136 + drivers/reset/reset-oxnas.c | 12 +- drivers/reset/reset-pistachio.c | 12 +- drivers/reset/reset-socfpga.c | 12 +- drivers/reset/reset-sunxi.c | 12 +- drivers/reset/reset-ti-syscon.c | 237 + drivers/reset/reset-zynq.c | 12 +- drivers/reset/sti/Kconfig | 1 - drivers/rtc/Kconfig | 26 +- drivers/rtc/Makefile | 2 + drivers/rtc/interface.c | 32 +- drivers/rtc/rtc-abx80x.c | 12 +- drivers/rtc/rtc-asm9260.c | 1 - drivers/rtc/rtc-at91sam9.c | 1 + drivers/rtc/rtc-cmos.c | 17 +- drivers/rtc/rtc-da9052.c | 1 + drivers/rtc/rtc-da9055.c | 1 + drivers/rtc/rtc-davinci.c | 2 + drivers/rtc/rtc-ds1286.c | 2 +- drivers/rtc/rtc-ds1305.c | 7 - drivers/rtc/rtc-ds1307.c | 5 - drivers/rtc/rtc-ds1343.c | 6 - drivers/rtc/rtc-ds1685.c | 53 +- drivers/rtc/rtc-ds2404.c | 2 +- drivers/rtc/rtc-ds3232.c | 6 - drivers/rtc/rtc-efi.c | 6 + drivers/rtc/rtc-generic.c | 36 +- drivers/rtc/rtc-hym8563.c | 5 +- drivers/rtc/rtc-isl12057.c | 33 +- drivers/rtc/rtc-m41t80.c | 34 +- drivers/rtc/rtc-m48t86.c | 2 +- drivers/rtc/rtc-max6916.c | 164 + drivers/rtc/rtc-mc146818-lib.c | 198 + drivers/rtc/rtc-mrst.c | 10 +- drivers/rtc/rtc-opal.c | 4 +- drivers/rtc/rtc-pcf2123.c | 4 +- drivers/rtc/rtc-pcf85063.c | 59 +- drivers/rtc/rtc-pcf8563.c | 5 +- drivers/rtc/rtc-rc5t583.c | 1 + drivers/rtc/rtc-rs5c372.c | 6 - drivers/rtc/rtc-rv8803.c | 205 +- drivers/rtc/rtc-rx8010.c | 8 +- drivers/rtc/rtc-rx8025.c | 5 - drivers/rtc/rtc-s35390a.c | 157 +- drivers/rtc/rtc-s3c.c | 14 - drivers/rtc/rtc-sh.c | 42 +- drivers/rtc/rtc-tegra.c | 6 - drivers/rtc/rtc-v3020.c | 2 +- drivers/s390/block/dasd_eckd.c | 14 +- drivers/s390/block/dasd_genhd.c | 3 +- drivers/s390/block/dcssblk.c | 10 +- drivers/s390/block/scm_blk.c | 3 +- drivers/s390/char/keyboard.c | 15 +- drivers/s390/char/sclp_con.c | 3 +- drivers/s390/char/sclp_config.c | 2 +- drivers/s390/char/sclp_early.c | 12 +- drivers/s390/char/sclp_ocf.c | 23 +- drivers/s390/char/zcore.c | 2 +- drivers/s390/cio/chp.c | 82 +- drivers/s390/cio/chp.h | 2 +- drivers/s390/cio/chsc.c | 25 +- drivers/s390/cio/chsc.h | 5 +- drivers/s390/cio/chsc_sch.c | 2 +- drivers/s390/cio/cmf.c | 15 +- drivers/s390/cio/device.c | 2 - drivers/s390/cio/device_ops.c | 22 +- drivers/s390/cio/device_status.c | 7 +- drivers/s390/cio/idset.h | 2 +- drivers/s390/cio/io_sch.h | 1 - drivers/s390/cio/ioasm.c | 91 +- drivers/s390/cio/qdio_main.c | 113 +- drivers/s390/crypto/ap_bus.c | 127 +- drivers/s390/crypto/ap_bus.h | 1 + drivers/s390/crypto/zcrypt_cex2a.c | 2 +- drivers/s390/crypto/zcrypt_cex4.c | 2 +- drivers/s390/crypto/zcrypt_pcixcc.c | 2 +- drivers/s390/net/qeth_core.h | 46 +- drivers/s390/net/qeth_core_main.c | 263 +- drivers/s390/net/qeth_core_sys.c | 4 + drivers/s390/net/qeth_l2.h | 7 + drivers/s390/net/qeth_l2_main.c | 105 +- drivers/s390/net/qeth_l3.h | 31 +- drivers/s390/net/qeth_l3_main.c | 1031 +- drivers/s390/net/qeth_l3_sys.c | 83 +- drivers/s390/virtio/Makefile | 6 +- drivers/s390/virtio/kvm_virtio.c | 4 +- drivers/scsi/53c700.c | 10 +- drivers/scsi/53c700.h | 15 +- drivers/scsi/Kconfig | 19 +- drivers/scsi/Makefile | 1 + drivers/scsi/aacraid/commctrl.c | 7 +- drivers/scsi/bnx2fc/bnx2fc_fcoe.c | 14 +- drivers/scsi/bnx2fc/bnx2fc_io.c | 2 +- drivers/scsi/bnx2i/bnx2i_hwi.c | 2 +- drivers/scsi/cxgbi/Makefile | 2 + drivers/scsi/cxgbi/cxgb3i/Kbuild | 1 + drivers/scsi/cxgbi/cxgb3i/Kconfig | 1 + drivers/scsi/cxgbi/cxgb3i/cxgb3i.c | 164 +- drivers/scsi/cxgbi/cxgb4i/Kbuild | 1 + drivers/scsi/cxgbi/cxgb4i/Kconfig | 1 + drivers/scsi/cxgbi/cxgb4i/cxgb4i.c | 203 +- drivers/scsi/cxgbi/libcxgbi.c | 734 +- drivers/scsi/cxgbi/libcxgbi.h | 188 +- drivers/scsi/cxlflash/main.c | 106 +- drivers/scsi/cxlflash/main.h | 6 +- drivers/scsi/cxlflash/sislite.h | 6 + drivers/scsi/fcoe/fcoe.c | 374 +- drivers/scsi/fcoe/fcoe.h | 1 + drivers/scsi/fcoe/fcoe_ctlr.c | 245 +- drivers/scsi/fcoe/fcoe_sysfs.c | 39 + drivers/scsi/fcoe/fcoe_transport.c | 4 +- drivers/scsi/fnic/fnic_fcs.c | 14 +- drivers/scsi/fnic/fnic_fip.h | 8 - drivers/scsi/hisi_sas/hisi_sas.h | 2 +- drivers/scsi/hisi_sas/hisi_sas_v2_hw.c | 73 +- drivers/scsi/hosts.c | 2 +- drivers/scsi/hpsa.c | 83 +- drivers/scsi/ibmvscsi/ibmvfc.c | 2 + drivers/scsi/ibmvscsi/ibmvfc.h | 2 +- drivers/scsi/ibmvscsi/ibmvscsi.h | 2 +- drivers/scsi/ibmvscsi_tgt/Makefile | 3 + drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c | 4087 +++++++ drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h | 346 + drivers/scsi/ibmvscsi_tgt/libsrp.c | 427 + drivers/scsi/ibmvscsi_tgt/libsrp.h | 123 + drivers/scsi/ipr.c | 26 +- drivers/scsi/ipr.h | 4 + drivers/scsi/libfc/fc_exch.c | 10 +- drivers/scsi/libfc/fc_lport.c | 24 +- drivers/scsi/libfc/fc_rport.c | 49 +- drivers/scsi/libsas/sas_ata.c | 12 +- drivers/scsi/lpfc/lpfc.h | 27 +- drivers/scsi/lpfc/lpfc_attr.c | 237 +- drivers/scsi/lpfc/lpfc_attr.h | 116 + drivers/scsi/lpfc/lpfc_crtn.h | 7 +- drivers/scsi/lpfc/lpfc_ct.c | 4 + drivers/scsi/lpfc/lpfc_els.c | 290 +- drivers/scsi/lpfc/lpfc_hw.h | 36 +- drivers/scsi/lpfc/lpfc_hw4.h | 41 +- drivers/scsi/lpfc/lpfc_ids.h | 122 + drivers/scsi/lpfc/lpfc_init.c | 292 +- drivers/scsi/lpfc/lpfc_scsi.c | 14 +- drivers/scsi/lpfc/lpfc_scsi.h | 3 +- drivers/scsi/lpfc/lpfc_sli.c | 116 +- drivers/scsi/lpfc/lpfc_sli.h | 3 +- drivers/scsi/lpfc/lpfc_sli4.h | 4 +- drivers/scsi/lpfc/lpfc_version.h | 2 +- drivers/scsi/mpt3sas/mpt3sas_scsih.c | 27 +- drivers/scsi/mpt3sas/mpt3sas_transport.c | 5 + drivers/scsi/osd/osd_initiator.c | 37 +- drivers/scsi/pm8001/pm8001_init.c | 2 +- drivers/scsi/qla2xxx/qla_attr.c | 170 +- drivers/scsi/qla2xxx/qla_bsg.c | 93 +- drivers/scsi/qla2xxx/qla_bsg.h | 13 + drivers/scsi/qla2xxx/qla_dbg.c | 50 +- drivers/scsi/qla2xxx/qla_def.h | 12 +- drivers/scsi/qla2xxx/qla_fw.h | 2 +- drivers/scsi/qla2xxx/qla_gbl.h | 5 +- drivers/scsi/qla2xxx/qla_init.c | 39 +- drivers/scsi/qla2xxx/qla_isr.c | 31 +- drivers/scsi/qla2xxx/qla_mbx.c | 123 +- drivers/scsi/qla2xxx/qla_nx.h | 1 - drivers/scsi/qla2xxx/qla_os.c | 127 +- drivers/scsi/qla2xxx/qla_target.c | 16 +- drivers/scsi/qla2xxx/qla_tmpl.c | 9 +- drivers/scsi/qla2xxx/qla_version.h | 2 +- drivers/scsi/scsi_debug.c | 93 +- drivers/scsi/scsi_devinfo.c | 4 + drivers/scsi/scsi_priv.h | 1 + drivers/scsi/scsi_transport_sas.c | 16 - drivers/scsi/sd.c | 26 +- drivers/scsi/ses.c | 5 +- drivers/scsi/snic/snic_disc.c | 4 +- drivers/scsi/snic/snic_fwint.h | 2 +- drivers/scsi/sr.c | 3 +- drivers/scsi/storvsc_drv.c | 2 + drivers/scsi/ufs/Kconfig | 16 + drivers/scsi/ufs/Makefile | 2 + drivers/scsi/ufs/tc-dwc-g210-pci.c | 181 + drivers/scsi/ufs/tc-dwc-g210-pltfrm.c | 113 + drivers/scsi/ufs/tc-dwc-g210.c | 319 + drivers/scsi/ufs/tc-dwc-g210.h | 19 + drivers/scsi/ufs/ufshcd-dwc.c | 154 + drivers/scsi/ufs/ufshcd-dwc.h | 26 + drivers/scsi/ufs/ufshcd-pltfrm.c | 2 +- drivers/scsi/ufs/ufshcd.c | 92 +- drivers/scsi/ufs/ufshcd.h | 7 + drivers/scsi/ufs/ufshci-dwc.h | 36 + drivers/scsi/ufs/ufshci.h | 11 + drivers/scsi/ufs/unipro.h | 39 + drivers/scsi/vmw_pvscsi.c | 2 +- drivers/scsi/vmw_pvscsi.h | 2 +- drivers/scsi/wd7000.c | 6 +- drivers/scsi/wd719x.c | 2 +- drivers/sh/pm_runtime.c | 9 - drivers/soc/Kconfig | 2 +- drivers/soc/Makefile | 2 +- drivers/soc/bcm/Kconfig | 18 +- drivers/soc/bcm/Makefile | 1 + drivers/soc/bcm/brcmstb/Makefile | 1 + drivers/soc/bcm/brcmstb/biuctrl.c | 117 + drivers/soc/bcm/brcmstb/common.c | 99 + drivers/soc/fsl/qe/Kconfig | 6 +- drivers/soc/fsl/qe/Makefile | 1 + drivers/soc/fsl/qe/qe.c | 6 + drivers/soc/fsl/qe/qe_tdm.c | 276 + drivers/soc/fsl/qe/ucc.c | 450 + drivers/soc/fsl/qe/ucc_fast.c | 36 + drivers/soc/qcom/smem_state.c | 12 +- drivers/soc/qcom/smp2p.c | 4 +- drivers/soc/qcom/smsm.c | 2 +- drivers/soc/qcom/wcnss_ctrl.c | 125 +- drivers/soc/renesas/Makefile | 2 + drivers/soc/renesas/r8a7792-sysc.c | 34 + drivers/soc/renesas/r8a7796-sysc.c | 48 + drivers/soc/renesas/rcar-sysc.c | 45 +- drivers/soc/renesas/rcar-sysc.h | 2 + drivers/soc/samsung/Kconfig | 4 + drivers/soc/samsung/Makefile | 1 + drivers/soc/samsung/exynos3250-pmu.c | 2 +- drivers/soc/samsung/exynos5420-pmu.c | 2 +- drivers/soc/samsung/pm_domains.c | 245 + drivers/soc/tegra/pmc.c | 141 +- drivers/soc/ux500/Kconfig | 7 + drivers/soc/ux500/Makefile | 1 + drivers/soc/ux500/ux500-soc-id.c | 222 + drivers/spi/Kconfig | 1 + drivers/spi/Makefile | 1 + drivers/spi/spi-bfin-sport.c | 15 +- drivers/spi/spi-bfin5xx.c | 15 +- drivers/spi/spi-cavium-octeon.c | 104 + drivers/spi/spi-cavium.c | 151 + drivers/spi/spi-cavium.h | 329 + drivers/spi/spi-clps711x.c | 69 +- drivers/spi/spi-img-spfi.c | 2 - drivers/spi/spi-imx.c | 191 +- drivers/spi/spi-loopback-test.c | 2 +- drivers/spi/spi-mpc52xx-psc.c | 17 +- drivers/spi/spi-mt65xx.c | 1 - drivers/spi/spi-omap2-mcspi.c | 145 +- drivers/spi/spi-orion.c | 88 + drivers/spi/spi-pic32-sqi.c | 7 +- drivers/spi/spi-pic32.c | 5 +- drivers/spi/spi-pxa2xx-dma.c | 170 +- drivers/spi/spi-pxa2xx-pci.c | 129 +- drivers/spi/spi-pxa2xx.c | 46 +- drivers/spi/spi-pxa2xx.h | 9 - drivers/spi/spi-qup.c | 1 - drivers/spi/spi-rockchip.c | 20 + drivers/spi/spi-s3c64xx.c | 206 +- drivers/spi/spi-sh-msiof.c | 21 +- drivers/spi/spi-sh.c | 16 +- drivers/spi/spi-st-ssc4.c | 36 +- drivers/spi/spi-sun4i.c | 8 + drivers/spi/spi-sun6i.c | 7 + drivers/spi/spi-ti-qspi.c | 2 +- drivers/spi/spi-topcliff-pch.c | 26 +- drivers/spi/spi-txx9.c | 11 +- drivers/spi/spi-xilinx.c | 8 +- drivers/spi/spi.c | 176 +- drivers/spi/spidev.c | 41 + drivers/ssb/driver_gpio.c | 22 +- drivers/staging/Kconfig | 2 + drivers/staging/Makefile | 1 + drivers/staging/android/Kconfig | 17 +- drivers/staging/android/Makefile | 3 +- drivers/staging/android/ashmem.c | 2 +- drivers/staging/android/lowmemorykiller.c | 12 +- drivers/staging/android/sw_sync.c | 341 +- drivers/staging/android/sync_debug.c | 165 +- drivers/staging/android/sync_debug.h | 84 + drivers/staging/android/trace/sync.h | 14 +- drivers/staging/comedi/comedi.h | 2 +- drivers/staging/comedi/comedi_fops.c | 21 +- drivers/staging/comedi/drivers/addi_apci_1564.c | 305 +- drivers/staging/comedi/drivers/adl_pci9118.c | 8 +- drivers/staging/comedi/drivers/adv_pci1760.c | 1 + drivers/staging/comedi/drivers/cb_pcidas64.c | 209 +- drivers/staging/comedi/drivers/comedi_bond.c | 10 +- drivers/staging/comedi/drivers/daqboard2000.c | 380 +- drivers/staging/comedi/drivers/das16.c | 39 +- drivers/staging/comedi/drivers/das16m1.c | 482 +- drivers/staging/comedi/drivers/das6402.c | 74 +- drivers/staging/comedi/drivers/das800.c | 106 +- drivers/staging/comedi/drivers/dmm32at.c | 98 +- drivers/staging/comedi/drivers/dt2801.c | 95 +- drivers/staging/comedi/drivers/dt2811.c | 852 +- drivers/staging/comedi/drivers/dt2814.c | 72 +- drivers/staging/comedi/drivers/dt2815.c | 140 +- drivers/staging/comedi/drivers/dt2817.c | 64 +- drivers/staging/comedi/drivers/gsc_hpdi.c | 87 +- drivers/staging/comedi/drivers/jr3_pci.c | 36 +- drivers/staging/comedi/drivers/me_daq.c | 2 +- drivers/staging/comedi/drivers/mpc624.c | 4 +- drivers/staging/comedi/drivers/ni_65xx.c | 18 +- drivers/staging/comedi/drivers/ni_pcidio.c | 4 +- drivers/staging/comedi/drivers/ni_pcimio.c | 8 +- drivers/staging/comedi/drivers/pcmmio.c | 40 +- drivers/staging/comedi/drivers/pcmuio.c | 2 +- drivers/staging/comedi/drivers/plx9080.h | 957 +- drivers/staging/comedi/drivers/quatech_daqp_cs.c | 2 +- drivers/staging/comedi/drivers/rtd520.c | 15 +- drivers/staging/comedi/drivers/s626.c | 8 +- drivers/staging/comedi/drivers/s626.h | 356 +- drivers/staging/comedi/drivers/serial2002.c | 7 +- drivers/staging/emxx_udc/Kconfig | 2 +- drivers/staging/emxx_udc/emxx_udc.c | 36 +- drivers/staging/fbtft/fbtft-core.c | 4 +- drivers/staging/fsl-mc/bus/dpbp.c | 132 +- drivers/staging/fsl-mc/bus/dpmcp-cmd.h | 86 +- drivers/staging/fsl-mc/bus/dpmcp.c | 89 +- drivers/staging/fsl-mc/bus/dpmng-cmd.h | 12 +- drivers/staging/fsl-mc/bus/dpmng.c | 15 +- drivers/staging/fsl-mc/bus/dprc-cmd.h | 379 +- drivers/staging/fsl-mc/bus/dprc-driver.c | 20 +- drivers/staging/fsl-mc/bus/dprc.c | 715 +- drivers/staging/fsl-mc/bus/mc-allocator.c | 2 +- drivers/staging/fsl-mc/bus/mc-bus.c | 71 +- drivers/staging/fsl-mc/bus/mc-msi.c | 17 +- drivers/staging/fsl-mc/bus/mc-sys.c | 46 +- drivers/staging/fsl-mc/include/dpbp-cmd.h | 125 +- drivers/staging/fsl-mc/include/mc-cmd.h | 91 +- drivers/staging/fsl-mc/include/mc.h | 21 +- drivers/staging/iio/accel/Kconfig | 14 - drivers/staging/iio/accel/Makefile | 4 - drivers/staging/iio/accel/sca3000_core.c | 4 +- drivers/staging/iio/adc/ad7280a.c | 8 +- drivers/staging/iio/adc/ad7606_ring.c | 3 +- drivers/staging/iio/adc/ad7816.c | 3 +- drivers/staging/iio/addac/adt7316.c | 4 +- drivers/staging/iio/cdc/ad7150.c | 2 +- drivers/staging/iio/impedance-analyzer/ad5933.c | 2 +- drivers/staging/iio/light/tsl2x7x_core.c | 2 +- drivers/staging/ks7010/Kconfig | 10 + drivers/staging/ks7010/Makefile | 4 + drivers/staging/ks7010/TODO | 36 + drivers/staging/ks7010/eap_packet.h | 129 + drivers/staging/ks7010/ks7010_sdio.c | 1236 +++ drivers/staging/ks7010/ks7010_sdio.h | 147 + drivers/staging/ks7010/ks_hostif.c | 2760 +++++ drivers/staging/ks7010/ks_hostif.h | 644 ++ drivers/staging/ks7010/ks_wlan.h | 505 + drivers/staging/ks7010/ks_wlan_ioctl.h | 67 + drivers/staging/ks7010/ks_wlan_net.c | 3528 ++++++ drivers/staging/ks7010/michael_mic.c | 139 + drivers/staging/ks7010/michael_mic.h | 26 + .../staging/lustre/include/linux/libcfs/curproc.h | 6 +- .../staging/lustre/include/linux/libcfs/libcfs.h | 6 +- .../lustre/include/linux/libcfs/libcfs_debug.h | 6 +- .../lustre/include/linux/libcfs/libcfs_fail.h | 4 - .../lustre/include/linux/libcfs/libcfs_hash.h | 6 +- .../lustre/include/linux/libcfs/libcfs_ioctl.h | 6 +- .../lustre/include/linux/libcfs/libcfs_prim.h | 6 +- .../lustre/include/linux/libcfs/libcfs_private.h | 6 +- .../lustre/include/linux/libcfs/libcfs_string.h | 6 +- .../lustre/include/linux/libcfs/libcfs_time.h | 6 +- .../lustre/include/linux/libcfs/libcfs_workitem.h | 6 +- .../lustre/include/linux/libcfs/linux/libcfs.h | 6 +- .../lustre/include/linux/libcfs/linux/linux-time.h | 6 +- .../staging/lustre/include/linux/lnet/lib-dlc.h | 2 +- .../staging/lustre/include/linux/lnet/lib-types.h | 3 +- drivers/staging/lustre/include/linux/lnet/types.h | 4 +- .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c | 354 +- .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h | 307 +- .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 384 +- .../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c | 8 +- .../staging/lustre/lnet/klnds/socklnd/socklnd.c | 308 +- .../staging/lustre/lnet/klnds/socklnd/socklnd.h | 209 +- .../staging/lustre/lnet/klnds/socklnd/socklnd_cb.c | 204 +- .../lustre/lnet/klnds/socklnd/socklnd_lib.c | 40 +- .../lustre/lnet/klnds/socklnd/socklnd_modparams.c | 2 +- .../lustre/lnet/klnds/socklnd/socklnd_proto.c | 74 +- drivers/staging/lustre/lnet/libcfs/debug.c | 8 +- drivers/staging/lustre/lnet/libcfs/fail.c | 4 - drivers/staging/lustre/lnet/libcfs/hash.c | 6 +- drivers/staging/lustre/lnet/libcfs/libcfs_string.c | 6 +- .../lustre/lnet/libcfs/linux/linux-crypto.c | 1 + .../lustre/lnet/libcfs/linux/linux-curproc.c | 6 +- .../staging/lustre/lnet/libcfs/linux/linux-debug.c | 6 +- .../staging/lustre/lnet/libcfs/linux/linux-mem.c | 2 +- .../lustre/lnet/libcfs/linux/linux-module.c | 6 +- .../staging/lustre/lnet/libcfs/linux/linux-prim.c | 6 +- .../lustre/lnet/libcfs/linux/linux-tracefile.c | 6 +- drivers/staging/lustre/lnet/libcfs/module.c | 6 +- drivers/staging/lustre/lnet/libcfs/prng.c | 6 +- drivers/staging/lustre/lnet/libcfs/tracefile.c | 6 +- drivers/staging/lustre/lnet/libcfs/tracefile.h | 6 +- drivers/staging/lustre/lnet/libcfs/workitem.c | 6 +- drivers/staging/lustre/lnet/lnet/acceptor.c | 6 +- drivers/staging/lustre/lnet/lnet/api-ni.c | 8 +- drivers/staging/lustre/lnet/lnet/config.c | 6 +- drivers/staging/lustre/lnet/lnet/lib-eq.c | 6 +- drivers/staging/lustre/lnet/lnet/lib-md.c | 6 +- drivers/staging/lustre/lnet/lnet/lib-me.c | 6 +- drivers/staging/lustre/lnet/lnet/lib-move.c | 6 +- drivers/staging/lustre/lnet/lnet/lib-msg.c | 6 +- drivers/staging/lustre/lnet/lnet/lo.c | 6 +- drivers/staging/lustre/lnet/lnet/module.c | 8 +- drivers/staging/lustre/lnet/lnet/net_fault.c | 4 +- drivers/staging/lustre/lnet/lnet/nidstrings.c | 6 +- drivers/staging/lustre/lnet/lnet/peer.c | 6 +- drivers/staging/lustre/lnet/lnet/router.c | 9 +- drivers/staging/lustre/lnet/selftest/brw_test.c | 6 +- drivers/staging/lustre/lnet/selftest/conctl.c | 6 +- drivers/staging/lustre/lnet/selftest/conrpc.c | 6 +- drivers/staging/lustre/lnet/selftest/conrpc.h | 6 +- drivers/staging/lustre/lnet/selftest/console.c | 6 +- drivers/staging/lustre/lnet/selftest/console.h | 6 +- drivers/staging/lustre/lnet/selftest/framework.c | 6 +- drivers/staging/lustre/lnet/selftest/module.c | 6 +- drivers/staging/lustre/lnet/selftest/ping_test.c | 6 +- drivers/staging/lustre/lnet/selftest/rpc.c | 6 +- drivers/staging/lustre/lnet/selftest/rpc.h | 6 +- drivers/staging/lustre/lnet/selftest/selftest.h | 7 +- drivers/staging/lustre/lnet/selftest/timer.c | 6 +- drivers/staging/lustre/lnet/selftest/timer.h | 6 +- drivers/staging/lustre/lustre/Kconfig | 6 - drivers/staging/lustre/lustre/fid/fid_internal.h | 6 +- drivers/staging/lustre/lustre/fid/fid_lib.c | 6 +- drivers/staging/lustre/lustre/fid/fid_request.c | 12 +- drivers/staging/lustre/lustre/fid/lproc_fid.c | 6 +- drivers/staging/lustre/lustre/fld/fld_cache.c | 6 +- drivers/staging/lustre/lustre/fld/fld_internal.h | 6 +- drivers/staging/lustre/lustre/fld/fld_request.c | 6 +- drivers/staging/lustre/lustre/fld/lproc_fld.c | 6 +- drivers/staging/lustre/lustre/include/cl_object.h | 16 +- .../staging/lustre/lustre/include/interval_tree.h | 6 +- .../lustre/lustre/include/linux/lustre_compat25.h | 6 +- .../lustre/lustre/include/linux/lustre_lite.h | 6 +- .../lustre/include/linux/lustre_patchless_compat.h | 6 +- .../lustre/lustre/include/linux/lustre_user.h | 6 +- .../staging/lustre/lustre/include/lprocfs_status.h | 6 +- drivers/staging/lustre/lustre/include/lu_object.h | 8 +- .../lustre/lustre/include/lustre/ll_fiemap.h | 6 +- .../lustre/lustre/include/lustre/lustre_idl.h | 50 +- .../lustre/lustre/include/lustre/lustre_user.h | 21 +- drivers/staging/lustre/lustre/include/lustre_acl.h | 6 +- drivers/staging/lustre/lustre/include/lustre_cfg.h | 6 +- .../staging/lustre/lustre/include/lustre_debug.h | 6 +- .../staging/lustre/lustre/include/lustre_disk.h | 6 +- drivers/staging/lustre/lustre/include/lustre_dlm.h | 24 +- .../staging/lustre/lustre/include/lustre_eacl.h | 17 +- .../staging/lustre/lustre/include/lustre_export.h | 19 +- drivers/staging/lustre/lustre/include/lustre_fid.h | 8 +- drivers/staging/lustre/lustre/include/lustre_fld.h | 6 +- drivers/staging/lustre/lustre/include/lustre_ha.h | 6 +- .../staging/lustre/lustre/include/lustre_handles.h | 6 +- .../staging/lustre/lustre/include/lustre_import.h | 6 +- .../staging/lustre/lustre/include/lustre_intent.h | 36 +- drivers/staging/lustre/lustre/include/lustre_lib.h | 6 +- .../staging/lustre/lustre/include/lustre_lite.h | 6 +- drivers/staging/lustre/lustre/include/lustre_log.h | 6 +- drivers/staging/lustre/lustre/include/lustre_mdc.h | 9 +- drivers/staging/lustre/lustre/include/lustre_mds.h | 6 +- drivers/staging/lustre/lustre/include/lustre_net.h | 421 +- .../staging/lustre/lustre/include/lustre_param.h | 6 +- .../lustre/lustre/include/lustre_req_layout.h | 8 +- drivers/staging/lustre/lustre/include/lustre_sec.h | 12 +- drivers/staging/lustre/lustre/include/obd.h | 17 +- drivers/staging/lustre/lustre/include/obd_cksum.h | 6 +- drivers/staging/lustre/lustre/include/obd_class.h | 16 +- .../staging/lustre/lustre/include/obd_support.h | 9 +- drivers/staging/lustre/lustre/ldlm/interval_tree.c | 6 +- drivers/staging/lustre/lustre/ldlm/l_lock.c | 6 +- drivers/staging/lustre/lustre/ldlm/ldlm_extent.c | 6 +- drivers/staging/lustre/lustre/ldlm/ldlm_flock.c | 15 +- .../staging/lustre/lustre/ldlm/ldlm_inodebits.c | 6 +- drivers/staging/lustre/lustre/ldlm/ldlm_internal.h | 6 +- drivers/staging/lustre/lustre/ldlm/ldlm_lib.c | 9 +- drivers/staging/lustre/lustre/ldlm/ldlm_lock.c | 22 +- drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c | 19 +- drivers/staging/lustre/lustre/ldlm/ldlm_plain.c | 6 +- drivers/staging/lustre/lustre/ldlm/ldlm_pool.c | 6 +- drivers/staging/lustre/lustre/ldlm/ldlm_request.c | 12 +- drivers/staging/lustre/lustre/ldlm/ldlm_resource.c | 8 +- drivers/staging/lustre/lustre/llite/Makefile | 6 +- drivers/staging/lustre/lustre/llite/dcache.c | 49 +- drivers/staging/lustre/lustre/llite/dir.c | 55 +- drivers/staging/lustre/lustre/llite/file.c | 97 +- drivers/staging/lustre/lustre/llite/glimpse.c | 6 +- drivers/staging/lustre/lustre/llite/lcommon_cl.c | 6 +- drivers/staging/lustre/lustre/llite/lcommon_misc.c | 14 +- drivers/staging/lustre/lustre/llite/llite_close.c | 6 +- .../staging/lustre/lustre/llite/llite_internal.h | 132 +- drivers/staging/lustre/lustre/llite/llite_lib.c | 125 +- drivers/staging/lustre/lustre/llite/llite_mmap.c | 18 +- drivers/staging/lustre/lustre/llite/llite_nfs.c | 24 +- drivers/staging/lustre/lustre/llite/lproc_llite.c | 18 +- drivers/staging/lustre/lustre/llite/namei.c | 34 +- drivers/staging/lustre/lustre/llite/rw.c | 151 +- drivers/staging/lustre/lustre/llite/rw26.c | 23 +- drivers/staging/lustre/lustre/llite/statahead.c | 28 +- drivers/staging/lustre/lustre/llite/super25.c | 25 +- drivers/staging/lustre/lustre/llite/symlink.c | 6 +- drivers/staging/lustre/lustre/llite/vvp_dev.c | 16 +- drivers/staging/lustre/lustre/llite/vvp_internal.h | 6 +- drivers/staging/lustre/lustre/llite/vvp_io.c | 15 +- drivers/staging/lustre/lustre/llite/vvp_lock.c | 6 +- drivers/staging/lustre/lustre/llite/vvp_object.c | 6 +- drivers/staging/lustre/lustre/llite/vvp_page.c | 6 +- drivers/staging/lustre/lustre/llite/vvp_req.c | 12 +- drivers/staging/lustre/lustre/llite/xattr.c | 103 +- drivers/staging/lustre/lustre/llite/xattr_cache.c | 16 +- drivers/staging/lustre/lustre/lmv/lmv_fld.c | 6 +- drivers/staging/lustre/lustre/lmv/lmv_intent.c | 32 +- drivers/staging/lustre/lustre/lmv/lmv_internal.h | 6 +- drivers/staging/lustre/lustre/lmv/lmv_obd.c | 39 +- drivers/staging/lustre/lustre/lmv/lproc_lmv.c | 6 +- .../staging/lustre/lustre/lov/lov_cl_internal.h | 6 +- drivers/staging/lustre/lustre/lov/lov_dev.c | 6 +- drivers/staging/lustre/lustre/lov/lov_ea.c | 6 +- drivers/staging/lustre/lustre/lov/lov_internal.h | 6 +- drivers/staging/lustre/lustre/lov/lov_io.c | 6 +- drivers/staging/lustre/lustre/lov/lov_lock.c | 6 +- drivers/staging/lustre/lustre/lov/lov_merge.c | 6 +- drivers/staging/lustre/lustre/lov/lov_obd.c | 22 +- drivers/staging/lustre/lustre/lov/lov_object.c | 10 +- drivers/staging/lustre/lustre/lov/lov_offset.c | 8 +- drivers/staging/lustre/lustre/lov/lov_pack.c | 6 +- drivers/staging/lustre/lustre/lov/lov_page.c | 6 +- drivers/staging/lustre/lustre/lov/lov_pool.c | 8 +- drivers/staging/lustre/lustre/lov/lov_request.c | 6 +- drivers/staging/lustre/lustre/lov/lovsub_dev.c | 6 +- drivers/staging/lustre/lustre/lov/lovsub_io.c | 6 +- drivers/staging/lustre/lustre/lov/lovsub_lock.c | 6 +- drivers/staging/lustre/lustre/lov/lovsub_object.c | 6 +- drivers/staging/lustre/lustre/lov/lovsub_page.c | 6 +- drivers/staging/lustre/lustre/lov/lproc_lov.c | 6 +- drivers/staging/lustre/lustre/mdc/lproc_mdc.c | 6 +- drivers/staging/lustre/lustre/mdc/mdc_internal.h | 6 +- drivers/staging/lustre/lustre/mdc/mdc_lib.c | 18 +- drivers/staging/lustre/lustre/mdc/mdc_locks.c | 120 +- drivers/staging/lustre/lustre/mdc/mdc_reint.c | 8 +- drivers/staging/lustre/lustre/mdc/mdc_request.c | 90 +- drivers/staging/lustre/lustre/mgc/lproc_mgc.c | 6 +- drivers/staging/lustre/lustre/mgc/mgc_internal.h | 6 +- drivers/staging/lustre/lustre/mgc/mgc_request.c | 14 +- drivers/staging/lustre/lustre/obdclass/Makefile | 3 +- .../staging/lustre/lustre/obdclass/cl_internal.h | 6 +- drivers/staging/lustre/lustre/obdclass/cl_io.c | 6 +- drivers/staging/lustre/lustre/obdclass/cl_lock.c | 6 +- drivers/staging/lustre/lustre/obdclass/cl_object.c | 12 +- drivers/staging/lustre/lustre/obdclass/cl_page.c | 52 +- drivers/staging/lustre/lustre/obdclass/class_obd.c | 6 +- drivers/staging/lustre/lustre/obdclass/debug.c | 6 +- drivers/staging/lustre/lustre/obdclass/genops.c | 6 +- .../staging/lustre/lustre/obdclass/kernelcomm.c | 6 +- .../lustre/lustre/obdclass/linux/linux-module.c | 6 +- .../lustre/lustre/obdclass/linux/linux-obdo.c | 6 +- .../lustre/lustre/obdclass/linux/linux-sysctl.c | 6 +- drivers/staging/lustre/lustre/obdclass/llog.c | 12 +- drivers/staging/lustre/lustre/obdclass/llog_cat.c | 6 +- .../staging/lustre/lustre/obdclass/llog_internal.h | 6 +- drivers/staging/lustre/lustre/obdclass/llog_obd.c | 6 +- drivers/staging/lustre/lustre/obdclass/llog_swab.c | 6 +- .../lustre/lustre/obdclass/lprocfs_status.c | 6 +- drivers/staging/lustre/lustre/obdclass/lu_object.c | 6 +- drivers/staging/lustre/lustre/obdclass/lu_ref.c | 6 +- .../lustre/lustre/obdclass/lustre_handles.c | 6 +- .../staging/lustre/lustre/obdclass/lustre_peer.c | 6 +- .../staging/lustre/lustre/obdclass/obd_config.c | 12 +- drivers/staging/lustre/lustre/obdclass/obd_mount.c | 8 +- drivers/staging/lustre/lustre/obdclass/obdo.c | 6 +- .../staging/lustre/lustre/obdclass/statfs_pack.c | 6 +- drivers/staging/lustre/lustre/obdclass/uuid.c | 6 +- .../staging/lustre/lustre/obdecho/echo_client.c | 6 +- drivers/staging/lustre/lustre/osc/lproc_osc.c | 6 +- drivers/staging/lustre/lustre/osc/osc_cache.c | 27 +- .../staging/lustre/lustre/osc/osc_cl_internal.h | 13 +- drivers/staging/lustre/lustre/osc/osc_dev.c | 6 +- drivers/staging/lustre/lustre/osc/osc_internal.h | 6 +- drivers/staging/lustre/lustre/osc/osc_io.c | 18 +- drivers/staging/lustre/lustre/osc/osc_lock.c | 27 +- drivers/staging/lustre/lustre/osc/osc_object.c | 6 +- drivers/staging/lustre/lustre/osc/osc_page.c | 36 +- drivers/staging/lustre/lustre/osc/osc_request.c | 24 +- drivers/staging/lustre/lustre/ptlrpc/client.c | 167 +- drivers/staging/lustre/lustre/ptlrpc/connection.c | 6 +- drivers/staging/lustre/lustre/ptlrpc/events.c | 36 +- drivers/staging/lustre/lustre/ptlrpc/import.c | 12 +- drivers/staging/lustre/lustre/ptlrpc/layout.c | 18 +- drivers/staging/lustre/lustre/ptlrpc/llog_client.c | 6 +- drivers/staging/lustre/lustre/ptlrpc/llog_net.c | 6 +- .../staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c | 10 +- drivers/staging/lustre/lustre/ptlrpc/niobuf.c | 26 +- drivers/staging/lustre/lustre/ptlrpc/nrs.c | 2 +- .../staging/lustre/lustre/ptlrpc/pack_generic.c | 19 +- drivers/staging/lustre/lustre/ptlrpc/pers.c | 6 +- drivers/staging/lustre/lustre/ptlrpc/pinger.c | 9 +- .../staging/lustre/lustre/ptlrpc/ptlrpc_internal.h | 49 +- .../staging/lustre/lustre/ptlrpc/ptlrpc_module.c | 6 +- drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c | 10 +- drivers/staging/lustre/lustre/ptlrpc/recover.c | 6 +- drivers/staging/lustre/lustre/ptlrpc/sec.c | 22 +- drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c | 8 +- drivers/staging/lustre/lustre/ptlrpc/sec_config.c | 10 +- drivers/staging/lustre/lustre/ptlrpc/sec_gc.c | 6 +- drivers/staging/lustre/lustre/ptlrpc/sec_lproc.c | 6 +- drivers/staging/lustre/lustre/ptlrpc/sec_null.c | 13 +- drivers/staging/lustre/lustre/ptlrpc/sec_plain.c | 24 +- drivers/staging/lustre/lustre/ptlrpc/service.c | 6 +- drivers/staging/lustre/lustre/ptlrpc/wiretest.c | 54 +- drivers/staging/lustre/sysfs-fs-lustre | 8 + drivers/staging/media/Kconfig | 16 +- drivers/staging/media/Makefile | 8 +- drivers/staging/media/cec/Kconfig | 15 + drivers/staging/media/cec/Makefile | 5 + drivers/staging/media/cec/TODO | 32 + drivers/staging/media/cec/cec-adap.c | 1664 +++ drivers/staging/media/cec/cec-api.c | 579 + drivers/staging/media/cec/cec-core.c | 412 + drivers/staging/media/cec/cec-priv.h | 56 + drivers/staging/media/davinci_vpfe/vpfe_video.c | 14 +- drivers/staging/media/davinci_vpfe/vpfe_video.h | 2 - drivers/staging/media/lirc/lirc_parallel.c | 8 +- drivers/staging/media/omap4iss/iss_video.c | 12 +- drivers/staging/media/omap4iss/iss_video.h | 1 - drivers/staging/media/pulse8-cec/Kconfig | 10 + drivers/staging/media/pulse8-cec/Makefile | 1 + drivers/staging/media/pulse8-cec/TODO | 52 + drivers/staging/media/pulse8-cec/pulse8-cec.c | 505 + drivers/staging/media/s5p-cec/Kconfig | 9 + drivers/staging/media/s5p-cec/Makefile | 2 + drivers/staging/media/s5p-cec/TODO | 7 + drivers/staging/media/s5p-cec/exynos_hdmi_cec.h | 38 + .../staging/media/s5p-cec/exynos_hdmi_cecctrl.c | 209 + drivers/staging/media/s5p-cec/regs-cec.h | 96 + drivers/staging/media/s5p-cec/s5p_cec.c | 294 + drivers/staging/media/s5p-cec/s5p_cec.h | 76 + drivers/staging/media/tw686x-kh/tw686x-kh-video.c | 12 +- drivers/staging/media/tw686x-kh/tw686x-kh.h | 1 - drivers/staging/rtl8192u/ieee80211/ieee80211.h | 4 +- .../staging/rtl8192u/ieee80211/ieee80211_softmac.c | 54 +- .../rtl8192u/ieee80211/ieee80211_softmac_wx.c | 34 +- drivers/staging/rtl8192u/ieee80211/ieee80211_wx.c | 6 +- drivers/staging/rtl8192u/r8180_93cx6.c | 30 +- drivers/staging/rtl8192u/r8180_93cx6.h | 2 +- drivers/staging/rtl8192u/r8192U.h | 11 +- drivers/staging/rtl8192u/r8192U_core.c | 198 +- drivers/staging/rtl8192u/r8192U_wx.c | 80 +- drivers/staging/rtl8723au/os_dep/ioctl_cfg80211.c | 11 +- drivers/staging/unisys/visorbus/iovmcall_gnuc.h | 4 +- drivers/staging/unisys/visorbus/visorbus_main.c | 12 +- drivers/staging/unisys/visorbus/visorchipset.c | 2 +- drivers/staging/unisys/visorhba/visorhba_main.c | 394 +- drivers/staging/unisys/visorinput/visorinput.c | 2 +- drivers/staging/unisys/visornic/visornic_main.c | 73 +- drivers/staging/wilc1000/Makefile | 1 - drivers/staging/wilc1000/TODO | 5 + drivers/staging/wilc1000/host_interface.c | 477 +- drivers/staging/wilc1000/host_interface.h | 4 - drivers/staging/wilc1000/linux_wlan.c | 37 +- drivers/staging/wilc1000/wilc_sdio.c | 2 +- drivers/staging/wilc1000/wilc_spi.c | 2 +- drivers/staging/wilc1000/wilc_wfi_cfgoperations.c | 19 +- drivers/staging/wilc1000/wilc_wfi_netdevice.h | 10 +- drivers/staging/wilc1000/wilc_wlan.c | 34 +- drivers/staging/wlan-ng/cfg80211.c | 5 +- drivers/target/iscsi/cxgbit/Kconfig | 2 +- drivers/target/iscsi/cxgbit/Makefile | 1 + drivers/target/iscsi/cxgbit/cxgbit.h | 2 +- drivers/target/iscsi/cxgbit/cxgbit_main.c | 2 + drivers/target/target_core_file.c | 2 +- drivers/target/target_core_iblock.c | 41 +- drivers/target/target_core_pscsi.c | 89 +- drivers/target/target_core_transport.c | 16 +- drivers/target/tcm_fc/tfc_sess.c | 2 + drivers/thermal/clock_cooling.c | 1 + drivers/thermal/cpu_cooling.c | 45 +- drivers/thermal/fair_share.c | 2 + drivers/thermal/gov_bang_bang.c | 2 + drivers/thermal/imx_thermal.c | 4 +- drivers/thermal/int340x_thermal/int3406_thermal.c | 1 - drivers/thermal/intel_pch_thermal.c | 60 +- drivers/thermal/intel_powerclamp.c | 11 +- drivers/thermal/intel_soc_dts_thermal.c | 4 +- drivers/thermal/power_allocator.c | 2 + drivers/thermal/rcar_thermal.c | 1 + drivers/thermal/step_wise.c | 2 + drivers/thermal/thermal_core.c | 10 +- drivers/thermal/thermal_hwmon.c | 2 + drivers/thunderbolt/nhi.c | 6 + drivers/thunderbolt/switch.c | 4 +- drivers/tty/cyclades.c | 4 - drivers/tty/hvc/hvc_console.h | 1 + drivers/tty/hvc/hvc_irq.c | 9 +- drivers/tty/hvc/hvc_opal.c | 11 +- drivers/tty/ipwireless/tty.c | 11 +- drivers/tty/metag_da.c | 4 +- drivers/tty/mips_ejtag_fdc.c | 4 +- drivers/tty/mxser.c | 1 - drivers/tty/serial/8250/8250.h | 18 +- drivers/tty/serial/8250/8250_core.c | 10 +- drivers/tty/serial/8250/8250_dma.c | 1 + drivers/tty/serial/8250/8250_early.c | 1 + drivers/tty/serial/8250/8250_fintek.c | 36 +- drivers/tty/serial/8250/8250_ingenic.c | 2 - drivers/tty/serial/8250/8250_mid.c | 24 +- drivers/tty/serial/8250/8250_mtk.c | 2 +- drivers/tty/serial/8250/8250_omap.c | 4 +- drivers/tty/serial/8250/8250_pci.c | 28 +- drivers/tty/serial/8250/8250_port.c | 64 +- drivers/tty/serial/8250/8250_uniphier.c | 2 +- drivers/tty/serial/8250/Kconfig | 5 +- drivers/tty/serial/Kconfig | 3 +- drivers/tty/serial/amba-pl011.c | 8 +- drivers/tty/serial/ar933x_uart.c | 4 +- drivers/tty/serial/atmel_serial.c | 129 +- drivers/tty/serial/bcm63xx_uart.c | 8 +- drivers/tty/serial/clps711x.c | 2 +- drivers/tty/serial/fsl_lpuart.c | 8 +- drivers/tty/serial/m32r_sio.c | 23 +- drivers/tty/serial/max310x.c | 176 +- drivers/tty/serial/mps2-uart.c | 29 +- drivers/tty/serial/msm_serial.c | 195 +- drivers/tty/serial/pic32_uart.c | 1 - drivers/tty/serial/pmac_zilog.c | 2 +- drivers/tty/serial/pxa.c | 30 +- drivers/tty/serial/samsung.c | 20 +- drivers/tty/serial/samsung.h | 38 +- drivers/tty/serial/serial-tegra.c | 7 +- drivers/tty/serial/serial_core.c | 8 +- drivers/tty/serial/serial_mctrl_gpio.c | 36 + drivers/tty/serial/serial_mctrl_gpio.h | 15 +- drivers/tty/serial/sh-sci.c | 196 +- drivers/tty/serial/sh-sci.h | 25 +- drivers/tty/serial/sirfsoc_uart.h | 4 +- drivers/tty/serial/sunhv.c | 6 - drivers/tty/serial/vt8500_serial.c | 30 +- drivers/tty/serial/xilinx_uartps.c | 30 +- drivers/tty/sysrq.c | 1 + drivers/tty/vt/consolemap.c | 13 +- drivers/tty/vt/keyboard.c | 16 +- drivers/tty/vt/vt.c | 431 +- drivers/tty/vt/vt_ioctl.c | 8 +- drivers/usb/chipidea/Kconfig | 5 +- drivers/usb/class/cdc-acm.c | 121 +- drivers/usb/class/cdc-wdm.c | 30 +- drivers/usb/class/usbtmc.c | 3 +- drivers/usb/common/common.c | 26 +- drivers/usb/core/devio.c | 4 +- drivers/usb/core/message.c | 153 + drivers/usb/dwc2/Kconfig | 1 + drivers/usb/dwc2/core.h | 9 +- drivers/usb/dwc2/gadget.c | 574 +- drivers/usb/dwc2/hcd_queue.c | 3 +- drivers/usb/dwc2/hw.h | 14 + drivers/usb/dwc2/platform.c | 22 + drivers/usb/dwc3/core.c | 389 +- drivers/usb/dwc3/core.h | 59 +- drivers/usb/dwc3/debug.h | 140 +- drivers/usb/dwc3/debugfs.c | 191 +- drivers/usb/dwc3/dwc3-of-simple.c | 1 + drivers/usb/dwc3/dwc3-omap.c | 53 +- drivers/usb/dwc3/dwc3-pci.c | 170 +- drivers/usb/dwc3/ep0.c | 26 +- drivers/usb/dwc3/gadget.c | 869 +- drivers/usb/dwc3/gadget.h | 4 +- drivers/usb/dwc3/host.c | 61 +- drivers/usb/dwc3/io.h | 7 +- drivers/usb/dwc3/trace.h | 96 +- drivers/usb/early/ehci-dbgp.c | 4 +- drivers/usb/gadget/Kconfig | 2 +- drivers/usb/gadget/composite.c | 6 +- drivers/usb/gadget/config.c | 2 +- drivers/usb/gadget/configfs.c | 2 + drivers/usb/gadget/function/f_eem.c | 12 +- drivers/usb/gadget/function/f_fs.c | 192 +- drivers/usb/gadget/function/f_mass_storage.c | 22 +- drivers/usb/gadget/function/f_rndis.c | 3 + drivers/usb/gadget/function/rndis.c | 6 + drivers/usb/gadget/function/u_ether.c | 3 +- drivers/usb/gadget/function/u_serial.c | 10 +- drivers/usb/gadget/function/uvc_configfs.c | 2 +- drivers/usb/gadget/function/uvc_queue.c | 2 +- drivers/usb/gadget/legacy/g_ffs.c | 15 +- drivers/usb/gadget/legacy/inode.c | 2 +- drivers/usb/gadget/udc/Kconfig | 4 +- drivers/usb/gadget/udc/Makefile | 5 + drivers/usb/gadget/udc/amd5536udc.c | 9 +- drivers/usb/gadget/udc/bdc/bdc_cmd.c | 3 - drivers/usb/gadget/udc/bdc/bdc_ep.c | 6 +- drivers/usb/gadget/udc/core.c | 1526 +++ drivers/usb/gadget/udc/dummy_hcd.c | 5 - drivers/usb/gadget/udc/fsl_qe_udc.c | 7 +- drivers/usb/gadget/udc/m66592-udc.c | 24 +- drivers/usb/gadget/udc/mv_u3d_core.c | 23 +- drivers/usb/gadget/udc/mv_udc_core.c | 9 +- drivers/usb/gadget/udc/net2272.c | 4 - drivers/usb/gadget/udc/net2280.c | 51 +- drivers/usb/gadget/udc/net2280.h | 1 + drivers/usb/gadget/udc/pch_udc.c | 20 +- drivers/usb/gadget/udc/pxa27x_udc.c | 9 +- drivers/usb/gadget/udc/r8a66597-udc.c | 24 +- drivers/usb/gadget/udc/trace.c | 18 + drivers/usb/gadget/udc/trace.h | 298 + drivers/usb/gadget/udc/udc-xilinx.c | 3 - drivers/usb/host/Kconfig | 2 +- drivers/usb/host/ehci-platform.c | 37 +- drivers/usb/host/max3421-hcd.c | 2 +- drivers/usb/host/ohci-hcd.c | 1 - drivers/usb/host/ohci-platform.c | 43 +- drivers/usb/host/xhci-mem.c | 74 +- drivers/usb/host/xhci-pci.c | 2 +- drivers/usb/host/xhci-plat.c | 8 +- drivers/usb/host/xhci-ring.c | 459 +- drivers/usb/host/xhci.c | 7 +- drivers/usb/host/xhci.h | 10 +- drivers/usb/image/microtek.h | 6 +- drivers/usb/misc/Kconfig | 20 +- drivers/usb/misc/Makefile | 1 - drivers/usb/misc/chaoskey.c | 21 +- drivers/usb/misc/ftdi-elan.c | 10 +- drivers/usb/misc/legousbtower.c | 35 +- drivers/usb/misc/sisusbvga/sisusb.c | 28 +- drivers/usb/misc/sisusbvga/sisusb_con.c | 87 +- drivers/usb/misc/sisusbvga/sisusb_init.h | 2 +- drivers/usb/misc/usb3503.c | 25 + drivers/usb/musb/Kconfig | 2 +- drivers/usb/musb/Makefile | 5 +- drivers/usb/musb/cppi_dma.c | 51 +- drivers/usb/musb/cppi_dma.h | 31 +- drivers/usb/musb/musb_core.c | 87 +- drivers/usb/musb/musb_cppi41.c | 47 +- drivers/usb/musb/musb_debug.h | 2 + drivers/usb/musb/musb_dsps.c | 112 +- drivers/usb/musb/musb_gadget.c | 122 +- drivers/usb/musb/musb_gadget_ep0.c | 22 +- drivers/usb/musb/musb_host.c | 133 +- drivers/usb/musb/musb_trace.c | 33 + drivers/usb/musb/musb_trace.h | 371 + drivers/usb/musb/musb_virthub.c | 31 +- drivers/usb/musb/musbhsdma.c | 10 +- drivers/usb/musb/sunxi.c | 74 +- drivers/usb/phy/Kconfig | 11 +- drivers/usb/phy/phy-am335x.c | 2 +- drivers/usb/phy/phy-generic.c | 8 +- drivers/usb/phy/phy-msm-usb.c | 178 +- drivers/usb/phy/phy-omap-otg.c | 4 +- drivers/usb/renesas_usbhs/common.c | 2 +- drivers/usb/renesas_usbhs/rcar3.c | 2 +- drivers/usb/serial/cp210x.c | 5 +- drivers/usb/serial/generic.c | 18 +- drivers/usb/serial/ti_usb_3410_5052.c | 271 +- drivers/usb/storage/usb.c | 6 +- drivers/usb/usbip/usbip_common.h | 2 +- drivers/usb/usbip/vudc_rx.c | 2 +- drivers/usb/usbip/vudc_sysfs.c | 2 +- drivers/vfio/pci/vfio_pci.c | 88 +- drivers/vfio/pci/vfio_pci_private.h | 8 + drivers/vfio/platform/vfio_amba.c | 1 + drivers/vfio/platform/vfio_platform.c | 5 + drivers/vfio/platform/vfio_platform_common.c | 198 +- drivers/vfio/platform/vfio_platform_private.h | 9 +- drivers/vfio/vfio.c | 2 +- drivers/vhost/Kconfig | 18 +- drivers/vhost/Kconfig.vringh | 5 + drivers/vhost/Makefile | 4 + drivers/vhost/net.c | 147 +- drivers/vhost/test.c | 8 +- drivers/vhost/vhost.c | 927 +- drivers/vhost/vhost.h | 64 +- drivers/vhost/vsock.c | 723 ++ drivers/video/backlight/lp855x_bl.c | 29 + drivers/video/console/dummycon.c | 3 - drivers/video/console/fbcon.c | 53 +- drivers/video/console/mdacon.c | 45 - drivers/video/console/newport_con.c | 42 - drivers/video/console/sticon.c | 29 - drivers/video/console/vgacon.c | 29 +- drivers/video/fbdev/bfin_adv7393fb.c | 2 + drivers/video/fbdev/bfin_adv7393fb.h | 2 - drivers/video/fbdev/clps711x-fb.c | 4 +- drivers/video/fbdev/core/fbmon.c | 1 - .../omap2/omapfb/displays/connector-analog-tv.c | 10 +- .../fbdev/omap2/omapfb/displays/connector-dvi.c | 60 +- .../fbdev/omap2/omapfb/displays/connector-hdmi.c | 44 +- .../fbdev/omap2/omapfb/displays/encoder-opa362.c | 2 +- .../fbdev/omap2/omapfb/displays/encoder-tfp410.c | 46 +- .../omap2/omapfb/displays/encoder-tpd12s015.c | 3 +- .../video/fbdev/omap2/omapfb/displays/panel-dpi.c | 2 +- .../fbdev/omap2/omapfb/displays/panel-dsi-cm.c | 54 +- .../omapfb/displays/panel-lgphilips-lb035q02.c | 58 +- .../omap2/omapfb/displays/panel-nec-nl8048hl11.c | 47 +- .../omapfb/displays/panel-sharp-ls037v7dw01.c | 83 +- .../omap2/omapfb/displays/panel-sony-acx565akm.c | 2 +- .../omap2/omapfb/displays/panel-tpo-td028ttec1.c | 45 +- .../omap2/omapfb/displays/panel-tpo-td043mtea1.c | 46 +- drivers/video/fbdev/omap2/omapfb/dss/apply.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/core.c | 4 +- .../video/fbdev/omap2/omapfb/dss/dispc-compat.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/dispc.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/dispc_coefs.c | 2 +- .../video/fbdev/omap2/omapfb/dss/display-sysfs.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/display.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/dpi.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/dsi.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/dss-of.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/dss.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/dss.h | 11 + .../video/fbdev/omap2/omapfb/dss/dss_features.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/hdmi.h | 3 +- drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/hdmi_common.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/hdmi_phy.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/hdmi_pll.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/hdmi_wp.c | 2 +- .../video/fbdev/omap2/omapfb/dss/manager-sysfs.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/manager.c | 3 +- drivers/video/fbdev/omap2/omapfb/dss/output.c | 2 +- .../video/fbdev/omap2/omapfb/dss/overlay-sysfs.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/overlay.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/pll.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/rfbi.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/sdi.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/venc.c | 2 +- drivers/video/fbdev/omap2/omapfb/dss/video-pll.c | 2 +- drivers/video/fbdev/omap2/omapfb/omapfb-ioctl.c | 2 +- drivers/video/fbdev/omap2/omapfb/omapfb-main.c | 14 +- drivers/video/fbdev/omap2/omapfb/omapfb-sysfs.c | 2 +- drivers/video/fbdev/omap2/omapfb/omapfb.h | 5 +- drivers/video/logo/logo.c | 4 +- drivers/virtio/virtio_balloon.c | 52 +- drivers/virtio/virtio_ring.c | 18 +- drivers/w1/slaves/w1_ds2406.c | 14 +- drivers/w1/slaves/w1_ds2408.c | 14 +- drivers/w1/slaves/w1_ds2413.c | 14 +- drivers/w1/slaves/w1_ds2423.c | 14 +- drivers/w1/slaves/w1_ds2431.c | 14 +- drivers/w1/slaves/w1_ds2433.c | 14 +- drivers/w1/slaves/w1_ds2760.c | 43 +- drivers/w1/slaves/w1_ds2780.c | 39 +- drivers/w1/slaves/w1_ds2781.c | 40 +- drivers/w1/slaves/w1_ds28e04.c | 14 +- drivers/w1/w1_family.h | 12 + drivers/watchdog/Kconfig | 33 +- drivers/watchdog/Makefile | 3 + drivers/watchdog/aspeed_wdt.c | 212 + drivers/watchdog/bcm2835_wdt.c | 11 +- drivers/watchdog/da9063_wdt.c | 2 + drivers/watchdog/f71808e_wdt.c | 28 +- drivers/watchdog/gpio_wdt.c | 2 + drivers/watchdog/iTCO_wdt.c | 2 + drivers/watchdog/max77620_wdt.c | 227 + drivers/watchdog/meson_gxbb_wdt.c | 270 + drivers/watchdog/nv_tco.c | 2 + drivers/watchdog/pcwd.c | 14 +- drivers/watchdog/pic32-dmt.c | 5 +- drivers/watchdog/pic32-wdt.c | 9 +- drivers/watchdog/qcom-wdt.c | 69 +- drivers/watchdog/sbsa_gwdt.c | 16 +- drivers/watchdog/sirfsoc_wdt.c | 15 +- drivers/watchdog/softdog.c | 92 +- drivers/watchdog/tangox_wdt.c | 4 +- drivers/watchdog/watchdog_core.c | 39 +- drivers/watchdog/watchdog_dev.c | 61 +- drivers/watchdog/ziirave_wdt.c | 2 +- drivers/xen/Kconfig | 2 +- drivers/xen/Makefile | 1 + drivers/xen/arm-device.c | 196 + drivers/xen/efi.c | 173 +- drivers/xen/events/events_base.c | 13 +- drivers/xen/events/events_fifo.c | 2 +- drivers/xen/evtchn.c | 43 +- drivers/xen/gntalloc.c | 2 +- drivers/xen/gntdev.c | 2 +- drivers/xen/privcmd.c | 2 +- drivers/xen/swiotlb-xen.c | 14 +- drivers/xen/time.c | 50 +- drivers/xen/xen-pciback/conf_space.c | 22 +- drivers/xen/xen-pciback/conf_space_header.c | 57 +- drivers/xen/xen-pciback/pciback.h | 1 - drivers/xen/xen-pciback/pciback_ops.c | 2 +- drivers/xen/xen-pciback/xenbus.c | 10 +- drivers/xen/xen-selfballoon.c | 4 +- drivers/xen/xenbus/xenbus_probe_frontend.c | 15 +- drivers/xen/xlate_mmu.c | 77 + 4533 files changed, 307275 insertions(+), 88961 deletions(-) create mode 100644 drivers/acpi/acpi_configfs.c create mode 100644 drivers/acpi/apei/bert.c create mode 100644 drivers/acpi/dptf/Kconfig create mode 100644 drivers/acpi/dptf/Makefile create mode 100644 drivers/acpi/dptf/dptf_power.c create mode 100644 drivers/acpi/dptf/int340x_thermal.c create mode 100644 drivers/acpi/nfit/Kconfig create mode 100644 drivers/acpi/nfit/Makefile create mode 100644 drivers/acpi/nfit/core.c create mode 100644 drivers/acpi/nfit/mce.c create mode 100644 drivers/acpi/nfit/nfit.h create mode 100644 drivers/acpi/pci_mcfg.c create mode 100644 drivers/acpi/pmic/intel_pmic_bxtwc.c create mode 100644 drivers/ata/ahci_brcm.c create mode 100644 drivers/bus/tegra-aconnect.c create mode 100644 drivers/char/hw_random/meson-rng.c create mode 100644 drivers/char/powernv-op-panel.c create mode 100644 drivers/char/tpm/tpm_tis_core.c create mode 100644 drivers/char/tpm/tpm_tis_core.h create mode 100644 drivers/char/tpm/tpm_tis_spi.c create mode 100644 drivers/char/tpm/tpm_vtpm_proxy.c create mode 100644 drivers/clk/meson/Kconfig create mode 100644 drivers/clk/meson/clk-mpll.c create mode 100644 drivers/clk/meson/gxbb.c create mode 100644 drivers/clk/meson/gxbb.h create mode 100644 drivers/clk/renesas/r8a7796-cpg-mssr.c create mode 100644 drivers/clk/renesas/rcar-gen3-cpg.c create mode 100644 drivers/clk/renesas/rcar-gen3-cpg.h create mode 100644 drivers/clk/sunxi-ng/Kconfig create mode 100644 drivers/clk/sunxi-ng/Makefile create mode 100644 drivers/clk/sunxi-ng/ccu-sun8i-h3.c create mode 100644 drivers/clk/sunxi-ng/ccu-sun8i-h3.h create mode 100644 drivers/clk/sunxi-ng/ccu_common.c create mode 100644 drivers/clk/sunxi-ng/ccu_common.h create mode 100644 drivers/clk/sunxi-ng/ccu_div.c create mode 100644 drivers/clk/sunxi-ng/ccu_div.h create mode 100644 drivers/clk/sunxi-ng/ccu_frac.c create mode 100644 drivers/clk/sunxi-ng/ccu_frac.h create mode 100644 drivers/clk/sunxi-ng/ccu_gate.c create mode 100644 drivers/clk/sunxi-ng/ccu_gate.h create mode 100644 drivers/clk/sunxi-ng/ccu_mp.c create mode 100644 drivers/clk/sunxi-ng/ccu_mp.h create mode 100644 drivers/clk/sunxi-ng/ccu_mult.h create mode 100644 drivers/clk/sunxi-ng/ccu_mux.c create mode 100644 drivers/clk/sunxi-ng/ccu_mux.h create mode 100644 drivers/clk/sunxi-ng/ccu_nk.c create mode 100644 drivers/clk/sunxi-ng/ccu_nk.h create mode 100644 drivers/clk/sunxi-ng/ccu_nkm.c create mode 100644 drivers/clk/sunxi-ng/ccu_nkm.h create mode 100644 drivers/clk/sunxi-ng/ccu_nkmp.c create mode 100644 drivers/clk/sunxi-ng/ccu_nkmp.h create mode 100644 drivers/clk/sunxi-ng/ccu_nm.c create mode 100644 drivers/clk/sunxi-ng/ccu_nm.h create mode 100644 drivers/clk/sunxi-ng/ccu_phase.c create mode 100644 drivers/clk/sunxi-ng/ccu_phase.h create mode 100644 drivers/clk/sunxi-ng/ccu_reset.c create mode 100644 drivers/clk/sunxi-ng/ccu_reset.h create mode 100644 drivers/clocksource/timer-oxnas-rps.c create mode 100644 drivers/crypto/caam/caampkc.c create mode 100644 drivers/crypto/caam/caampkc.h create mode 100644 drivers/crypto/caam/pkc_desc.c create mode 100644 drivers/crypto/vmx/.gitignore create mode 100644 drivers/crypto/vmx/aes_xts.c create mode 100644 drivers/dma-buf/fence-array.c create mode 100644 drivers/dma/mv_xor_v2.c create mode 100644 drivers/dma/xilinx/xilinx_dma.c create mode 100644 drivers/dma/xilinx/zynqmp_dma.c create mode 100644 drivers/edac/skx_edac.c create mode 100644 drivers/extcon/devres.c create mode 100644 drivers/firmware/scpi_pm_domain.c create mode 100644 drivers/gpio/gpio-max77620.c create mode 100644 drivers/gpio/gpio-merrifield.c create mode 100644 drivers/gpu/drm/amd/amdgpu/iceland_smum.h create mode 100644 drivers/gpu/drm/arc/arcpgu_sim.c create mode 100644 drivers/gpu/drm/arm/malidp_crtc.c create mode 100644 drivers/gpu/drm/arm/malidp_drv.c create mode 100644 drivers/gpu/drm/arm/malidp_drv.h create mode 100644 drivers/gpu/drm/arm/malidp_hw.c create mode 100644 drivers/gpu/drm/arm/malidp_hw.h create mode 100644 drivers/gpu/drm/arm/malidp_planes.c create mode 100644 drivers/gpu/drm/arm/malidp_regs.h create mode 100644 drivers/gpu/drm/bridge/adv7511/Kconfig create mode 100644 drivers/gpu/drm/bridge/adv7511/Makefile create mode 100644 drivers/gpu/drm/bridge/adv7511/adv7511.h create mode 100644 drivers/gpu/drm/bridge/adv7511/adv7511_drv.c create mode 100644 drivers/gpu/drm/bridge/adv7511/adv7533.c create mode 100644 drivers/gpu/drm/bridge/sii902x.c create mode 100644 drivers/gpu/drm/bridge/tc358767.c create mode 100644 drivers/gpu/drm/drm_blend.c create mode 100644 drivers/gpu/drm/drm_fourcc.c create mode 100644 drivers/gpu/drm/drm_simple_kms_helper.c create mode 100644 drivers/gpu/drm/i915/gvt/Makefile create mode 100644 drivers/gpu/drm/i915/gvt/debug.h create mode 100644 drivers/gpu/drm/i915/gvt/gvt.c create mode 100644 drivers/gpu/drm/i915/gvt/gvt.h create mode 100644 drivers/gpu/drm/i915/gvt/hypercall.h create mode 100644 drivers/gpu/drm/i915/gvt/mpt.h create mode 100644 drivers/gpu/drm/i915/i915_gem_dmabuf.h create mode 100644 drivers/gpu/drm/i915/i915_pci.c create mode 100644 drivers/gpu/drm/i915/i915_pvinfo.h create mode 100644 drivers/gpu/drm/i915/intel_breadcrumbs.c create mode 100644 drivers/gpu/drm/i915/intel_device_info.c create mode 100644 drivers/gpu/drm/i915/intel_dp_aux_backlight.c create mode 100644 drivers/gpu/drm/i915/intel_dpio_phy.c create mode 100644 drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c create mode 100644 drivers/gpu/drm/i915/intel_gvt.c create mode 100644 drivers/gpu/drm/i915/intel_gvt.h create mode 100644 drivers/gpu/drm/mediatek/mtk_cec.c create mode 100644 drivers/gpu/drm/mediatek/mtk_cec.h create mode 100644 drivers/gpu/drm/mediatek/mtk_hdmi.c create mode 100644 drivers/gpu/drm/mediatek/mtk_hdmi.h create mode 100644 drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c create mode 100644 drivers/gpu/drm/mediatek/mtk_hdmi_regs.h create mode 100644 drivers/gpu/drm/mediatek/mtk_mt8173_hdmi_phy.c create mode 100644 drivers/gpu/drm/msm/mdp/mdp5/mdp5_mdss.c create mode 100644 drivers/gpu/drm/msm/msm_gem_shrinker.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/ce/gp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/ce/gp104.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp104.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp104.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp104.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/gp104.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp104.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp104.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp104.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/pci/gp100.c create mode 100644 drivers/gpu/drm/vgem/vgem_fence.c create mode 100644 drivers/hid/hid-alps.c create mode 100644 drivers/hid/hid-led.c create mode 100644 drivers/hsi/hsi_core.c create mode 100644 drivers/hwmon/ftsteutates.c create mode 100644 drivers/hwmon/ina3221.c create mode 100644 drivers/hwmon/sht3x.c create mode 100644 drivers/iio/accel/bma220_spi.c create mode 100644 drivers/iio/accel/mma7660.c create mode 100644 drivers/iio/adc/bcm_iproc_adc.c create mode 100644 drivers/iio/industrialio-sw-device.c create mode 100644 drivers/iio/potentiometer/max5487.c create mode 100644 drivers/iio/pressure/bmp280-core.c create mode 100644 drivers/iio/pressure/bmp280-i2c.c create mode 100644 drivers/iio/pressure/bmp280-regmap.c create mode 100644 drivers/iio/pressure/bmp280-spi.c create mode 100644 drivers/iio/pressure/bmp280.h create mode 100644 drivers/iio/trigger/iio-trig-loop.c create mode 100644 drivers/infiniband/hw/hfi1/trace_ctxts.h create mode 100644 drivers/infiniband/hw/hfi1/trace_dbg.h create mode 100644 drivers/infiniband/hw/hfi1/trace_ibhdrs.h create mode 100644 drivers/infiniband/hw/hfi1/trace_misc.h create mode 100644 drivers/infiniband/hw/hfi1/trace_rc.h create mode 100644 drivers/infiniband/hw/hfi1/trace_rx.h create mode 100644 drivers/infiniband/hw/hfi1/trace_tx.h create mode 100644 drivers/infiniband/sw/rxe/Kconfig create mode 100644 drivers/infiniband/sw/rxe/Makefile create mode 100644 drivers/infiniband/sw/rxe/rxe.c create mode 100644 drivers/infiniband/sw/rxe/rxe.h create mode 100644 drivers/infiniband/sw/rxe/rxe_av.c create mode 100644 drivers/infiniband/sw/rxe/rxe_comp.c create mode 100644 drivers/infiniband/sw/rxe/rxe_cq.c create mode 100644 drivers/infiniband/sw/rxe/rxe_dma.c create mode 100644 drivers/infiniband/sw/rxe/rxe_hdr.h create mode 100644 drivers/infiniband/sw/rxe/rxe_icrc.c create mode 100644 drivers/infiniband/sw/rxe/rxe_loc.h create mode 100644 drivers/infiniband/sw/rxe/rxe_mcast.c create mode 100644 drivers/infiniband/sw/rxe/rxe_mmap.c create mode 100644 drivers/infiniband/sw/rxe/rxe_mr.c create mode 100644 drivers/infiniband/sw/rxe/rxe_net.c create mode 100644 drivers/infiniband/sw/rxe/rxe_net.h create mode 100644 drivers/infiniband/sw/rxe/rxe_opcode.c create mode 100644 drivers/infiniband/sw/rxe/rxe_opcode.h create mode 100644 drivers/infiniband/sw/rxe/rxe_param.h create mode 100644 drivers/infiniband/sw/rxe/rxe_pool.c create mode 100644 drivers/infiniband/sw/rxe/rxe_pool.h create mode 100644 drivers/infiniband/sw/rxe/rxe_qp.c create mode 100644 drivers/infiniband/sw/rxe/rxe_queue.c create mode 100644 drivers/infiniband/sw/rxe/rxe_queue.h create mode 100644 drivers/infiniband/sw/rxe/rxe_recv.c create mode 100644 drivers/infiniband/sw/rxe/rxe_req.c create mode 100644 drivers/infiniband/sw/rxe/rxe_resp.c create mode 100644 drivers/infiniband/sw/rxe/rxe_srq.c create mode 100644 drivers/infiniband/sw/rxe/rxe_sysfs.c create mode 100644 drivers/infiniband/sw/rxe/rxe_task.c create mode 100644 drivers/infiniband/sw/rxe/rxe_task.h create mode 100644 drivers/infiniband/sw/rxe/rxe_verbs.c create mode 100644 drivers/infiniband/sw/rxe/rxe_verbs.h create mode 100644 drivers/input/misc/atmel_captouch.c create mode 100644 drivers/input/misc/hisi_powerkey.c create mode 100644 drivers/input/tablet/pegasus_notetaker.c create mode 100644 drivers/input/touchscreen/raydium_i2c_ts.c create mode 100644 drivers/input/touchscreen/silead.c create mode 100644 drivers/input/touchscreen/sis_i2c.c create mode 100644 drivers/input/touchscreen/surface3_spi.c create mode 100644 drivers/iommu/mtk_iommu.h create mode 100644 drivers/iommu/mtk_iommu_v1.c create mode 100644 drivers/irqchip/irq-aspeed-vic.c create mode 100644 drivers/irqchip/irq-gic-pm.c create mode 100644 drivers/leds/leds-lp3952.c create mode 100644 drivers/leds/trigger/ledtrig-disk.c create mode 100644 drivers/mailbox/bcm-pdc-mailbox.c create mode 100644 drivers/md/dm-core.h create mode 100644 drivers/md/dm-rq.c create mode 100644 drivers/md/dm-rq.h create mode 100644 drivers/media/cec-edid.c create mode 100644 drivers/media/dvb-frontends/helene.c create mode 100644 drivers/media/dvb-frontends/helene.h create mode 100644 drivers/media/dvb-frontends/mn88472.c create mode 100644 drivers/media/dvb-frontends/mn88472_priv.h create mode 100644 drivers/media/platform/mtk-vcodec/Makefile create mode 100644 drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h create mode 100644 drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c create mode 100644 drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.h create mode 100644 drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c create mode 100644 drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_pm.c create mode 100644 drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_pm.h create mode 100644 drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.c create mode 100644 drivers/media/platform/mtk-vcodec/mtk_vcodec_intr.h create mode 100644 drivers/media/platform/mtk-vcodec/mtk_vcodec_util.c create mode 100644 drivers/media/platform/mtk-vcodec/mtk_vcodec_util.h create mode 100644 drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c create mode 100644 drivers/media/platform/mtk-vcodec/venc/venc_vp8_if.c create mode 100644 drivers/media/platform/mtk-vcodec/venc_drv_base.h create mode 100644 drivers/media/platform/mtk-vcodec/venc_drv_if.c create mode 100644 drivers/media/platform/mtk-vcodec/venc_drv_if.h create mode 100644 drivers/media/platform/mtk-vcodec/venc_ipi_msg.h create mode 100644 drivers/media/platform/mtk-vcodec/venc_vpu_if.c create mode 100644 drivers/media/platform/mtk-vcodec/venc_vpu_if.h create mode 100644 drivers/media/platform/mtk-vpu/Makefile create mode 100644 drivers/media/platform/mtk-vpu/mtk_vpu.c create mode 100644 drivers/media/platform/mtk-vpu/mtk_vpu.h create mode 100644 drivers/media/platform/rcar-fcp.c create mode 100644 drivers/media/platform/rcar-vin/Kconfig create mode 100644 drivers/media/platform/rcar-vin/Makefile create mode 100644 drivers/media/platform/rcar-vin/rcar-core.c create mode 100644 drivers/media/platform/rcar-vin/rcar-dma.c create mode 100644 drivers/media/platform/rcar-vin/rcar-v4l2.c create mode 100644 drivers/media/platform/rcar-vin/rcar-vin.h create mode 100644 drivers/media/platform/s5p-mfc/s5p_mfc_iommu.h create mode 100644 drivers/media/platform/vivid/vivid-cec.c create mode 100644 drivers/media/platform/vivid/vivid-cec.h create mode 100644 drivers/media/platform/vsp1/vsp1_clu.c create mode 100644 drivers/media/platform/vsp1/vsp1_clu.h create mode 100644 drivers/media/rc/keymaps/rc-cec.c create mode 100644 drivers/media/rc/keymaps/rc-dtt200u.c create mode 100644 drivers/memory/atmel-ebi.c create mode 100644 drivers/mfd/altera-a10sr.c create mode 100644 drivers/misc/cxl/phb.c create mode 100644 drivers/misc/lkdtm.h create mode 100644 drivers/misc/lkdtm_bugs.c create mode 100644 drivers/misc/lkdtm_core.c create mode 100644 drivers/misc/lkdtm_heap.c create mode 100644 drivers/misc/lkdtm_perms.c create mode 100644 drivers/misc/lkdtm_rodata.c create mode 100644 drivers/misc/lkdtm_usercopy.c create mode 100644 drivers/mmc/host/sdhci-brcmstb.c create mode 100644 drivers/mtd/nand/mtk_ecc.c create mode 100644 drivers/mtd/nand/mtk_ecc.h create mode 100644 drivers/mtd/nand/mtk_nand.c create mode 100644 drivers/mtd/spi-nor/atmel-quadspi.c create mode 100644 drivers/mtd/spi-nor/cadence-quadspi.c create mode 100644 drivers/mtd/spi-nor/hisi-sfc.c create mode 100644 drivers/net/can/rcar/Kconfig create mode 100644 drivers/net/can/rcar/Makefile create mode 100644 drivers/net/can/rcar/rcar_can.c create mode 100644 drivers/net/can/rcar/rcar_canfd.c create mode 100644 drivers/net/dsa/b53/Kconfig create mode 100644 drivers/net/dsa/b53/Makefile create mode 100644 drivers/net/dsa/b53/b53_common.c create mode 100644 drivers/net/dsa/b53/b53_mdio.c create mode 100644 drivers/net/dsa/b53/b53_mmap.c create mode 100644 drivers/net/dsa/b53/b53_priv.h create mode 100644 drivers/net/dsa/b53/b53_regs.h create mode 100644 drivers/net/dsa/b53/b53_spi.c create mode 100644 drivers/net/dsa/b53/b53_srab.c create mode 100644 drivers/net/dsa/mv88e6xxx/Kconfig create mode 100644 drivers/net/dsa/mv88e6xxx/Makefile create mode 100644 drivers/net/dsa/mv88e6xxx/chip.c create mode 100644 drivers/net/dsa/mv88e6xxx/mv88e6xxx.h create mode 100644 drivers/net/ethernet/broadcom/bgmac-bcma-mdio.c create mode 100644 drivers/net/ethernet/broadcom/bgmac-bcma.c create mode 100644 drivers/net/ethernet/broadcom/bgmac-platform.c create mode 100644 drivers/net/ethernet/chelsio/libcxgb/Makefile create mode 100644 drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.c create mode 100644 drivers/net/ethernet/chelsio/libcxgb/libcxgb_ppm.h create mode 100644 drivers/net/ethernet/hisilicon/hisi_femac.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_common.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_rep.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/rl.c create mode 100644 drivers/net/ethernet/mellanox/mlxsw/spectrum_kvdl.c create mode 100644 drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c create mode 100644 drivers/net/ethernet/qlogic/qede/qede_dcbnl.c create mode 100644 drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.c create mode 100644 drivers/net/ethernet/stmicro/stmmac/altr_tse_pcs.h create mode 100644 drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.h create mode 100644 drivers/net/phy/intel-xway.c create mode 100644 drivers/net/phy/mdio-hisi-femac.c create mode 100644 drivers/net/phy/mdio-mux-bcm-iproc.c create mode 100644 drivers/net/phy/mdio-xgene.c create mode 100644 drivers/net/phy/mdio-xgene.h create mode 100644 drivers/net/phy/swphy.c create mode 100644 drivers/net/phy/swphy.h create mode 100644 drivers/net/wan/fsl_ucc_hdlc.c create mode 100644 drivers/net/wan/fsl_ucc_hdlc.h create mode 100644 drivers/net/wan/slic_ds26522.c create mode 100644 drivers/net/wan/slic_ds26522.h create mode 100644 drivers/net/wireless/intel/iwlwifi/iwl-a000.c create mode 100644 drivers/nvdimm/pmem.h create mode 100644 drivers/nvme/host/fabrics.c create mode 100644 drivers/nvme/host/fabrics.h create mode 100644 drivers/nvme/host/rdma.c create mode 100644 drivers/nvme/target/Kconfig create mode 100644 drivers/nvme/target/Makefile create mode 100644 drivers/nvme/target/admin-cmd.c create mode 100644 drivers/nvme/target/configfs.c create mode 100644 drivers/nvme/target/core.c create mode 100644 drivers/nvme/target/discovery.c create mode 100644 drivers/nvme/target/fabrics-cmd.c create mode 100644 drivers/nvme/target/io-cmd.c create mode 100644 drivers/nvme/target/loop.c create mode 100644 drivers/nvme/target/nvmet.h create mode 100644 drivers/nvme/target/rdma.c create mode 100644 drivers/pci/host/pci-aardvark.c create mode 100644 drivers/pci/host/pcie-artpec6.c create mode 100644 drivers/pci/hotplug/pnv_php.c create mode 100644 drivers/pci/pci-mid.c create mode 100644 drivers/phy/phy-bcm-ns2-pcie.c create mode 100644 drivers/phy/phy-da8xx-usb.c create mode 100644 drivers/pinctrl/bcm/pinctrl-nsp-mux.c create mode 100644 drivers/pinctrl/intel/pinctrl-merrifield.c create mode 100644 drivers/pinctrl/pinctrl-max77620.c create mode 100644 drivers/pinctrl/pinctrl-oxnas.c create mode 100644 drivers/pinctrl/qcom/pinctrl-mdm9615.c create mode 100644 drivers/pinctrl/stm32/pinctrl-stm32f746.c create mode 100644 drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c create mode 100644 drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c create mode 100644 drivers/platform/x86/intel-vbtn.c create mode 100644 drivers/power/reset/brcm-kona-reset.c create mode 100644 drivers/power/reset/reboot-mode.c create mode 100644 drivers/power/reset/reboot-mode.h create mode 100644 drivers/power/reset/syscon-reboot-mode.c create mode 100644 drivers/pwm/pwm-bcm-iproc.c create mode 100644 drivers/pwm/pwm-cros-ec.c create mode 100644 drivers/pwm/pwm-stmpe.c create mode 100644 drivers/rapidio/rio_cm.c create mode 100644 drivers/rapidio/switches/idt_gen3.c create mode 100644 drivers/regulator/mt6323-regulator.c create mode 100644 drivers/remoteproc/qcom_mdt_loader.c create mode 100644 drivers/remoteproc/qcom_mdt_loader.h create mode 100644 drivers/remoteproc/qcom_q6v5_pil.c create mode 100644 drivers/reset/reset-meson.c create mode 100644 drivers/reset/reset-ti-syscon.c create mode 100644 drivers/rtc/rtc-max6916.c create mode 100644 drivers/rtc/rtc-mc146818-lib.c create mode 100644 drivers/scsi/ibmvscsi_tgt/Makefile create mode 100644 drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c create mode 100644 drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h create mode 100644 drivers/scsi/ibmvscsi_tgt/libsrp.c create mode 100644 drivers/scsi/ibmvscsi_tgt/libsrp.h create mode 100644 drivers/scsi/lpfc/lpfc_attr.h create mode 100644 drivers/scsi/lpfc/lpfc_ids.h create mode 100644 drivers/scsi/ufs/tc-dwc-g210-pci.c create mode 100644 drivers/scsi/ufs/tc-dwc-g210-pltfrm.c create mode 100644 drivers/scsi/ufs/tc-dwc-g210.c create mode 100644 drivers/scsi/ufs/tc-dwc-g210.h create mode 100644 drivers/scsi/ufs/ufshcd-dwc.c create mode 100644 drivers/scsi/ufs/ufshcd-dwc.h create mode 100644 drivers/scsi/ufs/ufshci-dwc.h create mode 100644 drivers/soc/bcm/brcmstb/Makefile create mode 100644 drivers/soc/bcm/brcmstb/biuctrl.c create mode 100644 drivers/soc/bcm/brcmstb/common.c create mode 100644 drivers/soc/fsl/qe/qe_tdm.c create mode 100644 drivers/soc/renesas/r8a7792-sysc.c create mode 100644 drivers/soc/renesas/r8a7796-sysc.c create mode 100644 drivers/soc/samsung/pm_domains.c create mode 100644 drivers/soc/ux500/Kconfig create mode 100644 drivers/soc/ux500/Makefile create mode 100644 drivers/soc/ux500/ux500-soc-id.c create mode 100644 drivers/spi/spi-cavium-octeon.c create mode 100644 drivers/spi/spi-cavium.c create mode 100644 drivers/spi/spi-cavium.h create mode 100644 drivers/staging/android/sync_debug.h create mode 100644 drivers/staging/ks7010/Kconfig create mode 100644 drivers/staging/ks7010/Makefile create mode 100644 drivers/staging/ks7010/TODO create mode 100644 drivers/staging/ks7010/eap_packet.h create mode 100644 drivers/staging/ks7010/ks7010_sdio.c create mode 100644 drivers/staging/ks7010/ks7010_sdio.h create mode 100644 drivers/staging/ks7010/ks_hostif.c create mode 100644 drivers/staging/ks7010/ks_hostif.h create mode 100644 drivers/staging/ks7010/ks_wlan.h create mode 100644 drivers/staging/ks7010/ks_wlan_ioctl.h create mode 100644 drivers/staging/ks7010/ks_wlan_net.c create mode 100644 drivers/staging/ks7010/michael_mic.c create mode 100644 drivers/staging/ks7010/michael_mic.h create mode 100644 drivers/staging/media/cec/Kconfig create mode 100644 drivers/staging/media/cec/Makefile create mode 100644 drivers/staging/media/cec/TODO create mode 100644 drivers/staging/media/cec/cec-adap.c create mode 100644 drivers/staging/media/cec/cec-api.c create mode 100644 drivers/staging/media/cec/cec-core.c create mode 100644 drivers/staging/media/cec/cec-priv.h create mode 100644 drivers/staging/media/pulse8-cec/Kconfig create mode 100644 drivers/staging/media/pulse8-cec/Makefile create mode 100644 drivers/staging/media/pulse8-cec/TODO create mode 100644 drivers/staging/media/pulse8-cec/pulse8-cec.c create mode 100644 drivers/staging/media/s5p-cec/Kconfig create mode 100644 drivers/staging/media/s5p-cec/Makefile create mode 100644 drivers/staging/media/s5p-cec/TODO create mode 100644 drivers/staging/media/s5p-cec/exynos_hdmi_cec.h create mode 100644 drivers/staging/media/s5p-cec/exynos_hdmi_cecctrl.c create mode 100644 drivers/staging/media/s5p-cec/regs-cec.h create mode 100644 drivers/staging/media/s5p-cec/s5p_cec.c create mode 100644 drivers/staging/media/s5p-cec/s5p_cec.h create mode 100644 drivers/usb/gadget/udc/core.c create mode 100644 drivers/usb/gadget/udc/trace.c create mode 100644 drivers/usb/gadget/udc/trace.h create mode 100644 drivers/usb/musb/musb_trace.c create mode 100644 drivers/usb/musb/musb_trace.h create mode 100644 drivers/vhost/Kconfig.vringh create mode 100644 drivers/vhost/vsock.c create mode 100644 drivers/watchdog/aspeed_wdt.c create mode 100644 drivers/watchdog/max77620_wdt.c create mode 100644 drivers/watchdog/meson_gxbb_wdt.c create mode 100644 drivers/xen/arm-device.c (limited to 'drivers') diff --git a/drivers/Makefile b/drivers/Makefile index 0b6f3d601..53abb4a5f 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -12,7 +12,7 @@ obj-$(CONFIG_GENERIC_PHY) += phy/ # GPIO must come after pinctrl as gpios may need to mux pins etc obj-$(CONFIG_PINCTRL) += pinctrl/ -obj-y += gpio/ +obj-$(CONFIG_GPIOLIB) += gpio/ obj-y += pwm/ obj-$(CONFIG_PCI) += pci/ obj-$(CONFIG_PARISC) += parisc/ @@ -78,7 +78,7 @@ obj-$(CONFIG_TARGET_CORE) += target/ obj-$(CONFIG_MTD) += mtd/ obj-$(CONFIG_SPI) += spi/ obj-$(CONFIG_SPMI) += spmi/ -obj-y += hsi/ +obj-$(CONFIG_HSI) += hsi/ obj-y += net/ obj-$(CONFIG_ATM) += atm/ obj-$(CONFIG_FUSION) += message/ @@ -122,13 +122,12 @@ obj-$(CONFIG_CPU_FREQ) += cpufreq/ obj-$(CONFIG_CPU_IDLE) += cpuidle/ obj-y += mmc/ obj-$(CONFIG_MEMSTICK) += memstick/ -obj-y += leds/ +obj-$(CONFIG_NEW_LEDS) += leds/ obj-$(CONFIG_INFINIBAND) += infiniband/ obj-$(CONFIG_SGI_SN) += sn/ obj-y += firmware/ obj-$(CONFIG_CRYPTO) += crypto/ obj-$(CONFIG_SUPERH) += sh/ -obj-$(CONFIG_ARCH_SHMOBILE) += sh/ ifndef CONFIG_ARCH_USES_GETTIMEOFFSET obj-y += clocksource/ endif @@ -139,6 +138,7 @@ obj-$(CONFIG_OF) += of/ obj-$(CONFIG_SSB) += ssb/ obj-$(CONFIG_BCMA) += bcma/ obj-$(CONFIG_VHOST_RING) += vhost/ +obj-$(CONFIG_VHOST) += vhost/ obj-$(CONFIG_VLYNQ) += vlynq/ obj-$(CONFIG_STAGING) += staging/ obj-y += platform/ diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index b7e2e7763..445ce2847 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -213,10 +213,17 @@ config ACPI_CPU_FREQ_PSS bool select THERMAL +config ACPI_PROCESSOR_CSTATE + def_bool y + depends on IA64 || X86 + config ACPI_PROCESSOR_IDLE bool select CPU_IDLE +config ACPI_MCFG + bool + config ACPI_CPPC_LIB bool depends on ACPI_PROCESSOR @@ -234,7 +241,7 @@ config ACPI_CPPC_LIB config ACPI_PROCESSOR tristate "Processor" depends on X86 || IA64 || ARM64 - select ACPI_PROCESSOR_IDLE if X86 || IA64 + select ACPI_PROCESSOR_IDLE select ACPI_CPU_FREQ_PSS if X86 || IA64 default y help @@ -291,8 +298,8 @@ config ACPI_THERMAL config ACPI_NUMA bool "NUMA support" depends on NUMA - depends on (X86 || IA64) - default y if IA64_GENERIC || IA64_SGI_SN2 + depends on (X86 || IA64 || ARM64) + default y if IA64_GENERIC || IA64_SGI_SN2 || ARM64 config ACPI_CUSTOM_DSDT_FILE string "Custom DSDT Table file to include" @@ -311,9 +318,12 @@ config ACPI_CUSTOM_DSDT bool default ACPI_CUSTOM_DSDT_FILE != "" +config ARCH_HAS_ACPI_TABLE_UPGRADE + def_bool n + config ACPI_TABLE_UPGRADE bool "Allow upgrading ACPI tables via initrd" - depends on BLK_DEV_INITRD && X86 + depends on BLK_DEV_INITRD && ARCH_HAS_ACPI_TABLE_UPGRADE default y help This option provides functionality to upgrade arbitrary ACPI tables @@ -447,34 +457,10 @@ config ACPI_REDUCED_HARDWARE_ONLY If you are unsure what to do, do not enable this option. -config ACPI_NFIT - tristate "ACPI NVDIMM Firmware Interface Table (NFIT)" - depends on PHYS_ADDR_T_64BIT - depends on BLK_DEV - depends on ARCH_HAS_MMIO_FLUSH - select LIBNVDIMM - help - Infrastructure to probe ACPI 6 compliant platforms for - NVDIMMs (NFIT) and register a libnvdimm device tree. In - addition to storage devices this also enables libnvdimm to pass - ACPI._DSM messages for platform/dimm configuration. - - To compile this driver as a module, choose M here: - the module will be called nfit. - -config ACPI_NFIT_DEBUG - bool "NFIT DSM debug" - depends on ACPI_NFIT - depends on DYNAMIC_DEBUG - default n - help - Enabling this option causes the nfit driver to dump the - input and output buffers of _DSM operations on the ACPI0012 - device and its children. This can be very verbose, so leave - it disabled unless you are debugging a hardware / firmware - issue. +source "drivers/acpi/nfit/Kconfig" source "drivers/acpi/apei/Kconfig" +source "drivers/acpi/dptf/Kconfig" config ACPI_EXTLOG tristate "Extended Error Log support" @@ -519,6 +505,20 @@ config XPOWER_PMIC_OPREGION help This config adds ACPI operation region support for XPower AXP288 PMIC. +config BXT_WC_PMIC_OPREGION + bool "ACPI operation region support for BXT WhiskeyCove PMIC" + depends on INTEL_SOC_PMIC + help + This config adds ACPI operation region support for BXT WhiskeyCove PMIC. + endif +config ACPI_CONFIGFS + tristate "ACPI configfs support" + select CONFIGFS_FS + help + Select this option to enable support for ACPI configuration from + userspace. The configurable ACPI groups will be visible under + /config/acpi, assuming configfs is mounted under /config. + endif # ACPI diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 251ce85a6..5ae9d85c5 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -40,11 +40,11 @@ acpi-$(CONFIG_ARCH_MIGHT_HAVE_ACPI_PDC) += processor_pdc.o acpi-y += ec.o acpi-$(CONFIG_ACPI_DOCK) += dock.o acpi-y += pci_root.o pci_link.o pci_irq.o +obj-$(CONFIG_ACPI_MCFG) += pci_mcfg.o acpi-y += acpi_lpss.o acpi_apd.o acpi-y += acpi_platform.o acpi-y += acpi_pnp.o acpi-$(CONFIG_ARM_AMBA) += acpi_amba.o -acpi-y += int340x_thermal.o acpi-y += power.o acpi-y += event.o acpi-$(CONFIG_ACPI_REDUCED_HARDWARE_ONLY) += evged.o @@ -70,7 +70,7 @@ obj-$(CONFIG_ACPI_PCI_SLOT) += pci_slot.o obj-$(CONFIG_ACPI_PROCESSOR) += processor.o obj-$(CONFIG_ACPI) += container.o obj-$(CONFIG_ACPI_THERMAL) += thermal.o -obj-$(CONFIG_ACPI_NFIT) += nfit.o +obj-$(CONFIG_ACPI_NFIT) += nfit/ obj-$(CONFIG_ACPI) += acpi_memhotplug.o obj-$(CONFIG_ACPI_HOTPLUG_IOAPIC) += ioapic.o obj-$(CONFIG_ACPI_BATTERY) += battery.o @@ -99,5 +99,9 @@ obj-$(CONFIG_ACPI_EXTLOG) += acpi_extlog.o obj-$(CONFIG_PMIC_OPREGION) += pmic/intel_pmic.o obj-$(CONFIG_CRC_PMIC_OPREGION) += pmic/intel_pmic_crc.o obj-$(CONFIG_XPOWER_PMIC_OPREGION) += pmic/intel_pmic_xpower.o +obj-$(CONFIG_BXT_WC_PMIC_OPREGION) += pmic/intel_pmic_bxtwc.o + +obj-$(CONFIG_ACPI_CONFIGFS) += acpi_configfs.o video-objs += acpi_video.o video_detect.o +obj-y += dptf/ diff --git a/drivers/acpi/acpi_cmos_rtc.c b/drivers/acpi/acpi_cmos_rtc.c index 81dc75033..0980a1339 100644 --- a/drivers/acpi/acpi_cmos_rtc.c +++ b/drivers/acpi/acpi_cmos_rtc.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include "internal.h" diff --git a/drivers/acpi/acpi_configfs.c b/drivers/acpi/acpi_configfs.c new file mode 100644 index 000000000..146a77fb7 --- /dev/null +++ b/drivers/acpi/acpi_configfs.c @@ -0,0 +1,267 @@ +/* + * ACPI configfs support + * + * Copyright (c) 2016 Intel Corporation + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + */ + +#define pr_fmt(fmt) "ACPI configfs: " fmt + +#include +#include +#include +#include + +static struct config_group *acpi_table_group; + +struct acpi_table { + struct config_item cfg; + struct acpi_table_header *header; +}; + +static ssize_t acpi_table_aml_write(struct config_item *cfg, + const void *data, size_t size) +{ + const struct acpi_table_header *header = data; + struct acpi_table *table; + int ret; + + table = container_of(cfg, struct acpi_table, cfg); + + if (table->header) { + pr_err("table already loaded\n"); + return -EBUSY; + } + + if (header->length != size) { + pr_err("invalid table length\n"); + return -EINVAL; + } + + if (memcmp(header->signature, ACPI_SIG_SSDT, 4)) { + pr_err("invalid table signature\n"); + return -EINVAL; + } + + table = container_of(cfg, struct acpi_table, cfg); + + table->header = kmemdup(header, header->length, GFP_KERNEL); + if (!table->header) + return -ENOMEM; + + ret = acpi_load_table(table->header); + if (ret) { + kfree(table->header); + table->header = NULL; + } + + return ret; +} + +static inline struct acpi_table_header *get_header(struct config_item *cfg) +{ + struct acpi_table *table = container_of(cfg, struct acpi_table, cfg); + + if (!table->header) + pr_err("table not loaded\n"); + + return table->header; +} + +static ssize_t acpi_table_aml_read(struct config_item *cfg, + void *data, size_t size) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + if (data) + memcpy(data, h, h->length); + + return h->length; +} + +#define MAX_ACPI_TABLE_SIZE (128 * 1024) + +CONFIGFS_BIN_ATTR(acpi_table_, aml, NULL, MAX_ACPI_TABLE_SIZE); + +struct configfs_bin_attribute *acpi_table_bin_attrs[] = { + &acpi_table_attr_aml, + NULL, +}; + +ssize_t acpi_table_signature_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%.*s\n", ACPI_NAME_SIZE, h->signature); +} + +ssize_t acpi_table_length_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%d\n", h->length); +} + +ssize_t acpi_table_revision_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%d\n", h->revision); +} + +ssize_t acpi_table_oem_id_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%.*s\n", ACPI_OEM_ID_SIZE, h->oem_id); +} + +ssize_t acpi_table_oem_table_id_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%.*s\n", ACPI_OEM_TABLE_ID_SIZE, h->oem_table_id); +} + +ssize_t acpi_table_oem_revision_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%d\n", h->oem_revision); +} + +ssize_t acpi_table_asl_compiler_id_show(struct config_item *cfg, char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%.*s\n", ACPI_NAME_SIZE, h->asl_compiler_id); +} + +ssize_t acpi_table_asl_compiler_revision_show(struct config_item *cfg, + char *str) +{ + struct acpi_table_header *h = get_header(cfg); + + if (!h) + return -EINVAL; + + return sprintf(str, "%d\n", h->asl_compiler_revision); +} + +CONFIGFS_ATTR_RO(acpi_table_, signature); +CONFIGFS_ATTR_RO(acpi_table_, length); +CONFIGFS_ATTR_RO(acpi_table_, revision); +CONFIGFS_ATTR_RO(acpi_table_, oem_id); +CONFIGFS_ATTR_RO(acpi_table_, oem_table_id); +CONFIGFS_ATTR_RO(acpi_table_, oem_revision); +CONFIGFS_ATTR_RO(acpi_table_, asl_compiler_id); +CONFIGFS_ATTR_RO(acpi_table_, asl_compiler_revision); + +struct configfs_attribute *acpi_table_attrs[] = { + &acpi_table_attr_signature, + &acpi_table_attr_length, + &acpi_table_attr_revision, + &acpi_table_attr_oem_id, + &acpi_table_attr_oem_table_id, + &acpi_table_attr_oem_revision, + &acpi_table_attr_asl_compiler_id, + &acpi_table_attr_asl_compiler_revision, + NULL, +}; + +static struct config_item_type acpi_table_type = { + .ct_owner = THIS_MODULE, + .ct_bin_attrs = acpi_table_bin_attrs, + .ct_attrs = acpi_table_attrs, +}; + +static struct config_item *acpi_table_make_item(struct config_group *group, + const char *name) +{ + struct acpi_table *table; + + table = kzalloc(sizeof(*table), GFP_KERNEL); + if (!table) + return ERR_PTR(-ENOMEM); + + config_item_init_type_name(&table->cfg, name, &acpi_table_type); + return &table->cfg; +} + +struct configfs_group_operations acpi_table_group_ops = { + .make_item = acpi_table_make_item, +}; + +static struct config_item_type acpi_tables_type = { + .ct_owner = THIS_MODULE, + .ct_group_ops = &acpi_table_group_ops, +}; + +static struct config_item_type acpi_root_group_type = { + .ct_owner = THIS_MODULE, +}; + +static struct configfs_subsystem acpi_configfs = { + .su_group = { + .cg_item = { + .ci_namebuf = "acpi", + .ci_type = &acpi_root_group_type, + }, + }, + .su_mutex = __MUTEX_INITIALIZER(acpi_configfs.su_mutex), +}; + +static int __init acpi_configfs_init(void) +{ + int ret; + struct config_group *root = &acpi_configfs.su_group; + + config_group_init(root); + + ret = configfs_register_subsystem(&acpi_configfs); + if (ret) + return ret; + + acpi_table_group = configfs_register_default_group(root, "table", + &acpi_tables_type); + return PTR_ERR_OR_ZERO(acpi_table_group); +} +module_init(acpi_configfs_init); + +static void __exit acpi_configfs_exit(void) +{ + configfs_unregister_default_group(acpi_table_group); + configfs_unregister_subsystem(&acpi_configfs); +} +module_exit(acpi_configfs_exit); + +MODULE_AUTHOR("Octavian Purdila "); +MODULE_DESCRIPTION("ACPI configfs support"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/acpi/acpi_lpat.c b/drivers/acpi/acpi_lpat.c index feb61c163..c1c4877ca 100644 --- a/drivers/acpi/acpi_lpat.c +++ b/drivers/acpi/acpi_lpat.c @@ -13,7 +13,7 @@ * GNU General Public License for more details. */ -#include +#include #include #include @@ -157,5 +157,3 @@ void acpi_lpat_free_conversion_table(struct acpi_lpat_conversion_table } } EXPORT_SYMBOL_GPL(acpi_lpat_free_conversion_table); - -MODULE_LICENSE("GPL"); diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c index 0872d5fec..357a0b8f8 100644 --- a/drivers/acpi/acpi_lpss.c +++ b/drivers/acpi/acpi_lpss.c @@ -29,6 +29,7 @@ ACPI_MODULE_NAME("acpi_lpss"); #ifdef CONFIG_X86_INTEL_LPSS #include +#include #include #include @@ -229,8 +230,8 @@ static const struct lpss_device_desc bsw_spi_dev_desc = { #define ICPU(model) { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, } static const struct x86_cpu_id lpss_cpu_ids[] = { - ICPU(0x37), /* Valleyview, Bay Trail */ - ICPU(0x4c), /* Braswell, Cherry Trail */ + ICPU(INTEL_FAM6_ATOM_SILVERMONT1), /* Valleyview, Bay Trail */ + ICPU(INTEL_FAM6_ATOM_AIRMONT), /* Braswell, Cherry Trail */ {} }; diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index c1d138e12..c5557d070 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -1246,6 +1246,9 @@ static int acpi_video_device_enumerate(struct acpi_video_bus *video) union acpi_object *dod = NULL; union acpi_object *obj; + if (!video->cap._DOD) + return AE_NOT_EXIST; + status = acpi_evaluate_object(video->device->handle, "_DOD", NULL, &buffer); if (!ACPI_SUCCESS(status)) { ACPI_EXCEPTION((AE_INFO, status, "Evaluating _DOD")); diff --git a/drivers/acpi/apei/Makefile b/drivers/acpi/apei/Makefile index 5d575a955..e50573de2 100644 --- a/drivers/acpi/apei/Makefile +++ b/drivers/acpi/apei/Makefile @@ -3,4 +3,4 @@ obj-$(CONFIG_ACPI_APEI_GHES) += ghes.o obj-$(CONFIG_ACPI_APEI_EINJ) += einj.o obj-$(CONFIG_ACPI_APEI_ERST_DEBUG) += erst-dbg.o -apei-y := apei-base.o hest.o erst.o +apei-y := apei-base.o hest.o erst.o bert.o diff --git a/drivers/acpi/apei/apei-internal.h b/drivers/acpi/apei/apei-internal.h index 16129c78b..6e9f14c0a 100644 --- a/drivers/acpi/apei/apei-internal.h +++ b/drivers/acpi/apei/apei-internal.h @@ -1,6 +1,6 @@ /* * apei-internal.h - ACPI Platform Error Interface internal - * definations. + * definitions. */ #ifndef APEI_INTERNAL_H diff --git a/drivers/acpi/apei/bert.c b/drivers/acpi/apei/bert.c new file mode 100644 index 000000000..a05b5c0cf --- /dev/null +++ b/drivers/acpi/apei/bert.c @@ -0,0 +1,150 @@ +/* + * APEI Boot Error Record Table (BERT) support + * + * Copyright 2011 Intel Corp. + * Author: Huang Ying + * + * Under normal circumstances, when a hardware error occurs, the error + * handler receives control and processes the error. This gives OSPM a + * chance to process the error condition, report it, and optionally attempt + * recovery. In some cases, the system is unable to process an error. + * For example, system firmware or a management controller may choose to + * reset the system or the system might experience an uncontrolled crash + * or reset.The boot error source is used to report unhandled errors that + * occurred in a previous boot. This mechanism is described in the BERT + * table. + * + * For more information about BERT, please refer to ACPI Specification + * version 4.0, section 17.3.1 + * + * This file is licensed under GPLv2. + * + */ + +#include +#include +#include +#include +#include + +#include "apei-internal.h" + +#undef pr_fmt +#define pr_fmt(fmt) "BERT: " fmt + +static int bert_disable; + +static void __init bert_print_all(struct acpi_bert_region *region, + unsigned int region_len) +{ + struct acpi_hest_generic_status *estatus = + (struct acpi_hest_generic_status *)region; + int remain = region_len; + u32 estatus_len; + + if (!estatus->block_status) + return; + + while (remain > sizeof(struct acpi_bert_region)) { + if (cper_estatus_check(estatus)) { + pr_err(FW_BUG "Invalid error record.\n"); + return; + } + + estatus_len = cper_estatus_len(estatus); + if (remain < estatus_len) { + pr_err(FW_BUG "Truncated status block (length: %u).\n", + estatus_len); + return; + } + + pr_info_once("Error records from previous boot:\n"); + + cper_estatus_print(KERN_INFO HW_ERR, estatus); + + /* + * Because the boot error source is "one-time polled" type, + * clear Block Status of current Generic Error Status Block, + * once it's printed. + */ + estatus->block_status = 0; + + estatus = (void *)estatus + estatus_len; + /* No more error records. */ + if (!estatus->block_status) + return; + + remain -= estatus_len; + } +} + +static int __init setup_bert_disable(char *str) +{ + bert_disable = 1; + + return 0; +} +__setup("bert_disable", setup_bert_disable); + +static int __init bert_check_table(struct acpi_table_bert *bert_tab) +{ + if (bert_tab->header.length < sizeof(struct acpi_table_bert) || + bert_tab->region_length < sizeof(struct acpi_bert_region)) + return -EINVAL; + + return 0; +} + +static int __init bert_init(void) +{ + struct acpi_bert_region *boot_error_region; + struct acpi_table_bert *bert_tab; + unsigned int region_len; + acpi_status status; + int rc = 0; + + if (acpi_disabled) + return 0; + + if (bert_disable) { + pr_info("Boot Error Record Table support is disabled.\n"); + return 0; + } + + status = acpi_get_table(ACPI_SIG_BERT, 0, (struct acpi_table_header **)&bert_tab); + if (status == AE_NOT_FOUND) + return 0; + + if (ACPI_FAILURE(status)) { + pr_err("get table failed, %s.\n", acpi_format_exception(status)); + return -EINVAL; + } + + rc = bert_check_table(bert_tab); + if (rc) { + pr_err(FW_BUG "table invalid.\n"); + return rc; + } + + region_len = bert_tab->region_length; + if (!request_mem_region(bert_tab->address, region_len, "APEI BERT")) { + pr_err("Can't request iomem region <%016llx-%016llx>.\n", + (unsigned long long)bert_tab->address, + (unsigned long long)bert_tab->address + region_len - 1); + return -EIO; + } + + boot_error_region = ioremap_cache(bert_tab->address, region_len); + if (boot_error_region) { + bert_print_all(boot_error_region, region_len); + iounmap(boot_error_region); + } else { + rc = -ENOMEM; + } + + release_mem_region(bert_tab->address, region_len); + + return rc; +} + +late_initcall(bert_init); diff --git a/drivers/acpi/apei/einj.c b/drivers/acpi/apei/einj.c index 559c1173d..eebb7e39c 100644 --- a/drivers/acpi/apei/einj.c +++ b/drivers/acpi/apei/einj.c @@ -33,7 +33,8 @@ #include "apei-internal.h" -#define EINJ_PFX "EINJ: " +#undef pr_fmt +#define pr_fmt(fmt) "EINJ: " fmt #define SPIN_UNIT 100 /* 100ns */ /* Firmware should respond within 1 milliseconds */ @@ -179,8 +180,7 @@ static int einj_get_available_error_type(u32 *type) static int einj_timedout(u64 *t) { if ((s64)*t < SPIN_UNIT) { - pr_warning(FW_WARN EINJ_PFX - "Firmware does not respond in time\n"); + pr_warning(FW_WARN "Firmware does not respond in time\n"); return 1; } *t -= SPIN_UNIT; @@ -307,8 +307,7 @@ static int __einj_error_trigger(u64 trigger_paddr, u32 type, r = request_mem_region(trigger_paddr, sizeof(*trigger_tab), "APEI EINJ Trigger Table"); if (!r) { - pr_err(EINJ_PFX - "Can not request [mem %#010llx-%#010llx] for Trigger table\n", + pr_err("Can not request [mem %#010llx-%#010llx] for Trigger table\n", (unsigned long long)trigger_paddr, (unsigned long long)trigger_paddr + sizeof(*trigger_tab) - 1); @@ -316,13 +315,12 @@ static int __einj_error_trigger(u64 trigger_paddr, u32 type, } trigger_tab = ioremap_cache(trigger_paddr, sizeof(*trigger_tab)); if (!trigger_tab) { - pr_err(EINJ_PFX "Failed to map trigger table!\n"); + pr_err("Failed to map trigger table!\n"); goto out_rel_header; } rc = einj_check_trigger_header(trigger_tab); if (rc) { - pr_warning(FW_BUG EINJ_PFX - "The trigger error action table is invalid\n"); + pr_warning(FW_BUG "Invalid trigger error action table.\n"); goto out_rel_header; } @@ -336,8 +334,7 @@ static int __einj_error_trigger(u64 trigger_paddr, u32 type, table_size - sizeof(*trigger_tab), "APEI EINJ Trigger Table"); if (!r) { - pr_err(EINJ_PFX -"Can not request [mem %#010llx-%#010llx] for Trigger Table Entry\n", + pr_err("Can not request [mem %#010llx-%#010llx] for Trigger Table Entry\n", (unsigned long long)trigger_paddr + sizeof(*trigger_tab), (unsigned long long)trigger_paddr + table_size - 1); goto out_rel_header; @@ -345,7 +342,7 @@ static int __einj_error_trigger(u64 trigger_paddr, u32 type, iounmap(trigger_tab); trigger_tab = ioremap_cache(trigger_paddr, table_size); if (!trigger_tab) { - pr_err(EINJ_PFX "Failed to map trigger table!\n"); + pr_err("Failed to map trigger table!\n"); goto out_rel_entry; } trigger_entry = (struct acpi_whea_header *) @@ -695,34 +692,42 @@ static int __init einj_init(void) struct dentry *fentry; struct apei_exec_context ctx; - if (acpi_disabled) + if (acpi_disabled) { + pr_warn("ACPI disabled.\n"); return -ENODEV; + } status = acpi_get_table(ACPI_SIG_EINJ, 0, (struct acpi_table_header **)&einj_tab); - if (status == AE_NOT_FOUND) + if (status == AE_NOT_FOUND) { + pr_warn("EINJ table not found.\n"); return -ENODEV; + } else if (ACPI_FAILURE(status)) { - const char *msg = acpi_format_exception(status); - pr_err(EINJ_PFX "Failed to get table, %s\n", msg); + pr_err("Failed to get EINJ table: %s\n", + acpi_format_exception(status)); return -EINVAL; } rc = einj_check_table(einj_tab); if (rc) { - pr_warning(FW_BUG EINJ_PFX "EINJ table is invalid\n"); + pr_warn(FW_BUG "Invalid EINJ table.n"); return -EINVAL; } rc = -ENOMEM; einj_debug_dir = debugfs_create_dir("einj", apei_get_debugfs_dir()); - if (!einj_debug_dir) + if (!einj_debug_dir) { + pr_err("Error creating debugfs node.\n"); goto err_cleanup; + } + fentry = debugfs_create_file("available_error_type", S_IRUSR, einj_debug_dir, NULL, &available_error_type_fops); if (!fentry) goto err_cleanup; + fentry = debugfs_create_file("error_type", S_IRUSR | S_IWUSR, einj_debug_dir, NULL, &error_type_fops); if (!fentry) @@ -735,14 +740,22 @@ static int __init einj_init(void) apei_resources_init(&einj_resources); einj_exec_ctx_init(&ctx); rc = apei_exec_collect_resources(&ctx, &einj_resources); - if (rc) + if (rc) { + pr_err("Error collecting EINJ resources.\n"); goto err_fini; + } + rc = apei_resources_request(&einj_resources, "APEI EINJ"); - if (rc) + if (rc) { + pr_err("Error requesting memory/port resources.\n"); goto err_fini; + } + rc = apei_exec_pre_map_gars(&ctx); - if (rc) + if (rc) { + pr_err("Error pre-mapping GARs.\n"); goto err_release; + } rc = -ENOMEM; einj_param = einj_get_parameter_address(); @@ -787,7 +800,7 @@ static int __init einj_init(void) goto err_unmap; } - pr_info(EINJ_PFX "Error INJection is initialized.\n"); + pr_info("Error INJection is initialized.\n"); return 0; @@ -798,6 +811,7 @@ err_unmap: sizeof(struct einj_parameter); acpi_os_unmap_iomem(einj_param, size); + pr_err("Error creating param extension debugfs nodes.\n"); } apei_exec_post_unmap_gars(&ctx); err_release: @@ -805,6 +819,7 @@ err_release: err_fini: apei_resources_fini(&einj_resources); err_cleanup: + pr_err("Error creating primary debugfs nodes.\n"); debugfs_remove_recursive(einj_debug_dir); return rc; diff --git a/drivers/acpi/apei/erst.c b/drivers/acpi/apei/erst.c index 006c3894c..f096ab3cb 100644 --- a/drivers/acpi/apei/erst.c +++ b/drivers/acpi/apei/erst.c @@ -927,7 +927,8 @@ static int erst_open_pstore(struct pstore_info *psi); static int erst_close_pstore(struct pstore_info *psi); static ssize_t erst_reader(u64 *id, enum pstore_type_id *type, int *count, struct timespec *time, char **buf, - bool *compressed, struct pstore_info *psi); + bool *compressed, ssize_t *ecc_notice_size, + struct pstore_info *psi); static int erst_writer(enum pstore_type_id type, enum kmsg_dump_reason reason, u64 *id, unsigned int part, int count, bool compressed, size_t size, struct pstore_info *psi); @@ -987,7 +988,8 @@ static int erst_close_pstore(struct pstore_info *psi) static ssize_t erst_reader(u64 *id, enum pstore_type_id *type, int *count, struct timespec *time, char **buf, - bool *compressed, struct pstore_info *psi) + bool *compressed, ssize_t *ecc_notice_size, + struct pstore_info *psi) { int rc; ssize_t len = 0; @@ -1033,6 +1035,7 @@ skip: memcpy(*buf, rcd->data, len - sizeof(*rcd)); *id = record_id; *compressed = false; + *ecc_notice_size = 0; if (uuid_le_cmp(rcd->sec_hdr.section_type, CPER_SECTION_TYPE_DMESG_Z) == 0) { *type = PSTORE_TYPE_DMESG; diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index 262ca31b8..85b7d07fe 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -30,6 +30,9 @@ #include #include #include +#include +#include +#include #ifdef CONFIG_X86 #include #endif @@ -174,22 +177,17 @@ void acpi_bus_detach_private_data(acpi_handle handle) EXPORT_SYMBOL_GPL(acpi_bus_detach_private_data); static void acpi_print_osc_error(acpi_handle handle, - struct acpi_osc_context *context, char *error) + struct acpi_osc_context *context, char *error) { - struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER}; int i; - if (ACPI_FAILURE(acpi_get_name(handle, ACPI_FULL_PATHNAME, &buffer))) - printk(KERN_DEBUG "%s: %s\n", context->uuid_str, error); - else { - printk(KERN_DEBUG "%s (%s): %s\n", - (char *)buffer.pointer, context->uuid_str, error); - kfree(buffer.pointer); - } - printk(KERN_DEBUG "_OSC request data:"); + acpi_handle_debug(handle, "(%s): %s\n", context->uuid_str, error); + + pr_debug("_OSC request data:"); for (i = 0; i < context->cap.length; i += sizeof(u32)) - printk(" %x", *((u32 *)(context->cap.pointer + i))); - printk("\n"); + pr_debug(" %x", *((u32 *)(context->cap.pointer + i))); + + pr_debug("\n"); } acpi_status acpi_str_to_uuid(char *str, u8 *uuid) @@ -302,6 +300,14 @@ out_kfree: EXPORT_SYMBOL(acpi_run_osc); bool osc_sb_apei_support_acked; + +/* + * ACPI 6.0 Section 8.4.4.2 Idle State Coordination + * OSPM supports platform coordinated low power idle(LPI) states + */ +bool osc_pc_lpi_support_confirmed; +EXPORT_SYMBOL_GPL(osc_pc_lpi_support_confirmed); + static u8 sb_uuid_str[] = "0811B06E-4A27-44F9-8D60-3CBBC22E7B48"; static void acpi_bus_osc_support(void) { @@ -322,6 +328,7 @@ static void acpi_bus_osc_support(void) capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PPC_OST_SUPPORT; capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_HOTPLUG_OST_SUPPORT; + capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_PCLPI_SUPPORT; if (!ghes_disable) capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_APEI_SUPPORT; @@ -329,9 +336,12 @@ static void acpi_bus_osc_support(void) return; if (ACPI_SUCCESS(acpi_run_osc(handle, &context))) { u32 *capbuf_ret = context.ret.pointer; - if (context.ret.length > OSC_SUPPORT_DWORD) + if (context.ret.length > OSC_SUPPORT_DWORD) { osc_sb_apei_support_acked = capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_APEI_SUPPORT; + osc_pc_lpi_support_confirmed = + capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_PCLPI_SUPPORT; + } kfree(context.ret.pointer); } /* do we need to check other returned cap? Sounds no */ @@ -475,6 +485,56 @@ static void acpi_device_remove_notify_handler(struct acpi_device *device) acpi_device_notify); } +/* Handle events targeting \_SB device (at present only graceful shutdown) */ + +#define ACPI_SB_NOTIFY_SHUTDOWN_REQUEST 0x81 +#define ACPI_SB_INDICATE_INTERVAL 10000 + +static void sb_notify_work(struct work_struct *dummy) +{ + acpi_handle sb_handle; + + orderly_poweroff(true); + + /* + * After initiating graceful shutdown, the ACPI spec requires OSPM + * to evaluate _OST method once every 10seconds to indicate that + * the shutdown is in progress + */ + acpi_get_handle(NULL, "\\_SB", &sb_handle); + while (1) { + pr_info("Graceful shutdown in progress.\n"); + acpi_evaluate_ost(sb_handle, ACPI_OST_EC_OSPM_SHUTDOWN, + ACPI_OST_SC_OS_SHUTDOWN_IN_PROGRESS, NULL); + msleep(ACPI_SB_INDICATE_INTERVAL); + } +} + +static void acpi_sb_notify(acpi_handle handle, u32 event, void *data) +{ + static DECLARE_WORK(acpi_sb_work, sb_notify_work); + + if (event == ACPI_SB_NOTIFY_SHUTDOWN_REQUEST) { + if (!work_busy(&acpi_sb_work)) + schedule_work(&acpi_sb_work); + } else + pr_warn("event %x is not supported by \\_SB device\n", event); +} + +static int __init acpi_setup_sb_notify_handler(void) +{ + acpi_handle sb_handle; + + if (ACPI_FAILURE(acpi_get_handle(NULL, "\\_SB", &sb_handle))) + return -ENXIO; + + if (ACPI_FAILURE(acpi_install_notify_handler(sb_handle, ACPI_DEVICE_NOTIFY, + acpi_sb_notify, NULL))) + return -EINVAL; + + return 0; +} + /* -------------------------------------------------------------------------- Device Matching -------------------------------------------------------------------------- */ @@ -961,8 +1021,7 @@ void __init acpi_early_init(void) /** * acpi_subsystem_init - Finalize the early initialization of ACPI. * - * Switch over the platform to the ACPI mode (if possible), initialize the - * handling of ACPI events, install the interrupt and global lock handlers. + * Switch over the platform to the ACPI mode (if possible). * * Doing this too early is generally unsafe, but at the same time it needs to be * done before all things that really depend on ACPI. The right spot appears to @@ -990,6 +1049,13 @@ void __init acpi_subsystem_init(void) } } +static acpi_status acpi_bus_table_handler(u32 event, void *table, void *context) +{ + acpi_scan_table_handler(event, table, context); + + return acpi_sysfs_table_handler(event, table, context); +} + static int __init acpi_bus_init(void) { int result; @@ -1043,6 +1109,8 @@ static int __init acpi_bus_init(void) * _PDC control method may load dynamic SSDT tables, * and we need to install the table handler before that. */ + status = acpi_install_table_handler(acpi_bus_table_handler, NULL); + acpi_sysfs_init(); acpi_early_processor_set_pdc(); @@ -1124,6 +1192,7 @@ static int __init acpi_init(void) acpi_sleep_proc_init(); acpi_wakeup_device_init(); acpi_debugger_init(); + acpi_setup_sb_notify_handler(); return 0; } diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c index 5c3b0918d..31abb0bdd 100644 --- a/drivers/acpi/button.c +++ b/drivers/acpi/button.c @@ -53,6 +53,10 @@ #define ACPI_BUTTON_DEVICE_NAME_LID "Lid Switch" #define ACPI_BUTTON_TYPE_LID 0x05 +#define ACPI_BUTTON_LID_INIT_IGNORE 0x00 +#define ACPI_BUTTON_LID_INIT_OPEN 0x01 +#define ACPI_BUTTON_LID_INIT_METHOD 0x02 + #define _COMPONENT ACPI_BUTTON_COMPONENT ACPI_MODULE_NAME("button"); @@ -105,6 +109,7 @@ struct acpi_button { static BLOCKING_NOTIFIER_HEAD(acpi_lid_notifier); static struct acpi_device *lid_device; +static u8 lid_init_state = ACPI_BUTTON_LID_INIT_METHOD; /* -------------------------------------------------------------------------- FS Interface (/proc) @@ -113,16 +118,52 @@ static struct acpi_device *lid_device; static struct proc_dir_entry *acpi_button_dir; static struct proc_dir_entry *acpi_lid_dir; +static int acpi_lid_evaluate_state(struct acpi_device *device) +{ + unsigned long long lid_state; + acpi_status status; + + status = acpi_evaluate_integer(device->handle, "_LID", NULL, &lid_state); + if (ACPI_FAILURE(status)) + return -ENODEV; + + return lid_state ? 1 : 0; +} + +static int acpi_lid_notify_state(struct acpi_device *device, int state) +{ + struct acpi_button *button = acpi_driver_data(device); + int ret; + + /* input layer checks if event is redundant */ + input_report_switch(button->input, SW_LID, !state); + input_sync(button->input); + + if (state) + pm_wakeup_event(&device->dev, 0); + + ret = blocking_notifier_call_chain(&acpi_lid_notifier, state, device); + if (ret == NOTIFY_DONE) + ret = blocking_notifier_call_chain(&acpi_lid_notifier, state, + device); + if (ret == NOTIFY_DONE || ret == NOTIFY_OK) { + /* + * It is also regarded as success if the notifier_chain + * returns NOTIFY_OK or NOTIFY_DONE. + */ + ret = 0; + } + return ret; +} + static int acpi_button_state_seq_show(struct seq_file *seq, void *offset) { struct acpi_device *device = seq->private; - acpi_status status; - unsigned long long state; + int state; - status = acpi_evaluate_integer(device->handle, "_LID", NULL, &state); + state = acpi_lid_evaluate_state(device); seq_printf(seq, "state: %s\n", - ACPI_FAILURE(status) ? "unsupported" : - (state ? "open" : "closed")); + state < 0 ? "unsupported" : (state ? "open" : "closed")); return 0; } @@ -191,8 +232,10 @@ remove_dev_dir: acpi_device_dir(device) = NULL; remove_lid_dir: remove_proc_entry(ACPI_BUTTON_SUBCLASS_LID, acpi_button_dir); + acpi_lid_dir = NULL; remove_button_dir: remove_proc_entry(ACPI_BUTTON_CLASS, acpi_root_dir); + acpi_button_dir = NULL; goto done; } @@ -209,7 +252,9 @@ static int acpi_button_remove_fs(struct acpi_device *device) acpi_lid_dir); acpi_device_dir(device) = NULL; remove_proc_entry(ACPI_BUTTON_SUBCLASS_LID, acpi_button_dir); + acpi_lid_dir = NULL; remove_proc_entry(ACPI_BUTTON_CLASS, acpi_root_dir); + acpi_button_dir = NULL; return 0; } @@ -231,51 +276,37 @@ EXPORT_SYMBOL(acpi_lid_notifier_unregister); int acpi_lid_open(void) { - acpi_status status; - unsigned long long state; - if (!lid_device) return -ENODEV; - status = acpi_evaluate_integer(lid_device->handle, "_LID", NULL, - &state); - if (ACPI_FAILURE(status)) - return -ENODEV; - - return !!state; + return acpi_lid_evaluate_state(lid_device); } EXPORT_SYMBOL(acpi_lid_open); -static int acpi_lid_send_state(struct acpi_device *device) +static int acpi_lid_update_state(struct acpi_device *device) { - struct acpi_button *button = acpi_driver_data(device); - unsigned long long state; - acpi_status status; - int ret; - - status = acpi_evaluate_integer(device->handle, "_LID", NULL, &state); - if (ACPI_FAILURE(status)) - return -ENODEV; + int state; - /* input layer checks if event is redundant */ - input_report_switch(button->input, SW_LID, !state); - input_sync(button->input); + state = acpi_lid_evaluate_state(device); + if (state < 0) + return state; - if (state) - pm_wakeup_event(&device->dev, 0); + return acpi_lid_notify_state(device, state); +} - ret = blocking_notifier_call_chain(&acpi_lid_notifier, state, device); - if (ret == NOTIFY_DONE) - ret = blocking_notifier_call_chain(&acpi_lid_notifier, state, - device); - if (ret == NOTIFY_DONE || ret == NOTIFY_OK) { - /* - * It is also regarded as success if the notifier_chain - * returns NOTIFY_OK or NOTIFY_DONE. - */ - ret = 0; +static void acpi_lid_initialize_state(struct acpi_device *device) +{ + switch (lid_init_state) { + case ACPI_BUTTON_LID_INIT_OPEN: + (void)acpi_lid_notify_state(device, 1); + break; + case ACPI_BUTTON_LID_INIT_METHOD: + (void)acpi_lid_update_state(device); + break; + case ACPI_BUTTON_LID_INIT_IGNORE: + default: + break; } - return ret; } static void acpi_button_notify(struct acpi_device *device, u32 event) @@ -290,7 +321,7 @@ static void acpi_button_notify(struct acpi_device *device, u32 event) case ACPI_BUTTON_NOTIFY_STATUS: input = button->input; if (button->type == ACPI_BUTTON_TYPE_LID) { - acpi_lid_send_state(device); + acpi_lid_update_state(device); } else { int keycode; @@ -335,7 +366,7 @@ static int acpi_button_resume(struct device *dev) button->suspended = false; if (button->type == ACPI_BUTTON_TYPE_LID) - return acpi_lid_send_state(device); + acpi_lid_initialize_state(device); return 0; } #endif @@ -416,7 +447,7 @@ static int acpi_button_add(struct acpi_device *device) if (error) goto err_remove_fs; if (button->type == ACPI_BUTTON_TYPE_LID) { - acpi_lid_send_state(device); + acpi_lid_initialize_state(device); /* * This assumes there's only one lid device, or if there are * more we only care about the last one... @@ -446,4 +477,42 @@ static int acpi_button_remove(struct acpi_device *device) return 0; } +static int param_set_lid_init_state(const char *val, struct kernel_param *kp) +{ + int result = 0; + + if (!strncmp(val, "open", sizeof("open") - 1)) { + lid_init_state = ACPI_BUTTON_LID_INIT_OPEN; + pr_info("Notify initial lid state as open\n"); + } else if (!strncmp(val, "method", sizeof("method") - 1)) { + lid_init_state = ACPI_BUTTON_LID_INIT_METHOD; + pr_info("Notify initial lid state with _LID return value\n"); + } else if (!strncmp(val, "ignore", sizeof("ignore") - 1)) { + lid_init_state = ACPI_BUTTON_LID_INIT_IGNORE; + pr_info("Do not notify initial lid state\n"); + } else + result = -EINVAL; + return result; +} + +static int param_get_lid_init_state(char *buffer, struct kernel_param *kp) +{ + switch (lid_init_state) { + case ACPI_BUTTON_LID_INIT_OPEN: + return sprintf(buffer, "open"); + case ACPI_BUTTON_LID_INIT_METHOD: + return sprintf(buffer, "method"); + case ACPI_BUTTON_LID_INIT_IGNORE: + return sprintf(buffer, "ignore"); + default: + return sprintf(buffer, "invalid"); + } + return 0; +} + +module_param_call(lid_init_state, + param_set_lid_init_state, param_get_lid_init_state, + NULL, 0644); +MODULE_PARM_DESC(lid_init_state, "Behavior for reporting LID initial state"); + module_acpi_driver(acpi_button_driver); diff --git a/drivers/acpi/dock.c b/drivers/acpi/dock.c index e8e128ded..0c00208b4 100644 --- a/drivers/acpi/dock.c +++ b/drivers/acpi/dock.c @@ -21,7 +21,7 @@ */ #include -#include +#include #include #include #include @@ -33,12 +33,7 @@ #include "internal.h" -#define ACPI_DOCK_DRIVER_DESCRIPTION "ACPI Dock Station Driver" - ACPI_MODULE_NAME("dock"); -MODULE_AUTHOR("Kristen Carlson Accardi"); -MODULE_DESCRIPTION(ACPI_DOCK_DRIVER_DESCRIPTION); -MODULE_LICENSE("GPL"); static bool immediate_undock = 1; module_param(immediate_undock, bool, 0644); diff --git a/drivers/acpi/dptf/Kconfig b/drivers/acpi/dptf/Kconfig new file mode 100644 index 000000000..ac0a6ed0c --- /dev/null +++ b/drivers/acpi/dptf/Kconfig @@ -0,0 +1,15 @@ +config DPTF_POWER + tristate "DPTF Platform Power Participant" + depends on X86 + help + This driver adds support for Dynamic Platform and Thermal Framework + (DPTF) Platform Power Participant device (INT3407) support. + This participant is responsible for exposing platform telemetry: + max_platform_power + platform_power_source + adapter_rating + battery_steady_power + charger_type + + To compile this driver as a module, choose M here: + the module will be called dptf_power. diff --git a/drivers/acpi/dptf/Makefile b/drivers/acpi/dptf/Makefile new file mode 100644 index 000000000..06ea88095 --- /dev/null +++ b/drivers/acpi/dptf/Makefile @@ -0,0 +1,4 @@ +obj-$(CONFIG_ACPI) += int340x_thermal.o +obj-$(CONFIG_DPTF_POWER) += dptf_power.o + +ccflags-y += -Idrivers/acpi diff --git a/drivers/acpi/dptf/dptf_power.c b/drivers/acpi/dptf/dptf_power.c new file mode 100644 index 000000000..734642dc5 --- /dev/null +++ b/drivers/acpi/dptf/dptf_power.c @@ -0,0 +1,128 @@ +/* + * dptf_power: DPTF platform power driver + * Copyright (c) 2016, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ + +#include +#include +#include +#include + +/* + * Presentation of attributes which are defined for INT3407. They are: + * PMAX : Maximum platform powe + * PSRC : Platform power source + * ARTG : Adapter rating + * CTYP : Charger type + * PBSS : Battery steady power + */ +#define DPTF_POWER_SHOW(name, object) \ +static ssize_t name##_show(struct device *dev,\ + struct device_attribute *attr,\ + char *buf)\ +{\ + struct platform_device *pdev = to_platform_device(dev);\ + struct acpi_device *acpi_dev = platform_get_drvdata(pdev);\ + unsigned long long val;\ + acpi_status status;\ +\ + status = acpi_evaluate_integer(acpi_dev->handle, #object,\ + NULL, &val);\ + if (ACPI_SUCCESS(status))\ + return sprintf(buf, "%d\n", (int)val);\ + else \ + return -EINVAL;\ +} + +DPTF_POWER_SHOW(max_platform_power_mw, PMAX) +DPTF_POWER_SHOW(platform_power_source, PSRC) +DPTF_POWER_SHOW(adapter_rating_mw, ARTG) +DPTF_POWER_SHOW(battery_steady_power_mw, PBSS) +DPTF_POWER_SHOW(charger_type, CTYP) + +static DEVICE_ATTR_RO(max_platform_power_mw); +static DEVICE_ATTR_RO(platform_power_source); +static DEVICE_ATTR_RO(adapter_rating_mw); +static DEVICE_ATTR_RO(battery_steady_power_mw); +static DEVICE_ATTR_RO(charger_type); + +static struct attribute *dptf_power_attrs[] = { + &dev_attr_max_platform_power_mw.attr, + &dev_attr_platform_power_source.attr, + &dev_attr_adapter_rating_mw.attr, + &dev_attr_battery_steady_power_mw.attr, + &dev_attr_charger_type.attr, + NULL +}; + +static struct attribute_group dptf_power_attribute_group = { + .attrs = dptf_power_attrs, + .name = "dptf_power" +}; + +static int dptf_power_add(struct platform_device *pdev) +{ + struct acpi_device *acpi_dev; + acpi_status status; + unsigned long long ptype; + int result; + + acpi_dev = ACPI_COMPANION(&(pdev->dev)); + if (!acpi_dev) + return -ENODEV; + + status = acpi_evaluate_integer(acpi_dev->handle, "PTYP", NULL, &ptype); + if (ACPI_FAILURE(status)) + return -ENODEV; + + if (ptype != 0x11) + return -ENODEV; + + result = sysfs_create_group(&pdev->dev.kobj, + &dptf_power_attribute_group); + if (result) + return result; + + platform_set_drvdata(pdev, acpi_dev); + + return 0; +} + +static int dptf_power_remove(struct platform_device *pdev) +{ + + sysfs_remove_group(&pdev->dev.kobj, &dptf_power_attribute_group); + + return 0; +} + +static const struct acpi_device_id int3407_device_ids[] = { + {"INT3407", 0}, + {"", 0}, +}; +MODULE_DEVICE_TABLE(acpi, int3407_device_ids); + +static struct platform_driver dptf_power_driver = { + .probe = dptf_power_add, + .remove = dptf_power_remove, + .driver = { + .name = "DPTF Platform Power", + .acpi_match_table = int3407_device_ids, + }, +}; + +module_platform_driver(dptf_power_driver); + +MODULE_AUTHOR("Srinivas Pandruvada "); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ACPI DPTF platform power driver"); diff --git a/drivers/acpi/dptf/int340x_thermal.c b/drivers/acpi/dptf/int340x_thermal.c new file mode 100644 index 000000000..33505c651 --- /dev/null +++ b/drivers/acpi/dptf/int340x_thermal.c @@ -0,0 +1,53 @@ +/* + * ACPI support for int340x thermal drivers + * + * Copyright (C) 2014, Intel Corporation + * Authors: Zhang Rui + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include + +#include "internal.h" + +#define INT3401_DEVICE 0X01 +static const struct acpi_device_id int340x_thermal_device_ids[] = { + {"INT3400"}, + {"INT3401", INT3401_DEVICE}, + {"INT3402"}, + {"INT3403"}, + {"INT3404"}, + {"INT3406"}, + {"INT3407"}, + {"INT3408"}, + {"INT3409"}, + {"INT340A"}, + {"INT340B"}, + {""}, +}; + +static int int340x_thermal_handler_attach(struct acpi_device *adev, + const struct acpi_device_id *id) +{ + if (IS_ENABLED(CONFIG_INT340X_THERMAL)) + acpi_create_platform_device(adev); + /* Intel SoC DTS thermal driver needs INT3401 to set IRQ descriptor */ + else if (IS_ENABLED(CONFIG_INTEL_SOC_DTS_THERMAL) && + id->driver_data == INT3401_DEVICE) + acpi_create_platform_device(adev); + return 1; +} + +static struct acpi_scan_handler int340x_thermal_handler = { + .ids = int340x_thermal_device_ids, + .attach = int340x_thermal_handler_attach, +}; + +void __init acpi_int340x_thermal_init(void) +{ + acpi_scan_add_handler(&int340x_thermal_handler); +} diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index f4218df00..e7bd57cc5 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -1365,13 +1365,9 @@ static void ec_remove_handlers(struct acpi_ec *ec) } } -static int acpi_ec_add(struct acpi_device *device) +static struct acpi_ec *acpi_ec_alloc(void) { - struct acpi_ec *ec = NULL; - int ret; - - strcpy(acpi_device_name(device), ACPI_EC_DEVICE_NAME); - strcpy(acpi_device_class(device), ACPI_EC_CLASS); + struct acpi_ec *ec; /* Check for boot EC */ if (boot_ec) { @@ -1382,9 +1378,21 @@ static int acpi_ec_add(struct acpi_device *device) first_ec = NULL; } else { ec = make_acpi_ec(); - if (!ec) - return -ENOMEM; } + return ec; +} + +static int acpi_ec_add(struct acpi_device *device) +{ + struct acpi_ec *ec = NULL; + int ret; + + strcpy(acpi_device_name(device), ACPI_EC_DEVICE_NAME); + strcpy(acpi_device_class(device), ACPI_EC_CLASS); + + ec = acpi_ec_alloc(); + if (!ec) + return -ENOMEM; if (ec_parse_device(device->handle, 0, ec, NULL) != AE_CTRL_TERMINATE) { kfree(ec); @@ -1471,27 +1479,31 @@ static const struct acpi_device_id ec_device_ids[] = { int __init acpi_ec_dsdt_probe(void) { acpi_status status; + struct acpi_ec *ec; + int ret; - if (boot_ec) - return 0; - + ec = acpi_ec_alloc(); + if (!ec) + return -ENOMEM; /* * Finding EC from DSDT if there is no ECDT EC available. When this * function is invoked, ACPI tables have been fully loaded, we can * walk namespace now. */ - boot_ec = make_acpi_ec(); - if (!boot_ec) - return -ENOMEM; status = acpi_get_devices(ec_device_ids[0].id, - ec_parse_device, boot_ec, NULL); - if (ACPI_FAILURE(status) || !boot_ec->handle) - return -ENODEV; - if (!ec_install_handlers(boot_ec)) { - first_ec = boot_ec; - return 0; + ec_parse_device, ec, NULL); + if (ACPI_FAILURE(status) || !ec->handle) { + ret = -ENODEV; + goto error; } - return -EFAULT; + ret = ec_install_handlers(ec); + +error: + if (ret) + kfree(ec); + else + first_ec = boot_ec = ec; + return ret; } #if 0 @@ -1535,6 +1547,11 @@ static int ec_clear_on_resume(const struct dmi_system_id *id) return 0; } +/* + * Some ECDTs contain wrong register addresses. + * MSI MS-171F + * https://bugzilla.kernel.org/show_bug.cgi?id=12461 + */ static int ec_correct_ecdt(const struct dmi_system_id *id) { pr_debug("Detected system needing ECDT address correction.\n"); @@ -1543,16 +1560,6 @@ static int ec_correct_ecdt(const struct dmi_system_id *id) } static struct dmi_system_id ec_dmi_table[] __initdata = { - { - ec_correct_ecdt, "Asus L4R", { - DMI_MATCH(DMI_BIOS_VERSION, "1008.006"), - DMI_MATCH(DMI_PRODUCT_NAME, "L4R"), - DMI_MATCH(DMI_BOARD_NAME, "L4R") }, NULL}, - { - ec_correct_ecdt, "Asus M6R", { - DMI_MATCH(DMI_BIOS_VERSION, "0207"), - DMI_MATCH(DMI_PRODUCT_NAME, "M6R"), - DMI_MATCH(DMI_BOARD_NAME, "M6R") }, NULL}, { ec_correct_ecdt, "MSI MS-171F", { DMI_MATCH(DMI_SYS_VENDOR, "Micro-Star"), @@ -1565,12 +1572,13 @@ static struct dmi_system_id ec_dmi_table[] __initdata = { int __init acpi_ec_ecdt_probe(void) { - int ret = 0; + int ret; acpi_status status; struct acpi_table_ecdt *ecdt_ptr; + struct acpi_ec *ec; - boot_ec = make_acpi_ec(); - if (!boot_ec) + ec = acpi_ec_alloc(); + if (!ec) return -ENOMEM; /* * Generate a boot ec context @@ -1594,28 +1602,20 @@ int __init acpi_ec_ecdt_probe(void) pr_info("EC description table is found, configuring boot EC\n"); if (EC_FLAGS_CORRECT_ECDT) { - /* - * Asus L4R, Asus M6R - * https://bugzilla.kernel.org/show_bug.cgi?id=9399 - * MSI MS-171F - * https://bugzilla.kernel.org/show_bug.cgi?id=12461 - */ - boot_ec->command_addr = ecdt_ptr->data.address; - boot_ec->data_addr = ecdt_ptr->control.address; + ec->command_addr = ecdt_ptr->data.address; + ec->data_addr = ecdt_ptr->control.address; } else { - boot_ec->command_addr = ecdt_ptr->control.address; - boot_ec->data_addr = ecdt_ptr->data.address; + ec->command_addr = ecdt_ptr->control.address; + ec->data_addr = ecdt_ptr->data.address; } - boot_ec->gpe = ecdt_ptr->gpe; - boot_ec->handle = ACPI_ROOT_OBJECT; - ret = ec_install_handlers(boot_ec); - if (!ret) - first_ec = boot_ec; + ec->gpe = ecdt_ptr->gpe; + ec->handle = ACPI_ROOT_OBJECT; + ret = ec_install_handlers(ec); error: - if (ret) { - kfree(boot_ec); - boot_ec = NULL; - } + if (ret) + kfree(ec); + else + first_ec = boot_ec = ec; return ret; } diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h index 27cc7feab..940218ff0 100644 --- a/drivers/acpi/internal.h +++ b/drivers/acpi/internal.h @@ -87,6 +87,9 @@ bool acpi_queue_hotplug_work(struct work_struct *work); void acpi_device_hotplug(struct acpi_device *adev, u32 src); bool acpi_scan_is_offline(struct acpi_device *adev, bool uevent); +acpi_status acpi_sysfs_table_handler(u32 event, void *table, void *context); +void acpi_scan_table_handler(u32 event, void *table, void *context); + /* -------------------------------------------------------------------------- Device Node Initialization / Removal -------------------------------------------------------------------------- */ diff --git a/drivers/acpi/nfit/Kconfig b/drivers/acpi/nfit/Kconfig new file mode 100644 index 000000000..dd0d53c52 --- /dev/null +++ b/drivers/acpi/nfit/Kconfig @@ -0,0 +1,26 @@ +config ACPI_NFIT + tristate "ACPI NVDIMM Firmware Interface Table (NFIT)" + depends on PHYS_ADDR_T_64BIT + depends on BLK_DEV + depends on ARCH_HAS_MMIO_FLUSH + select LIBNVDIMM + help + Infrastructure to probe ACPI 6 compliant platforms for + NVDIMMs (NFIT) and register a libnvdimm device tree. In + addition to storage devices this also enables libnvdimm to pass + ACPI._DSM messages for platform/dimm configuration. + + To compile this driver as a module, choose M here: + the module will be called nfit. + +config ACPI_NFIT_DEBUG + bool "NFIT DSM debug" + depends on ACPI_NFIT + depends on DYNAMIC_DEBUG + default n + help + Enabling this option causes the nfit driver to dump the + input and output buffers of _DSM operations on the ACPI0012 + device and its children. This can be very verbose, so leave + it disabled unless you are debugging a hardware / firmware + issue. diff --git a/drivers/acpi/nfit/Makefile b/drivers/acpi/nfit/Makefile new file mode 100644 index 000000000..a407e769f --- /dev/null +++ b/drivers/acpi/nfit/Makefile @@ -0,0 +1,3 @@ +obj-$(CONFIG_ACPI_NFIT) := nfit.o +nfit-y := core.o +nfit-$(CONFIG_X86_MCE) += mce.o diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c new file mode 100644 index 000000000..e1d5ea6d5 --- /dev/null +++ b/drivers/acpi/nfit/core.c @@ -0,0 +1,2793 @@ +/* + * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nfit.h" + +/* + * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is + * irrelevant. + */ +#include + +static bool force_enable_dimms; +module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR); +MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status"); + +static unsigned int scrub_timeout = NFIT_ARS_TIMEOUT; +module_param(scrub_timeout, uint, S_IRUGO|S_IWUSR); +MODULE_PARM_DESC(scrub_timeout, "Initial scrub timeout in seconds"); + +/* after three payloads of overflow, it's dead jim */ +static unsigned int scrub_overflow_abort = 3; +module_param(scrub_overflow_abort, uint, S_IRUGO|S_IWUSR); +MODULE_PARM_DESC(scrub_overflow_abort, + "Number of times we overflow ARS results before abort"); + +static bool disable_vendor_specific; +module_param(disable_vendor_specific, bool, S_IRUGO); +MODULE_PARM_DESC(disable_vendor_specific, + "Limit commands to the publicly specified set\n"); + +LIST_HEAD(acpi_descs); +DEFINE_MUTEX(acpi_desc_lock); + +static struct workqueue_struct *nfit_wq; + +struct nfit_table_prev { + struct list_head spas; + struct list_head memdevs; + struct list_head dcrs; + struct list_head bdws; + struct list_head idts; + struct list_head flushes; +}; + +static u8 nfit_uuid[NFIT_UUID_MAX][16]; + +const u8 *to_nfit_uuid(enum nfit_uuids id) +{ + return nfit_uuid[id]; +} +EXPORT_SYMBOL(to_nfit_uuid); + +static struct acpi_nfit_desc *to_acpi_nfit_desc( + struct nvdimm_bus_descriptor *nd_desc) +{ + return container_of(nd_desc, struct acpi_nfit_desc, nd_desc); +} + +static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc) +{ + struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; + + /* + * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct + * acpi_device. + */ + if (!nd_desc->provider_name + || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0) + return NULL; + + return to_acpi_device(acpi_desc->dev); +} + +static int xlat_status(void *buf, unsigned int cmd, u32 status) +{ + struct nd_cmd_clear_error *clear_err; + struct nd_cmd_ars_status *ars_status; + u16 flags; + + switch (cmd) { + case ND_CMD_ARS_CAP: + if ((status & 0xffff) == NFIT_ARS_CAP_NONE) + return -ENOTTY; + + /* Command failed */ + if (status & 0xffff) + return -EIO; + + /* No supported scan types for this range */ + flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE; + if ((status >> 16 & flags) == 0) + return -ENOTTY; + break; + case ND_CMD_ARS_START: + /* ARS is in progress */ + if ((status & 0xffff) == NFIT_ARS_START_BUSY) + return -EBUSY; + + /* Command failed */ + if (status & 0xffff) + return -EIO; + break; + case ND_CMD_ARS_STATUS: + ars_status = buf; + /* Command failed */ + if (status & 0xffff) + return -EIO; + /* Check extended status (Upper two bytes) */ + if (status == NFIT_ARS_STATUS_DONE) + return 0; + + /* ARS is in progress */ + if (status == NFIT_ARS_STATUS_BUSY) + return -EBUSY; + + /* No ARS performed for the current boot */ + if (status == NFIT_ARS_STATUS_NONE) + return -EAGAIN; + + /* + * ARS interrupted, either we overflowed or some other + * agent wants the scan to stop. If we didn't overflow + * then just continue with the returned results. + */ + if (status == NFIT_ARS_STATUS_INTR) { + if (ars_status->flags & NFIT_ARS_F_OVERFLOW) + return -ENOSPC; + return 0; + } + + /* Unknown status */ + if (status >> 16) + return -EIO; + break; + case ND_CMD_CLEAR_ERROR: + clear_err = buf; + if (status & 0xffff) + return -EIO; + if (!clear_err->cleared) + return -EIO; + if (clear_err->length > clear_err->cleared) + return clear_err->cleared; + break; + default: + break; + } + + /* all other non-zero status results in an error */ + if (status) + return -EIO; + return 0; +} + +static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, + struct nvdimm *nvdimm, unsigned int cmd, void *buf, + unsigned int buf_len, int *cmd_rc) +{ + struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); + union acpi_object in_obj, in_buf, *out_obj; + const struct nd_cmd_desc *desc = NULL; + struct device *dev = acpi_desc->dev; + struct nd_cmd_pkg *call_pkg = NULL; + const char *cmd_name, *dimm_name; + unsigned long cmd_mask, dsm_mask; + u32 offset, fw_status = 0; + acpi_handle handle; + unsigned int func; + const u8 *uuid; + int rc, i; + + func = cmd; + if (cmd == ND_CMD_CALL) { + call_pkg = buf; + func = call_pkg->nd_command; + } + + if (nvdimm) { + struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); + struct acpi_device *adev = nfit_mem->adev; + + if (!adev) + return -ENOTTY; + if (call_pkg && nfit_mem->family != call_pkg->nd_family) + return -ENOTTY; + + dimm_name = nvdimm_name(nvdimm); + cmd_name = nvdimm_cmd_name(cmd); + cmd_mask = nvdimm_cmd_mask(nvdimm); + dsm_mask = nfit_mem->dsm_mask; + desc = nd_cmd_dimm_desc(cmd); + uuid = to_nfit_uuid(nfit_mem->family); + handle = adev->handle; + } else { + struct acpi_device *adev = to_acpi_dev(acpi_desc); + + cmd_name = nvdimm_bus_cmd_name(cmd); + cmd_mask = nd_desc->cmd_mask; + dsm_mask = cmd_mask; + desc = nd_cmd_bus_desc(cmd); + uuid = to_nfit_uuid(NFIT_DEV_BUS); + handle = adev->handle; + dimm_name = "bus"; + } + + if (!desc || (cmd && (desc->out_num + desc->in_num == 0))) + return -ENOTTY; + + if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask)) + return -ENOTTY; + + in_obj.type = ACPI_TYPE_PACKAGE; + in_obj.package.count = 1; + in_obj.package.elements = &in_buf; + in_buf.type = ACPI_TYPE_BUFFER; + in_buf.buffer.pointer = buf; + in_buf.buffer.length = 0; + + /* libnvdimm has already validated the input envelope */ + for (i = 0; i < desc->in_num; i++) + in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc, + i, buf); + + if (call_pkg) { + /* skip over package wrapper */ + in_buf.buffer.pointer = (void *) &call_pkg->nd_payload; + in_buf.buffer.length = call_pkg->nd_size_in; + } + + if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) { + dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n", + __func__, dimm_name, cmd, func, + in_buf.buffer.length); + print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4, + in_buf.buffer.pointer, + min_t(u32, 256, in_buf.buffer.length), true); + } + + out_obj = acpi_evaluate_dsm(handle, uuid, 1, func, &in_obj); + if (!out_obj) { + dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name, + cmd_name); + return -EINVAL; + } + + if (call_pkg) { + call_pkg->nd_fw_size = out_obj->buffer.length; + memcpy(call_pkg->nd_payload + call_pkg->nd_size_in, + out_obj->buffer.pointer, + min(call_pkg->nd_fw_size, call_pkg->nd_size_out)); + + ACPI_FREE(out_obj); + /* + * Need to support FW function w/o known size in advance. + * Caller can determine required size based upon nd_fw_size. + * If we return an error (like elsewhere) then caller wouldn't + * be able to rely upon data returned to make calculation. + */ + return 0; + } + + if (out_obj->package.type != ACPI_TYPE_BUFFER) { + dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n", + __func__, dimm_name, cmd_name, out_obj->type); + rc = -EINVAL; + goto out; + } + + if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) { + dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__, + dimm_name, cmd_name, out_obj->buffer.length); + print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, + 4, out_obj->buffer.pointer, min_t(u32, 128, + out_obj->buffer.length), true); + } + + for (i = 0, offset = 0; i < desc->out_num; i++) { + u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf, + (u32 *) out_obj->buffer.pointer); + + if (offset + out_size > out_obj->buffer.length) { + dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n", + __func__, dimm_name, cmd_name, i); + break; + } + + if (in_buf.buffer.length + offset + out_size > buf_len) { + dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n", + __func__, dimm_name, cmd_name, i); + rc = -ENXIO; + goto out; + } + memcpy(buf + in_buf.buffer.length + offset, + out_obj->buffer.pointer + offset, out_size); + offset += out_size; + } + + /* + * Set fw_status for all the commands with a known format to be + * later interpreted by xlat_status(). + */ + if (i >= 1 && ((cmd >= ND_CMD_ARS_CAP && cmd <= ND_CMD_CLEAR_ERROR) + || (cmd >= ND_CMD_SMART && cmd <= ND_CMD_VENDOR))) + fw_status = *(u32 *) out_obj->buffer.pointer; + + if (offset + in_buf.buffer.length < buf_len) { + if (i >= 1) { + /* + * status valid, return the number of bytes left + * unfilled in the output buffer + */ + rc = buf_len - offset - in_buf.buffer.length; + if (cmd_rc) + *cmd_rc = xlat_status(buf, cmd, fw_status); + } else { + dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n", + __func__, dimm_name, cmd_name, buf_len, + offset); + rc = -ENXIO; + } + } else { + rc = 0; + if (cmd_rc) + *cmd_rc = xlat_status(buf, cmd, fw_status); + } + + out: + ACPI_FREE(out_obj); + + return rc; +} + +static const char *spa_type_name(u16 type) +{ + static const char *to_name[] = { + [NFIT_SPA_VOLATILE] = "volatile", + [NFIT_SPA_PM] = "pmem", + [NFIT_SPA_DCR] = "dimm-control-region", + [NFIT_SPA_BDW] = "block-data-window", + [NFIT_SPA_VDISK] = "volatile-disk", + [NFIT_SPA_VCD] = "volatile-cd", + [NFIT_SPA_PDISK] = "persistent-disk", + [NFIT_SPA_PCD] = "persistent-cd", + + }; + + if (type > NFIT_SPA_PCD) + return "unknown"; + + return to_name[type]; +} + +int nfit_spa_type(struct acpi_nfit_system_address *spa) +{ + int i; + + for (i = 0; i < NFIT_UUID_MAX; i++) + if (memcmp(to_nfit_uuid(i), spa->range_guid, 16) == 0) + return i; + return -1; +} + +static bool add_spa(struct acpi_nfit_desc *acpi_desc, + struct nfit_table_prev *prev, + struct acpi_nfit_system_address *spa) +{ + struct device *dev = acpi_desc->dev; + struct nfit_spa *nfit_spa; + + if (spa->header.length != sizeof(*spa)) + return false; + + list_for_each_entry(nfit_spa, &prev->spas, list) { + if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) { + list_move_tail(&nfit_spa->list, &acpi_desc->spas); + return true; + } + } + + nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa), + GFP_KERNEL); + if (!nfit_spa) + return false; + INIT_LIST_HEAD(&nfit_spa->list); + memcpy(nfit_spa->spa, spa, sizeof(*spa)); + list_add_tail(&nfit_spa->list, &acpi_desc->spas); + dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__, + spa->range_index, + spa_type_name(nfit_spa_type(spa))); + return true; +} + +static bool add_memdev(struct acpi_nfit_desc *acpi_desc, + struct nfit_table_prev *prev, + struct acpi_nfit_memory_map *memdev) +{ + struct device *dev = acpi_desc->dev; + struct nfit_memdev *nfit_memdev; + + if (memdev->header.length != sizeof(*memdev)) + return false; + + list_for_each_entry(nfit_memdev, &prev->memdevs, list) + if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) { + list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs); + return true; + } + + nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev), + GFP_KERNEL); + if (!nfit_memdev) + return false; + INIT_LIST_HEAD(&nfit_memdev->list); + memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev)); + list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs); + dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d\n", + __func__, memdev->device_handle, memdev->range_index, + memdev->region_index); + return true; +} + +/* + * An implementation may provide a truncated control region if no block windows + * are defined. + */ +static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr) +{ + if (dcr->header.length < offsetof(struct acpi_nfit_control_region, + window_size)) + return 0; + if (dcr->windows) + return sizeof(*dcr); + return offsetof(struct acpi_nfit_control_region, window_size); +} + +static bool add_dcr(struct acpi_nfit_desc *acpi_desc, + struct nfit_table_prev *prev, + struct acpi_nfit_control_region *dcr) +{ + struct device *dev = acpi_desc->dev; + struct nfit_dcr *nfit_dcr; + + if (!sizeof_dcr(dcr)) + return false; + + list_for_each_entry(nfit_dcr, &prev->dcrs, list) + if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) { + list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs); + return true; + } + + nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr), + GFP_KERNEL); + if (!nfit_dcr) + return false; + INIT_LIST_HEAD(&nfit_dcr->list); + memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)); + list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs); + dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__, + dcr->region_index, dcr->windows); + return true; +} + +static bool add_bdw(struct acpi_nfit_desc *acpi_desc, + struct nfit_table_prev *prev, + struct acpi_nfit_data_region *bdw) +{ + struct device *dev = acpi_desc->dev; + struct nfit_bdw *nfit_bdw; + + if (bdw->header.length != sizeof(*bdw)) + return false; + list_for_each_entry(nfit_bdw, &prev->bdws, list) + if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) { + list_move_tail(&nfit_bdw->list, &acpi_desc->bdws); + return true; + } + + nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw), + GFP_KERNEL); + if (!nfit_bdw) + return false; + INIT_LIST_HEAD(&nfit_bdw->list); + memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw)); + list_add_tail(&nfit_bdw->list, &acpi_desc->bdws); + dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__, + bdw->region_index, bdw->windows); + return true; +} + +static size_t sizeof_idt(struct acpi_nfit_interleave *idt) +{ + if (idt->header.length < sizeof(*idt)) + return 0; + return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1); +} + +static bool add_idt(struct acpi_nfit_desc *acpi_desc, + struct nfit_table_prev *prev, + struct acpi_nfit_interleave *idt) +{ + struct device *dev = acpi_desc->dev; + struct nfit_idt *nfit_idt; + + if (!sizeof_idt(idt)) + return false; + + list_for_each_entry(nfit_idt, &prev->idts, list) { + if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt)) + continue; + + if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) { + list_move_tail(&nfit_idt->list, &acpi_desc->idts); + return true; + } + } + + nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt), + GFP_KERNEL); + if (!nfit_idt) + return false; + INIT_LIST_HEAD(&nfit_idt->list); + memcpy(nfit_idt->idt, idt, sizeof_idt(idt)); + list_add_tail(&nfit_idt->list, &acpi_desc->idts); + dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__, + idt->interleave_index, idt->line_count); + return true; +} + +static size_t sizeof_flush(struct acpi_nfit_flush_address *flush) +{ + if (flush->header.length < sizeof(*flush)) + return 0; + return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1); +} + +static bool add_flush(struct acpi_nfit_desc *acpi_desc, + struct nfit_table_prev *prev, + struct acpi_nfit_flush_address *flush) +{ + struct device *dev = acpi_desc->dev; + struct nfit_flush *nfit_flush; + + if (!sizeof_flush(flush)) + return false; + + list_for_each_entry(nfit_flush, &prev->flushes, list) { + if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush)) + continue; + + if (memcmp(nfit_flush->flush, flush, + sizeof_flush(flush)) == 0) { + list_move_tail(&nfit_flush->list, &acpi_desc->flushes); + return true; + } + } + + nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush) + + sizeof_flush(flush), GFP_KERNEL); + if (!nfit_flush) + return false; + INIT_LIST_HEAD(&nfit_flush->list); + memcpy(nfit_flush->flush, flush, sizeof_flush(flush)); + list_add_tail(&nfit_flush->list, &acpi_desc->flushes); + dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__, + flush->device_handle, flush->hint_count); + return true; +} + +static void *add_table(struct acpi_nfit_desc *acpi_desc, + struct nfit_table_prev *prev, void *table, const void *end) +{ + struct device *dev = acpi_desc->dev; + struct acpi_nfit_header *hdr; + void *err = ERR_PTR(-ENOMEM); + + if (table >= end) + return NULL; + + hdr = table; + if (!hdr->length) { + dev_warn(dev, "found a zero length table '%d' parsing nfit\n", + hdr->type); + return NULL; + } + + switch (hdr->type) { + case ACPI_NFIT_TYPE_SYSTEM_ADDRESS: + if (!add_spa(acpi_desc, prev, table)) + return err; + break; + case ACPI_NFIT_TYPE_MEMORY_MAP: + if (!add_memdev(acpi_desc, prev, table)) + return err; + break; + case ACPI_NFIT_TYPE_CONTROL_REGION: + if (!add_dcr(acpi_desc, prev, table)) + return err; + break; + case ACPI_NFIT_TYPE_DATA_REGION: + if (!add_bdw(acpi_desc, prev, table)) + return err; + break; + case ACPI_NFIT_TYPE_INTERLEAVE: + if (!add_idt(acpi_desc, prev, table)) + return err; + break; + case ACPI_NFIT_TYPE_FLUSH_ADDRESS: + if (!add_flush(acpi_desc, prev, table)) + return err; + break; + case ACPI_NFIT_TYPE_SMBIOS: + dev_dbg(dev, "%s: smbios\n", __func__); + break; + default: + dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type); + break; + } + + return table + hdr->length; +} + +static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc, + struct nfit_mem *nfit_mem) +{ + u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle; + u16 dcr = nfit_mem->dcr->region_index; + struct nfit_spa *nfit_spa; + + list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { + u16 range_index = nfit_spa->spa->range_index; + int type = nfit_spa_type(nfit_spa->spa); + struct nfit_memdev *nfit_memdev; + + if (type != NFIT_SPA_BDW) + continue; + + list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { + if (nfit_memdev->memdev->range_index != range_index) + continue; + if (nfit_memdev->memdev->device_handle != device_handle) + continue; + if (nfit_memdev->memdev->region_index != dcr) + continue; + + nfit_mem->spa_bdw = nfit_spa->spa; + return; + } + } + + dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n", + nfit_mem->spa_dcr->range_index); + nfit_mem->bdw = NULL; +} + +static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc, + struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa) +{ + u16 dcr = __to_nfit_memdev(nfit_mem)->region_index; + struct nfit_memdev *nfit_memdev; + struct nfit_bdw *nfit_bdw; + struct nfit_idt *nfit_idt; + u16 idt_idx, range_index; + + list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) { + if (nfit_bdw->bdw->region_index != dcr) + continue; + nfit_mem->bdw = nfit_bdw->bdw; + break; + } + + if (!nfit_mem->bdw) + return; + + nfit_mem_find_spa_bdw(acpi_desc, nfit_mem); + + if (!nfit_mem->spa_bdw) + return; + + range_index = nfit_mem->spa_bdw->range_index; + list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { + if (nfit_memdev->memdev->range_index != range_index || + nfit_memdev->memdev->region_index != dcr) + continue; + nfit_mem->memdev_bdw = nfit_memdev->memdev; + idt_idx = nfit_memdev->memdev->interleave_index; + list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { + if (nfit_idt->idt->interleave_index != idt_idx) + continue; + nfit_mem->idt_bdw = nfit_idt->idt; + break; + } + break; + } +} + +static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc, + struct acpi_nfit_system_address *spa) +{ + struct nfit_mem *nfit_mem, *found; + struct nfit_memdev *nfit_memdev; + int type = nfit_spa_type(spa); + + switch (type) { + case NFIT_SPA_DCR: + case NFIT_SPA_PM: + break; + default: + return 0; + } + + list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { + struct nfit_flush *nfit_flush; + struct nfit_dcr *nfit_dcr; + u32 device_handle; + u16 dcr; + + if (nfit_memdev->memdev->range_index != spa->range_index) + continue; + found = NULL; + dcr = nfit_memdev->memdev->region_index; + device_handle = nfit_memdev->memdev->device_handle; + list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) + if (__to_nfit_memdev(nfit_mem)->device_handle + == device_handle) { + found = nfit_mem; + break; + } + + if (found) + nfit_mem = found; + else { + nfit_mem = devm_kzalloc(acpi_desc->dev, + sizeof(*nfit_mem), GFP_KERNEL); + if (!nfit_mem) + return -ENOMEM; + INIT_LIST_HEAD(&nfit_mem->list); + nfit_mem->acpi_desc = acpi_desc; + list_add(&nfit_mem->list, &acpi_desc->dimms); + } + + list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { + if (nfit_dcr->dcr->region_index != dcr) + continue; + /* + * Record the control region for the dimm. For + * the ACPI 6.1 case, where there are separate + * control regions for the pmem vs blk + * interfaces, be sure to record the extended + * blk details. + */ + if (!nfit_mem->dcr) + nfit_mem->dcr = nfit_dcr->dcr; + else if (nfit_mem->dcr->windows == 0 + && nfit_dcr->dcr->windows) + nfit_mem->dcr = nfit_dcr->dcr; + break; + } + + list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) { + struct acpi_nfit_flush_address *flush; + u16 i; + + if (nfit_flush->flush->device_handle != device_handle) + continue; + nfit_mem->nfit_flush = nfit_flush; + flush = nfit_flush->flush; + nfit_mem->flush_wpq = devm_kzalloc(acpi_desc->dev, + flush->hint_count + * sizeof(struct resource), GFP_KERNEL); + if (!nfit_mem->flush_wpq) + return -ENOMEM; + for (i = 0; i < flush->hint_count; i++) { + struct resource *res = &nfit_mem->flush_wpq[i]; + + res->start = flush->hint_address[i]; + res->end = res->start + 8 - 1; + } + break; + } + + if (dcr && !nfit_mem->dcr) { + dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n", + spa->range_index, dcr); + return -ENODEV; + } + + if (type == NFIT_SPA_DCR) { + struct nfit_idt *nfit_idt; + u16 idt_idx; + + /* multiple dimms may share a SPA when interleaved */ + nfit_mem->spa_dcr = spa; + nfit_mem->memdev_dcr = nfit_memdev->memdev; + idt_idx = nfit_memdev->memdev->interleave_index; + list_for_each_entry(nfit_idt, &acpi_desc->idts, list) { + if (nfit_idt->idt->interleave_index != idt_idx) + continue; + nfit_mem->idt_dcr = nfit_idt->idt; + break; + } + nfit_mem_init_bdw(acpi_desc, nfit_mem, spa); + } else { + /* + * A single dimm may belong to multiple SPA-PM + * ranges, record at least one in addition to + * any SPA-DCR range. + */ + nfit_mem->memdev_pmem = nfit_memdev->memdev; + } + } + + return 0; +} + +static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b) +{ + struct nfit_mem *a = container_of(_a, typeof(*a), list); + struct nfit_mem *b = container_of(_b, typeof(*b), list); + u32 handleA, handleB; + + handleA = __to_nfit_memdev(a)->device_handle; + handleB = __to_nfit_memdev(b)->device_handle; + if (handleA < handleB) + return -1; + else if (handleA > handleB) + return 1; + return 0; +} + +static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc) +{ + struct nfit_spa *nfit_spa; + + /* + * For each SPA-DCR or SPA-PMEM address range find its + * corresponding MEMDEV(s). From each MEMDEV find the + * corresponding DCR. Then, if we're operating on a SPA-DCR, + * try to find a SPA-BDW and a corresponding BDW that references + * the DCR. Throw it all into an nfit_mem object. Note, that + * BDWs are optional. + */ + list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { + int rc; + + rc = nfit_mem_dcr_init(acpi_desc, nfit_spa->spa); + if (rc) + return rc; + } + + list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp); + + return 0; +} + +static ssize_t revision_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); + struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); + struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); + + return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision); +} +static DEVICE_ATTR_RO(revision); + +/* + * This shows the number of full Address Range Scrubs that have been + * completed since driver load time. Userspace can wait on this using + * select/poll etc. A '+' at the end indicates an ARS is in progress + */ +static ssize_t scrub_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nvdimm_bus_descriptor *nd_desc; + ssize_t rc = -ENXIO; + + device_lock(dev); + nd_desc = dev_get_drvdata(dev); + if (nd_desc) { + struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); + + rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, + (work_busy(&acpi_desc->work)) ? "+\n" : "\n"); + } + device_unlock(dev); + return rc; +} + +static ssize_t scrub_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t size) +{ + struct nvdimm_bus_descriptor *nd_desc; + ssize_t rc; + long val; + + rc = kstrtol(buf, 0, &val); + if (rc) + return rc; + if (val != 1) + return -EINVAL; + + device_lock(dev); + nd_desc = dev_get_drvdata(dev); + if (nd_desc) { + struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc); + + rc = acpi_nfit_ars_rescan(acpi_desc); + } + device_unlock(dev); + if (rc) + return rc; + return size; +} +static DEVICE_ATTR_RW(scrub); + +static bool ars_supported(struct nvdimm_bus *nvdimm_bus) +{ + struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); + const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START + | 1 << ND_CMD_ARS_STATUS; + + return (nd_desc->cmd_mask & mask) == mask; +} + +static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev); + + if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus)) + return 0; + return a->mode; +} + +static struct attribute *acpi_nfit_attributes[] = { + &dev_attr_revision.attr, + &dev_attr_scrub.attr, + NULL, +}; + +static struct attribute_group acpi_nfit_attribute_group = { + .name = "nfit", + .attrs = acpi_nfit_attributes, + .is_visible = nfit_visible, +}; + +static const struct attribute_group *acpi_nfit_attribute_groups[] = { + &nvdimm_bus_attribute_group, + &acpi_nfit_attribute_group, + NULL, +}; + +static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev) +{ + struct nvdimm *nvdimm = to_nvdimm(dev); + struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); + + return __to_nfit_memdev(nfit_mem); +} + +static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev) +{ + struct nvdimm *nvdimm = to_nvdimm(dev); + struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); + + return nfit_mem->dcr; +} + +static ssize_t handle_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); + + return sprintf(buf, "%#x\n", memdev->device_handle); +} +static DEVICE_ATTR_RO(handle); + +static ssize_t phys_id_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev); + + return sprintf(buf, "%#x\n", memdev->physical_id); +} +static DEVICE_ATTR_RO(phys_id); + +static ssize_t vendor_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); + + return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id)); +} +static DEVICE_ATTR_RO(vendor); + +static ssize_t rev_id_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); + + return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id)); +} +static DEVICE_ATTR_RO(rev_id); + +static ssize_t device_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); + + return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id)); +} +static DEVICE_ATTR_RO(device); + +static ssize_t subsystem_vendor_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); + + return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id)); +} +static DEVICE_ATTR_RO(subsystem_vendor); + +static ssize_t subsystem_rev_id_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); + + return sprintf(buf, "0x%04x\n", + be16_to_cpu(dcr->subsystem_revision_id)); +} +static DEVICE_ATTR_RO(subsystem_rev_id); + +static ssize_t subsystem_device_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); + + return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id)); +} +static DEVICE_ATTR_RO(subsystem_device); + +static int num_nvdimm_formats(struct nvdimm *nvdimm) +{ + struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); + int formats = 0; + + if (nfit_mem->memdev_pmem) + formats++; + if (nfit_mem->memdev_bdw) + formats++; + return formats; +} + +static ssize_t format_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); + + return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code)); +} +static DEVICE_ATTR_RO(format); + +static ssize_t format1_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u32 handle; + ssize_t rc = -ENXIO; + struct nfit_mem *nfit_mem; + struct nfit_memdev *nfit_memdev; + struct acpi_nfit_desc *acpi_desc; + struct nvdimm *nvdimm = to_nvdimm(dev); + struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); + + nfit_mem = nvdimm_provider_data(nvdimm); + acpi_desc = nfit_mem->acpi_desc; + handle = to_nfit_memdev(dev)->device_handle; + + /* assumes DIMMs have at most 2 published interface codes */ + mutex_lock(&acpi_desc->init_mutex); + list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { + struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; + struct nfit_dcr *nfit_dcr; + + if (memdev->device_handle != handle) + continue; + + list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) { + if (nfit_dcr->dcr->region_index != memdev->region_index) + continue; + if (nfit_dcr->dcr->code == dcr->code) + continue; + rc = sprintf(buf, "0x%04x\n", + le16_to_cpu(nfit_dcr->dcr->code)); + break; + } + if (rc != ENXIO) + break; + } + mutex_unlock(&acpi_desc->init_mutex); + return rc; +} +static DEVICE_ATTR_RO(format1); + +static ssize_t formats_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nvdimm *nvdimm = to_nvdimm(dev); + + return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm)); +} +static DEVICE_ATTR_RO(formats); + +static ssize_t serial_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); + + return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number)); +} +static DEVICE_ATTR_RO(serial); + +static ssize_t family_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nvdimm *nvdimm = to_nvdimm(dev); + struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); + + if (nfit_mem->family < 0) + return -ENXIO; + return sprintf(buf, "%d\n", nfit_mem->family); +} +static DEVICE_ATTR_RO(family); + +static ssize_t dsm_mask_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nvdimm *nvdimm = to_nvdimm(dev); + struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); + + if (nfit_mem->family < 0) + return -ENXIO; + return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask); +} +static DEVICE_ATTR_RO(dsm_mask); + +static ssize_t flags_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u16 flags = to_nfit_memdev(dev)->flags; + + return sprintf(buf, "%s%s%s%s%s\n", + flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "", + flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "", + flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "", + flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "", + flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : ""); +} +static DEVICE_ATTR_RO(flags); + +static ssize_t id_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev); + + if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID) + return sprintf(buf, "%04x-%02x-%04x-%08x\n", + be16_to_cpu(dcr->vendor_id), + dcr->manufacturing_location, + be16_to_cpu(dcr->manufacturing_date), + be32_to_cpu(dcr->serial_number)); + else + return sprintf(buf, "%04x-%08x\n", + be16_to_cpu(dcr->vendor_id), + be32_to_cpu(dcr->serial_number)); +} +static DEVICE_ATTR_RO(id); + +static struct attribute *acpi_nfit_dimm_attributes[] = { + &dev_attr_handle.attr, + &dev_attr_phys_id.attr, + &dev_attr_vendor.attr, + &dev_attr_device.attr, + &dev_attr_rev_id.attr, + &dev_attr_subsystem_vendor.attr, + &dev_attr_subsystem_device.attr, + &dev_attr_subsystem_rev_id.attr, + &dev_attr_format.attr, + &dev_attr_formats.attr, + &dev_attr_format1.attr, + &dev_attr_serial.attr, + &dev_attr_flags.attr, + &dev_attr_id.attr, + &dev_attr_family.attr, + &dev_attr_dsm_mask.attr, + NULL, +}; + +static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj, + struct attribute *a, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct nvdimm *nvdimm = to_nvdimm(dev); + + if (!to_nfit_dcr(dev)) + return 0; + if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1) + return 0; + return a->mode; +} + +static struct attribute_group acpi_nfit_dimm_attribute_group = { + .name = "nfit", + .attrs = acpi_nfit_dimm_attributes, + .is_visible = acpi_nfit_dimm_attr_visible, +}; + +static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = { + &nvdimm_attribute_group, + &nd_device_attribute_group, + &acpi_nfit_dimm_attribute_group, + NULL, +}; + +static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc, + u32 device_handle) +{ + struct nfit_mem *nfit_mem; + + list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) + if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle) + return nfit_mem->nvdimm; + + return NULL; +} + +static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, + struct nfit_mem *nfit_mem, u32 device_handle) +{ + struct acpi_device *adev, *adev_dimm; + struct device *dev = acpi_desc->dev; + unsigned long dsm_mask; + const u8 *uuid; + int i; + + /* nfit test assumes 1:1 relationship between commands and dsms */ + nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en; + nfit_mem->family = NVDIMM_FAMILY_INTEL; + adev = to_acpi_dev(acpi_desc); + if (!adev) + return 0; + + adev_dimm = acpi_find_child_device(adev, device_handle, false); + nfit_mem->adev = adev_dimm; + if (!adev_dimm) { + dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n", + device_handle); + return force_enable_dimms ? 0 : -ENODEV; + } + + /* + * Until standardization materializes we need to consider 4 + * different command sets. Note, that checking for function0 (bit0) + * tells us if any commands are reachable through this uuid. + */ + for (i = NVDIMM_FAMILY_INTEL; i <= NVDIMM_FAMILY_MSFT; i++) + if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1)) + break; + + /* limit the supported commands to those that are publicly documented */ + nfit_mem->family = i; + if (nfit_mem->family == NVDIMM_FAMILY_INTEL) { + dsm_mask = 0x3fe; + if (disable_vendor_specific) + dsm_mask &= ~(1 << ND_CMD_VENDOR); + } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) { + dsm_mask = 0x1c3c76; + } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) { + dsm_mask = 0x1fe; + if (disable_vendor_specific) + dsm_mask &= ~(1 << 8); + } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) { + dsm_mask = 0xffffffff; + } else { + dev_dbg(dev, "unknown dimm command family\n"); + nfit_mem->family = -1; + /* DSMs are optional, continue loading the driver... */ + return 0; + } + + uuid = to_nfit_uuid(nfit_mem->family); + for_each_set_bit(i, &dsm_mask, BITS_PER_LONG) + if (acpi_check_dsm(adev_dimm->handle, uuid, 1, 1ULL << i)) + set_bit(i, &nfit_mem->dsm_mask); + + return 0; +} + +static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) +{ + struct nfit_mem *nfit_mem; + int dimm_count = 0; + + list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) { + struct acpi_nfit_flush_address *flush; + unsigned long flags = 0, cmd_mask; + struct nvdimm *nvdimm; + u32 device_handle; + u16 mem_flags; + int rc; + + device_handle = __to_nfit_memdev(nfit_mem)->device_handle; + nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle); + if (nvdimm) { + dimm_count++; + continue; + } + + if (nfit_mem->bdw && nfit_mem->memdev_pmem) + flags |= NDD_ALIASING; + + mem_flags = __to_nfit_memdev(nfit_mem)->flags; + if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED) + flags |= NDD_UNARMED; + + rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle); + if (rc) + continue; + + /* + * TODO: provide translation for non-NVDIMM_FAMILY_INTEL + * devices (i.e. from nd_cmd to acpi_dsm) to standardize the + * userspace interface. + */ + cmd_mask = 1UL << ND_CMD_CALL; + if (nfit_mem->family == NVDIMM_FAMILY_INTEL) + cmd_mask |= nfit_mem->dsm_mask; + + flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush + : NULL; + nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem, + acpi_nfit_dimm_attribute_groups, + flags, cmd_mask, flush ? flush->hint_count : 0, + nfit_mem->flush_wpq); + if (!nvdimm) + return -ENOMEM; + + nfit_mem->nvdimm = nvdimm; + dimm_count++; + + if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0) + continue; + + dev_info(acpi_desc->dev, "%s flags:%s%s%s%s\n", + nvdimm_name(nvdimm), + mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "", + mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"", + mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "", + mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : ""); + + } + + return nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count); +} + +static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc) +{ + struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; + const u8 *uuid = to_nfit_uuid(NFIT_DEV_BUS); + struct acpi_device *adev; + int i; + + nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en; + adev = to_acpi_dev(acpi_desc); + if (!adev) + return; + + for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++) + if (acpi_check_dsm(adev->handle, uuid, 1, 1ULL << i)) + set_bit(i, &nd_desc->cmd_mask); +} + +static ssize_t range_index_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct nd_region *nd_region = to_nd_region(dev); + struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region); + + return sprintf(buf, "%d\n", nfit_spa->spa->range_index); +} +static DEVICE_ATTR_RO(range_index); + +static struct attribute *acpi_nfit_region_attributes[] = { + &dev_attr_range_index.attr, + NULL, +}; + +static struct attribute_group acpi_nfit_region_attribute_group = { + .name = "nfit", + .attrs = acpi_nfit_region_attributes, +}; + +static const struct attribute_group *acpi_nfit_region_attribute_groups[] = { + &nd_region_attribute_group, + &nd_mapping_attribute_group, + &nd_device_attribute_group, + &nd_numa_attribute_group, + &acpi_nfit_region_attribute_group, + NULL, +}; + +/* enough info to uniquely specify an interleave set */ +struct nfit_set_info { + struct nfit_set_info_map { + u64 region_offset; + u32 serial_number; + u32 pad; + } mapping[0]; +}; + +static size_t sizeof_nfit_set_info(int num_mappings) +{ + return sizeof(struct nfit_set_info) + + num_mappings * sizeof(struct nfit_set_info_map); +} + +static int cmp_map(const void *m0, const void *m1) +{ + const struct nfit_set_info_map *map0 = m0; + const struct nfit_set_info_map *map1 = m1; + + return memcmp(&map0->region_offset, &map1->region_offset, + sizeof(u64)); +} + +/* Retrieve the nth entry referencing this spa */ +static struct acpi_nfit_memory_map *memdev_from_spa( + struct acpi_nfit_desc *acpi_desc, u16 range_index, int n) +{ + struct nfit_memdev *nfit_memdev; + + list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) + if (nfit_memdev->memdev->range_index == range_index) + if (n-- == 0) + return nfit_memdev->memdev; + return NULL; +} + +static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc, + struct nd_region_desc *ndr_desc, + struct acpi_nfit_system_address *spa) +{ + int i, spa_type = nfit_spa_type(spa); + struct device *dev = acpi_desc->dev; + struct nd_interleave_set *nd_set; + u16 nr = ndr_desc->num_mappings; + struct nfit_set_info *info; + + if (spa_type == NFIT_SPA_PM || spa_type == NFIT_SPA_VOLATILE) + /* pass */; + else + return 0; + + nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); + if (!nd_set) + return -ENOMEM; + + info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL); + if (!info) + return -ENOMEM; + for (i = 0; i < nr; i++) { + struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i]; + struct nfit_set_info_map *map = &info->mapping[i]; + struct nvdimm *nvdimm = nd_mapping->nvdimm; + struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm); + struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc, + spa->range_index, i); + + if (!memdev || !nfit_mem->dcr) { + dev_err(dev, "%s: failed to find DCR\n", __func__); + return -ENODEV; + } + + map->region_offset = memdev->region_offset; + map->serial_number = nfit_mem->dcr->serial_number; + } + + sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map), + cmp_map, NULL); + nd_set->cookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0); + ndr_desc->nd_set = nd_set; + devm_kfree(dev, info); + + return 0; +} + +static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio) +{ + struct acpi_nfit_interleave *idt = mmio->idt; + u32 sub_line_offset, line_index, line_offset; + u64 line_no, table_skip_count, table_offset; + + line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset); + table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index); + line_offset = idt->line_offset[line_index] + * mmio->line_size; + table_offset = table_skip_count * mmio->table_size; + + return mmio->base_offset + line_offset + table_offset + sub_line_offset; +} + +static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) +{ + struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; + u64 offset = nfit_blk->stat_offset + mmio->size * bw; + const u32 STATUS_MASK = 0x80000037; + + if (mmio->num_lines) + offset = to_interleave_offset(offset, mmio); + + return readl(mmio->addr.base + offset) & STATUS_MASK; +} + +static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, + resource_size_t dpa, unsigned int len, unsigned int write) +{ + u64 cmd, offset; + struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR]; + + enum { + BCW_OFFSET_MASK = (1ULL << 48)-1, + BCW_LEN_SHIFT = 48, + BCW_LEN_MASK = (1ULL << 8) - 1, + BCW_CMD_SHIFT = 56, + }; + + cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK; + len = len >> L1_CACHE_SHIFT; + cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT; + cmd |= ((u64) write) << BCW_CMD_SHIFT; + + offset = nfit_blk->cmd_offset + mmio->size * bw; + if (mmio->num_lines) + offset = to_interleave_offset(offset, mmio); + + writeq(cmd, mmio->addr.base + offset); + nvdimm_flush(nfit_blk->nd_region); + + if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH) + readq(mmio->addr.base + offset); +} + +static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, + resource_size_t dpa, void *iobuf, size_t len, int rw, + unsigned int lane) +{ + struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; + unsigned int copied = 0; + u64 base_offset; + int rc; + + base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES + + lane * mmio->size; + write_blk_ctl(nfit_blk, lane, dpa, len, rw); + while (len) { + unsigned int c; + u64 offset; + + if (mmio->num_lines) { + u32 line_offset; + + offset = to_interleave_offset(base_offset + copied, + mmio); + div_u64_rem(offset, mmio->line_size, &line_offset); + c = min_t(size_t, len, mmio->line_size - line_offset); + } else { + offset = base_offset + nfit_blk->bdw_offset; + c = len; + } + + if (rw) + memcpy_to_pmem(mmio->addr.aperture + offset, + iobuf + copied, c); + else { + if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH) + mmio_flush_range((void __force *) + mmio->addr.aperture + offset, c); + + memcpy_from_pmem(iobuf + copied, + mmio->addr.aperture + offset, c); + } + + copied += c; + len -= c; + } + + if (rw) + nvdimm_flush(nfit_blk->nd_region); + + rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0; + return rc; +} + +static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr, + resource_size_t dpa, void *iobuf, u64 len, int rw) +{ + struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr); + struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW]; + struct nd_region *nd_region = nfit_blk->nd_region; + unsigned int lane, copied = 0; + int rc = 0; + + lane = nd_region_acquire_lane(nd_region); + while (len) { + u64 c = min(len, mmio->size); + + rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied, + iobuf + copied, c, rw, lane); + if (rc) + break; + + copied += c; + len -= c; + } + nd_region_release_lane(nd_region, lane); + + return rc; +} + +static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio, + struct acpi_nfit_interleave *idt, u16 interleave_ways) +{ + if (idt) { + mmio->num_lines = idt->line_count; + mmio->line_size = idt->line_size; + if (interleave_ways == 0) + return -ENXIO; + mmio->table_size = mmio->num_lines * interleave_ways + * mmio->line_size; + } + + return 0; +} + +static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc, + struct nvdimm *nvdimm, struct nfit_blk *nfit_blk) +{ + struct nd_cmd_dimm_flags flags; + int rc; + + memset(&flags, 0, sizeof(flags)); + rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags, + sizeof(flags), NULL); + + if (rc >= 0 && flags.status == 0) + nfit_blk->dimm_flags = flags.flags; + else if (rc == -ENOTTY) { + /* fall back to a conservative default */ + nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH; + rc = 0; + } else + rc = -ENXIO; + + return rc; +} + +static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, + struct device *dev) +{ + struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus); + struct nd_blk_region *ndbr = to_nd_blk_region(dev); + struct nfit_blk_mmio *mmio; + struct nfit_blk *nfit_blk; + struct nfit_mem *nfit_mem; + struct nvdimm *nvdimm; + int rc; + + nvdimm = nd_blk_region_to_dimm(ndbr); + nfit_mem = nvdimm_provider_data(nvdimm); + if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) { + dev_dbg(dev, "%s: missing%s%s%s\n", __func__, + nfit_mem ? "" : " nfit_mem", + (nfit_mem && nfit_mem->dcr) ? "" : " dcr", + (nfit_mem && nfit_mem->bdw) ? "" : " bdw"); + return -ENXIO; + } + + nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL); + if (!nfit_blk) + return -ENOMEM; + nd_blk_region_set_provider_data(ndbr, nfit_blk); + nfit_blk->nd_region = to_nd_region(dev); + + /* map block aperture memory */ + nfit_blk->bdw_offset = nfit_mem->bdw->offset; + mmio = &nfit_blk->mmio[BDW]; + mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address, + nfit_mem->spa_bdw->length, ARCH_MEMREMAP_PMEM); + if (!mmio->addr.base) { + dev_dbg(dev, "%s: %s failed to map bdw\n", __func__, + nvdimm_name(nvdimm)); + return -ENOMEM; + } + mmio->size = nfit_mem->bdw->size; + mmio->base_offset = nfit_mem->memdev_bdw->region_offset; + mmio->idt = nfit_mem->idt_bdw; + mmio->spa = nfit_mem->spa_bdw; + rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw, + nfit_mem->memdev_bdw->interleave_ways); + if (rc) { + dev_dbg(dev, "%s: %s failed to init bdw interleave\n", + __func__, nvdimm_name(nvdimm)); + return rc; + } + + /* map block control memory */ + nfit_blk->cmd_offset = nfit_mem->dcr->command_offset; + nfit_blk->stat_offset = nfit_mem->dcr->status_offset; + mmio = &nfit_blk->mmio[DCR]; + mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address, + nfit_mem->spa_dcr->length); + if (!mmio->addr.base) { + dev_dbg(dev, "%s: %s failed to map dcr\n", __func__, + nvdimm_name(nvdimm)); + return -ENOMEM; + } + mmio->size = nfit_mem->dcr->window_size; + mmio->base_offset = nfit_mem->memdev_dcr->region_offset; + mmio->idt = nfit_mem->idt_dcr; + mmio->spa = nfit_mem->spa_dcr; + rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr, + nfit_mem->memdev_dcr->interleave_ways); + if (rc) { + dev_dbg(dev, "%s: %s failed to init dcr interleave\n", + __func__, nvdimm_name(nvdimm)); + return rc; + } + + rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk); + if (rc < 0) { + dev_dbg(dev, "%s: %s failed get DIMM flags\n", + __func__, nvdimm_name(nvdimm)); + return rc; + } + + if (nvdimm_has_flush(nfit_blk->nd_region) < 0) + dev_warn(dev, "unable to guarantee persistence of writes\n"); + + if (mmio->line_size == 0) + return 0; + + if ((u32) nfit_blk->cmd_offset % mmio->line_size + + 8 > mmio->line_size) { + dev_dbg(dev, "cmd_offset crosses interleave boundary\n"); + return -ENXIO; + } else if ((u32) nfit_blk->stat_offset % mmio->line_size + + 8 > mmio->line_size) { + dev_dbg(dev, "stat_offset crosses interleave boundary\n"); + return -ENXIO; + } + + return 0; +} + +static int ars_get_cap(struct acpi_nfit_desc *acpi_desc, + struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa) +{ + struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; + struct acpi_nfit_system_address *spa = nfit_spa->spa; + int cmd_rc, rc; + + cmd->address = spa->address; + cmd->length = spa->length; + rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd, + sizeof(*cmd), &cmd_rc); + if (rc < 0) + return rc; + return cmd_rc; +} + +static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa) +{ + int rc; + int cmd_rc; + struct nd_cmd_ars_start ars_start; + struct acpi_nfit_system_address *spa = nfit_spa->spa; + struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; + + memset(&ars_start, 0, sizeof(ars_start)); + ars_start.address = spa->address; + ars_start.length = spa->length; + if (nfit_spa_type(spa) == NFIT_SPA_PM) + ars_start.type = ND_ARS_PERSISTENT; + else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) + ars_start.type = ND_ARS_VOLATILE; + else + return -ENOTTY; + + rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, + sizeof(ars_start), &cmd_rc); + + if (rc < 0) + return rc; + return cmd_rc; +} + +static int ars_continue(struct acpi_nfit_desc *acpi_desc) +{ + int rc, cmd_rc; + struct nd_cmd_ars_start ars_start; + struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; + struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; + + memset(&ars_start, 0, sizeof(ars_start)); + ars_start.address = ars_status->restart_address; + ars_start.length = ars_status->restart_length; + ars_start.type = ars_status->type; + rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start, + sizeof(ars_start), &cmd_rc); + if (rc < 0) + return rc; + return cmd_rc; +} + +static int ars_get_status(struct acpi_nfit_desc *acpi_desc) +{ + struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc; + struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status; + int rc, cmd_rc; + + rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status, + acpi_desc->ars_status_size, &cmd_rc); + if (rc < 0) + return rc; + return cmd_rc; +} + +static int ars_status_process_records(struct nvdimm_bus *nvdimm_bus, + struct nd_cmd_ars_status *ars_status) +{ + int rc; + u32 i; + + for (i = 0; i < ars_status->num_records; i++) { + rc = nvdimm_bus_add_poison(nvdimm_bus, + ars_status->records[i].err_address, + ars_status->records[i].length); + if (rc) + return rc; + } + + return 0; +} + +static void acpi_nfit_remove_resource(void *data) +{ + struct resource *res = data; + + remove_resource(res); +} + +static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc, + struct nd_region_desc *ndr_desc) +{ + struct resource *res, *nd_res = ndr_desc->res; + int is_pmem, ret; + + /* No operation if the region is already registered as PMEM */ + is_pmem = region_intersects(nd_res->start, resource_size(nd_res), + IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY); + if (is_pmem == REGION_INTERSECTS) + return 0; + + res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL); + if (!res) + return -ENOMEM; + + res->name = "Persistent Memory"; + res->start = nd_res->start; + res->end = nd_res->end; + res->flags = IORESOURCE_MEM; + res->desc = IORES_DESC_PERSISTENT_MEMORY; + + ret = insert_resource(&iomem_resource, res); + if (ret) + return ret; + + ret = devm_add_action_or_reset(acpi_desc->dev, + acpi_nfit_remove_resource, + res); + if (ret) + return ret; + + return 0; +} + +static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc, + struct nd_mapping *nd_mapping, struct nd_region_desc *ndr_desc, + struct acpi_nfit_memory_map *memdev, + struct nfit_spa *nfit_spa) +{ + struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, + memdev->device_handle); + struct acpi_nfit_system_address *spa = nfit_spa->spa; + struct nd_blk_region_desc *ndbr_desc; + struct nfit_mem *nfit_mem; + int blk_valid = 0; + + if (!nvdimm) { + dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n", + spa->range_index, memdev->device_handle); + return -ENODEV; + } + + nd_mapping->nvdimm = nvdimm; + switch (nfit_spa_type(spa)) { + case NFIT_SPA_PM: + case NFIT_SPA_VOLATILE: + nd_mapping->start = memdev->address; + nd_mapping->size = memdev->region_size; + break; + case NFIT_SPA_DCR: + nfit_mem = nvdimm_provider_data(nvdimm); + if (!nfit_mem || !nfit_mem->bdw) { + dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n", + spa->range_index, nvdimm_name(nvdimm)); + } else { + nd_mapping->size = nfit_mem->bdw->capacity; + nd_mapping->start = nfit_mem->bdw->start_address; + ndr_desc->num_lanes = nfit_mem->bdw->windows; + blk_valid = 1; + } + + ndr_desc->nd_mapping = nd_mapping; + ndr_desc->num_mappings = blk_valid; + ndbr_desc = to_blk_region_desc(ndr_desc); + ndbr_desc->enable = acpi_nfit_blk_region_enable; + ndbr_desc->do_io = acpi_desc->blk_do_io; + nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus, + ndr_desc); + if (!nfit_spa->nd_region) + return -ENOMEM; + break; + } + + return 0; +} + +static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa) +{ + return (nfit_spa_type(spa) == NFIT_SPA_VDISK || + nfit_spa_type(spa) == NFIT_SPA_VCD || + nfit_spa_type(spa) == NFIT_SPA_PDISK || + nfit_spa_type(spa) == NFIT_SPA_PCD); +} + +static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc, + struct nfit_spa *nfit_spa) +{ + static struct nd_mapping nd_mappings[ND_MAX_MAPPINGS]; + struct acpi_nfit_system_address *spa = nfit_spa->spa; + struct nd_blk_region_desc ndbr_desc; + struct nd_region_desc *ndr_desc; + struct nfit_memdev *nfit_memdev; + struct nvdimm_bus *nvdimm_bus; + struct resource res; + int count = 0, rc; + + if (nfit_spa->nd_region) + return 0; + + if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) { + dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n", + __func__); + return 0; + } + + memset(&res, 0, sizeof(res)); + memset(&nd_mappings, 0, sizeof(nd_mappings)); + memset(&ndbr_desc, 0, sizeof(ndbr_desc)); + res.start = spa->address; + res.end = res.start + spa->length - 1; + ndr_desc = &ndbr_desc.ndr_desc; + ndr_desc->res = &res; + ndr_desc->provider_data = nfit_spa; + ndr_desc->attr_groups = acpi_nfit_region_attribute_groups; + if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) + ndr_desc->numa_node = acpi_map_pxm_to_online_node( + spa->proximity_domain); + else + ndr_desc->numa_node = NUMA_NO_NODE; + + list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) { + struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev; + struct nd_mapping *nd_mapping; + + if (memdev->range_index != spa->range_index) + continue; + if (count >= ND_MAX_MAPPINGS) { + dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n", + spa->range_index, ND_MAX_MAPPINGS); + return -ENXIO; + } + nd_mapping = &nd_mappings[count++]; + rc = acpi_nfit_init_mapping(acpi_desc, nd_mapping, ndr_desc, + memdev, nfit_spa); + if (rc) + goto out; + } + + ndr_desc->nd_mapping = nd_mappings; + ndr_desc->num_mappings = count; + rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa); + if (rc) + goto out; + + nvdimm_bus = acpi_desc->nvdimm_bus; + if (nfit_spa_type(spa) == NFIT_SPA_PM) { + rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc); + if (rc) { + dev_warn(acpi_desc->dev, + "failed to insert pmem resource to iomem: %d\n", + rc); + goto out; + } + + nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, + ndr_desc); + if (!nfit_spa->nd_region) + rc = -ENOMEM; + } else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) { + nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus, + ndr_desc); + if (!nfit_spa->nd_region) + rc = -ENOMEM; + } else if (nfit_spa_is_virtual(spa)) { + nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus, + ndr_desc); + if (!nfit_spa->nd_region) + rc = -ENOMEM; + } + + out: + if (rc) + dev_err(acpi_desc->dev, "failed to register spa range %d\n", + nfit_spa->spa->range_index); + return rc; +} + +static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc, + u32 max_ars) +{ + struct device *dev = acpi_desc->dev; + struct nd_cmd_ars_status *ars_status; + + if (acpi_desc->ars_status && acpi_desc->ars_status_size >= max_ars) { + memset(acpi_desc->ars_status, 0, acpi_desc->ars_status_size); + return 0; + } + + if (acpi_desc->ars_status) + devm_kfree(dev, acpi_desc->ars_status); + acpi_desc->ars_status = NULL; + ars_status = devm_kzalloc(dev, max_ars, GFP_KERNEL); + if (!ars_status) + return -ENOMEM; + acpi_desc->ars_status = ars_status; + acpi_desc->ars_status_size = max_ars; + return 0; +} + +static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc, + struct nfit_spa *nfit_spa) +{ + struct acpi_nfit_system_address *spa = nfit_spa->spa; + int rc; + + if (!nfit_spa->max_ars) { + struct nd_cmd_ars_cap ars_cap; + + memset(&ars_cap, 0, sizeof(ars_cap)); + rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa); + if (rc < 0) + return rc; + nfit_spa->max_ars = ars_cap.max_ars_out; + nfit_spa->clear_err_unit = ars_cap.clear_err_unit; + /* check that the supported scrub types match the spa type */ + if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE && + ((ars_cap.status >> 16) & ND_ARS_VOLATILE) == 0) + return -ENOTTY; + else if (nfit_spa_type(spa) == NFIT_SPA_PM && + ((ars_cap.status >> 16) & ND_ARS_PERSISTENT) == 0) + return -ENOTTY; + } + + if (ars_status_alloc(acpi_desc, nfit_spa->max_ars)) + return -ENOMEM; + + rc = ars_get_status(acpi_desc); + if (rc < 0 && rc != -ENOSPC) + return rc; + + if (ars_status_process_records(acpi_desc->nvdimm_bus, + acpi_desc->ars_status)) + return -ENOMEM; + + return 0; +} + +static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc, + struct nfit_spa *nfit_spa) +{ + struct acpi_nfit_system_address *spa = nfit_spa->spa; + unsigned int overflow_retry = scrub_overflow_abort; + u64 init_ars_start = 0, init_ars_len = 0; + struct device *dev = acpi_desc->dev; + unsigned int tmo = scrub_timeout; + int rc; + + if (!nfit_spa->ars_required || !nfit_spa->nd_region) + return; + + rc = ars_start(acpi_desc, nfit_spa); + /* + * If we timed out the initial scan we'll still be busy here, + * and will wait another timeout before giving up permanently. + */ + if (rc < 0 && rc != -EBUSY) + return; + + do { + u64 ars_start, ars_len; + + if (acpi_desc->cancel) + break; + rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); + if (rc == -ENOTTY) + break; + if (rc == -EBUSY && !tmo) { + dev_warn(dev, "range %d ars timeout, aborting\n", + spa->range_index); + break; + } + + if (rc == -EBUSY) { + /* + * Note, entries may be appended to the list + * while the lock is dropped, but the workqueue + * being active prevents entries being deleted / + * freed. + */ + mutex_unlock(&acpi_desc->init_mutex); + ssleep(1); + tmo--; + mutex_lock(&acpi_desc->init_mutex); + continue; + } + + /* we got some results, but there are more pending... */ + if (rc == -ENOSPC && overflow_retry--) { + if (!init_ars_len) { + init_ars_len = acpi_desc->ars_status->length; + init_ars_start = acpi_desc->ars_status->address; + } + rc = ars_continue(acpi_desc); + } + + if (rc < 0) { + dev_warn(dev, "range %d ars continuation failed\n", + spa->range_index); + break; + } + + if (init_ars_len) { + ars_start = init_ars_start; + ars_len = init_ars_len; + } else { + ars_start = acpi_desc->ars_status->address; + ars_len = acpi_desc->ars_status->length; + } + dev_dbg(dev, "spa range: %d ars from %#llx + %#llx complete\n", + spa->range_index, ars_start, ars_len); + /* notify the region about new poison entries */ + nvdimm_region_notify(nfit_spa->nd_region, + NVDIMM_REVALIDATE_POISON); + break; + } while (1); +} + +static void acpi_nfit_scrub(struct work_struct *work) +{ + struct device *dev; + u64 init_scrub_length = 0; + struct nfit_spa *nfit_spa; + u64 init_scrub_address = 0; + bool init_ars_done = false; + struct acpi_nfit_desc *acpi_desc; + unsigned int tmo = scrub_timeout; + unsigned int overflow_retry = scrub_overflow_abort; + + acpi_desc = container_of(work, typeof(*acpi_desc), work); + dev = acpi_desc->dev; + + /* + * We scrub in 2 phases. The first phase waits for any platform + * firmware initiated scrubs to complete and then we go search for the + * affected spa regions to mark them scanned. In the second phase we + * initiate a directed scrub for every range that was not scrubbed in + * phase 1. If we're called for a 'rescan', we harmlessly pass through + * the first phase, but really only care about running phase 2, where + * regions can be notified of new poison. + */ + + /* process platform firmware initiated scrubs */ + retry: + mutex_lock(&acpi_desc->init_mutex); + list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { + struct nd_cmd_ars_status *ars_status; + struct acpi_nfit_system_address *spa; + u64 ars_start, ars_len; + int rc; + + if (acpi_desc->cancel) + break; + + if (nfit_spa->nd_region) + continue; + + if (init_ars_done) { + /* + * No need to re-query, we're now just + * reconciling all the ranges covered by the + * initial scrub + */ + rc = 0; + } else + rc = acpi_nfit_query_poison(acpi_desc, nfit_spa); + + if (rc == -ENOTTY) { + /* no ars capability, just register spa and move on */ + acpi_nfit_register_region(acpi_desc, nfit_spa); + continue; + } + + if (rc == -EBUSY && !tmo) { + /* fallthrough to directed scrub in phase 2 */ + dev_warn(dev, "timeout awaiting ars results, continuing...\n"); + break; + } else if (rc == -EBUSY) { + mutex_unlock(&acpi_desc->init_mutex); + ssleep(1); + tmo--; + goto retry; + } + + /* we got some results, but there are more pending... */ + if (rc == -ENOSPC && overflow_retry--) { + ars_status = acpi_desc->ars_status; + /* + * Record the original scrub range, so that we + * can recall all the ranges impacted by the + * initial scrub. + */ + if (!init_scrub_length) { + init_scrub_length = ars_status->length; + init_scrub_address = ars_status->address; + } + rc = ars_continue(acpi_desc); + if (rc == 0) { + mutex_unlock(&acpi_desc->init_mutex); + goto retry; + } + } + + if (rc < 0) { + /* + * Initial scrub failed, we'll give it one more + * try below... + */ + break; + } + + /* We got some final results, record completed ranges */ + ars_status = acpi_desc->ars_status; + if (init_scrub_length) { + ars_start = init_scrub_address; + ars_len = ars_start + init_scrub_length; + } else { + ars_start = ars_status->address; + ars_len = ars_status->length; + } + spa = nfit_spa->spa; + + if (!init_ars_done) { + init_ars_done = true; + dev_dbg(dev, "init scrub %#llx + %#llx complete\n", + ars_start, ars_len); + } + if (ars_start <= spa->address && ars_start + ars_len + >= spa->address + spa->length) + acpi_nfit_register_region(acpi_desc, nfit_spa); + } + + /* + * For all the ranges not covered by an initial scrub we still + * want to see if there are errors, but it's ok to discover them + * asynchronously. + */ + list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { + /* + * Flag all the ranges that still need scrubbing, but + * register them now to make data available. + */ + if (!nfit_spa->nd_region) { + nfit_spa->ars_required = 1; + acpi_nfit_register_region(acpi_desc, nfit_spa); + } + } + + list_for_each_entry(nfit_spa, &acpi_desc->spas, list) + acpi_nfit_async_scrub(acpi_desc, nfit_spa); + acpi_desc->scrub_count++; + if (acpi_desc->scrub_count_state) + sysfs_notify_dirent(acpi_desc->scrub_count_state); + mutex_unlock(&acpi_desc->init_mutex); +} + +static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc) +{ + struct nfit_spa *nfit_spa; + int rc; + + list_for_each_entry(nfit_spa, &acpi_desc->spas, list) + if (nfit_spa_type(nfit_spa->spa) == NFIT_SPA_DCR) { + /* BLK regions don't need to wait for ars results */ + rc = acpi_nfit_register_region(acpi_desc, nfit_spa); + if (rc) + return rc; + } + + queue_work(nfit_wq, &acpi_desc->work); + return 0; +} + +static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc, + struct nfit_table_prev *prev) +{ + struct device *dev = acpi_desc->dev; + + if (!list_empty(&prev->spas) || + !list_empty(&prev->memdevs) || + !list_empty(&prev->dcrs) || + !list_empty(&prev->bdws) || + !list_empty(&prev->idts) || + !list_empty(&prev->flushes)) { + dev_err(dev, "new nfit deletes entries (unsupported)\n"); + return -ENXIO; + } + return 0; +} + +static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc) +{ + struct device *dev = acpi_desc->dev; + struct kernfs_node *nfit; + struct device *bus_dev; + + if (!ars_supported(acpi_desc->nvdimm_bus)) + return 0; + + bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); + nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit"); + if (!nfit) { + dev_err(dev, "sysfs_get_dirent 'nfit' failed\n"); + return -ENODEV; + } + acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub"); + sysfs_put(nfit); + if (!acpi_desc->scrub_count_state) { + dev_err(dev, "sysfs_get_dirent 'scrub' failed\n"); + return -ENODEV; + } + + return 0; +} + +static void acpi_nfit_destruct(void *data) +{ + struct acpi_nfit_desc *acpi_desc = data; + struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus); + + /* + * Destruct under acpi_desc_lock so that nfit_handle_mce does not + * race teardown + */ + mutex_lock(&acpi_desc_lock); + acpi_desc->cancel = 1; + /* + * Bounce the nvdimm bus lock to make sure any in-flight + * acpi_nfit_ars_rescan() submissions have had a chance to + * either submit or see ->cancel set. + */ + device_lock(bus_dev); + device_unlock(bus_dev); + + flush_workqueue(nfit_wq); + if (acpi_desc->scrub_count_state) + sysfs_put(acpi_desc->scrub_count_state); + nvdimm_bus_unregister(acpi_desc->nvdimm_bus); + acpi_desc->nvdimm_bus = NULL; + list_del(&acpi_desc->list); + mutex_unlock(&acpi_desc_lock); +} + +int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz) +{ + struct device *dev = acpi_desc->dev; + struct nfit_table_prev prev; + const void *end; + int rc; + + if (!acpi_desc->nvdimm_bus) { + acpi_nfit_init_dsms(acpi_desc); + + acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, + &acpi_desc->nd_desc); + if (!acpi_desc->nvdimm_bus) + return -ENOMEM; + + rc = devm_add_action_or_reset(dev, acpi_nfit_destruct, + acpi_desc); + if (rc) + return rc; + + rc = acpi_nfit_desc_init_scrub_attr(acpi_desc); + if (rc) + return rc; + + /* register this acpi_desc for mce notifications */ + mutex_lock(&acpi_desc_lock); + list_add_tail(&acpi_desc->list, &acpi_descs); + mutex_unlock(&acpi_desc_lock); + } + + mutex_lock(&acpi_desc->init_mutex); + + INIT_LIST_HEAD(&prev.spas); + INIT_LIST_HEAD(&prev.memdevs); + INIT_LIST_HEAD(&prev.dcrs); + INIT_LIST_HEAD(&prev.bdws); + INIT_LIST_HEAD(&prev.idts); + INIT_LIST_HEAD(&prev.flushes); + + list_cut_position(&prev.spas, &acpi_desc->spas, + acpi_desc->spas.prev); + list_cut_position(&prev.memdevs, &acpi_desc->memdevs, + acpi_desc->memdevs.prev); + list_cut_position(&prev.dcrs, &acpi_desc->dcrs, + acpi_desc->dcrs.prev); + list_cut_position(&prev.bdws, &acpi_desc->bdws, + acpi_desc->bdws.prev); + list_cut_position(&prev.idts, &acpi_desc->idts, + acpi_desc->idts.prev); + list_cut_position(&prev.flushes, &acpi_desc->flushes, + acpi_desc->flushes.prev); + + end = data + sz; + while (!IS_ERR_OR_NULL(data)) + data = add_table(acpi_desc, &prev, data, end); + + if (IS_ERR(data)) { + dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__, + PTR_ERR(data)); + rc = PTR_ERR(data); + goto out_unlock; + } + + rc = acpi_nfit_check_deletions(acpi_desc, &prev); + if (rc) + goto out_unlock; + + rc = nfit_mem_init(acpi_desc); + if (rc) + goto out_unlock; + + rc = acpi_nfit_register_dimms(acpi_desc); + if (rc) + goto out_unlock; + + rc = acpi_nfit_register_regions(acpi_desc); + + out_unlock: + mutex_unlock(&acpi_desc->init_mutex); + return rc; +} +EXPORT_SYMBOL_GPL(acpi_nfit_init); + +struct acpi_nfit_flush_work { + struct work_struct work; + struct completion cmp; +}; + +static void flush_probe(struct work_struct *work) +{ + struct acpi_nfit_flush_work *flush; + + flush = container_of(work, typeof(*flush), work); + complete(&flush->cmp); +} + +static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc) +{ + struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); + struct device *dev = acpi_desc->dev; + struct acpi_nfit_flush_work flush; + + /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */ + device_lock(dev); + device_unlock(dev); + + /* + * Scrub work could take 10s of seconds, userspace may give up so we + * need to be interruptible while waiting. + */ + INIT_WORK_ONSTACK(&flush.work, flush_probe); + COMPLETION_INITIALIZER_ONSTACK(flush.cmp); + queue_work(nfit_wq, &flush.work); + return wait_for_completion_interruptible(&flush.cmp); +} + +static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc, + struct nvdimm *nvdimm, unsigned int cmd) +{ + struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc); + + if (nvdimm) + return 0; + if (cmd != ND_CMD_ARS_START) + return 0; + + /* + * The kernel and userspace may race to initiate a scrub, but + * the scrub thread is prepared to lose that initial race. It + * just needs guarantees that any ars it initiates are not + * interrupted by any intervening start reqeusts from userspace. + */ + if (work_busy(&acpi_desc->work)) + return -EBUSY; + + return 0; +} + +int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc) +{ + struct device *dev = acpi_desc->dev; + struct nfit_spa *nfit_spa; + + if (work_busy(&acpi_desc->work)) + return -EBUSY; + + if (acpi_desc->cancel) + return 0; + + mutex_lock(&acpi_desc->init_mutex); + list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { + struct acpi_nfit_system_address *spa = nfit_spa->spa; + + if (nfit_spa_type(spa) != NFIT_SPA_PM) + continue; + + nfit_spa->ars_required = 1; + } + queue_work(nfit_wq, &acpi_desc->work); + dev_dbg(dev, "%s: ars_scan triggered\n", __func__); + mutex_unlock(&acpi_desc->init_mutex); + + return 0; +} + +void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev) +{ + struct nvdimm_bus_descriptor *nd_desc; + + dev_set_drvdata(dev, acpi_desc); + acpi_desc->dev = dev; + acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io; + nd_desc = &acpi_desc->nd_desc; + nd_desc->provider_name = "ACPI.NFIT"; + nd_desc->module = THIS_MODULE; + nd_desc->ndctl = acpi_nfit_ctl; + nd_desc->flush_probe = acpi_nfit_flush_probe; + nd_desc->clear_to_send = acpi_nfit_clear_to_send; + nd_desc->attr_groups = acpi_nfit_attribute_groups; + + INIT_LIST_HEAD(&acpi_desc->spas); + INIT_LIST_HEAD(&acpi_desc->dcrs); + INIT_LIST_HEAD(&acpi_desc->bdws); + INIT_LIST_HEAD(&acpi_desc->idts); + INIT_LIST_HEAD(&acpi_desc->flushes); + INIT_LIST_HEAD(&acpi_desc->memdevs); + INIT_LIST_HEAD(&acpi_desc->dimms); + INIT_LIST_HEAD(&acpi_desc->list); + mutex_init(&acpi_desc->init_mutex); + INIT_WORK(&acpi_desc->work, acpi_nfit_scrub); +} +EXPORT_SYMBOL_GPL(acpi_nfit_desc_init); + +static int acpi_nfit_add(struct acpi_device *adev) +{ + struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; + struct acpi_nfit_desc *acpi_desc; + struct device *dev = &adev->dev; + struct acpi_table_header *tbl; + acpi_status status = AE_OK; + acpi_size sz; + int rc = 0; + + status = acpi_get_table_with_size(ACPI_SIG_NFIT, 0, &tbl, &sz); + if (ACPI_FAILURE(status)) { + /* This is ok, we could have an nvdimm hotplugged later */ + dev_dbg(dev, "failed to find NFIT at startup\n"); + return 0; + } + + acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); + if (!acpi_desc) + return -ENOMEM; + acpi_nfit_desc_init(acpi_desc, &adev->dev); + + /* Save the acpi header for exporting the revision via sysfs */ + acpi_desc->acpi_header = *tbl; + + /* Evaluate _FIT and override with that if present */ + status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf); + if (ACPI_SUCCESS(status) && buf.length > 0) { + union acpi_object *obj = buf.pointer; + + if (obj->type == ACPI_TYPE_BUFFER) + rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer, + obj->buffer.length); + else + dev_dbg(dev, "%s invalid type %d, ignoring _FIT\n", + __func__, (int) obj->type); + kfree(buf.pointer); + } else + /* skip over the lead-in header table */ + rc = acpi_nfit_init(acpi_desc, (void *) tbl + + sizeof(struct acpi_table_nfit), + sz - sizeof(struct acpi_table_nfit)); + return rc; +} + +static int acpi_nfit_remove(struct acpi_device *adev) +{ + /* see acpi_nfit_destruct */ + return 0; +} + +static void acpi_nfit_notify(struct acpi_device *adev, u32 event) +{ + struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(&adev->dev); + struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL }; + struct device *dev = &adev->dev; + union acpi_object *obj; + acpi_status status; + int ret; + + dev_dbg(dev, "%s: event: %d\n", __func__, event); + + device_lock(dev); + if (!dev->driver) { + /* dev->driver may be null if we're being removed */ + dev_dbg(dev, "%s: no driver found for dev\n", __func__); + goto out_unlock; + } + + if (!acpi_desc) { + acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL); + if (!acpi_desc) + goto out_unlock; + acpi_nfit_desc_init(acpi_desc, &adev->dev); + } else { + /* + * Finish previous registration before considering new + * regions. + */ + flush_workqueue(nfit_wq); + } + + /* Evaluate _FIT */ + status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf); + if (ACPI_FAILURE(status)) { + dev_err(dev, "failed to evaluate _FIT\n"); + goto out_unlock; + } + + obj = buf.pointer; + if (obj->type == ACPI_TYPE_BUFFER) { + ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer, + obj->buffer.length); + if (ret) + dev_err(dev, "failed to merge updated NFIT\n"); + } else + dev_err(dev, "Invalid _FIT\n"); + kfree(buf.pointer); + + out_unlock: + device_unlock(dev); +} + +static const struct acpi_device_id acpi_nfit_ids[] = { + { "ACPI0012", 0 }, + { "", 0 }, +}; +MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids); + +static struct acpi_driver acpi_nfit_driver = { + .name = KBUILD_MODNAME, + .ids = acpi_nfit_ids, + .ops = { + .add = acpi_nfit_add, + .remove = acpi_nfit_remove, + .notify = acpi_nfit_notify, + }, +}; + +static __init int nfit_init(void) +{ + BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40); + BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56); + BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48); + BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20); + BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9); + BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80); + BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40); + + acpi_str_to_uuid(UUID_VOLATILE_MEMORY, nfit_uuid[NFIT_SPA_VOLATILE]); + acpi_str_to_uuid(UUID_PERSISTENT_MEMORY, nfit_uuid[NFIT_SPA_PM]); + acpi_str_to_uuid(UUID_CONTROL_REGION, nfit_uuid[NFIT_SPA_DCR]); + acpi_str_to_uuid(UUID_DATA_REGION, nfit_uuid[NFIT_SPA_BDW]); + acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_VDISK]); + acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD, nfit_uuid[NFIT_SPA_VCD]); + acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_PDISK]); + acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD, nfit_uuid[NFIT_SPA_PCD]); + acpi_str_to_uuid(UUID_NFIT_BUS, nfit_uuid[NFIT_DEV_BUS]); + acpi_str_to_uuid(UUID_NFIT_DIMM, nfit_uuid[NFIT_DEV_DIMM]); + acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE1, nfit_uuid[NFIT_DEV_DIMM_N_HPE1]); + acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE2, nfit_uuid[NFIT_DEV_DIMM_N_HPE2]); + acpi_str_to_uuid(UUID_NFIT_DIMM_N_MSFT, nfit_uuid[NFIT_DEV_DIMM_N_MSFT]); + + nfit_wq = create_singlethread_workqueue("nfit"); + if (!nfit_wq) + return -ENOMEM; + + nfit_mce_register(); + + return acpi_bus_register_driver(&acpi_nfit_driver); +} + +static __exit void nfit_exit(void) +{ + nfit_mce_unregister(); + acpi_bus_unregister_driver(&acpi_nfit_driver); + destroy_workqueue(nfit_wq); + WARN_ON(!list_empty(&acpi_descs)); +} + +module_init(nfit_init); +module_exit(nfit_exit); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Intel Corporation"); diff --git a/drivers/acpi/nfit/mce.c b/drivers/acpi/nfit/mce.c new file mode 100644 index 000000000..161f91539 --- /dev/null +++ b/drivers/acpi/nfit/mce.c @@ -0,0 +1,89 @@ +/* + * NFIT - Machine Check Handler + * + * Copyright(c) 2013-2016 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include "nfit.h" + +static int nfit_handle_mce(struct notifier_block *nb, unsigned long val, + void *data) +{ + struct mce *mce = (struct mce *)data; + struct acpi_nfit_desc *acpi_desc; + struct nfit_spa *nfit_spa; + + /* We only care about memory errors */ + if (!(mce->status & MCACOD)) + return NOTIFY_DONE; + + /* + * mce->addr contains the physical addr accessed that caused the + * machine check. We need to walk through the list of NFITs, and see + * if any of them matches that address, and only then start a scrub. + */ + mutex_lock(&acpi_desc_lock); + list_for_each_entry(acpi_desc, &acpi_descs, list) { + struct device *dev = acpi_desc->dev; + int found_match = 0; + + mutex_lock(&acpi_desc->init_mutex); + list_for_each_entry(nfit_spa, &acpi_desc->spas, list) { + struct acpi_nfit_system_address *spa = nfit_spa->spa; + + if (nfit_spa_type(spa) != NFIT_SPA_PM) + continue; + /* find the spa that covers the mce addr */ + if (spa->address > mce->addr) + continue; + if ((spa->address + spa->length - 1) < mce->addr) + continue; + found_match = 1; + dev_dbg(dev, "%s: addr in SPA %d (0x%llx, 0x%llx)\n", + __func__, spa->range_index, spa->address, + spa->length); + /* + * We can break at the first match because we're going + * to rescan all the SPA ranges. There shouldn't be any + * aliasing anyway. + */ + break; + } + mutex_unlock(&acpi_desc->init_mutex); + + /* + * We can ignore an -EBUSY here because if an ARS is already + * in progress, just let that be the last authoritative one + */ + if (found_match) + acpi_nfit_ars_rescan(acpi_desc); + } + + mutex_unlock(&acpi_desc_lock); + return NOTIFY_DONE; +} + +static struct notifier_block nfit_mce_dec = { + .notifier_call = nfit_handle_mce, +}; + +void nfit_mce_register(void) +{ + mce_register_decode_chain(&nfit_mce_dec); +} + +void nfit_mce_unregister(void) +{ + mce_unregister_decode_chain(&nfit_mce_dec); +} diff --git a/drivers/acpi/nfit/nfit.h b/drivers/acpi/nfit/nfit.h new file mode 100644 index 000000000..e894ded24 --- /dev/null +++ b/drivers/acpi/nfit/nfit.h @@ -0,0 +1,227 @@ +/* + * NVDIMM Firmware Interface Table - NFIT + * + * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#ifndef __NFIT_H__ +#define __NFIT_H__ +#include +#include +#include +#include +#include +#include +#include + +/* ACPI 6.1 */ +#define UUID_NFIT_BUS "2f10e7a4-9e91-11e4-89d3-123b93f75cba" + +/* http://pmem.io/documents/NVDIMM_DSM_Interface_Example.pdf */ +#define UUID_NFIT_DIMM "4309ac30-0d11-11e4-9191-0800200c9a66" + +/* https://github.com/HewlettPackard/hpe-nvm/blob/master/Documentation/ */ +#define UUID_NFIT_DIMM_N_HPE1 "9002c334-acf3-4c0e-9642-a235f0d53bc6" +#define UUID_NFIT_DIMM_N_HPE2 "5008664b-b758-41a0-a03c-27c2f2d04f7e" + +/* https://msdn.microsoft.com/library/windows/hardware/mt604741 */ +#define UUID_NFIT_DIMM_N_MSFT "1ee68b36-d4bd-4a1a-9a16-4f8e53d46e05" + +#define ACPI_NFIT_MEM_FAILED_MASK (ACPI_NFIT_MEM_SAVE_FAILED \ + | ACPI_NFIT_MEM_RESTORE_FAILED | ACPI_NFIT_MEM_FLUSH_FAILED \ + | ACPI_NFIT_MEM_NOT_ARMED) + +enum nfit_uuids { + /* for simplicity alias the uuid index with the family id */ + NFIT_DEV_DIMM = NVDIMM_FAMILY_INTEL, + NFIT_DEV_DIMM_N_HPE1 = NVDIMM_FAMILY_HPE1, + NFIT_DEV_DIMM_N_HPE2 = NVDIMM_FAMILY_HPE2, + NFIT_DEV_DIMM_N_MSFT = NVDIMM_FAMILY_MSFT, + NFIT_SPA_VOLATILE, + NFIT_SPA_PM, + NFIT_SPA_DCR, + NFIT_SPA_BDW, + NFIT_SPA_VDISK, + NFIT_SPA_VCD, + NFIT_SPA_PDISK, + NFIT_SPA_PCD, + NFIT_DEV_BUS, + NFIT_UUID_MAX, +}; + +/* + * Region format interface codes are stored with the interface as the + * LSB and the function as the MSB. + */ +#define NFIT_FIC_BYTE cpu_to_le16(0x101) /* byte-addressable energy backed */ +#define NFIT_FIC_BLK cpu_to_le16(0x201) /* block-addressable non-energy backed */ +#define NFIT_FIC_BYTEN cpu_to_le16(0x301) /* byte-addressable non-energy backed */ + +enum { + NFIT_BLK_READ_FLUSH = 1, + NFIT_BLK_DCR_LATCH = 2, + NFIT_ARS_STATUS_DONE = 0, + NFIT_ARS_STATUS_BUSY = 1 << 16, + NFIT_ARS_STATUS_NONE = 2 << 16, + NFIT_ARS_STATUS_INTR = 3 << 16, + NFIT_ARS_START_BUSY = 6, + NFIT_ARS_CAP_NONE = 1, + NFIT_ARS_F_OVERFLOW = 1, + NFIT_ARS_TIMEOUT = 90, +}; + +struct nfit_spa { + struct list_head list; + struct nd_region *nd_region; + unsigned int ars_required:1; + u32 clear_err_unit; + u32 max_ars; + struct acpi_nfit_system_address spa[0]; +}; + +struct nfit_dcr { + struct list_head list; + struct acpi_nfit_control_region dcr[0]; +}; + +struct nfit_bdw { + struct list_head list; + struct acpi_nfit_data_region bdw[0]; +}; + +struct nfit_idt { + struct list_head list; + struct acpi_nfit_interleave idt[0]; +}; + +struct nfit_flush { + struct list_head list; + struct acpi_nfit_flush_address flush[0]; +}; + +struct nfit_memdev { + struct list_head list; + struct acpi_nfit_memory_map memdev[0]; +}; + +/* assembled tables for a given dimm/memory-device */ +struct nfit_mem { + struct nvdimm *nvdimm; + struct acpi_nfit_memory_map *memdev_dcr; + struct acpi_nfit_memory_map *memdev_pmem; + struct acpi_nfit_memory_map *memdev_bdw; + struct acpi_nfit_control_region *dcr; + struct acpi_nfit_data_region *bdw; + struct acpi_nfit_system_address *spa_dcr; + struct acpi_nfit_system_address *spa_bdw; + struct acpi_nfit_interleave *idt_dcr; + struct acpi_nfit_interleave *idt_bdw; + struct nfit_flush *nfit_flush; + struct list_head list; + struct acpi_device *adev; + struct acpi_nfit_desc *acpi_desc; + struct resource *flush_wpq; + unsigned long dsm_mask; + int family; +}; + +struct acpi_nfit_desc { + struct nvdimm_bus_descriptor nd_desc; + struct acpi_table_header acpi_header; + struct mutex init_mutex; + struct list_head memdevs; + struct list_head flushes; + struct list_head dimms; + struct list_head spas; + struct list_head dcrs; + struct list_head bdws; + struct list_head idts; + struct nvdimm_bus *nvdimm_bus; + struct device *dev; + struct nd_cmd_ars_status *ars_status; + size_t ars_status_size; + struct work_struct work; + struct list_head list; + struct kernfs_node *scrub_count_state; + unsigned int scrub_count; + unsigned int cancel:1; + unsigned long dimm_cmd_force_en; + unsigned long bus_cmd_force_en; + int (*blk_do_io)(struct nd_blk_region *ndbr, resource_size_t dpa, + void *iobuf, u64 len, int rw); +}; + +enum nd_blk_mmio_selector { + BDW, + DCR, +}; + +struct nd_blk_addr { + union { + void __iomem *base; + void *aperture; + }; +}; + +struct nfit_blk { + struct nfit_blk_mmio { + struct nd_blk_addr addr; + u64 size; + u64 base_offset; + u32 line_size; + u32 num_lines; + u32 table_size; + struct acpi_nfit_interleave *idt; + struct acpi_nfit_system_address *spa; + } mmio[2]; + struct nd_region *nd_region; + u64 bdw_offset; /* post interleave offset */ + u64 stat_offset; + u64 cmd_offset; + u32 dimm_flags; +}; + +extern struct list_head acpi_descs; +extern struct mutex acpi_desc_lock; +int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc); + +#ifdef CONFIG_X86_MCE +void nfit_mce_register(void); +void nfit_mce_unregister(void); +#else +static inline void nfit_mce_register(void) +{ +} +static inline void nfit_mce_unregister(void) +{ +} +#endif + +int nfit_spa_type(struct acpi_nfit_system_address *spa); + +static inline struct acpi_nfit_memory_map *__to_nfit_memdev( + struct nfit_mem *nfit_mem) +{ + if (nfit_mem->memdev_dcr) + return nfit_mem->memdev_dcr; + return nfit_mem->memdev_pmem; +} + +static inline struct acpi_nfit_desc *to_acpi_desc( + struct nvdimm_bus_descriptor *nd_desc) +{ + return container_of(nd_desc, struct acpi_nfit_desc, nd_desc); +} + +const u8 *to_nfit_uuid(enum nfit_uuids id); +int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *nfit, acpi_size sz); +void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev); +#endif /* __NFIT_H__ */ diff --git a/drivers/acpi/numa.c b/drivers/acpi/numa.c index d176e0ece..ce3a7a16f 100644 --- a/drivers/acpi/numa.c +++ b/drivers/acpi/numa.c @@ -18,22 +18,21 @@ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * */ + +#define pr_fmt(fmt) "ACPI: " fmt + #include #include #include #include #include #include +#include +#include #include #include #include -#define PREFIX "ACPI: " - -#define ACPI_NUMA 0x80000000 -#define _COMPONENT ACPI_NUMA -ACPI_MODULE_NAME("numa"); - static nodemask_t nodes_found_map = NODE_MASK_NONE; /* maps to convert between proximity domain and logical node ID */ @@ -43,6 +42,7 @@ static int node_to_pxm_map[MAX_NUMNODES] = { [0 ... MAX_NUMNODES - 1] = PXM_INVAL }; unsigned char acpi_srat_revision __initdata; +int acpi_numa __initdata; int pxm_to_node(int pxm) { @@ -128,68 +128,63 @@ EXPORT_SYMBOL(acpi_map_pxm_to_online_node); static void __init acpi_table_print_srat_entry(struct acpi_subtable_header *header) { - - ACPI_FUNCTION_NAME("acpi_table_print_srat_entry"); - - if (!header) - return; - switch (header->type) { - case ACPI_SRAT_TYPE_CPU_AFFINITY: -#ifdef ACPI_DEBUG_OUTPUT { struct acpi_srat_cpu_affinity *p = (struct acpi_srat_cpu_affinity *)header; - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "SRAT Processor (id[0x%02x] eid[0x%02x]) in proximity domain %d %s\n", - p->apic_id, p->local_sapic_eid, - p->proximity_domain_lo, - (p->flags & ACPI_SRAT_CPU_ENABLED)? - "enabled" : "disabled")); + pr_debug("SRAT Processor (id[0x%02x] eid[0x%02x]) in proximity domain %d %s\n", + p->apic_id, p->local_sapic_eid, + p->proximity_domain_lo, + (p->flags & ACPI_SRAT_CPU_ENABLED) ? + "enabled" : "disabled"); } -#endif /* ACPI_DEBUG_OUTPUT */ break; case ACPI_SRAT_TYPE_MEMORY_AFFINITY: -#ifdef ACPI_DEBUG_OUTPUT { struct acpi_srat_mem_affinity *p = (struct acpi_srat_mem_affinity *)header; - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "SRAT Memory (0x%lx length 0x%lx) in proximity domain %d %s%s%s\n", - (unsigned long)p->base_address, - (unsigned long)p->length, - p->proximity_domain, - (p->flags & ACPI_SRAT_MEM_ENABLED)? - "enabled" : "disabled", - (p->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE)? - " hot-pluggable" : "", - (p->flags & ACPI_SRAT_MEM_NON_VOLATILE)? - " non-volatile" : "")); + pr_debug("SRAT Memory (0x%lx length 0x%lx) in proximity domain %d %s%s%s\n", + (unsigned long)p->base_address, + (unsigned long)p->length, + p->proximity_domain, + (p->flags & ACPI_SRAT_MEM_ENABLED) ? + "enabled" : "disabled", + (p->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) ? + " hot-pluggable" : "", + (p->flags & ACPI_SRAT_MEM_NON_VOLATILE) ? + " non-volatile" : ""); } -#endif /* ACPI_DEBUG_OUTPUT */ break; case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: -#ifdef ACPI_DEBUG_OUTPUT { struct acpi_srat_x2apic_cpu_affinity *p = (struct acpi_srat_x2apic_cpu_affinity *)header; - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "SRAT Processor (x2apicid[0x%08x]) in" - " proximity domain %d %s\n", - p->apic_id, - p->proximity_domain, - (p->flags & ACPI_SRAT_CPU_ENABLED) ? - "enabled" : "disabled")); + pr_debug("SRAT Processor (x2apicid[0x%08x]) in proximity domain %d %s\n", + p->apic_id, + p->proximity_domain, + (p->flags & ACPI_SRAT_CPU_ENABLED) ? + "enabled" : "disabled"); } -#endif /* ACPI_DEBUG_OUTPUT */ break; + + case ACPI_SRAT_TYPE_GICC_AFFINITY: + { + struct acpi_srat_gicc_affinity *p = + (struct acpi_srat_gicc_affinity *)header; + pr_debug("SRAT Processor (acpi id[0x%04x]) in proximity domain %d %s\n", + p->acpi_processor_uid, + p->proximity_domain, + (p->flags & ACPI_SRAT_GICC_ENABLED) ? + "enabled" : "disabled"); + } + break; + default: - printk(KERN_WARNING PREFIX - "Found unsupported SRAT entry (type = 0x%x)\n", - header->type); + pr_warn("Found unsupported SRAT entry (type = 0x%x)\n", + header->type); break; } } @@ -217,12 +212,117 @@ static int __init slit_valid(struct acpi_table_slit *slit) return 1; } +void __init bad_srat(void) +{ + pr_err("SRAT: SRAT not used.\n"); + acpi_numa = -1; +} + +int __init srat_disabled(void) +{ + return acpi_numa < 0; +} + +#if defined(CONFIG_X86) || defined(CONFIG_ARM64) +/* + * Callback for SLIT parsing. pxm_to_node() returns NUMA_NO_NODE for + * I/O localities since SRAT does not list them. I/O localities are + * not supported at this point. + */ +void __init acpi_numa_slit_init(struct acpi_table_slit *slit) +{ + int i, j; + + for (i = 0; i < slit->locality_count; i++) { + const int from_node = pxm_to_node(i); + + if (from_node == NUMA_NO_NODE) + continue; + + for (j = 0; j < slit->locality_count; j++) { + const int to_node = pxm_to_node(j); + + if (to_node == NUMA_NO_NODE) + continue; + + numa_set_distance(from_node, to_node, + slit->entry[slit->locality_count * i + j]); + } + } +} + +/* + * Default callback for parsing of the Proximity Domain <-> Memory + * Area mappings + */ +int __init +acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma) +{ + u64 start, end; + u32 hotpluggable; + int node, pxm; + + if (srat_disabled()) + goto out_err; + if (ma->header.length < sizeof(struct acpi_srat_mem_affinity)) { + pr_err("SRAT: Unexpected header length: %d\n", + ma->header.length); + goto out_err_bad_srat; + } + if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0) + goto out_err; + hotpluggable = ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE; + if (hotpluggable && !IS_ENABLED(CONFIG_MEMORY_HOTPLUG)) + goto out_err; + + start = ma->base_address; + end = start + ma->length; + pxm = ma->proximity_domain; + if (acpi_srat_revision <= 1) + pxm &= 0xff; + + node = acpi_map_pxm_to_node(pxm); + if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) { + pr_err("SRAT: Too many proximity domains.\n"); + goto out_err_bad_srat; + } + + if (numa_add_memblk(node, start, end) < 0) { + pr_err("SRAT: Failed to add memblk to node %u [mem %#010Lx-%#010Lx]\n", + node, (unsigned long long) start, + (unsigned long long) end - 1); + goto out_err_bad_srat; + } + + node_set(node, numa_nodes_parsed); + + pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s%s\n", + node, pxm, + (unsigned long long) start, (unsigned long long) end - 1, + hotpluggable ? " hotplug" : "", + ma->flags & ACPI_SRAT_MEM_NON_VOLATILE ? " non-volatile" : ""); + + /* Mark hotplug range in memblock. */ + if (hotpluggable && memblock_mark_hotplug(start, ma->length)) + pr_warn("SRAT: Failed to mark hotplug range [mem %#010Lx-%#010Lx] in memblock\n", + (unsigned long long)start, (unsigned long long)end - 1); + + max_possible_pfn = max(max_possible_pfn, PFN_UP(end - 1)); + + return 0; +out_err_bad_srat: + bad_srat(); +out_err: + return -EINVAL; +} +#endif /* defined(CONFIG_X86) || defined (CONFIG_ARM64) */ + static int __init acpi_parse_slit(struct acpi_table_header *table) { struct acpi_table_slit *slit = (struct acpi_table_slit *)table; if (!slit_valid(slit)) { - printk(KERN_INFO "ACPI: SLIT table looks invalid. Not used.\n"); + pr_info("SLIT table looks invalid. Not used.\n"); return -EINVAL; } acpi_numa_slit_init(slit); @@ -233,12 +333,9 @@ static int __init acpi_parse_slit(struct acpi_table_header *table) void __init __weak acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) { - printk(KERN_WARNING PREFIX - "Found unsupported x2apic [0x%08x] SRAT entry\n", pa->apic_id); - return; + pr_warn("Found unsupported x2apic [0x%08x] SRAT entry\n", pa->apic_id); } - static int __init acpi_parse_x2apic_affinity(struct acpi_subtable_header *header, const unsigned long end) @@ -275,6 +372,24 @@ acpi_parse_processor_affinity(struct acpi_subtable_header *header, return 0; } +static int __init +acpi_parse_gicc_affinity(struct acpi_subtable_header *header, + const unsigned long end) +{ + struct acpi_srat_gicc_affinity *processor_affinity; + + processor_affinity = (struct acpi_srat_gicc_affinity *)header; + if (!processor_affinity) + return -EINVAL; + + acpi_table_print_srat_entry(header); + + /* let architecture-dependent part to do it */ + acpi_numa_gicc_affinity_init(processor_affinity); + + return 0; +} + static int __initdata parsed_numa_memblks; static int __init @@ -319,6 +434,9 @@ int __init acpi_numa_init(void) { int cnt = 0; + if (acpi_disabled) + return -EINVAL; + /* * Should not limit number with cpu num that is from NR_CPUS or nr_cpus= * SRAT cpu entries could have different order with that in MADT. @@ -327,13 +445,15 @@ int __init acpi_numa_init(void) /* SRAT: Static Resource Affinity Table */ if (!acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat)) { - struct acpi_subtable_proc srat_proc[2]; + struct acpi_subtable_proc srat_proc[3]; memset(srat_proc, 0, sizeof(srat_proc)); srat_proc[0].id = ACPI_SRAT_TYPE_CPU_AFFINITY; srat_proc[0].handler = acpi_parse_processor_affinity; srat_proc[1].id = ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY; srat_proc[1].handler = acpi_parse_x2apic_affinity; + srat_proc[2].id = ACPI_SRAT_TYPE_GICC_AFFINITY; + srat_proc[2].handler = acpi_parse_gicc_affinity; acpi_table_parse_entries_array(ACPI_SIG_SRAT, sizeof(struct acpi_table_srat), @@ -347,8 +467,6 @@ int __init acpi_numa_init(void) /* SLIT: System Locality Information Table */ acpi_table_parse(ACPI_SIG_SLIT, acpi_parse_slit); - acpi_numa_arch_fixup(); - if (cnt < 0) return cnt; else if (!parsed_numa_memblks) diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c index b108f1358..4305ee9db 100644 --- a/drivers/acpi/osl.c +++ b/drivers/acpi/osl.c @@ -309,7 +309,7 @@ static void acpi_unmap(acpi_physical_address pg_off, void __iomem *vaddr) * During early init (when acpi_gbl_permanent_mmap has not been set yet) this * routine simply calls __acpi_map_table() to get the job done. */ -void __iomem *__init_refok +void __iomem *__ref acpi_os_map_iomem(acpi_physical_address phys, acpi_size size) { struct acpi_ioremap *map; @@ -362,8 +362,7 @@ out: } EXPORT_SYMBOL_GPL(acpi_os_map_iomem); -void *__init_refok -acpi_os_map_memory(acpi_physical_address phys, acpi_size size) +void *__ref acpi_os_map_memory(acpi_physical_address phys, acpi_size size) { return (void *)acpi_os_map_iomem(phys, size); } diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c new file mode 100644 index 000000000..b5b376e08 --- /dev/null +++ b/drivers/acpi/pci_mcfg.c @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2016 Broadcom + * Author: Jayachandran C + * Copyright (C) 2016 Semihalf + * Author: Tomasz Nowicki + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, as + * published by the Free Software Foundation (the "GPL"). + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 (GPLv2) for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 (GPLv2) along with this source code. + */ + +#define pr_fmt(fmt) "ACPI: " fmt + +#include +#include +#include + +/* Structure to hold entries from the MCFG table */ +struct mcfg_entry { + struct list_head list; + phys_addr_t addr; + u16 segment; + u8 bus_start; + u8 bus_end; +}; + +/* List to save MCFG entries */ +static LIST_HEAD(pci_mcfg_list); + +phys_addr_t pci_mcfg_lookup(u16 seg, struct resource *bus_res) +{ + struct mcfg_entry *e; + + /* + * We expect exact match, unless MCFG entry end bus covers more than + * specified by caller. + */ + list_for_each_entry(e, &pci_mcfg_list, list) { + if (e->segment == seg && e->bus_start == bus_res->start && + e->bus_end >= bus_res->end) + return e->addr; + } + + return 0; +} + +static __init int pci_mcfg_parse(struct acpi_table_header *header) +{ + struct acpi_table_mcfg *mcfg; + struct acpi_mcfg_allocation *mptr; + struct mcfg_entry *e, *arr; + int i, n; + + if (header->length < sizeof(struct acpi_table_mcfg)) + return -EINVAL; + + n = (header->length - sizeof(struct acpi_table_mcfg)) / + sizeof(struct acpi_mcfg_allocation); + mcfg = (struct acpi_table_mcfg *)header; + mptr = (struct acpi_mcfg_allocation *) &mcfg[1]; + + arr = kcalloc(n, sizeof(*arr), GFP_KERNEL); + if (!arr) + return -ENOMEM; + + for (i = 0, e = arr; i < n; i++, mptr++, e++) { + e->segment = mptr->pci_segment; + e->addr = mptr->address; + e->bus_start = mptr->start_bus_number; + e->bus_end = mptr->end_bus_number; + list_add(&e->list, &pci_mcfg_list); + } + + pr_info("MCFG table detected, %d entries\n", n); + return 0; +} + +/* Interface called by ACPI - parse and save MCFG table */ +void __init pci_mmcfg_late_init(void) +{ + int err = acpi_table_parse(ACPI_SIG_MCFG, pci_mcfg_parse); + if (err) + pr_err("Failed to parse MCFG (%d)\n", err); +} diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index ae3fe4e64..d144168d4 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -720,6 +720,36 @@ next: } } +static void acpi_pci_root_remap_iospace(struct resource_entry *entry) +{ +#ifdef PCI_IOBASE + struct resource *res = entry->res; + resource_size_t cpu_addr = res->start; + resource_size_t pci_addr = cpu_addr - entry->offset; + resource_size_t length = resource_size(res); + unsigned long port; + + if (pci_register_io_range(cpu_addr, length)) + goto err; + + port = pci_address_to_pio(cpu_addr); + if (port == (unsigned long)-1) + goto err; + + res->start = port; + res->end = port + length - 1; + entry->offset = port - pci_addr; + + if (pci_remap_iospace(res, cpu_addr) < 0) + goto err; + + pr_info("Remapped I/O %pa to %pR\n", &cpu_addr, res); + return; +err: + res->flags |= IORESOURCE_DISABLED; +#endif +} + int acpi_pci_probe_root_resources(struct acpi_pci_root_info *info) { int ret; @@ -740,6 +770,9 @@ int acpi_pci_probe_root_resources(struct acpi_pci_root_info *info) "no IO and memory resources present in _CRS\n"); else { resource_list_for_each_entry_safe(entry, tmp, list) { + if (entry->res->flags & IORESOURCE_IO) + acpi_pci_root_remap_iospace(entry); + if (entry->res->flags & IORESOURCE_DISABLED) resource_list_destroy_entry(entry); else @@ -811,6 +844,8 @@ static void acpi_pci_root_release_info(struct pci_host_bridge *bridge) resource_list_for_each_entry(entry, &bridge->windows) { res = entry->res; + if (res->flags & IORESOURCE_IO) + pci_unmap_iospace(res); if (res->parent && (res->flags & (IORESOURCE_MEM | IORESOURCE_IO))) release_resource(res); diff --git a/drivers/acpi/pci_slot.c b/drivers/acpi/pci_slot.c index 7188e53b6..f62c68e24 100644 --- a/drivers/acpi/pci_slot.c +++ b/drivers/acpi/pci_slot.c @@ -22,8 +22,9 @@ * General Public License for more details. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include -#include #include #include #include @@ -33,30 +34,11 @@ #include #include -static bool debug; static int check_sta_before_sun; -#define DRIVER_VERSION "0.1" -#define DRIVER_AUTHOR "Alex Chiang " -#define DRIVER_DESC "ACPI PCI Slot Detection Driver" -MODULE_AUTHOR(DRIVER_AUTHOR); -MODULE_DESCRIPTION(DRIVER_DESC); -MODULE_LICENSE("GPL"); -MODULE_PARM_DESC(debug, "Debugging mode enabled or not"); -module_param(debug, bool, 0644); - #define _COMPONENT ACPI_PCI_COMPONENT ACPI_MODULE_NAME("pci_slot"); -#define MY_NAME "pci_slot" -#define err(format, arg...) pr_err("%s: " format , MY_NAME , ## arg) -#define info(format, arg...) pr_info("%s: " format , MY_NAME , ## arg) -#define dbg(format, arg...) \ - do { \ - if (debug) \ - pr_debug("%s: " format, MY_NAME , ## arg); \ - } while (0) - #define SLOT_NAME_SIZE 21 /* Inspired by #define in acpiphp.h */ struct acpi_pci_slot { @@ -76,7 +58,7 @@ check_slot(acpi_handle handle, unsigned long long *sun) struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; acpi_get_name(handle, ACPI_FULL_PATHNAME, &buffer); - dbg("Checking slot on path: %s\n", (char *)buffer.pointer); + pr_debug("Checking slot on path: %s\n", (char *)buffer.pointer); if (check_sta_before_sun) { /* If SxFy doesn't have _STA, we just assume it's there */ @@ -87,14 +69,16 @@ check_slot(acpi_handle handle, unsigned long long *sun) status = acpi_evaluate_integer(handle, "_ADR", NULL, &adr); if (ACPI_FAILURE(status)) { - dbg("_ADR returned %d on %s\n", status, (char *)buffer.pointer); + pr_debug("_ADR returned %d on %s\n", + status, (char *)buffer.pointer); goto out; } /* No _SUN == not a slot == bail */ status = acpi_evaluate_integer(handle, "_SUN", NULL, sun); if (ACPI_FAILURE(status)) { - dbg("_SUN returned %d on %s\n", status, (char *)buffer.pointer); + pr_debug("_SUN returned %d on %s\n", + status, (char *)buffer.pointer); goto out; } @@ -132,15 +116,13 @@ register_slot(acpi_handle handle, u32 lvl, void *context, void **rv) } slot = kmalloc(sizeof(*slot), GFP_KERNEL); - if (!slot) { - err("%s: cannot allocate memory\n", __func__); + if (!slot) return AE_OK; - } snprintf(name, sizeof(name), "%llu", sun); pci_slot = pci_create_slot(pci_bus, device, name, NULL); if (IS_ERR(pci_slot)) { - err("pci_create_slot returned %ld\n", PTR_ERR(pci_slot)); + pr_err("pci_create_slot returned %ld\n", PTR_ERR(pci_slot)); kfree(slot); return AE_OK; } @@ -150,8 +132,8 @@ register_slot(acpi_handle handle, u32 lvl, void *context, void **rv) get_device(&pci_bus->dev); - dbg("pci_slot: %p, pci_bus: %x, device: %d, name: %s\n", - pci_slot, pci_bus->number, device, name); + pr_debug("%p, pci_bus: %x, device: %d, name: %s\n", + pci_slot, pci_bus->number, device, name); return AE_OK; } @@ -186,7 +168,8 @@ void acpi_pci_slot_remove(struct pci_bus *bus) static int do_sta_before_sun(const struct dmi_system_id *d) { - info("%s detected: will evaluate _STA before calling _SUN\n", d->ident); + pr_info("%s detected: will evaluate _STA before calling _SUN\n", + d->ident); check_sta_before_sun = 1; return 0; } diff --git a/drivers/acpi/pmic/intel_pmic.c b/drivers/acpi/pmic/intel_pmic.c index bd772cd56..ca18e0d23 100644 --- a/drivers/acpi/pmic/intel_pmic.c +++ b/drivers/acpi/pmic/intel_pmic.c @@ -13,7 +13,7 @@ * GNU General Public License for more details. */ -#include +#include #include #include #include @@ -21,12 +21,19 @@ #define PMIC_POWER_OPREGION_ID 0x8d #define PMIC_THERMAL_OPREGION_ID 0x8c +#define PMIC_REGS_OPREGION_ID 0x8f + +struct intel_pmic_regs_handler_ctx { + unsigned int val; + u16 addr; +}; struct intel_pmic_opregion { struct mutex lock; struct acpi_lpat_conversion_table *lpat_table; struct regmap *regmap; struct intel_pmic_opregion_data *data; + struct intel_pmic_regs_handler_ctx ctx; }; static int pmic_get_reg_bit(int address, struct pmic_table *table, @@ -131,7 +138,7 @@ static int pmic_thermal_aux(struct intel_pmic_opregion *opregion, int reg, } static int pmic_thermal_pen(struct intel_pmic_opregion *opregion, int reg, - u32 function, u64 *value) + int bit, u32 function, u64 *value) { struct intel_pmic_opregion_data *d = opregion->data; struct regmap *regmap = opregion->regmap; @@ -140,12 +147,12 @@ static int pmic_thermal_pen(struct intel_pmic_opregion *opregion, int reg, return -ENXIO; if (function == ACPI_READ) - return d->get_policy(regmap, reg, value); + return d->get_policy(regmap, reg, bit, value); if (*value != 0 && *value != 1) return -EINVAL; - return d->update_policy(regmap, reg, *value); + return d->update_policy(regmap, reg, bit, *value); } static bool pmic_thermal_is_temp(int address) @@ -170,13 +177,13 @@ static acpi_status intel_pmic_thermal_handler(u32 function, { struct intel_pmic_opregion *opregion = region_context; struct intel_pmic_opregion_data *d = opregion->data; - int reg, result; + int reg, bit, result; if (bits != 32 || !value64) return AE_BAD_PARAMETER; result = pmic_get_reg_bit(address, d->thermal_table, - d->thermal_table_count, ®, NULL); + d->thermal_table_count, ®, &bit); if (result == -ENOENT) return AE_BAD_PARAMETER; @@ -187,7 +194,8 @@ static acpi_status intel_pmic_thermal_handler(u32 function, else if (pmic_thermal_is_aux(address)) result = pmic_thermal_aux(opregion, reg, function, value64); else if (pmic_thermal_is_pen(address)) - result = pmic_thermal_pen(opregion, reg, function, value64); + result = pmic_thermal_pen(opregion, reg, bit, + function, value64); else result = -EINVAL; @@ -203,6 +211,48 @@ static acpi_status intel_pmic_thermal_handler(u32 function, return AE_OK; } +static acpi_status intel_pmic_regs_handler(u32 function, + acpi_physical_address address, u32 bits, u64 *value64, + void *handler_context, void *region_context) +{ + struct intel_pmic_opregion *opregion = region_context; + int result = 0; + + switch (address) { + case 0: + return AE_OK; + case 1: + opregion->ctx.addr |= (*value64 & 0xff) << 8; + return AE_OK; + case 2: + opregion->ctx.addr |= *value64 & 0xff; + return AE_OK; + case 3: + opregion->ctx.val = *value64 & 0xff; + return AE_OK; + case 4: + if (*value64) { + result = regmap_write(opregion->regmap, opregion->ctx.addr, + opregion->ctx.val); + } else { + result = regmap_read(opregion->regmap, opregion->ctx.addr, + &opregion->ctx.val); + if (result == 0) + *value64 = opregion->ctx.val; + } + memset(&opregion->ctx, 0x00, sizeof(opregion->ctx)); + } + + if (result < 0) { + if (result == -EINVAL) + return AE_BAD_PARAMETER; + else + return AE_ERROR; + } + + return AE_OK; +} + int intel_pmic_install_opregion_handler(struct device *dev, acpi_handle handle, struct regmap *regmap, struct intel_pmic_opregion_data *d) @@ -242,16 +292,30 @@ int intel_pmic_install_opregion_handler(struct device *dev, acpi_handle handle, acpi_remove_address_space_handler(handle, PMIC_POWER_OPREGION_ID, intel_pmic_power_handler); ret = -ENODEV; - goto out_error; + goto out_remove_power_handler; + } + + status = acpi_install_address_space_handler(handle, + PMIC_REGS_OPREGION_ID, intel_pmic_regs_handler, NULL, + opregion); + if (ACPI_FAILURE(status)) { + ret = -ENODEV; + goto out_remove_thermal_handler; } opregion->data = d; return 0; +out_remove_thermal_handler: + acpi_remove_address_space_handler(handle, PMIC_THERMAL_OPREGION_ID, + intel_pmic_thermal_handler); + +out_remove_power_handler: + acpi_remove_address_space_handler(handle, PMIC_POWER_OPREGION_ID, + intel_pmic_power_handler); + out_error: acpi_lpat_free_conversion_table(opregion->lpat_table); return ret; } EXPORT_SYMBOL_GPL(intel_pmic_install_opregion_handler); - -MODULE_LICENSE("GPL"); diff --git a/drivers/acpi/pmic/intel_pmic.h b/drivers/acpi/pmic/intel_pmic.h index d4e90af8f..e8bfa7b86 100644 --- a/drivers/acpi/pmic/intel_pmic.h +++ b/drivers/acpi/pmic/intel_pmic.h @@ -12,8 +12,8 @@ struct intel_pmic_opregion_data { int (*update_power)(struct regmap *r, int reg, int bit, bool on); int (*get_raw_temp)(struct regmap *r, int reg); int (*update_aux)(struct regmap *r, int reg, int raw_temp); - int (*get_policy)(struct regmap *r, int reg, u64 *value); - int (*update_policy)(struct regmap *r, int reg, int enable); + int (*get_policy)(struct regmap *r, int reg, int bit, u64 *value); + int (*update_policy)(struct regmap *r, int reg, int bit, int enable); struct pmic_table *power_table; int power_table_count; struct pmic_table *thermal_table; diff --git a/drivers/acpi/pmic/intel_pmic_bxtwc.c b/drivers/acpi/pmic/intel_pmic_bxtwc.c new file mode 100644 index 000000000..90011aad4 --- /dev/null +++ b/drivers/acpi/pmic/intel_pmic_bxtwc.c @@ -0,0 +1,420 @@ +/* + * intel_pmic_bxtwc.c - Intel BXT WhiskeyCove PMIC operation region driver + * + * Copyright (C) 2015 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version + * 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include "intel_pmic.h" + +#define WHISKEY_COVE_ALRT_HIGH_BIT_MASK 0x0F +#define WHISKEY_COVE_ADC_HIGH_BIT(x) (((x & 0x0F) << 8)) +#define WHISKEY_COVE_ADC_CURSRC(x) (((x & 0xF0) >> 4)) +#define VR_MODE_DISABLED 0 +#define VR_MODE_AUTO BIT(0) +#define VR_MODE_NORMAL BIT(1) +#define VR_MODE_SWITCH BIT(2) +#define VR_MODE_ECO (BIT(0)|BIT(1)) +#define VSWITCH2_OUTPUT BIT(5) +#define VSWITCH1_OUTPUT BIT(4) +#define VUSBPHY_CHARGE BIT(1) + +static struct pmic_table power_table[] = { + { + .address = 0x0, + .reg = 0x63, + .bit = VR_MODE_AUTO, + }, /* VDD1 -> VDD1CNT */ + { + .address = 0x04, + .reg = 0x65, + .bit = VR_MODE_AUTO, + }, /* VDD2 -> VDD2CNT */ + { + .address = 0x08, + .reg = 0x67, + .bit = VR_MODE_AUTO, + }, /* VDD3 -> VDD3CNT */ + { + .address = 0x0c, + .reg = 0x6d, + .bit = VR_MODE_AUTO, + }, /* VLFX -> VFLEXCNT */ + { + .address = 0x10, + .reg = 0x6f, + .bit = VR_MODE_NORMAL, + }, /* VP1A -> VPROG1ACNT */ + { + .address = 0x14, + .reg = 0x70, + .bit = VR_MODE_NORMAL, + }, /* VP1B -> VPROG1BCNT */ + { + .address = 0x18, + .reg = 0x71, + .bit = VR_MODE_NORMAL, + }, /* VP1C -> VPROG1CCNT */ + { + .address = 0x1c, + .reg = 0x72, + .bit = VR_MODE_NORMAL, + }, /* VP1D -> VPROG1DCNT */ + { + .address = 0x20, + .reg = 0x73, + .bit = VR_MODE_NORMAL, + }, /* VP2A -> VPROG2ACNT */ + { + .address = 0x24, + .reg = 0x74, + .bit = VR_MODE_NORMAL, + }, /* VP2B -> VPROG2BCNT */ + { + .address = 0x28, + .reg = 0x75, + .bit = VR_MODE_NORMAL, + }, /* VP2C -> VPROG2CCNT */ + { + .address = 0x2c, + .reg = 0x76, + .bit = VR_MODE_NORMAL, + }, /* VP3A -> VPROG3ACNT */ + { + .address = 0x30, + .reg = 0x77, + .bit = VR_MODE_NORMAL, + }, /* VP3B -> VPROG3BCNT */ + { + .address = 0x34, + .reg = 0x78, + .bit = VSWITCH2_OUTPUT, + }, /* VSW2 -> VLD0CNT Bit 5*/ + { + .address = 0x38, + .reg = 0x78, + .bit = VSWITCH1_OUTPUT, + }, /* VSW1 -> VLD0CNT Bit 4 */ + { + .address = 0x3c, + .reg = 0x78, + .bit = VUSBPHY_CHARGE, + }, /* VUPY -> VLDOCNT Bit 1 */ + { + .address = 0x40, + .reg = 0x7b, + .bit = VR_MODE_NORMAL, + }, /* VRSO -> VREFSOCCNT*/ + { + .address = 0x44, + .reg = 0xA0, + .bit = VR_MODE_NORMAL, + }, /* VP1E -> VPROG1ECNT */ + { + .address = 0x48, + .reg = 0xA1, + .bit = VR_MODE_NORMAL, + }, /* VP1F -> VPROG1FCNT */ + { + .address = 0x4c, + .reg = 0xA2, + .bit = VR_MODE_NORMAL, + }, /* VP2D -> VPROG2DCNT */ + { + .address = 0x50, + .reg = 0xA3, + .bit = VR_MODE_NORMAL, + }, /* VP4A -> VPROG4ACNT */ + { + .address = 0x54, + .reg = 0xA4, + .bit = VR_MODE_NORMAL, + }, /* VP4B -> VPROG4BCNT */ + { + .address = 0x58, + .reg = 0xA5, + .bit = VR_MODE_NORMAL, + }, /* VP4C -> VPROG4CCNT */ + { + .address = 0x5c, + .reg = 0xA6, + .bit = VR_MODE_NORMAL, + }, /* VP4D -> VPROG4DCNT */ + { + .address = 0x60, + .reg = 0xA7, + .bit = VR_MODE_NORMAL, + }, /* VP5A -> VPROG5ACNT */ + { + .address = 0x64, + .reg = 0xA8, + .bit = VR_MODE_NORMAL, + }, /* VP5B -> VPROG5BCNT */ + { + .address = 0x68, + .reg = 0xA9, + .bit = VR_MODE_NORMAL, + }, /* VP6A -> VPROG6ACNT */ + { + .address = 0x6c, + .reg = 0xAA, + .bit = VR_MODE_NORMAL, + }, /* VP6B -> VPROG6BCNT */ + { + .address = 0x70, + .reg = 0x36, + .bit = BIT(2), + }, /* SDWN_N -> MODEMCTRL Bit 2 */ + { + .address = 0x74, + .reg = 0x36, + .bit = BIT(0), + } /* MOFF -> MODEMCTRL Bit 0 */ +}; + +static struct pmic_table thermal_table[] = { + { + .address = 0x00, + .reg = 0x4F39 + }, + { + .address = 0x04, + .reg = 0x4F24 + }, + { + .address = 0x08, + .reg = 0x4F26 + }, + { + .address = 0x0c, + .reg = 0x4F3B + }, + { + .address = 0x10, + .reg = 0x4F28 + }, + { + .address = 0x14, + .reg = 0x4F2A + }, + { + .address = 0x18, + .reg = 0x4F3D + }, + { + .address = 0x1c, + .reg = 0x4F2C + }, + { + .address = 0x20, + .reg = 0x4F2E + }, + { + .address = 0x24, + .reg = 0x4F3F + }, + { + .address = 0x28, + .reg = 0x4F30 + }, + { + .address = 0x30, + .reg = 0x4F41 + }, + { + .address = 0x34, + .reg = 0x4F32 + }, + { + .address = 0x3c, + .reg = 0x4F43 + }, + { + .address = 0x40, + .reg = 0x4F34 + }, + { + .address = 0x48, + .reg = 0x4F6A, + .bit = 0, + }, + { + .address = 0x4C, + .reg = 0x4F6A, + .bit = 1 + }, + { + .address = 0x50, + .reg = 0x4F6A, + .bit = 2 + }, + { + .address = 0x54, + .reg = 0x4F6A, + .bit = 4 + }, + { + .address = 0x58, + .reg = 0x4F6A, + .bit = 5 + }, + { + .address = 0x5C, + .reg = 0x4F6A, + .bit = 3 + }, +}; + +static int intel_bxtwc_pmic_get_power(struct regmap *regmap, int reg, + int bit, u64 *value) +{ + int data; + + if (regmap_read(regmap, reg, &data)) + return -EIO; + + *value = (data & bit) ? 1 : 0; + return 0; +} + +static int intel_bxtwc_pmic_update_power(struct regmap *regmap, int reg, + int bit, bool on) +{ + u8 val, mask = bit; + + if (on) + val = 0xFF; + else + val = 0x0; + + return regmap_update_bits(regmap, reg, mask, val); +} + +static int intel_bxtwc_pmic_get_raw_temp(struct regmap *regmap, int reg) +{ + unsigned int val, adc_val, reg_val; + u8 temp_l, temp_h, cursrc; + unsigned long rlsb; + static const unsigned long rlsb_array[] = { + 0, 260420, 130210, 65100, 32550, 16280, + 8140, 4070, 2030, 0, 260420, 130210 }; + + if (regmap_read(regmap, reg, &val)) + return -EIO; + temp_l = (u8) val; + + if (regmap_read(regmap, (reg - 1), &val)) + return -EIO; + temp_h = (u8) val; + + reg_val = temp_l | WHISKEY_COVE_ADC_HIGH_BIT(temp_h); + cursrc = WHISKEY_COVE_ADC_CURSRC(temp_h); + rlsb = rlsb_array[cursrc]; + adc_val = reg_val * rlsb / 1000; + + return adc_val; +} + +static int +intel_bxtwc_pmic_update_aux(struct regmap *regmap, int reg, int raw) +{ + u32 bsr_num; + u16 resi_val, count = 0, thrsh = 0; + u8 alrt_h, alrt_l, cursel = 0; + + bsr_num = raw; + bsr_num /= (1 << 5); + + count = fls(bsr_num) - 1; + + cursel = clamp_t(s8, (count - 7), 0, 7); + thrsh = raw / (1 << (4 + cursel)); + + resi_val = (cursel << 9) | thrsh; + alrt_h = (resi_val >> 8) & WHISKEY_COVE_ALRT_HIGH_BIT_MASK; + if (regmap_update_bits(regmap, + reg - 1, + WHISKEY_COVE_ALRT_HIGH_BIT_MASK, + alrt_h)) + return -EIO; + + alrt_l = (u8)resi_val; + return regmap_write(regmap, reg, alrt_l); +} + +static int +intel_bxtwc_pmic_get_policy(struct regmap *regmap, int reg, int bit, u64 *value) +{ + u8 mask = BIT(bit); + unsigned int val; + + if (regmap_read(regmap, reg, &val)) + return -EIO; + + *value = (val & mask) >> bit; + return 0; +} + +static int +intel_bxtwc_pmic_update_policy(struct regmap *regmap, + int reg, int bit, int enable) +{ + u8 mask = BIT(bit), val = enable << bit; + + return regmap_update_bits(regmap, reg, mask, val); +} + +static struct intel_pmic_opregion_data intel_bxtwc_pmic_opregion_data = { + .get_power = intel_bxtwc_pmic_get_power, + .update_power = intel_bxtwc_pmic_update_power, + .get_raw_temp = intel_bxtwc_pmic_get_raw_temp, + .update_aux = intel_bxtwc_pmic_update_aux, + .get_policy = intel_bxtwc_pmic_get_policy, + .update_policy = intel_bxtwc_pmic_update_policy, + .power_table = power_table, + .power_table_count = ARRAY_SIZE(power_table), + .thermal_table = thermal_table, + .thermal_table_count = ARRAY_SIZE(thermal_table), +}; + +static int intel_bxtwc_pmic_opregion_probe(struct platform_device *pdev) +{ + struct intel_soc_pmic *pmic = dev_get_drvdata(pdev->dev.parent); + + return intel_pmic_install_opregion_handler(&pdev->dev, + ACPI_HANDLE(pdev->dev.parent), + pmic->regmap, + &intel_bxtwc_pmic_opregion_data); +} + +static struct platform_device_id bxt_wc_opregion_id_table[] = { + { .name = "bxt_wcove_region" }, + {}, +}; + +static struct platform_driver intel_bxtwc_pmic_opregion_driver = { + .probe = intel_bxtwc_pmic_opregion_probe, + .driver = { + .name = "bxt_whiskey_cove_pmic", + }, + .id_table = bxt_wc_opregion_id_table, +}; + +static int __init intel_bxtwc_pmic_opregion_driver_init(void) +{ + return platform_driver_register(&intel_bxtwc_pmic_opregion_driver); +} +device_initcall(intel_bxtwc_pmic_opregion_driver_init); diff --git a/drivers/acpi/pmic/intel_pmic_crc.c b/drivers/acpi/pmic/intel_pmic_crc.c index fcd1852dc..d7f1761ab 100644 --- a/drivers/acpi/pmic/intel_pmic_crc.c +++ b/drivers/acpi/pmic/intel_pmic_crc.c @@ -141,7 +141,8 @@ static int intel_crc_pmic_update_aux(struct regmap *regmap, int reg, int raw) regmap_update_bits(regmap, reg - 1, 0x3, raw >> 8) ? -EIO : 0; } -static int intel_crc_pmic_get_policy(struct regmap *regmap, int reg, u64 *value) +static int intel_crc_pmic_get_policy(struct regmap *regmap, + int reg, int bit, u64 *value) { int pen; @@ -152,7 +153,7 @@ static int intel_crc_pmic_get_policy(struct regmap *regmap, int reg, u64 *value) } static int intel_crc_pmic_update_policy(struct regmap *regmap, - int reg, int enable) + int reg, int bit, int enable) { int alert0; diff --git a/drivers/acpi/pmic/intel_pmic_xpower.c b/drivers/acpi/pmic/intel_pmic_xpower.c index 6a082d4de..e6e991ac2 100644 --- a/drivers/acpi/pmic/intel_pmic_xpower.c +++ b/drivers/acpi/pmic/intel_pmic_xpower.c @@ -13,7 +13,7 @@ * GNU General Public License for more details. */ -#include +#include #include #include #include @@ -262,7 +262,4 @@ static int __init intel_xpower_pmic_opregion_driver_init(void) { return platform_driver_register(&intel_xpower_pmic_opregion_driver); } -module_init(intel_xpower_pmic_opregion_driver_init); - -MODULE_DESCRIPTION("XPower AXP288 ACPI operation region driver"); -MODULE_LICENSE("GPL"); +device_initcall(intel_xpower_pmic_opregion_driver_init); diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c index 33a38d604..9125d7d96 100644 --- a/drivers/acpi/processor_core.c +++ b/drivers/acpi/processor_core.c @@ -108,13 +108,12 @@ static int map_gicc_mpidr(struct acpi_subtable_header *entry, return -EINVAL; } -static phys_cpuid_t map_madt_entry(int type, u32 acpi_id) +static phys_cpuid_t map_madt_entry(struct acpi_table_madt *madt, + int type, u32 acpi_id) { unsigned long madt_end, entry; phys_cpuid_t phys_id = PHYS_CPUID_INVALID; /* CPU hardware ID */ - struct acpi_table_madt *madt; - madt = get_madt_table(); if (!madt) return phys_id; @@ -145,6 +144,25 @@ static phys_cpuid_t map_madt_entry(int type, u32 acpi_id) return phys_id; } +phys_cpuid_t __init acpi_map_madt_entry(u32 acpi_id) +{ + struct acpi_table_madt *madt = NULL; + acpi_size tbl_size; + phys_cpuid_t rv; + + acpi_get_table_with_size(ACPI_SIG_MADT, 0, + (struct acpi_table_header **)&madt, + &tbl_size); + if (!madt) + return PHYS_CPUID_INVALID; + + rv = map_madt_entry(madt, 1, acpi_id); + + early_acpi_os_unmap_memory(madt, tbl_size); + + return rv; +} + static phys_cpuid_t map_mat_entry(acpi_handle handle, int type, u32 acpi_id) { struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; @@ -185,7 +203,7 @@ phys_cpuid_t acpi_get_phys_id(acpi_handle handle, int type, u32 acpi_id) phys_id = map_mat_entry(handle, type, acpi_id); if (invalid_phys_cpuid(phys_id)) - phys_id = map_madt_entry(type, acpi_id); + phys_id = map_madt_entry(get_madt_table(), type, acpi_id); return phys_id; } diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index d2fa8cb82..0553aeebb 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -90,7 +90,7 @@ static void acpi_processor_notify(acpi_handle handle, u32 event, void *data) pr->performance_platform_limit); break; case ACPI_PROCESSOR_NOTIFY_POWER: - acpi_processor_cst_has_changed(pr); + acpi_processor_power_state_has_changed(pr); acpi_bus_generate_netlink_event(device->pnp.device_class, dev_name(&device->dev), event, 0); break; @@ -118,12 +118,13 @@ static int acpi_cpu_soft_notify(struct notifier_block *nfb, struct acpi_device *device; action &= ~CPU_TASKS_FROZEN; - /* - * CPU_STARTING and CPU_DYING must not sleep. Return here since - * acpi_bus_get_device() may sleep. - */ - if (action == CPU_STARTING || action == CPU_DYING) + switch (action) { + case CPU_ONLINE: + case CPU_DEAD: + break; + default: return NOTIFY_DONE; + } if (!pr || acpi_bus_get_device(pr->handle, &device)) return NOTIFY_DONE; diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 444e3745c..cea52528a 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -59,6 +59,12 @@ module_param(latency_factor, uint, 0644); static DEFINE_PER_CPU(struct cpuidle_device *, acpi_cpuidle_device); +struct cpuidle_driver acpi_idle_driver = { + .name = "acpi_idle", + .owner = THIS_MODULE, +}; + +#ifdef CONFIG_ACPI_PROCESSOR_CSTATE static DEFINE_PER_CPU(struct acpi_processor_cx * [CPUIDLE_STATE_MAX], acpi_cstate); @@ -297,7 +303,6 @@ static int acpi_processor_get_power_info_cst(struct acpi_processor *pr) struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; union acpi_object *cst; - if (nocst) return -ENODEV; @@ -570,7 +575,7 @@ static int acpi_processor_power_verify(struct acpi_processor *pr) return (working); } -static int acpi_processor_get_power_info(struct acpi_processor *pr) +static int acpi_processor_get_cstate_info(struct acpi_processor *pr) { unsigned int i; int result; @@ -804,36 +809,12 @@ static void acpi_idle_enter_freeze(struct cpuidle_device *dev, acpi_idle_do_entry(cx); } -struct cpuidle_driver acpi_idle_driver = { - .name = "acpi_idle", - .owner = THIS_MODULE, -}; - -/** - * acpi_processor_setup_cpuidle_cx - prepares and configures CPUIDLE - * device i.e. per-cpu data - * - * @pr: the ACPI processor - * @dev : the cpuidle device - */ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, struct cpuidle_device *dev) { int i, count = CPUIDLE_DRIVER_STATE_START; struct acpi_processor_cx *cx; - if (!pr->flags.power_setup_done) - return -EINVAL; - - if (pr->flags.power == 0) { - return -EINVAL; - } - - if (!dev) - return -EINVAL; - - dev->cpu = pr->id; - if (max_cstate == 0) max_cstate = 1; @@ -856,31 +837,13 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, return 0; } -/** - * acpi_processor_setup_cpuidle states- prepares and configures cpuidle - * global state data i.e. idle routines - * - * @pr: the ACPI processor - */ -static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr) +static int acpi_processor_setup_cstates(struct acpi_processor *pr) { int i, count = CPUIDLE_DRIVER_STATE_START; struct acpi_processor_cx *cx; struct cpuidle_state *state; struct cpuidle_driver *drv = &acpi_idle_driver; - if (!pr->flags.power_setup_done) - return -EINVAL; - - if (pr->flags.power == 0) - return -EINVAL; - - drv->safe_state_index = -1; - for (i = CPUIDLE_DRIVER_STATE_START; i < CPUIDLE_STATE_MAX; i++) { - drv->states[i].name[0] = '\0'; - drv->states[i].desc[0] = '\0'; - } - if (max_cstate == 0) max_cstate = 1; @@ -892,7 +855,7 @@ static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr) state = &drv->states[count]; snprintf(state->name, CPUIDLE_NAME_LEN, "C%d", i); - strncpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); + strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); state->exit_latency = cx->latency; state->target_residency = cx->latency * latency_factor; state->enter = acpi_idle_enter; @@ -925,6 +888,450 @@ static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr) return 0; } +static inline void acpi_processor_cstate_first_run_checks(void) +{ + acpi_status status; + static int first_run; + + if (first_run) + return; + dmi_check_system(processor_power_dmi_table); + max_cstate = acpi_processor_cstate_check(max_cstate); + if (max_cstate < ACPI_C_STATES_MAX) + pr_notice("ACPI: processor limited to max C-state %d\n", + max_cstate); + first_run++; + + if (acpi_gbl_FADT.cst_control && !nocst) { + status = acpi_os_write_port(acpi_gbl_FADT.smi_command, + acpi_gbl_FADT.cst_control, 8); + if (ACPI_FAILURE(status)) + ACPI_EXCEPTION((AE_INFO, status, + "Notifying BIOS of _CST ability failed")); + } +} +#else + +static inline int disabled_by_idle_boot_param(void) { return 0; } +static inline void acpi_processor_cstate_first_run_checks(void) { } +static int acpi_processor_get_cstate_info(struct acpi_processor *pr) +{ + return -ENODEV; +} + +static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, + struct cpuidle_device *dev) +{ + return -EINVAL; +} + +static int acpi_processor_setup_cstates(struct acpi_processor *pr) +{ + return -EINVAL; +} + +#endif /* CONFIG_ACPI_PROCESSOR_CSTATE */ + +struct acpi_lpi_states_array { + unsigned int size; + unsigned int composite_states_size; + struct acpi_lpi_state *entries; + struct acpi_lpi_state *composite_states[ACPI_PROCESSOR_MAX_POWER]; +}; + +static int obj_get_integer(union acpi_object *obj, u32 *value) +{ + if (obj->type != ACPI_TYPE_INTEGER) + return -EINVAL; + + *value = obj->integer.value; + return 0; +} + +static int acpi_processor_evaluate_lpi(acpi_handle handle, + struct acpi_lpi_states_array *info) +{ + acpi_status status; + int ret = 0; + int pkg_count, state_idx = 1, loop; + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + union acpi_object *lpi_data; + struct acpi_lpi_state *lpi_state; + + status = acpi_evaluate_object(handle, "_LPI", NULL, &buffer); + if (ACPI_FAILURE(status)) { + ACPI_DEBUG_PRINT((ACPI_DB_INFO, "No _LPI, giving up\n")); + return -ENODEV; + } + + lpi_data = buffer.pointer; + + /* There must be at least 4 elements = 3 elements + 1 package */ + if (!lpi_data || lpi_data->type != ACPI_TYPE_PACKAGE || + lpi_data->package.count < 4) { + pr_debug("not enough elements in _LPI\n"); + ret = -ENODATA; + goto end; + } + + pkg_count = lpi_data->package.elements[2].integer.value; + + /* Validate number of power states. */ + if (pkg_count < 1 || pkg_count != lpi_data->package.count - 3) { + pr_debug("count given by _LPI is not valid\n"); + ret = -ENODATA; + goto end; + } + + lpi_state = kcalloc(pkg_count, sizeof(*lpi_state), GFP_KERNEL); + if (!lpi_state) { + ret = -ENOMEM; + goto end; + } + + info->size = pkg_count; + info->entries = lpi_state; + + /* LPI States start at index 3 */ + for (loop = 3; state_idx <= pkg_count; loop++, state_idx++, lpi_state++) { + union acpi_object *element, *pkg_elem, *obj; + + element = &lpi_data->package.elements[loop]; + if (element->type != ACPI_TYPE_PACKAGE || element->package.count < 7) + continue; + + pkg_elem = element->package.elements; + + obj = pkg_elem + 6; + if (obj->type == ACPI_TYPE_BUFFER) { + struct acpi_power_register *reg; + + reg = (struct acpi_power_register *)obj->buffer.pointer; + if (reg->space_id != ACPI_ADR_SPACE_SYSTEM_IO && + reg->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) + continue; + + lpi_state->address = reg->address; + lpi_state->entry_method = + reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE ? + ACPI_CSTATE_FFH : ACPI_CSTATE_SYSTEMIO; + } else if (obj->type == ACPI_TYPE_INTEGER) { + lpi_state->entry_method = ACPI_CSTATE_INTEGER; + lpi_state->address = obj->integer.value; + } else { + continue; + } + + /* elements[7,8] skipped for now i.e. Residency/Usage counter*/ + + obj = pkg_elem + 9; + if (obj->type == ACPI_TYPE_STRING) + strlcpy(lpi_state->desc, obj->string.pointer, + ACPI_CX_DESC_LEN); + + lpi_state->index = state_idx; + if (obj_get_integer(pkg_elem + 0, &lpi_state->min_residency)) { + pr_debug("No min. residency found, assuming 10 us\n"); + lpi_state->min_residency = 10; + } + + if (obj_get_integer(pkg_elem + 1, &lpi_state->wake_latency)) { + pr_debug("No wakeup residency found, assuming 10 us\n"); + lpi_state->wake_latency = 10; + } + + if (obj_get_integer(pkg_elem + 2, &lpi_state->flags)) + lpi_state->flags = 0; + + if (obj_get_integer(pkg_elem + 3, &lpi_state->arch_flags)) + lpi_state->arch_flags = 0; + + if (obj_get_integer(pkg_elem + 4, &lpi_state->res_cnt_freq)) + lpi_state->res_cnt_freq = 1; + + if (obj_get_integer(pkg_elem + 5, &lpi_state->enable_parent_state)) + lpi_state->enable_parent_state = 0; + } + + acpi_handle_debug(handle, "Found %d power states\n", state_idx); +end: + kfree(buffer.pointer); + return ret; +} + +/* + * flat_state_cnt - the number of composite LPI states after the process of flattening + */ +static int flat_state_cnt; + +/** + * combine_lpi_states - combine local and parent LPI states to form a composite LPI state + * + * @local: local LPI state + * @parent: parent LPI state + * @result: composite LPI state + */ +static bool combine_lpi_states(struct acpi_lpi_state *local, + struct acpi_lpi_state *parent, + struct acpi_lpi_state *result) +{ + if (parent->entry_method == ACPI_CSTATE_INTEGER) { + if (!parent->address) /* 0 means autopromotable */ + return false; + result->address = local->address + parent->address; + } else { + result->address = parent->address; + } + + result->min_residency = max(local->min_residency, parent->min_residency); + result->wake_latency = local->wake_latency + parent->wake_latency; + result->enable_parent_state = parent->enable_parent_state; + result->entry_method = local->entry_method; + + result->flags = parent->flags; + result->arch_flags = parent->arch_flags; + result->index = parent->index; + + strlcpy(result->desc, local->desc, ACPI_CX_DESC_LEN); + strlcat(result->desc, "+", ACPI_CX_DESC_LEN); + strlcat(result->desc, parent->desc, ACPI_CX_DESC_LEN); + return true; +} + +#define ACPI_LPI_STATE_FLAGS_ENABLED BIT(0) + +static void stash_composite_state(struct acpi_lpi_states_array *curr_level, + struct acpi_lpi_state *t) +{ + curr_level->composite_states[curr_level->composite_states_size++] = t; +} + +static int flatten_lpi_states(struct acpi_processor *pr, + struct acpi_lpi_states_array *curr_level, + struct acpi_lpi_states_array *prev_level) +{ + int i, j, state_count = curr_level->size; + struct acpi_lpi_state *p, *t = curr_level->entries; + + curr_level->composite_states_size = 0; + for (j = 0; j < state_count; j++, t++) { + struct acpi_lpi_state *flpi; + + if (!(t->flags & ACPI_LPI_STATE_FLAGS_ENABLED)) + continue; + + if (flat_state_cnt >= ACPI_PROCESSOR_MAX_POWER) { + pr_warn("Limiting number of LPI states to max (%d)\n", + ACPI_PROCESSOR_MAX_POWER); + pr_warn("Please increase ACPI_PROCESSOR_MAX_POWER if needed.\n"); + break; + } + + flpi = &pr->power.lpi_states[flat_state_cnt]; + + if (!prev_level) { /* leaf/processor node */ + memcpy(flpi, t, sizeof(*t)); + stash_composite_state(curr_level, flpi); + flat_state_cnt++; + continue; + } + + for (i = 0; i < prev_level->composite_states_size; i++) { + p = prev_level->composite_states[i]; + if (t->index <= p->enable_parent_state && + combine_lpi_states(p, t, flpi)) { + stash_composite_state(curr_level, flpi); + flat_state_cnt++; + flpi++; + } + } + } + + kfree(curr_level->entries); + return 0; +} + +static int acpi_processor_get_lpi_info(struct acpi_processor *pr) +{ + int ret, i; + acpi_status status; + acpi_handle handle = pr->handle, pr_ahandle; + struct acpi_device *d = NULL; + struct acpi_lpi_states_array info[2], *tmp, *prev, *curr; + + if (!osc_pc_lpi_support_confirmed) + return -EOPNOTSUPP; + + if (!acpi_has_method(handle, "_LPI")) + return -EINVAL; + + flat_state_cnt = 0; + prev = &info[0]; + curr = &info[1]; + handle = pr->handle; + ret = acpi_processor_evaluate_lpi(handle, prev); + if (ret) + return ret; + flatten_lpi_states(pr, prev, NULL); + + status = acpi_get_parent(handle, &pr_ahandle); + while (ACPI_SUCCESS(status)) { + acpi_bus_get_device(pr_ahandle, &d); + handle = pr_ahandle; + + if (strcmp(acpi_device_hid(d), ACPI_PROCESSOR_CONTAINER_HID)) + break; + + /* can be optional ? */ + if (!acpi_has_method(handle, "_LPI")) + break; + + ret = acpi_processor_evaluate_lpi(handle, curr); + if (ret) + break; + + /* flatten all the LPI states in this level of hierarchy */ + flatten_lpi_states(pr, curr, prev); + + tmp = prev, prev = curr, curr = tmp; + + status = acpi_get_parent(handle, &pr_ahandle); + } + + pr->power.count = flat_state_cnt; + /* reset the index after flattening */ + for (i = 0; i < pr->power.count; i++) + pr->power.lpi_states[i].index = i; + + /* Tell driver that _LPI is supported. */ + pr->flags.has_lpi = 1; + pr->flags.power = 1; + + return 0; +} + +int __weak acpi_processor_ffh_lpi_probe(unsigned int cpu) +{ + return -ENODEV; +} + +int __weak acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi) +{ + return -ENODEV; +} + +/** + * acpi_idle_lpi_enter - enters an ACPI any LPI state + * @dev: the target CPU + * @drv: cpuidle driver containing cpuidle state info + * @index: index of target state + * + * Return: 0 for success or negative value for error + */ +static int acpi_idle_lpi_enter(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int index) +{ + struct acpi_processor *pr; + struct acpi_lpi_state *lpi; + + pr = __this_cpu_read(processors); + + if (unlikely(!pr)) + return -EINVAL; + + lpi = &pr->power.lpi_states[index]; + if (lpi->entry_method == ACPI_CSTATE_FFH) + return acpi_processor_ffh_lpi_enter(lpi); + + return -EINVAL; +} + +static int acpi_processor_setup_lpi_states(struct acpi_processor *pr) +{ + int i; + struct acpi_lpi_state *lpi; + struct cpuidle_state *state; + struct cpuidle_driver *drv = &acpi_idle_driver; + + if (!pr->flags.has_lpi) + return -EOPNOTSUPP; + + for (i = 0; i < pr->power.count && i < CPUIDLE_STATE_MAX; i++) { + lpi = &pr->power.lpi_states[i]; + + state = &drv->states[i]; + snprintf(state->name, CPUIDLE_NAME_LEN, "LPI-%d", i); + strlcpy(state->desc, lpi->desc, CPUIDLE_DESC_LEN); + state->exit_latency = lpi->wake_latency; + state->target_residency = lpi->min_residency; + if (lpi->arch_flags) + state->flags |= CPUIDLE_FLAG_TIMER_STOP; + state->enter = acpi_idle_lpi_enter; + drv->safe_state_index = i; + } + + drv->state_count = i; + + return 0; +} + +/** + * acpi_processor_setup_cpuidle_states- prepares and configures cpuidle + * global state data i.e. idle routines + * + * @pr: the ACPI processor + */ +static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr) +{ + int i; + struct cpuidle_driver *drv = &acpi_idle_driver; + + if (!pr->flags.power_setup_done || !pr->flags.power) + return -EINVAL; + + drv->safe_state_index = -1; + for (i = CPUIDLE_DRIVER_STATE_START; i < CPUIDLE_STATE_MAX; i++) { + drv->states[i].name[0] = '\0'; + drv->states[i].desc[0] = '\0'; + } + + if (pr->flags.has_lpi) + return acpi_processor_setup_lpi_states(pr); + + return acpi_processor_setup_cstates(pr); +} + +/** + * acpi_processor_setup_cpuidle_dev - prepares and configures CPUIDLE + * device i.e. per-cpu data + * + * @pr: the ACPI processor + * @dev : the cpuidle device + */ +static int acpi_processor_setup_cpuidle_dev(struct acpi_processor *pr, + struct cpuidle_device *dev) +{ + if (!pr->flags.power_setup_done || !pr->flags.power || !dev) + return -EINVAL; + + dev->cpu = pr->id; + if (pr->flags.has_lpi) + return acpi_processor_ffh_lpi_probe(pr->id); + + return acpi_processor_setup_cpuidle_cx(pr, dev); +} + +static int acpi_processor_get_power_info(struct acpi_processor *pr) +{ + int ret; + + ret = acpi_processor_get_lpi_info(pr); + if (ret) + ret = acpi_processor_get_cstate_info(pr); + + return ret; +} + int acpi_processor_hotplug(struct acpi_processor *pr) { int ret = 0; @@ -933,18 +1340,15 @@ int acpi_processor_hotplug(struct acpi_processor *pr) if (disabled_by_idle_boot_param()) return 0; - if (nocst) - return -ENODEV; - if (!pr->flags.power_setup_done) return -ENODEV; dev = per_cpu(acpi_cpuidle_device, pr->id); cpuidle_pause_and_lock(); cpuidle_disable_device(dev); - acpi_processor_get_power_info(pr); - if (pr->flags.power) { - acpi_processor_setup_cpuidle_cx(pr, dev); + ret = acpi_processor_get_power_info(pr); + if (!ret && pr->flags.power) { + acpi_processor_setup_cpuidle_dev(pr, dev); ret = cpuidle_enable_device(dev); } cpuidle_resume_and_unlock(); @@ -952,7 +1356,7 @@ int acpi_processor_hotplug(struct acpi_processor *pr) return ret; } -int acpi_processor_cst_has_changed(struct acpi_processor *pr) +int acpi_processor_power_state_has_changed(struct acpi_processor *pr) { int cpu; struct acpi_processor *_pr; @@ -961,9 +1365,6 @@ int acpi_processor_cst_has_changed(struct acpi_processor *pr) if (disabled_by_idle_boot_param()) return 0; - if (nocst) - return -ENODEV; - if (!pr->flags.power_setup_done) return -ENODEV; @@ -1000,7 +1401,7 @@ int acpi_processor_cst_has_changed(struct acpi_processor *pr) acpi_processor_get_power_info(_pr); if (_pr->flags.power) { dev = per_cpu(acpi_cpuidle_device, cpu); - acpi_processor_setup_cpuidle_cx(_pr, dev); + acpi_processor_setup_cpuidle_dev(_pr, dev); cpuidle_enable_device(dev); } } @@ -1015,35 +1416,16 @@ static int acpi_processor_registered; int acpi_processor_power_init(struct acpi_processor *pr) { - acpi_status status; int retval; struct cpuidle_device *dev; - static int first_run; if (disabled_by_idle_boot_param()) return 0; - if (!first_run) { - dmi_check_system(processor_power_dmi_table); - max_cstate = acpi_processor_cstate_check(max_cstate); - if (max_cstate < ACPI_C_STATES_MAX) - printk(KERN_NOTICE - "ACPI: processor limited to max C-state %d\n", - max_cstate); - first_run++; - } - - if (acpi_gbl_FADT.cst_control && !nocst) { - status = - acpi_os_write_port(acpi_gbl_FADT.smi_command, acpi_gbl_FADT.cst_control, 8); - if (ACPI_FAILURE(status)) { - ACPI_EXCEPTION((AE_INFO, status, - "Notifying BIOS of _CST ability failed")); - } - } + acpi_processor_cstate_first_run_checks(); - acpi_processor_get_power_info(pr); - pr->flags.power_setup_done = 1; + if (!acpi_processor_get_power_info(pr)) + pr->flags.power_setup_done = 1; /* * Install the idle handler if processor power management is supported. @@ -1066,7 +1448,7 @@ int acpi_processor_power_init(struct acpi_processor *pr) return -ENOMEM; per_cpu(acpi_cpuidle_device, pr->id) = dev; - acpi_processor_setup_cpuidle_cx(pr, dev); + acpi_processor_setup_cpuidle_dev(pr, dev); /* Register per-cpu cpuidle_device. Cpuidle driver * must already be registered before registering device diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index f3c022292..e878fc799 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -46,6 +46,13 @@ DEFINE_MUTEX(acpi_device_lock); LIST_HEAD(acpi_wakeup_device_list); static DEFINE_MUTEX(acpi_hp_context_lock); +/* + * The UART device described by the SPCR table is the only object which needs + * special-casing. Everything else is covered by ACPI namespace paths in STAO + * table. + */ +static u64 spcr_uart_addr; + struct acpi_dep_data { struct list_head node; acpi_handle master; @@ -494,6 +501,8 @@ static void acpi_device_del(struct acpi_device *device) device_del(&device->dev); } +static BLOCKING_NOTIFIER_HEAD(acpi_reconfig_chain); + static LIST_HEAD(acpi_device_del_list); static DEFINE_MUTEX(acpi_device_del_lock); @@ -514,6 +523,9 @@ static void acpi_device_del_work_fn(struct work_struct *work_not_used) mutex_unlock(&acpi_device_del_lock); + blocking_notifier_call_chain(&acpi_reconfig_chain, + ACPI_RECONFIG_DEVICE_REMOVE, adev); + acpi_device_del(adev); /* * Drop references to all power resources that might have been @@ -1406,7 +1418,7 @@ void acpi_init_device_object(struct acpi_device *device, acpi_handle handle, acpi_bus_get_flags(device); device->flags.match_driver = false; device->flags.initialized = true; - device->flags.visited = false; + acpi_device_clear_enumerated(device); device_initialize(&device->dev); dev_set_uevent_suppress(&device->dev, true); acpi_init_coherency(device); @@ -1453,6 +1465,41 @@ static int acpi_add_single_object(struct acpi_device **child, return 0; } +static acpi_status acpi_get_resource_memory(struct acpi_resource *ares, + void *context) +{ + struct resource *res = context; + + if (acpi_dev_resource_memory(ares, res)) + return AE_CTRL_TERMINATE; + + return AE_OK; +} + +static bool acpi_device_should_be_hidden(acpi_handle handle) +{ + acpi_status status; + struct resource res; + + /* Check if it should ignore the UART device */ + if (!(spcr_uart_addr && acpi_has_method(handle, METHOD_NAME__CRS))) + return false; + + /* + * The UART device described in SPCR table is assumed to have only one + * memory resource present. So we only look for the first one here. + */ + status = acpi_walk_resources(handle, METHOD_NAME__CRS, + acpi_get_resource_memory, &res); + if (ACPI_FAILURE(status) || res.start != spcr_uart_addr) + return false; + + acpi_handle_info(handle, "The UART device @%pa in SPCR table will be hidden\n", + &res.start); + + return true; +} + static int acpi_bus_type_and_status(acpi_handle handle, int *type, unsigned long long *sta) { @@ -1466,6 +1513,9 @@ static int acpi_bus_type_and_status(acpi_handle handle, int *type, switch (acpi_type) { case ACPI_TYPE_ANY: /* for ACPI_ROOT_OBJECT */ case ACPI_TYPE_DEVICE: + if (acpi_device_should_be_hidden(handle)) + return -ENODEV; + *type = ACPI_BUS_TYPE_DEVICE; status = acpi_bus_get_status_handle(handle, sta); if (ACPI_FAILURE(status)) @@ -1676,15 +1726,20 @@ static void acpi_default_enumeration(struct acpi_device *device) bool is_spi_i2c_slave = false; /* - * Do not enemerate SPI/I2C slaves as they will be enuerated by their + * Do not enumerate SPI/I2C slaves as they will be enumerated by their * respective parents. */ INIT_LIST_HEAD(&resource_list); acpi_dev_get_resources(device, &resource_list, acpi_check_spi_i2c_slave, &is_spi_i2c_slave); acpi_dev_free_resource_list(&resource_list); - if (!is_spi_i2c_slave) + if (!is_spi_i2c_slave) { acpi_create_platform_device(device); + acpi_device_set_enumerated(device); + } else { + blocking_notifier_call_chain(&acpi_reconfig_chain, + ACPI_RECONFIG_DEVICE_ADD, device); + } } static const struct acpi_device_id generic_device_ids[] = { @@ -1751,7 +1806,7 @@ static void acpi_bus_attach(struct acpi_device *device) acpi_bus_get_status(device); /* Skip devices that are not present. */ if (!acpi_device_is_present(device)) { - device->flags.visited = false; + acpi_device_clear_enumerated(device); device->flags.power_manageable = 0; return; } @@ -1766,7 +1821,7 @@ static void acpi_bus_attach(struct acpi_device *device) device->flags.initialized = true; } - device->flags.visited = false; + ret = acpi_scan_attach_handler(device); if (ret < 0) return; @@ -1780,7 +1835,6 @@ static void acpi_bus_attach(struct acpi_device *device) if (!ret && device->pnp.type.platform_id) acpi_default_enumeration(device); } - device->flags.visited = true; ok: list_for_each_entry(child, &device->children, node) @@ -1872,7 +1926,7 @@ void acpi_bus_trim(struct acpi_device *adev) */ acpi_device_set_power(adev, ACPI_STATE_D3_COLD); adev->flags.initialized = false; - adev->flags.visited = false; + acpi_device_clear_enumerated(adev); } EXPORT_SYMBOL_GPL(acpi_bus_trim); @@ -1916,9 +1970,26 @@ static int acpi_bus_scan_fixed(void) return result < 0 ? result : 0; } +static void __init acpi_get_spcr_uart_addr(void) +{ + acpi_status status; + struct acpi_table_spcr *spcr_ptr; + + status = acpi_get_table(ACPI_SIG_SPCR, 0, + (struct acpi_table_header **)&spcr_ptr); + if (ACPI_SUCCESS(status)) + spcr_uart_addr = spcr_ptr->serial_port.address; + else + printk(KERN_WARNING PREFIX "STAO table present, but SPCR is missing\n"); +} + +static bool acpi_scan_initialized; + int __init acpi_scan_init(void) { int result; + acpi_status status; + struct acpi_table_stao *stao_ptr; acpi_pci_root_init(); acpi_pci_link_init(); @@ -1934,6 +2005,20 @@ int __init acpi_scan_init(void) acpi_scan_add_handler(&generic_device_handler); + /* + * If there is STAO table, check whether it needs to ignore the UART + * device in SPCR table. + */ + status = acpi_get_table(ACPI_SIG_STAO, 0, + (struct acpi_table_header **)&stao_ptr); + if (ACPI_SUCCESS(status)) { + if (stao_ptr->header.length > sizeof(struct acpi_table_stao)) + printk(KERN_INFO PREFIX "STAO Name List not yet supported."); + + if (stao_ptr->ignore_uart) + acpi_get_spcr_uart_addr(); + } + mutex_lock(&acpi_scan_lock); /* * Enumerate devices in the ACPI namespace. @@ -1960,6 +2045,8 @@ int __init acpi_scan_init(void) acpi_update_all_gpes(); + acpi_scan_initialized = true; + out: mutex_unlock(&acpi_scan_lock); return result; @@ -2003,3 +2090,57 @@ int __init __acpi_probe_device_table(struct acpi_probe_entry *ap_head, int nr) return count; } + +struct acpi_table_events_work { + struct work_struct work; + void *table; + u32 event; +}; + +static void acpi_table_events_fn(struct work_struct *work) +{ + struct acpi_table_events_work *tew; + + tew = container_of(work, struct acpi_table_events_work, work); + + if (tew->event == ACPI_TABLE_EVENT_LOAD) { + acpi_scan_lock_acquire(); + acpi_bus_scan(ACPI_ROOT_OBJECT); + acpi_scan_lock_release(); + } + + kfree(tew); +} + +void acpi_scan_table_handler(u32 event, void *table, void *context) +{ + struct acpi_table_events_work *tew; + + if (!acpi_scan_initialized) + return; + + if (event != ACPI_TABLE_EVENT_LOAD) + return; + + tew = kmalloc(sizeof(*tew), GFP_KERNEL); + if (!tew) + return; + + INIT_WORK(&tew->work, acpi_table_events_fn); + tew->table = table; + tew->event = event; + + schedule_work(&tew->work); +} + +int acpi_reconfig_notifier_register(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&acpi_reconfig_chain, nb); +} +EXPORT_SYMBOL(acpi_reconfig_notifier_register); + +int acpi_reconfig_notifier_unregister(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&acpi_reconfig_chain, nb); +} +EXPORT_SYMBOL(acpi_reconfig_notifier_unregister); diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c index 7a2e4d45b..2b38c1bb0 100644 --- a/drivers/acpi/sleep.c +++ b/drivers/acpi/sleep.c @@ -47,15 +47,32 @@ static void acpi_sleep_tts_switch(u32 acpi_state) } } -static int tts_notify_reboot(struct notifier_block *this, +static void acpi_sleep_pts_switch(u32 acpi_state) +{ + acpi_status status; + + status = acpi_execute_simple_method(NULL, "\\_PTS", acpi_state); + if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) { + /* + * OS can't evaluate the _PTS object correctly. Some warning + * message will be printed. But it won't break anything. + */ + printk(KERN_NOTICE "Failure in evaluating _PTS object\n"); + } +} + +static int sleep_notify_reboot(struct notifier_block *this, unsigned long code, void *x) { acpi_sleep_tts_switch(ACPI_STATE_S5); + + acpi_sleep_pts_switch(ACPI_STATE_S5); + return NOTIFY_DONE; } -static struct notifier_block tts_notifier = { - .notifier_call = tts_notify_reboot, +static struct notifier_block sleep_notifier = { + .notifier_call = sleep_notify_reboot, .next = NULL, .priority = 0, }; @@ -899,9 +916,9 @@ int __init acpi_sleep_init(void) pr_info(PREFIX "(supports%s)\n", supported); /* - * Register the tts_notifier to reboot notifier list so that the _TTS - * object can also be evaluated when the system enters S5. + * Register the sleep_notifier to reboot notifier list so that the _TTS + * and _PTS object can also be evaluated when the system enters S5. */ - register_reboot_notifier(&tts_notifier); + register_reboot_notifier(&sleep_notifier); return 0; } diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c index 4b3a9e27f..358165e9f 100644 --- a/drivers/acpi/sysfs.c +++ b/drivers/acpi/sysfs.c @@ -378,8 +378,7 @@ static void acpi_table_attr_init(struct acpi_table_attr *table_attr, return; } -static acpi_status -acpi_sysfs_table_handler(u32 event, void *table, void *context) +acpi_status acpi_sysfs_table_handler(u32 event, void *table, void *context) { struct acpi_table_attr *table_attr; @@ -452,9 +451,8 @@ static int acpi_tables_sysfs_init(void) kobject_uevent(tables_kobj, KOBJ_ADD); kobject_uevent(dynamic_tables_kobj, KOBJ_ADD); - status = acpi_install_table_handler(acpi_sysfs_table_handler, NULL); - return ACPI_FAILURE(status) ? -EINVAL : 0; + return 0; err_dynamic_tables: kobject_put(tables_kobj); err: diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index a372f9eaa..9f0ad6ebb 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -34,6 +34,8 @@ #include #include #include +#include +#include #include "internal.h" #ifdef CONFIG_ACPI_CUSTOM_DSDT @@ -481,8 +483,10 @@ static DECLARE_BITMAP(acpi_initrd_installed, NR_ACPI_INITRD_TABLES); #define MAP_CHUNK_SIZE (NR_FIX_BTMAPS << PAGE_SHIFT) -static void __init acpi_table_initrd_init(void *data, size_t size) +void __init acpi_table_upgrade(void) { + void *data = (void *)initrd_start; + size_t size = initrd_end - initrd_start; int sig, no, table_nr = 0, total_offset = 0; long offset = 0; struct acpi_table_header *table; @@ -540,7 +544,7 @@ static void __init acpi_table_initrd_init(void *data, size_t size) return; acpi_tables_addr = - memblock_find_in_range(0, max_low_pfn_mapped << PAGE_SHIFT, + memblock_find_in_range(0, ACPI_TABLE_UPGRADE_MAX_PHYS, all_tables_size, PAGE_SIZE); if (!acpi_tables_addr) { WARN_ON(1); @@ -578,10 +582,10 @@ static void __init acpi_table_initrd_init(void *data, size_t size) clen = size; if (clen > MAP_CHUNK_SIZE - slop) clen = MAP_CHUNK_SIZE - slop; - dest_p = early_ioremap(dest_addr & PAGE_MASK, - clen + slop); + dest_p = early_memremap(dest_addr & PAGE_MASK, + clen + slop); memcpy(dest_p + slop, src_p, clen); - early_iounmap(dest_p, clen + slop); + early_memunmap(dest_p, clen + slop); src_p += clen; dest_addr += clen; size -= clen; @@ -696,10 +700,6 @@ next_table: } } #else -static void __init acpi_table_initrd_init(void *data, size_t size) -{ -} - static acpi_status acpi_table_initrd_override(struct acpi_table_header *existing_table, acpi_physical_address *address, @@ -742,11 +742,6 @@ acpi_os_table_override(struct acpi_table_header *existing_table, return AE_OK; } -void __init early_acpi_table_init(void *data, size_t size) -{ - acpi_table_initrd_init(data, size); -} - /* * acpi_table_init() * diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 82707f982..f4ebe3953 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -1259,7 +1259,8 @@ static int __init acpi_thermal_init(void) return -ENODEV; } - acpi_thermal_pm_queue = create_workqueue("acpi_thermal_pm"); + acpi_thermal_pm_queue = alloc_workqueue("acpi_thermal_pm", + WQ_HIGHPRI | WQ_MEM_RECLAIM, 0); if (!acpi_thermal_pm_queue) return -ENODEV; diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 3d1327615..a6b36fc53 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -167,6 +167,14 @@ static const struct dmi_system_id video_detect_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad X201s"), }, }, + { + .callback = video_detect_force_video, + .ident = "ThinkPad X201T", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad X201T"), + }, + }, /* The native backlight controls do not work on some older machines */ { diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig index e2dc4c045..2c8be74f4 100644 --- a/drivers/ata/Kconfig +++ b/drivers/ata/Kconfig @@ -98,12 +98,12 @@ config SATA_AHCI_PLATFORM If unsure, say N. -config AHCI_BRCMSTB - tristate "Broadcom STB AHCI SATA support" - depends on ARCH_BRCMSTB || BMIPS_GENERIC +config AHCI_BRCM + tristate "Broadcom AHCI SATA support" + depends on ARCH_BRCMSTB || BMIPS_GENERIC || ARCH_BCM_NSP help This option enables support for the AHCI SATA3 controller found on - STB SoC's. + Broadcom SoC's. If unsure, say N. diff --git a/drivers/ata/Makefile b/drivers/ata/Makefile index 0b2afb7e5..a46e6b784 100644 --- a/drivers/ata/Makefile +++ b/drivers/ata/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_SATA_INIC162X) += sata_inic162x.o obj-$(CONFIG_SATA_SIL24) += sata_sil24.o obj-$(CONFIG_SATA_DWC) += sata_dwc_460ex.o obj-$(CONFIG_SATA_HIGHBANK) += sata_highbank.o libahci.o -obj-$(CONFIG_AHCI_BRCMSTB) += ahci_brcmstb.o libahci.o libahci_platform.o +obj-$(CONFIG_AHCI_BRCM) += ahci_brcm.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_CEVA) += ahci_ceva.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_DA850) += ahci_da850.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_IMX) += ahci_imx.o libahci.o libahci_platform.o diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index a83bbcc58..90eabaf81 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -580,7 +580,7 @@ static struct pci_driver ahci_pci_driver = { }, }; -#if defined(CONFIG_PATA_MARVELL) || defined(CONFIG_PATA_MARVELL_MODULE) +#if IS_ENABLED(CONFIG_PATA_MARVELL) static int marvell_enable; #else static int marvell_enable = 1; diff --git a/drivers/ata/ahci_brcm.c b/drivers/ata/ahci_brcm.c new file mode 100644 index 000000000..6f8a7341f --- /dev/null +++ b/drivers/ata/ahci_brcm.c @@ -0,0 +1,406 @@ +/* + * Broadcom SATA3 AHCI Controller Driver + * + * Copyright © 2009-2015 Broadcom Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ahci.h" + +#define DRV_NAME "brcm-ahci" + +#define SATA_TOP_CTRL_VERSION 0x0 +#define SATA_TOP_CTRL_BUS_CTRL 0x4 + #define MMIO_ENDIAN_SHIFT 0 /* CPU->AHCI */ + #define DMADESC_ENDIAN_SHIFT 2 /* AHCI->DDR */ + #define DMADATA_ENDIAN_SHIFT 4 /* AHCI->DDR */ + #define PIODATA_ENDIAN_SHIFT 6 + #define ENDIAN_SWAP_NONE 0 + #define ENDIAN_SWAP_FULL 2 + #define OVERRIDE_HWINIT BIT(16) +#define SATA_TOP_CTRL_TP_CTRL 0x8 +#define SATA_TOP_CTRL_PHY_CTRL 0xc + #define SATA_TOP_CTRL_PHY_CTRL_1 0x0 + #define SATA_TOP_CTRL_1_PHY_DEFAULT_POWER_STATE BIT(14) + #define SATA_TOP_CTRL_PHY_CTRL_2 0x4 + #define SATA_TOP_CTRL_2_SW_RST_MDIOREG BIT(0) + #define SATA_TOP_CTRL_2_SW_RST_OOB BIT(1) + #define SATA_TOP_CTRL_2_SW_RST_RX BIT(2) + #define SATA_TOP_CTRL_2_SW_RST_TX BIT(3) + #define SATA_TOP_CTRL_2_PHY_GLOBAL_RESET BIT(14) + #define SATA_TOP_CTRL_PHY_OFFS 0x8 + #define SATA_TOP_MAX_PHYS 2 + +#define SATA_FIRST_PORT_CTRL 0x700 +#define SATA_NEXT_PORT_CTRL_OFFSET 0x80 +#define SATA_PORT_PCTRL6(reg_base) (reg_base + 0x18) + +/* On big-endian MIPS, buses are reversed to big endian, so switch them back */ +#if defined(CONFIG_MIPS) && defined(__BIG_ENDIAN) +#define DATA_ENDIAN 2 /* AHCI->DDR inbound accesses */ +#define MMIO_ENDIAN 2 /* CPU->AHCI outbound accesses */ +#else +#define DATA_ENDIAN 0 +#define MMIO_ENDIAN 0 +#endif + +#define BUS_CTRL_ENDIAN_CONF \ + ((DATA_ENDIAN << DMADATA_ENDIAN_SHIFT) | \ + (DATA_ENDIAN << DMADESC_ENDIAN_SHIFT) | \ + (MMIO_ENDIAN << MMIO_ENDIAN_SHIFT)) + +enum brcm_ahci_version { + BRCM_SATA_BCM7425 = 1, + BRCM_SATA_BCM7445, + BRCM_SATA_NSP, +}; + +enum brcm_ahci_quirks { + BRCM_AHCI_QUIRK_NO_NCQ = BIT(0), + BRCM_AHCI_QUIRK_SKIP_PHY_ENABLE = BIT(1), +}; + +struct brcm_ahci_priv { + struct device *dev; + void __iomem *top_ctrl; + u32 port_mask; + u32 quirks; + enum brcm_ahci_version version; +}; + +static const struct ata_port_info ahci_brcm_port_info = { + .flags = AHCI_FLAG_COMMON | ATA_FLAG_NO_DIPM, + .link_flags = ATA_LFLAG_NO_DB_DELAY, + .pio_mask = ATA_PIO4, + .udma_mask = ATA_UDMA6, + .port_ops = &ahci_platform_ops, +}; + +static inline u32 brcm_sata_readreg(void __iomem *addr) +{ + /* + * MIPS endianness is configured by boot strap, which also reverses all + * bus endianness (i.e., big-endian CPU + big endian bus ==> native + * endian I/O). + * + * Other architectures (e.g., ARM) either do not support big endian, or + * else leave I/O in little endian mode. + */ + if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) + return __raw_readl(addr); + else + return readl_relaxed(addr); +} + +static inline void brcm_sata_writereg(u32 val, void __iomem *addr) +{ + /* See brcm_sata_readreg() comments */ + if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) + __raw_writel(val, addr); + else + writel_relaxed(val, addr); +} + +static void brcm_sata_alpm_init(struct ahci_host_priv *hpriv) +{ + struct brcm_ahci_priv *priv = hpriv->plat_data; + u32 bus_ctrl, port_ctrl, host_caps; + int i; + + /* Enable support for ALPM */ + bus_ctrl = brcm_sata_readreg(priv->top_ctrl + + SATA_TOP_CTRL_BUS_CTRL); + brcm_sata_writereg(bus_ctrl | OVERRIDE_HWINIT, + priv->top_ctrl + SATA_TOP_CTRL_BUS_CTRL); + host_caps = readl(hpriv->mmio + HOST_CAP); + writel(host_caps | HOST_CAP_ALPM, hpriv->mmio); + brcm_sata_writereg(bus_ctrl, priv->top_ctrl + SATA_TOP_CTRL_BUS_CTRL); + + /* + * Adjust timeout to allow PLL sufficient time to lock while waking + * up from slumber mode. + */ + for (i = 0, port_ctrl = SATA_FIRST_PORT_CTRL; + i < SATA_TOP_MAX_PHYS; + i++, port_ctrl += SATA_NEXT_PORT_CTRL_OFFSET) { + if (priv->port_mask & BIT(i)) + writel(0xff1003fc, + hpriv->mmio + SATA_PORT_PCTRL6(port_ctrl)); + } +} + +static void brcm_sata_phy_enable(struct brcm_ahci_priv *priv, int port) +{ + void __iomem *phyctrl = priv->top_ctrl + SATA_TOP_CTRL_PHY_CTRL + + (port * SATA_TOP_CTRL_PHY_OFFS); + void __iomem *p; + u32 reg; + + if (priv->quirks & BRCM_AHCI_QUIRK_SKIP_PHY_ENABLE) + return; + + /* clear PHY_DEFAULT_POWER_STATE */ + p = phyctrl + SATA_TOP_CTRL_PHY_CTRL_1; + reg = brcm_sata_readreg(p); + reg &= ~SATA_TOP_CTRL_1_PHY_DEFAULT_POWER_STATE; + brcm_sata_writereg(reg, p); + + /* reset the PHY digital logic */ + p = phyctrl + SATA_TOP_CTRL_PHY_CTRL_2; + reg = brcm_sata_readreg(p); + reg &= ~(SATA_TOP_CTRL_2_SW_RST_MDIOREG | SATA_TOP_CTRL_2_SW_RST_OOB | + SATA_TOP_CTRL_2_SW_RST_RX); + reg |= SATA_TOP_CTRL_2_SW_RST_TX; + brcm_sata_writereg(reg, p); + reg = brcm_sata_readreg(p); + reg |= SATA_TOP_CTRL_2_PHY_GLOBAL_RESET; + brcm_sata_writereg(reg, p); + reg = brcm_sata_readreg(p); + reg &= ~SATA_TOP_CTRL_2_PHY_GLOBAL_RESET; + brcm_sata_writereg(reg, p); + (void)brcm_sata_readreg(p); +} + +static void brcm_sata_phy_disable(struct brcm_ahci_priv *priv, int port) +{ + void __iomem *phyctrl = priv->top_ctrl + SATA_TOP_CTRL_PHY_CTRL + + (port * SATA_TOP_CTRL_PHY_OFFS); + void __iomem *p; + u32 reg; + + if (priv->quirks & BRCM_AHCI_QUIRK_SKIP_PHY_ENABLE) + return; + + /* power-off the PHY digital logic */ + p = phyctrl + SATA_TOP_CTRL_PHY_CTRL_2; + reg = brcm_sata_readreg(p); + reg |= (SATA_TOP_CTRL_2_SW_RST_MDIOREG | SATA_TOP_CTRL_2_SW_RST_OOB | + SATA_TOP_CTRL_2_SW_RST_RX | SATA_TOP_CTRL_2_SW_RST_TX | + SATA_TOP_CTRL_2_PHY_GLOBAL_RESET); + brcm_sata_writereg(reg, p); + + /* set PHY_DEFAULT_POWER_STATE */ + p = phyctrl + SATA_TOP_CTRL_PHY_CTRL_1; + reg = brcm_sata_readreg(p); + reg |= SATA_TOP_CTRL_1_PHY_DEFAULT_POWER_STATE; + brcm_sata_writereg(reg, p); +} + +static void brcm_sata_phys_enable(struct brcm_ahci_priv *priv) +{ + int i; + + for (i = 0; i < SATA_TOP_MAX_PHYS; i++) + if (priv->port_mask & BIT(i)) + brcm_sata_phy_enable(priv, i); +} + +static void brcm_sata_phys_disable(struct brcm_ahci_priv *priv) +{ + int i; + + for (i = 0; i < SATA_TOP_MAX_PHYS; i++) + if (priv->port_mask & BIT(i)) + brcm_sata_phy_disable(priv, i); +} + +static u32 brcm_ahci_get_portmask(struct platform_device *pdev, + struct brcm_ahci_priv *priv) +{ + void __iomem *ahci; + struct resource *res; + u32 impl; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ahci"); + ahci = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(ahci)) + return 0; + + impl = readl(ahci + HOST_PORTS_IMPL); + + if (fls(impl) > SATA_TOP_MAX_PHYS) + dev_warn(priv->dev, "warning: more ports than PHYs (%#x)\n", + impl); + else if (!impl) + dev_info(priv->dev, "no ports found\n"); + + devm_iounmap(&pdev->dev, ahci); + devm_release_mem_region(&pdev->dev, res->start, resource_size(res)); + + return impl; +} + +static void brcm_sata_init(struct brcm_ahci_priv *priv) +{ + void __iomem *ctrl = priv->top_ctrl + SATA_TOP_CTRL_BUS_CTRL; + + /* Configure endianness */ + if (priv->version == BRCM_SATA_NSP) { + u32 data = brcm_sata_readreg(ctrl); + + data &= ~((0x03 << DMADATA_ENDIAN_SHIFT) | + (0x03 << DMADESC_ENDIAN_SHIFT)); + data |= (0x02 << DMADATA_ENDIAN_SHIFT) | + (0x02 << DMADESC_ENDIAN_SHIFT); + brcm_sata_writereg(data, ctrl); + } else + brcm_sata_writereg(BUS_CTRL_ENDIAN_CONF, ctrl); +} + +#ifdef CONFIG_PM_SLEEP +static int brcm_ahci_suspend(struct device *dev) +{ + struct ata_host *host = dev_get_drvdata(dev); + struct ahci_host_priv *hpriv = host->private_data; + struct brcm_ahci_priv *priv = hpriv->plat_data; + int ret; + + ret = ahci_platform_suspend(dev); + brcm_sata_phys_disable(priv); + return ret; +} + +static int brcm_ahci_resume(struct device *dev) +{ + struct ata_host *host = dev_get_drvdata(dev); + struct ahci_host_priv *hpriv = host->private_data; + struct brcm_ahci_priv *priv = hpriv->plat_data; + + brcm_sata_init(priv); + brcm_sata_phys_enable(priv); + brcm_sata_alpm_init(hpriv); + return ahci_platform_resume(dev); +} +#endif + +static struct scsi_host_template ahci_platform_sht = { + AHCI_SHT(DRV_NAME), +}; + +static const struct of_device_id ahci_of_match[] = { + {.compatible = "brcm,bcm7425-ahci", .data = (void *)BRCM_SATA_BCM7425}, + {.compatible = "brcm,bcm7445-ahci", .data = (void *)BRCM_SATA_BCM7445}, + {.compatible = "brcm,bcm-nsp-ahci", .data = (void *)BRCM_SATA_NSP}, + {}, +}; +MODULE_DEVICE_TABLE(of, ahci_of_match); + +static int brcm_ahci_probe(struct platform_device *pdev) +{ + const struct of_device_id *of_id; + struct device *dev = &pdev->dev; + struct brcm_ahci_priv *priv; + struct ahci_host_priv *hpriv; + struct resource *res; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + of_id = of_match_node(ahci_of_match, pdev->dev.of_node); + if (!of_id) + return -ENODEV; + + priv->version = (enum brcm_ahci_version)of_id->data; + priv->dev = dev; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "top-ctrl"); + priv->top_ctrl = devm_ioremap_resource(dev, res); + if (IS_ERR(priv->top_ctrl)) + return PTR_ERR(priv->top_ctrl); + + if ((priv->version == BRCM_SATA_BCM7425) || + (priv->version == BRCM_SATA_NSP)) { + priv->quirks |= BRCM_AHCI_QUIRK_NO_NCQ; + priv->quirks |= BRCM_AHCI_QUIRK_SKIP_PHY_ENABLE; + } + + brcm_sata_init(priv); + + priv->port_mask = brcm_ahci_get_portmask(pdev, priv); + if (!priv->port_mask) + return -ENODEV; + + brcm_sata_phys_enable(priv); + + hpriv = ahci_platform_get_resources(pdev); + if (IS_ERR(hpriv)) + return PTR_ERR(hpriv); + hpriv->plat_data = priv; + hpriv->flags = AHCI_HFLAG_WAKE_BEFORE_STOP; + + brcm_sata_alpm_init(hpriv); + + ret = ahci_platform_enable_resources(hpriv); + if (ret) + return ret; + + if (priv->quirks & BRCM_AHCI_QUIRK_NO_NCQ) + hpriv->flags |= AHCI_HFLAG_NO_NCQ; + + ret = ahci_platform_init_host(pdev, hpriv, &ahci_brcm_port_info, + &ahci_platform_sht); + if (ret) + return ret; + + dev_info(dev, "Broadcom AHCI SATA3 registered\n"); + + return 0; +} + +static int brcm_ahci_remove(struct platform_device *pdev) +{ + struct ata_host *host = dev_get_drvdata(&pdev->dev); + struct ahci_host_priv *hpriv = host->private_data; + struct brcm_ahci_priv *priv = hpriv->plat_data; + int ret; + + ret = ata_platform_remove_one(pdev); + if (ret) + return ret; + + brcm_sata_phys_disable(priv); + + return 0; +} + +static SIMPLE_DEV_PM_OPS(ahci_brcm_pm_ops, brcm_ahci_suspend, brcm_ahci_resume); + +static struct platform_driver brcm_ahci_driver = { + .probe = brcm_ahci_probe, + .remove = brcm_ahci_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = ahci_of_match, + .pm = &ahci_brcm_pm_ops, + }, +}; +module_platform_driver(brcm_ahci_driver); + +MODULE_DESCRIPTION("Broadcom SATA3 AHCI Controller Driver"); +MODULE_AUTHOR("Brian Norris"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:sata-brcmstb"); diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index ccd8cc47c..dcf2c724f 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -1975,7 +1975,7 @@ unsigned int ahci_qc_issue(struct ata_queued_cmd *qc) */ pp->active_link = qc->dev->link; - if (qc->tf.protocol == ATA_PROT_NCQ) + if (ata_is_ncq(qc->tf.protocol)) writel(1 << qc->tag, port_mmio + PORT_SCR_ACT); if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) { @@ -2392,12 +2392,20 @@ static int ahci_port_start(struct ata_port *ap) static void ahci_port_stop(struct ata_port *ap) { const char *emsg = NULL; + struct ahci_host_priv *hpriv = ap->host->private_data; + void __iomem *host_mmio = hpriv->mmio; int rc; /* de-initialize port */ rc = ahci_deinit_port(ap, &emsg); if (rc) ata_port_warn(ap, "%s (%d)\n", emsg, rc); + + /* + * Clear GHC.IS to prevent stuck INTx after disabling MSI and + * re-enabling INTx. + */ + writel(1 << ap->port_no, host_mmio + HOST_IRQ_STAT); } void ahci_print_info(struct ata_host *host, const char *scc_s) diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 0506c49da..9f8b07029 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -69,6 +69,7 @@ #include #include #include +#include #include #include @@ -1238,7 +1239,7 @@ static int ata_read_native_max_address(struct ata_device *dev, u64 *max_sectors) } else tf.command = ATA_CMD_READ_NATIVE_MAX; - tf.protocol |= ATA_PROT_NODATA; + tf.protocol = ATA_PROT_NODATA; tf.device |= ATA_LBA; err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0); @@ -1297,7 +1298,7 @@ static int ata_set_max_sectors(struct ata_device *dev, u64 new_sectors) tf.device |= (new_sectors >> 24) & 0xf; } - tf.protocol |= ATA_PROT_NODATA; + tf.protocol = ATA_PROT_NODATA; tf.device |= ATA_LBA; tf.lbal = (new_sectors >> 0) & 0xff; @@ -4852,7 +4853,7 @@ int ata_std_qc_defer(struct ata_queued_cmd *qc) { struct ata_link *link = qc->dev->link; - if (qc->tf.protocol == ATA_PROT_NCQ) { + if (ata_is_ncq(qc->tf.protocol)) { if (!ata_tag_valid(link->active_tag)) return 0; } else { @@ -5017,7 +5018,7 @@ void __ata_qc_complete(struct ata_queued_cmd *qc) ata_sg_clean(qc); /* command should be marked inactive atomically with qc completion */ - if (qc->tf.protocol == ATA_PROT_NCQ) { + if (ata_is_ncq(qc->tf.protocol)) { link->sactive &= ~(1 << qc->tag); if (!link->sactive) ap->nr_active_links--; @@ -5054,7 +5055,7 @@ static void ata_verify_xfer(struct ata_queued_cmd *qc) { struct ata_device *dev = qc->dev; - if (ata_is_nodata(qc->tf.protocol)) + if (!ata_is_data(qc->tf.protocol)) return; if ((dev->mwdma_mask || dev->udma_mask) && ata_is_pio(qc->tf.protocol)) @@ -5082,6 +5083,9 @@ void ata_qc_complete(struct ata_queued_cmd *qc) { struct ata_port *ap = qc->ap; + /* Trigger the LED (if available) */ + ledtrig_disk_activity(); + /* XXX: New EH and old EH use different mechanisms to * synchronize EH with regular execution path. * @@ -5137,7 +5141,9 @@ void ata_qc_complete(struct ata_queued_cmd *qc) switch (qc->tf.command) { case ATA_CMD_SET_FEATURES: if (qc->tf.feature != SETFEATURES_WC_ON && - qc->tf.feature != SETFEATURES_WC_OFF) + qc->tf.feature != SETFEATURES_WC_OFF && + qc->tf.feature != SETFEATURES_RA_ON && + qc->tf.feature != SETFEATURES_RA_OFF) break; /* fall through */ case ATA_CMD_INIT_DEV_PARAMS: /* CHS translation changed */ diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c index c6f017458..0e1ec3707 100644 --- a/drivers/ata/libata-eh.c +++ b/drivers/ata/libata-eh.c @@ -2607,9 +2607,13 @@ static void ata_eh_link_report(struct ata_link *link) [DMA_FROM_DEVICE] = "in", }; static const char *prot_str[] = { + [ATA_PROT_UNKNOWN] = "unknown", + [ATA_PROT_NODATA] = "nodata", [ATA_PROT_PIO] = "pio", [ATA_PROT_DMA] = "dma", - [ATA_PROT_NCQ] = "ncq", + [ATA_PROT_NCQ] = "ncq dma", + [ATA_PROT_NCQ_NODATA] = "ncq nodata", + [ATAPI_PROT_NODATA] = "nodata", [ATAPI_PROT_PIO] = "pio", [ATAPI_PROT_DMA] = "dma", }; @@ -3177,7 +3181,7 @@ static void ata_eh_park_issue_cmd(struct ata_device *dev, int park) } tf.flags |= ATA_TFLAG_DEVICE | ATA_TFLAG_ISADDR; - tf.protocol |= ATA_PROT_NODATA; + tf.protocol = ATA_PROT_NODATA; err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0); if (park && (err_mask || tf.lbal != 0xc4)) { ata_dev_err(dev, "head unload failed!\n"); diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index bfec66fb2..e207b33e4 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -304,7 +304,7 @@ static void ata_scsi_set_invalid_field(struct ata_device *dev, struct scsi_cmnd *cmd, u16 field, u8 bit) { ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST, 0x24, 0x0); - /* "Invalid field in cbd" */ + /* "Invalid field in CDB" */ scsi_set_sense_field_pointer(cmd->sense_buffer, SCSI_SENSE_BUFFERSIZE, field, bit, 1); } @@ -1190,7 +1190,7 @@ static int atapi_drain_needed(struct request *rq) if (likely(rq->cmd_type != REQ_TYPE_BLOCK_PC)) return 0; - if (!blk_rq_bytes(rq) || (rq->cmd_flags & REQ_WRITE)) + if (!blk_rq_bytes(rq) || op_is_write(req_op(rq))) return 0; return atapi_cmd_type(rq->cmd[0]) == ATAPI_MISC; @@ -2075,8 +2075,8 @@ static unsigned int ata_scsiop_inq_std(struct ata_scsi_args *args, u8 *rbuf) 0x03, 0x20, /* SBC-2 (no version claimed) */ - 0x02, - 0x60 /* SPC-3 (no version claimed) */ + 0x03, + 0x00 /* SPC-3 (no version claimed) */ }; const u8 versions_zbc[] = { 0x00, @@ -2097,7 +2097,10 @@ static unsigned int ata_scsiop_inq_std(struct ata_scsi_args *args, u8 *rbuf) 0, 0x5, /* claim SPC-3 version compatibility */ 2, - 95 - 4 + 95 - 4, + 0, + 0, + 2 }; VPRINTK("ENTER\n"); @@ -2109,8 +2112,10 @@ static unsigned int ata_scsiop_inq_std(struct ata_scsi_args *args, u8 *rbuf) (args->dev->link->ap->pflags & ATA_PFLAG_EXTERNAL)) hdr[1] |= (1 << 7); - if (args->dev->class == ATA_DEV_ZAC) + if (args->dev->class == ATA_DEV_ZAC) { hdr[0] = TYPE_ZBC; + hdr[2] = 0x7; /* claim SPC-5 version compatibility */ + } memcpy(rbuf, hdr, sizeof(hdr)); memcpy(&rbuf[8], "ATA ", 8); @@ -2314,7 +2319,7 @@ static unsigned int ata_scsiop_inq_b0(struct ata_scsi_args *args, u8 *rbuf) * with the unmap bit set. */ if (ata_id_has_trim(args->id)) { - put_unaligned_be64(65535 * 512 / 8, &rbuf[36]); + put_unaligned_be64(65535 * ATA_MAX_TRIM_RNUM, &rbuf[36]); put_unaligned_be32(1, &rbuf[28]); } @@ -2424,15 +2429,17 @@ static void modecpy(u8 *dest, const u8 *src, int n, bool changeable) static unsigned int ata_msense_caching(u16 *id, u8 *buf, bool changeable) { modecpy(buf, def_cache_mpage, sizeof(def_cache_mpage), changeable); - if (changeable || ata_id_wcache_enabled(id)) - buf[2] |= (1 << 2); /* write cache enable */ - if (!changeable && !ata_id_rahead_enabled(id)) - buf[12] |= (1 << 5); /* disable read ahead */ + if (changeable) { + buf[2] |= (1 << 2); /* ata_mselect_caching() */ + } else { + buf[2] |= (ata_id_wcache_enabled(id) << 2); /* write cache enable */ + buf[12] |= (!ata_id_rahead_enabled(id) << 5); /* disable read ahead */ + } return sizeof(def_cache_mpage); } /** - * ata_msense_ctl_mode - Simulate MODE SENSE control mode page + * ata_msense_control - Simulate MODE SENSE control mode page * @dev: ATA device of interest * @buf: output buffer * @changeable: whether changeable parameters are requested @@ -2442,12 +2449,17 @@ static unsigned int ata_msense_caching(u16 *id, u8 *buf, bool changeable) * LOCKING: * None. */ -static unsigned int ata_msense_ctl_mode(struct ata_device *dev, u8 *buf, +static unsigned int ata_msense_control(struct ata_device *dev, u8 *buf, bool changeable) { modecpy(buf, def_control_mpage, sizeof(def_control_mpage), changeable); - if (changeable && (dev->flags & ATA_DFLAG_D_SENSE)) - buf[2] |= (1 << 2); /* Descriptor sense requested */ + if (changeable) { + buf[2] |= (1 << 2); /* ata_mselect_control() */ + } else { + bool d_sense = (dev->flags & ATA_DFLAG_D_SENSE); + + buf[2] |= (d_sense << 2); /* descriptor format sense data */ + } return sizeof(def_control_mpage); } @@ -2566,13 +2578,13 @@ static unsigned int ata_scsiop_mode_sense(struct ata_scsi_args *args, u8 *rbuf) break; case CONTROL_MPAGE: - p += ata_msense_ctl_mode(args->dev, p, page_control == 1); + p += ata_msense_control(args->dev, p, page_control == 1); break; case ALL_MPAGES: p += ata_msense_rw_recovery(p, page_control == 1); p += ata_msense_caching(args->id, p, page_control == 1); - p += ata_msense_ctl_mode(args->dev, p, page_control == 1); + p += ata_msense_control(args->dev, p, page_control == 1); break; default: /* invalid page code */ @@ -3077,6 +3089,9 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc) goto invalid_fld; } + if (ata_is_ncq(tf->protocol) && (cdb[2] & 0x3) == 0) + tf->protocol = ATA_PROT_NCQ_NODATA; + /* enable LBA */ tf->flags |= ATA_TFLAG_LBA; @@ -3125,8 +3140,8 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc) tf->command = cdb[9]; } - /* For NCQ commands with FPDMA protocol, copy the tag value */ - if (tf->protocol == ATA_PROT_NCQ) + /* For NCQ commands copy the tag value */ + if (ata_is_ncq(tf->protocol)) tf->nsect = qc->tag << 3; /* enforce correct master/slave bit */ @@ -3305,7 +3320,13 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc) goto invalid_param_len; buf = page_address(sg_page(scsi_sglist(scmd))); - size = ata_set_lba_range_entries(buf, 512, block, n_block); + + if (n_block <= 65535 * ATA_MAX_TRIM_RNUM) { + size = ata_set_lba_range_entries(buf, ATA_MAX_TRIM_RNUM, block, n_block); + } else { + fp = 2; + goto invalid_fld; + } if (ata_ncq_enabled(dev) && ata_fpdma_dsm_supported(dev)) { /* Newer devices support queued TRIM commands */ @@ -3454,7 +3475,7 @@ static unsigned int ata_scsi_zbc_in_xlat(struct ata_queued_cmd *qc) goto invalid_param_len; } sect = n_block / 512; - options = cdb[14]; + options = cdb[14] & 0xbf; if (ata_ncq_enabled(qc->dev) && ata_fpdma_zac_mgmt_in_supported(qc->dev)) { @@ -3464,7 +3485,7 @@ static unsigned int ata_scsi_zbc_in_xlat(struct ata_queued_cmd *qc) tf->nsect = qc->tag << 3; tf->feature = sect & 0xff; tf->hob_feature = (sect >> 8) & 0xff; - tf->auxiliary = ATA_SUBCMD_ZAC_MGMT_IN_REPORT_ZONES; + tf->auxiliary = ATA_SUBCMD_ZAC_MGMT_IN_REPORT_ZONES | (options << 8); } else { tf->command = ATA_CMD_ZAC_MGMT_IN; tf->feature = ATA_SUBCMD_ZAC_MGMT_IN_REPORT_ZONES; @@ -3506,7 +3527,7 @@ static unsigned int ata_scsi_zbc_out_xlat(struct ata_queued_cmd *qc) struct scsi_cmnd *scmd = qc->scsicmd; struct ata_device *dev = qc->dev; const u8 *cdb = scmd->cmnd; - u8 reset_all, sa; + u8 all, sa; u64 block; u32 n_block; u16 fp = (u16)-1; @@ -3533,20 +3554,20 @@ static unsigned int ata_scsi_zbc_out_xlat(struct ata_queued_cmd *qc) if (block > dev->n_sectors) goto out_of_range; - reset_all = cdb[14] & 0x1; + all = cdb[14] & 0x1; if (ata_ncq_enabled(qc->dev) && ata_fpdma_zac_mgmt_out_supported(qc->dev)) { - tf->protocol = ATA_PROT_NCQ; + tf->protocol = ATA_PROT_NCQ_NODATA; tf->command = ATA_CMD_NCQ_NON_DATA; - tf->hob_nsect = ATA_SUBCMD_NCQ_NON_DATA_ZAC_MGMT_OUT; + tf->feature = ATA_SUBCMD_NCQ_NON_DATA_ZAC_MGMT_OUT; tf->nsect = qc->tag << 3; - tf->auxiliary = sa | (reset_all & 0x1) << 8; + tf->auxiliary = sa | ((u16)all << 8); } else { tf->protocol = ATA_PROT_NODATA; tf->command = ATA_CMD_ZAC_MGMT_OUT; tf->feature = sa; - tf->hob_feature = reset_all & 0x1; + tf->hob_feature = all; } tf->lbah = (block >> 16) & 0xff; tf->lbam = (block >> 8) & 0xff; @@ -3667,7 +3688,7 @@ static int ata_mselect_control(struct ata_queued_cmd *qc, /* * Check that read-only bits are not modified. */ - ata_msense_ctl_mode(dev, mpage, false); + ata_msense_control(dev, mpage, false); for (i = 0; i < CONTROL_MPAGE_LEN - 2; i++) { if (i == 0) continue; @@ -4039,11 +4060,6 @@ void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd) args.done = cmd->scsi_done; switch(scsicmd[0]) { - /* TODO: worth improving? */ - case FORMAT_UNIT: - ata_scsi_invalid_field(dev, cmd, 0); - break; - case INQUIRY: if (scsicmd[1] & 2) /* is CmdDt set? */ ata_scsi_invalid_field(dev, cmd, 1); diff --git a/drivers/ata/libata-transport.c b/drivers/ata/libata-transport.c index e2d949729..7ef16c085 100644 --- a/drivers/ata/libata-transport.c +++ b/drivers/ata/libata-transport.c @@ -495,12 +495,13 @@ struct ata_show_ering_arg { static int ata_show_ering(struct ata_ering_entry *ent, void *void_arg) { struct ata_show_ering_arg* arg = void_arg; - struct timespec time; + u64 seconds; + u32 rem; - jiffies_to_timespec(ent->timestamp,&time); + seconds = div_u64_rem(ent->timestamp, HZ, &rem); arg->written += sprintf(arg->buf + arg->written, - "[%5lu.%06lu]", - time.tv_sec, time.tv_nsec); + "[%5llu.%09lu]", seconds, + rem * NSEC_PER_SEC / HZ); arg->written += get_ata_err_names(ent->err_mask, arg->buf + arg->written); return 0; diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c index 80fe0f6fe..b4d54771c 100644 --- a/drivers/ata/pata_arasan_cf.c +++ b/drivers/ata/pata_arasan_cf.c @@ -565,7 +565,7 @@ chan_request_fail: qc->ap->hsm_task_state = HSM_ST_ERR; cf_ctrl_reset(acdev); - spin_unlock_irqrestore(qc->ap->lock, flags); + spin_unlock_irqrestore(&acdev->host->lock, flags); sff_intr: dma_complete(acdev); } diff --git a/drivers/ata/pata_atiixp.c b/drivers/ata/pata_atiixp.c index 970f7767e..49d705c9f 100644 --- a/drivers/ata/pata_atiixp.c +++ b/drivers/ata/pata_atiixp.c @@ -183,8 +183,8 @@ static void atiixp_set_dmamode(struct ata_port *ap, struct ata_device *adev) * We must now look at the PIO mode situation. We may need to * adjust the PIO mode to keep the timings acceptable */ - if (adev->dma_mode >= XFER_MW_DMA_2) - wanted_pio = 4; + if (adev->dma_mode >= XFER_MW_DMA_2) + wanted_pio = 4; else if (adev->dma_mode == XFER_MW_DMA_1) wanted_pio = 3; else if (adev->dma_mode == XFER_MW_DMA_0) diff --git a/drivers/ata/pata_hpt366.c b/drivers/ata/pata_hpt366.c index e5fb7525a..a219a503c 100644 --- a/drivers/ata/pata_hpt366.c +++ b/drivers/ata/pata_hpt366.c @@ -368,7 +368,7 @@ static int hpt36x_init_one(struct pci_dev *dev, const struct pci_device_id *id) /* PCI clocking determines the ATA timing values to use */ /* info_hpt366 is safe against re-entry so we can scribble on it */ - switch ((reg1 & 0x700) >> 8) { + switch ((reg1 & 0xf00) >> 8) { case 9: hpriv = &hpt366_40; break; diff --git a/drivers/ata/pata_marvell.c b/drivers/ata/pata_marvell.c index ae9feb1ba..ff468a6fd 100644 --- a/drivers/ata/pata_marvell.c +++ b/drivers/ata/pata_marvell.c @@ -146,7 +146,7 @@ static int marvell_init_one (struct pci_dev *pdev, const struct pci_device_id *i if (pdev->device == 0x6101) ppi[1] = &ata_dummy_port_info; -#if defined(CONFIG_SATA_AHCI) || defined(CONFIG_SATA_AHCI_MODULE) +#if IS_ENABLED(CONFIG_SATA_AHCI) if (!marvell_pata_active(pdev)) { printk(KERN_INFO DRV_NAME ": PATA port not active, deferring to AHCI driver.\n"); return -ENODEV; diff --git a/drivers/ata/pata_ninja32.c b/drivers/ata/pata_ninja32.c index 633aa2934..44f97ad3c 100644 --- a/drivers/ata/pata_ninja32.c +++ b/drivers/ata/pata_ninja32.c @@ -144,7 +144,7 @@ static int ninja32_init_one(struct pci_dev *dev, const struct pci_device_id *id) ap->ioaddr.altstatus_addr = base + 0x1E; ap->ioaddr.bmdma_addr = base; ata_sff_std_ports(&ap->ioaddr); - ap->pflags = ATA_PFLAG_PIO32 | ATA_PFLAG_PIO32CHANGE; + ap->pflags |= ATA_PFLAG_PIO32 | ATA_PFLAG_PIO32CHANGE; ninja32_program(base); /* FIXME: Should we disable them at remove ? */ diff --git a/drivers/ata/sata_dwc_460ex.c b/drivers/ata/sata_dwc_460ex.c index 00c2af1d2..e0939bd5e 100644 --- a/drivers/ata/sata_dwc_460ex.c +++ b/drivers/ata/sata_dwc_460ex.c @@ -259,11 +259,8 @@ static int sata_dwc_dma_init_old(struct platform_device *pdev, /* Get physical SATA DMA register base address */ res = platform_get_resource(pdev, IORESOURCE_MEM, 1); hsdev->dma->regs = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(hsdev->dma->regs)) { - dev_err(&pdev->dev, - "ioremap failed for AHBDMA register address\n"); + if (IS_ERR(hsdev->dma->regs)) return PTR_ERR(hsdev->dma->regs); - } /* Initialize AHB DMAC */ return dw_dma_probe(hsdev->dma); @@ -281,7 +278,7 @@ static void sata_dwc_dma_exit_old(struct sata_dwc_device *hsdev) static const char *get_prot_descript(u8 protocol) { - switch ((enum ata_tf_protocols)protocol) { + switch (protocol) { case ATA_PROT_NODATA: return "ATA no data"; case ATA_PROT_PIO: @@ -290,6 +287,8 @@ static const char *get_prot_descript(u8 protocol) return "ATA DMA"; case ATA_PROT_NCQ: return "ATA NCQ"; + case ATA_PROT_NCQ_NODATA: + return "ATA NCQ no data"; case ATAPI_PROT_NODATA: return "ATAPI no data"; case ATAPI_PROT_PIO: @@ -1225,11 +1224,8 @@ static int sata_dwc_probe(struct platform_device *ofdev) /* Ioremap SATA registers */ res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); base = devm_ioremap_resource(&ofdev->dev, res); - if (IS_ERR(base)) { - dev_err(&ofdev->dev, - "ioremap failed for SATA register address\n"); + if (IS_ERR(base)) return PTR_ERR(base); - } dev_dbg(&ofdev->dev, "ioremap done for SATA register address\n"); /* Synopsys DWC SATA specific Registers */ diff --git a/drivers/atm/horizon.c b/drivers/atm/horizon.c index 527bbd595..5fc81e240 100644 --- a/drivers/atm/horizon.c +++ b/drivers/atm/horizon.c @@ -2795,9 +2795,7 @@ static int hrz_probe(struct pci_dev *pci_dev, dev->atm_dev->ci_range.vpi_bits = vpi_bits; dev->atm_dev->ci_range.vci_bits = 10-vpi_bits; - init_timer(&dev->housekeeping); - dev->housekeeping.function = do_housekeeping; - dev->housekeeping.data = (unsigned long) dev; + setup_timer(&dev->housekeeping, do_housekeeping, (unsigned long) dev); mod_timer(&dev->housekeeping, jiffies); out: diff --git a/drivers/atm/nicstar.c b/drivers/atm/nicstar.c index ddc4ceb85..700ed15c2 100644 --- a/drivers/atm/nicstar.c +++ b/drivers/atm/nicstar.c @@ -874,7 +874,8 @@ static scq_info *get_scq(ns_dev *card, int size, u32 scd) scq->skb = kmalloc(sizeof(struct sk_buff *) * (size / NS_SCQE_SIZE), GFP_KERNEL); if (!scq->skb) { - kfree(scq->org); + dma_free_coherent(&card->pcidev->dev, + 2 * size, scq->org, scq->dma); kfree(scq); return NULL; } diff --git a/drivers/base/firmware_class.c b/drivers/base/firmware_class.c index 11f768c70..acf228e9a 100644 --- a/drivers/base/firmware_class.c +++ b/drivers/base/firmware_class.c @@ -46,7 +46,8 @@ MODULE_LICENSE("GPL"); extern struct builtin_fw __start_builtin_fw[]; extern struct builtin_fw __end_builtin_fw[]; -static bool fw_get_builtin_firmware(struct firmware *fw, const char *name) +static bool fw_get_builtin_firmware(struct firmware *fw, const char *name, + void *buf, size_t size) { struct builtin_fw *b_fw; @@ -54,6 +55,9 @@ static bool fw_get_builtin_firmware(struct firmware *fw, const char *name) if (strcmp(name, b_fw->name) == 0) { fw->size = b_fw->size; fw->data = b_fw->data; + + if (buf && fw->size <= size) + memcpy(buf, fw->data, fw->size); return true; } } @@ -74,7 +78,9 @@ static bool fw_is_builtin_firmware(const struct firmware *fw) #else /* Module case - no builtin firmware support */ -static inline bool fw_get_builtin_firmware(struct firmware *fw, const char *name) +static inline bool fw_get_builtin_firmware(struct firmware *fw, + const char *name, void *buf, + size_t size) { return false; } @@ -112,6 +118,7 @@ static inline long firmware_loading_timeout(void) #define FW_OPT_FALLBACK 0 #endif #define FW_OPT_NO_WARN (1U << 3) +#define FW_OPT_NOCACHE (1U << 4) struct firmware_cache { /* firmware_buf instance will be added into the below list */ @@ -143,6 +150,7 @@ struct firmware_buf { unsigned long status; void *data; size_t size; + size_t allocated_size; #ifdef CONFIG_FW_LOADER_USER_HELPER bool is_paged_buf; bool need_uevent; @@ -178,7 +186,8 @@ static DEFINE_MUTEX(fw_lock); static struct firmware_cache fw_cache; static struct firmware_buf *__allocate_fw_buf(const char *fw_name, - struct firmware_cache *fwc) + struct firmware_cache *fwc, + void *dbuf, size_t size) { struct firmware_buf *buf; @@ -194,6 +203,8 @@ static struct firmware_buf *__allocate_fw_buf(const char *fw_name, kref_init(&buf->ref); buf->fwc = fwc; + buf->data = dbuf; + buf->allocated_size = size; init_completion(&buf->completion); #ifdef CONFIG_FW_LOADER_USER_HELPER INIT_LIST_HEAD(&buf->pending_list); @@ -217,7 +228,8 @@ static struct firmware_buf *__fw_lookup_buf(const char *fw_name) static int fw_lookup_and_allocate_buf(const char *fw_name, struct firmware_cache *fwc, - struct firmware_buf **buf) + struct firmware_buf **buf, void *dbuf, + size_t size) { struct firmware_buf *tmp; @@ -229,7 +241,7 @@ static int fw_lookup_and_allocate_buf(const char *fw_name, *buf = tmp; return 1; } - tmp = __allocate_fw_buf(fw_name, fwc); + tmp = __allocate_fw_buf(fw_name, fwc, dbuf, size); if (tmp) list_add(&tmp->list, &fwc->head); spin_unlock(&fwc->lock); @@ -261,6 +273,7 @@ static void __fw_free_buf(struct kref *ref) vfree(buf->pages); } else #endif + if (!buf->allocated_size) vfree(buf->data); kfree_const(buf->fw_id); kfree(buf); @@ -301,13 +314,21 @@ static void fw_finish_direct_load(struct device *device, mutex_unlock(&fw_lock); } -static int fw_get_filesystem_firmware(struct device *device, - struct firmware_buf *buf) +static int +fw_get_filesystem_firmware(struct device *device, struct firmware_buf *buf) { loff_t size; int i, len; int rc = -ENOENT; char *path; + enum kernel_read_file_id id = READING_FIRMWARE; + size_t msize = INT_MAX; + + /* Already populated data member means we're loading into a buffer */ + if (buf->data) { + id = READING_FIRMWARE_PREALLOC_BUFFER; + msize = buf->allocated_size; + } path = __getname(); if (!path) @@ -326,8 +347,8 @@ static int fw_get_filesystem_firmware(struct device *device, } buf->size = 0; - rc = kernel_read_file_from_path(path, &buf->data, &size, - INT_MAX, READING_FIRMWARE); + rc = kernel_read_file_from_path(path, &buf->data, &size, msize, + id); if (rc) { if (rc == -ENOENT) dev_dbg(device, "loading %s failed with error %d\n", @@ -691,6 +712,38 @@ out: static DEVICE_ATTR(loading, 0644, firmware_loading_show, firmware_loading_store); +static void firmware_rw_buf(struct firmware_buf *buf, char *buffer, + loff_t offset, size_t count, bool read) +{ + if (read) + memcpy(buffer, buf->data + offset, count); + else + memcpy(buf->data + offset, buffer, count); +} + +static void firmware_rw(struct firmware_buf *buf, char *buffer, + loff_t offset, size_t count, bool read) +{ + while (count) { + void *page_data; + int page_nr = offset >> PAGE_SHIFT; + int page_ofs = offset & (PAGE_SIZE-1); + int page_cnt = min_t(size_t, PAGE_SIZE - page_ofs, count); + + page_data = kmap(buf->pages[page_nr]); + + if (read) + memcpy(buffer, page_data + page_ofs, page_cnt); + else + memcpy(page_data + page_ofs, buffer, page_cnt); + + kunmap(buf->pages[page_nr]); + buffer += page_cnt; + offset += page_cnt; + count -= page_cnt; + } +} + static ssize_t firmware_data_read(struct file *filp, struct kobject *kobj, struct bin_attribute *bin_attr, char *buffer, loff_t offset, size_t count) @@ -715,21 +768,11 @@ static ssize_t firmware_data_read(struct file *filp, struct kobject *kobj, ret_count = count; - while (count) { - void *page_data; - int page_nr = offset >> PAGE_SHIFT; - int page_ofs = offset & (PAGE_SIZE-1); - int page_cnt = min_t(size_t, PAGE_SIZE - page_ofs, count); - - page_data = kmap(buf->pages[page_nr]); - - memcpy(buffer, page_data + page_ofs, page_cnt); + if (buf->data) + firmware_rw_buf(buf, buffer, offset, count, true); + else + firmware_rw(buf, buffer, offset, count, true); - kunmap(buf->pages[page_nr]); - buffer += page_cnt; - offset += page_cnt; - count -= page_cnt; - } out: mutex_unlock(&fw_lock); return ret_count; @@ -804,29 +847,23 @@ static ssize_t firmware_data_write(struct file *filp, struct kobject *kobj, goto out; } - retval = fw_realloc_buffer(fw_priv, offset + count); - if (retval) - goto out; - - retval = count; - - while (count) { - void *page_data; - int page_nr = offset >> PAGE_SHIFT; - int page_ofs = offset & (PAGE_SIZE - 1); - int page_cnt = min_t(size_t, PAGE_SIZE - page_ofs, count); - - page_data = kmap(buf->pages[page_nr]); - - memcpy(page_data + page_ofs, buffer, page_cnt); + if (buf->data) { + if (offset + count > buf->allocated_size) { + retval = -ENOMEM; + goto out; + } + firmware_rw_buf(buf, buffer, offset, count, false); + retval = count; + } else { + retval = fw_realloc_buffer(fw_priv, offset + count); + if (retval) + goto out; - kunmap(buf->pages[page_nr]); - buffer += page_cnt; - offset += page_cnt; - count -= page_cnt; + retval = count; + firmware_rw(buf, buffer, offset, count, false); } - buf->size = max_t(size_t, offset, buf->size); + buf->size = max_t(size_t, offset + count, buf->size); out: mutex_unlock(&fw_lock); return retval; @@ -894,7 +931,8 @@ static int _request_firmware_load(struct firmware_priv *fw_priv, struct firmware_buf *buf = fw_priv->buf; /* fall back on userspace loading */ - buf->is_paged_buf = true; + if (!buf->data) + buf->is_paged_buf = true; dev_set_uevent_suppress(f_dev, true); @@ -929,7 +967,7 @@ static int _request_firmware_load(struct firmware_priv *fw_priv, if (is_fw_load_aborted(buf)) retval = -EAGAIN; - else if (!buf->data) + else if (buf->is_paged_buf && !buf->data) retval = -ENOMEM; device_del(f_dev); @@ -1012,7 +1050,7 @@ static int sync_cached_firmware_buf(struct firmware_buf *buf) */ static int _request_firmware_prepare(struct firmware **firmware_p, const char *name, - struct device *device) + struct device *device, void *dbuf, size_t size) { struct firmware *firmware; struct firmware_buf *buf; @@ -1025,12 +1063,12 @@ _request_firmware_prepare(struct firmware **firmware_p, const char *name, return -ENOMEM; } - if (fw_get_builtin_firmware(firmware, name)) { + if (fw_get_builtin_firmware(firmware, name, dbuf, size)) { dev_dbg(device, "using built-in %s\n", name); return 0; /* assigned */ } - ret = fw_lookup_and_allocate_buf(name, &fw_cache, &buf); + ret = fw_lookup_and_allocate_buf(name, &fw_cache, &buf, dbuf, size); /* * bind with 'buf' now to avoid warning in failure path @@ -1070,14 +1108,16 @@ static int assign_firmware_buf(struct firmware *fw, struct device *device, * should be fixed in devres or driver core. */ /* don't cache firmware handled without uevent */ - if (device && (opt_flags & FW_OPT_UEVENT)) + if (device && (opt_flags & FW_OPT_UEVENT) && + !(opt_flags & FW_OPT_NOCACHE)) fw_add_devm_name(device, buf->fw_id); /* * After caching firmware image is started, let it piggyback * on request firmware. */ - if (buf->fwc->state == FW_LOADER_START_CACHE) { + if (!(opt_flags & FW_OPT_NOCACHE) && + buf->fwc->state == FW_LOADER_START_CACHE) { if (fw_cache_piggyback_on_request(buf->fw_id)) kref_get(&buf->ref); } @@ -1091,7 +1131,8 @@ static int assign_firmware_buf(struct firmware *fw, struct device *device, /* called from request_firmware() and request_firmware_work_func() */ static int _request_firmware(const struct firmware **firmware_p, const char *name, - struct device *device, unsigned int opt_flags) + struct device *device, void *buf, size_t size, + unsigned int opt_flags) { struct firmware *fw = NULL; long timeout; @@ -1105,7 +1146,7 @@ _request_firmware(const struct firmware **firmware_p, const char *name, goto out; } - ret = _request_firmware_prepare(&fw, name, device); + ret = _request_firmware_prepare(&fw, name, device, buf, size); if (ret <= 0) /* error or already assigned */ goto out; @@ -1184,7 +1225,7 @@ request_firmware(const struct firmware **firmware_p, const char *name, /* Need to pin this module until return */ __module_get(THIS_MODULE); - ret = _request_firmware(firmware_p, name, device, + ret = _request_firmware(firmware_p, name, device, NULL, 0, FW_OPT_UEVENT | FW_OPT_FALLBACK); module_put(THIS_MODULE); return ret; @@ -1208,13 +1249,43 @@ int request_firmware_direct(const struct firmware **firmware_p, int ret; __module_get(THIS_MODULE); - ret = _request_firmware(firmware_p, name, device, + ret = _request_firmware(firmware_p, name, device, NULL, 0, FW_OPT_UEVENT | FW_OPT_NO_WARN); module_put(THIS_MODULE); return ret; } EXPORT_SYMBOL_GPL(request_firmware_direct); +/** + * request_firmware_into_buf - load firmware into a previously allocated buffer + * @firmware_p: pointer to firmware image + * @name: name of firmware file + * @device: device for which firmware is being loaded and DMA region allocated + * @buf: address of buffer to load firmware into + * @size: size of buffer + * + * This function works pretty much like request_firmware(), but it doesn't + * allocate a buffer to hold the firmware data. Instead, the firmware + * is loaded directly into the buffer pointed to by @buf and the @firmware_p + * data member is pointed at @buf. + * + * This function doesn't cache firmware either. + */ +int +request_firmware_into_buf(const struct firmware **firmware_p, const char *name, + struct device *device, void *buf, size_t size) +{ + int ret; + + __module_get(THIS_MODULE); + ret = _request_firmware(firmware_p, name, device, buf, size, + FW_OPT_UEVENT | FW_OPT_FALLBACK | + FW_OPT_NOCACHE); + module_put(THIS_MODULE); + return ret; +} +EXPORT_SYMBOL(request_firmware_into_buf); + /** * release_firmware: - release the resource associated with a firmware image * @fw: firmware resource to release @@ -1247,7 +1318,7 @@ static void request_firmware_work_func(struct work_struct *work) fw_work = container_of(work, struct firmware_work, work); - _request_firmware(&fw, fw_work->name, fw_work->device, + _request_firmware(&fw, fw_work->name, fw_work->device, NULL, 0, fw_work->opt_flags); fw_work->cont(fw, fw_work->context); put_device(fw_work->device); /* taken in request_firmware_nowait() */ @@ -1380,7 +1451,7 @@ static int uncache_firmware(const char *fw_name) pr_debug("%s: %s\n", __func__, fw_name); - if (fw_get_builtin_firmware(&fw, fw_name)) + if (fw_get_builtin_firmware(&fw, fw_name, NULL, 0)) return 0; buf = fw_lookup_buf(fw_name); diff --git a/drivers/base/memory.c b/drivers/base/memory.c index f46dba8b7..dc75de905 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -391,6 +391,7 @@ static ssize_t show_valid_zones(struct device *dev, unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; struct page *first_page; struct zone *zone; + int zone_shift = 0; start_pfn = section_nr_to_pfn(mem->start_section_nr); end_pfn = start_pfn + nr_pages; @@ -402,21 +403,26 @@ static ssize_t show_valid_zones(struct device *dev, zone = page_zone(first_page); - if (zone_idx(zone) == ZONE_MOVABLE - 1) { - /*The mem block is the last memoryblock of this zone.*/ - if (end_pfn == zone_end_pfn(zone)) - return sprintf(buf, "%s %s\n", - zone->name, (zone + 1)->name); + /* MMOP_ONLINE_KEEP */ + sprintf(buf, "%s", zone->name); + + /* MMOP_ONLINE_KERNEL */ + zone_shift = zone_can_shift(start_pfn, nr_pages, ZONE_NORMAL); + if (zone_shift) { + strcat(buf, " "); + strcat(buf, (zone + zone_shift)->name); } - if (zone_idx(zone) == ZONE_MOVABLE) { - /*The mem block is the first memoryblock of ZONE_MOVABLE.*/ - if (start_pfn == zone->zone_start_pfn) - return sprintf(buf, "%s %s\n", - zone->name, (zone - 1)->name); + /* MMOP_ONLINE_MOVABLE */ + zone_shift = zone_can_shift(start_pfn, nr_pages, ZONE_MOVABLE); + if (zone_shift) { + strcat(buf, " "); + strcat(buf, (zone + zone_shift)->name); } - return sprintf(buf, "%s\n", zone->name); + strcat(buf, "\n"); + + return strlen(buf); } static DEVICE_ATTR(valid_zones, 0444, show_valid_zones, NULL); #endif diff --git a/drivers/base/node.c b/drivers/base/node.c index 560751bad..5548f9686 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -56,6 +56,7 @@ static ssize_t node_read_meminfo(struct device *dev, { int n; int nid = dev->id; + struct pglist_data *pgdat = NODE_DATA(nid); struct sysinfo i; si_meminfo_node(&i, nid); @@ -74,16 +75,16 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(i.totalram), nid, K(i.freeram), nid, K(i.totalram - i.freeram), - nid, K(node_page_state(nid, NR_ACTIVE_ANON) + - node_page_state(nid, NR_ACTIVE_FILE)), - nid, K(node_page_state(nid, NR_INACTIVE_ANON) + - node_page_state(nid, NR_INACTIVE_FILE)), - nid, K(node_page_state(nid, NR_ACTIVE_ANON)), - nid, K(node_page_state(nid, NR_INACTIVE_ANON)), - nid, K(node_page_state(nid, NR_ACTIVE_FILE)), - nid, K(node_page_state(nid, NR_INACTIVE_FILE)), - nid, K(node_page_state(nid, NR_UNEVICTABLE)), - nid, K(node_page_state(nid, NR_MLOCK))); + nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) + + node_page_state(pgdat, NR_ACTIVE_FILE)), + nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) + + node_page_state(pgdat, NR_INACTIVE_FILE)), + nid, K(node_page_state(pgdat, NR_ACTIVE_ANON)), + nid, K(node_page_state(pgdat, NR_INACTIVE_ANON)), + nid, K(node_page_state(pgdat, NR_ACTIVE_FILE)), + nid, K(node_page_state(pgdat, NR_INACTIVE_FILE)), + nid, K(node_page_state(pgdat, NR_UNEVICTABLE)), + nid, K(sum_zone_node_page_state(nid, NR_MLOCK))); #ifdef CONFIG_HIGHMEM n += sprintf(buf + n, @@ -113,30 +114,34 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d SUnreclaim: %8lu kB\n" #ifdef CONFIG_TRANSPARENT_HUGEPAGE "Node %d AnonHugePages: %8lu kB\n" + "Node %d ShmemHugePages: %8lu kB\n" + "Node %d ShmemPmdMapped: %8lu kB\n" #endif , - nid, K(node_page_state(nid, NR_FILE_DIRTY)), - nid, K(node_page_state(nid, NR_WRITEBACK)), - nid, K(node_page_state(nid, NR_FILE_PAGES)), - nid, K(node_page_state(nid, NR_FILE_MAPPED)), - nid, K(node_page_state(nid, NR_ANON_PAGES)), + nid, K(node_page_state(pgdat, NR_FILE_DIRTY)), + nid, K(node_page_state(pgdat, NR_WRITEBACK)), + nid, K(node_page_state(pgdat, NR_FILE_PAGES)), + nid, K(node_page_state(pgdat, NR_FILE_MAPPED)), + nid, K(node_page_state(pgdat, NR_ANON_MAPPED)), nid, K(i.sharedram), - nid, node_page_state(nid, NR_KERNEL_STACK) * - THREAD_SIZE / 1024, - nid, K(node_page_state(nid, NR_PAGETABLE)), - nid, K(node_page_state(nid, NR_UNSTABLE_NFS)), - nid, K(node_page_state(nid, NR_BOUNCE)), - nid, K(node_page_state(nid, NR_WRITEBACK_TEMP)), - nid, K(node_page_state(nid, NR_SLAB_RECLAIMABLE) + - node_page_state(nid, NR_SLAB_UNRECLAIMABLE)), - nid, K(node_page_state(nid, NR_SLAB_RECLAIMABLE)), + nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB), + nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)), + nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)), + nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)), + nid, K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), + nid, K(sum_zone_node_page_state(nid, NR_SLAB_RECLAIMABLE) + + sum_zone_node_page_state(nid, NR_SLAB_UNRECLAIMABLE)), + nid, K(sum_zone_node_page_state(nid, NR_SLAB_RECLAIMABLE)), #ifdef CONFIG_TRANSPARENT_HUGEPAGE - nid, K(node_page_state(nid, NR_SLAB_UNRECLAIMABLE)) - , nid, - K(node_page_state(nid, NR_ANON_TRANSPARENT_HUGEPAGES) * - HPAGE_PMD_NR)); + nid, K(sum_zone_node_page_state(nid, NR_SLAB_UNRECLAIMABLE)), + nid, K(node_page_state(pgdat, NR_ANON_THPS) * + HPAGE_PMD_NR), + nid, K(node_page_state(pgdat, NR_SHMEM_THPS) * + HPAGE_PMD_NR), + nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * + HPAGE_PMD_NR)); #else - nid, K(node_page_state(nid, NR_SLAB_UNRECLAIMABLE))); + nid, K(sum_zone_node_page_state(nid, NR_SLAB_UNRECLAIMABLE))); #endif n += hugetlb_report_node_meminfo(nid, buf + n); return n; @@ -155,12 +160,12 @@ static ssize_t node_read_numastat(struct device *dev, "interleave_hit %lu\n" "local_node %lu\n" "other_node %lu\n", - node_page_state(dev->id, NUMA_HIT), - node_page_state(dev->id, NUMA_MISS), - node_page_state(dev->id, NUMA_FOREIGN), - node_page_state(dev->id, NUMA_INTERLEAVE_HIT), - node_page_state(dev->id, NUMA_LOCAL), - node_page_state(dev->id, NUMA_OTHER)); + sum_zone_node_page_state(dev->id, NUMA_HIT), + sum_zone_node_page_state(dev->id, NUMA_MISS), + sum_zone_node_page_state(dev->id, NUMA_FOREIGN), + sum_zone_node_page_state(dev->id, NUMA_INTERLEAVE_HIT), + sum_zone_node_page_state(dev->id, NUMA_LOCAL), + sum_zone_node_page_state(dev->id, NUMA_OTHER)); } static DEVICE_ATTR(numastat, S_IRUGO, node_read_numastat, NULL); @@ -168,12 +173,18 @@ static ssize_t node_read_vmstat(struct device *dev, struct device_attribute *attr, char *buf) { int nid = dev->id; + struct pglist_data *pgdat = NODE_DATA(nid); int i; int n = 0; for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) n += sprintf(buf+n, "%s %lu\n", vmstat_text[i], - node_page_state(nid, i)); + sum_zone_node_page_state(nid, i)); + + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) + n += sprintf(buf+n, "%s %lu\n", + vmstat_text[i + NR_VM_ZONE_STAT_ITEMS], + node_page_state(pgdat, i)); return n; } @@ -359,7 +370,7 @@ int unregister_cpu_under_node(unsigned int cpu, unsigned int nid) #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE #define page_initialized(page) (page->lru.next) -static int __init_refok get_nid_for_pfn(unsigned long pfn) +static int __ref get_nid_for_pfn(unsigned long pfn) { struct page *page; diff --git a/drivers/base/power/clock_ops.c b/drivers/base/power/clock_ops.c index 3657ac1cb..8e2e4757a 100644 --- a/drivers/base/power/clock_ops.c +++ b/drivers/base/power/clock_ops.c @@ -121,6 +121,7 @@ int pm_clk_add(struct device *dev, const char *con_id) { return __pm_clk_add(dev, con_id, NULL); } +EXPORT_SYMBOL_GPL(pm_clk_add); /** * pm_clk_add_clk - Start using a device clock for power management. @@ -136,8 +137,41 @@ int pm_clk_add_clk(struct device *dev, struct clk *clk) { return __pm_clk_add(dev, NULL, clk); } +EXPORT_SYMBOL_GPL(pm_clk_add_clk); +/** + * of_pm_clk_add_clk - Start using a device clock for power management. + * @dev: Device whose clock is going to be used for power management. + * @name: Name of clock that is going to be used for power management. + * + * Add the clock described in the 'clocks' device-tree node that matches + * with the 'name' provided, to the list of clocks used for the power + * management of @dev. On success, returns 0. Returns a negative error + * code if the clock is not found or cannot be added. + */ +int of_pm_clk_add_clk(struct device *dev, const char *name) +{ + struct clk *clk; + int ret; + + if (!dev || !dev->of_node || !name) + return -EINVAL; + + clk = of_clk_get_by_name(dev->of_node, name); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + ret = pm_clk_add_clk(dev, clk); + if (ret) { + clk_put(clk); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(of_pm_clk_add_clk); + /** * of_pm_clk_add_clks - Start using device clock(s) for power management. * @dev: Device whose clock(s) is going to be used for power management. @@ -192,6 +226,7 @@ error: return ret; } +EXPORT_SYMBOL_GPL(of_pm_clk_add_clks); /** * __pm_clk_remove - Destroy PM clock entry. @@ -252,6 +287,7 @@ void pm_clk_remove(struct device *dev, const char *con_id) __pm_clk_remove(ce); } +EXPORT_SYMBOL_GPL(pm_clk_remove); /** * pm_clk_remove_clk - Stop using a device clock for power management. @@ -285,6 +321,7 @@ void pm_clk_remove_clk(struct device *dev, struct clk *clk) __pm_clk_remove(ce); } +EXPORT_SYMBOL_GPL(pm_clk_remove_clk); /** * pm_clk_init - Initialize a device's list of power management clocks. @@ -299,6 +336,7 @@ void pm_clk_init(struct device *dev) if (psd) INIT_LIST_HEAD(&psd->clock_list); } +EXPORT_SYMBOL_GPL(pm_clk_init); /** * pm_clk_create - Create and initialize a device's list of PM clocks. @@ -311,6 +349,7 @@ int pm_clk_create(struct device *dev) { return dev_pm_get_subsys_data(dev); } +EXPORT_SYMBOL_GPL(pm_clk_create); /** * pm_clk_destroy - Destroy a device's list of power management clocks. @@ -345,6 +384,7 @@ void pm_clk_destroy(struct device *dev) __pm_clk_remove(ce); } } +EXPORT_SYMBOL_GPL(pm_clk_destroy); /** * pm_clk_suspend - Disable clocks in a device's PM clock list. @@ -375,6 +415,7 @@ int pm_clk_suspend(struct device *dev) return 0; } +EXPORT_SYMBOL_GPL(pm_clk_suspend); /** * pm_clk_resume - Enable clocks in a device's PM clock list. @@ -400,6 +441,7 @@ int pm_clk_resume(struct device *dev) return 0; } +EXPORT_SYMBOL_GPL(pm_clk_resume); /** * pm_clk_notify - Notify routine for device addition and removal. @@ -480,6 +522,7 @@ int pm_clk_runtime_suspend(struct device *dev) return 0; } +EXPORT_SYMBOL_GPL(pm_clk_runtime_suspend); int pm_clk_runtime_resume(struct device *dev) { @@ -495,6 +538,7 @@ int pm_clk_runtime_resume(struct device *dev) return pm_generic_runtime_resume(dev); } +EXPORT_SYMBOL_GPL(pm_clk_runtime_resume); #else /* !CONFIG_PM_CLK */ @@ -598,3 +642,4 @@ void pm_clk_add_notifier(struct bus_type *bus, clknb->nb.notifier_call = pm_clk_notify; bus_register_notifier(bus, &clknb->nb); } +EXPORT_SYMBOL_GPL(pm_clk_add_notifier); diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index de23b648f..a1f2aff33 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -187,8 +187,7 @@ static int genpd_poweron(struct generic_pm_domain *genpd, unsigned int depth) struct gpd_link *link; int ret = 0; - if (genpd->status == GPD_STATE_ACTIVE - || (genpd->prepared_count > 0 && genpd->suspend_power_off)) + if (genpd->status == GPD_STATE_ACTIVE) return 0; /* @@ -735,81 +734,23 @@ static int pm_genpd_prepare(struct device *dev) mutex_lock(&genpd->lock); - if (genpd->prepared_count++ == 0) { + if (genpd->prepared_count++ == 0) genpd->suspended_count = 0; - genpd->suspend_power_off = genpd->status == GPD_STATE_POWER_OFF; - } mutex_unlock(&genpd->lock); - if (genpd->suspend_power_off) - return 0; - - /* - * The PM domain must be in the GPD_STATE_ACTIVE state at this point, - * so genpd_poweron() will return immediately, but if the device - * is suspended (e.g. it's been stopped by genpd_stop_dev()), we need - * to make it operational. - */ - pm_runtime_resume(dev); - __pm_runtime_disable(dev, false); - ret = pm_generic_prepare(dev); if (ret) { mutex_lock(&genpd->lock); - if (--genpd->prepared_count == 0) - genpd->suspend_power_off = false; + genpd->prepared_count--; mutex_unlock(&genpd->lock); - pm_runtime_enable(dev); } return ret; } -/** - * pm_genpd_suspend - Suspend a device belonging to an I/O PM domain. - * @dev: Device to suspend. - * - * Suspend a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a PM domain consisting of I/O devices. - */ -static int pm_genpd_suspend(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_suspend(dev); -} - -/** - * pm_genpd_suspend_late - Late suspend of a device from an I/O PM domain. - * @dev: Device to suspend. - * - * Carry out a late suspend of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a PM domain consisting of I/O devices. - */ -static int pm_genpd_suspend_late(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_suspend_late(dev); -} - /** * pm_genpd_suspend_noirq - Completion of suspend of device in an I/O PM domain. * @dev: Device to suspend. @@ -820,6 +761,7 @@ static int pm_genpd_suspend_late(struct device *dev) static int pm_genpd_suspend_noirq(struct device *dev) { struct generic_pm_domain *genpd; + int ret; dev_dbg(dev, "%s()\n", __func__); @@ -827,11 +769,14 @@ static int pm_genpd_suspend_noirq(struct device *dev) if (IS_ERR(genpd)) return -EINVAL; - if (genpd->suspend_power_off - || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev))) + if (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)) return 0; - genpd_stop_dev(genpd, dev); + if (genpd->dev_ops.stop && genpd->dev_ops.start) { + ret = pm_runtime_force_suspend(dev); + if (ret) + return ret; + } /* * Since all of the "noirq" callbacks are executed sequentially, it is @@ -853,6 +798,7 @@ static int pm_genpd_suspend_noirq(struct device *dev) static int pm_genpd_resume_noirq(struct device *dev) { struct generic_pm_domain *genpd; + int ret = 0; dev_dbg(dev, "%s()\n", __func__); @@ -860,8 +806,7 @@ static int pm_genpd_resume_noirq(struct device *dev) if (IS_ERR(genpd)) return -EINVAL; - if (genpd->suspend_power_off - || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev))) + if (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)) return 0; /* @@ -872,93 +817,10 @@ static int pm_genpd_resume_noirq(struct device *dev) pm_genpd_sync_poweron(genpd, true); genpd->suspended_count--; - return genpd_start_dev(genpd, dev); -} - -/** - * pm_genpd_resume_early - Early resume of a device in an I/O PM domain. - * @dev: Device to resume. - * - * Carry out an early resume of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int pm_genpd_resume_early(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_resume_early(dev); -} - -/** - * pm_genpd_resume - Resume of device in an I/O PM domain. - * @dev: Device to resume. - * - * Resume a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static int pm_genpd_resume(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_resume(dev); -} - -/** - * pm_genpd_freeze - Freezing a device in an I/O PM domain. - * @dev: Device to freeze. - * - * Freeze a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static int pm_genpd_freeze(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_freeze(dev); -} + if (genpd->dev_ops.stop && genpd->dev_ops.start) + ret = pm_runtime_force_resume(dev); -/** - * pm_genpd_freeze_late - Late freeze of a device in an I/O PM domain. - * @dev: Device to freeze. - * - * Carry out a late freeze of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int pm_genpd_freeze_late(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_freeze_late(dev); + return ret; } /** @@ -973,6 +835,7 @@ static int pm_genpd_freeze_late(struct device *dev) static int pm_genpd_freeze_noirq(struct device *dev) { struct generic_pm_domain *genpd; + int ret = 0; dev_dbg(dev, "%s()\n", __func__); @@ -980,7 +843,10 @@ static int pm_genpd_freeze_noirq(struct device *dev) if (IS_ERR(genpd)) return -EINVAL; - return genpd->suspend_power_off ? 0 : genpd_stop_dev(genpd, dev); + if (genpd->dev_ops.stop && genpd->dev_ops.start) + ret = pm_runtime_force_suspend(dev); + + return ret; } /** @@ -993,6 +859,7 @@ static int pm_genpd_freeze_noirq(struct device *dev) static int pm_genpd_thaw_noirq(struct device *dev) { struct generic_pm_domain *genpd; + int ret = 0; dev_dbg(dev, "%s()\n", __func__); @@ -1000,51 +867,10 @@ static int pm_genpd_thaw_noirq(struct device *dev) if (IS_ERR(genpd)) return -EINVAL; - return genpd->suspend_power_off ? - 0 : genpd_start_dev(genpd, dev); -} - -/** - * pm_genpd_thaw_early - Early thaw of device in an I/O PM domain. - * @dev: Device to thaw. - * - * Carry out an early thaw of a device under the assumption that its - * pm_domain field points to the domain member of an object of type - * struct generic_pm_domain representing a power domain consisting of I/O - * devices. - */ -static int pm_genpd_thaw_early(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); - - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_thaw_early(dev); -} - -/** - * pm_genpd_thaw - Thaw a device belonging to an I/O power domain. - * @dev: Device to thaw. - * - * Thaw a device under the assumption that its pm_domain field points to the - * domain member of an object of type struct generic_pm_domain representing - * a power domain consisting of I/O devices. - */ -static int pm_genpd_thaw(struct device *dev) -{ - struct generic_pm_domain *genpd; - - dev_dbg(dev, "%s()\n", __func__); + if (genpd->dev_ops.stop && genpd->dev_ops.start) + ret = pm_runtime_force_resume(dev); - genpd = dev_to_genpd(dev); - if (IS_ERR(genpd)) - return -EINVAL; - - return genpd->suspend_power_off ? 0 : pm_generic_thaw(dev); + return ret; } /** @@ -1057,6 +883,7 @@ static int pm_genpd_thaw(struct device *dev) static int pm_genpd_restore_noirq(struct device *dev) { struct generic_pm_domain *genpd; + int ret = 0; dev_dbg(dev, "%s()\n", __func__); @@ -1072,30 +899,20 @@ static int pm_genpd_restore_noirq(struct device *dev) * At this point suspended_count == 0 means we are being run for the * first time for the given domain in the present cycle. */ - if (genpd->suspended_count++ == 0) { + if (genpd->suspended_count++ == 0) /* * The boot kernel might put the domain into arbitrary state, * so make it appear as powered off to pm_genpd_sync_poweron(), * so that it tries to power it on in case it was really off. */ genpd->status = GPD_STATE_POWER_OFF; - if (genpd->suspend_power_off) { - /* - * If the domain was off before the hibernation, make - * sure it will be off going forward. - */ - genpd_power_off(genpd, true); - - return 0; - } - } - - if (genpd->suspend_power_off) - return 0; pm_genpd_sync_poweron(genpd, true); - return genpd_start_dev(genpd, dev); + if (genpd->dev_ops.stop && genpd->dev_ops.start) + ret = pm_runtime_force_resume(dev); + + return ret; } /** @@ -1110,7 +927,6 @@ static int pm_genpd_restore_noirq(struct device *dev) static void pm_genpd_complete(struct device *dev) { struct generic_pm_domain *genpd; - bool run_complete; dev_dbg(dev, "%s()\n", __func__); @@ -1118,20 +934,15 @@ static void pm_genpd_complete(struct device *dev) if (IS_ERR(genpd)) return; + pm_generic_complete(dev); + mutex_lock(&genpd->lock); - run_complete = !genpd->suspend_power_off; - if (--genpd->prepared_count == 0) - genpd->suspend_power_off = false; + genpd->prepared_count--; + if (!genpd->prepared_count) + genpd_queue_power_off_work(genpd); mutex_unlock(&genpd->lock); - - if (run_complete) { - pm_generic_complete(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); - pm_request_idle(dev); - } } /** @@ -1173,18 +984,10 @@ EXPORT_SYMBOL_GPL(pm_genpd_syscore_poweron); #else /* !CONFIG_PM_SLEEP */ #define pm_genpd_prepare NULL -#define pm_genpd_suspend NULL -#define pm_genpd_suspend_late NULL #define pm_genpd_suspend_noirq NULL -#define pm_genpd_resume_early NULL #define pm_genpd_resume_noirq NULL -#define pm_genpd_resume NULL -#define pm_genpd_freeze NULL -#define pm_genpd_freeze_late NULL #define pm_genpd_freeze_noirq NULL -#define pm_genpd_thaw_early NULL #define pm_genpd_thaw_noirq NULL -#define pm_genpd_thaw NULL #define pm_genpd_restore_noirq NULL #define pm_genpd_complete NULL @@ -1455,12 +1258,14 @@ EXPORT_SYMBOL_GPL(pm_genpd_remove_subdomain); * @genpd: PM domain object to initialize. * @gov: PM domain governor to associate with the domain (may be NULL). * @is_off: Initial value of the domain's power_is_off field. + * + * Returns 0 on successful initialization, else a negative error code. */ -void pm_genpd_init(struct generic_pm_domain *genpd, - struct dev_power_governor *gov, bool is_off) +int pm_genpd_init(struct generic_pm_domain *genpd, + struct dev_power_governor *gov, bool is_off) { if (IS_ERR_OR_NULL(genpd)) - return; + return -EINVAL; INIT_LIST_HEAD(&genpd->master_links); INIT_LIST_HEAD(&genpd->slave_links); @@ -1476,24 +1281,24 @@ void pm_genpd_init(struct generic_pm_domain *genpd, genpd->domain.ops.runtime_suspend = genpd_runtime_suspend; genpd->domain.ops.runtime_resume = genpd_runtime_resume; genpd->domain.ops.prepare = pm_genpd_prepare; - genpd->domain.ops.suspend = pm_genpd_suspend; - genpd->domain.ops.suspend_late = pm_genpd_suspend_late; + genpd->domain.ops.suspend = pm_generic_suspend; + genpd->domain.ops.suspend_late = pm_generic_suspend_late; genpd->domain.ops.suspend_noirq = pm_genpd_suspend_noirq; genpd->domain.ops.resume_noirq = pm_genpd_resume_noirq; - genpd->domain.ops.resume_early = pm_genpd_resume_early; - genpd->domain.ops.resume = pm_genpd_resume; - genpd->domain.ops.freeze = pm_genpd_freeze; - genpd->domain.ops.freeze_late = pm_genpd_freeze_late; + genpd->domain.ops.resume_early = pm_generic_resume_early; + genpd->domain.ops.resume = pm_generic_resume; + genpd->domain.ops.freeze = pm_generic_freeze; + genpd->domain.ops.freeze_late = pm_generic_freeze_late; genpd->domain.ops.freeze_noirq = pm_genpd_freeze_noirq; genpd->domain.ops.thaw_noirq = pm_genpd_thaw_noirq; - genpd->domain.ops.thaw_early = pm_genpd_thaw_early; - genpd->domain.ops.thaw = pm_genpd_thaw; - genpd->domain.ops.poweroff = pm_genpd_suspend; - genpd->domain.ops.poweroff_late = pm_genpd_suspend_late; + genpd->domain.ops.thaw_early = pm_generic_thaw_early; + genpd->domain.ops.thaw = pm_generic_thaw; + genpd->domain.ops.poweroff = pm_generic_poweroff; + genpd->domain.ops.poweroff_late = pm_generic_poweroff_late; genpd->domain.ops.poweroff_noirq = pm_genpd_suspend_noirq; genpd->domain.ops.restore_noirq = pm_genpd_restore_noirq; - genpd->domain.ops.restore_early = pm_genpd_resume_early; - genpd->domain.ops.restore = pm_genpd_resume; + genpd->domain.ops.restore_early = pm_generic_restore_early; + genpd->domain.ops.restore = pm_generic_restore; genpd->domain.ops.complete = pm_genpd_complete; if (genpd->flags & GENPD_FLAG_PM_CLK) { @@ -1518,6 +1323,8 @@ void pm_genpd_init(struct generic_pm_domain *genpd, mutex_lock(&gpd_list_lock); list_add(&genpd->gpd_list_node, &gpd_list); mutex_unlock(&gpd_list_lock); + + return 0; } EXPORT_SYMBOL_GPL(pm_genpd_init); diff --git a/drivers/base/power/opp/core.c b/drivers/base/power/opp/core.c index 7c04c8773..df0c70963 100644 --- a/drivers/base/power/opp/core.c +++ b/drivers/base/power/opp/core.c @@ -402,6 +402,22 @@ struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev, } EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_exact); +static noinline struct dev_pm_opp *_find_freq_ceil(struct opp_table *opp_table, + unsigned long *freq) +{ + struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); + + list_for_each_entry_rcu(temp_opp, &opp_table->opp_list, node) { + if (temp_opp->available && temp_opp->rate >= *freq) { + opp = temp_opp; + *freq = opp->rate; + break; + } + } + + return opp; +} + /** * dev_pm_opp_find_freq_ceil() - Search for an rounded ceil freq * @dev: device for which we do this operation @@ -427,7 +443,6 @@ struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev, unsigned long *freq) { struct opp_table *opp_table; - struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); opp_rcu_lockdep_assert(); @@ -440,15 +455,7 @@ struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev, if (IS_ERR(opp_table)) return ERR_CAST(opp_table); - list_for_each_entry_rcu(temp_opp, &opp_table->opp_list, node) { - if (temp_opp->available && temp_opp->rate >= *freq) { - opp = temp_opp; - *freq = opp->rate; - break; - } - } - - return opp; + return _find_freq_ceil(opp_table, freq); } EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil); @@ -612,7 +619,7 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) return PTR_ERR(opp_table); } - old_opp = dev_pm_opp_find_freq_ceil(dev, &old_freq); + old_opp = _find_freq_ceil(opp_table, &old_freq); if (!IS_ERR(old_opp)) { ou_volt = old_opp->u_volt; ou_volt_min = old_opp->u_volt_min; @@ -622,7 +629,7 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) __func__, old_freq, PTR_ERR(old_opp)); } - opp = dev_pm_opp_find_freq_ceil(dev, &freq); + opp = _find_freq_ceil(opp_table, &freq); if (IS_ERR(opp)) { ret = PTR_ERR(opp); dev_err(dev, "%s: failed to find OPP for freq %lu (%d)\n", diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index b74690418..82a081ea4 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -301,7 +301,7 @@ static int rpm_idle(struct device *dev, int rpmflags) int (*callback)(struct device *); int retval; - trace_rpm_idle(dev, rpmflags); + trace_rpm_idle_rcuidle(dev, rpmflags); retval = rpm_check_suspend_allowed(dev); if (retval < 0) ; /* Conditions are wrong. */ @@ -337,7 +337,7 @@ static int rpm_idle(struct device *dev, int rpmflags) dev->power.request_pending = true; queue_work(pm_wq, &dev->power.work); } - trace_rpm_return_int(dev, _THIS_IP_, 0); + trace_rpm_return_int_rcuidle(dev, _THIS_IP_, 0); return 0; } @@ -352,7 +352,7 @@ static int rpm_idle(struct device *dev, int rpmflags) wake_up_all(&dev->power.wait_queue); out: - trace_rpm_return_int(dev, _THIS_IP_, retval); + trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval); return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO); } @@ -419,7 +419,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) struct device *parent = NULL; int retval; - trace_rpm_suspend(dev, rpmflags); + trace_rpm_suspend_rcuidle(dev, rpmflags); repeat: retval = rpm_check_suspend_allowed(dev); @@ -549,7 +549,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) } out: - trace_rpm_return_int(dev, _THIS_IP_, retval); + trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval); return retval; @@ -601,7 +601,7 @@ static int rpm_resume(struct device *dev, int rpmflags) struct device *parent = NULL; int retval = 0; - trace_rpm_resume(dev, rpmflags); + trace_rpm_resume_rcuidle(dev, rpmflags); repeat: if (dev->power.runtime_error) @@ -764,7 +764,7 @@ static int rpm_resume(struct device *dev, int rpmflags) spin_lock_irq(&dev->power.lock); } - trace_rpm_return_int(dev, _THIS_IP_, retval); + trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval); return retval; } @@ -1045,10 +1045,14 @@ int __pm_runtime_set_status(struct device *dev, unsigned int status) */ if (!parent->power.disable_depth && !parent->power.ignore_children - && parent->power.runtime_status != RPM_ACTIVE) + && parent->power.runtime_status != RPM_ACTIVE) { + dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n", + dev_name(dev), + dev_name(parent)); error = -EBUSY; - else if (dev->power.runtime_status == RPM_SUSPENDED) + } else if (dev->power.runtime_status == RPM_SUSPENDED) { atomic_inc(&parent->power.child_count); + } spin_unlock(&parent->power.lock); @@ -1256,7 +1260,7 @@ void pm_runtime_allow(struct device *dev) dev->power.runtime_auto = true; if (atomic_dec_and_test(&dev->power.usage_count)) - rpm_idle(dev, RPM_AUTO); + rpm_idle(dev, RPM_AUTO | RPM_ASYNC); out: spin_unlock_irq(&dev->power.lock); @@ -1506,6 +1510,9 @@ int pm_runtime_force_resume(struct device *dev) goto out; } + if (!pm_runtime_status_suspended(dev)) + goto out; + ret = pm_runtime_set_active(dev); if (ret) goto out; diff --git a/drivers/base/power/trace.c b/drivers/base/power/trace.c index a6975795e..efec10b49 100644 --- a/drivers/base/power/trace.c +++ b/drivers/base/power/trace.c @@ -11,7 +11,7 @@ #include #include -#include +#include #include "power.h" @@ -103,7 +103,7 @@ static int set_magic_time(unsigned int user, unsigned int file, unsigned int dev n /= 24; time.tm_min = (n % 20) * 3; n /= 20; - set_rtc_time(&time); + mc146818_set_time(&time); return n ? -1 : 0; } @@ -112,7 +112,7 @@ static unsigned int read_magic_time(void) struct rtc_time time; unsigned int val; - get_rtc_time(&time); + mc146818_get_time(&time); pr_info("RTC time: %2d:%02d:%02d, date: %02d/%02d/%02d\n", time.tm_hour, time.tm_min, time.tm_sec, time.tm_mon + 1, time.tm_mday, time.tm_year % 100); diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index 5fb7718f2..62e4de2aa 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -334,10 +334,9 @@ void device_wakeup_arm_wake_irqs(void) struct wakeup_source *ws; rcu_read_lock(); - list_for_each_entry_rcu(ws, &wakeup_sources, entry) { - if (ws->wakeirq) - dev_pm_arm_wake_irq(ws->wakeirq); - } + list_for_each_entry_rcu(ws, &wakeup_sources, entry) + dev_pm_arm_wake_irq(ws->wakeirq); + rcu_read_unlock(); } @@ -351,10 +350,9 @@ void device_wakeup_disarm_wake_irqs(void) struct wakeup_source *ws; rcu_read_lock(); - list_for_each_entry_rcu(ws, &wakeup_sources, entry) { - if (ws->wakeirq) - dev_pm_disarm_wake_irq(ws->wakeirq); - } + list_for_each_entry_rcu(ws, &wakeup_sources, entry) + dev_pm_disarm_wake_irq(ws->wakeirq); + rcu_read_unlock(); } @@ -390,9 +388,7 @@ int device_wakeup_disable(struct device *dev) return -EINVAL; ws = device_wakeup_detach(dev); - if (ws) - wakeup_source_unregister(ws); - + wakeup_source_unregister(ws); return 0; } EXPORT_SYMBOL_GPL(device_wakeup_disable); diff --git a/drivers/base/property.c b/drivers/base/property.c index f38c21de2..43a36d68c 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -887,6 +887,34 @@ struct fwnode_handle *device_get_next_child_node(struct device *dev, } EXPORT_SYMBOL_GPL(device_get_next_child_node); +/** + * device_get_named_child_node - Return first matching named child node handle + * @dev: Device to find the named child node for. + * @childname: String to match child node name against. + */ +struct fwnode_handle *device_get_named_child_node(struct device *dev, + const char *childname) +{ + struct fwnode_handle *child; + + /* + * Find first matching named child node of this device. + * For ACPI this will be a data only sub-node. + */ + device_for_each_child_node(dev, child) { + if (is_of_node(child)) { + if (!of_node_cmp(to_of_node(child)->name, childname)) + return child; + } else if (is_acpi_data_node(child)) { + if (acpi_data_node_match(child, childname)) + return child; + } + } + + return NULL; +} +EXPORT_SYMBOL_GPL(device_get_named_child_node); + /** * fwnode_handle_put - Drop reference to a device node * @fwnode: Pointer to the device node to drop the reference to. diff --git a/drivers/base/regmap/regcache-rbtree.c b/drivers/base/regmap/regcache-rbtree.c index aa56af87d..b11af3f2c 100644 --- a/drivers/base/regmap/regcache-rbtree.c +++ b/drivers/base/regmap/regcache-rbtree.c @@ -404,6 +404,7 @@ static int regcache_rbtree_write(struct regmap *map, unsigned int reg, unsigned int new_base_reg, new_top_reg; unsigned int min, max; unsigned int max_dist; + unsigned int dist, best_dist = UINT_MAX; max_dist = map->reg_stride * sizeof(*rbnode_tmp) / map->cache_word_size; @@ -423,24 +424,41 @@ static int regcache_rbtree_write(struct regmap *map, unsigned int reg, &base_reg, &top_reg); if (base_reg <= max && top_reg >= min) { - new_base_reg = min(reg, base_reg); - new_top_reg = max(reg, top_reg); - } else { - if (max < base_reg) - node = node->rb_left; + if (reg < base_reg) + dist = base_reg - reg; + else if (reg > top_reg) + dist = reg - top_reg; else - node = node->rb_right; - - continue; + dist = 0; + if (dist < best_dist) { + rbnode = rbnode_tmp; + best_dist = dist; + new_base_reg = min(reg, base_reg); + new_top_reg = max(reg, top_reg); + } } - ret = regcache_rbtree_insert_to_block(map, rbnode_tmp, + /* + * Keep looking, we want to choose the closest block, + * otherwise we might end up creating overlapping + * blocks, which breaks the rbtree. + */ + if (reg < base_reg) + node = node->rb_left; + else if (reg > top_reg) + node = node->rb_right; + else + break; + } + + if (rbnode) { + ret = regcache_rbtree_insert_to_block(map, rbnode, new_base_reg, new_top_reg, reg, value); if (ret) return ret; - rbtree_ctx->cached_rbnode = rbnode_tmp; + rbtree_ctx->cached_rbnode = rbnode; return 0; } diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c index df7ff7290..4e582561e 100644 --- a/drivers/base/regmap/regcache.c +++ b/drivers/base/regmap/regcache.c @@ -38,10 +38,11 @@ static int regcache_hw_init(struct regmap *map) /* calculate the size of reg_defaults */ for (count = 0, i = 0; i < map->num_reg_defaults_raw; i++) - if (!regmap_volatile(map, i * map->reg_stride)) + if (regmap_readable(map, i * map->reg_stride) && + !regmap_volatile(map, i * map->reg_stride)) count++; - /* all registers are volatile, so just bypass */ + /* all registers are unreadable or volatile, so just bypass */ if (!count) { map->cache_bypass = true; return 0; diff --git a/drivers/base/regmap/regmap-i2c.c b/drivers/base/regmap/regmap-i2c.c index 1a8ec3b2b..4735318f4 100644 --- a/drivers/base/regmap/regmap-i2c.c +++ b/drivers/base/regmap/regmap-i2c.c @@ -259,7 +259,7 @@ static const struct regmap_bus *regmap_get_i2c_bus(struct i2c_client *i2c, { if (i2c_check_functionality(i2c->adapter, I2C_FUNC_I2C)) return ®map_i2c; - else if (config->reg_bits == 8 && + else if (config->val_bits == 8 && config->reg_bits == 8 && i2c_check_functionality(i2c->adapter, I2C_FUNC_SMBUS_I2C_BLOCK)) return ®map_i2c_smbus_i2c_block; diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index 26f799e71..ec262476d 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -268,13 +268,16 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) bool handled = false; u32 reg; + if (chip->handle_pre_irq) + chip->handle_pre_irq(chip->irq_drv_data); + if (chip->runtime_pm) { ret = pm_runtime_get_sync(map->dev); if (ret < 0) { dev_err(map->dev, "IRQ thread failed to resume: %d\n", ret); pm_runtime_put(map->dev); - return IRQ_NONE; + goto exit; } } @@ -296,7 +299,7 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) if (ret != 0) { dev_err(map->dev, "Failed to read IRQ status: %d\n", ret); - return IRQ_NONE; + goto exit; } for (i = 0; i < data->chip->num_regs; i++) { @@ -312,7 +315,7 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) break; default: BUG(); - return IRQ_NONE; + goto exit; } } @@ -329,7 +332,7 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) ret); if (chip->runtime_pm) pm_runtime_put(map->dev); - return IRQ_NONE; + goto exit; } } } @@ -365,6 +368,10 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) if (chip->runtime_pm) pm_runtime_put(map->dev); +exit: + if (chip->handle_post_irq) + chip->handle_post_irq(chip->irq_drv_data); + if (handled) return IRQ_HANDLED; else diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index df2d2ef5d..e964d0688 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -1474,6 +1474,12 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg, ret = map->bus->write(map->bus_context, buf, len); kfree(buf); + } else if (ret != 0 && !map->cache_bypass && map->format.parse_val) { + /* regcache_drop_region() takes lock that we already have, + * thus call map->cache_ops->drop() directly + */ + if (map->cache_ops && map->cache_ops->drop) + map->cache_ops->drop(map, reg, reg + 1); } trace_regmap_hw_write_done(map, reg, val_len / map->format.val_bytes); @@ -1777,8 +1783,6 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, size_t val_bytes = map->format.val_bytes; size_t total_size = val_bytes * val_count; - if (map->bus && !map->format.parse_inplace) - return -EINVAL; if (!IS_ALIGNED(reg, map->reg_stride)) return -EINVAL; @@ -1789,7 +1793,8 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, * * The first if block is used for memory mapped io. It does not allow * val_bytes of 3 for example. - * The second one is used for busses which do not have this limitation + * The second one is for busses that do not provide raw I/O. + * The third one is used for busses which do not have these limitations * and can write arbitrary value lengths. */ if (!map->bus) { @@ -1825,6 +1830,32 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, } out: map->unlock(map->lock_arg); + } else if (map->bus && !map->format.parse_inplace) { + const u8 *u8 = val; + const u16 *u16 = val; + const u32 *u32 = val; + unsigned int ival; + + for (i = 0; i < val_count; i++) { + switch (map->format.val_bytes) { + case 4: + ival = u32[i]; + break; + case 2: + ival = u16[i]; + break; + case 1: + ival = u8[i]; + break; + default: + return -EINVAL; + } + + ret = regmap_write(map, reg + (i * map->reg_stride), + ival); + if (ret) + return ret; + } } else if (map->use_single_write || (map->max_raw_write && map->max_raw_write < total_size)) { int chunk_stride = map->reg_stride; diff --git a/drivers/base/topology.c b/drivers/base/topology.c index 8b7d7f8e5..df3c97cb4 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -77,6 +77,14 @@ static DEVICE_ATTR_RO(book_siblings); static DEVICE_ATTR_RO(book_siblings_list); #endif +#ifdef CONFIG_SCHED_DRAWER +define_id_show_func(drawer_id); +static DEVICE_ATTR_RO(drawer_id); +define_siblings_show_func(drawer_siblings, drawer_cpumask); +static DEVICE_ATTR_RO(drawer_siblings); +static DEVICE_ATTR_RO(drawer_siblings_list); +#endif + static struct attribute *default_attrs[] = { &dev_attr_physical_package_id.attr, &dev_attr_core_id.attr, @@ -88,6 +96,11 @@ static struct attribute *default_attrs[] = { &dev_attr_book_id.attr, &dev_attr_book_siblings.attr, &dev_attr_book_siblings_list.attr, +#endif +#ifdef CONFIG_SCHED_DRAWER + &dev_attr_drawer_id.attr, + &dev_attr_drawer_siblings.attr, + &dev_attr_drawer_siblings_list.attr, #endif NULL }; diff --git a/drivers/bcma/Kconfig b/drivers/bcma/Kconfig index efdc2ae84..b5c48a8d4 100644 --- a/drivers/bcma/Kconfig +++ b/drivers/bcma/Kconfig @@ -76,9 +76,16 @@ config BCMA_PFLASH default y config BCMA_SFLASH - bool - depends on BCMA_DRIVER_MIPS + bool "ChipCommon-attached serial flash support" + depends on BCMA_HOST_SOC default y + help + Some cheap devices have serial flash connected to the ChipCommon + instead of independent SPI controller. It requires using a separated + driver that implements ChipCommon specific interface communication. + + Enabling this symbol will let bcma recognize serial flash and register + it as platform device. config BCMA_NFLASH bool diff --git a/drivers/bcma/driver_chipcommon_b.c b/drivers/bcma/driver_chipcommon_b.c index c20b5f4ff..57f10b58b 100644 --- a/drivers/bcma/driver_chipcommon_b.c +++ b/drivers/bcma/driver_chipcommon_b.c @@ -33,11 +33,12 @@ static bool bcma_wait_reg(struct bcma_bus *bus, void __iomem *addr, u32 mask, void bcma_chipco_b_mii_write(struct bcma_drv_cc_b *ccb, u32 offset, u32 value) { struct bcma_bus *bus = ccb->core->bus; + void __iomem *mii = ccb->mii; - writel(offset, ccb->mii + 0x00); - bcma_wait_reg(bus, ccb->mii + 0x00, 0x0100, 0x0000, 100); - writel(value, ccb->mii + 0x04); - bcma_wait_reg(bus, ccb->mii + 0x00, 0x0100, 0x0000, 100); + writel(offset, mii + BCMA_CCB_MII_MNG_CTL); + bcma_wait_reg(bus, mii + BCMA_CCB_MII_MNG_CTL, 0x0100, 0x0000, 100); + writel(value, mii + BCMA_CCB_MII_MNG_CMD_DATA); + bcma_wait_reg(bus, mii + BCMA_CCB_MII_MNG_CTL, 0x0100, 0x0000, 100); } EXPORT_SYMBOL_GPL(bcma_chipco_b_mii_write); diff --git a/drivers/block/brd.c b/drivers/block/brd.c index c04bd9bc3..0c76d4016 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -300,20 +300,20 @@ static void copy_from_brd(void *dst, struct brd_device *brd, * Process a single bvec of a bio. */ static int brd_do_bvec(struct brd_device *brd, struct page *page, - unsigned int len, unsigned int off, int rw, + unsigned int len, unsigned int off, bool is_write, sector_t sector) { void *mem; int err = 0; - if (rw != READ) { + if (is_write) { err = copy_to_brd_setup(brd, sector, len); if (err) goto out; } mem = kmap_atomic(page); - if (rw == READ) { + if (!is_write) { copy_from_brd(mem + off, brd, sector, len); flush_dcache_page(page); } else { @@ -330,7 +330,6 @@ static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio) { struct block_device *bdev = bio->bi_bdev; struct brd_device *brd = bdev->bd_disk->private_data; - int rw; struct bio_vec bvec; sector_t sector; struct bvec_iter iter; @@ -339,7 +338,7 @@ static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio) if (bio_end_sector(bio) > get_capacity(bdev->bd_disk)) goto io_error; - if (unlikely(bio->bi_rw & REQ_DISCARD)) { + if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) { if (sector & ((PAGE_SIZE >> SECTOR_SHIFT) - 1) || bio->bi_iter.bi_size & ~PAGE_MASK) goto io_error; @@ -347,16 +346,12 @@ static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio) goto out; } - rw = bio_rw(bio); - if (rw == READA) - rw = READ; - bio_for_each_segment(bvec, bio, iter) { unsigned int len = bvec.bv_len; int err; - err = brd_do_bvec(brd, bvec.bv_page, len, - bvec.bv_offset, rw, sector); + err = brd_do_bvec(brd, bvec.bv_page, len, bvec.bv_offset, + op_is_write(bio_op(bio)), sector); if (err) goto io_error; sector += len >> SECTOR_SHIFT; @@ -371,17 +366,17 @@ io_error: } static int brd_rw_page(struct block_device *bdev, sector_t sector, - struct page *page, int rw) + struct page *page, bool is_write) { struct brd_device *brd = bdev->bd_disk->private_data; - int err = brd_do_bvec(brd, page, PAGE_SIZE, 0, rw, sector); - page_endio(page, rw & WRITE, err); + int err = brd_do_bvec(brd, page, PAGE_SIZE, 0, is_write, sector); + page_endio(page, is_write, err); return err; } #ifdef CONFIG_BLK_DEV_RAM_DAX static long brd_direct_access(struct block_device *bdev, sector_t sector, - void __pmem **kaddr, pfn_t *pfn, long size) + void **kaddr, pfn_t *pfn, long size) { struct brd_device *brd = bdev->bd_disk->private_data; struct page *page; @@ -391,7 +386,7 @@ static long brd_direct_access(struct block_device *bdev, sector_t sector, page = brd_insert_page(brd, sector); if (!page) return -ENOSPC; - *kaddr = (void __pmem *)page_address(page); + *kaddr = page_address(page); *pfn = page_to_pfn_t(page); return PAGE_SIZE; @@ -509,7 +504,9 @@ static struct brd_device *brd_alloc(int i) blk_queue_max_discard_sectors(brd->brd_queue, UINT_MAX); brd->brd_queue->limits.discard_zeroes_data = 1; queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, brd->brd_queue); - +#ifdef CONFIG_BLK_DEV_RAM_DAX + queue_flag_set_unlocked(QUEUE_FLAG_DAX, brd->brd_queue); +#endif disk = brd->brd_disk = alloc_disk(max_part); if (!disk) goto out_free_queue; diff --git a/drivers/block/cciss.c b/drivers/block/cciss.c index 63c206468..db9d6bb63 100644 --- a/drivers/block/cciss.c +++ b/drivers/block/cciss.c @@ -1951,7 +1951,6 @@ static int cciss_add_disk(ctlr_info_t *h, struct gendisk *disk, if (cciss_create_ld_sysfs_entry(h, drv_index)) goto cleanup_queue; disk->private_data = h->drv[drv_index]; - disk->driverfs_dev = &h->drv[drv_index]->dev; /* Set up queue information */ blk_queue_bounce_limit(disk->queue, h->pdev->dma_mask); @@ -1973,7 +1972,7 @@ static int cciss_add_disk(ctlr_info_t *h, struct gendisk *disk, /* allows the interrupt handler to start the queue */ wmb(); h->drv[drv_index]->queue = disk->queue; - add_disk(disk); + device_add_disk(&h->drv[drv_index]->dev, disk); return 0; cleanup_queue: diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c index 10459a145..2d3d50ab7 100644 --- a/drivers/block/drbd/drbd_actlog.c +++ b/drivers/block/drbd/drbd_actlog.c @@ -27,7 +27,6 @@ #include #include #include -#include #include "drbd_int.h" @@ -137,19 +136,19 @@ void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_b static int _drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bdev, - sector_t sector, int rw) + sector_t sector, int op) { struct bio *bio; /* we do all our meta data IO in aligned 4k blocks. */ const int size = 4096; - int err; + int err, op_flags = 0; device->md_io.done = 0; device->md_io.error = -ENODEV; - if ((rw & WRITE) && !test_bit(MD_NO_FUA, &device->flags)) - rw |= REQ_FUA | REQ_FLUSH; - rw |= REQ_SYNC | REQ_NOIDLE; + if ((op == REQ_OP_WRITE) && !test_bit(MD_NO_FUA, &device->flags)) + op_flags |= REQ_FUA | REQ_PREFLUSH; + op_flags |= REQ_SYNC | REQ_NOIDLE; bio = bio_alloc_drbd(GFP_NOIO); bio->bi_bdev = bdev->md_bdev; @@ -159,9 +158,9 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, goto out; bio->bi_private = device; bio->bi_end_io = drbd_md_endio; - bio->bi_rw = rw; + bio_set_op_attrs(bio, op, op_flags); - if (!(rw & WRITE) && device->state.disk == D_DISKLESS && device->ldev == NULL) + if (op != REQ_OP_WRITE && device->state.disk == D_DISKLESS && device->ldev == NULL) /* special case, drbd_md_read() during drbd_adm_attach(): no get_ldev */ ; else if (!get_ldev_if_state(device, D_ATTACHING)) { @@ -174,10 +173,10 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, bio_get(bio); /* one bio_put() is in the completion handler */ atomic_inc(&device->md_io.in_use); /* drbd_md_put_buffer() is in the completion handler */ device->md_io.submit_jif = jiffies; - if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) + if (drbd_insert_fault(device, (op == REQ_OP_WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) bio_io_error(bio); else - submit_bio(rw, bio); + submit_bio(bio); wait_until_done_or_force_detached(device, bdev, &device->md_io.done); if (!bio->bi_error) err = device->md_io.error; @@ -188,7 +187,7 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, } int drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bdev, - sector_t sector, int rw) + sector_t sector, int op) { int err; D_ASSERT(device, atomic_read(&device->md_io.in_use) == 1); @@ -197,19 +196,21 @@ int drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bd dynamic_drbd_dbg(device, "meta_data io: %s [%d]:%s(,%llus,%s) %pS\n", current->comm, current->pid, __func__, - (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ", + (unsigned long long)sector, (op == REQ_OP_WRITE) ? "WRITE" : "READ", (void*)_RET_IP_ ); if (sector < drbd_md_first_sector(bdev) || sector + 7 > drbd_md_last_sector(bdev)) drbd_alert(device, "%s [%d]:%s(,%llus,%s) out of range md access!\n", current->comm, current->pid, __func__, - (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ"); + (unsigned long long)sector, + (op == REQ_OP_WRITE) ? "WRITE" : "READ"); - err = _drbd_md_sync_page_io(device, bdev, sector, rw); + err = _drbd_md_sync_page_io(device, bdev, sector, op); if (err) { drbd_err(device, "drbd_md_sync_page_io(,%llus,%s) failed with error %d\n", - (unsigned long long)sector, (rw & WRITE) ? "WRITE" : "READ", err); + (unsigned long long)sector, + (op == REQ_OP_WRITE) ? "WRITE" : "READ", err); } return err; } @@ -256,7 +257,7 @@ bool drbd_al_begin_io_fastpath(struct drbd_device *device, struct drbd_interval unsigned first = i->sector >> (AL_EXTENT_SHIFT-9); unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9); - D_ASSERT(device, (unsigned)(last - first) <= 1); + D_ASSERT(device, first <= last); D_ASSERT(device, atomic_read(&device->local_cnt) > 0); /* FIXME figure out a fast path for bios crossing AL extent boundaries */ @@ -339,6 +340,8 @@ static int __al_write_transaction(struct drbd_device *device, struct al_transact i = 0; + drbd_bm_reset_al_hints(device); + /* Even though no one can start to change this list * once we set the LC_LOCKED -- from drbd_al_begin_io(), * lc_try_lock_for_transaction() --, someone may still @@ -768,10 +771,18 @@ static bool lazy_bitmap_update_due(struct drbd_device *device) static void maybe_schedule_on_disk_bitmap_update(struct drbd_device *device, bool rs_done) { - if (rs_done) - set_bit(RS_DONE, &device->flags); - /* and also set RS_PROGRESS below */ - else if (!lazy_bitmap_update_due(device)) + if (rs_done) { + struct drbd_connection *connection = first_peer_device(device)->connection; + if (connection->agreed_pro_version <= 95 || + is_sync_target_state(device->state.conn)) + set_bit(RS_DONE, &device->flags); + /* and also set RS_PROGRESS below */ + + /* Else: rather wait for explicit notification via receive_state, + * to avoid uuids-rotated-too-fast causing full resync + * in next handshake, in case the replication link breaks + * at the most unfortunate time... */ + } else if (!lazy_bitmap_update_due(device)) return; drbd_device_post_work(device, RS_PROGRESS); @@ -830,6 +841,13 @@ static int update_sync_bits(struct drbd_device *device, return count; } +static bool plausible_request_size(int size) +{ + return size > 0 + && size <= DRBD_MAX_BATCH_BIO_SIZE + && IS_ALIGNED(size, 512); +} + /* clear the bit corresponding to the piece of storage in question: * size byte of data starting from sector. Only clear a bits of the affected * one ore more _aligned_ BM_BLOCK_SIZE blocks. @@ -845,11 +863,11 @@ int __drbd_change_sync(struct drbd_device *device, sector_t sector, int size, unsigned long count = 0; sector_t esector, nr_sectors; - /* This would be an empty REQ_FLUSH, be silent. */ + /* This would be an empty REQ_PREFLUSH, be silent. */ if ((mode == SET_OUT_OF_SYNC) && size == 0) return 0; - if (size <= 0 || !IS_ALIGNED(size, 512) || size > DRBD_MAX_DISCARD_SIZE) { + if (!plausible_request_size(size)) { drbd_err(device, "%s: sector=%llus size=%d nonsense!\n", drbd_change_sync_fname[mode], (unsigned long long)sector, size); diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c index 92d6fc020..ab62b81c2 100644 --- a/drivers/block/drbd/drbd_bitmap.c +++ b/drivers/block/drbd/drbd_bitmap.c @@ -96,6 +96,13 @@ struct drbd_bitmap { struct page **bm_pages; spinlock_t bm_lock; + /* exclusively to be used by __al_write_transaction(), + * drbd_bm_mark_for_writeout() and + * and drbd_bm_write_hinted() -> bm_rw() called from there. + */ + unsigned int n_bitmap_hints; + unsigned int al_bitmap_hints[AL_UPDATES_PER_TRANSACTION]; + /* see LIMITATIONS: above */ unsigned long bm_set; /* nr of set bits; THINK maybe atomic_t? */ @@ -242,6 +249,11 @@ static void bm_set_page_need_writeout(struct page *page) set_bit(BM_PAGE_NEED_WRITEOUT, &page_private(page)); } +void drbd_bm_reset_al_hints(struct drbd_device *device) +{ + device->bitmap->n_bitmap_hints = 0; +} + /** * drbd_bm_mark_for_writeout() - mark a page with a "hint" to be considered for writeout * @device: DRBD device. @@ -253,6 +265,7 @@ static void bm_set_page_need_writeout(struct page *page) */ void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr) { + struct drbd_bitmap *b = device->bitmap; struct page *page; if (page_nr >= device->bitmap->bm_number_of_pages) { drbd_warn(device, "BAD: page_nr: %u, number_of_pages: %u\n", @@ -260,7 +273,9 @@ void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr) return; } page = device->bitmap->bm_pages[page_nr]; - set_bit(BM_PAGE_HINT_WRITEOUT, &page_private(page)); + BUG_ON(b->n_bitmap_hints >= ARRAY_SIZE(b->al_bitmap_hints)); + if (!test_and_set_bit(BM_PAGE_HINT_WRITEOUT, &page_private(page))) + b->al_bitmap_hints[b->n_bitmap_hints++] = page_nr; } static int bm_test_page_unchanged(struct page *page) @@ -427,8 +442,7 @@ static struct page **bm_realloc_pages(struct drbd_bitmap *b, unsigned long want) } /* - * called on driver init only. TODO call when a device is created. - * allocates the drbd_bitmap, and stores it in device->bitmap. + * allocates the drbd_bitmap and stores it in device->bitmap. */ int drbd_bm_init(struct drbd_device *device) { @@ -633,7 +647,8 @@ int drbd_bm_resize(struct drbd_device *device, sector_t capacity, int set_new_bi unsigned long bits, words, owords, obits; unsigned long want, have, onpages; /* number of pages */ struct page **npages, **opages = NULL; - int err = 0, growing; + int err = 0; + bool growing; if (!expect(b)) return -ENOMEM; @@ -980,7 +995,7 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho struct drbd_bitmap *b = device->bitmap; struct page *page; unsigned int len; - unsigned int rw = (ctx->flags & BM_AIO_READ) ? READ : WRITE; + unsigned int op = (ctx->flags & BM_AIO_READ) ? REQ_OP_READ : REQ_OP_WRITE; sector_t on_disk_sector = device->ldev->md.md_offset + device->ldev->md.bm_offset; @@ -1011,12 +1026,12 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho bio_add_page(bio, page, len, 0); bio->bi_private = ctx; bio->bi_end_io = drbd_bm_endio; + bio_set_op_attrs(bio, op, 0); - if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) { - bio->bi_rw |= rw; + if (drbd_insert_fault(device, (op == REQ_OP_WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) { bio_io_error(bio); } else { - submit_bio(rw, bio); + submit_bio(bio); /* this should not count as user activity and cause the * resync to throttle -- see drbd_rs_should_slow_down(). */ atomic_add(len >> 9, &device->rs_sect_ev); @@ -1030,7 +1045,7 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned { struct drbd_bm_aio_ctx *ctx; struct drbd_bitmap *b = device->bitmap; - int num_pages, i, count = 0; + unsigned int num_pages, i, count = 0; unsigned long now; char ppb[10]; int err = 0; @@ -1078,16 +1093,37 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned now = jiffies; /* let the layers below us try to merge these bios... */ - for (i = 0; i < num_pages; i++) { - /* ignore completely unchanged pages */ - if (lazy_writeout_upper_idx && i == lazy_writeout_upper_idx) - break; - if (!(flags & BM_AIO_READ)) { - if ((flags & BM_AIO_WRITE_HINTED) && - !test_and_clear_bit(BM_PAGE_HINT_WRITEOUT, - &page_private(b->bm_pages[i]))) - continue; + if (flags & BM_AIO_READ) { + for (i = 0; i < num_pages; i++) { + atomic_inc(&ctx->in_flight); + bm_page_io_async(ctx, i); + ++count; + cond_resched(); + } + } else if (flags & BM_AIO_WRITE_HINTED) { + /* ASSERT: BM_AIO_WRITE_ALL_PAGES is not set. */ + unsigned int hint; + for (hint = 0; hint < b->n_bitmap_hints; hint++) { + i = b->al_bitmap_hints[hint]; + if (i >= num_pages) /* == -1U: no hint here. */ + continue; + /* Several AL-extents may point to the same page. */ + if (!test_and_clear_bit(BM_PAGE_HINT_WRITEOUT, + &page_private(b->bm_pages[i]))) + continue; + /* Has it even changed? */ + if (bm_test_page_unchanged(b->bm_pages[i])) + continue; + atomic_inc(&ctx->in_flight); + bm_page_io_async(ctx, i); + ++count; + } + } else { + for (i = 0; i < num_pages; i++) { + /* ignore completely unchanged pages */ + if (lazy_writeout_upper_idx && i == lazy_writeout_upper_idx) + break; if (!(flags & BM_AIO_WRITE_ALL_PAGES) && bm_test_page_unchanged(b->bm_pages[i])) { dynamic_drbd_dbg(device, "skipped bm write for idx %u\n", i); @@ -1100,11 +1136,11 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned dynamic_drbd_dbg(device, "skipped bm lazy write for idx %u\n", i); continue; } + atomic_inc(&ctx->in_flight); + bm_page_io_async(ctx, i); + ++count; + cond_resched(); } - atomic_inc(&ctx->in_flight); - bm_page_io_async(ctx, i); - ++count; - cond_resched(); } /* @@ -1121,10 +1157,14 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned kref_put(&ctx->kref, &drbd_bm_aio_ctx_destroy); /* summary for global bitmap IO */ - if (flags == 0) - drbd_info(device, "bitmap %s of %u pages took %lu jiffies\n", - (flags & BM_AIO_READ) ? "READ" : "WRITE", - count, jiffies - now); + if (flags == 0) { + unsigned int ms = jiffies_to_msecs(jiffies - now); + if (ms > 5) { + drbd_info(device, "bitmap %s of %u pages took %u ms\n", + (flags & BM_AIO_READ) ? "READ" : "WRITE", + count, ms); + } + } if (ctx->error) { drbd_alert(device, "we had at least one MD IO ERROR during bitmap IO\n"); diff --git a/drivers/block/drbd/drbd_debugfs.c b/drivers/block/drbd/drbd_debugfs.c index 4de95bbff..de5c3ee8a 100644 --- a/drivers/block/drbd/drbd_debugfs.c +++ b/drivers/block/drbd/drbd_debugfs.c @@ -237,14 +237,9 @@ static void seq_print_peer_request_flags(struct seq_file *m, struct drbd_peer_re seq_print_rq_state_bit(m, f & EE_SEND_WRITE_ACK, &sep, "C"); seq_print_rq_state_bit(m, f & EE_MAY_SET_IN_SYNC, &sep, "set-in-sync"); - if (f & EE_IS_TRIM) { - seq_putc(m, sep); - sep = '|'; - if (f & EE_IS_TRIM_USE_ZEROOUT) - seq_puts(m, "zero-out"); - else - seq_puts(m, "trim"); - } + if (f & EE_IS_TRIM) + __seq_print_rq_state_bit(m, f & EE_IS_TRIM_USE_ZEROOUT, &sep, "zero-out", "trim"); + seq_print_rq_state_bit(m, f & EE_WRITE_SAME, &sep, "write-same"); seq_putc(m, '\n'); } @@ -430,9 +425,6 @@ static int drbd_single_open(struct file *file, int (*show)(struct seq_file *, vo /* Are we still linked, * or has debugfs_remove() already been called? */ parent = file->f_path.dentry->d_parent; - /* not sure if this can happen: */ - if (!parent || d_really_is_negative(parent)) - goto out; /* serialize with d_delete() */ inode_lock(d_inode(parent)); /* Make sure the object is still alive */ @@ -445,7 +437,6 @@ static int drbd_single_open(struct file *file, int (*show)(struct seq_file *, vo if (ret) kref_put(kref, release); } -out: return ret; } @@ -908,7 +899,7 @@ static int drbd_version_open(struct inode *inode, struct file *file) return single_open(file, drbd_version_show, NULL); } -static struct file_operations drbd_version_fops = { +static const struct file_operations drbd_version_fops = { .owner = THIS_MODULE, .open = drbd_version_open, .llseek = seq_lseek, diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index 7a1cf7eaa..4cb8f21ff 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -468,9 +469,15 @@ enum { /* this is/was a write request */ __EE_WRITE, + /* this is/was a write same request */ + __EE_WRITE_SAME, + /* this originates from application on peer * (not some resync or verify or other DRBD internal request) */ __EE_APPLICATION, + + /* If it contains only 0 bytes, send back P_RS_DEALLOCATED */ + __EE_RS_THIN_REQ, }; #define EE_CALL_AL_COMPLETE_IO (1<<__EE_CALL_AL_COMPLETE_IO) #define EE_MAY_SET_IN_SYNC (1<<__EE_MAY_SET_IN_SYNC) @@ -484,7 +491,9 @@ enum { #define EE_IN_INTERVAL_TREE (1<<__EE_IN_INTERVAL_TREE) #define EE_SUBMITTED (1<<__EE_SUBMITTED) #define EE_WRITE (1<<__EE_WRITE) +#define EE_WRITE_SAME (1<<__EE_WRITE_SAME) #define EE_APPLICATION (1<<__EE_APPLICATION) +#define EE_RS_THIN_REQ (1<<__EE_RS_THIN_REQ) /* flag bits per device */ enum { @@ -1123,6 +1132,7 @@ extern int drbd_send_ov_request(struct drbd_peer_device *, sector_t sector, int extern int drbd_send_bitmap(struct drbd_device *device); extern void drbd_send_sr_reply(struct drbd_peer_device *, enum drbd_state_rv retcode); extern void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_state_rv retcode); +extern int drbd_send_rs_deallocated(struct drbd_peer_device *, struct drbd_peer_request *); extern void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *ldev); extern void drbd_device_cleanup(struct drbd_device *device); void drbd_print_uuids(struct drbd_device *device, const char *text); @@ -1327,14 +1337,14 @@ struct bm_extent { #endif #endif -/* BIO_MAX_SIZE is 256 * PAGE_SIZE, +/* Estimate max bio size as 256 * PAGE_SIZE, * so for typical PAGE_SIZE of 4k, that is (1<<20) Byte. * Since we may live in a mixed-platform cluster, * we limit us to a platform agnostic constant here for now. * A followup commit may allow even bigger BIO sizes, * once we thought that through. */ #define DRBD_MAX_BIO_SIZE (1U << 20) -#if DRBD_MAX_BIO_SIZE > BIO_MAX_SIZE +#if DRBD_MAX_BIO_SIZE > (BIO_MAX_PAGES << PAGE_SHIFT) #error Architecture not supported: DRBD_MAX_BIO_SIZE > BIO_MAX_SIZE #endif #define DRBD_MAX_BIO_SIZE_SAFE (1U << 12) /* Works always = 4k */ @@ -1342,11 +1352,11 @@ struct bm_extent { #define DRBD_MAX_SIZE_H80_PACKET (1U << 15) /* Header 80 only allows packets up to 32KiB data */ #define DRBD_MAX_BIO_SIZE_P95 (1U << 17) /* Protocol 95 to 99 allows bios up to 128KiB */ -/* For now, don't allow more than one activity log extent worth of data - * to be discarded in one go. We may need to rework drbd_al_begin_io() - * to allow for even larger discard ranges */ -#define DRBD_MAX_DISCARD_SIZE AL_EXTENT_SIZE -#define DRBD_MAX_DISCARD_SECTORS (DRBD_MAX_DISCARD_SIZE >> 9) +/* For now, don't allow more than half of what we can "activate" in one + * activity log transaction to be discarded in one go. We may need to rework + * drbd_al_begin_io() to allow for even larger discard ranges */ +#define DRBD_MAX_BATCH_BIO_SIZE (AL_UPDATES_PER_TRANSACTION/2*AL_EXTENT_SIZE) +#define DRBD_MAX_BBIO_SECTORS (DRBD_MAX_BATCH_BIO_SIZE >> 9) extern int drbd_bm_init(struct drbd_device *device); extern int drbd_bm_resize(struct drbd_device *device, sector_t sectors, int set_new_bits); @@ -1369,6 +1379,7 @@ extern int drbd_bm_e_weight(struct drbd_device *device, unsigned long enr); extern int drbd_bm_read(struct drbd_device *device) __must_hold(local); extern void drbd_bm_mark_for_writeout(struct drbd_device *device, int page_nr); extern int drbd_bm_write(struct drbd_device *device) __must_hold(local); +extern void drbd_bm_reset_al_hints(struct drbd_device *device) __must_hold(local); extern int drbd_bm_write_hinted(struct drbd_device *device) __must_hold(local); extern int drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_hold(local); extern int drbd_bm_write_all(struct drbd_device *device) __must_hold(local); @@ -1483,12 +1494,14 @@ enum determine_dev_size { extern enum determine_dev_size drbd_determine_dev_size(struct drbd_device *, enum dds_flags, struct resize_parms *) __must_hold(local); extern void resync_after_online_grow(struct drbd_device *); -extern void drbd_reconsider_max_bio_size(struct drbd_device *device, struct drbd_backing_dev *bdev); +extern void drbd_reconsider_queue_parameters(struct drbd_device *device, + struct drbd_backing_dev *bdev, struct o_qlim *o); extern enum drbd_state_rv drbd_set_role(struct drbd_device *device, enum drbd_role new_role, int force); extern bool conn_try_outdate_peer(struct drbd_connection *connection); extern void conn_try_outdate_peer_async(struct drbd_connection *connection); +extern enum drbd_peer_state conn_khelper(struct drbd_connection *connection, char *cmd); extern int drbd_khelper(struct drbd_device *device, char *cmd); /* drbd_worker.c */ @@ -1507,7 +1520,7 @@ extern int drbd_resync_finished(struct drbd_device *device); extern void *drbd_md_get_buffer(struct drbd_device *device, const char *intent); extern void drbd_md_put_buffer(struct drbd_device *device); extern int drbd_md_sync_page_io(struct drbd_device *device, - struct drbd_backing_dev *bdev, sector_t sector, int rw); + struct drbd_backing_dev *bdev, sector_t sector, int op); extern void drbd_ov_out_of_sync_found(struct drbd_device *, sector_t, int); extern void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_backing_dev *bdev, unsigned int *done); @@ -1548,6 +1561,8 @@ extern void start_resync_timer_fn(unsigned long data); extern void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req); /* drbd_receiver.c */ +extern int drbd_issue_discard_or_zero_out(struct drbd_device *device, + sector_t start, unsigned int nr_sectors, bool discard); extern int drbd_receiver(struct drbd_thread *thi); extern int drbd_ack_receiver(struct drbd_thread *thi); extern void drbd_send_ping_wf(struct work_struct *ws); @@ -1557,11 +1572,11 @@ extern bool drbd_rs_should_slow_down(struct drbd_device *device, sector_t sector bool throttle_if_app_is_waiting); extern int drbd_submit_peer_request(struct drbd_device *, struct drbd_peer_request *, const unsigned, - const int); + const unsigned, const int); extern int drbd_free_peer_reqs(struct drbd_device *, struct list_head *); extern struct drbd_peer_request *drbd_alloc_peer_req(struct drbd_peer_device *, u64, sector_t, unsigned int, - bool, + unsigned int, gfp_t) __must_hold(local); extern void __drbd_free_peer_req(struct drbd_device *, struct drbd_peer_request *, int); @@ -1635,8 +1650,6 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin /* drbd_proc.c */ extern struct proc_dir_entry *drbd_proc; extern const struct file_operations drbd_proc_fops; -extern const char *drbd_conn_str(enum drbd_conns s); -extern const char *drbd_role_str(enum drbd_role s); /* drbd_actlog.c */ extern bool drbd_al_begin_io_prepare(struct drbd_device *device, struct drbd_interval *i); @@ -2095,13 +2108,22 @@ static inline void _sub_unacked(struct drbd_device *device, int n, const char *f ERR_IF_CNT_IS_NEGATIVE(unacked_cnt, func, line); } +static inline bool is_sync_target_state(enum drbd_conns connection_state) +{ + return connection_state == C_SYNC_TARGET || + connection_state == C_PAUSED_SYNC_T; +} + +static inline bool is_sync_source_state(enum drbd_conns connection_state) +{ + return connection_state == C_SYNC_SOURCE || + connection_state == C_PAUSED_SYNC_S; +} + static inline bool is_sync_state(enum drbd_conns connection_state) { - return - (connection_state == C_SYNC_SOURCE - || connection_state == C_SYNC_TARGET - || connection_state == C_PAUSED_SYNC_S - || connection_state == C_PAUSED_SYNC_T); + return is_sync_source_state(connection_state) || + is_sync_target_state(connection_state); } /** diff --git a/drivers/block/drbd/drbd_interval.h b/drivers/block/drbd/drbd_interval.h index f210543f0..23c5a9442 100644 --- a/drivers/block/drbd/drbd_interval.h +++ b/drivers/block/drbd/drbd_interval.h @@ -6,13 +6,13 @@ struct drbd_interval { struct rb_node rb; - sector_t sector; /* start sector of the interval */ - unsigned int size; /* size in bytes */ - sector_t end; /* highest interval end in subtree */ - int local:1 /* local or remote request? */; - int waiting:1; /* someone is waiting for this to complete */ - int completed:1; /* this has been completed already; - * ignore for conflict detection */ + sector_t sector; /* start sector of the interval */ + unsigned int size; /* size in bytes */ + sector_t end; /* highest interval end in subtree */ + unsigned int local:1 /* local or remote request? */; + unsigned int waiting:1; /* someone is waiting for completion */ + unsigned int completed:1; /* this has been completed already; + * ignore for conflict detection */ }; static inline void drbd_clear_interval(struct drbd_interval *i) diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index 2ba1494b2..100be556e 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include #include @@ -920,6 +920,31 @@ void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device) } } +/* communicated if (agreed_features & DRBD_FF_WSAME) */ +void assign_p_sizes_qlim(struct drbd_device *device, struct p_sizes *p, struct request_queue *q) +{ + if (q) { + p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q)); + p->qlim->logical_block_size = cpu_to_be32(queue_logical_block_size(q)); + p->qlim->alignment_offset = cpu_to_be32(queue_alignment_offset(q)); + p->qlim->io_min = cpu_to_be32(queue_io_min(q)); + p->qlim->io_opt = cpu_to_be32(queue_io_opt(q)); + p->qlim->discard_enabled = blk_queue_discard(q); + p->qlim->discard_zeroes_data = queue_discard_zeroes_data(q); + p->qlim->write_same_capable = !!q->limits.max_write_same_sectors; + } else { + q = device->rq_queue; + p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q)); + p->qlim->logical_block_size = cpu_to_be32(queue_logical_block_size(q)); + p->qlim->alignment_offset = 0; + p->qlim->io_min = cpu_to_be32(queue_io_min(q)); + p->qlim->io_opt = cpu_to_be32(queue_io_opt(q)); + p->qlim->discard_enabled = 0; + p->qlim->discard_zeroes_data = 0; + p->qlim->write_same_capable = 0; + } +} + int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enum dds_flags flags) { struct drbd_device *device = peer_device->device; @@ -928,29 +953,37 @@ int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enu sector_t d_size, u_size; int q_order_type; unsigned int max_bio_size; + unsigned int packet_size; + + sock = &peer_device->connection->data; + p = drbd_prepare_command(peer_device, sock); + if (!p) + return -EIO; + packet_size = sizeof(*p); + if (peer_device->connection->agreed_features & DRBD_FF_WSAME) + packet_size += sizeof(p->qlim[0]); + + memset(p, 0, packet_size); if (get_ldev_if_state(device, D_NEGOTIATING)) { - D_ASSERT(device, device->ldev->backing_bdev); + struct request_queue *q = bdev_get_queue(device->ldev->backing_bdev); d_size = drbd_get_max_capacity(device->ldev); rcu_read_lock(); u_size = rcu_dereference(device->ldev->disk_conf)->disk_size; rcu_read_unlock(); q_order_type = drbd_queue_order_type(device); - max_bio_size = queue_max_hw_sectors(device->ldev->backing_bdev->bd_disk->queue) << 9; + max_bio_size = queue_max_hw_sectors(q) << 9; max_bio_size = min(max_bio_size, DRBD_MAX_BIO_SIZE); + assign_p_sizes_qlim(device, p, q); put_ldev(device); } else { d_size = 0; u_size = 0; q_order_type = QUEUE_ORDERED_NONE; max_bio_size = DRBD_MAX_BIO_SIZE; /* ... multiple BIOs per peer_request */ + assign_p_sizes_qlim(device, p, NULL); } - sock = &peer_device->connection->data; - p = drbd_prepare_command(peer_device, sock); - if (!p) - return -EIO; - if (peer_device->connection->agreed_pro_version <= 94) max_bio_size = min(max_bio_size, DRBD_MAX_SIZE_H80_PACKET); else if (peer_device->connection->agreed_pro_version < 100) @@ -962,7 +995,8 @@ int drbd_send_sizes(struct drbd_peer_device *peer_device, int trigger_reply, enu p->max_bio_size = cpu_to_be32(max_bio_size); p->queue_order_type = cpu_to_be16(q_order_type); p->dds_flags = cpu_to_be16(flags); - return drbd_send_command(peer_device, sock, P_SIZES, sizeof(*p), NULL, 0); + + return drbd_send_command(peer_device, sock, P_SIZES, packet_size, NULL, 0); } /** @@ -1377,6 +1411,22 @@ int drbd_send_ack_ex(struct drbd_peer_device *peer_device, enum drbd_packet cmd, cpu_to_be64(block_id)); } +int drbd_send_rs_deallocated(struct drbd_peer_device *peer_device, + struct drbd_peer_request *peer_req) +{ + struct drbd_socket *sock; + struct p_block_desc *p; + + sock = &peer_device->connection->data; + p = drbd_prepare_command(peer_device, sock); + if (!p) + return -EIO; + p->sector = cpu_to_be64(peer_req->i.sector); + p->blksize = cpu_to_be32(peer_req->i.size); + p->pad = 0; + return drbd_send_command(peer_device, sock, P_RS_DEALLOCATED, sizeof(*p), NULL, 0); +} + int drbd_send_drequest(struct drbd_peer_device *peer_device, int cmd, sector_t sector, int size, u64 block_id) { @@ -1561,6 +1611,9 @@ static int _drbd_send_bio(struct drbd_peer_device *peer_device, struct bio *bio) ? 0 : MSG_MORE); if (err) return err; + /* REQ_OP_WRITE_SAME has only one segment */ + if (bio_op(bio) == REQ_OP_WRITE_SAME) + break; } return 0; } @@ -1579,6 +1632,9 @@ static int _drbd_send_zc_bio(struct drbd_peer_device *peer_device, struct bio *b bio_iter_last(bvec, iter) ? 0 : MSG_MORE); if (err) return err; + /* REQ_OP_WRITE_SAME has only one segment */ + if (bio_op(bio) == REQ_OP_WRITE_SAME) + break; } return 0; } @@ -1603,15 +1659,17 @@ static int _drbd_send_zc_ee(struct drbd_peer_device *peer_device, return 0; } -static u32 bio_flags_to_wire(struct drbd_connection *connection, unsigned long bi_rw) +static u32 bio_flags_to_wire(struct drbd_connection *connection, + struct bio *bio) { if (connection->agreed_pro_version >= 95) - return (bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) | - (bi_rw & REQ_FUA ? DP_FUA : 0) | - (bi_rw & REQ_FLUSH ? DP_FLUSH : 0) | - (bi_rw & REQ_DISCARD ? DP_DISCARD : 0); + return (bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0) | + (bio->bi_opf & REQ_FUA ? DP_FUA : 0) | + (bio->bi_opf & REQ_PREFLUSH ? DP_FLUSH : 0) | + (bio_op(bio) == REQ_OP_WRITE_SAME ? DP_WSAME : 0) | + (bio_op(bio) == REQ_OP_DISCARD ? DP_DISCARD : 0); else - return bi_rw & REQ_SYNC ? DP_RW_SYNC : 0; + return bio->bi_opf & REQ_SYNC ? DP_RW_SYNC : 0; } /* Used to send write or TRIM aka REQ_DISCARD requests @@ -1622,6 +1680,8 @@ int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request * struct drbd_device *device = peer_device->device; struct drbd_socket *sock; struct p_data *p; + struct p_wsame *wsame = NULL; + void *digest_out; unsigned int dp_flags = 0; int digest_size; int err; @@ -1636,7 +1696,7 @@ int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request * p->sector = cpu_to_be64(req->i.sector); p->block_id = (unsigned long)req; p->seq_num = cpu_to_be32(atomic_inc_return(&device->packet_seq)); - dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio->bi_rw); + dp_flags = bio_flags_to_wire(peer_device->connection, req->master_bio); if (device->state.conn >= C_SYNC_SOURCE && device->state.conn <= C_PAUSED_SYNC_T) dp_flags |= DP_MAY_SET_IN_SYNC; @@ -1657,12 +1717,29 @@ int drbd_send_dblock(struct drbd_peer_device *peer_device, struct drbd_request * err = __send_command(peer_device->connection, device->vnr, sock, P_TRIM, sizeof(*t), NULL, 0); goto out; } + if (dp_flags & DP_WSAME) { + /* this will only work if DRBD_FF_WSAME is set AND the + * handshake agreed that all nodes and backend devices are + * WRITE_SAME capable and agree on logical_block_size */ + wsame = (struct p_wsame*)p; + digest_out = wsame + 1; + wsame->size = cpu_to_be32(req->i.size); + } else + digest_out = p + 1; /* our digest is still only over the payload. * TRIM does not carry any payload. */ if (digest_size) - drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, p + 1); - err = __send_command(peer_device->connection, device->vnr, sock, P_DATA, sizeof(*p) + digest_size, NULL, req->i.size); + drbd_csum_bio(peer_device->connection->integrity_tfm, req->master_bio, digest_out); + if (wsame) { + err = + __send_command(peer_device->connection, device->vnr, sock, P_WSAME, + sizeof(*wsame) + digest_size, NULL, + bio_iovec(req->master_bio).bv_len); + } else + err = + __send_command(peer_device->connection, device->vnr, sock, P_DATA, + sizeof(*p) + digest_size, NULL, req->i.size); if (!err) { /* For protocol A, we have to memcpy the payload into * socket buffers, as we may complete right away @@ -3061,7 +3138,7 @@ void drbd_md_write(struct drbd_device *device, void *b) D_ASSERT(device, drbd_md_ss(device->ldev) == device->ldev->md.md_offset); sector = device->ldev->md.md_offset; - if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) { + if (drbd_md_sync_page_io(device, device->ldev, sector, REQ_OP_WRITE)) { /* this was a try anyways ... */ drbd_err(device, "meta data update failed!\n"); drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR); @@ -3263,7 +3340,8 @@ int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev) * Affects the paranoia out-of-range access check in drbd_md_sync_page_io(). */ bdev->md.md_size_sect = 8; - if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset, READ)) { + if (drbd_md_sync_page_io(device, bdev, bdev->md.md_offset, + REQ_OP_READ)) { /* NOTE: can't do normal error processing here as this is called BEFORE disk is attached */ drbd_err(device, "Error while reading metadata.\n"); @@ -3505,7 +3583,12 @@ static int w_bitmap_io(struct drbd_work *w, int unused) struct bm_io_work *work = &device->bm_io_work; int rv = -EIO; - D_ASSERT(device, atomic_read(&device->ap_bio_cnt) == 0); + if (work->flags != BM_LOCKED_CHANGE_ALLOWED) { + int cnt = atomic_read(&device->ap_bio_cnt); + if (cnt) + drbd_err(device, "FIXME: ap_bio_cnt %d, expected 0; queued for '%s'\n", + cnt, work->why); + } if (get_ldev(device)) { drbd_bm_lock(device, work->why, work->flags); @@ -3585,18 +3668,20 @@ void drbd_queue_bitmap_io(struct drbd_device *device, int drbd_bitmap_io(struct drbd_device *device, int (*io_fn)(struct drbd_device *), char *why, enum bm_flag flags) { + /* Only suspend io, if some operation is supposed to be locked out */ + const bool do_suspend_io = flags & (BM_DONT_CLEAR|BM_DONT_SET|BM_DONT_TEST); int rv; D_ASSERT(device, current != first_peer_device(device)->connection->worker.task); - if ((flags & BM_LOCKED_SET_ALLOWED) == 0) + if (do_suspend_io) drbd_suspend_io(device); drbd_bm_lock(device, why, flags); rv = io_fn(device); drbd_bm_unlock(device); - if ((flags & BM_LOCKED_SET_ALLOWED) == 0) + if (do_suspend_io) drbd_resume_io(device); return rv; @@ -3635,6 +3720,8 @@ const char *cmdname(enum drbd_packet cmd) * one PRO_VERSION */ static const char *cmdnames[] = { [P_DATA] = "Data", + [P_WSAME] = "WriteSame", + [P_TRIM] = "Trim", [P_DATA_REPLY] = "DataReply", [P_RS_DATA_REPLY] = "RSDataReply", [P_BARRIER] = "Barrier", @@ -3679,6 +3766,8 @@ const char *cmdname(enum drbd_packet cmd) [P_CONN_ST_CHG_REPLY] = "conn_st_chg_reply", [P_RETRY_WRITE] = "retry_write", [P_PROTOCOL_UPDATE] = "protocol_update", + [P_RS_THIN_REQ] = "rs_thin_req", + [P_RS_DEALLOCATED] = "rs_deallocated", /* enum drbd_packet, but not commands - obsoleted flags: * P_MAY_IGNORE diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index 0bac9c824..f35db29ca 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -343,7 +343,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd) (char[20]) { }, /* address family */ (char[60]) { }, /* address */ NULL }; - char mb[12]; + char mb[14]; char *argv[] = {usermode_helper, cmd, mb, NULL }; struct drbd_connection *connection = first_peer_device(device)->connection; struct sib_info sib; @@ -352,7 +352,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd) if (current == connection->worker.task) set_bit(CALLBACK_PENDING, &connection->flags); - snprintf(mb, 12, "minor-%d", device_to_minor(device)); + snprintf(mb, 14, "minor-%d", device_to_minor(device)); setup_khelper_env(connection, envp); /* The helper may take some time. @@ -387,7 +387,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd) return ret; } -static int conn_khelper(struct drbd_connection *connection, char *cmd) +enum drbd_peer_state conn_khelper(struct drbd_connection *connection, char *cmd) { char *envp[] = { "HOME=/", "TERM=linux", @@ -442,19 +442,17 @@ static enum drbd_fencing_p highest_fencing_policy(struct drbd_connection *connec } rcu_read_unlock(); - if (fp == FP_NOT_AVAIL) { - /* IO Suspending works on the whole resource. - Do it only for one device. */ - vnr = 0; - peer_device = idr_get_next(&connection->peer_devices, &vnr); - drbd_change_state(peer_device->device, CS_VERBOSE | CS_HARD, NS(susp_fen, 0)); - } - return fp; } +static bool resource_is_supended(struct drbd_resource *resource) +{ + return resource->susp || resource->susp_fen || resource->susp_nod; +} + bool conn_try_outdate_peer(struct drbd_connection *connection) { + struct drbd_resource * const resource = connection->resource; unsigned int connect_cnt; union drbd_state mask = { }; union drbd_state val = { }; @@ -462,21 +460,41 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) char *ex_to_string; int r; - spin_lock_irq(&connection->resource->req_lock); + spin_lock_irq(&resource->req_lock); if (connection->cstate >= C_WF_REPORT_PARAMS) { drbd_err(connection, "Expected cstate < C_WF_REPORT_PARAMS\n"); - spin_unlock_irq(&connection->resource->req_lock); + spin_unlock_irq(&resource->req_lock); return false; } connect_cnt = connection->connect_cnt; - spin_unlock_irq(&connection->resource->req_lock); + spin_unlock_irq(&resource->req_lock); fp = highest_fencing_policy(connection); switch (fp) { case FP_NOT_AVAIL: drbd_warn(connection, "Not fencing peer, I'm not even Consistent myself.\n"); - goto out; + spin_lock_irq(&resource->req_lock); + if (connection->cstate < C_WF_REPORT_PARAMS) { + _conn_request_state(connection, + (union drbd_state) { { .susp_fen = 1 } }, + (union drbd_state) { { .susp_fen = 0 } }, + CS_VERBOSE | CS_HARD | CS_DC_SUSP); + /* We are no longer suspended due to the fencing policy. + * We may still be suspended due to the on-no-data-accessible policy. + * If that was OND_IO_ERROR, fail pending requests. */ + if (!resource_is_supended(resource)) + _tl_restart(connection, CONNECTION_LOST_WHILE_PENDING); + } + /* Else: in case we raced with a connection handshake, + * let the handshake figure out if we maybe can RESEND, + * and do not resume/fail pending requests here. + * Worst case is we stay suspended for now, which may be + * resolved by either re-establishing the replication link, or + * the next link failure, or eventually the administrator. */ + spin_unlock_irq(&resource->req_lock); + return false; + case FP_DONT_CARE: return true; default: ; @@ -485,17 +503,17 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) r = conn_khelper(connection, "fence-peer"); switch ((r>>8) & 0xff) { - case 3: /* peer is inconsistent */ + case P_INCONSISTENT: /* peer is inconsistent */ ex_to_string = "peer is inconsistent or worse"; mask.pdsk = D_MASK; val.pdsk = D_INCONSISTENT; break; - case 4: /* peer got outdated, or was already outdated */ + case P_OUTDATED: /* peer got outdated, or was already outdated */ ex_to_string = "peer was fenced"; mask.pdsk = D_MASK; val.pdsk = D_OUTDATED; break; - case 5: /* peer was down */ + case P_DOWN: /* peer was down */ if (conn_highest_disk(connection) == D_UP_TO_DATE) { /* we will(have) create(d) a new UUID anyways... */ ex_to_string = "peer is unreachable, assumed to be dead"; @@ -505,7 +523,7 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) ex_to_string = "peer unreachable, doing nothing since disk != UpToDate"; } break; - case 6: /* Peer is primary, voluntarily outdate myself. + case P_PRIMARY: /* Peer is primary, voluntarily outdate myself. * This is useful when an unconnected R_SECONDARY is asked to * become R_PRIMARY, but finds the other peer being active. */ ex_to_string = "peer is active"; @@ -513,7 +531,9 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) mask.disk = D_MASK; val.disk = D_OUTDATED; break; - case 7: + case P_FENCING: + /* THINK: do we need to handle this + * like case 4, or more like case 5? */ if (fp != FP_STONITH) drbd_err(connection, "fence-peer() = 7 && fencing != Stonith !!!\n"); ex_to_string = "peer was stonithed"; @@ -529,13 +549,11 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) drbd_info(connection, "fence-peer helper returned %d (%s)\n", (r>>8) & 0xff, ex_to_string); - out: - /* Not using conn_request_state(connection, mask, val, CS_VERBOSE); here, because we might were able to re-establish the connection in the meantime. */ - spin_lock_irq(&connection->resource->req_lock); + spin_lock_irq(&resource->req_lock); if (connection->cstate < C_WF_REPORT_PARAMS && !test_bit(STATE_SENT, &connection->flags)) { if (connection->connect_cnt != connect_cnt) /* In case the connection was established and droped @@ -544,7 +562,7 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) else _conn_request_state(connection, mask, val, CS_VERBOSE); } - spin_unlock_irq(&connection->resource->req_lock); + spin_unlock_irq(&resource->req_lock); return conn_highest_pdsk(connection) <= D_OUTDATED; } @@ -1154,51 +1172,160 @@ static int drbd_check_al_size(struct drbd_device *device, struct disk_conf *dc) return 0; } +static void blk_queue_discard_granularity(struct request_queue *q, unsigned int granularity) +{ + q->limits.discard_granularity = granularity; +} + +static unsigned int drbd_max_discard_sectors(struct drbd_connection *connection) +{ + /* when we introduced REQ_WRITE_SAME support, we also bumped + * our maximum supported batch bio size used for discards. */ + if (connection->agreed_features & DRBD_FF_WSAME) + return DRBD_MAX_BBIO_SECTORS; + /* before, with DRBD <= 8.4.6, we only allowed up to one AL_EXTENT_SIZE. */ + return AL_EXTENT_SIZE >> 9; +} + +static void decide_on_discard_support(struct drbd_device *device, + struct request_queue *q, + struct request_queue *b, + bool discard_zeroes_if_aligned) +{ + /* q = drbd device queue (device->rq_queue) + * b = backing device queue (device->ldev->backing_bdev->bd_disk->queue), + * or NULL if diskless + */ + struct drbd_connection *connection = first_peer_device(device)->connection; + bool can_do = b ? blk_queue_discard(b) : true; + + if (can_do && b && !b->limits.discard_zeroes_data && !discard_zeroes_if_aligned) { + can_do = false; + drbd_info(device, "discard_zeroes_data=0 and discard_zeroes_if_aligned=no: disabling discards\n"); + } + if (can_do && connection->cstate >= C_CONNECTED && !(connection->agreed_features & DRBD_FF_TRIM)) { + can_do = false; + drbd_info(connection, "peer DRBD too old, does not support TRIM: disabling discards\n"); + } + if (can_do) { + /* We don't care for the granularity, really. + * Stacking limits below should fix it for the local + * device. Whether or not it is a suitable granularity + * on the remote device is not our problem, really. If + * you care, you need to use devices with similar + * topology on all peers. */ + blk_queue_discard_granularity(q, 512); + q->limits.max_discard_sectors = drbd_max_discard_sectors(connection); + queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); + } else { + queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); + blk_queue_discard_granularity(q, 0); + q->limits.max_discard_sectors = 0; + } +} + +static void fixup_discard_if_not_supported(struct request_queue *q) +{ + /* To avoid confusion, if this queue does not support discard, clear + * max_discard_sectors, which is what lsblk -D reports to the user. + * Older kernels got this wrong in "stack limits". + * */ + if (!blk_queue_discard(q)) { + blk_queue_max_discard_sectors(q, 0); + blk_queue_discard_granularity(q, 0); + } +} + +static void decide_on_write_same_support(struct drbd_device *device, + struct request_queue *q, + struct request_queue *b, struct o_qlim *o) +{ + struct drbd_peer_device *peer_device = first_peer_device(device); + struct drbd_connection *connection = peer_device->connection; + bool can_do = b ? b->limits.max_write_same_sectors : true; + + if (can_do && connection->cstate >= C_CONNECTED && !(connection->agreed_features & DRBD_FF_WSAME)) { + can_do = false; + drbd_info(peer_device, "peer does not support WRITE_SAME\n"); + } + + if (o) { + /* logical block size; queue_logical_block_size(NULL) is 512 */ + unsigned int peer_lbs = be32_to_cpu(o->logical_block_size); + unsigned int me_lbs_b = queue_logical_block_size(b); + unsigned int me_lbs = queue_logical_block_size(q); + + if (me_lbs_b != me_lbs) { + drbd_warn(device, + "logical block size of local backend does not match (drbd:%u, backend:%u); was this a late attach?\n", + me_lbs, me_lbs_b); + /* rather disable write same than trigger some BUG_ON later in the scsi layer. */ + can_do = false; + } + if (me_lbs_b != peer_lbs) { + drbd_warn(peer_device, "logical block sizes do not match (me:%u, peer:%u); this may cause problems.\n", + me_lbs, peer_lbs); + if (can_do) { + drbd_dbg(peer_device, "logical block size mismatch: WRITE_SAME disabled.\n"); + can_do = false; + } + me_lbs = max(me_lbs, me_lbs_b); + /* We cannot change the logical block size of an in-use queue. + * We can only hope that access happens to be properly aligned. + * If not, the peer will likely produce an IO error, and detach. */ + if (peer_lbs > me_lbs) { + if (device->state.role != R_PRIMARY) { + blk_queue_logical_block_size(q, peer_lbs); + drbd_warn(peer_device, "logical block size set to %u\n", peer_lbs); + } else { + drbd_warn(peer_device, + "current Primary must NOT adjust logical block size (%u -> %u); hope for the best.\n", + me_lbs, peer_lbs); + } + } + } + if (can_do && !o->write_same_capable) { + /* If we introduce an open-coded write-same loop on the receiving side, + * the peer would present itself as "capable". */ + drbd_dbg(peer_device, "WRITE_SAME disabled (peer device not capable)\n"); + can_do = false; + } + } + + blk_queue_max_write_same_sectors(q, can_do ? DRBD_MAX_BBIO_SECTORS : 0); +} + static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backing_dev *bdev, - unsigned int max_bio_size) + unsigned int max_bio_size, struct o_qlim *o) { struct request_queue * const q = device->rq_queue; unsigned int max_hw_sectors = max_bio_size >> 9; unsigned int max_segments = 0; struct request_queue *b = NULL; + struct disk_conf *dc; + bool discard_zeroes_if_aligned = true; if (bdev) { b = bdev->backing_bdev->bd_disk->queue; max_hw_sectors = min(queue_max_hw_sectors(b), max_bio_size >> 9); rcu_read_lock(); - max_segments = rcu_dereference(device->ldev->disk_conf)->max_bio_bvecs; + dc = rcu_dereference(device->ldev->disk_conf); + max_segments = dc->max_bio_bvecs; + discard_zeroes_if_aligned = dc->discard_zeroes_if_aligned; rcu_read_unlock(); blk_set_stacking_limits(&q->limits); - blk_queue_max_write_same_sectors(q, 0); } - blk_queue_logical_block_size(q, 512); blk_queue_max_hw_sectors(q, max_hw_sectors); /* This is the workaround for "bio would need to, but cannot, be split" */ blk_queue_max_segments(q, max_segments ? max_segments : BLK_MAX_SEGMENTS); blk_queue_segment_boundary(q, PAGE_SIZE-1); + decide_on_discard_support(device, q, b, discard_zeroes_if_aligned); + decide_on_write_same_support(device, q, b, o); if (b) { - struct drbd_connection *connection = first_peer_device(device)->connection; - - blk_queue_max_discard_sectors(q, DRBD_MAX_DISCARD_SECTORS); - - if (blk_queue_discard(b) && - (connection->cstate < C_CONNECTED || connection->agreed_features & FF_TRIM)) { - /* We don't care, stacking below should fix it for the local device. - * Whether or not it is a suitable granularity on the remote device - * is not our problem, really. If you care, you need to - * use devices with similar topology on all peers. */ - q->limits.discard_granularity = 512; - queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); - } else { - blk_queue_max_discard_sectors(q, 0); - queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); - q->limits.discard_granularity = 0; - } - blk_queue_stack_limits(q, b); if (q->backing_dev_info.ra_pages != b->backing_dev_info.ra_pages) { @@ -1208,15 +1335,10 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi q->backing_dev_info.ra_pages = b->backing_dev_info.ra_pages; } } - /* To avoid confusion, if this queue does not support discard, clear - * max_discard_sectors, which is what lsblk -D reports to the user. */ - if (!blk_queue_discard(q)) { - blk_queue_max_discard_sectors(q, 0); - q->limits.discard_granularity = 0; - } + fixup_discard_if_not_supported(q); } -void drbd_reconsider_max_bio_size(struct drbd_device *device, struct drbd_backing_dev *bdev) +void drbd_reconsider_queue_parameters(struct drbd_device *device, struct drbd_backing_dev *bdev, struct o_qlim *o) { unsigned int now, new, local, peer; @@ -1259,7 +1381,7 @@ void drbd_reconsider_max_bio_size(struct drbd_device *device, struct drbd_backin if (new != now) drbd_info(device, "max BIO size = %u\n", new); - drbd_setup_queue_param(device, bdev, new); + drbd_setup_queue_param(device, bdev, new, o); } /* Starts the worker thread */ @@ -1348,6 +1470,43 @@ static bool write_ordering_changed(struct disk_conf *a, struct disk_conf *b) a->disk_drain != b->disk_drain; } +static void sanitize_disk_conf(struct drbd_device *device, struct disk_conf *disk_conf, + struct drbd_backing_dev *nbc) +{ + struct request_queue * const q = nbc->backing_bdev->bd_disk->queue; + + if (disk_conf->al_extents < DRBD_AL_EXTENTS_MIN) + disk_conf->al_extents = DRBD_AL_EXTENTS_MIN; + if (disk_conf->al_extents > drbd_al_extents_max(nbc)) + disk_conf->al_extents = drbd_al_extents_max(nbc); + + if (!blk_queue_discard(q) + || (!q->limits.discard_zeroes_data && !disk_conf->discard_zeroes_if_aligned)) { + if (disk_conf->rs_discard_granularity) { + disk_conf->rs_discard_granularity = 0; /* disable feature */ + drbd_info(device, "rs_discard_granularity feature disabled\n"); + } + } + + if (disk_conf->rs_discard_granularity) { + int orig_value = disk_conf->rs_discard_granularity; + int remainder; + + if (q->limits.discard_granularity > disk_conf->rs_discard_granularity) + disk_conf->rs_discard_granularity = q->limits.discard_granularity; + + remainder = disk_conf->rs_discard_granularity % q->limits.discard_granularity; + disk_conf->rs_discard_granularity += remainder; + + if (disk_conf->rs_discard_granularity > q->limits.max_discard_sectors << 9) + disk_conf->rs_discard_granularity = q->limits.max_discard_sectors << 9; + + if (disk_conf->rs_discard_granularity != orig_value) + drbd_info(device, "rs_discard_granularity changed to %d\n", + disk_conf->rs_discard_granularity); + } +} + int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) { struct drbd_config_context adm_ctx; @@ -1395,10 +1554,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) if (!expect(new_disk_conf->resync_rate >= 1)) new_disk_conf->resync_rate = 1; - if (new_disk_conf->al_extents < DRBD_AL_EXTENTS_MIN) - new_disk_conf->al_extents = DRBD_AL_EXTENTS_MIN; - if (new_disk_conf->al_extents > drbd_al_extents_max(device->ldev)) - new_disk_conf->al_extents = drbd_al_extents_max(device->ldev); + sanitize_disk_conf(device, new_disk_conf, device->ldev); if (new_disk_conf->c_plan_ahead > DRBD_C_PLAN_AHEAD_MAX) new_disk_conf->c_plan_ahead = DRBD_C_PLAN_AHEAD_MAX; @@ -1457,6 +1613,9 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) if (write_ordering_changed(old_disk_conf, new_disk_conf)) drbd_bump_write_ordering(device->resource, NULL, WO_BDEV_FLUSH); + if (old_disk_conf->discard_zeroes_if_aligned != new_disk_conf->discard_zeroes_if_aligned) + drbd_reconsider_queue_parameters(device, device->ldev, NULL); + drbd_md_sync(device); if (device->state.conn >= C_CONNECTED) { @@ -1693,10 +1852,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) if (retcode != NO_ERROR) goto fail; - if (new_disk_conf->al_extents < DRBD_AL_EXTENTS_MIN) - new_disk_conf->al_extents = DRBD_AL_EXTENTS_MIN; - if (new_disk_conf->al_extents > drbd_al_extents_max(nbc)) - new_disk_conf->al_extents = drbd_al_extents_max(nbc); + sanitize_disk_conf(device, new_disk_conf, nbc); if (drbd_get_max_capacity(nbc) < new_disk_conf->disk_size) { drbd_err(device, "max capacity %llu smaller than disk size %llu\n", @@ -1838,7 +1994,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) device->read_cnt = 0; device->writ_cnt = 0; - drbd_reconsider_max_bio_size(device, device->ldev); + drbd_reconsider_queue_parameters(device, device->ldev, NULL); /* If I am currently not R_PRIMARY, * but meta data primary indicator is set, diff --git a/drivers/block/drbd/drbd_proc.c b/drivers/block/drbd/drbd_proc.c index 6537b25db..be2b93fd2 100644 --- a/drivers/block/drbd/drbd_proc.c +++ b/drivers/block/drbd/drbd_proc.c @@ -25,7 +25,7 @@ #include -#include +#include #include #include #include @@ -122,18 +122,18 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se x = res/50; y = 20-x; - seq_printf(seq, "\t["); + seq_puts(seq, "\t["); for (i = 1; i < x; i++) - seq_printf(seq, "="); - seq_printf(seq, ">"); + seq_putc(seq, '='); + seq_putc(seq, '>'); for (i = 0; i < y; i++) seq_printf(seq, "."); - seq_printf(seq, "] "); + seq_puts(seq, "] "); if (state.conn == C_VERIFY_S || state.conn == C_VERIFY_T) - seq_printf(seq, "verified:"); + seq_puts(seq, "verified:"); else - seq_printf(seq, "sync'ed:"); + seq_puts(seq, "sync'ed:"); seq_printf(seq, "%3u.%u%% ", res / 10, res % 10); /* if more than a few GB, display in MB */ @@ -146,7 +146,7 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se (unsigned long) Bit2KB(rs_left), (unsigned long) Bit2KB(rs_total)); - seq_printf(seq, "\n\t"); + seq_puts(seq, "\n\t"); /* see drivers/md/md.c * We do not want to overflow, so the order of operands and @@ -175,9 +175,9 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se rt / 3600, (rt % 3600) / 60, rt % 60); dbdt = Bit2KB(db/dt); - seq_printf(seq, " speed: "); + seq_puts(seq, " speed: "); seq_printf_with_thousands_grouping(seq, dbdt); - seq_printf(seq, " ("); + seq_puts(seq, " ("); /* ------------------------- ~3s average ------------------------ */ if (proc_details >= 1) { /* this is what drbd_rs_should_slow_down() uses */ @@ -188,7 +188,7 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se db = device->rs_mark_left[i] - rs_left; dbdt = Bit2KB(db/dt); seq_printf_with_thousands_grouping(seq, dbdt); - seq_printf(seq, " -- "); + seq_puts(seq, " -- "); } /* --------------------- long term average ---------------------- */ @@ -200,11 +200,11 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se db = rs_total - rs_left; dbdt = Bit2KB(db/dt); seq_printf_with_thousands_grouping(seq, dbdt); - seq_printf(seq, ")"); + seq_putc(seq, ')'); if (state.conn == C_SYNC_TARGET || state.conn == C_VERIFY_S) { - seq_printf(seq, " want: "); + seq_puts(seq, " want: "); seq_printf_with_thousands_grouping(seq, device->c_sync_rate); } seq_printf(seq, " K/sec%s\n", stalled ? " (stalled)" : ""); @@ -231,7 +231,7 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se (unsigned long long)bm_bits * BM_SECT_PER_BIT); if (stop_sector != 0 && stop_sector != ULLONG_MAX) seq_printf(seq, " stop sector: %llu", stop_sector); - seq_printf(seq, "\n"); + seq_putc(seq, '\n'); } } @@ -276,7 +276,7 @@ static int drbd_seq_show(struct seq_file *seq, void *v) rcu_read_lock(); idr_for_each_entry(&drbd_devices, device, i) { if (prev_i != i - 1) - seq_printf(seq, "\n"); + seq_putc(seq, '\n'); prev_i = i; state = device->state; diff --git a/drivers/block/drbd/drbd_protocol.h b/drivers/block/drbd/drbd_protocol.h index ef9245363..4d296800f 100644 --- a/drivers/block/drbd/drbd_protocol.h +++ b/drivers/block/drbd/drbd_protocol.h @@ -60,6 +60,15 @@ enum drbd_packet { * which is why I chose TRIM here, to disambiguate. */ P_TRIM = 0x31, + /* Only use these two if both support FF_THIN_RESYNC */ + P_RS_THIN_REQ = 0x32, /* Request a block for resync or reply P_RS_DEALLOCATED */ + P_RS_DEALLOCATED = 0x33, /* Contains only zeros on sync source node */ + + /* REQ_WRITE_SAME. + * On a receiving side without REQ_WRITE_SAME, + * we may fall back to an opencoded loop instead. */ + P_WSAME = 0x34, + P_MAY_IGNORE = 0x100, /* Flag to test if (cmd > P_MAY_IGNORE) ... */ P_MAX_OPT_CMD = 0x101, @@ -106,16 +115,20 @@ struct p_header100 { u32 pad; } __packed; -/* these defines must not be changed without changing the protocol version */ -#define DP_HARDBARRIER 1 /* depricated */ +/* These defines must not be changed without changing the protocol version. + * New defines may only be introduced together with protocol version bump or + * new protocol feature flags. + */ +#define DP_HARDBARRIER 1 /* no longer used */ #define DP_RW_SYNC 2 /* equals REQ_SYNC */ #define DP_MAY_SET_IN_SYNC 4 #define DP_UNPLUG 8 /* not used anymore */ #define DP_FUA 16 /* equals REQ_FUA */ -#define DP_FLUSH 32 /* equals REQ_FLUSH */ +#define DP_FLUSH 32 /* equals REQ_PREFLUSH */ #define DP_DISCARD 64 /* equals REQ_DISCARD */ #define DP_SEND_RECEIVE_ACK 128 /* This is a proto B write request */ #define DP_SEND_WRITE_ACK 256 /* This is a proto C write request */ +#define DP_WSAME 512 /* equiv. REQ_WRITE_SAME */ struct p_data { u64 sector; /* 64 bits sector number */ @@ -129,6 +142,11 @@ struct p_trim { u32 size; /* == bio->bi_size */ } __packed; +struct p_wsame { + struct p_data p_data; + u32 size; /* == bio->bi_size */ +} __packed; + /* * commands which share a struct: * p_block_ack: @@ -160,7 +178,23 @@ struct p_block_req { * ReportParams */ -#define FF_TRIM 1 +/* supports TRIM/DISCARD on the "wire" protocol */ +#define DRBD_FF_TRIM 1 + +/* Detect all-zeros during resync, and rather TRIM/UNMAP/DISCARD those blocks + * instead of fully allocate a supposedly thin volume on initial resync */ +#define DRBD_FF_THIN_RESYNC 2 + +/* supports REQ_WRITE_SAME on the "wire" protocol. + * Note: this flag is overloaded, + * its presence also + * - indicates support for 128 MiB "batch bios", + * max discard size of 128 MiB + * instead of 4M before that. + * - indicates that we exchange additional settings in p_sizes + * drbd_send_sizes()/receive_sizes() + */ +#define DRBD_FF_WSAME 4 struct p_connection_features { u32 protocol_min; @@ -235,6 +269,40 @@ struct p_rs_uuid { u64 uuid; } __packed; +/* optional queue_limits if (agreed_features & DRBD_FF_WSAME) + * see also struct queue_limits, as of late 2015 */ +struct o_qlim { + /* we don't need it yet, but we may as well communicate it now */ + u32 physical_block_size; + + /* so the original in struct queue_limits is unsigned short, + * but I'd have to put in padding anyways. */ + u32 logical_block_size; + + /* One incoming bio becomes one DRBD request, + * which may be translated to several bio on the receiving side. + * We don't need to communicate chunk/boundary/segment ... limits. + */ + + /* various IO hints may be useful with "diskless client" setups */ + u32 alignment_offset; + u32 io_min; + u32 io_opt; + + /* We may need to communicate integrity stuff at some point, + * but let's not get ahead of ourselves. */ + + /* Backend discard capabilities. + * Receiving side uses "blkdev_issue_discard()", no need to communicate + * more specifics. If the backend cannot do discards, the DRBD peer + * may fall back to blkdev_issue_zeroout(). + */ + u8 discard_enabled; + u8 discard_zeroes_data; + u8 write_same_capable; + u8 _pad; +} __packed; + struct p_sizes { u64 d_size; /* size of disk */ u64 u_size; /* user requested size */ @@ -242,6 +310,9 @@ struct p_sizes { u32 max_bio_size; /* Maximal size of a BIO */ u16 queue_order_type; /* not yet implemented in DRBD*/ u16 dds_flags; /* use enum dds_flags here. */ + + /* optional queue_limits if (agreed_features & DRBD_FF_WSAME) */ + struct o_qlim qlim[0]; } __packed; struct p_state { diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index 050aaa1c0..942384f34 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -25,7 +25,7 @@ #include -#include +#include #include #include @@ -48,7 +48,7 @@ #include "drbd_req.h" #include "drbd_vli.h" -#define PRO_FEATURES (FF_TRIM) +#define PRO_FEATURES (DRBD_FF_TRIM|DRBD_FF_THIN_RESYNC|DRBD_FF_WSAME) struct packet_info { enum drbd_packet cmd; @@ -361,14 +361,17 @@ You must not have the req_lock: drbd_wait_ee_list_empty() */ +/* normal: payload_size == request size (bi_size) + * w_same: payload_size == logical_block_size + * trim: payload_size == 0 */ struct drbd_peer_request * drbd_alloc_peer_req(struct drbd_peer_device *peer_device, u64 id, sector_t sector, - unsigned int data_size, bool has_payload, gfp_t gfp_mask) __must_hold(local) + unsigned int request_size, unsigned int payload_size, gfp_t gfp_mask) __must_hold(local) { struct drbd_device *device = peer_device->device; struct drbd_peer_request *peer_req; struct page *page = NULL; - unsigned nr_pages = (data_size + PAGE_SIZE -1) >> PAGE_SHIFT; + unsigned nr_pages = (payload_size + PAGE_SIZE -1) >> PAGE_SHIFT; if (drbd_insert_fault(device, DRBD_FAULT_AL_EE)) return NULL; @@ -380,7 +383,7 @@ drbd_alloc_peer_req(struct drbd_peer_device *peer_device, u64 id, sector_t secto return NULL; } - if (has_payload && data_size) { + if (nr_pages) { page = drbd_alloc_pages(peer_device, nr_pages, gfpflags_allow_blocking(gfp_mask)); if (!page) @@ -390,7 +393,7 @@ drbd_alloc_peer_req(struct drbd_peer_device *peer_device, u64 id, sector_t secto memset(peer_req, 0, sizeof(*peer_req)); INIT_LIST_HEAD(&peer_req->w.list); drbd_clear_interval(&peer_req->i); - peer_req->i.size = data_size; + peer_req->i.size = request_size; peer_req->i.sector = sector; peer_req->submit_jif = jiffies; peer_req->peer_device = peer_device; @@ -1204,13 +1207,84 @@ static int drbd_recv_header(struct drbd_connection *connection, struct packet_in return err; } -static void drbd_flush(struct drbd_connection *connection) +/* This is blkdev_issue_flush, but asynchronous. + * We want to submit to all component volumes in parallel, + * then wait for all completions. + */ +struct issue_flush_context { + atomic_t pending; + int error; + struct completion done; +}; +struct one_flush_context { + struct drbd_device *device; + struct issue_flush_context *ctx; +}; + +void one_flush_endio(struct bio *bio) { - int rv; - struct drbd_peer_device *peer_device; - int vnr; + struct one_flush_context *octx = bio->bi_private; + struct drbd_device *device = octx->device; + struct issue_flush_context *ctx = octx->ctx; + + if (bio->bi_error) { + ctx->error = bio->bi_error; + drbd_info(device, "local disk FLUSH FAILED with status %d\n", bio->bi_error); + } + kfree(octx); + bio_put(bio); + + clear_bit(FLUSH_PENDING, &device->flags); + put_ldev(device); + kref_put(&device->kref, drbd_destroy_device); + + if (atomic_dec_and_test(&ctx->pending)) + complete(&ctx->done); +} + +static void submit_one_flush(struct drbd_device *device, struct issue_flush_context *ctx) +{ + struct bio *bio = bio_alloc(GFP_NOIO, 0); + struct one_flush_context *octx = kmalloc(sizeof(*octx), GFP_NOIO); + if (!bio || !octx) { + drbd_warn(device, "Could not allocate a bio, CANNOT ISSUE FLUSH\n"); + /* FIXME: what else can I do now? disconnecting or detaching + * really does not help to improve the state of the world, either. + */ + kfree(octx); + if (bio) + bio_put(bio); + + ctx->error = -ENOMEM; + put_ldev(device); + kref_put(&device->kref, drbd_destroy_device); + return; + } + octx->device = device; + octx->ctx = ctx; + bio->bi_bdev = device->ldev->backing_bdev; + bio->bi_private = octx; + bio->bi_end_io = one_flush_endio; + bio_set_op_attrs(bio, REQ_OP_FLUSH, WRITE_FLUSH); + + device->flush_jif = jiffies; + set_bit(FLUSH_PENDING, &device->flags); + atomic_inc(&ctx->pending); + submit_bio(bio); +} + +static void drbd_flush(struct drbd_connection *connection) +{ if (connection->resource->write_ordering >= WO_BDEV_FLUSH) { + struct drbd_peer_device *peer_device; + struct issue_flush_context ctx; + int vnr; + + atomic_set(&ctx.pending, 1); + ctx.error = 0; + init_completion(&ctx.done); + rcu_read_lock(); idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { struct drbd_device *device = peer_device->device; @@ -1220,31 +1294,24 @@ static void drbd_flush(struct drbd_connection *connection) kref_get(&device->kref); rcu_read_unlock(); - /* Right now, we have only this one synchronous code path - * for flushes between request epochs. - * We may want to make those asynchronous, - * or at least parallelize the flushes to the volume devices. - */ - device->flush_jif = jiffies; - set_bit(FLUSH_PENDING, &device->flags); - rv = blkdev_issue_flush(device->ldev->backing_bdev, - GFP_NOIO, NULL); - clear_bit(FLUSH_PENDING, &device->flags); - if (rv) { - drbd_info(device, "local disk flush failed with status %d\n", rv); - /* would rather check on EOPNOTSUPP, but that is not reliable. - * don't try again for ANY return value != 0 - * if (rv == -EOPNOTSUPP) */ - drbd_bump_write_ordering(connection->resource, NULL, WO_DRAIN_IO); - } - put_ldev(device); - kref_put(&device->kref, drbd_destroy_device); + submit_one_flush(device, &ctx); rcu_read_lock(); - if (rv) - break; } rcu_read_unlock(); + + /* Do we want to add a timeout, + * if disk-timeout is set? */ + if (!atomic_dec_and_test(&ctx.pending)) + wait_for_completion(&ctx.done); + + if (ctx.error) { + /* would rather check on EOPNOTSUPP, but that is not reliable. + * don't try again for ANY return value != 0 + * if (rv == -EOPNOTSUPP) */ + /* Any error is already reported by bio_endio callback. */ + drbd_bump_write_ordering(connection->resource, NULL, WO_DRAIN_IO); + } } } @@ -1379,11 +1446,125 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin drbd_info(resource, "Method to ensure write ordering: %s\n", write_ordering_str[resource->write_ordering]); } +/* + * We *may* ignore the discard-zeroes-data setting, if so configured. + * + * Assumption is that it "discard_zeroes_data=0" is only because the backend + * may ignore partial unaligned discards. + * + * LVM/DM thin as of at least + * LVM version: 2.02.115(2)-RHEL7 (2015-01-28) + * Library version: 1.02.93-RHEL7 (2015-01-28) + * Driver version: 4.29.0 + * still behaves this way. + * + * For unaligned (wrt. alignment and granularity) or too small discards, + * we zero-out the initial (and/or) trailing unaligned partial chunks, + * but discard all the aligned full chunks. + * + * At least for LVM/DM thin, the result is effectively "discard_zeroes_data=1". + */ +int drbd_issue_discard_or_zero_out(struct drbd_device *device, sector_t start, unsigned int nr_sectors, bool discard) +{ + struct block_device *bdev = device->ldev->backing_bdev; + struct request_queue *q = bdev_get_queue(bdev); + sector_t tmp, nr; + unsigned int max_discard_sectors, granularity; + int alignment; + int err = 0; + + if (!discard) + goto zero_out; + + /* Zero-sector (unknown) and one-sector granularities are the same. */ + granularity = max(q->limits.discard_granularity >> 9, 1U); + alignment = (bdev_discard_alignment(bdev) >> 9) % granularity; + + max_discard_sectors = min(q->limits.max_discard_sectors, (1U << 22)); + max_discard_sectors -= max_discard_sectors % granularity; + if (unlikely(!max_discard_sectors)) + goto zero_out; + + if (nr_sectors < granularity) + goto zero_out; + + tmp = start; + if (sector_div(tmp, granularity) != alignment) { + if (nr_sectors < 2*granularity) + goto zero_out; + /* start + gran - (start + gran - align) % gran */ + tmp = start + granularity - alignment; + tmp = start + granularity - sector_div(tmp, granularity); + + nr = tmp - start; + err |= blkdev_issue_zeroout(bdev, start, nr, GFP_NOIO, 0); + nr_sectors -= nr; + start = tmp; + } + while (nr_sectors >= granularity) { + nr = min_t(sector_t, nr_sectors, max_discard_sectors); + err |= blkdev_issue_discard(bdev, start, nr, GFP_NOIO, 0); + nr_sectors -= nr; + start += nr; + } + zero_out: + if (nr_sectors) { + err |= blkdev_issue_zeroout(bdev, start, nr_sectors, GFP_NOIO, 0); + } + return err != 0; +} + +static bool can_do_reliable_discards(struct drbd_device *device) +{ + struct request_queue *q = bdev_get_queue(device->ldev->backing_bdev); + struct disk_conf *dc; + bool can_do; + + if (!blk_queue_discard(q)) + return false; + + if (q->limits.discard_zeroes_data) + return true; + + rcu_read_lock(); + dc = rcu_dereference(device->ldev->disk_conf); + can_do = dc->discard_zeroes_if_aligned; + rcu_read_unlock(); + return can_do; +} + +static void drbd_issue_peer_discard(struct drbd_device *device, struct drbd_peer_request *peer_req) +{ + /* If the backend cannot discard, or does not guarantee + * read-back zeroes in discarded ranges, we fall back to + * zero-out. Unless configuration specifically requested + * otherwise. */ + if (!can_do_reliable_discards(device)) + peer_req->flags |= EE_IS_TRIM_USE_ZEROOUT; + + if (drbd_issue_discard_or_zero_out(device, peer_req->i.sector, + peer_req->i.size >> 9, !(peer_req->flags & EE_IS_TRIM_USE_ZEROOUT))) + peer_req->flags |= EE_WAS_ERROR; + drbd_endio_write_sec_final(peer_req); +} + +static void drbd_issue_peer_wsame(struct drbd_device *device, + struct drbd_peer_request *peer_req) +{ + struct block_device *bdev = device->ldev->backing_bdev; + sector_t s = peer_req->i.sector; + sector_t nr = peer_req->i.size >> 9; + if (blkdev_issue_write_same(bdev, s, nr, GFP_NOIO, peer_req->pages)) + peer_req->flags |= EE_WAS_ERROR; + drbd_endio_write_sec_final(peer_req); +} + + /** * drbd_submit_peer_request() * @device: DRBD device. * @peer_req: peer request - * @rw: flag field, see bio->bi_rw + * @rw: flag field, see bio->bi_opf * * May spread the pages to multiple bios, * depending on bio_add_page restrictions. @@ -1398,7 +1579,8 @@ void drbd_bump_write_ordering(struct drbd_resource *resource, struct drbd_backin /* TODO allocate from our own bio_set. */ int drbd_submit_peer_request(struct drbd_device *device, struct drbd_peer_request *peer_req, - const unsigned rw, const int fault_type) + const unsigned op, const unsigned op_flags, + const int fault_type) { struct bio *bios = NULL; struct bio *bio; @@ -1409,7 +1591,13 @@ int drbd_submit_peer_request(struct drbd_device *device, unsigned nr_pages = (data_size + PAGE_SIZE -1) >> PAGE_SHIFT; int err = -ENOMEM; - if (peer_req->flags & EE_IS_TRIM_USE_ZEROOUT) { + /* TRIM/DISCARD: for now, always use the helper function + * blkdev_issue_zeroout(..., discard=true). + * It's synchronous, but it does the right thing wrt. bio splitting. + * Correctness first, performance later. Next step is to code an + * asynchronous variant of the same. + */ + if (peer_req->flags & (EE_IS_TRIM|EE_WRITE_SAME)) { /* wait for all pending IO completions, before we start * zeroing things out. */ conn_wait_active_ee_empty(peer_req->peer_device->connection); @@ -1417,22 +1605,22 @@ int drbd_submit_peer_request(struct drbd_device *device, * so we can find it to present it in debugfs */ peer_req->submit_jif = jiffies; peer_req->flags |= EE_SUBMITTED; - spin_lock_irq(&device->resource->req_lock); - list_add_tail(&peer_req->w.list, &device->active_ee); - spin_unlock_irq(&device->resource->req_lock); - if (blkdev_issue_zeroout(device->ldev->backing_bdev, - sector, data_size >> 9, GFP_NOIO, false)) - peer_req->flags |= EE_WAS_ERROR; - drbd_endio_write_sec_final(peer_req); + + /* If this was a resync request from receive_rs_deallocated(), + * it is already on the sync_ee list */ + if (list_empty(&peer_req->w.list)) { + spin_lock_irq(&device->resource->req_lock); + list_add_tail(&peer_req->w.list, &device->active_ee); + spin_unlock_irq(&device->resource->req_lock); + } + + if (peer_req->flags & EE_IS_TRIM) + drbd_issue_peer_discard(device, peer_req); + else /* EE_WRITE_SAME */ + drbd_issue_peer_wsame(device, peer_req); return 0; } - /* Discards don't have any payload. - * But the scsi layer still expects a bio_vec it can use internally, - * see sd_setup_discard_cmnd() and blk_add_request_payload(). */ - if (peer_req->flags & EE_IS_TRIM) - nr_pages = 1; - /* In most cases, we will only need one bio. But in case the lower * level restrictions happen to be different at this offset on this * side than those of the sending peer, we may need to submit the @@ -1450,7 +1638,7 @@ next_bio: /* > peer_req->i.sector, unless this is the first bio */ bio->bi_iter.bi_sector = sector; bio->bi_bdev = device->ldev->backing_bdev; - bio->bi_rw = rw; + bio_set_op_attrs(bio, op, op_flags); bio->bi_private = peer_req; bio->bi_end_io = drbd_peer_request_endio; @@ -1458,11 +1646,6 @@ next_bio: bios = bio; ++n_bios; - if (rw & REQ_DISCARD) { - bio->bi_iter.bi_size = data_size; - goto submit; - } - page_chain_for_each(page) { unsigned len = min_t(unsigned, data_size, PAGE_SIZE); if (!bio_add_page(bio, page, len, 0)) { @@ -1484,7 +1667,6 @@ next_bio: --nr_pages; } D_ASSERT(device, data_size == 0); -submit: D_ASSERT(device, page == NULL); atomic_set(&peer_req->pending_bios, n_bios); @@ -1608,8 +1790,26 @@ static int receive_Barrier(struct drbd_connection *connection, struct packet_inf return 0; } +/* quick wrapper in case payload size != request_size (write same) */ +static void drbd_csum_ee_size(struct crypto_ahash *h, + struct drbd_peer_request *r, void *d, + unsigned int payload_size) +{ + unsigned int tmp = r->i.size; + r->i.size = payload_size; + drbd_csum_ee(h, r, d); + r->i.size = tmp; +} + /* used from receive_RSDataReply (recv_resync_read) - * and from receive_Data */ + * and from receive_Data. + * data_size: actual payload ("data in") + * for normal writes that is bi_size. + * for discards, that is zero. + * for write same, it is logical_block_size. + * both trim and write same have the bi_size ("data len to be affected") + * as extra argument in the packet header. + */ static struct drbd_peer_request * read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, struct packet_info *pi) __must_hold(local) @@ -1624,6 +1824,7 @@ read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, void *dig_vv = peer_device->connection->int_dig_vv; unsigned long *data; struct p_trim *trim = (pi->cmd == P_TRIM) ? pi->data : NULL; + struct p_trim *wsame = (pi->cmd == P_WSAME) ? pi->data : NULL; digest_size = 0; if (!trim && peer_device->connection->peer_integrity_tfm) { @@ -1638,38 +1839,60 @@ read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, data_size -= digest_size; } + /* assume request_size == data_size, but special case trim and wsame. */ + ds = data_size; if (trim) { - D_ASSERT(peer_device, data_size == 0); - data_size = be32_to_cpu(trim->size); + if (!expect(data_size == 0)) + return NULL; + ds = be32_to_cpu(trim->size); + } else if (wsame) { + if (data_size != queue_logical_block_size(device->rq_queue)) { + drbd_err(peer_device, "data size (%u) != drbd logical block size (%u)\n", + data_size, queue_logical_block_size(device->rq_queue)); + return NULL; + } + if (data_size != bdev_logical_block_size(device->ldev->backing_bdev)) { + drbd_err(peer_device, "data size (%u) != backend logical block size (%u)\n", + data_size, bdev_logical_block_size(device->ldev->backing_bdev)); + return NULL; + } + ds = be32_to_cpu(wsame->size); } - if (!expect(IS_ALIGNED(data_size, 512))) + if (!expect(IS_ALIGNED(ds, 512))) return NULL; - /* prepare for larger trim requests. */ - if (!trim && !expect(data_size <= DRBD_MAX_BIO_SIZE)) + if (trim || wsame) { + if (!expect(ds <= (DRBD_MAX_BBIO_SECTORS << 9))) + return NULL; + } else if (!expect(ds <= DRBD_MAX_BIO_SIZE)) return NULL; /* even though we trust out peer, * we sometimes have to double check. */ - if (sector + (data_size>>9) > capacity) { + if (sector + (ds>>9) > capacity) { drbd_err(device, "request from peer beyond end of local disk: " "capacity: %llus < sector: %llus + size: %u\n", (unsigned long long)capacity, - (unsigned long long)sector, data_size); + (unsigned long long)sector, ds); return NULL; } /* GFP_NOIO, because we must not cause arbitrary write-out: in a DRBD * "criss-cross" setup, that might cause write-out on some other DRBD, * which in turn might block on the other node at this very place. */ - peer_req = drbd_alloc_peer_req(peer_device, id, sector, data_size, trim == NULL, GFP_NOIO); + peer_req = drbd_alloc_peer_req(peer_device, id, sector, ds, data_size, GFP_NOIO); if (!peer_req) return NULL; peer_req->flags |= EE_WRITE; - if (trim) + if (trim) { + peer_req->flags |= EE_IS_TRIM; return peer_req; + } + if (wsame) + peer_req->flags |= EE_WRITE_SAME; + /* receive payload size bytes into page chain */ ds = data_size; page = peer_req->pages; page_chain_for_each(page) { @@ -1689,7 +1912,7 @@ read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, } if (digest_size) { - drbd_csum_ee(peer_device->connection->peer_integrity_tfm, peer_req, dig_vv); + drbd_csum_ee_size(peer_device->connection->peer_integrity_tfm, peer_req, dig_vv, data_size); if (memcmp(dig_in, dig_vv, digest_size)) { drbd_err(device, "Digest integrity check FAILED: %llus +%u\n", (unsigned long long)sector, data_size); @@ -1830,7 +2053,8 @@ static int recv_resync_read(struct drbd_peer_device *peer_device, sector_t secto spin_unlock_irq(&device->resource->req_lock); atomic_add(pi->size >> 9, &device->rs_sect_ev); - if (drbd_submit_peer_request(device, peer_req, WRITE, DRBD_FAULT_RS_WR) == 0) + if (drbd_submit_peer_request(device, peer_req, REQ_OP_WRITE, 0, + DRBD_FAULT_RS_WR) == 0) return 0; /* don't care for the reason here */ @@ -2065,13 +2289,13 @@ static inline int overlaps(sector_t s1, int l1, sector_t s2, int l2) static bool overlapping_resync_write(struct drbd_device *device, struct drbd_peer_request *peer_req) { struct drbd_peer_request *rs_req; - bool rv = 0; + bool rv = false; spin_lock_irq(&device->resource->req_lock); list_for_each_entry(rs_req, &device->sync_ee, w.list) { if (overlaps(peer_req->i.sector, peer_req->i.size, rs_req->i.sector, rs_req->i.size)) { - rv = 1; + rv = true; break; } } @@ -2152,12 +2376,19 @@ static int wait_for_and_update_peer_seq(struct drbd_peer_device *peer_device, co /* see also bio_flags_to_wire() * DRBD_REQ_*, because we need to semantically map the flags to data packet * flags and back. We may replicate to other kernel versions. */ -static unsigned long wire_flags_to_bio(u32 dpf) +static unsigned long wire_flags_to_bio_flags(u32 dpf) { return (dpf & DP_RW_SYNC ? REQ_SYNC : 0) | (dpf & DP_FUA ? REQ_FUA : 0) | - (dpf & DP_FLUSH ? REQ_FLUSH : 0) | - (dpf & DP_DISCARD ? REQ_DISCARD : 0); + (dpf & DP_FLUSH ? REQ_PREFLUSH : 0); +} + +static unsigned long wire_flags_to_bio_op(u32 dpf) +{ + if (dpf & DP_DISCARD) + return REQ_OP_DISCARD; + else + return REQ_OP_WRITE; } static void fail_postponed_requests(struct drbd_device *device, sector_t sector, @@ -2303,7 +2534,7 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * struct drbd_peer_request *peer_req; struct p_data *p = pi->data; u32 peer_seq = be32_to_cpu(p->seq_num); - int rw = WRITE; + int op, op_flags; u32 dp_flags; int err, tp; @@ -2342,14 +2573,11 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * peer_req->flags |= EE_APPLICATION; dp_flags = be32_to_cpu(p->dp_flags); - rw |= wire_flags_to_bio(dp_flags); + op = wire_flags_to_bio_op(dp_flags); + op_flags = wire_flags_to_bio_flags(dp_flags); if (pi->cmd == P_TRIM) { - struct request_queue *q = bdev_get_queue(device->ldev->backing_bdev); - peer_req->flags |= EE_IS_TRIM; - if (!blk_queue_discard(q)) - peer_req->flags |= EE_IS_TRIM_USE_ZEROOUT; D_ASSERT(peer_device, peer_req->i.size > 0); - D_ASSERT(peer_device, rw & REQ_DISCARD); + D_ASSERT(peer_device, op == REQ_OP_DISCARD); D_ASSERT(peer_device, peer_req->pages == NULL); } else if (peer_req->pages == NULL) { D_ASSERT(device, peer_req->i.size == 0); @@ -2414,11 +2642,11 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * update_peer_seq(peer_device, peer_seq); spin_lock_irq(&device->resource->req_lock); } - /* if we use the zeroout fallback code, we process synchronously - * and we wait for all pending requests, respectively wait for + /* TRIM and WRITE_SAME are processed synchronously, + * we wait for all pending requests, respectively wait for * active_ee to become empty in drbd_submit_peer_request(); * better not add ourselves here. */ - if ((peer_req->flags & EE_IS_TRIM_USE_ZEROOUT) == 0) + if ((peer_req->flags & (EE_IS_TRIM|EE_WRITE_SAME)) == 0) list_add_tail(&peer_req->w.list, &device->active_ee); spin_unlock_irq(&device->resource->req_lock); @@ -2433,7 +2661,8 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * peer_req->flags |= EE_CALL_AL_COMPLETE_IO; } - err = drbd_submit_peer_request(device, peer_req, rw, DRBD_FAULT_DT_WR); + err = drbd_submit_peer_request(device, peer_req, op, op_flags, + DRBD_FAULT_DT_WR); if (!err) return 0; @@ -2449,7 +2678,7 @@ static int receive_Data(struct drbd_connection *connection, struct packet_info * } out_interrupted: - drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT + EV_CLEANUP); + drbd_may_finish_epoch(connection, peer_req->epoch, EV_PUT | EV_CLEANUP); put_ldev(device); drbd_free_peer_req(device, peer_req); return err; @@ -2574,6 +2803,7 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet case P_DATA_REQUEST: drbd_send_ack_rp(peer_device, P_NEG_DREPLY, p); break; + case P_RS_THIN_REQ: case P_RS_DATA_REQUEST: case P_CSUM_RS_REQUEST: case P_OV_REQUEST: @@ -2599,7 +2829,7 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet * "criss-cross" setup, that might cause write-out on some other DRBD, * which in turn might block on the other node at this very place. */ peer_req = drbd_alloc_peer_req(peer_device, p->block_id, sector, size, - true /* has real payload */, GFP_NOIO); + size, GFP_NOIO); if (!peer_req) { put_ldev(device); return -ENOMEM; @@ -2613,6 +2843,12 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet peer_req->flags |= EE_APPLICATION; goto submit; + case P_RS_THIN_REQ: + /* If at some point in the future we have a smart way to + find out if this data block is completely deallocated, + then we would do something smarter here than reading + the block... */ + peer_req->flags |= EE_RS_THIN_REQ; case P_RS_DATA_REQUEST: peer_req->w.cb = w_e_end_rsdata_req; fault_type = DRBD_FAULT_RS_RD; @@ -2723,7 +2959,8 @@ submit_for_resync: submit: update_receiver_timing_details(connection, drbd_submit_peer_request); inc_unacked(device); - if (drbd_submit_peer_request(device, peer_req, READ, fault_type) == 0) + if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ, 0, + fault_type) == 0) return 0; /* don't care for the reason here */ @@ -2957,7 +3194,8 @@ static void drbd_uuid_dump(struct drbd_device *device, char *text, u64 *uuid, -1091 requires proto 91 -1096 requires proto 96 */ -static int drbd_uuid_compare(struct drbd_device *const device, int *rule_nr) __must_hold(local) + +static int drbd_uuid_compare(struct drbd_device *const device, enum drbd_role const peer_role, int *rule_nr) __must_hold(local) { struct drbd_peer_device *const peer_device = first_peer_device(device); struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL; @@ -3037,8 +3275,39 @@ static int drbd_uuid_compare(struct drbd_device *const device, int *rule_nr) __m * next bit (weight 2) is set when peer was primary */ *rule_nr = 40; + /* Neither has the "crashed primary" flag set, + * only a replication link hickup. */ + if (rct == 0) + return 0; + + /* Current UUID equal and no bitmap uuid; does not necessarily + * mean this was a "simultaneous hard crash", maybe IO was + * frozen, so no UUID-bump happened. + * This is a protocol change, overload DRBD_FF_WSAME as flag + * for "new-enough" peer DRBD version. */ + if (device->state.role == R_PRIMARY || peer_role == R_PRIMARY) { + *rule_nr = 41; + if (!(connection->agreed_features & DRBD_FF_WSAME)) { + drbd_warn(peer_device, "Equivalent unrotated UUIDs, but current primary present.\n"); + return -(0x10000 | PRO_VERSION_MAX | (DRBD_FF_WSAME << 8)); + } + if (device->state.role == R_PRIMARY && peer_role == R_PRIMARY) { + /* At least one has the "crashed primary" bit set, + * both are primary now, but neither has rotated its UUIDs? + * "Can not happen." */ + drbd_err(peer_device, "Equivalent unrotated UUIDs, but both are primary. Can not resolve this.\n"); + return -100; + } + if (device->state.role == R_PRIMARY) + return 1; + return -1; + } + + /* Both are secondary. + * Really looks like recovery from simultaneous hard crash. + * Check which had been primary before, and arbitrate. */ switch (rct) { - case 0: /* !self_pri && !peer_pri */ return 0; + case 0: /* !self_pri && !peer_pri */ return 0; /* already handled */ case 1: /* self_pri && !peer_pri */ return 1; case 2: /* !self_pri && peer_pri */ return -1; case 3: /* self_pri && peer_pri */ @@ -3165,7 +3434,7 @@ static enum drbd_conns drbd_sync_handshake(struct drbd_peer_device *peer_device, drbd_uuid_dump(device, "peer", device->p_uuid, device->p_uuid[UI_SIZE], device->p_uuid[UI_FLAGS]); - hg = drbd_uuid_compare(device, &rule_nr); + hg = drbd_uuid_compare(device, peer_role, &rule_nr); spin_unlock_irq(&device->ldev->md.uuid_lock); drbd_info(device, "uuid_compare()=%d by rule %d\n", hg, rule_nr); @@ -3174,6 +3443,15 @@ static enum drbd_conns drbd_sync_handshake(struct drbd_peer_device *peer_device, drbd_alert(device, "Unrelated data, aborting!\n"); return C_MASK; } + if (hg < -0x10000) { + int proto, fflags; + hg = -hg; + proto = hg & 0xff; + fflags = (hg >> 8) & 0xff; + drbd_alert(device, "To resolve this both sides have to support at least protocol %d and feature flags 0x%x\n", + proto, fflags); + return C_MASK; + } if (hg < -1000) { drbd_alert(device, "To resolve this both sides have to support at least protocol %d\n", -hg - 1000); return C_MASK; @@ -3403,7 +3681,8 @@ static int receive_protocol(struct drbd_connection *connection, struct packet_in */ peer_integrity_tfm = crypto_alloc_ahash(integrity_alg, 0, CRYPTO_ALG_ASYNC); - if (!peer_integrity_tfm) { + if (IS_ERR(peer_integrity_tfm)) { + peer_integrity_tfm = NULL; drbd_err(connection, "peer data-integrity-alg %s not supported\n", integrity_alg); goto disconnect; @@ -3754,6 +4033,7 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info struct drbd_peer_device *peer_device; struct drbd_device *device; struct p_sizes *p = pi->data; + struct o_qlim *o = (connection->agreed_features & DRBD_FF_WSAME) ? p->qlim : NULL; enum determine_dev_size dd = DS_UNCHANGED; sector_t p_size, p_usize, p_csize, my_usize; int ldsc = 0; /* local disk size changed */ @@ -3773,6 +4053,7 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info device->p_size = p_size; if (get_ldev(device)) { + sector_t new_size, cur_size; rcu_read_lock(); my_usize = rcu_dereference(device->ldev->disk_conf)->disk_size; rcu_read_unlock(); @@ -3789,11 +4070,13 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info /* Never shrink a device with usable data during connect. But allow online shrinking if we are connected. */ - if (drbd_new_dev_size(device, device->ldev, p_usize, 0) < - drbd_get_capacity(device->this_bdev) && + new_size = drbd_new_dev_size(device, device->ldev, p_usize, 0); + cur_size = drbd_get_capacity(device->this_bdev); + if (new_size < cur_size && device->state.disk >= D_OUTDATED && device->state.conn < C_CONNECTED) { - drbd_err(device, "The peer's disk size is too small!\n"); + drbd_err(device, "The peer's disk size is too small! (%llu < %llu sectors)\n", + (unsigned long long)new_size, (unsigned long long)cur_size); conn_request_state(peer_device->connection, NS(conn, C_DISCONNECTING), CS_HARD); put_ldev(device); return -EIO; @@ -3827,14 +4110,14 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info } device->peer_max_bio_size = be32_to_cpu(p->max_bio_size); - /* Leave drbd_reconsider_max_bio_size() before drbd_determine_dev_size(). + /* Leave drbd_reconsider_queue_parameters() before drbd_determine_dev_size(). In case we cleared the QUEUE_FLAG_DISCARD from our queue in - drbd_reconsider_max_bio_size(), we can be sure that after + drbd_reconsider_queue_parameters(), we can be sure that after drbd_determine_dev_size() no REQ_DISCARDs are in the queue. */ ddsf = be16_to_cpu(p->dds_flags); if (get_ldev(device)) { - drbd_reconsider_max_bio_size(device, device->ldev); + drbd_reconsider_queue_parameters(device, device->ldev, o); dd = drbd_determine_dev_size(device, ddsf, NULL); put_ldev(device); if (dd == DS_ERROR) @@ -3854,7 +4137,7 @@ static int receive_sizes(struct drbd_connection *connection, struct packet_info * However, if he sends a zero current size, * take his (user-capped or) backing disk size anyways. */ - drbd_reconsider_max_bio_size(device, NULL); + drbd_reconsider_queue_parameters(device, NULL, o); drbd_set_my_capacity(device, p_csize ?: p_usize ?: p_size); } @@ -4587,9 +4870,75 @@ static int receive_out_of_sync(struct drbd_connection *connection, struct packet return 0; } +static int receive_rs_deallocated(struct drbd_connection *connection, struct packet_info *pi) +{ + struct drbd_peer_device *peer_device; + struct p_block_desc *p = pi->data; + struct drbd_device *device; + sector_t sector; + int size, err = 0; + + peer_device = conn_peer_device(connection, pi->vnr); + if (!peer_device) + return -EIO; + device = peer_device->device; + + sector = be64_to_cpu(p->sector); + size = be32_to_cpu(p->blksize); + + dec_rs_pending(device); + + if (get_ldev(device)) { + struct drbd_peer_request *peer_req; + const int op = REQ_OP_DISCARD; + + peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER, sector, + size, 0, GFP_NOIO); + if (!peer_req) { + put_ldev(device); + return -ENOMEM; + } + + peer_req->w.cb = e_end_resync_block; + peer_req->submit_jif = jiffies; + peer_req->flags |= EE_IS_TRIM; + + spin_lock_irq(&device->resource->req_lock); + list_add_tail(&peer_req->w.list, &device->sync_ee); + spin_unlock_irq(&device->resource->req_lock); + + atomic_add(pi->size >> 9, &device->rs_sect_ev); + err = drbd_submit_peer_request(device, peer_req, op, 0, DRBD_FAULT_RS_WR); + + if (err) { + spin_lock_irq(&device->resource->req_lock); + list_del(&peer_req->w.list); + spin_unlock_irq(&device->resource->req_lock); + + drbd_free_peer_req(device, peer_req); + put_ldev(device); + err = 0; + goto fail; + } + + inc_unacked(device); + + /* No put_ldev() here. Gets called in drbd_endio_write_sec_final(), + as well as drbd_rs_complete_io() */ + } else { + fail: + drbd_rs_complete_io(device, sector); + drbd_send_ack_ex(peer_device, P_NEG_ACK, sector, size, ID_SYNCER); + } + + atomic_add(size >> 9, &device->rs_sect_in); + + return err; +} + struct data_cmd { int expect_payload; - size_t pkt_size; + unsigned int pkt_size; int (*fn)(struct drbd_connection *, struct packet_info *); }; @@ -4614,11 +4963,14 @@ static struct data_cmd drbd_cmd_handler[] = { [P_OV_REQUEST] = { 0, sizeof(struct p_block_req), receive_DataRequest }, [P_OV_REPLY] = { 1, sizeof(struct p_block_req), receive_DataRequest }, [P_CSUM_RS_REQUEST] = { 1, sizeof(struct p_block_req), receive_DataRequest }, + [P_RS_THIN_REQ] = { 0, sizeof(struct p_block_req), receive_DataRequest }, [P_DELAY_PROBE] = { 0, sizeof(struct p_delay_probe93), receive_skip }, [P_OUT_OF_SYNC] = { 0, sizeof(struct p_block_desc), receive_out_of_sync }, [P_CONN_ST_CHG_REQ] = { 0, sizeof(struct p_req_state), receive_req_conn_state }, [P_PROTOCOL_UPDATE] = { 1, sizeof(struct p_protocol), receive_protocol }, [P_TRIM] = { 0, sizeof(struct p_trim), receive_Data }, + [P_RS_DEALLOCATED] = { 0, sizeof(struct p_block_desc), receive_rs_deallocated }, + [P_WSAME] = { 1, sizeof(struct p_wsame), receive_Data }, }; static void drbdd(struct drbd_connection *connection) @@ -4628,7 +4980,7 @@ static void drbdd(struct drbd_connection *connection) int err; while (get_t_state(&connection->receiver) == RUNNING) { - struct data_cmd *cmd; + struct data_cmd const *cmd; drbd_thread_current_set_cpu(&connection->receiver); update_receiver_timing_details(connection, drbd_recv_header); @@ -4643,11 +4995,18 @@ static void drbdd(struct drbd_connection *connection) } shs = cmd->pkt_size; + if (pi.cmd == P_SIZES && connection->agreed_features & DRBD_FF_WSAME) + shs += sizeof(struct o_qlim); if (pi.size > shs && !cmd->expect_payload) { drbd_err(connection, "No payload expected %s l:%d\n", cmdname(pi.cmd), pi.size); goto err_out; } + if (pi.size < shs) { + drbd_err(connection, "%s: unexpected packet size, expected:%d received:%d\n", + cmdname(pi.cmd), (int)shs, pi.size); + goto err_out; + } if (shs) { update_receiver_timing_details(connection, drbd_recv_all_warn); @@ -4783,9 +5142,11 @@ static int drbd_disconnected(struct drbd_peer_device *peer_device) drbd_md_sync(device); - /* serialize with bitmap writeout triggered by the state change, - * if any. */ - wait_event(device->misc_wait, !test_bit(BITMAP_IO, &device->flags)); + if (get_ldev(device)) { + drbd_bitmap_io(device, &drbd_bm_write_copy_pages, + "write from disconnected", BM_LOCKED_CHANGE_ALLOWED); + put_ldev(device); + } /* tcp_close and release of sendpage pages can be deferred. I don't * want to use SO_LINGER, because apparently it can be deferred for @@ -4892,8 +5253,12 @@ static int drbd_do_features(struct drbd_connection *connection) drbd_info(connection, "Handshake successful: " "Agreed network protocol version %d\n", connection->agreed_pro_version); - drbd_info(connection, "Agreed to%ssupport TRIM on protocol level\n", - connection->agreed_features & FF_TRIM ? " " : " not "); + drbd_info(connection, "Feature flags enabled on protocol level: 0x%x%s%s%s.\n", + connection->agreed_features, + connection->agreed_features & DRBD_FF_TRIM ? " TRIM" : "", + connection->agreed_features & DRBD_FF_THIN_RESYNC ? " THIN_RESYNC" : "", + connection->agreed_features & DRBD_FF_WSAME ? " WRITE_SAME" : + connection->agreed_features ? "" : " none"); return 1; diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index 2255dcfeb..de279fe4e 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -47,8 +47,7 @@ static void _drbd_end_io_acct(struct drbd_device *device, struct drbd_request *r &device->vdisk->part0, req->start_jif); } -static struct drbd_request *drbd_req_new(struct drbd_device *device, - struct bio *bio_src) +static struct drbd_request *drbd_req_new(struct drbd_device *device, struct bio *bio_src) { struct drbd_request *req; @@ -58,10 +57,12 @@ static struct drbd_request *drbd_req_new(struct drbd_device *device, memset(req, 0, sizeof(*req)); drbd_req_make_private_bio(req, bio_src); - req->rq_state = bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0; - req->device = device; - req->master_bio = bio_src; - req->epoch = 0; + req->rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0) + | (bio_op(bio_src) == REQ_OP_WRITE_SAME ? RQ_WSAME : 0) + | (bio_op(bio_src) == REQ_OP_DISCARD ? RQ_UNMAP : 0); + req->device = device; + req->master_bio = bio_src; + req->epoch = 0; drbd_clear_interval(&req->i); req->i.sector = bio_src->bi_iter.bi_sector; @@ -218,7 +219,6 @@ void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m) { const unsigned s = req->rq_state; struct drbd_device *device = req->device; - int rw; int error, ok; /* we must not complete the master bio, while it is @@ -242,8 +242,6 @@ void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m) return; } - rw = bio_rw(req->master_bio); - /* * figure out whether to report success or failure. * @@ -267,7 +265,7 @@ void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m) * epoch number. If they match, increase the current_tle_nr, * and reset the transfer log epoch write_cnt. */ - if (rw == WRITE && + if (op_is_write(bio_op(req->master_bio)) && req->epoch == atomic_read(&first_peer_device(device)->connection->current_tle_nr)) start_new_tl_epoch(first_peer_device(device)->connection); @@ -284,11 +282,14 @@ void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m) * because no path was available, in which case * it was not even added to the transfer_log. * - * READA may fail, and will not be retried. + * read-ahead may fail, and will not be retried. * * WRITE should have used all available paths already. */ - if (!ok && rw == READ && !list_empty(&req->tl_requests)) + if (!ok && + bio_op(req->master_bio) == REQ_OP_READ && + !(req->master_bio->bi_opf & REQ_RAHEAD) && + !list_empty(&req->tl_requests)) req->rq_state |= RQ_POSTPONED; if (!(req->rq_state & RQ_POSTPONED)) { @@ -644,7 +645,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what, __drbd_chk_io_error(device, DRBD_READ_ERROR); /* fall through. */ case READ_AHEAD_COMPLETED_WITH_ERROR: - /* it is legal to fail READA, no __drbd_chk_io_error in that case. */ + /* it is legal to fail read-ahead, no __drbd_chk_io_error in that case. */ mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED); break; @@ -656,7 +657,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what, break; case QUEUE_FOR_NET_READ: - /* READ or READA, and + /* READ, and * no local disk, * or target area marked as invalid, * or just got an io-error. */ @@ -977,16 +978,20 @@ static void complete_conflicting_writes(struct drbd_request *req) sector_t sector = req->i.sector; int size = req->i.size; - i = drbd_find_overlap(&device->write_requests, sector, size); - if (!i) - return; - for (;;) { - prepare_to_wait(&device->misc_wait, &wait, TASK_UNINTERRUPTIBLE); - i = drbd_find_overlap(&device->write_requests, sector, size); - if (!i) + drbd_for_each_overlap(i, &device->write_requests, sector, size) { + /* Ignore, if already completed to upper layers. */ + if (i->completed) + continue; + /* Handle the first found overlap. After the schedule + * we have to restart the tree walk. */ + break; + } + if (!i) /* if any */ break; + /* Indicate to wake up device->misc_wait on progress. */ + prepare_to_wait(&device->misc_wait, &wait, TASK_UNINTERRUPTIBLE); i->waiting = true; spin_unlock_irq(&device->resource->req_lock); schedule(); @@ -995,7 +1000,7 @@ static void complete_conflicting_writes(struct drbd_request *req) finish_wait(&device->misc_wait, &wait); } -/* called within req_lock and rcu_read_lock() */ +/* called within req_lock */ static void maybe_pull_ahead(struct drbd_device *device) { struct drbd_connection *connection = first_peer_device(device)->connection; @@ -1132,7 +1137,7 @@ static int drbd_process_write_request(struct drbd_request *req) * replicating, in which case there is no point. */ if (unlikely(req->i.size == 0)) { /* The only size==0 bios we expect are empty flushes. */ - D_ASSERT(device, req->master_bio->bi_rw & REQ_FLUSH); + D_ASSERT(device, req->master_bio->bi_opf & REQ_PREFLUSH); if (remote) _req_mod(req, QUEUE_AS_DRBD_BARRIER); return remote; @@ -1152,12 +1157,29 @@ static int drbd_process_write_request(struct drbd_request *req) return remote; } +static void drbd_process_discard_req(struct drbd_request *req) +{ + int err = drbd_issue_discard_or_zero_out(req->device, + req->i.sector, req->i.size >> 9, true); + + if (err) + req->private_bio->bi_error = -EIO; + bio_endio(req->private_bio); +} + static void drbd_submit_req_private_bio(struct drbd_request *req) { struct drbd_device *device = req->device; struct bio *bio = req->private_bio; - const int rw = bio_rw(bio); + unsigned int type; + + if (bio_op(bio) != REQ_OP_READ) + type = DRBD_FAULT_DT_WR; + else if (bio->bi_opf & REQ_RAHEAD) + type = DRBD_FAULT_DT_RA; + else + type = DRBD_FAULT_DT_RD; bio->bi_bdev = device->ldev->backing_bdev; @@ -1167,11 +1189,10 @@ drbd_submit_req_private_bio(struct drbd_request *req) * stable storage, and this is a WRITE, we may not even submit * this bio. */ if (get_ldev(device)) { - if (drbd_insert_fault(device, - rw == WRITE ? DRBD_FAULT_DT_WR - : rw == READ ? DRBD_FAULT_DT_RD - : DRBD_FAULT_DT_RA)) + if (drbd_insert_fault(device, type)) bio_io_error(bio); + else if (bio_op(bio) == REQ_OP_DISCARD) + drbd_process_discard_req(req); else generic_make_request(bio); put_ldev(device); @@ -1223,24 +1244,45 @@ drbd_request_prepare(struct drbd_device *device, struct bio *bio, unsigned long /* Update disk stats */ _drbd_start_io_acct(device, req); + /* process discards always from our submitter thread */ + if (bio_op(bio) & REQ_OP_DISCARD) + goto queue_for_submitter_thread; + if (rw == WRITE && req->private_bio && req->i.size && !test_bit(AL_SUSPENDED, &device->flags)) { - if (!drbd_al_begin_io_fastpath(device, &req->i)) { - atomic_inc(&device->ap_actlog_cnt); - drbd_queue_write(device, req); - return NULL; - } + if (!drbd_al_begin_io_fastpath(device, &req->i)) + goto queue_for_submitter_thread; req->rq_state |= RQ_IN_ACT_LOG; req->in_actlog_jif = jiffies; } - return req; + + queue_for_submitter_thread: + atomic_inc(&device->ap_actlog_cnt); + drbd_queue_write(device, req); + return NULL; +} + +/* Require at least one path to current data. + * We don't want to allow writes on C_STANDALONE D_INCONSISTENT: + * We would not allow to read what was written, + * we would not have bumped the data generation uuids, + * we would cause data divergence for all the wrong reasons. + * + * If we don't see at least one D_UP_TO_DATE, we will fail this request, + * which either returns EIO, or, if OND_SUSPEND_IO is set, suspends IO, + * and queues for retry later. + */ +static bool may_do_writes(struct drbd_device *device) +{ + const union drbd_dev_state s = device->state; + return s.disk == D_UP_TO_DATE || s.pdsk == D_UP_TO_DATE; } static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request *req) { struct drbd_resource *resource = device->resource; - const int rw = bio_rw(req->master_bio); + const int rw = bio_data_dir(req->master_bio); struct bio_and_error m = { NULL, }; bool no_remote = false; bool submit_private_bio = false; @@ -1270,7 +1312,7 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request goto out; } - /* We fail READ/READA early, if we can not serve it. + /* We fail READ early, if we can not serve it. * We must do this before req is registered on any lists. * Otherwise, drbd_req_complete() will queue failed READ for retry. */ if (rw != WRITE) { @@ -1291,6 +1333,12 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request } if (rw == WRITE) { + if (req->private_bio && !may_do_writes(device)) { + bio_put(req->private_bio); + req->private_bio = NULL; + put_ldev(device); + goto nodata; + } if (!drbd_process_write_request(req)) no_remote = true; } else { diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h index bb2ef7816..eb49e7f2d 100644 --- a/drivers/block/drbd/drbd_req.h +++ b/drivers/block/drbd/drbd_req.h @@ -206,6 +206,8 @@ enum drbd_req_state_bits { /* Set when this is a write, clear for a read */ __RQ_WRITE, + __RQ_WSAME, + __RQ_UNMAP, /* Should call drbd_al_complete_io() for this request... */ __RQ_IN_ACT_LOG, @@ -241,10 +243,11 @@ enum drbd_req_state_bits { #define RQ_NET_OK (1UL << __RQ_NET_OK) #define RQ_NET_SIS (1UL << __RQ_NET_SIS) -/* 0x1f8 */ #define RQ_NET_MASK (((1UL << __RQ_NET_MAX)-1) & ~RQ_LOCAL_MASK) #define RQ_WRITE (1UL << __RQ_WRITE) +#define RQ_WSAME (1UL << __RQ_WSAME) +#define RQ_UNMAP (1UL << __RQ_UNMAP) #define RQ_IN_ACT_LOG (1UL << __RQ_IN_ACT_LOG) #define RQ_POSTPONED (1UL << __RQ_POSTPONED) #define RQ_COMPLETION_SUSP (1UL << __RQ_COMPLETION_SUSP) diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c index 5a7ef7873..eea0c4aec 100644 --- a/drivers/block/drbd/drbd_state.c +++ b/drivers/block/drbd/drbd_state.c @@ -814,7 +814,7 @@ is_valid_state(struct drbd_device *device, union drbd_state ns) } if (rv <= 0) - /* already found a reason to abort */; + goto out; /* already found a reason to abort */ else if (ns.role == R_SECONDARY && device->open_cnt) rv = SS_DEVICE_IN_USE; @@ -862,6 +862,7 @@ is_valid_state(struct drbd_device *device, union drbd_state ns) else if (ns.conn >= C_CONNECTED && ns.pdsk == D_UNKNOWN) rv = SS_CONNECTED_OUTDATES; +out: rcu_read_unlock(); return rv; @@ -906,6 +907,15 @@ is_valid_soft_transition(union drbd_state os, union drbd_state ns, struct drbd_c (ns.conn >= C_CONNECTED && os.conn == C_WF_REPORT_PARAMS))) rv = SS_IN_TRANSIENT_STATE; + /* Do not promote during resync handshake triggered by "force primary". + * This is a hack. It should really be rejected by the peer during the + * cluster wide state change request. */ + if (os.role != R_PRIMARY && ns.role == R_PRIMARY + && ns.pdsk == D_UP_TO_DATE + && ns.disk != D_UP_TO_DATE && ns.disk != D_DISKLESS + && (ns.conn <= C_WF_SYNC_UUID || ns.conn != os.conn)) + rv = SS_IN_TRANSIENT_STATE; + if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && os.conn < C_CONNECTED) rv = SS_NEED_CONNECTION; @@ -1628,6 +1638,26 @@ static void broadcast_state_change(struct drbd_state_change *state_change) #undef REMEMBER_STATE_CHANGE } +/* takes old and new peer disk state */ +static bool lost_contact_to_peer_data(enum drbd_disk_state os, enum drbd_disk_state ns) +{ + if ((os >= D_INCONSISTENT && os != D_UNKNOWN && os != D_OUTDATED) + && (ns < D_INCONSISTENT || ns == D_UNKNOWN || ns == D_OUTDATED)) + return true; + + /* Scenario, starting with normal operation + * Connected Primary/Secondary UpToDate/UpToDate + * NetworkFailure Primary/Unknown UpToDate/DUnknown (frozen) + * ... + * Connected Primary/Secondary UpToDate/Diskless (resumed; needs to bump uuid!) + */ + if (os == D_UNKNOWN + && (ns == D_DISKLESS || ns == D_FAILED || ns == D_OUTDATED)) + return true; + + return false; +} + /** * after_state_ch() - Perform after state change actions that may sleep * @device: DRBD device. @@ -1675,7 +1705,7 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os, what = RESEND; if ((os.disk == D_ATTACHING || os.disk == D_NEGOTIATING) && - conn_lowest_disk(connection) > D_NEGOTIATING) + conn_lowest_disk(connection) == D_UP_TO_DATE) what = RESTART_FROZEN_DISK_IO; if (resource->susp_nod && what != NOTHING) { @@ -1699,6 +1729,13 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os, idr_for_each_entry(&connection->peer_devices, peer_device, vnr) clear_bit(NEW_CUR_UUID, &peer_device->device->flags); rcu_read_unlock(); + + /* We should actively create a new uuid, _before_ + * we resume/resent, if the peer is diskless + * (recovery from a multiple error scenario). + * Currently, this happens with a slight delay + * below when checking lost_contact_to_peer_data() ... + */ _tl_restart(connection, RESEND); _conn_request_state(connection, (union drbd_state) { { .susp_fen = 1 } }, @@ -1742,12 +1779,7 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os, BM_LOCKED_TEST_ALLOWED); /* Lost contact to peer's copy of the data */ - if ((os.pdsk >= D_INCONSISTENT && - os.pdsk != D_UNKNOWN && - os.pdsk != D_OUTDATED) - && (ns.pdsk < D_INCONSISTENT || - ns.pdsk == D_UNKNOWN || - ns.pdsk == D_OUTDATED)) { + if (lost_contact_to_peer_data(os.pdsk, ns.pdsk)) { if (get_ldev(device)) { if ((ns.role == R_PRIMARY || ns.peer == R_PRIMARY) && device->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) { @@ -1934,12 +1966,17 @@ static void after_state_ch(struct drbd_device *device, union drbd_state os, /* This triggers bitmap writeout of potentially still unwritten pages * if the resync finished cleanly, or aborted because of peer disk - * failure, or because of connection loss. + * failure, or on transition from resync back to AHEAD/BEHIND. + * + * Connection loss is handled in drbd_disconnected() by the receiver. + * * For resync aborted because of local disk failure, we cannot do * any bitmap writeout anymore. + * * No harm done if some bits change during this phase. */ - if (os.conn > C_CONNECTED && ns.conn <= C_CONNECTED && get_ldev(device)) { + if ((os.conn > C_CONNECTED && os.conn < C_AHEAD) && + (ns.conn == C_CONNECTED || ns.conn >= C_AHEAD) && get_ldev(device)) { drbd_queue_bitmap_io(device, &drbd_bm_write_copy_pages, NULL, "write from resync_finished", BM_LOCKED_CHANGE_ALLOWED); put_ldev(device); @@ -2160,9 +2197,7 @@ conn_set_state(struct drbd_connection *connection, union drbd_state mask, union ns.disk = os.disk; rv = _drbd_set_state(device, ns, flags, NULL); - if (rv < SS_SUCCESS) - BUG(); - + BUG_ON(rv < SS_SUCCESS); ns.i = device->state.i; ns_max.role = max_role(ns.role, ns_max.role); ns_max.peer = max_role(ns.peer, ns_max.peer); diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h index bd989536f..6c9d5d4a8 100644 --- a/drivers/block/drbd/drbd_state.h +++ b/drivers/block/drbd/drbd_state.h @@ -140,7 +140,7 @@ extern void drbd_resume_al(struct drbd_device *device); extern bool conn_all_vols_unconf(struct drbd_connection *connection); /** - * drbd_request_state() - Reqest a state change + * drbd_request_state() - Request a state change * @device: DRBD device. * @mask: mask of state bits to change. * @val: value of new state bits. diff --git a/drivers/block/drbd/drbd_strings.c b/drivers/block/drbd/drbd_strings.c index 80b0f63c7..0eeab1477 100644 --- a/drivers/block/drbd/drbd_strings.c +++ b/drivers/block/drbd/drbd_strings.c @@ -26,7 +26,7 @@ #include #include "drbd_strings.h" -static const char *drbd_conn_s_names[] = { +static const char * const drbd_conn_s_names[] = { [C_STANDALONE] = "StandAlone", [C_DISCONNECTING] = "Disconnecting", [C_UNCONNECTED] = "Unconnected", @@ -53,13 +53,13 @@ static const char *drbd_conn_s_names[] = { [C_BEHIND] = "Behind", }; -static const char *drbd_role_s_names[] = { +static const char * const drbd_role_s_names[] = { [R_PRIMARY] = "Primary", [R_SECONDARY] = "Secondary", [R_UNKNOWN] = "Unknown" }; -static const char *drbd_disk_s_names[] = { +static const char * const drbd_disk_s_names[] = { [D_DISKLESS] = "Diskless", [D_ATTACHING] = "Attaching", [D_FAILED] = "Failed", @@ -71,7 +71,7 @@ static const char *drbd_disk_s_names[] = { [D_UP_TO_DATE] = "UpToDate", }; -static const char *drbd_state_sw_errors[] = { +static const char * const drbd_state_sw_errors[] = { [-SS_TWO_PRIMARIES] = "Multiple primaries not allowed by config", [-SS_NO_UP_TO_DATE_DISK] = "Need access to UpToDate data", [-SS_NO_LOCAL_DISK] = "Can not resync without local disk", diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c index 4d87499f0..c6755c9a0 100644 --- a/drivers/block/drbd/drbd_worker.c +++ b/drivers/block/drbd/drbd_worker.c @@ -173,8 +173,8 @@ void drbd_peer_request_endio(struct bio *bio) { struct drbd_peer_request *peer_req = bio->bi_private; struct drbd_device *device = peer_req->peer_device->device; - int is_write = bio_data_dir(bio) == WRITE; - int is_discard = !!(bio->bi_rw & REQ_DISCARD); + bool is_write = bio_data_dir(bio) == WRITE; + bool is_discard = !!(bio_op(bio) == REQ_OP_DISCARD); if (bio->bi_error && __ratelimit(&drbd_ratelimit_state)) drbd_warn(device, "%s: error=%d s=%llus\n", @@ -248,18 +248,26 @@ void drbd_request_endio(struct bio *bio) /* to avoid recursion in __req_mod */ if (unlikely(bio->bi_error)) { - if (bio->bi_rw & REQ_DISCARD) - what = (bio->bi_error == -EOPNOTSUPP) - ? DISCARD_COMPLETED_NOTSUPP - : DISCARD_COMPLETED_WITH_ERROR; - else - what = (bio_data_dir(bio) == WRITE) - ? WRITE_COMPLETED_WITH_ERROR - : (bio_rw(bio) == READ) - ? READ_COMPLETED_WITH_ERROR - : READ_AHEAD_COMPLETED_WITH_ERROR; - } else + switch (bio_op(bio)) { + case REQ_OP_DISCARD: + if (bio->bi_error == -EOPNOTSUPP) + what = DISCARD_COMPLETED_NOTSUPP; + else + what = DISCARD_COMPLETED_WITH_ERROR; + break; + case REQ_OP_READ: + if (bio->bi_opf & REQ_RAHEAD) + what = READ_AHEAD_COMPLETED_WITH_ERROR; + else + what = READ_COMPLETED_WITH_ERROR; + break; + default: + what = WRITE_COMPLETED_WITH_ERROR; + break; + } + } else { what = COMPLETED_OK; + } bio_put(req->private_bio); req->private_bio = ERR_PTR(bio->bi_error); @@ -320,6 +328,10 @@ void drbd_csum_bio(struct crypto_ahash *tfm, struct bio *bio, void *digest) sg_set_page(&sg, bvec.bv_page, bvec.bv_len, bvec.bv_offset); ahash_request_set_crypt(req, &sg, NULL, sg.length); crypto_ahash_update(req); + /* REQ_OP_WRITE_SAME has only one segment, + * checksum the payload only once. */ + if (bio_op(bio) == REQ_OP_WRITE_SAME) + break; } ahash_request_set_crypt(req, NULL, digest, 0); crypto_ahash_final(req); @@ -387,7 +399,7 @@ static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector, /* GFP_TRY, because if there is no memory available right now, this may * be rescheduled for later. It is "only" background resync, after all. */ peer_req = drbd_alloc_peer_req(peer_device, ID_SYNCER /* unused */, sector, - size, true /* has real payload */, GFP_TRY); + size, size, GFP_TRY); if (!peer_req) goto defer; @@ -397,7 +409,8 @@ static int read_for_csum(struct drbd_peer_device *peer_device, sector_t sector, spin_unlock_irq(&device->resource->req_lock); atomic_add(size >> 9, &device->rs_sect_ev); - if (drbd_submit_peer_request(device, peer_req, READ, DRBD_FAULT_RS_RD) == 0) + if (drbd_submit_peer_request(device, peer_req, REQ_OP_READ, 0, + DRBD_FAULT_RS_RD) == 0) return 0; /* If it failed because of ENOMEM, retry should help. If it failed @@ -582,6 +595,7 @@ static int make_resync_request(struct drbd_device *const device, int cancel) int number, rollback_i, size; int align, requeue = 0; int i = 0; + int discard_granularity = 0; if (unlikely(cancel)) return 0; @@ -601,6 +615,12 @@ static int make_resync_request(struct drbd_device *const device, int cancel) return 0; } + if (connection->agreed_features & DRBD_FF_THIN_RESYNC) { + rcu_read_lock(); + discard_granularity = rcu_dereference(device->ldev->disk_conf)->rs_discard_granularity; + rcu_read_unlock(); + } + max_bio_size = queue_max_hw_sectors(device->rq_queue) << 9; number = drbd_rs_number_requests(device); if (number <= 0) @@ -665,6 +685,9 @@ next_sector: if (sector & ((1<<(align+3))-1)) break; + if (discard_granularity && size == discard_granularity) + break; + /* do not cross extent boundaries */ if (((bit+1) & BM_BLOCKS_PER_BM_EXT_MASK) == 0) break; @@ -711,7 +734,8 @@ next_sector: int err; inc_rs_pending(device); - err = drbd_send_drequest(peer_device, P_RS_DATA_REQUEST, + err = drbd_send_drequest(peer_device, + size == discard_granularity ? P_RS_THIN_REQ : P_RS_DATA_REQUEST, sector, size, ID_SYNCER); if (err) { drbd_err(device, "drbd_send_drequest() failed, aborting...\n"); @@ -828,6 +852,7 @@ static void ping_peer(struct drbd_device *device) int drbd_resync_finished(struct drbd_device *device) { + struct drbd_connection *connection = first_peer_device(device)->connection; unsigned long db, dt, dbdt; unsigned long n_oos; union drbd_state os, ns; @@ -849,8 +874,7 @@ int drbd_resync_finished(struct drbd_device *device) if (dw) { dw->w.cb = w_resync_finished; dw->device = device; - drbd_queue_work(&first_peer_device(device)->connection->sender_work, - &dw->w); + drbd_queue_work(&connection->sender_work, &dw->w); return 1; } drbd_err(device, "Warn failed to drbd_rs_del_all() and to kmalloc(dw).\n"); @@ -963,6 +987,30 @@ int drbd_resync_finished(struct drbd_device *device) _drbd_set_state(device, ns, CS_VERBOSE, NULL); out_unlock: spin_unlock_irq(&device->resource->req_lock); + + /* If we have been sync source, and have an effective fencing-policy, + * once *all* volumes are back in sync, call "unfence". */ + if (os.conn == C_SYNC_SOURCE) { + enum drbd_disk_state disk_state = D_MASK; + enum drbd_disk_state pdsk_state = D_MASK; + enum drbd_fencing_p fp = FP_DONT_CARE; + + rcu_read_lock(); + fp = rcu_dereference(device->ldev->disk_conf)->fencing; + if (fp != FP_DONT_CARE) { + struct drbd_peer_device *peer_device; + int vnr; + idr_for_each_entry(&connection->peer_devices, peer_device, vnr) { + struct drbd_device *device = peer_device->device; + disk_state = min_t(enum drbd_disk_state, disk_state, device->state.disk); + pdsk_state = min_t(enum drbd_disk_state, pdsk_state, device->state.pdsk); + } + } + rcu_read_unlock(); + if (disk_state == D_UP_TO_DATE && pdsk_state == D_UP_TO_DATE) + conn_khelper(connection, "unfence-peer"); + } + put_ldev(device); out: device->rs_total = 0; @@ -999,7 +1047,6 @@ static void move_to_net_ee_or_free(struct drbd_device *device, struct drbd_peer_ /** * w_e_end_data_req() - Worker callback, to send a P_DATA_REPLY packet in response to a P_DATA_REQUEST - * @device: DRBD device. * @w: work object. * @cancel: The connection will be closed anyways */ @@ -1035,6 +1082,30 @@ int w_e_end_data_req(struct drbd_work *w, int cancel) return err; } +static bool all_zero(struct drbd_peer_request *peer_req) +{ + struct page *page = peer_req->pages; + unsigned int len = peer_req->i.size; + + page_chain_for_each(page) { + unsigned int l = min_t(unsigned int, len, PAGE_SIZE); + unsigned int i, words = l / sizeof(long); + unsigned long *d; + + d = kmap_atomic(page); + for (i = 0; i < words; i++) { + if (d[i]) { + kunmap_atomic(d); + return false; + } + } + kunmap_atomic(d); + len -= l; + } + + return true; +} + /** * w_e_end_rsdata_req() - Worker callback to send a P_RS_DATA_REPLY packet in response to a P_RS_DATA_REQUEST * @w: work object. @@ -1063,7 +1134,10 @@ int w_e_end_rsdata_req(struct drbd_work *w, int cancel) } else if (likely((peer_req->flags & EE_WAS_ERROR) == 0)) { if (likely(device->state.pdsk >= D_INCONSISTENT)) { inc_rs_pending(device); - err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req); + if (peer_req->flags & EE_RS_THIN_REQ && all_zero(peer_req)) + err = drbd_send_rs_deallocated(peer_device, peer_req); + else + err = drbd_send_block(peer_device, P_RS_DATA_REPLY, peer_req); } else { if (__ratelimit(&drbd_ratelimit_state)) drbd_err(device, "Not sending RSDataReply, " @@ -1633,7 +1707,7 @@ static bool use_checksum_based_resync(struct drbd_connection *connection, struct rcu_read_unlock(); return connection->agreed_pro_version >= 89 && /* supported? */ connection->csums_tfm && /* configured? */ - (csums_after_crash_only == 0 /* use for each resync? */ + (csums_after_crash_only == false /* use for each resync? */ || test_bit(CRASHED_PRIMARY, &device->flags)); /* or only after Primary crash? */ } @@ -1768,7 +1842,7 @@ void drbd_start_resync(struct drbd_device *device, enum drbd_conns side) device->bm_resync_fo = 0; device->use_csums = use_checksum_based_resync(connection, device); } else { - device->use_csums = 0; + device->use_csums = false; } /* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index b206115d7..e3d8e4ced 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -3818,8 +3818,9 @@ static int __floppy_read_block_0(struct block_device *bdev, int drive) bio.bi_flags |= (1 << BIO_QUIET); bio.bi_private = &cbdata; bio.bi_end_io = floppy_rb0_cb; + bio_set_op_attrs(&bio, REQ_OP_READ, 0); - submit_bio(READ, &bio); + submit_bio(&bio); process_fd_request(); init_completion(&cbdata.complete); @@ -4345,8 +4346,7 @@ static int __init do_floppy_init(void) /* to be cleaned up... */ disks[drive]->private_data = (void *)(long)drive; disks[drive]->flags |= GENHD_FL_REMOVABLE; - disks[drive]->driverfs_dev = &floppy_device[drive].dev; - add_disk(disks[drive]); + device_add_disk(&floppy_device[drive].dev, disks[drive]); } return 0; diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 7339e65f6..005e2921a 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -447,7 +447,7 @@ static int lo_req_flush(struct loop_device *lo, struct request *rq) static inline void handle_partial_read(struct loop_cmd *cmd, long bytes) { - if (bytes < 0 || (cmd->rq->cmd_flags & REQ_WRITE)) + if (bytes < 0 || op_is_write(req_op(cmd->rq))) return; if (unlikely(bytes < blk_rq_bytes(cmd->rq))) { @@ -510,14 +510,10 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, return 0; } - -static inline int lo_rw_simple(struct loop_device *lo, - struct request *rq, loff_t pos, bool rw) +static int do_req_filebacked(struct loop_device *lo, struct request *rq) { struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); - - if (cmd->use_aio) - return lo_rw_aio(lo, cmd, pos, rw); + loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset; /* * lo_write_simple and lo_read_simple should have been covered @@ -528,37 +524,30 @@ static inline int lo_rw_simple(struct loop_device *lo, * of the req at one time. And direct read IO doesn't need to * run flush_dcache_page(). */ - if (rw == WRITE) - return lo_write_simple(lo, rq, pos); - else - return lo_read_simple(lo, rq, pos); -} - -static int do_req_filebacked(struct loop_device *lo, struct request *rq) -{ - loff_t pos; - int ret; - - pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset; - - if (rq->cmd_flags & REQ_WRITE) { - if (rq->cmd_flags & REQ_FLUSH) - ret = lo_req_flush(lo, rq); - else if (rq->cmd_flags & REQ_DISCARD) - ret = lo_discard(lo, rq, pos); - else if (lo->transfer) - ret = lo_write_transfer(lo, rq, pos); + switch (req_op(rq)) { + case REQ_OP_FLUSH: + return lo_req_flush(lo, rq); + case REQ_OP_DISCARD: + return lo_discard(lo, rq, pos); + case REQ_OP_WRITE: + if (lo->transfer) + return lo_write_transfer(lo, rq, pos); + else if (cmd->use_aio) + return lo_rw_aio(lo, cmd, pos, WRITE); else - ret = lo_rw_simple(lo, rq, pos, WRITE); - - } else { + return lo_write_simple(lo, rq, pos); + case REQ_OP_READ: if (lo->transfer) - ret = lo_read_transfer(lo, rq, pos); + return lo_read_transfer(lo, rq, pos); + else if (cmd->use_aio) + return lo_rw_aio(lo, cmd, pos, READ); else - ret = lo_rw_simple(lo, rq, pos, READ); + return lo_read_simple(lo, rq, pos); + default: + WARN_ON_ONCE(1); + return -EIO; + break; } - - return ret; } struct switch_request { @@ -1677,11 +1666,15 @@ static int loop_queue_rq(struct blk_mq_hw_ctx *hctx, if (lo->lo_state != Lo_bound) return -EIO; - if (lo->use_dio && !(cmd->rq->cmd_flags & (REQ_FLUSH | - REQ_DISCARD))) - cmd->use_aio = true; - else + switch (req_op(cmd->rq)) { + case REQ_OP_FLUSH: + case REQ_OP_DISCARD: cmd->use_aio = false; + break; + default: + cmd->use_aio = lo->use_dio; + break; + } queue_kthread_work(&lo->worker, &cmd->work); @@ -1690,7 +1683,7 @@ static int loop_queue_rq(struct blk_mq_hw_ctx *hctx, static void loop_handle_cmd(struct loop_cmd *cmd) { - const bool write = cmd->rq->cmd_flags & REQ_WRITE; + const bool write = op_is_write(req_op(cmd->rq)); struct loop_device *lo = cmd->rq->q->queuedata; int ret = 0; @@ -1783,6 +1776,7 @@ static int loop_add(struct loop_device **l, int i) */ queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, lo->lo_queue); + err = -ENOMEM; disk = lo->lo_disk = alloc_disk(1 << part_shift); if (!disk) goto out_free_queue; diff --git a/drivers/block/mg_disk.c b/drivers/block/mg_disk.c index 145ce2aa2..e937fcf71 100644 --- a/drivers/block/mg_disk.c +++ b/drivers/block/mg_disk.c @@ -687,15 +687,13 @@ static unsigned int mg_issue_req(struct request *req, unsigned int sect_num, unsigned int sect_cnt) { - switch (rq_data_dir(req)) { - case READ: + if (rq_data_dir(req) == READ) { if (mg_out(host, sect_num, sect_cnt, MG_CMD_RD, &mg_read_intr) != MG_ERR_NONE) { mg_bad_rw_intr(host); return host->error; } - break; - case WRITE: + } else { /* TODO : handler */ outb(ATA_NIEN, (unsigned long)host->dev_base + MG_REG_DRV_CTRL); if (mg_out(host, sect_num, sect_cnt, MG_CMD_WR, &mg_write_intr) @@ -714,7 +712,6 @@ static unsigned int mg_issue_req(struct request *req, mod_timer(&host->timer, jiffies + 3 * HZ); outb(MG_CMD_WR_CONF, (unsigned long)host->dev_base + MG_REG_COMMAND); - break; } return MG_ERR_NONE; } @@ -1018,7 +1015,7 @@ probe_err_7: probe_err_6: blk_cleanup_queue(host->breq); probe_err_5: - unregister_blkdev(MG_DISK_MAJ, MG_DISK_NAME); + unregister_blkdev(host->major, MG_DISK_NAME); probe_err_4: if (!prv_data->use_polling) free_irq(host->irq, host); diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 6053e4659..2aca98e8e 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3765,7 +3765,7 @@ static int mtip_submit_request(struct blk_mq_hw_ctx *hctx, struct request *rq) return -ENODATA; } - if (rq->cmd_flags & REQ_DISCARD) { + if (req_op(rq) == REQ_OP_DISCARD) { int err; err = mtip_send_trim(dd, blk_rq_pos(rq), blk_rq_sectors(rq)); @@ -3956,7 +3956,6 @@ static int mtip_block_initialize(struct driver_data *dd) if (rv) goto disk_index_error; - dd->disk->driverfs_dev = &dd->pdev->dev; dd->disk->major = dd->major; dd->disk->first_minor = index * MTIP_MAX_MINORS; dd->disk->minors = MTIP_MAX_MINORS; @@ -4008,7 +4007,7 @@ skip_create_disk: /* * if rebuild pending, start the service thread, and delay the block - * queue creation and add_disk() + * queue creation and device_add_disk() */ if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) goto start_service_thread; @@ -4042,7 +4041,7 @@ skip_create_disk: set_capacity(dd->disk, capacity); /* Enable the block device and add it to /dev */ - add_disk(dd->disk); + device_add_disk(&dd->pdev->dev, dd->disk); dd->bdev = bdget_disk(dd->disk, 0); /* diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 6a48ed419..a9e398019 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -282,9 +282,9 @@ static int nbd_send_req(struct nbd_device *nbd, struct request *req) if (req->cmd_type == REQ_TYPE_DRV_PRIV) type = NBD_CMD_DISC; - else if (req->cmd_flags & REQ_DISCARD) + else if (req_op(req) == REQ_OP_DISCARD) type = NBD_CMD_TRIM; - else if (req->cmd_flags & REQ_FLUSH) + else if (req_op(req) == REQ_OP_FLUSH) type = NBD_CMD_FLUSH; else if (rq_data_dir(req) == WRITE) type = NBD_CMD_WRITE; @@ -451,14 +451,9 @@ static int nbd_thread_recv(struct nbd_device *nbd, struct block_device *bdev) sk_set_memalloc(nbd->sock->sk); - nbd->task_recv = current; - ret = device_create_file(disk_to_dev(nbd->disk), &pid_attr); if (ret) { dev_err(disk_to_dev(nbd->disk), "device_create_file failed!\n"); - - nbd->task_recv = NULL; - return ret; } @@ -477,9 +472,6 @@ static int nbd_thread_recv(struct nbd_device *nbd, struct block_device *bdev) nbd_size_clear(nbd, bdev); device_remove_file(disk_to_dev(nbd->disk), &pid_attr); - - nbd->task_recv = NULL; - return ret; } @@ -788,6 +780,8 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, if (!nbd->sock) return -EINVAL; + /* We have to claim the device under the lock */ + nbd->task_recv = current; mutex_unlock(&nbd->tx_lock); nbd_parse_flags(nbd, bdev); @@ -796,6 +790,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, nbd_name(nbd)); if (IS_ERR(thread)) { mutex_lock(&nbd->tx_lock); + nbd->task_recv = NULL; return PTR_ERR(thread); } @@ -805,6 +800,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, kthread_stop(thread); mutex_lock(&nbd->tx_lock); + nbd->task_recv = NULL; sock_shutdown(nbd); nbd_clear_que(nbd); diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c index cab97593b..75a7f88d6 100644 --- a/drivers/block/null_blk.c +++ b/drivers/block/null_blk.c @@ -448,7 +448,7 @@ static int null_lnvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) struct request *rq; struct bio *bio = rqd->bio; - rq = blk_mq_alloc_request(q, bio_rw(bio), 0); + rq = blk_mq_alloc_request(q, bio_data_dir(bio), 0); if (IS_ERR(rq)) return -ENOMEM; diff --git a/drivers/block/osdblk.c b/drivers/block/osdblk.c index c2854a2bf..92900f5f0 100644 --- a/drivers/block/osdblk.c +++ b/drivers/block/osdblk.c @@ -321,7 +321,7 @@ static void osdblk_rq_fn(struct request_queue *q) * driver-specific, etc. */ - do_flush = rq->cmd_flags & REQ_FLUSH; + do_flush = (req_op(rq) == REQ_OP_FLUSH); do_write = (rq_data_dir(rq) == WRITE); if (!do_flush) { /* osd_flush does not use a bio */ diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index d06c62ecc..90fa4ac14 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -1074,7 +1074,7 @@ static void pkt_gather_data(struct pktcdvd_device *pd, struct packet_data *pkt) BUG(); atomic_inc(&pkt->io_wait); - bio->bi_rw = READ; + bio_set_op_attrs(bio, REQ_OP_READ, 0); pkt_queue_bio(pd, bio); frames_read++; } @@ -1157,7 +1157,7 @@ static int pkt_start_recovery(struct packet_data *pkt) bio_reset(pkt->bio); pkt->bio->bi_bdev = pd->bdev; - pkt->bio->bi_rw = REQ_WRITE; + bio_set_op_attrs(pkt->bio, REQ_OP_WRITE, 0); pkt->bio->bi_iter.bi_sector = new_sector; pkt->bio->bi_iter.bi_size = pkt->frames * CD_FRAMESIZE; pkt->bio->bi_vcnt = pkt->frames; @@ -1336,7 +1336,7 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt) /* Start the write request */ atomic_set(&pkt->io_wait, 1); - pkt->w_bio->bi_rw = WRITE; + bio_set_op_attrs(pkt->w_bio, REQ_OP_WRITE, 0); pkt_queue_bio(pd, pkt->w_bio); } diff --git a/drivers/block/ps3disk.c b/drivers/block/ps3disk.c index 4b7e40583..76f33c84c 100644 --- a/drivers/block/ps3disk.c +++ b/drivers/block/ps3disk.c @@ -196,7 +196,7 @@ static void ps3disk_do_request(struct ps3_storage_device *dev, dev_dbg(&dev->sbd.core, "%s:%u\n", __func__, __LINE__); while ((req = blk_fetch_request(q))) { - if (req->cmd_flags & REQ_FLUSH) { + if (req_op(req) == REQ_OP_FLUSH) { if (ps3disk_submit_flush_request(dev, req)) break; } else if (req->cmd_type == REQ_TYPE_FS) { @@ -256,7 +256,7 @@ static irqreturn_t ps3disk_interrupt(int irq, void *data) return IRQ_HANDLED; } - if (req->cmd_flags & REQ_FLUSH) { + if (req_op(req) == REQ_OP_FLUSH) { read = 0; op = "flush"; } else { @@ -487,7 +487,6 @@ static int ps3disk_probe(struct ps3_system_bus_device *_dev) gendisk->fops = &ps3disk_fops; gendisk->queue = queue; gendisk->private_data = dev; - gendisk->driverfs_dev = &dev->sbd.core; snprintf(gendisk->disk_name, sizeof(gendisk->disk_name), PS3DISK_NAME, devidx+'a'); priv->blocking_factor = dev->blk_size >> 9; @@ -499,7 +498,7 @@ static int ps3disk_probe(struct ps3_system_bus_device *_dev) gendisk->disk_name, priv->model, priv->raw_capacity >> 11, get_capacity(gendisk) >> 11); - add_disk(gendisk); + device_add_disk(&dev->sbd.core, gendisk); return 0; fail_cleanup_queue: diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c index 56847fcda..456b4fe21 100644 --- a/drivers/block/ps3vram.c +++ b/drivers/block/ps3vram.c @@ -773,14 +773,13 @@ static int ps3vram_probe(struct ps3_system_bus_device *dev) gendisk->fops = &ps3vram_fops; gendisk->queue = queue; gendisk->private_data = dev; - gendisk->driverfs_dev = &dev->core; strlcpy(gendisk->disk_name, DEVICE_NAME, sizeof(gendisk->disk_name)); set_capacity(gendisk, priv->size >> 9); dev_info(&dev->core, "%s: Using %lu MiB of GPU memory\n", gendisk->disk_name, get_capacity(gendisk) >> 11); - add_disk(gendisk); + device_add_disk(&dev->core, gendisk); return 0; fail_cleanup_queue: diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 81666a564..6c6519f64 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -1937,7 +1937,7 @@ static struct ceph_osd_request *rbd_osd_req_create( osd_req->r_callback = rbd_osd_req_callback; osd_req->r_priv = obj_request; - osd_req->r_base_oloc.pool = ceph_file_layout_pg_pool(rbd_dev->layout); + osd_req->r_base_oloc.pool = rbd_dev->layout.pool_id; if (ceph_oid_aprintf(&osd_req->r_base_oid, GFP_NOIO, "%s", obj_request->object_name)) goto fail; @@ -1991,7 +1991,7 @@ rbd_osd_req_create_copyup(struct rbd_obj_request *obj_request) osd_req->r_callback = rbd_osd_req_callback; osd_req->r_priv = obj_request; - osd_req->r_base_oloc.pool = ceph_file_layout_pg_pool(rbd_dev->layout); + osd_req->r_base_oloc.pool = rbd_dev->layout.pool_id; if (ceph_oid_aprintf(&osd_req->r_base_oid, GFP_NOIO, "%s", obj_request->object_name)) goto fail; @@ -3286,9 +3286,9 @@ static void rbd_queue_workfn(struct work_struct *work) goto err; } - if (rq->cmd_flags & REQ_DISCARD) + if (req_op(rq) == REQ_OP_DISCARD) op_type = OBJ_OP_DISCARD; - else if (rq->cmd_flags & REQ_WRITE) + else if (req_op(rq) == REQ_OP_WRITE) op_type = OBJ_OP_WRITE; else op_type = OBJ_OP_READ; @@ -3950,6 +3950,7 @@ static void rbd_dev_release(struct device *dev) bool need_put = !!rbd_dev->opts; ceph_oid_destroy(&rbd_dev->header_oid); + ceph_oloc_destroy(&rbd_dev->header_oloc); rbd_put_client(rbd_dev->rbd_client); rbd_spec_put(rbd_dev->spec); @@ -3995,10 +3996,11 @@ static struct rbd_device *rbd_dev_create(struct rbd_client *rbdc, /* Initialize the layout used for all rbd requests */ - rbd_dev->layout.fl_stripe_unit = cpu_to_le32(1 << RBD_MAX_OBJ_ORDER); - rbd_dev->layout.fl_stripe_count = cpu_to_le32(1); - rbd_dev->layout.fl_object_size = cpu_to_le32(1 << RBD_MAX_OBJ_ORDER); - rbd_dev->layout.fl_pg_pool = cpu_to_le32((u32) spec->pool_id); + rbd_dev->layout.stripe_unit = 1 << RBD_MAX_OBJ_ORDER; + rbd_dev->layout.stripe_count = 1; + rbd_dev->layout.object_size = 1 << RBD_MAX_OBJ_ORDER; + rbd_dev->layout.pool_id = spec->pool_id; + RCU_INIT_POINTER(rbd_dev->layout.pool_ns, NULL); /* * If this is a mapping rbd_dev (as opposed to a parent one), @@ -5187,7 +5189,7 @@ static int rbd_dev_header_name(struct rbd_device *rbd_dev) rbd_assert(rbd_image_format_valid(rbd_dev->image_format)); - rbd_dev->header_oloc.pool = ceph_file_layout_pg_pool(rbd_dev->layout); + rbd_dev->header_oloc.pool = rbd_dev->layout.pool_id; if (rbd_dev->image_format == 1) ret = ceph_oid_aprintf(&rbd_dev->header_oid, GFP_KERNEL, "%s%s", spec->image_name, RBD_SUFFIX); @@ -5335,15 +5337,6 @@ static ssize_t do_rbd_add(struct bus_type *bus, } spec->pool_id = (u64)rc; - /* The ceph file layout needs to fit pool id in 32 bits */ - - if (spec->pool_id > (u64)U32_MAX) { - rbd_warn(NULL, "pool id too large (%llu > %u)", - (unsigned long long)spec->pool_id, U32_MAX); - rc = -EIO; - goto err_out_client; - } - rbd_dev = rbd_dev_create(rbdc, spec, rbd_opts); if (!rbd_dev) { rc = -ENOMEM; diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c index e1b8b7061..f81d70b39 100644 --- a/drivers/block/rsxx/dev.c +++ b/drivers/block/rsxx/dev.c @@ -230,8 +230,7 @@ int rsxx_attach_dev(struct rsxx_cardinfo *card) set_capacity(card->gendisk, card->size8 >> 9); else set_capacity(card->gendisk, 0); - add_disk(card->gendisk); - + device_add_disk(CARD_TO_DEV(card), card->gendisk); card->bdev_attached = 1; } @@ -308,7 +307,6 @@ int rsxx_setup_dev(struct rsxx_cardinfo *card) snprintf(card->gendisk->disk_name, sizeof(card->gendisk->disk_name), "rsxx%d", card->disk_id); - card->gendisk->driverfs_dev = &card->dev->dev; card->gendisk->major = card->major; card->gendisk->first_minor = 0; card->gendisk->fops = &rsxx_fops; diff --git a/drivers/block/rsxx/dma.c b/drivers/block/rsxx/dma.c index cf8cd293a..5a20385f8 100644 --- a/drivers/block/rsxx/dma.c +++ b/drivers/block/rsxx/dma.c @@ -705,7 +705,7 @@ int rsxx_dma_queue_bio(struct rsxx_cardinfo *card, dma_cnt[i] = 0; } - if (bio->bi_rw & REQ_DISCARD) { + if (bio_op(bio) == REQ_OP_DISCARD) { bv_len = bio->bi_iter.bi_size; while (bv_len > 0) { diff --git a/drivers/block/skd_main.c b/drivers/block/skd_main.c index 910e06591..3822eae10 100644 --- a/drivers/block/skd_main.c +++ b/drivers/block/skd_main.c @@ -597,7 +597,7 @@ static void skd_request_fn(struct request_queue *q) data_dir = rq_data_dir(req); io_flags = req->cmd_flags; - if (io_flags & REQ_FLUSH) + if (req_op(req) == REQ_OP_FLUSH) flush++; if (io_flags & REQ_FUA) @@ -4690,10 +4690,10 @@ static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo) return -EIO; } -static int skd_bdev_attach(struct skd_device *skdev) +static int skd_bdev_attach(struct device *parent, struct skd_device *skdev) { pr_debug("%s:%s:%d add_disk\n", skdev->name, __func__, __LINE__); - add_disk(skdev->disk); + device_add_disk(parent, skdev->disk); return 0; } @@ -4812,8 +4812,6 @@ static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_drvdata(pdev, skdev); - skdev->disk->driverfs_dev = &pdev->dev; - for (i = 0; i < SKD_MAX_BARS; i++) { skdev->mem_phys[i] = pci_resource_start(pdev, i); skdev->mem_size[i] = (u32)pci_resource_len(pdev, i); @@ -4851,7 +4849,7 @@ static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) (SKD_START_WAIT_SECONDS * HZ)); if (skdev->gendisk_on > 0) { /* device came on-line after reset */ - skd_bdev_attach(skdev); + skd_bdev_attach(&pdev->dev, skdev); rc = 0; } else { /* we timed out, something is wrong with the device, diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c index 4b911ed96..cab157331 100644 --- a/drivers/block/sunvdc.c +++ b/drivers/block/sunvdc.c @@ -804,7 +804,6 @@ static int probe_disk(struct vdc_port *port) g->fops = &vdc_fops; g->queue = q; g->private_data = port; - g->driverfs_dev = &port->vio.vdev->dev; set_capacity(g, port->vdisk_size); @@ -835,7 +834,7 @@ static int probe_disk(struct vdc_port *port) port->vdisk_size, (port->vdisk_size >> (20 - 9)), port->vio.ver.major, port->vio.ver.minor); - add_disk(g); + device_add_disk(&port->vio.vdev->dev, g); return 0; } diff --git a/drivers/block/umem.c b/drivers/block/umem.c index 7939b9f87..be90e1585 100644 --- a/drivers/block/umem.c +++ b/drivers/block/umem.c @@ -344,7 +344,6 @@ static int add_bio(struct cardinfo *card) int offset; struct bio *bio; struct bio_vec vec; - int rw; bio = card->currentbio; if (!bio && card->bio) { @@ -359,7 +358,6 @@ static int add_bio(struct cardinfo *card) if (!bio) return 0; - rw = bio_rw(bio); if (card->mm_pages[card->Ready].cnt >= DESC_PER_PAGE) return 0; @@ -369,7 +367,7 @@ static int add_bio(struct cardinfo *card) vec.bv_page, vec.bv_offset, vec.bv_len, - (rw == READ) ? + bio_op(bio) == REQ_OP_READ ? PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE); p = &card->mm_pages[card->Ready]; @@ -398,7 +396,7 @@ static int add_bio(struct cardinfo *card) DMASCR_CHAIN_EN | DMASCR_SEM_EN | pci_cmds); - if (rw == WRITE) + if (bio_op(bio) == REQ_OP_WRITE) desc->control_bits |= cpu_to_le32(DMASCR_TRANSFER_READ); desc->sem_control_bits = desc->control_bits; @@ -462,7 +460,7 @@ static void process_page(unsigned long data) le32_to_cpu(desc->local_addr)>>9, le32_to_cpu(desc->transfer_size)); dump_dmastat(card, control); - } else if ((bio->bi_rw & REQ_WRITE) && + } else if (op_is_write(bio_op(bio)) && le32_to_cpu(desc->local_addr) >> 9 == card->init_size) { card->init_size += le32_to_cpu(desc->transfer_size) >> 9; @@ -537,7 +535,7 @@ static blk_qc_t mm_make_request(struct request_queue *q, struct bio *bio) *card->biotail = bio; bio->bi_next = NULL; card->biotail = &bio->bi_next; - if (bio->bi_rw & REQ_SYNC || !mm_check_plugged(card)) + if (bio->bi_opf & REQ_SYNC || !mm_check_plugged(card)) activate(card); spin_unlock_irq(&card->lock); diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 42758b527..93b1aaa5b 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -172,7 +172,7 @@ static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems); vbr->req = req; - if (req->cmd_flags & REQ_FLUSH) { + if (req_op(req) == REQ_OP_FLUSH) { vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_FLUSH); vbr->out_hdr.sector = 0; vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req)); @@ -236,25 +236,22 @@ static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, static int virtblk_get_id(struct gendisk *disk, char *id_str) { struct virtio_blk *vblk = disk->private_data; + struct request_queue *q = vblk->disk->queue; struct request *req; - struct bio *bio; int err; - bio = bio_map_kern(vblk->disk->queue, id_str, VIRTIO_BLK_ID_BYTES, - GFP_KERNEL); - if (IS_ERR(bio)) - return PTR_ERR(bio); - - req = blk_make_request(vblk->disk->queue, bio, GFP_KERNEL); - if (IS_ERR(req)) { - bio_put(bio); + req = blk_get_request(q, READ, GFP_KERNEL); + if (IS_ERR(req)) return PTR_ERR(req); - } - req->cmd_type = REQ_TYPE_DRV_PRIV; + + err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL); + if (err) + goto out; + err = blk_execute_rq(vblk->disk->queue, vblk->disk, req, false); +out: blk_put_request(req); - return err; } @@ -394,22 +391,16 @@ static int init_vq(struct virtio_blk *vblk) num_vqs = 1; vblk->vqs = kmalloc(sizeof(*vblk->vqs) * num_vqs, GFP_KERNEL); - if (!vblk->vqs) { - err = -ENOMEM; - goto out; - } + if (!vblk->vqs) + return -ENOMEM; names = kmalloc(sizeof(*names) * num_vqs, GFP_KERNEL); - if (!names) - goto err_names; - callbacks = kmalloc(sizeof(*callbacks) * num_vqs, GFP_KERNEL); - if (!callbacks) - goto err_callbacks; - vqs = kmalloc(sizeof(*vqs) * num_vqs, GFP_KERNEL); - if (!vqs) - goto err_vqs; + if (!names || !callbacks || !vqs) { + err = -ENOMEM; + goto out; + } for (i = 0; i < num_vqs; i++) { callbacks[i] = virtblk_done; @@ -420,7 +411,7 @@ static int init_vq(struct virtio_blk *vblk) /* Discover virtqueues and write information to configuration. */ err = vdev->config->find_vqs(vdev, num_vqs, vqs, callbacks, names); if (err) - goto err_find_vqs; + goto out; for (i = 0; i < num_vqs; i++) { spin_lock_init(&vblk->vqs[i].lock); @@ -428,16 +419,12 @@ static int init_vq(struct virtio_blk *vblk) } vblk->num_vqs = num_vqs; - err_find_vqs: +out: kfree(vqs); - err_vqs: kfree(callbacks); - err_callbacks: kfree(names); - err_names: if (err) kfree(vblk->vqs); - out: return err; } @@ -656,7 +643,6 @@ static int virtblk_probe(struct virtio_device *vdev) vblk->disk->first_minor = index_to_minor(index); vblk->disk->private_data = vblk; vblk->disk->fops = &virtblk_fops; - vblk->disk->driverfs_dev = &vdev->dev; vblk->disk->flags |= GENHD_FL_EXT_DEVT; vblk->index = index; @@ -733,7 +719,7 @@ static int virtblk_probe(struct virtio_device *vdev) virtio_device_ready(vdev); - add_disk(vblk->disk); + device_add_disk(&vdev->dev, vblk->disk); err = device_create_file(disk_to_dev(vblk->disk), &dev_attr_serial); if (err) goto out_del_disk; diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index 4809c1501..4a80ee752 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c @@ -501,7 +501,7 @@ static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif, struct xen_vbd *vbd = &blkif->vbd; int rc = -EACCES; - if ((operation != READ) && vbd->readonly) + if ((operation != REQ_OP_READ) && vbd->readonly) goto out; if (likely(req->nr_sects)) { @@ -1014,7 +1014,7 @@ static int dispatch_discard_io(struct xen_blkif_ring *ring, preq.sector_number = req->u.discard.sector_number; preq.nr_sects = req->u.discard.nr_sectors; - err = xen_vbd_translate(&preq, blkif, WRITE); + err = xen_vbd_translate(&preq, blkif, REQ_OP_WRITE); if (err) { pr_warn("access denied: DISCARD [%llu->%llu] on dev=%04x\n", preq.sector_number, @@ -1229,6 +1229,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, struct bio **biolist = pending_req->biolist; int i, nbio = 0; int operation; + int operation_flags = 0; struct blk_plug plug; bool drain = false; struct grant_page **pages = pending_req->segments; @@ -1247,17 +1248,19 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, switch (req_operation) { case BLKIF_OP_READ: ring->st_rd_req++; - operation = READ; + operation = REQ_OP_READ; break; case BLKIF_OP_WRITE: ring->st_wr_req++; - operation = WRITE_ODIRECT; + operation = REQ_OP_WRITE; + operation_flags = WRITE_ODIRECT; break; case BLKIF_OP_WRITE_BARRIER: drain = true; case BLKIF_OP_FLUSH_DISKCACHE: ring->st_f_req++; - operation = WRITE_FLUSH; + operation = REQ_OP_WRITE; + operation_flags = WRITE_FLUSH; break; default: operation = 0; /* make gcc happy */ @@ -1269,7 +1272,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, nseg = req->operation == BLKIF_OP_INDIRECT ? req->u.indirect.nr_segments : req->u.rw.nr_segments; - if (unlikely(nseg == 0 && operation != WRITE_FLUSH) || + if (unlikely(nseg == 0 && operation_flags != WRITE_FLUSH) || unlikely((req->operation != BLKIF_OP_INDIRECT) && (nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) || unlikely((req->operation == BLKIF_OP_INDIRECT) && @@ -1310,7 +1313,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, if (xen_vbd_translate(&preq, ring->blkif, operation) != 0) { pr_debug("access denied: %s of [%llu,%llu] on dev=%04x\n", - operation == READ ? "read" : "write", + operation == REQ_OP_READ ? "read" : "write", preq.sector_number, preq.sector_number + preq.nr_sects, ring->blkif->vbd.pdevice); @@ -1369,6 +1372,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, bio->bi_private = pending_req; bio->bi_end_io = end_block_io_op; bio->bi_iter.bi_sector = preq.sector_number; + bio_set_op_attrs(bio, operation, operation_flags); } preq.sector_number += seg[i].nsec; @@ -1376,7 +1380,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, /* This will be hit if the operation was a flush or discard. */ if (!bio) { - BUG_ON(operation != WRITE_FLUSH); + BUG_ON(operation_flags != WRITE_FLUSH); bio = bio_alloc(GFP_KERNEL, 0); if (unlikely(bio == NULL)) @@ -1386,20 +1390,21 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, bio->bi_bdev = preq.bdev; bio->bi_private = pending_req; bio->bi_end_io = end_block_io_op; + bio_set_op_attrs(bio, operation, operation_flags); } atomic_set(&pending_req->pendcnt, nbio); blk_start_plug(&plug); for (i = 0; i < nbio; i++) - submit_bio(operation, biolist[i]); + submit_bio(biolist[i]); /* Let the I/Os go.. */ blk_finish_plug(&plug); - if (operation == READ) + if (operation == REQ_OP_READ) ring->st_rd_sect += preq.nr_sects; - else if (operation & WRITE) + else if (operation == REQ_OP_WRITE) ring->st_wr_sect += preq.nr_sects; return 0; diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c index 3355f1cdd..3cc6d1d86 100644 --- a/drivers/block/xen-blkback/xenbus.c +++ b/drivers/block/xen-blkback/xenbus.c @@ -379,7 +379,7 @@ static struct attribute *xen_vbdstat_attrs[] = { NULL }; -static struct attribute_group xen_vbdstat_group = { +static const struct attribute_group xen_vbdstat_group = { .name = "statistics", .attrs = xen_vbdstat_attrs, }; @@ -480,7 +480,7 @@ static int xen_vbd_create(struct xen_blkif *blkif, blkif_vdev_t handle, if (q && test_bit(QUEUE_FLAG_WC, &q->queue_flags)) vbd->flush_support = true; - if (q && blk_queue_secdiscard(q)) + if (q && blk_queue_secure_erase(q)) vbd->discard_secure = true; pr_debug("Successful creation of handle=%04x (dom=%u)\n", @@ -715,8 +715,11 @@ static void backend_changed(struct xenbus_watch *watch, /* Front end dir is a number, which is used as the handle. */ err = kstrtoul(strrchr(dev->otherend, '/') + 1, 0, &handle); - if (err) + if (err) { + kfree(be->mode); + be->mode = NULL; return; + } be->major = major; be->minor = minor; @@ -1022,9 +1025,9 @@ static int connect_ring(struct backend_info *be) pr_debug("%s %s\n", __func__, dev->otherend); be->blkif->blk_protocol = BLKIF_PROTOCOL_DEFAULT; - err = xenbus_gather(XBT_NIL, dev->otherend, "protocol", - "%63s", protocol, NULL); - if (err) + err = xenbus_scanf(XBT_NIL, dev->otherend, "protocol", + "%63s", protocol); + if (err <= 0) strcpy(protocol, "unspecified, assuming default"); else if (0 == strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE)) be->blkif->blk_protocol = BLKIF_PROTOCOL_NATIVE; @@ -1036,10 +1039,9 @@ static int connect_ring(struct backend_info *be) xenbus_dev_fatal(dev, err, "unknown fe protocol %s", protocol); return -ENOSYS; } - err = xenbus_gather(XBT_NIL, dev->otherend, - "feature-persistent", "%u", - &pers_grants, NULL); - if (err) + err = xenbus_scanf(XBT_NIL, dev->otherend, + "feature-persistent", "%u", &pers_grants); + if (err <= 0) pers_grants = 0; be->blkif->vbd.feature_gnt_persistent = pers_grants; diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index fcc5b4e0a..88ef6d472 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -189,6 +189,8 @@ struct blkfront_info struct mutex mutex; struct xenbus_device *xbdev; struct gendisk *gd; + u16 sector_size; + unsigned int physical_sector_size; int vdevice; blkif_vdev_t handle; enum blkif_state connected; @@ -196,6 +198,7 @@ struct blkfront_info unsigned int nr_ring_pages; struct request_queue *rq; unsigned int feature_flush; + unsigned int feature_fua; unsigned int feature_discard:1; unsigned int feature_secdiscard:1; unsigned int discard_granularity; @@ -547,7 +550,7 @@ static int blkif_queue_discard_req(struct request *req, struct blkfront_ring_inf ring_req->u.discard.nr_sectors = blk_rq_sectors(req); ring_req->u.discard.id = id; ring_req->u.discard.sector_number = (blkif_sector_t)blk_rq_pos(req); - if ((req->cmd_flags & REQ_SECURE) && info->feature_secdiscard) + if (req_op(req) == REQ_OP_SECURE_ERASE && info->feature_secdiscard) ring_req->u.discard.flag = BLKIF_DISCARD_SECURE; else ring_req->u.discard.flag = 0; @@ -746,7 +749,7 @@ static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *ri * The indirect operation can only be a BLKIF_OP_READ or * BLKIF_OP_WRITE */ - BUG_ON(req->cmd_flags & (REQ_FLUSH | REQ_FUA)); + BUG_ON(req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA); ring_req->operation = BLKIF_OP_INDIRECT; ring_req->u.indirect.indirect_op = rq_data_dir(req) ? BLKIF_OP_WRITE : BLKIF_OP_READ; @@ -758,7 +761,7 @@ static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *ri ring_req->u.rw.handle = info->handle; ring_req->operation = rq_data_dir(req) ? BLKIF_OP_WRITE : BLKIF_OP_READ; - if (req->cmd_flags & (REQ_FLUSH | REQ_FUA)) { + if (req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA) { /* * Ideally we can do an unordered flush-to-disk. * In case the backend onlysupports barriers, use that. @@ -766,19 +769,14 @@ static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *ri * implement it the same way. (It's also a FLUSH+FUA, * since it is guaranteed ordered WRT previous writes.) */ - switch (info->feature_flush & - ((REQ_FLUSH|REQ_FUA))) { - case REQ_FLUSH|REQ_FUA: + if (info->feature_flush && info->feature_fua) ring_req->operation = BLKIF_OP_WRITE_BARRIER; - break; - case REQ_FLUSH: + else if (info->feature_flush) ring_req->operation = BLKIF_OP_FLUSH_DISKCACHE; - break; - default: + else ring_req->operation = 0; - } } ring_req->u.rw.nr_segments = num_grant; if (unlikely(require_extra_req)) { @@ -847,7 +845,8 @@ static int blkif_queue_request(struct request *req, struct blkfront_ring_info *r if (unlikely(rinfo->dev_info->connected != BLKIF_STATE_CONNECTED)) return 1; - if (unlikely(req->cmd_flags & (REQ_DISCARD | REQ_SECURE))) + if (unlikely(req_op(req) == REQ_OP_DISCARD || + req_op(req) == REQ_OP_SECURE_ERASE)) return blkif_queue_discard_req(req, rinfo); else return blkif_queue_rw_req(req, rinfo); @@ -867,10 +866,10 @@ static inline bool blkif_request_flush_invalid(struct request *req, struct blkfront_info *info) { return ((req->cmd_type != REQ_TYPE_FS) || - ((req->cmd_flags & REQ_FLUSH) && - !(info->feature_flush & REQ_FLUSH)) || + ((req_op(req) == REQ_OP_FLUSH) && + !info->feature_flush) || ((req->cmd_flags & REQ_FUA) && - !(info->feature_flush & REQ_FUA))); + !info->feature_fua)); } static int blkif_queue_rq(struct blk_mq_hw_ctx *hctx, @@ -913,9 +912,45 @@ static struct blk_mq_ops blkfront_mq_ops = { .map_queue = blk_mq_map_queue, }; +static void blkif_set_queue_limits(struct blkfront_info *info) +{ + struct request_queue *rq = info->rq; + struct gendisk *gd = info->gd; + unsigned int segments = info->max_indirect_segments ? : + BLKIF_MAX_SEGMENTS_PER_REQUEST; + + queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq); + + if (info->feature_discard) { + queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, rq); + blk_queue_max_discard_sectors(rq, get_capacity(gd)); + rq->limits.discard_granularity = info->discard_granularity; + rq->limits.discard_alignment = info->discard_alignment; + if (info->feature_secdiscard) + queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, rq); + } + + /* Hard sector size and max sectors impersonate the equiv. hardware. */ + blk_queue_logical_block_size(rq, info->sector_size); + blk_queue_physical_block_size(rq, info->physical_sector_size); + blk_queue_max_hw_sectors(rq, (segments * XEN_PAGE_SIZE) / 512); + + /* Each segment in a request is up to an aligned page in size. */ + blk_queue_segment_boundary(rq, PAGE_SIZE - 1); + blk_queue_max_segment_size(rq, PAGE_SIZE); + + /* Ensure a merged request will fit in a single I/O ring slot. */ + blk_queue_max_segments(rq, segments / GRANTS_PER_PSEG); + + /* Make sure buffer addresses are sector-aligned. */ + blk_queue_dma_alignment(rq, 511); + + /* Make sure we don't use bounce buffers. */ + blk_queue_bounce_limit(rq, BLK_BOUNCE_ANY); +} + static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size, - unsigned int physical_sector_size, - unsigned int segments) + unsigned int physical_sector_size) { struct request_queue *rq; struct blkfront_info *info = gd->private_data; @@ -947,58 +982,31 @@ static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size, } rq->queuedata = info; - queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq); - - if (info->feature_discard) { - queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, rq); - blk_queue_max_discard_sectors(rq, get_capacity(gd)); - rq->limits.discard_granularity = info->discard_granularity; - rq->limits.discard_alignment = info->discard_alignment; - if (info->feature_secdiscard) - queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, rq); - } - - /* Hard sector size and max sectors impersonate the equiv. hardware. */ - blk_queue_logical_block_size(rq, sector_size); - blk_queue_physical_block_size(rq, physical_sector_size); - blk_queue_max_hw_sectors(rq, (segments * XEN_PAGE_SIZE) / 512); - - /* Each segment in a request is up to an aligned page in size. */ - blk_queue_segment_boundary(rq, PAGE_SIZE - 1); - blk_queue_max_segment_size(rq, PAGE_SIZE); - - /* Ensure a merged request will fit in a single I/O ring slot. */ - blk_queue_max_segments(rq, segments / GRANTS_PER_PSEG); - - /* Make sure buffer addresses are sector-aligned. */ - blk_queue_dma_alignment(rq, 511); - - /* Make sure we don't use bounce buffers. */ - blk_queue_bounce_limit(rq, BLK_BOUNCE_ANY); - - gd->queue = rq; + info->rq = gd->queue = rq; + info->gd = gd; + info->sector_size = sector_size; + info->physical_sector_size = physical_sector_size; + blkif_set_queue_limits(info); return 0; } -static const char *flush_info(unsigned int feature_flush) +static const char *flush_info(struct blkfront_info *info) { - switch (feature_flush & ((REQ_FLUSH | REQ_FUA))) { - case REQ_FLUSH|REQ_FUA: + if (info->feature_flush && info->feature_fua) return "barrier: enabled;"; - case REQ_FLUSH: + else if (info->feature_flush) return "flush diskcache: enabled;"; - default: + else return "barrier or flush: disabled;"; - } } static void xlvbd_flush(struct blkfront_info *info) { - blk_queue_write_cache(info->rq, info->feature_flush & REQ_FLUSH, - info->feature_flush & REQ_FUA); + blk_queue_write_cache(info->rq, info->feature_flush ? true : false, + info->feature_fua ? true : false); pr_info("blkfront: %s: %s %s %s %s %s\n", - info->gd->disk_name, flush_info(info->feature_flush), + info->gd->disk_name, flush_info(info), "persistent grants:", info->feature_persistent ? "enabled;" : "disabled;", "indirect descriptors:", info->max_indirect_segments ? "enabled;" : "disabled;"); @@ -1139,19 +1147,13 @@ static int xlvbd_alloc_gendisk(blkif_sector_t capacity, gd->first_minor = minor; gd->fops = &xlvbd_block_fops; gd->private_data = info; - gd->driverfs_dev = &(info->xbdev->dev); set_capacity(gd, capacity); - if (xlvbd_init_blk_queue(gd, sector_size, physical_sector_size, - info->max_indirect_segments ? : - BLKIF_MAX_SEGMENTS_PER_REQUEST)) { + if (xlvbd_init_blk_queue(gd, sector_size, physical_sector_size)) { del_gendisk(gd); goto release; } - info->rq = gd->queue; - info->gd = gd; - xlvbd_flush(info); if (vdisk_info & VDISK_READONLY) @@ -1321,7 +1323,7 @@ free_shadow: rinfo->ring_ref[i] = GRANT_INVALID_REF; } } - free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * PAGE_SIZE)); + free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * XEN_PAGE_SIZE)); rinfo->ring.sring = NULL; if (rinfo->irq) @@ -1597,7 +1599,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) info->feature_discard = 0; info->feature_secdiscard = 0; queue_flag_clear(QUEUE_FLAG_DISCARD, rq); - queue_flag_clear(QUEUE_FLAG_SECDISCARD, rq); + queue_flag_clear(QUEUE_FLAG_SECERASE, rq); } blk_mq_complete_request(req, error); break; @@ -1617,6 +1619,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) if (unlikely(error)) { if (error == -EOPNOTSUPP) error = 0; + info->feature_fua = 0; info->feature_flush = 0; xlvbd_flush(info); } @@ -2012,8 +2015,10 @@ static int blkif_recover(struct blkfront_info *info) struct split_bio *split_bio; blkfront_gather_backend_features(info); + /* Reset limits changed by blk_mq_update_nr_hw_queues(). */ + blkif_set_queue_limits(info); segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST; - blk_queue_max_segments(info->rq, segs); + blk_queue_max_segments(info->rq, segs / GRANTS_PER_PSEG); for (r_index = 0; r_index < info->nr_rings; r_index++) { struct blkfront_ring_info *rinfo = &info->rinfo[r_index]; @@ -2064,7 +2069,7 @@ static int blkif_recover(struct blkfront_info *info) bio_trim(cloned_bio, offset, size); cloned_bio->bi_private = split_bio; cloned_bio->bi_end_io = split_bio_end; - submit_bio(cloned_bio->bi_rw, cloned_bio); + submit_bio(cloned_bio); } /* * Now we have to wait for all those smaller bios to @@ -2073,7 +2078,7 @@ static int blkif_recover(struct blkfront_info *info) continue; } /* We don't need to split this bio */ - submit_bio(bio->bi_rw, bio); + submit_bio(bio); } return 0; @@ -2108,11 +2113,16 @@ static int blkfront_resume(struct xenbus_device *dev) /* * Get the bios in the request so we can re-queue them. */ - if (shadow[j].request->cmd_flags & - (REQ_FLUSH | REQ_FUA | REQ_DISCARD | REQ_SECURE)) { + if (req_op(shadow[i].request) == REQ_OP_FLUSH || + req_op(shadow[i].request) == REQ_OP_DISCARD || + req_op(shadow[i].request) == REQ_OP_SECURE_ERASE || + shadow[j].request->cmd_flags & REQ_FUA) { /* * Flush operations don't contain bios, so * we need to requeue the whole request + * + * XXX: but this doesn't make any sense for a + * write with the FUA flag set.. */ list_add(&shadow[j].request->queuelist, &info->requests); continue; @@ -2197,10 +2207,9 @@ static void blkfront_setup_discard(struct blkfront_info *info) info->discard_granularity = discard_granularity; info->discard_alignment = discard_alignment; } - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "discard-secure", "%d", &discard_secure, - NULL); - if (!err) + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "discard-secure", "%u", &discard_secure); + if (err > 0) info->feature_secdiscard = !!discard_secure; } @@ -2298,10 +2307,10 @@ static void blkfront_gather_backend_features(struct blkfront_info *info) unsigned int indirect_segments; info->feature_flush = 0; + info->feature_fua = 0; - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "feature-barrier", "%d", &barrier, - NULL); + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "feature-barrier", "%d", &barrier); /* * If there's no "feature-barrier" defined, then it means @@ -2310,38 +2319,40 @@ static void blkfront_gather_backend_features(struct blkfront_info *info) * * If there are barriers, then we use flush. */ - if (!err && barrier) - info->feature_flush = REQ_FLUSH | REQ_FUA; + if (err > 0 && barrier) { + info->feature_flush = 1; + info->feature_fua = 1; + } + /* * And if there is "feature-flush-cache" use that above * barriers. */ - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "feature-flush-cache", "%d", &flush, - NULL); + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "feature-flush-cache", "%d", &flush); - if (!err && flush) - info->feature_flush = REQ_FLUSH; + if (err > 0 && flush) { + info->feature_flush = 1; + info->feature_fua = 0; + } - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "feature-discard", "%d", &discard, - NULL); + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "feature-discard", "%d", &discard); - if (!err && discard) + if (err > 0 && discard) blkfront_setup_discard(info); - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "feature-persistent", "%u", &persistent, - NULL); - if (err) + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "feature-persistent", "%d", &persistent); + if (err <= 0) info->feature_persistent = 0; else info->feature_persistent = persistent; - err = xenbus_gather(XBT_NIL, info->xbdev->otherend, - "feature-max-indirect-segments", "%u", &indirect_segments, - NULL); - if (err) + err = xenbus_scanf(XBT_NIL, info->xbdev->otherend, + "feature-max-indirect-segments", "%u", + &indirect_segments); + if (err <= 0) info->max_indirect_segments = 0; else info->max_indirect_segments = min(indirect_segments, @@ -2431,7 +2442,7 @@ static void blkfront_connect(struct blkfront_info *info) if (err) { xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s", info->xbdev->otherend); - return; + goto fail; } xenbus_switch_state(info->xbdev, XenbusStateConnected); @@ -2441,9 +2452,14 @@ static void blkfront_connect(struct blkfront_info *info) for (i = 0; i < info->nr_rings; i++) kick_pending_request_queues(&info->rinfo[i]); - add_disk(info->gd); + device_add_disk(&info->xbdev->dev, info->gd); info->is_ready = 1; + return; + +fail: + blkif_free(info, 0); + return; } /** diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig index 386ba3d1a..b8ecba6dc 100644 --- a/drivers/block/zram/Kconfig +++ b/drivers/block/zram/Kconfig @@ -1,8 +1,7 @@ config ZRAM tristate "Compressed RAM block device support" - depends on BLOCK && SYSFS && ZSMALLOC - select LZO_COMPRESS - select LZO_DECOMPRESS + depends on BLOCK && SYSFS && ZSMALLOC && CRYPTO + select CRYPTO_LZO default n help Creates virtual block devices called /dev/zramX (X = 0, 1, ...). @@ -14,13 +13,3 @@ config ZRAM disks and maybe many more. See zram.txt for more information. - -config ZRAM_LZ4_COMPRESS - bool "Enable LZ4 algorithm support" - depends on ZRAM - select LZ4_COMPRESS - select LZ4_DECOMPRESS - default n - help - This option enables LZ4 compression algorithm support. Compression - algorithm can be changed using `comp_algorithm' device attribute. \ No newline at end of file diff --git a/drivers/block/zram/Makefile b/drivers/block/zram/Makefile index be0763ff5..9e2b79e9a 100644 --- a/drivers/block/zram/Makefile +++ b/drivers/block/zram/Makefile @@ -1,5 +1,3 @@ -zram-y := zcomp_lzo.o zcomp.o zram_drv.o - -zram-$(CONFIG_ZRAM_LZ4_COMPRESS) += zcomp_lz4.o +zram-y := zcomp.o zram_drv.o obj-$(CONFIG_ZRAM) += zram.o diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c index b51a816d7..4b5cd3a7b 100644 --- a/drivers/block/zram/zcomp.c +++ b/drivers/block/zram/zcomp.c @@ -14,108 +14,150 @@ #include #include #include +#include #include "zcomp.h" -#include "zcomp_lzo.h" -#ifdef CONFIG_ZRAM_LZ4_COMPRESS -#include "zcomp_lz4.h" -#endif -static struct zcomp_backend *backends[] = { - &zcomp_lzo, -#ifdef CONFIG_ZRAM_LZ4_COMPRESS - &zcomp_lz4, +static const char * const backends[] = { + "lzo", +#if IS_ENABLED(CONFIG_CRYPTO_LZ4) + "lz4", +#endif +#if IS_ENABLED(CONFIG_CRYPTO_DEFLATE) + "deflate", +#endif +#if IS_ENABLED(CONFIG_CRYPTO_LZ4HC) + "lz4hc", +#endif +#if IS_ENABLED(CONFIG_CRYPTO_842) + "842", #endif NULL }; -static struct zcomp_backend *find_backend(const char *compress) -{ - int i = 0; - while (backends[i]) { - if (sysfs_streq(compress, backends[i]->name)) - break; - i++; - } - return backends[i]; -} - -static void zcomp_strm_free(struct zcomp *comp, struct zcomp_strm *zstrm) +static void zcomp_strm_free(struct zcomp_strm *zstrm) { - if (zstrm->private) - comp->backend->destroy(zstrm->private); + if (!IS_ERR_OR_NULL(zstrm->tfm)) + crypto_free_comp(zstrm->tfm); free_pages((unsigned long)zstrm->buffer, 1); kfree(zstrm); } /* - * allocate new zcomp_strm structure with ->private initialized by + * allocate new zcomp_strm structure with ->tfm initialized by * backend, return NULL on error */ -static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp, gfp_t flags) +static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp) { - struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), flags); + struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL); if (!zstrm) return NULL; - zstrm->private = comp->backend->create(flags); + zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0); /* * allocate 2 pages. 1 for compressed data, plus 1 extra for the * case when compressed size is larger than the original one */ - zstrm->buffer = (void *)__get_free_pages(flags | __GFP_ZERO, 1); - if (!zstrm->private || !zstrm->buffer) { - zcomp_strm_free(comp, zstrm); + zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); + if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) { + zcomp_strm_free(zstrm); zstrm = NULL; } return zstrm; } +bool zcomp_available_algorithm(const char *comp) +{ + int i = 0; + + while (backends[i]) { + if (sysfs_streq(comp, backends[i])) + return true; + i++; + } + + /* + * Crypto does not ignore a trailing new line symbol, + * so make sure you don't supply a string containing + * one. + * This also means that we permit zcomp initialisation + * with any compressing algorithm known to crypto api. + */ + return crypto_has_comp(comp, 0, 0) == 1; +} + /* show available compressors */ ssize_t zcomp_available_show(const char *comp, char *buf) { + bool known_algorithm = false; ssize_t sz = 0; int i = 0; - while (backends[i]) { - if (!strcmp(comp, backends[i]->name)) + for (; backends[i]; i++) { + if (!strcmp(comp, backends[i])) { + known_algorithm = true; sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, - "[%s] ", backends[i]->name); - else + "[%s] ", backends[i]); + } else { sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, - "%s ", backends[i]->name); - i++; + "%s ", backends[i]); + } } + + /* + * Out-of-tree module known to crypto api or a missing + * entry in `backends'. + */ + if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1) + sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, + "[%s] ", comp); + sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); return sz; } -bool zcomp_available_algorithm(const char *comp) -{ - return find_backend(comp) != NULL; -} - -struct zcomp_strm *zcomp_strm_find(struct zcomp *comp) +struct zcomp_strm *zcomp_stream_get(struct zcomp *comp) { return *get_cpu_ptr(comp->stream); } -void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm) +void zcomp_stream_put(struct zcomp *comp) { put_cpu_ptr(comp->stream); } -int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, - const unsigned char *src, size_t *dst_len) +int zcomp_compress(struct zcomp_strm *zstrm, + const void *src, unsigned int *dst_len) { - return comp->backend->compress(src, zstrm->buffer, dst_len, - zstrm->private); + /* + * Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized + * because sometimes we can endup having a bigger compressed data + * due to various reasons: for example compression algorithms tend + * to add some padding to the compressed buffer. Speaking of padding, + * comp algorithm `842' pads the compressed length to multiple of 8 + * and returns -ENOSP when the dst memory is not big enough, which + * is not something that ZRAM wants to see. We can handle the + * `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we + * receive -ERRNO from the compressing backend we can't help it + * anymore. To make `842' happy we need to tell the exact size of + * the dst buffer, zram_drv will take care of the fact that + * compressed buffer is too big. + */ + *dst_len = PAGE_SIZE * 2; + + return crypto_comp_compress(zstrm->tfm, + src, PAGE_SIZE, + zstrm->buffer, dst_len); } -int zcomp_decompress(struct zcomp *comp, const unsigned char *src, - size_t src_len, unsigned char *dst) +int zcomp_decompress(struct zcomp_strm *zstrm, + const void *src, unsigned int src_len, void *dst) { - return comp->backend->decompress(src, src_len, dst); + unsigned int dst_len = PAGE_SIZE; + + return crypto_comp_decompress(zstrm->tfm, + src, src_len, + dst, &dst_len); } static int __zcomp_cpu_notifier(struct zcomp *comp, @@ -127,7 +169,7 @@ static int __zcomp_cpu_notifier(struct zcomp *comp, case CPU_UP_PREPARE: if (WARN_ON(*per_cpu_ptr(comp->stream, cpu))) break; - zstrm = zcomp_strm_alloc(comp, GFP_KERNEL); + zstrm = zcomp_strm_alloc(comp); if (IS_ERR_OR_NULL(zstrm)) { pr_err("Can't allocate a compression stream\n"); return NOTIFY_BAD; @@ -138,7 +180,7 @@ static int __zcomp_cpu_notifier(struct zcomp *comp, case CPU_UP_CANCELED: zstrm = *per_cpu_ptr(comp->stream, cpu); if (!IS_ERR_OR_NULL(zstrm)) - zcomp_strm_free(comp, zstrm); + zcomp_strm_free(zstrm); *per_cpu_ptr(comp->stream, cpu) = NULL; break; default: @@ -209,18 +251,16 @@ void zcomp_destroy(struct zcomp *comp) struct zcomp *zcomp_create(const char *compress) { struct zcomp *comp; - struct zcomp_backend *backend; int error; - backend = find_backend(compress); - if (!backend) + if (!zcomp_available_algorithm(compress)) return ERR_PTR(-EINVAL); comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); if (!comp) return ERR_PTR(-ENOMEM); - comp->backend = backend; + comp->name = compress; error = zcomp_init(comp); if (error) { kfree(comp); diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h index ffd88cb74..478cac2ed 100644 --- a/drivers/block/zram/zcomp.h +++ b/drivers/block/zram/zcomp.h @@ -13,33 +13,15 @@ struct zcomp_strm { /* compression/decompression buffer */ void *buffer; - /* - * The private data of the compression stream, only compression - * stream backend can touch this (e.g. compression algorithm - * working memory) - */ - void *private; -}; - -/* static compression backend */ -struct zcomp_backend { - int (*compress)(const unsigned char *src, unsigned char *dst, - size_t *dst_len, void *private); - - int (*decompress)(const unsigned char *src, size_t src_len, - unsigned char *dst); - - void *(*create)(gfp_t flags); - void (*destroy)(void *private); - - const char *name; + struct crypto_comp *tfm; }; /* dynamic per-device compression frontend */ struct zcomp { struct zcomp_strm * __percpu *stream; - struct zcomp_backend *backend; struct notifier_block notifier; + + const char *name; }; ssize_t zcomp_available_show(const char *comp, char *buf); @@ -48,14 +30,14 @@ bool zcomp_available_algorithm(const char *comp); struct zcomp *zcomp_create(const char *comp); void zcomp_destroy(struct zcomp *comp); -struct zcomp_strm *zcomp_strm_find(struct zcomp *comp); -void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm); +struct zcomp_strm *zcomp_stream_get(struct zcomp *comp); +void zcomp_stream_put(struct zcomp *comp); -int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, - const unsigned char *src, size_t *dst_len); +int zcomp_compress(struct zcomp_strm *zstrm, + const void *src, unsigned int *dst_len); -int zcomp_decompress(struct zcomp *comp, const unsigned char *src, - size_t src_len, unsigned char *dst); +int zcomp_decompress(struct zcomp_strm *zstrm, + const void *src, unsigned int src_len, void *dst); bool zcomp_set_max_streams(struct zcomp *comp, int num_strm); #endif /* _ZCOMP_H_ */ diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 8fcad8b76..04365b17e 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -342,9 +342,16 @@ static ssize_t comp_algorithm_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); + char compressor[CRYPTO_MAX_ALG_NAME]; size_t sz; - if (!zcomp_available_algorithm(buf)) + strlcpy(compressor, buf, sizeof(compressor)); + /* ignore trailing newline */ + sz = strlen(compressor); + if (sz > 0 && compressor[sz - 1] == '\n') + compressor[sz - 1] = 0x00; + + if (!zcomp_available_algorithm(compressor)) return -EINVAL; down_write(&zram->init_lock); @@ -353,13 +360,8 @@ static ssize_t comp_algorithm_store(struct device *dev, pr_info("Can't change algorithm for initialized device\n"); return -EBUSY; } - strlcpy(zram->compressor, buf, sizeof(zram->compressor)); - - /* ignore trailing newline */ - sz = strlen(zram->compressor); - if (sz > 0 && zram->compressor[sz - 1] == '\n') - zram->compressor[sz - 1] = 0x00; + strlcpy(zram->compressor, compressor, sizeof(compressor)); up_write(&zram->init_lock); return len; } @@ -563,7 +565,7 @@ static int zram_decompress_page(struct zram *zram, char *mem, u32 index) unsigned char *cmem; struct zram_meta *meta = zram->meta; unsigned long handle; - size_t size; + unsigned int size; bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value); handle = meta->table[index].handle; @@ -576,10 +578,14 @@ static int zram_decompress_page(struct zram *zram, char *mem, u32 index) } cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO); - if (size == PAGE_SIZE) + if (size == PAGE_SIZE) { copy_page(mem, cmem); - else - ret = zcomp_decompress(zram->comp, cmem, size, mem); + } else { + struct zcomp_strm *zstrm = zcomp_stream_get(zram->comp); + + ret = zcomp_decompress(zstrm, cmem, size, mem); + zcomp_stream_put(zram->comp); + } zs_unmap_object(meta->mem_pool, handle); bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value); @@ -646,7 +652,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, int offset) { int ret = 0; - size_t clen; + unsigned int clen; unsigned long handle = 0; struct page *page; unsigned char *user_mem, *cmem, *src, *uncmem = NULL; @@ -695,8 +701,8 @@ compress_again: goto out; } - zstrm = zcomp_strm_find(zram->comp); - ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen); + zstrm = zcomp_stream_get(zram->comp); + ret = zcomp_compress(zstrm, uncmem, &clen); if (!is_partial_io(bvec)) { kunmap_atomic(user_mem); user_mem = NULL; @@ -732,19 +738,21 @@ compress_again: handle = zs_malloc(meta->mem_pool, clen, __GFP_KSWAPD_RECLAIM | __GFP_NOWARN | - __GFP_HIGHMEM); + __GFP_HIGHMEM | + __GFP_MOVABLE); if (!handle) { - zcomp_strm_release(zram->comp, zstrm); + zcomp_stream_put(zram->comp); zstrm = NULL; atomic64_inc(&zram->stats.writestall); handle = zs_malloc(meta->mem_pool, clen, - GFP_NOIO | __GFP_HIGHMEM); + GFP_NOIO | __GFP_HIGHMEM | + __GFP_MOVABLE); if (handle) goto compress_again; - pr_err("Error allocating memory for compressed page: %u, size=%zu\n", + pr_err("Error allocating memory for compressed page: %u, size=%u\n", index, clen); ret = -ENOMEM; goto out; @@ -769,7 +777,7 @@ compress_again: memcpy(cmem, src, clen); } - zcomp_strm_release(zram->comp, zstrm); + zcomp_stream_put(zram->comp); zstrm = NULL; zs_unmap_object(meta->mem_pool, handle); @@ -789,7 +797,7 @@ compress_again: atomic64_inc(&zram->stats.pages_stored); out: if (zstrm) - zcomp_strm_release(zram->comp, zstrm); + zcomp_stream_put(zram->comp); if (is_partial_io(bvec)) kfree(uncmem); return ret; @@ -835,15 +843,16 @@ static void zram_bio_discard(struct zram *zram, u32 index, } static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, - int offset, int rw) + int offset, bool is_write) { unsigned long start_time = jiffies; + int rw_acct = is_write ? REQ_OP_WRITE : REQ_OP_READ; int ret; - generic_start_io_acct(rw, bvec->bv_len >> SECTOR_SHIFT, + generic_start_io_acct(rw_acct, bvec->bv_len >> SECTOR_SHIFT, &zram->disk->part0); - if (rw == READ) { + if (!is_write) { atomic64_inc(&zram->stats.num_reads); ret = zram_bvec_read(zram, bvec, index, offset); } else { @@ -851,10 +860,10 @@ static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, ret = zram_bvec_write(zram, bvec, index, offset); } - generic_end_io_acct(rw, &zram->disk->part0, start_time); + generic_end_io_acct(rw_acct, &zram->disk->part0, start_time); if (unlikely(ret)) { - if (rw == READ) + if (!is_write) atomic64_inc(&zram->stats.failed_reads); else atomic64_inc(&zram->stats.failed_writes); @@ -865,7 +874,7 @@ static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, static void __zram_make_request(struct zram *zram, struct bio *bio) { - int offset, rw; + int offset; u32 index; struct bio_vec bvec; struct bvec_iter iter; @@ -874,13 +883,12 @@ static void __zram_make_request(struct zram *zram, struct bio *bio) offset = (bio->bi_iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; - if (unlikely(bio->bi_rw & REQ_DISCARD)) { + if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) { zram_bio_discard(zram, index, offset, bio); bio_endio(bio); return; } - rw = bio_data_dir(bio); bio_for_each_segment(bvec, bio, iter) { int max_transfer_size = PAGE_SIZE - offset; @@ -895,15 +903,18 @@ static void __zram_make_request(struct zram *zram, struct bio *bio) bv.bv_len = max_transfer_size; bv.bv_offset = bvec.bv_offset; - if (zram_bvec_rw(zram, &bv, index, offset, rw) < 0) + if (zram_bvec_rw(zram, &bv, index, offset, + op_is_write(bio_op(bio))) < 0) goto out; bv.bv_len = bvec.bv_len - max_transfer_size; bv.bv_offset += max_transfer_size; - if (zram_bvec_rw(zram, &bv, index + 1, 0, rw) < 0) + if (zram_bvec_rw(zram, &bv, index + 1, 0, + op_is_write(bio_op(bio))) < 0) goto out; } else - if (zram_bvec_rw(zram, &bvec, index, offset, rw) < 0) + if (zram_bvec_rw(zram, &bvec, index, offset, + op_is_write(bio_op(bio))) < 0) goto out; update_position(&index, &offset, &bvec); @@ -960,7 +971,7 @@ static void zram_slot_free_notify(struct block_device *bdev, } static int zram_rw_page(struct block_device *bdev, sector_t sector, - struct page *page, int rw) + struct page *page, bool is_write) { int offset, err = -EIO; u32 index; @@ -984,7 +995,7 @@ static int zram_rw_page(struct block_device *bdev, sector_t sector, bv.bv_len = PAGE_SIZE; bv.bv_offset = 0; - err = zram_bvec_rw(zram, &bv, index, offset, rw); + err = zram_bvec_rw(zram, &bv, index, offset, is_write); put_zram: zram_meta_put(zram); out: @@ -997,7 +1008,7 @@ out: * (e.g., SetPageError, set_page_dirty and extra works). */ if (err == 0) - page_endio(page, rw, 0); + page_endio(page, is_write, 0); return err; } diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 3f5bf66a2..74fcf10da 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -15,8 +15,9 @@ #ifndef _ZRAM_DRV_H_ #define _ZRAM_DRV_H_ -#include +#include #include +#include #include "zcomp.h" @@ -113,7 +114,7 @@ struct zram { * we can store in a disk. */ u64 disksize; /* bytes */ - char compressor[10]; + char compressor[CRYPTO_MAX_ALG_NAME]; /* * zram is claimed so open request will be failed */ diff --git a/drivers/bluetooth/bpa10x.c b/drivers/bluetooth/bpa10x.c index fd6b53e9b..a9932fe57 100644 --- a/drivers/bluetooth/bpa10x.c +++ b/drivers/bluetooth/bpa10x.c @@ -274,6 +274,8 @@ static int bpa10x_setup(struct hci_dev *hdev) BT_INFO("%s: %s", hdev->name, (char *)(skb->data + 1)); + hci_set_fw_info(hdev, "%s", skb->data + 1); + kfree_skb(skb); return 0; } diff --git a/drivers/bluetooth/btmrvl_main.c b/drivers/bluetooth/btmrvl_main.c index 7ad8d61c0..e6a85f0e6 100644 --- a/drivers/bluetooth/btmrvl_main.c +++ b/drivers/bluetooth/btmrvl_main.c @@ -138,7 +138,7 @@ int btmrvl_process_event(struct btmrvl_private *priv, struct sk_buff *skb) if (event->length > 3 && event->data[3]) priv->btmrvl_dev.dev_type = HCI_AMP; else - priv->btmrvl_dev.dev_type = HCI_BREDR; + priv->btmrvl_dev.dev_type = HCI_PRIMARY; BT_DBG("dev_type: %d", priv->btmrvl_dev.dev_type); } else if (priv->btmrvl_dev.sendcmdflag && diff --git a/drivers/bluetooth/btmrvl_sdio.c b/drivers/bluetooth/btmrvl_sdio.c index e569ff003..73eea4f21 100644 --- a/drivers/bluetooth/btmrvl_sdio.c +++ b/drivers/bluetooth/btmrvl_sdio.c @@ -1071,7 +1071,6 @@ static int btmrvl_sdio_host_to_card(struct btmrvl_private *priv, { struct btmrvl_sdio_card *card = priv->btmrvl_dev.card; int ret = 0; - int buf_block_len; int blksz; int i = 0; u8 *buf = NULL; @@ -1083,9 +1082,13 @@ static int btmrvl_sdio_host_to_card(struct btmrvl_private *priv, return -EINVAL; } + blksz = DIV_ROUND_UP(nb, SDIO_BLOCK_SIZE) * SDIO_BLOCK_SIZE; + buf = payload; - if ((unsigned long) payload & (BTSDIO_DMA_ALIGN - 1)) { - tmpbufsz = ALIGN_SZ(nb, BTSDIO_DMA_ALIGN); + if ((unsigned long) payload & (BTSDIO_DMA_ALIGN - 1) || + nb < blksz) { + tmpbufsz = ALIGN_SZ(blksz, BTSDIO_DMA_ALIGN) + + BTSDIO_DMA_ALIGN; tmpbuf = kzalloc(tmpbufsz, GFP_KERNEL); if (!tmpbuf) return -ENOMEM; @@ -1093,15 +1096,12 @@ static int btmrvl_sdio_host_to_card(struct btmrvl_private *priv, memcpy(buf, payload, nb); } - blksz = SDIO_BLOCK_SIZE; - buf_block_len = DIV_ROUND_UP(nb, blksz); - sdio_claim_host(card->func); do { /* Transfer data to card */ ret = sdio_writesb(card->func, card->ioport, buf, - buf_block_len * blksz); + blksz); if (ret < 0) { i++; BT_ERR("i=%d writesb failed: %d", i, ret); @@ -1625,6 +1625,7 @@ static int btmrvl_sdio_suspend(struct device *dev) if (priv->adapter->hs_state != HS_ACTIVATED) { if (btmrvl_enable_hs(priv)) { BT_ERR("HS not actived, suspend failed!"); + priv->adapter->is_suspending = false; return -EBUSY; } } diff --git a/drivers/bluetooth/btsdio.c b/drivers/bluetooth/btsdio.c index 2b05661e3..1cb958e19 100644 --- a/drivers/bluetooth/btsdio.c +++ b/drivers/bluetooth/btsdio.c @@ -311,7 +311,7 @@ static int btsdio_probe(struct sdio_func *func, if (id->class == SDIO_CLASS_BT_AMP) hdev->dev_type = HCI_AMP; else - hdev->dev_type = HCI_BREDR; + hdev->dev_type = HCI_PRIMARY; data->hdev = hdev; diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index 0e09dec8e..77dd700fd 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -250,6 +250,8 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x0cf3, 0xe007), .driver_info = BTUSB_QCA_ROME }, { USB_DEVICE(0x0cf3, 0xe300), .driver_info = BTUSB_QCA_ROME }, { USB_DEVICE(0x0cf3, 0xe360), .driver_info = BTUSB_QCA_ROME }, + { USB_DEVICE(0x0489, 0xe092), .driver_info = BTUSB_QCA_ROME }, + { USB_DEVICE(0x04ca, 0x3011), .driver_info = BTUSB_QCA_ROME }, /* Broadcom BCM2035 */ { USB_DEVICE(0x0a5c, 0x2009), .driver_info = BTUSB_BCM92035 }, @@ -315,6 +317,7 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x8087, 0x07dc), .driver_info = BTUSB_INTEL }, { USB_DEVICE(0x8087, 0x0a2a), .driver_info = BTUSB_INTEL }, { USB_DEVICE(0x8087, 0x0a2b), .driver_info = BTUSB_INTEL_NEW }, + { USB_DEVICE(0x8087, 0x0aa7), .driver_info = BTUSB_INTEL }, /* Other Intel Bluetooth devices */ { USB_VENDOR_AND_INTERFACE_INFO(0x8087, 0xe0, 0x01, 0x01), @@ -2104,10 +2107,14 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) /* With this Intel bootloader only the hardware variant and device * revision information are used to select the right firmware. * - * Currently this bootloader support is limited to hardware variant - * iBT 3.0 (LnP/SfP) which is identified by the value 11 (0x0b). + * The firmware filename is ibt--.sfi. + * + * Currently the supported hardware variants are: + * 11 (0x0b) for iBT3.0 (LnP/SfP) + * 12 (0x0c) for iBT3.5 (WsP) */ snprintf(fwname, sizeof(fwname), "/*(DEBLOBBED)*/", + le16_to_cpu(ver.hw_variant), le16_to_cpu(params->dev_revid)); err = reject_firmware(&fw, fwname, &hdev->dev); @@ -2123,7 +2130,8 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) /* Save the DDC file name for later use to apply once the firmware * downloading is done. */ - snprintf(fwname, sizeof(fwname), "intel/ibt-11-%u.ddc", + snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.ddc", + le16_to_cpu(ver.hw_variant), le16_to_cpu(params->dev_revid)); kfree_skb(skb); @@ -2826,7 +2834,7 @@ static int btusb_probe(struct usb_interface *intf, if (id->driver_info & BTUSB_AMP) hdev->dev_type = HCI_AMP; else - hdev->dev_type = HCI_BREDR; + hdev->dev_type = HCI_PRIMARY; data->hdev = hdev; diff --git a/drivers/bluetooth/btwilink.c b/drivers/bluetooth/btwilink.c index 24a652f92..485281b3f 100644 --- a/drivers/bluetooth/btwilink.c +++ b/drivers/bluetooth/btwilink.c @@ -51,7 +51,7 @@ */ struct ti_st { struct hci_dev *hdev; - char reg_status; + int reg_status; long (*st_write) (struct sk_buff *); struct completion wait_reg_completion; }; @@ -83,7 +83,7 @@ static inline void ti_st_tx_complete(struct ti_st *hst, int pkt_type) * status.ti_st_open() function will wait for signal from this * API when st_register() function returns ST_PENDING. */ -static void st_reg_completion_cb(void *priv_data, char data) +static void st_reg_completion_cb(void *priv_data, int data) { struct ti_st *lhst = priv_data; diff --git a/drivers/bluetooth/hci_intel.c b/drivers/bluetooth/hci_intel.c index 4124269aa..affc8e4fa 100644 --- a/drivers/bluetooth/hci_intel.c +++ b/drivers/bluetooth/hci_intel.c @@ -537,9 +537,7 @@ static int intel_setup(struct hci_uart *hu) { static const u8 reset_param[] = { 0x00, 0x01, 0x00, 0x01, 0x00, 0x08, 0x04, 0x00 }; - static const u8 lpm_param[] = { 0x03, 0x07, 0x01, 0x0b }; struct intel_data *intel = hu->priv; - struct intel_device *idev = NULL; struct hci_dev *hdev = hu->hdev; struct sk_buff *skb; struct intel_version ver; @@ -884,35 +882,23 @@ done: bt_dev_info(hdev, "Device booted in %llu usecs", duration); - /* Enable LPM if matching pdev with wakeup enabled */ + /* Enable LPM if matching pdev with wakeup enabled, set TX active + * until further LPM TX notification. + */ mutex_lock(&intel_device_list_lock); list_for_each(p, &intel_device_list) { struct intel_device *dev = list_entry(p, struct intel_device, list); if (hu->tty->dev->parent == dev->pdev->dev.parent) { - if (device_may_wakeup(&dev->pdev->dev)) - idev = dev; + if (device_may_wakeup(&dev->pdev->dev)) { + set_bit(STATE_LPM_ENABLED, &intel->flags); + set_bit(STATE_TX_ACTIVE, &intel->flags); + } break; } } mutex_unlock(&intel_device_list_lock); - if (!idev) - goto no_lpm; - - bt_dev_info(hdev, "Enabling LPM"); - - skb = __hci_cmd_sync(hdev, 0xfc8b, sizeof(lpm_param), lpm_param, - HCI_CMD_TIMEOUT); - if (IS_ERR(skb)) { - bt_dev_err(hdev, "Failed to enable LPM"); - goto no_lpm; - } - kfree_skb(skb); - - set_bit(STATE_LPM_ENABLED, &intel->flags); - -no_lpm: /* Ignore errors, device can work without DDC parameters */ btintel_load_ddc_config(hdev, fwname); diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index 49b3e1e2d..dda97398c 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -609,7 +609,7 @@ static int hci_uart_register_dev(struct hci_uart *hu) if (test_bit(HCI_UART_CREATE_AMP, &hu->hdev_flags)) hdev->dev_type = HCI_AMP; else - hdev->dev_type = HCI_BREDR; + hdev->dev_type = HCI_PRIMARY; if (test_bit(HCI_UART_INIT_PENDING, &hu->hdev_flags)) return 0; diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c index aba31210c..3ff229b2e 100644 --- a/drivers/bluetooth/hci_vhci.c +++ b/drivers/bluetooth/hci_vhci.c @@ -97,10 +97,10 @@ static int __vhci_create_device(struct vhci_data *data, __u8 opcode) if (data->hdev) return -EBADFD; - /* bits 0-1 are dev_type (BR/EDR or AMP) */ + /* bits 0-1 are dev_type (Primary or AMP) */ dev_type = opcode & 0x03; - if (dev_type != HCI_BREDR && dev_type != HCI_AMP) + if (dev_type != HCI_PRIMARY && dev_type != HCI_AMP) return -EINVAL; /* bits 2-5 are reserved (must be zero) */ @@ -316,7 +316,7 @@ static void vhci_open_timeout(struct work_struct *work) struct vhci_data *data = container_of(work, struct vhci_data, open_timeout.work); - vhci_create_device(data, amp ? HCI_AMP : HCI_BREDR); + vhci_create_device(data, amp ? HCI_AMP : HCI_PRIMARY); } static int vhci_open(struct inode *inode, struct file *file) diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig index c5a7de9bc..3b205e212 100644 --- a/drivers/bus/Kconfig +++ b/drivers/bus/Kconfig @@ -132,6 +132,19 @@ config SUNXI_RSB with various RSB based devices, such as AXP223, AXP8XX PMICs, and AC100/AC200 ICs. +# TODO: This uses pm_clk_*() symbols that aren't exported in v4.7 and hence +# the driver will fail to build as a module. However there are patches to +# address that queued for v4.8, so this can be turned into a tristate symbol +# after v4.8-rc1. +config TEGRA_ACONNECT + bool "Tegra ACONNECT Bus Driver" + depends on ARCH_TEGRA_210_SOC + depends on OF && PM + select PM_CLK + help + Driver for the Tegra ACONNECT bus which is used to interface with + the devices inside the Audio Processing Engine (APE) for Tegra210. + config UNIPHIER_SYSTEM_BUS tristate "UniPhier System Bus driver" depends on ARCH_UNIPHIER && OF diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile index ccff007ee..ac84cc434 100644 --- a/drivers/bus/Makefile +++ b/drivers/bus/Makefile @@ -17,5 +17,6 @@ obj-$(CONFIG_OMAP_INTERCONNECT) += omap_l3_smx.o omap_l3_noc.o obj-$(CONFIG_OMAP_OCP2SCP) += omap-ocp2scp.o obj-$(CONFIG_SUNXI_RSB) += sunxi-rsb.o obj-$(CONFIG_SIMPLE_PM_BUS) += simple-pm-bus.o +obj-$(CONFIG_TEGRA_ACONNECT) += tegra-aconnect.o obj-$(CONFIG_UNIPHIER_SYSTEM_BUS) += uniphier-system-bus.o obj-$(CONFIG_VEXPRESS_CONFIG) += vexpress-config.o diff --git a/drivers/bus/arm-cci.c b/drivers/bus/arm-cci.c index a49b28378..ffa7c9dcb 100644 --- a/drivers/bus/arm-cci.c +++ b/drivers/bus/arm-cci.c @@ -144,12 +144,15 @@ struct cci_pmu { int num_cntrs; atomic_t active_events; struct mutex reserve_mutex; - struct notifier_block cpu_nb; + struct list_head entry; cpumask_t cpus; }; #define to_cci_pmu(c) (container_of(c, struct cci_pmu, pmu)) +static DEFINE_MUTEX(cci_pmu_mutex); +static LIST_HEAD(cci_pmu_list); + enum cci_models { #ifdef CONFIG_ARM_CCI400_PMU CCI400_R0, @@ -548,7 +551,7 @@ static struct attribute *cci5xx_pmu_event_attrs[] = { CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq, 0xB), CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs, 0xC), CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard, 0xD), - CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snopp_rq_stall_tt_full, 0xE), + CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_stall_tt_full, 0xE), CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot, 0xF), NULL }; @@ -1503,31 +1506,26 @@ static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev) return perf_pmu_register(&cci_pmu->pmu, name, -1); } -static int cci_pmu_cpu_notifier(struct notifier_block *self, - unsigned long action, void *hcpu) +static int cci_pmu_offline_cpu(unsigned int cpu) { - struct cci_pmu *cci_pmu = container_of(self, - struct cci_pmu, cpu_nb); - unsigned int cpu = (long)hcpu; + struct cci_pmu *cci_pmu; unsigned int target; - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_DOWN_PREPARE: + mutex_lock(&cci_pmu_mutex); + list_for_each_entry(cci_pmu, &cci_pmu_list, entry) { if (!cpumask_test_and_clear_cpu(cpu, &cci_pmu->cpus)) - break; + continue; target = cpumask_any_but(cpu_online_mask, cpu); - if (target >= nr_cpu_ids) // UP, last CPU - break; + if (target >= nr_cpu_ids) + continue; /* * TODO: migrate context once core races on event->ctx have * been fixed. */ cpumask_set_cpu(target, &cci_pmu->cpus); - default: - break; } - - return NOTIFY_OK; + mutex_unlock(&cci_pmu_mutex); + return 0; } static struct cci_pmu_model cci_pmu_models[] = { @@ -1766,24 +1764,13 @@ static int cci_pmu_probe(struct platform_device *pdev) atomic_set(&cci_pmu->active_events, 0); cpumask_set_cpu(smp_processor_id(), &cci_pmu->cpus); - cci_pmu->cpu_nb = (struct notifier_block) { - .notifier_call = cci_pmu_cpu_notifier, - /* - * to migrate uncore events, our notifier should be executed - * before perf core's notifier. - */ - .priority = CPU_PRI_PERF + 1, - }; - - ret = register_cpu_notifier(&cci_pmu->cpu_nb); + ret = cci_pmu_init(cci_pmu, pdev); if (ret) return ret; - ret = cci_pmu_init(cci_pmu, pdev); - if (ret) { - unregister_cpu_notifier(&cci_pmu->cpu_nb); - return ret; - } + mutex_lock(&cci_pmu_mutex); + list_add(&cci_pmu->entry, &cci_pmu_list); + mutex_unlock(&cci_pmu_mutex); pr_info("ARM %s PMU driver probed", cci_pmu->model->name); return 0; @@ -1817,6 +1804,12 @@ static int __init cci_platform_init(void) { int ret; + ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE, + "AP_PERF_ARM_CCI_ONLINE", NULL, + cci_pmu_offline_cpu); + if (ret) + return ret; + ret = platform_driver_register(&cci_pmu_driver); if (ret) return ret; diff --git a/drivers/bus/arm-ccn.c b/drivers/bus/arm-ccn.c index d88372b79..884c0305e 100644 --- a/drivers/bus/arm-ccn.c +++ b/drivers/bus/arm-ccn.c @@ -167,7 +167,7 @@ struct arm_ccn_dt { struct hrtimer hrtimer; cpumask_t cpu; - struct notifier_block cpu_nb; + struct list_head entry; struct pmu pmu; }; @@ -190,6 +190,8 @@ struct arm_ccn { int mn_id; }; +static DEFINE_MUTEX(arm_ccn_mutex); +static LIST_HEAD(arm_ccn_list); static int arm_ccn_node_to_xp(int node) { @@ -211,6 +213,7 @@ static int arm_ccn_node_to_xp_port(int node) #define CCN_CONFIG_TYPE(_config) (((_config) >> 8) & 0xff) #define CCN_CONFIG_EVENT(_config) (((_config) >> 16) & 0xff) #define CCN_CONFIG_PORT(_config) (((_config) >> 24) & 0x3) +#define CCN_CONFIG_BUS(_config) (((_config) >> 24) & 0x3) #define CCN_CONFIG_VC(_config) (((_config) >> 26) & 0x7) #define CCN_CONFIG_DIR(_config) (((_config) >> 29) & 0x1) #define CCN_CONFIG_MASK(_config) (((_config) >> 30) & 0xf) @@ -240,6 +243,7 @@ static CCN_FORMAT_ATTR(xp, "config:0-7"); static CCN_FORMAT_ATTR(type, "config:8-15"); static CCN_FORMAT_ATTR(event, "config:16-23"); static CCN_FORMAT_ATTR(port, "config:24-25"); +static CCN_FORMAT_ATTR(bus, "config:24-25"); static CCN_FORMAT_ATTR(vc, "config:26-28"); static CCN_FORMAT_ATTR(dir, "config:29-29"); static CCN_FORMAT_ATTR(mask, "config:30-33"); @@ -252,6 +256,7 @@ static struct attribute *arm_ccn_pmu_format_attrs[] = { &arm_ccn_pmu_format_attr_type.attr.attr, &arm_ccn_pmu_format_attr_event.attr.attr, &arm_ccn_pmu_format_attr_port.attr.attr, + &arm_ccn_pmu_format_attr_bus.attr.attr, &arm_ccn_pmu_format_attr_vc.attr.attr, &arm_ccn_pmu_format_attr_dir.attr.attr, &arm_ccn_pmu_format_attr_mask.attr.attr, @@ -349,10 +354,14 @@ static ssize_t arm_ccn_pmu_event_show(struct device *dev, break; case CCN_TYPE_XP: res += snprintf(buf + res, PAGE_SIZE - res, - ",xp=?,port=?,vc=?,dir=?"); + ",xp=?,vc=?"); if (event->event == CCN_EVENT_WATCHPOINT) res += snprintf(buf + res, PAGE_SIZE - res, - ",cmp_l=?,cmp_h=?,mask=?"); + ",port=?,dir=?,cmp_l=?,cmp_h=?,mask=?"); + else + res += snprintf(buf + res, PAGE_SIZE - res, + ",bus=?"); + break; case CCN_TYPE_MN: res += snprintf(buf + res, PAGE_SIZE - res, ",node=%d", ccn->mn_id); @@ -736,9 +745,10 @@ static int arm_ccn_pmu_event_init(struct perf_event *event) if (has_branch_stack(event) || event->attr.exclude_user || event->attr.exclude_kernel || event->attr.exclude_hv || - event->attr.exclude_idle) { + event->attr.exclude_idle || event->attr.exclude_host || + event->attr.exclude_guest) { dev_warn(ccn->dev, "Can't exclude execution levels!\n"); - return -EOPNOTSUPP; + return -EINVAL; } if (event->cpu < 0) { @@ -930,38 +940,17 @@ static void arm_ccn_pmu_event_start(struct perf_event *event, int flags) arm_ccn_pmu_read_counter(ccn, hw->idx)); hw->state = 0; - /* - * Pin the timer, so that the overflows are handled by the chosen - * event->cpu (this is the same one as presented in "cpumask" - * attribute). - */ - if (!ccn->irq) - hrtimer_start(&ccn->dt.hrtimer, arm_ccn_pmu_timer_period(), - HRTIMER_MODE_REL_PINNED); - /* Set the DT bus input, engaging the counter */ arm_ccn_pmu_xp_dt_config(event, 1); } static void arm_ccn_pmu_event_stop(struct perf_event *event, int flags) { - struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); struct hw_perf_event *hw = &event->hw; - u64 timeout; /* Disable counting, setting the DT bus to pass-through mode */ arm_ccn_pmu_xp_dt_config(event, 0); - if (!ccn->irq) - hrtimer_cancel(&ccn->dt.hrtimer); - - /* Let the DT bus drain */ - timeout = arm_ccn_pmu_read_counter(ccn, CCN_IDX_PMU_CYCLE_COUNTER) + - ccn->num_xps; - while (arm_ccn_pmu_read_counter(ccn, CCN_IDX_PMU_CYCLE_COUNTER) < - timeout) - cpu_relax(); - if (flags & PERF_EF_UPDATE) arm_ccn_pmu_event_update(event); @@ -1027,7 +1016,7 @@ static void arm_ccn_pmu_xp_event_config(struct perf_event *event) hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__XP_PMU_EVENT(hw->config_base); id = (CCN_CONFIG_VC(event->attr.config) << 4) | - (CCN_CONFIG_PORT(event->attr.config) << 3) | + (CCN_CONFIG_BUS(event->attr.config) << 3) | (CCN_CONFIG_EVENT(event->attr.config) << 0); val = readl(source->base + CCN_XP_PMU_EVENT_SEL); @@ -1112,15 +1101,31 @@ static void arm_ccn_pmu_event_config(struct perf_event *event) spin_unlock(&ccn->dt.config_lock); } +static int arm_ccn_pmu_active_counters(struct arm_ccn *ccn) +{ + return bitmap_weight(ccn->dt.pmu_counters_mask, + CCN_NUM_PMU_EVENT_COUNTERS + 1); +} + static int arm_ccn_pmu_event_add(struct perf_event *event, int flags) { int err; struct hw_perf_event *hw = &event->hw; + struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); err = arm_ccn_pmu_event_alloc(event); if (err) return err; + /* + * Pin the timer, so that the overflows are handled by the chosen + * event->cpu (this is the same one as presented in "cpumask" + * attribute). + */ + if (!ccn->irq && arm_ccn_pmu_active_counters(ccn) == 1) + hrtimer_start(&ccn->dt.hrtimer, arm_ccn_pmu_timer_period(), + HRTIMER_MODE_REL_PINNED); + arm_ccn_pmu_event_config(event); hw->state = PERF_HES_STOPPED; @@ -1133,9 +1138,14 @@ static int arm_ccn_pmu_event_add(struct perf_event *event, int flags) static void arm_ccn_pmu_event_del(struct perf_event *event, int flags) { + struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu); + arm_ccn_pmu_event_stop(event, PERF_EF_UPDATE); arm_ccn_pmu_event_release(event); + + if (!ccn->irq && arm_ccn_pmu_active_counters(ccn) == 0) + hrtimer_cancel(&ccn->dt.hrtimer); } static void arm_ccn_pmu_event_read(struct perf_event *event) @@ -1143,6 +1153,24 @@ static void arm_ccn_pmu_event_read(struct perf_event *event) arm_ccn_pmu_event_update(event); } +static void arm_ccn_pmu_enable(struct pmu *pmu) +{ + struct arm_ccn *ccn = pmu_to_arm_ccn(pmu); + + u32 val = readl(ccn->dt.base + CCN_DT_PMCR); + val |= CCN_DT_PMCR__PMU_EN; + writel(val, ccn->dt.base + CCN_DT_PMCR); +} + +static void arm_ccn_pmu_disable(struct pmu *pmu) +{ + struct arm_ccn *ccn = pmu_to_arm_ccn(pmu); + + u32 val = readl(ccn->dt.base + CCN_DT_PMCR); + val &= ~CCN_DT_PMCR__PMU_EN; + writel(val, ccn->dt.base + CCN_DT_PMCR); +} + static irqreturn_t arm_ccn_pmu_overflow_handler(struct arm_ccn_dt *dt) { u32 pmovsr = readl(dt->base + CCN_DT_PMOVSR); @@ -1186,30 +1214,27 @@ static enum hrtimer_restart arm_ccn_pmu_timer_handler(struct hrtimer *hrtimer) } -static int arm_ccn_pmu_cpu_notifier(struct notifier_block *nb, - unsigned long action, void *hcpu) +static int arm_ccn_pmu_offline_cpu(unsigned int cpu) { - struct arm_ccn_dt *dt = container_of(nb, struct arm_ccn_dt, cpu_nb); - struct arm_ccn *ccn = container_of(dt, struct arm_ccn, dt); - unsigned int cpu = (long)hcpu; /* for (long) see kernel/cpu.c */ + struct arm_ccn_dt *dt; unsigned int target; - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_DOWN_PREPARE: + mutex_lock(&arm_ccn_mutex); + list_for_each_entry(dt, &arm_ccn_list, entry) { + struct arm_ccn *ccn = container_of(dt, struct arm_ccn, dt); + if (!cpumask_test_and_clear_cpu(cpu, &dt->cpu)) - break; + continue; target = cpumask_any_but(cpu_online_mask, cpu); if (target >= nr_cpu_ids) - break; + continue; perf_pmu_migrate_context(&dt->pmu, cpu, target); cpumask_set_cpu(target, &dt->cpu); if (ccn->irq) WARN_ON(irq_set_affinity_hint(ccn->irq, &dt->cpu) != 0); - default: - break; } - - return NOTIFY_OK; + mutex_unlock(&arm_ccn_mutex); + return 0; } @@ -1268,6 +1293,8 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn) .start = arm_ccn_pmu_event_start, .stop = arm_ccn_pmu_event_stop, .read = arm_ccn_pmu_event_read, + .pmu_enable = arm_ccn_pmu_enable, + .pmu_disable = arm_ccn_pmu_disable, }; /* No overflow interrupt? Have to use a timer instead. */ @@ -1281,16 +1308,6 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn) /* Pick one CPU which we will use to collect data from CCN... */ cpumask_set_cpu(smp_processor_id(), &ccn->dt.cpu); - /* - * ... and change the selection when it goes offline. Priority is - * picked to have a chance to migrate events before perf is notified. - */ - ccn->dt.cpu_nb.notifier_call = arm_ccn_pmu_cpu_notifier; - ccn->dt.cpu_nb.priority = CPU_PRI_PERF + 1, - err = register_cpu_notifier(&ccn->dt.cpu_nb); - if (err) - goto error_cpu_notifier; - /* Also make sure that the overflow interrupt is handled by this CPU */ if (ccn->irq) { err = irq_set_affinity_hint(ccn->irq, &ccn->dt.cpu); @@ -1304,12 +1321,13 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn) if (err) goto error_pmu_register; + mutex_lock(&arm_ccn_mutex); + list_add(&ccn->dt.entry, &arm_ccn_list); + mutex_unlock(&arm_ccn_mutex); return 0; error_pmu_register: error_set_affinity: - unregister_cpu_notifier(&ccn->dt.cpu_nb); -error_cpu_notifier: ida_simple_remove(&arm_ccn_pmu_ida, ccn->dt.id); for (i = 0; i < ccn->num_xps; i++) writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL); @@ -1321,9 +1339,12 @@ static void arm_ccn_pmu_cleanup(struct arm_ccn *ccn) { int i; + mutex_lock(&arm_ccn_mutex); + list_del(&ccn->dt.entry); + mutex_unlock(&arm_ccn_mutex); + if (ccn->irq) irq_set_affinity_hint(ccn->irq, NULL); - unregister_cpu_notifier(&ccn->dt.cpu_nb); for (i = 0; i < ccn->num_xps; i++) writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL); writel(0, ccn->dt.base + CCN_DT_PMCR); @@ -1331,7 +1352,6 @@ static void arm_ccn_pmu_cleanup(struct arm_ccn *ccn) ida_simple_remove(&arm_ccn_pmu_ida, ccn->dt.id); } - static int arm_ccn_for_each_valid_region(struct arm_ccn *ccn, int (*callback)(struct arm_ccn *ccn, int region, void __iomem *base, u32 type, u32 id)) @@ -1496,8 +1516,9 @@ static int arm_ccn_probe(struct platform_device *pdev) /* Can set 'disable' bits, so can acknowledge interrupts */ writel(CCN_MN_ERRINT_STATUS__PMU_EVENTS__ENABLE, ccn->base + CCN_MN_ERRINT_STATUS); - err = devm_request_irq(ccn->dev, irq, arm_ccn_irq_handler, 0, - dev_name(ccn->dev), ccn); + err = devm_request_irq(ccn->dev, irq, arm_ccn_irq_handler, + IRQF_NOBALANCING | IRQF_NO_THREAD, + dev_name(ccn->dev), ccn); if (err) return err; @@ -1550,7 +1571,13 @@ static struct platform_driver arm_ccn_driver = { static int __init arm_ccn_init(void) { - int i; + int i, ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE, + "AP_PERF_ARM_CCN_ONLINE", NULL, + arm_ccn_pmu_offline_cpu); + if (ret) + return ret; for (i = 0; i < ARRAY_SIZE(arm_ccn_pmu_events); i++) arm_ccn_pmu_events_attrs[i] = &arm_ccn_pmu_events[i].attr.attr; @@ -1560,6 +1587,7 @@ static int __init arm_ccn_init(void) static void __exit arm_ccn_exit(void) { + cpuhp_remove_state_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE); platform_driver_unregister(&arm_ccn_driver); } diff --git a/drivers/bus/imx-weim.c b/drivers/bus/imx-weim.c index 1827fc4d1..4bd361d64 100644 --- a/drivers/bus/imx-weim.c +++ b/drivers/bus/imx-weim.c @@ -163,9 +163,8 @@ static int __init weim_parse_dt(struct platform_device *pdev, } if (have_child) - ret = of_platform_populate(pdev->dev.of_node, - of_default_bus_match_table, - NULL, &pdev->dev); + ret = of_platform_default_populate(pdev->dev.of_node, + NULL, &pdev->dev); if (ret) dev_err(&pdev->dev, "%s fail to create devices.\n", pdev->dev.of_node->full_name); diff --git a/drivers/bus/mvebu-mbus.c b/drivers/bus/mvebu-mbus.c index ce54a0160..c7f396903 100644 --- a/drivers/bus/mvebu-mbus.c +++ b/drivers/bus/mvebu-mbus.c @@ -117,7 +117,7 @@ struct mvebu_mbus_soc_data { unsigned int (*win_remap_offset)(const int win); void (*setup_cpu_target)(struct mvebu_mbus_state *s); int (*save_cpu_target)(struct mvebu_mbus_state *s, - u32 *store_addr); + u32 __iomem *store_addr); int (*show_cpu_target)(struct mvebu_mbus_state *s, struct seq_file *seq, void *v); }; @@ -728,7 +728,7 @@ mvebu_mbus_default_setup_cpu_target(struct mvebu_mbus_state *mbus) static int mvebu_mbus_default_save_cpu_target(struct mvebu_mbus_state *mbus, - u32 *store_addr) + u32 __iomem *store_addr) { int i; @@ -780,7 +780,7 @@ mvebu_mbus_dove_setup_cpu_target(struct mvebu_mbus_state *mbus) static int mvebu_mbus_dove_save_cpu_target(struct mvebu_mbus_state *mbus, - u32 *store_addr) + u32 __iomem *store_addr) { int i; @@ -796,7 +796,7 @@ mvebu_mbus_dove_save_cpu_target(struct mvebu_mbus_state *mbus, return 4; } -int mvebu_mbus_save_cpu_target(u32 *store_addr) +int mvebu_mbus_save_cpu_target(u32 __iomem *store_addr) { return mbus_state.soc->save_cpu_target(&mbus_state, store_addr); } @@ -1089,7 +1089,7 @@ static void mvebu_mbus_resume(void) } } -struct syscore_ops mvebu_mbus_syscore_ops = { +static struct syscore_ops mvebu_mbus_syscore_ops = { .suspend = mvebu_mbus_suspend, .resume = mvebu_mbus_resume, }; diff --git a/drivers/bus/tegra-aconnect.c b/drivers/bus/tegra-aconnect.c new file mode 100644 index 000000000..7e4104b74 --- /dev/null +++ b/drivers/bus/tegra-aconnect.c @@ -0,0 +1,112 @@ +/* + * Tegra ACONNECT Bus Driver + * + * Copyright (C) 2016, NVIDIA CORPORATION. All rights reserved. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#include +#include +#include +#include +#include +#include + +static int tegra_aconnect_add_clock(struct device *dev, char *name) +{ + struct clk *clk; + int ret; + + clk = clk_get(dev, name); + if (IS_ERR(clk)) { + dev_err(dev, "%s clock not found\n", name); + return PTR_ERR(clk); + } + + ret = pm_clk_add_clk(dev, clk); + if (ret) + clk_put(clk); + + return ret; +} + +static int tegra_aconnect_probe(struct platform_device *pdev) +{ + int ret; + + if (!pdev->dev.of_node) + return -EINVAL; + + ret = pm_clk_create(&pdev->dev); + if (ret) + return ret; + + ret = tegra_aconnect_add_clock(&pdev->dev, "ape"); + if (ret) + goto clk_destroy; + + ret = tegra_aconnect_add_clock(&pdev->dev, "apb2ape"); + if (ret) + goto clk_destroy; + + pm_runtime_enable(&pdev->dev); + + of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev); + + dev_info(&pdev->dev, "Tegra ACONNECT bus registered\n"); + + return 0; + +clk_destroy: + pm_clk_destroy(&pdev->dev); + + return ret; +} + +static int tegra_aconnect_remove(struct platform_device *pdev) +{ + pm_runtime_disable(&pdev->dev); + + pm_clk_destroy(&pdev->dev); + + return 0; +} + +static int tegra_aconnect_runtime_resume(struct device *dev) +{ + return pm_clk_resume(dev); +} + +static int tegra_aconnect_runtime_suspend(struct device *dev) +{ + return pm_clk_suspend(dev); +} + +static const struct dev_pm_ops tegra_aconnect_pm_ops = { + SET_RUNTIME_PM_OPS(tegra_aconnect_runtime_suspend, + tegra_aconnect_runtime_resume, NULL) +}; + +static const struct of_device_id tegra_aconnect_of_match[] = { + { .compatible = "nvidia,tegra210-aconnect", }, + { } +}; +MODULE_DEVICE_TABLE(of, tegra_aconnect_of_match); + +static struct platform_driver tegra_aconnect_driver = { + .probe = tegra_aconnect_probe, + .remove = tegra_aconnect_remove, + .driver = { + .name = "tegra-aconnect", + .of_match_table = tegra_aconnect_of_match, + .pm = &tegra_aconnect_pm_ops, + }, +}; +module_platform_driver(tegra_aconnect_driver); + +MODULE_DESCRIPTION("NVIDIA Tegra ACONNECT Bus Driver"); +MODULE_AUTHOR("Jon Hunter "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/bus/uniphier-system-bus.c b/drivers/bus/uniphier-system-bus.c index 350b7309c..1e6e0269e 100644 --- a/drivers/bus/uniphier-system-bus.c +++ b/drivers/bus/uniphier-system-bus.c @@ -257,8 +257,7 @@ static int uniphier_system_bus_probe(struct platform_device *pdev) uniphier_system_bus_set_reg(priv); /* Now, the bus is configured. Populate platform_devices below it */ - return of_platform_populate(dev->of_node, of_default_bus_match_table, - NULL, dev); + return of_platform_default_populate(dev->of_node, NULL, dev); } static const struct of_device_id uniphier_system_bus_match[] = { diff --git a/drivers/bus/vexpress-config.c b/drivers/bus/vexpress-config.c index c3cb76b36..9efdf1de4 100644 --- a/drivers/bus/vexpress-config.c +++ b/drivers/bus/vexpress-config.c @@ -178,6 +178,7 @@ static int vexpress_config_populate(struct device_node *node) parent = class_find_device(vexpress_config_class, NULL, bridge, vexpress_config_node_match); + of_node_put(bridge); if (WARN_ON(!parent)) return -ENODEV; diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c index 1b257ea97..5d475b3a0 100644 --- a/drivers/cdrom/cdrom.c +++ b/drivers/cdrom/cdrom.c @@ -2032,7 +2032,7 @@ static int cdrom_read_subchannel(struct cdrom_device_info *cdi, init_cdrom_command(&cgc, buffer, 16, CGC_DATA_READ); cgc.cmd[0] = GPCMD_READ_SUBCHANNEL; - cgc.cmd[1] = 2; /* MSF addressing */ + cgc.cmd[1] = subchnl->cdsc_format;/* MSF or LBA addressing */ cgc.cmd[2] = 0x40; /* request subQ data */ cgc.cmd[3] = mcn ? 2 : 1; cgc.cmd[8] = 16; @@ -2041,17 +2041,27 @@ static int cdrom_read_subchannel(struct cdrom_device_info *cdi, return ret; subchnl->cdsc_audiostatus = cgc.buffer[1]; - subchnl->cdsc_format = CDROM_MSF; subchnl->cdsc_ctrl = cgc.buffer[5] & 0xf; subchnl->cdsc_trk = cgc.buffer[6]; subchnl->cdsc_ind = cgc.buffer[7]; - subchnl->cdsc_reladdr.msf.minute = cgc.buffer[13]; - subchnl->cdsc_reladdr.msf.second = cgc.buffer[14]; - subchnl->cdsc_reladdr.msf.frame = cgc.buffer[15]; - subchnl->cdsc_absaddr.msf.minute = cgc.buffer[9]; - subchnl->cdsc_absaddr.msf.second = cgc.buffer[10]; - subchnl->cdsc_absaddr.msf.frame = cgc.buffer[11]; + if (subchnl->cdsc_format == CDROM_LBA) { + subchnl->cdsc_absaddr.lba = ((cgc.buffer[8] << 24) | + (cgc.buffer[9] << 16) | + (cgc.buffer[10] << 8) | + (cgc.buffer[11])); + subchnl->cdsc_reladdr.lba = ((cgc.buffer[12] << 24) | + (cgc.buffer[13] << 16) | + (cgc.buffer[14] << 8) | + (cgc.buffer[15])); + } else { + subchnl->cdsc_reladdr.msf.minute = cgc.buffer[13]; + subchnl->cdsc_reladdr.msf.second = cgc.buffer[14]; + subchnl->cdsc_reladdr.msf.frame = cgc.buffer[15]; + subchnl->cdsc_absaddr.msf.minute = cgc.buffer[9]; + subchnl->cdsc_absaddr.msf.second = cgc.buffer[10]; + subchnl->cdsc_absaddr.msf.frame = cgc.buffer[11]; + } return 0; } @@ -3022,7 +3032,7 @@ static noinline int mmc_ioctl_cdrom_subchannel(struct cdrom_device_info *cdi, if (!((requested == CDROM_MSF) || (requested == CDROM_LBA))) return -EINVAL; - q.cdsc_format = CDROM_MSF; + ret = cdrom_read_subchannel(cdi, &q, 0); if (ret) return ret; diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 601f64fcc..dcc09739a 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -178,6 +178,20 @@ config IBM_BSR of threads across a large system which avoids bouncing a cacheline between several cores on a system +config POWERNV_OP_PANEL + tristate "IBM POWERNV Operator Panel Display support" + depends on PPC_POWERNV + default m + help + If you say Y here, a special character device node, /dev/op_panel, + will be created which exposes the operator panel display on IBM + Power Systems machines with FSPs. + + If you don't require access to the operator panel display from user + space, say N. + + If unsure, say M here to build it as a module called powernv-op-panel. + source "drivers/char/ipmi/Kconfig" config DS1620 @@ -279,7 +293,7 @@ if RTC_LIB=n config RTC tristate "Enhanced Real Time Clock Support (legacy PC RTC driver)" - depends on ALPHA || (MIPS && MACH_LOONGSON64) || MN10300 + depends on ALPHA || (MIPS && MACH_LOONGSON64) ---help--- If you say Y here and create a character special file /dev/rtc with major number 10 and minor number 135 using mknod ("man mknod"), you @@ -325,32 +339,6 @@ config JS_RTC To compile this driver as a module, choose M here: the module will be called js-rtc. -config GEN_RTC - tristate "Generic /dev/rtc emulation" - depends on RTC!=y - depends on ALPHA || M68K || MN10300 || PARISC || PPC || X86 - ---help--- - If you say Y here and create a character special file /dev/rtc with - major number 10 and minor number 135 using mknod ("man mknod"), you - will get access to the real time clock (or hardware clock) built - into your computer. - - It reports status information via the file /proc/driver/rtc and its - behaviour is set by various ioctls on /dev/rtc. If you enable the - "extended RTC operation" below it will also provide an emulation - for RTC_UIE which is required by some programs and may improve - precision in some cases. - - To compile this driver as a module, choose M here: the - module will be called genrtc. - -config GEN_RTC_X - bool "Extended RTC operation" - depends on GEN_RTC - help - Provides an emulation for RTC_UIE which is required by some programs - and may improve precision of the generic RTC support in some cases. - config EFI_RTC bool "EFI Real Time Clock Services" depends on IA64 diff --git a/drivers/char/Makefile b/drivers/char/Makefile index d8a757930..6e6c244a6 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -25,7 +25,6 @@ obj-$(CONFIG_APPLICOM) += applicom.o obj-$(CONFIG_SONYPI) += sonypi.o obj-$(CONFIG_RTC) += rtc.o obj-$(CONFIG_HPET) += hpet.o -obj-$(CONFIG_GEN_RTC) += genrtc.o obj-$(CONFIG_EFI_RTC) += efirtc.o obj-$(CONFIG_DS1302) += ds1302.o obj-$(CONFIG_XILINX_HWICAP) += xilinx_hwicap/ @@ -60,3 +59,4 @@ js-rtc-y = rtc.o obj-$(CONFIG_TILE_SROM) += tile-srom.o obj-$(CONFIG_XILLYBUS) += xillybus/ +obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c index aef87fdbd..44311296e 100644 --- a/drivers/char/agp/intel-gtt.c +++ b/drivers/char/agp/intel-gtt.c @@ -840,6 +840,14 @@ static bool i830_check_flags(unsigned int flags) return false; } +void intel_gtt_insert_page(dma_addr_t addr, + unsigned int pg, + unsigned int flags) +{ + intel_private.driver->write_entry(addr, pg, flags); +} +EXPORT_SYMBOL(intel_gtt_insert_page); + void intel_gtt_insert_sg_entries(struct sg_table *st, unsigned int pg_start, unsigned int flags) diff --git a/drivers/char/dsp56k.c b/drivers/char/dsp56k.c index 8bf70e8c3..50aa9ba91 100644 --- a/drivers/char/dsp56k.c +++ b/drivers/char/dsp56k.c @@ -325,7 +325,7 @@ static long dsp56k_ioctl(struct file *file, unsigned int cmd, if(get_user(bin, &binary->bin) < 0) return -EFAULT; - if (len == 0) { + if (len <= 0) { return -EINVAL; /* nothing to upload?!? */ } if (len > DSP56K_MAX_BINARY_LENGTH) { diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index ac51149e9..8c0770bf8 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -90,7 +90,7 @@ config HW_RANDOM_BCM63XX config HW_RANDOM_BCM2835 tristate "Broadcom BCM2835 Random Number Generator support" - depends on ARCH_BCM2835 + depends on ARCH_BCM2835 || ARCH_BCM_NSP || ARCH_BCM_5301X default HW_RANDOM ---help--- This driver provides kernel-side support for the Random Number @@ -244,7 +244,7 @@ config HW_RANDOM_TX4939 config HW_RANDOM_MXC_RNGA tristate "Freescale i.MX RNGA Random Number Generator" - depends on ARCH_HAS_RNGA + depends on SOC_IMX31 default HW_RANDOM ---help--- This driver provides kernel-side support for the Random Number @@ -396,6 +396,20 @@ config HW_RANDOM_PIC32 If unsure, say Y. +config HW_RANDOM_MESON + tristate "Amlogic Meson Random Number Generator support" + depends on HW_RANDOM + depends on ARCH_MESON || COMPILE_TEST + default y + ---help--- + This driver provides kernel-side support for the Random Number + Generator hardware found on Amlogic Meson SoCs. + + To compile this driver as a module, choose M here. the + module will be called meson-rng. + + If unsure, say Y. + endif # HW_RANDOM config UML_RANDOM diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 63022b49f..04bb0b033 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -34,3 +34,4 @@ obj-$(CONFIG_HW_RANDOM_ST) += st-rng.o obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o obj-$(CONFIG_HW_RANDOM_STM32) += stm32-rng.o obj-$(CONFIG_HW_RANDOM_PIC32) += pic32-rng.o +obj-$(CONFIG_HW_RANDOM_MESON) += meson-rng.o diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index 7192ec25f..af2149273 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -19,6 +19,7 @@ #define RNG_CTRL 0x0 #define RNG_STATUS 0x4 #define RNG_DATA 0x8 +#define RNG_INT_MASK 0x10 /* enable rng */ #define RNG_RBGEN 0x1 @@ -26,10 +27,24 @@ /* the initial numbers generated are "less random" so will be discarded */ #define RNG_WARMUP_COUNT 0x40000 +#define RNG_INT_OFF 0x1 + +static void __init nsp_rng_init(void __iomem *base) +{ + u32 val; + + /* mask the interrupt */ + val = readl(base + RNG_INT_MASK); + val |= RNG_INT_OFF; + writel(val, base + RNG_INT_MASK); +} + static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) { void __iomem *rng_base = (void __iomem *)rng->priv; + u32 max_words = max / sizeof(u32); + u32 num_words, count; while ((__raw_readl(rng_base + RNG_STATUS) >> 24) == 0) { if (!wait) @@ -37,8 +52,14 @@ static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max, cpu_relax(); } - *(u32 *)buf = __raw_readl(rng_base + RNG_DATA); - return sizeof(u32); + num_words = readl(rng_base + RNG_STATUS) >> 24; + if (num_words > max_words) + num_words = max_words; + + for (count = 0; count < num_words; count++) + ((u32 *)buf)[count] = readl(rng_base + RNG_DATA); + + return num_words * sizeof(u32); } static struct hwrng bcm2835_rng_ops = { @@ -46,10 +67,19 @@ static struct hwrng bcm2835_rng_ops = { .read = bcm2835_rng_read, }; +static const struct of_device_id bcm2835_rng_of_match[] = { + { .compatible = "brcm,bcm2835-rng"}, + { .compatible = "brcm,bcm-nsp-rng", .data = nsp_rng_init}, + { .compatible = "brcm,bcm5301x-rng", .data = nsp_rng_init}, + {}, +}; + static int bcm2835_rng_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; + void (*rng_setup)(void __iomem *base); + const struct of_device_id *rng_id; void __iomem *rng_base; int err; @@ -61,6 +91,15 @@ static int bcm2835_rng_probe(struct platform_device *pdev) } bcm2835_rng_ops.priv = (unsigned long)rng_base; + rng_id = of_match_node(bcm2835_rng_of_match, np); + if (!rng_id) + return -EINVAL; + + /* Check for rng init function, execute it */ + rng_setup = rng_id->data; + if (rng_setup) + rng_setup(rng_base); + /* set warm-up count & enable */ __raw_writel(RNG_WARMUP_COUNT, rng_base + RNG_STATUS); __raw_writel(RNG_RBGEN, rng_base + RNG_CTRL); @@ -90,10 +129,6 @@ static int bcm2835_rng_remove(struct platform_device *pdev) return 0; } -static const struct of_device_id bcm2835_rng_of_match[] = { - { .compatible = "brcm,bcm2835-rng", }, - {}, -}; MODULE_DEVICE_TABLE(of, bcm2835_rng_of_match); static struct platform_driver bcm2835_rng_driver = { diff --git a/drivers/char/hw_random/exynos-rng.c b/drivers/char/hw_random/exynos-rng.c index ed44561ea..23d358553 100644 --- a/drivers/char/hw_random/exynos-rng.c +++ b/drivers/char/hw_random/exynos-rng.c @@ -45,12 +45,12 @@ struct exynos_rng { static u32 exynos_rng_readl(struct exynos_rng *rng, u32 offset) { - return __raw_readl(rng->mem + offset); + return readl_relaxed(rng->mem + offset); } static void exynos_rng_writel(struct exynos_rng *rng, u32 val, u32 offset) { - __raw_writel(val, rng->mem + offset); + writel_relaxed(val, rng->mem + offset); } static int exynos_rng_configure(struct exynos_rng *exynos_rng) diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c new file mode 100644 index 000000000..0cfd81bca --- /dev/null +++ b/drivers/char/hw_random/meson-rng.c @@ -0,0 +1,131 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright (c) 2016 BayLibre, SAS. + * Author: Neil Armstrong + * Copyright (C) 2014 Amlogic, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * The full GNU General Public License is included in this distribution + * in the file called COPYING. + * + * BSD LICENSE + * + * Copyright (c) 2016 BayLibre, SAS. + * Author: Neil Armstrong + * Copyright (C) 2014 Amlogic, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define RNG_DATA 0x00 + +struct meson_rng_data { + void __iomem *base; + struct platform_device *pdev; + struct hwrng rng; +}; + +static int meson_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct meson_rng_data *data = + container_of(rng, struct meson_rng_data, rng); + + if (max < sizeof(u32)) + return 0; + + *(u32 *)buf = readl_relaxed(data->base + RNG_DATA); + + return sizeof(u32); +} + +static int meson_rng_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct meson_rng_data *data; + struct resource *res; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->pdev = pdev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + data->base = devm_ioremap_resource(dev, res); + if (IS_ERR(data->base)) + return PTR_ERR(data->base); + + data->rng.name = pdev->name; + data->rng.read = meson_rng_read; + + platform_set_drvdata(pdev, data); + + return devm_hwrng_register(dev, &data->rng); +} + +static const struct of_device_id meson_rng_of_match[] = { + { .compatible = "amlogic,meson-rng", }, + {}, +}; + +static struct platform_driver meson_rng_driver = { + .probe = meson_rng_probe, + .driver = { + .name = "meson-rng", + .of_match_table = meson_rng_of_match, + }, +}; + +module_platform_driver(meson_rng_driver); + +MODULE_ALIAS("platform:meson-rng"); +MODULE_DESCRIPTION("Meson H/W Random Number Generator driver"); +MODULE_AUTHOR("Lawrence Mok "); +MODULE_AUTHOR("Neil Armstrong "); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index 8a1432e8b..01d4be2c3 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -384,7 +384,12 @@ static int omap_rng_probe(struct platform_device *pdev) } pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_get_sync(&pdev->dev); + if (ret) { + dev_err(&pdev->dev, "Failed to runtime_get device: %d\n", ret); + pm_runtime_put_noidle(&pdev->dev); + goto err_ioremap; + } ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) : get_omap_rng_device_details(priv); @@ -435,8 +440,15 @@ static int __maybe_unused omap_rng_suspend(struct device *dev) static int __maybe_unused omap_rng_resume(struct device *dev) { struct omap_rng_dev *priv = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_get_sync(dev); + if (ret) { + dev_err(dev, "Failed to runtime_get device: %d\n", ret); + pm_runtime_put_noidle(dev); + return ret; + } - pm_runtime_get_sync(dev); priv->pdata->init(priv); return 0; diff --git a/drivers/char/hw_random/stm32-rng.c b/drivers/char/hw_random/stm32-rng.c index 92a810648..63d84e6f1 100644 --- a/drivers/char/hw_random/stm32-rng.c +++ b/drivers/char/hw_random/stm32-rng.c @@ -69,8 +69,12 @@ static int stm32_rng_read(struct hwrng *rng, void *data, size_t max, bool wait) } /* If error detected or data not ready... */ - if (sr != RNG_SR_DRDY) + if (sr != RNG_SR_DRDY) { + if (WARN_ONCE(sr & (RNG_SR_SEIS | RNG_SR_CEIS), + "bad RNG status - %x\n", sr)) + writel_relaxed(0, priv->base + RNG_SR); break; + } *(u32 *)data = readl_relaxed(priv->base + RNG_DR); @@ -79,10 +83,6 @@ static int stm32_rng_read(struct hwrng *rng, void *data, size_t max, bool wait) max -= sizeof(u32); } - if (WARN_ONCE(sr & (RNG_SR_SEIS | RNG_SR_CEIS), - "bad RNG status - %x\n", sr)) - writel_relaxed(0, priv->base + RNG_SR); - pm_runtime_mark_last_busy((struct device *) priv->rng.priv); pm_runtime_put_sync_autosuspend((struct device *) priv->rng.priv); diff --git a/drivers/char/ipmi/Kconfig b/drivers/char/ipmi/Kconfig index 6ed9e9fe5..5a9350b10 100644 --- a/drivers/char/ipmi/Kconfig +++ b/drivers/char/ipmi/Kconfig @@ -50,18 +50,6 @@ config IPMI_SI Currently, only KCS and SMIC are supported. If you are using IPMI, you should probably say "y" here. -config IPMI_SI_PROBE_DEFAULTS - bool 'Probe for all possible IPMI system interfaces by default' - default n - depends on IPMI_SI - help - Modern systems will usually expose IPMI interfaces via a discoverable - firmware mechanism such as ACPI or DMI. Older systems do not, and so - the driver is forced to probe hardware manually. This may cause boot - delays. Say "n" here to disable this manual probing. IPMI will then - only be available on older systems if the "ipmi_si_intf.trydefaults=1" - boot argument is passed. - config IPMI_SSIF tristate 'IPMI SMBus handler (SSIF)' select I2C diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index 44b1bd6ba..d8619998c 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -474,12 +474,12 @@ static DEFINE_MUTEX(smi_watchers_mutex); static const char * const addr_src_to_str[] = { "invalid", "hotmod", "hardcoded", "SPMI", "ACPI", "SMBIOS", "PCI", - "device-tree", "default" + "device-tree" }; const char *ipmi_addr_src_to_str(enum ipmi_addr_src src) { - if (src > SI_DEFAULT) + if (src >= SI_LAST) src = 0; /* Invalid */ return addr_src_to_str[src]; } diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c index 7b1c412b4..a112c0146 100644 --- a/drivers/char/ipmi/ipmi_si_intf.c +++ b/drivers/char/ipmi/ipmi_si_intf.c @@ -1322,7 +1322,6 @@ static bool si_tryplatform = true; #ifdef CONFIG_PCI static bool si_trypci = true; #endif -static bool si_trydefaults = IS_ENABLED(CONFIG_IPMI_SI_PROBE_DEFAULTS); static char *si_type[SI_MAX_PARMS]; #define MAX_SI_TYPE_STR 30 static char si_type_str[MAX_SI_TYPE_STR]; @@ -1371,10 +1370,6 @@ module_param_named(trypci, si_trypci, bool, 0); MODULE_PARM_DESC(trypci, "Setting this to zero will disable the" " default scan of the interfaces identified via pci"); #endif -module_param_named(trydefaults, si_trydefaults, bool, 0); -MODULE_PARM_DESC(trydefaults, "Setting this to 'false' will disable the" - " default scan of the KCS and SMIC interface at the standard" - " address"); module_param_string(type, si_type_str, MAX_SI_TYPE_STR, 0); MODULE_PARM_DESC(type, "Defines the type of each interface, each" " interface separated by commas. The types are 'kcs'," @@ -3461,62 +3456,6 @@ static inline void wait_for_timer_and_thread(struct smi_info *smi_info) del_timer_sync(&smi_info->si_timer); } -static const struct ipmi_default_vals -{ - const int type; - const int port; -} ipmi_defaults[] = -{ - { .type = SI_KCS, .port = 0xca2 }, - { .type = SI_SMIC, .port = 0xca9 }, - { .type = SI_BT, .port = 0xe4 }, - { .port = 0 } -}; - -static void default_find_bmc(void) -{ - struct smi_info *info; - int i; - - for (i = 0; ; i++) { - if (!ipmi_defaults[i].port) - break; -#ifdef CONFIG_PPC - if (check_legacy_ioport(ipmi_defaults[i].port)) - continue; -#endif - info = smi_info_alloc(); - if (!info) - return; - - info->addr_source = SI_DEFAULT; - - info->si_type = ipmi_defaults[i].type; - info->io_setup = port_setup; - info->io.addr_data = ipmi_defaults[i].port; - info->io.addr_type = IPMI_IO_ADDR_SPACE; - - info->io.addr = NULL; - info->io.regspacing = DEFAULT_REGSPACING; - info->io.regsize = DEFAULT_REGSPACING; - info->io.regshift = 0; - - if (add_smi(info) == 0) { - if ((try_smi_init(info)) == 0) { - /* Found one... */ - printk(KERN_INFO PFX "Found default %s" - " state machine at %s address 0x%lx\n", - si_to_str[info->si_type], - addr_space_to_str[info->io.addr_type], - info->io.addr_data); - } else - cleanup_one_si(info); - } else { - kfree(info); - } - } -} - static int is_new_interface(struct smi_info *info) { struct smi_info *e; @@ -3844,8 +3783,6 @@ static int init_ipmi_si(void) #ifdef CONFIG_PARISC register_parisc_driver(&ipmi_parisc_driver); parisc_registered = true; - /* poking PC IO addresses will crash machine, don't do it */ - si_trydefaults = 0; #endif /* We prefer devices with interrupts, but in the case of a machine @@ -3885,16 +3822,6 @@ static int init_ipmi_si(void) if (type) return 0; - if (si_trydefaults) { - mutex_lock(&smi_infos_lock); - if (list_empty(&smi_infos)) { - /* No BMC was found, try defaults. */ - mutex_unlock(&smi_infos_lock); - default_find_bmc(); - } else - mutex_unlock(&smi_infos_lock); - } - mutex_lock(&smi_infos_lock); if (unload_when_empty && list_empty(&smi_infos)) { mutex_unlock(&smi_infos_lock); diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c index 097c86898..5673ffff0 100644 --- a/drivers/char/ipmi/ipmi_ssif.c +++ b/drivers/char/ipmi/ipmi_ssif.c @@ -568,12 +568,16 @@ static void retry_timeout(unsigned long data) } -static void ssif_alert(struct i2c_client *client, unsigned int data) +static void ssif_alert(struct i2c_client *client, enum i2c_alert_protocol type, + unsigned int data) { struct ssif_info *ssif_info = i2c_get_clientdata(client); unsigned long oflags, *flags; bool do_get = false; + if (type != I2C_PROTOCOL_SMBUS_ALERT) + return; + ssif_inc_stat(ssif_info, alerts); flags = ipmi_ssif_lock_cond(ssif_info, &oflags); diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 71025c2f6..a33163dbb 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -66,12 +67,8 @@ static inline int range_is_allowed(unsigned long pfn, unsigned long size) u64 cursor = from; while (cursor < to) { - if (!devmem_is_allowed(pfn)) { - printk(KERN_INFO - "Program %s tried to access /dev/mem between %Lx->%Lx.\n", - current->comm, from, to); + if (!devmem_is_allowed(pfn)) return 0; - } cursor += PAGE_SIZE; pfn++; } @@ -661,6 +658,28 @@ static int mmap_zero(struct file *file, struct vm_area_struct *vma) return 0; } +static unsigned long get_unmapped_area_zero(struct file *file, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ +#ifdef CONFIG_MMU + if (flags & MAP_SHARED) { + /* + * mmap_zero() will call shmem_zero_setup() to create a file, + * so use shmem's get_unmapped_area in case it can be huge; + * and pass NULL for file as in mmap.c's get_unmapped_area(), + * so as not to confuse shmem with our handle on "/dev/zero". + */ + return shmem_get_unmapped_area(NULL, addr, len, pgoff, flags); + } + + /* Otherwise flags & MAP_PRIVATE: with no shmem object beneath it */ + return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); +#else + return -ENOSYS; +#endif +} + static ssize_t write_full(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { @@ -768,6 +787,7 @@ static const struct file_operations zero_fops = { .read_iter = read_iter_zero, .write_iter = write_iter_zero, .mmap = mmap_zero, + .get_unmapped_area = get_unmapped_area_zero, #ifndef CONFIG_MMU .mmap_capabilities = zero_mmap_capabilities, #endif diff --git a/drivers/char/powernv-op-panel.c b/drivers/char/powernv-op-panel.c new file mode 100644 index 000000000..a45dabcc8 --- /dev/null +++ b/drivers/char/powernv-op-panel.c @@ -0,0 +1,223 @@ +/* + * OPAL Operator Panel Display Driver + * + * Copyright 2016, Suraj Jitindar Singh, IBM Corporation. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * This driver creates a character device (/dev/op_panel) which exposes the + * operator panel (character LCD display) on IBM Power Systems machines + * with FSPs. + * A character buffer written to the device will be displayed on the + * operator panel. + */ + +static DEFINE_MUTEX(oppanel_mutex); + +static u32 num_lines, oppanel_size; +static oppanel_line_t *oppanel_lines; +static char *oppanel_data; + +static loff_t oppanel_llseek(struct file *filp, loff_t offset, int whence) +{ + return fixed_size_llseek(filp, offset, whence, oppanel_size); +} + +static ssize_t oppanel_read(struct file *filp, char __user *userbuf, size_t len, + loff_t *f_pos) +{ + return simple_read_from_buffer(userbuf, len, f_pos, oppanel_data, + oppanel_size); +} + +static int __op_panel_update_display(void) +{ + struct opal_msg msg; + int rc, token; + + token = opal_async_get_token_interruptible(); + if (token < 0) { + if (token != -ERESTARTSYS) + pr_debug("Couldn't get OPAL async token [token=%d]\n", + token); + return token; + } + + rc = opal_write_oppanel_async(token, oppanel_lines, num_lines); + switch (rc) { + case OPAL_ASYNC_COMPLETION: + rc = opal_async_wait_response(token, &msg); + if (rc) { + pr_debug("Failed to wait for async response [rc=%d]\n", + rc); + break; + } + rc = opal_get_async_rc(msg); + if (rc != OPAL_SUCCESS) { + pr_debug("OPAL async call returned failed [rc=%d]\n", + rc); + break; + } + case OPAL_SUCCESS: + break; + default: + pr_debug("OPAL write op-panel call failed [rc=%d]\n", rc); + } + + opal_async_release_token(token); + return rc; +} + +static ssize_t oppanel_write(struct file *filp, const char __user *userbuf, + size_t len, loff_t *f_pos) +{ + loff_t f_pos_prev = *f_pos; + ssize_t ret; + int rc; + + if (!*f_pos) + memset(oppanel_data, ' ', oppanel_size); + else if (*f_pos >= oppanel_size) + return -EFBIG; + + ret = simple_write_to_buffer(oppanel_data, oppanel_size, f_pos, userbuf, + len); + if (ret > 0) { + rc = __op_panel_update_display(); + if (rc != OPAL_SUCCESS) { + pr_err_ratelimited("OPAL call failed to write to op panel display [rc=%d]\n", + rc); + *f_pos = f_pos_prev; + return -EIO; + } + } + return ret; +} + +static int oppanel_open(struct inode *inode, struct file *filp) +{ + if (!mutex_trylock(&oppanel_mutex)) { + pr_debug("Device Busy\n"); + return -EBUSY; + } + return 0; +} + +static int oppanel_release(struct inode *inode, struct file *filp) +{ + mutex_unlock(&oppanel_mutex); + return 0; +} + +static const struct file_operations oppanel_fops = { + .owner = THIS_MODULE, + .llseek = oppanel_llseek, + .read = oppanel_read, + .write = oppanel_write, + .open = oppanel_open, + .release = oppanel_release +}; + +static struct miscdevice oppanel_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "op_panel", + .fops = &oppanel_fops +}; + +static int oppanel_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + u32 line_len; + int rc, i; + + rc = of_property_read_u32(np, "#length", &line_len); + if (rc) { + pr_err_ratelimited("Operator panel length property not found\n"); + return rc; + } + rc = of_property_read_u32(np, "#lines", &num_lines); + if (rc) { + pr_err_ratelimited("Operator panel lines property not found\n"); + return rc; + } + oppanel_size = line_len * num_lines; + + pr_devel("Operator panel of size %u found with %u lines of length %u\n", + oppanel_size, num_lines, line_len); + + oppanel_data = kcalloc(oppanel_size, sizeof(*oppanel_data), GFP_KERNEL); + if (!oppanel_data) + return -ENOMEM; + + oppanel_lines = kcalloc(num_lines, sizeof(oppanel_line_t), GFP_KERNEL); + if (!oppanel_lines) { + rc = -ENOMEM; + goto free_oppanel_data; + } + + memset(oppanel_data, ' ', oppanel_size); + for (i = 0; i < num_lines; i++) { + oppanel_lines[i].line_len = cpu_to_be64(line_len); + oppanel_lines[i].line = cpu_to_be64(__pa(&oppanel_data[i * + line_len])); + } + + rc = misc_register(&oppanel_dev); + if (rc) { + pr_err_ratelimited("Failed to register as misc device\n"); + goto free_oppanel; + } + + return 0; + +free_oppanel: + kfree(oppanel_lines); +free_oppanel_data: + kfree(oppanel_data); + return rc; +} + +static int oppanel_remove(struct platform_device *pdev) +{ + misc_deregister(&oppanel_dev); + kfree(oppanel_lines); + kfree(oppanel_data); + return 0; +} + +static const struct of_device_id oppanel_match[] = { + { .compatible = "ibm,opal-oppanel" }, + { }, +}; + +static struct platform_driver oppanel_driver = { + .driver = { + .name = "powernv-op-panel", + .of_match_table = oppanel_match, + }, + .probe = oppanel_probe, + .remove = oppanel_remove, +}; + +module_platform_driver(oppanel_driver); + +MODULE_DEVICE_TABLE(of, oppanel_match); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("PowerNV Operator Panel LCD Display Driver"); +MODULE_AUTHOR("Suraj Jitindar Singh "); diff --git a/drivers/char/random.c b/drivers/char/random.c index d72c6d14a..3efb3bf0a 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -249,6 +249,7 @@ #include #include #include +#include #include #include #include @@ -261,6 +262,7 @@ #include #include #include +#include #include #include @@ -413,6 +415,34 @@ static struct fasync_struct *fasync; static DEFINE_SPINLOCK(random_ready_list_lock); static LIST_HEAD(random_ready_list); +struct crng_state { + __u32 state[16]; + unsigned long init_time; + spinlock_t lock; +}; + +struct crng_state primary_crng = { + .lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock), +}; + +/* + * crng_init = 0 --> Uninitialized + * 1 --> Initialized + * 2 --> Initialized from input_pool + * + * crng_init is protected by primary_crng->lock, and only increases + * its value (from 0->1->2). + */ +static int crng_init = 0; +#define crng_ready() (likely(crng_init > 0)) +static int crng_init_cnt = 0; +#define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE) +static void _extract_crng(struct crng_state *crng, + __u8 out[CHACHA20_BLOCK_SIZE]); +static void _crng_backtrack_protect(struct crng_state *crng, + __u8 tmp[CHACHA20_BLOCK_SIZE], int used); +static void process_random_ready_list(void); + /********************************************************************** * * OS independent entropy store. Here are the functions which handle @@ -442,10 +472,15 @@ struct entropy_store { __u8 last_data[EXTRACT_SIZE]; }; +static ssize_t extract_entropy(struct entropy_store *r, void *buf, + size_t nbytes, int min, int rsvd); +static ssize_t _extract_entropy(struct entropy_store *r, void *buf, + size_t nbytes, int fips); + +static void crng_reseed(struct crng_state *crng, struct entropy_store *r); static void push_to_pool(struct work_struct *work); static __u32 input_pool_data[INPUT_POOL_WORDS]; static __u32 blocking_pool_data[OUTPUT_POOL_WORDS]; -static __u32 nonblocking_pool_data[OUTPUT_POOL_WORDS]; static struct entropy_store input_pool = { .poolinfo = &poolinfo_table[0], @@ -466,16 +501,6 @@ static struct entropy_store blocking_pool = { push_to_pool), }; -static struct entropy_store nonblocking_pool = { - .poolinfo = &poolinfo_table[1], - .name = "nonblocking", - .pull = &input_pool, - .lock = __SPIN_LOCK_UNLOCKED(nonblocking_pool.lock), - .pool = nonblocking_pool_data, - .push_work = __WORK_INITIALIZER(nonblocking_pool.push_work, - push_to_pool), -}; - static __u32 const twist_table[8] = { 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158, 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 }; @@ -678,12 +703,6 @@ retry: if (!r->initialized && r->entropy_total > 128) { r->initialized = 1; r->entropy_total = 0; - if (r == &nonblocking_pool) { - prandom_reseed_late(); - process_random_ready_list(); - wake_up_all(&urandom_init_wait); - pr_notice("random: %s pool is initialized\n", r->name); - } } trace_credit_entropy_bits(r->name, nbits, @@ -693,30 +712,27 @@ retry: if (r == &input_pool) { int entropy_bits = entropy_count >> ENTROPY_SHIFT; + if (crng_init < 2 && entropy_bits >= 128) { + crng_reseed(&primary_crng, r); + entropy_bits = r->entropy_count >> ENTROPY_SHIFT; + } + /* should we wake readers? */ if (entropy_bits >= random_read_wakeup_bits) { wake_up_interruptible(&random_read_wait); kill_fasync(&fasync, SIGIO, POLL_IN); } /* If the input pool is getting full, send some - * entropy to the two output pools, flipping back and - * forth between them, until the output pools are 75% - * full. + * entropy to the blocking pool until it is 75% full. */ if (entropy_bits > random_write_wakeup_bits && r->initialized && r->entropy_total >= 2*random_read_wakeup_bits) { - static struct entropy_store *last = &blocking_pool; struct entropy_store *other = &blocking_pool; - if (last == &blocking_pool) - other = &nonblocking_pool; if (other->entropy_count <= - 3 * other->poolinfo->poolfracbits / 4) - last = other; - if (last->entropy_count <= - 3 * last->poolinfo->poolfracbits / 4) { - schedule_work(&last->push_work); + 3 * other->poolinfo->poolfracbits / 4) { + schedule_work(&other->push_work); r->entropy_total = 0; } } @@ -737,6 +753,223 @@ static int credit_entropy_bits_safe(struct entropy_store *r, int nbits) return 0; } +/********************************************************************* + * + * CRNG using CHACHA20 + * + *********************************************************************/ + +#define CRNG_RESEED_INTERVAL (300*HZ) + +static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait); + +#ifdef CONFIG_NUMA +/* + * Hack to deal with crazy userspace progams when they are all trying + * to access /dev/urandom in parallel. The programs are almost + * certainly doing something terribly wrong, but we'll work around + * their brain damage. + */ +static struct crng_state **crng_node_pool __read_mostly; +#endif + +static void crng_initialize(struct crng_state *crng) +{ + int i; + unsigned long rv; + + memcpy(&crng->state[0], "expand 32-byte k", 16); + if (crng == &primary_crng) + _extract_entropy(&input_pool, &crng->state[4], + sizeof(__u32) * 12, 0); + else + get_random_bytes(&crng->state[4], sizeof(__u32) * 12); + for (i = 4; i < 16; i++) { + if (!arch_get_random_seed_long(&rv) && + !arch_get_random_long(&rv)) + rv = random_get_entropy(); + crng->state[i] ^= rv; + } + crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; +} + +static int crng_fast_load(const char *cp, size_t len) +{ + unsigned long flags; + char *p; + + if (!spin_trylock_irqsave(&primary_crng.lock, flags)) + return 0; + if (crng_ready()) { + spin_unlock_irqrestore(&primary_crng.lock, flags); + return 0; + } + p = (unsigned char *) &primary_crng.state[4]; + while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) { + p[crng_init_cnt % CHACHA20_KEY_SIZE] ^= *cp; + cp++; crng_init_cnt++; len--; + } + if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { + crng_init = 1; + wake_up_interruptible(&crng_init_wait); + pr_notice("random: fast init done\n"); + } + spin_unlock_irqrestore(&primary_crng.lock, flags); + return 1; +} + +static void crng_reseed(struct crng_state *crng, struct entropy_store *r) +{ + unsigned long flags; + int i, num; + union { + __u8 block[CHACHA20_BLOCK_SIZE]; + __u32 key[8]; + } buf; + + if (r) { + num = extract_entropy(r, &buf, 32, 16, 0); + if (num == 0) + return; + } else { + _extract_crng(&primary_crng, buf.block); + _crng_backtrack_protect(&primary_crng, buf.block, + CHACHA20_KEY_SIZE); + } + spin_lock_irqsave(&primary_crng.lock, flags); + for (i = 0; i < 8; i++) { + unsigned long rv; + if (!arch_get_random_seed_long(&rv) && + !arch_get_random_long(&rv)) + rv = random_get_entropy(); + crng->state[i+4] ^= buf.key[i] ^ rv; + } + memzero_explicit(&buf, sizeof(buf)); + crng->init_time = jiffies; + if (crng == &primary_crng && crng_init < 2) { + crng_init = 2; + process_random_ready_list(); + wake_up_interruptible(&crng_init_wait); + pr_notice("random: crng init done\n"); + } + spin_unlock_irqrestore(&primary_crng.lock, flags); +} + +static inline void maybe_reseed_primary_crng(void) +{ + if (crng_init > 2 && + time_after(jiffies, primary_crng.init_time + CRNG_RESEED_INTERVAL)) + crng_reseed(&primary_crng, &input_pool); +} + +static inline void crng_wait_ready(void) +{ + wait_event_interruptible(crng_init_wait, crng_ready()); +} + +static void _extract_crng(struct crng_state *crng, + __u8 out[CHACHA20_BLOCK_SIZE]) +{ + unsigned long v, flags; + + if (crng_init > 1 && + time_after(jiffies, crng->init_time + CRNG_RESEED_INTERVAL)) + crng_reseed(crng, crng == &primary_crng ? &input_pool : NULL); + spin_lock_irqsave(&crng->lock, flags); + if (arch_get_random_long(&v)) + crng->state[14] ^= v; + chacha20_block(&crng->state[0], out); + if (crng->state[12] == 0) + crng->state[13]++; + spin_unlock_irqrestore(&crng->lock, flags); +} + +static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE]) +{ + struct crng_state *crng = NULL; + +#ifdef CONFIG_NUMA + if (crng_node_pool) + crng = crng_node_pool[numa_node_id()]; + if (crng == NULL) +#endif + crng = &primary_crng; + _extract_crng(crng, out); +} + +/* + * Use the leftover bytes from the CRNG block output (if there is + * enough) to mutate the CRNG key to provide backtracking protection. + */ +static void _crng_backtrack_protect(struct crng_state *crng, + __u8 tmp[CHACHA20_BLOCK_SIZE], int used) +{ + unsigned long flags; + __u32 *s, *d; + int i; + + used = round_up(used, sizeof(__u32)); + if (used + CHACHA20_KEY_SIZE > CHACHA20_BLOCK_SIZE) { + extract_crng(tmp); + used = 0; + } + spin_lock_irqsave(&crng->lock, flags); + s = (__u32 *) &tmp[used]; + d = &crng->state[4]; + for (i=0; i < 8; i++) + *d++ ^= *s++; + spin_unlock_irqrestore(&crng->lock, flags); +} + +static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used) +{ + struct crng_state *crng = NULL; + +#ifdef CONFIG_NUMA + if (crng_node_pool) + crng = crng_node_pool[numa_node_id()]; + if (crng == NULL) +#endif + crng = &primary_crng; + _crng_backtrack_protect(crng, tmp, used); +} + +static ssize_t extract_crng_user(void __user *buf, size_t nbytes) +{ + ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE; + __u8 tmp[CHACHA20_BLOCK_SIZE]; + int large_request = (nbytes > 256); + + while (nbytes) { + if (large_request && need_resched()) { + if (signal_pending(current)) { + if (ret == 0) + ret = -ERESTARTSYS; + break; + } + schedule(); + } + + extract_crng(tmp); + i = min_t(int, nbytes, CHACHA20_BLOCK_SIZE); + if (copy_to_user(buf, tmp, i)) { + ret = -EFAULT; + break; + } + + nbytes -= i; + buf += i; + ret += i; + } + crng_backtrack_protect(tmp, i); + + /* Wipe data just written to memory */ + memzero_explicit(tmp, sizeof(tmp)); + + return ret; +} + + /********************************************************************* * * Entropy input management @@ -753,12 +986,12 @@ struct timer_rand_state { #define INIT_TIMER_RAND_STATE { INITIAL_JIFFIES, }; /* - * Add device- or boot-specific data to the input and nonblocking - * pools to help initialize them to unique values. + * Add device- or boot-specific data to the input pool to help + * initialize it. * - * None of this adds any entropy, it is meant to avoid the - * problem of the nonblocking pool having similar initial state - * across largely identical devices. + * None of this adds any entropy; it is meant to avoid the problem of + * the entropy pool having similar initial state across largely + * identical devices. */ void add_device_randomness(const void *buf, unsigned int size) { @@ -770,11 +1003,6 @@ void add_device_randomness(const void *buf, unsigned int size) _mix_pool_bytes(&input_pool, buf, size); _mix_pool_bytes(&input_pool, &time, sizeof(time)); spin_unlock_irqrestore(&input_pool.lock, flags); - - spin_lock_irqsave(&nonblocking_pool.lock, flags); - _mix_pool_bytes(&nonblocking_pool, buf, size); - _mix_pool_bytes(&nonblocking_pool, &time, sizeof(time)); - spin_unlock_irqrestore(&nonblocking_pool.lock, flags); } EXPORT_SYMBOL(add_device_randomness); @@ -805,7 +1033,7 @@ static void add_timer_randomness(struct timer_rand_state *state, unsigned num) sample.jiffies = jiffies; sample.cycles = random_get_entropy(); sample.num = num; - r = nonblocking_pool.initialized ? &input_pool : &nonblocking_pool; + r = &input_pool; mix_pool_bytes(r, &sample, sizeof(sample)); /* @@ -921,11 +1149,21 @@ void add_interrupt_randomness(int irq, int irq_flags) fast_mix(fast_pool); add_interrupt_bench(cycles); + if (!crng_ready()) { + if ((fast_pool->count >= 64) && + crng_fast_load((char *) fast_pool->pool, + sizeof(fast_pool->pool))) { + fast_pool->count = 0; + fast_pool->last = now; + } + return; + } + if ((fast_pool->count < 64) && !time_after(now, fast_pool->last + HZ)) return; - r = nonblocking_pool.initialized ? &input_pool : &nonblocking_pool; + r = &input_pool; if (!spin_trylock(&r->lock)) return; @@ -969,9 +1207,6 @@ EXPORT_SYMBOL_GPL(add_disk_randomness); * *********************************************************************/ -static ssize_t extract_entropy(struct entropy_store *r, void *buf, - size_t nbytes, int min, int rsvd); - /* * This utility inline function is responsible for transferring entropy * from the primary pool to the secondary extraction pool. We make @@ -1146,6 +1381,36 @@ static void extract_buf(struct entropy_store *r, __u8 *out) memzero_explicit(&hash, sizeof(hash)); } +static ssize_t _extract_entropy(struct entropy_store *r, void *buf, + size_t nbytes, int fips) +{ + ssize_t ret = 0, i; + __u8 tmp[EXTRACT_SIZE]; + unsigned long flags; + + while (nbytes) { + extract_buf(r, tmp); + + if (fips) { + spin_lock_irqsave(&r->lock, flags); + if (!memcmp(tmp, r->last_data, EXTRACT_SIZE)) + panic("Hardware RNG duplicated output!\n"); + memcpy(r->last_data, tmp, EXTRACT_SIZE); + spin_unlock_irqrestore(&r->lock, flags); + } + i = min_t(int, nbytes, EXTRACT_SIZE); + memcpy(buf, tmp, i); + nbytes -= i; + buf += i; + ret += i; + } + + /* Wipe data just returned from memory */ + memzero_explicit(tmp, sizeof(tmp)); + + return ret; +} + /* * This function extracts randomness from the "entropy pool", and * returns it in a buffer. @@ -1158,7 +1423,6 @@ static void extract_buf(struct entropy_store *r, __u8 *out) static ssize_t extract_entropy(struct entropy_store *r, void *buf, size_t nbytes, int min, int reserved) { - ssize_t ret = 0, i; __u8 tmp[EXTRACT_SIZE]; unsigned long flags; @@ -1182,27 +1446,7 @@ static ssize_t extract_entropy(struct entropy_store *r, void *buf, xfer_secondary_pool(r, nbytes); nbytes = account(r, nbytes, min, reserved); - while (nbytes) { - extract_buf(r, tmp); - - if (fips_enabled) { - spin_lock_irqsave(&r->lock, flags); - if (!memcmp(tmp, r->last_data, EXTRACT_SIZE)) - panic("Hardware RNG duplicated output!\n"); - memcpy(r->last_data, tmp, EXTRACT_SIZE); - spin_unlock_irqrestore(&r->lock, flags); - } - i = min_t(int, nbytes, EXTRACT_SIZE); - memcpy(buf, tmp, i); - nbytes -= i; - buf += i; - ret += i; - } - - /* Wipe data just returned from memory */ - memzero_explicit(tmp, sizeof(tmp)); - - return ret; + return _extract_entropy(r, buf, nbytes, fips_enabled); } /* @@ -1257,15 +1501,28 @@ static ssize_t extract_entropy_user(struct entropy_store *r, void __user *buf, */ void get_random_bytes(void *buf, int nbytes) { + __u8 tmp[CHACHA20_BLOCK_SIZE]; + #if DEBUG_RANDOM_BOOT > 0 - if (unlikely(nonblocking_pool.initialized == 0)) + if (!crng_ready()) printk(KERN_NOTICE "random: %pF get_random_bytes called " - "with %d bits of entropy available\n", - (void *) _RET_IP_, - nonblocking_pool.entropy_total); + "with crng_init = %d\n", (void *) _RET_IP_, crng_init); #endif trace_get_random_bytes(nbytes, _RET_IP_); - extract_entropy(&nonblocking_pool, buf, nbytes, 0, 0); + + while (nbytes >= CHACHA20_BLOCK_SIZE) { + extract_crng(buf); + buf += CHACHA20_BLOCK_SIZE; + nbytes -= CHACHA20_BLOCK_SIZE; + } + + if (nbytes > 0) { + extract_crng(tmp); + memcpy(buf, tmp, nbytes); + crng_backtrack_protect(tmp, nbytes); + } else + crng_backtrack_protect(tmp, CHACHA20_BLOCK_SIZE); + memzero_explicit(tmp, sizeof(tmp)); } EXPORT_SYMBOL(get_random_bytes); @@ -1283,7 +1540,7 @@ int add_random_ready_callback(struct random_ready_callback *rdy) unsigned long flags; int err = -EALREADY; - if (likely(nonblocking_pool.initialized)) + if (crng_ready()) return err; owner = rdy->owner; @@ -1291,7 +1548,7 @@ int add_random_ready_callback(struct random_ready_callback *rdy) return -ENOENT; spin_lock_irqsave(&random_ready_list_lock, flags); - if (nonblocking_pool.initialized) + if (crng_ready()) goto out; owner = NULL; @@ -1355,7 +1612,7 @@ void get_random_bytes_arch(void *buf, int nbytes) } if (nbytes) - extract_entropy(&nonblocking_pool, p, nbytes, 0, 0); + get_random_bytes(p, nbytes); } EXPORT_SYMBOL(get_random_bytes_arch); @@ -1398,9 +1655,28 @@ static void init_std_data(struct entropy_store *r) */ static int rand_initialize(void) { +#ifdef CONFIG_NUMA + int i; + struct crng_state *crng; + struct crng_state **pool; +#endif + init_std_data(&input_pool); init_std_data(&blocking_pool); - init_std_data(&nonblocking_pool); + crng_initialize(&primary_crng); + +#ifdef CONFIG_NUMA + pool = kcalloc(nr_node_ids, sizeof(*pool), GFP_KERNEL|__GFP_NOFAIL); + for_each_online_node(i) { + crng = kmalloc_node(sizeof(struct crng_state), + GFP_KERNEL | __GFP_NOFAIL, i); + spin_lock_init(&crng->lock); + crng_initialize(crng); + pool[i] = crng; + } + mb(); + crng_node_pool = pool; +#endif return 0; } early_initcall(rand_initialize); @@ -1462,22 +1738,22 @@ random_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) static ssize_t urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { + unsigned long flags; static int maxwarn = 10; int ret; - if (unlikely(nonblocking_pool.initialized == 0) && - maxwarn > 0) { + if (!crng_ready() && maxwarn > 0) { maxwarn--; printk(KERN_NOTICE "random: %s: uninitialized urandom read " - "(%zd bytes read, %d bits of entropy available)\n", - current->comm, nbytes, nonblocking_pool.entropy_total); + "(%zd bytes read)\n", + current->comm, nbytes); + spin_lock_irqsave(&primary_crng.lock, flags); + crng_init_cnt = 0; + spin_unlock_irqrestore(&primary_crng.lock, flags); } - nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3)); - ret = extract_entropy_user(&nonblocking_pool, buf, nbytes); - - trace_urandom_read(8 * nbytes, ENTROPY_BITS(&nonblocking_pool), - ENTROPY_BITS(&input_pool)); + ret = extract_crng_user(buf, nbytes); + trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool)); return ret; } @@ -1523,10 +1799,7 @@ static ssize_t random_write(struct file *file, const char __user *buffer, { size_t ret; - ret = write_pool(&blocking_pool, buffer, count); - if (ret) - return ret; - ret = write_pool(&nonblocking_pool, buffer, count); + ret = write_pool(&input_pool, buffer, count); if (ret) return ret; @@ -1575,7 +1848,6 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg) if (!capable(CAP_SYS_ADMIN)) return -EPERM; input_pool.entropy_count = 0; - nonblocking_pool.entropy_count = 0; blocking_pool.entropy_count = 0; return 0; default: @@ -1617,11 +1889,10 @@ SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, if (flags & GRND_RANDOM) return _random_read(flags & GRND_NONBLOCK, buf, count); - if (unlikely(nonblocking_pool.initialized == 0)) { + if (!crng_ready()) { if (flags & GRND_NONBLOCK) return -EAGAIN; - wait_event_interruptible(urandom_init_wait, - nonblocking_pool.initialized); + crng_wait_ready(); if (signal_pending(current)) return -ERESTARTSYS; } @@ -1857,18 +2128,17 @@ void add_hwgenerator_randomness(const char *buffer, size_t count, { struct entropy_store *poolp = &input_pool; - if (unlikely(nonblocking_pool.initialized == 0)) - poolp = &nonblocking_pool; - else { - /* Suspend writing if we're above the trickle - * threshold. We'll be woken up again once below - * random_write_wakeup_thresh, or when the calling - * thread is about to terminate. - */ - wait_event_interruptible(random_write_wait, - kthread_should_stop() || - ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits); + if (!crng_ready()) { + crng_fast_load(buffer, count); + return; } + + /* Suspend writing if we're above the trickle threshold. + * We'll be woken up again once below random_write_wakeup_thresh, + * or when the calling thread is about to terminate. + */ + wait_event_interruptible(random_write_wait, kthread_should_stop() || + ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits); mix_pool_bytes(poolp, buffer, count); credit_entropy_bits(poolp, entropy); } diff --git a/drivers/char/tpm/Kconfig b/drivers/char/tpm/Kconfig index 3b84a8b1b..9faa0b1e7 100644 --- a/drivers/char/tpm/Kconfig +++ b/drivers/char/tpm/Kconfig @@ -24,9 +24,16 @@ menuconfig TCG_TPM if TCG_TPM +config TCG_TIS_CORE + tristate + ---help--- + TCG TIS TPM core driver. It implements the TPM TCG TIS logic and hooks + into the TPM kernel APIs. Physical layers will register against it. + config TCG_TIS tristate "TPM Interface Specification 1.2 Interface / TPM 2.0 FIFO Interface" depends on X86 + select TCG_TIS_CORE ---help--- If you have a TPM security chip that is compliant with the TCG TIS 1.2 TPM specification (TPM1.2) or the TCG PTP FIFO @@ -34,6 +41,18 @@ config TCG_TIS within Linux. To compile this driver as a module, choose M here; the module will be called tpm_tis. +config TCG_TIS_SPI + tristate "TPM Interface Specification 1.3 Interface / TPM 2.0 FIFO Interface - (SPI)" + depends on SPI + select TCG_TIS_CORE + ---help--- + If you have a TPM security chip which is connected to a regular, + non-tcg SPI master (i.e. most embedded platforms) that is compliant with the + TCG TIS 1.3 TPM specification (TPM1.2) or the TCG PTP FIFO + specification (TPM2.0) say Yes and it will be accessible from + within Linux. To compile this driver as a module, choose M here; + the module will be called tpm_tis_spi. + config TCG_TIS_I2C_ATMEL tristate "TPM Interface Specification 1.2 Interface (I2C - Atmel)" depends on I2C @@ -122,5 +141,16 @@ config TCG_CRB from within Linux. To compile this driver as a module, choose M here; the module will be called tpm_crb. +config TCG_VTPM_PROXY + tristate "VTPM Proxy Interface" + depends on TCG_TPM + select ANON_INODES + ---help--- + This driver proxies for an emulated TPM (vTPM) running in userspace. + A device /dev/vtpmx is provided that creates a device pair + /dev/vtpmX and a server-side file descriptor on which the vTPM + can receive commands. + + source "drivers/char/tpm/st33zp24/Kconfig" endif # TCG_TPM diff --git a/drivers/char/tpm/Makefile b/drivers/char/tpm/Makefile index 56e8f1f3d..a385fb8c1 100644 --- a/drivers/char/tpm/Makefile +++ b/drivers/char/tpm/Makefile @@ -12,7 +12,9 @@ ifdef CONFIG_TCG_IBMVTPM tpm-y += tpm_eventlog.o tpm_of.o endif endif +obj-$(CONFIG_TCG_TIS_CORE) += tpm_tis_core.o obj-$(CONFIG_TCG_TIS) += tpm_tis.o +obj-$(CONFIG_TCG_TIS_SPI) += tpm_tis_spi.o obj-$(CONFIG_TCG_TIS_I2C_ATMEL) += tpm_i2c_atmel.o obj-$(CONFIG_TCG_TIS_I2C_INFINEON) += tpm_i2c_infineon.o obj-$(CONFIG_TCG_TIS_I2C_NUVOTON) += tpm_i2c_nuvoton.o @@ -23,3 +25,4 @@ obj-$(CONFIG_TCG_IBMVTPM) += tpm_ibmvtpm.o obj-$(CONFIG_TCG_TIS_ST33ZP24) += st33zp24/ obj-$(CONFIG_TCG_XEN) += xen-tpmfront.o obj-$(CONFIG_TCG_CRB) += tpm_crb.o +obj-$(CONFIG_TCG_VTPM_PROXY) += tpm_vtpm_proxy.o diff --git a/drivers/char/tpm/st33zp24/Kconfig b/drivers/char/tpm/st33zp24/Kconfig index 19c007461..e74c6f29f 100644 --- a/drivers/char/tpm/st33zp24/Kconfig +++ b/drivers/char/tpm/st33zp24/Kconfig @@ -1,6 +1,5 @@ config TCG_TIS_ST33ZP24 - tristate "STMicroelectronics TPM Interface Specification 1.2 Interface" - depends on GPIOLIB || COMPILE_TEST + tristate ---help--- STMicroelectronics ST33ZP24 core driver. It implements the core TPM1.2 logic and hooks into the TPM kernel APIs. Physical layers will @@ -10,9 +9,9 @@ config TCG_TIS_ST33ZP24 tpm_st33zp24. config TCG_TIS_ST33ZP24_I2C - tristate "TPM 1.2 ST33ZP24 I2C support" - depends on TCG_TIS_ST33ZP24 + tristate "STMicroelectronics TPM Interface Specification 1.2 Interface (I2C)" depends on I2C + select TCG_TIS_ST33ZP24 ---help--- This module adds support for the STMicroelectronics TPM security chip ST33ZP24 with i2c interface. @@ -20,9 +19,9 @@ config TCG_TIS_ST33ZP24_I2C called tpm_st33zp24_i2c. config TCG_TIS_ST33ZP24_SPI - tristate "TPM 1.2 ST33ZP24 SPI support" - depends on TCG_TIS_ST33ZP24 + tristate "STMicroelectronics TPM Interface Specification 1.2 Interface (SPI)" depends on SPI + select TCG_TIS_ST33ZP24 ---help--- This module adds support for the STMicroelectronics TPM security chip ST33ZP24 with spi interface. diff --git a/drivers/char/tpm/st33zp24/i2c.c b/drivers/char/tpm/st33zp24/i2c.c index 309d2767c..028a9cd76 100644 --- a/drivers/char/tpm/st33zp24/i2c.c +++ b/drivers/char/tpm/st33zp24/i2c.c @@ -1,6 +1,6 @@ /* * STMicroelectronics TPM I2C Linux driver for TPM ST33ZP24 - * Copyright (C) 2009 - 2015 STMicroelectronics + * Copyright (C) 2009 - 2016 STMicroelectronics * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,11 +19,14 @@ #include #include #include +#include #include #include +#include #include #include +#include "../tpm.h" #include "st33zp24.h" #define TPM_DUMMY_BYTE 0xAA @@ -108,11 +111,40 @@ static const struct st33zp24_phy_ops i2c_phy_ops = { .recv = st33zp24_i2c_recv, }; -#ifdef CONFIG_OF -static int st33zp24_i2c_of_request_resources(struct st33zp24_i2c_phy *phy) +static int st33zp24_i2c_acpi_request_resources(struct i2c_client *client) { + struct tpm_chip *chip = i2c_get_clientdata(client); + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); + struct st33zp24_i2c_phy *phy = tpm_dev->phy_id; + struct gpio_desc *gpiod_lpcpd; + struct device *dev = &client->dev; + + /* Get LPCPD GPIO from ACPI */ + gpiod_lpcpd = devm_gpiod_get_index(dev, "TPM IO LPCPD", 1, + GPIOD_OUT_HIGH); + if (IS_ERR(gpiod_lpcpd)) { + dev_err(&client->dev, + "Failed to retrieve lpcpd-gpios from acpi.\n"); + phy->io_lpcpd = -1; + /* + * lpcpd pin is not specified. This is not an issue as + * power management can be also managed by TPM specific + * commands. So leave with a success status code. + */ + return 0; + } + + phy->io_lpcpd = desc_to_gpio(gpiod_lpcpd); + + return 0; +} + +static int st33zp24_i2c_of_request_resources(struct i2c_client *client) +{ + struct tpm_chip *chip = i2c_get_clientdata(client); + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); + struct st33zp24_i2c_phy *phy = tpm_dev->phy_id; struct device_node *pp; - struct i2c_client *client = phy->client; int gpio; int ret; @@ -146,16 +178,12 @@ static int st33zp24_i2c_of_request_resources(struct st33zp24_i2c_phy *phy) return 0; } -#else -static int st33zp24_i2c_of_request_resources(struct st33zp24_i2c_phy *phy) -{ - return -ENODEV; -} -#endif -static int st33zp24_i2c_request_resources(struct i2c_client *client, - struct st33zp24_i2c_phy *phy) +static int st33zp24_i2c_request_resources(struct i2c_client *client) { + struct tpm_chip *chip = i2c_get_clientdata(client); + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); + struct st33zp24_i2c_phy *phy = tpm_dev->phy_id; struct st33zp24_platform_data *pdata; int ret; @@ -212,13 +240,18 @@ static int st33zp24_i2c_probe(struct i2c_client *client, return -ENOMEM; phy->client = client; + pdata = client->dev.platform_data; if (!pdata && client->dev.of_node) { - ret = st33zp24_i2c_of_request_resources(phy); + ret = st33zp24_i2c_of_request_resources(client); if (ret) return ret; } else if (pdata) { - ret = st33zp24_i2c_request_resources(client, phy); + ret = st33zp24_i2c_request_resources(client); + if (ret) + return ret; + } else if (ACPI_HANDLE(&client->dev)) { + ret = st33zp24_i2c_acpi_request_resources(client); if (ret) return ret; } @@ -245,13 +278,17 @@ static const struct i2c_device_id st33zp24_i2c_id[] = { }; MODULE_DEVICE_TABLE(i2c, st33zp24_i2c_id); -#ifdef CONFIG_OF static const struct of_device_id of_st33zp24_i2c_match[] = { { .compatible = "st,st33zp24-i2c", }, {} }; MODULE_DEVICE_TABLE(of, of_st33zp24_i2c_match); -#endif + +static const struct acpi_device_id st33zp24_i2c_acpi_match[] = { + {"SMO3324"}, + {} +}; +MODULE_DEVICE_TABLE(acpi, st33zp24_i2c_acpi_match); static SIMPLE_DEV_PM_OPS(st33zp24_i2c_ops, st33zp24_pm_suspend, st33zp24_pm_resume); @@ -261,6 +298,7 @@ static struct i2c_driver st33zp24_i2c_driver = { .name = TPM_ST33_I2C, .pm = &st33zp24_i2c_ops, .of_match_table = of_match_ptr(of_st33zp24_i2c_match), + .acpi_match_table = ACPI_PTR(st33zp24_i2c_acpi_match), }, .probe = st33zp24_i2c_probe, .remove = st33zp24_i2c_remove, diff --git a/drivers/char/tpm/st33zp24/spi.c b/drivers/char/tpm/st33zp24/spi.c index f974c945c..9f5a01170 100644 --- a/drivers/char/tpm/st33zp24/spi.c +++ b/drivers/char/tpm/st33zp24/spi.c @@ -1,6 +1,6 @@ /* * STMicroelectronics TPM SPI Linux driver for TPM ST33ZP24 - * Copyright (C) 2009 - 2015 STMicroelectronics + * Copyright (C) 2009 - 2016 STMicroelectronics * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,11 +19,14 @@ #include #include #include +#include #include #include +#include #include #include +#include "../tpm.h" #include "st33zp24.h" #define TPM_DATA_FIFO 0x24 @@ -66,7 +69,7 @@ struct st33zp24_spi_phy { struct spi_device *spi_device; - struct spi_transfer spi_xfer; + u8 tx_buf[ST33ZP24_SPI_BUFFER_SIZE]; u8 rx_buf[ST33ZP24_SPI_BUFFER_SIZE]; @@ -110,43 +113,39 @@ static int st33zp24_status_to_errno(u8 code) static int st33zp24_spi_send(void *phy_id, u8 tpm_register, u8 *tpm_data, int tpm_size) { - u8 data = 0; - int total_length = 0, nbr_dummy_bytes = 0, ret = 0; + int total_length = 0, ret = 0; struct st33zp24_spi_phy *phy = phy_id; struct spi_device *dev = phy->spi_device; - u8 *tx_buf = (u8 *)phy->spi_xfer.tx_buf; - u8 *rx_buf = phy->spi_xfer.rx_buf; + struct spi_transfer spi_xfer = { + .tx_buf = phy->tx_buf, + .rx_buf = phy->rx_buf, + }; /* Pre-Header */ - data = TPM_WRITE_DIRECTION | LOCALITY0; - memcpy(tx_buf + total_length, &data, sizeof(data)); - total_length++; - data = tpm_register; - memcpy(tx_buf + total_length, &data, sizeof(data)); - total_length++; + phy->tx_buf[total_length++] = TPM_WRITE_DIRECTION | LOCALITY0; + phy->tx_buf[total_length++] = tpm_register; if (tpm_size > 0 && tpm_register == TPM_DATA_FIFO) { - tx_buf[total_length++] = tpm_size >> 8; - tx_buf[total_length++] = tpm_size; + phy->tx_buf[total_length++] = tpm_size >> 8; + phy->tx_buf[total_length++] = tpm_size; } - memcpy(&tx_buf[total_length], tpm_data, tpm_size); + memcpy(&phy->tx_buf[total_length], tpm_data, tpm_size); total_length += tpm_size; - nbr_dummy_bytes = phy->latency; - memset(&tx_buf[total_length], TPM_DUMMY_BYTE, nbr_dummy_bytes); + memset(&phy->tx_buf[total_length], TPM_DUMMY_BYTE, phy->latency); - phy->spi_xfer.len = total_length + nbr_dummy_bytes; + spi_xfer.len = total_length + phy->latency; - ret = spi_sync_transfer(dev, &phy->spi_xfer, 1); + ret = spi_sync_transfer(dev, &spi_xfer, 1); if (ret == 0) - ret = rx_buf[total_length + nbr_dummy_bytes - 1]; + ret = phy->rx_buf[total_length + phy->latency - 1]; return st33zp24_status_to_errno(ret); } /* st33zp24_spi_send() */ /* - * read8_recv + * st33zp24_spi_read8_recv * Recv byte from the TIS register according to the ST33ZP24 SPI protocol. * @param: phy_id, the phy description * @param: tpm_register, the tpm tis register where the data should be read @@ -154,40 +153,37 @@ static int st33zp24_spi_send(void *phy_id, u8 tpm_register, u8 *tpm_data, * @param: tpm_size, tpm TPM response size to read. * @return: should be zero if success else a negative error code. */ -static int read8_reg(void *phy_id, u8 tpm_register, u8 *tpm_data, int tpm_size) +static int st33zp24_spi_read8_reg(void *phy_id, u8 tpm_register, u8 *tpm_data, + int tpm_size) { - u8 data = 0; - int total_length = 0, nbr_dummy_bytes, ret; + int total_length = 0, ret; struct st33zp24_spi_phy *phy = phy_id; struct spi_device *dev = phy->spi_device; - u8 *tx_buf = (u8 *)phy->spi_xfer.tx_buf; - u8 *rx_buf = phy->spi_xfer.rx_buf; + struct spi_transfer spi_xfer = { + .tx_buf = phy->tx_buf, + .rx_buf = phy->rx_buf, + }; /* Pre-Header */ - data = LOCALITY0; - memcpy(tx_buf + total_length, &data, sizeof(data)); - total_length++; - data = tpm_register; - memcpy(tx_buf + total_length, &data, sizeof(data)); - total_length++; + phy->tx_buf[total_length++] = LOCALITY0; + phy->tx_buf[total_length++] = tpm_register; - nbr_dummy_bytes = phy->latency; - memset(&tx_buf[total_length], TPM_DUMMY_BYTE, - nbr_dummy_bytes + tpm_size); + memset(&phy->tx_buf[total_length], TPM_DUMMY_BYTE, + phy->latency + tpm_size); - phy->spi_xfer.len = total_length + nbr_dummy_bytes + tpm_size; + spi_xfer.len = total_length + phy->latency + tpm_size; /* header + status byte + size of the data + status byte */ - ret = spi_sync_transfer(dev, &phy->spi_xfer, 1); + ret = spi_sync_transfer(dev, &spi_xfer, 1); if (tpm_size > 0 && ret == 0) { - ret = rx_buf[total_length + nbr_dummy_bytes - 1]; + ret = phy->rx_buf[total_length + phy->latency - 1]; - memcpy(tpm_data, rx_buf + total_length + nbr_dummy_bytes, + memcpy(tpm_data, phy->rx_buf + total_length + phy->latency, tpm_size); } return ret; -} /* read8_reg() */ +} /* st33zp24_spi_read8_reg() */ /* * st33zp24_spi_recv @@ -203,13 +199,13 @@ static int st33zp24_spi_recv(void *phy_id, u8 tpm_register, u8 *tpm_data, { int ret; - ret = read8_reg(phy_id, tpm_register, tpm_data, tpm_size); + ret = st33zp24_spi_read8_reg(phy_id, tpm_register, tpm_data, tpm_size); if (!st33zp24_status_to_errno(ret)) return tpm_size; return ret; } /* st33zp24_spi_recv() */ -static int evaluate_latency(void *phy_id) +static int st33zp24_spi_evaluate_latency(void *phy_id) { struct st33zp24_spi_phy *phy = phy_id; int latency = 1, status = 0; @@ -217,9 +213,15 @@ static int evaluate_latency(void *phy_id) while (!status && latency < MAX_SPI_LATENCY) { phy->latency = latency; - status = read8_reg(phy_id, TPM_INTF_CAPABILITY, &data, 1); + status = st33zp24_spi_read8_reg(phy_id, TPM_INTF_CAPABILITY, + &data, 1); latency++; } + if (status < 0) + return status; + if (latency == MAX_SPI_LATENCY) + return -ENODEV; + return latency - 1; } /* evaluate_latency() */ @@ -228,24 +230,52 @@ static const struct st33zp24_phy_ops spi_phy_ops = { .recv = st33zp24_spi_recv, }; -#ifdef CONFIG_OF -static int tpm_stm_spi_of_request_resources(struct st33zp24_spi_phy *phy) +static int st33zp24_spi_acpi_request_resources(struct spi_device *spi_dev) { + struct tpm_chip *chip = spi_get_drvdata(spi_dev); + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); + struct st33zp24_spi_phy *phy = tpm_dev->phy_id; + struct gpio_desc *gpiod_lpcpd; + struct device *dev = &spi_dev->dev; + + /* Get LPCPD GPIO from ACPI */ + gpiod_lpcpd = devm_gpiod_get_index(dev, "TPM IO LPCPD", 1, + GPIOD_OUT_HIGH); + if (IS_ERR(gpiod_lpcpd)) { + dev_err(dev, "Failed to retrieve lpcpd-gpios from acpi.\n"); + phy->io_lpcpd = -1; + /* + * lpcpd pin is not specified. This is not an issue as + * power management can be also managed by TPM specific + * commands. So leave with a success status code. + */ + return 0; + } + + phy->io_lpcpd = desc_to_gpio(gpiod_lpcpd); + + return 0; +} + +static int st33zp24_spi_of_request_resources(struct spi_device *spi_dev) +{ + struct tpm_chip *chip = spi_get_drvdata(spi_dev); + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); + struct st33zp24_spi_phy *phy = tpm_dev->phy_id; struct device_node *pp; - struct spi_device *dev = phy->spi_device; int gpio; int ret; - pp = dev->dev.of_node; + pp = spi_dev->dev.of_node; if (!pp) { - dev_err(&dev->dev, "No platform data\n"); + dev_err(&spi_dev->dev, "No platform data\n"); return -ENODEV; } /* Get GPIO from device tree */ gpio = of_get_named_gpio(pp, "lpcpd-gpios", 0); if (gpio < 0) { - dev_err(&dev->dev, + dev_err(&spi_dev->dev, "Failed to retrieve lpcpd-gpios from dts.\n"); phy->io_lpcpd = -1; /* @@ -256,26 +286,22 @@ static int tpm_stm_spi_of_request_resources(struct st33zp24_spi_phy *phy) return 0; } /* GPIO request and configuration */ - ret = devm_gpio_request_one(&dev->dev, gpio, + ret = devm_gpio_request_one(&spi_dev->dev, gpio, GPIOF_OUT_INIT_HIGH, "TPM IO LPCPD"); if (ret) { - dev_err(&dev->dev, "Failed to request lpcpd pin\n"); + dev_err(&spi_dev->dev, "Failed to request lpcpd pin\n"); return -ENODEV; } phy->io_lpcpd = gpio; return 0; } -#else -static int tpm_stm_spi_of_request_resources(struct st33zp24_spi_phy *phy) -{ - return -ENODEV; -} -#endif -static int tpm_stm_spi_request_resources(struct spi_device *dev, - struct st33zp24_spi_phy *phy) +static int st33zp24_spi_request_resources(struct spi_device *dev) { + struct tpm_chip *chip = spi_get_drvdata(dev); + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); + struct st33zp24_spi_phy *phy = tpm_dev->phy_id; struct st33zp24_platform_data *pdata; int ret; @@ -303,13 +329,12 @@ static int tpm_stm_spi_request_resources(struct spi_device *dev, } /* - * tpm_st33_spi_probe initialize the TPM device + * st33zp24_spi_probe initialize the TPM device * @param: dev, the spi_device drescription (TPM SPI description). * @return: 0 in case of success. * or a negative value describing the error. */ -static int -tpm_st33_spi_probe(struct spi_device *dev) +static int st33zp24_spi_probe(struct spi_device *dev) { int ret; struct st33zp24_platform_data *pdata; @@ -328,21 +353,23 @@ tpm_st33_spi_probe(struct spi_device *dev) return -ENOMEM; phy->spi_device = dev; + pdata = dev->dev.platform_data; if (!pdata && dev->dev.of_node) { - ret = tpm_stm_spi_of_request_resources(phy); + ret = st33zp24_spi_of_request_resources(dev); if (ret) return ret; } else if (pdata) { - ret = tpm_stm_spi_request_resources(dev, phy); + ret = st33zp24_spi_request_resources(dev); + if (ret) + return ret; + } else if (ACPI_HANDLE(&dev->dev)) { + ret = st33zp24_spi_acpi_request_resources(dev); if (ret) return ret; } - phy->spi_xfer.tx_buf = phy->tx_buf; - phy->spi_xfer.rx_buf = phy->rx_buf; - - phy->latency = evaluate_latency(phy); + phy->latency = st33zp24_spi_evaluate_latency(phy); if (phy->latency <= 0) return -ENODEV; @@ -351,11 +378,11 @@ tpm_st33_spi_probe(struct spi_device *dev) } /* - * tpm_st33_spi_remove remove the TPM device + * st33zp24_spi_remove remove the TPM device * @param: client, the spi_device drescription (TPM SPI description). * @return: 0 in case of success. */ -static int tpm_st33_spi_remove(struct spi_device *dev) +static int st33zp24_spi_remove(struct spi_device *dev) { struct tpm_chip *chip = spi_get_drvdata(dev); @@ -368,29 +395,34 @@ static const struct spi_device_id st33zp24_spi_id[] = { }; MODULE_DEVICE_TABLE(spi, st33zp24_spi_id); -#ifdef CONFIG_OF static const struct of_device_id of_st33zp24_spi_match[] = { { .compatible = "st,st33zp24-spi", }, {} }; MODULE_DEVICE_TABLE(of, of_st33zp24_spi_match); -#endif + +static const struct acpi_device_id st33zp24_spi_acpi_match[] = { + {"SMO3324"}, + {} +}; +MODULE_DEVICE_TABLE(acpi, st33zp24_spi_acpi_match); static SIMPLE_DEV_PM_OPS(st33zp24_spi_ops, st33zp24_pm_suspend, st33zp24_pm_resume); -static struct spi_driver tpm_st33_spi_driver = { +static struct spi_driver st33zp24_spi_driver = { .driver = { .name = TPM_ST33_SPI, .pm = &st33zp24_spi_ops, .of_match_table = of_match_ptr(of_st33zp24_spi_match), + .acpi_match_table = ACPI_PTR(st33zp24_spi_acpi_match), }, - .probe = tpm_st33_spi_probe, - .remove = tpm_st33_spi_remove, + .probe = st33zp24_spi_probe, + .remove = st33zp24_spi_remove, .id_table = st33zp24_spi_id, }; -module_spi_driver(tpm_st33_spi_driver); +module_spi_driver(st33zp24_spi_driver); MODULE_AUTHOR("TPM support (TPMsupport@list.st.com)"); MODULE_DESCRIPTION("STM TPM 1.2 SPI ST33 Driver"); diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c index 8d626784c..c2ee30451 100644 --- a/drivers/char/tpm/st33zp24/st33zp24.c +++ b/drivers/char/tpm/st33zp24/st33zp24.c @@ -1,6 +1,6 @@ /* * STMicroelectronics TPM Linux driver for TPM ST33ZP24 - * Copyright (C) 2009 - 2015 STMicroelectronics + * Copyright (C) 2009 - 2016 STMicroelectronics * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -73,14 +73,6 @@ enum tis_defaults { TIS_LONG_TIMEOUT = 2000, }; -struct st33zp24_dev { - struct tpm_chip *chip; - void *phy_id; - const struct st33zp24_phy_ops *ops; - u32 intrs; - int io_lpcpd; -}; - /* * clear_interruption clear the pending interrupt. * @param: tpm_dev, the tpm device device. @@ -102,11 +94,9 @@ static u8 clear_interruption(struct st33zp24_dev *tpm_dev) */ static void st33zp24_cancel(struct tpm_chip *chip) { - struct st33zp24_dev *tpm_dev; + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u8 data; - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); - data = TPM_STS_COMMAND_READY; tpm_dev->ops->send(tpm_dev->phy_id, TPM_STS, &data, 1); } /* st33zp24_cancel() */ @@ -118,11 +108,9 @@ static void st33zp24_cancel(struct tpm_chip *chip) */ static u8 st33zp24_status(struct tpm_chip *chip) { - struct st33zp24_dev *tpm_dev; + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u8 data; - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); - tpm_dev->ops->recv(tpm_dev->phy_id, TPM_STS, &data, 1); return data; } /* st33zp24_status() */ @@ -134,17 +122,15 @@ static u8 st33zp24_status(struct tpm_chip *chip) */ static int check_locality(struct tpm_chip *chip) { - struct st33zp24_dev *tpm_dev; + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u8 data; u8 status; - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); - status = tpm_dev->ops->recv(tpm_dev->phy_id, TPM_ACCESS, &data, 1); if (status && (data & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) == (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) - return chip->vendor.locality; + return tpm_dev->locality; return -EACCES; } /* check_locality() */ @@ -156,27 +142,25 @@ static int check_locality(struct tpm_chip *chip) */ static int request_locality(struct tpm_chip *chip) { + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); unsigned long stop; long ret; - struct st33zp24_dev *tpm_dev; u8 data; - if (check_locality(chip) == chip->vendor.locality) - return chip->vendor.locality; - - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); + if (check_locality(chip) == tpm_dev->locality) + return tpm_dev->locality; data = TPM_ACCESS_REQUEST_USE; ret = tpm_dev->ops->send(tpm_dev->phy_id, TPM_ACCESS, &data, 1); if (ret < 0) return ret; - stop = jiffies + chip->vendor.timeout_a; + stop = jiffies + chip->timeout_a; /* Request locality is usually effective after the request */ do { if (check_locality(chip) >= 0) - return chip->vendor.locality; + return tpm_dev->locality; msleep(TPM_TIMEOUT); } while (time_before(jiffies, stop)); @@ -190,10 +174,9 @@ static int request_locality(struct tpm_chip *chip) */ static void release_locality(struct tpm_chip *chip) { - struct st33zp24_dev *tpm_dev; + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u8 data; - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); data = TPM_ACCESS_ACTIVE_LOCALITY; tpm_dev->ops->send(tpm_dev->phy_id, TPM_ACCESS, &data, 1); @@ -206,23 +189,21 @@ static void release_locality(struct tpm_chip *chip) */ static int get_burstcount(struct tpm_chip *chip) { + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); unsigned long stop; int burstcnt, status; - u8 tpm_reg, temp; - struct st33zp24_dev *tpm_dev; - - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); + u8 temp; - stop = jiffies + chip->vendor.timeout_d; + stop = jiffies + chip->timeout_d; do { - tpm_reg = TPM_STS + 1; - status = tpm_dev->ops->recv(tpm_dev->phy_id, tpm_reg, &temp, 1); + status = tpm_dev->ops->recv(tpm_dev->phy_id, TPM_STS + 1, + &temp, 1); if (status < 0) return -EBUSY; - tpm_reg = TPM_STS + 2; burstcnt = temp; - status = tpm_dev->ops->recv(tpm_dev->phy_id, tpm_reg, &temp, 1); + status = tpm_dev->ops->recv(tpm_dev->phy_id, TPM_STS + 2, + &temp, 1); if (status < 0) return -EBUSY; @@ -271,15 +252,13 @@ static bool wait_for_tpm_stat_cond(struct tpm_chip *chip, u8 mask, static int wait_for_stat(struct tpm_chip *chip, u8 mask, unsigned long timeout, wait_queue_head_t *queue, bool check_cancel) { + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); unsigned long stop; int ret = 0; bool canceled = false; bool condition; u32 cur_intrs; u8 status; - struct st33zp24_dev *tpm_dev; - - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); /* check current status */ status = st33zp24_status(chip); @@ -288,10 +267,10 @@ static int wait_for_stat(struct tpm_chip *chip, u8 mask, unsigned long timeout, stop = jiffies + timeout; - if (chip->vendor.irq) { + if (chip->flags & TPM_CHIP_FLAG_IRQ) { cur_intrs = tpm_dev->intrs; clear_interruption(tpm_dev); - enable_irq(chip->vendor.irq); + enable_irq(tpm_dev->irq); do { if (ret == -ERESTARTSYS && freezing(current)) @@ -314,7 +293,7 @@ static int wait_for_stat(struct tpm_chip *chip, u8 mask, unsigned long timeout, } } while (ret == -ERESTARTSYS && freezing(current)); - disable_irq_nosync(chip->vendor.irq); + disable_irq_nosync(tpm_dev->irq); } else { do { @@ -337,16 +316,14 @@ static int wait_for_stat(struct tpm_chip *chip, u8 mask, unsigned long timeout, */ static int recv_data(struct tpm_chip *chip, u8 *buf, size_t count) { + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); int size = 0, burstcnt, len, ret; - struct st33zp24_dev *tpm_dev; - - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); while (size < count && wait_for_stat(chip, TPM_STS_DATA_AVAIL | TPM_STS_VALID, - chip->vendor.timeout_c, - &chip->vendor.read_queue, true) == 0) { + chip->timeout_c, + &tpm_dev->read_queue, true) == 0) { burstcnt = get_burstcount(chip); if (burstcnt < 0) return burstcnt; @@ -370,13 +347,11 @@ static int recv_data(struct tpm_chip *chip, u8 *buf, size_t count) static irqreturn_t tpm_ioserirq_handler(int irq, void *dev_id) { struct tpm_chip *chip = dev_id; - struct st33zp24_dev *tpm_dev; - - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); tpm_dev->intrs++; - wake_up_interruptible(&chip->vendor.read_queue); - disable_irq_nosync(chip->vendor.irq); + wake_up_interruptible(&tpm_dev->read_queue); + disable_irq_nosync(tpm_dev->irq); return IRQ_HANDLED; } /* tpm_ioserirq_handler() */ @@ -393,19 +368,17 @@ static irqreturn_t tpm_ioserirq_handler(int irq, void *dev_id) static int st33zp24_send(struct tpm_chip *chip, unsigned char *buf, size_t len) { + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u32 status, i, size, ordinal; int burstcnt = 0; int ret; u8 data; - struct st33zp24_dev *tpm_dev; if (!chip) return -EBUSY; if (len < TPM_HEADER_SIZE) return -EBUSY; - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); - ret = request_locality(chip); if (ret < 0) return ret; @@ -414,8 +387,8 @@ static int st33zp24_send(struct tpm_chip *chip, unsigned char *buf, if ((status & TPM_STS_COMMAND_READY) == 0) { st33zp24_cancel(chip); if (wait_for_stat - (chip, TPM_STS_COMMAND_READY, chip->vendor.timeout_b, - &chip->vendor.read_queue, false) < 0) { + (chip, TPM_STS_COMMAND_READY, chip->timeout_b, + &tpm_dev->read_queue, false) < 0) { ret = -ETIME; goto out_err; } @@ -456,12 +429,12 @@ static int st33zp24_send(struct tpm_chip *chip, unsigned char *buf, if (ret < 0) goto out_err; - if (chip->vendor.irq) { + if (chip->flags & TPM_CHIP_FLAG_IRQ) { ordinal = be32_to_cpu(*((__be32 *) (buf + 6))); ret = wait_for_stat(chip, TPM_STS_DATA_AVAIL | TPM_STS_VALID, tpm_calc_ordinal_duration(chip, ordinal), - &chip->vendor.read_queue, false); + &tpm_dev->read_queue, false); if (ret < 0) goto out_err; } @@ -532,6 +505,7 @@ static bool st33zp24_req_canceled(struct tpm_chip *chip, u8 status) } static const struct tpm_class_ops st33zp24_tpm = { + .flags = TPM_OPS_AUTO_STARTUP, .send = st33zp24_send, .recv = st33zp24_recv, .cancel = st33zp24_cancel, @@ -565,20 +539,20 @@ int st33zp24_probe(void *phy_id, const struct st33zp24_phy_ops *ops, if (!tpm_dev) return -ENOMEM; - TPM_VPRIV(chip) = tpm_dev; tpm_dev->phy_id = phy_id; tpm_dev->ops = ops; + dev_set_drvdata(&chip->dev, tpm_dev); - chip->vendor.timeout_a = msecs_to_jiffies(TIS_SHORT_TIMEOUT); - chip->vendor.timeout_b = msecs_to_jiffies(TIS_LONG_TIMEOUT); - chip->vendor.timeout_c = msecs_to_jiffies(TIS_SHORT_TIMEOUT); - chip->vendor.timeout_d = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->timeout_a = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->timeout_b = msecs_to_jiffies(TIS_LONG_TIMEOUT); + chip->timeout_c = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->timeout_d = msecs_to_jiffies(TIS_SHORT_TIMEOUT); - chip->vendor.locality = LOCALITY0; + tpm_dev->locality = LOCALITY0; if (irq) { /* INTERRUPT Setup */ - init_waitqueue_head(&chip->vendor.read_queue); + init_waitqueue_head(&tpm_dev->read_queue); tpm_dev->intrs = 0; if (request_locality(chip) != LOCALITY0) { @@ -611,16 +585,14 @@ int st33zp24_probe(void *phy_id, const struct st33zp24_phy_ops *ops, if (ret < 0) goto _tpm_clean_answer; - chip->vendor.irq = irq; + tpm_dev->irq = irq; + chip->flags |= TPM_CHIP_FLAG_IRQ; - disable_irq_nosync(chip->vendor.irq); + disable_irq_nosync(tpm_dev->irq); tpm_gen_interrupt(chip); } - tpm_get_timeouts(chip); - tpm_do_selftest(chip); - return tpm_chip_register(chip); _tpm_clean_answer: dev_info(&chip->dev, "TPM initialization fail\n"); @@ -650,10 +622,9 @@ EXPORT_SYMBOL(st33zp24_remove); int st33zp24_pm_suspend(struct device *dev) { struct tpm_chip *chip = dev_get_drvdata(dev); - struct st33zp24_dev *tpm_dev; - int ret = 0; + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); + int ret = 0; if (gpio_is_valid(tpm_dev->io_lpcpd)) gpio_set_value(tpm_dev->io_lpcpd, 0); @@ -672,16 +643,14 @@ EXPORT_SYMBOL(st33zp24_pm_suspend); int st33zp24_pm_resume(struct device *dev) { struct tpm_chip *chip = dev_get_drvdata(dev); - struct st33zp24_dev *tpm_dev; + struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); int ret = 0; - tpm_dev = (struct st33zp24_dev *)TPM_VPRIV(chip); - if (gpio_is_valid(tpm_dev->io_lpcpd)) { gpio_set_value(tpm_dev->io_lpcpd, 1); ret = wait_for_stat(chip, - TPM_STS_VALID, chip->vendor.timeout_b, - &chip->vendor.read_queue, false); + TPM_STS_VALID, chip->timeout_b, + &tpm_dev->read_queue, false); } else { ret = tpm_pm_resume(dev); if (!ret) diff --git a/drivers/char/tpm/st33zp24/st33zp24.h b/drivers/char/tpm/st33zp24/st33zp24.h index c207cebf6..6f4a4198a 100644 --- a/drivers/char/tpm/st33zp24/st33zp24.h +++ b/drivers/char/tpm/st33zp24/st33zp24.h @@ -1,6 +1,6 @@ /* * STMicroelectronics TPM Linux driver for TPM ST33ZP24 - * Copyright (C) 2009 - 2015 STMicroelectronics + * Copyright (C) 2009 - 2016 STMicroelectronics * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -21,6 +21,18 @@ #define TPM_WRITE_DIRECTION 0x80 #define TPM_BUFSIZE 2048 +struct st33zp24_dev { + struct tpm_chip *chip; + void *phy_id; + const struct st33zp24_phy_ops *ops; + int locality; + int irq; + u32 intrs; + int io_lpcpd; + wait_queue_head_t read_queue; +}; + + struct st33zp24_phy_ops { int (*send)(void *phy_id, u8 tpm_register, u8 *tpm_data, int tpm_size); int (*recv)(void *phy_id, u8 tpm_register, u8 *tpm_data, int tpm_size); diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index 274dd0123..e5950131b 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -29,33 +29,88 @@ #include "tpm.h" #include "tpm_eventlog.h" -static DECLARE_BITMAP(dev_mask, TPM_NUM_DEVICES); -static LIST_HEAD(tpm_chip_list); -static DEFINE_SPINLOCK(driver_lock); +DEFINE_IDR(dev_nums_idr); +static DEFINE_MUTEX(idr_lock); struct class *tpm_class; dev_t tpm_devt; -/* - * tpm_chip_find_get - return tpm_chip for a given chip number - * @chip_num the device number for the chip +/** + * tpm_try_get_ops() - Get a ref to the tpm_chip + * @chip: Chip to ref + * + * The caller must already have some kind of locking to ensure that chip is + * valid. This function will lock the chip so that the ops member can be + * accessed safely. The locking prevents tpm_chip_unregister from + * completing, so it should not be held for long periods. + * + * Returns -ERRNO if the chip could not be got. */ -struct tpm_chip *tpm_chip_find_get(int chip_num) +int tpm_try_get_ops(struct tpm_chip *chip) { - struct tpm_chip *pos, *chip = NULL; + int rc = -EIO; - rcu_read_lock(); - list_for_each_entry_rcu(pos, &tpm_chip_list, list) { - if (chip_num != TPM_ANY_NUM && chip_num != pos->dev_num) - continue; + get_device(&chip->dev); - if (try_module_get(pos->pdev->driver->owner)) { - chip = pos; - break; - } + down_read(&chip->ops_sem); + if (!chip->ops) + goto out_lock; + + return 0; +out_lock: + up_read(&chip->ops_sem); + put_device(&chip->dev); + return rc; +} +EXPORT_SYMBOL_GPL(tpm_try_get_ops); + +/** + * tpm_put_ops() - Release a ref to the tpm_chip + * @chip: Chip to put + * + * This is the opposite pair to tpm_try_get_ops(). After this returns chip may + * be kfree'd. + */ +void tpm_put_ops(struct tpm_chip *chip) +{ + up_read(&chip->ops_sem); + put_device(&chip->dev); +} +EXPORT_SYMBOL_GPL(tpm_put_ops); + +/** + * tpm_chip_find_get() - return tpm_chip for a given chip number + * @chip_num: id to find + * + * The return'd chip has been tpm_try_get_ops'd and must be released via + * tpm_put_ops + */ +struct tpm_chip *tpm_chip_find_get(int chip_num) +{ + struct tpm_chip *chip, *res = NULL; + int chip_prev; + + mutex_lock(&idr_lock); + + if (chip_num == TPM_ANY_NUM) { + chip_num = 0; + do { + chip_prev = chip_num; + chip = idr_get_next(&dev_nums_idr, &chip_num); + if (chip && !tpm_try_get_ops(chip)) { + res = chip; + break; + } + } while (chip_prev != chip_num); + } else { + chip = idr_find_slowpath(&dev_nums_idr, chip_num); + if (chip && !tpm_try_get_ops(chip)) + res = chip; } - rcu_read_unlock(); - return chip; + + mutex_unlock(&idr_lock); + + return res; } /** @@ -68,24 +123,25 @@ static void tpm_dev_release(struct device *dev) { struct tpm_chip *chip = container_of(dev, struct tpm_chip, dev); - spin_lock(&driver_lock); - clear_bit(chip->dev_num, dev_mask); - spin_unlock(&driver_lock); + mutex_lock(&idr_lock); + idr_remove(&dev_nums_idr, chip->dev_num); + mutex_unlock(&idr_lock); + kfree(chip); } /** - * tpmm_chip_alloc() - allocate a new struct tpm_chip instance - * @dev: device to which the chip is associated + * tpm_chip_alloc() - allocate a new struct tpm_chip instance + * @pdev: device to which the chip is associated + * At this point pdev mst be initialized, but does not have to + * be registered * @ops: struct tpm_class_ops instance * * Allocates a new struct tpm_chip instance and assigns a free - * device number for it. Caller does not have to worry about - * freeing the allocated resources. When the devices is removed - * devres calls tpmm_chip_remove() to do the job. + * device number for it. Must be paired with put_device(&chip->dev). */ -struct tpm_chip *tpmm_chip_alloc(struct device *dev, - const struct tpm_class_ops *ops) +struct tpm_chip *tpm_chip_alloc(struct device *dev, + const struct tpm_class_ops *ops) { struct tpm_chip *chip; int rc; @@ -95,53 +151,75 @@ struct tpm_chip *tpmm_chip_alloc(struct device *dev, return ERR_PTR(-ENOMEM); mutex_init(&chip->tpm_mutex); - INIT_LIST_HEAD(&chip->list); + init_rwsem(&chip->ops_sem); chip->ops = ops; - spin_lock(&driver_lock); - chip->dev_num = find_first_zero_bit(dev_mask, TPM_NUM_DEVICES); - spin_unlock(&driver_lock); - - if (chip->dev_num >= TPM_NUM_DEVICES) { + mutex_lock(&idr_lock); + rc = idr_alloc(&dev_nums_idr, NULL, 0, TPM_NUM_DEVICES, GFP_KERNEL); + mutex_unlock(&idr_lock); + if (rc < 0) { dev_err(dev, "No available tpm device numbers\n"); kfree(chip); - return ERR_PTR(-ENOMEM); + return ERR_PTR(rc); } + chip->dev_num = rc; - set_bit(chip->dev_num, dev_mask); - - scnprintf(chip->devname, sizeof(chip->devname), "tpm%d", chip->dev_num); - - chip->pdev = dev; - - dev_set_drvdata(dev, chip); + device_initialize(&chip->dev); chip->dev.class = tpm_class; chip->dev.release = tpm_dev_release; - chip->dev.parent = chip->pdev; -#ifdef CONFIG_ACPI + chip->dev.parent = dev; chip->dev.groups = chip->groups; -#endif if (chip->dev_num == 0) chip->dev.devt = MKDEV(MISC_MAJOR, TPM_MINOR); else chip->dev.devt = MKDEV(MAJOR(tpm_devt), chip->dev_num); - dev_set_name(&chip->dev, "%s", chip->devname); + rc = dev_set_name(&chip->dev, "tpm%d", chip->dev_num); + if (rc) + goto out; - device_initialize(&chip->dev); + if (!dev) + chip->flags |= TPM_CHIP_FLAG_VIRTUAL; cdev_init(&chip->cdev, &tpm_fops); - chip->cdev.owner = chip->pdev->driver->owner; + chip->cdev.owner = THIS_MODULE; chip->cdev.kobj.parent = &chip->dev.kobj; - rc = devm_add_action(dev, (void (*)(void *)) put_device, &chip->dev); - if (rc) { - put_device(&chip->dev); + return chip; + +out: + put_device(&chip->dev); + return ERR_PTR(rc); +} +EXPORT_SYMBOL_GPL(tpm_chip_alloc); + +/** + * tpmm_chip_alloc() - allocate a new struct tpm_chip instance + * @pdev: parent device to which the chip is associated + * @ops: struct tpm_class_ops instance + * + * Same as tpm_chip_alloc except devm is used to do the put_device + */ +struct tpm_chip *tpmm_chip_alloc(struct device *pdev, + const struct tpm_class_ops *ops) +{ + struct tpm_chip *chip; + int rc; + + chip = tpm_chip_alloc(pdev, ops); + if (IS_ERR(chip)) + return chip; + + rc = devm_add_action_or_reset(pdev, + (void (*)(void *)) put_device, + &chip->dev); + if (rc) return ERR_PTR(rc); - } + + dev_set_drvdata(pdev, chip); return chip; } @@ -155,7 +233,7 @@ static int tpm_add_char_device(struct tpm_chip *chip) if (rc) { dev_err(&chip->dev, "unable to cdev_add() %s, major %d, minor %d, err=%d\n", - chip->devname, MAJOR(chip->dev.devt), + dev_name(&chip->dev), MAJOR(chip->dev.devt), MINOR(chip->dev.devt), rc); return rc; @@ -165,13 +243,18 @@ static int tpm_add_char_device(struct tpm_chip *chip) if (rc) { dev_err(&chip->dev, "unable to device_register() %s, major %d, minor %d, err=%d\n", - chip->devname, MAJOR(chip->dev.devt), + dev_name(&chip->dev), MAJOR(chip->dev.devt), MINOR(chip->dev.devt), rc); cdev_del(&chip->cdev); return rc; } + /* Make the chip available. */ + mutex_lock(&idr_lock); + idr_replace(&dev_nums_idr, chip, chip->dev_num); + mutex_unlock(&idr_lock); + return rc; } @@ -179,20 +262,28 @@ static void tpm_del_char_device(struct tpm_chip *chip) { cdev_del(&chip->cdev); device_del(&chip->dev); + + /* Make the chip unavailable. */ + mutex_lock(&idr_lock); + idr_replace(&dev_nums_idr, NULL, chip->dev_num); + mutex_unlock(&idr_lock); + + /* Make the driver uncallable. */ + down_write(&chip->ops_sem); + if (chip->flags & TPM_CHIP_FLAG_TPM2) + tpm2_shutdown(chip, TPM2_SU_CLEAR); + chip->ops = NULL; + up_write(&chip->ops_sem); } static int tpm1_chip_register(struct tpm_chip *chip) { - int rc; - if (chip->flags & TPM_CHIP_FLAG_TPM2) return 0; - rc = tpm_sysfs_add_device(chip); - if (rc) - return rc; + tpm_sysfs_add_device(chip); - chip->bios_dir = tpm_bios_log_setup(chip->devname); + chip->bios_dir = tpm_bios_log_setup(dev_name(&chip->dev)); return 0; } @@ -204,10 +295,50 @@ static void tpm1_chip_unregister(struct tpm_chip *chip) if (chip->bios_dir) tpm_bios_log_teardown(chip->bios_dir); +} + +static void tpm_del_legacy_sysfs(struct tpm_chip *chip) +{ + struct attribute **i; + + if (chip->flags & (TPM_CHIP_FLAG_TPM2 | TPM_CHIP_FLAG_VIRTUAL)) + return; + + sysfs_remove_link(&chip->dev.parent->kobj, "ppi"); - tpm_sysfs_del_device(chip); + for (i = chip->groups[0]->attrs; *i != NULL; ++i) + sysfs_remove_link(&chip->dev.parent->kobj, (*i)->name); } +/* For compatibility with legacy sysfs paths we provide symlinks from the + * parent dev directory to selected names within the tpm chip directory. Old + * kernel versions created these files directly under the parent. + */ +static int tpm_add_legacy_sysfs(struct tpm_chip *chip) +{ + struct attribute **i; + int rc; + + if (chip->flags & (TPM_CHIP_FLAG_TPM2 | TPM_CHIP_FLAG_VIRTUAL)) + return 0; + + rc = __compat_only_sysfs_link_entry_to_kobj( + &chip->dev.parent->kobj, &chip->dev.kobj, "ppi"); + if (rc && rc != -ENOENT) + return rc; + + /* All the names from tpm-sysfs */ + for (i = chip->groups[0]->attrs; *i != NULL; ++i) { + rc = __compat_only_sysfs_link_entry_to_kobj( + &chip->dev.parent->kobj, &chip->dev.kobj, (*i)->name); + if (rc) { + tpm_del_legacy_sysfs(chip); + return rc; + } + } + + return 0; +} /* * tpm_chip_register() - create a character device for the TPM chip * @chip: TPM chip to use. @@ -223,6 +354,15 @@ int tpm_chip_register(struct tpm_chip *chip) { int rc; + if (chip->ops->flags & TPM_OPS_AUTO_STARTUP) { + if (chip->flags & TPM_CHIP_FLAG_TPM2) + rc = tpm2_auto_startup(chip); + else + rc = tpm1_auto_startup(chip); + if (rc) + return rc; + } + rc = tpm1_chip_register(chip); if (rc) return rc; @@ -230,30 +370,20 @@ int tpm_chip_register(struct tpm_chip *chip) tpm_add_ppi(chip); rc = tpm_add_char_device(chip); - if (rc) - goto out_err; - - /* Make the chip available. */ - spin_lock(&driver_lock); - list_add_tail_rcu(&chip->list, &tpm_chip_list); - spin_unlock(&driver_lock); + if (rc) { + tpm1_chip_unregister(chip); + return rc; + } chip->flags |= TPM_CHIP_FLAG_REGISTERED; - if (!(chip->flags & TPM_CHIP_FLAG_TPM2)) { - rc = __compat_only_sysfs_link_entry_to_kobj(&chip->pdev->kobj, - &chip->dev.kobj, - "ppi"); - if (rc && rc != -ENOENT) { - tpm_chip_unregister(chip); - return rc; - } + rc = tpm_add_legacy_sysfs(chip); + if (rc) { + tpm_chip_unregister(chip); + return rc; } return 0; -out_err: - tpm1_chip_unregister(chip); - return rc; } EXPORT_SYMBOL_GPL(tpm_chip_register); @@ -264,6 +394,9 @@ EXPORT_SYMBOL_GPL(tpm_chip_register); * Takes the chip first away from the list of available TPM chips and then * cleans up all the resources reserved by tpm_chip_register(). * + * Once this function returns the driver call backs in 'op's will not be + * running and will no longer start. + * * NOTE: This function should be only called before deinitializing chip * resources. */ @@ -272,13 +405,7 @@ void tpm_chip_unregister(struct tpm_chip *chip) if (!(chip->flags & TPM_CHIP_FLAG_REGISTERED)) return; - spin_lock(&driver_lock); - list_del_rcu(&chip->list); - spin_unlock(&driver_lock); - synchronize_rcu(); - - if (!(chip->flags & TPM_CHIP_FLAG_TPM2)) - sysfs_remove_link(&chip->pdev->kobj, "ppi"); + tpm_del_legacy_sysfs(chip); tpm1_chip_unregister(chip); tpm_del_char_device(chip); diff --git a/drivers/char/tpm/tpm-dev.c b/drivers/char/tpm/tpm-dev.c index de0337ebd..912ad30be 100644 --- a/drivers/char/tpm/tpm-dev.c +++ b/drivers/char/tpm/tpm-dev.c @@ -61,7 +61,7 @@ static int tpm_open(struct inode *inode, struct file *file) * by the check of is_open variable, which is protected * by driver_lock. */ if (test_and_set_bit(0, &chip->is_open)) { - dev_dbg(chip->pdev, "Another process owns this TPM\n"); + dev_dbg(&chip->dev, "Another process owns this TPM\n"); return -EBUSY; } @@ -79,7 +79,6 @@ static int tpm_open(struct inode *inode, struct file *file) INIT_WORK(&priv->work, timeout_work); file->private_data = priv; - get_device(chip->pdev); return 0; } @@ -137,9 +136,18 @@ static ssize_t tpm_write(struct file *file, const char __user *buf, return -EFAULT; } - /* atomic tpm command send and result receive */ + /* atomic tpm command send and result receive. We only hold the ops + * lock during this period so that the tpm can be unregistered even if + * the char dev is held open. + */ + if (tpm_try_get_ops(priv->chip)) { + mutex_unlock(&priv->buffer_mutex); + return -EPIPE; + } out_size = tpm_transmit(priv->chip, priv->data_buffer, - sizeof(priv->data_buffer)); + sizeof(priv->data_buffer), 0); + + tpm_put_ops(priv->chip); if (out_size < 0) { mutex_unlock(&priv->buffer_mutex); return out_size; @@ -166,7 +174,6 @@ static int tpm_release(struct inode *inode, struct file *file) file->private_data = NULL; atomic_set(&priv->data_pending, 0); clear_bit(0, &priv->chip->is_open); - put_device(priv->chip->pdev); kfree(priv); return 0; } diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c index e2fa89c88..aef20ee23 100644 --- a/drivers/char/tpm/tpm-interface.c +++ b/drivers/char/tpm/tpm-interface.c @@ -319,7 +319,7 @@ unsigned long tpm_calc_ordinal_duration(struct tpm_chip *chip, duration_idx = tpm_ordinal_duration[ordinal]; if (duration_idx != TPM_UNDEFINED) - duration = chip->vendor.duration[duration_idx]; + duration = chip->duration[duration_idx]; if (duration <= 0) return 2 * 60 * HZ; else @@ -330,8 +330,8 @@ EXPORT_SYMBOL_GPL(tpm_calc_ordinal_duration); /* * Internal kernel interface to transmit TPM commands */ -ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf, - size_t bufsiz) +ssize_t tpm_transmit(struct tpm_chip *chip, const u8 *buf, size_t bufsiz, + unsigned int flags) { ssize_t rc; u32 count, ordinal; @@ -345,21 +345,22 @@ ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf, if (count == 0) return -ENODATA; if (count > bufsiz) { - dev_err(chip->pdev, + dev_err(&chip->dev, "invalid count value %x %zx\n", count, bufsiz); return -E2BIG; } - mutex_lock(&chip->tpm_mutex); + if (!(flags & TPM_TRANSMIT_UNLOCKED)) + mutex_lock(&chip->tpm_mutex); rc = chip->ops->send(chip, (u8 *) buf, count); if (rc < 0) { - dev_err(chip->pdev, + dev_err(&chip->dev, "tpm_transmit: tpm_send: error %zd\n", rc); goto out; } - if (chip->vendor.irq) + if (chip->flags & TPM_CHIP_FLAG_IRQ) goto out_recv; if (chip->flags & TPM_CHIP_FLAG_TPM2) @@ -373,7 +374,7 @@ ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf, goto out_recv; if (chip->ops->req_canceled(chip, status)) { - dev_err(chip->pdev, "Operation Canceled\n"); + dev_err(&chip->dev, "Operation Canceled\n"); rc = -ECANCELED; goto out; } @@ -383,30 +384,31 @@ ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf, } while (time_before(jiffies, stop)); chip->ops->cancel(chip); - dev_err(chip->pdev, "Operation Timed out\n"); + dev_err(&chip->dev, "Operation Timed out\n"); rc = -ETIME; goto out; out_recv: rc = chip->ops->recv(chip, (u8 *) buf, bufsiz); if (rc < 0) - dev_err(chip->pdev, + dev_err(&chip->dev, "tpm_transmit: tpm_recv: error %zd\n", rc); out: - mutex_unlock(&chip->tpm_mutex); + if (!(flags & TPM_TRANSMIT_UNLOCKED)) + mutex_unlock(&chip->tpm_mutex); return rc; } #define TPM_DIGEST_SIZE 20 #define TPM_RET_CODE_IDX 6 -ssize_t tpm_transmit_cmd(struct tpm_chip *chip, void *cmd, - int len, const char *desc) +ssize_t tpm_transmit_cmd(struct tpm_chip *chip, const void *cmd, + int len, unsigned int flags, const char *desc) { - struct tpm_output_header *header; + const struct tpm_output_header *header; int err; - len = tpm_transmit(chip, (u8 *) cmd, len); + len = tpm_transmit(chip, (const u8 *)cmd, len, flags); if (len < 0) return len; else if (len < TPM_HEADER_SIZE) @@ -416,7 +418,7 @@ ssize_t tpm_transmit_cmd(struct tpm_chip *chip, void *cmd, err = be32_to_cpu(header->return_code); if (err != 0 && desc) - dev_err(chip->pdev, "A TPM error (%d) occurred %s\n", err, + dev_err(&chip->dev, "A TPM error (%d) occurred %s\n", err, desc); return err; @@ -432,12 +434,11 @@ static const struct tpm_input_header tpm_getcap_header = { .ordinal = TPM_ORD_GET_CAP }; -ssize_t tpm_getcap(struct device *dev, __be32 subcap_id, cap_t *cap, +ssize_t tpm_getcap(struct tpm_chip *chip, __be32 subcap_id, cap_t *cap, const char *desc) { struct tpm_cmd_t tpm_cmd; int rc; - struct tpm_chip *chip = dev_get_drvdata(dev); tpm_cmd.header.in = tpm_getcap_header; if (subcap_id == CAP_VERSION_1_1 || subcap_id == CAP_VERSION_1_2) { @@ -454,7 +455,8 @@ ssize_t tpm_getcap(struct device *dev, __be32 subcap_id, cap_t *cap, tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4); tpm_cmd.params.getcap_in.subcap = subcap_id; } - rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, desc); + rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0, + desc); if (!rc) *cap = tpm_cmd.params.getcap_out.cap; return rc; @@ -470,7 +472,7 @@ void tpm_gen_interrupt(struct tpm_chip *chip) tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4); tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT; - rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0, "attempting to determine the timeouts"); } EXPORT_SYMBOL_GPL(tpm_gen_interrupt); @@ -491,7 +493,7 @@ static int tpm_startup(struct tpm_chip *chip, __be16 startup_type) start_cmd.header.in = tpm_startup_header; start_cmd.params.startup_in.startup_type = startup_type; - return tpm_transmit_cmd(chip, &start_cmd, TPM_INTERNAL_RESULT_SIZE, + return tpm_transmit_cmd(chip, &start_cmd, TPM_INTERNAL_RESULT_SIZE, 0, "attempting to start the TPM"); } @@ -505,15 +507,15 @@ int tpm_get_timeouts(struct tpm_chip *chip) if (chip->flags & TPM_CHIP_FLAG_TPM2) { /* Fixed timeouts for TPM2 */ - chip->vendor.timeout_a = msecs_to_jiffies(TPM2_TIMEOUT_A); - chip->vendor.timeout_b = msecs_to_jiffies(TPM2_TIMEOUT_B); - chip->vendor.timeout_c = msecs_to_jiffies(TPM2_TIMEOUT_C); - chip->vendor.timeout_d = msecs_to_jiffies(TPM2_TIMEOUT_D); - chip->vendor.duration[TPM_SHORT] = + chip->timeout_a = msecs_to_jiffies(TPM2_TIMEOUT_A); + chip->timeout_b = msecs_to_jiffies(TPM2_TIMEOUT_B); + chip->timeout_c = msecs_to_jiffies(TPM2_TIMEOUT_C); + chip->timeout_d = msecs_to_jiffies(TPM2_TIMEOUT_D); + chip->duration[TPM_SHORT] = msecs_to_jiffies(TPM2_DURATION_SHORT); - chip->vendor.duration[TPM_MEDIUM] = + chip->duration[TPM_MEDIUM] = msecs_to_jiffies(TPM2_DURATION_MEDIUM); - chip->vendor.duration[TPM_LONG] = + chip->duration[TPM_LONG] = msecs_to_jiffies(TPM2_DURATION_LONG); return 0; } @@ -522,12 +524,13 @@ int tpm_get_timeouts(struct tpm_chip *chip) tpm_cmd.params.getcap_in.cap = TPM_CAP_PROP; tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4); tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT; - rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, NULL); + rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0, + NULL); if (rc == TPM_ERR_INVALID_POSTINIT) { /* The TPM is not started, we are the first to talk to it. Execute a startup command. */ - dev_info(chip->pdev, "Issuing TPM_STARTUP"); + dev_info(&chip->dev, "Issuing TPM_STARTUP"); if (tpm_startup(chip, TPM_ST_CLEAR)) return rc; @@ -536,10 +539,10 @@ int tpm_get_timeouts(struct tpm_chip *chip) tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4); tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_TIMEOUT; rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, - NULL); + 0, NULL); } if (rc) { - dev_err(chip->pdev, + dev_err(&chip->dev, "A TPM error (%zd) occurred attempting to determine the timeouts\n", rc); goto duration; @@ -561,10 +564,10 @@ int tpm_get_timeouts(struct tpm_chip *chip) * of misreporting. */ if (chip->ops->update_timeouts != NULL) - chip->vendor.timeout_adjusted = + chip->timeout_adjusted = chip->ops->update_timeouts(chip, new_timeout); - if (!chip->vendor.timeout_adjusted) { + if (!chip->timeout_adjusted) { /* Don't overwrite default if value is 0 */ if (new_timeout[0] != 0 && new_timeout[0] < 1000) { int i; @@ -572,13 +575,13 @@ int tpm_get_timeouts(struct tpm_chip *chip) /* timeouts in msec rather usec */ for (i = 0; i != ARRAY_SIZE(new_timeout); i++) new_timeout[i] *= 1000; - chip->vendor.timeout_adjusted = true; + chip->timeout_adjusted = true; } } /* Report adjusted timeouts */ - if (chip->vendor.timeout_adjusted) { - dev_info(chip->pdev, + if (chip->timeout_adjusted) { + dev_info(&chip->dev, HW_ERR "Adjusting reported timeouts: A %lu->%luus B %lu->%luus C %lu->%luus D %lu->%luus\n", old_timeout[0], new_timeout[0], old_timeout[1], new_timeout[1], @@ -586,10 +589,10 @@ int tpm_get_timeouts(struct tpm_chip *chip) old_timeout[3], new_timeout[3]); } - chip->vendor.timeout_a = usecs_to_jiffies(new_timeout[0]); - chip->vendor.timeout_b = usecs_to_jiffies(new_timeout[1]); - chip->vendor.timeout_c = usecs_to_jiffies(new_timeout[2]); - chip->vendor.timeout_d = usecs_to_jiffies(new_timeout[3]); + chip->timeout_a = usecs_to_jiffies(new_timeout[0]); + chip->timeout_b = usecs_to_jiffies(new_timeout[1]); + chip->timeout_c = usecs_to_jiffies(new_timeout[2]); + chip->timeout_d = usecs_to_jiffies(new_timeout[3]); duration: tpm_cmd.header.in = tpm_getcap_header; @@ -597,7 +600,7 @@ duration: tpm_cmd.params.getcap_in.subcap_size = cpu_to_be32(4); tpm_cmd.params.getcap_in.subcap = TPM_CAP_PROP_TIS_DURATION; - rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &tpm_cmd, TPM_INTERNAL_RESULT_SIZE, 0, "attempting to determine the durations"); if (rc) return rc; @@ -608,11 +611,11 @@ duration: return -EINVAL; duration_cap = &tpm_cmd.params.getcap_out.cap.duration; - chip->vendor.duration[TPM_SHORT] = + chip->duration[TPM_SHORT] = usecs_to_jiffies(be32_to_cpu(duration_cap->tpm_short)); - chip->vendor.duration[TPM_MEDIUM] = + chip->duration[TPM_MEDIUM] = usecs_to_jiffies(be32_to_cpu(duration_cap->tpm_medium)); - chip->vendor.duration[TPM_LONG] = + chip->duration[TPM_LONG] = usecs_to_jiffies(be32_to_cpu(duration_cap->tpm_long)); /* The Broadcom BCM0102 chipset in a Dell Latitude D820 gets the above @@ -620,12 +623,12 @@ duration: * fix up the resulting too-small TPM_SHORT value to make things work. * We also scale the TPM_MEDIUM and -_LONG values by 1000. */ - if (chip->vendor.duration[TPM_SHORT] < (HZ / 100)) { - chip->vendor.duration[TPM_SHORT] = HZ; - chip->vendor.duration[TPM_MEDIUM] *= 1000; - chip->vendor.duration[TPM_LONG] *= 1000; - chip->vendor.duration_adjusted = true; - dev_info(chip->pdev, "Adjusting TPM timeout parameters."); + if (chip->duration[TPM_SHORT] < (HZ / 100)) { + chip->duration[TPM_SHORT] = HZ; + chip->duration[TPM_MEDIUM] *= 1000; + chip->duration[TPM_LONG] *= 1000; + chip->duration_adjusted = true; + dev_info(&chip->dev, "Adjusting TPM timeout parameters."); } return 0; } @@ -653,7 +656,7 @@ static int tpm_continue_selftest(struct tpm_chip *chip) struct tpm_cmd_t cmd; cmd.header.in = continue_selftest_header; - rc = tpm_transmit_cmd(chip, &cmd, CONTINUE_SELFTEST_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &cmd, CONTINUE_SELFTEST_RESULT_SIZE, 0, "continue selftest"); return rc; } @@ -673,7 +676,7 @@ int tpm_pcr_read_dev(struct tpm_chip *chip, int pcr_idx, u8 *res_buf) cmd.header.in = pcrread_header; cmd.params.pcrread_in.pcr_idx = cpu_to_be32(pcr_idx); - rc = tpm_transmit_cmd(chip, &cmd, READ_PCR_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &cmd, READ_PCR_RESULT_SIZE, 0, "attempting to read a pcr value"); if (rc == 0) @@ -700,7 +703,7 @@ int tpm_is_tpm2(u32 chip_num) rc = (chip->flags & TPM_CHIP_FLAG_TPM2) != 0; - tpm_chip_put(chip); + tpm_put_ops(chip); return rc; } @@ -729,7 +732,7 @@ int tpm_pcr_read(u32 chip_num, int pcr_idx, u8 *res_buf) rc = tpm2_pcr_read(chip, pcr_idx, res_buf); else rc = tpm_pcr_read_dev(chip, pcr_idx, res_buf); - tpm_chip_put(chip); + tpm_put_ops(chip); return rc; } EXPORT_SYMBOL_GPL(tpm_pcr_read); @@ -764,17 +767,17 @@ int tpm_pcr_extend(u32 chip_num, int pcr_idx, const u8 *hash) if (chip->flags & TPM_CHIP_FLAG_TPM2) { rc = tpm2_pcr_extend(chip, pcr_idx, hash); - tpm_chip_put(chip); + tpm_put_ops(chip); return rc; } cmd.header.in = pcrextend_header; cmd.params.pcrextend_in.pcr_idx = cpu_to_be32(pcr_idx); memcpy(cmd.params.pcrextend_in.hash, hash, TPM_DIGEST_SIZE); - rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE, 0, "attempting extend a PCR value"); - tpm_chip_put(chip); + tpm_put_ops(chip); return rc; } EXPORT_SYMBOL_GPL(tpm_pcr_extend); @@ -810,12 +813,14 @@ int tpm_do_selftest(struct tpm_chip *chip) /* Attempt to read a PCR value */ cmd.header.in = pcrread_header; cmd.params.pcrread_in.pcr_idx = cpu_to_be32(0); - rc = tpm_transmit(chip, (u8 *) &cmd, READ_PCR_RESULT_SIZE); + rc = tpm_transmit(chip, (u8 *) &cmd, READ_PCR_RESULT_SIZE, 0); /* Some buggy TPMs will not respond to tpm_tis_ready() for * around 300ms while the self test is ongoing, keep trying * until the self test duration expires. */ if (rc == -ETIME) { - dev_info(chip->pdev, HW_ERR "TPM command timed out during continue self test"); + dev_info( + &chip->dev, HW_ERR + "TPM command timed out during continue self test"); msleep(delay_msec); continue; } @@ -825,7 +830,7 @@ int tpm_do_selftest(struct tpm_chip *chip) rc = be32_to_cpu(cmd.header.out.return_code); if (rc == TPM_ERR_DISABLED || rc == TPM_ERR_DEACTIVATED) { - dev_info(chip->pdev, + dev_info(&chip->dev, "TPM is disabled/deactivated (0x%X)\n", rc); /* TPM is disabled and/or deactivated; driver can * proceed and TPM does handle commands for @@ -842,6 +847,33 @@ int tpm_do_selftest(struct tpm_chip *chip) } EXPORT_SYMBOL_GPL(tpm_do_selftest); +/** + * tpm1_auto_startup - Perform the standard automatic TPM initialization + * sequence + * @chip: TPM chip to use + * + * Returns 0 on success, < 0 in case of fatal error. + */ +int tpm1_auto_startup(struct tpm_chip *chip) +{ + int rc; + + rc = tpm_get_timeouts(chip); + if (rc) + goto out; + rc = tpm_do_selftest(chip); + if (rc) { + dev_err(&chip->dev, "TPM self test failed\n"); + goto out; + } + + return rc; +out: + if (rc > 0) + rc = -ENODEV; + return rc; +} + int tpm_send(u32 chip_num, void *cmd, size_t buflen) { struct tpm_chip *chip; @@ -851,9 +883,9 @@ int tpm_send(u32 chip_num, void *cmd, size_t buflen) if (chip == NULL) return -ENODEV; - rc = tpm_transmit_cmd(chip, cmd, buflen, "attempting tpm_cmd"); + rc = tpm_transmit_cmd(chip, cmd, buflen, 0, "attempting tpm_cmd"); - tpm_chip_put(chip); + tpm_put_ops(chip); return rc; } EXPORT_SYMBOL_GPL(tpm_send); @@ -888,7 +920,7 @@ int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask, unsigned long timeout, stop = jiffies + timeout; - if (chip->vendor.irq) { + if (chip->flags & TPM_CHIP_FLAG_IRQ) { again: timeout = stop - jiffies; if ((long)timeout <= 0) @@ -953,14 +985,15 @@ int tpm_pm_suspend(struct device *dev) cmd.params.pcrextend_in.pcr_idx = cpu_to_be32(tpm_suspend_pcr); memcpy(cmd.params.pcrextend_in.hash, dummy_hash, TPM_DIGEST_SIZE); - rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE, + rc = tpm_transmit_cmd(chip, &cmd, EXTEND_PCR_RESULT_SIZE, 0, "extending dummy pcr before suspend"); } /* now do the actual savestate */ for (try = 0; try < TPM_RETRY; try++) { cmd.header.in = savestate_header; - rc = tpm_transmit_cmd(chip, &cmd, SAVESTATE_RESULT_SIZE, NULL); + rc = tpm_transmit_cmd(chip, &cmd, SAVESTATE_RESULT_SIZE, 0, + NULL); /* * If the TPM indicates that it is too busy to respond to @@ -978,10 +1011,10 @@ int tpm_pm_suspend(struct device *dev) } if (rc) - dev_err(chip->pdev, + dev_err(&chip->dev, "Error (%d) sending savestate before suspend\n", rc); else if (try > 0) - dev_warn(chip->pdev, "TPM savestate took %dms\n", + dev_warn(&chip->dev, "TPM savestate took %dms\n", try * TPM_TIMEOUT_RETRY); return rc; @@ -1035,7 +1068,7 @@ int tpm_get_random(u32 chip_num, u8 *out, size_t max) if (chip->flags & TPM_CHIP_FLAG_TPM2) { err = tpm2_get_random(chip, out, max); - tpm_chip_put(chip); + tpm_put_ops(chip); return err; } @@ -1044,8 +1077,8 @@ int tpm_get_random(u32 chip_num, u8 *out, size_t max) tpm_cmd.params.getrandom_in.num_bytes = cpu_to_be32(num_bytes); err = tpm_transmit_cmd(chip, &tpm_cmd, - TPM_GETRANDOM_RESULT_SIZE + num_bytes, - "attempting get random"); + TPM_GETRANDOM_RESULT_SIZE + num_bytes, + 0, "attempting get random"); if (err) break; @@ -1057,7 +1090,7 @@ int tpm_get_random(u32 chip_num, u8 *out, size_t max) num_bytes -= recd; } while (retries-- && total < max); - tpm_chip_put(chip); + tpm_put_ops(chip); return total ? total : -EIO; } EXPORT_SYMBOL_GPL(tpm_get_random); @@ -1083,7 +1116,7 @@ int tpm_seal_trusted(u32 chip_num, struct trusted_key_payload *payload, rc = tpm2_seal_trusted(chip, payload, options); - tpm_chip_put(chip); + tpm_put_ops(chip); return rc; } EXPORT_SYMBOL_GPL(tpm_seal_trusted); @@ -1109,7 +1142,8 @@ int tpm_unseal_trusted(u32 chip_num, struct trusted_key_payload *payload, rc = tpm2_unseal_trusted(chip, payload, options); - tpm_chip_put(chip); + tpm_put_ops(chip); + return rc; } EXPORT_SYMBOL_GPL(tpm_unseal_trusted); @@ -1136,6 +1170,7 @@ static int __init tpm_init(void) static void __exit tpm_exit(void) { + idr_destroy(&dev_nums_idr); class_destroy(tpm_class); unregister_chrdev_region(tpm_devt, TPM_NUM_DEVICES); } diff --git a/drivers/char/tpm/tpm-sysfs.c b/drivers/char/tpm/tpm-sysfs.c index ee66fd467..e1f7236c1 100644 --- a/drivers/char/tpm/tpm-sysfs.c +++ b/drivers/char/tpm/tpm-sysfs.c @@ -36,10 +36,10 @@ static ssize_t pubek_show(struct device *dev, struct device_attribute *attr, int i, rc; char *str = buf; - struct tpm_chip *chip = dev_get_drvdata(dev); + struct tpm_chip *chip = to_tpm_chip(dev); tpm_cmd.header.in = tpm_readpubek_header; - err = tpm_transmit_cmd(chip, &tpm_cmd, READ_PUBEK_RESULT_SIZE, + err = tpm_transmit_cmd(chip, &tpm_cmd, READ_PUBEK_RESULT_SIZE, 0, "attempting to read the PUBEK"); if (err) goto out; @@ -92,9 +92,9 @@ static ssize_t pcrs_show(struct device *dev, struct device_attribute *attr, ssize_t rc; int i, j, num_pcrs; char *str = buf; - struct tpm_chip *chip = dev_get_drvdata(dev); + struct tpm_chip *chip = to_tpm_chip(dev); - rc = tpm_getcap(dev, TPM_CAP_PROP_PCR, &cap, + rc = tpm_getcap(chip, TPM_CAP_PROP_PCR, &cap, "attempting to determine the number of PCRS"); if (rc) return 0; @@ -119,8 +119,8 @@ static ssize_t enabled_show(struct device *dev, struct device_attribute *attr, cap_t cap; ssize_t rc; - rc = tpm_getcap(dev, TPM_CAP_FLAG_PERM, &cap, - "attempting to determine the permanent enabled state"); + rc = tpm_getcap(to_tpm_chip(dev), TPM_CAP_FLAG_PERM, &cap, + "attempting to determine the permanent enabled state"); if (rc) return 0; @@ -135,8 +135,8 @@ static ssize_t active_show(struct device *dev, struct device_attribute *attr, cap_t cap; ssize_t rc; - rc = tpm_getcap(dev, TPM_CAP_FLAG_PERM, &cap, - "attempting to determine the permanent active state"); + rc = tpm_getcap(to_tpm_chip(dev), TPM_CAP_FLAG_PERM, &cap, + "attempting to determine the permanent active state"); if (rc) return 0; @@ -151,8 +151,8 @@ static ssize_t owned_show(struct device *dev, struct device_attribute *attr, cap_t cap; ssize_t rc; - rc = tpm_getcap(dev, TPM_CAP_PROP_OWNER, &cap, - "attempting to determine the owner state"); + rc = tpm_getcap(to_tpm_chip(dev), TPM_CAP_PROP_OWNER, &cap, + "attempting to determine the owner state"); if (rc) return 0; @@ -167,8 +167,8 @@ static ssize_t temp_deactivated_show(struct device *dev, cap_t cap; ssize_t rc; - rc = tpm_getcap(dev, TPM_CAP_FLAG_VOL, &cap, - "attempting to determine the temporary state"); + rc = tpm_getcap(to_tpm_chip(dev), TPM_CAP_FLAG_VOL, &cap, + "attempting to determine the temporary state"); if (rc) return 0; @@ -180,11 +180,12 @@ static DEVICE_ATTR_RO(temp_deactivated); static ssize_t caps_show(struct device *dev, struct device_attribute *attr, char *buf) { + struct tpm_chip *chip = to_tpm_chip(dev); cap_t cap; ssize_t rc; char *str = buf; - rc = tpm_getcap(dev, TPM_CAP_PROP_MANUFACTURER, &cap, + rc = tpm_getcap(chip, TPM_CAP_PROP_MANUFACTURER, &cap, "attempting to determine the manufacturer"); if (rc) return 0; @@ -192,8 +193,8 @@ static ssize_t caps_show(struct device *dev, struct device_attribute *attr, be32_to_cpu(cap.manufacturer_id)); /* Try to get a TPM version 1.2 TPM_CAP_VERSION_INFO */ - rc = tpm_getcap(dev, CAP_VERSION_1_2, &cap, - "attempting to determine the 1.2 version"); + rc = tpm_getcap(chip, CAP_VERSION_1_2, &cap, + "attempting to determine the 1.2 version"); if (!rc) { str += sprintf(str, "TCG version: %d.%d\nFirmware version: %d.%d\n", @@ -203,7 +204,7 @@ static ssize_t caps_show(struct device *dev, struct device_attribute *attr, cap.tpm_version_1_2.revMinor); } else { /* Otherwise just use TPM_STRUCT_VER */ - rc = tpm_getcap(dev, CAP_VERSION_1_1, &cap, + rc = tpm_getcap(chip, CAP_VERSION_1_1, &cap, "attempting to determine the 1.1 version"); if (rc) return 0; @@ -222,7 +223,7 @@ static DEVICE_ATTR_RO(caps); static ssize_t cancel_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct tpm_chip *chip = dev_get_drvdata(dev); + struct tpm_chip *chip = to_tpm_chip(dev); if (chip == NULL) return 0; @@ -234,16 +235,16 @@ static DEVICE_ATTR_WO(cancel); static ssize_t durations_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct tpm_chip *chip = dev_get_drvdata(dev); + struct tpm_chip *chip = to_tpm_chip(dev); - if (chip->vendor.duration[TPM_LONG] == 0) + if (chip->duration[TPM_LONG] == 0) return 0; return sprintf(buf, "%d %d %d [%s]\n", - jiffies_to_usecs(chip->vendor.duration[TPM_SHORT]), - jiffies_to_usecs(chip->vendor.duration[TPM_MEDIUM]), - jiffies_to_usecs(chip->vendor.duration[TPM_LONG]), - chip->vendor.duration_adjusted + jiffies_to_usecs(chip->duration[TPM_SHORT]), + jiffies_to_usecs(chip->duration[TPM_MEDIUM]), + jiffies_to_usecs(chip->duration[TPM_LONG]), + chip->duration_adjusted ? "adjusted" : "original"); } static DEVICE_ATTR_RO(durations); @@ -251,14 +252,14 @@ static DEVICE_ATTR_RO(durations); static ssize_t timeouts_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct tpm_chip *chip = dev_get_drvdata(dev); + struct tpm_chip *chip = to_tpm_chip(dev); return sprintf(buf, "%d %d %d %d [%s]\n", - jiffies_to_usecs(chip->vendor.timeout_a), - jiffies_to_usecs(chip->vendor.timeout_b), - jiffies_to_usecs(chip->vendor.timeout_c), - jiffies_to_usecs(chip->vendor.timeout_d), - chip->vendor.timeout_adjusted + jiffies_to_usecs(chip->timeout_a), + jiffies_to_usecs(chip->timeout_b), + jiffies_to_usecs(chip->timeout_c), + jiffies_to_usecs(chip->timeout_d), + chip->timeout_adjusted ? "adjusted" : "original"); } static DEVICE_ATTR_RO(timeouts); @@ -281,19 +282,12 @@ static const struct attribute_group tpm_dev_group = { .attrs = tpm_dev_attrs, }; -int tpm_sysfs_add_device(struct tpm_chip *chip) +void tpm_sysfs_add_device(struct tpm_chip *chip) { - int err; - err = sysfs_create_group(&chip->pdev->kobj, - &tpm_dev_group); - - if (err) - dev_err(chip->pdev, - "failed to create sysfs attributes, %d\n", err); - return err; -} - -void tpm_sysfs_del_device(struct tpm_chip *chip) -{ - sysfs_remove_group(&chip->pdev->kobj, &tpm_dev_group); + /* The sysfs routines rely on an implicit tpm_try_get_ops, device_del + * is called before ops is null'd and the sysfs core synchronizes this + * removal so that no callbacks are running or can run again + */ + WARN_ON(chip->groups_cnt != 0); + chip->groups[chip->groups_cnt++] = &tpm_dev_group; } diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h index 28b477e8d..b0585e99d 100644 --- a/drivers/char/tpm/tpm.h +++ b/drivers/char/tpm/tpm.h @@ -19,6 +19,10 @@ * License. * */ + +#ifndef __TPM_H__ +#define __TPM_H__ + #include #include #include @@ -34,7 +38,7 @@ enum tpm_const { TPM_MINOR = 224, /* officially assigned */ TPM_BUFSIZE = 4096, - TPM_NUM_DEVICES = 256, + TPM_NUM_DEVICES = 65536, TPM_RETRY = 50, /* 5 seconds */ }; @@ -128,33 +132,6 @@ enum tpm2_startup_types { TPM2_SU_STATE = 0x0001, }; -struct tpm_chip; - -struct tpm_vendor_specific { - void __iomem *iobase; /* ioremapped address */ - unsigned long base; /* TPM base address */ - - int irq; - - int region_size; - int have_region; - - struct list_head list; - int locality; - unsigned long timeout_a, timeout_b, timeout_c, timeout_d; /* jiffies */ - bool timeout_adjusted; - unsigned long duration[3]; /* jiffies */ - bool duration_adjusted; - void *priv; - - wait_queue_head_t read_queue; - wait_queue_head_t int_queue; - - u16 manufacturer_id; -}; - -#define TPM_VPRIV(c) ((c)->vendor.priv) - #define TPM_VID_INTEL 0x8086 #define TPM_VID_WINBOND 0x1050 #define TPM_VID_STM 0x104A @@ -164,44 +141,48 @@ struct tpm_vendor_specific { enum tpm_chip_flags { TPM_CHIP_FLAG_REGISTERED = BIT(0), TPM_CHIP_FLAG_TPM2 = BIT(1), + TPM_CHIP_FLAG_IRQ = BIT(2), + TPM_CHIP_FLAG_VIRTUAL = BIT(3), }; struct tpm_chip { - struct device *pdev; /* Device stuff */ struct device dev; struct cdev cdev; + /* A driver callback under ops cannot be run unless ops_sem is held + * (sometimes implicitly, eg for the sysfs code). ops becomes null + * when the driver is unregistered, see tpm_try_get_ops. + */ + struct rw_semaphore ops_sem; const struct tpm_class_ops *ops; + unsigned int flags; int dev_num; /* /dev/tpm# */ - char devname[7]; unsigned long is_open; /* only one allowed */ - int time_expired; struct mutex tpm_mutex; /* tpm is processing */ - struct tpm_vendor_specific vendor; + unsigned long timeout_a; /* jiffies */ + unsigned long timeout_b; /* jiffies */ + unsigned long timeout_c; /* jiffies */ + unsigned long timeout_d; /* jiffies */ + bool timeout_adjusted; + unsigned long duration[3]; /* jiffies */ + bool duration_adjusted; struct dentry **bios_dir; -#ifdef CONFIG_ACPI - const struct attribute_group *groups[2]; + const struct attribute_group *groups[3]; unsigned int groups_cnt; +#ifdef CONFIG_ACPI acpi_handle acpi_dev_handle; char ppi_version[TPM_PPI_VERSION_LEN + 1]; #endif /* CONFIG_ACPI */ - - struct list_head list; }; #define to_tpm_chip(d) container_of(d, struct tpm_chip, dev) -static inline void tpm_chip_put(struct tpm_chip *chip) -{ - module_put(chip->pdev->driver->owner); -} - static inline int tpm_read_index(int base, int index) { outb(index, base); @@ -493,14 +474,21 @@ static inline void tpm_buf_append_u32(struct tpm_buf *buf, const u32 value) extern struct class *tpm_class; extern dev_t tpm_devt; extern const struct file_operations tpm_fops; +extern struct idr dev_nums_idr; + +enum tpm_transmit_flags { + TPM_TRANSMIT_UNLOCKED = BIT(0), +}; -ssize_t tpm_getcap(struct device *, __be32, cap_t *, const char *); -ssize_t tpm_transmit(struct tpm_chip *chip, const char *buf, - size_t bufsiz); -ssize_t tpm_transmit_cmd(struct tpm_chip *chip, void *cmd, int len, - const char *desc); +ssize_t tpm_transmit(struct tpm_chip *chip, const u8 *buf, size_t bufsiz, + unsigned int flags); +ssize_t tpm_transmit_cmd(struct tpm_chip *chip, const void *cmd, int len, + unsigned int flags, const char *desc); +ssize_t tpm_getcap(struct tpm_chip *chip, __be32 subcap_id, cap_t *cap, + const char *desc); extern int tpm_get_timeouts(struct tpm_chip *); extern void tpm_gen_interrupt(struct tpm_chip *); +int tpm1_auto_startup(struct tpm_chip *chip); extern int tpm_do_selftest(struct tpm_chip *); extern unsigned long tpm_calc_ordinal_duration(struct tpm_chip *, u32); extern int tpm_pm_suspend(struct device *); @@ -509,13 +497,17 @@ extern int wait_for_tpm_stat(struct tpm_chip *, u8, unsigned long, wait_queue_head_t *, bool); struct tpm_chip *tpm_chip_find_get(int chip_num); -extern struct tpm_chip *tpmm_chip_alloc(struct device *dev, +__must_check int tpm_try_get_ops(struct tpm_chip *chip); +void tpm_put_ops(struct tpm_chip *chip); + +extern struct tpm_chip *tpm_chip_alloc(struct device *dev, + const struct tpm_class_ops *ops); +extern struct tpm_chip *tpmm_chip_alloc(struct device *pdev, const struct tpm_class_ops *ops); extern int tpm_chip_register(struct tpm_chip *chip); extern void tpm_chip_unregister(struct tpm_chip *chip); -int tpm_sysfs_add_device(struct tpm_chip *chip); -void tpm_sysfs_del_device(struct tpm_chip *chip); +void tpm_sysfs_add_device(struct tpm_chip *chip); int tpm_pcr_read_dev(struct tpm_chip *chip, int pcr_idx, u8 *res_buf); @@ -539,9 +531,9 @@ int tpm2_unseal_trusted(struct tpm_chip *chip, ssize_t tpm2_get_tpm_pt(struct tpm_chip *chip, u32 property_id, u32 *value, const char *desc); -extern int tpm2_startup(struct tpm_chip *chip, u16 startup_type); +int tpm2_auto_startup(struct tpm_chip *chip); extern void tpm2_shutdown(struct tpm_chip *chip, u16 shutdown_type); extern unsigned long tpm2_calc_ordinal_duration(struct tpm_chip *, u32); -extern int tpm2_do_selftest(struct tpm_chip *chip); extern int tpm2_gen_interrupt(struct tpm_chip *chip); extern int tpm2_probe(struct tpm_chip *chip); +#endif diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index b28e4da3d..ef5a58b98 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -282,7 +282,7 @@ int tpm2_pcr_read(struct tpm_chip *chip, int pcr_idx, u8 *res_buf) sizeof(cmd.params.pcrread_in.pcr_select)); cmd.params.pcrread_in.pcr_select[pcr_idx >> 3] = 1 << (pcr_idx & 0x7); - rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), + rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "attempting to read a pcr value"); if (rc == 0) { buf = cmd.params.pcrread_out.digest; @@ -330,7 +330,7 @@ int tpm2_pcr_extend(struct tpm_chip *chip, int pcr_idx, const u8 *hash) cmd.params.pcrextend_in.hash_alg = cpu_to_be16(TPM2_ALG_SHA1); memcpy(cmd.params.pcrextend_in.digest, hash, TPM_DIGEST_SIZE); - rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), + rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "attempting extend a PCR value"); return rc; @@ -376,7 +376,7 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *out, size_t max) cmd.header.in = tpm2_getrandom_header; cmd.params.getrandom_in.size = cpu_to_be16(num_bytes); - err = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), + err = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "attempting get random"); if (err) break; @@ -434,12 +434,12 @@ static void tpm2_buf_append_auth(struct tpm_buf *buf, u32 session_handle, } /** - * tpm2_seal_trusted() - seal a trusted key - * @chip_num: A specific chip number for the request or TPM_ANY_NUM - * @options: authentication values and other options + * tpm2_seal_trusted() - seal the payload of a trusted key + * @chip_num: TPM chip to use * @payload: the key data in clear and encrypted form + * @options: authentication values and other options * - * Returns < 0 on error and 0 on success. + * Return: < 0 on error and 0 on success. */ int tpm2_seal_trusted(struct tpm_chip *chip, struct trusted_key_payload *payload, @@ -512,7 +512,7 @@ int tpm2_seal_trusted(struct tpm_chip *chip, goto out; } - rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "sealing data"); + rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, 0, "sealing data"); if (rc) goto out; @@ -538,10 +538,18 @@ out: return rc; } -static int tpm2_load(struct tpm_chip *chip, - struct trusted_key_payload *payload, - struct trusted_key_options *options, - u32 *blob_handle) +/** + * tpm2_load_cmd() - execute a TPM2_Load command + * @chip_num: TPM chip to use + * @payload: the key data in clear and encrypted form + * @options: authentication values and other options + * + * Return: same as with tpm_transmit_cmd + */ +static int tpm2_load_cmd(struct tpm_chip *chip, + struct trusted_key_payload *payload, + struct trusted_key_options *options, + u32 *blob_handle, unsigned int flags) { struct tpm_buf buf; unsigned int private_len; @@ -576,7 +584,7 @@ static int tpm2_load(struct tpm_chip *chip, goto out; } - rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "loading blob"); + rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, flags, "loading blob"); if (!rc) *blob_handle = be32_to_cpup( (__be32 *) &buf.data[TPM_HEADER_SIZE]); @@ -590,32 +598,50 @@ out: return rc; } -static void tpm2_flush_context(struct tpm_chip *chip, u32 handle) +/** + * tpm2_flush_context_cmd() - execute a TPM2_FlushContext command + * @chip_num: TPM chip to use + * @payload: the key data in clear and encrypted form + * @options: authentication values and other options + * + * Return: same as with tpm_transmit_cmd + */ +static void tpm2_flush_context_cmd(struct tpm_chip *chip, u32 handle, + unsigned int flags) { struct tpm_buf buf; int rc; rc = tpm_buf_init(&buf, TPM2_ST_NO_SESSIONS, TPM2_CC_FLUSH_CONTEXT); if (rc) { - dev_warn(chip->pdev, "0x%08x was not flushed, out of memory\n", + dev_warn(&chip->dev, "0x%08x was not flushed, out of memory\n", handle); return; } tpm_buf_append_u32(&buf, handle); - rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "flushing context"); + rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, flags, + "flushing context"); if (rc) - dev_warn(chip->pdev, "0x%08x was not flushed, rc=%d\n", handle, + dev_warn(&chip->dev, "0x%08x was not flushed, rc=%d\n", handle, rc); tpm_buf_destroy(&buf); } -static int tpm2_unseal(struct tpm_chip *chip, - struct trusted_key_payload *payload, - struct trusted_key_options *options, - u32 blob_handle) +/** + * tpm2_unseal_cmd() - execute a TPM2_Unload command + * @chip_num: TPM chip to use + * @payload: the key data in clear and encrypted form + * @options: authentication values and other options + * + * Return: same as with tpm_transmit_cmd + */ +static int tpm2_unseal_cmd(struct tpm_chip *chip, + struct trusted_key_payload *payload, + struct trusted_key_options *options, + u32 blob_handle, unsigned int flags) { struct tpm_buf buf; u16 data_len; @@ -635,7 +661,7 @@ static int tpm2_unseal(struct tpm_chip *chip, options->blobauth /* hmac */, TPM_DIGEST_SIZE); - rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, "unsealing"); + rc = tpm_transmit_cmd(chip, buf.data, PAGE_SIZE, flags, "unsealing"); if (rc > 0) rc = -EPERM; @@ -654,12 +680,12 @@ static int tpm2_unseal(struct tpm_chip *chip, } /** - * tpm_unseal_trusted() - unseal a trusted key - * @chip_num: A specific chip number for the request or TPM_ANY_NUM - * @options: authentication values and other options + * tpm_unseal_trusted() - unseal the payload of a trusted key + * @chip_num: TPM chip to use * @payload: the key data in clear and encrypted form + * @options: authentication values and other options * - * Returns < 0 on error and 0 on success. + * Return: < 0 on error and 0 on success. */ int tpm2_unseal_trusted(struct tpm_chip *chip, struct trusted_key_payload *payload, @@ -668,14 +694,17 @@ int tpm2_unseal_trusted(struct tpm_chip *chip, u32 blob_handle; int rc; - rc = tpm2_load(chip, payload, options, &blob_handle); + mutex_lock(&chip->tpm_mutex); + rc = tpm2_load_cmd(chip, payload, options, &blob_handle, + TPM_TRANSMIT_UNLOCKED); if (rc) - return rc; - - rc = tpm2_unseal(chip, payload, options, blob_handle); - - tpm2_flush_context(chip, blob_handle); + goto out; + rc = tpm2_unseal_cmd(chip, payload, options, blob_handle, + TPM_TRANSMIT_UNLOCKED); + tpm2_flush_context_cmd(chip, blob_handle, TPM_TRANSMIT_UNLOCKED); +out: + mutex_unlock(&chip->tpm_mutex); return rc; } @@ -701,9 +730,9 @@ ssize_t tpm2_get_tpm_pt(struct tpm_chip *chip, u32 property_id, u32 *value, cmd.params.get_tpm_pt_in.property_id = cpu_to_be32(property_id); cmd.params.get_tpm_pt_in.property_cnt = cpu_to_be32(1); - rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), desc); + rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, desc); if (!rc) - *value = cmd.params.get_tpm_pt_out.value; + *value = be32_to_cpu(cmd.params.get_tpm_pt_out.value); return rc; } @@ -728,17 +757,16 @@ static const struct tpm_input_header tpm2_startup_header = { * returned it remarks a POSIX error code. If a positive number is returned * it remarks a TPM error. */ -int tpm2_startup(struct tpm_chip *chip, u16 startup_type) +static int tpm2_startup(struct tpm_chip *chip, u16 startup_type) { struct tpm2_cmd cmd; cmd.header.in = tpm2_startup_header; cmd.params.startup_in.startup_type = cpu_to_be16(startup_type); - return tpm_transmit_cmd(chip, &cmd, sizeof(cmd), + return tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "attempting to start the TPM"); } -EXPORT_SYMBOL_GPL(tpm2_startup); #define TPM2_SHUTDOWN_IN_SIZE \ (sizeof(struct tpm_input_header) + \ @@ -764,16 +792,15 @@ void tpm2_shutdown(struct tpm_chip *chip, u16 shutdown_type) cmd.header.in = tpm2_shutdown_header; cmd.params.startup_in.startup_type = cpu_to_be16(shutdown_type); - rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), "stopping the TPM"); + rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, "stopping the TPM"); /* In places where shutdown command is sent there's no much we can do * except print the error code on a system failure. */ if (rc < 0) - dev_warn(chip->pdev, "transmit returned %d while stopping the TPM", + dev_warn(&chip->dev, "transmit returned %d while stopping the TPM", rc); } -EXPORT_SYMBOL_GPL(tpm2_shutdown); /* * tpm2_calc_ordinal_duration() - maximum duration for a command @@ -793,7 +820,7 @@ unsigned long tpm2_calc_ordinal_duration(struct tpm_chip *chip, u32 ordinal) index = tpm2_ordinal_duration[ordinal - TPM2_CC_FIRST]; if (index != TPM_UNDEFINED) - duration = chip->vendor.duration[index]; + duration = chip->duration[index]; if (duration <= 0) duration = 2 * 60 * HZ; @@ -830,14 +857,14 @@ static int tpm2_start_selftest(struct tpm_chip *chip, bool full) cmd.header.in = tpm2_selftest_header; cmd.params.selftest_in.full_test = full; - rc = tpm_transmit_cmd(chip, &cmd, TPM2_SELF_TEST_IN_SIZE, + rc = tpm_transmit_cmd(chip, &cmd, TPM2_SELF_TEST_IN_SIZE, 0, "continue selftest"); /* At least some prototype chips seem to give RC_TESTING error * immediately. This is a workaround for that. */ if (rc == TPM2_RC_TESTING) { - dev_warn(chip->pdev, "Got RC_TESTING, ignoring\n"); + dev_warn(&chip->dev, "Got RC_TESTING, ignoring\n"); rc = 0; } @@ -855,7 +882,7 @@ static int tpm2_start_selftest(struct tpm_chip *chip, bool full) * returned it remarks a POSIX error code. If a positive number is returned * it remarks a TPM error. */ -int tpm2_do_selftest(struct tpm_chip *chip) +static int tpm2_do_selftest(struct tpm_chip *chip) { int rc; unsigned int loops; @@ -882,7 +909,7 @@ int tpm2_do_selftest(struct tpm_chip *chip) cmd.params.pcrread_in.pcr_select[1] = 0x00; cmd.params.pcrread_in.pcr_select[2] = 0x00; - rc = tpm_transmit_cmd(chip, (u8 *) &cmd, sizeof(cmd), NULL); + rc = tpm_transmit_cmd(chip, &cmd, sizeof(cmd), 0, NULL); if (rc < 0) break; @@ -895,7 +922,6 @@ int tpm2_do_selftest(struct tpm_chip *chip) return rc; } -EXPORT_SYMBOL_GPL(tpm2_do_selftest); /** * tpm2_gen_interrupt() - generate an interrupt @@ -931,7 +957,7 @@ int tpm2_probe(struct tpm_chip *chip) cmd.params.get_tpm_pt_in.property_id = cpu_to_be32(0x100); cmd.params.get_tpm_pt_in.property_cnt = cpu_to_be32(1); - rc = tpm_transmit(chip, (const char *) &cmd, sizeof(cmd)); + rc = tpm_transmit(chip, (const u8 *)&cmd, sizeof(cmd), 0); if (rc < 0) return rc; else if (rc < TPM_HEADER_SIZE) @@ -943,3 +969,42 @@ int tpm2_probe(struct tpm_chip *chip) return 0; } EXPORT_SYMBOL_GPL(tpm2_probe); + +/** + * tpm2_auto_startup - Perform the standard automatic TPM initialization + * sequence + * @chip: TPM chip to use + * + * Returns 0 on success, < 0 in case of fatal error. + */ +int tpm2_auto_startup(struct tpm_chip *chip) +{ + int rc; + + rc = tpm_get_timeouts(chip); + if (rc) + goto out; + + rc = tpm2_do_selftest(chip); + if (rc != 0 && rc != TPM2_RC_INITIALIZE) { + dev_err(&chip->dev, "TPM self test failed\n"); + goto out; + } + + if (rc == TPM2_RC_INITIALIZE) { + rc = tpm2_startup(chip, TPM2_SU_CLEAR); + if (rc) + goto out; + + rc = tpm2_do_selftest(chip); + if (rc) { + dev_err(&chip->dev, "TPM self test failed\n"); + goto out; + } + } + +out: + if (rc > 0) + rc = -ENODEV; + return rc; +} diff --git a/drivers/char/tpm/tpm_atmel.c b/drivers/char/tpm/tpm_atmel.c index dfadad091..0d322ab11 100644 --- a/drivers/char/tpm/tpm_atmel.c +++ b/drivers/char/tpm/tpm_atmel.c @@ -37,6 +37,7 @@ enum tpm_atmel_read_status { static int tpm_atml_recv(struct tpm_chip *chip, u8 *buf, size_t count) { + struct tpm_atmel_priv *priv = dev_get_drvdata(&chip->dev); u8 status, *hdr = buf; u32 size; int i; @@ -47,12 +48,12 @@ static int tpm_atml_recv(struct tpm_chip *chip, u8 *buf, size_t count) return -EIO; for (i = 0; i < 6; i++) { - status = ioread8(chip->vendor.iobase + 1); + status = ioread8(priv->iobase + 1); if ((status & ATML_STATUS_DATA_AVAIL) == 0) { - dev_err(chip->pdev, "error reading header\n"); + dev_err(&chip->dev, "error reading header\n"); return -EIO; } - *buf++ = ioread8(chip->vendor.iobase); + *buf++ = ioread8(priv->iobase); } /* size of the data received */ @@ -60,12 +61,12 @@ static int tpm_atml_recv(struct tpm_chip *chip, u8 *buf, size_t count) size = be32_to_cpu(*native_size); if (count < size) { - dev_err(chip->pdev, + dev_err(&chip->dev, "Recv size(%d) less than available space\n", size); for (; i < size; i++) { /* clear the waiting data anyway */ - status = ioread8(chip->vendor.iobase + 1); + status = ioread8(priv->iobase + 1); if ((status & ATML_STATUS_DATA_AVAIL) == 0) { - dev_err(chip->pdev, "error reading data\n"); + dev_err(&chip->dev, "error reading data\n"); return -EIO; } } @@ -74,19 +75,19 @@ static int tpm_atml_recv(struct tpm_chip *chip, u8 *buf, size_t count) /* read all the data available */ for (; i < size; i++) { - status = ioread8(chip->vendor.iobase + 1); + status = ioread8(priv->iobase + 1); if ((status & ATML_STATUS_DATA_AVAIL) == 0) { - dev_err(chip->pdev, "error reading data\n"); + dev_err(&chip->dev, "error reading data\n"); return -EIO; } - *buf++ = ioread8(chip->vendor.iobase); + *buf++ = ioread8(priv->iobase); } /* make sure data available is gone */ - status = ioread8(chip->vendor.iobase + 1); + status = ioread8(priv->iobase + 1); if (status & ATML_STATUS_DATA_AVAIL) { - dev_err(chip->pdev, "data available is stuck\n"); + dev_err(&chip->dev, "data available is stuck\n"); return -EIO; } @@ -95,12 +96,13 @@ static int tpm_atml_recv(struct tpm_chip *chip, u8 *buf, size_t count) static int tpm_atml_send(struct tpm_chip *chip, u8 *buf, size_t count) { + struct tpm_atmel_priv *priv = dev_get_drvdata(&chip->dev); int i; - dev_dbg(chip->pdev, "tpm_atml_send:\n"); + dev_dbg(&chip->dev, "tpm_atml_send:\n"); for (i = 0; i < count; i++) { - dev_dbg(chip->pdev, "%d 0x%x(%d)\n", i, buf[i], buf[i]); - iowrite8(buf[i], chip->vendor.iobase); + dev_dbg(&chip->dev, "%d 0x%x(%d)\n", i, buf[i], buf[i]); + iowrite8(buf[i], priv->iobase); } return count; @@ -108,12 +110,16 @@ static int tpm_atml_send(struct tpm_chip *chip, u8 *buf, size_t count) static void tpm_atml_cancel(struct tpm_chip *chip) { - iowrite8(ATML_STATUS_ABORT, chip->vendor.iobase + 1); + struct tpm_atmel_priv *priv = dev_get_drvdata(&chip->dev); + + iowrite8(ATML_STATUS_ABORT, priv->iobase + 1); } static u8 tpm_atml_status(struct tpm_chip *chip) { - return ioread8(chip->vendor.iobase + 1); + struct tpm_atmel_priv *priv = dev_get_drvdata(&chip->dev); + + return ioread8(priv->iobase + 1); } static bool tpm_atml_req_canceled(struct tpm_chip *chip, u8 status) @@ -136,13 +142,13 @@ static struct platform_device *pdev; static void atml_plat_remove(void) { struct tpm_chip *chip = dev_get_drvdata(&pdev->dev); + struct tpm_atmel_priv *priv = dev_get_drvdata(&chip->dev); if (chip) { tpm_chip_unregister(chip); - if (chip->vendor.have_region) - atmel_release_region(chip->vendor.base, - chip->vendor.region_size); - atmel_put_base_addr(chip->vendor.iobase); + if (priv->have_region) + atmel_release_region(priv->base, priv->region_size); + atmel_put_base_addr(priv->iobase); platform_device_unregister(pdev); } } @@ -163,6 +169,7 @@ static int __init init_atmel(void) int have_region, region_size; unsigned long base; struct tpm_chip *chip; + struct tpm_atmel_priv *priv; rc = platform_driver_register(&atml_drv); if (rc) @@ -183,16 +190,24 @@ static int __init init_atmel(void) goto err_rel_reg; } + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) { + rc = -ENOMEM; + goto err_unreg_dev; + } + + priv->iobase = iobase; + priv->base = base; + priv->have_region = have_region; + priv->region_size = region_size; + chip = tpmm_chip_alloc(&pdev->dev, &tpm_atmel); if (IS_ERR(chip)) { rc = PTR_ERR(chip); goto err_unreg_dev; } - chip->vendor.iobase = iobase; - chip->vendor.base = base; - chip->vendor.have_region = have_region; - chip->vendor.region_size = region_size; + dev_set_drvdata(&chip->dev, priv); rc = tpm_chip_register(chip); if (rc) diff --git a/drivers/char/tpm/tpm_atmel.h b/drivers/char/tpm/tpm_atmel.h index 6c831f946..4f96d80cd 100644 --- a/drivers/char/tpm/tpm_atmel.h +++ b/drivers/char/tpm/tpm_atmel.h @@ -22,12 +22,19 @@ * */ +struct tpm_atmel_priv { + int region_size; + int have_region; + unsigned long base; + void __iomem *iobase; +}; + #ifdef CONFIG_PPC64 #include -#define atmel_getb(chip, offset) readb(chip->vendor->iobase + offset); -#define atmel_putb(val, chip, offset) writeb(val, chip->vendor->iobase + offset) +#define atmel_getb(priv, offset) readb(priv->iobase + offset) +#define atmel_putb(val, priv, offset) writeb(val, priv->iobase + offset) #define atmel_request_region request_mem_region #define atmel_release_region release_mem_region @@ -78,8 +85,9 @@ static void __iomem * atmel_get_base_addr(unsigned long *base, int *region_size) return ioremap(*base, *region_size); } #else -#define atmel_getb(chip, offset) inb(chip->vendor->base + offset) -#define atmel_putb(val, chip, offset) outb(val, chip->vendor->base + offset) +#define atmel_getb(chip, offset) inb(atmel_get_priv(chip)->base + offset) +#define atmel_putb(val, chip, offset) \ + outb(val, atmel_get_priv(chip)->base + offset) #define atmel_request_region request_region #define atmel_release_region release_region /* Atmel definitions */ diff --git a/drivers/char/tpm/tpm_crb.c b/drivers/char/tpm/tpm_crb.c index e9fd1d83f..1801f3823 100644 --- a/drivers/char/tpm/tpm_crb.c +++ b/drivers/char/tpm/tpm_crb.c @@ -77,7 +77,6 @@ enum crb_flags { struct crb_priv { unsigned int flags; - struct resource res; void __iomem *iobase; struct crb_control_area __iomem *cca; u8 __iomem *cmd; @@ -88,7 +87,7 @@ static SIMPLE_DEV_PM_OPS(crb_pm, tpm_pm_suspend, tpm_pm_resume); static u8 crb_status(struct tpm_chip *chip) { - struct crb_priv *priv = chip->vendor.priv; + struct crb_priv *priv = dev_get_drvdata(&chip->dev); u8 sts = 0; if ((ioread32(&priv->cca->start) & CRB_START_INVOKE) != @@ -100,7 +99,7 @@ static u8 crb_status(struct tpm_chip *chip) static int crb_recv(struct tpm_chip *chip, u8 *buf, size_t count) { - struct crb_priv *priv = chip->vendor.priv; + struct crb_priv *priv = dev_get_drvdata(&chip->dev); unsigned int expected; /* sanity check */ @@ -140,9 +139,14 @@ static int crb_do_acpi_start(struct tpm_chip *chip) static int crb_send(struct tpm_chip *chip, u8 *buf, size_t len) { - struct crb_priv *priv = chip->vendor.priv; + struct crb_priv *priv = dev_get_drvdata(&chip->dev); int rc = 0; + /* Zero the cancel register so that the next command will not get + * canceled. + */ + iowrite32(0, &priv->cca->cancel); + if (len > ioread32(&priv->cca->cmd_size)) { dev_err(&chip->dev, "invalid command count value %x %zx\n", @@ -167,7 +171,7 @@ static int crb_send(struct tpm_chip *chip, u8 *buf, size_t len) static void crb_cancel(struct tpm_chip *chip) { - struct crb_priv *priv = chip->vendor.priv; + struct crb_priv *priv = dev_get_drvdata(&chip->dev); iowrite32(cpu_to_le32(CRB_CANCEL_INVOKE), &priv->cca->cancel); @@ -176,19 +180,18 @@ static void crb_cancel(struct tpm_chip *chip) if ((priv->flags & CRB_FL_ACPI_START) && crb_do_acpi_start(chip)) dev_err(&chip->dev, "ACPI Start failed\n"); - - iowrite32(0, &priv->cca->cancel); } static bool crb_req_canceled(struct tpm_chip *chip, u8 status) { - struct crb_priv *priv = chip->vendor.priv; + struct crb_priv *priv = dev_get_drvdata(&chip->dev); u32 cancel = ioread32(&priv->cca->cancel); return (cancel & CRB_CANCEL_INVOKE) == CRB_CANCEL_INVOKE; } static const struct tpm_class_ops tpm_crb = { + .flags = TPM_OPS_AUTO_STARTUP, .status = crb_status, .recv = crb_recv, .send = crb_send, @@ -201,42 +204,33 @@ static const struct tpm_class_ops tpm_crb = { static int crb_init(struct acpi_device *device, struct crb_priv *priv) { struct tpm_chip *chip; - int rc; chip = tpmm_chip_alloc(&device->dev, &tpm_crb); if (IS_ERR(chip)) return PTR_ERR(chip); - chip->vendor.priv = priv; + dev_set_drvdata(&chip->dev, priv); chip->acpi_dev_handle = device->handle; chip->flags = TPM_CHIP_FLAG_TPM2; - rc = tpm_get_timeouts(chip); - if (rc) - return rc; - - rc = tpm2_do_selftest(chip); - if (rc) - return rc; - return tpm_chip_register(chip); } static int crb_check_resource(struct acpi_resource *ares, void *data) { - struct crb_priv *priv = data; + struct resource *io_res = data; struct resource res; if (acpi_dev_resource_memory(ares, &res)) { - priv->res = res; - priv->res.name = NULL; + *io_res = res; + io_res->name = NULL; } return 1; } static void __iomem *crb_map_res(struct device *dev, struct crb_priv *priv, - u64 start, u32 size) + struct resource *io_res, u64 start, u32 size) { struct resource new_res = { .start = start, @@ -248,51 +242,72 @@ static void __iomem *crb_map_res(struct device *dev, struct crb_priv *priv, if (start != new_res.start) return (void __iomem *) ERR_PTR(-EINVAL); - if (!resource_contains(&priv->res, &new_res)) + if (!resource_contains(io_res, &new_res)) return devm_ioremap_resource(dev, &new_res); - return priv->iobase + (new_res.start - priv->res.start); + return priv->iobase + (new_res.start - io_res->start); } static int crb_map_io(struct acpi_device *device, struct crb_priv *priv, struct acpi_table_tpm2 *buf) { struct list_head resources; + struct resource io_res; struct device *dev = &device->dev; - u64 pa; + u64 cmd_pa; + u32 cmd_size; + u64 rsp_pa; + u32 rsp_size; int ret; INIT_LIST_HEAD(&resources); ret = acpi_dev_get_resources(device, &resources, crb_check_resource, - priv); + &io_res); if (ret < 0) return ret; acpi_dev_free_resource_list(&resources); - if (resource_type(&priv->res) != IORESOURCE_MEM) { + if (resource_type(&io_res) != IORESOURCE_MEM) { dev_err(dev, FW_BUG "TPM2 ACPI table does not define a memory resource\n"); return -EINVAL; } - priv->iobase = devm_ioremap_resource(dev, &priv->res); + priv->iobase = devm_ioremap_resource(dev, &io_res); if (IS_ERR(priv->iobase)) return PTR_ERR(priv->iobase); - priv->cca = crb_map_res(dev, priv, buf->control_address, 0x1000); + priv->cca = crb_map_res(dev, priv, &io_res, buf->control_address, + sizeof(struct crb_control_area)); if (IS_ERR(priv->cca)) return PTR_ERR(priv->cca); - pa = ((u64) ioread32(&priv->cca->cmd_pa_high) << 32) | - (u64) ioread32(&priv->cca->cmd_pa_low); - priv->cmd = crb_map_res(dev, priv, pa, ioread32(&priv->cca->cmd_size)); + cmd_pa = ((u64) ioread32(&priv->cca->cmd_pa_high) << 32) | + (u64) ioread32(&priv->cca->cmd_pa_low); + cmd_size = ioread32(&priv->cca->cmd_size); + priv->cmd = crb_map_res(dev, priv, &io_res, cmd_pa, cmd_size); if (IS_ERR(priv->cmd)) return PTR_ERR(priv->cmd); - memcpy_fromio(&pa, &priv->cca->rsp_pa, 8); - pa = le64_to_cpu(pa); - priv->rsp = crb_map_res(dev, priv, pa, ioread32(&priv->cca->rsp_size)); - return PTR_ERR_OR_ZERO(priv->rsp); + memcpy_fromio(&rsp_pa, &priv->cca->rsp_pa, 8); + rsp_pa = le64_to_cpu(rsp_pa); + rsp_size = ioread32(&priv->cca->rsp_size); + + if (cmd_pa != rsp_pa) { + priv->rsp = crb_map_res(dev, priv, &io_res, rsp_pa, rsp_size); + return PTR_ERR_OR_ZERO(priv->rsp); + } + + /* According to the PTP specification, overlapping command and response + * buffer sizes must be identical. + */ + if (cmd_size != rsp_size) { + dev_err(dev, FW_BUG "overlapping command and response buffer sizes are not identical"); + return -EINVAL; + } + + priv->rsp = priv->cmd; + return 0; } static int crb_acpi_add(struct acpi_device *device) @@ -344,9 +359,6 @@ static int crb_acpi_remove(struct acpi_device *device) struct device *dev = &device->dev; struct tpm_chip *chip = dev_get_drvdata(dev); - if (chip->flags & TPM_CHIP_FLAG_TPM2) - tpm2_shutdown(chip, TPM2_SU_CLEAR); - tpm_chip_unregister(chip); return 0; diff --git a/drivers/char/tpm/tpm_eventlog.c b/drivers/char/tpm/tpm_eventlog.c index 4e6940acf..e72288632 100644 --- a/drivers/char/tpm/tpm_eventlog.c +++ b/drivers/char/tpm/tpm_eventlog.c @@ -403,7 +403,7 @@ static int is_bad(void *p) return 0; } -struct dentry **tpm_bios_log_setup(char *name) +struct dentry **tpm_bios_log_setup(const char *name) { struct dentry **ret = NULL, *tpm_dir, *bin_file, *ascii_file; diff --git a/drivers/char/tpm/tpm_eventlog.h b/drivers/char/tpm/tpm_eventlog.h index 267bfbd1b..8de62b09b 100644 --- a/drivers/char/tpm/tpm_eventlog.h +++ b/drivers/char/tpm/tpm_eventlog.h @@ -77,10 +77,10 @@ int read_log(struct tpm_bios_log *log); #if defined(CONFIG_TCG_IBMVTPM) || defined(CONFIG_TCG_IBMVTPM_MODULE) || \ defined(CONFIG_ACPI) -extern struct dentry **tpm_bios_log_setup(char *); +extern struct dentry **tpm_bios_log_setup(const char *); extern void tpm_bios_log_teardown(struct dentry **); #else -static inline struct dentry **tpm_bios_log_setup(char *name) +static inline struct dentry **tpm_bios_log_setup(const char *name) { return NULL; } diff --git a/drivers/char/tpm/tpm_i2c_atmel.c b/drivers/char/tpm/tpm_i2c_atmel.c index 8dfb88b97..95ce2e9cc 100644 --- a/drivers/char/tpm/tpm_i2c_atmel.c +++ b/drivers/char/tpm/tpm_i2c_atmel.c @@ -51,8 +51,8 @@ struct priv_data { static int i2c_atmel_send(struct tpm_chip *chip, u8 *buf, size_t len) { - struct priv_data *priv = chip->vendor.priv; - struct i2c_client *client = to_i2c_client(chip->pdev); + struct priv_data *priv = dev_get_drvdata(&chip->dev); + struct i2c_client *client = to_i2c_client(chip->dev.parent); s32 status; priv->len = 0; @@ -62,7 +62,7 @@ static int i2c_atmel_send(struct tpm_chip *chip, u8 *buf, size_t len) status = i2c_master_send(client, buf, len); - dev_dbg(chip->pdev, + dev_dbg(&chip->dev, "%s(buf=%*ph len=%0zx) -> sts=%d\n", __func__, (int)min_t(size_t, 64, len), buf, len, status); return status; @@ -70,8 +70,8 @@ static int i2c_atmel_send(struct tpm_chip *chip, u8 *buf, size_t len) static int i2c_atmel_recv(struct tpm_chip *chip, u8 *buf, size_t count) { - struct priv_data *priv = chip->vendor.priv; - struct i2c_client *client = to_i2c_client(chip->pdev); + struct priv_data *priv = dev_get_drvdata(&chip->dev); + struct i2c_client *client = to_i2c_client(chip->dev.parent); struct tpm_output_header *hdr = (struct tpm_output_header *)priv->buffer; u32 expected_len; @@ -88,7 +88,7 @@ static int i2c_atmel_recv(struct tpm_chip *chip, u8 *buf, size_t count) return -ENOMEM; if (priv->len >= expected_len) { - dev_dbg(chip->pdev, + dev_dbg(&chip->dev, "%s early(buf=%*ph count=%0zx) -> ret=%d\n", __func__, (int)min_t(size_t, 64, expected_len), buf, count, expected_len); @@ -97,7 +97,7 @@ static int i2c_atmel_recv(struct tpm_chip *chip, u8 *buf, size_t count) } rc = i2c_master_recv(client, buf, expected_len); - dev_dbg(chip->pdev, + dev_dbg(&chip->dev, "%s reread(buf=%*ph count=%0zx) -> ret=%d\n", __func__, (int)min_t(size_t, 64, expected_len), buf, count, expected_len); @@ -106,13 +106,13 @@ static int i2c_atmel_recv(struct tpm_chip *chip, u8 *buf, size_t count) static void i2c_atmel_cancel(struct tpm_chip *chip) { - dev_err(chip->pdev, "TPM operation cancellation was requested, but is not supported"); + dev_err(&chip->dev, "TPM operation cancellation was requested, but is not supported"); } static u8 i2c_atmel_read_status(struct tpm_chip *chip) { - struct priv_data *priv = chip->vendor.priv; - struct i2c_client *client = to_i2c_client(chip->pdev); + struct priv_data *priv = dev_get_drvdata(&chip->dev); + struct i2c_client *client = to_i2c_client(chip->dev.parent); int rc; /* The TPM fails the I2C read until it is ready, so we do the entire @@ -125,7 +125,7 @@ static u8 i2c_atmel_read_status(struct tpm_chip *chip) /* Once the TPM has completed the command the command remains readable * until another command is issued. */ rc = i2c_master_recv(client, priv->buffer, sizeof(priv->buffer)); - dev_dbg(chip->pdev, + dev_dbg(&chip->dev, "%s: sts=%d", __func__, rc); if (rc <= 0) return 0; @@ -141,6 +141,7 @@ static bool i2c_atmel_req_canceled(struct tpm_chip *chip, u8 status) } static const struct tpm_class_ops i2c_atmel = { + .flags = TPM_OPS_AUTO_STARTUP, .status = i2c_atmel_read_status, .recv = i2c_atmel_recv, .send = i2c_atmel_send, @@ -155,6 +156,7 @@ static int i2c_atmel_probe(struct i2c_client *client, { struct tpm_chip *chip; struct device *dev = &client->dev; + struct priv_data *priv; if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) return -ENODEV; @@ -163,26 +165,21 @@ static int i2c_atmel_probe(struct i2c_client *client, if (IS_ERR(chip)) return PTR_ERR(chip); - chip->vendor.priv = devm_kzalloc(dev, sizeof(struct priv_data), - GFP_KERNEL); - if (!chip->vendor.priv) + priv = devm_kzalloc(dev, sizeof(struct priv_data), GFP_KERNEL); + if (!priv) return -ENOMEM; /* Default timeouts */ - chip->vendor.timeout_a = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); - chip->vendor.timeout_b = msecs_to_jiffies(TPM_I2C_LONG_TIMEOUT); - chip->vendor.timeout_c = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); - chip->vendor.timeout_d = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); - chip->vendor.irq = 0; + chip->timeout_a = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); + chip->timeout_b = msecs_to_jiffies(TPM_I2C_LONG_TIMEOUT); + chip->timeout_c = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); + chip->timeout_d = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); + + dev_set_drvdata(&chip->dev, priv); /* There is no known way to probe for this device, and all version * information seems to be read via TPM commands. Thus we rely on the * TPM startup process in the common code to detect the device. */ - if (tpm_get_timeouts(chip)) - return -ENODEV; - - if (tpm_do_selftest(chip)) - return -ENODEV; return tpm_chip_register(chip); } diff --git a/drivers/char/tpm/tpm_i2c_infineon.c b/drivers/char/tpm/tpm_i2c_infineon.c index 63d5d22e9..62ee44e57 100644 --- a/drivers/char/tpm/tpm_i2c_infineon.c +++ b/drivers/char/tpm/tpm_i2c_infineon.c @@ -66,6 +66,7 @@ enum i2c_chip_type { /* Structure to store I2C TPM specific stuff */ struct tpm_inf_dev { struct i2c_client *client; + int locality; u8 buf[TPM_BUFSIZE + sizeof(u8)]; /* max. buffer size + addr */ struct tpm_chip *chip; enum i2c_chip_type chip_type; @@ -288,7 +289,7 @@ static int check_locality(struct tpm_chip *chip, int loc) if ((buf & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) == (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) { - chip->vendor.locality = loc; + tpm_dev.locality = loc; return loc; } @@ -320,7 +321,7 @@ static int request_locality(struct tpm_chip *chip, int loc) iic_tpm_write(TPM_ACCESS(loc), &buf, 1); /* wait for burstcount */ - stop = jiffies + chip->vendor.timeout_a; + stop = jiffies + chip->timeout_a; do { if (check_locality(chip, loc) >= 0) return loc; @@ -337,7 +338,7 @@ static u8 tpm_tis_i2c_status(struct tpm_chip *chip) u8 i = 0; do { - if (iic_tpm_read(TPM_STS(chip->vendor.locality), &buf, 1) < 0) + if (iic_tpm_read(TPM_STS(tpm_dev.locality), &buf, 1) < 0) return 0; i++; @@ -351,7 +352,7 @@ static void tpm_tis_i2c_ready(struct tpm_chip *chip) { /* this causes the current command to be aborted */ u8 buf = TPM_STS_COMMAND_READY; - iic_tpm_write_long(TPM_STS(chip->vendor.locality), &buf, 1); + iic_tpm_write_long(TPM_STS(tpm_dev.locality), &buf, 1); } static ssize_t get_burstcount(struct tpm_chip *chip) @@ -362,10 +363,10 @@ static ssize_t get_burstcount(struct tpm_chip *chip) /* wait for burstcount */ /* which timeout value, spec has 2 answers (c & d) */ - stop = jiffies + chip->vendor.timeout_d; + stop = jiffies + chip->timeout_d; do { /* Note: STS is little endian */ - if (iic_tpm_read(TPM_STS(chip->vendor.locality)+1, buf, 3) < 0) + if (iic_tpm_read(TPM_STS(tpm_dev.locality)+1, buf, 3) < 0) burstcnt = 0; else burstcnt = (buf[2] << 16) + (buf[1] << 8) + buf[0]; @@ -419,7 +420,7 @@ static int recv_data(struct tpm_chip *chip, u8 *buf, size_t count) if (burstcnt > (count - size)) burstcnt = count - size; - rc = iic_tpm_read(TPM_DATA_FIFO(chip->vendor.locality), + rc = iic_tpm_read(TPM_DATA_FIFO(tpm_dev.locality), &(buf[size]), burstcnt); if (rc == 0) size += burstcnt; @@ -446,7 +447,7 @@ static int tpm_tis_i2c_recv(struct tpm_chip *chip, u8 *buf, size_t count) /* read first 10 bytes, including tag, paramsize, and result */ size = recv_data(chip, buf, TPM_HEADER_SIZE); if (size < TPM_HEADER_SIZE) { - dev_err(chip->pdev, "Unable to read header\n"); + dev_err(&chip->dev, "Unable to read header\n"); goto out; } @@ -459,14 +460,14 @@ static int tpm_tis_i2c_recv(struct tpm_chip *chip, u8 *buf, size_t count) size += recv_data(chip, &buf[TPM_HEADER_SIZE], expected - TPM_HEADER_SIZE); if (size < expected) { - dev_err(chip->pdev, "Unable to read remainder of result\n"); + dev_err(&chip->dev, "Unable to read remainder of result\n"); size = -ETIME; goto out; } - wait_for_stat(chip, TPM_STS_VALID, chip->vendor.timeout_c, &status); + wait_for_stat(chip, TPM_STS_VALID, chip->timeout_c, &status); if (status & TPM_STS_DATA_AVAIL) { /* retry? */ - dev_err(chip->pdev, "Error left over data\n"); + dev_err(&chip->dev, "Error left over data\n"); size = -EIO; goto out; } @@ -477,7 +478,7 @@ out: * so we sleep rather than keeping the bus busy */ usleep_range(SLEEP_DURATION_RESET_LOW, SLEEP_DURATION_RESET_HI); - release_locality(chip, chip->vendor.locality, 0); + release_locality(chip, tpm_dev.locality, 0); return size; } @@ -500,7 +501,7 @@ static int tpm_tis_i2c_send(struct tpm_chip *chip, u8 *buf, size_t len) tpm_tis_i2c_ready(chip); if (wait_for_stat (chip, TPM_STS_COMMAND_READY, - chip->vendor.timeout_b, &status) < 0) { + chip->timeout_b, &status) < 0) { rc = -ETIME; goto out_err; } @@ -516,7 +517,7 @@ static int tpm_tis_i2c_send(struct tpm_chip *chip, u8 *buf, size_t len) if (burstcnt > (len - 1 - count)) burstcnt = len - 1 - count; - rc = iic_tpm_write(TPM_DATA_FIFO(chip->vendor.locality), + rc = iic_tpm_write(TPM_DATA_FIFO(tpm_dev.locality), &(buf[count]), burstcnt); if (rc == 0) count += burstcnt; @@ -530,7 +531,7 @@ static int tpm_tis_i2c_send(struct tpm_chip *chip, u8 *buf, size_t len) } wait_for_stat(chip, TPM_STS_VALID, - chip->vendor.timeout_c, &status); + chip->timeout_c, &status); if ((status & TPM_STS_DATA_EXPECT) == 0) { rc = -EIO; @@ -539,15 +540,15 @@ static int tpm_tis_i2c_send(struct tpm_chip *chip, u8 *buf, size_t len) } /* write last byte */ - iic_tpm_write(TPM_DATA_FIFO(chip->vendor.locality), &(buf[count]), 1); - wait_for_stat(chip, TPM_STS_VALID, chip->vendor.timeout_c, &status); + iic_tpm_write(TPM_DATA_FIFO(tpm_dev.locality), &(buf[count]), 1); + wait_for_stat(chip, TPM_STS_VALID, chip->timeout_c, &status); if ((status & TPM_STS_DATA_EXPECT) != 0) { rc = -EIO; goto out_err; } /* go and do it */ - iic_tpm_write(TPM_STS(chip->vendor.locality), &sts, 1); + iic_tpm_write(TPM_STS(tpm_dev.locality), &sts, 1); return len; out_err: @@ -556,7 +557,7 @@ out_err: * so we sleep rather than keeping the bus busy */ usleep_range(SLEEP_DURATION_RESET_LOW, SLEEP_DURATION_RESET_HI); - release_locality(chip, chip->vendor.locality, 0); + release_locality(chip, tpm_dev.locality, 0); return rc; } @@ -566,6 +567,7 @@ static bool tpm_tis_i2c_req_canceled(struct tpm_chip *chip, u8 status) } static const struct tpm_class_ops tpm_tis_i2c = { + .flags = TPM_OPS_AUTO_STARTUP, .status = tpm_tis_i2c_status, .recv = tpm_tis_i2c_recv, .send = tpm_tis_i2c_send, @@ -585,14 +587,11 @@ static int tpm_tis_i2c_init(struct device *dev) if (IS_ERR(chip)) return PTR_ERR(chip); - /* Disable interrupts */ - chip->vendor.irq = 0; - /* Default timeouts */ - chip->vendor.timeout_a = msecs_to_jiffies(TIS_SHORT_TIMEOUT); - chip->vendor.timeout_b = msecs_to_jiffies(TIS_LONG_TIMEOUT); - chip->vendor.timeout_c = msecs_to_jiffies(TIS_SHORT_TIMEOUT); - chip->vendor.timeout_d = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->timeout_a = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->timeout_b = msecs_to_jiffies(TIS_LONG_TIMEOUT); + chip->timeout_c = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->timeout_d = msecs_to_jiffies(TIS_SHORT_TIMEOUT); if (request_locality(chip, 0) != 0) { dev_err(dev, "could not request locality\n"); @@ -619,15 +618,11 @@ static int tpm_tis_i2c_init(struct device *dev) dev_info(dev, "1.2 TPM (device-id 0x%X)\n", vendor >> 16); - INIT_LIST_HEAD(&chip->vendor.list); tpm_dev.chip = chip; - tpm_get_timeouts(chip); - tpm_do_selftest(chip); - return tpm_chip_register(chip); out_release: - release_locality(chip, chip->vendor.locality, 1); + release_locality(chip, tpm_dev.locality, 1); tpm_dev.client = NULL; out_err: return rc; @@ -699,7 +694,7 @@ static int tpm_tis_i2c_remove(struct i2c_client *client) struct tpm_chip *chip = tpm_dev.chip; tpm_chip_unregister(chip); - release_locality(chip, chip->vendor.locality, 1); + release_locality(chip, tpm_dev.locality, 1); tpm_dev.client = NULL; return 0; diff --git a/drivers/char/tpm/tpm_i2c_nuvoton.c b/drivers/char/tpm/tpm_i2c_nuvoton.c index 847f1597f..e3a9155ee 100644 --- a/drivers/char/tpm/tpm_i2c_nuvoton.c +++ b/drivers/char/tpm/tpm_i2c_nuvoton.c @@ -1,5 +1,5 @@ -/****************************************************************************** - * Nuvoton TPM I2C Device Driver Interface for WPCT301/NPCT501, + /****************************************************************************** + * Nuvoton TPM I2C Device Driver Interface for WPCT301/NPCT501/NPCT6XX, * based on the TCG TPM Interface Spec version 1.2. * Specifications at www.trustedcomputinggroup.org * @@ -31,6 +31,7 @@ #include #include #include +#include #include "tpm.h" /* I2C interface offsets */ @@ -52,10 +53,13 @@ #define TPM_I2C_RETRY_DELAY_SHORT 2 /* msec */ #define TPM_I2C_RETRY_DELAY_LONG 10 /* msec */ -#define I2C_DRIVER_NAME "tpm_i2c_nuvoton" +#define OF_IS_TPM2 ((void *)1) +#define I2C_IS_TPM2 1 struct priv_data { + int irq; unsigned int intrs; + wait_queue_head_t read_queue; }; static s32 i2c_nuvoton_read_buf(struct i2c_client *client, u8 offset, u8 size, @@ -96,13 +100,13 @@ static s32 i2c_nuvoton_write_buf(struct i2c_client *client, u8 offset, u8 size, /* read TPM_STS register */ static u8 i2c_nuvoton_read_status(struct tpm_chip *chip) { - struct i2c_client *client = to_i2c_client(chip->pdev); + struct i2c_client *client = to_i2c_client(chip->dev.parent); s32 status; u8 data; status = i2c_nuvoton_read_buf(client, TPM_STS, 1, &data); if (status <= 0) { - dev_err(chip->pdev, "%s() error return %d\n", __func__, + dev_err(&chip->dev, "%s() error return %d\n", __func__, status); data = TPM_STS_ERR_VAL; } @@ -127,13 +131,13 @@ static s32 i2c_nuvoton_write_status(struct i2c_client *client, u8 data) /* write commandReady to TPM_STS register */ static void i2c_nuvoton_ready(struct tpm_chip *chip) { - struct i2c_client *client = to_i2c_client(chip->pdev); + struct i2c_client *client = to_i2c_client(chip->dev.parent); s32 status; /* this causes the current command to be aborted */ status = i2c_nuvoton_write_status(client, TPM_STS_COMMAND_READY); if (status < 0) - dev_err(chip->pdev, + dev_err(&chip->dev, "%s() fail to write TPM_STS.commandReady\n", __func__); } @@ -142,7 +146,7 @@ static void i2c_nuvoton_ready(struct tpm_chip *chip) static int i2c_nuvoton_get_burstcount(struct i2c_client *client, struct tpm_chip *chip) { - unsigned long stop = jiffies + chip->vendor.timeout_d; + unsigned long stop = jiffies + chip->timeout_d; s32 status; int burst_count = -1; u8 data; @@ -163,7 +167,7 @@ static int i2c_nuvoton_get_burstcount(struct i2c_client *client, } /* - * WPCT301/NPCT501 SINT# supports only dataAvail + * WPCT301/NPCT501/NPCT6XX SINT# supports only dataAvail * any call to this function which is not waiting for dataAvail will * set queue to NULL to avoid waiting for interrupt */ @@ -176,12 +180,12 @@ static bool i2c_nuvoton_check_status(struct tpm_chip *chip, u8 mask, u8 value) static int i2c_nuvoton_wait_for_stat(struct tpm_chip *chip, u8 mask, u8 value, u32 timeout, wait_queue_head_t *queue) { - if (chip->vendor.irq && queue) { + if ((chip->flags & TPM_CHIP_FLAG_IRQ) && queue) { s32 rc; - struct priv_data *priv = chip->vendor.priv; + struct priv_data *priv = dev_get_drvdata(&chip->dev); unsigned int cur_intrs = priv->intrs; - enable_irq(chip->vendor.irq); + enable_irq(priv->irq); rc = wait_event_interruptible_timeout(*queue, cur_intrs != priv->intrs, timeout); @@ -212,7 +216,7 @@ static int i2c_nuvoton_wait_for_stat(struct tpm_chip *chip, u8 mask, u8 value, return 0; } while (time_before(jiffies, stop)); } - dev_err(chip->pdev, "%s(%02x, %02x) -> timeout\n", __func__, mask, + dev_err(&chip->dev, "%s(%02x, %02x) -> timeout\n", __func__, mask, value); return -ETIMEDOUT; } @@ -231,16 +235,17 @@ static int i2c_nuvoton_wait_for_data_avail(struct tpm_chip *chip, u32 timeout, static int i2c_nuvoton_recv_data(struct i2c_client *client, struct tpm_chip *chip, u8 *buf, size_t count) { + struct priv_data *priv = dev_get_drvdata(&chip->dev); s32 rc; int burst_count, bytes2read, size = 0; while (size < count && i2c_nuvoton_wait_for_data_avail(chip, - chip->vendor.timeout_c, - &chip->vendor.read_queue) == 0) { + chip->timeout_c, + &priv->read_queue) == 0) { burst_count = i2c_nuvoton_get_burstcount(client, chip); if (burst_count < 0) { - dev_err(chip->pdev, + dev_err(&chip->dev, "%s() fail to read burstCount=%d\n", __func__, burst_count); return -EIO; @@ -249,12 +254,12 @@ static int i2c_nuvoton_recv_data(struct i2c_client *client, rc = i2c_nuvoton_read_buf(client, TPM_DATA_FIFO_R, bytes2read, &buf[size]); if (rc < 0) { - dev_err(chip->pdev, + dev_err(&chip->dev, "%s() fail on i2c_nuvoton_read_buf()=%d\n", __func__, rc); return -EIO; } - dev_dbg(chip->pdev, "%s(%d):", __func__, bytes2read); + dev_dbg(&chip->dev, "%s(%d):", __func__, bytes2read); size += bytes2read; } @@ -264,7 +269,8 @@ static int i2c_nuvoton_recv_data(struct i2c_client *client, /* Read TPM command results */ static int i2c_nuvoton_recv(struct tpm_chip *chip, u8 *buf, size_t count) { - struct device *dev = chip->pdev; + struct priv_data *priv = dev_get_drvdata(&chip->dev); + struct device *dev = chip->dev.parent; struct i2c_client *client = to_i2c_client(dev); s32 rc; int expected, status, burst_count, retries, size = 0; @@ -285,7 +291,7 @@ static int i2c_nuvoton_recv(struct tpm_chip *chip, u8 *buf, size_t count) * tag, paramsize, and result */ status = i2c_nuvoton_wait_for_data_avail( - chip, chip->vendor.timeout_c, &chip->vendor.read_queue); + chip, chip->timeout_c, &priv->read_queue); if (status != 0) { dev_err(dev, "%s() timeout on dataAvail\n", __func__); size = -ETIMEDOUT; @@ -325,7 +331,7 @@ static int i2c_nuvoton_recv(struct tpm_chip *chip, u8 *buf, size_t count) } if (i2c_nuvoton_wait_for_stat( chip, TPM_STS_VALID | TPM_STS_DATA_AVAIL, - TPM_STS_VALID, chip->vendor.timeout_c, + TPM_STS_VALID, chip->timeout_c, NULL)) { dev_err(dev, "%s() error left over data\n", __func__); size = -ETIMEDOUT; @@ -334,7 +340,7 @@ static int i2c_nuvoton_recv(struct tpm_chip *chip, u8 *buf, size_t count) break; } i2c_nuvoton_ready(chip); - dev_dbg(chip->pdev, "%s() -> %d\n", __func__, size); + dev_dbg(&chip->dev, "%s() -> %d\n", __func__, size); return size; } @@ -347,7 +353,8 @@ static int i2c_nuvoton_recv(struct tpm_chip *chip, u8 *buf, size_t count) */ static int i2c_nuvoton_send(struct tpm_chip *chip, u8 *buf, size_t len) { - struct device *dev = chip->pdev; + struct priv_data *priv = dev_get_drvdata(&chip->dev); + struct device *dev = chip->dev.parent; struct i2c_client *client = to_i2c_client(dev); u32 ordinal; size_t count = 0; @@ -357,7 +364,7 @@ static int i2c_nuvoton_send(struct tpm_chip *chip, u8 *buf, size_t len) i2c_nuvoton_ready(chip); if (i2c_nuvoton_wait_for_stat(chip, TPM_STS_COMMAND_READY, TPM_STS_COMMAND_READY, - chip->vendor.timeout_b, NULL)) { + chip->timeout_b, NULL)) { dev_err(dev, "%s() timeout on commandReady\n", __func__); rc = -EIO; @@ -389,7 +396,7 @@ static int i2c_nuvoton_send(struct tpm_chip *chip, u8 *buf, size_t len) TPM_STS_EXPECT, TPM_STS_VALID | TPM_STS_EXPECT, - chip->vendor.timeout_c, + chip->timeout_c, NULL); if (rc < 0) { dev_err(dev, "%s() timeout on Expect\n", @@ -414,7 +421,7 @@ static int i2c_nuvoton_send(struct tpm_chip *chip, u8 *buf, size_t len) rc = i2c_nuvoton_wait_for_stat(chip, TPM_STS_VALID | TPM_STS_EXPECT, TPM_STS_VALID, - chip->vendor.timeout_c, NULL); + chip->timeout_c, NULL); if (rc) { dev_err(dev, "%s() timeout on Expect to clear\n", __func__); @@ -439,7 +446,7 @@ static int i2c_nuvoton_send(struct tpm_chip *chip, u8 *buf, size_t len) rc = i2c_nuvoton_wait_for_data_avail(chip, tpm_calc_ordinal_duration(chip, ordinal), - &chip->vendor.read_queue); + &priv->read_queue); if (rc) { dev_err(dev, "%s() timeout command duration\n", __func__); i2c_nuvoton_ready(chip); @@ -456,6 +463,7 @@ static bool i2c_nuvoton_req_canceled(struct tpm_chip *chip, u8 status) } static const struct tpm_class_ops tpm_i2c = { + .flags = TPM_OPS_AUTO_STARTUP, .status = i2c_nuvoton_read_status, .recv = i2c_nuvoton_recv, .send = i2c_nuvoton_send, @@ -473,11 +481,11 @@ static const struct tpm_class_ops tpm_i2c = { static irqreturn_t i2c_nuvoton_int_handler(int dummy, void *dev_id) { struct tpm_chip *chip = dev_id; - struct priv_data *priv = chip->vendor.priv; + struct priv_data *priv = dev_get_drvdata(&chip->dev); priv->intrs++; - wake_up(&chip->vendor.read_queue); - disable_irq_nosync(chip->vendor.irq); + wake_up(&priv->read_queue); + disable_irq_nosync(priv->irq); return IRQ_HANDLED; } @@ -521,6 +529,7 @@ static int i2c_nuvoton_probe(struct i2c_client *client, int rc; struct tpm_chip *chip; struct device *dev = &client->dev; + struct priv_data *priv; u32 vid = 0; rc = get_vid(client, &vid); @@ -534,46 +543,56 @@ static int i2c_nuvoton_probe(struct i2c_client *client, if (IS_ERR(chip)) return PTR_ERR(chip); - chip->vendor.priv = devm_kzalloc(dev, sizeof(struct priv_data), - GFP_KERNEL); - if (!chip->vendor.priv) + priv = devm_kzalloc(dev, sizeof(struct priv_data), GFP_KERNEL); + if (!priv) return -ENOMEM; - init_waitqueue_head(&chip->vendor.read_queue); - init_waitqueue_head(&chip->vendor.int_queue); + if (dev->of_node) { + const struct of_device_id *of_id; + + of_id = of_match_device(dev->driver->of_match_table, dev); + if (of_id && of_id->data == OF_IS_TPM2) + chip->flags |= TPM_CHIP_FLAG_TPM2; + } else + if (id->driver_data == I2C_IS_TPM2) + chip->flags |= TPM_CHIP_FLAG_TPM2; + + init_waitqueue_head(&priv->read_queue); /* Default timeouts */ - chip->vendor.timeout_a = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); - chip->vendor.timeout_b = msecs_to_jiffies(TPM_I2C_LONG_TIMEOUT); - chip->vendor.timeout_c = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); - chip->vendor.timeout_d = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); + chip->timeout_a = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); + chip->timeout_b = msecs_to_jiffies(TPM_I2C_LONG_TIMEOUT); + chip->timeout_c = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); + chip->timeout_d = msecs_to_jiffies(TPM_I2C_SHORT_TIMEOUT); + + dev_set_drvdata(&chip->dev, priv); /* * I2C intfcaps (interrupt capabilitieis) in the chip are hard coded to: * TPM_INTF_INT_LEVEL_LOW | TPM_INTF_DATA_AVAIL_INT * The IRQ should be set in the i2c_board_info (which is done * automatically in of_i2c_register_devices, for device tree users */ - chip->vendor.irq = client->irq; - - if (chip->vendor.irq) { - dev_dbg(dev, "%s() chip-vendor.irq\n", __func__); - rc = devm_request_irq(dev, chip->vendor.irq, + priv->irq = client->irq; + if (client->irq) { + dev_dbg(dev, "%s() priv->irq\n", __func__); + rc = devm_request_irq(dev, client->irq, i2c_nuvoton_int_handler, IRQF_TRIGGER_LOW, - chip->devname, + dev_name(&chip->dev), chip); if (rc) { dev_err(dev, "%s() Unable to request irq: %d for use\n", - __func__, chip->vendor.irq); - chip->vendor.irq = 0; + __func__, priv->irq); + priv->irq = 0; } else { + chip->flags |= TPM_CHIP_FLAG_IRQ; /* Clear any pending interrupt */ i2c_nuvoton_ready(chip); /* - wait for TPM_STS==0xA0 (stsValid, commandReady) */ rc = i2c_nuvoton_wait_for_stat(chip, TPM_STS_COMMAND_READY, TPM_STS_COMMAND_READY, - chip->vendor.timeout_b, + chip->timeout_b, NULL); if (rc == 0) { /* @@ -601,25 +620,20 @@ static int i2c_nuvoton_probe(struct i2c_client *client, } } - if (tpm_get_timeouts(chip)) - return -ENODEV; - - if (tpm_do_selftest(chip)) - return -ENODEV; - return tpm_chip_register(chip); } static int i2c_nuvoton_remove(struct i2c_client *client) { - struct device *dev = &(client->dev); - struct tpm_chip *chip = dev_get_drvdata(dev); + struct tpm_chip *chip = i2c_get_clientdata(client); + tpm_chip_unregister(chip); return 0; } static const struct i2c_device_id i2c_nuvoton_id[] = { - {I2C_DRIVER_NAME, 0}, + {"tpm_i2c_nuvoton"}, + {"tpm2_i2c_nuvoton", .driver_data = I2C_IS_TPM2}, {} }; MODULE_DEVICE_TABLE(i2c, i2c_nuvoton_id); @@ -628,6 +642,7 @@ MODULE_DEVICE_TABLE(i2c, i2c_nuvoton_id); static const struct of_device_id i2c_nuvoton_of_match[] = { {.compatible = "nuvoton,npct501"}, {.compatible = "winbond,wpct301"}, + {.compatible = "nuvoton,npct601", .data = OF_IS_TPM2}, {}, }; MODULE_DEVICE_TABLE(of, i2c_nuvoton_of_match); @@ -640,7 +655,7 @@ static struct i2c_driver i2c_nuvoton_driver = { .probe = i2c_nuvoton_probe, .remove = i2c_nuvoton_remove, .driver = { - .name = I2C_DRIVER_NAME, + .name = "tpm_i2c_nuvoton", .pm = &i2c_nuvoton_pm_ops, .of_match_table = of_match_ptr(i2c_nuvoton_of_match), }, diff --git a/drivers/char/tpm/tpm_ibmvtpm.c b/drivers/char/tpm/tpm_ibmvtpm.c index b0a9a9e34..946025a74 100644 --- a/drivers/char/tpm/tpm_ibmvtpm.c +++ b/drivers/char/tpm/tpm_ibmvtpm.c @@ -53,21 +53,6 @@ static int ibmvtpm_send_crq(struct vio_dev *vdev, u64 w1, u64 w2) return plpar_hcall_norets(H_SEND_CRQ, vdev->unit_address, w1, w2); } -/** - * ibmvtpm_get_data - Retrieve ibm vtpm data - * @dev: device struct - * - * Return value: - * vtpm device struct - */ -static struct ibmvtpm_dev *ibmvtpm_get_data(const struct device *dev) -{ - struct tpm_chip *chip = dev_get_drvdata(dev); - if (chip) - return (struct ibmvtpm_dev *)TPM_VPRIV(chip); - return NULL; -} - /** * tpm_ibmvtpm_recv - Receive data after send * @chip: tpm chip struct @@ -79,12 +64,10 @@ static struct ibmvtpm_dev *ibmvtpm_get_data(const struct device *dev) */ static int tpm_ibmvtpm_recv(struct tpm_chip *chip, u8 *buf, size_t count) { - struct ibmvtpm_dev *ibmvtpm; + struct ibmvtpm_dev *ibmvtpm = dev_get_drvdata(&chip->dev); u16 len; int sig; - ibmvtpm = (struct ibmvtpm_dev *)TPM_VPRIV(chip); - if (!ibmvtpm->rtce_buf) { dev_err(ibmvtpm->dev, "ibmvtpm device is not ready\n"); return 0; @@ -122,13 +105,11 @@ static int tpm_ibmvtpm_recv(struct tpm_chip *chip, u8 *buf, size_t count) */ static int tpm_ibmvtpm_send(struct tpm_chip *chip, u8 *buf, size_t count) { - struct ibmvtpm_dev *ibmvtpm; + struct ibmvtpm_dev *ibmvtpm = dev_get_drvdata(&chip->dev); struct ibmvtpm_crq crq; __be64 *word = (__be64 *)&crq; int rc, sig; - ibmvtpm = (struct ibmvtpm_dev *)TPM_VPRIV(chip); - if (!ibmvtpm->rtce_buf) { dev_err(ibmvtpm->dev, "ibmvtpm device is not ready\n"); return 0; @@ -289,8 +270,8 @@ static int ibmvtpm_crq_send_init(struct ibmvtpm_dev *ibmvtpm) */ static int tpm_ibmvtpm_remove(struct vio_dev *vdev) { - struct ibmvtpm_dev *ibmvtpm = ibmvtpm_get_data(&vdev->dev); - struct tpm_chip *chip = dev_get_drvdata(ibmvtpm->dev); + struct tpm_chip *chip = dev_get_drvdata(&vdev->dev); + struct ibmvtpm_dev *ibmvtpm = dev_get_drvdata(&chip->dev); int rc = 0; tpm_chip_unregister(chip); @@ -327,7 +308,8 @@ static int tpm_ibmvtpm_remove(struct vio_dev *vdev) */ static unsigned long tpm_ibmvtpm_get_desired_dma(struct vio_dev *vdev) { - struct ibmvtpm_dev *ibmvtpm = ibmvtpm_get_data(&vdev->dev); + struct tpm_chip *chip = dev_get_drvdata(&vdev->dev); + struct ibmvtpm_dev *ibmvtpm = dev_get_drvdata(&chip->dev); /* ibmvtpm initializes at probe time, so the data we are * asking for may not be set yet. Estimate that 4K required @@ -348,7 +330,8 @@ static unsigned long tpm_ibmvtpm_get_desired_dma(struct vio_dev *vdev) */ static int tpm_ibmvtpm_suspend(struct device *dev) { - struct ibmvtpm_dev *ibmvtpm = ibmvtpm_get_data(dev); + struct tpm_chip *chip = dev_get_drvdata(dev); + struct ibmvtpm_dev *ibmvtpm = dev_get_drvdata(&chip->dev); struct ibmvtpm_crq crq; u64 *buf = (u64 *) &crq; int rc = 0; @@ -400,7 +383,8 @@ static int ibmvtpm_reset_crq(struct ibmvtpm_dev *ibmvtpm) */ static int tpm_ibmvtpm_resume(struct device *dev) { - struct ibmvtpm_dev *ibmvtpm = ibmvtpm_get_data(dev); + struct tpm_chip *chip = dev_get_drvdata(dev); + struct ibmvtpm_dev *ibmvtpm = dev_get_drvdata(&chip->dev); int rc = 0; do { @@ -643,7 +627,7 @@ static int tpm_ibmvtpm_probe(struct vio_dev *vio_dev, crq_q->index = 0; - TPM_VPRIV(chip) = (void *)ibmvtpm; + dev_set_drvdata(&chip->dev, ibmvtpm); spin_lock_init(&ibmvtpm->rtce_lock); diff --git a/drivers/char/tpm/tpm_infineon.c b/drivers/char/tpm/tpm_infineon.c index 6c488e635..e3cf9f354 100644 --- a/drivers/char/tpm/tpm_infineon.c +++ b/drivers/char/tpm/tpm_infineon.c @@ -195,9 +195,9 @@ static int wait(struct tpm_chip *chip, int wait_for_bit) } if (i == TPM_MAX_TRIES) { /* timeout occurs */ if (wait_for_bit == STAT_XFE) - dev_err(chip->pdev, "Timeout in wait(STAT_XFE)\n"); + dev_err(&chip->dev, "Timeout in wait(STAT_XFE)\n"); if (wait_for_bit == STAT_RDA) - dev_err(chip->pdev, "Timeout in wait(STAT_RDA)\n"); + dev_err(&chip->dev, "Timeout in wait(STAT_RDA)\n"); return -EIO; } return 0; @@ -220,7 +220,7 @@ static void wait_and_send(struct tpm_chip *chip, u8 sendbyte) static void tpm_wtx(struct tpm_chip *chip) { number_of_wtx++; - dev_info(chip->pdev, "Granting WTX (%02d / %02d)\n", + dev_info(&chip->dev, "Granting WTX (%02d / %02d)\n", number_of_wtx, TPM_MAX_WTX_PACKAGES); wait_and_send(chip, TPM_VL_VER); wait_and_send(chip, TPM_CTRL_WTX); @@ -231,7 +231,7 @@ static void tpm_wtx(struct tpm_chip *chip) static void tpm_wtx_abort(struct tpm_chip *chip) { - dev_info(chip->pdev, "Aborting WTX\n"); + dev_info(&chip->dev, "Aborting WTX\n"); wait_and_send(chip, TPM_VL_VER); wait_and_send(chip, TPM_CTRL_WTX_ABORT); wait_and_send(chip, 0x00); @@ -257,7 +257,7 @@ recv_begin: } if (buf[0] != TPM_VL_VER) { - dev_err(chip->pdev, + dev_err(&chip->dev, "Wrong transport protocol implementation!\n"); return -EIO; } @@ -272,7 +272,7 @@ recv_begin: } if ((size == 0x6D00) && (buf[1] == 0x80)) { - dev_err(chip->pdev, "Error handling on vendor layer!\n"); + dev_err(&chip->dev, "Error handling on vendor layer!\n"); return -EIO; } @@ -284,7 +284,7 @@ recv_begin: } if (buf[1] == TPM_CTRL_WTX) { - dev_info(chip->pdev, "WTX-package received\n"); + dev_info(&chip->dev, "WTX-package received\n"); if (number_of_wtx < TPM_MAX_WTX_PACKAGES) { tpm_wtx(chip); goto recv_begin; @@ -295,14 +295,14 @@ recv_begin: } if (buf[1] == TPM_CTRL_WTX_ABORT_ACK) { - dev_info(chip->pdev, "WTX-abort acknowledged\n"); + dev_info(&chip->dev, "WTX-abort acknowledged\n"); return size; } if (buf[1] == TPM_CTRL_ERROR) { - dev_err(chip->pdev, "ERROR-package received:\n"); + dev_err(&chip->dev, "ERROR-package received:\n"); if (buf[4] == TPM_INF_NAK) - dev_err(chip->pdev, + dev_err(&chip->dev, "-> Negative acknowledgement" " - retransmit command!\n"); return -EIO; @@ -321,7 +321,7 @@ static int tpm_inf_send(struct tpm_chip *chip, u8 * buf, size_t count) ret = empty_fifo(chip, 1); if (ret) { - dev_err(chip->pdev, "Timeout while clearing FIFO\n"); + dev_err(&chip->dev, "Timeout while clearing FIFO\n"); return -EIO; } diff --git a/drivers/char/tpm/tpm_nsc.c b/drivers/char/tpm/tpm_nsc.c index 289389ece..9ff0e072c 100644 --- a/drivers/char/tpm/tpm_nsc.c +++ b/drivers/char/tpm/tpm_nsc.c @@ -64,15 +64,21 @@ enum tpm_nsc_cmd_mode { NSC_COMMAND_EOC = 0x03, NSC_COMMAND_CANCEL = 0x22 }; + +struct tpm_nsc_priv { + unsigned long base; +}; + /* * Wait for a certain status to appear */ static int wait_for_stat(struct tpm_chip *chip, u8 mask, u8 val, u8 * data) { + struct tpm_nsc_priv *priv = dev_get_drvdata(&chip->dev); unsigned long stop; /* status immediately available check */ - *data = inb(chip->vendor.base + NSC_STATUS); + *data = inb(priv->base + NSC_STATUS); if ((*data & mask) == val) return 0; @@ -80,7 +86,7 @@ static int wait_for_stat(struct tpm_chip *chip, u8 mask, u8 val, u8 * data) stop = jiffies + 10 * HZ; do { msleep(TPM_TIMEOUT); - *data = inb(chip->vendor.base + 1); + *data = inb(priv->base + 1); if ((*data & mask) == val) return 0; } @@ -91,13 +97,14 @@ static int wait_for_stat(struct tpm_chip *chip, u8 mask, u8 val, u8 * data) static int nsc_wait_for_ready(struct tpm_chip *chip) { + struct tpm_nsc_priv *priv = dev_get_drvdata(&chip->dev); int status; unsigned long stop; /* status immediately available check */ - status = inb(chip->vendor.base + NSC_STATUS); + status = inb(priv->base + NSC_STATUS); if (status & NSC_STATUS_OBF) - status = inb(chip->vendor.base + NSC_DATA); + status = inb(priv->base + NSC_DATA); if (status & NSC_STATUS_RDY) return 0; @@ -105,21 +112,22 @@ static int nsc_wait_for_ready(struct tpm_chip *chip) stop = jiffies + 100; do { msleep(TPM_TIMEOUT); - status = inb(chip->vendor.base + NSC_STATUS); + status = inb(priv->base + NSC_STATUS); if (status & NSC_STATUS_OBF) - status = inb(chip->vendor.base + NSC_DATA); + status = inb(priv->base + NSC_DATA); if (status & NSC_STATUS_RDY) return 0; } while (time_before(jiffies, stop)); - dev_info(chip->pdev, "wait for ready failed\n"); + dev_info(&chip->dev, "wait for ready failed\n"); return -EBUSY; } static int tpm_nsc_recv(struct tpm_chip *chip, u8 * buf, size_t count) { + struct tpm_nsc_priv *priv = dev_get_drvdata(&chip->dev); u8 *buffer = buf; u8 data, *p; u32 size; @@ -129,12 +137,13 @@ static int tpm_nsc_recv(struct tpm_chip *chip, u8 * buf, size_t count) return -EIO; if (wait_for_stat(chip, NSC_STATUS_F0, NSC_STATUS_F0, &data) < 0) { - dev_err(chip->pdev, "F0 timeout\n"); + dev_err(&chip->dev, "F0 timeout\n"); return -EIO; } - if ((data = - inb(chip->vendor.base + NSC_DATA)) != NSC_COMMAND_NORMAL) { - dev_err(chip->pdev, "not in normal mode (0x%x)\n", + + data = inb(priv->base + NSC_DATA); + if (data != NSC_COMMAND_NORMAL) { + dev_err(&chip->dev, "not in normal mode (0x%x)\n", data); return -EIO; } @@ -143,22 +152,24 @@ static int tpm_nsc_recv(struct tpm_chip *chip, u8 * buf, size_t count) for (p = buffer; p < &buffer[count]; p++) { if (wait_for_stat (chip, NSC_STATUS_OBF, NSC_STATUS_OBF, &data) < 0) { - dev_err(chip->pdev, + dev_err(&chip->dev, "OBF timeout (while reading data)\n"); return -EIO; } if (data & NSC_STATUS_F0) break; - *p = inb(chip->vendor.base + NSC_DATA); + *p = inb(priv->base + NSC_DATA); } if ((data & NSC_STATUS_F0) == 0 && (wait_for_stat(chip, NSC_STATUS_F0, NSC_STATUS_F0, &data) < 0)) { - dev_err(chip->pdev, "F0 not set\n"); + dev_err(&chip->dev, "F0 not set\n"); return -EIO; } - if ((data = inb(chip->vendor.base + NSC_DATA)) != NSC_COMMAND_EOC) { - dev_err(chip->pdev, + + data = inb(priv->base + NSC_DATA); + if (data != NSC_COMMAND_EOC) { + dev_err(&chip->dev, "expected end of command(0x%x)\n", data); return -EIO; } @@ -174,6 +185,7 @@ static int tpm_nsc_recv(struct tpm_chip *chip, u8 * buf, size_t count) static int tpm_nsc_send(struct tpm_chip *chip, u8 * buf, size_t count) { + struct tpm_nsc_priv *priv = dev_get_drvdata(&chip->dev); u8 data; int i; @@ -183,48 +195,52 @@ static int tpm_nsc_send(struct tpm_chip *chip, u8 * buf, size_t count) * fix it. Not sure why this is needed, we followed the flow * chart in the manual to the letter. */ - outb(NSC_COMMAND_CANCEL, chip->vendor.base + NSC_COMMAND); + outb(NSC_COMMAND_CANCEL, priv->base + NSC_COMMAND); if (nsc_wait_for_ready(chip) != 0) return -EIO; if (wait_for_stat(chip, NSC_STATUS_IBF, 0, &data) < 0) { - dev_err(chip->pdev, "IBF timeout\n"); + dev_err(&chip->dev, "IBF timeout\n"); return -EIO; } - outb(NSC_COMMAND_NORMAL, chip->vendor.base + NSC_COMMAND); + outb(NSC_COMMAND_NORMAL, priv->base + NSC_COMMAND); if (wait_for_stat(chip, NSC_STATUS_IBR, NSC_STATUS_IBR, &data) < 0) { - dev_err(chip->pdev, "IBR timeout\n"); + dev_err(&chip->dev, "IBR timeout\n"); return -EIO; } for (i = 0; i < count; i++) { if (wait_for_stat(chip, NSC_STATUS_IBF, 0, &data) < 0) { - dev_err(chip->pdev, + dev_err(&chip->dev, "IBF timeout (while writing data)\n"); return -EIO; } - outb(buf[i], chip->vendor.base + NSC_DATA); + outb(buf[i], priv->base + NSC_DATA); } if (wait_for_stat(chip, NSC_STATUS_IBF, 0, &data) < 0) { - dev_err(chip->pdev, "IBF timeout\n"); + dev_err(&chip->dev, "IBF timeout\n"); return -EIO; } - outb(NSC_COMMAND_EOC, chip->vendor.base + NSC_COMMAND); + outb(NSC_COMMAND_EOC, priv->base + NSC_COMMAND); return count; } static void tpm_nsc_cancel(struct tpm_chip *chip) { - outb(NSC_COMMAND_CANCEL, chip->vendor.base + NSC_COMMAND); + struct tpm_nsc_priv *priv = dev_get_drvdata(&chip->dev); + + outb(NSC_COMMAND_CANCEL, priv->base + NSC_COMMAND); } static u8 tpm_nsc_status(struct tpm_chip *chip) { - return inb(chip->vendor.base + NSC_STATUS); + struct tpm_nsc_priv *priv = dev_get_drvdata(&chip->dev); + + return inb(priv->base + NSC_STATUS); } static bool tpm_nsc_req_canceled(struct tpm_chip *chip, u8 status) @@ -247,9 +263,10 @@ static struct platform_device *pdev = NULL; static void tpm_nsc_remove(struct device *dev) { struct tpm_chip *chip = dev_get_drvdata(dev); + struct tpm_nsc_priv *priv = dev_get_drvdata(&chip->dev); tpm_chip_unregister(chip); - release_region(chip->vendor.base, 2); + release_region(priv->base, 2); } static SIMPLE_DEV_PM_OPS(tpm_nsc_pm, tpm_pm_suspend, tpm_pm_resume); @@ -268,6 +285,7 @@ static int __init init_nsc(void) int nscAddrBase = TPM_ADDR; struct tpm_chip *chip; unsigned long base; + struct tpm_nsc_priv *priv; /* verify that it is a National part (SID) */ if (tpm_read_index(TPM_ADDR, NSC_SID_INDEX) != 0xEF) { @@ -301,6 +319,14 @@ static int __init init_nsc(void) if ((rc = platform_device_add(pdev)) < 0) goto err_put_dev; + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) { + rc = -ENOMEM; + goto err_del_dev; + } + + priv->base = base; + if (request_region(base, 2, "tpm_nsc0") == NULL ) { rc = -EBUSY; goto err_del_dev; @@ -312,6 +338,8 @@ static int __init init_nsc(void) goto err_rel_reg; } + dev_set_drvdata(&chip->dev, priv); + rc = tpm_chip_register(chip); if (rc) goto err_rel_reg; @@ -349,8 +377,6 @@ static int __init init_nsc(void) "NSC TPM revision %d\n", tpm_read_index(nscAddrBase, 0x27) & 0x1F); - chip->vendor.base = base; - return 0; err_rel_reg: diff --git a/drivers/char/tpm/tpm_tis.c b/drivers/char/tpm/tpm_tis.c index a50700672..eaf5730d7 100644 --- a/drivers/char/tpm/tpm_tis.c +++ b/drivers/char/tpm/tpm_tis.c @@ -29,40 +29,7 @@ #include #include #include "tpm.h" - -enum tis_access { - TPM_ACCESS_VALID = 0x80, - TPM_ACCESS_ACTIVE_LOCALITY = 0x20, - TPM_ACCESS_REQUEST_PENDING = 0x04, - TPM_ACCESS_REQUEST_USE = 0x02, -}; - -enum tis_status { - TPM_STS_VALID = 0x80, - TPM_STS_COMMAND_READY = 0x40, - TPM_STS_GO = 0x20, - TPM_STS_DATA_AVAIL = 0x10, - TPM_STS_DATA_EXPECT = 0x08, -}; - -enum tis_int_flags { - TPM_GLOBAL_INT_ENABLE = 0x80000000, - TPM_INTF_BURST_COUNT_STATIC = 0x100, - TPM_INTF_CMD_READY_INT = 0x080, - TPM_INTF_INT_EDGE_FALLING = 0x040, - TPM_INTF_INT_EDGE_RISING = 0x020, - TPM_INTF_INT_LEVEL_LOW = 0x010, - TPM_INTF_INT_LEVEL_HIGH = 0x008, - TPM_INTF_LOCALITY_CHANGE_INT = 0x004, - TPM_INTF_STS_VALID_INT = 0x002, - TPM_INTF_DATA_AVAIL_INT = 0x001, -}; - -enum tis_defaults { - TIS_MEM_LEN = 0x5000, - TIS_SHORT_TIMEOUT = 750, /* ms */ - TIS_LONG_TIMEOUT = 2000, /* 2 sec */ -}; +#include "tpm_tis_core.h" struct tpm_info { struct resource res; @@ -73,30 +40,30 @@ struct tpm_info { int irq; }; -/* Some timeout values are needed before it is known whether the chip is - * TPM 1.0 or TPM 2.0. - */ -#define TIS_TIMEOUT_A_MAX max(TIS_SHORT_TIMEOUT, TPM2_TIMEOUT_A) -#define TIS_TIMEOUT_B_MAX max(TIS_LONG_TIMEOUT, TPM2_TIMEOUT_B) -#define TIS_TIMEOUT_C_MAX max(TIS_SHORT_TIMEOUT, TPM2_TIMEOUT_C) -#define TIS_TIMEOUT_D_MAX max(TIS_SHORT_TIMEOUT, TPM2_TIMEOUT_D) - -#define TPM_ACCESS(l) (0x0000 | ((l) << 12)) -#define TPM_INT_ENABLE(l) (0x0008 | ((l) << 12)) -#define TPM_INT_VECTOR(l) (0x000C | ((l) << 12)) -#define TPM_INT_STATUS(l) (0x0010 | ((l) << 12)) -#define TPM_INTF_CAPS(l) (0x0014 | ((l) << 12)) -#define TPM_STS(l) (0x0018 | ((l) << 12)) -#define TPM_STS3(l) (0x001b | ((l) << 12)) -#define TPM_DATA_FIFO(l) (0x0024 | ((l) << 12)) - -#define TPM_DID_VID(l) (0x0F00 | ((l) << 12)) -#define TPM_RID(l) (0x0F04 | ((l) << 12)) - -struct priv_data { - bool irq_tested; +struct tpm_tis_tcg_phy { + struct tpm_tis_data priv; + void __iomem *iobase; }; +static inline struct tpm_tis_tcg_phy *to_tpm_tis_tcg_phy(struct tpm_tis_data *data) +{ + return container_of(data, struct tpm_tis_tcg_phy, priv); +} + +static bool interrupts = true; +module_param(interrupts, bool, 0444); +MODULE_PARM_DESC(interrupts, "Enable interrupts"); + +static bool itpm; +module_param(itpm, bool, 0444); +MODULE_PARM_DESC(itpm, "Force iTPM workarounds (found on some Lenovo laptops)"); + +static bool force; +#ifdef CONFIG_X86 +module_param(force, bool, 0444); +MODULE_PARM_DESC(force, "Force device probe rather than using ACPI entry"); +#endif + #if defined(CONFIG_PNP) && defined(CONFIG_ACPI) static int has_hid(struct acpi_device *dev, const char *hid) { @@ -120,744 +87,82 @@ static inline int is_itpm(struct acpi_device *dev) } #endif -/* Before we attempt to access the TPM we must see that the valid bit is set. - * The specification says that this bit is 0 at reset and remains 0 until the - * 'TPM has gone through its self test and initialization and has established - * correct values in the other bits.' */ -static int wait_startup(struct tpm_chip *chip, int l) -{ - unsigned long stop = jiffies + chip->vendor.timeout_a; - do { - if (ioread8(chip->vendor.iobase + TPM_ACCESS(l)) & - TPM_ACCESS_VALID) - return 0; - msleep(TPM_TIMEOUT); - } while (time_before(jiffies, stop)); - return -1; -} - -static int check_locality(struct tpm_chip *chip, int l) -{ - if ((ioread8(chip->vendor.iobase + TPM_ACCESS(l)) & - (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) == - (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) - return chip->vendor.locality = l; - - return -1; -} - -static void release_locality(struct tpm_chip *chip, int l, int force) -{ - if (force || (ioread8(chip->vendor.iobase + TPM_ACCESS(l)) & - (TPM_ACCESS_REQUEST_PENDING | TPM_ACCESS_VALID)) == - (TPM_ACCESS_REQUEST_PENDING | TPM_ACCESS_VALID)) - iowrite8(TPM_ACCESS_ACTIVE_LOCALITY, - chip->vendor.iobase + TPM_ACCESS(l)); -} - -static int request_locality(struct tpm_chip *chip, int l) -{ - unsigned long stop, timeout; - long rc; - - if (check_locality(chip, l) >= 0) - return l; - - iowrite8(TPM_ACCESS_REQUEST_USE, - chip->vendor.iobase + TPM_ACCESS(l)); - - stop = jiffies + chip->vendor.timeout_a; - - if (chip->vendor.irq) { -again: - timeout = stop - jiffies; - if ((long)timeout <= 0) - return -1; - rc = wait_event_interruptible_timeout(chip->vendor.int_queue, - (check_locality - (chip, l) >= 0), - timeout); - if (rc > 0) - return l; - if (rc == -ERESTARTSYS && freezing(current)) { - clear_thread_flag(TIF_SIGPENDING); - goto again; - } - } else { - /* wait for burstcount */ - do { - if (check_locality(chip, l) >= 0) - return l; - msleep(TPM_TIMEOUT); - } - while (time_before(jiffies, stop)); - } - return -1; -} - -static u8 tpm_tis_status(struct tpm_chip *chip) -{ - return ioread8(chip->vendor.iobase + - TPM_STS(chip->vendor.locality)); -} - -static void tpm_tis_ready(struct tpm_chip *chip) -{ - /* this causes the current command to be aborted */ - iowrite8(TPM_STS_COMMAND_READY, - chip->vendor.iobase + TPM_STS(chip->vendor.locality)); -} - -static int get_burstcount(struct tpm_chip *chip) -{ - unsigned long stop; - int burstcnt; - - /* wait for burstcount */ - /* which timeout value, spec has 2 answers (c & d) */ - stop = jiffies + chip->vendor.timeout_d; - do { - burstcnt = ioread8(chip->vendor.iobase + - TPM_STS(chip->vendor.locality) + 1); - burstcnt += ioread8(chip->vendor.iobase + - TPM_STS(chip->vendor.locality) + - 2) << 8; - if (burstcnt) - return burstcnt; - msleep(TPM_TIMEOUT); - } while (time_before(jiffies, stop)); - return -EBUSY; -} - -static int recv_data(struct tpm_chip *chip, u8 *buf, size_t count) +static int tpm_tcg_read_bytes(struct tpm_tis_data *data, u32 addr, u16 len, + u8 *result) { - int size = 0, burstcnt; - while (size < count && - wait_for_tpm_stat(chip, - TPM_STS_DATA_AVAIL | TPM_STS_VALID, - chip->vendor.timeout_c, - &chip->vendor.read_queue, true) - == 0) { - burstcnt = get_burstcount(chip); - for (; burstcnt > 0 && size < count; burstcnt--) - buf[size++] = ioread8(chip->vendor.iobase + - TPM_DATA_FIFO(chip->vendor. - locality)); - } - return size; -} - -static int tpm_tis_recv(struct tpm_chip *chip, u8 *buf, size_t count) -{ - int size = 0; - int expected, status; - - if (count < TPM_HEADER_SIZE) { - size = -EIO; - goto out; - } - - /* read first 10 bytes, including tag, paramsize, and result */ - if ((size = - recv_data(chip, buf, TPM_HEADER_SIZE)) < TPM_HEADER_SIZE) { - dev_err(chip->pdev, "Unable to read header\n"); - goto out; - } - - expected = be32_to_cpu(*(__be32 *) (buf + 2)); - if (expected > count) { - size = -EIO; - goto out; - } - - if ((size += - recv_data(chip, &buf[TPM_HEADER_SIZE], - expected - TPM_HEADER_SIZE)) < expected) { - dev_err(chip->pdev, "Unable to read remainder of result\n"); - size = -ETIME; - goto out; - } - - wait_for_tpm_stat(chip, TPM_STS_VALID, chip->vendor.timeout_c, - &chip->vendor.int_queue, false); - status = tpm_tis_status(chip); - if (status & TPM_STS_DATA_AVAIL) { /* retry? */ - dev_err(chip->pdev, "Error left over data\n"); - size = -EIO; - goto out; - } - -out: - tpm_tis_ready(chip); - release_locality(chip, chip->vendor.locality, 0); - return size; -} - -static bool itpm; -module_param(itpm, bool, 0444); -MODULE_PARM_DESC(itpm, "Force iTPM workarounds (found on some Lenovo laptops)"); - -/* - * If interrupts are used (signaled by an irq set in the vendor structure) - * tpm.c can skip polling for the data to be available as the interrupt is - * waited for here - */ -static int tpm_tis_send_data(struct tpm_chip *chip, u8 *buf, size_t len) -{ - int rc, status, burstcnt; - size_t count = 0; - - if (request_locality(chip, 0) < 0) - return -EBUSY; - - status = tpm_tis_status(chip); - if ((status & TPM_STS_COMMAND_READY) == 0) { - tpm_tis_ready(chip); - if (wait_for_tpm_stat - (chip, TPM_STS_COMMAND_READY, chip->vendor.timeout_b, - &chip->vendor.int_queue, false) < 0) { - rc = -ETIME; - goto out_err; - } - } - - while (count < len - 1) { - burstcnt = get_burstcount(chip); - for (; burstcnt > 0 && count < len - 1; burstcnt--) { - iowrite8(buf[count], chip->vendor.iobase + - TPM_DATA_FIFO(chip->vendor.locality)); - count++; - } - - wait_for_tpm_stat(chip, TPM_STS_VALID, chip->vendor.timeout_c, - &chip->vendor.int_queue, false); - status = tpm_tis_status(chip); - if (!itpm && (status & TPM_STS_DATA_EXPECT) == 0) { - rc = -EIO; - goto out_err; - } - } - - /* write last byte */ - iowrite8(buf[count], - chip->vendor.iobase + TPM_DATA_FIFO(chip->vendor.locality)); - wait_for_tpm_stat(chip, TPM_STS_VALID, chip->vendor.timeout_c, - &chip->vendor.int_queue, false); - status = tpm_tis_status(chip); - if ((status & TPM_STS_DATA_EXPECT) != 0) { - rc = -EIO; - goto out_err; - } + struct tpm_tis_tcg_phy *phy = to_tpm_tis_tcg_phy(data); + while (len--) + *result++ = ioread8(phy->iobase + addr); return 0; - -out_err: - tpm_tis_ready(chip); - release_locality(chip, chip->vendor.locality, 0); - return rc; -} - -static void disable_interrupts(struct tpm_chip *chip) -{ - u32 intmask; - - intmask = - ioread32(chip->vendor.iobase + - TPM_INT_ENABLE(chip->vendor.locality)); - intmask &= ~TPM_GLOBAL_INT_ENABLE; - iowrite32(intmask, - chip->vendor.iobase + - TPM_INT_ENABLE(chip->vendor.locality)); - devm_free_irq(chip->pdev, chip->vendor.irq, chip); - chip->vendor.irq = 0; -} - -/* - * If interrupts are used (signaled by an irq set in the vendor structure) - * tpm.c can skip polling for the data to be available as the interrupt is - * waited for here - */ -static int tpm_tis_send_main(struct tpm_chip *chip, u8 *buf, size_t len) -{ - int rc; - u32 ordinal; - unsigned long dur; - - rc = tpm_tis_send_data(chip, buf, len); - if (rc < 0) - return rc; - - /* go and do it */ - iowrite8(TPM_STS_GO, - chip->vendor.iobase + TPM_STS(chip->vendor.locality)); - - if (chip->vendor.irq) { - ordinal = be32_to_cpu(*((__be32 *) (buf + 6))); - - if (chip->flags & TPM_CHIP_FLAG_TPM2) - dur = tpm2_calc_ordinal_duration(chip, ordinal); - else - dur = tpm_calc_ordinal_duration(chip, ordinal); - - if (wait_for_tpm_stat - (chip, TPM_STS_DATA_AVAIL | TPM_STS_VALID, dur, - &chip->vendor.read_queue, false) < 0) { - rc = -ETIME; - goto out_err; - } - } - return len; -out_err: - tpm_tis_ready(chip); - release_locality(chip, chip->vendor.locality, 0); - return rc; -} - -static int tpm_tis_send(struct tpm_chip *chip, u8 *buf, size_t len) -{ - int rc, irq; - struct priv_data *priv = chip->vendor.priv; - - if (!chip->vendor.irq || priv->irq_tested) - return tpm_tis_send_main(chip, buf, len); - - /* Verify receipt of the expected IRQ */ - irq = chip->vendor.irq; - chip->vendor.irq = 0; - rc = tpm_tis_send_main(chip, buf, len); - chip->vendor.irq = irq; - if (!priv->irq_tested) - msleep(1); - if (!priv->irq_tested) - disable_interrupts(chip); - priv->irq_tested = true; - return rc; } -struct tis_vendor_timeout_override { - u32 did_vid; - unsigned long timeout_us[4]; -}; - -static const struct tis_vendor_timeout_override vendor_timeout_overrides[] = { - /* Atmel 3204 */ - { 0x32041114, { (TIS_SHORT_TIMEOUT*1000), (TIS_LONG_TIMEOUT*1000), - (TIS_SHORT_TIMEOUT*1000), (TIS_SHORT_TIMEOUT*1000) } }, -}; - -static bool tpm_tis_update_timeouts(struct tpm_chip *chip, - unsigned long *timeout_cap) +static int tpm_tcg_write_bytes(struct tpm_tis_data *data, u32 addr, u16 len, + u8 *value) { - int i; - u32 did_vid; + struct tpm_tis_tcg_phy *phy = to_tpm_tis_tcg_phy(data); - did_vid = ioread32(chip->vendor.iobase + TPM_DID_VID(0)); - - for (i = 0; i != ARRAY_SIZE(vendor_timeout_overrides); i++) { - if (vendor_timeout_overrides[i].did_vid != did_vid) - continue; - memcpy(timeout_cap, vendor_timeout_overrides[i].timeout_us, - sizeof(vendor_timeout_overrides[i].timeout_us)); - return true; - } - - return false; + while (len--) + iowrite8(*value++, phy->iobase + addr); + return 0; } -/* - * Early probing for iTPM with STS_DATA_EXPECT flaw. - * Try sending command without itpm flag set and if that - * fails, repeat with itpm flag set. - */ -static int probe_itpm(struct tpm_chip *chip) +static int tpm_tcg_read16(struct tpm_tis_data *data, u32 addr, u16 *result) { - int rc = 0; - u8 cmd_getticks[] = { - 0x00, 0xc1, 0x00, 0x00, 0x00, 0x0a, - 0x00, 0x00, 0x00, 0xf1 - }; - size_t len = sizeof(cmd_getticks); - bool rem_itpm = itpm; - u16 vendor = ioread16(chip->vendor.iobase + TPM_DID_VID(0)); - - /* probe only iTPMS */ - if (vendor != TPM_VID_INTEL) - return 0; - - itpm = false; - - rc = tpm_tis_send_data(chip, cmd_getticks, len); - if (rc == 0) - goto out; - - tpm_tis_ready(chip); - release_locality(chip, chip->vendor.locality, 0); + struct tpm_tis_tcg_phy *phy = to_tpm_tis_tcg_phy(data); - itpm = true; - - rc = tpm_tis_send_data(chip, cmd_getticks, len); - if (rc == 0) { - dev_info(chip->pdev, "Detected an iTPM.\n"); - rc = 1; - } else - rc = -EFAULT; - -out: - itpm = rem_itpm; - tpm_tis_ready(chip); - release_locality(chip, chip->vendor.locality, 0); - - return rc; + *result = ioread16(phy->iobase + addr); + return 0; } -static bool tpm_tis_req_canceled(struct tpm_chip *chip, u8 status) +static int tpm_tcg_read32(struct tpm_tis_data *data, u32 addr, u32 *result) { - switch (chip->vendor.manufacturer_id) { - case TPM_VID_WINBOND: - return ((status == TPM_STS_VALID) || - (status == (TPM_STS_VALID | TPM_STS_COMMAND_READY))); - case TPM_VID_STM: - return (status == (TPM_STS_VALID | TPM_STS_COMMAND_READY)); - default: - return (status == TPM_STS_COMMAND_READY); - } -} - -static const struct tpm_class_ops tpm_tis = { - .status = tpm_tis_status, - .recv = tpm_tis_recv, - .send = tpm_tis_send, - .cancel = tpm_tis_ready, - .update_timeouts = tpm_tis_update_timeouts, - .req_complete_mask = TPM_STS_DATA_AVAIL | TPM_STS_VALID, - .req_complete_val = TPM_STS_DATA_AVAIL | TPM_STS_VALID, - .req_canceled = tpm_tis_req_canceled, -}; + struct tpm_tis_tcg_phy *phy = to_tpm_tis_tcg_phy(data); -static irqreturn_t tis_int_handler(int dummy, void *dev_id) -{ - struct tpm_chip *chip = dev_id; - u32 interrupt; - int i; - - interrupt = ioread32(chip->vendor.iobase + - TPM_INT_STATUS(chip->vendor.locality)); - - if (interrupt == 0) - return IRQ_NONE; - - ((struct priv_data *)chip->vendor.priv)->irq_tested = true; - if (interrupt & TPM_INTF_DATA_AVAIL_INT) - wake_up_interruptible(&chip->vendor.read_queue); - if (interrupt & TPM_INTF_LOCALITY_CHANGE_INT) - for (i = 0; i < 5; i++) - if (check_locality(chip, i) >= 0) - break; - if (interrupt & - (TPM_INTF_LOCALITY_CHANGE_INT | TPM_INTF_STS_VALID_INT | - TPM_INTF_CMD_READY_INT)) - wake_up_interruptible(&chip->vendor.int_queue); - - /* Clear interrupts handled with TPM_EOI */ - iowrite32(interrupt, - chip->vendor.iobase + - TPM_INT_STATUS(chip->vendor.locality)); - ioread32(chip->vendor.iobase + TPM_INT_STATUS(chip->vendor.locality)); - return IRQ_HANDLED; + *result = ioread32(phy->iobase + addr); + return 0; } -/* Register the IRQ and issue a command that will cause an interrupt. If an - * irq is seen then leave the chip setup for IRQ operation, otherwise reverse - * everything and leave in polling mode. Returns 0 on success. - */ -static int tpm_tis_probe_irq_single(struct tpm_chip *chip, u32 intmask, - int flags, int irq) +static int tpm_tcg_write32(struct tpm_tis_data *data, u32 addr, u32 value) { - struct priv_data *priv = chip->vendor.priv; - u8 original_int_vec; - - if (devm_request_irq(chip->pdev, irq, tis_int_handler, flags, - chip->devname, chip) != 0) { - dev_info(chip->pdev, "Unable to request irq: %d for probe\n", - irq); - return -1; - } - chip->vendor.irq = irq; - - original_int_vec = ioread8(chip->vendor.iobase + - TPM_INT_VECTOR(chip->vendor.locality)); - iowrite8(irq, - chip->vendor.iobase + TPM_INT_VECTOR(chip->vendor.locality)); - - /* Clear all existing */ - iowrite32(ioread32(chip->vendor.iobase + - TPM_INT_STATUS(chip->vendor.locality)), - chip->vendor.iobase + TPM_INT_STATUS(chip->vendor.locality)); - - /* Turn on */ - iowrite32(intmask | TPM_GLOBAL_INT_ENABLE, - chip->vendor.iobase + TPM_INT_ENABLE(chip->vendor.locality)); - - priv->irq_tested = false; - - /* Generate an interrupt by having the core call through to - * tpm_tis_send - */ - if (chip->flags & TPM_CHIP_FLAG_TPM2) - tpm2_gen_interrupt(chip); - else - tpm_gen_interrupt(chip); - - /* tpm_tis_send will either confirm the interrupt is working or it - * will call disable_irq which undoes all of the above. - */ - if (!chip->vendor.irq) { - iowrite8(original_int_vec, - chip->vendor.iobase + - TPM_INT_VECTOR(chip->vendor.locality)); - return 1; - } + struct tpm_tis_tcg_phy *phy = to_tpm_tis_tcg_phy(data); + iowrite32(value, phy->iobase + addr); return 0; } -/* Try to find the IRQ the TPM is using. This is for legacy x86 systems that - * do not have ACPI/etc. We typically expect the interrupt to be declared if - * present. - */ -static void tpm_tis_probe_irq(struct tpm_chip *chip, u32 intmask) -{ - u8 original_int_vec; - int i; - - original_int_vec = ioread8(chip->vendor.iobase + - TPM_INT_VECTOR(chip->vendor.locality)); - - if (!original_int_vec) { - if (IS_ENABLED(CONFIG_X86)) - for (i = 3; i <= 15; i++) - if (!tpm_tis_probe_irq_single(chip, intmask, 0, - i)) - return; - } else if (!tpm_tis_probe_irq_single(chip, intmask, 0, - original_int_vec)) - return; -} - -static bool interrupts = true; -module_param(interrupts, bool, 0444); -MODULE_PARM_DESC(interrupts, "Enable interrupts"); - -static void tpm_tis_remove(struct tpm_chip *chip) -{ - if (chip->flags & TPM_CHIP_FLAG_TPM2) - tpm2_shutdown(chip, TPM2_SU_CLEAR); - - iowrite32(~TPM_GLOBAL_INT_ENABLE & - ioread32(chip->vendor.iobase + - TPM_INT_ENABLE(chip->vendor. - locality)), - chip->vendor.iobase + - TPM_INT_ENABLE(chip->vendor.locality)); - release_locality(chip, chip->vendor.locality, 1); -} +static const struct tpm_tis_phy_ops tpm_tcg = { + .read_bytes = tpm_tcg_read_bytes, + .write_bytes = tpm_tcg_write_bytes, + .read16 = tpm_tcg_read16, + .read32 = tpm_tcg_read32, + .write32 = tpm_tcg_write32, +}; static int tpm_tis_init(struct device *dev, struct tpm_info *tpm_info, acpi_handle acpi_dev_handle) { - u32 vendor, intfcaps, intmask; - int rc, probe; - struct tpm_chip *chip; - struct priv_data *priv; + struct tpm_tis_tcg_phy *phy; + int irq = -1; - priv = devm_kzalloc(dev, sizeof(struct priv_data), GFP_KERNEL); - if (priv == NULL) + phy = devm_kzalloc(dev, sizeof(struct tpm_tis_tcg_phy), GFP_KERNEL); + if (phy == NULL) return -ENOMEM; - chip = tpmm_chip_alloc(dev, &tpm_tis); - if (IS_ERR(chip)) - return PTR_ERR(chip); - - chip->vendor.priv = priv; -#ifdef CONFIG_ACPI - chip->acpi_dev_handle = acpi_dev_handle; -#endif + phy->iobase = devm_ioremap_resource(dev, &tpm_info->res); + if (IS_ERR(phy->iobase)) + return PTR_ERR(phy->iobase); - chip->vendor.iobase = devm_ioremap_resource(dev, &tpm_info->res); - if (IS_ERR(chip->vendor.iobase)) - return PTR_ERR(chip->vendor.iobase); - - /* Maximum timeouts */ - chip->vendor.timeout_a = TIS_TIMEOUT_A_MAX; - chip->vendor.timeout_b = TIS_TIMEOUT_B_MAX; - chip->vendor.timeout_c = TIS_TIMEOUT_C_MAX; - chip->vendor.timeout_d = TIS_TIMEOUT_D_MAX; - - if (wait_startup(chip, 0) != 0) { - rc = -ENODEV; - goto out_err; - } - - /* Take control of the TPM's interrupt hardware and shut it off */ - intmask = ioread32(chip->vendor.iobase + - TPM_INT_ENABLE(chip->vendor.locality)); - intmask |= TPM_INTF_CMD_READY_INT | TPM_INTF_LOCALITY_CHANGE_INT | - TPM_INTF_DATA_AVAIL_INT | TPM_INTF_STS_VALID_INT; - intmask &= ~TPM_GLOBAL_INT_ENABLE; - iowrite32(intmask, - chip->vendor.iobase + TPM_INT_ENABLE(chip->vendor.locality)); - - if (request_locality(chip, 0) != 0) { - rc = -ENODEV; - goto out_err; - } - - rc = tpm2_probe(chip); - if (rc) - goto out_err; - - vendor = ioread32(chip->vendor.iobase + TPM_DID_VID(0)); - chip->vendor.manufacturer_id = vendor; - - dev_info(dev, "%s TPM (device-id 0x%X, rev-id %d)\n", - (chip->flags & TPM_CHIP_FLAG_TPM2) ? "2.0" : "1.2", - vendor >> 16, ioread8(chip->vendor.iobase + TPM_RID(0))); - - if (!itpm) { - probe = probe_itpm(chip); - if (probe < 0) { - rc = -ENODEV; - goto out_err; - } - itpm = !!probe; - } + if (interrupts) + irq = tpm_info->irq; if (itpm) - dev_info(dev, "Intel iTPM workaround enabled\n"); - - - /* Figure out the capabilities */ - intfcaps = - ioread32(chip->vendor.iobase + - TPM_INTF_CAPS(chip->vendor.locality)); - dev_dbg(dev, "TPM interface capabilities (0x%x):\n", - intfcaps); - if (intfcaps & TPM_INTF_BURST_COUNT_STATIC) - dev_dbg(dev, "\tBurst Count Static\n"); - if (intfcaps & TPM_INTF_CMD_READY_INT) - dev_dbg(dev, "\tCommand Ready Int Support\n"); - if (intfcaps & TPM_INTF_INT_EDGE_FALLING) - dev_dbg(dev, "\tInterrupt Edge Falling\n"); - if (intfcaps & TPM_INTF_INT_EDGE_RISING) - dev_dbg(dev, "\tInterrupt Edge Rising\n"); - if (intfcaps & TPM_INTF_INT_LEVEL_LOW) - dev_dbg(dev, "\tInterrupt Level Low\n"); - if (intfcaps & TPM_INTF_INT_LEVEL_HIGH) - dev_dbg(dev, "\tInterrupt Level High\n"); - if (intfcaps & TPM_INTF_LOCALITY_CHANGE_INT) - dev_dbg(dev, "\tLocality Change Int Support\n"); - if (intfcaps & TPM_INTF_STS_VALID_INT) - dev_dbg(dev, "\tSts Valid Int Support\n"); - if (intfcaps & TPM_INTF_DATA_AVAIL_INT) - dev_dbg(dev, "\tData Avail Int Support\n"); - - /* Very early on issue a command to the TPM in polling mode to make - * sure it works. May as well use that command to set the proper - * timeouts for the driver. - */ - if (tpm_get_timeouts(chip)) { - dev_err(dev, "Could not get TPM timeouts and durations\n"); - rc = -ENODEV; - goto out_err; - } - - /* INTERRUPT Setup */ - init_waitqueue_head(&chip->vendor.read_queue); - init_waitqueue_head(&chip->vendor.int_queue); - if (interrupts && tpm_info->irq != -1) { - if (tpm_info->irq) { - tpm_tis_probe_irq_single(chip, intmask, IRQF_SHARED, - tpm_info->irq); - if (!chip->vendor.irq) - dev_err(chip->pdev, FW_BUG - "TPM interrupt not working, polling instead\n"); - } else - tpm_tis_probe_irq(chip, intmask); - } + phy->priv.flags |= TPM_TIS_ITPM_POSSIBLE; - if (chip->flags & TPM_CHIP_FLAG_TPM2) { - rc = tpm2_do_selftest(chip); - if (rc == TPM2_RC_INITIALIZE) { - dev_warn(dev, "Firmware has not started TPM\n"); - rc = tpm2_startup(chip, TPM2_SU_CLEAR); - if (!rc) - rc = tpm2_do_selftest(chip); - } - - if (rc) { - dev_err(dev, "TPM self test failed\n"); - if (rc > 0) - rc = -ENODEV; - goto out_err; - } - } else { - if (tpm_do_selftest(chip)) { - dev_err(dev, "TPM self test failed\n"); - rc = -ENODEV; - goto out_err; - } - } - - return tpm_chip_register(chip); -out_err: - tpm_tis_remove(chip); - return rc; + return tpm_tis_core_init(dev, &phy->priv, irq, &tpm_tcg, + acpi_dev_handle); } -#ifdef CONFIG_PM_SLEEP -static void tpm_tis_reenable_interrupts(struct tpm_chip *chip) -{ - u32 intmask; - - /* reenable interrupts that device may have lost or - BIOS/firmware may have disabled */ - iowrite8(chip->vendor.irq, chip->vendor.iobase + - TPM_INT_VECTOR(chip->vendor.locality)); - - intmask = - ioread32(chip->vendor.iobase + - TPM_INT_ENABLE(chip->vendor.locality)); - - intmask |= TPM_INTF_CMD_READY_INT - | TPM_INTF_LOCALITY_CHANGE_INT | TPM_INTF_DATA_AVAIL_INT - | TPM_INTF_STS_VALID_INT | TPM_GLOBAL_INT_ENABLE; - - iowrite32(intmask, - chip->vendor.iobase + TPM_INT_ENABLE(chip->vendor.locality)); -} - -static int tpm_tis_resume(struct device *dev) -{ - struct tpm_chip *chip = dev_get_drvdata(dev); - int ret; - - if (chip->vendor.irq) - tpm_tis_reenable_interrupts(chip); - - ret = tpm_pm_resume(dev); - if (ret) - return ret; - - /* TPM 1.2 requires self-test on resume. This function actually returns - * an error code but for unknown reason it isn't handled. - */ - if (!(chip->flags & TPM_CHIP_FLAG_TPM2)) - tpm_do_selftest(chip); - - return 0; -} -#endif - static SIMPLE_DEV_PM_OPS(tpm_tis_pm, tpm_pm_suspend, tpm_tis_resume); static int tpm_tis_pnp_init(struct pnp_dev *pnp_dev, @@ -1058,12 +363,6 @@ static struct platform_driver tis_drv = { }, }; -static bool force; -#ifdef CONFIG_X86 -module_param(force, bool, 0444); -MODULE_PARM_DESC(force, "Force device probe rather than using ACPI entry"); -#endif - static int tpm_tis_force_device(void) { struct platform_device *pdev; diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c new file mode 100644 index 000000000..d66f51b36 --- /dev/null +++ b/drivers/char/tpm/tpm_tis_core.c @@ -0,0 +1,835 @@ +/* + * Copyright (C) 2005, 2006 IBM Corporation + * Copyright (C) 2014, 2015 Intel Corporation + * + * Authors: + * Leendert van Doorn + * Kylene Hall + * + * Maintained by: + * + * Device driver for TCG/TCPA TPM (trusted platform module). + * Specifications at www.trustedcomputinggroup.org + * + * This device driver implements the TPM interface as defined in + * the TCG TPM Interface Spec version 1.2, revision 1.0. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, version 2 of the + * License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "tpm.h" +#include "tpm_tis_core.h" + +/* Before we attempt to access the TPM we must see that the valid bit is set. + * The specification says that this bit is 0 at reset and remains 0 until the + * 'TPM has gone through its self test and initialization and has established + * correct values in the other bits.' + */ +static int wait_startup(struct tpm_chip *chip, int l) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + unsigned long stop = jiffies + chip->timeout_a; + + do { + int rc; + u8 access; + + rc = tpm_tis_read8(priv, TPM_ACCESS(l), &access); + if (rc < 0) + return rc; + + if (access & TPM_ACCESS_VALID) + return 0; + msleep(TPM_TIMEOUT); + } while (time_before(jiffies, stop)); + return -1; +} + +static int check_locality(struct tpm_chip *chip, int l) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + int rc; + u8 access; + + rc = tpm_tis_read8(priv, TPM_ACCESS(l), &access); + if (rc < 0) + return rc; + + if ((access & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) == + (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) + return priv->locality = l; + + return -1; +} + +static void release_locality(struct tpm_chip *chip, int l, int force) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + int rc; + u8 access; + + rc = tpm_tis_read8(priv, TPM_ACCESS(l), &access); + if (rc < 0) + return; + + if (force || (access & + (TPM_ACCESS_REQUEST_PENDING | TPM_ACCESS_VALID)) == + (TPM_ACCESS_REQUEST_PENDING | TPM_ACCESS_VALID)) + tpm_tis_write8(priv, TPM_ACCESS(l), TPM_ACCESS_ACTIVE_LOCALITY); + +} + +static int request_locality(struct tpm_chip *chip, int l) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + unsigned long stop, timeout; + long rc; + + if (check_locality(chip, l) >= 0) + return l; + + rc = tpm_tis_write8(priv, TPM_ACCESS(l), TPM_ACCESS_REQUEST_USE); + if (rc < 0) + return rc; + + stop = jiffies + chip->timeout_a; + + if (chip->flags & TPM_CHIP_FLAG_IRQ) { +again: + timeout = stop - jiffies; + if ((long)timeout <= 0) + return -1; + rc = wait_event_interruptible_timeout(priv->int_queue, + (check_locality + (chip, l) >= 0), + timeout); + if (rc > 0) + return l; + if (rc == -ERESTARTSYS && freezing(current)) { + clear_thread_flag(TIF_SIGPENDING); + goto again; + } + } else { + /* wait for burstcount */ + do { + if (check_locality(chip, l) >= 0) + return l; + msleep(TPM_TIMEOUT); + } while (time_before(jiffies, stop)); + } + return -1; +} + +static u8 tpm_tis_status(struct tpm_chip *chip) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + int rc; + u8 status; + + rc = tpm_tis_read8(priv, TPM_STS(priv->locality), &status); + if (rc < 0) + return 0; + + return status; +} + +static void tpm_tis_ready(struct tpm_chip *chip) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + + /* this causes the current command to be aborted */ + tpm_tis_write8(priv, TPM_STS(priv->locality), TPM_STS_COMMAND_READY); +} + +static int get_burstcount(struct tpm_chip *chip) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + unsigned long stop; + int burstcnt, rc; + u32 value; + + /* wait for burstcount */ + /* which timeout value, spec has 2 answers (c & d) */ + stop = jiffies + chip->timeout_d; + do { + rc = tpm_tis_read32(priv, TPM_STS(priv->locality), &value); + if (rc < 0) + return rc; + + burstcnt = (value >> 8) & 0xFFFF; + if (burstcnt) + return burstcnt; + msleep(TPM_TIMEOUT); + } while (time_before(jiffies, stop)); + return -EBUSY; +} + +static int recv_data(struct tpm_chip *chip, u8 *buf, size_t count) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + int size = 0, burstcnt, rc; + + while (size < count && + wait_for_tpm_stat(chip, + TPM_STS_DATA_AVAIL | TPM_STS_VALID, + chip->timeout_c, + &priv->read_queue, true) == 0) { + burstcnt = min_t(int, get_burstcount(chip), count - size); + + rc = tpm_tis_read_bytes(priv, TPM_DATA_FIFO(priv->locality), + burstcnt, buf + size); + if (rc < 0) + return rc; + + size += burstcnt; + } + return size; +} + +static int tpm_tis_recv(struct tpm_chip *chip, u8 *buf, size_t count) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + int size = 0; + int expected, status; + + if (count < TPM_HEADER_SIZE) { + size = -EIO; + goto out; + } + + size = recv_data(chip, buf, TPM_HEADER_SIZE); + /* read first 10 bytes, including tag, paramsize, and result */ + if (size < TPM_HEADER_SIZE) { + dev_err(&chip->dev, "Unable to read header\n"); + goto out; + } + + expected = be32_to_cpu(*(__be32 *) (buf + 2)); + if (expected > count) { + size = -EIO; + goto out; + } + + size += recv_data(chip, &buf[TPM_HEADER_SIZE], + expected - TPM_HEADER_SIZE); + if (size < expected) { + dev_err(&chip->dev, "Unable to read remainder of result\n"); + size = -ETIME; + goto out; + } + + wait_for_tpm_stat(chip, TPM_STS_VALID, chip->timeout_c, + &priv->int_queue, false); + status = tpm_tis_status(chip); + if (status & TPM_STS_DATA_AVAIL) { /* retry? */ + dev_err(&chip->dev, "Error left over data\n"); + size = -EIO; + goto out; + } + +out: + tpm_tis_ready(chip); + release_locality(chip, priv->locality, 0); + return size; +} + +/* + * If interrupts are used (signaled by an irq set in the vendor structure) + * tpm.c can skip polling for the data to be available as the interrupt is + * waited for here + */ +static int tpm_tis_send_data(struct tpm_chip *chip, u8 *buf, size_t len) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + int rc, status, burstcnt; + size_t count = 0; + bool itpm = priv->flags & TPM_TIS_ITPM_POSSIBLE; + + if (request_locality(chip, 0) < 0) + return -EBUSY; + + status = tpm_tis_status(chip); + if ((status & TPM_STS_COMMAND_READY) == 0) { + tpm_tis_ready(chip); + if (wait_for_tpm_stat + (chip, TPM_STS_COMMAND_READY, chip->timeout_b, + &priv->int_queue, false) < 0) { + rc = -ETIME; + goto out_err; + } + } + + while (count < len - 1) { + burstcnt = min_t(int, get_burstcount(chip), len - count - 1); + rc = tpm_tis_write_bytes(priv, TPM_DATA_FIFO(priv->locality), + burstcnt, buf + count); + if (rc < 0) + goto out_err; + + count += burstcnt; + + wait_for_tpm_stat(chip, TPM_STS_VALID, chip->timeout_c, + &priv->int_queue, false); + status = tpm_tis_status(chip); + if (!itpm && (status & TPM_STS_DATA_EXPECT) == 0) { + rc = -EIO; + goto out_err; + } + } + + /* write last byte */ + rc = tpm_tis_write8(priv, TPM_DATA_FIFO(priv->locality), buf[count]); + if (rc < 0) + goto out_err; + + wait_for_tpm_stat(chip, TPM_STS_VALID, chip->timeout_c, + &priv->int_queue, false); + status = tpm_tis_status(chip); + if (!itpm && (status & TPM_STS_DATA_EXPECT) != 0) { + rc = -EIO; + goto out_err; + } + + return 0; + +out_err: + tpm_tis_ready(chip); + release_locality(chip, priv->locality, 0); + return rc; +} + +static void disable_interrupts(struct tpm_chip *chip) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + u32 intmask; + int rc; + + rc = tpm_tis_read32(priv, TPM_INT_ENABLE(priv->locality), &intmask); + if (rc < 0) + intmask = 0; + + intmask &= ~TPM_GLOBAL_INT_ENABLE; + rc = tpm_tis_write32(priv, TPM_INT_ENABLE(priv->locality), intmask); + + devm_free_irq(chip->dev.parent, priv->irq, chip); + priv->irq = 0; + chip->flags &= ~TPM_CHIP_FLAG_IRQ; +} + +/* + * If interrupts are used (signaled by an irq set in the vendor structure) + * tpm.c can skip polling for the data to be available as the interrupt is + * waited for here + */ +static int tpm_tis_send_main(struct tpm_chip *chip, u8 *buf, size_t len) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + int rc; + u32 ordinal; + unsigned long dur; + + rc = tpm_tis_send_data(chip, buf, len); + if (rc < 0) + return rc; + + /* go and do it */ + rc = tpm_tis_write8(priv, TPM_STS(priv->locality), TPM_STS_GO); + if (rc < 0) + goto out_err; + + if (chip->flags & TPM_CHIP_FLAG_IRQ) { + ordinal = be32_to_cpu(*((__be32 *) (buf + 6))); + + if (chip->flags & TPM_CHIP_FLAG_TPM2) + dur = tpm2_calc_ordinal_duration(chip, ordinal); + else + dur = tpm_calc_ordinal_duration(chip, ordinal); + + if (wait_for_tpm_stat + (chip, TPM_STS_DATA_AVAIL | TPM_STS_VALID, dur, + &priv->read_queue, false) < 0) { + rc = -ETIME; + goto out_err; + } + } + return len; +out_err: + tpm_tis_ready(chip); + release_locality(chip, priv->locality, 0); + return rc; +} + +static int tpm_tis_send(struct tpm_chip *chip, u8 *buf, size_t len) +{ + int rc, irq; + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + + if (!(chip->flags & TPM_CHIP_FLAG_IRQ) || priv->irq_tested) + return tpm_tis_send_main(chip, buf, len); + + /* Verify receipt of the expected IRQ */ + irq = priv->irq; + priv->irq = 0; + chip->flags &= ~TPM_CHIP_FLAG_IRQ; + rc = tpm_tis_send_main(chip, buf, len); + priv->irq = irq; + chip->flags |= TPM_CHIP_FLAG_IRQ; + if (!priv->irq_tested) + msleep(1); + if (!priv->irq_tested) + disable_interrupts(chip); + priv->irq_tested = true; + return rc; +} + +struct tis_vendor_timeout_override { + u32 did_vid; + unsigned long timeout_us[4]; +}; + +static const struct tis_vendor_timeout_override vendor_timeout_overrides[] = { + /* Atmel 3204 */ + { 0x32041114, { (TIS_SHORT_TIMEOUT*1000), (TIS_LONG_TIMEOUT*1000), + (TIS_SHORT_TIMEOUT*1000), (TIS_SHORT_TIMEOUT*1000) } }, +}; + +static bool tpm_tis_update_timeouts(struct tpm_chip *chip, + unsigned long *timeout_cap) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + int i, rc; + u32 did_vid; + + rc = tpm_tis_read32(priv, TPM_DID_VID(0), &did_vid); + if (rc < 0) + return rc; + + for (i = 0; i != ARRAY_SIZE(vendor_timeout_overrides); i++) { + if (vendor_timeout_overrides[i].did_vid != did_vid) + continue; + memcpy(timeout_cap, vendor_timeout_overrides[i].timeout_us, + sizeof(vendor_timeout_overrides[i].timeout_us)); + return true; + } + + return false; +} + +/* + * Early probing for iTPM with STS_DATA_EXPECT flaw. + * Try sending command without itpm flag set and if that + * fails, repeat with itpm flag set. + */ +static int probe_itpm(struct tpm_chip *chip) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + int rc = 0; + u8 cmd_getticks[] = { + 0x00, 0xc1, 0x00, 0x00, 0x00, 0x0a, + 0x00, 0x00, 0x00, 0xf1 + }; + size_t len = sizeof(cmd_getticks); + bool itpm; + u16 vendor; + + rc = tpm_tis_read16(priv, TPM_DID_VID(0), &vendor); + if (rc < 0) + return rc; + + /* probe only iTPMS */ + if (vendor != TPM_VID_INTEL) + return 0; + + itpm = false; + + rc = tpm_tis_send_data(chip, cmd_getticks, len); + if (rc == 0) + goto out; + + tpm_tis_ready(chip); + release_locality(chip, priv->locality, 0); + + itpm = true; + + rc = tpm_tis_send_data(chip, cmd_getticks, len); + if (rc == 0) { + dev_info(&chip->dev, "Detected an iTPM.\n"); + rc = 1; + } else + rc = -EFAULT; + +out: + tpm_tis_ready(chip); + release_locality(chip, priv->locality, 0); + + return rc; +} + +static bool tpm_tis_req_canceled(struct tpm_chip *chip, u8 status) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + + switch (priv->manufacturer_id) { + case TPM_VID_WINBOND: + return ((status == TPM_STS_VALID) || + (status == (TPM_STS_VALID | TPM_STS_COMMAND_READY))); + case TPM_VID_STM: + return (status == (TPM_STS_VALID | TPM_STS_COMMAND_READY)); + default: + return (status == TPM_STS_COMMAND_READY); + } +} + +static irqreturn_t tis_int_handler(int dummy, void *dev_id) +{ + struct tpm_chip *chip = dev_id; + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + u32 interrupt; + int i, rc; + + rc = tpm_tis_read32(priv, TPM_INT_STATUS(priv->locality), &interrupt); + if (rc < 0) + return IRQ_NONE; + + if (interrupt == 0) + return IRQ_NONE; + + priv->irq_tested = true; + if (interrupt & TPM_INTF_DATA_AVAIL_INT) + wake_up_interruptible(&priv->read_queue); + if (interrupt & TPM_INTF_LOCALITY_CHANGE_INT) + for (i = 0; i < 5; i++) + if (check_locality(chip, i) >= 0) + break; + if (interrupt & + (TPM_INTF_LOCALITY_CHANGE_INT | TPM_INTF_STS_VALID_INT | + TPM_INTF_CMD_READY_INT)) + wake_up_interruptible(&priv->int_queue); + + /* Clear interrupts handled with TPM_EOI */ + rc = tpm_tis_write32(priv, TPM_INT_STATUS(priv->locality), interrupt); + if (rc < 0) + return IRQ_NONE; + + tpm_tis_read32(priv, TPM_INT_STATUS(priv->locality), &interrupt); + return IRQ_HANDLED; +} + +/* Register the IRQ and issue a command that will cause an interrupt. If an + * irq is seen then leave the chip setup for IRQ operation, otherwise reverse + * everything and leave in polling mode. Returns 0 on success. + */ +static int tpm_tis_probe_irq_single(struct tpm_chip *chip, u32 intmask, + int flags, int irq) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + u8 original_int_vec; + int rc; + u32 int_status; + + if (devm_request_irq(chip->dev.parent, irq, tis_int_handler, flags, + dev_name(&chip->dev), chip) != 0) { + dev_info(&chip->dev, "Unable to request irq: %d for probe\n", + irq); + return -1; + } + priv->irq = irq; + + rc = tpm_tis_read8(priv, TPM_INT_VECTOR(priv->locality), + &original_int_vec); + if (rc < 0) + return rc; + + rc = tpm_tis_write8(priv, TPM_INT_VECTOR(priv->locality), irq); + if (rc < 0) + return rc; + + rc = tpm_tis_read32(priv, TPM_INT_STATUS(priv->locality), &int_status); + if (rc < 0) + return rc; + + /* Clear all existing */ + rc = tpm_tis_write32(priv, TPM_INT_STATUS(priv->locality), int_status); + if (rc < 0) + return rc; + + /* Turn on */ + rc = tpm_tis_write32(priv, TPM_INT_ENABLE(priv->locality), + intmask | TPM_GLOBAL_INT_ENABLE); + if (rc < 0) + return rc; + + priv->irq_tested = false; + + /* Generate an interrupt by having the core call through to + * tpm_tis_send + */ + if (chip->flags & TPM_CHIP_FLAG_TPM2) + tpm2_gen_interrupt(chip); + else + tpm_gen_interrupt(chip); + + /* tpm_tis_send will either confirm the interrupt is working or it + * will call disable_irq which undoes all of the above. + */ + if (!(chip->flags & TPM_CHIP_FLAG_IRQ)) { + rc = tpm_tis_write8(priv, original_int_vec, + TPM_INT_VECTOR(priv->locality)); + if (rc < 0) + return rc; + + return 1; + } + + return 0; +} + +/* Try to find the IRQ the TPM is using. This is for legacy x86 systems that + * do not have ACPI/etc. We typically expect the interrupt to be declared if + * present. + */ +static void tpm_tis_probe_irq(struct tpm_chip *chip, u32 intmask) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + u8 original_int_vec; + int i, rc; + + rc = tpm_tis_read8(priv, TPM_INT_VECTOR(priv->locality), + &original_int_vec); + if (rc < 0) + return; + + if (!original_int_vec) { + if (IS_ENABLED(CONFIG_X86)) + for (i = 3; i <= 15; i++) + if (!tpm_tis_probe_irq_single(chip, intmask, 0, + i)) + return; + } else if (!tpm_tis_probe_irq_single(chip, intmask, 0, + original_int_vec)) + return; +} + +void tpm_tis_remove(struct tpm_chip *chip) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + u32 reg = TPM_INT_ENABLE(priv->locality); + u32 interrupt; + int rc; + + rc = tpm_tis_read32(priv, reg, &interrupt); + if (rc < 0) + interrupt = 0; + + tpm_tis_write32(priv, reg, ~TPM_GLOBAL_INT_ENABLE & interrupt); + release_locality(chip, priv->locality, 1); +} +EXPORT_SYMBOL_GPL(tpm_tis_remove); + +static const struct tpm_class_ops tpm_tis = { + .flags = TPM_OPS_AUTO_STARTUP, + .status = tpm_tis_status, + .recv = tpm_tis_recv, + .send = tpm_tis_send, + .cancel = tpm_tis_ready, + .update_timeouts = tpm_tis_update_timeouts, + .req_complete_mask = TPM_STS_DATA_AVAIL | TPM_STS_VALID, + .req_complete_val = TPM_STS_DATA_AVAIL | TPM_STS_VALID, + .req_canceled = tpm_tis_req_canceled, +}; + +int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, + const struct tpm_tis_phy_ops *phy_ops, + acpi_handle acpi_dev_handle) +{ + u32 vendor, intfcaps, intmask; + u8 rid; + int rc, probe; + struct tpm_chip *chip; + + chip = tpmm_chip_alloc(dev, &tpm_tis); + if (IS_ERR(chip)) + return PTR_ERR(chip); + +#ifdef CONFIG_ACPI + chip->acpi_dev_handle = acpi_dev_handle; +#endif + + /* Maximum timeouts */ + chip->timeout_a = msecs_to_jiffies(TIS_TIMEOUT_A_MAX); + chip->timeout_b = msecs_to_jiffies(TIS_TIMEOUT_B_MAX); + chip->timeout_c = msecs_to_jiffies(TIS_TIMEOUT_C_MAX); + chip->timeout_d = msecs_to_jiffies(TIS_TIMEOUT_D_MAX); + priv->phy_ops = phy_ops; + dev_set_drvdata(&chip->dev, priv); + + if (wait_startup(chip, 0) != 0) { + rc = -ENODEV; + goto out_err; + } + + /* Take control of the TPM's interrupt hardware and shut it off */ + rc = tpm_tis_read32(priv, TPM_INT_ENABLE(priv->locality), &intmask); + if (rc < 0) + goto out_err; + + intmask |= TPM_INTF_CMD_READY_INT | TPM_INTF_LOCALITY_CHANGE_INT | + TPM_INTF_DATA_AVAIL_INT | TPM_INTF_STS_VALID_INT; + intmask &= ~TPM_GLOBAL_INT_ENABLE; + tpm_tis_write32(priv, TPM_INT_ENABLE(priv->locality), intmask); + + if (request_locality(chip, 0) != 0) { + rc = -ENODEV; + goto out_err; + } + + rc = tpm2_probe(chip); + if (rc) + goto out_err; + + rc = tpm_tis_read32(priv, TPM_DID_VID(0), &vendor); + if (rc < 0) + goto out_err; + + priv->manufacturer_id = vendor; + + rc = tpm_tis_read8(priv, TPM_RID(0), &rid); + if (rc < 0) + goto out_err; + + dev_info(dev, "%s TPM (device-id 0x%X, rev-id %d)\n", + (chip->flags & TPM_CHIP_FLAG_TPM2) ? "2.0" : "1.2", + vendor >> 16, rid); + + if (!(priv->flags & TPM_TIS_ITPM_POSSIBLE)) { + probe = probe_itpm(chip); + if (probe < 0) { + rc = -ENODEV; + goto out_err; + } + + if (!!probe) + priv->flags |= TPM_TIS_ITPM_POSSIBLE; + } + + /* Figure out the capabilities */ + rc = tpm_tis_read32(priv, TPM_INTF_CAPS(priv->locality), &intfcaps); + if (rc < 0) + goto out_err; + + dev_dbg(dev, "TPM interface capabilities (0x%x):\n", + intfcaps); + if (intfcaps & TPM_INTF_BURST_COUNT_STATIC) + dev_dbg(dev, "\tBurst Count Static\n"); + if (intfcaps & TPM_INTF_CMD_READY_INT) + dev_dbg(dev, "\tCommand Ready Int Support\n"); + if (intfcaps & TPM_INTF_INT_EDGE_FALLING) + dev_dbg(dev, "\tInterrupt Edge Falling\n"); + if (intfcaps & TPM_INTF_INT_EDGE_RISING) + dev_dbg(dev, "\tInterrupt Edge Rising\n"); + if (intfcaps & TPM_INTF_INT_LEVEL_LOW) + dev_dbg(dev, "\tInterrupt Level Low\n"); + if (intfcaps & TPM_INTF_INT_LEVEL_HIGH) + dev_dbg(dev, "\tInterrupt Level High\n"); + if (intfcaps & TPM_INTF_LOCALITY_CHANGE_INT) + dev_dbg(dev, "\tLocality Change Int Support\n"); + if (intfcaps & TPM_INTF_STS_VALID_INT) + dev_dbg(dev, "\tSts Valid Int Support\n"); + if (intfcaps & TPM_INTF_DATA_AVAIL_INT) + dev_dbg(dev, "\tData Avail Int Support\n"); + + /* Very early on issue a command to the TPM in polling mode to make + * sure it works. May as well use that command to set the proper + * timeouts for the driver. + */ + if (tpm_get_timeouts(chip)) { + dev_err(dev, "Could not get TPM timeouts and durations\n"); + rc = -ENODEV; + goto out_err; + } + + /* INTERRUPT Setup */ + init_waitqueue_head(&priv->read_queue); + init_waitqueue_head(&priv->int_queue); + if (irq != -1) { + if (irq) { + tpm_tis_probe_irq_single(chip, intmask, IRQF_SHARED, + irq); + if (!(chip->flags & TPM_CHIP_FLAG_IRQ)) + dev_err(&chip->dev, FW_BUG + "TPM interrupt not working, polling instead\n"); + } else { + tpm_tis_probe_irq(chip, intmask); + } + } + + return tpm_chip_register(chip); +out_err: + tpm_tis_remove(chip); + return rc; +} +EXPORT_SYMBOL_GPL(tpm_tis_core_init); + +#ifdef CONFIG_PM_SLEEP +static void tpm_tis_reenable_interrupts(struct tpm_chip *chip) +{ + struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); + u32 intmask; + int rc; + + /* reenable interrupts that device may have lost or + * BIOS/firmware may have disabled + */ + rc = tpm_tis_write8(priv, TPM_INT_VECTOR(priv->locality), priv->irq); + if (rc < 0) + return; + + rc = tpm_tis_read32(priv, TPM_INT_ENABLE(priv->locality), &intmask); + if (rc < 0) + return; + + intmask |= TPM_INTF_CMD_READY_INT + | TPM_INTF_LOCALITY_CHANGE_INT | TPM_INTF_DATA_AVAIL_INT + | TPM_INTF_STS_VALID_INT | TPM_GLOBAL_INT_ENABLE; + + tpm_tis_write32(priv, TPM_INT_ENABLE(priv->locality), intmask); +} + +int tpm_tis_resume(struct device *dev) +{ + struct tpm_chip *chip = dev_get_drvdata(dev); + int ret; + + if (chip->flags & TPM_CHIP_FLAG_IRQ) + tpm_tis_reenable_interrupts(chip); + + ret = tpm_pm_resume(dev); + if (ret) + return ret; + + /* TPM 1.2 requires self-test on resume. This function actually returns + * an error code but for unknown reason it isn't handled. + */ + if (!(chip->flags & TPM_CHIP_FLAG_TPM2)) + tpm_do_selftest(chip); + + return 0; +} +EXPORT_SYMBOL_GPL(tpm_tis_resume); +#endif + +MODULE_AUTHOR("Leendert van Doorn (leendert@watson.ibm.com)"); +MODULE_DESCRIPTION("TPM Driver"); +MODULE_VERSION("2.0"); +MODULE_LICENSE("GPL"); diff --git a/drivers/char/tpm/tpm_tis_core.h b/drivers/char/tpm/tpm_tis_core.h new file mode 100644 index 000000000..9191aabbf --- /dev/null +++ b/drivers/char/tpm/tpm_tis_core.h @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2005, 2006 IBM Corporation + * Copyright (C) 2014, 2015 Intel Corporation + * + * Authors: + * Leendert van Doorn + * Kylene Hall + * + * Maintained by: + * + * Device driver for TCG/TCPA TPM (trusted platform module). + * Specifications at www.trustedcomputinggroup.org + * + * This device driver implements the TPM interface as defined in + * the TCG TPM Interface Spec version 1.2, revision 1.0. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, version 2 of the + * License. + */ + +#ifndef __TPM_TIS_CORE_H__ +#define __TPM_TIS_CORE_H__ + +#include "tpm.h" + +enum tis_access { + TPM_ACCESS_VALID = 0x80, + TPM_ACCESS_ACTIVE_LOCALITY = 0x20, + TPM_ACCESS_REQUEST_PENDING = 0x04, + TPM_ACCESS_REQUEST_USE = 0x02, +}; + +enum tis_status { + TPM_STS_VALID = 0x80, + TPM_STS_COMMAND_READY = 0x40, + TPM_STS_GO = 0x20, + TPM_STS_DATA_AVAIL = 0x10, + TPM_STS_DATA_EXPECT = 0x08, +}; + +enum tis_int_flags { + TPM_GLOBAL_INT_ENABLE = 0x80000000, + TPM_INTF_BURST_COUNT_STATIC = 0x100, + TPM_INTF_CMD_READY_INT = 0x080, + TPM_INTF_INT_EDGE_FALLING = 0x040, + TPM_INTF_INT_EDGE_RISING = 0x020, + TPM_INTF_INT_LEVEL_LOW = 0x010, + TPM_INTF_INT_LEVEL_HIGH = 0x008, + TPM_INTF_LOCALITY_CHANGE_INT = 0x004, + TPM_INTF_STS_VALID_INT = 0x002, + TPM_INTF_DATA_AVAIL_INT = 0x001, +}; + +enum tis_defaults { + TIS_MEM_LEN = 0x5000, + TIS_SHORT_TIMEOUT = 750, /* ms */ + TIS_LONG_TIMEOUT = 2000, /* 2 sec */ +}; + +/* Some timeout values are needed before it is known whether the chip is + * TPM 1.0 or TPM 2.0. + */ +#define TIS_TIMEOUT_A_MAX max(TIS_SHORT_TIMEOUT, TPM2_TIMEOUT_A) +#define TIS_TIMEOUT_B_MAX max(TIS_LONG_TIMEOUT, TPM2_TIMEOUT_B) +#define TIS_TIMEOUT_C_MAX max(TIS_SHORT_TIMEOUT, TPM2_TIMEOUT_C) +#define TIS_TIMEOUT_D_MAX max(TIS_SHORT_TIMEOUT, TPM2_TIMEOUT_D) + +#define TPM_ACCESS(l) (0x0000 | ((l) << 12)) +#define TPM_INT_ENABLE(l) (0x0008 | ((l) << 12)) +#define TPM_INT_VECTOR(l) (0x000C | ((l) << 12)) +#define TPM_INT_STATUS(l) (0x0010 | ((l) << 12)) +#define TPM_INTF_CAPS(l) (0x0014 | ((l) << 12)) +#define TPM_STS(l) (0x0018 | ((l) << 12)) +#define TPM_STS3(l) (0x001b | ((l) << 12)) +#define TPM_DATA_FIFO(l) (0x0024 | ((l) << 12)) + +#define TPM_DID_VID(l) (0x0F00 | ((l) << 12)) +#define TPM_RID(l) (0x0F04 | ((l) << 12)) + +enum tpm_tis_flags { + TPM_TIS_ITPM_POSSIBLE = BIT(0), +}; + +struct tpm_tis_data { + u16 manufacturer_id; + int locality; + int irq; + bool irq_tested; + unsigned int flags; + wait_queue_head_t int_queue; + wait_queue_head_t read_queue; + const struct tpm_tis_phy_ops *phy_ops; +}; + +struct tpm_tis_phy_ops { + int (*read_bytes)(struct tpm_tis_data *data, u32 addr, u16 len, + u8 *result); + int (*write_bytes)(struct tpm_tis_data *data, u32 addr, u16 len, + u8 *value); + int (*read16)(struct tpm_tis_data *data, u32 addr, u16 *result); + int (*read32)(struct tpm_tis_data *data, u32 addr, u32 *result); + int (*write32)(struct tpm_tis_data *data, u32 addr, u32 src); +}; + +static inline int tpm_tis_read_bytes(struct tpm_tis_data *data, u32 addr, + u16 len, u8 *result) +{ + return data->phy_ops->read_bytes(data, addr, len, result); +} + +static inline int tpm_tis_read8(struct tpm_tis_data *data, u32 addr, u8 *result) +{ + return data->phy_ops->read_bytes(data, addr, 1, result); +} + +static inline int tpm_tis_read16(struct tpm_tis_data *data, u32 addr, + u16 *result) +{ + return data->phy_ops->read16(data, addr, result); +} + +static inline int tpm_tis_read32(struct tpm_tis_data *data, u32 addr, + u32 *result) +{ + return data->phy_ops->read32(data, addr, result); +} + +static inline int tpm_tis_write_bytes(struct tpm_tis_data *data, u32 addr, + u16 len, u8 *value) +{ + return data->phy_ops->write_bytes(data, addr, len, value); +} + +static inline int tpm_tis_write8(struct tpm_tis_data *data, u32 addr, u8 value) +{ + return data->phy_ops->write_bytes(data, addr, 1, &value); +} + +static inline int tpm_tis_write32(struct tpm_tis_data *data, u32 addr, + u32 value) +{ + return data->phy_ops->write32(data, addr, value); +} + +void tpm_tis_remove(struct tpm_chip *chip); +int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, + const struct tpm_tis_phy_ops *phy_ops, + acpi_handle acpi_dev_handle); + +#ifdef CONFIG_PM_SLEEP +int tpm_tis_resume(struct device *dev); +#endif + +#endif diff --git a/drivers/char/tpm/tpm_tis_spi.c b/drivers/char/tpm/tpm_tis_spi.c new file mode 100644 index 000000000..dbaad9c68 --- /dev/null +++ b/drivers/char/tpm/tpm_tis_spi.c @@ -0,0 +1,272 @@ +/* + * Copyright (C) 2015 Infineon Technologies AG + * Copyright (C) 2016 STMicroelectronics SAS + * + * Authors: + * Peter Huewe + * Christophe Ricard + * + * Maintained by: + * + * Device driver for TCG/TCPA TPM (trusted platform module). + * Specifications at www.trustedcomputinggroup.org + * + * This device driver implements the TPM interface as defined in + * the TCG TPM Interface Spec version 1.3, revision 27 via _raw/native + * SPI access_. + * + * It is based on the original tpm_tis device driver from Leendert van + * Dorn and Kyleen Hall and Jarko Sakkinnen. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, version 2 of the + * License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include "tpm.h" +#include "tpm_tis_core.h" + +#define MAX_SPI_FRAMESIZE 64 + +struct tpm_tis_spi_phy { + struct tpm_tis_data priv; + struct spi_device *spi_device; + + u8 tx_buf[MAX_SPI_FRAMESIZE + 4]; + u8 rx_buf[MAX_SPI_FRAMESIZE + 4]; +}; + +static inline struct tpm_tis_spi_phy *to_tpm_tis_spi_phy(struct tpm_tis_data *data) +{ + return container_of(data, struct tpm_tis_spi_phy, priv); +} + +static int tpm_tis_spi_read_bytes(struct tpm_tis_data *data, u32 addr, + u16 len, u8 *result) +{ + struct tpm_tis_spi_phy *phy = to_tpm_tis_spi_phy(data); + int ret, i; + struct spi_message m; + struct spi_transfer spi_xfer = { + .tx_buf = phy->tx_buf, + .rx_buf = phy->rx_buf, + .len = 4, + }; + + if (len > MAX_SPI_FRAMESIZE) + return -ENOMEM; + + phy->tx_buf[0] = 0x80 | (len - 1); + phy->tx_buf[1] = 0xd4; + phy->tx_buf[2] = (addr >> 8) & 0xFF; + phy->tx_buf[3] = addr & 0xFF; + + spi_xfer.cs_change = 1; + spi_message_init(&m); + spi_message_add_tail(&spi_xfer, &m); + + spi_bus_lock(phy->spi_device->master); + ret = spi_sync_locked(phy->spi_device, &m); + if (ret < 0) + goto exit; + + memset(phy->tx_buf, 0, len); + + /* According to TCG PTP specification, if there is no TPM present at + * all, then the design has a weak pull-up on MISO. If a TPM is not + * present, a pull-up on MISO means that the SB controller sees a 1, + * and will latch in 0xFF on the read. + */ + for (i = 0; (phy->rx_buf[0] & 0x01) == 0 && i < TPM_RETRY; i++) { + spi_xfer.len = 1; + spi_message_init(&m); + spi_message_add_tail(&spi_xfer, &m); + ret = spi_sync_locked(phy->spi_device, &m); + if (ret < 0) + goto exit; + } + + spi_xfer.cs_change = 0; + spi_xfer.len = len; + spi_xfer.rx_buf = result; + + spi_message_init(&m); + spi_message_add_tail(&spi_xfer, &m); + ret = spi_sync_locked(phy->spi_device, &m); + +exit: + spi_bus_unlock(phy->spi_device->master); + return ret; +} + +static int tpm_tis_spi_write_bytes(struct tpm_tis_data *data, u32 addr, + u16 len, u8 *value) +{ + struct tpm_tis_spi_phy *phy = to_tpm_tis_spi_phy(data); + int ret, i; + struct spi_message m; + struct spi_transfer spi_xfer = { + .tx_buf = phy->tx_buf, + .rx_buf = phy->rx_buf, + .len = 4, + }; + + if (len > MAX_SPI_FRAMESIZE) + return -ENOMEM; + + phy->tx_buf[0] = len - 1; + phy->tx_buf[1] = 0xd4; + phy->tx_buf[2] = (addr >> 8) & 0xFF; + phy->tx_buf[3] = addr & 0xFF; + + spi_xfer.cs_change = 1; + spi_message_init(&m); + spi_message_add_tail(&spi_xfer, &m); + + spi_bus_lock(phy->spi_device->master); + ret = spi_sync_locked(phy->spi_device, &m); + if (ret < 0) + goto exit; + + memset(phy->tx_buf, 0, len); + + /* According to TCG PTP specification, if there is no TPM present at + * all, then the design has a weak pull-up on MISO. If a TPM is not + * present, a pull-up on MISO means that the SB controller sees a 1, + * and will latch in 0xFF on the read. + */ + for (i = 0; (phy->rx_buf[0] & 0x01) == 0 && i < TPM_RETRY; i++) { + spi_xfer.len = 1; + spi_message_init(&m); + spi_message_add_tail(&spi_xfer, &m); + ret = spi_sync_locked(phy->spi_device, &m); + if (ret < 0) + goto exit; + } + + spi_xfer.len = len; + spi_xfer.tx_buf = value; + spi_xfer.cs_change = 0; + spi_xfer.tx_buf = value; + spi_message_init(&m); + spi_message_add_tail(&spi_xfer, &m); + ret = spi_sync_locked(phy->spi_device, &m); + +exit: + spi_bus_unlock(phy->spi_device->master); + return ret; +} + +static int tpm_tis_spi_read16(struct tpm_tis_data *data, u32 addr, u16 *result) +{ + int rc; + + rc = data->phy_ops->read_bytes(data, addr, sizeof(u16), (u8 *)result); + if (!rc) + *result = le16_to_cpu(*result); + return rc; +} + +static int tpm_tis_spi_read32(struct tpm_tis_data *data, u32 addr, u32 *result) +{ + int rc; + + rc = data->phy_ops->read_bytes(data, addr, sizeof(u32), (u8 *)result); + if (!rc) + *result = le32_to_cpu(*result); + return rc; +} + +static int tpm_tis_spi_write32(struct tpm_tis_data *data, u32 addr, u32 value) +{ + value = cpu_to_le32(value); + return data->phy_ops->write_bytes(data, addr, sizeof(u32), + (u8 *)&value); +} + +static const struct tpm_tis_phy_ops tpm_spi_phy_ops = { + .read_bytes = tpm_tis_spi_read_bytes, + .write_bytes = tpm_tis_spi_write_bytes, + .read16 = tpm_tis_spi_read16, + .read32 = tpm_tis_spi_read32, + .write32 = tpm_tis_spi_write32, +}; + +static int tpm_tis_spi_probe(struct spi_device *dev) +{ + struct tpm_tis_spi_phy *phy; + + phy = devm_kzalloc(&dev->dev, sizeof(struct tpm_tis_spi_phy), + GFP_KERNEL); + if (!phy) + return -ENOMEM; + + phy->spi_device = dev; + + return tpm_tis_core_init(&dev->dev, &phy->priv, -1, &tpm_spi_phy_ops, + NULL); +} + +static SIMPLE_DEV_PM_OPS(tpm_tis_pm, tpm_pm_suspend, tpm_tis_resume); + +static int tpm_tis_spi_remove(struct spi_device *dev) +{ + struct tpm_chip *chip = spi_get_drvdata(dev); + + tpm_chip_unregister(chip); + tpm_tis_remove(chip); + return 0; +} + +static const struct spi_device_id tpm_tis_spi_id[] = { + {"tpm_tis_spi", 0}, + {} +}; +MODULE_DEVICE_TABLE(spi, tpm_tis_spi_id); + +static const struct of_device_id of_tis_spi_match[] = { + { .compatible = "st,st33htpm-spi", }, + { .compatible = "infineon,slb9670", }, + { .compatible = "tcg,tpm_tis-spi", }, + {} +}; +MODULE_DEVICE_TABLE(of, of_tis_spi_match); + +static const struct acpi_device_id acpi_tis_spi_match[] = { + {"SMO0768", 0}, + {} +}; +MODULE_DEVICE_TABLE(acpi, acpi_tis_spi_match); + +static struct spi_driver tpm_tis_spi_driver = { + .driver = { + .owner = THIS_MODULE, + .name = "tpm_tis_spi", + .pm = &tpm_tis_pm, + .of_match_table = of_match_ptr(of_tis_spi_match), + .acpi_match_table = ACPI_PTR(acpi_tis_spi_match), + }, + .probe = tpm_tis_spi_probe, + .remove = tpm_tis_spi_remove, + .id_table = tpm_tis_spi_id, +}; +module_spi_driver(tpm_tis_spi_driver); + +MODULE_DESCRIPTION("TPM Driver for native SPI access"); +MODULE_LICENSE("GPL"); diff --git a/drivers/char/tpm/tpm_vtpm_proxy.c b/drivers/char/tpm/tpm_vtpm_proxy.c new file mode 100644 index 000000000..9a940332c --- /dev/null +++ b/drivers/char/tpm/tpm_vtpm_proxy.c @@ -0,0 +1,637 @@ +/* + * Copyright (C) 2015, 2016 IBM Corporation + * + * Author: Stefan Berger + * + * Maintained by: + * + * Device driver for vTPM (vTPM proxy driver) + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, version 2 of the + * License. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tpm.h" + +#define VTPM_PROXY_REQ_COMPLETE_FLAG BIT(0) + +struct proxy_dev { + struct tpm_chip *chip; + + u32 flags; /* public API flags */ + + wait_queue_head_t wq; + + struct mutex buf_lock; /* protect buffer and flags */ + + long state; /* internal state */ +#define STATE_OPENED_FLAG BIT(0) +#define STATE_WAIT_RESPONSE_FLAG BIT(1) /* waiting for emulator response */ + + size_t req_len; /* length of queued TPM request */ + size_t resp_len; /* length of queued TPM response */ + u8 buffer[TPM_BUFSIZE]; /* request/response buffer */ + + struct work_struct work; /* task that retrieves TPM timeouts */ +}; + +/* all supported flags */ +#define VTPM_PROXY_FLAGS_ALL (VTPM_PROXY_FLAG_TPM2) + +static struct workqueue_struct *workqueue; + +static void vtpm_proxy_delete_device(struct proxy_dev *proxy_dev); + +/* + * Functions related to 'server side' + */ + +/** + * vtpm_proxy_fops_read - Read TPM commands on 'server side' + * + * Return value: + * Number of bytes read or negative error code + */ +static ssize_t vtpm_proxy_fops_read(struct file *filp, char __user *buf, + size_t count, loff_t *off) +{ + struct proxy_dev *proxy_dev = filp->private_data; + size_t len; + int sig, rc; + + sig = wait_event_interruptible(proxy_dev->wq, + proxy_dev->req_len != 0 || + !(proxy_dev->state & STATE_OPENED_FLAG)); + if (sig) + return -EINTR; + + mutex_lock(&proxy_dev->buf_lock); + + if (!(proxy_dev->state & STATE_OPENED_FLAG)) { + mutex_unlock(&proxy_dev->buf_lock); + return -EPIPE; + } + + len = proxy_dev->req_len; + + if (count < len) { + mutex_unlock(&proxy_dev->buf_lock); + pr_debug("Invalid size in recv: count=%zd, req_len=%zd\n", + count, len); + return -EIO; + } + + rc = copy_to_user(buf, proxy_dev->buffer, len); + memset(proxy_dev->buffer, 0, len); + proxy_dev->req_len = 0; + + if (!rc) + proxy_dev->state |= STATE_WAIT_RESPONSE_FLAG; + + mutex_unlock(&proxy_dev->buf_lock); + + if (rc) + return -EFAULT; + + return len; +} + +/** + * vtpm_proxy_fops_write - Write TPM responses on 'server side' + * + * Return value: + * Number of bytes read or negative error value + */ +static ssize_t vtpm_proxy_fops_write(struct file *filp, const char __user *buf, + size_t count, loff_t *off) +{ + struct proxy_dev *proxy_dev = filp->private_data; + + mutex_lock(&proxy_dev->buf_lock); + + if (!(proxy_dev->state & STATE_OPENED_FLAG)) { + mutex_unlock(&proxy_dev->buf_lock); + return -EPIPE; + } + + if (count > sizeof(proxy_dev->buffer) || + !(proxy_dev->state & STATE_WAIT_RESPONSE_FLAG)) { + mutex_unlock(&proxy_dev->buf_lock); + return -EIO; + } + + proxy_dev->state &= ~STATE_WAIT_RESPONSE_FLAG; + + proxy_dev->req_len = 0; + + if (copy_from_user(proxy_dev->buffer, buf, count)) { + mutex_unlock(&proxy_dev->buf_lock); + return -EFAULT; + } + + proxy_dev->resp_len = count; + + mutex_unlock(&proxy_dev->buf_lock); + + wake_up_interruptible(&proxy_dev->wq); + + return count; +} + +/* + * vtpm_proxy_fops_poll: Poll status on 'server side' + * + * Return value: + * Poll flags + */ +static unsigned int vtpm_proxy_fops_poll(struct file *filp, poll_table *wait) +{ + struct proxy_dev *proxy_dev = filp->private_data; + unsigned ret; + + poll_wait(filp, &proxy_dev->wq, wait); + + ret = POLLOUT; + + mutex_lock(&proxy_dev->buf_lock); + + if (proxy_dev->req_len) + ret |= POLLIN | POLLRDNORM; + + if (!(proxy_dev->state & STATE_OPENED_FLAG)) + ret |= POLLHUP; + + mutex_unlock(&proxy_dev->buf_lock); + + return ret; +} + +/* + * vtpm_proxy_fops_open - Open vTPM device on 'server side' + * + * Called when setting up the anonymous file descriptor + */ +static void vtpm_proxy_fops_open(struct file *filp) +{ + struct proxy_dev *proxy_dev = filp->private_data; + + proxy_dev->state |= STATE_OPENED_FLAG; +} + +/** + * vtpm_proxy_fops_undo_open - counter-part to vtpm_fops_open + * + * Call to undo vtpm_proxy_fops_open + */ +static void vtpm_proxy_fops_undo_open(struct proxy_dev *proxy_dev) +{ + mutex_lock(&proxy_dev->buf_lock); + + proxy_dev->state &= ~STATE_OPENED_FLAG; + + mutex_unlock(&proxy_dev->buf_lock); + + /* no more TPM responses -- wake up anyone waiting for them */ + wake_up_interruptible(&proxy_dev->wq); +} + +/* + * vtpm_proxy_fops_release: Close 'server side' + * + * Return value: + * Always returns 0. + */ +static int vtpm_proxy_fops_release(struct inode *inode, struct file *filp) +{ + struct proxy_dev *proxy_dev = filp->private_data; + + filp->private_data = NULL; + + vtpm_proxy_delete_device(proxy_dev); + + return 0; +} + +static const struct file_operations vtpm_proxy_fops = { + .owner = THIS_MODULE, + .llseek = no_llseek, + .read = vtpm_proxy_fops_read, + .write = vtpm_proxy_fops_write, + .poll = vtpm_proxy_fops_poll, + .release = vtpm_proxy_fops_release, +}; + +/* + * Functions invoked by the core TPM driver to send TPM commands to + * 'server side' and receive responses from there. + */ + +/* + * Called when core TPM driver reads TPM responses from 'server side' + * + * Return value: + * Number of TPM response bytes read, negative error value otherwise + */ +static int vtpm_proxy_tpm_op_recv(struct tpm_chip *chip, u8 *buf, size_t count) +{ + struct proxy_dev *proxy_dev = dev_get_drvdata(&chip->dev); + size_t len; + + /* process gone ? */ + mutex_lock(&proxy_dev->buf_lock); + + if (!(proxy_dev->state & STATE_OPENED_FLAG)) { + mutex_unlock(&proxy_dev->buf_lock); + return -EPIPE; + } + + len = proxy_dev->resp_len; + if (count < len) { + dev_err(&chip->dev, + "Invalid size in recv: count=%zd, resp_len=%zd\n", + count, len); + len = -EIO; + goto out; + } + + memcpy(buf, proxy_dev->buffer, len); + proxy_dev->resp_len = 0; + +out: + mutex_unlock(&proxy_dev->buf_lock); + + return len; +} + +/* + * Called when core TPM driver forwards TPM requests to 'server side'. + * + * Return value: + * 0 in case of success, negative error value otherwise. + */ +static int vtpm_proxy_tpm_op_send(struct tpm_chip *chip, u8 *buf, size_t count) +{ + struct proxy_dev *proxy_dev = dev_get_drvdata(&chip->dev); + int rc = 0; + + if (count > sizeof(proxy_dev->buffer)) { + dev_err(&chip->dev, + "Invalid size in send: count=%zd, buffer size=%zd\n", + count, sizeof(proxy_dev->buffer)); + return -EIO; + } + + mutex_lock(&proxy_dev->buf_lock); + + if (!(proxy_dev->state & STATE_OPENED_FLAG)) { + mutex_unlock(&proxy_dev->buf_lock); + return -EPIPE; + } + + proxy_dev->resp_len = 0; + + proxy_dev->req_len = count; + memcpy(proxy_dev->buffer, buf, count); + + proxy_dev->state &= ~STATE_WAIT_RESPONSE_FLAG; + + mutex_unlock(&proxy_dev->buf_lock); + + wake_up_interruptible(&proxy_dev->wq); + + return rc; +} + +static void vtpm_proxy_tpm_op_cancel(struct tpm_chip *chip) +{ + /* not supported */ +} + +static u8 vtpm_proxy_tpm_op_status(struct tpm_chip *chip) +{ + struct proxy_dev *proxy_dev = dev_get_drvdata(&chip->dev); + + if (proxy_dev->resp_len) + return VTPM_PROXY_REQ_COMPLETE_FLAG; + + return 0; +} + +static bool vtpm_proxy_tpm_req_canceled(struct tpm_chip *chip, u8 status) +{ + struct proxy_dev *proxy_dev = dev_get_drvdata(&chip->dev); + bool ret; + + mutex_lock(&proxy_dev->buf_lock); + + ret = !(proxy_dev->state & STATE_OPENED_FLAG); + + mutex_unlock(&proxy_dev->buf_lock); + + return ret; +} + +static const struct tpm_class_ops vtpm_proxy_tpm_ops = { + .flags = TPM_OPS_AUTO_STARTUP, + .recv = vtpm_proxy_tpm_op_recv, + .send = vtpm_proxy_tpm_op_send, + .cancel = vtpm_proxy_tpm_op_cancel, + .status = vtpm_proxy_tpm_op_status, + .req_complete_mask = VTPM_PROXY_REQ_COMPLETE_FLAG, + .req_complete_val = VTPM_PROXY_REQ_COMPLETE_FLAG, + .req_canceled = vtpm_proxy_tpm_req_canceled, +}; + +/* + * Code related to the startup of the TPM 2 and startup of TPM 1.2 + + * retrieval of timeouts and durations. + */ + +static void vtpm_proxy_work(struct work_struct *work) +{ + struct proxy_dev *proxy_dev = container_of(work, struct proxy_dev, + work); + int rc; + + rc = tpm_chip_register(proxy_dev->chip); + if (rc) + goto err; + + return; + +err: + vtpm_proxy_fops_undo_open(proxy_dev); +} + +/* + * vtpm_proxy_work_stop: make sure the work has finished + * + * This function is useful when user space closed the fd + * while the driver still determines timeouts. + */ +static void vtpm_proxy_work_stop(struct proxy_dev *proxy_dev) +{ + vtpm_proxy_fops_undo_open(proxy_dev); + flush_work(&proxy_dev->work); +} + +/* + * vtpm_proxy_work_start: Schedule the work for TPM 1.2 & 2 initialization + */ +static inline void vtpm_proxy_work_start(struct proxy_dev *proxy_dev) +{ + queue_work(workqueue, &proxy_dev->work); +} + +/* + * Code related to creation and deletion of device pairs + */ +static struct proxy_dev *vtpm_proxy_create_proxy_dev(void) +{ + struct proxy_dev *proxy_dev; + struct tpm_chip *chip; + int err; + + proxy_dev = kzalloc(sizeof(*proxy_dev), GFP_KERNEL); + if (proxy_dev == NULL) + return ERR_PTR(-ENOMEM); + + init_waitqueue_head(&proxy_dev->wq); + mutex_init(&proxy_dev->buf_lock); + INIT_WORK(&proxy_dev->work, vtpm_proxy_work); + + chip = tpm_chip_alloc(NULL, &vtpm_proxy_tpm_ops); + if (IS_ERR(chip)) { + err = PTR_ERR(chip); + goto err_proxy_dev_free; + } + dev_set_drvdata(&chip->dev, proxy_dev); + + proxy_dev->chip = chip; + + return proxy_dev; + +err_proxy_dev_free: + kfree(proxy_dev); + + return ERR_PTR(err); +} + +/* + * Undo what has been done in vtpm_create_proxy_dev + */ +static inline void vtpm_proxy_delete_proxy_dev(struct proxy_dev *proxy_dev) +{ + put_device(&proxy_dev->chip->dev); /* frees chip */ + kfree(proxy_dev); +} + +/* + * Create a /dev/tpm%d and 'server side' file descriptor pair + * + * Return value: + * Returns file pointer on success, an error value otherwise + */ +static struct file *vtpm_proxy_create_device( + struct vtpm_proxy_new_dev *vtpm_new_dev) +{ + struct proxy_dev *proxy_dev; + int rc, fd; + struct file *file; + + if (vtpm_new_dev->flags & ~VTPM_PROXY_FLAGS_ALL) + return ERR_PTR(-EOPNOTSUPP); + + proxy_dev = vtpm_proxy_create_proxy_dev(); + if (IS_ERR(proxy_dev)) + return ERR_CAST(proxy_dev); + + proxy_dev->flags = vtpm_new_dev->flags; + + /* setup an anonymous file for the server-side */ + fd = get_unused_fd_flags(O_RDWR); + if (fd < 0) { + rc = fd; + goto err_delete_proxy_dev; + } + + file = anon_inode_getfile("[vtpms]", &vtpm_proxy_fops, proxy_dev, + O_RDWR); + if (IS_ERR(file)) { + rc = PTR_ERR(file); + goto err_put_unused_fd; + } + + /* from now on we can unwind with put_unused_fd() + fput() */ + /* simulate an open() on the server side */ + vtpm_proxy_fops_open(file); + + if (proxy_dev->flags & VTPM_PROXY_FLAG_TPM2) + proxy_dev->chip->flags |= TPM_CHIP_FLAG_TPM2; + + vtpm_proxy_work_start(proxy_dev); + + vtpm_new_dev->fd = fd; + vtpm_new_dev->major = MAJOR(proxy_dev->chip->dev.devt); + vtpm_new_dev->minor = MINOR(proxy_dev->chip->dev.devt); + vtpm_new_dev->tpm_num = proxy_dev->chip->dev_num; + + return file; + +err_put_unused_fd: + put_unused_fd(fd); + +err_delete_proxy_dev: + vtpm_proxy_delete_proxy_dev(proxy_dev); + + return ERR_PTR(rc); +} + +/* + * Counter part to vtpm_create_device. + */ +static void vtpm_proxy_delete_device(struct proxy_dev *proxy_dev) +{ + vtpm_proxy_work_stop(proxy_dev); + + /* + * A client may hold the 'ops' lock, so let it know that the server + * side shuts down before we try to grab the 'ops' lock when + * unregistering the chip. + */ + vtpm_proxy_fops_undo_open(proxy_dev); + + tpm_chip_unregister(proxy_dev->chip); + + vtpm_proxy_delete_proxy_dev(proxy_dev); +} + +/* + * Code related to the control device /dev/vtpmx + */ + +/* + * vtpmx_fops_ioctl: ioctl on /dev/vtpmx + * + * Return value: + * Returns 0 on success, a negative error code otherwise. + */ +static long vtpmx_fops_ioctl(struct file *f, unsigned int ioctl, + unsigned long arg) +{ + void __user *argp = (void __user *)arg; + struct vtpm_proxy_new_dev __user *vtpm_new_dev_p; + struct vtpm_proxy_new_dev vtpm_new_dev; + struct file *file; + + switch (ioctl) { + case VTPM_PROXY_IOC_NEW_DEV: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + vtpm_new_dev_p = argp; + if (copy_from_user(&vtpm_new_dev, vtpm_new_dev_p, + sizeof(vtpm_new_dev))) + return -EFAULT; + file = vtpm_proxy_create_device(&vtpm_new_dev); + if (IS_ERR(file)) + return PTR_ERR(file); + if (copy_to_user(vtpm_new_dev_p, &vtpm_new_dev, + sizeof(vtpm_new_dev))) { + put_unused_fd(vtpm_new_dev.fd); + fput(file); + return -EFAULT; + } + + fd_install(vtpm_new_dev.fd, file); + return 0; + + default: + return -ENOIOCTLCMD; + } +} + +#ifdef CONFIG_COMPAT +static long vtpmx_fops_compat_ioctl(struct file *f, unsigned int ioctl, + unsigned long arg) +{ + return vtpmx_fops_ioctl(f, ioctl, (unsigned long)compat_ptr(arg)); +} +#endif + +static const struct file_operations vtpmx_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = vtpmx_fops_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = vtpmx_fops_compat_ioctl, +#endif + .llseek = noop_llseek, +}; + +static struct miscdevice vtpmx_miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "vtpmx", + .fops = &vtpmx_fops, +}; + +static int vtpmx_init(void) +{ + return misc_register(&vtpmx_miscdev); +} + +static void vtpmx_cleanup(void) +{ + misc_deregister(&vtpmx_miscdev); +} + +static int __init vtpm_module_init(void) +{ + int rc; + + rc = vtpmx_init(); + if (rc) { + pr_err("couldn't create vtpmx device\n"); + return rc; + } + + workqueue = create_workqueue("tpm-vtpm"); + if (!workqueue) { + pr_err("couldn't create workqueue\n"); + rc = -ENOMEM; + goto err_vtpmx_cleanup; + } + + return 0; + +err_vtpmx_cleanup: + vtpmx_cleanup(); + + return rc; +} + +static void __exit vtpm_module_exit(void) +{ + destroy_workqueue(workqueue); + vtpmx_cleanup(); +} + +module_init(vtpm_module_init); +module_exit(vtpm_module_exit); + +MODULE_AUTHOR("Stefan Berger (stefanb@us.ibm.com)"); +MODULE_DESCRIPTION("vTPM Driver"); +MODULE_VERSION("0.1"); +MODULE_LICENSE("GPL"); diff --git a/drivers/char/tpm/xen-tpmfront.c b/drivers/char/tpm/xen-tpmfront.c index 3111f2778..62028f483 100644 --- a/drivers/char/tpm/xen-tpmfront.c +++ b/drivers/char/tpm/xen-tpmfront.c @@ -28,6 +28,8 @@ struct tpm_private { unsigned int evtchn; int ring_ref; domid_t backend_id; + int irq; + wait_queue_head_t read_queue; }; enum status_bits { @@ -39,7 +41,7 @@ enum status_bits { static u8 vtpm_status(struct tpm_chip *chip) { - struct tpm_private *priv = TPM_VPRIV(chip); + struct tpm_private *priv = dev_get_drvdata(&chip->dev); switch (priv->shr->state) { case VTPM_STATE_IDLE: return VTPM_STATUS_IDLE | VTPM_STATUS_CANCELED; @@ -60,7 +62,7 @@ static bool vtpm_req_canceled(struct tpm_chip *chip, u8 status) static void vtpm_cancel(struct tpm_chip *chip) { - struct tpm_private *priv = TPM_VPRIV(chip); + struct tpm_private *priv = dev_get_drvdata(&chip->dev); priv->shr->state = VTPM_STATE_CANCEL; wmb(); notify_remote_via_evtchn(priv->evtchn); @@ -73,7 +75,7 @@ static unsigned int shr_data_offset(struct vtpm_shared_page *shr) static int vtpm_send(struct tpm_chip *chip, u8 *buf, size_t count) { - struct tpm_private *priv = TPM_VPRIV(chip); + struct tpm_private *priv = dev_get_drvdata(&chip->dev); struct vtpm_shared_page *shr = priv->shr; unsigned int offset = shr_data_offset(shr); @@ -87,8 +89,8 @@ static int vtpm_send(struct tpm_chip *chip, u8 *buf, size_t count) return -EINVAL; /* Wait for completion of any existing command or cancellation */ - if (wait_for_tpm_stat(chip, VTPM_STATUS_IDLE, chip->vendor.timeout_c, - &chip->vendor.read_queue, true) < 0) { + if (wait_for_tpm_stat(chip, VTPM_STATUS_IDLE, chip->timeout_c, + &priv->read_queue, true) < 0) { vtpm_cancel(chip); return -ETIME; } @@ -104,7 +106,7 @@ static int vtpm_send(struct tpm_chip *chip, u8 *buf, size_t count) duration = tpm_calc_ordinal_duration(chip, ordinal); if (wait_for_tpm_stat(chip, VTPM_STATUS_IDLE, duration, - &chip->vendor.read_queue, true) < 0) { + &priv->read_queue, true) < 0) { /* got a signal or timeout, try to cancel */ vtpm_cancel(chip); return -ETIME; @@ -115,7 +117,7 @@ static int vtpm_send(struct tpm_chip *chip, u8 *buf, size_t count) static int vtpm_recv(struct tpm_chip *chip, u8 *buf, size_t count) { - struct tpm_private *priv = TPM_VPRIV(chip); + struct tpm_private *priv = dev_get_drvdata(&chip->dev); struct vtpm_shared_page *shr = priv->shr; unsigned int offset = shr_data_offset(shr); size_t length = shr->length; @@ -124,8 +126,8 @@ static int vtpm_recv(struct tpm_chip *chip, u8 *buf, size_t count) return -ECANCELED; /* In theory the wait at the end of _send makes this one unnecessary */ - if (wait_for_tpm_stat(chip, VTPM_STATUS_RESULT, chip->vendor.timeout_c, - &chip->vendor.read_queue, true) < 0) { + if (wait_for_tpm_stat(chip, VTPM_STATUS_RESULT, chip->timeout_c, + &priv->read_queue, true) < 0) { vtpm_cancel(chip); return -ETIME; } @@ -161,7 +163,7 @@ static irqreturn_t tpmif_interrupt(int dummy, void *dev_id) switch (priv->shr->state) { case VTPM_STATE_IDLE: case VTPM_STATE_FINISH: - wake_up_interruptible(&priv->chip->vendor.read_queue); + wake_up_interruptible(&priv->read_queue); break; case VTPM_STATE_SUBMIT: case VTPM_STATE_CANCEL: @@ -179,10 +181,10 @@ static int setup_chip(struct device *dev, struct tpm_private *priv) if (IS_ERR(chip)) return PTR_ERR(chip); - init_waitqueue_head(&chip->vendor.read_queue); + init_waitqueue_head(&priv->read_queue); priv->chip = chip; - TPM_VPRIV(chip) = priv; + dev_set_drvdata(&chip->dev, priv); return 0; } @@ -217,7 +219,7 @@ static int setup_ring(struct xenbus_device *dev, struct tpm_private *priv) xenbus_dev_fatal(dev, rv, "allocating TPM irq"); return rv; } - priv->chip->vendor.irq = rv; + priv->irq = rv; again: rv = xenbus_transaction_start(&xbt); @@ -277,8 +279,8 @@ static void ring_free(struct tpm_private *priv) else free_page((unsigned long)priv->shr); - if (priv->chip && priv->chip->vendor.irq) - unbind_from_irqhandler(priv->chip->vendor.irq, priv); + if (priv->irq) + unbind_from_irqhandler(priv->irq, priv); kfree(priv); } @@ -318,10 +320,10 @@ static int tpmfront_probe(struct xenbus_device *dev, static int tpmfront_remove(struct xenbus_device *dev) { struct tpm_chip *chip = dev_get_drvdata(&dev->dev); - struct tpm_private *priv = TPM_VPRIV(chip); + struct tpm_private *priv = dev_get_drvdata(&chip->dev); tpm_chip_unregister(chip); ring_free(priv); - TPM_VPRIV(chip) = NULL; + dev_set_drvdata(&chip->dev, NULL); return 0; } diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index d2406fe25..5da47e26a 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -165,6 +165,12 @@ struct ports_device { */ struct virtqueue *c_ivq, *c_ovq; + /* + * A control packet buffer for guest->host requests, protected + * by c_ovq_lock. + */ + struct virtio_console_control cpkt; + /* Array of per-port IO virtqueues */ struct virtqueue **in_vqs, **out_vqs; @@ -560,28 +566,29 @@ static ssize_t __send_control_msg(struct ports_device *portdev, u32 port_id, unsigned int event, unsigned int value) { struct scatterlist sg[1]; - struct virtio_console_control cpkt; struct virtqueue *vq; unsigned int len; if (!use_multiport(portdev)) return 0; - cpkt.id = cpu_to_virtio32(portdev->vdev, port_id); - cpkt.event = cpu_to_virtio16(portdev->vdev, event); - cpkt.value = cpu_to_virtio16(portdev->vdev, value); - vq = portdev->c_ovq; - sg_init_one(sg, &cpkt, sizeof(cpkt)); - spin_lock(&portdev->c_ovq_lock); - if (virtqueue_add_outbuf(vq, sg, 1, &cpkt, GFP_ATOMIC) == 0) { + + portdev->cpkt.id = cpu_to_virtio32(portdev->vdev, port_id); + portdev->cpkt.event = cpu_to_virtio16(portdev->vdev, event); + portdev->cpkt.value = cpu_to_virtio16(portdev->vdev, value); + + sg_init_one(sg, &portdev->cpkt, sizeof(struct virtio_console_control)); + + if (virtqueue_add_outbuf(vq, sg, 1, &portdev->cpkt, GFP_ATOMIC) == 0) { virtqueue_kick(vq); while (!virtqueue_get_buf(vq, &len) && !virtqueue_is_broken(vq)) cpu_relax(); } + spin_unlock(&portdev->c_ovq_lock); return 0; } diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 98efbfcdb..e2d9bd760 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -49,10 +49,10 @@ config COMMON_CLK_MAX77802 This driver supports Maxim 77802 crystal oscillator clock. config COMMON_CLK_RK808 - tristate "Clock driver for RK808" + tristate "Clock driver for RK808/RK818" depends on MFD_RK808 ---help--- - This driver supports RK808 crystal oscillator clock. These + This driver supports RK808 and RK818 crystal oscillator clock. These multi-function devices have two fixed-rate oscillators, clocked at 32KHz each. Clkout1 is always on, Clkout2 can off by control register. @@ -203,16 +203,19 @@ config COMMON_CLK_PIC32 config COMMON_CLK_OXNAS bool "Clock driver for the OXNAS SoC Family" + depends on ARCH_OXNAS || COMPILE_TEST select MFD_SYSCON ---help--- Support for the OXNAS SoC Family clocks. source "drivers/clk/bcm/Kconfig" source "drivers/clk/hisilicon/Kconfig" +source "drivers/clk/meson/Kconfig" source "drivers/clk/mvebu/Kconfig" source "drivers/clk/qcom/Kconfig" source "drivers/clk/renesas/Kconfig" source "drivers/clk/samsung/Kconfig" +source "drivers/clk/sunxi-ng/Kconfig" source "drivers/clk/tegra/Kconfig" source "drivers/clk/ti/Kconfig" diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index dcc5e698f..3b6f9cf34 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -16,13 +16,14 @@ obj-$(CONFIG_COMMON_CLK) += clk-conf.o endif # hardware specific clock types -# please keep this section sorted lexicographically by file/directory path name +# please keep this section sorted lexicographically by file path name obj-$(CONFIG_MACH_ASM9260) += clk-asm9260.o obj-$(CONFIG_COMMON_CLK_AXI_CLKGEN) += clk-axi-clkgen.o obj-$(CONFIG_ARCH_AXXIA) += clk-axm5516.o obj-$(CONFIG_COMMON_CLK_CDCE706) += clk-cdce706.o -obj-$(CONFIG_COMMON_CLK_CS2000_CP) += clk-cs2000-cp.o +obj-$(CONFIG_COMMON_CLK_CDCE925) += clk-cdce925.o obj-$(CONFIG_ARCH_CLPS711X) += clk-clps711x.o +obj-$(CONFIG_COMMON_CLK_CS2000_CP) += clk-cs2000-cp.o obj-$(CONFIG_ARCH_EFM32) += clk-efm32gg.o obj-$(CONFIG_ARCH_HIGHBANK) += clk-highbank.o obj-$(CONFIG_MACH_LOONGSON32) += clk-ls1x.o @@ -35,6 +36,7 @@ obj-$(CONFIG_ARCH_NOMADIK) += clk-nomadik.o obj-$(CONFIG_ARCH_NSPIRE) += clk-nspire.o obj-$(CONFIG_COMMON_CLK_OXNAS) += clk-oxnas.o obj-$(CONFIG_COMMON_CLK_PALMAS) += clk-palmas.o +obj-$(CONFIG_COMMON_CLK_PWM) += clk-pwm.o obj-$(CONFIG_CLK_QORIQ) += clk-qoriq.o obj-$(CONFIG_COMMON_CLK_RK808) += clk-rk808.o obj-$(CONFIG_COMMON_CLK_S2MPS11) += clk-s2mps11.o @@ -42,7 +44,6 @@ obj-$(CONFIG_COMMON_CLK_SCPI) += clk-scpi.o obj-$(CONFIG_COMMON_CLK_SI5351) += clk-si5351.o obj-$(CONFIG_COMMON_CLK_SI514) += clk-si514.o obj-$(CONFIG_COMMON_CLK_SI570) += clk-si570.o -obj-$(CONFIG_COMMON_CLK_CDCE925) += clk-cdce925.o obj-$(CONFIG_ARCH_STM32) += clk-stm32f4.o obj-$(CONFIG_ARCH_TANGO) += clk-tango4.o obj-$(CONFIG_CLK_TWL6040) += clk-twl6040.o @@ -50,35 +51,39 @@ obj-$(CONFIG_ARCH_U300) += clk-u300.o obj-$(CONFIG_ARCH_VT8500) += clk-vt8500.o obj-$(CONFIG_COMMON_CLK_WM831X) += clk-wm831x.o obj-$(CONFIG_COMMON_CLK_XGENE) += clk-xgene.o -obj-$(CONFIG_COMMON_CLK_PWM) += clk-pwm.o + +# please keep this section sorted lexicographically by directory path name obj-$(CONFIG_COMMON_CLK_AT91) += at91/ obj-$(CONFIG_ARCH_ARTPEC) += axis/ +obj-$(CONFIG_ARC_PLAT_AXS10X) += axs10x/ obj-y += bcm/ obj-$(CONFIG_ARCH_BERLIN) += berlin/ +obj-$(CONFIG_H8300) += h8300/ obj-$(CONFIG_ARCH_HISI) += hisilicon/ obj-$(CONFIG_ARCH_MXC) += imx/ obj-$(CONFIG_MACH_INGENIC) += ingenic/ obj-$(CONFIG_COMMON_CLK_KEYSTONE) += keystone/ obj-$(CONFIG_ARCH_MEDIATEK) += mediatek/ +obj-$(CONFIG_COMMON_CLK_AMLOGIC) += meson/ obj-$(CONFIG_MACH_PIC32) += microchip/ ifeq ($(CONFIG_COMMON_CLK), y) obj-$(CONFIG_ARCH_MMP) += mmp/ endif obj-y += mvebu/ -obj-$(CONFIG_ARCH_MESON) += meson/ obj-$(CONFIG_ARCH_MXS) += mxs/ -obj-$(CONFIG_MACH_PISTACHIO) += pistachio/ obj-$(CONFIG_COMMON_CLK_NXP) += nxp/ +obj-$(CONFIG_MACH_PISTACHIO) += pistachio/ obj-$(CONFIG_COMMON_CLK_PXA) += pxa/ obj-$(CONFIG_COMMON_CLK_QCOM) += qcom/ +obj-$(CONFIG_ARCH_RENESAS) += renesas/ obj-$(CONFIG_ARCH_ROCKCHIP) += rockchip/ obj-$(CONFIG_COMMON_CLK_SAMSUNG) += samsung/ -obj-$(CONFIG_ARCH_RENESAS) += renesas/ obj-$(CONFIG_ARCH_SIRF) += sirf/ obj-$(CONFIG_ARCH_SOCFPGA) += socfpga/ obj-$(CONFIG_PLAT_SPEAR) += spear/ obj-$(CONFIG_ARCH_STI) += st/ obj-$(CONFIG_ARCH_SUNXI) += sunxi/ +obj-$(CONFIG_ARCH_SUNXI) += sunxi-ng/ obj-$(CONFIG_ARCH_TEGRA) += tegra/ obj-y += ti/ obj-$(CONFIG_ARCH_U8500) += ux500/ @@ -86,5 +91,3 @@ obj-$(CONFIG_COMMON_CLK_VERSATILE) += versatile/ obj-$(CONFIG_X86) += x86/ obj-$(CONFIG_ARCH_ZX) += zte/ obj-$(CONFIG_ARCH_ZYNQ) += zynq/ -obj-$(CONFIG_H8300) += h8300/ -obj-$(CONFIG_ARC_PLAT_AXS10X) += axs10x/ diff --git a/drivers/clk/at91/clk-generated.c b/drivers/clk/at91/clk-generated.c index e1aa210dd..7f6bec883 100644 --- a/drivers/clk/at91/clk-generated.c +++ b/drivers/clk/at91/clk-generated.c @@ -267,7 +267,7 @@ at91_clk_register_generated(struct regmap *regmap, spinlock_t *lock, const char return clk; } -void __init of_sama5d2_clk_generated_setup(struct device_node *np) +static void __init of_sama5d2_clk_generated_setup(struct device_node *np) { int num; u32 id; diff --git a/drivers/clk/bcm/clk-iproc-armpll.c b/drivers/clk/bcm/clk-iproc-armpll.c index a196ee28a..d7d628214 100644 --- a/drivers/clk/bcm/clk-iproc-armpll.c +++ b/drivers/clk/bcm/clk-iproc-armpll.c @@ -20,6 +20,8 @@ #include #include +#include "clk-iproc.h" + #define IPROC_CLK_MAX_FREQ_POLICY 0x3 #define IPROC_CLK_POLICY_FREQ_OFFSET 0x008 #define IPROC_CLK_POLICY_FREQ_POLICY_FREQ_SHIFT 8 @@ -242,7 +244,6 @@ static const struct clk_ops iproc_arm_pll_ops = { void __init iproc_armpll_setup(struct device_node *node) { int ret; - struct clk *clk; struct iproc_arm_pll *pll; struct clk_init_data init; const char *parent_name; @@ -263,18 +264,18 @@ void __init iproc_armpll_setup(struct device_node *node) init.num_parents = (parent_name ? 1 : 0); pll->hw.init = &init; - clk = clk_register(NULL, &pll->hw); - if (WARN_ON(IS_ERR(clk))) + ret = clk_hw_register(NULL, &pll->hw); + if (WARN_ON(ret)) goto err_iounmap; - ret = of_clk_add_provider(node, of_clk_src_simple_get, clk); + ret = of_clk_add_hw_provider(node, of_clk_hw_simple_get, &pll->hw); if (WARN_ON(ret)) goto err_clk_unregister; return; err_clk_unregister: - clk_unregister(clk); + clk_hw_unregister(&pll->hw); err_iounmap: iounmap(pll->base); err_free_pll: diff --git a/drivers/clk/bcm/clk-iproc-asiu.c b/drivers/clk/bcm/clk-iproc-asiu.c index f630e1bbd..4360e4813 100644 --- a/drivers/clk/bcm/clk-iproc-asiu.c +++ b/drivers/clk/bcm/clk-iproc-asiu.c @@ -37,7 +37,7 @@ struct iproc_asiu { void __iomem *div_base; void __iomem *gate_base; - struct clk_onecell_data clk_data; + struct clk_hw_onecell_data *clk_data; struct iproc_asiu_clk *clks; }; @@ -197,11 +197,11 @@ void __init iproc_asiu_setup(struct device_node *node, if (WARN_ON(!asiu)) return; - asiu->clk_data.clk_num = num_clks; - asiu->clk_data.clks = kcalloc(num_clks, sizeof(*asiu->clk_data.clks), - GFP_KERNEL); - if (WARN_ON(!asiu->clk_data.clks)) + asiu->clk_data = kzalloc(sizeof(*asiu->clk_data->hws) * num_clks + + sizeof(*asiu->clk_data), GFP_KERNEL); + if (WARN_ON(!asiu->clk_data)) goto err_clks; + asiu->clk_data->num = num_clks; asiu->clks = kcalloc(num_clks, sizeof(*asiu->clks), GFP_KERNEL); if (WARN_ON(!asiu->clks)) @@ -217,7 +217,6 @@ void __init iproc_asiu_setup(struct device_node *node, for (i = 0; i < num_clks; i++) { struct clk_init_data init; - struct clk *clk; const char *parent_name; struct iproc_asiu_clk *asiu_clk; const char *clk_name; @@ -240,22 +239,22 @@ void __init iproc_asiu_setup(struct device_node *node, init.num_parents = (parent_name ? 1 : 0); asiu_clk->hw.init = &init; - clk = clk_register(NULL, &asiu_clk->hw); - if (WARN_ON(IS_ERR(clk))) + ret = clk_hw_register(NULL, &asiu_clk->hw); + if (WARN_ON(ret)) goto err_clk_register; - asiu->clk_data.clks[i] = clk; + asiu->clk_data->hws[i] = &asiu_clk->hw; } - ret = of_clk_add_provider(node, of_clk_src_onecell_get, - &asiu->clk_data); + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, + asiu->clk_data); if (WARN_ON(ret)) goto err_clk_register; return; err_clk_register: - for (i = 0; i < num_clks; i++) - clk_unregister(asiu->clk_data.clks[i]); + while (--i >= 0) + clk_hw_unregister(asiu->clk_data->hws[i]); iounmap(asiu->gate_base); err_iomap_gate: @@ -265,7 +264,7 @@ err_iomap_div: kfree(asiu->clks); err_asiu_clks: - kfree(asiu->clk_data.clks); + kfree(asiu->clk_data); err_clks: kfree(asiu); diff --git a/drivers/clk/bcm/clk-iproc-pll.c b/drivers/clk/bcm/clk-iproc-pll.c index fd492a5da..e04634c46 100644 --- a/drivers/clk/bcm/clk-iproc-pll.c +++ b/drivers/clk/bcm/clk-iproc-pll.c @@ -89,7 +89,7 @@ struct iproc_pll { const struct iproc_pll_vco_param *vco_param; unsigned int num_vco_entries; - struct clk_onecell_data clk_data; + struct clk_hw_onecell_data *clk_data; struct iproc_clk *clks; }; @@ -625,7 +625,6 @@ void __init iproc_pll_clk_setup(struct device_node *node, unsigned int num_clks) { int i, ret; - struct clk *clk; struct iproc_pll *pll; struct iproc_clk *iclk; struct clk_init_data init; @@ -638,11 +637,11 @@ void __init iproc_pll_clk_setup(struct device_node *node, if (WARN_ON(!pll)) return; - pll->clk_data.clk_num = num_clks; - pll->clk_data.clks = kcalloc(num_clks, sizeof(*pll->clk_data.clks), - GFP_KERNEL); - if (WARN_ON(!pll->clk_data.clks)) + pll->clk_data = kzalloc(sizeof(*pll->clk_data->hws) * num_clks + + sizeof(*pll->clk_data), GFP_KERNEL); + if (WARN_ON(!pll->clk_data)) goto err_clk_data; + pll->clk_data->num = num_clks; pll->clks = kcalloc(num_clks, sizeof(*pll->clks), GFP_KERNEL); if (WARN_ON(!pll->clks)) @@ -694,11 +693,11 @@ void __init iproc_pll_clk_setup(struct device_node *node, iproc_pll_sw_cfg(pll); - clk = clk_register(NULL, &iclk->hw); - if (WARN_ON(IS_ERR(clk))) + ret = clk_hw_register(NULL, &iclk->hw); + if (WARN_ON(ret)) goto err_pll_register; - pll->clk_data.clks[0] = clk; + pll->clk_data->hws[0] = &iclk->hw; /* now initialize and register all leaf clocks */ for (i = 1; i < num_clks; i++) { @@ -724,22 +723,23 @@ void __init iproc_pll_clk_setup(struct device_node *node, init.num_parents = (parent_name ? 1 : 0); iclk->hw.init = &init; - clk = clk_register(NULL, &iclk->hw); - if (WARN_ON(IS_ERR(clk))) + ret = clk_hw_register(NULL, &iclk->hw); + if (WARN_ON(ret)) goto err_clk_register; - pll->clk_data.clks[i] = clk; + pll->clk_data->hws[i] = &iclk->hw; } - ret = of_clk_add_provider(node, of_clk_src_onecell_get, &pll->clk_data); + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, + pll->clk_data); if (WARN_ON(ret)) goto err_clk_register; return; err_clk_register: - for (i = 0; i < num_clks; i++) - clk_unregister(pll->clk_data.clks[i]); + while (--i >= 0) + clk_hw_unregister(pll->clk_data->hws[i]); err_pll_register: if (pll->status_base != pll->control_base) @@ -759,7 +759,7 @@ err_pll_iomap: kfree(pll->clks); err_clks: - kfree(pll->clk_data.clks); + kfree(pll->clk_data); err_clk_data: kfree(pll); diff --git a/drivers/clk/clk-clps711x.c b/drivers/clk/clk-clps711x.c index 1f60b0241..adaf109f2 100644 --- a/drivers/clk/clk-clps711x.c +++ b/drivers/clk/clk-clps711x.c @@ -184,5 +184,5 @@ static void __init clps711x_clk_init_dt(struct device_node *np) of_clk_add_provider(np, of_clk_src_onecell_get, &clps711x_clk->clk_data); } -CLK_OF_DECLARE(clps711x, "cirrus,clps711x-clk", clps711x_clk_init_dt); +CLK_OF_DECLARE(clps711x, "cirrus,ep7209-clk", clps711x_clk_init_dt); #endif diff --git a/drivers/clk/clk-conf.c b/drivers/clk/clk-conf.c index 43a218f35..674785d96 100644 --- a/drivers/clk/clk-conf.c +++ b/drivers/clk/clk-conf.c @@ -55,7 +55,7 @@ static int __set_clk_parents(struct device_node *node, bool clk_supplier) } clk = of_clk_get_from_provider(&clkspec); if (IS_ERR(clk)) { - pr_warn("clk: couldn't get parent clock %d for %s\n", + pr_warn("clk: couldn't get assigned clock %d for %s\n", index, node->full_name); rc = PTR_ERR(clk); goto err; diff --git a/drivers/clk/clk-fixed-factor.c b/drivers/clk/clk-fixed-factor.c index 75cd6c792..4db3be214 100644 --- a/drivers/clk/clk-fixed-factor.c +++ b/drivers/clk/clk-fixed-factor.c @@ -142,6 +142,11 @@ void clk_hw_unregister_fixed_factor(struct clk_hw *hw) EXPORT_SYMBOL_GPL(clk_hw_unregister_fixed_factor); #ifdef CONFIG_OF +static const struct of_device_id set_rate_parent_matches[] = { + { .compatible = "allwinner,sun4i-a10-pll3-2x-clk" }, + { /* Sentinel */ }, +}; + /** * of_fixed_factor_clk_setup() - Setup function for simple fixed factor clock */ @@ -150,6 +155,7 @@ void __init of_fixed_factor_clk_setup(struct device_node *node) struct clk *clk; const char *clk_name = node->name; const char *parent_name; + unsigned long flags = 0; u32 div, mult; if (of_property_read_u32(node, "clock-div", &div)) { @@ -167,7 +173,10 @@ void __init of_fixed_factor_clk_setup(struct device_node *node) of_property_read_string(node, "clock-output-names", &clk_name); parent_name = of_clk_get_parent_name(node, 0); - clk = clk_register_fixed_factor(NULL, clk_name, parent_name, 0, + if (of_match_node(set_rate_parent_matches, node)) + flags |= CLK_SET_RATE_PARENT; + + clk = clk_register_fixed_factor(NULL, clk_name, parent_name, flags, mult, div); if (!IS_ERR(clk)) of_clk_add_provider(node, of_clk_src_simple_get, clk); diff --git a/drivers/clk/clk-fixed-rate.c b/drivers/clk/clk-fixed-rate.c index 8e4453eb5..2edb39342 100644 --- a/drivers/clk/clk-fixed-rate.c +++ b/drivers/clk/clk-fixed-rate.c @@ -145,6 +145,17 @@ void clk_unregister_fixed_rate(struct clk *clk) } EXPORT_SYMBOL_GPL(clk_unregister_fixed_rate); +void clk_hw_unregister_fixed_rate(struct clk_hw *hw) +{ + struct clk_fixed_rate *fixed; + + fixed = to_clk_fixed_rate(hw); + + clk_hw_unregister(hw); + kfree(fixed); +} +EXPORT_SYMBOL_GPL(clk_hw_unregister_fixed_rate); + #ifdef CONFIG_OF /** * of_fixed_clk_setup() - Setup function for simple fixed rate clock diff --git a/drivers/clk/clk-highbank.c b/drivers/clk/clk-highbank.c index be3a21abb..727ed8e1b 100644 --- a/drivers/clk/clk-highbank.c +++ b/drivers/clk/clk-highbank.c @@ -275,7 +275,6 @@ static const struct clk_ops periclk_ops = { static __init struct clk *hb_clk_init(struct device_node *node, const struct clk_ops *ops) { u32 reg; - struct clk *clk; struct hb_clk *hb_clk; const char *clk_name = node->name; const char *parent_name; @@ -308,13 +307,13 @@ static __init struct clk *hb_clk_init(struct device_node *node, const struct clk hb_clk->hw.init = &init; - clk = clk_register(NULL, &hb_clk->hw); - if (WARN_ON(IS_ERR(clk))) { + rc = clk_hw_register(NULL, &hb_clk->hw); + if (WARN_ON(rc)) { kfree(hb_clk); return NULL; } - rc = of_clk_add_provider(node, of_clk_src_simple_get, clk); - return clk; + rc = of_clk_add_hw_provider(node, of_clk_hw_simple_get, &hb_clk->hw); + return hb_clk->hw.clk; } static void __init hb_pll_init(struct device_node *node) diff --git a/drivers/clk/clk-multiplier.c b/drivers/clk/clk-multiplier.c index 9e449c7b7..dc037c957 100644 --- a/drivers/clk/clk-multiplier.c +++ b/drivers/clk/clk-multiplier.c @@ -52,14 +52,28 @@ static unsigned long __bestmult(struct clk_hw *hw, unsigned long rate, unsigned long *best_parent_rate, u8 width, unsigned long flags) { + struct clk_multiplier *mult = to_clk_multiplier(hw); unsigned long orig_parent_rate = *best_parent_rate; unsigned long parent_rate, current_rate, best_rate = ~0; unsigned int i, bestmult = 0; + unsigned int maxmult = (1 << width) - 1; + + if (!(clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT)) { + bestmult = rate / orig_parent_rate; + + /* Make sure we don't end up with a 0 multiplier */ + if ((bestmult == 0) && + !(mult->flags & CLK_MULTIPLIER_ZERO_BYPASS)) + bestmult = 1; - if (!(clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT)) - return rate / *best_parent_rate; + /* Make sure we don't overflow the multiplier */ + if (bestmult > maxmult) + bestmult = maxmult; + + return bestmult; + } - for (i = 1; i < ((1 << width) - 1); i++) { + for (i = 1; i < maxmult; i++) { if (rate == orig_parent_rate * i) { /* * This is the best case for us if we have a diff --git a/drivers/clk/clk-nomadik.c b/drivers/clk/clk-nomadik.c index e4d8a991c..71677eb12 100644 --- a/drivers/clk/clk-nomadik.c +++ b/drivers/clk/clk-nomadik.c @@ -253,11 +253,11 @@ static const struct clk_ops pll_clk_ops = { .recalc_rate = pll_clk_recalc_rate, }; -static struct clk * __init +static struct clk_hw * __init pll_clk_register(struct device *dev, const char *name, const char *parent_name, u32 id) { - struct clk *clk; + int ret; struct clk_pll *pll; struct clk_init_data init; @@ -281,11 +281,13 @@ pll_clk_register(struct device *dev, const char *name, pr_debug("register PLL1 clock \"%s\"\n", name); - clk = clk_register(dev, &pll->hw); - if (IS_ERR(clk)) + ret = clk_hw_register(dev, &pll->hw); + if (ret) { kfree(pll); + return ERR_PTR(ret); + } - return clk; + return &pll->hw; } /* @@ -345,11 +347,11 @@ static const struct clk_ops src_clk_ops = { .recalc_rate = src_clk_recalc_rate, }; -static struct clk * __init +static struct clk_hw * __init src_clk_register(struct device *dev, const char *name, const char *parent_name, u8 id) { - struct clk *clk; + int ret; struct clk_src *sclk; struct clk_init_data init; @@ -376,11 +378,13 @@ src_clk_register(struct device *dev, const char *name, pr_debug("register clock \"%s\" ID: %d group: %d bits: %08x\n", name, id, sclk->group1, sclk->clkbit); - clk = clk_register(dev, &sclk->hw); - if (IS_ERR(clk)) + ret = clk_hw_register(dev, &sclk->hw); + if (ret) { kfree(sclk); + return ERR_PTR(ret); + } - return clk; + return &sclk->hw; } #ifdef CONFIG_DEBUG_FS @@ -508,7 +512,7 @@ device_initcall(nomadik_src_clk_init_debugfs); static void __init of_nomadik_pll_setup(struct device_node *np) { - struct clk *clk = ERR_PTR(-EINVAL); + struct clk_hw *hw; const char *clk_name = np->name; const char *parent_name; u32 pll_id; @@ -522,16 +526,16 @@ static void __init of_nomadik_pll_setup(struct device_node *np) return; } parent_name = of_clk_get_parent_name(np, 0); - clk = pll_clk_register(NULL, clk_name, parent_name, pll_id); - if (!IS_ERR(clk)) - of_clk_add_provider(np, of_clk_src_simple_get, clk); + hw = pll_clk_register(NULL, clk_name, parent_name, pll_id); + if (!IS_ERR(hw)) + of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); } CLK_OF_DECLARE(nomadik_pll_clk, "st,nomadik-pll-clock", of_nomadik_pll_setup); static void __init of_nomadik_hclk_setup(struct device_node *np) { - struct clk *clk = ERR_PTR(-EINVAL); + struct clk_hw *hw; const char *clk_name = np->name; const char *parent_name; @@ -542,20 +546,20 @@ static void __init of_nomadik_hclk_setup(struct device_node *np) /* * The HCLK divides PLL1 with 1 (passthru), 2, 3 or 4. */ - clk = clk_register_divider(NULL, clk_name, parent_name, + hw = clk_hw_register_divider(NULL, clk_name, parent_name, 0, src_base + SRC_CR, 13, 2, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &src_lock); - if (!IS_ERR(clk)) - of_clk_add_provider(np, of_clk_src_simple_get, clk); + if (!IS_ERR(hw)) + of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); } CLK_OF_DECLARE(nomadik_hclk_clk, "st,nomadik-hclk-clock", of_nomadik_hclk_setup); static void __init of_nomadik_src_clk_setup(struct device_node *np) { - struct clk *clk = ERR_PTR(-EINVAL); + struct clk_hw *hw; const char *clk_name = np->name; const char *parent_name; u32 clk_id; @@ -569,9 +573,9 @@ static void __init of_nomadik_src_clk_setup(struct device_node *np) return; } parent_name = of_clk_get_parent_name(np, 0); - clk = src_clk_register(NULL, clk_name, parent_name, clk_id); - if (!IS_ERR(clk)) - of_clk_add_provider(np, of_clk_src_simple_get, clk); + hw = src_clk_register(NULL, clk_name, parent_name, clk_id); + if (!IS_ERR(hw)) + of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); } CLK_OF_DECLARE(nomadik_src_clk, "st,nomadik-src-clock", of_nomadik_src_clk_setup); diff --git a/drivers/clk/clk-oxnas.c b/drivers/clk/clk-oxnas.c index 79bcb2e42..47649ac5d 100644 --- a/drivers/clk/clk-oxnas.c +++ b/drivers/clk/clk-oxnas.c @@ -18,7 +18,7 @@ #include #include -#include +#include #include #include #include @@ -170,26 +170,17 @@ static int oxnas_stdclk_probe(struct platform_device *pdev) clk_oxnas->onecell_data); } -static int oxnas_stdclk_remove(struct platform_device *pdev) -{ - of_clk_del_provider(pdev->dev.of_node); - - return 0; -} - static const struct of_device_id oxnas_stdclk_dt_ids[] = { { .compatible = "oxsemi,ox810se-stdclk" }, { } }; -MODULE_DEVICE_TABLE(of, oxnas_stdclk_dt_ids); static struct platform_driver oxnas_stdclk_driver = { .probe = oxnas_stdclk_probe, - .remove = oxnas_stdclk_remove, .driver = { .name = "oxnas-stdclk", + .suppress_bind_attrs = true, .of_match_table = oxnas_stdclk_dt_ids, }, }; - -module_platform_driver(oxnas_stdclk_driver); +builtin_platform_driver(oxnas_stdclk_driver); diff --git a/drivers/clk/clk-s2mps11.c b/drivers/clk/clk-s2mps11.c index f8c83977c..fbaa84a33 100644 --- a/drivers/clk/clk-s2mps11.c +++ b/drivers/clk/clk-s2mps11.c @@ -137,7 +137,7 @@ static int s2mps11_clk_probe(struct platform_device *pdev) { struct sec_pmic_dev *iodev = dev_get_drvdata(pdev->dev.parent); struct s2mps11_clk *s2mps11_clks; - struct clk_onecell_data *clk_data; + struct clk_hw_onecell_data *clk_data; unsigned int s2mps11_reg; int i, ret = 0; enum sec_device_type hwid = platform_get_device_id(pdev)->driver_data; @@ -147,15 +147,12 @@ static int s2mps11_clk_probe(struct platform_device *pdev) if (!s2mps11_clks) return -ENOMEM; - clk_data = devm_kzalloc(&pdev->dev, sizeof(*clk_data), GFP_KERNEL); + clk_data = devm_kzalloc(&pdev->dev, sizeof(*clk_data) + + sizeof(*clk_data->hws) * S2MPS11_CLKS_NUM, + GFP_KERNEL); if (!clk_data) return -ENOMEM; - clk_data->clks = devm_kcalloc(&pdev->dev, S2MPS11_CLKS_NUM, - sizeof(struct clk *), GFP_KERNEL); - if (!clk_data->clks) - return -ENOMEM; - switch (hwid) { case S2MPS11X: s2mps11_reg = S2MPS11_REG_RTC_CTRL; @@ -196,18 +193,18 @@ static int s2mps11_clk_probe(struct platform_device *pdev) goto err_reg; } - s2mps11_clks[i].lookup = clkdev_create(s2mps11_clks[i].clk, + s2mps11_clks[i].lookup = clkdev_hw_create(&s2mps11_clks[i].hw, s2mps11_clks_init[i].name, NULL); if (!s2mps11_clks[i].lookup) { ret = -ENOMEM; goto err_reg; } - clk_data->clks[i] = s2mps11_clks[i].clk; + clk_data->hws[i] = &s2mps11_clks[i].hw; } - clk_data->clk_num = S2MPS11_CLKS_NUM; - of_clk_add_provider(s2mps11_clks->clk_np, of_clk_src_onecell_get, - clk_data); + clk_data->num = S2MPS11_CLKS_NUM; + of_clk_add_hw_provider(s2mps11_clks->clk_np, of_clk_hw_onecell_get, + clk_data); platform_set_drvdata(pdev, s2mps11_clks); diff --git a/drivers/clk/clk-stm32f4.c b/drivers/clk/clk-stm32f4.c index fd89e7711..02d681008 100644 --- a/drivers/clk/clk-stm32f4.c +++ b/drivers/clk/clk-stm32f4.c @@ -136,7 +136,7 @@ static const u64 stm32f42xx_gate_map[] = { 0x000000f17ef417ffull, 0x0000000000000001ull, 0x04777f33f6fec9ffull }; -static struct clk *clks[MAX_CLKS]; +static struct clk_hw *clks[MAX_CLKS]; static DEFINE_SPINLOCK(stm32f4_clk_lock); static void __iomem *base; @@ -281,7 +281,7 @@ static int stm32f4_rcc_lookup_clk_idx(u8 primary, u8 secondary) (BIT_ULL_WORD(secondary) >= 2 ? hweight64(table[2]) : 0); } -static struct clk * +static struct clk_hw * stm32f4_rcc_lookup_clk(struct of_phandle_args *clkspec, void *data) { int i = stm32f4_rcc_lookup_clk_idx(clkspec->args[0], clkspec->args[1]); @@ -346,9 +346,9 @@ static void __init stm32f4_rcc_init(struct device_node *np) clk_register_apb_mul(NULL, "apb2_mul", "apb2_div", CLK_SET_RATE_PARENT, 15); - clks[SYSTICK] = clk_register_fixed_factor(NULL, "systick", "ahb_div", + clks[SYSTICK] = clk_hw_register_fixed_factor(NULL, "systick", "ahb_div", 0, 1, 8); - clks[FCLK] = clk_register_fixed_factor(NULL, "fclk", "ahb_div", + clks[FCLK] = clk_hw_register_fixed_factor(NULL, "fclk", "ahb_div", 0, 1, 1); for (n = 0; n < ARRAY_SIZE(stm32f4_gates); n++) { @@ -360,18 +360,18 @@ static void __init stm32f4_rcc_init(struct device_node *np) if (idx < 0) goto fail; - clks[idx] = clk_register_gate( + clks[idx] = clk_hw_register_gate( NULL, gd->name, gd->parent_name, gd->flags, base + gd->offset, gd->bit_idx, 0, &stm32f4_clk_lock); - if (IS_ERR(clks[n])) { + if (IS_ERR(clks[idx])) { pr_err("%s: Unable to register leaf clock %s\n", np->full_name, gd->name); goto fail; } } - of_clk_add_provider(np, stm32f4_rcc_lookup_clk, NULL); + of_clk_add_hw_provider(np, stm32f4_rcc_lookup_clk, NULL); return; fail: iounmap(base); diff --git a/drivers/clk/clk-u300.c b/drivers/clk/clk-u300.c index 95d1742da..ec8aafda6 100644 --- a/drivers/clk/clk-u300.c +++ b/drivers/clk/clk-u300.c @@ -689,7 +689,7 @@ static const struct clk_ops syscon_clk_ops = { .set_rate = syscon_clk_set_rate, }; -static struct clk * __init +static struct clk_hw * __init syscon_clk_register(struct device *dev, const char *name, const char *parent_name, unsigned long flags, bool hw_ctrld, @@ -697,9 +697,10 @@ syscon_clk_register(struct device *dev, const char *name, void __iomem *en_reg, u8 en_bit, u16 clk_val) { - struct clk *clk; + struct clk_hw *hw; struct clk_syscon *sclk; struct clk_init_data init; + int ret; sclk = kzalloc(sizeof(struct clk_syscon), GFP_KERNEL); if (!sclk) { @@ -722,11 +723,14 @@ syscon_clk_register(struct device *dev, const char *name, sclk->en_bit = en_bit; sclk->clk_val = clk_val; - clk = clk_register(dev, &sclk->hw); - if (IS_ERR(clk)) + hw = &sclk->hw; + ret = clk_hw_register(dev, hw); + if (ret) { kfree(sclk); + hw = ERR_PTR(ret); + } - return clk; + return hw; } #define U300_CLK_TYPE_SLOW 0 @@ -868,7 +872,7 @@ static struct u300_clock const u300_clk_lookup[] __initconst = { static void __init of_u300_syscon_clk_init(struct device_node *np) { - struct clk *clk = ERR_PTR(-EINVAL); + struct clk_hw *hw = ERR_PTR(-EINVAL); const char *clk_name = np->name; const char *parent_name; void __iomem *res_reg; @@ -911,16 +915,15 @@ static void __init of_u300_syscon_clk_init(struct device_node *np) const struct u300_clock *u3clk = &u300_clk_lookup[i]; if (u3clk->type == clk_type && u3clk->id == clk_id) - clk = syscon_clk_register(NULL, - clk_name, parent_name, - 0, u3clk->hw_ctrld, - res_reg, u3clk->id, - en_reg, u3clk->id, - u3clk->clk_val); + hw = syscon_clk_register(NULL, clk_name, parent_name, + 0, u3clk->hw_ctrld, + res_reg, u3clk->id, + en_reg, u3clk->id, + u3clk->clk_val); } - if (!IS_ERR(clk)) { - of_clk_add_provider(np, of_clk_src_simple_get, clk); + if (!IS_ERR(hw)) { + of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); /* * Some few system clocks - device tree does not @@ -928,11 +931,11 @@ static void __init of_u300_syscon_clk_init(struct device_node *np) * for now we add these three clocks here. */ if (clk_type == U300_CLK_TYPE_REST && clk_id == 5) - clk_register_clkdev(clk, NULL, "pl172"); + clk_hw_register_clkdev(hw, NULL, "pl172"); if (clk_type == U300_CLK_TYPE_REST && clk_id == 9) - clk_register_clkdev(clk, NULL, "semi"); + clk_hw_register_clkdev(hw, NULL, "semi"); if (clk_type == U300_CLK_TYPE_REST && clk_id == 12) - clk_register_clkdev(clk, NULL, "intcon"); + clk_hw_register_clkdev(hw, NULL, "intcon"); } } @@ -1111,13 +1114,14 @@ static const struct clk_ops mclk_ops = { .set_rate = mclk_clk_set_rate, }; -static struct clk * __init +static struct clk_hw * __init mclk_clk_register(struct device *dev, const char *name, const char *parent_name, bool is_mspro) { - struct clk *clk; + struct clk_hw *hw; struct clk_mclk *mclk; struct clk_init_data init; + int ret; mclk = kzalloc(sizeof(struct clk_mclk), GFP_KERNEL); if (!mclk) { @@ -1133,23 +1137,26 @@ mclk_clk_register(struct device *dev, const char *name, mclk->hw.init = &init; mclk->is_mspro = is_mspro; - clk = clk_register(dev, &mclk->hw); - if (IS_ERR(clk)) + hw = &mclk->hw; + ret = clk_hw_register(dev, hw); + if (ret) { kfree(mclk); + hw = ERR_PTR(ret); + } - return clk; + return hw; } static void __init of_u300_syscon_mclk_init(struct device_node *np) { - struct clk *clk = ERR_PTR(-EINVAL); + struct clk_hw *hw; const char *clk_name = np->name; const char *parent_name; parent_name = of_clk_get_parent_name(np, 0); - clk = mclk_clk_register(NULL, clk_name, parent_name, false); - if (!IS_ERR(clk)) - of_clk_add_provider(np, of_clk_src_simple_get, clk); + hw = mclk_clk_register(NULL, clk_name, parent_name, false); + if (!IS_ERR(hw)) + of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); } static const struct of_device_id u300_clk_match[] __initconst = { diff --git a/drivers/clk/clk-vt8500.c b/drivers/clk/clk-vt8500.c index b0f76a84f..37368a399 100644 --- a/drivers/clk/clk-vt8500.c +++ b/drivers/clk/clk-vt8500.c @@ -383,51 +383,49 @@ static int vt8500_find_pll_bits(unsigned long rate, unsigned long parent_rate, return 0; } -static int wm8650_find_pll_bits(unsigned long rate, unsigned long parent_rate, - u32 *multiplier, u32 *divisor1, u32 *divisor2) +/* + * M * parent [O1] => / P [O2] => / D [O3] + * Where O1 is 900MHz...3GHz; + * O2 is 600MHz >= (M * parent) / P >= 300MHz; + * M is 36...120 [25MHz parent]; D is 1 or 2 or 4 or 8. + * Possible ranges (O3): + * D = 8: 37,5MHz...75MHz + * D = 4: 75MHz...150MHz + * D = 2: 150MHz...300MHz + * D = 1: 300MHz...600MHz + */ +static int wm8650_find_pll_bits(unsigned long rate, + unsigned long parent_rate, u32 *multiplier, u32 *divisor1, + u32 *divisor2) { - u32 mul, div1; - int div2; - u32 best_mul, best_div1, best_div2; - unsigned long tclk, rate_err, best_err; - - best_err = (unsigned long)-1; + unsigned long O1, min_err, rate_err; - /* Find the closest match (lower or equal to requested) */ - for (div1 = 5; div1 >= 3; div1--) - for (div2 = 3; div2 >= 0; div2--) - for (mul = 3; mul <= 1023; mul++) { - tclk = parent_rate * mul / (div1 * (1 << div2)); - if (tclk > rate) - continue; - /* error will always be +ve */ - rate_err = rate - tclk; - if (rate_err == 0) { - *multiplier = mul; - *divisor1 = div1; - *divisor2 = div2; - return 0; - } + if (!parent_rate || (rate < 37500000) || (rate > 600000000)) + return -EINVAL; - if (rate_err < best_err) { - best_err = rate_err; - best_mul = mul; - best_div1 = div1; - best_div2 = div2; - } - } + *divisor2 = rate <= 75000000 ? 3 : rate <= 150000000 ? 2 : + rate <= 300000000 ? 1 : 0; + /* + * Divisor P cannot be calculated. Test all divisors and find where M + * will be as close as possible to the requested rate. + */ + min_err = ULONG_MAX; + for (*divisor1 = 5; *divisor1 >= 3; (*divisor1)--) { + O1 = rate * *divisor1 * (1 << (*divisor2)); + rate_err = O1 % parent_rate; + if (rate_err < min_err) { + *multiplier = O1 / parent_rate; + if (rate_err == 0) + return 0; + + min_err = rate_err; + } + } - if (best_err == (unsigned long)-1) { - pr_warn("%s: impossible rate %lu\n", __func__, rate); + if ((*multiplier < 3) || (*multiplier > 1023)) return -EINVAL; - } - /* if we got here, it wasn't an exact match */ - pr_warn("%s: requested rate %lu, found rate %lu\n", __func__, rate, - rate - best_err); - *multiplier = best_mul; - *divisor1 = best_div1; - *divisor2 = best_div2; + pr_warn("%s: rate error is %lu\n", __func__, min_err); return 0; } @@ -464,7 +462,6 @@ static int wm8750_find_pll_bits(unsigned long rate, unsigned long parent_rate, { u32 mul; int div1, div2; - u32 best_mul, best_div1, best_div2; unsigned long tclk, rate_err, best_err; best_err = (unsigned long)-1; @@ -488,9 +485,9 @@ static int wm8750_find_pll_bits(unsigned long rate, unsigned long parent_rate, if (rate_err < best_err) { best_err = rate_err; - best_mul = mul; - best_div1 = div1; - best_div2 = div2; + *multiplier = mul; + *divisor1 = div1; + *divisor2 = div2; } } @@ -503,10 +500,7 @@ static int wm8750_find_pll_bits(unsigned long rate, unsigned long parent_rate, pr_warn("%s: requested rate %lu, found rate %lu\n", __func__, rate, rate - best_err); - *filter = wm8750_get_filter(parent_rate, best_div1); - *multiplier = best_mul; - *divisor1 = best_div1; - *divisor2 = best_div2; + *filter = wm8750_get_filter(parent_rate, *divisor1); return 0; } @@ -516,7 +510,6 @@ static int wm8850_find_pll_bits(unsigned long rate, unsigned long parent_rate, { u32 mul; int div1, div2; - u32 best_mul, best_div1, best_div2; unsigned long tclk, rate_err, best_err; best_err = (unsigned long)-1; @@ -540,9 +533,9 @@ static int wm8850_find_pll_bits(unsigned long rate, unsigned long parent_rate, if (rate_err < best_err) { best_err = rate_err; - best_mul = mul; - best_div1 = div1; - best_div2 = div2; + *multiplier = mul; + *divisor1 = div1; + *divisor2 = div2; } } @@ -555,10 +548,6 @@ static int wm8850_find_pll_bits(unsigned long rate, unsigned long parent_rate, pr_warn("%s: requested rate %lu, found rate %lu\n", __func__, rate, rate - best_err); - *multiplier = best_mul; - *divisor1 = best_div1; - *divisor2 = best_div2; - return 0; } diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index d584004f7..820a939fb 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -172,104 +172,6 @@ static bool clk_core_is_enabled(struct clk_core *core) return core->ops->is_enabled(core->hw); } -static void clk_unprepare_unused_subtree(struct clk_core *core) -{ - struct clk_core *child; - - lockdep_assert_held(&prepare_lock); - - hlist_for_each_entry(child, &core->children, child_node) - clk_unprepare_unused_subtree(child); - - if (core->prepare_count) - return; - - if (core->flags & CLK_IGNORE_UNUSED) - return; - - if (clk_core_is_prepared(core)) { - trace_clk_unprepare(core); - if (core->ops->unprepare_unused) - core->ops->unprepare_unused(core->hw); - else if (core->ops->unprepare) - core->ops->unprepare(core->hw); - trace_clk_unprepare_complete(core); - } -} - -static void clk_disable_unused_subtree(struct clk_core *core) -{ - struct clk_core *child; - unsigned long flags; - - lockdep_assert_held(&prepare_lock); - - hlist_for_each_entry(child, &core->children, child_node) - clk_disable_unused_subtree(child); - - flags = clk_enable_lock(); - - if (core->enable_count) - goto unlock_out; - - if (core->flags & CLK_IGNORE_UNUSED) - goto unlock_out; - - /* - * some gate clocks have special needs during the disable-unused - * sequence. call .disable_unused if available, otherwise fall - * back to .disable - */ - if (clk_core_is_enabled(core)) { - trace_clk_disable(core); - if (core->ops->disable_unused) - core->ops->disable_unused(core->hw); - else if (core->ops->disable) - core->ops->disable(core->hw); - trace_clk_disable_complete(core); - } - -unlock_out: - clk_enable_unlock(flags); -} - -static bool clk_ignore_unused; -static int __init clk_ignore_unused_setup(char *__unused) -{ - clk_ignore_unused = true; - return 1; -} -__setup("clk_ignore_unused", clk_ignore_unused_setup); - -static int clk_disable_unused(void) -{ - struct clk_core *core; - - if (clk_ignore_unused) { - pr_warn("clk: Not disabling unused clocks\n"); - return 0; - } - - clk_prepare_lock(); - - hlist_for_each_entry(core, &clk_root_list, child_node) - clk_disable_unused_subtree(core); - - hlist_for_each_entry(core, &clk_orphan_list, child_node) - clk_disable_unused_subtree(core); - - hlist_for_each_entry(core, &clk_root_list, child_node) - clk_unprepare_unused_subtree(core); - - hlist_for_each_entry(core, &clk_orphan_list, child_node) - clk_unprepare_unused_subtree(core); - - clk_prepare_unlock(); - - return 0; -} -late_initcall_sync(clk_disable_unused); - /*** helper functions ***/ const char *__clk_get_name(const struct clk *clk) @@ -591,6 +493,13 @@ static void clk_core_unprepare(struct clk_core *core) clk_core_unprepare(core->parent); } +static void clk_core_unprepare_lock(struct clk_core *core) +{ + clk_prepare_lock(); + clk_core_unprepare(core); + clk_prepare_unlock(); +} + /** * clk_unprepare - undo preparation of a clock source * @clk: the clk being unprepared @@ -607,9 +516,7 @@ void clk_unprepare(struct clk *clk) if (IS_ERR_OR_NULL(clk)) return; - clk_prepare_lock(); - clk_core_unprepare(clk->core); - clk_prepare_unlock(); + clk_core_unprepare_lock(clk->core); } EXPORT_SYMBOL_GPL(clk_unprepare); @@ -645,6 +552,17 @@ static int clk_core_prepare(struct clk_core *core) return 0; } +static int clk_core_prepare_lock(struct clk_core *core) +{ + int ret; + + clk_prepare_lock(); + ret = clk_core_prepare(core); + clk_prepare_unlock(); + + return ret; +} + /** * clk_prepare - prepare a clock source * @clk: the clk being prepared @@ -659,16 +577,10 @@ static int clk_core_prepare(struct clk_core *core) */ int clk_prepare(struct clk *clk) { - int ret; - if (!clk) return 0; - clk_prepare_lock(); - ret = clk_core_prepare(clk->core); - clk_prepare_unlock(); - - return ret; + return clk_core_prepare_lock(clk->core); } EXPORT_SYMBOL_GPL(clk_prepare); @@ -688,16 +600,25 @@ static void clk_core_disable(struct clk_core *core) if (--core->enable_count > 0) return; - trace_clk_disable(core); + trace_clk_disable_rcuidle(core); if (core->ops->disable) core->ops->disable(core->hw); - trace_clk_disable_complete(core); + trace_clk_disable_complete_rcuidle(core); clk_core_disable(core->parent); } +static void clk_core_disable_lock(struct clk_core *core) +{ + unsigned long flags; + + flags = clk_enable_lock(); + clk_core_disable(core); + clk_enable_unlock(flags); +} + /** * clk_disable - gate a clock * @clk: the clk being gated @@ -712,14 +633,10 @@ static void clk_core_disable(struct clk_core *core) */ void clk_disable(struct clk *clk) { - unsigned long flags; - if (IS_ERR_OR_NULL(clk)) return; - flags = clk_enable_lock(); - clk_core_disable(clk->core); - clk_enable_unlock(flags); + clk_core_disable_lock(clk->core); } EXPORT_SYMBOL_GPL(clk_disable); @@ -741,12 +658,12 @@ static int clk_core_enable(struct clk_core *core) if (ret) return ret; - trace_clk_enable(core); + trace_clk_enable_rcuidle(core); if (core->ops->enable) ret = core->ops->enable(core->hw); - trace_clk_enable_complete(core); + trace_clk_enable_complete_rcuidle(core); if (ret) { clk_core_disable(core->parent); @@ -758,6 +675,18 @@ static int clk_core_enable(struct clk_core *core) return 0; } +static int clk_core_enable_lock(struct clk_core *core) +{ + unsigned long flags; + int ret; + + flags = clk_enable_lock(); + ret = clk_core_enable(core); + clk_enable_unlock(flags); + + return ret; +} + /** * clk_enable - ungate a clock * @clk: the clk being ungated @@ -773,19 +702,136 @@ static int clk_core_enable(struct clk_core *core) */ int clk_enable(struct clk *clk) { - unsigned long flags; - int ret; - if (!clk) return 0; + return clk_core_enable_lock(clk->core); +} +EXPORT_SYMBOL_GPL(clk_enable); + +static int clk_core_prepare_enable(struct clk_core *core) +{ + int ret; + + ret = clk_core_prepare_lock(core); + if (ret) + return ret; + + ret = clk_core_enable_lock(core); + if (ret) + clk_core_unprepare_lock(core); + + return ret; +} + +static void clk_core_disable_unprepare(struct clk_core *core) +{ + clk_core_disable_lock(core); + clk_core_unprepare_lock(core); +} + +static void clk_unprepare_unused_subtree(struct clk_core *core) +{ + struct clk_core *child; + + lockdep_assert_held(&prepare_lock); + + hlist_for_each_entry(child, &core->children, child_node) + clk_unprepare_unused_subtree(child); + + if (core->prepare_count) + return; + + if (core->flags & CLK_IGNORE_UNUSED) + return; + + if (clk_core_is_prepared(core)) { + trace_clk_unprepare(core); + if (core->ops->unprepare_unused) + core->ops->unprepare_unused(core->hw); + else if (core->ops->unprepare) + core->ops->unprepare(core->hw); + trace_clk_unprepare_complete(core); + } +} + +static void clk_disable_unused_subtree(struct clk_core *core) +{ + struct clk_core *child; + unsigned long flags; + + lockdep_assert_held(&prepare_lock); + + hlist_for_each_entry(child, &core->children, child_node) + clk_disable_unused_subtree(child); + + if (core->flags & CLK_OPS_PARENT_ENABLE) + clk_core_prepare_enable(core->parent); + flags = clk_enable_lock(); - ret = clk_core_enable(clk->core); + + if (core->enable_count) + goto unlock_out; + + if (core->flags & CLK_IGNORE_UNUSED) + goto unlock_out; + + /* + * some gate clocks have special needs during the disable-unused + * sequence. call .disable_unused if available, otherwise fall + * back to .disable + */ + if (clk_core_is_enabled(core)) { + trace_clk_disable(core); + if (core->ops->disable_unused) + core->ops->disable_unused(core->hw); + else if (core->ops->disable) + core->ops->disable(core->hw); + trace_clk_disable_complete(core); + } + +unlock_out: clk_enable_unlock(flags); + if (core->flags & CLK_OPS_PARENT_ENABLE) + clk_core_disable_unprepare(core->parent); +} - return ret; +static bool clk_ignore_unused; +static int __init clk_ignore_unused_setup(char *__unused) +{ + clk_ignore_unused = true; + return 1; } -EXPORT_SYMBOL_GPL(clk_enable); +__setup("clk_ignore_unused", clk_ignore_unused_setup); + +static int clk_disable_unused(void) +{ + struct clk_core *core; + + if (clk_ignore_unused) { + pr_warn("clk: Not disabling unused clocks\n"); + return 0; + } + + clk_prepare_lock(); + + hlist_for_each_entry(core, &clk_root_list, child_node) + clk_disable_unused_subtree(core); + + hlist_for_each_entry(core, &clk_orphan_list, child_node) + clk_disable_unused_subtree(core); + + hlist_for_each_entry(core, &clk_root_list, child_node) + clk_unprepare_unused_subtree(core); + + hlist_for_each_entry(core, &clk_orphan_list, child_node) + clk_unprepare_unused_subtree(core); + + clk_prepare_unlock(); + + return 0; +} +late_initcall_sync(clk_disable_unused); static int clk_core_round_rate_nolock(struct clk_core *core, struct clk_rate_request *req) @@ -828,9 +874,7 @@ static int clk_core_round_rate_nolock(struct clk_core *core, /** * __clk_determine_rate - get the closest rate actually supported by a clock * @hw: determine the rate of this clock - * @rate: target rate - * @min_rate: returned rate must be greater than this rate - * @max_rate: returned rate must be less than this rate + * @req: target rate request * * Useful for clk_ops such as .set_rate and .determine_rate. */ @@ -1128,7 +1172,9 @@ static struct clk_core *__clk_set_parent_before(struct clk_core *core, struct clk_core *old_parent = core->parent; /* - * Migrate prepare state between parents and prevent race with + * 1. enable parents for CLK_OPS_PARENT_ENABLE clock + * + * 2. Migrate prepare state between parents and prevent race with * clk_enable(). * * If the clock is not prepared, then a race with @@ -1144,12 +1190,17 @@ static struct clk_core *__clk_set_parent_before(struct clk_core *core, * * See also: Comment for clk_set_parent() below. */ + + /* enable old_parent & parent if CLK_OPS_PARENT_ENABLE is set */ + if (core->flags & CLK_OPS_PARENT_ENABLE) { + clk_core_prepare_enable(old_parent); + clk_core_prepare_enable(parent); + } + + /* migrate prepare count if > 0 */ if (core->prepare_count) { - clk_core_prepare(parent); - flags = clk_enable_lock(); - clk_core_enable(parent); - clk_core_enable(core); - clk_enable_unlock(flags); + clk_core_prepare_enable(parent); + clk_core_enable_lock(core); } /* update the clk tree topology */ @@ -1164,18 +1215,19 @@ static void __clk_set_parent_after(struct clk_core *core, struct clk_core *parent, struct clk_core *old_parent) { - unsigned long flags; - /* * Finish the migration of prepare state and undo the changes done * for preventing a race with clk_enable(). */ if (core->prepare_count) { - flags = clk_enable_lock(); - clk_core_disable(core); - clk_core_disable(old_parent); - clk_enable_unlock(flags); - clk_core_unprepare(old_parent); + clk_core_disable_lock(core); + clk_core_disable_unprepare(old_parent); + } + + /* re-balance ref counting if CLK_OPS_PARENT_ENABLE is set */ + if (core->flags & CLK_OPS_PARENT_ENABLE) { + clk_core_disable_unprepare(parent); + clk_core_disable_unprepare(old_parent); } } @@ -1422,13 +1474,17 @@ static void clk_change_rate(struct clk_core *core) unsigned long best_parent_rate = 0; bool skip_set_rate = false; struct clk_core *old_parent; + struct clk_core *parent = NULL; old_rate = core->rate; - if (core->new_parent) + if (core->new_parent) { + parent = core->new_parent; best_parent_rate = core->new_parent->rate; - else if (core->parent) + } else if (core->parent) { + parent = core->parent; best_parent_rate = core->parent->rate; + } if (core->flags & CLK_SET_RATE_UNGATE) { unsigned long flags; @@ -1456,6 +1512,9 @@ static void clk_change_rate(struct clk_core *core) __clk_set_parent_after(core, core->new_parent, old_parent); } + if (core->flags & CLK_OPS_PARENT_ENABLE) + clk_core_prepare_enable(parent); + trace_clk_set_rate(core, core->new_rate); if (!skip_set_rate && core->ops->set_rate) @@ -1474,6 +1533,9 @@ static void clk_change_rate(struct clk_core *core) clk_core_unprepare(core); } + if (core->flags & CLK_OPS_PARENT_ENABLE) + clk_core_disable_unprepare(parent); + if (core->notifier_count && old_rate != core->rate) __clk_notify(core, POST_RATE_CHANGE, old_rate, core->rate); @@ -1501,7 +1563,6 @@ static int clk_core_set_rate_nolock(struct clk_core *core, { struct clk_core *top, *fail_clk; unsigned long rate = req_rate; - int ret = 0; if (!core) return 0; @@ -1532,7 +1593,7 @@ static int clk_core_set_rate_nolock(struct clk_core *core, core->req_rate = req_rate; - return ret; + return 0; } /** diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c index 89cc700fb..97ae60fa1 100644 --- a/drivers/clk/clkdev.c +++ b/drivers/clk/clkdev.c @@ -250,7 +250,7 @@ struct clk_lookup_alloc { char con_id[MAX_CON_ID]; }; -static struct clk_lookup * __init_refok +static struct clk_lookup * __ref vclkdev_alloc(struct clk_hw *hw, const char *con_id, const char *dev_fmt, va_list ap) { @@ -287,7 +287,7 @@ vclkdev_create(struct clk_hw *hw, const char *con_id, const char *dev_fmt, return cl; } -struct clk_lookup * __init_refok +struct clk_lookup * __ref clkdev_alloc(struct clk *clk, const char *con_id, const char *dev_fmt, ...) { struct clk_lookup *cl; diff --git a/drivers/clk/hisilicon/clk-hi3519.c b/drivers/clk/hisilicon/clk-hi3519.c index 715c7301a..51b173ef1 100644 --- a/drivers/clk/hisilicon/clk-hi3519.c +++ b/drivers/clk/hisilicon/clk-hi3519.c @@ -38,6 +38,11 @@ #define HI3519_NR_CLKS 128 +struct hi3519_crg_data { + struct hisi_clock_data *clk_data; + struct hisi_reset_controller *rstc; +}; + static const struct hisi_fixed_rate_clock hi3519_fixed_rate_clks[] = { { HI3519_FIXED_24M, "24m", NULL, 0, 24000000, }, { HI3519_FIXED_50M, "50m", NULL, 0, 50000000, }, @@ -80,33 +85,105 @@ static const struct hisi_gate_clock hi3519_gate_clks[] = { CLK_SET_RATE_PARENT, 0xe4, 18, 0, }, }; -static int hi3519_clk_probe(struct platform_device *pdev) +static struct hisi_clock_data *hi3519_clk_register(struct platform_device *pdev) { - struct device_node *np = pdev->dev.of_node; struct hisi_clock_data *clk_data; - struct hisi_reset_controller *rstc; + int ret; - rstc = hisi_reset_init(np); - if (!rstc) + clk_data = hisi_clk_alloc(pdev, HI3519_NR_CLKS); + if (!clk_data) + return ERR_PTR(-ENOMEM); + + ret = hisi_clk_register_fixed_rate(hi3519_fixed_rate_clks, + ARRAY_SIZE(hi3519_fixed_rate_clks), + clk_data); + if (ret) + return ERR_PTR(ret); + + ret = hisi_clk_register_mux(hi3519_mux_clks, + ARRAY_SIZE(hi3519_mux_clks), + clk_data); + if (ret) + goto unregister_fixed_rate; + + ret = hisi_clk_register_gate(hi3519_gate_clks, + ARRAY_SIZE(hi3519_gate_clks), + clk_data); + if (ret) + goto unregister_mux; + + ret = of_clk_add_provider(pdev->dev.of_node, + of_clk_src_onecell_get, &clk_data->clk_data); + if (ret) + goto unregister_gate; + + return clk_data; + +unregister_fixed_rate: + hisi_clk_unregister_fixed_rate(hi3519_fixed_rate_clks, + ARRAY_SIZE(hi3519_fixed_rate_clks), + clk_data); + +unregister_mux: + hisi_clk_unregister_mux(hi3519_mux_clks, + ARRAY_SIZE(hi3519_mux_clks), + clk_data); +unregister_gate: + hisi_clk_unregister_gate(hi3519_gate_clks, + ARRAY_SIZE(hi3519_gate_clks), + clk_data); + return ERR_PTR(ret); +} + +static void hi3519_clk_unregister(struct platform_device *pdev) +{ + struct hi3519_crg_data *crg = platform_get_drvdata(pdev); + + of_clk_del_provider(pdev->dev.of_node); + + hisi_clk_unregister_gate(hi3519_gate_clks, + ARRAY_SIZE(hi3519_mux_clks), + crg->clk_data); + hisi_clk_unregister_mux(hi3519_mux_clks, + ARRAY_SIZE(hi3519_mux_clks), + crg->clk_data); + hisi_clk_unregister_fixed_rate(hi3519_fixed_rate_clks, + ARRAY_SIZE(hi3519_fixed_rate_clks), + crg->clk_data); +} + +static int hi3519_clk_probe(struct platform_device *pdev) +{ + struct hi3519_crg_data *crg; + + crg = devm_kmalloc(&pdev->dev, sizeof(*crg), GFP_KERNEL); + if (!crg) + return -ENOMEM; + + crg->rstc = hisi_reset_init(pdev); + if (!crg->rstc) return -ENOMEM; - clk_data = hisi_clk_init(np, HI3519_NR_CLKS); - if (!clk_data) { - hisi_reset_exit(rstc); - return -ENODEV; + crg->clk_data = hi3519_clk_register(pdev); + if (IS_ERR(crg->clk_data)) { + hisi_reset_exit(crg->rstc); + return PTR_ERR(crg->clk_data); } - hisi_clk_register_fixed_rate(hi3519_fixed_rate_clks, - ARRAY_SIZE(hi3519_fixed_rate_clks), - clk_data); - hisi_clk_register_mux(hi3519_mux_clks, ARRAY_SIZE(hi3519_mux_clks), - clk_data); - hisi_clk_register_gate(hi3519_gate_clks, - ARRAY_SIZE(hi3519_gate_clks), clk_data); + platform_set_drvdata(pdev, crg); + return 0; +} + +static int hi3519_clk_remove(struct platform_device *pdev) +{ + struct hi3519_crg_data *crg = platform_get_drvdata(pdev); + hisi_reset_exit(crg->rstc); + hi3519_clk_unregister(pdev); return 0; } + static const struct of_device_id hi3519_clk_match_table[] = { { .compatible = "hisilicon,hi3519-crg" }, { } @@ -115,6 +192,7 @@ MODULE_DEVICE_TABLE(of, hi3519_clk_match_table); static struct platform_driver hi3519_clk_driver = { .probe = hi3519_clk_probe, + .remove = hi3519_clk_remove, .driver = { .name = "hi3519-clk", .of_match_table = hi3519_clk_match_table, @@ -127,5 +205,11 @@ static int __init hi3519_clk_init(void) } core_initcall(hi3519_clk_init); +static void __exit hi3519_clk_exit(void) +{ + platform_driver_unregister(&hi3519_clk_driver); +} +module_exit(hi3519_clk_exit); + MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("HiSilicon Hi3519 Clock Driver"); diff --git a/drivers/clk/hisilicon/clk-hi6220.c b/drivers/clk/hisilicon/clk-hi6220.c index f02cb41d4..fe364e63f 100644 --- a/drivers/clk/hisilicon/clk-hi6220.c +++ b/drivers/clk/hisilicon/clk-hi6220.c @@ -34,8 +34,8 @@ static struct hisi_fixed_rate_clock hi6220_fixed_rate_clks[] __initdata = { { HI6220_PLL_BBP, "bbppll0", NULL, 0, 245760000, }, { HI6220_PLL_GPU, "gpupll", NULL, 0, 1000000000,}, { HI6220_PLL1_DDR, "ddrpll1", NULL, 0, 1066000000,}, - { HI6220_PLL_SYS, "syspll", NULL, 0, 1200000000,}, - { HI6220_PLL_SYS_MEDIA, "media_syspll", NULL, 0, 1200000000,}, + { HI6220_PLL_SYS, "syspll", NULL, 0, 1190400000,}, + { HI6220_PLL_SYS_MEDIA, "media_syspll", NULL, 0, 1190400000,}, { HI6220_DDR_SRC, "ddr_sel_src", NULL, 0, 1200000000,}, { HI6220_PLL_MEDIA, "media_pll", NULL, 0, 1440000000,}, { HI6220_PLL_DDR, "ddrpll0", NULL, 0, 1600000000,}, @@ -68,6 +68,8 @@ static struct hisi_gate_clock hi6220_separated_gate_clks_ao[] __initdata = { { HI6220_TIMER7_PCLK, "timer7_pclk", "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 22, 0, }, { HI6220_TIMER8_PCLK, "timer8_pclk", "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 23, 0, }, { HI6220_UART0_PCLK, "uart0_pclk", "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 24, 0, }, + { HI6220_RTC0_PCLK, "rtc0_pclk", "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 25, 0, }, + { HI6220_RTC1_PCLK, "rtc1_pclk", "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 26, 0, }, }; static void __init hi6220_clk_ao_init(struct device_node *np) diff --git a/drivers/clk/hisilicon/clk.c b/drivers/clk/hisilicon/clk.c index 9b15adbfc..9ba2d91f4 100644 --- a/drivers/clk/hisilicon/clk.c +++ b/drivers/clk/hisilicon/clk.c @@ -37,6 +37,35 @@ static DEFINE_SPINLOCK(hisi_clk_lock); +struct hisi_clock_data *hisi_clk_alloc(struct platform_device *pdev, + int nr_clks) +{ + struct hisi_clock_data *clk_data; + struct resource *res; + struct clk **clk_table; + + clk_data = devm_kmalloc(&pdev->dev, sizeof(*clk_data), GFP_KERNEL); + if (!clk_data) + return NULL; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + clk_data->base = devm_ioremap(&pdev->dev, + res->start, resource_size(res)); + if (!clk_data->base) + return NULL; + + clk_table = devm_kmalloc(&pdev->dev, sizeof(struct clk *) * nr_clks, + GFP_KERNEL); + if (!clk_table) + return NULL; + + clk_data->clk_data.clks = clk_table; + clk_data->clk_data.clk_num = nr_clks; + + return clk_data; +} +EXPORT_SYMBOL_GPL(hisi_clk_alloc); + struct hisi_clock_data *hisi_clk_init(struct device_node *np, int nr_clks) { @@ -73,7 +102,7 @@ err: } EXPORT_SYMBOL_GPL(hisi_clk_init); -void hisi_clk_register_fixed_rate(const struct hisi_fixed_rate_clock *clks, +int hisi_clk_register_fixed_rate(const struct hisi_fixed_rate_clock *clks, int nums, struct hisi_clock_data *data) { struct clk *clk; @@ -87,14 +116,22 @@ void hisi_clk_register_fixed_rate(const struct hisi_fixed_rate_clock *clks, if (IS_ERR(clk)) { pr_err("%s: failed to register clock %s\n", __func__, clks[i].name); - continue; + goto err; } data->clk_data.clks[clks[i].id] = clk; } + + return 0; + +err: + while (i--) + clk_unregister_fixed_rate(data->clk_data.clks[clks[i].id]); + + return PTR_ERR(clk); } EXPORT_SYMBOL_GPL(hisi_clk_register_fixed_rate); -void hisi_clk_register_fixed_factor(const struct hisi_fixed_factor_clock *clks, +int hisi_clk_register_fixed_factor(const struct hisi_fixed_factor_clock *clks, int nums, struct hisi_clock_data *data) { @@ -109,14 +146,22 @@ void hisi_clk_register_fixed_factor(const struct hisi_fixed_factor_clock *clks, if (IS_ERR(clk)) { pr_err("%s: failed to register clock %s\n", __func__, clks[i].name); - continue; + goto err; } data->clk_data.clks[clks[i].id] = clk; } + + return 0; + +err: + while (i--) + clk_unregister_fixed_factor(data->clk_data.clks[clks[i].id]); + + return PTR_ERR(clk); } EXPORT_SYMBOL_GPL(hisi_clk_register_fixed_factor); -void hisi_clk_register_mux(const struct hisi_mux_clock *clks, +int hisi_clk_register_mux(const struct hisi_mux_clock *clks, int nums, struct hisi_clock_data *data) { struct clk *clk; @@ -135,7 +180,7 @@ void hisi_clk_register_mux(const struct hisi_mux_clock *clks, if (IS_ERR(clk)) { pr_err("%s: failed to register clock %s\n", __func__, clks[i].name); - continue; + goto err; } if (clks[i].alias) @@ -143,10 +188,18 @@ void hisi_clk_register_mux(const struct hisi_mux_clock *clks, data->clk_data.clks[clks[i].id] = clk; } + + return 0; + +err: + while (i--) + clk_unregister_mux(data->clk_data.clks[clks[i].id]); + + return PTR_ERR(clk); } EXPORT_SYMBOL_GPL(hisi_clk_register_mux); -void hisi_clk_register_divider(const struct hisi_divider_clock *clks, +int hisi_clk_register_divider(const struct hisi_divider_clock *clks, int nums, struct hisi_clock_data *data) { struct clk *clk; @@ -165,7 +218,7 @@ void hisi_clk_register_divider(const struct hisi_divider_clock *clks, if (IS_ERR(clk)) { pr_err("%s: failed to register clock %s\n", __func__, clks[i].name); - continue; + goto err; } if (clks[i].alias) @@ -173,10 +226,18 @@ void hisi_clk_register_divider(const struct hisi_divider_clock *clks, data->clk_data.clks[clks[i].id] = clk; } + + return 0; + +err: + while (i--) + clk_unregister_divider(data->clk_data.clks[clks[i].id]); + + return PTR_ERR(clk); } EXPORT_SYMBOL_GPL(hisi_clk_register_divider); -void hisi_clk_register_gate(const struct hisi_gate_clock *clks, +int hisi_clk_register_gate(const struct hisi_gate_clock *clks, int nums, struct hisi_clock_data *data) { struct clk *clk; @@ -194,7 +255,7 @@ void hisi_clk_register_gate(const struct hisi_gate_clock *clks, if (IS_ERR(clk)) { pr_err("%s: failed to register clock %s\n", __func__, clks[i].name); - continue; + goto err; } if (clks[i].alias) @@ -202,6 +263,14 @@ void hisi_clk_register_gate(const struct hisi_gate_clock *clks, data->clk_data.clks[clks[i].id] = clk; } + + return 0; + +err: + while (i--) + clk_unregister_gate(data->clk_data.clks[clks[i].id]); + + return PTR_ERR(clk); } EXPORT_SYMBOL_GPL(hisi_clk_register_gate); diff --git a/drivers/clk/hisilicon/clk.h b/drivers/clk/hisilicon/clk.h index 20d64afe4..4e1d1affc 100644 --- a/drivers/clk/hisilicon/clk.h +++ b/drivers/clk/hisilicon/clk.h @@ -30,6 +30,8 @@ #include #include +struct platform_device; + struct hisi_clock_data { struct clk_onecell_data clk_data; void __iomem *base; @@ -110,19 +112,41 @@ struct clk *hi6220_register_clkdiv(struct device *dev, const char *name, const char *parent_name, unsigned long flags, void __iomem *reg, u8 shift, u8 width, u32 mask_bit, spinlock_t *lock); +struct hisi_clock_data *hisi_clk_alloc(struct platform_device *, int); struct hisi_clock_data *hisi_clk_init(struct device_node *, int); -void hisi_clk_register_fixed_rate(const struct hisi_fixed_rate_clock *, +int hisi_clk_register_fixed_rate(const struct hisi_fixed_rate_clock *, int, struct hisi_clock_data *); -void hisi_clk_register_fixed_factor(const struct hisi_fixed_factor_clock *, +int hisi_clk_register_fixed_factor(const struct hisi_fixed_factor_clock *, int, struct hisi_clock_data *); -void hisi_clk_register_mux(const struct hisi_mux_clock *, int, +int hisi_clk_register_mux(const struct hisi_mux_clock *, int, struct hisi_clock_data *); -void hisi_clk_register_divider(const struct hisi_divider_clock *, +int hisi_clk_register_divider(const struct hisi_divider_clock *, int, struct hisi_clock_data *); -void hisi_clk_register_gate(const struct hisi_gate_clock *, +int hisi_clk_register_gate(const struct hisi_gate_clock *, int, struct hisi_clock_data *); void hisi_clk_register_gate_sep(const struct hisi_gate_clock *, int, struct hisi_clock_data *); void hi6220_clk_register_divider(const struct hi6220_divider_clock *, int, struct hisi_clock_data *); + +#define hisi_clk_unregister(type) \ +static inline \ +void hisi_clk_unregister_##type(const struct hisi_##type##_clock *clks, \ + int nums, struct hisi_clock_data *data) \ +{ \ + struct clk **clocks = data->clk_data.clks; \ + int i; \ + for (i = 0; i < nums; i++) { \ + int id = clks[i].id; \ + if (clocks[id]) \ + clk_unregister_##type(clocks[id]); \ + } \ +} + +hisi_clk_unregister(fixed_rate) +hisi_clk_unregister(fixed_factor) +hisi_clk_unregister(mux) +hisi_clk_unregister(divider) +hisi_clk_unregister(gate) + #endif /* __HISI_CLK_H */ diff --git a/drivers/clk/hisilicon/clkdivider-hi6220.c b/drivers/clk/hisilicon/clkdivider-hi6220.c index 113eee8ed..a1c1f684a 100644 --- a/drivers/clk/hisilicon/clkdivider-hi6220.c +++ b/drivers/clk/hisilicon/clkdivider-hi6220.c @@ -18,6 +18,8 @@ #include #include +#include "clk.h" + #define div_mask(width) ((1 << (width)) - 1) /** diff --git a/drivers/clk/hisilicon/reset.c b/drivers/clk/hisilicon/reset.c index 6aa49c220..2a5015c73 100644 --- a/drivers/clk/hisilicon/reset.c +++ b/drivers/clk/hisilicon/reset.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -98,25 +99,25 @@ static const struct reset_control_ops hisi_reset_ops = { .deassert = hisi_reset_deassert, }; -struct hisi_reset_controller *hisi_reset_init(struct device_node *np) +struct hisi_reset_controller *hisi_reset_init(struct platform_device *pdev) { struct hisi_reset_controller *rstc; + struct resource *res; - rstc = kzalloc(sizeof(*rstc), GFP_KERNEL); + rstc = devm_kmalloc(&pdev->dev, sizeof(*rstc), GFP_KERNEL); if (!rstc) return NULL; - rstc->membase = of_iomap(np, 0); - if (!rstc->membase) { - kfree(rstc); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + rstc->membase = devm_ioremap(&pdev->dev, + res->start, resource_size(res)); + if (!rstc->membase) return NULL; - } spin_lock_init(&rstc->lock); - rstc->rcdev.owner = THIS_MODULE; rstc->rcdev.ops = &hisi_reset_ops; - rstc->rcdev.of_node = np; + rstc->rcdev.of_node = pdev->dev.of_node; rstc->rcdev.of_reset_n_cells = 2; rstc->rcdev.of_xlate = hisi_reset_of_xlate; reset_controller_register(&rstc->rcdev); @@ -128,7 +129,5 @@ EXPORT_SYMBOL_GPL(hisi_reset_init); void hisi_reset_exit(struct hisi_reset_controller *rstc) { reset_controller_unregister(&rstc->rcdev); - iounmap(rstc->membase); - kfree(rstc); } EXPORT_SYMBOL_GPL(hisi_reset_exit); diff --git a/drivers/clk/hisilicon/reset.h b/drivers/clk/hisilicon/reset.h index 677d773ed..9a69374a0 100644 --- a/drivers/clk/hisilicon/reset.h +++ b/drivers/clk/hisilicon/reset.h @@ -22,10 +22,11 @@ struct device_node; struct hisi_reset_controller; #ifdef CONFIG_RESET_CONTROLLER -struct hisi_reset_controller *hisi_reset_init(struct device_node *np); +struct hisi_reset_controller *hisi_reset_init(struct platform_device *pdev); void hisi_reset_exit(struct hisi_reset_controller *rstc); #else -static inline hisi_reset_controller *hisi_reset_init(struct device_node *np) +static inline +struct hisi_reset_controller *hisi_reset_init(struct platform_device *pdev) { return 0; } diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c index 2beb396fe..ba1c1ae72 100644 --- a/drivers/clk/imx/clk-imx6q.c +++ b/drivers/clk/imx/clk-imx6q.c @@ -192,13 +192,13 @@ static void __init imx6q_clocks_init(struct device_node *ccm_node) clk[IMX6QDL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 2, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); /* type name parent_name base div_mask */ - clk[IMX6QDL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "pll1_bypass_src", base + 0x00, 0x7f); - clk[IMX6QDL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "pll2_bypass_src", base + 0x30, 0x1); - clk[IMX6QDL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "pll3_bypass_src", base + 0x10, 0x3); - clk[IMX6QDL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "pll4_bypass_src", base + 0x70, 0x7f); - clk[IMX6QDL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "pll5_bypass_src", base + 0xa0, 0x7f); - clk[IMX6QDL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "pll6_bypass_src", base + 0xe0, 0x3); - clk[IMX6QDL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "pll7_bypass_src", base + 0x20, 0x3); + clk[IMX6QDL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); + clk[IMX6QDL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); + clk[IMX6QDL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); + clk[IMX6QDL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); + clk[IMX6QDL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); + clk[IMX6QDL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); + clk[IMX6QDL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); clk[IMX6QDL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); clk[IMX6QDL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); diff --git a/drivers/clk/imx/clk-imx6sl.c b/drivers/clk/imx/clk-imx6sl.c index 1be6230a0..5fd4ddac1 100644 --- a/drivers/clk/imx/clk-imx6sl.c +++ b/drivers/clk/imx/clk-imx6sl.c @@ -218,13 +218,13 @@ static void __init imx6sl_clocks_init(struct device_node *ccm_node) clks[IMX6SL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); /* type name parent_name base div_mask */ - clks[IMX6SL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "pll1_bypass_src", base + 0x00, 0x7f); - clks[IMX6SL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "pll2_bypass_src", base + 0x30, 0x1); - clks[IMX6SL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "pll3_bypass_src", base + 0x10, 0x3); - clks[IMX6SL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "pll4_bypass_src", base + 0x70, 0x7f); - clks[IMX6SL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "pll5_bypass_src", base + 0xa0, 0x7f); - clks[IMX6SL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "pll6_bypass_src", base + 0xe0, 0x3); - clks[IMX6SL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "pll7_bypass_src", base + 0x20, 0x3); + clks[IMX6SL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); + clks[IMX6SL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); + clks[IMX6SL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); + clks[IMX6SL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); + clks[IMX6SL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); + clks[IMX6SL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); + clks[IMX6SL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); clks[IMX6SL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); clks[IMX6SL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); diff --git a/drivers/clk/imx/clk-imx6sx.c b/drivers/clk/imx/clk-imx6sx.c index 97e742a8b..b5c96de41 100644 --- a/drivers/clk/imx/clk-imx6sx.c +++ b/drivers/clk/imx/clk-imx6sx.c @@ -174,13 +174,13 @@ static void __init imx6sx_clocks_init(struct device_node *ccm_node) clks[IMX6SX_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); /* type name parent_name base div_mask */ - clks[IMX6SX_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "pll1_bypass_src", base + 0x00, 0x7f); - clks[IMX6SX_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "pll2_bypass_src", base + 0x30, 0x1); - clks[IMX6SX_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "pll3_bypass_src", base + 0x10, 0x3); - clks[IMX6SX_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "pll4_bypass_src", base + 0x70, 0x7f); - clks[IMX6SX_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "pll5_bypass_src", base + 0xa0, 0x7f); - clks[IMX6SX_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "pll6_bypass_src", base + 0xe0, 0x3); - clks[IMX6SX_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "pll7_bypass_src", base + 0x20, 0x3); + clks[IMX6SX_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); + clks[IMX6SX_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); + clks[IMX6SX_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); + clks[IMX6SX_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); + clks[IMX6SX_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); + clks[IMX6SX_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); + clks[IMX6SX_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); clks[IMX6SX_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); clks[IMX6SX_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); diff --git a/drivers/clk/imx/clk-imx6ul.c b/drivers/clk/imx/clk-imx6ul.c index 0f1f17a8f..d1d7787ce 100644 --- a/drivers/clk/imx/clk-imx6ul.c +++ b/drivers/clk/imx/clk-imx6ul.c @@ -130,13 +130,13 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node) clks[IMX6UL_PLL6_BYPASS_SRC] = imx_clk_mux("pll6_bypass_src", base + 0xe0, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); clks[IMX6UL_PLL7_BYPASS_SRC] = imx_clk_mux("pll7_bypass_src", base + 0x20, 14, 1, pll_bypass_src_sels, ARRAY_SIZE(pll_bypass_src_sels)); - clks[IMX6UL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "pll1_bypass_src", base + 0x00, 0x7f); - clks[IMX6UL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "pll2_bypass_src", base + 0x30, 0x1); - clks[IMX6UL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "pll3_bypass_src", base + 0x10, 0x3); - clks[IMX6UL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "pll4_bypass_src", base + 0x70, 0x7f); - clks[IMX6UL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "pll5_bypass_src", base + 0xa0, 0x7f); - clks[IMX6UL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "pll6_bypass_src", base + 0xe0, 0x3); - clks[IMX6UL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "pll7_bypass_src", base + 0x20, 0x3); + clks[IMX6UL_CLK_PLL1] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll1", "osc", base + 0x00, 0x7f); + clks[IMX6UL_CLK_PLL2] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll2", "osc", base + 0x30, 0x1); + clks[IMX6UL_CLK_PLL3] = imx_clk_pllv3(IMX_PLLV3_USB, "pll3", "osc", base + 0x10, 0x3); + clks[IMX6UL_CLK_PLL4] = imx_clk_pllv3(IMX_PLLV3_AV, "pll4", "osc", base + 0x70, 0x7f); + clks[IMX6UL_CLK_PLL5] = imx_clk_pllv3(IMX_PLLV3_AV, "pll5", "osc", base + 0xa0, 0x7f); + clks[IMX6UL_CLK_PLL6] = imx_clk_pllv3(IMX_PLLV3_ENET, "pll6", "osc", base + 0xe0, 0x3); + clks[IMX6UL_CLK_PLL7] = imx_clk_pllv3(IMX_PLLV3_USB, "pll7", "osc", base + 0x20, 0x3); clks[IMX6UL_PLL1_BYPASS] = imx_clk_mux_flags("pll1_bypass", base + 0x00, 16, 1, pll1_bypass_sels, ARRAY_SIZE(pll1_bypass_sels), CLK_SET_RATE_PARENT); clks[IMX6UL_PLL2_BYPASS] = imx_clk_mux_flags("pll2_bypass", base + 0x30, 16, 1, pll2_bypass_sels, ARRAY_SIZE(pll2_bypass_sels), CLK_SET_RATE_PARENT); @@ -305,8 +305,8 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node) clks[IMX6UL_CLK_CAN1_SERIAL] = imx_clk_gate2("can1_serial", "can_podf", base + 0x68, 16); clks[IMX6UL_CLK_CAN2_IPG] = imx_clk_gate2("can2_ipg", "ipg", base + 0x68, 18); clks[IMX6UL_CLK_CAN2_SERIAL] = imx_clk_gate2("can2_serial", "can_podf", base + 0x68, 20); - clks[IMX6UL_CLK_GPT2_BUS] = imx_clk_gate2("gpt_bus", "perclk", base + 0x68, 24); - clks[IMX6UL_CLK_GPT2_SERIAL] = imx_clk_gate2("gpt_serial", "perclk", base + 0x68, 26); + clks[IMX6UL_CLK_GPT2_BUS] = imx_clk_gate2("gpt2_bus", "perclk", base + 0x68, 24); + clks[IMX6UL_CLK_GPT2_SERIAL] = imx_clk_gate2("gpt2_serial", "perclk", base + 0x68, 26); clks[IMX6UL_CLK_UART2_IPG] = imx_clk_gate2("uart2_ipg", "ipg", base + 0x68, 28); clks[IMX6UL_CLK_UART2_SERIAL] = imx_clk_gate2("uart2_serial", "uart_podf", base + 0x68, 28); clks[IMX6UL_CLK_AIPSTZ3] = imx_clk_gate2("aips_tz3", "ahb", base + 0x68, 30); diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c index 522996800..6ed4f8fa0 100644 --- a/drivers/clk/imx/clk-imx7d.c +++ b/drivers/clk/imx/clk-imx7d.c @@ -65,7 +65,7 @@ static const char *dram_phym_sel[] = { "pll_dram_main_clk", "dram_phym_alt_clk", }; static const char *dram_sel[] = { "pll_dram_main_clk", - "dram_alt_clk", }; + "dram_alt_root_clk", }; static const char *dram_phym_alt_sel[] = { "osc", "pll_dram_533m_clk", "pll_sys_main_clk", "pll_enet_500m_clk", @@ -361,6 +361,14 @@ static const char *pll_enet_bypass_sel[] = { "pll_enet_main", "pll_enet_main_src static const char *pll_audio_bypass_sel[] = { "pll_audio_main", "pll_audio_main_src", }; static const char *pll_video_bypass_sel[] = { "pll_video_main", "pll_video_main_src", }; +static int const clks_init_on[] __initconst = { + IMX7D_ARM_A7_ROOT_CLK, IMX7D_MAIN_AXI_ROOT_CLK, + IMX7D_PLL_SYS_MAIN_480M_CLK, IMX7D_NAND_USDHC_BUS_ROOT_CLK, + IMX7D_DRAM_PHYM_ROOT_CLK, IMX7D_DRAM_ROOT_CLK, + IMX7D_DRAM_PHYM_ALT_ROOT_CLK, IMX7D_DRAM_ALT_ROOT_CLK, + IMX7D_AHB_CHANNEL_ROOT_CLK, +}; + static struct clk_onecell_data clk_data; static struct clk ** const uart_clks[] __initconst = { @@ -395,12 +403,12 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node) clks[IMX7D_PLL_AUDIO_MAIN_SRC] = imx_clk_mux("pll_audio_main_src", base + 0xf0, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); clks[IMX7D_PLL_VIDEO_MAIN_SRC] = imx_clk_mux("pll_video_main_src", base + 0x130, 14, 2, pll_bypass_src_sel, ARRAY_SIZE(pll_bypass_src_sel)); - clks[IMX7D_PLL_ARM_MAIN] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll_arm_main", "pll_arm_main_src", base + 0x60, 0x7f); - clks[IMX7D_PLL_DRAM_MAIN] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll_dram_main", "pll_dram_main_src", base + 0x70, 0x7f); - clks[IMX7D_PLL_SYS_MAIN] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll_sys_main", "pll_sys_main_src", base + 0xb0, 0x1); - clks[IMX7D_PLL_ENET_MAIN] = imx_clk_pllv3(IMX_PLLV3_ENET_IMX7, "pll_enet_main", "pll_enet_main_src", base + 0xe0, 0x0); - clks[IMX7D_PLL_AUDIO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV, "pll_audio_main", "pll_audio_main_src", base + 0xf0, 0x7f); - clks[IMX7D_PLL_VIDEO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV, "pll_video_main", "pll_video_main_src", base + 0x130, 0x7f); + clks[IMX7D_PLL_ARM_MAIN] = imx_clk_pllv3(IMX_PLLV3_SYS, "pll_arm_main", "osc", base + 0x60, 0x7f); + clks[IMX7D_PLL_DRAM_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV, "pll_dram_main", "osc", base + 0x70, 0x7f); + clks[IMX7D_PLL_SYS_MAIN] = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll_sys_main", "osc", base + 0xb0, 0x1); + clks[IMX7D_PLL_ENET_MAIN] = imx_clk_pllv3(IMX_PLLV3_ENET_IMX7, "pll_enet_main", "osc", base + 0xe0, 0x0); + clks[IMX7D_PLL_AUDIO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV, "pll_audio_main", "osc", base + 0xf0, 0x7f); + clks[IMX7D_PLL_VIDEO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV, "pll_video_main", "osc", base + 0x130, 0x7f); clks[IMX7D_PLL_ARM_MAIN_BYPASS] = imx_clk_mux_flags("pll_arm_main_bypass", base + 0x60, 16, 1, pll_arm_bypass_sel, ARRAY_SIZE(pll_arm_bypass_sel), CLK_SET_RATE_PARENT); clks[IMX7D_PLL_DRAM_MAIN_BYPASS] = imx_clk_mux_flags("pll_dram_main_bypass", base + 0x70, 16, 1, pll_dram_bypass_sel, ARRAY_SIZE(pll_dram_bypass_sel), CLK_SET_RATE_PARENT); @@ -474,363 +482,363 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node) base = of_iomap(np, 0); WARN_ON(!base); - clks[IMX7D_ARM_A7_ROOT_SRC] = imx_clk_mux("arm_a7_src", base + 0x8000, 24, 3, arm_a7_sel, ARRAY_SIZE(arm_a7_sel)); - clks[IMX7D_ARM_M4_ROOT_SRC] = imx_clk_mux("arm_m4_src", base + 0x8080, 24, 3, arm_m4_sel, ARRAY_SIZE(arm_m4_sel)); - clks[IMX7D_ARM_M0_ROOT_SRC] = imx_clk_mux("arm_m0_src", base + 0x8100, 24, 3, arm_m0_sel, ARRAY_SIZE(arm_m0_sel)); - clks[IMX7D_MAIN_AXI_ROOT_SRC] = imx_clk_mux("axi_src", base + 0x8800, 24, 3, axi_sel, ARRAY_SIZE(axi_sel)); - clks[IMX7D_DISP_AXI_ROOT_SRC] = imx_clk_mux("disp_axi_src", base + 0x8880, 24, 3, disp_axi_sel, ARRAY_SIZE(disp_axi_sel)); - clks[IMX7D_ENET_AXI_ROOT_SRC] = imx_clk_mux("enet_axi_src", base + 0x8900, 24, 3, enet_axi_sel, ARRAY_SIZE(enet_axi_sel)); - clks[IMX7D_NAND_USDHC_BUS_ROOT_SRC] = imx_clk_mux("nand_usdhc_src", base + 0x8980, 24, 3, nand_usdhc_bus_sel, ARRAY_SIZE(nand_usdhc_bus_sel)); - clks[IMX7D_AHB_CHANNEL_ROOT_SRC] = imx_clk_mux("ahb_src", base + 0x9000, 24, 3, ahb_channel_sel, ARRAY_SIZE(ahb_channel_sel)); - clks[IMX7D_DRAM_PHYM_ROOT_SRC] = imx_clk_mux("dram_phym_src", base + 0x9800, 24, 1, dram_phym_sel, ARRAY_SIZE(dram_phym_sel)); - clks[IMX7D_DRAM_ROOT_SRC] = imx_clk_mux("dram_src", base + 0x9880, 24, 1, dram_sel, ARRAY_SIZE(dram_sel)); - clks[IMX7D_DRAM_PHYM_ALT_ROOT_SRC] = imx_clk_mux("dram_phym_alt_src", base + 0xa000, 24, 3, dram_phym_alt_sel, ARRAY_SIZE(dram_phym_alt_sel)); - clks[IMX7D_DRAM_ALT_ROOT_SRC] = imx_clk_mux("dram_alt_src", base + 0xa080, 24, 3, dram_alt_sel, ARRAY_SIZE(dram_alt_sel)); - clks[IMX7D_USB_HSIC_ROOT_SRC] = imx_clk_mux("usb_hsic_src", base + 0xa100, 24, 3, usb_hsic_sel, ARRAY_SIZE(usb_hsic_sel)); - clks[IMX7D_PCIE_CTRL_ROOT_SRC] = imx_clk_mux("pcie_ctrl_src", base + 0xa180, 24, 3, pcie_ctrl_sel, ARRAY_SIZE(pcie_ctrl_sel)); - clks[IMX7D_PCIE_PHY_ROOT_SRC] = imx_clk_mux("pcie_phy_src", base + 0xa200, 24, 3, pcie_phy_sel, ARRAY_SIZE(pcie_phy_sel)); - clks[IMX7D_EPDC_PIXEL_ROOT_SRC] = imx_clk_mux("epdc_pixel_src", base + 0xa280, 24, 3, epdc_pixel_sel, ARRAY_SIZE(epdc_pixel_sel)); - clks[IMX7D_LCDIF_PIXEL_ROOT_SRC] = imx_clk_mux("lcdif_pixel_src", base + 0xa300, 24, 3, lcdif_pixel_sel, ARRAY_SIZE(lcdif_pixel_sel)); - clks[IMX7D_MIPI_DSI_ROOT_SRC] = imx_clk_mux("mipi_dsi_src", base + 0xa380, 24, 3, mipi_dsi_sel, ARRAY_SIZE(mipi_dsi_sel)); - clks[IMX7D_MIPI_CSI_ROOT_SRC] = imx_clk_mux("mipi_csi_src", base + 0xa400, 24, 3, mipi_csi_sel, ARRAY_SIZE(mipi_csi_sel)); - clks[IMX7D_MIPI_DPHY_ROOT_SRC] = imx_clk_mux("mipi_dphy_src", base + 0xa480, 24, 3, mipi_dphy_sel, ARRAY_SIZE(mipi_dphy_sel)); - clks[IMX7D_SAI1_ROOT_SRC] = imx_clk_mux("sai1_src", base + 0xa500, 24, 3, sai1_sel, ARRAY_SIZE(sai1_sel)); - clks[IMX7D_SAI2_ROOT_SRC] = imx_clk_mux("sai2_src", base + 0xa580, 24, 3, sai2_sel, ARRAY_SIZE(sai2_sel)); - clks[IMX7D_SAI3_ROOT_SRC] = imx_clk_mux("sai3_src", base + 0xa600, 24, 3, sai3_sel, ARRAY_SIZE(sai3_sel)); - clks[IMX7D_SPDIF_ROOT_SRC] = imx_clk_mux("spdif_src", base + 0xa680, 24, 3, spdif_sel, ARRAY_SIZE(spdif_sel)); - clks[IMX7D_ENET1_REF_ROOT_SRC] = imx_clk_mux("enet1_ref_src", base + 0xa700, 24, 3, enet1_ref_sel, ARRAY_SIZE(enet1_ref_sel)); - clks[IMX7D_ENET1_TIME_ROOT_SRC] = imx_clk_mux("enet1_time_src", base + 0xa780, 24, 3, enet1_time_sel, ARRAY_SIZE(enet1_time_sel)); - clks[IMX7D_ENET2_REF_ROOT_SRC] = imx_clk_mux("enet2_ref_src", base + 0xa800, 24, 3, enet2_ref_sel, ARRAY_SIZE(enet2_ref_sel)); - clks[IMX7D_ENET2_TIME_ROOT_SRC] = imx_clk_mux("enet2_time_src", base + 0xa880, 24, 3, enet2_time_sel, ARRAY_SIZE(enet2_time_sel)); - clks[IMX7D_ENET_PHY_REF_ROOT_SRC] = imx_clk_mux("enet_phy_ref_src", base + 0xa900, 24, 3, enet_phy_ref_sel, ARRAY_SIZE(enet_phy_ref_sel)); - clks[IMX7D_EIM_ROOT_SRC] = imx_clk_mux("eim_src", base + 0xa980, 24, 3, eim_sel, ARRAY_SIZE(eim_sel)); - clks[IMX7D_NAND_ROOT_SRC] = imx_clk_mux("nand_src", base + 0xaa00, 24, 3, nand_sel, ARRAY_SIZE(nand_sel)); - clks[IMX7D_QSPI_ROOT_SRC] = imx_clk_mux("qspi_src", base + 0xaa80, 24, 3, qspi_sel, ARRAY_SIZE(qspi_sel)); - clks[IMX7D_USDHC1_ROOT_SRC] = imx_clk_mux("usdhc1_src", base + 0xab00, 24, 3, usdhc1_sel, ARRAY_SIZE(usdhc1_sel)); - clks[IMX7D_USDHC2_ROOT_SRC] = imx_clk_mux("usdhc2_src", base + 0xab80, 24, 3, usdhc2_sel, ARRAY_SIZE(usdhc2_sel)); - clks[IMX7D_USDHC3_ROOT_SRC] = imx_clk_mux("usdhc3_src", base + 0xac00, 24, 3, usdhc3_sel, ARRAY_SIZE(usdhc3_sel)); - clks[IMX7D_CAN1_ROOT_SRC] = imx_clk_mux("can1_src", base + 0xac80, 24, 3, can1_sel, ARRAY_SIZE(can1_sel)); - clks[IMX7D_CAN2_ROOT_SRC] = imx_clk_mux("can2_src", base + 0xad00, 24, 3, can2_sel, ARRAY_SIZE(can2_sel)); - clks[IMX7D_I2C1_ROOT_SRC] = imx_clk_mux("i2c1_src", base + 0xad80, 24, 3, i2c1_sel, ARRAY_SIZE(i2c1_sel)); - clks[IMX7D_I2C2_ROOT_SRC] = imx_clk_mux("i2c2_src", base + 0xae00, 24, 3, i2c2_sel, ARRAY_SIZE(i2c2_sel)); - clks[IMX7D_I2C3_ROOT_SRC] = imx_clk_mux("i2c3_src", base + 0xae80, 24, 3, i2c3_sel, ARRAY_SIZE(i2c3_sel)); - clks[IMX7D_I2C4_ROOT_SRC] = imx_clk_mux("i2c4_src", base + 0xaf00, 24, 3, i2c4_sel, ARRAY_SIZE(i2c4_sel)); - clks[IMX7D_UART1_ROOT_SRC] = imx_clk_mux("uart1_src", base + 0xaf80, 24, 3, uart1_sel, ARRAY_SIZE(uart1_sel)); - clks[IMX7D_UART2_ROOT_SRC] = imx_clk_mux("uart2_src", base + 0xb000, 24, 3, uart2_sel, ARRAY_SIZE(uart2_sel)); - clks[IMX7D_UART3_ROOT_SRC] = imx_clk_mux("uart3_src", base + 0xb080, 24, 3, uart3_sel, ARRAY_SIZE(uart3_sel)); - clks[IMX7D_UART4_ROOT_SRC] = imx_clk_mux("uart4_src", base + 0xb100, 24, 3, uart4_sel, ARRAY_SIZE(uart4_sel)); - clks[IMX7D_UART5_ROOT_SRC] = imx_clk_mux("uart5_src", base + 0xb180, 24, 3, uart5_sel, ARRAY_SIZE(uart5_sel)); - clks[IMX7D_UART6_ROOT_SRC] = imx_clk_mux("uart6_src", base + 0xb200, 24, 3, uart6_sel, ARRAY_SIZE(uart6_sel)); - clks[IMX7D_UART7_ROOT_SRC] = imx_clk_mux("uart7_src", base + 0xb280, 24, 3, uart7_sel, ARRAY_SIZE(uart7_sel)); - clks[IMX7D_ECSPI1_ROOT_SRC] = imx_clk_mux("ecspi1_src", base + 0xb300, 24, 3, ecspi1_sel, ARRAY_SIZE(ecspi1_sel)); - clks[IMX7D_ECSPI2_ROOT_SRC] = imx_clk_mux("ecspi2_src", base + 0xb380, 24, 3, ecspi2_sel, ARRAY_SIZE(ecspi2_sel)); - clks[IMX7D_ECSPI3_ROOT_SRC] = imx_clk_mux("ecspi3_src", base + 0xb400, 24, 3, ecspi3_sel, ARRAY_SIZE(ecspi3_sel)); - clks[IMX7D_ECSPI4_ROOT_SRC] = imx_clk_mux("ecspi4_src", base + 0xb480, 24, 3, ecspi4_sel, ARRAY_SIZE(ecspi4_sel)); - clks[IMX7D_PWM1_ROOT_SRC] = imx_clk_mux("pwm1_src", base + 0xb500, 24, 3, pwm1_sel, ARRAY_SIZE(pwm1_sel)); - clks[IMX7D_PWM2_ROOT_SRC] = imx_clk_mux("pwm2_src", base + 0xb580, 24, 3, pwm2_sel, ARRAY_SIZE(pwm2_sel)); - clks[IMX7D_PWM3_ROOT_SRC] = imx_clk_mux("pwm3_src", base + 0xb600, 24, 3, pwm3_sel, ARRAY_SIZE(pwm3_sel)); - clks[IMX7D_PWM4_ROOT_SRC] = imx_clk_mux("pwm4_src", base + 0xb680, 24, 3, pwm4_sel, ARRAY_SIZE(pwm4_sel)); - clks[IMX7D_FLEXTIMER1_ROOT_SRC] = imx_clk_mux("flextimer1_src", base + 0xb700, 24, 3, flextimer1_sel, ARRAY_SIZE(flextimer1_sel)); - clks[IMX7D_FLEXTIMER2_ROOT_SRC] = imx_clk_mux("flextimer2_src", base + 0xb780, 24, 3, flextimer2_sel, ARRAY_SIZE(flextimer2_sel)); - clks[IMX7D_SIM1_ROOT_SRC] = imx_clk_mux("sim1_src", base + 0xb800, 24, 3, sim1_sel, ARRAY_SIZE(sim1_sel)); - clks[IMX7D_SIM2_ROOT_SRC] = imx_clk_mux("sim2_src", base + 0xb880, 24, 3, sim2_sel, ARRAY_SIZE(sim2_sel)); - clks[IMX7D_GPT1_ROOT_SRC] = imx_clk_mux("gpt1_src", base + 0xb900, 24, 3, gpt1_sel, ARRAY_SIZE(gpt1_sel)); - clks[IMX7D_GPT2_ROOT_SRC] = imx_clk_mux("gpt2_src", base + 0xb980, 24, 3, gpt2_sel, ARRAY_SIZE(gpt2_sel)); - clks[IMX7D_GPT3_ROOT_SRC] = imx_clk_mux("gpt3_src", base + 0xba00, 24, 3, gpt3_sel, ARRAY_SIZE(gpt3_sel)); - clks[IMX7D_GPT4_ROOT_SRC] = imx_clk_mux("gpt4_src", base + 0xba80, 24, 3, gpt4_sel, ARRAY_SIZE(gpt4_sel)); - clks[IMX7D_TRACE_ROOT_SRC] = imx_clk_mux("trace_src", base + 0xbb00, 24, 3, trace_sel, ARRAY_SIZE(trace_sel)); - clks[IMX7D_WDOG_ROOT_SRC] = imx_clk_mux("wdog_src", base + 0xbb80, 24, 3, wdog_sel, ARRAY_SIZE(wdog_sel)); - clks[IMX7D_CSI_MCLK_ROOT_SRC] = imx_clk_mux("csi_mclk_src", base + 0xbc00, 24, 3, csi_mclk_sel, ARRAY_SIZE(csi_mclk_sel)); - clks[IMX7D_AUDIO_MCLK_ROOT_SRC] = imx_clk_mux("audio_mclk_src", base + 0xbc80, 24, 3, audio_mclk_sel, ARRAY_SIZE(audio_mclk_sel)); - clks[IMX7D_WRCLK_ROOT_SRC] = imx_clk_mux("wrclk_src", base + 0xbd00, 24, 3, wrclk_sel, ARRAY_SIZE(wrclk_sel)); - clks[IMX7D_CLKO1_ROOT_SRC] = imx_clk_mux("clko1_src", base + 0xbd80, 24, 3, clko1_sel, ARRAY_SIZE(clko1_sel)); - clks[IMX7D_CLKO2_ROOT_SRC] = imx_clk_mux("clko2_src", base + 0xbe00, 24, 3, clko2_sel, ARRAY_SIZE(clko2_sel)); - - clks[IMX7D_ARM_A7_ROOT_CG] = imx_clk_gate("arm_a7_cg", "arm_a7_src", base + 0x8000, 28); - clks[IMX7D_ARM_M4_ROOT_CG] = imx_clk_gate("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); - clks[IMX7D_ARM_M0_ROOT_CG] = imx_clk_gate("arm_m0_cg", "arm_m0_src", base + 0x8100, 28); - clks[IMX7D_MAIN_AXI_ROOT_CG] = imx_clk_gate("axi_cg", "axi_src", base + 0x8800, 28); - clks[IMX7D_DISP_AXI_ROOT_CG] = imx_clk_gate("disp_axi_cg", "disp_axi_src", base + 0x8880, 28); - clks[IMX7D_ENET_AXI_ROOT_CG] = imx_clk_gate("enet_axi_cg", "enet_axi_src", base + 0x8900, 28); - clks[IMX7D_NAND_USDHC_BUS_ROOT_CG] = imx_clk_gate("nand_usdhc_cg", "nand_usdhc_src", base + 0x8980, 28); - clks[IMX7D_AHB_CHANNEL_ROOT_CG] = imx_clk_gate("ahb_cg", "ahb_src", base + 0x9000, 28); - clks[IMX7D_DRAM_PHYM_ROOT_CG] = imx_clk_gate("dram_phym_cg", "dram_phym_src", base + 0x9800, 28); - clks[IMX7D_DRAM_ROOT_CG] = imx_clk_gate("dram_cg", "dram_src", base + 0x9880, 28); - clks[IMX7D_DRAM_PHYM_ALT_ROOT_CG] = imx_clk_gate("dram_phym_alt_cg", "dram_phym_alt_src", base + 0xa000, 28); - clks[IMX7D_DRAM_ALT_ROOT_CG] = imx_clk_gate("dram_alt_cg", "dram_alt_src", base + 0xa080, 28); - clks[IMX7D_USB_HSIC_ROOT_CG] = imx_clk_gate("usb_hsic_cg", "usb_hsic_src", base + 0xa100, 28); - clks[IMX7D_PCIE_CTRL_ROOT_CG] = imx_clk_gate("pcie_ctrl_cg", "pcie_ctrl_src", base + 0xa180, 28); - clks[IMX7D_PCIE_PHY_ROOT_CG] = imx_clk_gate("pcie_phy_cg", "pcie_phy_src", base + 0xa200, 28); - clks[IMX7D_EPDC_PIXEL_ROOT_CG] = imx_clk_gate("epdc_pixel_cg", "epdc_pixel_src", base + 0xa280, 28); - clks[IMX7D_LCDIF_PIXEL_ROOT_CG] = imx_clk_gate("lcdif_pixel_cg", "lcdif_pixel_src", base + 0xa300, 28); - clks[IMX7D_MIPI_DSI_ROOT_CG] = imx_clk_gate("mipi_dsi_cg", "mipi_dsi_src", base + 0xa380, 28); - clks[IMX7D_MIPI_CSI_ROOT_CG] = imx_clk_gate("mipi_csi_cg", "mipi_csi_src", base + 0xa400, 28); - clks[IMX7D_MIPI_DPHY_ROOT_CG] = imx_clk_gate("mipi_dphy_cg", "mipi_dphy_src", base + 0xa480, 28); - clks[IMX7D_SAI1_ROOT_CG] = imx_clk_gate("sai1_cg", "sai1_src", base + 0xa500, 28); - clks[IMX7D_SAI2_ROOT_CG] = imx_clk_gate("sai2_cg", "sai2_src", base + 0xa580, 28); - clks[IMX7D_SAI3_ROOT_CG] = imx_clk_gate("sai3_cg", "sai3_src", base + 0xa600, 28); - clks[IMX7D_SPDIF_ROOT_CG] = imx_clk_gate("spdif_cg", "spdif_src", base + 0xa680, 28); - clks[IMX7D_ENET1_REF_ROOT_CG] = imx_clk_gate("enet1_ref_cg", "enet1_ref_src", base + 0xa700, 28); - clks[IMX7D_ENET1_TIME_ROOT_CG] = imx_clk_gate("enet1_time_cg", "enet1_time_src", base + 0xa780, 28); - clks[IMX7D_ENET2_REF_ROOT_CG] = imx_clk_gate("enet2_ref_cg", "enet2_ref_src", base + 0xa800, 28); - clks[IMX7D_ENET2_TIME_ROOT_CG] = imx_clk_gate("enet2_time_cg", "enet2_time_src", base + 0xa880, 28); - clks[IMX7D_ENET_PHY_REF_ROOT_CG] = imx_clk_gate("enet_phy_ref_cg", "enet_phy_ref_src", base + 0xa900, 28); - clks[IMX7D_EIM_ROOT_CG] = imx_clk_gate("eim_cg", "eim_src", base + 0xa980, 28); - clks[IMX7D_NAND_ROOT_CG] = imx_clk_gate("nand_cg", "nand_src", base + 0xaa00, 28); - clks[IMX7D_QSPI_ROOT_CG] = imx_clk_gate("qspi_cg", "qspi_src", base + 0xaa80, 28); - clks[IMX7D_USDHC1_ROOT_CG] = imx_clk_gate("usdhc1_cg", "usdhc1_src", base + 0xab00, 28); - clks[IMX7D_USDHC2_ROOT_CG] = imx_clk_gate("usdhc2_cg", "usdhc2_src", base + 0xab80, 28); - clks[IMX7D_USDHC3_ROOT_CG] = imx_clk_gate("usdhc3_cg", "usdhc3_src", base + 0xac00, 28); - clks[IMX7D_CAN1_ROOT_CG] = imx_clk_gate("can1_cg", "can1_src", base + 0xac80, 28); - clks[IMX7D_CAN2_ROOT_CG] = imx_clk_gate("can2_cg", "can2_src", base + 0xad00, 28); - clks[IMX7D_I2C1_ROOT_CG] = imx_clk_gate("i2c1_cg", "i2c1_src", base + 0xad80, 28); - clks[IMX7D_I2C2_ROOT_CG] = imx_clk_gate("i2c2_cg", "i2c2_src", base + 0xae00, 28); - clks[IMX7D_I2C3_ROOT_CG] = imx_clk_gate("i2c3_cg", "i2c3_src", base + 0xae80, 28); - clks[IMX7D_I2C4_ROOT_CG] = imx_clk_gate("i2c4_cg", "i2c4_src", base + 0xaf00, 28); - clks[IMX7D_UART1_ROOT_CG] = imx_clk_gate("uart1_cg", "uart1_src", base + 0xaf80, 28); - clks[IMX7D_UART2_ROOT_CG] = imx_clk_gate("uart2_cg", "uart2_src", base + 0xb000, 28); - clks[IMX7D_UART3_ROOT_CG] = imx_clk_gate("uart3_cg", "uart3_src", base + 0xb080, 28); - clks[IMX7D_UART4_ROOT_CG] = imx_clk_gate("uart4_cg", "uart4_src", base + 0xb100, 28); - clks[IMX7D_UART5_ROOT_CG] = imx_clk_gate("uart5_cg", "uart5_src", base + 0xb180, 28); - clks[IMX7D_UART6_ROOT_CG] = imx_clk_gate("uart6_cg", "uart6_src", base + 0xb200, 28); - clks[IMX7D_UART7_ROOT_CG] = imx_clk_gate("uart7_cg", "uart7_src", base + 0xb280, 28); - clks[IMX7D_ECSPI1_ROOT_CG] = imx_clk_gate("ecspi1_cg", "ecspi1_src", base + 0xb300, 28); - clks[IMX7D_ECSPI2_ROOT_CG] = imx_clk_gate("ecspi2_cg", "ecspi2_src", base + 0xb380, 28); - clks[IMX7D_ECSPI3_ROOT_CG] = imx_clk_gate("ecspi3_cg", "ecspi3_src", base + 0xb400, 28); - clks[IMX7D_ECSPI4_ROOT_CG] = imx_clk_gate("ecspi4_cg", "ecspi4_src", base + 0xb480, 28); - clks[IMX7D_PWM1_ROOT_CG] = imx_clk_gate("pwm1_cg", "pwm1_src", base + 0xb500, 28); - clks[IMX7D_PWM2_ROOT_CG] = imx_clk_gate("pwm2_cg", "pwm2_src", base + 0xb580, 28); - clks[IMX7D_PWM3_ROOT_CG] = imx_clk_gate("pwm3_cg", "pwm3_src", base + 0xb600, 28); - clks[IMX7D_PWM4_ROOT_CG] = imx_clk_gate("pwm4_cg", "pwm4_src", base + 0xb680, 28); - clks[IMX7D_FLEXTIMER1_ROOT_CG] = imx_clk_gate("flextimer1_cg", "flextimer1_src", base + 0xb700, 28); - clks[IMX7D_FLEXTIMER2_ROOT_CG] = imx_clk_gate("flextimer2_cg", "flextimer2_src", base + 0xb780, 28); - clks[IMX7D_SIM1_ROOT_CG] = imx_clk_gate("sim1_cg", "sim1_src", base + 0xb800, 28); - clks[IMX7D_SIM2_ROOT_CG] = imx_clk_gate("sim2_cg", "sim2_src", base + 0xb880, 28); - clks[IMX7D_GPT1_ROOT_CG] = imx_clk_gate("gpt1_cg", "gpt1_src", base + 0xb900, 28); - clks[IMX7D_GPT2_ROOT_CG] = imx_clk_gate("gpt2_cg", "gpt2_src", base + 0xb980, 28); - clks[IMX7D_GPT3_ROOT_CG] = imx_clk_gate("gpt3_cg", "gpt3_src", base + 0xbA00, 28); - clks[IMX7D_GPT4_ROOT_CG] = imx_clk_gate("gpt4_cg", "gpt4_src", base + 0xbA80, 28); - clks[IMX7D_TRACE_ROOT_CG] = imx_clk_gate("trace_cg", "trace_src", base + 0xbb00, 28); - clks[IMX7D_WDOG_ROOT_CG] = imx_clk_gate("wdog_cg", "wdog_src", base + 0xbb80, 28); - clks[IMX7D_CSI_MCLK_ROOT_CG] = imx_clk_gate("csi_mclk_cg", "csi_mclk_src", base + 0xbc00, 28); - clks[IMX7D_AUDIO_MCLK_ROOT_CG] = imx_clk_gate("audio_mclk_cg", "audio_mclk_src", base + 0xbc80, 28); - clks[IMX7D_WRCLK_ROOT_CG] = imx_clk_gate("wrclk_cg", "wrclk_src", base + 0xbd00, 28); - clks[IMX7D_CLKO1_ROOT_CG] = imx_clk_gate("clko1_cg", "clko1_src", base + 0xbd80, 28); - clks[IMX7D_CLKO2_ROOT_CG] = imx_clk_gate("clko2_cg", "clko2_src", base + 0xbe00, 28); - - clks[IMX7D_MAIN_AXI_ROOT_PRE_DIV] = imx_clk_divider("axi_pre_div", "axi_cg", base + 0x8800, 16, 3); - clks[IMX7D_DISP_AXI_ROOT_PRE_DIV] = imx_clk_divider("disp_axi_pre_div", "disp_axi_cg", base + 0x8880, 16, 3); - clks[IMX7D_ENET_AXI_ROOT_PRE_DIV] = imx_clk_divider("enet_axi_pre_div", "enet_axi_cg", base + 0x8900, 16, 3); - clks[IMX7D_NAND_USDHC_BUS_ROOT_PRE_DIV] = imx_clk_divider("nand_usdhc_pre_div", "nand_usdhc_cg", base + 0x8980, 16, 3); - clks[IMX7D_AHB_CHANNEL_ROOT_PRE_DIV] = imx_clk_divider("ahb_pre_div", "ahb_cg", base + 0x9000, 16, 3); - clks[IMX7D_DRAM_PHYM_ALT_ROOT_PRE_DIV] = imx_clk_divider("dram_phym_alt_pre_div", "dram_phym_alt_cg", base + 0xa000, 16, 3); - clks[IMX7D_DRAM_ALT_ROOT_PRE_DIV] = imx_clk_divider("dram_alt_pre_div", "dram_alt_cg", base + 0xa080, 16, 3); - clks[IMX7D_USB_HSIC_ROOT_PRE_DIV] = imx_clk_divider("usb_hsic_pre_div", "usb_hsic_cg", base + 0xa100, 16, 3); - clks[IMX7D_PCIE_CTRL_ROOT_PRE_DIV] = imx_clk_divider("pcie_ctrl_pre_div", "pcie_ctrl_cg", base + 0xa180, 16, 3); - clks[IMX7D_PCIE_PHY_ROOT_PRE_DIV] = imx_clk_divider("pcie_phy_pre_div", "pcie_phy_cg", base + 0xa200, 16, 3); - clks[IMX7D_EPDC_PIXEL_ROOT_PRE_DIV] = imx_clk_divider("epdc_pixel_pre_div", "epdc_pixel_cg", base + 0xa280, 16, 3); - clks[IMX7D_LCDIF_PIXEL_ROOT_PRE_DIV] = imx_clk_divider("lcdif_pixel_pre_div", "lcdif_pixel_cg", base + 0xa300, 16, 3); - clks[IMX7D_MIPI_DSI_ROOT_PRE_DIV] = imx_clk_divider("mipi_dsi_pre_div", "mipi_dsi_cg", base + 0xa380, 16, 3); - clks[IMX7D_MIPI_CSI_ROOT_PRE_DIV] = imx_clk_divider("mipi_csi_pre_div", "mipi_csi_cg", base + 0xa400, 16, 3); - clks[IMX7D_MIPI_DPHY_ROOT_PRE_DIV] = imx_clk_divider("mipi_dphy_pre_div", "mipi_dphy_cg", base + 0xa480, 16, 3); - clks[IMX7D_SAI1_ROOT_PRE_DIV] = imx_clk_divider("sai1_pre_div", "sai1_cg", base + 0xa500, 16, 3); - clks[IMX7D_SAI2_ROOT_PRE_DIV] = imx_clk_divider("sai2_pre_div", "sai2_cg", base + 0xa580, 16, 3); - clks[IMX7D_SAI3_ROOT_PRE_DIV] = imx_clk_divider("sai3_pre_div", "sai3_cg", base + 0xa600, 16, 3); - clks[IMX7D_SPDIF_ROOT_PRE_DIV] = imx_clk_divider("spdif_pre_div", "spdif_cg", base + 0xa680, 16, 3); - clks[IMX7D_ENET1_REF_ROOT_PRE_DIV] = imx_clk_divider("enet1_ref_pre_div", "enet1_ref_cg", base + 0xa700, 16, 3); - clks[IMX7D_ENET1_TIME_ROOT_PRE_DIV] = imx_clk_divider("enet1_time_pre_div", "enet1_time_cg", base + 0xa780, 16, 3); - clks[IMX7D_ENET2_REF_ROOT_PRE_DIV] = imx_clk_divider("enet2_ref_pre_div", "enet2_ref_cg", base + 0xa800, 16, 3); - clks[IMX7D_ENET2_TIME_ROOT_PRE_DIV] = imx_clk_divider("enet2_time_pre_div", "enet2_time_cg", base + 0xa880, 16, 3); - clks[IMX7D_ENET_PHY_REF_ROOT_PRE_DIV] = imx_clk_divider("enet_phy_ref_pre_div", "enet_phy_ref_cg", base + 0xa900, 16, 3); - clks[IMX7D_EIM_ROOT_PRE_DIV] = imx_clk_divider("eim_pre_div", "eim_cg", base + 0xa980, 16, 3); - clks[IMX7D_NAND_ROOT_PRE_DIV] = imx_clk_divider("nand_pre_div", "nand_cg", base + 0xaa00, 16, 3); - clks[IMX7D_QSPI_ROOT_PRE_DIV] = imx_clk_divider("qspi_pre_div", "qspi_cg", base + 0xaa80, 16, 3); - clks[IMX7D_USDHC1_ROOT_PRE_DIV] = imx_clk_divider("usdhc1_pre_div", "usdhc1_cg", base + 0xab00, 16, 3); - clks[IMX7D_USDHC2_ROOT_PRE_DIV] = imx_clk_divider("usdhc2_pre_div", "usdhc2_cg", base + 0xab80, 16, 3); - clks[IMX7D_USDHC3_ROOT_PRE_DIV] = imx_clk_divider("usdhc3_pre_div", "usdhc3_cg", base + 0xac00, 16, 3); - clks[IMX7D_CAN1_ROOT_PRE_DIV] = imx_clk_divider("can1_pre_div", "can1_cg", base + 0xac80, 16, 3); - clks[IMX7D_CAN2_ROOT_PRE_DIV] = imx_clk_divider("can2_pre_div", "can2_cg", base + 0xad00, 16, 3); - clks[IMX7D_I2C1_ROOT_PRE_DIV] = imx_clk_divider("i2c1_pre_div", "i2c1_cg", base + 0xad80, 16, 3); - clks[IMX7D_I2C2_ROOT_PRE_DIV] = imx_clk_divider("i2c2_pre_div", "i2c2_cg", base + 0xae00, 16, 3); - clks[IMX7D_I2C3_ROOT_PRE_DIV] = imx_clk_divider("i2c3_pre_div", "i2c3_cg", base + 0xae80, 16, 3); - clks[IMX7D_I2C4_ROOT_PRE_DIV] = imx_clk_divider("i2c4_pre_div", "i2c4_cg", base + 0xaf00, 16, 3); - clks[IMX7D_UART1_ROOT_PRE_DIV] = imx_clk_divider("uart1_pre_div", "uart1_cg", base + 0xaf80, 16, 3); - clks[IMX7D_UART2_ROOT_PRE_DIV] = imx_clk_divider("uart2_pre_div", "uart2_cg", base + 0xb000, 16, 3); - clks[IMX7D_UART3_ROOT_PRE_DIV] = imx_clk_divider("uart3_pre_div", "uart3_cg", base + 0xb080, 16, 3); - clks[IMX7D_UART4_ROOT_PRE_DIV] = imx_clk_divider("uart4_pre_div", "uart4_cg", base + 0xb100, 16, 3); - clks[IMX7D_UART5_ROOT_PRE_DIV] = imx_clk_divider("uart5_pre_div", "uart5_cg", base + 0xb180, 16, 3); - clks[IMX7D_UART6_ROOT_PRE_DIV] = imx_clk_divider("uart6_pre_div", "uart6_cg", base + 0xb200, 16, 3); - clks[IMX7D_UART7_ROOT_PRE_DIV] = imx_clk_divider("uart7_pre_div", "uart7_cg", base + 0xb280, 16, 3); - clks[IMX7D_ECSPI1_ROOT_PRE_DIV] = imx_clk_divider("ecspi1_pre_div", "ecspi1_cg", base + 0xb300, 16, 3); - clks[IMX7D_ECSPI2_ROOT_PRE_DIV] = imx_clk_divider("ecspi2_pre_div", "ecspi2_cg", base + 0xb380, 16, 3); - clks[IMX7D_ECSPI3_ROOT_PRE_DIV] = imx_clk_divider("ecspi3_pre_div", "ecspi3_cg", base + 0xb400, 16, 3); - clks[IMX7D_ECSPI4_ROOT_PRE_DIV] = imx_clk_divider("ecspi4_pre_div", "ecspi4_cg", base + 0xb480, 16, 3); - clks[IMX7D_PWM1_ROOT_PRE_DIV] = imx_clk_divider("pwm1_pre_div", "pwm1_cg", base + 0xb500, 16, 3); - clks[IMX7D_PWM2_ROOT_PRE_DIV] = imx_clk_divider("pwm2_pre_div", "pwm2_cg", base + 0xb580, 16, 3); - clks[IMX7D_PWM3_ROOT_PRE_DIV] = imx_clk_divider("pwm3_pre_div", "pwm3_cg", base + 0xb600, 16, 3); - clks[IMX7D_PWM4_ROOT_PRE_DIV] = imx_clk_divider("pwm4_pre_div", "pwm4_cg", base + 0xb680, 16, 3); - clks[IMX7D_FLEXTIMER1_ROOT_PRE_DIV] = imx_clk_divider("flextimer1_pre_div", "flextimer1_cg", base + 0xb700, 16, 3); - clks[IMX7D_FLEXTIMER2_ROOT_PRE_DIV] = imx_clk_divider("flextimer2_pre_div", "flextimer2_cg", base + 0xb780, 16, 3); - clks[IMX7D_SIM1_ROOT_PRE_DIV] = imx_clk_divider("sim1_pre_div", "sim1_cg", base + 0xb800, 16, 3); - clks[IMX7D_SIM2_ROOT_PRE_DIV] = imx_clk_divider("sim2_pre_div", "sim2_cg", base + 0xb880, 16, 3); - clks[IMX7D_GPT1_ROOT_PRE_DIV] = imx_clk_divider("gpt1_pre_div", "gpt1_cg", base + 0xb900, 16, 3); - clks[IMX7D_GPT2_ROOT_PRE_DIV] = imx_clk_divider("gpt2_pre_div", "gpt2_cg", base + 0xb980, 16, 3); - clks[IMX7D_GPT3_ROOT_PRE_DIV] = imx_clk_divider("gpt3_pre_div", "gpt3_cg", base + 0xba00, 16, 3); - clks[IMX7D_GPT4_ROOT_PRE_DIV] = imx_clk_divider("gpt4_pre_div", "gpt4_cg", base + 0xba80, 16, 3); - clks[IMX7D_TRACE_ROOT_PRE_DIV] = imx_clk_divider("trace_pre_div", "trace_cg", base + 0xbb00, 16, 3); - clks[IMX7D_WDOG_ROOT_PRE_DIV] = imx_clk_divider("wdog_pre_div", "wdog_cg", base + 0xbb80, 16, 3); - clks[IMX7D_CSI_MCLK_ROOT_PRE_DIV] = imx_clk_divider("csi_mclk_pre_div", "csi_mclk_cg", base + 0xbc00, 16, 3); - clks[IMX7D_AUDIO_MCLK_ROOT_PRE_DIV] = imx_clk_divider("audio_mclk_pre_div", "audio_mclk_cg", base + 0xbc80, 16, 3); - clks[IMX7D_WRCLK_ROOT_PRE_DIV] = imx_clk_divider("wrclk_pre_div", "wrclk_cg", base + 0xbd00, 16, 3); - clks[IMX7D_CLKO1_ROOT_PRE_DIV] = imx_clk_divider("clko1_pre_div", "clko1_cg", base + 0xbd80, 16, 3); - clks[IMX7D_CLKO2_ROOT_PRE_DIV] = imx_clk_divider("clko2_pre_div", "clko2_cg", base + 0xbe00, 16, 3); - - clks[IMX7D_ARM_A7_ROOT_DIV] = imx_clk_divider("arm_a7_div", "arm_a7_cg", base + 0x8000, 0, 3); - clks[IMX7D_ARM_M4_ROOT_DIV] = imx_clk_divider("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3); - clks[IMX7D_ARM_M0_ROOT_DIV] = imx_clk_divider("arm_m0_div", "arm_m0_cg", base + 0x8100, 0, 3); - clks[IMX7D_MAIN_AXI_ROOT_DIV] = imx_clk_divider("axi_post_div", "axi_pre_div", base + 0x8800, 0, 6); - clks[IMX7D_DISP_AXI_ROOT_DIV] = imx_clk_divider("disp_axi_post_div", "disp_axi_pre_div", base + 0x8880, 0, 6); - clks[IMX7D_ENET_AXI_ROOT_DIV] = imx_clk_divider("enet_axi_post_div", "enet_axi_pre_div", base + 0x8900, 0, 6); - clks[IMX7D_NAND_USDHC_BUS_ROOT_DIV] = imx_clk_divider("nand_usdhc_post_div", "nand_usdhc_pre_div", base + 0x8980, 0, 6); - clks[IMX7D_AHB_CHANNEL_ROOT_DIV] = imx_clk_divider("ahb_post_div", "ahb_pre_div", base + 0x9000, 0, 6); - clks[IMX7D_DRAM_ROOT_DIV] = imx_clk_divider("dram_post_div", "dram_cg", base + 0x9880, 0, 3); - clks[IMX7D_DRAM_PHYM_ALT_ROOT_DIV] = imx_clk_divider("dram_phym_alt_post_div", "dram_phym_alt_pre_div", base + 0xa000, 0, 3); - clks[IMX7D_DRAM_ALT_ROOT_DIV] = imx_clk_divider("dram_alt_post_div", "dram_alt_pre_div", base + 0xa080, 0, 3); - clks[IMX7D_USB_HSIC_ROOT_DIV] = imx_clk_divider("usb_hsic_post_div", "usb_hsic_pre_div", base + 0xa100, 0, 6); - clks[IMX7D_PCIE_CTRL_ROOT_DIV] = imx_clk_divider("pcie_ctrl_post_div", "pcie_ctrl_pre_div", base + 0xa180, 0, 6); - clks[IMX7D_PCIE_PHY_ROOT_DIV] = imx_clk_divider("pcie_phy_post_div", "pcie_phy_pre_div", base + 0xa200, 0, 6); - clks[IMX7D_EPDC_PIXEL_ROOT_DIV] = imx_clk_divider("epdc_pixel_post_div", "epdc_pixel_pre_div", base + 0xa280, 0, 6); - clks[IMX7D_LCDIF_PIXEL_ROOT_DIV] = imx_clk_divider("lcdif_pixel_post_div", "lcdif_pixel_pre_div", base + 0xa300, 0, 6); - clks[IMX7D_MIPI_DSI_ROOT_DIV] = imx_clk_divider("mipi_dsi_post_div", "mipi_dsi_pre_div", base + 0xa380, 0, 6); - clks[IMX7D_MIPI_CSI_ROOT_DIV] = imx_clk_divider("mipi_csi_post_div", "mipi_csi_pre_div", base + 0xa400, 0, 6); - clks[IMX7D_MIPI_DPHY_ROOT_DIV] = imx_clk_divider("mipi_dphy_post_div", "mipi_csi_dphy_div", base + 0xa480, 0, 6); - clks[IMX7D_SAI1_ROOT_DIV] = imx_clk_divider("sai1_post_div", "sai1_pre_div", base + 0xa500, 0, 6); - clks[IMX7D_SAI2_ROOT_DIV] = imx_clk_divider("sai2_post_div", "sai2_pre_div", base + 0xa580, 0, 6); - clks[IMX7D_SAI3_ROOT_DIV] = imx_clk_divider("sai3_post_div", "sai3_pre_div", base + 0xa600, 0, 6); - clks[IMX7D_SPDIF_ROOT_DIV] = imx_clk_divider("spdif_post_div", "spdif_pre_div", base + 0xa680, 0, 6); - clks[IMX7D_ENET1_REF_ROOT_DIV] = imx_clk_divider("enet1_ref_post_div", "enet1_ref_pre_div", base + 0xa700, 0, 6); - clks[IMX7D_ENET1_TIME_ROOT_DIV] = imx_clk_divider("enet1_time_post_div", "enet1_time_pre_div", base + 0xa780, 0, 6); - clks[IMX7D_ENET2_REF_ROOT_DIV] = imx_clk_divider("enet2_ref_post_div", "enet2_ref_pre_div", base + 0xa800, 0, 6); - clks[IMX7D_ENET2_TIME_ROOT_DIV] = imx_clk_divider("enet2_time_post_div", "enet2_time_pre_div", base + 0xa880, 0, 6); - clks[IMX7D_ENET_PHY_REF_ROOT_DIV] = imx_clk_divider("enet_phy_ref_post_div", "enet_phy_ref_pre_div", base + 0xa900, 0, 6); - clks[IMX7D_EIM_ROOT_DIV] = imx_clk_divider("eim_post_div", "eim_pre_div", base + 0xa980, 0, 6); - clks[IMX7D_NAND_ROOT_DIV] = imx_clk_divider("nand_post_div", "nand_pre_div", base + 0xaa00, 0, 6); - clks[IMX7D_QSPI_ROOT_DIV] = imx_clk_divider("qspi_post_div", "qspi_pre_div", base + 0xaa80, 0, 6); - clks[IMX7D_USDHC1_ROOT_DIV] = imx_clk_divider("usdhc1_post_div", "usdhc1_pre_div", base + 0xab00, 0, 6); - clks[IMX7D_USDHC2_ROOT_DIV] = imx_clk_divider("usdhc2_post_div", "usdhc2_pre_div", base + 0xab80, 0, 6); - clks[IMX7D_USDHC3_ROOT_DIV] = imx_clk_divider("usdhc3_post_div", "usdhc3_pre_div", base + 0xac00, 0, 6); - clks[IMX7D_CAN1_ROOT_DIV] = imx_clk_divider("can1_post_div", "can1_pre_div", base + 0xac80, 0, 6); - clks[IMX7D_CAN2_ROOT_DIV] = imx_clk_divider("can2_post_div", "can2_pre_div", base + 0xad00, 0, 6); - clks[IMX7D_I2C1_ROOT_DIV] = imx_clk_divider("i2c1_post_div", "i2c1_pre_div", base + 0xad80, 0, 6); - clks[IMX7D_I2C2_ROOT_DIV] = imx_clk_divider("i2c2_post_div", "i2c2_pre_div", base + 0xae00, 0, 6); - clks[IMX7D_I2C3_ROOT_DIV] = imx_clk_divider("i2c3_post_div", "i2c3_pre_div", base + 0xae80, 0, 6); - clks[IMX7D_I2C4_ROOT_DIV] = imx_clk_divider("i2c4_post_div", "i2c4_pre_div", base + 0xaf00, 0, 6); - clks[IMX7D_UART1_ROOT_DIV] = imx_clk_divider("uart1_post_div", "uart1_pre_div", base + 0xaf80, 0, 6); - clks[IMX7D_UART2_ROOT_DIV] = imx_clk_divider("uart2_post_div", "uart2_pre_div", base + 0xb000, 0, 6); - clks[IMX7D_UART3_ROOT_DIV] = imx_clk_divider("uart3_post_div", "uart3_pre_div", base + 0xb080, 0, 6); - clks[IMX7D_UART4_ROOT_DIV] = imx_clk_divider("uart4_post_div", "uart4_pre_div", base + 0xb100, 0, 6); - clks[IMX7D_UART5_ROOT_DIV] = imx_clk_divider("uart5_post_div", "uart5_pre_div", base + 0xb180, 0, 6); - clks[IMX7D_UART6_ROOT_DIV] = imx_clk_divider("uart6_post_div", "uart6_pre_div", base + 0xb200, 0, 6); - clks[IMX7D_UART7_ROOT_DIV] = imx_clk_divider("uart7_post_div", "uart7_pre_div", base + 0xb280, 0, 6); - clks[IMX7D_ECSPI1_ROOT_DIV] = imx_clk_divider("ecspi1_post_div", "ecspi1_pre_div", base + 0xb300, 0, 6); - clks[IMX7D_ECSPI2_ROOT_DIV] = imx_clk_divider("ecspi2_post_div", "ecspi2_pre_div", base + 0xb380, 0, 6); - clks[IMX7D_ECSPI3_ROOT_DIV] = imx_clk_divider("ecspi3_post_div", "ecspi3_pre_div", base + 0xb400, 0, 6); - clks[IMX7D_ECSPI4_ROOT_DIV] = imx_clk_divider("ecspi4_post_div", "ecspi4_pre_div", base + 0xb480, 0, 6); - clks[IMX7D_PWM1_ROOT_DIV] = imx_clk_divider("pwm1_post_div", "pwm1_pre_div", base + 0xb500, 0, 6); - clks[IMX7D_PWM2_ROOT_DIV] = imx_clk_divider("pwm2_post_div", "pwm2_pre_div", base + 0xb580, 0, 6); - clks[IMX7D_PWM3_ROOT_DIV] = imx_clk_divider("pwm3_post_div", "pwm3_pre_div", base + 0xb600, 0, 6); - clks[IMX7D_PWM4_ROOT_DIV] = imx_clk_divider("pwm4_post_div", "pwm4_pre_div", base + 0xb680, 0, 6); - clks[IMX7D_FLEXTIMER1_ROOT_DIV] = imx_clk_divider("flextimer1_post_div", "flextimer1_pre_div", base + 0xb700, 0, 6); - clks[IMX7D_FLEXTIMER2_ROOT_DIV] = imx_clk_divider("flextimer2_post_div", "flextimer2_pre_div", base + 0xb780, 0, 6); - clks[IMX7D_SIM1_ROOT_DIV] = imx_clk_divider("sim1_post_div", "sim1_pre_div", base + 0xb800, 0, 6); - clks[IMX7D_SIM2_ROOT_DIV] = imx_clk_divider("sim2_post_div", "sim2_pre_div", base + 0xb880, 0, 6); - clks[IMX7D_GPT1_ROOT_DIV] = imx_clk_divider("gpt1_post_div", "gpt1_pre_div", base + 0xb900, 0, 6); - clks[IMX7D_GPT2_ROOT_DIV] = imx_clk_divider("gpt2_post_div", "gpt2_pre_div", base + 0xb980, 0, 6); - clks[IMX7D_GPT3_ROOT_DIV] = imx_clk_divider("gpt3_post_div", "gpt3_pre_div", base + 0xba00, 0, 6); - clks[IMX7D_GPT4_ROOT_DIV] = imx_clk_divider("gpt4_post_div", "gpt4_pre_div", base + 0xba80, 0, 6); - clks[IMX7D_TRACE_ROOT_DIV] = imx_clk_divider("trace_post_div", "trace_pre_div", base + 0xbb00, 0, 6); - clks[IMX7D_WDOG_ROOT_DIV] = imx_clk_divider("wdog_post_div", "wdog_pre_div", base + 0xbb80, 0, 6); - clks[IMX7D_CSI_MCLK_ROOT_DIV] = imx_clk_divider("csi_mclk_post_div", "csi_mclk_pre_div", base + 0xbc00, 0, 6); - clks[IMX7D_AUDIO_MCLK_ROOT_DIV] = imx_clk_divider("audio_mclk_post_div", "audio_mclk_pre_div", base + 0xbc80, 0, 6); - clks[IMX7D_WRCLK_ROOT_DIV] = imx_clk_divider("wrclk_post_div", "wrclk_pre_div", base + 0xbd00, 0, 6); - clks[IMX7D_CLKO1_ROOT_DIV] = imx_clk_divider("clko1_post_div", "clko1_pre_div", base + 0xbd80, 0, 6); - clks[IMX7D_CLKO2_ROOT_DIV] = imx_clk_divider("clko2_post_div", "clko2_pre_div", base + 0xbe00, 0, 6); - - clks[IMX7D_ARM_A7_ROOT_CLK] = imx_clk_gate2("arm_a7_root_clk", "arm_a7_div", base + 0x4000, 0); - clks[IMX7D_ARM_M4_ROOT_CLK] = imx_clk_gate2("arm_m4_root_clk", "arm_m4_div", base + 0x4010, 0); - clks[IMX7D_ARM_M0_ROOT_CLK] = imx_clk_gate2("arm_m0_root_clk", "arm_m0_div", base + 0x4020, 0); - clks[IMX7D_MAIN_AXI_ROOT_CLK] = imx_clk_gate2("main_axi_root_clk", "axi_post_div", base + 0x4040, 0); - clks[IMX7D_DISP_AXI_ROOT_CLK] = imx_clk_gate2("disp_axi_root_clk", "disp_axi_post_div", base + 0x4050, 0); - clks[IMX7D_ENET_AXI_ROOT_CLK] = imx_clk_gate2("enet_axi_root_clk", "enet_axi_post_div", base + 0x4060, 0); - clks[IMX7D_OCRAM_CLK] = imx_clk_gate2("ocram_clk", "axi_post_div", base + 0x4110, 0); - clks[IMX7D_OCRAM_S_CLK] = imx_clk_gate2("ocram_s_clk", "ahb_post_div", base + 0x4120, 0); - clks[IMX7D_NAND_USDHC_BUS_ROOT_CLK] = imx_clk_gate2("nand_usdhc_root_clk", "nand_usdhc_post_div", base + 0x4130, 0); - clks[IMX7D_AHB_CHANNEL_ROOT_CLK] = imx_clk_gate2("ahb_root_clk", "ahb_post_div", base + 0x4200, 0); - clks[IMX7D_DRAM_ROOT_CLK] = imx_clk_gate2("dram_root_clk", "dram_post_div", base + 0x4130, 0); - clks[IMX7D_DRAM_PHYM_ROOT_CLK] = imx_clk_gate2("dram_phym_root_clk", "dram_phym_cg", base + 0x4130, 0); - clks[IMX7D_DRAM_PHYM_ALT_ROOT_CLK] = imx_clk_gate2("dram_phym_alt_root_clk", "dram_phym_alt_post_div", base + 0x4130, 0); - clks[IMX7D_DRAM_ALT_ROOT_CLK] = imx_clk_gate2("dram_alt_root_clk", "dram_alt_post_div", base + 0x4130, 0); - clks[IMX7D_USB_HSIC_ROOT_CLK] = imx_clk_gate2("usb_hsic_root_clk", "usb_hsic_post_div", base + 0x4420, 0); - clks[IMX7D_PCIE_CTRL_ROOT_CLK] = imx_clk_gate2("pcie_ctrl_root_clk", "pcie_ctrl_post_div", base + 0x4600, 0); - clks[IMX7D_PCIE_PHY_ROOT_CLK] = imx_clk_gate2("pcie_phy_root_clk", "pcie_phy_post_div", base + 0x4600, 0); - clks[IMX7D_EPDC_PIXEL_ROOT_CLK] = imx_clk_gate2("epdc_pixel_root_clk", "epdc_pixel_post_div", base + 0x44a0, 0); - clks[IMX7D_LCDIF_PIXEL_ROOT_CLK] = imx_clk_gate2("lcdif_pixel_root_clk", "lcdif_pixel_post_div", base + 0x44b0, 0); - clks[IMX7D_MIPI_DSI_ROOT_CLK] = imx_clk_gate2("mipi_dsi_root_clk", "mipi_dsi_post_div", base + 0x4650, 0); - clks[IMX7D_MIPI_CSI_ROOT_CLK] = imx_clk_gate2("mipi_csi_root_clk", "mipi_csi_post_div", base + 0x4640, 0); - clks[IMX7D_MIPI_DPHY_ROOT_CLK] = imx_clk_gate2("mipi_dphy_root_clk", "mipi_dphy_post_div", base + 0x4660, 0); - clks[IMX7D_SAI1_ROOT_CLK] = imx_clk_gate2("sai1_root_clk", "sai1_post_div", base + 0x48c0, 0); - clks[IMX7D_SAI2_ROOT_CLK] = imx_clk_gate2("sai2_root_clk", "sai2_post_div", base + 0x48d0, 0); - clks[IMX7D_SAI3_ROOT_CLK] = imx_clk_gate2("sai3_root_clk", "sai3_post_div", base + 0x48e0, 0); - clks[IMX7D_SPDIF_ROOT_CLK] = imx_clk_gate2("spdif_root_clk", "spdif_post_div", base + 0x44d0, 0); - clks[IMX7D_ENET1_REF_ROOT_CLK] = imx_clk_gate2("enet1_ref_root_clk", "enet1_ref_post_div", base + 0x44e0, 0); - clks[IMX7D_ENET1_TIME_ROOT_CLK] = imx_clk_gate2("enet1_time_root_clk", "enet1_time_post_div", base + 0x44f0, 0); - clks[IMX7D_ENET2_REF_ROOT_CLK] = imx_clk_gate2("enet2_ref_root_clk", "enet2_ref_post_div", base + 0x4500, 0); - clks[IMX7D_ENET2_TIME_ROOT_CLK] = imx_clk_gate2("enet2_time_root_clk", "enet2_time_post_div", base + 0x4510, 0); - clks[IMX7D_ENET_PHY_REF_ROOT_CLK] = imx_clk_gate2("enet_phy_ref_root_clk", "enet_phy_ref_post_div", base + 0x4520, 0); - clks[IMX7D_EIM_ROOT_CLK] = imx_clk_gate2("eim_root_clk", "eim_post_div", base + 0x4160, 0); - clks[IMX7D_NAND_ROOT_CLK] = imx_clk_gate2("nand_root_clk", "nand_post_div", base + 0x4140, 0); - clks[IMX7D_QSPI_ROOT_CLK] = imx_clk_gate2("qspi_root_clk", "qspi_post_div", base + 0x4150, 0); - clks[IMX7D_USDHC1_ROOT_CLK] = imx_clk_gate2("usdhc1_root_clk", "usdhc1_post_div", base + 0x46c0, 0); - clks[IMX7D_USDHC2_ROOT_CLK] = imx_clk_gate2("usdhc2_root_clk", "usdhc2_post_div", base + 0x46d0, 0); - clks[IMX7D_USDHC3_ROOT_CLK] = imx_clk_gate2("usdhc3_root_clk", "usdhc3_post_div", base + 0x46e0, 0); - clks[IMX7D_CAN1_ROOT_CLK] = imx_clk_gate2("can1_root_clk", "can1_post_div", base + 0x4740, 0); - clks[IMX7D_CAN2_ROOT_CLK] = imx_clk_gate2("can2_root_clk", "can2_post_div", base + 0x4750, 0); - clks[IMX7D_I2C1_ROOT_CLK] = imx_clk_gate2("i2c1_root_clk", "i2c1_post_div", base + 0x4880, 0); - clks[IMX7D_I2C2_ROOT_CLK] = imx_clk_gate2("i2c2_root_clk", "i2c2_post_div", base + 0x4890, 0); - clks[IMX7D_I2C3_ROOT_CLK] = imx_clk_gate2("i2c3_root_clk", "i2c3_post_div", base + 0x48a0, 0); - clks[IMX7D_I2C4_ROOT_CLK] = imx_clk_gate2("i2c4_root_clk", "i2c4_post_div", base + 0x48b0, 0); - clks[IMX7D_UART1_ROOT_CLK] = imx_clk_gate2("uart1_root_clk", "uart1_post_div", base + 0x4940, 0); - clks[IMX7D_UART2_ROOT_CLK] = imx_clk_gate2("uart2_root_clk", "uart2_post_div", base + 0x4950, 0); - clks[IMX7D_UART3_ROOT_CLK] = imx_clk_gate2("uart3_root_clk", "uart3_post_div", base + 0x4960, 0); - clks[IMX7D_UART4_ROOT_CLK] = imx_clk_gate2("uart4_root_clk", "uart4_post_div", base + 0x4970, 0); - clks[IMX7D_UART5_ROOT_CLK] = imx_clk_gate2("uart5_root_clk", "uart5_post_div", base + 0x4980, 0); - clks[IMX7D_UART6_ROOT_CLK] = imx_clk_gate2("uart6_root_clk", "uart6_post_div", base + 0x4990, 0); - clks[IMX7D_UART7_ROOT_CLK] = imx_clk_gate2("uart7_root_clk", "uart7_post_div", base + 0x49a0, 0); - clks[IMX7D_ECSPI1_ROOT_CLK] = imx_clk_gate2("ecspi1_root_clk", "ecspi1_post_div", base + 0x4780, 0); - clks[IMX7D_ECSPI2_ROOT_CLK] = imx_clk_gate2("ecspi2_root_clk", "ecspi2_post_div", base + 0x4790, 0); - clks[IMX7D_ECSPI3_ROOT_CLK] = imx_clk_gate2("ecspi3_root_clk", "ecspi3_post_div", base + 0x47a0, 0); - clks[IMX7D_ECSPI4_ROOT_CLK] = imx_clk_gate2("ecspi4_root_clk", "ecspi4_post_div", base + 0x47b0, 0); - clks[IMX7D_PWM1_ROOT_CLK] = imx_clk_gate2("pwm1_root_clk", "pwm1_post_div", base + 0x4840, 0); - clks[IMX7D_PWM2_ROOT_CLK] = imx_clk_gate2("pwm2_root_clk", "pwm2_post_div", base + 0x4850, 0); - clks[IMX7D_PWM3_ROOT_CLK] = imx_clk_gate2("pwm3_root_clk", "pwm3_post_div", base + 0x4860, 0); - clks[IMX7D_PWM4_ROOT_CLK] = imx_clk_gate2("pwm4_root_clk", "pwm4_post_div", base + 0x4870, 0); - clks[IMX7D_FLEXTIMER1_ROOT_CLK] = imx_clk_gate2("flextimer1_root_clk", "flextimer1_post_div", base + 0x4800, 0); - clks[IMX7D_FLEXTIMER2_ROOT_CLK] = imx_clk_gate2("flextimer2_root_clk", "flextimer2_post_div", base + 0x4810, 0); - clks[IMX7D_SIM1_ROOT_CLK] = imx_clk_gate2("sim1_root_clk", "sim1_post_div", base + 0x4900, 0); - clks[IMX7D_SIM2_ROOT_CLK] = imx_clk_gate2("sim2_root_clk", "sim2_post_div", base + 0x4910, 0); - clks[IMX7D_GPT1_ROOT_CLK] = imx_clk_gate2("gpt1_root_clk", "gpt1_post_div", base + 0x47c0, 0); - clks[IMX7D_GPT2_ROOT_CLK] = imx_clk_gate2("gpt2_root_clk", "gpt2_post_div", base + 0x47d0, 0); - clks[IMX7D_GPT3_ROOT_CLK] = imx_clk_gate2("gpt3_root_clk", "gpt3_post_div", base + 0x47e0, 0); - clks[IMX7D_GPT4_ROOT_CLK] = imx_clk_gate2("gpt4_root_clk", "gpt4_post_div", base + 0x47f0, 0); - clks[IMX7D_TRACE_ROOT_CLK] = imx_clk_gate2("trace_root_clk", "trace_post_div", base + 0x4300, 0); - clks[IMX7D_WDOG1_ROOT_CLK] = imx_clk_gate2("wdog1_root_clk", "wdog_post_div", base + 0x49c0, 0); - clks[IMX7D_WDOG2_ROOT_CLK] = imx_clk_gate2("wdog2_root_clk", "wdog_post_div", base + 0x49d0, 0); - clks[IMX7D_WDOG3_ROOT_CLK] = imx_clk_gate2("wdog3_root_clk", "wdog_post_div", base + 0x49e0, 0); - clks[IMX7D_WDOG4_ROOT_CLK] = imx_clk_gate2("wdog4_root_clk", "wdog_post_div", base + 0x49f0, 0); - clks[IMX7D_CSI_MCLK_ROOT_CLK] = imx_clk_gate2("csi_mclk_root_clk", "csi_mclk_post_div", base + 0x4490, 0); - clks[IMX7D_AUDIO_MCLK_ROOT_CLK] = imx_clk_gate2("audio_mclk_root_clk", "audio_mclk_post_div", base + 0x4790, 0); - clks[IMX7D_WRCLK_ROOT_CLK] = imx_clk_gate2("wrclk_root_clk", "wrclk_post_div", base + 0x47a0, 0); - clks[IMX7D_ADC_ROOT_CLK] = imx_clk_gate2("adc_root_clk", "ipg_root_clk", base + 0x4200, 0); + clks[IMX7D_ARM_A7_ROOT_SRC] = imx_clk_mux2("arm_a7_src", base + 0x8000, 24, 3, arm_a7_sel, ARRAY_SIZE(arm_a7_sel)); + clks[IMX7D_ARM_M4_ROOT_SRC] = imx_clk_mux2("arm_m4_src", base + 0x8080, 24, 3, arm_m4_sel, ARRAY_SIZE(arm_m4_sel)); + clks[IMX7D_ARM_M0_ROOT_SRC] = imx_clk_mux2("arm_m0_src", base + 0x8100, 24, 3, arm_m0_sel, ARRAY_SIZE(arm_m0_sel)); + clks[IMX7D_MAIN_AXI_ROOT_SRC] = imx_clk_mux2("axi_src", base + 0x8800, 24, 3, axi_sel, ARRAY_SIZE(axi_sel)); + clks[IMX7D_DISP_AXI_ROOT_SRC] = imx_clk_mux2("disp_axi_src", base + 0x8880, 24, 3, disp_axi_sel, ARRAY_SIZE(disp_axi_sel)); + clks[IMX7D_ENET_AXI_ROOT_SRC] = imx_clk_mux2("enet_axi_src", base + 0x8900, 24, 3, enet_axi_sel, ARRAY_SIZE(enet_axi_sel)); + clks[IMX7D_NAND_USDHC_BUS_ROOT_SRC] = imx_clk_mux2("nand_usdhc_src", base + 0x8980, 24, 3, nand_usdhc_bus_sel, ARRAY_SIZE(nand_usdhc_bus_sel)); + clks[IMX7D_AHB_CHANNEL_ROOT_SRC] = imx_clk_mux2("ahb_src", base + 0x9000, 24, 3, ahb_channel_sel, ARRAY_SIZE(ahb_channel_sel)); + clks[IMX7D_DRAM_PHYM_ROOT_SRC] = imx_clk_mux2("dram_phym_src", base + 0x9800, 24, 1, dram_phym_sel, ARRAY_SIZE(dram_phym_sel)); + clks[IMX7D_DRAM_ROOT_SRC] = imx_clk_mux2("dram_src", base + 0x9880, 24, 1, dram_sel, ARRAY_SIZE(dram_sel)); + clks[IMX7D_DRAM_PHYM_ALT_ROOT_SRC] = imx_clk_mux2("dram_phym_alt_src", base + 0xa000, 24, 3, dram_phym_alt_sel, ARRAY_SIZE(dram_phym_alt_sel)); + clks[IMX7D_DRAM_ALT_ROOT_SRC] = imx_clk_mux2("dram_alt_src", base + 0xa080, 24, 3, dram_alt_sel, ARRAY_SIZE(dram_alt_sel)); + clks[IMX7D_USB_HSIC_ROOT_SRC] = imx_clk_mux2("usb_hsic_src", base + 0xa100, 24, 3, usb_hsic_sel, ARRAY_SIZE(usb_hsic_sel)); + clks[IMX7D_PCIE_CTRL_ROOT_SRC] = imx_clk_mux2("pcie_ctrl_src", base + 0xa180, 24, 3, pcie_ctrl_sel, ARRAY_SIZE(pcie_ctrl_sel)); + clks[IMX7D_PCIE_PHY_ROOT_SRC] = imx_clk_mux2("pcie_phy_src", base + 0xa200, 24, 3, pcie_phy_sel, ARRAY_SIZE(pcie_phy_sel)); + clks[IMX7D_EPDC_PIXEL_ROOT_SRC] = imx_clk_mux2("epdc_pixel_src", base + 0xa280, 24, 3, epdc_pixel_sel, ARRAY_SIZE(epdc_pixel_sel)); + clks[IMX7D_LCDIF_PIXEL_ROOT_SRC] = imx_clk_mux2("lcdif_pixel_src", base + 0xa300, 24, 3, lcdif_pixel_sel, ARRAY_SIZE(lcdif_pixel_sel)); + clks[IMX7D_MIPI_DSI_ROOT_SRC] = imx_clk_mux2("mipi_dsi_src", base + 0xa380, 24, 3, mipi_dsi_sel, ARRAY_SIZE(mipi_dsi_sel)); + clks[IMX7D_MIPI_CSI_ROOT_SRC] = imx_clk_mux2("mipi_csi_src", base + 0xa400, 24, 3, mipi_csi_sel, ARRAY_SIZE(mipi_csi_sel)); + clks[IMX7D_MIPI_DPHY_ROOT_SRC] = imx_clk_mux2("mipi_dphy_src", base + 0xa480, 24, 3, mipi_dphy_sel, ARRAY_SIZE(mipi_dphy_sel)); + clks[IMX7D_SAI1_ROOT_SRC] = imx_clk_mux2("sai1_src", base + 0xa500, 24, 3, sai1_sel, ARRAY_SIZE(sai1_sel)); + clks[IMX7D_SAI2_ROOT_SRC] = imx_clk_mux2("sai2_src", base + 0xa580, 24, 3, sai2_sel, ARRAY_SIZE(sai2_sel)); + clks[IMX7D_SAI3_ROOT_SRC] = imx_clk_mux2("sai3_src", base + 0xa600, 24, 3, sai3_sel, ARRAY_SIZE(sai3_sel)); + clks[IMX7D_SPDIF_ROOT_SRC] = imx_clk_mux2("spdif_src", base + 0xa680, 24, 3, spdif_sel, ARRAY_SIZE(spdif_sel)); + clks[IMX7D_ENET1_REF_ROOT_SRC] = imx_clk_mux2("enet1_ref_src", base + 0xa700, 24, 3, enet1_ref_sel, ARRAY_SIZE(enet1_ref_sel)); + clks[IMX7D_ENET1_TIME_ROOT_SRC] = imx_clk_mux2("enet1_time_src", base + 0xa780, 24, 3, enet1_time_sel, ARRAY_SIZE(enet1_time_sel)); + clks[IMX7D_ENET2_REF_ROOT_SRC] = imx_clk_mux2("enet2_ref_src", base + 0xa800, 24, 3, enet2_ref_sel, ARRAY_SIZE(enet2_ref_sel)); + clks[IMX7D_ENET2_TIME_ROOT_SRC] = imx_clk_mux2("enet2_time_src", base + 0xa880, 24, 3, enet2_time_sel, ARRAY_SIZE(enet2_time_sel)); + clks[IMX7D_ENET_PHY_REF_ROOT_SRC] = imx_clk_mux2("enet_phy_ref_src", base + 0xa900, 24, 3, enet_phy_ref_sel, ARRAY_SIZE(enet_phy_ref_sel)); + clks[IMX7D_EIM_ROOT_SRC] = imx_clk_mux2("eim_src", base + 0xa980, 24, 3, eim_sel, ARRAY_SIZE(eim_sel)); + clks[IMX7D_NAND_ROOT_SRC] = imx_clk_mux2("nand_src", base + 0xaa00, 24, 3, nand_sel, ARRAY_SIZE(nand_sel)); + clks[IMX7D_QSPI_ROOT_SRC] = imx_clk_mux2("qspi_src", base + 0xaa80, 24, 3, qspi_sel, ARRAY_SIZE(qspi_sel)); + clks[IMX7D_USDHC1_ROOT_SRC] = imx_clk_mux2("usdhc1_src", base + 0xab00, 24, 3, usdhc1_sel, ARRAY_SIZE(usdhc1_sel)); + clks[IMX7D_USDHC2_ROOT_SRC] = imx_clk_mux2("usdhc2_src", base + 0xab80, 24, 3, usdhc2_sel, ARRAY_SIZE(usdhc2_sel)); + clks[IMX7D_USDHC3_ROOT_SRC] = imx_clk_mux2("usdhc3_src", base + 0xac00, 24, 3, usdhc3_sel, ARRAY_SIZE(usdhc3_sel)); + clks[IMX7D_CAN1_ROOT_SRC] = imx_clk_mux2("can1_src", base + 0xac80, 24, 3, can1_sel, ARRAY_SIZE(can1_sel)); + clks[IMX7D_CAN2_ROOT_SRC] = imx_clk_mux2("can2_src", base + 0xad00, 24, 3, can2_sel, ARRAY_SIZE(can2_sel)); + clks[IMX7D_I2C1_ROOT_SRC] = imx_clk_mux2("i2c1_src", base + 0xad80, 24, 3, i2c1_sel, ARRAY_SIZE(i2c1_sel)); + clks[IMX7D_I2C2_ROOT_SRC] = imx_clk_mux2("i2c2_src", base + 0xae00, 24, 3, i2c2_sel, ARRAY_SIZE(i2c2_sel)); + clks[IMX7D_I2C3_ROOT_SRC] = imx_clk_mux2("i2c3_src", base + 0xae80, 24, 3, i2c3_sel, ARRAY_SIZE(i2c3_sel)); + clks[IMX7D_I2C4_ROOT_SRC] = imx_clk_mux2("i2c4_src", base + 0xaf00, 24, 3, i2c4_sel, ARRAY_SIZE(i2c4_sel)); + clks[IMX7D_UART1_ROOT_SRC] = imx_clk_mux2("uart1_src", base + 0xaf80, 24, 3, uart1_sel, ARRAY_SIZE(uart1_sel)); + clks[IMX7D_UART2_ROOT_SRC] = imx_clk_mux2("uart2_src", base + 0xb000, 24, 3, uart2_sel, ARRAY_SIZE(uart2_sel)); + clks[IMX7D_UART3_ROOT_SRC] = imx_clk_mux2("uart3_src", base + 0xb080, 24, 3, uart3_sel, ARRAY_SIZE(uart3_sel)); + clks[IMX7D_UART4_ROOT_SRC] = imx_clk_mux2("uart4_src", base + 0xb100, 24, 3, uart4_sel, ARRAY_SIZE(uart4_sel)); + clks[IMX7D_UART5_ROOT_SRC] = imx_clk_mux2("uart5_src", base + 0xb180, 24, 3, uart5_sel, ARRAY_SIZE(uart5_sel)); + clks[IMX7D_UART6_ROOT_SRC] = imx_clk_mux2("uart6_src", base + 0xb200, 24, 3, uart6_sel, ARRAY_SIZE(uart6_sel)); + clks[IMX7D_UART7_ROOT_SRC] = imx_clk_mux2("uart7_src", base + 0xb280, 24, 3, uart7_sel, ARRAY_SIZE(uart7_sel)); + clks[IMX7D_ECSPI1_ROOT_SRC] = imx_clk_mux2("ecspi1_src", base + 0xb300, 24, 3, ecspi1_sel, ARRAY_SIZE(ecspi1_sel)); + clks[IMX7D_ECSPI2_ROOT_SRC] = imx_clk_mux2("ecspi2_src", base + 0xb380, 24, 3, ecspi2_sel, ARRAY_SIZE(ecspi2_sel)); + clks[IMX7D_ECSPI3_ROOT_SRC] = imx_clk_mux2("ecspi3_src", base + 0xb400, 24, 3, ecspi3_sel, ARRAY_SIZE(ecspi3_sel)); + clks[IMX7D_ECSPI4_ROOT_SRC] = imx_clk_mux2("ecspi4_src", base + 0xb480, 24, 3, ecspi4_sel, ARRAY_SIZE(ecspi4_sel)); + clks[IMX7D_PWM1_ROOT_SRC] = imx_clk_mux2("pwm1_src", base + 0xb500, 24, 3, pwm1_sel, ARRAY_SIZE(pwm1_sel)); + clks[IMX7D_PWM2_ROOT_SRC] = imx_clk_mux2("pwm2_src", base + 0xb580, 24, 3, pwm2_sel, ARRAY_SIZE(pwm2_sel)); + clks[IMX7D_PWM3_ROOT_SRC] = imx_clk_mux2("pwm3_src", base + 0xb600, 24, 3, pwm3_sel, ARRAY_SIZE(pwm3_sel)); + clks[IMX7D_PWM4_ROOT_SRC] = imx_clk_mux2("pwm4_src", base + 0xb680, 24, 3, pwm4_sel, ARRAY_SIZE(pwm4_sel)); + clks[IMX7D_FLEXTIMER1_ROOT_SRC] = imx_clk_mux2("flextimer1_src", base + 0xb700, 24, 3, flextimer1_sel, ARRAY_SIZE(flextimer1_sel)); + clks[IMX7D_FLEXTIMER2_ROOT_SRC] = imx_clk_mux2("flextimer2_src", base + 0xb780, 24, 3, flextimer2_sel, ARRAY_SIZE(flextimer2_sel)); + clks[IMX7D_SIM1_ROOT_SRC] = imx_clk_mux2("sim1_src", base + 0xb800, 24, 3, sim1_sel, ARRAY_SIZE(sim1_sel)); + clks[IMX7D_SIM2_ROOT_SRC] = imx_clk_mux2("sim2_src", base + 0xb880, 24, 3, sim2_sel, ARRAY_SIZE(sim2_sel)); + clks[IMX7D_GPT1_ROOT_SRC] = imx_clk_mux2("gpt1_src", base + 0xb900, 24, 3, gpt1_sel, ARRAY_SIZE(gpt1_sel)); + clks[IMX7D_GPT2_ROOT_SRC] = imx_clk_mux2("gpt2_src", base + 0xb980, 24, 3, gpt2_sel, ARRAY_SIZE(gpt2_sel)); + clks[IMX7D_GPT3_ROOT_SRC] = imx_clk_mux2("gpt3_src", base + 0xba00, 24, 3, gpt3_sel, ARRAY_SIZE(gpt3_sel)); + clks[IMX7D_GPT4_ROOT_SRC] = imx_clk_mux2("gpt4_src", base + 0xba80, 24, 3, gpt4_sel, ARRAY_SIZE(gpt4_sel)); + clks[IMX7D_TRACE_ROOT_SRC] = imx_clk_mux2("trace_src", base + 0xbb00, 24, 3, trace_sel, ARRAY_SIZE(trace_sel)); + clks[IMX7D_WDOG_ROOT_SRC] = imx_clk_mux2("wdog_src", base + 0xbb80, 24, 3, wdog_sel, ARRAY_SIZE(wdog_sel)); + clks[IMX7D_CSI_MCLK_ROOT_SRC] = imx_clk_mux2("csi_mclk_src", base + 0xbc00, 24, 3, csi_mclk_sel, ARRAY_SIZE(csi_mclk_sel)); + clks[IMX7D_AUDIO_MCLK_ROOT_SRC] = imx_clk_mux2("audio_mclk_src", base + 0xbc80, 24, 3, audio_mclk_sel, ARRAY_SIZE(audio_mclk_sel)); + clks[IMX7D_WRCLK_ROOT_SRC] = imx_clk_mux2("wrclk_src", base + 0xbd00, 24, 3, wrclk_sel, ARRAY_SIZE(wrclk_sel)); + clks[IMX7D_CLKO1_ROOT_SRC] = imx_clk_mux2("clko1_src", base + 0xbd80, 24, 3, clko1_sel, ARRAY_SIZE(clko1_sel)); + clks[IMX7D_CLKO2_ROOT_SRC] = imx_clk_mux2("clko2_src", base + 0xbe00, 24, 3, clko2_sel, ARRAY_SIZE(clko2_sel)); + + clks[IMX7D_ARM_A7_ROOT_CG] = imx_clk_gate3("arm_a7_cg", "arm_a7_src", base + 0x8000, 28); + clks[IMX7D_ARM_M4_ROOT_CG] = imx_clk_gate3("arm_m4_cg", "arm_m4_src", base + 0x8080, 28); + clks[IMX7D_ARM_M0_ROOT_CG] = imx_clk_gate3("arm_m0_cg", "arm_m0_src", base + 0x8100, 28); + clks[IMX7D_MAIN_AXI_ROOT_CG] = imx_clk_gate3("axi_cg", "axi_src", base + 0x8800, 28); + clks[IMX7D_DISP_AXI_ROOT_CG] = imx_clk_gate3("disp_axi_cg", "disp_axi_src", base + 0x8880, 28); + clks[IMX7D_ENET_AXI_ROOT_CG] = imx_clk_gate3("enet_axi_cg", "enet_axi_src", base + 0x8900, 28); + clks[IMX7D_NAND_USDHC_BUS_ROOT_CG] = imx_clk_gate3("nand_usdhc_cg", "nand_usdhc_src", base + 0x8980, 28); + clks[IMX7D_AHB_CHANNEL_ROOT_CG] = imx_clk_gate3("ahb_cg", "ahb_src", base + 0x9000, 28); + clks[IMX7D_DRAM_PHYM_ROOT_CG] = imx_clk_gate3("dram_phym_cg", "dram_phym_src", base + 0x9800, 28); + clks[IMX7D_DRAM_ROOT_CG] = imx_clk_gate3("dram_cg", "dram_src", base + 0x9880, 28); + clks[IMX7D_DRAM_PHYM_ALT_ROOT_CG] = imx_clk_gate3("dram_phym_alt_cg", "dram_phym_alt_src", base + 0xa000, 28); + clks[IMX7D_DRAM_ALT_ROOT_CG] = imx_clk_gate3("dram_alt_cg", "dram_alt_src", base + 0xa080, 28); + clks[IMX7D_USB_HSIC_ROOT_CG] = imx_clk_gate3("usb_hsic_cg", "usb_hsic_src", base + 0xa100, 28); + clks[IMX7D_PCIE_CTRL_ROOT_CG] = imx_clk_gate3("pcie_ctrl_cg", "pcie_ctrl_src", base + 0xa180, 28); + clks[IMX7D_PCIE_PHY_ROOT_CG] = imx_clk_gate3("pcie_phy_cg", "pcie_phy_src", base + 0xa200, 28); + clks[IMX7D_EPDC_PIXEL_ROOT_CG] = imx_clk_gate3("epdc_pixel_cg", "epdc_pixel_src", base + 0xa280, 28); + clks[IMX7D_LCDIF_PIXEL_ROOT_CG] = imx_clk_gate3("lcdif_pixel_cg", "lcdif_pixel_src", base + 0xa300, 28); + clks[IMX7D_MIPI_DSI_ROOT_CG] = imx_clk_gate3("mipi_dsi_cg", "mipi_dsi_src", base + 0xa380, 28); + clks[IMX7D_MIPI_CSI_ROOT_CG] = imx_clk_gate3("mipi_csi_cg", "mipi_csi_src", base + 0xa400, 28); + clks[IMX7D_MIPI_DPHY_ROOT_CG] = imx_clk_gate3("mipi_dphy_cg", "mipi_dphy_src", base + 0xa480, 28); + clks[IMX7D_SAI1_ROOT_CG] = imx_clk_gate3("sai1_cg", "sai1_src", base + 0xa500, 28); + clks[IMX7D_SAI2_ROOT_CG] = imx_clk_gate3("sai2_cg", "sai2_src", base + 0xa580, 28); + clks[IMX7D_SAI3_ROOT_CG] = imx_clk_gate3("sai3_cg", "sai3_src", base + 0xa600, 28); + clks[IMX7D_SPDIF_ROOT_CG] = imx_clk_gate3("spdif_cg", "spdif_src", base + 0xa680, 28); + clks[IMX7D_ENET1_REF_ROOT_CG] = imx_clk_gate3("enet1_ref_cg", "enet1_ref_src", base + 0xa700, 28); + clks[IMX7D_ENET1_TIME_ROOT_CG] = imx_clk_gate3("enet1_time_cg", "enet1_time_src", base + 0xa780, 28); + clks[IMX7D_ENET2_REF_ROOT_CG] = imx_clk_gate3("enet2_ref_cg", "enet2_ref_src", base + 0xa800, 28); + clks[IMX7D_ENET2_TIME_ROOT_CG] = imx_clk_gate3("enet2_time_cg", "enet2_time_src", base + 0xa880, 28); + clks[IMX7D_ENET_PHY_REF_ROOT_CG] = imx_clk_gate3("enet_phy_ref_cg", "enet_phy_ref_src", base + 0xa900, 28); + clks[IMX7D_EIM_ROOT_CG] = imx_clk_gate3("eim_cg", "eim_src", base + 0xa980, 28); + clks[IMX7D_NAND_ROOT_CG] = imx_clk_gate3("nand_cg", "nand_src", base + 0xaa00, 28); + clks[IMX7D_QSPI_ROOT_CG] = imx_clk_gate3("qspi_cg", "qspi_src", base + 0xaa80, 28); + clks[IMX7D_USDHC1_ROOT_CG] = imx_clk_gate3("usdhc1_cg", "usdhc1_src", base + 0xab00, 28); + clks[IMX7D_USDHC2_ROOT_CG] = imx_clk_gate3("usdhc2_cg", "usdhc2_src", base + 0xab80, 28); + clks[IMX7D_USDHC3_ROOT_CG] = imx_clk_gate3("usdhc3_cg", "usdhc3_src", base + 0xac00, 28); + clks[IMX7D_CAN1_ROOT_CG] = imx_clk_gate3("can1_cg", "can1_src", base + 0xac80, 28); + clks[IMX7D_CAN2_ROOT_CG] = imx_clk_gate3("can2_cg", "can2_src", base + 0xad00, 28); + clks[IMX7D_I2C1_ROOT_CG] = imx_clk_gate3("i2c1_cg", "i2c1_src", base + 0xad80, 28); + clks[IMX7D_I2C2_ROOT_CG] = imx_clk_gate3("i2c2_cg", "i2c2_src", base + 0xae00, 28); + clks[IMX7D_I2C3_ROOT_CG] = imx_clk_gate3("i2c3_cg", "i2c3_src", base + 0xae80, 28); + clks[IMX7D_I2C4_ROOT_CG] = imx_clk_gate3("i2c4_cg", "i2c4_src", base + 0xaf00, 28); + clks[IMX7D_UART1_ROOT_CG] = imx_clk_gate3("uart1_cg", "uart1_src", base + 0xaf80, 28); + clks[IMX7D_UART2_ROOT_CG] = imx_clk_gate3("uart2_cg", "uart2_src", base + 0xb000, 28); + clks[IMX7D_UART3_ROOT_CG] = imx_clk_gate3("uart3_cg", "uart3_src", base + 0xb080, 28); + clks[IMX7D_UART4_ROOT_CG] = imx_clk_gate3("uart4_cg", "uart4_src", base + 0xb100, 28); + clks[IMX7D_UART5_ROOT_CG] = imx_clk_gate3("uart5_cg", "uart5_src", base + 0xb180, 28); + clks[IMX7D_UART6_ROOT_CG] = imx_clk_gate3("uart6_cg", "uart6_src", base + 0xb200, 28); + clks[IMX7D_UART7_ROOT_CG] = imx_clk_gate3("uart7_cg", "uart7_src", base + 0xb280, 28); + clks[IMX7D_ECSPI1_ROOT_CG] = imx_clk_gate3("ecspi1_cg", "ecspi1_src", base + 0xb300, 28); + clks[IMX7D_ECSPI2_ROOT_CG] = imx_clk_gate3("ecspi2_cg", "ecspi2_src", base + 0xb380, 28); + clks[IMX7D_ECSPI3_ROOT_CG] = imx_clk_gate3("ecspi3_cg", "ecspi3_src", base + 0xb400, 28); + clks[IMX7D_ECSPI4_ROOT_CG] = imx_clk_gate3("ecspi4_cg", "ecspi4_src", base + 0xb480, 28); + clks[IMX7D_PWM1_ROOT_CG] = imx_clk_gate3("pwm1_cg", "pwm1_src", base + 0xb500, 28); + clks[IMX7D_PWM2_ROOT_CG] = imx_clk_gate3("pwm2_cg", "pwm2_src", base + 0xb580, 28); + clks[IMX7D_PWM3_ROOT_CG] = imx_clk_gate3("pwm3_cg", "pwm3_src", base + 0xb600, 28); + clks[IMX7D_PWM4_ROOT_CG] = imx_clk_gate3("pwm4_cg", "pwm4_src", base + 0xb680, 28); + clks[IMX7D_FLEXTIMER1_ROOT_CG] = imx_clk_gate3("flextimer1_cg", "flextimer1_src", base + 0xb700, 28); + clks[IMX7D_FLEXTIMER2_ROOT_CG] = imx_clk_gate3("flextimer2_cg", "flextimer2_src", base + 0xb780, 28); + clks[IMX7D_SIM1_ROOT_CG] = imx_clk_gate3("sim1_cg", "sim1_src", base + 0xb800, 28); + clks[IMX7D_SIM2_ROOT_CG] = imx_clk_gate3("sim2_cg", "sim2_src", base + 0xb880, 28); + clks[IMX7D_GPT1_ROOT_CG] = imx_clk_gate3("gpt1_cg", "gpt1_src", base + 0xb900, 28); + clks[IMX7D_GPT2_ROOT_CG] = imx_clk_gate3("gpt2_cg", "gpt2_src", base + 0xb980, 28); + clks[IMX7D_GPT3_ROOT_CG] = imx_clk_gate3("gpt3_cg", "gpt3_src", base + 0xbA00, 28); + clks[IMX7D_GPT4_ROOT_CG] = imx_clk_gate3("gpt4_cg", "gpt4_src", base + 0xbA80, 28); + clks[IMX7D_TRACE_ROOT_CG] = imx_clk_gate3("trace_cg", "trace_src", base + 0xbb00, 28); + clks[IMX7D_WDOG_ROOT_CG] = imx_clk_gate3("wdog_cg", "wdog_src", base + 0xbb80, 28); + clks[IMX7D_CSI_MCLK_ROOT_CG] = imx_clk_gate3("csi_mclk_cg", "csi_mclk_src", base + 0xbc00, 28); + clks[IMX7D_AUDIO_MCLK_ROOT_CG] = imx_clk_gate3("audio_mclk_cg", "audio_mclk_src", base + 0xbc80, 28); + clks[IMX7D_WRCLK_ROOT_CG] = imx_clk_gate3("wrclk_cg", "wrclk_src", base + 0xbd00, 28); + clks[IMX7D_CLKO1_ROOT_CG] = imx_clk_gate3("clko1_cg", "clko1_src", base + 0xbd80, 28); + clks[IMX7D_CLKO2_ROOT_CG] = imx_clk_gate3("clko2_cg", "clko2_src", base + 0xbe00, 28); + + clks[IMX7D_MAIN_AXI_ROOT_PRE_DIV] = imx_clk_divider2("axi_pre_div", "axi_cg", base + 0x8800, 16, 3); + clks[IMX7D_DISP_AXI_ROOT_PRE_DIV] = imx_clk_divider2("disp_axi_pre_div", "disp_axi_cg", base + 0x8880, 16, 3); + clks[IMX7D_ENET_AXI_ROOT_PRE_DIV] = imx_clk_divider2("enet_axi_pre_div", "enet_axi_cg", base + 0x8900, 16, 3); + clks[IMX7D_NAND_USDHC_BUS_ROOT_PRE_DIV] = imx_clk_divider2("nand_usdhc_pre_div", "nand_usdhc_cg", base + 0x8980, 16, 3); + clks[IMX7D_AHB_CHANNEL_ROOT_PRE_DIV] = imx_clk_divider2("ahb_pre_div", "ahb_cg", base + 0x9000, 16, 3); + clks[IMX7D_DRAM_PHYM_ALT_ROOT_PRE_DIV] = imx_clk_divider2("dram_phym_alt_pre_div", "dram_phym_alt_cg", base + 0xa000, 16, 3); + clks[IMX7D_DRAM_ALT_ROOT_PRE_DIV] = imx_clk_divider2("dram_alt_pre_div", "dram_alt_cg", base + 0xa080, 16, 3); + clks[IMX7D_USB_HSIC_ROOT_PRE_DIV] = imx_clk_divider2("usb_hsic_pre_div", "usb_hsic_cg", base + 0xa100, 16, 3); + clks[IMX7D_PCIE_CTRL_ROOT_PRE_DIV] = imx_clk_divider2("pcie_ctrl_pre_div", "pcie_ctrl_cg", base + 0xa180, 16, 3); + clks[IMX7D_PCIE_PHY_ROOT_PRE_DIV] = imx_clk_divider2("pcie_phy_pre_div", "pcie_phy_cg", base + 0xa200, 16, 3); + clks[IMX7D_EPDC_PIXEL_ROOT_PRE_DIV] = imx_clk_divider2("epdc_pixel_pre_div", "epdc_pixel_cg", base + 0xa280, 16, 3); + clks[IMX7D_LCDIF_PIXEL_ROOT_PRE_DIV] = imx_clk_divider2("lcdif_pixel_pre_div", "lcdif_pixel_cg", base + 0xa300, 16, 3); + clks[IMX7D_MIPI_DSI_ROOT_PRE_DIV] = imx_clk_divider2("mipi_dsi_pre_div", "mipi_dsi_cg", base + 0xa380, 16, 3); + clks[IMX7D_MIPI_CSI_ROOT_PRE_DIV] = imx_clk_divider2("mipi_csi_pre_div", "mipi_csi_cg", base + 0xa400, 16, 3); + clks[IMX7D_MIPI_DPHY_ROOT_PRE_DIV] = imx_clk_divider2("mipi_dphy_pre_div", "mipi_dphy_cg", base + 0xa480, 16, 3); + clks[IMX7D_SAI1_ROOT_PRE_DIV] = imx_clk_divider2("sai1_pre_div", "sai1_cg", base + 0xa500, 16, 3); + clks[IMX7D_SAI2_ROOT_PRE_DIV] = imx_clk_divider2("sai2_pre_div", "sai2_cg", base + 0xa580, 16, 3); + clks[IMX7D_SAI3_ROOT_PRE_DIV] = imx_clk_divider2("sai3_pre_div", "sai3_cg", base + 0xa600, 16, 3); + clks[IMX7D_SPDIF_ROOT_PRE_DIV] = imx_clk_divider2("spdif_pre_div", "spdif_cg", base + 0xa680, 16, 3); + clks[IMX7D_ENET1_REF_ROOT_PRE_DIV] = imx_clk_divider2("enet1_ref_pre_div", "enet1_ref_cg", base + 0xa700, 16, 3); + clks[IMX7D_ENET1_TIME_ROOT_PRE_DIV] = imx_clk_divider2("enet1_time_pre_div", "enet1_time_cg", base + 0xa780, 16, 3); + clks[IMX7D_ENET2_REF_ROOT_PRE_DIV] = imx_clk_divider2("enet2_ref_pre_div", "enet2_ref_cg", base + 0xa800, 16, 3); + clks[IMX7D_ENET2_TIME_ROOT_PRE_DIV] = imx_clk_divider2("enet2_time_pre_div", "enet2_time_cg", base + 0xa880, 16, 3); + clks[IMX7D_ENET_PHY_REF_ROOT_PRE_DIV] = imx_clk_divider2("enet_phy_ref_pre_div", "enet_phy_ref_cg", base + 0xa900, 16, 3); + clks[IMX7D_EIM_ROOT_PRE_DIV] = imx_clk_divider2("eim_pre_div", "eim_cg", base + 0xa980, 16, 3); + clks[IMX7D_NAND_ROOT_PRE_DIV] = imx_clk_divider2("nand_pre_div", "nand_cg", base + 0xaa00, 16, 3); + clks[IMX7D_QSPI_ROOT_PRE_DIV] = imx_clk_divider2("qspi_pre_div", "qspi_cg", base + 0xaa80, 16, 3); + clks[IMX7D_USDHC1_ROOT_PRE_DIV] = imx_clk_divider2("usdhc1_pre_div", "usdhc1_cg", base + 0xab00, 16, 3); + clks[IMX7D_USDHC2_ROOT_PRE_DIV] = imx_clk_divider2("usdhc2_pre_div", "usdhc2_cg", base + 0xab80, 16, 3); + clks[IMX7D_USDHC3_ROOT_PRE_DIV] = imx_clk_divider2("usdhc3_pre_div", "usdhc3_cg", base + 0xac00, 16, 3); + clks[IMX7D_CAN1_ROOT_PRE_DIV] = imx_clk_divider2("can1_pre_div", "can1_cg", base + 0xac80, 16, 3); + clks[IMX7D_CAN2_ROOT_PRE_DIV] = imx_clk_divider2("can2_pre_div", "can2_cg", base + 0xad00, 16, 3); + clks[IMX7D_I2C1_ROOT_PRE_DIV] = imx_clk_divider2("i2c1_pre_div", "i2c1_cg", base + 0xad80, 16, 3); + clks[IMX7D_I2C2_ROOT_PRE_DIV] = imx_clk_divider2("i2c2_pre_div", "i2c2_cg", base + 0xae00, 16, 3); + clks[IMX7D_I2C3_ROOT_PRE_DIV] = imx_clk_divider2("i2c3_pre_div", "i2c3_cg", base + 0xae80, 16, 3); + clks[IMX7D_I2C4_ROOT_PRE_DIV] = imx_clk_divider2("i2c4_pre_div", "i2c4_cg", base + 0xaf00, 16, 3); + clks[IMX7D_UART1_ROOT_PRE_DIV] = imx_clk_divider2("uart1_pre_div", "uart1_cg", base + 0xaf80, 16, 3); + clks[IMX7D_UART2_ROOT_PRE_DIV] = imx_clk_divider2("uart2_pre_div", "uart2_cg", base + 0xb000, 16, 3); + clks[IMX7D_UART3_ROOT_PRE_DIV] = imx_clk_divider2("uart3_pre_div", "uart3_cg", base + 0xb080, 16, 3); + clks[IMX7D_UART4_ROOT_PRE_DIV] = imx_clk_divider2("uart4_pre_div", "uart4_cg", base + 0xb100, 16, 3); + clks[IMX7D_UART5_ROOT_PRE_DIV] = imx_clk_divider2("uart5_pre_div", "uart5_cg", base + 0xb180, 16, 3); + clks[IMX7D_UART6_ROOT_PRE_DIV] = imx_clk_divider2("uart6_pre_div", "uart6_cg", base + 0xb200, 16, 3); + clks[IMX7D_UART7_ROOT_PRE_DIV] = imx_clk_divider2("uart7_pre_div", "uart7_cg", base + 0xb280, 16, 3); + clks[IMX7D_ECSPI1_ROOT_PRE_DIV] = imx_clk_divider2("ecspi1_pre_div", "ecspi1_cg", base + 0xb300, 16, 3); + clks[IMX7D_ECSPI2_ROOT_PRE_DIV] = imx_clk_divider2("ecspi2_pre_div", "ecspi2_cg", base + 0xb380, 16, 3); + clks[IMX7D_ECSPI3_ROOT_PRE_DIV] = imx_clk_divider2("ecspi3_pre_div", "ecspi3_cg", base + 0xb400, 16, 3); + clks[IMX7D_ECSPI4_ROOT_PRE_DIV] = imx_clk_divider2("ecspi4_pre_div", "ecspi4_cg", base + 0xb480, 16, 3); + clks[IMX7D_PWM1_ROOT_PRE_DIV] = imx_clk_divider2("pwm1_pre_div", "pwm1_cg", base + 0xb500, 16, 3); + clks[IMX7D_PWM2_ROOT_PRE_DIV] = imx_clk_divider2("pwm2_pre_div", "pwm2_cg", base + 0xb580, 16, 3); + clks[IMX7D_PWM3_ROOT_PRE_DIV] = imx_clk_divider2("pwm3_pre_div", "pwm3_cg", base + 0xb600, 16, 3); + clks[IMX7D_PWM4_ROOT_PRE_DIV] = imx_clk_divider2("pwm4_pre_div", "pwm4_cg", base + 0xb680, 16, 3); + clks[IMX7D_FLEXTIMER1_ROOT_PRE_DIV] = imx_clk_divider2("flextimer1_pre_div", "flextimer1_cg", base + 0xb700, 16, 3); + clks[IMX7D_FLEXTIMER2_ROOT_PRE_DIV] = imx_clk_divider2("flextimer2_pre_div", "flextimer2_cg", base + 0xb780, 16, 3); + clks[IMX7D_SIM1_ROOT_PRE_DIV] = imx_clk_divider2("sim1_pre_div", "sim1_cg", base + 0xb800, 16, 3); + clks[IMX7D_SIM2_ROOT_PRE_DIV] = imx_clk_divider2("sim2_pre_div", "sim2_cg", base + 0xb880, 16, 3); + clks[IMX7D_GPT1_ROOT_PRE_DIV] = imx_clk_divider2("gpt1_pre_div", "gpt1_cg", base + 0xb900, 16, 3); + clks[IMX7D_GPT2_ROOT_PRE_DIV] = imx_clk_divider2("gpt2_pre_div", "gpt2_cg", base + 0xb980, 16, 3); + clks[IMX7D_GPT3_ROOT_PRE_DIV] = imx_clk_divider2("gpt3_pre_div", "gpt3_cg", base + 0xba00, 16, 3); + clks[IMX7D_GPT4_ROOT_PRE_DIV] = imx_clk_divider2("gpt4_pre_div", "gpt4_cg", base + 0xba80, 16, 3); + clks[IMX7D_TRACE_ROOT_PRE_DIV] = imx_clk_divider2("trace_pre_div", "trace_cg", base + 0xbb00, 16, 3); + clks[IMX7D_WDOG_ROOT_PRE_DIV] = imx_clk_divider2("wdog_pre_div", "wdog_cg", base + 0xbb80, 16, 3); + clks[IMX7D_CSI_MCLK_ROOT_PRE_DIV] = imx_clk_divider2("csi_mclk_pre_div", "csi_mclk_cg", base + 0xbc00, 16, 3); + clks[IMX7D_AUDIO_MCLK_ROOT_PRE_DIV] = imx_clk_divider2("audio_mclk_pre_div", "audio_mclk_cg", base + 0xbc80, 16, 3); + clks[IMX7D_WRCLK_ROOT_PRE_DIV] = imx_clk_divider2("wrclk_pre_div", "wrclk_cg", base + 0xbd00, 16, 3); + clks[IMX7D_CLKO1_ROOT_PRE_DIV] = imx_clk_divider2("clko1_pre_div", "clko1_cg", base + 0xbd80, 16, 3); + clks[IMX7D_CLKO2_ROOT_PRE_DIV] = imx_clk_divider2("clko2_pre_div", "clko2_cg", base + 0xbe00, 16, 3); + + clks[IMX7D_ARM_A7_ROOT_DIV] = imx_clk_divider2("arm_a7_div", "arm_a7_cg", base + 0x8000, 0, 3); + clks[IMX7D_ARM_M4_ROOT_DIV] = imx_clk_divider2("arm_m4_div", "arm_m4_cg", base + 0x8080, 0, 3); + clks[IMX7D_ARM_M0_ROOT_DIV] = imx_clk_divider2("arm_m0_div", "arm_m0_cg", base + 0x8100, 0, 3); + clks[IMX7D_MAIN_AXI_ROOT_DIV] = imx_clk_divider2("axi_post_div", "axi_pre_div", base + 0x8800, 0, 6); + clks[IMX7D_DISP_AXI_ROOT_DIV] = imx_clk_divider2("disp_axi_post_div", "disp_axi_pre_div", base + 0x8880, 0, 6); + clks[IMX7D_ENET_AXI_ROOT_DIV] = imx_clk_divider2("enet_axi_post_div", "enet_axi_pre_div", base + 0x8900, 0, 6); + clks[IMX7D_NAND_USDHC_BUS_ROOT_DIV] = imx_clk_divider2("nand_usdhc_post_div", "nand_usdhc_pre_div", base + 0x8980, 0, 6); + clks[IMX7D_AHB_CHANNEL_ROOT_DIV] = imx_clk_divider2("ahb_post_div", "ahb_pre_div", base + 0x9000, 0, 6); + clks[IMX7D_DRAM_ROOT_DIV] = imx_clk_divider2("dram_post_div", "dram_cg", base + 0x9880, 0, 3); + clks[IMX7D_DRAM_PHYM_ALT_ROOT_DIV] = imx_clk_divider2("dram_phym_alt_post_div", "dram_phym_alt_pre_div", base + 0xa000, 0, 3); + clks[IMX7D_DRAM_ALT_ROOT_DIV] = imx_clk_divider2("dram_alt_post_div", "dram_alt_pre_div", base + 0xa080, 0, 3); + clks[IMX7D_USB_HSIC_ROOT_DIV] = imx_clk_divider2("usb_hsic_post_div", "usb_hsic_pre_div", base + 0xa100, 0, 6); + clks[IMX7D_PCIE_CTRL_ROOT_DIV] = imx_clk_divider2("pcie_ctrl_post_div", "pcie_ctrl_pre_div", base + 0xa180, 0, 6); + clks[IMX7D_PCIE_PHY_ROOT_DIV] = imx_clk_divider2("pcie_phy_post_div", "pcie_phy_pre_div", base + 0xa200, 0, 6); + clks[IMX7D_EPDC_PIXEL_ROOT_DIV] = imx_clk_divider2("epdc_pixel_post_div", "epdc_pixel_pre_div", base + 0xa280, 0, 6); + clks[IMX7D_LCDIF_PIXEL_ROOT_DIV] = imx_clk_divider2("lcdif_pixel_post_div", "lcdif_pixel_pre_div", base + 0xa300, 0, 6); + clks[IMX7D_MIPI_DSI_ROOT_DIV] = imx_clk_divider2("mipi_dsi_post_div", "mipi_dsi_pre_div", base + 0xa380, 0, 6); + clks[IMX7D_MIPI_CSI_ROOT_DIV] = imx_clk_divider2("mipi_csi_post_div", "mipi_csi_pre_div", base + 0xa400, 0, 6); + clks[IMX7D_MIPI_DPHY_ROOT_DIV] = imx_clk_divider2("mipi_dphy_post_div", "mipi_csi_dphy_div", base + 0xa480, 0, 6); + clks[IMX7D_SAI1_ROOT_DIV] = imx_clk_divider2("sai1_post_div", "sai1_pre_div", base + 0xa500, 0, 6); + clks[IMX7D_SAI2_ROOT_DIV] = imx_clk_divider2("sai2_post_div", "sai2_pre_div", base + 0xa580, 0, 6); + clks[IMX7D_SAI3_ROOT_DIV] = imx_clk_divider2("sai3_post_div", "sai3_pre_div", base + 0xa600, 0, 6); + clks[IMX7D_SPDIF_ROOT_DIV] = imx_clk_divider2("spdif_post_div", "spdif_pre_div", base + 0xa680, 0, 6); + clks[IMX7D_ENET1_REF_ROOT_DIV] = imx_clk_divider2("enet1_ref_post_div", "enet1_ref_pre_div", base + 0xa700, 0, 6); + clks[IMX7D_ENET1_TIME_ROOT_DIV] = imx_clk_divider2("enet1_time_post_div", "enet1_time_pre_div", base + 0xa780, 0, 6); + clks[IMX7D_ENET2_REF_ROOT_DIV] = imx_clk_divider2("enet2_ref_post_div", "enet2_ref_pre_div", base + 0xa800, 0, 6); + clks[IMX7D_ENET2_TIME_ROOT_DIV] = imx_clk_divider2("enet2_time_post_div", "enet2_time_pre_div", base + 0xa880, 0, 6); + clks[IMX7D_ENET_PHY_REF_ROOT_DIV] = imx_clk_divider2("enet_phy_ref_post_div", "enet_phy_ref_pre_div", base + 0xa900, 0, 6); + clks[IMX7D_EIM_ROOT_DIV] = imx_clk_divider2("eim_post_div", "eim_pre_div", base + 0xa980, 0, 6); + clks[IMX7D_NAND_ROOT_DIV] = imx_clk_divider2("nand_post_div", "nand_pre_div", base + 0xaa00, 0, 6); + clks[IMX7D_QSPI_ROOT_DIV] = imx_clk_divider2("qspi_post_div", "qspi_pre_div", base + 0xaa80, 0, 6); + clks[IMX7D_USDHC1_ROOT_DIV] = imx_clk_divider2("usdhc1_post_div", "usdhc1_pre_div", base + 0xab00, 0, 6); + clks[IMX7D_USDHC2_ROOT_DIV] = imx_clk_divider2("usdhc2_post_div", "usdhc2_pre_div", base + 0xab80, 0, 6); + clks[IMX7D_USDHC3_ROOT_DIV] = imx_clk_divider2("usdhc3_post_div", "usdhc3_pre_div", base + 0xac00, 0, 6); + clks[IMX7D_CAN1_ROOT_DIV] = imx_clk_divider2("can1_post_div", "can1_pre_div", base + 0xac80, 0, 6); + clks[IMX7D_CAN2_ROOT_DIV] = imx_clk_divider2("can2_post_div", "can2_pre_div", base + 0xad00, 0, 6); + clks[IMX7D_I2C1_ROOT_DIV] = imx_clk_divider2("i2c1_post_div", "i2c1_pre_div", base + 0xad80, 0, 6); + clks[IMX7D_I2C2_ROOT_DIV] = imx_clk_divider2("i2c2_post_div", "i2c2_pre_div", base + 0xae00, 0, 6); + clks[IMX7D_I2C3_ROOT_DIV] = imx_clk_divider2("i2c3_post_div", "i2c3_pre_div", base + 0xae80, 0, 6); + clks[IMX7D_I2C4_ROOT_DIV] = imx_clk_divider2("i2c4_post_div", "i2c4_pre_div", base + 0xaf00, 0, 6); + clks[IMX7D_UART1_ROOT_DIV] = imx_clk_divider2("uart1_post_div", "uart1_pre_div", base + 0xaf80, 0, 6); + clks[IMX7D_UART2_ROOT_DIV] = imx_clk_divider2("uart2_post_div", "uart2_pre_div", base + 0xb000, 0, 6); + clks[IMX7D_UART3_ROOT_DIV] = imx_clk_divider2("uart3_post_div", "uart3_pre_div", base + 0xb080, 0, 6); + clks[IMX7D_UART4_ROOT_DIV] = imx_clk_divider2("uart4_post_div", "uart4_pre_div", base + 0xb100, 0, 6); + clks[IMX7D_UART5_ROOT_DIV] = imx_clk_divider2("uart5_post_div", "uart5_pre_div", base + 0xb180, 0, 6); + clks[IMX7D_UART6_ROOT_DIV] = imx_clk_divider2("uart6_post_div", "uart6_pre_div", base + 0xb200, 0, 6); + clks[IMX7D_UART7_ROOT_DIV] = imx_clk_divider2("uart7_post_div", "uart7_pre_div", base + 0xb280, 0, 6); + clks[IMX7D_ECSPI1_ROOT_DIV] = imx_clk_divider2("ecspi1_post_div", "ecspi1_pre_div", base + 0xb300, 0, 6); + clks[IMX7D_ECSPI2_ROOT_DIV] = imx_clk_divider2("ecspi2_post_div", "ecspi2_pre_div", base + 0xb380, 0, 6); + clks[IMX7D_ECSPI3_ROOT_DIV] = imx_clk_divider2("ecspi3_post_div", "ecspi3_pre_div", base + 0xb400, 0, 6); + clks[IMX7D_ECSPI4_ROOT_DIV] = imx_clk_divider2("ecspi4_post_div", "ecspi4_pre_div", base + 0xb480, 0, 6); + clks[IMX7D_PWM1_ROOT_DIV] = imx_clk_divider2("pwm1_post_div", "pwm1_pre_div", base + 0xb500, 0, 6); + clks[IMX7D_PWM2_ROOT_DIV] = imx_clk_divider2("pwm2_post_div", "pwm2_pre_div", base + 0xb580, 0, 6); + clks[IMX7D_PWM3_ROOT_DIV] = imx_clk_divider2("pwm3_post_div", "pwm3_pre_div", base + 0xb600, 0, 6); + clks[IMX7D_PWM4_ROOT_DIV] = imx_clk_divider2("pwm4_post_div", "pwm4_pre_div", base + 0xb680, 0, 6); + clks[IMX7D_FLEXTIMER1_ROOT_DIV] = imx_clk_divider2("flextimer1_post_div", "flextimer1_pre_div", base + 0xb700, 0, 6); + clks[IMX7D_FLEXTIMER2_ROOT_DIV] = imx_clk_divider2("flextimer2_post_div", "flextimer2_pre_div", base + 0xb780, 0, 6); + clks[IMX7D_SIM1_ROOT_DIV] = imx_clk_divider2("sim1_post_div", "sim1_pre_div", base + 0xb800, 0, 6); + clks[IMX7D_SIM2_ROOT_DIV] = imx_clk_divider2("sim2_post_div", "sim2_pre_div", base + 0xb880, 0, 6); + clks[IMX7D_GPT1_ROOT_DIV] = imx_clk_divider2("gpt1_post_div", "gpt1_pre_div", base + 0xb900, 0, 6); + clks[IMX7D_GPT2_ROOT_DIV] = imx_clk_divider2("gpt2_post_div", "gpt2_pre_div", base + 0xb980, 0, 6); + clks[IMX7D_GPT3_ROOT_DIV] = imx_clk_divider2("gpt3_post_div", "gpt3_pre_div", base + 0xba00, 0, 6); + clks[IMX7D_GPT4_ROOT_DIV] = imx_clk_divider2("gpt4_post_div", "gpt4_pre_div", base + 0xba80, 0, 6); + clks[IMX7D_TRACE_ROOT_DIV] = imx_clk_divider2("trace_post_div", "trace_pre_div", base + 0xbb00, 0, 6); + clks[IMX7D_WDOG_ROOT_DIV] = imx_clk_divider2("wdog_post_div", "wdog_pre_div", base + 0xbb80, 0, 6); + clks[IMX7D_CSI_MCLK_ROOT_DIV] = imx_clk_divider2("csi_mclk_post_div", "csi_mclk_pre_div", base + 0xbc00, 0, 6); + clks[IMX7D_AUDIO_MCLK_ROOT_DIV] = imx_clk_divider2("audio_mclk_post_div", "audio_mclk_pre_div", base + 0xbc80, 0, 6); + clks[IMX7D_WRCLK_ROOT_DIV] = imx_clk_divider2("wrclk_post_div", "wrclk_pre_div", base + 0xbd00, 0, 6); + clks[IMX7D_CLKO1_ROOT_DIV] = imx_clk_divider2("clko1_post_div", "clko1_pre_div", base + 0xbd80, 0, 6); + clks[IMX7D_CLKO2_ROOT_DIV] = imx_clk_divider2("clko2_post_div", "clko2_pre_div", base + 0xbe00, 0, 6); + + clks[IMX7D_ARM_A7_ROOT_CLK] = imx_clk_gate4("arm_a7_root_clk", "arm_a7_div", base + 0x4000, 0); + clks[IMX7D_ARM_M4_ROOT_CLK] = imx_clk_gate4("arm_m4_root_clk", "arm_m4_div", base + 0x4010, 0); + clks[IMX7D_ARM_M0_ROOT_CLK] = imx_clk_gate4("arm_m0_root_clk", "arm_m0_div", base + 0x4020, 0); + clks[IMX7D_MAIN_AXI_ROOT_CLK] = imx_clk_gate4("main_axi_root_clk", "axi_post_div", base + 0x4040, 0); + clks[IMX7D_DISP_AXI_ROOT_CLK] = imx_clk_gate4("disp_axi_root_clk", "disp_axi_post_div", base + 0x4050, 0); + clks[IMX7D_ENET_AXI_ROOT_CLK] = imx_clk_gate4("enet_axi_root_clk", "enet_axi_post_div", base + 0x4060, 0); + clks[IMX7D_OCRAM_CLK] = imx_clk_gate4("ocram_clk", "axi_post_div", base + 0x4110, 0); + clks[IMX7D_OCRAM_S_CLK] = imx_clk_gate4("ocram_s_clk", "ahb_post_div", base + 0x4120, 0); + clks[IMX7D_NAND_USDHC_BUS_ROOT_CLK] = imx_clk_gate4("nand_usdhc_root_clk", "nand_usdhc_post_div", base + 0x4130, 0); + clks[IMX7D_AHB_CHANNEL_ROOT_CLK] = imx_clk_gate4("ahb_root_clk", "ahb_post_div", base + 0x4200, 0); + clks[IMX7D_DRAM_ROOT_CLK] = imx_clk_gate4("dram_root_clk", "dram_post_div", base + 0x4130, 0); + clks[IMX7D_DRAM_PHYM_ROOT_CLK] = imx_clk_gate4("dram_phym_root_clk", "dram_phym_cg", base + 0x4130, 0); + clks[IMX7D_DRAM_PHYM_ALT_ROOT_CLK] = imx_clk_gate4("dram_phym_alt_root_clk", "dram_phym_alt_post_div", base + 0x4130, 0); + clks[IMX7D_DRAM_ALT_ROOT_CLK] = imx_clk_gate4("dram_alt_root_clk", "dram_alt_post_div", base + 0x4130, 0); + clks[IMX7D_USB_HSIC_ROOT_CLK] = imx_clk_gate4("usb_hsic_root_clk", "usb_hsic_post_div", base + 0x4420, 0); + clks[IMX7D_PCIE_CTRL_ROOT_CLK] = imx_clk_gate4("pcie_ctrl_root_clk", "pcie_ctrl_post_div", base + 0x4600, 0); + clks[IMX7D_PCIE_PHY_ROOT_CLK] = imx_clk_gate4("pcie_phy_root_clk", "pcie_phy_post_div", base + 0x4600, 0); + clks[IMX7D_EPDC_PIXEL_ROOT_CLK] = imx_clk_gate4("epdc_pixel_root_clk", "epdc_pixel_post_div", base + 0x44a0, 0); + clks[IMX7D_LCDIF_PIXEL_ROOT_CLK] = imx_clk_gate4("lcdif_pixel_root_clk", "lcdif_pixel_post_div", base + 0x44b0, 0); + clks[IMX7D_MIPI_DSI_ROOT_CLK] = imx_clk_gate4("mipi_dsi_root_clk", "mipi_dsi_post_div", base + 0x4650, 0); + clks[IMX7D_MIPI_CSI_ROOT_CLK] = imx_clk_gate4("mipi_csi_root_clk", "mipi_csi_post_div", base + 0x4640, 0); + clks[IMX7D_MIPI_DPHY_ROOT_CLK] = imx_clk_gate4("mipi_dphy_root_clk", "mipi_dphy_post_div", base + 0x4660, 0); + clks[IMX7D_SAI1_ROOT_CLK] = imx_clk_gate4("sai1_root_clk", "sai1_post_div", base + 0x48c0, 0); + clks[IMX7D_SAI2_ROOT_CLK] = imx_clk_gate4("sai2_root_clk", "sai2_post_div", base + 0x48d0, 0); + clks[IMX7D_SAI3_ROOT_CLK] = imx_clk_gate4("sai3_root_clk", "sai3_post_div", base + 0x48e0, 0); + clks[IMX7D_SPDIF_ROOT_CLK] = imx_clk_gate4("spdif_root_clk", "spdif_post_div", base + 0x44d0, 0); + clks[IMX7D_ENET1_REF_ROOT_CLK] = imx_clk_gate4("enet1_ref_root_clk", "enet1_ref_post_div", base + 0x44e0, 0); + clks[IMX7D_ENET1_TIME_ROOT_CLK] = imx_clk_gate4("enet1_time_root_clk", "enet1_time_post_div", base + 0x44f0, 0); + clks[IMX7D_ENET2_REF_ROOT_CLK] = imx_clk_gate4("enet2_ref_root_clk", "enet2_ref_post_div", base + 0x4500, 0); + clks[IMX7D_ENET2_TIME_ROOT_CLK] = imx_clk_gate4("enet2_time_root_clk", "enet2_time_post_div", base + 0x4510, 0); + clks[IMX7D_ENET_PHY_REF_ROOT_CLK] = imx_clk_gate4("enet_phy_ref_root_clk", "enet_phy_ref_post_div", base + 0x4520, 0); + clks[IMX7D_EIM_ROOT_CLK] = imx_clk_gate4("eim_root_clk", "eim_post_div", base + 0x4160, 0); + clks[IMX7D_NAND_ROOT_CLK] = imx_clk_gate4("nand_root_clk", "nand_post_div", base + 0x4140, 0); + clks[IMX7D_QSPI_ROOT_CLK] = imx_clk_gate4("qspi_root_clk", "qspi_post_div", base + 0x4150, 0); + clks[IMX7D_USDHC1_ROOT_CLK] = imx_clk_gate4("usdhc1_root_clk", "usdhc1_post_div", base + 0x46c0, 0); + clks[IMX7D_USDHC2_ROOT_CLK] = imx_clk_gate4("usdhc2_root_clk", "usdhc2_post_div", base + 0x46d0, 0); + clks[IMX7D_USDHC3_ROOT_CLK] = imx_clk_gate4("usdhc3_root_clk", "usdhc3_post_div", base + 0x46e0, 0); + clks[IMX7D_CAN1_ROOT_CLK] = imx_clk_gate4("can1_root_clk", "can1_post_div", base + 0x4740, 0); + clks[IMX7D_CAN2_ROOT_CLK] = imx_clk_gate4("can2_root_clk", "can2_post_div", base + 0x4750, 0); + clks[IMX7D_I2C1_ROOT_CLK] = imx_clk_gate4("i2c1_root_clk", "i2c1_post_div", base + 0x4880, 0); + clks[IMX7D_I2C2_ROOT_CLK] = imx_clk_gate4("i2c2_root_clk", "i2c2_post_div", base + 0x4890, 0); + clks[IMX7D_I2C3_ROOT_CLK] = imx_clk_gate4("i2c3_root_clk", "i2c3_post_div", base + 0x48a0, 0); + clks[IMX7D_I2C4_ROOT_CLK] = imx_clk_gate4("i2c4_root_clk", "i2c4_post_div", base + 0x48b0, 0); + clks[IMX7D_UART1_ROOT_CLK] = imx_clk_gate4("uart1_root_clk", "uart1_post_div", base + 0x4940, 0); + clks[IMX7D_UART2_ROOT_CLK] = imx_clk_gate4("uart2_root_clk", "uart2_post_div", base + 0x4950, 0); + clks[IMX7D_UART3_ROOT_CLK] = imx_clk_gate4("uart3_root_clk", "uart3_post_div", base + 0x4960, 0); + clks[IMX7D_UART4_ROOT_CLK] = imx_clk_gate4("uart4_root_clk", "uart4_post_div", base + 0x4970, 0); + clks[IMX7D_UART5_ROOT_CLK] = imx_clk_gate4("uart5_root_clk", "uart5_post_div", base + 0x4980, 0); + clks[IMX7D_UART6_ROOT_CLK] = imx_clk_gate4("uart6_root_clk", "uart6_post_div", base + 0x4990, 0); + clks[IMX7D_UART7_ROOT_CLK] = imx_clk_gate4("uart7_root_clk", "uart7_post_div", base + 0x49a0, 0); + clks[IMX7D_ECSPI1_ROOT_CLK] = imx_clk_gate4("ecspi1_root_clk", "ecspi1_post_div", base + 0x4780, 0); + clks[IMX7D_ECSPI2_ROOT_CLK] = imx_clk_gate4("ecspi2_root_clk", "ecspi2_post_div", base + 0x4790, 0); + clks[IMX7D_ECSPI3_ROOT_CLK] = imx_clk_gate4("ecspi3_root_clk", "ecspi3_post_div", base + 0x47a0, 0); + clks[IMX7D_ECSPI4_ROOT_CLK] = imx_clk_gate4("ecspi4_root_clk", "ecspi4_post_div", base + 0x47b0, 0); + clks[IMX7D_PWM1_ROOT_CLK] = imx_clk_gate4("pwm1_root_clk", "pwm1_post_div", base + 0x4840, 0); + clks[IMX7D_PWM2_ROOT_CLK] = imx_clk_gate4("pwm2_root_clk", "pwm2_post_div", base + 0x4850, 0); + clks[IMX7D_PWM3_ROOT_CLK] = imx_clk_gate4("pwm3_root_clk", "pwm3_post_div", base + 0x4860, 0); + clks[IMX7D_PWM4_ROOT_CLK] = imx_clk_gate4("pwm4_root_clk", "pwm4_post_div", base + 0x4870, 0); + clks[IMX7D_FLEXTIMER1_ROOT_CLK] = imx_clk_gate4("flextimer1_root_clk", "flextimer1_post_div", base + 0x4800, 0); + clks[IMX7D_FLEXTIMER2_ROOT_CLK] = imx_clk_gate4("flextimer2_root_clk", "flextimer2_post_div", base + 0x4810, 0); + clks[IMX7D_SIM1_ROOT_CLK] = imx_clk_gate4("sim1_root_clk", "sim1_post_div", base + 0x4900, 0); + clks[IMX7D_SIM2_ROOT_CLK] = imx_clk_gate4("sim2_root_clk", "sim2_post_div", base + 0x4910, 0); + clks[IMX7D_GPT1_ROOT_CLK] = imx_clk_gate4("gpt1_root_clk", "gpt1_post_div", base + 0x47c0, 0); + clks[IMX7D_GPT2_ROOT_CLK] = imx_clk_gate4("gpt2_root_clk", "gpt2_post_div", base + 0x47d0, 0); + clks[IMX7D_GPT3_ROOT_CLK] = imx_clk_gate4("gpt3_root_clk", "gpt3_post_div", base + 0x47e0, 0); + clks[IMX7D_GPT4_ROOT_CLK] = imx_clk_gate4("gpt4_root_clk", "gpt4_post_div", base + 0x47f0, 0); + clks[IMX7D_TRACE_ROOT_CLK] = imx_clk_gate4("trace_root_clk", "trace_post_div", base + 0x4300, 0); + clks[IMX7D_WDOG1_ROOT_CLK] = imx_clk_gate4("wdog1_root_clk", "wdog_post_div", base + 0x49c0, 0); + clks[IMX7D_WDOG2_ROOT_CLK] = imx_clk_gate4("wdog2_root_clk", "wdog_post_div", base + 0x49d0, 0); + clks[IMX7D_WDOG3_ROOT_CLK] = imx_clk_gate4("wdog3_root_clk", "wdog_post_div", base + 0x49e0, 0); + clks[IMX7D_WDOG4_ROOT_CLK] = imx_clk_gate4("wdog4_root_clk", "wdog_post_div", base + 0x49f0, 0); + clks[IMX7D_CSI_MCLK_ROOT_CLK] = imx_clk_gate4("csi_mclk_root_clk", "csi_mclk_post_div", base + 0x4490, 0); + clks[IMX7D_AUDIO_MCLK_ROOT_CLK] = imx_clk_gate4("audio_mclk_root_clk", "audio_mclk_post_div", base + 0x4790, 0); + clks[IMX7D_WRCLK_ROOT_CLK] = imx_clk_gate4("wrclk_root_clk", "wrclk_post_div", base + 0x47a0, 0); + clks[IMX7D_ADC_ROOT_CLK] = imx_clk_gate4("adc_root_clk", "ipg_root_clk", base + 0x4200, 0); clks[IMX7D_GPT_3M_CLK] = imx_clk_fixed_factor("gpt_3m", "osc", 1, 8); @@ -846,28 +854,13 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node) clk_data.clk_num = ARRAY_SIZE(clks); of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); - /* TO BE FIXED LATER - * Enable all clock to bring up imx7, otherwise system will be halt and block - * the other part upstream Because imx7d clock design changed, clock framework - * need do a little modify. - * Dong Aisheng is working on this. After that, this part need be changed. - */ - for (i = 0; i < IMX7D_CLK_END; i++) - clk_prepare_enable(clks[i]); + for (i = 0; i < ARRAY_SIZE(clks_init_on); i++) + clk_prepare_enable(clks[clks_init_on[i]]); /* use old gpt clk setting, gpt1 root clk must be twice as gpt counter freq */ clk_set_parent(clks[IMX7D_GPT1_ROOT_SRC], clks[IMX7D_OSC_24M_CLK]); - /* - * init enet clock source: - * AXI clock source is 250MHz - * Phy refrence clock is 25MHz - * 1588 time clock source is 100MHz - */ clk_set_parent(clks[IMX7D_ENET_AXI_ROOT_SRC], clks[IMX7D_PLL_ENET_MAIN_250M_CLK]); - clk_set_parent(clks[IMX7D_ENET_PHY_REF_ROOT_SRC], clks[IMX7D_PLL_ENET_MAIN_25M_CLK]); - clk_set_parent(clks[IMX7D_ENET1_TIME_ROOT_SRC], clks[IMX7D_PLL_ENET_MAIN_100M_CLK]); - clk_set_parent(clks[IMX7D_ENET2_TIME_ROOT_SRC], clks[IMX7D_PLL_ENET_MAIN_100M_CLK]); /* set uart module clock's parent clock source that must be great then 80MHz */ clk_set_parent(clks[IMX7D_UART1_ROOT_SRC], clks[IMX7D_OSC_24M_CLK]); diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c index 4826b3c9e..19f9b6229 100644 --- a/drivers/clk/imx/clk-pllv3.c +++ b/drivers/clk/imx/clk-pllv3.c @@ -29,8 +29,8 @@ * struct clk_pllv3 - IMX PLL clock version 3 * @clk_hw: clock source * @base: base address of PLL registers - * @powerup_set: set POWER bit to power up the PLL - * @powerdown: pll powerdown offset bit + * @power_bit: pll power bit mask + * @powerup_set: set power_bit to power up the PLL * @div_mask: mask of divider bits * @div_shift: shift of divider bits * @@ -40,8 +40,8 @@ struct clk_pllv3 { struct clk_hw hw; void __iomem *base; + u32 power_bit; bool powerup_set; - u32 powerdown; u32 div_mask; u32 div_shift; unsigned long ref_clock; @@ -52,7 +52,7 @@ struct clk_pllv3 { static int clk_pllv3_wait_lock(struct clk_pllv3 *pll) { unsigned long timeout = jiffies + msecs_to_jiffies(10); - u32 val = readl_relaxed(pll->base) & pll->powerdown; + u32 val = readl_relaxed(pll->base) & pll->power_bit; /* No need to wait for lock when pll is not powered up */ if ((pll->powerup_set && !val) || (!pll->powerup_set && val)) @@ -77,9 +77,9 @@ static int clk_pllv3_prepare(struct clk_hw *hw) val = readl_relaxed(pll->base); if (pll->powerup_set) - val |= BM_PLL_POWER; + val |= pll->power_bit; else - val &= ~BM_PLL_POWER; + val &= ~pll->power_bit; writel_relaxed(val, pll->base); return clk_pllv3_wait_lock(pll); @@ -92,9 +92,9 @@ static void clk_pllv3_unprepare(struct clk_hw *hw) val = readl_relaxed(pll->base); if (pll->powerup_set) - val &= ~BM_PLL_POWER; + val &= ~pll->power_bit; else - val |= BM_PLL_POWER; + val |= pll->power_bit; writel_relaxed(val, pll->base); } @@ -218,8 +218,12 @@ static unsigned long clk_pllv3_av_recalc_rate(struct clk_hw *hw, u32 mfn = readl_relaxed(pll->base + PLL_NUM_OFFSET); u32 mfd = readl_relaxed(pll->base + PLL_DENOM_OFFSET); u32 div = readl_relaxed(pll->base) & pll->div_mask; + u64 temp64 = (u64)parent_rate; - return (parent_rate * div) + ((parent_rate / mfd) * mfn); + temp64 *= mfn; + do_div(temp64, mfd); + + return (parent_rate * div) + (u32)temp64; } static long clk_pllv3_av_round_rate(struct clk_hw *hw, unsigned long rate, @@ -243,7 +247,7 @@ static long clk_pllv3_av_round_rate(struct clk_hw *hw, unsigned long rate, do_div(temp64, parent_rate); mfn = temp64; - return parent_rate * div + parent_rate / mfd * mfn; + return parent_rate * div + parent_rate * mfn / mfd; } static int clk_pllv3_av_set_rate(struct clk_hw *hw, unsigned long rate, @@ -312,7 +316,7 @@ struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name, if (!pll) return ERR_PTR(-ENOMEM); - pll->powerdown = BM_PLL_POWER; + pll->power_bit = BM_PLL_POWER; switch (type) { case IMX_PLLV3_SYS: @@ -328,7 +332,7 @@ struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name, ops = &clk_pllv3_av_ops; break; case IMX_PLLV3_ENET_IMX7: - pll->powerdown = IMX7_ENET_PLL_POWER; + pll->power_bit = IMX7_ENET_PLL_POWER; pll->ref_clock = 1000000000; ops = &clk_pllv3_enet_ops; break; diff --git a/drivers/clk/imx/clk-vf610.c b/drivers/clk/imx/clk-vf610.c index 3a1f24475..0476353ab 100644 --- a/drivers/clk/imx/clk-vf610.c +++ b/drivers/clk/imx/clk-vf610.c @@ -315,12 +315,12 @@ static void __init vf610_clocks_init(struct device_node *ccm_node) clk[VF610_CLK_PIT] = imx_clk_gate2("pit", "ipg_bus", CCM_CCGR1, CCM_CCGRx_CGn(7)); - clk[VF610_CLK_UART0] = imx_clk_gate2("uart0", "ipg_bus", CCM_CCGR0, CCM_CCGRx_CGn(7)); - clk[VF610_CLK_UART1] = imx_clk_gate2("uart1", "ipg_bus", CCM_CCGR0, CCM_CCGRx_CGn(8)); - clk[VF610_CLK_UART2] = imx_clk_gate2("uart2", "ipg_bus", CCM_CCGR0, CCM_CCGRx_CGn(9)); - clk[VF610_CLK_UART3] = imx_clk_gate2("uart3", "ipg_bus", CCM_CCGR0, CCM_CCGRx_CGn(10)); - clk[VF610_CLK_UART4] = imx_clk_gate2("uart4", "ipg_bus", CCM_CCGR6, CCM_CCGRx_CGn(9)); - clk[VF610_CLK_UART5] = imx_clk_gate2("uart5", "ipg_bus", CCM_CCGR6, CCM_CCGRx_CGn(10)); + clk[VF610_CLK_UART0] = imx_clk_gate2_cgr("uart0", "ipg_bus", CCM_CCGR0, CCM_CCGRx_CGn(7), 0x2); + clk[VF610_CLK_UART1] = imx_clk_gate2_cgr("uart1", "ipg_bus", CCM_CCGR0, CCM_CCGRx_CGn(8), 0x2); + clk[VF610_CLK_UART2] = imx_clk_gate2_cgr("uart2", "ipg_bus", CCM_CCGR0, CCM_CCGRx_CGn(9), 0x2); + clk[VF610_CLK_UART3] = imx_clk_gate2_cgr("uart3", "ipg_bus", CCM_CCGR0, CCM_CCGRx_CGn(10), 0x2); + clk[VF610_CLK_UART4] = imx_clk_gate2_cgr("uart4", "ipg_bus", CCM_CCGR6, CCM_CCGRx_CGn(9), 0x2); + clk[VF610_CLK_UART5] = imx_clk_gate2_cgr("uart5", "ipg_bus", CCM_CCGR6, CCM_CCGRx_CGn(10), 0x2); clk[VF610_CLK_I2C0] = imx_clk_gate2("i2c0", "ipg_bus", CCM_CCGR4, CCM_CCGRx_CGn(6)); clk[VF610_CLK_I2C1] = imx_clk_gate2("i2c1", "ipg_bus", CCM_CCGR4, CCM_CCGRx_CGn(7)); diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h index 508d0fad8..a81c0385e 100644 --- a/drivers/clk/imx/clk.h +++ b/drivers/clk/imx/clk.h @@ -51,28 +51,6 @@ struct clk * imx_obtain_fixed_clock( struct clk *imx_clk_gate_exclusive(const char *name, const char *parent, void __iomem *reg, u8 shift, u32 exclusive_mask); -static inline struct clk *imx_clk_gate2(const char *name, const char *parent, - void __iomem *reg, u8 shift) -{ - return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg, - shift, 0x3, 0, &imx_ccm_lock, NULL); -} - -static inline struct clk *imx_clk_gate2_shared(const char *name, - const char *parent, void __iomem *reg, u8 shift, - unsigned int *share_count) -{ - return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg, - shift, 0x3, 0, &imx_ccm_lock, share_count); -} - -static inline struct clk *imx_clk_gate2_cgr(const char *name, const char *parent, - void __iomem *reg, u8 shift, u8 cgr_val) -{ - return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg, - shift, cgr_val, 0, &imx_ccm_lock, NULL); -} - struct clk *imx_clk_pfd(const char *name, const char *parent_name, void __iomem *reg, u8 idx); @@ -97,6 +75,13 @@ static inline struct clk *imx_clk_fixed(const char *name, int rate) return clk_register_fixed_rate(NULL, name, NULL, 0, rate); } +static inline struct clk *imx_clk_fixed_factor(const char *name, + const char *parent, unsigned int mult, unsigned int div) +{ + return clk_register_fixed_factor(NULL, name, parent, + CLK_SET_RATE_PARENT, mult, div); +} + static inline struct clk *imx_clk_divider(const char *name, const char *parent, void __iomem *reg, u8 shift, u8 width) { @@ -112,6 +97,14 @@ static inline struct clk *imx_clk_divider_flags(const char *name, reg, shift, width, 0, &imx_ccm_lock); } +static inline struct clk *imx_clk_divider2(const char *name, const char *parent, + void __iomem *reg, u8 shift, u8 width) +{ + return clk_register_divider(NULL, name, parent, + CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + reg, shift, width, 0, &imx_ccm_lock); +} + static inline struct clk *imx_clk_gate(const char *name, const char *parent, void __iomem *reg, u8 shift) { @@ -126,6 +119,44 @@ static inline struct clk *imx_clk_gate_dis(const char *name, const char *parent, shift, CLK_GATE_SET_TO_DISABLE, &imx_ccm_lock); } +static inline struct clk *imx_clk_gate2(const char *name, const char *parent, + void __iomem *reg, u8 shift) +{ + return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg, + shift, 0x3, 0, &imx_ccm_lock, NULL); +} + +static inline struct clk *imx_clk_gate2_shared(const char *name, + const char *parent, void __iomem *reg, u8 shift, + unsigned int *share_count) +{ + return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg, + shift, 0x3, 0, &imx_ccm_lock, share_count); +} + +static inline struct clk *imx_clk_gate2_cgr(const char *name, + const char *parent, void __iomem *reg, u8 shift, u8 cgr_val) +{ + return clk_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT, reg, + shift, cgr_val, 0, &imx_ccm_lock, NULL); +} + +static inline struct clk *imx_clk_gate3(const char *name, const char *parent, + void __iomem *reg, u8 shift) +{ + return clk_register_gate(NULL, name, parent, + CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + reg, shift, 0, &imx_ccm_lock); +} + +static inline struct clk *imx_clk_gate4(const char *name, const char *parent, + void __iomem *reg, u8 shift) +{ + return clk_register_gate2(NULL, name, parent, + CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + reg, shift, 0x3, 0, &imx_ccm_lock, NULL); +} + static inline struct clk *imx_clk_mux(const char *name, void __iomem *reg, u8 shift, u8 width, const char **parents, int num_parents) { @@ -134,6 +165,14 @@ static inline struct clk *imx_clk_mux(const char *name, void __iomem *reg, width, 0, &imx_ccm_lock); } +static inline struct clk *imx_clk_mux2(const char *name, void __iomem *reg, + u8 shift, u8 width, const char **parents, int num_parents) +{ + return clk_register_mux(NULL, name, parents, num_parents, + CLK_SET_RATE_NO_REPARENT | CLK_OPS_PARENT_ENABLE, + reg, shift, width, 0, &imx_ccm_lock); +} + static inline struct clk *imx_clk_mux_flags(const char *name, void __iomem *reg, u8 shift, u8 width, const char **parents, int num_parents, unsigned long flags) @@ -143,13 +182,6 @@ static inline struct clk *imx_clk_mux_flags(const char *name, &imx_ccm_lock); } -static inline struct clk *imx_clk_fixed_factor(const char *name, - const char *parent, unsigned int mult, unsigned int div) -{ - return clk_register_fixed_factor(NULL, name, parent, - CLK_SET_RATE_PARENT, mult, div); -} - struct clk *imx_clk_cpu(const char *name, const char *parent_name, struct clk *div, struct clk *mux, struct clk *pll, struct clk *step); diff --git a/drivers/clk/meson/Kconfig b/drivers/clk/meson/Kconfig new file mode 100644 index 000000000..19480bcc7 --- /dev/null +++ b/drivers/clk/meson/Kconfig @@ -0,0 +1,19 @@ +config COMMON_CLK_AMLOGIC + bool + depends on OF + depends on ARCH_MESON || COMPILE_TEST + +config COMMON_CLK_MESON8B + bool + depends on COMMON_CLK_AMLOGIC + help + Support for the clock controller on AmLogic S805 devices, aka + meson8b. Say Y if you want peripherals and CPU frequency scaling to + work. + +config COMMON_CLK_GXBB + bool + depends on COMMON_CLK_AMLOGIC + help + Support for the clock controller on AmLogic S905 devices, aka gxbb. + Say Y if you want peripherals and CPU frequency scaling to work. diff --git a/drivers/clk/meson/Makefile b/drivers/clk/meson/Makefile index 6d45531df..197e40175 100644 --- a/drivers/clk/meson/Makefile +++ b/drivers/clk/meson/Makefile @@ -2,5 +2,6 @@ # Makefile for Meson specific clk # -obj-y += clkc.o clk-pll.o clk-cpu.o -obj-y += meson8b-clkc.o +obj-$(CONFIG_COMMON_CLK_AMLOGIC) += clk-pll.o clk-cpu.o clk-mpll.o +obj-$(CONFIG_COMMON_CLK_MESON8B) += meson8b-clkc.o +obj-$(CONFIG_COMMON_CLK_GXBB) += gxbb.o diff --git a/drivers/clk/meson/clk-cpu.c b/drivers/clk/meson/clk-cpu.c index f7c30ea54..f8b2b7efd 100644 --- a/drivers/clk/meson/clk-cpu.c +++ b/drivers/clk/meson/clk-cpu.c @@ -51,13 +51,6 @@ #include "clkc.h" -struct meson_clk_cpu { - struct notifier_block clk_nb; - const struct clk_div_table *div_table; - struct clk_hw hw; - void __iomem *base; - u16 reg_off; -}; #define to_meson_clk_cpu_hw(_hw) container_of(_hw, struct meson_clk_cpu, hw) #define to_meson_clk_cpu_nb(_nb) container_of(_nb, struct meson_clk_cpu, clk_nb) @@ -119,6 +112,7 @@ static unsigned long meson_clk_cpu_recalc_rate(struct clk_hw *hw, return parent_rate / div; } +/* FIXME MUX1 & MUX2 should be struct clk_hw objects */ static int meson_clk_cpu_pre_rate_change(struct meson_clk_cpu *clk_cpu, struct clk_notifier_data *ndata) { @@ -140,6 +134,7 @@ static int meson_clk_cpu_pre_rate_change(struct meson_clk_cpu *clk_cpu, return 0; } +/* FIXME MUX1 & MUX2 should be struct clk_hw objects */ static int meson_clk_cpu_post_rate_change(struct meson_clk_cpu *clk_cpu, struct clk_notifier_data *ndata) { @@ -161,7 +156,7 @@ static int meson_clk_cpu_post_rate_change(struct meson_clk_cpu *clk_cpu, * PLL clock is to be changed. We use the xtal input as temporary parent * while the PLL frequency is stabilized. */ -static int meson_clk_cpu_notifier_cb(struct notifier_block *nb, +int meson_clk_cpu_notifier_cb(struct notifier_block *nb, unsigned long event, void *data) { struct clk_notifier_data *ndata = data; @@ -176,68 +171,8 @@ static int meson_clk_cpu_notifier_cb(struct notifier_block *nb, return notifier_from_errno(ret); } -static const struct clk_ops meson_clk_cpu_ops = { +const struct clk_ops meson_clk_cpu_ops = { .recalc_rate = meson_clk_cpu_recalc_rate, .round_rate = meson_clk_cpu_round_rate, .set_rate = meson_clk_cpu_set_rate, }; - -struct clk *meson_clk_register_cpu(const struct clk_conf *clk_conf, - void __iomem *reg_base, - spinlock_t *lock) -{ - struct clk *clk; - struct clk *pclk; - struct meson_clk_cpu *clk_cpu; - struct clk_init_data init; - int ret; - - clk_cpu = kzalloc(sizeof(*clk_cpu), GFP_KERNEL); - if (!clk_cpu) - return ERR_PTR(-ENOMEM); - - clk_cpu->base = reg_base; - clk_cpu->reg_off = clk_conf->reg_off; - clk_cpu->div_table = clk_conf->conf.div_table; - clk_cpu->clk_nb.notifier_call = meson_clk_cpu_notifier_cb; - - init.name = clk_conf->clk_name; - init.ops = &meson_clk_cpu_ops; - init.flags = clk_conf->flags | CLK_GET_RATE_NOCACHE; - init.flags |= CLK_SET_RATE_PARENT; - init.parent_names = clk_conf->clks_parent; - init.num_parents = 1; - - clk_cpu->hw.init = &init; - - pclk = __clk_lookup(clk_conf->clks_parent[0]); - if (!pclk) { - pr_err("%s: could not lookup parent clock %s\n", - __func__, clk_conf->clks_parent[0]); - ret = -EINVAL; - goto free_clk; - } - - ret = clk_notifier_register(pclk, &clk_cpu->clk_nb); - if (ret) { - pr_err("%s: failed to register clock notifier for %s\n", - __func__, clk_conf->clk_name); - goto free_clk; - } - - clk = clk_register(NULL, &clk_cpu->hw); - if (IS_ERR(clk)) { - ret = PTR_ERR(clk); - goto unregister_clk_nb; - } - - return clk; - -unregister_clk_nb: - clk_notifier_unregister(pclk, &clk_cpu->clk_nb); -free_clk: - kfree(clk_cpu); - - return ERR_PTR(ret); -} - diff --git a/drivers/clk/meson/clk-mpll.c b/drivers/clk/meson/clk-mpll.c new file mode 100644 index 000000000..03af79005 --- /dev/null +++ b/drivers/clk/meson/clk-mpll.c @@ -0,0 +1,94 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright (c) 2016 AmLogic, Inc. + * Author: Michael Turquette + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * The full GNU General Public License is included in this distribution + * in the file called COPYING + * + * BSD LICENSE + * + * Copyright (c) 2016 AmLogic, Inc. + * Author: Michael Turquette + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * MultiPhase Locked Loops are outputs from a PLL with additional frequency + * scaling capabilities. MPLL rates are calculated as: + * + * f(N2_integer, SDM_IN ) = 2.0G/(N2_integer + SDM_IN/16384) + */ + +#include +#include "clkc.h" + +#define SDM_MAX 16384 + +#define to_meson_clk_mpll(_hw) container_of(_hw, struct meson_clk_mpll, hw) + +static unsigned long mpll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct meson_clk_mpll *mpll = to_meson_clk_mpll(hw); + struct parm *p; + unsigned long rate = 0; + unsigned long reg, sdm, n2; + + p = &mpll->sdm; + reg = readl(mpll->base + p->reg_off); + sdm = PARM_GET(p->width, p->shift, reg); + + p = &mpll->n2; + reg = readl(mpll->base + p->reg_off); + n2 = PARM_GET(p->width, p->shift, reg); + + rate = (parent_rate * SDM_MAX) / ((SDM_MAX * n2) + sdm); + + return rate; +} + +const struct clk_ops meson_clk_mpll_ro_ops = { + .recalc_rate = mpll_recalc_rate, +}; diff --git a/drivers/clk/meson/clk-pll.c b/drivers/clk/meson/clk-pll.c index 664edf070..4adc1e892 100644 --- a/drivers/clk/meson/clk-pll.c +++ b/drivers/clk/meson/clk-pll.c @@ -44,13 +44,6 @@ #define MESON_PLL_RESET BIT(29) #define MESON_PLL_LOCK BIT(31) -struct meson_clk_pll { - struct clk_hw hw; - void __iomem *base; - struct pll_conf *conf; - unsigned int rate_count; - spinlock_t *lock; -}; #define to_meson_clk_pll(_hw) container_of(_hw, struct meson_clk_pll, hw) static unsigned long meson_clk_pll_recalc_rate(struct clk_hw *hw, @@ -60,22 +53,36 @@ static unsigned long meson_clk_pll_recalc_rate(struct clk_hw *hw, struct parm *p; unsigned long parent_rate_mhz = parent_rate / 1000000; unsigned long rate_mhz; - u16 n, m, od; + u16 n, m, frac = 0, od, od2 = 0; u32 reg; - p = &pll->conf->n; + p = &pll->n; reg = readl(pll->base + p->reg_off); n = PARM_GET(p->width, p->shift, reg); - p = &pll->conf->m; + p = &pll->m; reg = readl(pll->base + p->reg_off); m = PARM_GET(p->width, p->shift, reg); - p = &pll->conf->od; + p = &pll->od; reg = readl(pll->base + p->reg_off); od = PARM_GET(p->width, p->shift, reg); - rate_mhz = (parent_rate_mhz * m / n) >> od; + p = &pll->od2; + if (p->width) { + reg = readl(pll->base + p->reg_off); + od2 = PARM_GET(p->width, p->shift, reg); + } + + p = &pll->frac; + if (p->width) { + reg = readl(pll->base + p->reg_off); + frac = PARM_GET(p->width, p->shift, reg); + rate_mhz = (parent_rate_mhz * m + \ + (parent_rate_mhz * frac >> 12)) * 2 / n; + rate_mhz = rate_mhz >> od >> od2; + } else + rate_mhz = (parent_rate_mhz * m / n) >> od >> od2; return rate_mhz * 1000000; } @@ -84,7 +91,7 @@ static long meson_clk_pll_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *parent_rate) { struct meson_clk_pll *pll = to_meson_clk_pll(hw); - const struct pll_rate_table *rate_table = pll->conf->rate_table; + const struct pll_rate_table *rate_table = pll->rate_table; int i; for (i = 0; i < pll->rate_count; i++) { @@ -99,7 +106,7 @@ static long meson_clk_pll_round_rate(struct clk_hw *hw, unsigned long rate, static const struct pll_rate_table *meson_clk_get_pll_settings(struct meson_clk_pll *pll, unsigned long rate) { - const struct pll_rate_table *rate_table = pll->conf->rate_table; + const struct pll_rate_table *rate_table = pll->rate_table; int i; for (i = 0; i < pll->rate_count; i++) { @@ -145,24 +152,38 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, return -EINVAL; /* PLL reset */ - p = &pll->conf->n; + p = &pll->n; reg = readl(pll->base + p->reg_off); writel(reg | MESON_PLL_RESET, pll->base + p->reg_off); reg = PARM_SET(p->width, p->shift, reg, rate_set->n); writel(reg, pll->base + p->reg_off); - p = &pll->conf->m; + p = &pll->m; reg = readl(pll->base + p->reg_off); reg = PARM_SET(p->width, p->shift, reg, rate_set->m); writel(reg, pll->base + p->reg_off); - p = &pll->conf->od; + p = &pll->od; reg = readl(pll->base + p->reg_off); reg = PARM_SET(p->width, p->shift, reg, rate_set->od); writel(reg, pll->base + p->reg_off); - p = &pll->conf->n; + p = &pll->od2; + if (p->width) { + reg = readl(pll->base + p->reg_off); + reg = PARM_SET(p->width, p->shift, reg, rate_set->od2); + writel(reg, pll->base + p->reg_off); + } + + p = &pll->frac; + if (p->width) { + reg = readl(pll->base + p->reg_off); + reg = PARM_SET(p->width, p->shift, reg, rate_set->frac); + writel(reg, pll->base + p->reg_off); + } + + p = &pll->n; ret = meson_clk_pll_wait_lock(pll, p); if (ret) { pr_warn("%s: pll did not lock, trying to restore old rate %lu\n", @@ -173,55 +194,12 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, return ret; } -static const struct clk_ops meson_clk_pll_ops = { +const struct clk_ops meson_clk_pll_ops = { .recalc_rate = meson_clk_pll_recalc_rate, .round_rate = meson_clk_pll_round_rate, .set_rate = meson_clk_pll_set_rate, }; -static const struct clk_ops meson_clk_pll_ro_ops = { +const struct clk_ops meson_clk_pll_ro_ops = { .recalc_rate = meson_clk_pll_recalc_rate, }; - -struct clk *meson_clk_register_pll(const struct clk_conf *clk_conf, - void __iomem *reg_base, - spinlock_t *lock) -{ - struct clk *clk; - struct meson_clk_pll *clk_pll; - struct clk_init_data init; - - clk_pll = kzalloc(sizeof(*clk_pll), GFP_KERNEL); - if (!clk_pll) - return ERR_PTR(-ENOMEM); - - clk_pll->base = reg_base + clk_conf->reg_off; - clk_pll->lock = lock; - clk_pll->conf = clk_conf->conf.pll; - - init.name = clk_conf->clk_name; - init.flags = clk_conf->flags | CLK_GET_RATE_NOCACHE; - - init.parent_names = &clk_conf->clks_parent[0]; - init.num_parents = 1; - init.ops = &meson_clk_pll_ro_ops; - - /* If no rate_table is specified we assume the PLL is read-only */ - if (clk_pll->conf->rate_table) { - int len; - - for (len = 0; clk_pll->conf->rate_table[len].rate != 0; ) - len++; - - clk_pll->rate_count = len; - init.ops = &meson_clk_pll_ops; - } - - clk_pll->hw.init = &init; - - clk = clk_register(NULL, &clk_pll->hw); - if (IS_ERR(clk)) - kfree(clk_pll); - - return clk; -} diff --git a/drivers/clk/meson/clkc.h b/drivers/clk/meson/clkc.h index 609ae92cc..53326c32e 100644 --- a/drivers/clk/meson/clkc.h +++ b/drivers/clk/meson/clkc.h @@ -34,19 +34,16 @@ struct parm { u8 shift; u8 width; }; -#define PARM(_r, _s, _w) \ - { \ - .reg_off = (_r), \ - .shift = (_s), \ - .width = (_w), \ - } \ struct pll_rate_table { unsigned long rate; u16 m; u16 n; u16 od; + u16 od2; + u16 frac; }; + #define PLL_RATE(_r, _m, _n, _od) \ { \ .rate = (_r), \ @@ -55,133 +52,69 @@ struct pll_rate_table { .od = (_od), \ } \ -struct pll_conf { - const struct pll_rate_table *rate_table; - struct parm m; - struct parm n; - struct parm od; -}; +#define PLL_FRAC_RATE(_r, _m, _n, _od, _od2, _frac) \ + { \ + .rate = (_r), \ + .m = (_m), \ + .n = (_n), \ + .od = (_od), \ + .od2 = (_od2), \ + .frac = (_frac), \ + } \ -struct fixed_fact_conf { - unsigned int div; - unsigned int mult; - struct parm div_parm; - struct parm mult_parm; +struct meson_clk_pll { + struct clk_hw hw; + void __iomem *base; + struct parm m; + struct parm n; + struct parm frac; + struct parm od; + struct parm od2; + const struct pll_rate_table *rate_table; + unsigned int rate_count; + spinlock_t *lock; }; -struct fixed_rate_conf { - unsigned long rate; - struct parm rate_parm; -}; +#define to_meson_clk_pll(_hw) container_of(_hw, struct meson_clk_pll, hw) -struct composite_conf { - struct parm mux_parm; - struct parm div_parm; - struct parm gate_parm; - struct clk_div_table *div_table; - u32 *mux_table; - u8 mux_flags; - u8 div_flags; - u8 gate_flags; +struct meson_clk_cpu { + struct clk_hw hw; + void __iomem *base; + u16 reg_off; + struct notifier_block clk_nb; + const struct clk_div_table *div_table; }; -#define PNAME(x) static const char *x[] +int meson_clk_cpu_notifier_cb(struct notifier_block *nb, unsigned long event, + void *data); -enum clk_type { - CLK_FIXED_FACTOR, - CLK_FIXED_RATE, - CLK_COMPOSITE, - CLK_CPU, - CLK_PLL, +struct meson_clk_mpll { + struct clk_hw hw; + void __iomem *base; + struct parm sdm; + struct parm n2; + /* FIXME ssen gate control? */ + spinlock_t *lock; }; -struct clk_conf { - u16 reg_off; - enum clk_type clk_type; - unsigned int clk_id; - const char *clk_name; - const char **clks_parent; - int num_parents; - unsigned long flags; - union { - struct fixed_fact_conf fixed_fact; - struct fixed_rate_conf fixed_rate; - const struct composite_conf *composite; - struct pll_conf *pll; - const struct clk_div_table *div_table; - } conf; +#define MESON_GATE(_name, _reg, _bit) \ +struct clk_gate gxbb_##_name = { \ + .reg = (void __iomem *) _reg, \ + .bit_idx = (_bit), \ + .lock = &clk_lock, \ + .hw.init = &(struct clk_init_data) { \ + .name = #_name, \ + .ops = &clk_gate_ops, \ + .parent_names = (const char *[]){ "clk81" }, \ + .num_parents = 1, \ + .flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED), \ + }, \ }; -#define FIXED_RATE_P(_ro, _ci, _cn, _f, _c) \ - { \ - .reg_off = (_ro), \ - .clk_type = CLK_FIXED_RATE, \ - .clk_id = (_ci), \ - .clk_name = (_cn), \ - .flags = (_f), \ - .conf.fixed_rate.rate_parm = _c, \ - } \ - -#define FIXED_RATE(_ci, _cn, _f, _r) \ - { \ - .clk_type = CLK_FIXED_RATE, \ - .clk_id = (_ci), \ - .clk_name = (_cn), \ - .flags = (_f), \ - .conf.fixed_rate.rate = (_r), \ - } \ - -#define PLL(_ro, _ci, _cn, _cp, _f, _c) \ - { \ - .reg_off = (_ro), \ - .clk_type = CLK_PLL, \ - .clk_id = (_ci), \ - .clk_name = (_cn), \ - .clks_parent = (_cp), \ - .num_parents = ARRAY_SIZE(_cp), \ - .flags = (_f), \ - .conf.pll = (_c), \ - } \ - -#define FIXED_FACTOR_DIV(_ci, _cn, _cp, _f, _d) \ - { \ - .clk_type = CLK_FIXED_FACTOR, \ - .clk_id = (_ci), \ - .clk_name = (_cn), \ - .clks_parent = (_cp), \ - .num_parents = ARRAY_SIZE(_cp), \ - .conf.fixed_fact.div = (_d), \ - } \ - -#define CPU(_ro, _ci, _cn, _cp, _dt) \ - { \ - .reg_off = (_ro), \ - .clk_type = CLK_CPU, \ - .clk_id = (_ci), \ - .clk_name = (_cn), \ - .clks_parent = (_cp), \ - .num_parents = ARRAY_SIZE(_cp), \ - .conf.div_table = (_dt), \ - } \ - -#define COMPOSITE(_ro, _ci, _cn, _cp, _f, _c) \ - { \ - .reg_off = (_ro), \ - .clk_type = CLK_COMPOSITE, \ - .clk_id = (_ci), \ - .clk_name = (_cn), \ - .clks_parent = (_cp), \ - .num_parents = ARRAY_SIZE(_cp), \ - .flags = (_f), \ - .conf.composite = (_c), \ - } \ - -struct clk **meson_clk_init(struct device_node *np, unsigned long nr_clks); -void meson_clk_register_clks(const struct clk_conf *clk_confs, - unsigned int nr_confs, void __iomem *clk_base); -struct clk *meson_clk_register_cpu(const struct clk_conf *clk_conf, - void __iomem *reg_base, spinlock_t *lock); -struct clk *meson_clk_register_pll(const struct clk_conf *clk_conf, - void __iomem *reg_base, spinlock_t *lock); +/* clk_ops */ +extern const struct clk_ops meson_clk_pll_ro_ops; +extern const struct clk_ops meson_clk_pll_ops; +extern const struct clk_ops meson_clk_cpu_ops; +extern const struct clk_ops meson_clk_mpll_ro_ops; #endif /* __CLKC_H */ diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c new file mode 100644 index 000000000..a4c6684b3 --- /dev/null +++ b/drivers/clk/meson/gxbb.c @@ -0,0 +1,944 @@ +/* + * AmLogic S905 / GXBB Clock Controller Driver + * + * Copyright (c) 2016 AmLogic, Inc. + * Michael Turquette + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +#include +#include +#include +#include +#include + +#include "clkc.h" +#include "gxbb.h" + +static DEFINE_SPINLOCK(clk_lock); + +static const struct pll_rate_table sys_pll_rate_table[] = { + PLL_RATE(24000000, 56, 1, 2), + PLL_RATE(48000000, 64, 1, 2), + PLL_RATE(72000000, 72, 1, 2), + PLL_RATE(96000000, 64, 1, 2), + PLL_RATE(120000000, 80, 1, 2), + PLL_RATE(144000000, 96, 1, 2), + PLL_RATE(168000000, 56, 1, 1), + PLL_RATE(192000000, 64, 1, 1), + PLL_RATE(216000000, 72, 1, 1), + PLL_RATE(240000000, 80, 1, 1), + PLL_RATE(264000000, 88, 1, 1), + PLL_RATE(288000000, 96, 1, 1), + PLL_RATE(312000000, 52, 1, 2), + PLL_RATE(336000000, 56, 1, 2), + PLL_RATE(360000000, 60, 1, 2), + PLL_RATE(384000000, 64, 1, 2), + PLL_RATE(408000000, 68, 1, 2), + PLL_RATE(432000000, 72, 1, 2), + PLL_RATE(456000000, 76, 1, 2), + PLL_RATE(480000000, 80, 1, 2), + PLL_RATE(504000000, 84, 1, 2), + PLL_RATE(528000000, 88, 1, 2), + PLL_RATE(552000000, 92, 1, 2), + PLL_RATE(576000000, 96, 1, 2), + PLL_RATE(600000000, 50, 1, 1), + PLL_RATE(624000000, 52, 1, 1), + PLL_RATE(648000000, 54, 1, 1), + PLL_RATE(672000000, 56, 1, 1), + PLL_RATE(696000000, 58, 1, 1), + PLL_RATE(720000000, 60, 1, 1), + PLL_RATE(744000000, 62, 1, 1), + PLL_RATE(768000000, 64, 1, 1), + PLL_RATE(792000000, 66, 1, 1), + PLL_RATE(816000000, 68, 1, 1), + PLL_RATE(840000000, 70, 1, 1), + PLL_RATE(864000000, 72, 1, 1), + PLL_RATE(888000000, 74, 1, 1), + PLL_RATE(912000000, 76, 1, 1), + PLL_RATE(936000000, 78, 1, 1), + PLL_RATE(960000000, 80, 1, 1), + PLL_RATE(984000000, 82, 1, 1), + PLL_RATE(1008000000, 84, 1, 1), + PLL_RATE(1032000000, 86, 1, 1), + PLL_RATE(1056000000, 88, 1, 1), + PLL_RATE(1080000000, 90, 1, 1), + PLL_RATE(1104000000, 92, 1, 1), + PLL_RATE(1128000000, 94, 1, 1), + PLL_RATE(1152000000, 96, 1, 1), + PLL_RATE(1176000000, 98, 1, 1), + PLL_RATE(1200000000, 50, 1, 0), + PLL_RATE(1224000000, 51, 1, 0), + PLL_RATE(1248000000, 52, 1, 0), + PLL_RATE(1272000000, 53, 1, 0), + PLL_RATE(1296000000, 54, 1, 0), + PLL_RATE(1320000000, 55, 1, 0), + PLL_RATE(1344000000, 56, 1, 0), + PLL_RATE(1368000000, 57, 1, 0), + PLL_RATE(1392000000, 58, 1, 0), + PLL_RATE(1416000000, 59, 1, 0), + PLL_RATE(1440000000, 60, 1, 0), + PLL_RATE(1464000000, 61, 1, 0), + PLL_RATE(1488000000, 62, 1, 0), + PLL_RATE(1512000000, 63, 1, 0), + PLL_RATE(1536000000, 64, 1, 0), + PLL_RATE(1560000000, 65, 1, 0), + PLL_RATE(1584000000, 66, 1, 0), + PLL_RATE(1608000000, 67, 1, 0), + PLL_RATE(1632000000, 68, 1, 0), + PLL_RATE(1656000000, 68, 1, 0), + PLL_RATE(1680000000, 68, 1, 0), + PLL_RATE(1704000000, 68, 1, 0), + PLL_RATE(1728000000, 69, 1, 0), + PLL_RATE(1752000000, 69, 1, 0), + PLL_RATE(1776000000, 69, 1, 0), + PLL_RATE(1800000000, 69, 1, 0), + PLL_RATE(1824000000, 70, 1, 0), + PLL_RATE(1848000000, 70, 1, 0), + PLL_RATE(1872000000, 70, 1, 0), + PLL_RATE(1896000000, 70, 1, 0), + PLL_RATE(1920000000, 71, 1, 0), + PLL_RATE(1944000000, 71, 1, 0), + PLL_RATE(1968000000, 71, 1, 0), + PLL_RATE(1992000000, 71, 1, 0), + PLL_RATE(2016000000, 72, 1, 0), + PLL_RATE(2040000000, 72, 1, 0), + PLL_RATE(2064000000, 72, 1, 0), + PLL_RATE(2088000000, 72, 1, 0), + PLL_RATE(2112000000, 73, 1, 0), + { /* sentinel */ }, +}; + +static const struct pll_rate_table gp0_pll_rate_table[] = { + PLL_RATE(96000000, 32, 1, 3), + PLL_RATE(99000000, 33, 1, 3), + PLL_RATE(102000000, 34, 1, 3), + PLL_RATE(105000000, 35, 1, 3), + PLL_RATE(108000000, 36, 1, 3), + PLL_RATE(111000000, 37, 1, 3), + PLL_RATE(114000000, 38, 1, 3), + PLL_RATE(117000000, 39, 1, 3), + PLL_RATE(120000000, 40, 1, 3), + PLL_RATE(123000000, 41, 1, 3), + PLL_RATE(126000000, 42, 1, 3), + PLL_RATE(129000000, 43, 1, 3), + PLL_RATE(132000000, 44, 1, 3), + PLL_RATE(135000000, 45, 1, 3), + PLL_RATE(138000000, 46, 1, 3), + PLL_RATE(141000000, 47, 1, 3), + PLL_RATE(144000000, 48, 1, 3), + PLL_RATE(147000000, 49, 1, 3), + PLL_RATE(150000000, 50, 1, 3), + PLL_RATE(153000000, 51, 1, 3), + PLL_RATE(156000000, 52, 1, 3), + PLL_RATE(159000000, 53, 1, 3), + PLL_RATE(162000000, 54, 1, 3), + PLL_RATE(165000000, 55, 1, 3), + PLL_RATE(168000000, 56, 1, 3), + PLL_RATE(171000000, 57, 1, 3), + PLL_RATE(174000000, 58, 1, 3), + PLL_RATE(177000000, 59, 1, 3), + PLL_RATE(180000000, 60, 1, 3), + PLL_RATE(183000000, 61, 1, 3), + PLL_RATE(186000000, 62, 1, 3), + PLL_RATE(192000000, 32, 1, 2), + PLL_RATE(198000000, 33, 1, 2), + PLL_RATE(204000000, 34, 1, 2), + PLL_RATE(210000000, 35, 1, 2), + PLL_RATE(216000000, 36, 1, 2), + PLL_RATE(222000000, 37, 1, 2), + PLL_RATE(228000000, 38, 1, 2), + PLL_RATE(234000000, 39, 1, 2), + PLL_RATE(240000000, 40, 1, 2), + PLL_RATE(246000000, 41, 1, 2), + PLL_RATE(252000000, 42, 1, 2), + PLL_RATE(258000000, 43, 1, 2), + PLL_RATE(264000000, 44, 1, 2), + PLL_RATE(270000000, 45, 1, 2), + PLL_RATE(276000000, 46, 1, 2), + PLL_RATE(282000000, 47, 1, 2), + PLL_RATE(288000000, 48, 1, 2), + PLL_RATE(294000000, 49, 1, 2), + PLL_RATE(300000000, 50, 1, 2), + PLL_RATE(306000000, 51, 1, 2), + PLL_RATE(312000000, 52, 1, 2), + PLL_RATE(318000000, 53, 1, 2), + PLL_RATE(324000000, 54, 1, 2), + PLL_RATE(330000000, 55, 1, 2), + PLL_RATE(336000000, 56, 1, 2), + PLL_RATE(342000000, 57, 1, 2), + PLL_RATE(348000000, 58, 1, 2), + PLL_RATE(354000000, 59, 1, 2), + PLL_RATE(360000000, 60, 1, 2), + PLL_RATE(366000000, 61, 1, 2), + PLL_RATE(372000000, 62, 1, 2), + PLL_RATE(384000000, 32, 1, 1), + PLL_RATE(396000000, 33, 1, 1), + PLL_RATE(408000000, 34, 1, 1), + PLL_RATE(420000000, 35, 1, 1), + PLL_RATE(432000000, 36, 1, 1), + PLL_RATE(444000000, 37, 1, 1), + PLL_RATE(456000000, 38, 1, 1), + PLL_RATE(468000000, 39, 1, 1), + PLL_RATE(480000000, 40, 1, 1), + PLL_RATE(492000000, 41, 1, 1), + PLL_RATE(504000000, 42, 1, 1), + PLL_RATE(516000000, 43, 1, 1), + PLL_RATE(528000000, 44, 1, 1), + PLL_RATE(540000000, 45, 1, 1), + PLL_RATE(552000000, 46, 1, 1), + PLL_RATE(564000000, 47, 1, 1), + PLL_RATE(576000000, 48, 1, 1), + PLL_RATE(588000000, 49, 1, 1), + PLL_RATE(600000000, 50, 1, 1), + PLL_RATE(612000000, 51, 1, 1), + PLL_RATE(624000000, 52, 1, 1), + PLL_RATE(636000000, 53, 1, 1), + PLL_RATE(648000000, 54, 1, 1), + PLL_RATE(660000000, 55, 1, 1), + PLL_RATE(672000000, 56, 1, 1), + PLL_RATE(684000000, 57, 1, 1), + PLL_RATE(696000000, 58, 1, 1), + PLL_RATE(708000000, 59, 1, 1), + PLL_RATE(720000000, 60, 1, 1), + PLL_RATE(732000000, 61, 1, 1), + PLL_RATE(744000000, 62, 1, 1), + PLL_RATE(768000000, 32, 1, 0), + PLL_RATE(792000000, 33, 1, 0), + PLL_RATE(816000000, 34, 1, 0), + PLL_RATE(840000000, 35, 1, 0), + PLL_RATE(864000000, 36, 1, 0), + PLL_RATE(888000000, 37, 1, 0), + PLL_RATE(912000000, 38, 1, 0), + PLL_RATE(936000000, 39, 1, 0), + PLL_RATE(960000000, 40, 1, 0), + PLL_RATE(984000000, 41, 1, 0), + PLL_RATE(1008000000, 42, 1, 0), + PLL_RATE(1032000000, 43, 1, 0), + PLL_RATE(1056000000, 44, 1, 0), + PLL_RATE(1080000000, 45, 1, 0), + PLL_RATE(1104000000, 46, 1, 0), + PLL_RATE(1128000000, 47, 1, 0), + PLL_RATE(1152000000, 48, 1, 0), + PLL_RATE(1176000000, 49, 1, 0), + PLL_RATE(1200000000, 50, 1, 0), + PLL_RATE(1224000000, 51, 1, 0), + PLL_RATE(1248000000, 52, 1, 0), + PLL_RATE(1272000000, 53, 1, 0), + PLL_RATE(1296000000, 54, 1, 0), + PLL_RATE(1320000000, 55, 1, 0), + PLL_RATE(1344000000, 56, 1, 0), + PLL_RATE(1368000000, 57, 1, 0), + PLL_RATE(1392000000, 58, 1, 0), + PLL_RATE(1416000000, 59, 1, 0), + PLL_RATE(1440000000, 60, 1, 0), + PLL_RATE(1464000000, 61, 1, 0), + PLL_RATE(1488000000, 62, 1, 0), + { /* sentinel */ }, +}; + +static const struct clk_div_table cpu_div_table[] = { + { .val = 1, .div = 1 }, + { .val = 2, .div = 2 }, + { .val = 3, .div = 3 }, + { .val = 2, .div = 4 }, + { .val = 3, .div = 6 }, + { .val = 4, .div = 8 }, + { .val = 5, .div = 10 }, + { .val = 6, .div = 12 }, + { .val = 7, .div = 14 }, + { .val = 8, .div = 16 }, + { /* sentinel */ }, +}; + +static struct meson_clk_pll gxbb_fixed_pll = { + .m = { + .reg_off = HHI_MPLL_CNTL, + .shift = 0, + .width = 9, + }, + .n = { + .reg_off = HHI_MPLL_CNTL, + .shift = 9, + .width = 5, + }, + .od = { + .reg_off = HHI_MPLL_CNTL, + .shift = 16, + .width = 2, + }, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "fixed_pll", + .ops = &meson_clk_pll_ro_ops, + .parent_names = (const char *[]){ "xtal" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct meson_clk_pll gxbb_hdmi_pll = { + .m = { + .reg_off = HHI_HDMI_PLL_CNTL, + .shift = 0, + .width = 9, + }, + .n = { + .reg_off = HHI_HDMI_PLL_CNTL, + .shift = 9, + .width = 5, + }, + .frac = { + .reg_off = HHI_HDMI_PLL_CNTL2, + .shift = 0, + .width = 12, + }, + .od = { + .reg_off = HHI_HDMI_PLL_CNTL2, + .shift = 16, + .width = 2, + }, + .od2 = { + .reg_off = HHI_HDMI_PLL_CNTL2, + .shift = 22, + .width = 2, + }, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "hdmi_pll", + .ops = &meson_clk_pll_ro_ops, + .parent_names = (const char *[]){ "xtal" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct meson_clk_pll gxbb_sys_pll = { + .m = { + .reg_off = HHI_SYS_PLL_CNTL, + .shift = 0, + .width = 9, + }, + .n = { + .reg_off = HHI_SYS_PLL_CNTL, + .shift = 9, + .width = 5, + }, + .od = { + .reg_off = HHI_SYS_PLL_CNTL, + .shift = 10, + .width = 2, + }, + .rate_table = sys_pll_rate_table, + .rate_count = ARRAY_SIZE(sys_pll_rate_table), + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "sys_pll", + .ops = &meson_clk_pll_ro_ops, + .parent_names = (const char *[]){ "xtal" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct meson_clk_pll gxbb_gp0_pll = { + .m = { + .reg_off = HHI_GP0_PLL_CNTL, + .shift = 0, + .width = 9, + }, + .n = { + .reg_off = HHI_GP0_PLL_CNTL, + .shift = 9, + .width = 5, + }, + .od = { + .reg_off = HHI_GP0_PLL_CNTL, + .shift = 16, + .width = 2, + }, + .rate_table = gp0_pll_rate_table, + .rate_count = ARRAY_SIZE(gp0_pll_rate_table), + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "gp0_pll", + .ops = &meson_clk_pll_ops, + .parent_names = (const char *[]){ "xtal" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_fixed_factor gxbb_fclk_div2 = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div2", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor gxbb_fclk_div3 = { + .mult = 1, + .div = 3, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div3", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor gxbb_fclk_div4 = { + .mult = 1, + .div = 4, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div4", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor gxbb_fclk_div5 = { + .mult = 1, + .div = 5, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div5", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor gxbb_fclk_div7 = { + .mult = 1, + .div = 7, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div7", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct meson_clk_mpll gxbb_mpll0 = { + .sdm = { + .reg_off = HHI_MPLL_CNTL7, + .shift = 0, + .width = 14, + }, + .n2 = { + .reg_off = HHI_MPLL_CNTL7, + .shift = 16, + .width = 9, + }, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "mpll0", + .ops = &meson_clk_mpll_ro_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct meson_clk_mpll gxbb_mpll1 = { + .sdm = { + .reg_off = HHI_MPLL_CNTL8, + .shift = 0, + .width = 14, + }, + .n2 = { + .reg_off = HHI_MPLL_CNTL8, + .shift = 16, + .width = 9, + }, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "mpll1", + .ops = &meson_clk_mpll_ro_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct meson_clk_mpll gxbb_mpll2 = { + .sdm = { + .reg_off = HHI_MPLL_CNTL9, + .shift = 0, + .width = 14, + }, + .n2 = { + .reg_off = HHI_MPLL_CNTL9, + .shift = 16, + .width = 9, + }, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "mpll2", + .ops = &meson_clk_mpll_ro_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +/* + * FIXME cpu clocks and the legacy composite clocks (e.g. clk81) are both PLL + * post-dividers and should be modeled with their respective PLLs via the + * forthcoming coordinated clock rates feature + */ +static struct meson_clk_cpu gxbb_cpu_clk = { + .reg_off = HHI_SYS_CPU_CLK_CNTL1, + .div_table = cpu_div_table, + .clk_nb.notifier_call = meson_clk_cpu_notifier_cb, + .hw.init = &(struct clk_init_data){ + .name = "cpu_clk", + .ops = &meson_clk_cpu_ops, + .parent_names = (const char *[]){ "sys_pll" }, + .num_parents = 1, + }, +}; + +static u32 mux_table_clk81[] = { 6, 5, 7 }; + +static struct clk_mux gxbb_mpeg_clk_sel = { + .reg = (void *)HHI_MPEG_CLK_CNTL, + .mask = 0x7, + .shift = 12, + .flags = CLK_MUX_READ_ONLY, + .table = mux_table_clk81, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "mpeg_clk_sel", + .ops = &clk_mux_ro_ops, + /* + * FIXME bits 14:12 selects from 8 possible parents: + * xtal, 1'b0 (wtf), fclk_div7, mpll_clkout1, mpll_clkout2, + * fclk_div4, fclk_div3, fclk_div5 + */ + .parent_names = (const char *[]){ "fclk_div3", "fclk_div4", + "fclk_div5" }, + .num_parents = 3, + .flags = (CLK_SET_RATE_NO_REPARENT | CLK_IGNORE_UNUSED), + }, +}; + +static struct clk_divider gxbb_mpeg_clk_div = { + .reg = (void *)HHI_MPEG_CLK_CNTL, + .shift = 0, + .width = 7, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "mpeg_clk_div", + .ops = &clk_divider_ops, + .parent_names = (const char *[]){ "mpeg_clk_sel" }, + .num_parents = 1, + .flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED), + }, +}; + +/* the mother of dragons^W gates */ +static struct clk_gate gxbb_clk81 = { + .reg = (void *)HHI_MPEG_CLK_CNTL, + .bit_idx = 7, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "clk81", + .ops = &clk_gate_ops, + .parent_names = (const char *[]){ "mpeg_clk_div" }, + .num_parents = 1, + .flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED | CLK_IS_CRITICAL), + }, +}; + +/* Everything Else (EE) domain gates */ +static MESON_GATE(ddr, HHI_GCLK_MPEG0, 0); +static MESON_GATE(dos, HHI_GCLK_MPEG0, 1); +static MESON_GATE(isa, HHI_GCLK_MPEG0, 5); +static MESON_GATE(pl301, HHI_GCLK_MPEG0, 6); +static MESON_GATE(periphs, HHI_GCLK_MPEG0, 7); +static MESON_GATE(spicc, HHI_GCLK_MPEG0, 8); +static MESON_GATE(i2c, HHI_GCLK_MPEG0, 9); +static MESON_GATE(sar_adc, HHI_GCLK_MPEG0, 10); +static MESON_GATE(smart_card, HHI_GCLK_MPEG0, 11); +static MESON_GATE(rng0, HHI_GCLK_MPEG0, 12); +static MESON_GATE(uart0, HHI_GCLK_MPEG0, 13); +static MESON_GATE(sdhc, HHI_GCLK_MPEG0, 14); +static MESON_GATE(stream, HHI_GCLK_MPEG0, 15); +static MESON_GATE(async_fifo, HHI_GCLK_MPEG0, 16); +static MESON_GATE(sdio, HHI_GCLK_MPEG0, 17); +static MESON_GATE(abuf, HHI_GCLK_MPEG0, 18); +static MESON_GATE(hiu_iface, HHI_GCLK_MPEG0, 19); +static MESON_GATE(assist_misc, HHI_GCLK_MPEG0, 23); +static MESON_GATE(spi, HHI_GCLK_MPEG0, 30); + +static MESON_GATE(i2s_spdif, HHI_GCLK_MPEG1, 2); +static MESON_GATE(eth, HHI_GCLK_MPEG1, 3); +static MESON_GATE(demux, HHI_GCLK_MPEG1, 4); +static MESON_GATE(aiu_glue, HHI_GCLK_MPEG1, 6); +static MESON_GATE(iec958, HHI_GCLK_MPEG1, 7); +static MESON_GATE(i2s_out, HHI_GCLK_MPEG1, 8); +static MESON_GATE(amclk, HHI_GCLK_MPEG1, 9); +static MESON_GATE(aififo2, HHI_GCLK_MPEG1, 10); +static MESON_GATE(mixer, HHI_GCLK_MPEG1, 11); +static MESON_GATE(mixer_iface, HHI_GCLK_MPEG1, 12); +static MESON_GATE(adc, HHI_GCLK_MPEG1, 13); +static MESON_GATE(blkmv, HHI_GCLK_MPEG1, 14); +static MESON_GATE(aiu, HHI_GCLK_MPEG1, 15); +static MESON_GATE(uart1, HHI_GCLK_MPEG1, 16); +static MESON_GATE(g2d, HHI_GCLK_MPEG1, 20); +static MESON_GATE(usb0, HHI_GCLK_MPEG1, 21); +static MESON_GATE(usb1, HHI_GCLK_MPEG1, 22); +static MESON_GATE(reset, HHI_GCLK_MPEG1, 23); +static MESON_GATE(nand, HHI_GCLK_MPEG1, 24); +static MESON_GATE(dos_parser, HHI_GCLK_MPEG1, 25); +static MESON_GATE(usb, HHI_GCLK_MPEG1, 26); +static MESON_GATE(vdin1, HHI_GCLK_MPEG1, 28); +static MESON_GATE(ahb_arb0, HHI_GCLK_MPEG1, 29); +static MESON_GATE(efuse, HHI_GCLK_MPEG1, 30); +static MESON_GATE(boot_rom, HHI_GCLK_MPEG1, 31); + +static MESON_GATE(ahb_data_bus, HHI_GCLK_MPEG2, 1); +static MESON_GATE(ahb_ctrl_bus, HHI_GCLK_MPEG2, 2); +static MESON_GATE(hdmi_intr_sync, HHI_GCLK_MPEG2, 3); +static MESON_GATE(hdmi_pclk, HHI_GCLK_MPEG2, 4); +static MESON_GATE(usb1_ddr_bridge, HHI_GCLK_MPEG2, 8); +static MESON_GATE(usb0_ddr_bridge, HHI_GCLK_MPEG2, 9); +static MESON_GATE(mmc_pclk, HHI_GCLK_MPEG2, 11); +static MESON_GATE(dvin, HHI_GCLK_MPEG2, 12); +static MESON_GATE(uart2, HHI_GCLK_MPEG2, 15); +static MESON_GATE(sana, HHI_GCLK_MPEG2, 22); +static MESON_GATE(vpu_intr, HHI_GCLK_MPEG2, 25); +static MESON_GATE(sec_ahb_ahb3_bridge, HHI_GCLK_MPEG2, 26); +static MESON_GATE(clk81_a53, HHI_GCLK_MPEG2, 29); + +static MESON_GATE(vclk2_venci0, HHI_GCLK_OTHER, 1); +static MESON_GATE(vclk2_venci1, HHI_GCLK_OTHER, 2); +static MESON_GATE(vclk2_vencp0, HHI_GCLK_OTHER, 3); +static MESON_GATE(vclk2_vencp1, HHI_GCLK_OTHER, 4); +static MESON_GATE(gclk_venci_int0, HHI_GCLK_OTHER, 8); +static MESON_GATE(gclk_vencp_int, HHI_GCLK_OTHER, 9); +static MESON_GATE(dac_clk, HHI_GCLK_OTHER, 10); +static MESON_GATE(aoclk_gate, HHI_GCLK_OTHER, 14); +static MESON_GATE(iec958_gate, HHI_GCLK_OTHER, 16); +static MESON_GATE(enc480p, HHI_GCLK_OTHER, 20); +static MESON_GATE(rng1, HHI_GCLK_OTHER, 21); +static MESON_GATE(gclk_venci_int1, HHI_GCLK_OTHER, 22); +static MESON_GATE(vclk2_venclmcc, HHI_GCLK_OTHER, 24); +static MESON_GATE(vclk2_vencl, HHI_GCLK_OTHER, 25); +static MESON_GATE(vclk_other, HHI_GCLK_OTHER, 26); +static MESON_GATE(edp, HHI_GCLK_OTHER, 31); + +/* Always On (AO) domain gates */ + +static MESON_GATE(ao_media_cpu, HHI_GCLK_AO, 0); +static MESON_GATE(ao_ahb_sram, HHI_GCLK_AO, 1); +static MESON_GATE(ao_ahb_bus, HHI_GCLK_AO, 2); +static MESON_GATE(ao_iface, HHI_GCLK_AO, 3); +static MESON_GATE(ao_i2c, HHI_GCLK_AO, 4); + +/* Array of all clocks provided by this provider */ + +static struct clk_hw_onecell_data gxbb_hw_onecell_data = { + .hws = { + [CLKID_SYS_PLL] = &gxbb_sys_pll.hw, + [CLKID_CPUCLK] = &gxbb_cpu_clk.hw, + [CLKID_HDMI_PLL] = &gxbb_hdmi_pll.hw, + [CLKID_FIXED_PLL] = &gxbb_fixed_pll.hw, + [CLKID_FCLK_DIV2] = &gxbb_fclk_div2.hw, + [CLKID_FCLK_DIV3] = &gxbb_fclk_div3.hw, + [CLKID_FCLK_DIV4] = &gxbb_fclk_div4.hw, + [CLKID_FCLK_DIV5] = &gxbb_fclk_div5.hw, + [CLKID_FCLK_DIV7] = &gxbb_fclk_div7.hw, + [CLKID_GP0_PLL] = &gxbb_gp0_pll.hw, + [CLKID_MPEG_SEL] = &gxbb_mpeg_clk_sel.hw, + [CLKID_MPEG_DIV] = &gxbb_mpeg_clk_div.hw, + [CLKID_CLK81] = &gxbb_clk81.hw, + [CLKID_MPLL0] = &gxbb_mpll0.hw, + [CLKID_MPLL1] = &gxbb_mpll1.hw, + [CLKID_MPLL2] = &gxbb_mpll2.hw, + [CLKID_DDR] = &gxbb_ddr.hw, + [CLKID_DOS] = &gxbb_dos.hw, + [CLKID_ISA] = &gxbb_isa.hw, + [CLKID_PL301] = &gxbb_pl301.hw, + [CLKID_PERIPHS] = &gxbb_periphs.hw, + [CLKID_SPICC] = &gxbb_spicc.hw, + [CLKID_I2C] = &gxbb_i2c.hw, + [CLKID_SAR_ADC] = &gxbb_sar_adc.hw, + [CLKID_SMART_CARD] = &gxbb_smart_card.hw, + [CLKID_RNG0] = &gxbb_rng0.hw, + [CLKID_UART0] = &gxbb_uart0.hw, + [CLKID_SDHC] = &gxbb_sdhc.hw, + [CLKID_STREAM] = &gxbb_stream.hw, + [CLKID_ASYNC_FIFO] = &gxbb_async_fifo.hw, + [CLKID_SDIO] = &gxbb_sdio.hw, + [CLKID_ABUF] = &gxbb_abuf.hw, + [CLKID_HIU_IFACE] = &gxbb_hiu_iface.hw, + [CLKID_ASSIST_MISC] = &gxbb_assist_misc.hw, + [CLKID_SPI] = &gxbb_spi.hw, + [CLKID_I2S_SPDIF] = &gxbb_i2s_spdif.hw, + [CLKID_ETH] = &gxbb_eth.hw, + [CLKID_DEMUX] = &gxbb_demux.hw, + [CLKID_AIU_GLUE] = &gxbb_aiu_glue.hw, + [CLKID_IEC958] = &gxbb_iec958.hw, + [CLKID_I2S_OUT] = &gxbb_i2s_out.hw, + [CLKID_AMCLK] = &gxbb_amclk.hw, + [CLKID_AIFIFO2] = &gxbb_aififo2.hw, + [CLKID_MIXER] = &gxbb_mixer.hw, + [CLKID_MIXER_IFACE] = &gxbb_mixer_iface.hw, + [CLKID_ADC] = &gxbb_adc.hw, + [CLKID_BLKMV] = &gxbb_blkmv.hw, + [CLKID_AIU] = &gxbb_aiu.hw, + [CLKID_UART1] = &gxbb_uart1.hw, + [CLKID_G2D] = &gxbb_g2d.hw, + [CLKID_USB0] = &gxbb_usb0.hw, + [CLKID_USB1] = &gxbb_usb1.hw, + [CLKID_RESET] = &gxbb_reset.hw, + [CLKID_NAND] = &gxbb_nand.hw, + [CLKID_DOS_PARSER] = &gxbb_dos_parser.hw, + [CLKID_USB] = &gxbb_usb.hw, + [CLKID_VDIN1] = &gxbb_vdin1.hw, + [CLKID_AHB_ARB0] = &gxbb_ahb_arb0.hw, + [CLKID_EFUSE] = &gxbb_efuse.hw, + [CLKID_BOOT_ROM] = &gxbb_boot_rom.hw, + [CLKID_AHB_DATA_BUS] = &gxbb_ahb_data_bus.hw, + [CLKID_AHB_CTRL_BUS] = &gxbb_ahb_ctrl_bus.hw, + [CLKID_HDMI_INTR_SYNC] = &gxbb_hdmi_intr_sync.hw, + [CLKID_HDMI_PCLK] = &gxbb_hdmi_pclk.hw, + [CLKID_USB1_DDR_BRIDGE] = &gxbb_usb1_ddr_bridge.hw, + [CLKID_USB0_DDR_BRIDGE] = &gxbb_usb0_ddr_bridge.hw, + [CLKID_MMC_PCLK] = &gxbb_mmc_pclk.hw, + [CLKID_DVIN] = &gxbb_dvin.hw, + [CLKID_UART2] = &gxbb_uart2.hw, + [CLKID_SANA] = &gxbb_sana.hw, + [CLKID_VPU_INTR] = &gxbb_vpu_intr.hw, + [CLKID_SEC_AHB_AHB3_BRIDGE] = &gxbb_sec_ahb_ahb3_bridge.hw, + [CLKID_CLK81_A53] = &gxbb_clk81_a53.hw, + [CLKID_VCLK2_VENCI0] = &gxbb_vclk2_venci0.hw, + [CLKID_VCLK2_VENCI1] = &gxbb_vclk2_venci1.hw, + [CLKID_VCLK2_VENCP0] = &gxbb_vclk2_vencp0.hw, + [CLKID_VCLK2_VENCP1] = &gxbb_vclk2_vencp1.hw, + [CLKID_GCLK_VENCI_INT0] = &gxbb_gclk_venci_int0.hw, + [CLKID_GCLK_VENCI_INT] = &gxbb_gclk_vencp_int.hw, + [CLKID_DAC_CLK] = &gxbb_dac_clk.hw, + [CLKID_AOCLK_GATE] = &gxbb_aoclk_gate.hw, + [CLKID_IEC958_GATE] = &gxbb_iec958_gate.hw, + [CLKID_ENC480P] = &gxbb_enc480p.hw, + [CLKID_RNG1] = &gxbb_rng1.hw, + [CLKID_GCLK_VENCI_INT1] = &gxbb_gclk_venci_int1.hw, + [CLKID_VCLK2_VENCLMCC] = &gxbb_vclk2_venclmcc.hw, + [CLKID_VCLK2_VENCL] = &gxbb_vclk2_vencl.hw, + [CLKID_VCLK_OTHER] = &gxbb_vclk_other.hw, + [CLKID_EDP] = &gxbb_edp.hw, + [CLKID_AO_MEDIA_CPU] = &gxbb_ao_media_cpu.hw, + [CLKID_AO_AHB_SRAM] = &gxbb_ao_ahb_sram.hw, + [CLKID_AO_AHB_BUS] = &gxbb_ao_ahb_bus.hw, + [CLKID_AO_IFACE] = &gxbb_ao_iface.hw, + [CLKID_AO_I2C] = &gxbb_ao_i2c.hw, + }, + .num = NR_CLKS, +}; + +/* Convenience tables to populate base addresses in .probe */ + +static struct meson_clk_pll *const gxbb_clk_plls[] = { + &gxbb_fixed_pll, + &gxbb_hdmi_pll, + &gxbb_sys_pll, + &gxbb_gp0_pll, +}; + +static struct meson_clk_mpll *const gxbb_clk_mplls[] = { + &gxbb_mpll0, + &gxbb_mpll1, + &gxbb_mpll2, +}; + +static struct clk_gate *gxbb_clk_gates[] = { + &gxbb_clk81, + &gxbb_ddr, + &gxbb_dos, + &gxbb_isa, + &gxbb_pl301, + &gxbb_periphs, + &gxbb_spicc, + &gxbb_i2c, + &gxbb_sar_adc, + &gxbb_smart_card, + &gxbb_rng0, + &gxbb_uart0, + &gxbb_sdhc, + &gxbb_stream, + &gxbb_async_fifo, + &gxbb_sdio, + &gxbb_abuf, + &gxbb_hiu_iface, + &gxbb_assist_misc, + &gxbb_spi, + &gxbb_i2s_spdif, + &gxbb_eth, + &gxbb_demux, + &gxbb_aiu_glue, + &gxbb_iec958, + &gxbb_i2s_out, + &gxbb_amclk, + &gxbb_aififo2, + &gxbb_mixer, + &gxbb_mixer_iface, + &gxbb_adc, + &gxbb_blkmv, + &gxbb_aiu, + &gxbb_uart1, + &gxbb_g2d, + &gxbb_usb0, + &gxbb_usb1, + &gxbb_reset, + &gxbb_nand, + &gxbb_dos_parser, + &gxbb_usb, + &gxbb_vdin1, + &gxbb_ahb_arb0, + &gxbb_efuse, + &gxbb_boot_rom, + &gxbb_ahb_data_bus, + &gxbb_ahb_ctrl_bus, + &gxbb_hdmi_intr_sync, + &gxbb_hdmi_pclk, + &gxbb_usb1_ddr_bridge, + &gxbb_usb0_ddr_bridge, + &gxbb_mmc_pclk, + &gxbb_dvin, + &gxbb_uart2, + &gxbb_sana, + &gxbb_vpu_intr, + &gxbb_sec_ahb_ahb3_bridge, + &gxbb_clk81_a53, + &gxbb_vclk2_venci0, + &gxbb_vclk2_venci1, + &gxbb_vclk2_vencp0, + &gxbb_vclk2_vencp1, + &gxbb_gclk_venci_int0, + &gxbb_gclk_vencp_int, + &gxbb_dac_clk, + &gxbb_aoclk_gate, + &gxbb_iec958_gate, + &gxbb_enc480p, + &gxbb_rng1, + &gxbb_gclk_venci_int1, + &gxbb_vclk2_venclmcc, + &gxbb_vclk2_vencl, + &gxbb_vclk_other, + &gxbb_edp, + &gxbb_ao_media_cpu, + &gxbb_ao_ahb_sram, + &gxbb_ao_ahb_bus, + &gxbb_ao_iface, + &gxbb_ao_i2c, +}; + +static int gxbb_clkc_probe(struct platform_device *pdev) +{ + void __iomem *clk_base; + int ret, clkid, i; + struct clk_hw *parent_hw; + struct clk *parent_clk; + struct device *dev = &pdev->dev; + + /* Generic clocks and PLLs */ + clk_base = of_iomap(dev->of_node, 0); + if (!clk_base) { + pr_err("%s: Unable to map clk base\n", __func__); + return -ENXIO; + } + + /* Populate base address for PLLs */ + for (i = 0; i < ARRAY_SIZE(gxbb_clk_plls); i++) + gxbb_clk_plls[i]->base = clk_base; + + /* Populate base address for MPLLs */ + for (i = 0; i < ARRAY_SIZE(gxbb_clk_mplls); i++) + gxbb_clk_mplls[i]->base = clk_base; + + /* Populate the base address for CPU clk */ + gxbb_cpu_clk.base = clk_base; + + /* Populate the base address for the MPEG clks */ + gxbb_mpeg_clk_sel.reg = clk_base + (u64)gxbb_mpeg_clk_sel.reg; + gxbb_mpeg_clk_div.reg = clk_base + (u64)gxbb_mpeg_clk_div.reg; + + /* Populate base address for gates */ + for (i = 0; i < ARRAY_SIZE(gxbb_clk_gates); i++) + gxbb_clk_gates[i]->reg = clk_base + + (u64)gxbb_clk_gates[i]->reg; + + /* + * register all clks + */ + for (clkid = 0; clkid < NR_CLKS; clkid++) { + ret = devm_clk_hw_register(dev, gxbb_hw_onecell_data.hws[clkid]); + if (ret) + goto iounmap; + } + + /* + * Register CPU clk notifier + * + * FIXME this is wrong for a lot of reasons. First, the muxes should be + * struct clk_hw objects. Second, we shouldn't program the muxes in + * notifier handlers. The tricky programming sequence will be handled + * by the forthcoming coordinated clock rates mechanism once that + * feature is released. + * + * Furthermore, looking up the parent this way is terrible. At some + * point we will stop allocating a default struct clk when registering + * a new clk_hw, and this hack will no longer work. Releasing the ccr + * feature before that time solves the problem :-) + */ + parent_hw = clk_hw_get_parent(&gxbb_cpu_clk.hw); + parent_clk = parent_hw->clk; + ret = clk_notifier_register(parent_clk, &gxbb_cpu_clk.clk_nb); + if (ret) { + pr_err("%s: failed to register clock notifier for cpu_clk\n", + __func__); + goto iounmap; + } + + return of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get, + &gxbb_hw_onecell_data); + +iounmap: + iounmap(clk_base); + return ret; +} + +static const struct of_device_id gxbb_clkc_match_table[] = { + { .compatible = "amlogic,gxbb-clkc" }, + { } +}; + +static struct platform_driver gxbb_driver = { + .probe = gxbb_clkc_probe, + .driver = { + .name = "gxbb-clkc", + .of_match_table = gxbb_clkc_match_table, + }, +}; + +static int __init gxbb_clkc_init(void) +{ + return platform_driver_register(&gxbb_driver); +} +device_initcall(gxbb_clkc_init); diff --git a/drivers/clk/meson/gxbb.h b/drivers/clk/meson/gxbb.h new file mode 100644 index 000000000..a2adf3448 --- /dev/null +++ b/drivers/clk/meson/gxbb.h @@ -0,0 +1,271 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright (c) 2016 AmLogic, Inc. + * Author: Michael Turquette + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * The full GNU General Public License is included in this distribution + * in the file called COPYING + * + * BSD LICENSE + * + * Copyright (c) 2016 BayLibre, Inc. + * Author: Michael Turquette + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __GXBB_H +#define __GXBB_H + +/* + * Clock controller register offsets + * + * Register offsets from the data sheet are listed in comment blocks below. + * Those offsets must be multiplied by 4 before adding them to the base address + * to get the right value + */ +#define SCR 0x2C /* 0x0b offset in data sheet */ +#define TIMEOUT_VALUE 0x3c /* 0x0f offset in data sheet */ + +#define HHI_GP0_PLL_CNTL 0x40 /* 0x10 offset in data sheet */ +#define HHI_GP0_PLL_CNTL2 0x44 /* 0x11 offset in data sheet */ +#define HHI_GP0_PLL_CNTL3 0x48 /* 0x12 offset in data sheet */ +#define HHI_GP0_PLL_CNTL4 0x4c /* 0x13 offset in data sheet */ + +#define HHI_XTAL_DIVN_CNTL 0xbc /* 0x2f offset in data sheet */ +#define HHI_TIMER90K 0xec /* 0x3b offset in data sheet */ + +#define HHI_MEM_PD_REG0 0x100 /* 0x40 offset in data sheet */ +#define HHI_MEM_PD_REG1 0x104 /* 0x41 offset in data sheet */ +#define HHI_VPU_MEM_PD_REG1 0x108 /* 0x42 offset in data sheet */ +#define HHI_VIID_CLK_DIV 0x128 /* 0x4a offset in data sheet */ +#define HHI_VIID_CLK_CNTL 0x12c /* 0x4b offset in data sheet */ + +#define HHI_GCLK_MPEG0 0x140 /* 0x50 offset in data sheet */ +#define HHI_GCLK_MPEG1 0x144 /* 0x51 offset in data sheet */ +#define HHI_GCLK_MPEG2 0x148 /* 0x52 offset in data sheet */ +#define HHI_GCLK_OTHER 0x150 /* 0x54 offset in data sheet */ +#define HHI_GCLK_AO 0x154 /* 0x55 offset in data sheet */ +#define HHI_SYS_OSCIN_CNTL 0x158 /* 0x56 offset in data sheet */ +#define HHI_SYS_CPU_CLK_CNTL1 0x15c /* 0x57 offset in data sheet */ +#define HHI_SYS_CPU_RESET_CNTL 0x160 /* 0x58 offset in data sheet */ +#define HHI_VID_CLK_DIV 0x164 /* 0x59 offset in data sheet */ + +#define HHI_MPEG_CLK_CNTL 0x174 /* 0x5d offset in data sheet */ +#define HHI_AUD_CLK_CNTL 0x178 /* 0x5e offset in data sheet */ +#define HHI_VID_CLK_CNTL 0x17c /* 0x5f offset in data sheet */ +#define HHI_AUD_CLK_CNTL2 0x190 /* 0x64 offset in data sheet */ +#define HHI_VID_CLK_CNTL2 0x194 /* 0x65 offset in data sheet */ +#define HHI_SYS_CPU_CLK_CNTL0 0x19c /* 0x67 offset in data sheet */ +#define HHI_VID_PLL_CLK_DIV 0x1a0 /* 0x68 offset in data sheet */ +#define HHI_AUD_CLK_CNTL3 0x1a4 /* 0x69 offset in data sheet */ +#define HHI_MALI_CLK_CNTL 0x1b0 /* 0x6c offset in data sheet */ +#define HHI_VPU_CLK_CNTL 0x1bC /* 0x6f offset in data sheet */ + +#define HHI_HDMI_CLK_CNTL 0x1CC /* 0x73 offset in data sheet */ +#define HHI_VDEC_CLK_CNTL 0x1E0 /* 0x78 offset in data sheet */ +#define HHI_VDEC2_CLK_CNTL 0x1E4 /* 0x79 offset in data sheet */ +#define HHI_VDEC3_CLK_CNTL 0x1E8 /* 0x7a offset in data sheet */ +#define HHI_VDEC4_CLK_CNTL 0x1EC /* 0x7b offset in data sheet */ +#define HHI_HDCP22_CLK_CNTL 0x1F0 /* 0x7c offset in data sheet */ +#define HHI_VAPBCLK_CNTL 0x1F4 /* 0x7d offset in data sheet */ + +#define HHI_VPU_CLKB_CNTL 0x20C /* 0x83 offset in data sheet */ +#define HHI_USB_CLK_CNTL 0x220 /* 0x88 offset in data sheet */ +#define HHI_32K_CLK_CNTL 0x224 /* 0x89 offset in data sheet */ +#define HHI_GEN_CLK_CNTL 0x228 /* 0x8a offset in data sheet */ +#define HHI_GEN_CLK_CNTL 0x228 /* 0x8a offset in data sheet */ + +#define HHI_PCM_CLK_CNTL 0x258 /* 0x96 offset in data sheet */ +#define HHI_NAND_CLK_CNTL 0x25C /* 0x97 offset in data sheet */ +#define HHI_SD_EMMC_CLK_CNTL 0x264 /* 0x99 offset in data sheet */ + +#define HHI_MPLL_CNTL 0x280 /* 0xa0 offset in data sheet */ +#define HHI_MPLL_CNTL2 0x284 /* 0xa1 offset in data sheet */ +#define HHI_MPLL_CNTL3 0x288 /* 0xa2 offset in data sheet */ +#define HHI_MPLL_CNTL4 0x28C /* 0xa3 offset in data sheet */ +#define HHI_MPLL_CNTL5 0x290 /* 0xa4 offset in data sheet */ +#define HHI_MPLL_CNTL6 0x294 /* 0xa5 offset in data sheet */ +#define HHI_MPLL_CNTL7 0x298 /* MP0, 0xa6 offset in data sheet */ +#define HHI_MPLL_CNTL8 0x29C /* MP1, 0xa7 offset in data sheet */ +#define HHI_MPLL_CNTL9 0x2A0 /* MP2, 0xa8 offset in data sheet */ +#define HHI_MPLL_CNTL10 0x2A4 /* MP2, 0xa9 offset in data sheet */ + +#define HHI_MPLL3_CNTL0 0x2E0 /* 0xb8 offset in data sheet */ +#define HHI_MPLL3_CNTL1 0x2E4 /* 0xb9 offset in data sheet */ +#define HHI_VDAC_CNTL0 0x2F4 /* 0xbd offset in data sheet */ +#define HHI_VDAC_CNTL1 0x2F8 /* 0xbe offset in data sheet */ + +#define HHI_SYS_PLL_CNTL 0x300 /* 0xc0 offset in data sheet */ +#define HHI_SYS_PLL_CNTL2 0x304 /* 0xc1 offset in data sheet */ +#define HHI_SYS_PLL_CNTL3 0x308 /* 0xc2 offset in data sheet */ +#define HHI_SYS_PLL_CNTL4 0x30c /* 0xc3 offset in data sheet */ +#define HHI_SYS_PLL_CNTL5 0x310 /* 0xc4 offset in data sheet */ +#define HHI_DPLL_TOP_I 0x318 /* 0xc6 offset in data sheet */ +#define HHI_DPLL_TOP2_I 0x31C /* 0xc7 offset in data sheet */ +#define HHI_HDMI_PLL_CNTL 0x320 /* 0xc8 offset in data sheet */ +#define HHI_HDMI_PLL_CNTL2 0x324 /* 0xc9 offset in data sheet */ +#define HHI_HDMI_PLL_CNTL3 0x328 /* 0xca offset in data sheet */ +#define HHI_HDMI_PLL_CNTL4 0x32C /* 0xcb offset in data sheet */ +#define HHI_HDMI_PLL_CNTL5 0x330 /* 0xcc offset in data sheet */ +#define HHI_HDMI_PLL_CNTL6 0x334 /* 0xcd offset in data sheet */ +#define HHI_HDMI_PLL_CNTL_I 0x338 /* 0xce offset in data sheet */ +#define HHI_HDMI_PLL_CNTL7 0x33C /* 0xcf offset in data sheet */ + +#define HHI_HDMI_PHY_CNTL0 0x3A0 /* 0xe8 offset in data sheet */ +#define HHI_HDMI_PHY_CNTL1 0x3A4 /* 0xe9 offset in data sheet */ +#define HHI_HDMI_PHY_CNTL2 0x3A8 /* 0xea offset in data sheet */ +#define HHI_HDMI_PHY_CNTL3 0x3AC /* 0xeb offset in data sheet */ + +#define HHI_VID_LOCK_CLK_CNTL 0x3C8 /* 0xf2 offset in data sheet */ +#define HHI_BT656_CLK_CNTL 0x3D4 /* 0xf5 offset in data sheet */ +#define HHI_SAR_CLK_CNTL 0x3D8 /* 0xf6 offset in data sheet */ + +/* + * CLKID index values + * + * These indices are entirely contrived and do not map onto the hardware. + * Migrate them out of this header and into the DT header file when they need + * to be exposed to client nodes in DT: include/dt-bindings/clock/gxbb-clkc.h + */ +#define CLKID_SYS_PLL 0 +/* CLKID_CPUCLK */ +#define CLKID_HDMI_PLL 2 +#define CLKID_FIXED_PLL 3 +#define CLKID_FCLK_DIV2 4 +#define CLKID_FCLK_DIV3 5 +#define CLKID_FCLK_DIV4 6 +#define CLKID_FCLK_DIV5 7 +#define CLKID_FCLK_DIV7 8 +#define CLKID_GP0_PLL 9 +#define CLKID_MPEG_SEL 10 +#define CLKID_MPEG_DIV 11 +/* CLKID_CLK81 */ +#define CLKID_MPLL0 13 +#define CLKID_MPLL1 14 +#define CLKID_MPLL2 15 +#define CLKID_DDR 16 +#define CLKID_DOS 17 +#define CLKID_ISA 18 +#define CLKID_PL301 19 +#define CLKID_PERIPHS 20 +#define CLKID_SPICC 21 +#define CLKID_I2C 22 +#define CLKID_SAR_ADC 23 +#define CLKID_SMART_CARD 24 +#define CLKID_RNG0 25 +#define CLKID_UART0 26 +#define CLKID_SDHC 27 +#define CLKID_STREAM 28 +#define CLKID_ASYNC_FIFO 29 +#define CLKID_SDIO 30 +#define CLKID_ABUF 31 +#define CLKID_HIU_IFACE 32 +#define CLKID_ASSIST_MISC 33 +#define CLKID_SPI 34 +#define CLKID_I2S_SPDIF 35 +#define CLKID_ETH 36 +#define CLKID_DEMUX 37 +#define CLKID_AIU_GLUE 38 +#define CLKID_IEC958 39 +#define CLKID_I2S_OUT 40 +#define CLKID_AMCLK 41 +#define CLKID_AIFIFO2 42 +#define CLKID_MIXER 43 +#define CLKID_MIXER_IFACE 44 +#define CLKID_ADC 45 +#define CLKID_BLKMV 46 +#define CLKID_AIU 47 +#define CLKID_UART1 48 +#define CLKID_G2D 49 +#define CLKID_USB0 50 +#define CLKID_USB1 51 +#define CLKID_RESET 52 +#define CLKID_NAND 53 +#define CLKID_DOS_PARSER 54 +#define CLKID_USB 55 +#define CLKID_VDIN1 56 +#define CLKID_AHB_ARB0 57 +#define CLKID_EFUSE 58 +#define CLKID_BOOT_ROM 59 +#define CLKID_AHB_DATA_BUS 60 +#define CLKID_AHB_CTRL_BUS 61 +#define CLKID_HDMI_INTR_SYNC 62 +#define CLKID_HDMI_PCLK 63 +#define CLKID_USB1_DDR_BRIDGE 64 +#define CLKID_USB0_DDR_BRIDGE 65 +#define CLKID_MMC_PCLK 66 +#define CLKID_DVIN 67 +#define CLKID_UART2 68 +#define CLKID_SANA 69 +#define CLKID_VPU_INTR 70 +#define CLKID_SEC_AHB_AHB3_BRIDGE 71 +#define CLKID_CLK81_A53 72 +#define CLKID_VCLK2_VENCI0 73 +#define CLKID_VCLK2_VENCI1 74 +#define CLKID_VCLK2_VENCP0 75 +#define CLKID_VCLK2_VENCP1 76 +#define CLKID_GCLK_VENCI_INT0 77 +#define CLKID_GCLK_VENCI_INT 78 +#define CLKID_DAC_CLK 79 +#define CLKID_AOCLK_GATE 80 +#define CLKID_IEC958_GATE 81 +#define CLKID_ENC480P 82 +#define CLKID_RNG1 83 +#define CLKID_GCLK_VENCI_INT1 84 +#define CLKID_VCLK2_VENCLMCC 85 +#define CLKID_VCLK2_VENCL 86 +#define CLKID_VCLK_OTHER 87 +#define CLKID_EDP 88 +#define CLKID_AO_MEDIA_CPU 89 +#define CLKID_AO_AHB_SRAM 90 +#define CLKID_AO_AHB_BUS 91 +#define CLKID_AO_IFACE 92 +#define CLKID_AO_I2C 93 + +#define NR_CLKS 94 + +/* include the CLKIDs that have been made part of the stable DT binding */ +#include + +#endif /* __GXBB_H */ diff --git a/drivers/clk/meson/meson8b-clkc.c b/drivers/clk/meson/meson8b-clkc.c index 4d057b3e2..4c9413cdf 100644 --- a/drivers/clk/meson/meson8b-clkc.c +++ b/drivers/clk/meson/meson8b-clkc.c @@ -1,7 +1,12 @@ /* + * AmLogic S805 / Meson8b Clock Controller Driver + * * Copyright (c) 2015 Endless Mobile, Inc. * Author: Carlo Caione * + * Copyright (c) 2016 BayLibre, Inc. + * Michael Turquette + * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. @@ -15,23 +20,33 @@ * this program. If not, see . */ +#include #include -#include -#include #include -#include #include +#include +#include #include "clkc.h" -#define MESON8B_REG_CTL0_ADDR 0x0000 -#define MESON8B_REG_SYS_CPU_CNTL1 0x015c -#define MESON8B_REG_HHI_MPEG 0x0174 -#define MESON8B_REG_MALI 0x01b0 +/* + * Clock controller register offsets + * + * Register offsets from the HardKernel[0] data sheet are listed in comment + * blocks below. Those offsets must be multiplied by 4 before adding them to + * the base address to get the right value + * + * [0] http://dn.odroid.com/S805/Datasheet/S805_Datasheet%20V0.8%2020150126.pdf + */ +#define MESON8B_REG_SYS_CPU_CNTL1 0x015c /* 0x57 offset in data sheet */ +#define MESON8B_REG_HHI_MPEG 0x0174 /* 0x5d offset in data sheet */ +#define MESON8B_REG_MALI 0x01b0 /* 0x6c offset in data sheet */ #define MESON8B_REG_PLL_FIXED 0x0280 #define MESON8B_REG_PLL_SYS 0x0300 #define MESON8B_REG_PLL_VID 0x0320 +static DEFINE_SPINLOCK(clk_lock); + static const struct pll_rate_table sys_pll_rate_table[] = { PLL_RATE(312000000, 52, 1, 2), PLL_RATE(336000000, 56, 1, 2), @@ -102,95 +117,331 @@ static const struct clk_div_table cpu_div_table[] = { { /* sentinel */ }, }; -PNAME(p_xtal) = { "xtal" }; -PNAME(p_fclk_div) = { "fixed_pll" }; -PNAME(p_cpu_clk) = { "sys_pll" }; -PNAME(p_clk81) = { "fclk_div3", "fclk_div4", "fclk_div5" }; -PNAME(p_mali) = { "fclk_div3", "fclk_div4", "fclk_div5", - "fclk_div7", "zero" }; +static struct clk_fixed_rate meson8b_xtal = { + .fixed_rate = 24000000, + .hw.init = &(struct clk_init_data){ + .name = "xtal", + .num_parents = 0, + .ops = &clk_fixed_rate_ops, + }, +}; + +static struct meson_clk_pll meson8b_fixed_pll = { + .m = { + .reg_off = MESON8B_REG_PLL_FIXED, + .shift = 0, + .width = 9, + }, + .n = { + .reg_off = MESON8B_REG_PLL_FIXED, + .shift = 9, + .width = 5, + }, + .od = { + .reg_off = MESON8B_REG_PLL_FIXED, + .shift = 16, + .width = 2, + }, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "fixed_pll", + .ops = &meson_clk_pll_ro_ops, + .parent_names = (const char *[]){ "xtal" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct meson_clk_pll meson8b_vid_pll = { + .m = { + .reg_off = MESON8B_REG_PLL_VID, + .shift = 0, + .width = 9, + }, + .n = { + .reg_off = MESON8B_REG_PLL_VID, + .shift = 9, + .width = 5, + }, + .od = { + .reg_off = MESON8B_REG_PLL_VID, + .shift = 16, + .width = 2, + }, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "vid_pll", + .ops = &meson_clk_pll_ro_ops, + .parent_names = (const char *[]){ "xtal" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct meson_clk_pll meson8b_sys_pll = { + .m = { + .reg_off = MESON8B_REG_PLL_SYS, + .shift = 0, + .width = 9, + }, + .n = { + .reg_off = MESON8B_REG_PLL_SYS, + .shift = 9, + .width = 5, + }, + .od = { + .reg_off = MESON8B_REG_PLL_SYS, + .shift = 16, + .width = 2, + }, + .rate_table = sys_pll_rate_table, + .rate_count = ARRAY_SIZE(sys_pll_rate_table), + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "sys_pll", + .ops = &meson_clk_pll_ops, + .parent_names = (const char *[]){ "xtal" }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_fixed_factor meson8b_fclk_div2 = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div2", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor meson8b_fclk_div3 = { + .mult = 1, + .div = 3, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div3", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor meson8b_fclk_div4 = { + .mult = 1, + .div = 4, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div4", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor meson8b_fclk_div5 = { + .mult = 1, + .div = 5, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div5", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +static struct clk_fixed_factor meson8b_fclk_div7 = { + .mult = 1, + .div = 7, + .hw.init = &(struct clk_init_data){ + .name = "fclk_div7", + .ops = &clk_fixed_factor_ops, + .parent_names = (const char *[]){ "fixed_pll" }, + .num_parents = 1, + }, +}; + +/* + * FIXME cpu clocks and the legacy composite clocks (e.g. clk81) are both PLL + * post-dividers and should be modeled with their respective PLLs via the + * forthcoming coordinated clock rates feature + */ +static struct meson_clk_cpu meson8b_cpu_clk = { + .reg_off = MESON8B_REG_SYS_CPU_CNTL1, + .div_table = cpu_div_table, + .clk_nb.notifier_call = meson_clk_cpu_notifier_cb, + .hw.init = &(struct clk_init_data){ + .name = "cpu_clk", + .ops = &meson_clk_cpu_ops, + .parent_names = (const char *[]){ "sys_pll" }, + .num_parents = 1, + }, +}; static u32 mux_table_clk81[] = { 6, 5, 7 }; -static u32 mux_table_mali[] = { 6, 5, 7, 4, 0 }; - -static struct pll_conf pll_confs = { - .m = PARM(0x00, 0, 9), - .n = PARM(0x00, 9, 5), - .od = PARM(0x00, 16, 2), -}; - -static struct pll_conf sys_pll_conf = { - .m = PARM(0x00, 0, 9), - .n = PARM(0x00, 9, 5), - .od = PARM(0x00, 16, 2), - .rate_table = sys_pll_rate_table, -}; - -static const struct composite_conf clk81_conf __initconst = { - .mux_table = mux_table_clk81, - .mux_flags = CLK_MUX_READ_ONLY, - .mux_parm = PARM(0x00, 12, 3), - .div_parm = PARM(0x00, 0, 7), - .gate_parm = PARM(0x00, 7, 1), -}; - -static const struct composite_conf mali_conf __initconst = { - .mux_table = mux_table_mali, - .mux_parm = PARM(0x00, 9, 3), - .div_parm = PARM(0x00, 0, 7), - .gate_parm = PARM(0x00, 8, 1), -}; - -static const struct clk_conf meson8b_xtal_conf __initconst = - FIXED_RATE_P(MESON8B_REG_CTL0_ADDR, CLKID_XTAL, "xtal", 0, - PARM(0x00, 4, 7)); - -static const struct clk_conf meson8b_clk_confs[] __initconst = { - FIXED_RATE(CLKID_ZERO, "zero", 0, 0), - PLL(MESON8B_REG_PLL_FIXED, CLKID_PLL_FIXED, "fixed_pll", - p_xtal, 0, &pll_confs), - PLL(MESON8B_REG_PLL_VID, CLKID_PLL_VID, "vid_pll", - p_xtal, 0, &pll_confs), - PLL(MESON8B_REG_PLL_SYS, CLKID_PLL_SYS, "sys_pll", - p_xtal, 0, &sys_pll_conf), - FIXED_FACTOR_DIV(CLKID_FCLK_DIV2, "fclk_div2", p_fclk_div, 0, 2), - FIXED_FACTOR_DIV(CLKID_FCLK_DIV3, "fclk_div3", p_fclk_div, 0, 3), - FIXED_FACTOR_DIV(CLKID_FCLK_DIV4, "fclk_div4", p_fclk_div, 0, 4), - FIXED_FACTOR_DIV(CLKID_FCLK_DIV5, "fclk_div5", p_fclk_div, 0, 5), - FIXED_FACTOR_DIV(CLKID_FCLK_DIV7, "fclk_div7", p_fclk_div, 0, 7), - CPU(MESON8B_REG_SYS_CPU_CNTL1, CLKID_CPUCLK, "a5_clk", p_cpu_clk, - cpu_div_table), - COMPOSITE(MESON8B_REG_HHI_MPEG, CLKID_CLK81, "clk81", p_clk81, - CLK_SET_RATE_NO_REPARENT | CLK_IGNORE_UNUSED, &clk81_conf), - COMPOSITE(MESON8B_REG_MALI, CLKID_MALI, "mali", p_mali, - CLK_IGNORE_UNUSED, &mali_conf), -}; - -static void __init meson8b_clkc_init(struct device_node *np) -{ - void __iomem *clk_base; - if (!meson_clk_init(np, CLK_NR_CLKS)) - return; +struct clk_mux meson8b_mpeg_clk_sel = { + .reg = (void *)MESON8B_REG_HHI_MPEG, + .mask = 0x7, + .shift = 12, + .flags = CLK_MUX_READ_ONLY, + .table = mux_table_clk81, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "mpeg_clk_sel", + .ops = &clk_mux_ro_ops, + /* + * FIXME bits 14:12 selects from 8 possible parents: + * xtal, 1'b0 (wtf), fclk_div7, mpll_clkout1, mpll_clkout2, + * fclk_div4, fclk_div3, fclk_div5 + */ + .parent_names = (const char *[]){ "fclk_div3", "fclk_div4", + "fclk_div5" }, + .num_parents = 3, + .flags = (CLK_SET_RATE_NO_REPARENT | CLK_IGNORE_UNUSED), + }, +}; - /* XTAL */ - clk_base = of_iomap(np, 0); - if (!clk_base) { - pr_err("%s: Unable to map xtal base\n", __func__); - return; - } +struct clk_divider meson8b_mpeg_clk_div = { + .reg = (void *)MESON8B_REG_HHI_MPEG, + .shift = 0, + .width = 7, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "mpeg_clk_div", + .ops = &clk_divider_ops, + .parent_names = (const char *[]){ "mpeg_clk_sel" }, + .num_parents = 1, + .flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED), + }, +}; - meson_clk_register_clks(&meson8b_xtal_conf, 1, clk_base); - iounmap(clk_base); +struct clk_gate meson8b_clk81 = { + .reg = (void *)MESON8B_REG_HHI_MPEG, + .bit_idx = 7, + .lock = &clk_lock, + .hw.init = &(struct clk_init_data){ + .name = "clk81", + .ops = &clk_gate_ops, + .parent_names = (const char *[]){ "mpeg_clk_div" }, + .num_parents = 1, + .flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED), + }, +}; + +static struct clk_hw_onecell_data meson8b_hw_onecell_data = { + .hws = { + [CLKID_XTAL] = &meson8b_xtal.hw, + [CLKID_PLL_FIXED] = &meson8b_fixed_pll.hw, + [CLKID_PLL_VID] = &meson8b_vid_pll.hw, + [CLKID_PLL_SYS] = &meson8b_sys_pll.hw, + [CLKID_FCLK_DIV2] = &meson8b_fclk_div2.hw, + [CLKID_FCLK_DIV3] = &meson8b_fclk_div3.hw, + [CLKID_FCLK_DIV4] = &meson8b_fclk_div4.hw, + [CLKID_FCLK_DIV5] = &meson8b_fclk_div5.hw, + [CLKID_FCLK_DIV7] = &meson8b_fclk_div7.hw, + [CLKID_CPUCLK] = &meson8b_cpu_clk.hw, + [CLKID_MPEG_SEL] = &meson8b_mpeg_clk_sel.hw, + [CLKID_MPEG_DIV] = &meson8b_mpeg_clk_div.hw, + [CLKID_CLK81] = &meson8b_clk81.hw, + }, + .num = CLK_NR_CLKS, +}; + +static struct meson_clk_pll *const meson8b_clk_plls[] = { + &meson8b_fixed_pll, + &meson8b_vid_pll, + &meson8b_sys_pll, +}; + +static int meson8b_clkc_probe(struct platform_device *pdev) +{ + void __iomem *clk_base; + int ret, clkid, i; + struct clk_hw *parent_hw; + struct clk *parent_clk; + struct device *dev = &pdev->dev; /* Generic clocks and PLLs */ - clk_base = of_iomap(np, 1); + clk_base = of_iomap(dev->of_node, 1); if (!clk_base) { pr_err("%s: Unable to map clk base\n", __func__); - return; + return -ENXIO; + } + + /* Populate base address for PLLs */ + for (i = 0; i < ARRAY_SIZE(meson8b_clk_plls); i++) + meson8b_clk_plls[i]->base = clk_base; + + /* Populate the base address for CPU clk */ + meson8b_cpu_clk.base = clk_base; + + /* Populate the base address for the MPEG clks */ + meson8b_mpeg_clk_sel.reg = clk_base + (u32)meson8b_mpeg_clk_sel.reg; + meson8b_mpeg_clk_div.reg = clk_base + (u32)meson8b_mpeg_clk_div.reg; + meson8b_clk81.reg = clk_base + (u32)meson8b_clk81.reg; + + /* + * register all clks + * CLKID_UNUSED = 0, so skip it and start with CLKID_XTAL = 1 + */ + for (clkid = CLKID_XTAL; clkid < CLK_NR_CLKS; clkid++) { + /* array might be sparse */ + if (!meson8b_hw_onecell_data.hws[clkid]) + continue; + + /* FIXME convert to devm_clk_register */ + ret = devm_clk_hw_register(dev, meson8b_hw_onecell_data.hws[clkid]); + if (ret) + goto iounmap; } - meson_clk_register_clks(meson8b_clk_confs, - ARRAY_SIZE(meson8b_clk_confs), - clk_base); + /* + * Register CPU clk notifier + * + * FIXME this is wrong for a lot of reasons. First, the muxes should be + * struct clk_hw objects. Second, we shouldn't program the muxes in + * notifier handlers. The tricky programming sequence will be handled + * by the forthcoming coordinated clock rates mechanism once that + * feature is released. + * + * Furthermore, looking up the parent this way is terrible. At some + * point we will stop allocating a default struct clk when registering + * a new clk_hw, and this hack will no longer work. Releasing the ccr + * feature before that time solves the problem :-) + */ + parent_hw = clk_hw_get_parent(&meson8b_cpu_clk.hw); + parent_clk = parent_hw->clk; + ret = clk_notifier_register(parent_clk, &meson8b_cpu_clk.clk_nb); + if (ret) { + pr_err("%s: failed to register clock notifier for cpu_clk\n", + __func__); + goto iounmap; + } + + return of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get, + &meson8b_hw_onecell_data); + +iounmap: + iounmap(clk_base); + return ret; +} + +static const struct of_device_id meson8b_clkc_match_table[] = { + { .compatible = "amlogic,meson8b-clkc" }, + { } +}; + +static struct platform_driver meson8b_driver = { + .probe = meson8b_clkc_probe, + .driver = { + .name = "meson8b-clkc", + .of_match_table = meson8b_clkc_match_table, + }, +}; + +static int __init meson8b_clkc_init(void) +{ + return platform_driver_register(&meson8b_driver); } -CLK_OF_DECLARE(meson8b_clock, "amlogic,meson8b-clkc", meson8b_clkc_init); +device_initcall(meson8b_clkc_init); diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c index 481b2646b..90d740a2f 100644 --- a/drivers/clk/nxp/clk-lpc32xx.c +++ b/drivers/clk/nxp/clk-lpc32xx.c @@ -87,7 +87,7 @@ enum { enum { /* Start from the last defined clock in dt bindings */ - LPC32XX_CLK_ADC_DIV = LPC32XX_CLK_HCLK_PLL + 1, + LPC32XX_CLK_ADC_DIV = LPC32XX_CLK_PERIPH + 1, LPC32XX_CLK_ADC_RTC, LPC32XX_CLK_TEST1, LPC32XX_CLK_TEST2, @@ -99,7 +99,6 @@ enum { LPC32XX_CLK_HCLK_DIV_PERIPH, LPC32XX_CLK_HCLK_DIV, LPC32XX_CLK_HCLK, - LPC32XX_CLK_PERIPH, LPC32XX_CLK_ARM, LPC32XX_CLK_ARM_VFP, diff --git a/drivers/clk/qcom/gcc-msm8660.c b/drivers/clk/qcom/gcc-msm8660.c index 6dc558649..c347a0d44 100644 --- a/drivers/clk/qcom/gcc-msm8660.c +++ b/drivers/clk/qcom/gcc-msm8660.c @@ -2290,6 +2290,32 @@ static struct clk_branch sdc5_h_clk = { }, }; +static struct clk_branch ebi2_2x_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 18, + .clkr = { + .enable_reg = 0x2660, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "ebi2_2x_clk", + .ops = &clk_branch_ops, + }, + }, +}; + +static struct clk_branch ebi2_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 19, + .clkr = { + .enable_reg = 0x2664, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "ebi2_clk", + .ops = &clk_branch_ops, + }, + }, +}; + static struct clk_branch adm0_clk = { .halt_reg = 0x2fdc, .halt_check = BRANCH_HALT_VOTED, @@ -2533,6 +2559,8 @@ static struct clk_regmap *gcc_msm8660_clks[] = { [SDC3_H_CLK] = &sdc3_h_clk.clkr, [SDC4_H_CLK] = &sdc4_h_clk.clkr, [SDC5_H_CLK] = &sdc5_h_clk.clkr, + [EBI2_2X_CLK] = &ebi2_2x_clk.clkr, + [EBI2_CLK] = &ebi2_clk.clkr, [ADM0_CLK] = &adm0_clk.clkr, [ADM0_PBUS_CLK] = &adm0_pbus_clk.clkr, [ADM1_CLK] = &adm1_clk.clkr, diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c index c9b96f318..bbf732bbc 100644 --- a/drivers/clk/qcom/gcc-msm8996.c +++ b/drivers/clk/qcom/gcc-msm8996.c @@ -2891,21 +2891,6 @@ static struct clk_branch gcc_smmu_aggre0_ahb_clk = { }, }; -static struct clk_branch gcc_aggre1_pnoc_ahb_clk = { - .halt_reg = 0x82014, - .clkr = { - .enable_reg = 0x82014, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gcc_aggre1_pnoc_ahb_clk", - .parent_names = (const char *[]){ "periph_noc_clk_src" }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_aggre2_ufs_axi_clk = { .halt_reg = 0x83014, .clkr = { @@ -3308,7 +3293,6 @@ static struct clk_regmap *gcc_msm8996_clocks[] = { [GCC_AGGRE0_CNOC_AHB_CLK] = &gcc_aggre0_cnoc_ahb_clk.clkr, [GCC_SMMU_AGGRE0_AXI_CLK] = &gcc_smmu_aggre0_axi_clk.clkr, [GCC_SMMU_AGGRE0_AHB_CLK] = &gcc_smmu_aggre0_ahb_clk.clkr, - [GCC_AGGRE1_PNOC_AHB_CLK] = &gcc_aggre1_pnoc_ahb_clk.clkr, [GCC_AGGRE2_UFS_AXI_CLK] = &gcc_aggre2_ufs_axi_clk.clkr, [GCC_AGGRE2_USB3_AXI_CLK] = &gcc_aggre2_usb3_axi_clk.clkr, [GCC_QSPI_AHB_CLK] = &gcc_qspi_ahb_clk.clkr, diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig index 2115ce410..41a12d376 100644 --- a/drivers/clk/renesas/Kconfig +++ b/drivers/clk/renesas/Kconfig @@ -1,6 +1,7 @@ config CLK_RENESAS_CPG_MSSR bool default y if ARCH_R8A7795 + default y if ARCH_R8A7796 config CLK_RENESAS_CPG_MSTP bool @@ -11,6 +12,7 @@ config CLK_RENESAS_CPG_MSTP default y if ARCH_R8A7779 default y if ARCH_R8A7790 default y if ARCH_R8A7791 + default y if ARCH_R8A7792 default y if ARCH_R8A7793 default y if ARCH_R8A7794 default y if ARCH_SH73A0 diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile index ead8bb843..90dd0db7d 100644 --- a/drivers/clk/renesas/Makefile +++ b/drivers/clk/renesas/Makefile @@ -6,9 +6,11 @@ obj-$(CONFIG_ARCH_R8A7778) += clk-r8a7778.o obj-$(CONFIG_ARCH_R8A7779) += clk-r8a7779.o obj-$(CONFIG_ARCH_R8A7790) += clk-rcar-gen2.o clk-div6.o obj-$(CONFIG_ARCH_R8A7791) += clk-rcar-gen2.o clk-div6.o +obj-$(CONFIG_ARCH_R8A7792) += clk-rcar-gen2.o clk-div6.o obj-$(CONFIG_ARCH_R8A7793) += clk-rcar-gen2.o clk-div6.o obj-$(CONFIG_ARCH_R8A7794) += clk-rcar-gen2.o clk-div6.o -obj-$(CONFIG_ARCH_R8A7795) += r8a7795-cpg-mssr.o +obj-$(CONFIG_ARCH_R8A7795) += r8a7795-cpg-mssr.o rcar-gen3-cpg.o +obj-$(CONFIG_ARCH_R8A7796) += r8a7796-cpg-mssr.o rcar-gen3-cpg.o obj-$(CONFIG_ARCH_SH73A0) += clk-sh73a0.o clk-div6.o obj-$(CONFIG_CLK_RENESAS_CPG_MSSR) += renesas-cpg-mssr.o clk-div6.o diff --git a/drivers/clk/renesas/r8a7795-cpg-mssr.c b/drivers/clk/renesas/r8a7795-cpg-mssr.c index e7b98c4d4..e38bf60c0 100644 --- a/drivers/clk/renesas/r8a7795-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7795-cpg-mssr.c @@ -12,22 +12,14 @@ * the Free Software Foundation; version 2 of the License. */ -#include -#include -#include #include -#include #include -#include #include -#include -#include #include #include "renesas-cpg-mssr.h" - -#define CPG_RCKCR 0x240 +#include "rcar-gen3-cpg.h" enum clk_ids { /* Core Clock Outputs exported to DT */ @@ -58,20 +50,6 @@ enum clk_ids { MOD_CLK_BASE }; -enum r8a7795_clk_types { - CLK_TYPE_GEN3_MAIN = CLK_TYPE_CUSTOM, - CLK_TYPE_GEN3_PLL0, - CLK_TYPE_GEN3_PLL1, - CLK_TYPE_GEN3_PLL2, - CLK_TYPE_GEN3_PLL3, - CLK_TYPE_GEN3_PLL4, - CLK_TYPE_GEN3_SD, - CLK_TYPE_GEN3_R, -}; - -#define DEF_GEN3_SD(_name, _id, _parent, _offset) \ - DEF_BASE(_name, _id, CLK_TYPE_GEN3_SD, _parent, .offset = _offset) - static const struct cpg_core_clk r8a7795_core_clks[] __initconst = { /* External Clock Inputs */ DEF_INPUT("extal", CLK_EXTAL), @@ -130,6 +108,9 @@ static const struct cpg_core_clk r8a7795_core_clks[] __initconst = { }; static const struct mssr_mod_clk r8a7795_mod_clks[] __initconst = { + DEF_MOD("fdp1-2", 117, R8A7795_CLK_S2D1), + DEF_MOD("fdp1-1", 118, R8A7795_CLK_S2D1), + DEF_MOD("fdp1-0", 119, R8A7795_CLK_S2D1), DEF_MOD("scif5", 202, R8A7795_CLK_S3D4), DEF_MOD("scif4", 203, R8A7795_CLK_S3D4), DEF_MOD("scif3", 204, R8A7795_CLK_S3D4), @@ -158,11 +139,20 @@ static const struct mssr_mod_clk r8a7795_mod_clks[] __initconst = { DEF_MOD("intc-ap", 408, R8A7795_CLK_S3D1), DEF_MOD("audmac0", 502, R8A7795_CLK_S3D4), DEF_MOD("audmac1", 501, R8A7795_CLK_S3D4), + DEF_MOD("drif7", 508, R8A7795_CLK_S3D2), + DEF_MOD("drif6", 509, R8A7795_CLK_S3D2), + DEF_MOD("drif5", 510, R8A7795_CLK_S3D2), + DEF_MOD("drif4", 511, R8A7795_CLK_S3D2), + DEF_MOD("drif3", 512, R8A7795_CLK_S3D2), + DEF_MOD("drif2", 513, R8A7795_CLK_S3D2), + DEF_MOD("drif1", 514, R8A7795_CLK_S3D2), + DEF_MOD("drif0", 515, R8A7795_CLK_S3D2), DEF_MOD("hscif4", 516, R8A7795_CLK_S3D1), DEF_MOD("hscif3", 517, R8A7795_CLK_S3D1), DEF_MOD("hscif2", 518, R8A7795_CLK_S3D1), DEF_MOD("hscif1", 519, R8A7795_CLK_S3D1), DEF_MOD("hscif0", 520, R8A7795_CLK_S3D1), + DEF_MOD("thermal", 522, R8A7795_CLK_CP), DEF_MOD("pwm", 523, R8A7795_CLK_S3D4), DEF_MOD("fcpvd3", 600, R8A7795_CLK_S2D1), DEF_MOD("fcpvd2", 601, R8A7795_CLK_S2D1), @@ -200,7 +190,7 @@ static const struct mssr_mod_clk r8a7795_mod_clks[] __initconst = { DEF_MOD("du2", 722, R8A7795_CLK_S2D1), DEF_MOD("du1", 723, R8A7795_CLK_S2D1), DEF_MOD("du0", 724, R8A7795_CLK_S2D1), - DEF_MOD("lvds", 727, R8A7795_CLK_S2D1), + DEF_MOD("lvds", 727, R8A7795_CLK_S0D4), DEF_MOD("hdmi1", 728, R8A7795_CLK_HDMI), DEF_MOD("hdmi0", 729, R8A7795_CLK_HDMI), DEF_MOD("vin7", 804, R8A7795_CLK_S2D1), @@ -263,225 +253,6 @@ static const unsigned int r8a7795_crit_mod_clks[] __initconst = { MOD_CLK_ID(408), /* INTC-AP (GIC) */ }; -/* ----------------------------------------------------------------------------- - * SDn Clock - * - */ -#define CPG_SD_STP_HCK BIT(9) -#define CPG_SD_STP_CK BIT(8) - -#define CPG_SD_STP_MASK (CPG_SD_STP_HCK | CPG_SD_STP_CK) -#define CPG_SD_FC_MASK (0x7 << 2 | 0x3 << 0) - -#define CPG_SD_DIV_TABLE_DATA(stp_hck, stp_ck, sd_srcfc, sd_fc, sd_div) \ -{ \ - .val = ((stp_hck) ? CPG_SD_STP_HCK : 0) | \ - ((stp_ck) ? CPG_SD_STP_CK : 0) | \ - ((sd_srcfc) << 2) | \ - ((sd_fc) << 0), \ - .div = (sd_div), \ -} - -struct sd_div_table { - u32 val; - unsigned int div; -}; - -struct sd_clock { - struct clk_hw hw; - void __iomem *reg; - const struct sd_div_table *div_table; - unsigned int div_num; - unsigned int div_min; - unsigned int div_max; -}; - -/* SDn divider - * sd_srcfc sd_fc div - * stp_hck stp_ck (div) (div) = sd_srcfc x sd_fc - *------------------------------------------------------------------- - * 0 0 0 (1) 1 (4) 4 - * 0 0 1 (2) 1 (4) 8 - * 1 0 2 (4) 1 (4) 16 - * 1 0 3 (8) 1 (4) 32 - * 1 0 4 (16) 1 (4) 64 - * 0 0 0 (1) 0 (2) 2 - * 0 0 1 (2) 0 (2) 4 - * 1 0 2 (4) 0 (2) 8 - * 1 0 3 (8) 0 (2) 16 - * 1 0 4 (16) 0 (2) 32 - */ -static const struct sd_div_table cpg_sd_div_table[] = { -/* CPG_SD_DIV_TABLE_DATA(stp_hck, stp_ck, sd_srcfc, sd_fc, sd_div) */ - CPG_SD_DIV_TABLE_DATA(0, 0, 0, 1, 4), - CPG_SD_DIV_TABLE_DATA(0, 0, 1, 1, 8), - CPG_SD_DIV_TABLE_DATA(1, 0, 2, 1, 16), - CPG_SD_DIV_TABLE_DATA(1, 0, 3, 1, 32), - CPG_SD_DIV_TABLE_DATA(1, 0, 4, 1, 64), - CPG_SD_DIV_TABLE_DATA(0, 0, 0, 0, 2), - CPG_SD_DIV_TABLE_DATA(0, 0, 1, 0, 4), - CPG_SD_DIV_TABLE_DATA(1, 0, 2, 0, 8), - CPG_SD_DIV_TABLE_DATA(1, 0, 3, 0, 16), - CPG_SD_DIV_TABLE_DATA(1, 0, 4, 0, 32), -}; - -#define to_sd_clock(_hw) container_of(_hw, struct sd_clock, hw) - -static int cpg_sd_clock_enable(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - u32 val, sd_fc; - unsigned int i; - - val = clk_readl(clock->reg); - - sd_fc = val & CPG_SD_FC_MASK; - for (i = 0; i < clock->div_num; i++) - if (sd_fc == (clock->div_table[i].val & CPG_SD_FC_MASK)) - break; - - if (i >= clock->div_num) - return -EINVAL; - - val &= ~(CPG_SD_STP_MASK); - val |= clock->div_table[i].val & CPG_SD_STP_MASK; - - clk_writel(val, clock->reg); - - return 0; -} - -static void cpg_sd_clock_disable(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - - clk_writel(clk_readl(clock->reg) | CPG_SD_STP_MASK, clock->reg); -} - -static int cpg_sd_clock_is_enabled(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - - return !(clk_readl(clock->reg) & CPG_SD_STP_MASK); -} - -static unsigned long cpg_sd_clock_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct sd_clock *clock = to_sd_clock(hw); - unsigned long rate = parent_rate; - u32 val, sd_fc; - unsigned int i; - - val = clk_readl(clock->reg); - - sd_fc = val & CPG_SD_FC_MASK; - for (i = 0; i < clock->div_num; i++) - if (sd_fc == (clock->div_table[i].val & CPG_SD_FC_MASK)) - break; - - if (i >= clock->div_num) - return -EINVAL; - - return DIV_ROUND_CLOSEST(rate, clock->div_table[i].div); -} - -static unsigned int cpg_sd_clock_calc_div(struct sd_clock *clock, - unsigned long rate, - unsigned long parent_rate) -{ - unsigned int div; - - if (!rate) - rate = 1; - - div = DIV_ROUND_CLOSEST(parent_rate, rate); - - return clamp_t(unsigned int, div, clock->div_min, clock->div_max); -} - -static long cpg_sd_clock_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate) -{ - struct sd_clock *clock = to_sd_clock(hw); - unsigned int div = cpg_sd_clock_calc_div(clock, rate, *parent_rate); - - return DIV_ROUND_CLOSEST(*parent_rate, div); -} - -static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct sd_clock *clock = to_sd_clock(hw); - unsigned int div = cpg_sd_clock_calc_div(clock, rate, parent_rate); - u32 val; - unsigned int i; - - for (i = 0; i < clock->div_num; i++) - if (div == clock->div_table[i].div) - break; - - if (i >= clock->div_num) - return -EINVAL; - - val = clk_readl(clock->reg); - val &= ~(CPG_SD_STP_MASK | CPG_SD_FC_MASK); - val |= clock->div_table[i].val & (CPG_SD_STP_MASK | CPG_SD_FC_MASK); - clk_writel(val, clock->reg); - - return 0; -} - -static const struct clk_ops cpg_sd_clock_ops = { - .enable = cpg_sd_clock_enable, - .disable = cpg_sd_clock_disable, - .is_enabled = cpg_sd_clock_is_enabled, - .recalc_rate = cpg_sd_clock_recalc_rate, - .round_rate = cpg_sd_clock_round_rate, - .set_rate = cpg_sd_clock_set_rate, -}; - -static struct clk * __init cpg_sd_clk_register(const struct cpg_core_clk *core, - void __iomem *base, - const char *parent_name) -{ - struct clk_init_data init; - struct sd_clock *clock; - struct clk *clk; - unsigned int i; - - clock = kzalloc(sizeof(*clock), GFP_KERNEL); - if (!clock) - return ERR_PTR(-ENOMEM); - - init.name = core->name; - init.ops = &cpg_sd_clock_ops; - init.flags = CLK_IS_BASIC | CLK_SET_RATE_PARENT; - init.parent_names = &parent_name; - init.num_parents = 1; - - clock->reg = base + core->offset; - clock->hw.init = &init; - clock->div_table = cpg_sd_div_table; - clock->div_num = ARRAY_SIZE(cpg_sd_div_table); - - clock->div_max = clock->div_table[0].div; - clock->div_min = clock->div_max; - for (i = 1; i < clock->div_num; i++) { - clock->div_max = max(clock->div_max, clock->div_table[i].div); - clock->div_min = min(clock->div_min, clock->div_table[i].div); - } - - clk = clk_register(NULL, &clock->hw); - if (IS_ERR(clk)) - kfree(clock); - - return clk; -} - -#define CPG_PLL0CR 0x00d8 -#define CPG_PLL2CR 0x002c -#define CPG_PLL4CR 0x01f4 /* * CPG Clock Data @@ -513,13 +284,7 @@ static struct clk * __init cpg_sd_clk_register(const struct cpg_core_clk *core, (((md) & BIT(19)) >> 18) | \ (((md) & BIT(17)) >> 17)) -struct cpg_pll_config { - unsigned int extal_div; - unsigned int pll1_mult; - unsigned int pll3_mult; -}; - -static const struct cpg_pll_config cpg_pll_configs[16] __initconst = { +static const struct rcar_gen3_cpg_pll_config cpg_pll_configs[16] __initconst = { /* EXTAL div PLL1 mult PLL3 mult */ { 1, 192, 192, }, { 1, 192, 128, }, @@ -539,112 +304,9 @@ static const struct cpg_pll_config cpg_pll_configs[16] __initconst = { { 2, 192, 192, }, }; -static const struct cpg_pll_config *cpg_pll_config __initdata; - -static -struct clk * __init r8a7795_cpg_clk_register(struct device *dev, - const struct cpg_core_clk *core, - const struct cpg_mssr_info *info, - struct clk **clks, - void __iomem *base) -{ - const struct clk *parent; - unsigned int mult = 1; - unsigned int div = 1; - u32 value; - - parent = clks[core->parent]; - if (IS_ERR(parent)) - return ERR_CAST(parent); - - switch (core->type) { - case CLK_TYPE_GEN3_MAIN: - div = cpg_pll_config->extal_div; - break; - - case CLK_TYPE_GEN3_PLL0: - /* - * PLL0 is a configurable multiplier clock. Register it as a - * fixed factor clock for now as there's no generic multiplier - * clock implementation and we currently have no need to change - * the multiplier value. - */ - value = readl(base + CPG_PLL0CR); - mult = (((value >> 24) & 0x7f) + 1) * 2; - break; - - case CLK_TYPE_GEN3_PLL1: - mult = cpg_pll_config->pll1_mult; - break; - - case CLK_TYPE_GEN3_PLL2: - /* - * PLL2 is a configurable multiplier clock. Register it as a - * fixed factor clock for now as there's no generic multiplier - * clock implementation and we currently have no need to change - * the multiplier value. - */ - value = readl(base + CPG_PLL2CR); - mult = (((value >> 24) & 0x7f) + 1) * 2; - break; - - case CLK_TYPE_GEN3_PLL3: - mult = cpg_pll_config->pll3_mult; - break; - - case CLK_TYPE_GEN3_PLL4: - /* - * PLL4 is a configurable multiplier clock. Register it as a - * fixed factor clock for now as there's no generic multiplier - * clock implementation and we currently have no need to change - * the multiplier value. - */ - value = readl(base + CPG_PLL4CR); - mult = (((value >> 24) & 0x7f) + 1) * 2; - break; - - case CLK_TYPE_GEN3_SD: - return cpg_sd_clk_register(core, base, __clk_get_name(parent)); - - case CLK_TYPE_GEN3_R: - /* RINT is default. Only if EXTALR is populated, we switch to it */ - value = readl(base + CPG_RCKCR) & 0x3f; - - if (clk_get_rate(clks[CLK_EXTALR])) { - parent = clks[CLK_EXTALR]; - value |= BIT(15); - } - - writel(value, base + CPG_RCKCR); - break; - - default: - return ERR_PTR(-EINVAL); - } - - return clk_register_fixed_factor(NULL, core->name, - __clk_get_name(parent), 0, mult, div); -} - -/* - * Reset register definitions. - */ -#define MODEMR 0xe6160060 - -static u32 rcar_gen3_read_mode_pins(void) -{ - void __iomem *modemr = ioremap_nocache(MODEMR, 4); - u32 mode; - - BUG_ON(!modemr); - mode = ioread32(modemr); - iounmap(modemr); - - return mode; -} - static int __init r8a7795_cpg_mssr_init(struct device *dev) { + const struct rcar_gen3_cpg_pll_config *cpg_pll_config; u32 cpg_mode = rcar_gen3_read_mode_pins(); cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)]; @@ -653,7 +315,7 @@ static int __init r8a7795_cpg_mssr_init(struct device *dev) return -EINVAL; } - return 0; + return rcar_gen3_cpg_init(cpg_pll_config, CLK_EXTALR); } const struct cpg_mssr_info r8a7795_cpg_mssr_info __initconst = { @@ -674,5 +336,5 @@ const struct cpg_mssr_info r8a7795_cpg_mssr_info __initconst = { /* Callbacks */ .init = r8a7795_cpg_mssr_init, - .cpg_clk_register = r8a7795_cpg_clk_register, + .cpg_clk_register = rcar_gen3_cpg_clk_register, }; diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c new file mode 100644 index 000000000..c84b549c1 --- /dev/null +++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c @@ -0,0 +1,192 @@ +/* + * r8a7796 Clock Pulse Generator / Module Standby and Software Reset + * + * Copyright (C) 2016 Glider bvba + * + * Based on r8a7795-cpg-mssr.c + * + * Copyright (C) 2015 Glider bvba + * Copyright (C) 2015 Renesas Electronics Corp. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#include +#include +#include + +#include + +#include "renesas-cpg-mssr.h" +#include "rcar-gen3-cpg.h" + +enum clk_ids { + /* Core Clock Outputs exported to DT */ + LAST_DT_CORE_CLK = R8A7796_CLK_OSC, + + /* External Input Clocks */ + CLK_EXTAL, + CLK_EXTALR, + + /* Internal Core Clocks */ + CLK_MAIN, + CLK_PLL0, + CLK_PLL1, + CLK_PLL2, + CLK_PLL3, + CLK_PLL4, + CLK_PLL1_DIV2, + CLK_PLL1_DIV4, + CLK_S0, + CLK_S1, + CLK_S2, + CLK_S3, + CLK_SDSRC, + CLK_SSPSRC, + + /* Module Clocks */ + MOD_CLK_BASE +}; + +static const struct cpg_core_clk r8a7796_core_clks[] __initconst = { + /* External Clock Inputs */ + DEF_INPUT("extal", CLK_EXTAL), + DEF_INPUT("extalr", CLK_EXTALR), + + /* Internal Core Clocks */ + DEF_BASE(".main", CLK_MAIN, CLK_TYPE_GEN3_MAIN, CLK_EXTAL), + DEF_BASE(".pll0", CLK_PLL0, CLK_TYPE_GEN3_PLL0, CLK_MAIN), + DEF_BASE(".pll1", CLK_PLL1, CLK_TYPE_GEN3_PLL1, CLK_MAIN), + DEF_BASE(".pll2", CLK_PLL2, CLK_TYPE_GEN3_PLL2, CLK_MAIN), + DEF_BASE(".pll3", CLK_PLL3, CLK_TYPE_GEN3_PLL3, CLK_MAIN), + DEF_BASE(".pll4", CLK_PLL4, CLK_TYPE_GEN3_PLL4, CLK_MAIN), + + DEF_FIXED(".pll1_div2", CLK_PLL1_DIV2, CLK_PLL1, 2, 1), + DEF_FIXED(".pll1_div4", CLK_PLL1_DIV4, CLK_PLL1_DIV2, 2, 1), + DEF_FIXED(".s0", CLK_S0, CLK_PLL1_DIV2, 2, 1), + DEF_FIXED(".s1", CLK_S1, CLK_PLL1_DIV2, 3, 1), + DEF_FIXED(".s2", CLK_S2, CLK_PLL1_DIV2, 4, 1), + DEF_FIXED(".s3", CLK_S3, CLK_PLL1_DIV2, 6, 1), + + /* Core Clock Outputs */ + DEF_FIXED("ztr", R8A7796_CLK_ZTR, CLK_PLL1_DIV2, 6, 1), + DEF_FIXED("ztrd2", R8A7796_CLK_ZTRD2, CLK_PLL1_DIV2, 12, 1), + DEF_FIXED("zt", R8A7796_CLK_ZT, CLK_PLL1_DIV2, 4, 1), + DEF_FIXED("zx", R8A7796_CLK_ZX, CLK_PLL1_DIV2, 2, 1), + DEF_FIXED("s0d1", R8A7796_CLK_S0D1, CLK_S0, 1, 1), + DEF_FIXED("s0d2", R8A7796_CLK_S0D2, CLK_S0, 2, 1), + DEF_FIXED("s0d3", R8A7796_CLK_S0D3, CLK_S0, 3, 1), + DEF_FIXED("s0d4", R8A7796_CLK_S0D4, CLK_S0, 4, 1), + DEF_FIXED("s0d6", R8A7796_CLK_S0D6, CLK_S0, 6, 1), + DEF_FIXED("s0d8", R8A7796_CLK_S0D8, CLK_S0, 8, 1), + DEF_FIXED("s0d12", R8A7796_CLK_S0D12, CLK_S0, 12, 1), + DEF_FIXED("s1d1", R8A7796_CLK_S1D1, CLK_S1, 1, 1), + DEF_FIXED("s1d2", R8A7796_CLK_S1D2, CLK_S1, 2, 1), + DEF_FIXED("s1d4", R8A7796_CLK_S1D4, CLK_S1, 4, 1), + DEF_FIXED("s2d1", R8A7796_CLK_S2D1, CLK_S2, 1, 1), + DEF_FIXED("s2d2", R8A7796_CLK_S2D2, CLK_S2, 2, 1), + DEF_FIXED("s2d4", R8A7796_CLK_S2D4, CLK_S2, 4, 1), + DEF_FIXED("s3d1", R8A7796_CLK_S3D1, CLK_S3, 1, 1), + DEF_FIXED("s3d2", R8A7796_CLK_S3D2, CLK_S3, 2, 1), + DEF_FIXED("s3d4", R8A7796_CLK_S3D4, CLK_S3, 4, 1), + + DEF_FIXED("cl", R8A7796_CLK_CL, CLK_PLL1_DIV2, 48, 1), + DEF_FIXED("cp", R8A7796_CLK_CP, CLK_EXTAL, 2, 1), +}; + +static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = { + DEF_MOD("scif2", 310, R8A7796_CLK_S3D4), + DEF_MOD("intc-ap", 408, R8A7796_CLK_S3D1), +}; + +static const unsigned int r8a7796_crit_mod_clks[] __initconst = { + MOD_CLK_ID(408), /* INTC-AP (GIC) */ +}; + + +/* + * CPG Clock Data + */ + +/* + * MD EXTAL PLL0 PLL1 PLL2 PLL3 PLL4 + * 14 13 19 17 (MHz) + *------------------------------------------------------------------- + * 0 0 0 0 16.66 x 1 x180 x192 x144 x192 x144 + * 0 0 0 1 16.66 x 1 x180 x192 x144 x128 x144 + * 0 0 1 0 Prohibited setting + * 0 0 1 1 16.66 x 1 x180 x192 x144 x192 x144 + * 0 1 0 0 20 x 1 x150 x160 x120 x160 x120 + * 0 1 0 1 20 x 1 x150 x160 x120 x106 x120 + * 0 1 1 0 Prohibited setting + * 0 1 1 1 20 x 1 x150 x160 x120 x160 x120 + * 1 0 0 0 25 x 1 x120 x128 x96 x128 x96 + * 1 0 0 1 25 x 1 x120 x128 x96 x84 x96 + * 1 0 1 0 Prohibited setting + * 1 0 1 1 25 x 1 x120 x128 x96 x128 x96 + * 1 1 0 0 33.33 / 2 x180 x192 x144 x192 x144 + * 1 1 0 1 33.33 / 2 x180 x192 x144 x128 x144 + * 1 1 1 0 Prohibited setting + * 1 1 1 1 33.33 / 2 x180 x192 x144 x192 x144 + */ +#define CPG_PLL_CONFIG_INDEX(md) ((((md) & BIT(14)) >> 11) | \ + (((md) & BIT(13)) >> 11) | \ + (((md) & BIT(19)) >> 18) | \ + (((md) & BIT(17)) >> 17)) + +static const struct rcar_gen3_cpg_pll_config cpg_pll_configs[16] __initconst = { + /* EXTAL div PLL1 mult PLL3 mult */ + { 1, 192, 192, }, + { 1, 192, 128, }, + { 0, /* Prohibited setting */ }, + { 1, 192, 192, }, + { 1, 160, 160, }, + { 1, 160, 106, }, + { 0, /* Prohibited setting */ }, + { 1, 160, 160, }, + { 1, 128, 128, }, + { 1, 128, 84, }, + { 0, /* Prohibited setting */ }, + { 1, 128, 128, }, + { 2, 192, 192, }, + { 2, 192, 128, }, + { 0, /* Prohibited setting */ }, + { 2, 192, 192, }, +}; + +static int __init r8a7796_cpg_mssr_init(struct device *dev) +{ + const struct rcar_gen3_cpg_pll_config *cpg_pll_config; + u32 cpg_mode = rcar_gen3_read_mode_pins(); + + cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)]; + if (!cpg_pll_config->extal_div) { + dev_err(dev, "Prohibited setting (cpg_mode=0x%x)\n", cpg_mode); + return -EINVAL; + } + + return rcar_gen3_cpg_init(cpg_pll_config, CLK_EXTALR); +} + +const struct cpg_mssr_info r8a7796_cpg_mssr_info __initconst = { + /* Core Clocks */ + .core_clks = r8a7796_core_clks, + .num_core_clks = ARRAY_SIZE(r8a7796_core_clks), + .last_dt_core_clk = LAST_DT_CORE_CLK, + .num_total_core_clks = MOD_CLK_BASE, + + /* Module Clocks */ + .mod_clks = r8a7796_mod_clks, + .num_mod_clks = ARRAY_SIZE(r8a7796_mod_clks), + .num_hw_mod_clks = 12 * 32, + + /* Critical Module Clocks */ + .crit_mod_clks = r8a7796_crit_mod_clks, + .num_crit_mod_clks = ARRAY_SIZE(r8a7796_crit_mod_clks), + + /* Callbacks */ + .init = r8a7796_cpg_mssr_init, + .cpg_clk_register = rcar_gen3_cpg_clk_register, +}; diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c new file mode 100644 index 000000000..bb4f2f9a8 --- /dev/null +++ b/drivers/clk/renesas/rcar-gen3-cpg.c @@ -0,0 +1,359 @@ +/* + * R-Car Gen3 Clock Pulse Generator + * + * Copyright (C) 2015-2016 Glider bvba + * + * Based on clk-rcar-gen3.c + * + * Copyright (C) 2015 Renesas Electronics Corp. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "renesas-cpg-mssr.h" +#include "rcar-gen3-cpg.h" + +#define CPG_PLL0CR 0x00d8 +#define CPG_PLL2CR 0x002c +#define CPG_PLL4CR 0x01f4 + + +/* + * SDn Clock + */ +#define CPG_SD_STP_HCK BIT(9) +#define CPG_SD_STP_CK BIT(8) + +#define CPG_SD_STP_MASK (CPG_SD_STP_HCK | CPG_SD_STP_CK) +#define CPG_SD_FC_MASK (0x7 << 2 | 0x3 << 0) + +#define CPG_SD_DIV_TABLE_DATA(stp_hck, stp_ck, sd_srcfc, sd_fc, sd_div) \ +{ \ + .val = ((stp_hck) ? CPG_SD_STP_HCK : 0) | \ + ((stp_ck) ? CPG_SD_STP_CK : 0) | \ + ((sd_srcfc) << 2) | \ + ((sd_fc) << 0), \ + .div = (sd_div), \ +} + +struct sd_div_table { + u32 val; + unsigned int div; +}; + +struct sd_clock { + struct clk_hw hw; + void __iomem *reg; + const struct sd_div_table *div_table; + unsigned int div_num; + unsigned int div_min; + unsigned int div_max; +}; + +/* SDn divider + * sd_srcfc sd_fc div + * stp_hck stp_ck (div) (div) = sd_srcfc x sd_fc + *------------------------------------------------------------------- + * 0 0 0 (1) 1 (4) 4 + * 0 0 1 (2) 1 (4) 8 + * 1 0 2 (4) 1 (4) 16 + * 1 0 3 (8) 1 (4) 32 + * 1 0 4 (16) 1 (4) 64 + * 0 0 0 (1) 0 (2) 2 + * 0 0 1 (2) 0 (2) 4 + * 1 0 2 (4) 0 (2) 8 + * 1 0 3 (8) 0 (2) 16 + * 1 0 4 (16) 0 (2) 32 + */ +static const struct sd_div_table cpg_sd_div_table[] = { +/* CPG_SD_DIV_TABLE_DATA(stp_hck, stp_ck, sd_srcfc, sd_fc, sd_div) */ + CPG_SD_DIV_TABLE_DATA(0, 0, 0, 1, 4), + CPG_SD_DIV_TABLE_DATA(0, 0, 1, 1, 8), + CPG_SD_DIV_TABLE_DATA(1, 0, 2, 1, 16), + CPG_SD_DIV_TABLE_DATA(1, 0, 3, 1, 32), + CPG_SD_DIV_TABLE_DATA(1, 0, 4, 1, 64), + CPG_SD_DIV_TABLE_DATA(0, 0, 0, 0, 2), + CPG_SD_DIV_TABLE_DATA(0, 0, 1, 0, 4), + CPG_SD_DIV_TABLE_DATA(1, 0, 2, 0, 8), + CPG_SD_DIV_TABLE_DATA(1, 0, 3, 0, 16), + CPG_SD_DIV_TABLE_DATA(1, 0, 4, 0, 32), +}; + +#define to_sd_clock(_hw) container_of(_hw, struct sd_clock, hw) + +static int cpg_sd_clock_enable(struct clk_hw *hw) +{ + struct sd_clock *clock = to_sd_clock(hw); + u32 val, sd_fc; + unsigned int i; + + val = clk_readl(clock->reg); + + sd_fc = val & CPG_SD_FC_MASK; + for (i = 0; i < clock->div_num; i++) + if (sd_fc == (clock->div_table[i].val & CPG_SD_FC_MASK)) + break; + + if (i >= clock->div_num) + return -EINVAL; + + val &= ~(CPG_SD_STP_MASK); + val |= clock->div_table[i].val & CPG_SD_STP_MASK; + + clk_writel(val, clock->reg); + + return 0; +} + +static void cpg_sd_clock_disable(struct clk_hw *hw) +{ + struct sd_clock *clock = to_sd_clock(hw); + + clk_writel(clk_readl(clock->reg) | CPG_SD_STP_MASK, clock->reg); +} + +static int cpg_sd_clock_is_enabled(struct clk_hw *hw) +{ + struct sd_clock *clock = to_sd_clock(hw); + + return !(clk_readl(clock->reg) & CPG_SD_STP_MASK); +} + +static unsigned long cpg_sd_clock_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct sd_clock *clock = to_sd_clock(hw); + unsigned long rate = parent_rate; + u32 val, sd_fc; + unsigned int i; + + val = clk_readl(clock->reg); + + sd_fc = val & CPG_SD_FC_MASK; + for (i = 0; i < clock->div_num; i++) + if (sd_fc == (clock->div_table[i].val & CPG_SD_FC_MASK)) + break; + + if (i >= clock->div_num) + return -EINVAL; + + return DIV_ROUND_CLOSEST(rate, clock->div_table[i].div); +} + +static unsigned int cpg_sd_clock_calc_div(struct sd_clock *clock, + unsigned long rate, + unsigned long parent_rate) +{ + unsigned int div; + + if (!rate) + rate = 1; + + div = DIV_ROUND_CLOSEST(parent_rate, rate); + + return clamp_t(unsigned int, div, clock->div_min, clock->div_max); +} + +static long cpg_sd_clock_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + struct sd_clock *clock = to_sd_clock(hw); + unsigned int div = cpg_sd_clock_calc_div(clock, rate, *parent_rate); + + return DIV_ROUND_CLOSEST(*parent_rate, div); +} + +static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct sd_clock *clock = to_sd_clock(hw); + unsigned int div = cpg_sd_clock_calc_div(clock, rate, parent_rate); + u32 val; + unsigned int i; + + for (i = 0; i < clock->div_num; i++) + if (div == clock->div_table[i].div) + break; + + if (i >= clock->div_num) + return -EINVAL; + + val = clk_readl(clock->reg); + val &= ~(CPG_SD_STP_MASK | CPG_SD_FC_MASK); + val |= clock->div_table[i].val & (CPG_SD_STP_MASK | CPG_SD_FC_MASK); + clk_writel(val, clock->reg); + + return 0; +} + +static const struct clk_ops cpg_sd_clock_ops = { + .enable = cpg_sd_clock_enable, + .disable = cpg_sd_clock_disable, + .is_enabled = cpg_sd_clock_is_enabled, + .recalc_rate = cpg_sd_clock_recalc_rate, + .round_rate = cpg_sd_clock_round_rate, + .set_rate = cpg_sd_clock_set_rate, +}; + +static struct clk * __init cpg_sd_clk_register(const struct cpg_core_clk *core, + void __iomem *base, + const char *parent_name) +{ + struct clk_init_data init; + struct sd_clock *clock; + struct clk *clk; + unsigned int i; + + clock = kzalloc(sizeof(*clock), GFP_KERNEL); + if (!clock) + return ERR_PTR(-ENOMEM); + + init.name = core->name; + init.ops = &cpg_sd_clock_ops; + init.flags = CLK_IS_BASIC | CLK_SET_RATE_PARENT; + init.parent_names = &parent_name; + init.num_parents = 1; + + clock->reg = base + core->offset; + clock->hw.init = &init; + clock->div_table = cpg_sd_div_table; + clock->div_num = ARRAY_SIZE(cpg_sd_div_table); + + clock->div_max = clock->div_table[0].div; + clock->div_min = clock->div_max; + for (i = 1; i < clock->div_num; i++) { + clock->div_max = max(clock->div_max, clock->div_table[i].div); + clock->div_min = min(clock->div_min, clock->div_table[i].div); + } + + clk = clk_register(NULL, &clock->hw); + if (IS_ERR(clk)) + kfree(clock); + + return clk; +} + + +static const struct rcar_gen3_cpg_pll_config *cpg_pll_config __initdata; +static unsigned int cpg_clk_extalr __initdata; + +struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev, + const struct cpg_core_clk *core, const struct cpg_mssr_info *info, + struct clk **clks, void __iomem *base) +{ + const struct clk *parent; + unsigned int mult = 1; + unsigned int div = 1; + u32 value; + + parent = clks[core->parent]; + if (IS_ERR(parent)) + return ERR_CAST(parent); + + switch (core->type) { + case CLK_TYPE_GEN3_MAIN: + div = cpg_pll_config->extal_div; + break; + + case CLK_TYPE_GEN3_PLL0: + /* + * PLL0 is a configurable multiplier clock. Register it as a + * fixed factor clock for now as there's no generic multiplier + * clock implementation and we currently have no need to change + * the multiplier value. + */ + value = readl(base + CPG_PLL0CR); + mult = (((value >> 24) & 0x7f) + 1) * 2; + break; + + case CLK_TYPE_GEN3_PLL1: + mult = cpg_pll_config->pll1_mult; + break; + + case CLK_TYPE_GEN3_PLL2: + /* + * PLL2 is a configurable multiplier clock. Register it as a + * fixed factor clock for now as there's no generic multiplier + * clock implementation and we currently have no need to change + * the multiplier value. + */ + value = readl(base + CPG_PLL2CR); + mult = (((value >> 24) & 0x7f) + 1) * 2; + break; + + case CLK_TYPE_GEN3_PLL3: + mult = cpg_pll_config->pll3_mult; + break; + + case CLK_TYPE_GEN3_PLL4: + /* + * PLL4 is a configurable multiplier clock. Register it as a + * fixed factor clock for now as there's no generic multiplier + * clock implementation and we currently have no need to change + * the multiplier value. + */ + value = readl(base + CPG_PLL4CR); + mult = (((value >> 24) & 0x7f) + 1) * 2; + break; + + case CLK_TYPE_GEN3_SD: + return cpg_sd_clk_register(core, base, __clk_get_name(parent)); + + case CLK_TYPE_GEN3_R: + /* + * RINT is default. + * Only if EXTALR is populated, we switch to it. + */ + value = readl(base + CPG_RCKCR) & 0x3f; + + if (clk_get_rate(clks[cpg_clk_extalr])) { + parent = clks[cpg_clk_extalr]; + value |= BIT(15); + } + + writel(value, base + CPG_RCKCR); + break; + + default: + return ERR_PTR(-EINVAL); + } + + return clk_register_fixed_factor(NULL, core->name, + __clk_get_name(parent), 0, mult, div); +} + +/* + * Reset register definitions. + */ +#define MODEMR 0xe6160060 + +u32 __init rcar_gen3_read_mode_pins(void) +{ + void __iomem *modemr = ioremap_nocache(MODEMR, 4); + u32 mode; + + BUG_ON(!modemr); + mode = ioread32(modemr); + iounmap(modemr); + + return mode; +} + +int __init rcar_gen3_cpg_init(const struct rcar_gen3_cpg_pll_config *config, + unsigned int clk_extalr) +{ + cpg_pll_config = config; + cpg_clk_extalr = clk_extalr; + return 0; +} diff --git a/drivers/clk/renesas/rcar-gen3-cpg.h b/drivers/clk/renesas/rcar-gen3-cpg.h new file mode 100644 index 000000000..f69908514 --- /dev/null +++ b/drivers/clk/renesas/rcar-gen3-cpg.h @@ -0,0 +1,43 @@ +/* + * R-Car Gen3 Clock Pulse Generator + * + * Copyright (C) 2015-2016 Glider bvba + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#ifndef __CLK_RENESAS_RCAR_GEN3_CPG_H__ +#define __CLK_RENESAS_RCAR_GEN3_CPG_H__ + +enum rcar_gen3_clk_types { + CLK_TYPE_GEN3_MAIN = CLK_TYPE_CUSTOM, + CLK_TYPE_GEN3_PLL0, + CLK_TYPE_GEN3_PLL1, + CLK_TYPE_GEN3_PLL2, + CLK_TYPE_GEN3_PLL3, + CLK_TYPE_GEN3_PLL4, + CLK_TYPE_GEN3_SD, + CLK_TYPE_GEN3_R, +}; + +#define DEF_GEN3_SD(_name, _id, _parent, _offset) \ + DEF_BASE(_name, _id, CLK_TYPE_GEN3_SD, _parent, .offset = _offset) + +struct rcar_gen3_cpg_pll_config { + unsigned int extal_div; + unsigned int pll1_mult; + unsigned int pll3_mult; +}; + +#define CPG_RCKCR 0x240 + +u32 rcar_gen3_read_mode_pins(void); +struct clk *rcar_gen3_cpg_clk_register(struct device *dev, + const struct cpg_core_clk *core, const struct cpg_mssr_info *info, + struct clk **clks, void __iomem *base); +int rcar_gen3_cpg_init(const struct rcar_gen3_cpg_pll_config *config, + unsigned int clk_extalr); + +#endif diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c index 210cd744a..e1365e749 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.c +++ b/drivers/clk/renesas/renesas-cpg-mssr.c @@ -508,6 +508,12 @@ static const struct of_device_id cpg_mssr_match[] = { .compatible = "renesas,r8a7795-cpg-mssr", .data = &r8a7795_cpg_mssr_info, }, +#endif +#ifdef CONFIG_ARCH_R8A7796 + { + .compatible = "renesas,r8a7796-cpg-mssr", + .data = &r8a7796_cpg_mssr_info, + }, #endif { /* sentinel */ } }; diff --git a/drivers/clk/renesas/renesas-cpg-mssr.h b/drivers/clk/renesas/renesas-cpg-mssr.h index 0d1e3e811..ee7edfaf1 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.h +++ b/drivers/clk/renesas/renesas-cpg-mssr.h @@ -131,4 +131,5 @@ struct cpg_mssr_info { }; extern const struct cpg_mssr_info r8a7795_cpg_mssr_info; +extern const struct cpg_mssr_info r8a7796_cpg_mssr_info; #endif diff --git a/drivers/clk/rockchip/clk-rk3228.c b/drivers/clk/rockchip/clk-rk3228.c index 016bdb0b7..db6e5a9e6 100644 --- a/drivers/clk/rockchip/clk-rk3228.c +++ b/drivers/clk/rockchip/clk-rk3228.c @@ -151,8 +151,8 @@ PNAME(mux_uart0_p) = { "uart0_src", "uart0_frac", "xin24m" }; PNAME(mux_uart1_p) = { "uart1_src", "uart1_frac", "xin24m" }; PNAME(mux_uart2_p) = { "uart2_src", "uart2_frac", "xin24m" }; -PNAME(mux_sclk_macphy_50m_p) = { "ext_gmac", "phy_50m_out" }; -PNAME(mux_sclk_gmac_pre_p) = { "sclk_gmac_src", "sclk_macphy_50m" }; +PNAME(mux_sclk_mac_extclk_p) = { "ext_gmac", "phy_50m_out" }; +PNAME(mux_sclk_gmac_pre_p) = { "sclk_gmac_src", "sclk_mac_extclk" }; PNAME(mux_sclk_macphy_p) = { "sclk_gmac_src", "ext_gmac" }; static struct rockchip_pll_clock rk3228_pll_clks[] __initdata = { @@ -170,6 +170,34 @@ static struct rockchip_pll_clock rk3228_pll_clks[] __initdata = { #define DFLAGS CLK_DIVIDER_HIWORD_MASK #define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE) +static struct rockchip_clk_branch rk3228_i2s0_fracmux __initdata = + MUX(0, "i2s0_pre", mux_i2s0_p, CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(9), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rk3228_i2s1_fracmux __initdata = + MUX(0, "i2s1_pre", mux_i2s1_pre_p, CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(3), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rk3228_i2s2_fracmux __initdata = + MUX(0, "i2s2_pre", mux_i2s2_p, CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(16), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rk3228_spdif_fracmux __initdata = + MUX(SCLK_SPDIF, "sclk_spdif", mux_sclk_spdif_p, CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(6), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rk3228_uart0_fracmux __initdata = + MUX(SCLK_UART0, "sclk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(13), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rk3228_uart1_fracmux __initdata = + MUX(SCLK_UART1, "sclk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(14), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rk3228_uart2_fracmux __initdata = + MUX(SCLK_UART2, "sclk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(15), 8, 2, MFLAGS); + static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { /* * Clock-Architecture Diagram 1 @@ -335,7 +363,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { RK2928_CLKGATE_CON(2), 6, GFLAGS), GATE(0, "sclk_hsadc", "ext_hsadc", 0, - RK3288_CLKGATE_CON(10), 12, GFLAGS), + RK2928_CLKGATE_CON(10), 12, GFLAGS), COMPOSITE(0, "sclk_wifi", mux_pll_src_cpll_gpll_usb480m_p, 0, RK2928_CLKSEL_CON(23), 5, 2, MFLAGS, 0, 6, DFLAGS, @@ -379,22 +407,21 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { COMPOSITE(0, "i2s0_src", mux_pll_src_2plls_p, 0, RK2928_CLKSEL_CON(9), 15, 1, MFLAGS, 0, 7, DFLAGS, RK2928_CLKGATE_CON(0), 3, GFLAGS), - COMPOSITE_FRAC(0, "i2s0_frac", "i2s0_src", CLK_SET_RATE_PARENT, - RK3288_CLKSEL_CON(8), 0, - RK3288_CLKGATE_CON(0), 4, GFLAGS), - COMPOSITE_NODIV(SCLK_I2S0, "sclk_i2s0", mux_i2s0_p, 0, - RK2928_CLKSEL_CON(9), 8, 2, MFLAGS, + COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(8), 0, + RK2928_CLKGATE_CON(0), 4, GFLAGS, + &rk3228_i2s0_fracmux), + GATE(SCLK_I2S0, "sclk_i2s0", "i2s0_pre", CLK_SET_RATE_PARENT, RK2928_CLKGATE_CON(0), 5, GFLAGS), COMPOSITE(0, "i2s1_src", mux_pll_src_2plls_p, 0, RK2928_CLKSEL_CON(3), 15, 1, MFLAGS, 0, 7, DFLAGS, RK2928_CLKGATE_CON(0), 10, GFLAGS), - COMPOSITE_FRAC(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT, - RK3288_CLKSEL_CON(7), 0, - RK3288_CLKGATE_CON(0), 11, GFLAGS), - MUX(0, "i2s1_pre", mux_i2s1_pre_p, 0, - RK2928_CLKSEL_CON(3), 8, 2, MFLAGS), - GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", 0, + COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(7), 0, + RK2928_CLKGATE_CON(0), 11, GFLAGS, + &rk3228_i2s1_fracmux), + GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT, RK2928_CLKGATE_CON(0), 14, GFLAGS), COMPOSITE_NODIV(SCLK_I2S_OUT, "i2s_out", mux_i2s_out_p, 0, RK2928_CLKSEL_CON(3), 12, 1, MFLAGS, @@ -403,21 +430,20 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { COMPOSITE(0, "i2s2_src", mux_pll_src_2plls_p, 0, RK2928_CLKSEL_CON(16), 15, 1, MFLAGS, 0, 7, DFLAGS, RK2928_CLKGATE_CON(0), 7, GFLAGS), - COMPOSITE_FRAC(0, "i2s2_frac", "i2s2_src", CLK_SET_RATE_PARENT, - RK3288_CLKSEL_CON(30), 0, - RK3288_CLKGATE_CON(0), 8, GFLAGS), - COMPOSITE_NODIV(SCLK_I2S2, "sclk_i2s2", mux_i2s2_p, 0, - RK2928_CLKSEL_CON(16), 8, 2, MFLAGS, + COMPOSITE_FRACMUX(0, "i2s2_frac", "i2s2_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(30), 0, + RK2928_CLKGATE_CON(0), 8, GFLAGS, + &rk3228_i2s2_fracmux), + GATE(SCLK_I2S2, "sclk_i2s2", "i2s2_pre", CLK_SET_RATE_PARENT, RK2928_CLKGATE_CON(0), 9, GFLAGS), COMPOSITE(0, "sclk_spdif_src", mux_pll_src_2plls_p, 0, RK2928_CLKSEL_CON(6), 15, 1, MFLAGS, 0, 7, DFLAGS, RK2928_CLKGATE_CON(2), 10, GFLAGS), - COMPOSITE_FRAC(0, "spdif_frac", "sclk_spdif_src", CLK_SET_RATE_PARENT, - RK3288_CLKSEL_CON(20), 0, - RK3288_CLKGATE_CON(2), 12, GFLAGS), - MUX(SCLK_SPDIF, "sclk_spdif", mux_sclk_spdif_p, 0, - RK2928_CLKSEL_CON(6), 8, 2, MFLAGS), + COMPOSITE_FRACMUX(0, "spdif_frac", "sclk_spdif_src", CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(20), 0, + RK2928_CLKGATE_CON(2), 12, GFLAGS, + &rk3228_spdif_fracmux), GATE(0, "jtag", "ext_jtag", 0, RK2928_CLKGATE_CON(1), 3, GFLAGS), @@ -456,45 +482,42 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { COMPOSITE(0, "uart2_src", mux_pll_src_cpll_gpll_usb480m_p, 0, RK2928_CLKSEL_CON(15), 12, 2, MFLAGS, 0, 7, DFLAGS, RK2928_CLKGATE_CON(1), 12, GFLAGS), - COMPOSITE_FRAC(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, + COMPOSITE_FRACMUX(0, "uart0_frac", "uart0_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(17), 0, - RK2928_CLKGATE_CON(1), 9, GFLAGS), - COMPOSITE_FRAC(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, + RK2928_CLKGATE_CON(1), 9, GFLAGS, + &rk3228_uart0_fracmux), + COMPOSITE_FRACMUX(0, "uart1_frac", "uart1_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(18), 0, - RK2928_CLKGATE_CON(1), 11, GFLAGS), - COMPOSITE_FRAC(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, + RK2928_CLKGATE_CON(1), 11, GFLAGS, + &rk3228_uart1_fracmux), + COMPOSITE_FRACMUX(0, "uart2_frac", "uart2_src", CLK_SET_RATE_PARENT, RK2928_CLKSEL_CON(19), 0, - RK2928_CLKGATE_CON(1), 13, GFLAGS), - MUX(SCLK_UART0, "sclk_uart0", mux_uart0_p, CLK_SET_RATE_PARENT, - RK2928_CLKSEL_CON(13), 8, 2, MFLAGS), - MUX(SCLK_UART1, "sclk_uart1", mux_uart1_p, CLK_SET_RATE_PARENT, - RK2928_CLKSEL_CON(14), 8, 2, MFLAGS), - MUX(SCLK_UART2, "sclk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT, - RK2928_CLKSEL_CON(15), 8, 2, MFLAGS), + RK2928_CLKGATE_CON(1), 13, GFLAGS, + &rk3228_uart2_fracmux), COMPOSITE(SCLK_NANDC, "sclk_nandc", mux_pll_src_2plls_p, 0, RK2928_CLKSEL_CON(2), 14, 1, MFLAGS, 8, 5, DFLAGS, RK2928_CLKGATE_CON(1), 0, GFLAGS), - COMPOSITE(0, "sclk_gmac_src", mux_pll_src_2plls_p, 0, + COMPOSITE(SCLK_MAC_SRC, "sclk_gmac_src", mux_pll_src_2plls_p, 0, RK2928_CLKSEL_CON(5), 7, 1, MFLAGS, 0, 5, DFLAGS, RK2928_CLKGATE_CON(1), 7, GFLAGS), - MUX(0, "sclk_macphy_50m", mux_sclk_macphy_50m_p, 0, + MUX(SCLK_MAC_EXTCLK, "sclk_mac_extclk", mux_sclk_mac_extclk_p, 0, RK2928_CLKSEL_CON(29), 10, 1, MFLAGS), - MUX(0, "sclk_gmac_pre", mux_sclk_gmac_pre_p, 0, + MUX(SCLK_MAC, "sclk_gmac_pre", mux_sclk_gmac_pre_p, 0, RK2928_CLKSEL_CON(5), 5, 1, MFLAGS), - GATE(0, "sclk_mac_refout", "sclk_gmac_pre", 0, + GATE(SCLK_MAC_REFOUT, "sclk_mac_refout", "sclk_gmac_pre", 0, RK2928_CLKGATE_CON(5), 4, GFLAGS), - GATE(0, "sclk_mac_ref", "sclk_gmac_pre", 0, + GATE(SCLK_MAC_REF, "sclk_mac_ref", "sclk_gmac_pre", 0, RK2928_CLKGATE_CON(5), 3, GFLAGS), - GATE(0, "sclk_mac_rx", "sclk_gmac_pre", 0, + GATE(SCLK_MAC_RX, "sclk_mac_rx", "sclk_gmac_pre", 0, RK2928_CLKGATE_CON(5), 5, GFLAGS), - GATE(0, "sclk_mac_tx", "sclk_gmac_pre", 0, + GATE(SCLK_MAC_TX, "sclk_mac_tx", "sclk_gmac_pre", 0, RK2928_CLKGATE_CON(5), 6, GFLAGS), - COMPOSITE(0, "sclk_macphy", mux_sclk_macphy_p, 0, + COMPOSITE(SCLK_MAC_PHY, "sclk_macphy", mux_sclk_macphy_p, 0, RK2928_CLKSEL_CON(29), 12, 1, MFLAGS, 8, 2, DFLAGS, RK2928_CLKGATE_CON(5), 7, GFLAGS), - COMPOSITE(0, "sclk_gmac_out", mux_pll_src_2plls_p, 0, + COMPOSITE(SCLK_MAC_OUT, "sclk_gmac_out", mux_pll_src_2plls_p, 0, RK2928_CLKSEL_CON(5), 15, 1, MFLAGS, 8, 5, DFLAGS, RK2928_CLKGATE_CON(2), 2, GFLAGS), @@ -528,7 +551,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { /* PD_PERI */ GATE(0, "aclk_peri_noc", "aclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(12), 0, GFLAGS), - GATE(0, "aclk_gmac", "aclk_peri", 0, RK2928_CLKGATE_CON(11), 4, GFLAGS), + GATE(ACLK_GMAC, "aclk_gmac", "aclk_peri", 0, RK2928_CLKGATE_CON(11), 4, GFLAGS), GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 0, GFLAGS), GATE(HCLK_SDIO, "hclk_sdio", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 1, GFLAGS), @@ -544,7 +567,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { GATE(0, "hclk_host2_arb", "hclk_peri", 0, RK2928_CLKGATE_CON(11), 14, GFLAGS), GATE(0, "hclk_peri_noc", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(12), 1, GFLAGS), - GATE(0, "pclk_gmac", "pclk_peri", 0, RK2928_CLKGATE_CON(11), 5, GFLAGS), + GATE(PCLK_GMAC, "pclk_gmac", "pclk_peri", 0, RK2928_CLKGATE_CON(11), 5, GFLAGS), GATE(0, "pclk_peri_noc", "pclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(12), 2, GFLAGS), /* PD_GPU */ @@ -558,10 +581,10 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = { GATE(0, "aclk_bus_noc", "aclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(10), 1, GFLAGS), GATE(0, "hclk_rom", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 3, GFLAGS), - GATE(0, "hclk_i2s0_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 7, GFLAGS), - GATE(0, "hclk_i2s1_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 8, GFLAGS), - GATE(0, "hclk_i2s2_2ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 9, GFLAGS), - GATE(0, "hclk_spdif_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 10, GFLAGS), + GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 7, GFLAGS), + GATE(HCLK_I2S1_8CH, "hclk_i2s1_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 8, GFLAGS), + GATE(HCLK_I2S2_2CH, "hclk_i2s2_2ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 9, GFLAGS), + GATE(HCLK_SPDIF_8CH, "hclk_spdif_8ch", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 10, GFLAGS), GATE(0, "hclk_tsp", "hclk_cpu", 0, RK2928_CLKGATE_CON(10), 11, GFLAGS), GATE(0, "hclk_crypto_mst", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 11, GFLAGS), GATE(0, "hclk_crypto_slv", "hclk_cpu", 0, RK2928_CLKGATE_CON(8), 12, GFLAGS), diff --git a/drivers/clk/rockchip/clk-rk3399.c b/drivers/clk/rockchip/clk-rk3399.c index 31b77f713..cdfabeb9a 100644 --- a/drivers/clk/rockchip/clk-rk3399.c +++ b/drivers/clk/rockchip/clk-rk3399.c @@ -833,9 +833,9 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { /* perihp */ GATE(0, "cpll_aclk_perihp_src", "cpll", CLK_IGNORE_UNUSED, - RK3399_CLKGATE_CON(5), 0, GFLAGS), - GATE(0, "gpll_aclk_perihp_src", "gpll", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(5), 1, GFLAGS), + GATE(0, "gpll_aclk_perihp_src", "gpll", CLK_IGNORE_UNUSED, + RK3399_CLKGATE_CON(5), 0, GFLAGS), COMPOSITE(ACLK_PERIHP, "aclk_perihp", mux_aclk_perihp_p, CLK_IGNORE_UNUSED, RK3399_CLKSEL_CON(14), 7, 1, MFLAGS, 0, 5, DFLAGS, RK3399_CLKGATE_CON(5), 2, GFLAGS), @@ -923,9 +923,9 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { RK3399_CLKGATE_CON(6), 14, GFLAGS), GATE(0, "cpll_aclk_emmc_src", "cpll", CLK_IGNORE_UNUSED, - RK3399_CLKGATE_CON(6), 12, GFLAGS), - GATE(0, "gpll_aclk_emmc_src", "gpll", CLK_IGNORE_UNUSED, RK3399_CLKGATE_CON(6), 13, GFLAGS), + GATE(0, "gpll_aclk_emmc_src", "gpll", CLK_IGNORE_UNUSED, + RK3399_CLKGATE_CON(6), 12, GFLAGS), COMPOSITE_NOGATE(ACLK_EMMC, "aclk_emmc", mux_aclk_emmc_p, CLK_IGNORE_UNUSED, RK3399_CLKSEL_CON(21), 7, 1, MFLAGS, 0, 5, DFLAGS), GATE(ACLK_EMMC_CORE, "aclk_emmccore", "aclk_emmc", CLK_IGNORE_UNUSED, @@ -1071,7 +1071,7 @@ static struct rockchip_clk_branch rk3399_clk_branches[] __initdata = { /* vio */ COMPOSITE(ACLK_VIO, "aclk_vio", mux_pll_src_cpll_gpll_ppll_p, CLK_IGNORE_UNUSED, RK3399_CLKSEL_CON(42), 6, 2, MFLAGS, 0, 5, DFLAGS, - RK3399_CLKGATE_CON(11), 10, GFLAGS), + RK3399_CLKGATE_CON(11), 0, GFLAGS), COMPOSITE_NOMUX(PCLK_VIO, "pclk_vio", "aclk_vio", 0, RK3399_CLKSEL_CON(43), 0, 5, DFLAGS, RK3399_CLKGATE_CON(11), 1, GFLAGS), @@ -1484,6 +1484,7 @@ static const char *const rk3399_cru_critical_clocks[] __initconst = { "hclk_perilp1", "hclk_perilp1_noc", "aclk_dmac0_perilp", + "aclk_emmc_noc", "gpll_hclk_perilp1_src", "gpll_aclk_perilp0_src", "gpll_aclk_perihp_src", @@ -1500,6 +1501,7 @@ static void __init rk3399_clk_init(struct device_node *np) { struct rockchip_clk_provider *ctx; void __iomem *reg_base; + struct clk *clk; reg_base = of_iomap(np, 0); if (!reg_base) { @@ -1514,6 +1516,14 @@ static void __init rk3399_clk_init(struct device_node *np) return; } + /* Watchdog pclk is controlled by RK3399 SECURE_GRF_SOC_CON3[8]. */ + clk = clk_register_fixed_factor(NULL, "pclk_wdt", "pclk_alive", 0, 1, 1); + if (IS_ERR(clk)) + pr_warn("%s: could not register clock pclk_wdt: %ld\n", + __func__, PTR_ERR(clk)); + else + rockchip_clk_add_lookup(ctx, clk, PCLK_WDT); + rockchip_clk_register_plls(ctx, rk3399_pll_clks, ARRAY_SIZE(rk3399_pll_clks), -1); diff --git a/drivers/clk/samsung/Kconfig b/drivers/clk/samsung/Kconfig index 20c5fe92a..addc65270 100644 --- a/drivers/clk/samsung/Kconfig +++ b/drivers/clk/samsung/Kconfig @@ -9,6 +9,15 @@ config EXYNOS_ARM64_COMMON_CLK bool "Samsung Exynos ARMv8-family clock controller support" if COMPILE_TEST depends on COMMON_CLK_SAMSUNG +config EXYNOS_AUDSS_CLK_CON + tristate "Samsung Exynos AUDSS clock controller support" + depends on COMMON_CLK_SAMSUNG + default y if ARCH_EXYNOS + help + Support for the Audio Subsystem CLKCON clock controller present + on some Exynos SoC variants. Choose M or Y here if you want to + use audio devices such as I2S, PCM, etc. + # For S3C24XX platforms, select following symbols: config S3C2410_COMMON_CLK bool "Samsung S3C2410 clock controller support" if COMPILE_TEST diff --git a/drivers/clk/samsung/Makefile b/drivers/clk/samsung/Makefile index fc367d4b2..57f4dc6dc 100644 --- a/drivers/clk/samsung/Makefile +++ b/drivers/clk/samsung/Makefile @@ -12,7 +12,7 @@ obj-$(CONFIG_SOC_EXYNOS5410) += clk-exynos5410.o obj-$(CONFIG_SOC_EXYNOS5420) += clk-exynos5420.o obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK) += clk-exynos5433.o obj-$(CONFIG_SOC_EXYNOS5440) += clk-exynos5440.o -obj-$(CONFIG_ARCH_EXYNOS) += clk-exynos-audss.o +obj-$(CONFIG_EXYNOS_AUDSS_CLK_CON) += clk-exynos-audss.o obj-$(CONFIG_ARCH_EXYNOS) += clk-exynos-clkout.o obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK) += clk-exynos7.o obj-$(CONFIG_S3C2410_COMMON_CLK)+= clk-s3c2410.o diff --git a/drivers/clk/samsung/clk-cpu.c b/drivers/clk/samsung/clk-cpu.c index 813003d6c..8bf7e805f 100644 --- a/drivers/clk/samsung/clk-cpu.c +++ b/drivers/clk/samsung/clk-cpu.c @@ -45,6 +45,13 @@ #define E4210_DIV_STAT_CPU0 0x400 #define E4210_DIV_STAT_CPU1 0x404 +#define E5433_MUX_SEL2 0x008 +#define E5433_MUX_STAT2 0x208 +#define E5433_DIV_CPU0 0x400 +#define E5433_DIV_CPU1 0x404 +#define E5433_DIV_STAT_CPU0 0x500 +#define E5433_DIV_STAT_CPU1 0x504 + #define E4210_DIV0_RATIO0_MASK 0x7 #define E4210_DIV1_HPM_MASK (0x7 << 4) #define E4210_DIV1_COPY_MASK (0x7 << 0) @@ -252,6 +259,102 @@ static int exynos_cpuclk_post_rate_change(struct clk_notifier_data *ndata, return 0; } +/* + * Helper function to set the 'safe' dividers for the CPU clock. The parameters + * div and mask contain the divider value and the register bit mask of the + * dividers to be programmed. + */ +static void exynos5433_set_safe_div(void __iomem *base, unsigned long div, + unsigned long mask) +{ + unsigned long div0; + + div0 = readl(base + E5433_DIV_CPU0); + div0 = (div0 & ~mask) | (div & mask); + writel(div0, base + E5433_DIV_CPU0); + wait_until_divider_stable(base + E5433_DIV_STAT_CPU0, mask); +} + +/* handler for pre-rate change notification from parent clock */ +static int exynos5433_cpuclk_pre_rate_change(struct clk_notifier_data *ndata, + struct exynos_cpuclk *cpuclk, void __iomem *base) +{ + const struct exynos_cpuclk_cfg_data *cfg_data = cpuclk->cfg; + unsigned long alt_prate = clk_get_rate(cpuclk->alt_parent); + unsigned long alt_div = 0, alt_div_mask = DIV_MASK; + unsigned long div0, div1 = 0, mux_reg; + unsigned long flags; + + /* find out the divider values to use for clock data */ + while ((cfg_data->prate * 1000) != ndata->new_rate) { + if (cfg_data->prate == 0) + return -EINVAL; + cfg_data++; + } + + spin_lock_irqsave(cpuclk->lock, flags); + + /* + * For the selected PLL clock frequency, get the pre-defined divider + * values. + */ + div0 = cfg_data->div0; + div1 = cfg_data->div1; + + /* + * If the old parent clock speed is less than the clock speed of + * the alternate parent, then it should be ensured that at no point + * the armclk speed is more than the old_prate until the dividers are + * set. Also workaround the issue of the dividers being set to lower + * values before the parent clock speed is set to new lower speed + * (this can result in too high speed of armclk output clocks). + */ + if (alt_prate > ndata->old_rate || ndata->old_rate > ndata->new_rate) { + unsigned long tmp_rate = min(ndata->old_rate, ndata->new_rate); + + alt_div = DIV_ROUND_UP(alt_prate, tmp_rate) - 1; + WARN_ON(alt_div >= MAX_DIV); + + exynos5433_set_safe_div(base, alt_div, alt_div_mask); + div0 |= alt_div; + } + + /* select the alternate parent */ + mux_reg = readl(base + E5433_MUX_SEL2); + writel(mux_reg | 1, base + E5433_MUX_SEL2); + wait_until_mux_stable(base + E5433_MUX_STAT2, 0, 2); + + /* alternate parent is active now. set the dividers */ + writel(div0, base + E5433_DIV_CPU0); + wait_until_divider_stable(base + E5433_DIV_STAT_CPU0, DIV_MASK_ALL); + + writel(div1, base + E5433_DIV_CPU1); + wait_until_divider_stable(base + E5433_DIV_STAT_CPU1, DIV_MASK_ALL); + + spin_unlock_irqrestore(cpuclk->lock, flags); + return 0; +} + +/* handler for post-rate change notification from parent clock */ +static int exynos5433_cpuclk_post_rate_change(struct clk_notifier_data *ndata, + struct exynos_cpuclk *cpuclk, void __iomem *base) +{ + unsigned long div = 0, div_mask = DIV_MASK; + unsigned long mux_reg; + unsigned long flags; + + spin_lock_irqsave(cpuclk->lock, flags); + + /* select apll as the alternate parent */ + mux_reg = readl(base + E5433_MUX_SEL2); + writel(mux_reg & ~1, base + E5433_MUX_SEL2); + wait_until_mux_stable(base + E5433_MUX_STAT2, 0, 1); + + exynos5433_set_safe_div(base, div, div_mask); + spin_unlock_irqrestore(cpuclk->lock, flags); + return 0; +} + /* * This notifier function is called for the pre-rate and post-rate change * notifications of the parent clock of cpuclk. @@ -275,6 +378,29 @@ static int exynos_cpuclk_notifier_cb(struct notifier_block *nb, return notifier_from_errno(err); } +/* + * This notifier function is called for the pre-rate and post-rate change + * notifications of the parent clock of cpuclk. + */ +static int exynos5433_cpuclk_notifier_cb(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct clk_notifier_data *ndata = data; + struct exynos_cpuclk *cpuclk; + void __iomem *base; + int err = 0; + + cpuclk = container_of(nb, struct exynos_cpuclk, clk_nb); + base = cpuclk->ctrl_base; + + if (event == PRE_RATE_CHANGE) + err = exynos5433_cpuclk_pre_rate_change(ndata, cpuclk, base); + else if (event == POST_RATE_CHANGE) + err = exynos5433_cpuclk_post_rate_change(ndata, cpuclk, base); + + return notifier_from_errno(err); +} + /* helper function to register a CPU clock */ int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, unsigned int lookup_id, const char *name, const char *parent, @@ -301,7 +427,10 @@ int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, cpuclk->ctrl_base = ctx->reg_base + offset; cpuclk->lock = &ctx->lock; cpuclk->flags = flags; - cpuclk->clk_nb.notifier_call = exynos_cpuclk_notifier_cb; + if (flags & CLK_CPU_HAS_E5433_REGS_LAYOUT) + cpuclk->clk_nb.notifier_call = exynos5433_cpuclk_notifier_cb; + else + cpuclk->clk_nb.notifier_call = exynos_cpuclk_notifier_cb; cpuclk->alt_parent = __clk_lookup(alt_parent); if (!cpuclk->alt_parent) { diff --git a/drivers/clk/samsung/clk-cpu.h b/drivers/clk/samsung/clk-cpu.h index 37874d3c3..d4b6b517f 100644 --- a/drivers/clk/samsung/clk-cpu.h +++ b/drivers/clk/samsung/clk-cpu.h @@ -57,10 +57,12 @@ struct exynos_cpuclk { struct notifier_block clk_nb; unsigned long flags; -/* The CPU clock registers has DIV1 configuration register */ +/* The CPU clock registers have DIV1 configuration register */ #define CLK_CPU_HAS_DIV1 (1 << 0) /* When ALT parent is active, debug clocks need safe divider values */ #define CLK_CPU_NEEDS_DEBUG_ALT_DIV (1 << 1) +/* The CPU clock registers have Exynos5433-compatible layout */ +#define CLK_CPU_HAS_E5433_REGS_LAYOUT (1 << 2) }; extern int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx, diff --git a/drivers/clk/samsung/clk-exynos-audss.c b/drivers/clk/samsung/clk-exynos-audss.c index 4e9584d79..bdf8b971f 100644 --- a/drivers/clk/samsung/clk-exynos-audss.c +++ b/drivers/clk/samsung/clk-exynos-audss.c @@ -273,17 +273,7 @@ static struct platform_driver exynos_audss_clk_driver = { .remove = exynos_audss_clk_remove, }; -static int __init exynos_audss_clk_init(void) -{ - return platform_driver_register(&exynos_audss_clk_driver); -} -core_initcall(exynos_audss_clk_init); - -static void __exit exynos_audss_clk_exit(void) -{ - platform_driver_unregister(&exynos_audss_clk_driver); -} -module_exit(exynos_audss_clk_exit); +module_platform_driver(exynos_audss_clk_driver); MODULE_AUTHOR("Padmavathi Venna "); MODULE_DESCRIPTION("Exynos Audio Subsystem Clock Controller"); diff --git a/drivers/clk/samsung/clk-exynos-clkout.c b/drivers/clk/samsung/clk-exynos-clkout.c index 7cd02ff37..96fab6cfb 100644 --- a/drivers/clk/samsung/clk-exynos-clkout.c +++ b/drivers/clk/samsung/clk-exynos-clkout.c @@ -151,6 +151,8 @@ static void __init exynos5_clkout_init(struct device_node *node) } CLK_OF_DECLARE(exynos5250_clkout, "samsung,exynos5250-pmu", exynos5_clkout_init); +CLK_OF_DECLARE(exynos5410_clkout, "samsung,exynos5410-pmu", + exynos5_clkout_init); CLK_OF_DECLARE(exynos5420_clkout, "samsung,exynos5420-pmu", exynos5_clkout_init); CLK_OF_DECLARE(exynos5433_clkout, "samsung,exynos5433-pmu", diff --git a/drivers/clk/samsung/clk-exynos3250.c b/drivers/clk/samsung/clk-exynos3250.c index 16575ee87..1b81e283f 100644 --- a/drivers/clk/samsung/clk-exynos3250.c +++ b/drivers/clk/samsung/clk-exynos3250.c @@ -103,7 +103,7 @@ #define PWR_CTRL1_USE_CORE1_WFI (1 << 1) #define PWR_CTRL1_USE_CORE0_WFI (1 << 0) -static unsigned long exynos3250_cmu_clk_regs[] __initdata = { +static const unsigned long exynos3250_cmu_clk_regs[] __initconst = { SRC_LEFTBUS, DIV_LEFTBUS, GATE_IP_LEFTBUS, @@ -226,7 +226,7 @@ PNAME(group_sclk_fimd0_p) = { "xxti", "xusbxti", PNAME(mout_mfc_p) = { "mout_mfc_0", "mout_mfc_1" }; PNAME(mout_g3d_p) = { "mout_g3d_0", "mout_g3d_1" }; -static struct samsung_fixed_factor_clock fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock fixed_factor_clks[] __initconst = { FFACTOR(0, "sclk_mpll_1600", "mout_mpll", 1, 1, 0), FFACTOR(0, "sclk_mpll_mif", "mout_mpll", 1, 2, 0), FFACTOR(0, "sclk_bpll", "fout_bpll", 1, 2, 0), @@ -237,7 +237,7 @@ static struct samsung_fixed_factor_clock fixed_factor_clks[] __initdata = { FFACTOR(CLK_FIN_PLL, "fin_pll", "xusbxti", 1, 1, 0), }; -static struct samsung_mux_clock mux_clks[] __initdata = { +static const struct samsung_mux_clock mux_clks[] __initconst = { /* * NOTE: Following table is sorted by register address in ascending * order and then bitfield shift in descending order, as it is done @@ -326,7 +326,7 @@ static struct samsung_mux_clock mux_clks[] __initdata = { CLK_SET_RATE_PARENT, 0), }; -static struct samsung_div_clock div_clks[] __initdata = { +static const struct samsung_div_clock div_clks[] __initconst = { /* * NOTE: Following table is sorted by register address in ascending * order and then bitfield shift in descending order, as it is done @@ -429,7 +429,7 @@ static struct samsung_div_clock div_clks[] __initdata = { DIV(CLK_DIV_COPY, "div_copy", "mout_hpm", DIV_CPU1, 0, 3), }; -static struct samsung_gate_clock gate_clks[] __initdata = { +static const struct samsung_gate_clock gate_clks[] __initconst = { /* * NOTE: Following table is sorted by register address in ascending * order and then bitfield shift in descending order, as it is done @@ -669,7 +669,7 @@ static struct samsung_gate_clock gate_clks[] __initdata = { }; /* APLL & MPLL & BPLL & UPLL */ -static struct samsung_pll_rate_table exynos3250_pll_rates[] = { +static const struct samsung_pll_rate_table exynos3250_pll_rates[] __initconst = { PLL_35XX_RATE(1200000000, 400, 4, 1), PLL_35XX_RATE(1100000000, 275, 3, 1), PLL_35XX_RATE(1066000000, 533, 6, 1), @@ -691,7 +691,7 @@ static struct samsung_pll_rate_table exynos3250_pll_rates[] = { }; /* EPLL */ -static struct samsung_pll_rate_table exynos3250_epll_rates[] = { +static const struct samsung_pll_rate_table exynos3250_epll_rates[] __initconst = { PLL_36XX_RATE(800000000, 200, 3, 1, 0), PLL_36XX_RATE(288000000, 96, 2, 2, 0), PLL_36XX_RATE(192000000, 128, 2, 3, 0), @@ -710,7 +710,7 @@ static struct samsung_pll_rate_table exynos3250_epll_rates[] = { }; /* VPLL */ -static struct samsung_pll_rate_table exynos3250_vpll_rates[] = { +static const struct samsung_pll_rate_table exynos3250_vpll_rates[] __initconst = { PLL_36XX_RATE(600000000, 100, 2, 1, 0), PLL_36XX_RATE(533000000, 266, 3, 2, 32768), PLL_36XX_RATE(519230987, 173, 2, 2, 5046), @@ -740,7 +740,7 @@ static struct samsung_pll_rate_table exynos3250_vpll_rates[] = { { /* sentinel */ } }; -static struct samsung_pll_clock exynos3250_plls[] __initdata = { +static const struct samsung_pll_clock exynos3250_plls[] __initconst = { PLL(pll_35xx, CLK_FOUT_APLL, "fout_apll", "fin_pll", APLL_LOCK, APLL_CON0, exynos3250_pll_rates), PLL(pll_35xx, CLK_FOUT_MPLL, "fout_mpll", "fin_pll", @@ -772,7 +772,7 @@ static void __init exynos3_core_down_clock(void __iomem *reg_base) __raw_writel(0x0, reg_base + PWR_CTRL2); } -static struct samsung_cmu_info cmu_info __initdata = { +static const struct samsung_cmu_info cmu_info __initconst = { .pll_clks = exynos3250_plls, .nr_pll_clks = ARRAY_SIZE(exynos3250_plls), .mux_clks = mux_clks, @@ -848,7 +848,7 @@ CLK_OF_DECLARE(exynos3250_cmu, "samsung,exynos3250-cmu", exynos3250_cmu_init); #define EPLL_CON2 0x111c #define SRC_EPLL 0x1120 -static unsigned long exynos3250_cmu_dmc_clk_regs[] __initdata = { +static const unsigned long exynos3250_cmu_dmc_clk_regs[] __initconst = { BPLL_LOCK, BPLL_CON0, BPLL_CON1, @@ -874,7 +874,7 @@ PNAME(mout_bpll_p) = { "fin_pll", "fout_bpll", }; PNAME(mout_mpll_mif_p) = { "fin_pll", "sclk_mpll_mif", }; PNAME(mout_dphy_p) = { "mout_mpll_mif", "mout_bpll", }; -static struct samsung_mux_clock dmc_mux_clks[] __initdata = { +static const struct samsung_mux_clock dmc_mux_clks[] __initconst = { /* * NOTE: Following table is sorted by register address in ascending * order and then bitfield shift in descending order, as it is done @@ -893,7 +893,7 @@ static struct samsung_mux_clock dmc_mux_clks[] __initdata = { MUX(CLK_MOUT_EPLL, "mout_epll", mout_epll_p, SRC_EPLL, 4, 1), }; -static struct samsung_div_clock dmc_div_clks[] __initdata = { +static const struct samsung_div_clock dmc_div_clks[] __initconst = { /* * NOTE: Following table is sorted by register address in ascending * order and then bitfield shift in descending order, as it is done @@ -910,14 +910,14 @@ static struct samsung_div_clock dmc_div_clks[] __initdata = { DIV(CLK_DIV_DMCD, "div_dmcd", "div_dmc", DIV_DMC1, 11, 3), }; -static struct samsung_pll_clock exynos3250_dmc_plls[] __initdata = { +static const struct samsung_pll_clock exynos3250_dmc_plls[] __initconst = { PLL(pll_35xx, CLK_FOUT_BPLL, "fout_bpll", "fin_pll", BPLL_LOCK, BPLL_CON0, exynos3250_pll_rates), PLL(pll_36xx, CLK_FOUT_EPLL, "fout_epll", "fin_pll", EPLL_LOCK, EPLL_CON0, exynos3250_epll_rates), }; -static struct samsung_cmu_info dmc_cmu_info __initdata = { +static const struct samsung_cmu_info dmc_cmu_info __initconst = { .pll_clks = exynos3250_dmc_plls, .nr_pll_clks = ARRAY_SIZE(exynos3250_dmc_plls), .mux_clks = dmc_mux_clks, @@ -947,7 +947,7 @@ CLK_OF_DECLARE(exynos3250_cmu_dmc, "samsung,exynos3250-cmu-dmc", #define GATE_IP_ISP1 0x804 #define GATE_SCLK_ISP 0x900 -static struct samsung_div_clock isp_div_clks[] __initdata = { +static const struct samsung_div_clock isp_div_clks[] __initconst = { /* * NOTE: Following table is sorted by register address in ascending * order and then bitfield shift in descending order, as it is done @@ -967,7 +967,7 @@ static struct samsung_div_clock isp_div_clks[] __initdata = { DIV(CLK_DIV_MPWM, "div_mpwm", "div_isp1", DIV_ISP1, 0, 3), }; -static struct samsung_gate_clock isp_gate_clks[] __initdata = { +static const struct samsung_gate_clock isp_gate_clks[] __initconst = { /* * NOTE: Following table is sorted by register address in ascending * order and then bitfield shift in descending order, as it is done @@ -1063,7 +1063,7 @@ static struct samsung_gate_clock isp_gate_clks[] __initdata = { GATE_SCLK_ISP, 0, CLK_IGNORE_UNUSED, 0), }; -static struct samsung_cmu_info isp_cmu_info __initdata = { +static const struct samsung_cmu_info isp_cmu_info __initconst = { .div_clks = isp_div_clks, .nr_div_clks = ARRAY_SIZE(isp_div_clks), .gate_clks = isp_gate_clks, @@ -1079,14 +1079,15 @@ static int __init exynos3250_cmu_isp_probe(struct platform_device *pdev) return 0; } -static const struct of_device_id exynos3250_cmu_isp_of_match[] = { +static const struct of_device_id exynos3250_cmu_isp_of_match[] __initconst = { { .compatible = "samsung,exynos3250-cmu-isp", }, { /* sentinel */ } }; -static struct platform_driver exynos3250_cmu_isp_driver = { +static struct platform_driver exynos3250_cmu_isp_driver __initdata = { .driver = { .name = "exynos3250-cmu-isp", + .suppress_bind_attrs = true, .of_match_table = exynos3250_cmu_isp_of_match, }, }; diff --git a/drivers/clk/samsung/clk-exynos4.c b/drivers/clk/samsung/clk-exynos4.c index 7b3d0f975..faab9b31b 100644 --- a/drivers/clk/samsung/clk-exynos4.c +++ b/drivers/clk/samsung/clk-exynos4.c @@ -169,7 +169,7 @@ static struct samsung_clk_reg_dump *exynos4_save_pll; * list of controller registers to be saved and restored during a * suspend/resume cycle. */ -static unsigned long exynos4210_clk_save[] __initdata = { +static const unsigned long exynos4210_clk_save[] __initconst = { E4210_SRC_IMAGE, E4210_SRC_LCD1, E4210_SRC_MASK_LCD1, @@ -181,7 +181,7 @@ static unsigned long exynos4210_clk_save[] __initdata = { PWR_CTRL1, }; -static unsigned long exynos4x12_clk_save[] __initdata = { +static const unsigned long exynos4x12_clk_save[] __initconst = { E4X12_GATE_IP_IMAGE, E4X12_GATE_IP_PERIR, E4X12_SRC_CAM1, @@ -192,7 +192,7 @@ static unsigned long exynos4x12_clk_save[] __initdata = { E4X12_PWR_CTRL2, }; -static unsigned long exynos4_clk_pll_regs[] __initdata = { +static const unsigned long exynos4_clk_pll_regs[] __initconst = { EPLL_LOCK, VPLL_LOCK, EPLL_CON0, @@ -203,7 +203,7 @@ static unsigned long exynos4_clk_pll_regs[] __initdata = { VPLL_CON2, }; -static unsigned long exynos4_clk_regs[] __initdata = { +static const unsigned long exynos4_clk_regs[] __initconst = { SRC_LEFTBUS, DIV_LEFTBUS, GATE_IP_LEFTBUS, @@ -505,28 +505,28 @@ static struct samsung_fixed_rate_clock exynos4_fixed_rate_ext_clks[] __initdata }; /* fixed rate clocks generated inside the soc */ -static struct samsung_fixed_rate_clock exynos4_fixed_rate_clks[] __initdata = { +static const struct samsung_fixed_rate_clock exynos4_fixed_rate_clks[] __initconst = { FRATE(0, "sclk_hdmi24m", NULL, 0, 24000000), FRATE(CLK_SCLK_HDMIPHY, "sclk_hdmiphy", "hdmi", 0, 27000000), FRATE(0, "sclk_usbphy0", NULL, 0, 48000000), }; -static struct samsung_fixed_rate_clock exynos4210_fixed_rate_clks[] __initdata = { +static const struct samsung_fixed_rate_clock exynos4210_fixed_rate_clks[] __initconst = { FRATE(0, "sclk_usbphy1", NULL, 0, 48000000), }; -static struct samsung_fixed_factor_clock exynos4_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock exynos4_fixed_factor_clks[] __initconst = { FFACTOR(0, "sclk_apll_div_2", "sclk_apll", 1, 2, 0), FFACTOR(0, "fout_mpll_div_2", "fout_mpll", 1, 2, 0), FFACTOR(0, "fout_apll_div_2", "fout_apll", 1, 2, 0), FFACTOR(0, "arm_clk_div_2", "div_core2", 1, 2, 0), }; -static struct samsung_fixed_factor_clock exynos4210_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock exynos4210_fixed_factor_clks[] __initconst = { FFACTOR(0, "sclk_mpll_div_2", "sclk_mpll", 1, 2, 0), }; -static struct samsung_fixed_factor_clock exynos4x12_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock exynos4x12_fixed_factor_clks[] __initconst = { FFACTOR(0, "sclk_mpll_user_l_div_2", "mout_mpll_user_l", 1, 2, 0), FFACTOR(0, "sclk_mpll_user_r_div_2", "mout_mpll_user_r", 1, 2, 0), FFACTOR(0, "sclk_mpll_user_t_div_2", "mout_mpll_user_t", 1, 2, 0), @@ -534,7 +534,7 @@ static struct samsung_fixed_factor_clock exynos4x12_fixed_factor_clks[] __initda }; /* list of mux clocks supported in all exynos4 soc's */ -static struct samsung_mux_clock exynos4_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos4_mux_clks[] __initconst = { MUX_FA(CLK_MOUT_APLL, "mout_apll", mout_apll_p, SRC_CPU, 0, 1, CLK_SET_RATE_PARENT | CLK_RECALC_NEW_RATES, 0, "mout_apll"), @@ -555,11 +555,11 @@ static struct samsung_mux_clock exynos4_mux_clks[] __initdata = { }; /* list of mux clocks supported in exynos4210 soc */ -static struct samsung_mux_clock exynos4210_mux_early[] __initdata = { +static const struct samsung_mux_clock exynos4210_mux_early[] __initconst = { MUX(0, "mout_vpllsrc", mout_vpllsrc_p, SRC_TOP1, 0, 1), }; -static struct samsung_mux_clock exynos4210_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos4210_mux_clks[] __initconst = { MUX(0, "mout_gdl", sclk_ampll_p4210, SRC_LEFTBUS, 0, 1), MUX(0, "mout_clkout_leftbus", clkout_left_p4210, CLKOUT_CMU_LEFTBUS, 0, 5), @@ -622,7 +622,7 @@ static struct samsung_mux_clock exynos4210_mux_clks[] __initdata = { }; /* list of mux clocks supported in exynos4x12 soc */ -static struct samsung_mux_clock exynos4x12_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos4x12_mux_clks[] __initconst = { MUX(0, "mout_mpll_user_l", mout_mpll_p, SRC_LEFTBUS, 4, 1), MUX(0, "mout_gdl", mout_gdl_p4x12, SRC_LEFTBUS, 0, 1), MUX(0, "mout_clkout_leftbus", clkout_left_p4x12, @@ -705,7 +705,7 @@ static struct samsung_mux_clock exynos4x12_mux_clks[] __initdata = { }; /* list of divider clocks supported in all exynos4 soc's */ -static struct samsung_div_clock exynos4_div_clks[] __initdata = { +static const struct samsung_div_clock exynos4_div_clks[] __initconst = { DIV(CLK_DIV_GDL, "div_gdl", "mout_gdl", DIV_LEFTBUS, 0, 3), DIV(0, "div_gpl", "div_gdl", DIV_LEFTBUS, 4, 3), DIV(0, "div_clkout_leftbus", "mout_clkout_leftbus", @@ -795,7 +795,7 @@ static struct samsung_div_clock exynos4_div_clks[] __initdata = { }; /* list of divider clocks supported in exynos4210 soc */ -static struct samsung_div_clock exynos4210_div_clks[] __initdata = { +static const struct samsung_div_clock exynos4210_div_clks[] __initconst = { DIV(CLK_ACLK200, "aclk200", "mout_aclk200", DIV_TOP, 0, 3), DIV(CLK_SCLK_FIMG2D, "sclk_fimg2d", "mout_g2d", DIV_IMAGE, 0, 4), DIV(0, "div_fimd1", "mout_fimd1", E4210_DIV_LCD1, 0, 4), @@ -806,7 +806,7 @@ static struct samsung_div_clock exynos4210_div_clks[] __initdata = { }; /* list of divider clocks supported in exynos4x12 soc */ -static struct samsung_div_clock exynos4x12_div_clks[] __initdata = { +static const struct samsung_div_clock exynos4x12_div_clks[] __initconst = { DIV(0, "div_mdnie0", "mout_mdnie0", DIV_LCD0, 4, 4), DIV(0, "div_mdnie_pwm0", "mout_mdnie_pwm0", DIV_LCD0, 8, 4), DIV(0, "div_mdnie_pwm_pre0", "div_mdnie_pwm0", DIV_LCD0, 12, 4), @@ -837,7 +837,7 @@ static struct samsung_div_clock exynos4x12_div_clks[] __initdata = { }; /* list of gate clocks supported in all exynos4 soc's */ -static struct samsung_gate_clock exynos4_gate_clks[] __initdata = { +static const struct samsung_gate_clock exynos4_gate_clks[] __initconst = { /* * After all Exynos4 based platforms are migrated to use device tree, * the device name and clock alias names specified below for some @@ -1043,7 +1043,7 @@ static struct samsung_gate_clock exynos4_gate_clks[] __initdata = { }; /* list of gate clocks supported in exynos4210 soc */ -static struct samsung_gate_clock exynos4210_gate_clks[] __initdata = { +static const struct samsung_gate_clock exynos4210_gate_clks[] __initconst = { GATE(CLK_TVENC, "tvenc", "aclk160", GATE_IP_TV, 2, 0, 0), GATE(CLK_G2D, "g2d", "aclk200", E4210_GATE_IP_IMAGE, 0, 0, 0), GATE(CLK_ROTATOR, "rotator", "aclk200", E4210_GATE_IP_IMAGE, 1, 0, 0), @@ -1090,7 +1090,7 @@ static struct samsung_gate_clock exynos4210_gate_clks[] __initdata = { }; /* list of gate clocks supported in exynos4x12 soc */ -static struct samsung_gate_clock exynos4x12_gate_clks[] __initdata = { +static const struct samsung_gate_clock exynos4x12_gate_clks[] __initconst = { GATE(CLK_AUDSS, "audss", "sclk_epll", E4X12_GATE_IP_MAUDIO, 0, 0, 0), GATE(CLK_MDNIE0, "mdnie0", "aclk160", GATE_IP_LCD0, 2, 0, 0), GATE(CLK_ROTATOR, "rotator", "aclk200", E4X12_GATE_IP_IMAGE, 1, 0, 0), @@ -1190,17 +1190,17 @@ static struct samsung_gate_clock exynos4x12_gate_clks[] __initdata = { 0), }; -static struct samsung_clock_alias exynos4_aliases[] __initdata = { +static const struct samsung_clock_alias exynos4_aliases[] __initconst = { ALIAS(CLK_MOUT_CORE, NULL, "moutcore"), ALIAS(CLK_ARM_CLK, NULL, "armclk"), ALIAS(CLK_SCLK_APLL, NULL, "mout_apll"), }; -static struct samsung_clock_alias exynos4210_aliases[] __initdata = { +static const struct samsung_clock_alias exynos4210_aliases[] __initconst = { ALIAS(CLK_SCLK_MPLL, NULL, "mout_mpll"), }; -static struct samsung_clock_alias exynos4x12_aliases[] __initdata = { +static const struct samsung_clock_alias exynos4x12_aliases[] __initconst = { ALIAS(CLK_MOUT_MPLL_USER_C, NULL, "mout_mpll"), }; @@ -1211,7 +1211,7 @@ static struct samsung_clock_alias exynos4x12_aliases[] __initdata = { * controller is first remapped and the value of XOM[0] bit is read to * determine the parent clock. */ -static unsigned long exynos4_get_xom(void) +static unsigned long __init exynos4_get_xom(void) { unsigned long xom = 0; void __iomem *chipid_base; @@ -1264,7 +1264,7 @@ static const struct of_device_id ext_clk_match[] __initconst = { }; /* PLLs PMS values */ -static struct samsung_pll_rate_table exynos4210_apll_rates[] __initdata = { +static const struct samsung_pll_rate_table exynos4210_apll_rates[] __initconst = { PLL_45XX_RATE(1200000000, 150, 3, 1, 28), PLL_45XX_RATE(1000000000, 250, 6, 1, 28), PLL_45XX_RATE( 800000000, 200, 6, 1, 28), @@ -1277,7 +1277,7 @@ static struct samsung_pll_rate_table exynos4210_apll_rates[] __initdata = { { /* sentinel */ } }; -static struct samsung_pll_rate_table exynos4210_epll_rates[] __initdata = { +static const struct samsung_pll_rate_table exynos4210_epll_rates[] __initconst = { PLL_4600_RATE(192000000, 48, 3, 1, 0, 0), PLL_4600_RATE(180633605, 45, 3, 1, 10381, 0), PLL_4600_RATE(180000000, 45, 3, 1, 0, 0), @@ -1288,7 +1288,7 @@ static struct samsung_pll_rate_table exynos4210_epll_rates[] __initdata = { { /* sentinel */ } }; -static struct samsung_pll_rate_table exynos4210_vpll_rates[] __initdata = { +static const struct samsung_pll_rate_table exynos4210_vpll_rates[] __initconst = { PLL_4650_RATE(360000000, 44, 3, 0, 1024, 0, 14, 0), PLL_4650_RATE(324000000, 53, 2, 1, 1024, 1, 1, 1), PLL_4650_RATE(259617187, 63, 3, 1, 1950, 0, 20, 1), @@ -1297,7 +1297,7 @@ static struct samsung_pll_rate_table exynos4210_vpll_rates[] __initdata = { { /* sentinel */ } }; -static struct samsung_pll_rate_table exynos4x12_apll_rates[] __initdata = { +static const struct samsung_pll_rate_table exynos4x12_apll_rates[] __initconst = { PLL_35XX_RATE(1500000000, 250, 4, 0), PLL_35XX_RATE(1400000000, 175, 3, 0), PLL_35XX_RATE(1300000000, 325, 6, 0), @@ -1315,7 +1315,7 @@ static struct samsung_pll_rate_table exynos4x12_apll_rates[] __initdata = { { /* sentinel */ } }; -static struct samsung_pll_rate_table exynos4x12_epll_rates[] __initdata = { +static const struct samsung_pll_rate_table exynos4x12_epll_rates[] __initconst = { PLL_36XX_RATE(192000000, 48, 3, 1, 0), PLL_36XX_RATE(180633605, 45, 3, 1, 10381), PLL_36XX_RATE(180000000, 45, 3, 1, 0), @@ -1326,7 +1326,7 @@ static struct samsung_pll_rate_table exynos4x12_epll_rates[] __initdata = { { /* sentinel */ } }; -static struct samsung_pll_rate_table exynos4x12_vpll_rates[] __initdata = { +static const struct samsung_pll_rate_table exynos4x12_vpll_rates[] __initconst = { PLL_36XX_RATE(533000000, 133, 3, 1, 16384), PLL_36XX_RATE(440000000, 110, 3, 1, 0), PLL_36XX_RATE(350000000, 175, 3, 2, 0), @@ -1375,12 +1375,12 @@ static void __init exynos4x12_core_down_clock(void) if (num_possible_cpus() == 4) tmp |= PWR_CTRL1_USE_CORE3_WFE | PWR_CTRL1_USE_CORE2_WFE | PWR_CTRL1_USE_CORE3_WFI | PWR_CTRL1_USE_CORE2_WFI; - __raw_writel(tmp, reg_base + PWR_CTRL1); + writel_relaxed(tmp, reg_base + PWR_CTRL1); /* * Disable the clock up feature in case it was enabled by bootloader. */ - __raw_writel(0x0, reg_base + E4X12_PWR_CTRL2); + writel_relaxed(0x0, reg_base + E4X12_PWR_CTRL2); } #define E4210_CPU_DIV0(apll, pclk_dbg, atb, periph, corem1, corem0) \ @@ -1450,8 +1450,6 @@ static void __init exynos4_clk_init(struct device_node *np, panic("%s: failed to map registers\n", __func__); ctx = samsung_clk_init(np, reg_base, CLK_NR_CLKS); - if (!ctx) - panic("%s: unable to allocate context.\n", __func__); samsung_clk_of_register_fixed_ext(ctx, exynos4_fixed_rate_ext_clks, ARRAY_SIZE(exynos4_fixed_rate_ext_clks), diff --git a/drivers/clk/samsung/clk-exynos4415.c b/drivers/clk/samsung/clk-exynos4415.c index 86ee06b22..6c9063159 100644 --- a/drivers/clk/samsung/clk-exynos4415.c +++ b/drivers/clk/samsung/clk-exynos4415.c @@ -111,7 +111,7 @@ #define DIV_CPU0 0x14500 #define DIV_CPU1 0x14504 -static unsigned long exynos4415_cmu_clk_regs[] __initdata = { +static const unsigned long exynos4415_cmu_clk_regs[] __initconst = { SRC_LEFTBUS, DIV_LEFTBUS, GATE_IP_LEFTBUS, @@ -268,16 +268,16 @@ PNAME(group_aclk_isp0_300_user_p) = { "fin_pll", "mout_aclk_isp0_300" }; PNAME(group_aclk_isp1_300_user_p) = { "fin_pll", "mout_aclk_isp1_300" }; PNAME(group_mout_mpll_user_t_p) = { "mout_mpll_user_t" }; -static struct samsung_fixed_factor_clock exynos4415_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock exynos4415_fixed_factor_clks[] __initconst = { /* HACK: fin_pll hardcoded to xusbxti until detection is implemented. */ FFACTOR(CLK_FIN_PLL, "fin_pll", "xusbxti", 1, 1, 0), }; -static struct samsung_fixed_rate_clock exynos4415_fixed_rate_clks[] __initdata = { +static const struct samsung_fixed_rate_clock exynos4415_fixed_rate_clks[] __initconst = { FRATE(CLK_SCLK_HDMIPHY, "sclk_hdmiphy", NULL, 0, 27000000), }; -static struct samsung_mux_clock exynos4415_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos4415_mux_clks[] __initconst = { /* * NOTE: Following table is sorted by register address in ascending * order and then bitfield shift in descending order, as it is done @@ -427,7 +427,7 @@ static struct samsung_mux_clock exynos4415_mux_clks[] __initdata = { group_aclk_isp1_300_user_p, SRC_TOP_ISP1, 0, 1), }; -static struct samsung_div_clock exynos4415_div_clks[] __initdata = { +static const struct samsung_div_clock exynos4415_div_clks[] __initconst = { /* * NOTE: Following table is sorted by register address in ascending * order and then bitfield shift in descending order, as it is done @@ -566,7 +566,7 @@ static struct samsung_div_clock exynos4415_div_clks[] __initdata = { DIV(CLK_DIV_COPY, "div_copy", "mout_hpm", DIV_CPU1, 0, 3), }; -static struct samsung_gate_clock exynos4415_gate_clks[] __initdata = { +static const struct samsung_gate_clock exynos4415_gate_clks[] __initconst = { /* * NOTE: Following table is sorted by register address in ascending * order and then bitfield shift in descending order, as it is done @@ -859,7 +859,7 @@ static struct samsung_gate_clock exynos4415_gate_clks[] __initdata = { /* * APLL & MPLL & BPLL & ISP_PLL & DISP_PLL & G3D_PLL */ -static struct samsung_pll_rate_table exynos4415_pll_rates[] = { +static const struct samsung_pll_rate_table exynos4415_pll_rates[] __initconst = { PLL_35XX_RATE(1600000000, 400, 3, 1), PLL_35XX_RATE(1500000000, 250, 2, 1), PLL_35XX_RATE(1400000000, 175, 3, 0), @@ -891,7 +891,7 @@ static struct samsung_pll_rate_table exynos4415_pll_rates[] = { }; /* EPLL */ -static struct samsung_pll_rate_table exynos4415_epll_rates[] = { +static const struct samsung_pll_rate_table exynos4415_epll_rates[] __initconst = { PLL_36XX_RATE(800000000, 200, 3, 1, 0), PLL_36XX_RATE(288000000, 96, 2, 2, 0), PLL_36XX_RATE(192000000, 128, 2, 3, 0), @@ -909,7 +909,7 @@ static struct samsung_pll_rate_table exynos4415_epll_rates[] = { { /* sentinel */ } }; -static struct samsung_pll_clock exynos4415_plls[] __initdata = { +static const struct samsung_pll_clock exynos4415_plls[] __initconst = { PLL(pll_35xx, CLK_FOUT_APLL, "fout_apll", "fin_pll", APLL_LOCK, APLL_CON0, exynos4415_pll_rates), PLL(pll_36xx, CLK_FOUT_EPLL, "fout_epll", "fin_pll", @@ -922,7 +922,7 @@ static struct samsung_pll_clock exynos4415_plls[] __initdata = { "fin_pll", DISP_PLL_LOCK, DISP_PLL_CON0, exynos4415_pll_rates), }; -static struct samsung_cmu_info cmu_info __initdata = { +static const struct samsung_cmu_info cmu_info __initconst = { .pll_clks = exynos4415_plls, .nr_pll_clks = ARRAY_SIZE(exynos4415_plls), .mux_clks = exynos4415_mux_clks, @@ -961,7 +961,7 @@ CLK_OF_DECLARE(exynos4415_cmu, "samsung,exynos4415-cmu", exynos4415_cmu_init); #define SRC_DMC 0x300 #define DIV_DMC1 0x504 -static unsigned long exynos4415_cmu_dmc_clk_regs[] __initdata = { +static const unsigned long exynos4415_cmu_dmc_clk_regs[] __initconst = { MPLL_LOCK, MPLL_CON0, MPLL_CON1, @@ -978,14 +978,14 @@ PNAME(mout_mpll_p) = { "fin_pll", "fout_mpll", }; PNAME(mout_bpll_p) = { "fin_pll", "fout_bpll", }; PNAME(mbpll_p) = { "mout_mpll", "mout_bpll", }; -static struct samsung_mux_clock exynos4415_dmc_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos4415_dmc_mux_clks[] __initconst = { MUX(CLK_DMC_MOUT_MPLL, "mout_mpll", mout_mpll_p, SRC_DMC, 12, 1), MUX(CLK_DMC_MOUT_BPLL, "mout_bpll", mout_bpll_p, SRC_DMC, 10, 1), MUX(CLK_DMC_MOUT_DPHY, "mout_dphy", mbpll_p, SRC_DMC, 8, 1), MUX(CLK_DMC_MOUT_DMC_BUS, "mout_dmc_bus", mbpll_p, SRC_DMC, 4, 1), }; -static struct samsung_div_clock exynos4415_dmc_div_clks[] __initdata = { +static const struct samsung_div_clock exynos4415_dmc_div_clks[] __initconst = { DIV(CLK_DMC_DIV_DMC, "div_dmc", "div_dmc_pre", DIV_DMC1, 27, 3), DIV(CLK_DMC_DIV_DPHY, "div_dphy", "mout_dphy", DIV_DMC1, 23, 3), DIV(CLK_DMC_DIV_DMC_PRE, "div_dmc_pre", "mout_dmc_bus", @@ -995,14 +995,14 @@ static struct samsung_div_clock exynos4415_dmc_div_clks[] __initdata = { DIV(CLK_DMC_DIV_MPLL_PRE, "div_mpll_pre", "mout_mpll", DIV_DMC1, 8, 2), }; -static struct samsung_pll_clock exynos4415_dmc_plls[] __initdata = { +static const struct samsung_pll_clock exynos4415_dmc_plls[] __initconst = { PLL(pll_35xx, CLK_DMC_FOUT_MPLL, "fout_mpll", "fin_pll", MPLL_LOCK, MPLL_CON0, exynos4415_pll_rates), PLL(pll_35xx, CLK_DMC_FOUT_BPLL, "fout_bpll", "fin_pll", BPLL_LOCK, BPLL_CON0, exynos4415_pll_rates), }; -static struct samsung_cmu_info cmu_dmc_info __initdata = { +static const struct samsung_cmu_info cmu_dmc_info __initconst = { .pll_clks = exynos4415_dmc_plls, .nr_pll_clks = ARRAY_SIZE(exynos4415_dmc_plls), .mux_clks = exynos4415_dmc_mux_clks, diff --git a/drivers/clk/samsung/clk-exynos5250.c b/drivers/clk/samsung/clk-exynos5250.c index 837197db4..27a227d66 100644 --- a/drivers/clk/samsung/clk-exynos5250.c +++ b/drivers/clk/samsung/clk-exynos5250.c @@ -117,7 +117,7 @@ static struct samsung_clk_reg_dump *exynos5250_save; * list of controller registers to be saved and restored during a * suspend/resume cycle. */ -static unsigned long exynos5250_clk_regs[] __initdata = { +static const unsigned long exynos5250_clk_regs[] __initconst = { SRC_CPU, DIV_CPU0, PWR_CTRL1, @@ -190,7 +190,7 @@ static struct syscore_ops exynos5250_clk_syscore_ops = { .resume = exynos5250_clk_resume, }; -static void exynos5250_clk_sleep_init(void) +static void __init exynos5250_clk_sleep_init(void) { exynos5250_save = samsung_clk_alloc_reg_dump(exynos5250_clk_regs, ARRAY_SIZE(exynos5250_clk_regs)); @@ -203,7 +203,7 @@ static void exynos5250_clk_sleep_init(void) register_syscore_ops(&exynos5250_clk_syscore_ops); } #else -static void exynos5250_clk_sleep_init(void) {} +static void __init exynos5250_clk_sleep_init(void) {} #endif /* list of all parent clock list */ @@ -266,23 +266,23 @@ static struct samsung_fixed_rate_clock exynos5250_fixed_rate_ext_clks[] __initda }; /* fixed rate clocks generated inside the soc */ -static struct samsung_fixed_rate_clock exynos5250_fixed_rate_clks[] __initdata = { +static const struct samsung_fixed_rate_clock exynos5250_fixed_rate_clks[] __initconst = { FRATE(CLK_SCLK_HDMIPHY, "sclk_hdmiphy", NULL, 0, 24000000), FRATE(0, "sclk_hdmi27m", NULL, 0, 27000000), FRATE(0, "sclk_dptxphy", NULL, 0, 24000000), FRATE(0, "sclk_uhostphy", NULL, 0, 48000000), }; -static struct samsung_fixed_factor_clock exynos5250_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock exynos5250_fixed_factor_clks[] __initconst = { FFACTOR(0, "fout_mplldiv2", "fout_mpll", 1, 2, 0), FFACTOR(0, "fout_bplldiv2", "fout_bpll", 1, 2, 0), }; -static struct samsung_mux_clock exynos5250_pll_pmux_clks[] __initdata = { +static const struct samsung_mux_clock exynos5250_pll_pmux_clks[] __initconst = { MUX(0, "mout_vpllsrc", mout_vpllsrc_p, SRC_TOP2, 0, 1), }; -static struct samsung_mux_clock exynos5250_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos5250_mux_clks[] __initconst = { /* * NOTE: Following table is sorted by (clock domain, register address, * bitfield shift) triplet in ascending order. When adding new entries, @@ -378,7 +378,7 @@ static struct samsung_mux_clock exynos5250_mux_clks[] __initdata = { MUX(0, "mout_bpll_fout", mout_bpll_fout_p, PLL_DIV2_SEL, 0, 1), }; -static struct samsung_div_clock exynos5250_div_clks[] __initdata = { +static const struct samsung_div_clock exynos5250_div_clks[] __initconst = { /* * NOTE: Following table is sorted by (clock domain, register address, * bitfield shift) triplet in ascending order. When adding new entries, @@ -470,7 +470,7 @@ static struct samsung_div_clock exynos5250_div_clks[] __initdata = { DIV(CLK_DIV_I2S2, "div_i2s2", "sclk_audio2", DIV_PERIC5, 8, 6), }; -static struct samsung_gate_clock exynos5250_gate_clks[] __initdata = { +static const struct samsung_gate_clock exynos5250_gate_clks[] __initconst = { /* * NOTE: Following table is sorted by (clock domain, register address, * bitfield shift) triplet in ascending order. When adding new entries, @@ -698,7 +698,7 @@ static struct samsung_gate_clock exynos5250_gate_clks[] __initdata = { GATE_IP_ISP1, 7, 0, 0), }; -static struct samsung_pll_rate_table vpll_24mhz_tbl[] __initdata = { +static const struct samsung_pll_rate_table vpll_24mhz_tbl[] __initconst = { /* sorted in descending order */ /* PLL_36XX_RATE(rate, m, p, s, k) */ PLL_36XX_RATE(266000000, 266, 3, 3, 0), @@ -707,7 +707,7 @@ static struct samsung_pll_rate_table vpll_24mhz_tbl[] __initdata = { { }, }; -static struct samsung_pll_rate_table epll_24mhz_tbl[] __initdata = { +static const struct samsung_pll_rate_table epll_24mhz_tbl[] __initconst = { /* sorted in descending order */ /* PLL_36XX_RATE(rate, m, p, s, k) */ PLL_36XX_RATE(192000000, 64, 2, 2, 0), @@ -721,7 +721,7 @@ static struct samsung_pll_rate_table epll_24mhz_tbl[] __initdata = { { }, }; -static struct samsung_pll_rate_table apll_24mhz_tbl[] __initdata = { +static const struct samsung_pll_rate_table apll_24mhz_tbl[] __initconst = { /* sorted in descending order */ /* PLL_35XX_RATE(rate, m, p, s) */ PLL_35XX_RATE(1700000000, 425, 6, 0), @@ -805,8 +805,7 @@ static void __init exynos5250_clk_init(struct device_node *np) } ctx = samsung_clk_init(np, reg_base, CLK_NR_CLKS); - if (!ctx) - panic("%s: unable to allocate context.\n", __func__); + samsung_clk_of_register_fixed_ext(ctx, exynos5250_fixed_rate_ext_clks, ARRAY_SIZE(exynos5250_fixed_rate_ext_clks), ext_clk_match); diff --git a/drivers/clk/samsung/clk-exynos5260.c b/drivers/clk/samsung/clk-exynos5260.c index 7a7ed075a..a43642c36 100644 --- a/drivers/clk/samsung/clk-exynos5260.c +++ b/drivers/clk/samsung/clk-exynos5260.c @@ -22,7 +22,7 @@ * Applicable for all 2550 Type PLLS for Exynos5260, listed below * DISP_PLL, EGL_PLL, KFC_PLL, MEM_PLL, BUS_PLL, MEDIA_PLL, G3D_PLL. */ -static struct samsung_pll_rate_table pll2550_24mhz_tbl[] __initdata = { +static const struct samsung_pll_rate_table pll2550_24mhz_tbl[] __initconst = { PLL_35XX_RATE(1700000000, 425, 6, 0), PLL_35XX_RATE(1600000000, 200, 3, 0), PLL_35XX_RATE(1500000000, 250, 4, 0), @@ -55,7 +55,7 @@ static struct samsung_pll_rate_table pll2550_24mhz_tbl[] __initdata = { /* * Applicable for 2650 Type PLL for AUD_PLL. */ -static struct samsung_pll_rate_table pll2650_24mhz_tbl[] __initdata = { +static const struct samsung_pll_rate_table pll2650_24mhz_tbl[] __initconst = { PLL_36XX_RATE(1600000000, 200, 3, 0, 0), PLL_36XX_RATE(1200000000, 100, 2, 0, 0), PLL_36XX_RATE(1000000000, 250, 3, 1, 0), @@ -78,7 +78,7 @@ static struct samsung_pll_rate_table pll2650_24mhz_tbl[] __initdata = { /* CMU_AUD */ -static unsigned long aud_clk_regs[] __initdata = { +static const unsigned long aud_clk_regs[] __initconst = { MUX_SEL_AUD, DIV_AUD0, DIV_AUD1, @@ -92,7 +92,7 @@ PNAME(mout_aud_pll_user_p) = {"fin_pll", "fout_aud_pll"}; PNAME(mout_sclk_aud_i2s_p) = {"mout_aud_pll_user", "ioclk_i2s_cdclk"}; PNAME(mout_sclk_aud_pcm_p) = {"mout_aud_pll_user", "ioclk_pcm_extclk"}; -static struct samsung_mux_clock aud_mux_clks[] __initdata = { +static const struct samsung_mux_clock aud_mux_clks[] __initconst = { MUX(AUD_MOUT_AUD_PLL_USER, "mout_aud_pll_user", mout_aud_pll_user_p, MUX_SEL_AUD, 0, 1), MUX(AUD_MOUT_SCLK_AUD_I2S, "mout_sclk_aud_i2s", mout_sclk_aud_i2s_p, @@ -101,7 +101,7 @@ static struct samsung_mux_clock aud_mux_clks[] __initdata = { MUX_SEL_AUD, 8, 1), }; -static struct samsung_div_clock aud_div_clks[] __initdata = { +static const struct samsung_div_clock aud_div_clks[] __initconst = { DIV(AUD_DOUT_ACLK_AUD_131, "dout_aclk_aud_131", "mout_aud_pll_user", DIV_AUD0, 0, 4), @@ -113,7 +113,7 @@ static struct samsung_div_clock aud_div_clks[] __initdata = { DIV_AUD1, 12, 4), }; -static struct samsung_gate_clock aud_gate_clks[] __initdata = { +static const struct samsung_gate_clock aud_gate_clks[] __initconst = { GATE(AUD_SCLK_I2S, "sclk_aud_i2s", "dout_sclk_aud_i2s", EN_SCLK_AUD, 0, CLK_SET_RATE_PARENT, 0), GATE(AUD_SCLK_PCM, "sclk_aud_pcm", "dout_sclk_aud_pcm", @@ -154,7 +154,7 @@ CLK_OF_DECLARE(exynos5260_clk_aud, "samsung,exynos5260-clock-aud", /* CMU_DISP */ -static unsigned long disp_clk_regs[] __initdata = { +static const unsigned long disp_clk_regs[] __initconst = { MUX_SEL_DISP0, MUX_SEL_DISP1, MUX_SEL_DISP2, @@ -201,7 +201,7 @@ PNAME(mout_phyclk_mipi_dphy_4lmrxclk_esc0_user_p) = {"fin_pll", PNAME(mout_sclk_hdmi_spdif_p) = {"fin_pll", "ioclk_spdif_extclk", "dout_aclk_peri_aud", "phyclk_hdmi_phy_ref_cko"}; -static struct samsung_mux_clock disp_mux_clks[] __initdata = { +static const struct samsung_mux_clock disp_mux_clks[] __initconst = { MUX(DISP_MOUT_ACLK_DISP_333_USER, "mout_aclk_disp_333_user", mout_aclk_disp_333_user_p, MUX_SEL_DISP0, 0, 1), @@ -270,7 +270,7 @@ static struct samsung_mux_clock disp_mux_clks[] __initdata = { MUX_SEL_DISP4, 4, 2), }; -static struct samsung_div_clock disp_div_clks[] __initdata = { +static const struct samsung_div_clock disp_div_clks[] __initconst = { DIV(DISP_DOUT_PCLK_DISP_111, "dout_pclk_disp_111", "mout_aclk_disp_222_user", DIV_DISP, 8, 4), @@ -283,7 +283,7 @@ static struct samsung_div_clock disp_div_clks[] __initdata = { DIV_DISP, 16, 4), }; -static struct samsung_gate_clock disp_gate_clks[] __initdata = { +static const struct samsung_gate_clock disp_gate_clks[] __initconst = { GATE(DISP_MOUT_HDMI_PHY_PIXEL_USER, "sclk_hdmi_link_i_pixel", "mout_phyclk_hdmi_phy_pixel_clko_user", EN_SCLK_DISP0, 26, CLK_SET_RATE_PARENT, 0), @@ -344,7 +344,7 @@ CLK_OF_DECLARE(exynos5260_clk_disp, "samsung,exynos5260-clock-disp", /* CMU_EGL */ -static unsigned long egl_clk_regs[] __initdata = { +static const unsigned long egl_clk_regs[] __initconst = { EGL_PLL_LOCK, EGL_PLL_CON0, EGL_PLL_CON1, @@ -361,13 +361,13 @@ static unsigned long egl_clk_regs[] __initdata = { PNAME(mout_egl_b_p) = {"mout_egl_pll", "dout_bus_pll"}; PNAME(mout_egl_pll_p) = {"fin_pll", "fout_egl_pll"}; -static struct samsung_mux_clock egl_mux_clks[] __initdata = { +static const struct samsung_mux_clock egl_mux_clks[] __initconst = { MUX(EGL_MOUT_EGL_PLL, "mout_egl_pll", mout_egl_pll_p, MUX_SEL_EGL, 4, 1), MUX(EGL_MOUT_EGL_B, "mout_egl_b", mout_egl_b_p, MUX_SEL_EGL, 16, 1), }; -static struct samsung_div_clock egl_div_clks[] __initdata = { +static const struct samsung_div_clock egl_div_clks[] __initconst = { DIV(EGL_DOUT_EGL1, "dout_egl1", "mout_egl_b", DIV_EGL, 0, 3), DIV(EGL_DOUT_EGL2, "dout_egl2", "dout_egl1", DIV_EGL, 4, 3), DIV(EGL_DOUT_ACLK_EGL, "dout_aclk_egl", "dout_egl2", DIV_EGL, 8, 3), @@ -379,7 +379,7 @@ static struct samsung_div_clock egl_div_clks[] __initdata = { DIV(EGL_DOUT_EGL_PLL, "dout_egl_pll", "mout_egl_b", DIV_EGL, 24, 3), }; -static struct samsung_pll_clock egl_pll_clks[] __initdata = { +static const struct samsung_pll_clock egl_pll_clks[] __initconst = { PLL(pll_2550xx, EGL_FOUT_EGL_PLL, "fout_egl_pll", "fin_pll", EGL_PLL_LOCK, EGL_PLL_CON0, pll2550_24mhz_tbl), @@ -408,7 +408,7 @@ CLK_OF_DECLARE(exynos5260_clk_egl, "samsung,exynos5260-clock-egl", /* CMU_FSYS */ -static unsigned long fsys_clk_regs[] __initdata = { +static const unsigned long fsys_clk_regs[] __initconst = { MUX_SEL_FSYS0, MUX_SEL_FSYS1, EN_ACLK_FSYS, @@ -431,7 +431,7 @@ PNAME(mout_phyclk_usbdrd30_pipe_pclk_user_p) = {"fin_pll", PNAME(mout_phyclk_usbdrd30_phyclock_user_p) = {"fin_pll", "phyclk_usbdrd30_udrd30_phyclock"}; -static struct samsung_mux_clock fsys_mux_clks[] __initdata = { +static const struct samsung_mux_clock fsys_mux_clks[] __initconst = { MUX(FSYS_MOUT_PHYCLK_USBDRD30_PHYCLOCK_USER, "mout_phyclk_usbdrd30_phyclock_user", mout_phyclk_usbdrd30_phyclock_user_p, @@ -454,7 +454,7 @@ static struct samsung_mux_clock fsys_mux_clks[] __initdata = { MUX_SEL_FSYS1, 16, 1), }; -static struct samsung_gate_clock fsys_gate_clks[] __initdata = { +static const struct samsung_gate_clock fsys_gate_clks[] __initconst = { GATE(FSYS_PHYCLK_USBHOST20, "phyclk_usbhost20_phyclock", "mout_phyclk_usbdrd30_phyclock_user", EN_SCLK_FSYS, 1, 0, 0), @@ -508,7 +508,7 @@ CLK_OF_DECLARE(exynos5260_clk_fsys, "samsung,exynos5260-clock-fsys", /* CMU_G2D */ -static unsigned long g2d_clk_regs[] __initdata = { +static const unsigned long g2d_clk_regs[] __initconst = { MUX_SEL_G2D, MUX_STAT_G2D, DIV_G2D, @@ -535,18 +535,18 @@ static unsigned long g2d_clk_regs[] __initdata = { PNAME(mout_aclk_g2d_333_user_p) = {"fin_pll", "dout_aclk_g2d_333"}; -static struct samsung_mux_clock g2d_mux_clks[] __initdata = { +static const struct samsung_mux_clock g2d_mux_clks[] __initconst = { MUX(G2D_MOUT_ACLK_G2D_333_USER, "mout_aclk_g2d_333_user", mout_aclk_g2d_333_user_p, MUX_SEL_G2D, 0, 1), }; -static struct samsung_div_clock g2d_div_clks[] __initdata = { +static const struct samsung_div_clock g2d_div_clks[] __initconst = { DIV(G2D_DOUT_PCLK_G2D_83, "dout_pclk_g2d_83", "mout_aclk_g2d_333_user", DIV_G2D, 0, 3), }; -static struct samsung_gate_clock g2d_gate_clks[] __initdata = { +static const struct samsung_gate_clock g2d_gate_clks[] __initconst = { GATE(G2D_CLK_G2D, "clk_g2d", "mout_aclk_g2d_333_user", EN_IP_G2D, 4, 0, 0), GATE(G2D_CLK_JPEG, "clk_jpeg", "mout_aclk_g2d_333_user", @@ -599,7 +599,7 @@ CLK_OF_DECLARE(exynos5260_clk_g2d, "samsung,exynos5260-clock-g2d", /* CMU_G3D */ -static unsigned long g3d_clk_regs[] __initdata = { +static const unsigned long g3d_clk_regs[] __initconst = { G3D_PLL_LOCK, G3D_PLL_CON0, G3D_PLL_CON1, @@ -615,23 +615,23 @@ static unsigned long g3d_clk_regs[] __initdata = { PNAME(mout_g3d_pll_p) = {"fin_pll", "fout_g3d_pll"}; -static struct samsung_mux_clock g3d_mux_clks[] __initdata = { +static const struct samsung_mux_clock g3d_mux_clks[] __initconst = { MUX(G3D_MOUT_G3D_PLL, "mout_g3d_pll", mout_g3d_pll_p, MUX_SEL_G3D, 0, 1), }; -static struct samsung_div_clock g3d_div_clks[] __initdata = { +static const struct samsung_div_clock g3d_div_clks[] __initconst = { DIV(G3D_DOUT_PCLK_G3D, "dout_pclk_g3d", "dout_aclk_g3d", DIV_G3D, 0, 3), DIV(G3D_DOUT_ACLK_G3D, "dout_aclk_g3d", "mout_g3d_pll", DIV_G3D, 4, 3), }; -static struct samsung_gate_clock g3d_gate_clks[] __initdata = { +static const struct samsung_gate_clock g3d_gate_clks[] __initconst = { GATE(G3D_CLK_G3D, "clk_g3d", "dout_aclk_g3d", EN_IP_G3D, 2, 0, 0), GATE(G3D_CLK_G3D_HPM, "clk_g3d_hpm", "dout_aclk_g3d", EN_IP_G3D, 3, 0, 0), }; -static struct samsung_pll_clock g3d_pll_clks[] __initdata = { +static const struct samsung_pll_clock g3d_pll_clks[] __initconst = { PLL(pll_2550, G3D_FOUT_G3D_PLL, "fout_g3d_pll", "fin_pll", G3D_PLL_LOCK, G3D_PLL_CON0, pll2550_24mhz_tbl), @@ -662,7 +662,7 @@ CLK_OF_DECLARE(exynos5260_clk_g3d, "samsung,exynos5260-clock-g3d", /* CMU_GSCL */ -static unsigned long gscl_clk_regs[] __initdata = { +static const unsigned long gscl_clk_regs[] __initconst = { MUX_SEL_GSCL, DIV_GSCL, EN_ACLK_GSCL, @@ -692,7 +692,7 @@ PNAME(mout_aclk_m2m_400_user_p) = {"fin_pll", "dout_aclk_gscl_400"}; PNAME(mout_aclk_gscl_fimc_user_p) = {"fin_pll", "dout_aclk_gscl_400"}; PNAME(mout_aclk_csis_p) = {"dout_aclk_csis_200", "mout_aclk_gscl_fimc_user"}; -static struct samsung_mux_clock gscl_mux_clks[] __initdata = { +static const struct samsung_mux_clock gscl_mux_clks[] __initconst = { MUX(GSCL_MOUT_ACLK_GSCL_333_USER, "mout_aclk_gscl_333_user", mout_aclk_gscl_333_user_p, MUX_SEL_GSCL, 0, 1), @@ -706,7 +706,7 @@ static struct samsung_mux_clock gscl_mux_clks[] __initdata = { MUX_SEL_GSCL, 24, 1), }; -static struct samsung_div_clock gscl_div_clks[] __initdata = { +static const struct samsung_div_clock gscl_div_clks[] __initconst = { DIV(GSCL_DOUT_PCLK_M2M_100, "dout_pclk_m2m_100", "mout_aclk_m2m_400_user", DIV_GSCL, 0, 3), @@ -715,7 +715,7 @@ static struct samsung_div_clock gscl_div_clks[] __initdata = { DIV_GSCL, 4, 3), }; -static struct samsung_gate_clock gscl_gate_clks[] __initdata = { +static const struct samsung_gate_clock gscl_gate_clks[] __initconst = { GATE(GSCL_SCLK_CSIS0_WRAP, "sclk_csis0_wrap", "dout_aclk_csis_200", EN_SCLK_GSCL_FIMC, 0, CLK_SET_RATE_PARENT, 0), GATE(GSCL_SCLK_CSIS1_WRAP, "sclk_csis1_wrap", "dout_aclk_csis_200", @@ -795,7 +795,7 @@ CLK_OF_DECLARE(exynos5260_clk_gscl, "samsung,exynos5260-clock-gscl", /* CMU_ISP */ -static unsigned long isp_clk_regs[] __initdata = { +static const unsigned long isp_clk_regs[] __initconst = { MUX_SEL_ISP0, MUX_SEL_ISP1, DIV_ISP, @@ -811,14 +811,14 @@ static unsigned long isp_clk_regs[] __initdata = { PNAME(mout_isp_400_user_p) = {"fin_pll", "dout_aclk_isp1_400"}; PNAME(mout_isp_266_user_p) = {"fin_pll", "dout_aclk_isp1_266"}; -static struct samsung_mux_clock isp_mux_clks[] __initdata = { +static const struct samsung_mux_clock isp_mux_clks[] __initconst = { MUX(ISP_MOUT_ISP_266_USER, "mout_isp_266_user", mout_isp_266_user_p, MUX_SEL_ISP0, 0, 1), MUX(ISP_MOUT_ISP_400_USER, "mout_isp_400_user", mout_isp_400_user_p, MUX_SEL_ISP0, 4, 1), }; -static struct samsung_div_clock isp_div_clks[] __initdata = { +static const struct samsung_div_clock isp_div_clks[] __initconst = { DIV(ISP_DOUT_PCLK_ISP_66, "dout_pclk_isp_66", "mout_kfc", DIV_ISP, 0, 3), DIV(ISP_DOUT_PCLK_ISP_133, "dout_pclk_isp_133", "mout_kfc", @@ -830,7 +830,7 @@ static struct samsung_div_clock isp_div_clks[] __initdata = { DIV(ISP_DOUT_SCLK_MPWM, "dout_sclk_mpwm", "mout_kfc", DIV_ISP, 20, 2), }; -static struct samsung_gate_clock isp_gate_clks[] __initdata = { +static const struct samsung_gate_clock isp_gate_clks[] __initconst = { GATE(ISP_CLK_GIC, "clk_isp_gic", "mout_aclk_isp1_266", EN_IP_ISP0, 15, 0, 0), @@ -914,7 +914,7 @@ CLK_OF_DECLARE(exynos5260_clk_isp, "samsung,exynos5260-clock-isp", /* CMU_KFC */ -static unsigned long kfc_clk_regs[] __initdata = { +static const unsigned long kfc_clk_regs[] __initconst = { KFC_PLL_LOCK, KFC_PLL_CON0, KFC_PLL_CON1, @@ -932,13 +932,13 @@ static unsigned long kfc_clk_regs[] __initdata = { PNAME(mout_kfc_pll_p) = {"fin_pll", "fout_kfc_pll"}; PNAME(mout_kfc_p) = {"mout_kfc_pll", "dout_media_pll"}; -static struct samsung_mux_clock kfc_mux_clks[] __initdata = { +static const struct samsung_mux_clock kfc_mux_clks[] __initconst = { MUX(KFC_MOUT_KFC_PLL, "mout_kfc_pll", mout_kfc_pll_p, MUX_SEL_KFC0, 0, 1), MUX(KFC_MOUT_KFC, "mout_kfc", mout_kfc_p, MUX_SEL_KFC2, 0, 1), }; -static struct samsung_div_clock kfc_div_clks[] __initdata = { +static const struct samsung_div_clock kfc_div_clks[] __initconst = { DIV(KFC_DOUT_KFC1, "dout_kfc1", "mout_kfc", DIV_KFC, 0, 3), DIV(KFC_DOUT_KFC2, "dout_kfc2", "dout_kfc1", DIV_KFC, 4, 3), DIV(KFC_DOUT_KFC_ATCLK, "dout_kfc_atclk", "dout_kfc2", DIV_KFC, 8, 3), @@ -949,7 +949,7 @@ static struct samsung_div_clock kfc_div_clks[] __initdata = { DIV(KFC_DOUT_KFC_PLL, "dout_kfc_pll", "mout_kfc", DIV_KFC, 24, 3), }; -static struct samsung_pll_clock kfc_pll_clks[] __initdata = { +static const struct samsung_pll_clock kfc_pll_clks[] __initconst = { PLL(pll_2550xx, KFC_FOUT_KFC_PLL, "fout_kfc_pll", "fin_pll", KFC_PLL_LOCK, KFC_PLL_CON0, pll2550_24mhz_tbl), @@ -978,7 +978,7 @@ CLK_OF_DECLARE(exynos5260_clk_kfc, "samsung,exynos5260-clock-kfc", /* CMU_MFC */ -static unsigned long mfc_clk_regs[] __initdata = { +static const unsigned long mfc_clk_regs[] __initconst = { MUX_SEL_MFC, DIV_MFC, EN_ACLK_MFC, @@ -991,18 +991,18 @@ static unsigned long mfc_clk_regs[] __initdata = { PNAME(mout_aclk_mfc_333_user_p) = {"fin_pll", "dout_aclk_mfc_333"}; -static struct samsung_mux_clock mfc_mux_clks[] __initdata = { +static const struct samsung_mux_clock mfc_mux_clks[] __initconst = { MUX(MFC_MOUT_ACLK_MFC_333_USER, "mout_aclk_mfc_333_user", mout_aclk_mfc_333_user_p, MUX_SEL_MFC, 0, 1), }; -static struct samsung_div_clock mfc_div_clks[] __initdata = { +static const struct samsung_div_clock mfc_div_clks[] __initconst = { DIV(MFC_DOUT_PCLK_MFC_83, "dout_pclk_mfc_83", "mout_aclk_mfc_333_user", DIV_MFC, 0, 3), }; -static struct samsung_gate_clock mfc_gate_clks[] __initdata = { +static const struct samsung_gate_clock mfc_gate_clks[] __initconst = { GATE(MFC_CLK_MFC, "clk_mfc", "mout_aclk_mfc_333_user", EN_IP_MFC, 1, 0, 0), GATE(MFC_CLK_SMMU2_MFCM0, "clk_smmu2_mfcm0", "mout_aclk_mfc_333_user", @@ -1034,7 +1034,7 @@ CLK_OF_DECLARE(exynos5260_clk_mfc, "samsung,exynos5260-clock-mfc", /* CMU_MIF */ -static unsigned long mif_clk_regs[] __initdata = { +static const unsigned long mif_clk_regs[] __initconst = { MEM_PLL_LOCK, BUS_PLL_LOCK, MEDIA_PLL_LOCK, @@ -1076,7 +1076,7 @@ PNAME(mout_mif_drex2x_p) = {"dout_mem_pll", "dout_bus_pll"}; PNAME(mout_clkm_phy_p) = {"mout_mif_drex", "dout_media_pll"}; PNAME(mout_clk2x_phy_p) = {"mout_mif_drex2x", "dout_media_pll"}; -static struct samsung_mux_clock mif_mux_clks[] __initdata = { +static const struct samsung_mux_clock mif_mux_clks[] __initconst = { MUX(MIF_MOUT_MEM_PLL, "mout_mem_pll", mout_mem_pll_p, MUX_SEL_MIF, 0, 1), MUX(MIF_MOUT_BUS_PLL, "mout_bus_pll", mout_bus_pll_p, @@ -1093,7 +1093,7 @@ static struct samsung_mux_clock mif_mux_clks[] __initdata = { MUX_SEL_MIF, 24, 1), }; -static struct samsung_div_clock mif_div_clks[] __initdata = { +static const struct samsung_div_clock mif_div_clks[] __initconst = { DIV(MIF_DOUT_MEDIA_PLL, "dout_media_pll", "mout_media_pll", DIV_MIF, 0, 3), DIV(MIF_DOUT_MEM_PLL, "dout_mem_pll", "mout_mem_pll", @@ -1112,7 +1112,7 @@ static struct samsung_div_clock mif_div_clks[] __initdata = { DIV_MIF, 28, 4), }; -static struct samsung_gate_clock mif_gate_clks[] __initdata = { +static const struct samsung_gate_clock mif_gate_clks[] __initconst = { GATE(MIF_CLK_LPDDR3PHY_WRAP0, "clk_lpddr3phy_wrap0", "dout_clk2x_phy", EN_IP_MIF, 12, CLK_IGNORE_UNUSED, 0), GATE(MIF_CLK_LPDDR3PHY_WRAP1, "clk_lpddr3phy_wrap1", "dout_clk2x_phy", @@ -1146,7 +1146,7 @@ static struct samsung_gate_clock mif_gate_clks[] __initdata = { CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0), }; -static struct samsung_pll_clock mif_pll_clks[] __initdata = { +static const struct samsung_pll_clock mif_pll_clks[] __initconst = { PLL(pll_2550xx, MIF_FOUT_MEM_PLL, "fout_mem_pll", "fin_pll", MEM_PLL_LOCK, MEM_PLL_CON0, pll2550_24mhz_tbl), @@ -1183,7 +1183,7 @@ CLK_OF_DECLARE(exynos5260_clk_mif, "samsung,exynos5260-clock-mif", /* CMU_PERI */ -static unsigned long peri_clk_regs[] __initdata = { +static const unsigned long peri_clk_regs[] __initconst = { MUX_SEL_PERI, MUX_SEL_PERI1, DIV_PERI, @@ -1219,7 +1219,7 @@ PNAME(mout_sclk_i2scod_p) = {"ioclk_i2s_cdclk", "fin_pll", "dout_aclk_peri_aud", PNAME(mout_sclk_spdif_p) = {"ioclk_spdif_extclk", "fin_pll", "dout_aclk_peri_aud", "phyclk_hdmi_phy_ref_cko"}; -static struct samsung_mux_clock peri_mux_clks[] __initdata = { +static const struct samsung_mux_clock peri_mux_clks[] __initconst = { MUX(PERI_MOUT_SCLK_PCM, "mout_sclk_pcm", mout_sclk_pcm_p, MUX_SEL_PERI1, 4, 2), MUX(PERI_MOUT_SCLK_I2SCOD, "mout_sclk_i2scod", mout_sclk_i2scod_p, @@ -1228,12 +1228,12 @@ static struct samsung_mux_clock peri_mux_clks[] __initdata = { MUX_SEL_PERI1, 20, 2), }; -static struct samsung_div_clock peri_div_clks[] __initdata = { +static const struct samsung_div_clock peri_div_clks[] __initconst = { DIV(PERI_DOUT_PCM, "dout_pcm", "mout_sclk_pcm", DIV_PERI, 0, 8), DIV(PERI_DOUT_I2S, "dout_i2s", "mout_sclk_i2scod", DIV_PERI, 8, 6), }; -static struct samsung_gate_clock peri_gate_clks[] __initdata = { +static const struct samsung_gate_clock peri_gate_clks[] __initconst = { GATE(PERI_SCLK_PCM1, "sclk_pcm1", "dout_pcm", EN_SCLK_PERI, 0, CLK_SET_RATE_PARENT, 0), GATE(PERI_SCLK_I2S, "sclk_i2s", "dout_i2s", EN_SCLK_PERI, 1, @@ -1389,7 +1389,7 @@ CLK_OF_DECLARE(exynos5260_clk_peri, "samsung,exynos5260-clock-peri", /* CMU_TOP */ -static unsigned long top_clk_regs[] __initdata = { +static const unsigned long top_clk_regs[] __initconst = { DISP_PLL_LOCK, AUD_PLL_LOCK, DISP_PLL_CON0, @@ -1430,7 +1430,7 @@ static unsigned long top_clk_regs[] __initdata = { }; /* fixed rate clocks generated inside the soc */ -static struct samsung_fixed_rate_clock fixed_rate_clks[] __initdata = { +static const struct samsung_fixed_rate_clock fixed_rate_clks[] __initconst = { FRATE(PHYCLK_DPTX_PHY_CH3_TXD_CLK, "phyclk_dptx_phy_ch3_txd_clk", NULL, 0, 270000000), FRATE(PHYCLK_DPTX_PHY_CH2_TXD_CLK, "phyclk_dptx_phy_ch2_txd_clk", NULL, @@ -1513,7 +1513,7 @@ PNAME(mout_sclk_fsys_mmc1_sdclkin_b_p) = {"mout_sclk_fsys_mmc1_sdclkin_a", PNAME(mout_sclk_fsys_mmc2_sdclkin_b_p) = {"mout_sclk_fsys_mmc2_sdclkin_a", "mout_mediatop_pll_user"}; -static struct samsung_mux_clock top_mux_clks[] __initdata = { +static const struct samsung_mux_clock top_mux_clks[] __initconst = { MUX(TOP_MOUT_MEDIATOP_PLL_USER, "mout_mediatop_pll_user", mout_mediatop_pll_user_p, MUX_SEL_TOP_PLL0, 0, 1), @@ -1673,7 +1673,7 @@ static struct samsung_mux_clock top_mux_clks[] __initdata = { MUX_SEL_TOP_GSCL, 20, 1), }; -static struct samsung_div_clock top_div_clks[] __initdata = { +static const struct samsung_div_clock top_div_clks[] __initconst = { DIV(TOP_DOUT_ACLK_G2D_333, "dout_aclk_g2d_333", "mout_aclk_g2d_333", DIV_TOP_G2D_MFC, 0, 3), DIV(TOP_DOUT_ACLK_MFC_333, "dout_aclk_mfc_333", "mout_aclk_mfc_333", @@ -1794,7 +1794,7 @@ static struct samsung_div_clock top_div_clks[] __initdata = { }; -static struct samsung_gate_clock top_gate_clks[] __initdata = { +static const struct samsung_gate_clock top_gate_clks[] __initconst = { GATE(TOP_SCLK_MMC0, "sclk_fsys_mmc0_sdclkin", "dout_sclk_fsys_mmc0_sdclkin_b", EN_SCLK_TOP, 7, CLK_SET_RATE_PARENT, 0), @@ -1809,7 +1809,7 @@ static struct samsung_gate_clock top_gate_clks[] __initdata = { CLK_SET_RATE_PARENT, 0), }; -static struct samsung_pll_clock top_pll_clks[] __initdata = { +static const struct samsung_pll_clock top_pll_clks[] __initconst = { PLL(pll_2550xx, TOP_FOUT_DISP_PLL, "fout_disp_pll", "fin_pll", DISP_PLL_LOCK, DISP_PLL_CON0, pll2550_24mhz_tbl), diff --git a/drivers/clk/samsung/clk-exynos5410.c b/drivers/clk/samsung/clk-exynos5410.c index d5d5dcabc..54ec486a5 100644 --- a/drivers/clk/samsung/clk-exynos5410.c +++ b/drivers/clk/samsung/clk-exynos5410.c @@ -31,11 +31,14 @@ #define SRC_CPU 0x200 #define DIV_CPU0 0x500 #define SRC_CPERI1 0x4204 +#define GATE_IP_G2D 0x8800 #define DIV_TOP0 0x10510 #define DIV_TOP1 0x10514 +#define DIV_FSYS0 0x10548 #define DIV_FSYS1 0x1054c #define DIV_FSYS2 0x10550 #define DIV_PERIC0 0x10558 +#define DIV_PERIC3 0x10564 #define SRC_TOP0 0x10210 #define SRC_TOP1 0x10214 #define SRC_TOP2 0x10218 @@ -44,6 +47,8 @@ #define SRC_MASK_FSYS 0x10340 #define SRC_MASK_PERIC0 0x10350 #define GATE_BUS_FSYS0 0x10740 +#define GATE_TOP_SCLK_FSYS 0x10840 +#define GATE_TOP_SCLK_PERIC 0x10850 #define GATE_IP_FSYS 0x10944 #define GATE_IP_PERIC 0x10950 #define GATE_IP_PERIS 0x10960 @@ -71,12 +76,13 @@ PNAME(mout_kfc_p) = { "mout_kpll", "sclk_mpll", }; PNAME(mpll_user_p) = { "fin_pll", "sclk_mpll", }; PNAME(bpll_user_p) = { "fin_pll", "sclk_bpll", }; PNAME(mpll_bpll_p) = { "sclk_mpll_muxed", "sclk_bpll_muxed", }; +PNAME(sclk_mpll_bpll_p) = { "sclk_mpll_bpll", "fin_pll", }; PNAME(group2_p) = { "fin_pll", "fin_pll", "none", "none", "none", "none", "sclk_mpll_bpll", "none", "none", "sclk_cpll" }; -static struct samsung_mux_clock exynos5410_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos5410_mux_clks[] __initconst = { MUX(0, "mout_apll", apll_p, SRC_CPU, 0, 1), MUX(0, "mout_cpu", mout_cpu_p, SRC_CPU, 16, 1), @@ -96,16 +102,20 @@ static struct samsung_mux_clock exynos5410_mux_clks[] __initdata = { MUX(0, "mout_mmc0", group2_p, SRC_FSYS, 0, 4), MUX(0, "mout_mmc1", group2_p, SRC_FSYS, 4, 4), MUX(0, "mout_mmc2", group2_p, SRC_FSYS, 8, 4), + MUX(0, "mout_usbd300", sclk_mpll_bpll_p, SRC_FSYS, 28, 1), + MUX(0, "mout_usbd301", sclk_mpll_bpll_p, SRC_FSYS, 29, 1), MUX(0, "mout_uart0", group2_p, SRC_PERIC0, 0, 4), MUX(0, "mout_uart1", group2_p, SRC_PERIC0, 4, 4), MUX(0, "mout_uart2", group2_p, SRC_PERIC0, 8, 4), + MUX(0, "mout_uart3", group2_p, SRC_PERIC0, 12, 4), + MUX(0, "mout_pwm", group2_p, SRC_PERIC0, 24, 4), MUX(0, "mout_aclk200", mpll_bpll_p, SRC_TOP0, 12, 1), MUX(0, "mout_aclk400", mpll_bpll_p, SRC_TOP0, 20, 1), }; -static struct samsung_div_clock exynos5410_div_clks[] __initdata = { +static const struct samsung_div_clock exynos5410_div_clks[] __initconst = { DIV(0, "div_arm", "mout_cpu", DIV_CPU0, 0, 3), DIV(0, "div_arm2", "div_arm", DIV_CPU0, 28, 3), @@ -121,6 +131,11 @@ static struct samsung_div_clock exynos5410_div_clks[] __initdata = { DIV(0, "aclk66_pre", "sclk_mpll_muxed", DIV_TOP1, 24, 3), DIV(0, "aclk66", "aclk66_pre", DIV_TOP0, 0, 3), + DIV(0, "dout_usbphy300", "mout_usbd300", DIV_FSYS0, 16, 4), + DIV(0, "dout_usbphy301", "mout_usbd301", DIV_FSYS0, 20, 4), + DIV(0, "dout_usbd300", "mout_usbd300", DIV_FSYS0, 24, 4), + DIV(0, "dout_usbd301", "mout_usbd301", DIV_FSYS0, 28, 4), + DIV(0, "div_mmc0", "mout_mmc0", DIV_FSYS1, 0, 4), DIV(0, "div_mmc1", "mout_mmc1", DIV_FSYS1, 16, 4), DIV(0, "div_mmc2", "mout_mmc2", DIV_FSYS2, 0, 4), @@ -137,12 +152,19 @@ static struct samsung_div_clock exynos5410_div_clks[] __initdata = { DIV(0, "div_uart2", "mout_uart2", DIV_PERIC0, 8, 4), DIV(0, "div_uart3", "mout_uart3", DIV_PERIC0, 12, 4), + DIV(0, "dout_pwm", "mout_pwm", DIV_PERIC3, 0, 4), + DIV(0, "aclk200", "mout_aclk200", DIV_TOP0, 12, 3), + DIV(0, "aclk266", "mpll_user_p", DIV_TOP0, 16, 3), DIV(0, "aclk400", "mout_aclk400", DIV_TOP0, 24, 3), }; -static struct samsung_gate_clock exynos5410_gate_clks[] __initdata = { +static const struct samsung_gate_clock exynos5410_gate_clks[] __initconst = { + GATE(CLK_SSS, "sss", "aclk266", GATE_IP_G2D, 2, 0, 0), GATE(CLK_MCT, "mct", "aclk66", GATE_IP_PERIS, 18, 0, 0), + GATE(CLK_WDT, "wdt", "aclk66", GATE_IP_PERIS, 19, 0, 0), + GATE(CLK_RTC, "rtc", "aclk66", GATE_IP_PERIS, 20, 0, 0), + GATE(CLK_TMU, "tmu", "aclk66", GATE_IP_PERIS, 21, 0, 0), GATE(CLK_SCLK_MMC0, "sclk_mmc0", "div_mmc_pre0", SRC_MASK_FSYS, 0, CLK_SET_RATE_PARENT, 0), @@ -155,9 +177,31 @@ static struct samsung_gate_clock exynos5410_gate_clks[] __initdata = { GATE(CLK_MMC1, "sdmmc1", "aclk200", GATE_BUS_FSYS0, 13, 0, 0), GATE(CLK_MMC2, "sdmmc2", "aclk200", GATE_BUS_FSYS0, 14, 0, 0), + GATE(CLK_SCLK_USBPHY301, "sclk_usbphy301", "dout_usbphy301", + GATE_TOP_SCLK_FSYS, 7, CLK_SET_RATE_PARENT, 0), + GATE(CLK_SCLK_USBPHY300, "sclk_usbphy300", "dout_usbphy300", + GATE_TOP_SCLK_FSYS, 8, CLK_SET_RATE_PARENT, 0), + GATE(CLK_SCLK_USBD300, "sclk_usbd300", "dout_usbd300", + GATE_TOP_SCLK_FSYS, 9, CLK_SET_RATE_PARENT, 0), + GATE(CLK_SCLK_USBD301, "sclk_usbd301", "dout_usbd301", + GATE_TOP_SCLK_FSYS, 10, CLK_SET_RATE_PARENT, 0), + + GATE(CLK_SCLK_PWM, "sclk_pwm", "dout_pwm", + GATE_TOP_SCLK_PERIC, 11, CLK_SET_RATE_PARENT, 0), + GATE(CLK_UART0, "uart0", "aclk66", GATE_IP_PERIC, 0, 0, 0), GATE(CLK_UART1, "uart1", "aclk66", GATE_IP_PERIC, 1, 0, 0), GATE(CLK_UART2, "uart2", "aclk66", GATE_IP_PERIC, 2, 0, 0), + GATE(CLK_UART3, "uart3", "aclk66", GATE_IP_PERIC, 3, 0, 0), + GATE(CLK_I2C0, "i2c0", "aclk66", GATE_IP_PERIC, 6, 0, 0), + GATE(CLK_I2C1, "i2c1", "aclk66", GATE_IP_PERIC, 7, 0, 0), + GATE(CLK_I2C2, "i2c2", "aclk66", GATE_IP_PERIC, 8, 0, 0), + GATE(CLK_I2C3, "i2c3", "aclk66", GATE_IP_PERIC, 9, 0, 0), + GATE(CLK_USI0, "usi0", "aclk66", GATE_IP_PERIC, 10, 0, 0), + GATE(CLK_USI1, "usi1", "aclk66", GATE_IP_PERIC, 11, 0, 0), + GATE(CLK_USI2, "usi2", "aclk66", GATE_IP_PERIC, 12, 0, 0), + GATE(CLK_USI3, "usi3", "aclk66", GATE_IP_PERIC, 13, 0, 0), + GATE(CLK_PWM, "pwm", "aclk66", GATE_IP_PERIC, 24, 0, 0), GATE(CLK_SCLK_UART0, "sclk_uart0", "div_uart0", SRC_MASK_PERIC0, 0, CLK_SET_RATE_PARENT, 0), @@ -165,9 +209,15 @@ static struct samsung_gate_clock exynos5410_gate_clks[] __initdata = { SRC_MASK_PERIC0, 4, CLK_SET_RATE_PARENT, 0), GATE(CLK_SCLK_UART2, "sclk_uart2", "div_uart2", SRC_MASK_PERIC0, 8, CLK_SET_RATE_PARENT, 0), + GATE(CLK_SCLK_UART3, "sclk_uart3", "div_uart3", + SRC_MASK_PERIC0, 12, CLK_SET_RATE_PARENT, 0), + + GATE(CLK_USBH20, "usbh20", "aclk200_fsys", GATE_IP_FSYS, 18, 0, 0), + GATE(CLK_USBD300, "usbd300", "aclk200_fsys", GATE_IP_FSYS, 19, 0, 0), + GATE(CLK_USBD301, "usbd301", "aclk200_fsys", GATE_IP_FSYS, 20, 0, 0), }; -static struct samsung_pll_clock exynos5410_plls[nr_plls] __initdata = { +static const struct samsung_pll_clock exynos5410_plls[nr_plls] __initconst = { [apll] = PLL(pll_35xx, CLK_FOUT_APLL, "fout_apll", "fin_pll", APLL_LOCK, APLL_CON0, NULL), [cpll] = PLL(pll_35xx, CLK_FOUT_CPLL, "fout_cpll", "fin_pll", CPLL_LOCK, diff --git a/drivers/clk/samsung/clk-exynos5420.c b/drivers/clk/samsung/clk-exynos5420.c index 92382cef9..bb196ca21 100644 --- a/drivers/clk/samsung/clk-exynos5420.c +++ b/drivers/clk/samsung/clk-exynos5420.c @@ -160,7 +160,7 @@ static struct samsung_clk_reg_dump *exynos5800_save; * list of controller registers to be saved and restored during a * suspend/resume cycle. */ -static unsigned long exynos5x_clk_regs[] __initdata = { +static const unsigned long exynos5x_clk_regs[] __initconst = { SRC_CPU, DIV_CPU0, DIV_CPU1, @@ -248,7 +248,7 @@ static unsigned long exynos5x_clk_regs[] __initdata = { DIV_KFC0, }; -static unsigned long exynos5800_clk_regs[] __initdata = { +static const unsigned long exynos5800_clk_regs[] __initconst = { SRC_TOP8, SRC_TOP9, SRC_CAM, @@ -306,7 +306,7 @@ static struct syscore_ops exynos5420_clk_syscore_ops = { .resume = exynos5420_clk_resume, }; -static void exynos5420_clk_sleep_init(void) +static void __init exynos5420_clk_sleep_init(void) { exynos5x_save = samsung_clk_alloc_reg_dump(exynos5x_clk_regs, ARRAY_SIZE(exynos5x_clk_regs)); @@ -333,7 +333,7 @@ err_soc: return; } #else -static void exynos5420_clk_sleep_init(void) {} +static void __init exynos5420_clk_sleep_init(void) {} #endif /* list of all parent clocks */ @@ -484,7 +484,7 @@ static struct samsung_fixed_rate_clock }; /* fixed rate clocks generated inside the soc */ -static struct samsung_fixed_rate_clock exynos5x_fixed_rate_clks[] __initdata = { +static const struct samsung_fixed_rate_clock exynos5x_fixed_rate_clks[] __initconst = { FRATE(CLK_SCLK_HDMIPHY, "sclk_hdmiphy", NULL, 0, 24000000), FRATE(0, "sclk_pwi", NULL, 0, 24000000), FRATE(0, "sclk_usbh20", NULL, 0, 48000000), @@ -492,19 +492,19 @@ static struct samsung_fixed_rate_clock exynos5x_fixed_rate_clks[] __initdata = { FRATE(0, "sclk_usbh20_scan_clk", NULL, 0, 480000000), }; -static struct samsung_fixed_factor_clock - exynos5x_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock + exynos5x_fixed_factor_clks[] __initconst = { FFACTOR(0, "ff_hsic_12m", "fin_pll", 1, 2, 0), FFACTOR(0, "ff_sw_aclk66", "mout_sw_aclk66", 1, 2, 0), }; -static struct samsung_fixed_factor_clock - exynos5800_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock + exynos5800_fixed_factor_clks[] __initconst = { FFACTOR(0, "ff_dout_epll2", "mout_sclk_epll", 1, 2, 0), FFACTOR(0, "ff_dout_spll2", "mout_sclk_spll", 1, 2, 0), }; -static struct samsung_mux_clock exynos5800_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos5800_mux_clks[] __initconst = { MUX(0, "mout_aclk400_isp", mout_group3_5800_p, SRC_TOP0, 0, 3), MUX(0, "mout_aclk400_mscl", mout_group3_5800_p, SRC_TOP0, 4, 3), MUX(0, "mout_aclk400_wcore", mout_group2_5800_p, SRC_TOP0, 16, 3), @@ -553,7 +553,7 @@ static struct samsung_mux_clock exynos5800_mux_clks[] __initdata = { MUX(0, "mout_fimd1", mout_group2_p, SRC_DISP10, 4, 3), }; -static struct samsung_div_clock exynos5800_div_clks[] __initdata = { +static const struct samsung_div_clock exynos5800_div_clks[] __initconst = { DIV(CLK_DOUT_ACLK400_WCORE, "dout_aclk400_wcore", "mout_aclk400_wcore", DIV_TOP0, 16, 3), DIV(0, "dout_aclk550_cam", "mout_aclk550_cam", @@ -569,14 +569,14 @@ static struct samsung_div_clock exynos5800_div_clks[] __initdata = { DIV(0, "dout_sclk_sw", "sclk_spll", DIV_TOP9, 24, 6), }; -static struct samsung_gate_clock exynos5800_gate_clks[] __initdata = { +static const struct samsung_gate_clock exynos5800_gate_clks[] __initconst = { GATE(CLK_ACLK550_CAM, "aclk550_cam", "mout_user_aclk550_cam", GATE_BUS_TOP, 24, 0, 0), GATE(CLK_ACLK432_SCALER, "aclk432_scaler", "mout_user_aclk432_scaler", GATE_BUS_TOP, 27, 0, 0), }; -static struct samsung_mux_clock exynos5420_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos5420_mux_clks[] __initconst = { MUX(0, "sclk_bpll", mout_bpll_p, TOP_SPARE2, 0, 1), MUX(0, "mout_aclk400_wcore_bpll", mout_aclk400_wcore_bpll_p, TOP_SPARE2, 4, 1), @@ -606,12 +606,12 @@ static struct samsung_mux_clock exynos5420_mux_clks[] __initdata = { MUX(0, "mout_fimd1", mout_group3_p, SRC_DISP10, 4, 1), }; -static struct samsung_div_clock exynos5420_div_clks[] __initdata = { +static const struct samsung_div_clock exynos5420_div_clks[] __initconst = { DIV(CLK_DOUT_ACLK400_WCORE, "dout_aclk400_wcore", "mout_aclk400_wcore_bpll", DIV_TOP0, 16, 3), }; -static struct samsung_mux_clock exynos5x_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos5x_mux_clks[] __initconst = { MUX(0, "mout_user_pclk66_gpio", mout_user_pclk66_gpio_p, SRC_TOP7, 4, 1), MUX(0, "mout_mspll_kfc", mout_mspll_cpu_p, SRC_TOP7, 8, 2), @@ -778,7 +778,7 @@ static struct samsung_mux_clock exynos5x_mux_clks[] __initdata = { MUX(0, "mout_isp_sensor", mout_group2_p, SRC_ISP, 28, 3), }; -static struct samsung_div_clock exynos5x_div_clks[] __initdata = { +static const struct samsung_div_clock exynos5x_div_clks[] __initconst = { DIV(0, "div_arm", "mout_cpu", DIV_CPU0, 0, 3), DIV(0, "sclk_apll", "mout_apll", DIV_CPU0, 24, 3), DIV(0, "armclk2", "div_arm", DIV_CPU0, 28, 3), @@ -911,7 +911,7 @@ static struct samsung_div_clock exynos5x_div_clks[] __initdata = { CLK_SET_RATE_PARENT, 0), }; -static struct samsung_gate_clock exynos5x_gate_clks[] __initdata = { +static const struct samsung_gate_clock exynos5x_gate_clks[] __initconst = { /* G2D */ GATE(CLK_MDMA0, "mdma0", "aclk266_g2d", GATE_IP_G2D, 1, 0, 0), GATE(CLK_SSS, "sss", "aclk266_g2d", GATE_IP_G2D, 2, 0, 0), @@ -946,7 +946,7 @@ static struct samsung_gate_clock exynos5x_gate_clks[] __initdata = { GATE_BUS_TOP, 13, 0, 0), GATE(0, "aclk166", "mout_user_aclk166", GATE_BUS_TOP, 14, CLK_IGNORE_UNUSED, 0), - GATE(0, "aclk333", "mout_user_aclk333", + GATE(CLK_ACLK333, "aclk333", "mout_user_aclk333", GATE_BUS_TOP, 15, CLK_IGNORE_UNUSED, 0), GATE(0, "aclk400_isp", "mout_user_aclk400_isp", GATE_BUS_TOP, 16, 0, 0), @@ -1219,7 +1219,7 @@ static struct samsung_gate_clock exynos5x_gate_clks[] __initdata = { GATE(CLK_G3D, "g3d", "mout_user_aclk_g3d", GATE_IP_G3D, 9, 0, 0), }; -static const struct samsung_pll_rate_table exynos5420_pll2550x_24mhz_tbl[] = { +static const struct samsung_pll_rate_table exynos5420_pll2550x_24mhz_tbl[] __initconst = { PLL_35XX_RATE(2000000000, 250, 3, 0), PLL_35XX_RATE(1900000000, 475, 6, 0), PLL_35XX_RATE(1800000000, 225, 3, 0), @@ -1356,8 +1356,6 @@ static void __init exynos5x_clk_init(struct device_node *np, exynos5x_soc = soc; ctx = samsung_clk_init(np, reg_base, CLK_NR_CLKS); - if (!ctx) - panic("%s: unable to allocate context.\n", __func__); samsung_clk_of_register_fixed_ext(ctx, exynos5x_fixed_rate_ext_clks, ARRAY_SIZE(exynos5x_fixed_rate_ext_clks), diff --git a/drivers/clk/samsung/clk-exynos5433.c b/drivers/clk/samsung/clk-exynos5433.c index 128527b8f..ea1608682 100644 --- a/drivers/clk/samsung/clk-exynos5433.c +++ b/drivers/clk/samsung/clk-exynos5433.c @@ -11,10 +11,12 @@ #include #include +#include #include #include "clk.h" +#include "clk-cpu.h" #include "clk-pll.h" /* @@ -108,7 +110,7 @@ #define ENABLE_CMU_TOP 0x0c00 #define ENABLE_CMU_TOP_DIV_STAT 0x0c04 -static unsigned long top_clk_regs[] __initdata = { +static const unsigned long top_clk_regs[] __initconst = { ISP_PLL_LOCK, AUD_PLL_LOCK, ISP_PLL_CON0, @@ -218,11 +220,11 @@ PNAME(mout_sclk_audio0_p) = { "ioclk_audiocdclk0", "oscclk", PNAME(mout_sclk_hdmi_spdif_p) = { "sclk_audio1", "ioclk_spdif_extclk", }; -static struct samsung_fixed_factor_clock top_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock top_fixed_factor_clks[] __initconst = { FFACTOR(0, "oscclk_efuse_common", "oscclk", 1, 1, 0), }; -static struct samsung_fixed_rate_clock top_fixed_clks[] __initdata = { +static const struct samsung_fixed_rate_clock top_fixed_clks[] __initconst = { /* Xi2s{0|1}CDCLK input clock for I2S/PCM */ FRATE(0, "ioclk_audiocdclk1", NULL, 0, 100000000), FRATE(0, "ioclk_audiocdclk0", NULL, 0, 100000000), @@ -238,7 +240,7 @@ static struct samsung_fixed_rate_clock top_fixed_clks[] __initdata = { FRATE(0, "ioclk_i2s1_bclk_in", NULL, 0, 12288000), }; -static struct samsung_mux_clock top_mux_clks[] __initdata = { +static const struct samsung_mux_clock top_mux_clks[] __initconst = { /* MUX_SEL_TOP0 */ MUX(CLK_MOUT_AUD_PLL, "mout_aud_pll", mout_aud_pll_p, MUX_SEL_TOP0, 4, 1), @@ -374,7 +376,7 @@ static struct samsung_mux_clock top_mux_clks[] __initdata = { mout_sclk_hdmi_spdif_p, MUX_SEL_TOP_DISP, 0, 1), }; -static struct samsung_div_clock top_div_clks[] __initdata = { +static const struct samsung_div_clock top_div_clks[] __initconst = { /* DIV_TOP0 */ DIV(CLK_DIV_ACLK_CAM1_333, "div_aclk_cam1_333", "mout_aclk_cam1_333", DIV_TOP0, 28, 3), @@ -538,7 +540,7 @@ static struct samsung_div_clock top_div_clks[] __initdata = { DIV_TOP_PERIC4, 0, 4), }; -static struct samsung_gate_clock top_gate_clks[] __initdata = { +static const struct samsung_gate_clock top_gate_clks[] __initconst = { /* ENABLE_ACLK_TOP */ GATE(CLK_ACLK_G3D_400, "aclk_g3d_400", "div_aclk_g3d_400", ENABLE_ACLK_TOP, 30, 0, 0), @@ -639,7 +641,7 @@ static struct samsung_gate_clock top_gate_clks[] __initdata = { /* ENABLE_SCLK_TOP_FSYS */ GATE(CLK_SCLK_PCIE_100_FSYS, "sclk_pcie_100_fsys", "div_sclk_pcie_100", - ENABLE_SCLK_TOP_FSYS, 7, 0, 0), + ENABLE_SCLK_TOP_FSYS, 7, CLK_IGNORE_UNUSED, 0), GATE(CLK_SCLK_MMC2_FSYS, "sclk_mmc2_fsys", "div_sclk_mmc2_b", ENABLE_SCLK_TOP_FSYS, 6, CLK_SET_RATE_PARENT, 0), GATE(CLK_SCLK_MMC1_FSYS, "sclk_mmc1_fsys", "div_sclk_mmc1_b", @@ -668,11 +670,14 @@ static struct samsung_gate_clock top_gate_clks[] __initdata = { GATE(CLK_SCLK_PCM1_PERIC, "sclk_pcm1_peric", "div_sclk_pcm1", ENABLE_SCLK_TOP_PERIC, 7, CLK_SET_RATE_PARENT, 0), GATE(CLK_SCLK_UART2_PERIC, "sclk_uart2_peric", "div_sclk_uart2", - ENABLE_SCLK_TOP_PERIC, 5, CLK_SET_RATE_PARENT, 0), + ENABLE_SCLK_TOP_PERIC, 5, CLK_SET_RATE_PARENT | + CLK_IGNORE_UNUSED, 0), GATE(CLK_SCLK_UART1_PERIC, "sclk_uart1_peric", "div_sclk_uart1", - ENABLE_SCLK_TOP_PERIC, 4, CLK_SET_RATE_PARENT, 0), + ENABLE_SCLK_TOP_PERIC, 4, CLK_SET_RATE_PARENT | + CLK_IGNORE_UNUSED, 0), GATE(CLK_SCLK_UART0_PERIC, "sclk_uart0_peric", "div_sclk_uart0", - ENABLE_SCLK_TOP_PERIC, 3, CLK_SET_RATE_PARENT, 0), + ENABLE_SCLK_TOP_PERIC, 3, CLK_SET_RATE_PARENT | + CLK_IGNORE_UNUSED, 0), GATE(CLK_SCLK_SPI2_PERIC, "sclk_spi2_peric", "div_sclk_spi2_b", ENABLE_SCLK_TOP_PERIC, 2, CLK_SET_RATE_PARENT, 0), GATE(CLK_SCLK_SPI1_PERIC, "sclk_spi1_peric", "div_sclk_spi1_b", @@ -693,7 +698,7 @@ static struct samsung_gate_clock top_gate_clks[] __initdata = { * ATLAS_PLL & APOLLO_PLL & MEM0_PLL & MEM1_PLL & BUS_PLL & MFC_PLL * & MPHY_PLL & G3D_PLL & DISP_PLL & ISP_PLL */ -static struct samsung_pll_rate_table exynos5443_pll_rates[] = { +static const struct samsung_pll_rate_table exynos5443_pll_rates[] __initconst = { PLL_35XX_RATE(2500000000U, 625, 6, 0), PLL_35XX_RATE(2400000000U, 500, 5, 0), PLL_35XX_RATE(2300000000U, 575, 6, 0), @@ -744,7 +749,7 @@ static struct samsung_pll_rate_table exynos5443_pll_rates[] = { }; /* AUD_PLL */ -static struct samsung_pll_rate_table exynos5443_aud_pll_rates[] = { +static const struct samsung_pll_rate_table exynos5443_aud_pll_rates[] __initconst = { PLL_36XX_RATE(400000000U, 200, 3, 2, 0), PLL_36XX_RATE(393216000U, 197, 3, 2, -25690), PLL_36XX_RATE(384000000U, 128, 2, 2, 0), @@ -757,14 +762,14 @@ static struct samsung_pll_rate_table exynos5443_aud_pll_rates[] = { { /* sentinel */ } }; -static struct samsung_pll_clock top_pll_clks[] __initdata = { +static const struct samsung_pll_clock top_pll_clks[] __initconst = { PLL(pll_35xx, CLK_FOUT_ISP_PLL, "fout_isp_pll", "oscclk", ISP_PLL_LOCK, ISP_PLL_CON0, exynos5443_pll_rates), PLL(pll_36xx, CLK_FOUT_AUD_PLL, "fout_aud_pll", "oscclk", AUD_PLL_LOCK, AUD_PLL_CON0, exynos5443_aud_pll_rates), }; -static struct samsung_cmu_info top_cmu_info __initdata = { +static const struct samsung_cmu_info top_cmu_info __initconst = { .pll_clks = top_pll_clks, .nr_pll_clks = ARRAY_SIZE(top_pll_clks), .mux_clks = top_mux_clks, @@ -800,7 +805,7 @@ CLK_OF_DECLARE(exynos5433_cmu_top, "samsung,exynos5433-cmu-top", #define DIV_CPIF 0x0600 #define ENABLE_SCLK_CPIF 0x0a00 -static unsigned long cpif_clk_regs[] __initdata = { +static const unsigned long cpif_clk_regs[] __initconst = { MPHY_PLL_LOCK, MPHY_PLL_CON0, MPHY_PLL_CON1, @@ -813,32 +818,32 @@ static unsigned long cpif_clk_regs[] __initdata = { /* list of all parent clock list */ PNAME(mout_mphy_pll_p) = { "oscclk", "fout_mphy_pll", }; -static struct samsung_pll_clock cpif_pll_clks[] __initdata = { +static const struct samsung_pll_clock cpif_pll_clks[] __initconst = { PLL(pll_35xx, CLK_FOUT_MPHY_PLL, "fout_mphy_pll", "oscclk", MPHY_PLL_LOCK, MPHY_PLL_CON0, exynos5443_pll_rates), }; -static struct samsung_mux_clock cpif_mux_clks[] __initdata = { +static const struct samsung_mux_clock cpif_mux_clks[] __initconst = { /* MUX_SEL_CPIF0 */ MUX(CLK_MOUT_MPHY_PLL, "mout_mphy_pll", mout_mphy_pll_p, MUX_SEL_CPIF0, 0, 1), }; -static struct samsung_div_clock cpif_div_clks[] __initdata = { +static const struct samsung_div_clock cpif_div_clks[] __initconst = { /* DIV_CPIF */ DIV(CLK_DIV_SCLK_MPHY, "div_sclk_mphy", "mout_mphy_pll", DIV_CPIF, 0, 6), }; -static struct samsung_gate_clock cpif_gate_clks[] __initdata = { +static const struct samsung_gate_clock cpif_gate_clks[] __initconst = { /* ENABLE_SCLK_CPIF */ GATE(CLK_SCLK_MPHY_PLL, "sclk_mphy_pll", "mout_mphy_pll", - ENABLE_SCLK_CPIF, 9, 0, 0), + ENABLE_SCLK_CPIF, 9, CLK_IGNORE_UNUSED, 0), GATE(CLK_SCLK_UFS_MPHY, "sclk_ufs_mphy", "div_sclk_mphy", ENABLE_SCLK_CPIF, 4, 0, 0), }; -static struct samsung_cmu_info cpif_cmu_info __initdata = { +static const struct samsung_cmu_info cpif_cmu_info __initconst = { .pll_clks = cpif_pll_clks, .nr_pll_clks = ARRAY_SIZE(cpif_pll_clks), .mux_clks = cpif_mux_clks, @@ -939,7 +944,7 @@ CLK_OF_DECLARE(exynos5433_cmu_cpif, "samsung,exynos5433-cmu-cpif", #define PAUSE 0x1008 #define DDRPHY_LOCK_CTRL 0x100c -static unsigned long mif_clk_regs[] __initdata = { +static const unsigned long mif_clk_regs[] __initconst = { MEM0_PLL_LOCK, MEM1_PLL_LOCK, BUS_PLL_LOCK, @@ -1004,7 +1009,7 @@ static unsigned long mif_clk_regs[] __initdata = { DDRPHY_LOCK_CTRL, }; -static struct samsung_pll_clock mif_pll_clks[] __initdata = { +static const struct samsung_pll_clock mif_pll_clks[] __initconst = { PLL(pll_35xx, CLK_FOUT_MEM0_PLL, "fout_mem0_pll", "oscclk", MEM0_PLL_LOCK, MEM0_PLL_CON0, exynos5443_pll_rates), PLL(pll_35xx, CLK_FOUT_MEM1_PLL, "fout_mem1_pll", "oscclk", @@ -1065,7 +1070,7 @@ PNAME(mout_sclk_decon_tv_vclk_b_p) = { "mout_sclk_decon_tv_vclk_a", PNAME(mout_sclk_dsim1_c_p) = { "mout_sclk_dsim1_b", "sclk_mphy_pll", }; PNAME(mout_sclk_dsim1_b_p) = { "mout_sclk_dsim1_a", "mout_mfc_pll_div2",}; -static struct samsung_fixed_factor_clock mif_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock mif_fixed_factor_clks[] __initconst = { /* dout_{mfc|bus|mem1|mem0}_pll is half fixed rate from parent mux */ FFACTOR(CLK_DOUT_MFC_PLL, "dout_mfc_pll", "mout_mfc_pll", 1, 1, 0), FFACTOR(CLK_DOUT_BUS_PLL, "dout_bus_pll", "mout_bus_pll", 1, 1, 0), @@ -1073,7 +1078,7 @@ static struct samsung_fixed_factor_clock mif_fixed_factor_clks[] __initdata = { FFACTOR(CLK_DOUT_MEM0_PLL, "dout_mem0_pll", "mout_mem0_pll", 1, 1, 0), }; -static struct samsung_mux_clock mif_mux_clks[] __initdata = { +static const struct samsung_mux_clock mif_mux_clks[] __initconst = { /* MUX_SEL_MIF0 */ MUX(CLK_MOUT_MFC_PLL_DIV2, "mout_mfc_pll_div2", mout_mfc_pll_div2_p, MUX_SEL_MIF0, 28, 1), @@ -1169,7 +1174,7 @@ static struct samsung_mux_clock mif_mux_clks[] __initdata = { MUX_SEL_MIF7, 0, 1), }; -static struct samsung_div_clock mif_div_clks[] __initdata = { +static const struct samsung_div_clock mif_div_clks[] __initconst = { /* DIV_MIF1 */ DIV(CLK_DIV_SCLK_HPM_MIF, "div_sclk_hpm_mif", "div_clk2x_phy", DIV_MIF1, 16, 2), @@ -1223,7 +1228,7 @@ static struct samsung_div_clock mif_div_clks[] __initdata = { 0, 3), }; -static struct samsung_gate_clock mif_gate_clks[] __initdata = { +static const struct samsung_gate_clock mif_gate_clks[] __initconst = { /* ENABLE_ACLK_MIF0 */ GATE(CLK_CLK2X_PHY1, "clk2k_phy1", "div_clk2x_phy", ENABLE_ACLK_MIF0, 19, CLK_IGNORE_UNUSED, 0), @@ -1440,11 +1445,13 @@ static struct samsung_gate_clock mif_gate_clks[] __initdata = { /* ENABLE_PCLK_MIF_SECURE_DREX0_TZ */ GATE(CLK_PCLK_DREX0_TZ, "pclk_drex0_tz", "div_aclk_mif_133", - ENABLE_PCLK_MIF_SECURE_DREX0_TZ, 0, 0, 0), + ENABLE_PCLK_MIF_SECURE_DREX0_TZ, 0, + CLK_IGNORE_UNUSED, 0), /* ENABLE_PCLK_MIF_SECURE_DREX1_TZ */ GATE(CLK_PCLK_DREX1_TZ, "pclk_drex1_tz", "div_aclk_mif_133", - ENABLE_PCLK_MIF_SECURE_DREX1_TZ, 0, 0, 0), + ENABLE_PCLK_MIF_SECURE_DREX1_TZ, 0, + CLK_IGNORE_UNUSED, 0), /* ENABLE_PCLK_MIF_SECURE_MONOTONIC_CNT */ GATE(CLK_PCLK_MONOTONIC_CNT, "pclk_monotonic_cnt", "div_aclk_mif_133", @@ -1486,7 +1493,7 @@ static struct samsung_gate_clock mif_gate_clks[] __initdata = { ENABLE_SCLK_MIF, 0, CLK_IGNORE_UNUSED, 0), }; -static struct samsung_cmu_info mif_cmu_info __initdata = { +static const struct samsung_cmu_info mif_cmu_info __initconst = { .pll_clks = mif_pll_clks, .nr_pll_clks = ARRAY_SIZE(mif_pll_clks), .mux_clks = mif_mux_clks, @@ -1522,7 +1529,7 @@ CLK_OF_DECLARE(exynos5433_cmu_mif, "samsung,exynos5433-cmu-mif", #define ENABLE_IP_PERIC1 0x0B04 #define ENABLE_IP_PERIC2 0x0B08 -static unsigned long peric_clk_regs[] __initdata = { +static const unsigned long peric_clk_regs[] __initconst = { DIV_PERIC, ENABLE_ACLK_PERIC, ENABLE_PCLK_PERIC0, @@ -1533,13 +1540,13 @@ static unsigned long peric_clk_regs[] __initdata = { ENABLE_IP_PERIC2, }; -static struct samsung_div_clock peric_div_clks[] __initdata = { +static const struct samsung_div_clock peric_div_clks[] __initconst = { /* DIV_PERIC */ DIV(CLK_DIV_SCLK_SCI, "div_sclk_sci", "oscclk", DIV_PERIC, 4, 4), DIV(CLK_DIV_SCLK_SC_IN, "div_sclk_sc_in", "oscclk", DIV_PERIC, 0, 4), }; -static struct samsung_gate_clock peric_gate_clks[] __initdata = { +static const struct samsung_gate_clock peric_gate_clks[] __initconst = { /* ENABLE_ACLK_PERIC */ GATE(CLK_ACLK_AHB2APB_PERIC2P, "aclk_ahb2apb_peric2p", "aclk_peric_66", ENABLE_ACLK_PERIC, 3, CLK_IGNORE_UNUSED, 0), @@ -1654,8 +1661,7 @@ static struct samsung_gate_clock peric_gate_clks[] __initdata = { GATE(CLK_SCLK_IOCLK_SPI2, "sclk_ioclk_spi2", "ioclk_spi2_clk_in", ENABLE_SCLK_PERIC, 13, CLK_SET_RATE_PARENT, 0), GATE(CLK_SCLK_IOCLK_SPI1, "sclk_ioclk_spi1", "ioclk_spi1_clk_in", - ENABLE_SCLK_PERIC, 12, - CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0), + ENABLE_SCLK_PERIC, 12, CLK_SET_RATE_PARENT, 0), GATE(CLK_SCLK_IOCLK_SPI0, "sclk_ioclk_spi0", "ioclk_spi0_clk_in", ENABLE_SCLK_PERIC, 11, CLK_SET_RATE_PARENT, 0), GATE(CLK_SCLK_IOCLK_I2S1_BCLK, "sclk_ioclk_i2s1_bclk", @@ -1670,18 +1676,21 @@ static struct samsung_gate_clock peric_gate_clks[] __initdata = { GATE(CLK_SCLK_SPI2, "sclk_spi2", "sclk_spi2_peric", ENABLE_SCLK_PERIC, 5, CLK_SET_RATE_PARENT, 0), GATE(CLK_SCLK_SPI1, "sclk_spi1", "sclk_spi1_peric", ENABLE_SCLK_PERIC, - 4, CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0), + 4, CLK_SET_RATE_PARENT, 0), GATE(CLK_SCLK_SPI0, "sclk_spi0", "sclk_spi0_peric", ENABLE_SCLK_PERIC, 3, CLK_SET_RATE_PARENT, 0), GATE(CLK_SCLK_UART2, "sclk_uart2", "sclk_uart2_peric", - ENABLE_SCLK_PERIC, 2, CLK_SET_RATE_PARENT, 0), + ENABLE_SCLK_PERIC, 2, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0), GATE(CLK_SCLK_UART1, "sclk_uart1", "sclk_uart1_peric", - ENABLE_SCLK_PERIC, 1, CLK_SET_RATE_PARENT, 0), + ENABLE_SCLK_PERIC, 1, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0), GATE(CLK_SCLK_UART0, "sclk_uart0", "sclk_uart0_peric", - ENABLE_SCLK_PERIC, 0, CLK_SET_RATE_PARENT, 0), + ENABLE_SCLK_PERIC, 0, + CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0), }; -static struct samsung_cmu_info peric_cmu_info __initdata = { +static const struct samsung_cmu_info peric_cmu_info __initconst = { .div_clks = peric_div_clks, .nr_div_clks = ARRAY_SIZE(peric_div_clks), .gate_clks = peric_gate_clks, @@ -1728,7 +1737,7 @@ CLK_OF_DECLARE(exynos5433_cmu_peric, "samsung,exynos5433-cmu-peric", #define ENABLE_IP_PERIS_SECURE_ANTIBRK_CNT 0x0b1c #define ENABLE_IP_PERIS_SECURE_OTP_CON 0x0b20 -static unsigned long peris_clk_regs[] __initdata = { +static const unsigned long peris_clk_regs[] __initconst = { ENABLE_ACLK_PERIS, ENABLE_PCLK_PERIS, ENABLE_PCLK_PERIS_SECURE_TZPC, @@ -1756,7 +1765,7 @@ static unsigned long peris_clk_regs[] __initdata = { ENABLE_IP_PERIS_SECURE_OTP_CON, }; -static struct samsung_gate_clock peris_gate_clks[] __initdata = { +static const struct samsung_gate_clock peris_gate_clks[] __initconst = { /* ENABLE_ACLK_PERIS */ GATE(CLK_ACLK_AHB2APB_PERIS1P, "aclk_ahb2apb_peris1p", "aclk_peris_66", ENABLE_ACLK_PERIS, 2, CLK_IGNORE_UNUSED, 0), @@ -1875,7 +1884,7 @@ static struct samsung_gate_clock peris_gate_clks[] __initdata = { ENABLE_SCLK_PERIS_SECURE_OTP_CON, 0, 0, 0), }; -static struct samsung_cmu_info peris_cmu_info __initdata = { +static const struct samsung_cmu_info peris_cmu_info __initconst = { .gate_clks = peris_gate_clks, .nr_gate_clks = ARRAY_SIZE(peris_gate_clks), .nr_clk_ids = PERIS_NR_CLK, @@ -1959,7 +1968,7 @@ PNAME(mout_sclk_mphy_p) = { "mout_sclk_ufs_mphy_user", "mout_phyclk_lli_mphy_to_ufs_user", }; -static unsigned long fsys_clk_regs[] __initdata = { +static const unsigned long fsys_clk_regs[] __initconst = { MUX_SEL_FSYS0, MUX_SEL_FSYS1, MUX_SEL_FSYS2, @@ -1980,7 +1989,7 @@ static unsigned long fsys_clk_regs[] __initdata = { ENABLE_IP_FSYS1, }; -static struct samsung_fixed_rate_clock fsys_fixed_clks[] __initdata = { +static const struct samsung_fixed_rate_clock fsys_fixed_clks[] __initconst = { /* PHY clocks from USBDRD30_PHY */ FRATE(CLK_PHYCLK_USBDRD30_UDRD30_PHYCLOCK_PHY, "phyclk_usbdrd30_udrd30_phyclock_phy", NULL, @@ -2020,7 +2029,7 @@ static struct samsung_fixed_rate_clock fsys_fixed_clks[] __initdata = { NULL, 0, 26000000), }; -static struct samsung_mux_clock fsys_mux_clks[] __initdata = { +static const struct samsung_mux_clock fsys_mux_clks[] __initconst = { /* MUX_SEL_FSYS0 */ MUX(CLK_MOUT_SCLK_UFS_MPHY_USER, "mout_sclk_ufs_mphy_user", mout_sclk_ufs_mphy_user_p, MUX_SEL_FSYS0, 4, 1), @@ -2104,7 +2113,7 @@ static struct samsung_mux_clock fsys_mux_clks[] __initdata = { MUX_SEL_FSYS4, 0, 1), }; -static struct samsung_gate_clock fsys_gate_clks[] __initdata = { +static const struct samsung_gate_clock fsys_gate_clks[] __initconst = { /* ENABLE_ACLK_FSYS0 */ GATE(CLK_ACLK_PCIE, "aclk_pcie", "mout_aclk_fsys_200_user", ENABLE_ACLK_FSYS0, 13, CLK_IGNORE_UNUSED, 0), @@ -2138,7 +2147,7 @@ static struct samsung_gate_clock fsys_gate_clks[] __initdata = { GATE(CLK_ACLK_SMMU_PDMA1, "aclk_smmu_pdma1", "mout_aclk_fsys_200_user", ENABLE_ACLK_FSYS1, 25, CLK_IGNORE_UNUSED, 0), GATE(CLK_ACLK_BTS_PCIE, "aclk_bts_pcie", "mout_aclk_fsys_200_user", - ENABLE_ACLK_FSYS1, 24, 0, 0), + ENABLE_ACLK_FSYS1, 24, CLK_IGNORE_UNUSED, 0), GATE(CLK_ACLK_AXIUS_PDMA1, "aclk_axius_pdma1", "mout_aclk_fsys_200_user", ENABLE_ACLK_FSYS1, 22, CLK_IGNORE_UNUSED, 0), @@ -2185,13 +2194,13 @@ static struct samsung_gate_clock fsys_gate_clks[] __initdata = { /* ENABLE_PCLK_FSYS */ GATE(CLK_PCLK_PCIE_CTRL, "pclk_pcie_ctrl", "mout_aclk_fsys_200_user", - ENABLE_PCLK_FSYS, 17, 0, 0), + ENABLE_PCLK_FSYS, 17, CLK_IGNORE_UNUSED, 0), GATE(CLK_PCLK_SMMU_PDMA1, "pclk_smmu_pdma1", "mout_aclk_fsys_200_user", ENABLE_PCLK_FSYS, 16, CLK_IGNORE_UNUSED, 0), GATE(CLK_PCLK_PCIE_PHY, "pclk_pcie_phy", "mout_aclk_fsys_200_user", - ENABLE_PCLK_FSYS, 14, 0, 0), + ENABLE_PCLK_FSYS, 14, CLK_IGNORE_UNUSED, 0), GATE(CLK_PCLK_BTS_PCIE, "pclk_bts_pcie", "mout_aclk_fsys_200_user", - ENABLE_PCLK_FSYS, 13, 0, 0), + ENABLE_PCLK_FSYS, 13, CLK_IGNORE_UNUSED, 0), GATE(CLK_PCLK_SMMU_PDMA0, "pclk_smmu_pdma0", "mout_aclk_fsys_200_user", ENABLE_PCLK_FSYS, 8, CLK_IGNORE_UNUSED, 0), GATE(CLK_PCLK_BTS_UFS, "pclk_bts_ufs", "mout_aclk_fsys_200_user", @@ -2270,11 +2279,12 @@ static struct samsung_gate_clock fsys_gate_clks[] __initdata = { ENABLE_SCLK_FSYS, 0, 0, 0), /* ENABLE_IP_FSYS0 */ + GATE(CLK_PCIE, "pcie", "sclk_pcie_100", ENABLE_IP_FSYS0, 17, 0, 0), GATE(CLK_PDMA1, "pdma1", "aclk_pdma1", ENABLE_IP_FSYS0, 15, 0, 0), GATE(CLK_PDMA0, "pdma0", "aclk_pdma0", ENABLE_IP_FSYS0, 0, 0, 0), }; -static struct samsung_cmu_info fsys_cmu_info __initdata = { +static const struct samsung_cmu_info fsys_cmu_info __initconst = { .mux_clks = fsys_mux_clks, .nr_mux_clks = ARRAY_SIZE(fsys_mux_clks), .gate_clks = fsys_gate_clks, @@ -2310,7 +2320,7 @@ CLK_OF_DECLARE(exynos5433_cmu_fsys, "samsung,exynos5433-cmu-fsys", #define DIV_ENABLE_IP_G2D1 0x0b04 #define DIV_ENABLE_IP_G2D_SECURE_SMMU_G2D 0x0b08 -static unsigned long g2d_clk_regs[] __initdata = { +static const unsigned long g2d_clk_regs[] __initconst = { MUX_SEL_G2D0, MUX_SEL_ENABLE_G2D0, DIV_G2D, @@ -2327,7 +2337,7 @@ static unsigned long g2d_clk_regs[] __initdata = { PNAME(mout_aclk_g2d_266_user_p) = { "oscclk", "aclk_g2d_266", }; PNAME(mout_aclk_g2d_400_user_p) = { "oscclk", "aclk_g2d_400", }; -static struct samsung_mux_clock g2d_mux_clks[] __initdata = { +static const struct samsung_mux_clock g2d_mux_clks[] __initconst = { /* MUX_SEL_G2D0 */ MUX(CLK_MUX_ACLK_G2D_266_USER, "mout_aclk_g2d_266_user", mout_aclk_g2d_266_user_p, MUX_SEL_G2D0, 4, 1), @@ -2335,13 +2345,13 @@ static struct samsung_mux_clock g2d_mux_clks[] __initdata = { mout_aclk_g2d_400_user_p, MUX_SEL_G2D0, 0, 1), }; -static struct samsung_div_clock g2d_div_clks[] __initdata = { +static const struct samsung_div_clock g2d_div_clks[] __initconst = { /* DIV_G2D */ DIV(CLK_DIV_PCLK_G2D, "div_pclk_g2d", "mout_aclk_g2d_266_user", DIV_G2D, 0, 2), }; -static struct samsung_gate_clock g2d_gate_clks[] __initdata = { +static const struct samsung_gate_clock g2d_gate_clks[] __initconst = { /* DIV_ENABLE_ACLK_G2D */ GATE(CLK_ACLK_SMMU_MDMA1, "aclk_smmu_mdma1", "mout_aclk_g2d_266_user", DIV_ENABLE_ACLK_G2D, 12, 0, 0), @@ -2398,7 +2408,7 @@ static struct samsung_gate_clock g2d_gate_clks[] __initdata = { DIV_ENABLE_PCLK_G2D_SECURE_SMMU_G2D, 0, 0, 0), }; -static struct samsung_cmu_info g2d_cmu_info __initdata = { +static const struct samsung_cmu_info g2d_cmu_info __initconst = { .mux_clks = g2d_mux_clks, .nr_mux_clks = ARRAY_SIZE(g2d_mux_clks), .div_clks = g2d_div_clks, @@ -2454,7 +2464,7 @@ CLK_OF_DECLARE(exynos5433_cmu_g2d, "samsung,exynos5433-cmu-g2d", #define CLKOUT_CMU_DISP 0x0c00 #define CLKOUT_CMU_DISP_DIV_STAT 0x0c04 -static unsigned long disp_clk_regs[] __initdata = { +static const unsigned long disp_clk_regs[] __initconst = { DISP_PLL_LOCK, DISP_PLL_CON0, DISP_PLL_CON1, @@ -2527,12 +2537,12 @@ PNAME(mout_sclk_decon_tv_vclk_c_disp_p) = { PNAME(mout_sclk_decon_tv_vclk_b_disp_p) = { "mout_sclk_decon_tv_vclk_a_disp", "mout_sclk_decon_tv_vclk_user", }; -static struct samsung_pll_clock disp_pll_clks[] __initdata = { +static const struct samsung_pll_clock disp_pll_clks[] __initconst = { PLL(pll_35xx, CLK_FOUT_DISP_PLL, "fout_disp_pll", "oscclk", DISP_PLL_LOCK, DISP_PLL_CON0, exynos5443_pll_rates), }; -static struct samsung_fixed_factor_clock disp_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock disp_fixed_factor_clks[] __initconst = { /* * sclk_rgb_{vclk|tv_vclk} is half clock of sclk_decon_{vclk|tv_vclk}. * The divider has fixed value (2) between sclk_rgb_{vclk|tv_vclk} @@ -2544,7 +2554,7 @@ static struct samsung_fixed_factor_clock disp_fixed_factor_clks[] __initdata = { 1, 2, 0), }; -static struct samsung_fixed_rate_clock disp_fixed_clks[] __initdata = { +static const struct samsung_fixed_rate_clock disp_fixed_clks[] __initconst = { /* PHY clocks from MIPI_DPHY1 */ FRATE(0, "phyclk_mipidphy1_bitclkdiv8_phy", NULL, 0, 188000000), FRATE(0, "phyclk_mipidphy1_rxclkesc0_phy", NULL, 0, 100000000), @@ -2558,7 +2568,7 @@ static struct samsung_fixed_rate_clock disp_fixed_clks[] __initdata = { NULL, 0, 166000000), }; -static struct samsung_mux_clock disp_mux_clks[] __initdata = { +static const struct samsung_mux_clock disp_mux_clks[] __initconst = { /* MUX_SEL_DISP0 */ MUX(CLK_MOUT_DISP_PLL, "mout_disp_pll", mout_disp_pll_p, MUX_SEL_DISP0, 0, 1), @@ -2633,7 +2643,7 @@ static struct samsung_mux_clock disp_mux_clks[] __initdata = { mout_sclk_decon_vclk_p, MUX_SEL_DISP4, 0, 1), }; -static struct samsung_div_clock disp_div_clks[] __initdata = { +static const struct samsung_div_clock disp_div_clks[] __initconst = { /* DIV_DISP */ DIV(CLK_DIV_SCLK_DSIM1_DISP, "div_sclk_dsim1_disp", "mout_sclk_dsim1_b_disp", DIV_DISP, 24, 3), @@ -2651,7 +2661,7 @@ static struct samsung_div_clock disp_div_clks[] __initdata = { DIV_DISP, 0, 2), }; -static struct samsung_gate_clock disp_gate_clks[] __initdata = { +static const struct samsung_gate_clock disp_gate_clks[] __initconst = { /* ENABLE_ACLK_DISP0 */ GATE(CLK_ACLK_DECON_TV, "aclk_decon_tv", "mout_aclk_disp_333_user", ENABLE_ACLK_DISP0, 2, 0, 0), @@ -2811,7 +2821,7 @@ static struct samsung_gate_clock disp_gate_clks[] __initdata = { "div_sclk_decon_eclk_disp", ENABLE_SCLK_DISP, 2, 0, 0), }; -static struct samsung_cmu_info disp_cmu_info __initdata = { +static const struct samsung_cmu_info disp_cmu_info __initconst = { .pll_clks = disp_pll_clks, .nr_pll_clks = ARRAY_SIZE(disp_pll_clks), .mux_clks = disp_mux_clks, @@ -2856,7 +2866,7 @@ CLK_OF_DECLARE(exynos5433_cmu_disp, "samsung,exynos5433-cmu-disp", #define ENABLE_IP_AUD0 0x0b00 #define ENABLE_IP_AUD1 0x0b04 -static unsigned long aud_clk_regs[] __initdata = { +static const unsigned long aud_clk_regs[] __initconst = { MUX_SEL_AUD0, MUX_SEL_AUD1, MUX_ENABLE_AUD0, @@ -2875,13 +2885,13 @@ static unsigned long aud_clk_regs[] __initdata = { PNAME(mout_aud_pll_user_aud_p) = { "oscclk", "fout_aud_pll", }; PNAME(mout_sclk_aud_pcm_p) = { "mout_aud_pll_user", "ioclk_audiocdclk0",}; -static struct samsung_fixed_rate_clock aud_fixed_clks[] __initdata = { +static const struct samsung_fixed_rate_clock aud_fixed_clks[] __initconst = { FRATE(0, "ioclk_jtag_tclk", NULL, 0, 33000000), FRATE(0, "ioclk_slimbus_clk", NULL, 0, 25000000), FRATE(0, "ioclk_i2s_bclk", NULL, 0, 50000000), }; -static struct samsung_mux_clock aud_mux_clks[] __initdata = { +static const struct samsung_mux_clock aud_mux_clks[] __initconst = { /* MUX_SEL_AUD0 */ MUX(CLK_MOUT_AUD_PLL_USER, "mout_aud_pll_user", mout_aud_pll_user_aud_p, MUX_SEL_AUD0, 0, 1), @@ -2893,7 +2903,7 @@ static struct samsung_mux_clock aud_mux_clks[] __initdata = { MUX_SEL_AUD1, 0, 1), }; -static struct samsung_div_clock aud_div_clks[] __initdata = { +static const struct samsung_div_clock aud_div_clks[] __initconst = { /* DIV_AUD0 */ DIV(CLK_DIV_ATCLK_AUD, "div_atclk_aud", "div_aud_ca5", DIV_AUD0, 12, 4), @@ -2915,7 +2925,7 @@ static struct samsung_div_clock aud_div_clks[] __initdata = { DIV_AUD1, 0, 4), }; -static struct samsung_gate_clock aud_gate_clks[] __initdata = { +static const struct samsung_gate_clock aud_gate_clks[] __initconst = { /* ENABLE_ACLK_AUD */ GATE(CLK_ACLK_INTR_CTRL, "aclk_intr_ctrl", "div_aclk_aud", ENABLE_ACLK_AUD, 12, 0, 0), @@ -2962,7 +2972,7 @@ static struct samsung_gate_clock aud_gate_clks[] __initdata = { /* ENABLE_SCLK_AUD0 */ GATE(CLK_ATCLK_AUD, "atclk_aud", "div_atclk_aud", ENABLE_SCLK_AUD0, - 2, 0, 0), + 2, CLK_IGNORE_UNUSED, 0), GATE(CLK_PCLK_DBG_AUD, "pclk_dbg_aud", "div_pclk_dbg_aud", ENABLE_SCLK_AUD0, 1, 0, 0), GATE(CLK_SCLK_AUD_CA5, "sclk_aud_ca5", "div_aud_ca5", ENABLE_SCLK_AUD0, @@ -2976,7 +2986,7 @@ static struct samsung_gate_clock aud_gate_clks[] __initdata = { GATE(CLK_SCLK_AUD_SLIMBUS, "sclk_aud_slimbus", "div_sclk_aud_slimbus", ENABLE_SCLK_AUD1, 4, 0, 0), GATE(CLK_SCLK_AUD_UART, "sclk_aud_uart", "div_sclk_aud_uart", - ENABLE_SCLK_AUD1, 3, 0, 0), + ENABLE_SCLK_AUD1, 3, CLK_IGNORE_UNUSED, 0), GATE(CLK_SCLK_AUD_PCM, "sclk_aud_pcm", "div_sclk_aud_pcm", ENABLE_SCLK_AUD1, 2, 0, 0), GATE(CLK_SCLK_I2S_BCLK, "sclk_i2s_bclk", "ioclk_i2s_bclk", @@ -2985,7 +2995,7 @@ static struct samsung_gate_clock aud_gate_clks[] __initdata = { ENABLE_SCLK_AUD1, 0, CLK_IGNORE_UNUSED, 0), }; -static struct samsung_cmu_info aud_cmu_info __initdata = { +static const struct samsung_cmu_info aud_cmu_info __initconst = { .mux_clks = aud_mux_clks, .nr_mux_clks = ARRAY_SIZE(aud_mux_clks), .div_clks = aud_div_clks, @@ -3031,24 +3041,24 @@ PNAME(mout_aclk_bus2_400_p) = { "oscclk", "aclk_bus2_400", }; ENABLE_IP_BUS0, \ ENABLE_IP_BUS1 -static unsigned long bus01_clk_regs[] __initdata = { +static const unsigned long bus01_clk_regs[] __initconst = { CMU_BUS_COMMON_CLK_REGS, }; -static unsigned long bus2_clk_regs[] __initdata = { +static const unsigned long bus2_clk_regs[] __initconst = { MUX_SEL_BUS2, MUX_ENABLE_BUS2, CMU_BUS_COMMON_CLK_REGS, }; -static struct samsung_div_clock bus0_div_clks[] __initdata = { +static const struct samsung_div_clock bus0_div_clks[] __initconst = { /* DIV_BUS0 */ DIV(CLK_DIV_PCLK_BUS_133, "div_pclk_bus0_133", "aclk_bus0_400", DIV_BUS, 0, 3), }; /* CMU_BUS0 clocks */ -static struct samsung_gate_clock bus0_gate_clks[] __initdata = { +static const struct samsung_gate_clock bus0_gate_clks[] __initconst = { /* ENABLE_ACLK_BUS0 */ GATE(CLK_ACLK_AHB2APB_BUSP, "aclk_ahb2apb_bus0p", "div_pclk_bus0_133", ENABLE_ACLK_BUS, 4, CLK_IGNORE_UNUSED, 0), @@ -3067,13 +3077,13 @@ static struct samsung_gate_clock bus0_gate_clks[] __initdata = { }; /* CMU_BUS1 clocks */ -static struct samsung_div_clock bus1_div_clks[] __initdata = { +static const struct samsung_div_clock bus1_div_clks[] __initconst = { /* DIV_BUS1 */ DIV(CLK_DIV_PCLK_BUS_133, "div_pclk_bus1_133", "aclk_bus1_400", DIV_BUS, 0, 3), }; -static struct samsung_gate_clock bus1_gate_clks[] __initdata = { +static const struct samsung_gate_clock bus1_gate_clks[] __initconst = { /* ENABLE_ACLK_BUS1 */ GATE(CLK_ACLK_AHB2APB_BUSP, "aclk_ahb2apb_bus1p", "div_pclk_bus1_133", ENABLE_ACLK_BUS, 4, CLK_IGNORE_UNUSED, 0), @@ -3092,19 +3102,19 @@ static struct samsung_gate_clock bus1_gate_clks[] __initdata = { }; /* CMU_BUS2 clocks */ -static struct samsung_mux_clock bus2_mux_clks[] __initdata = { +static const struct samsung_mux_clock bus2_mux_clks[] __initconst = { /* MUX_SEL_BUS2 */ MUX(CLK_MOUT_ACLK_BUS2_400_USER, "mout_aclk_bus2_400_user", mout_aclk_bus2_400_p, MUX_SEL_BUS2, 0, 1), }; -static struct samsung_div_clock bus2_div_clks[] __initdata = { +static const struct samsung_div_clock bus2_div_clks[] __initconst = { /* DIV_BUS2 */ DIV(CLK_DIV_PCLK_BUS_133, "div_pclk_bus2_133", "mout_aclk_bus2_400_user", DIV_BUS, 0, 3), }; -static struct samsung_gate_clock bus2_gate_clks[] __initdata = { +static const struct samsung_gate_clock bus2_gate_clks[] __initconst = { /* ENABLE_ACLK_BUS2 */ GATE(CLK_ACLK_AHB2APB_BUSP, "aclk_ahb2apb_bus2p", "div_pclk_bus2_133", ENABLE_ACLK_BUS, 3, CLK_IGNORE_UNUSED, 0), @@ -3133,19 +3143,19 @@ static struct samsung_gate_clock bus2_gate_clks[] __initdata = { .nr_gate_clks = ARRAY_SIZE(bus##id##_gate_clks), \ .nr_clk_ids = BUSx_NR_CLK -static struct samsung_cmu_info bus0_cmu_info __initdata = { +static const struct samsung_cmu_info bus0_cmu_info __initconst = { CMU_BUS_INFO_CLKS(0), .clk_regs = bus01_clk_regs, .nr_clk_regs = ARRAY_SIZE(bus01_clk_regs), }; -static struct samsung_cmu_info bus1_cmu_info __initdata = { +static const struct samsung_cmu_info bus1_cmu_info __initconst = { CMU_BUS_INFO_CLKS(1), .clk_regs = bus01_clk_regs, .nr_clk_regs = ARRAY_SIZE(bus01_clk_regs), }; -static struct samsung_cmu_info bus2_cmu_info __initdata = { +static const struct samsung_cmu_info bus2_cmu_info __initconst = { CMU_BUS_INFO_CLKS(2), .mux_clks = bus2_mux_clks, .nr_mux_clks = ARRAY_SIZE(bus2_mux_clks), @@ -3189,7 +3199,7 @@ exynos5433_cmu_bus_init(2); #define CLKOUT_CMU_G3D_DIV_STAT 0x0c04 #define CLK_STOPCTRL 0x1000 -static unsigned long g3d_clk_regs[] __initdata = { +static const unsigned long g3d_clk_regs[] __initconst = { G3D_PLL_LOCK, G3D_PLL_CON0, G3D_PLL_CON1, @@ -3212,12 +3222,12 @@ static unsigned long g3d_clk_regs[] __initdata = { PNAME(mout_aclk_g3d_400_p) = { "mout_g3d_pll", "aclk_g3d_400", }; PNAME(mout_g3d_pll_p) = { "oscclk", "fout_g3d_pll", }; -static struct samsung_pll_clock g3d_pll_clks[] __initdata = { +static const struct samsung_pll_clock g3d_pll_clks[] __initconst = { PLL(pll_35xx, CLK_FOUT_G3D_PLL, "fout_g3d_pll", "oscclk", G3D_PLL_LOCK, G3D_PLL_CON0, exynos5443_pll_rates), }; -static struct samsung_mux_clock g3d_mux_clks[] __initdata = { +static const struct samsung_mux_clock g3d_mux_clks[] __initconst = { /* MUX_SEL_G3D */ MUX_F(CLK_MOUT_ACLK_G3D_400, "mout_aclk_g3d_400", mout_aclk_g3d_400_p, MUX_SEL_G3D, 8, 1, CLK_SET_RATE_PARENT, 0), @@ -3225,7 +3235,7 @@ static struct samsung_mux_clock g3d_mux_clks[] __initdata = { MUX_SEL_G3D, 0, 1, CLK_SET_RATE_PARENT, 0), }; -static struct samsung_div_clock g3d_div_clks[] __initdata = { +static const struct samsung_div_clock g3d_div_clks[] __initconst = { /* DIV_G3D */ DIV(CLK_DIV_SCLK_HPM_G3D, "div_sclk_hpm_g3d", "mout_g3d_pll", DIV_G3D, 8, 2), @@ -3235,7 +3245,7 @@ static struct samsung_div_clock g3d_div_clks[] __initdata = { 0, 3, CLK_SET_RATE_PARENT, 0), }; -static struct samsung_gate_clock g3d_gate_clks[] __initdata = { +static const struct samsung_gate_clock g3d_gate_clks[] __initconst = { /* ENABLE_ACLK_G3D */ GATE(CLK_ACLK_BTS_G3D1, "aclk_bts_g3d1", "div_aclk_g3d", ENABLE_ACLK_G3D, 7, 0, 0), @@ -3269,7 +3279,7 @@ static struct samsung_gate_clock g3d_gate_clks[] __initdata = { ENABLE_SCLK_G3D, 0, 0, 0), }; -static struct samsung_cmu_info g3d_cmu_info __initdata = { +static const struct samsung_cmu_info g3d_cmu_info __initconst = { .pll_clks = g3d_pll_clks, .nr_pll_clks = ARRAY_SIZE(g3d_pll_clks), .mux_clks = g3d_mux_clks, @@ -3310,7 +3320,7 @@ CLK_OF_DECLARE(exynos5433_cmu_g3d, "samsung,exynos5433-cmu-g3d", #define ENABLE_IP_GSCL_SECURE_SMMU_GSCL1 0x0b0c #define ENABLE_IP_GSCL_SECURE_SMMU_GSCL2 0x0b10 -static unsigned long gscl_clk_regs[] __initdata = { +static const unsigned long gscl_clk_regs[] __initconst = { MUX_SEL_GSCL, MUX_ENABLE_GSCL, ENABLE_ACLK_GSCL, @@ -3332,7 +3342,7 @@ static unsigned long gscl_clk_regs[] __initdata = { PNAME(aclk_gscl_111_user_p) = { "oscclk", "aclk_gscl_111", }; PNAME(aclk_gscl_333_user_p) = { "oscclk", "aclk_gscl_333", }; -static struct samsung_mux_clock gscl_mux_clks[] __initdata = { +static const struct samsung_mux_clock gscl_mux_clks[] __initconst = { /* MUX_SEL_GSCL */ MUX(CLK_MOUT_ACLK_GSCL_111_USER, "mout_aclk_gscl_111_user", aclk_gscl_111_user_p, MUX_SEL_GSCL, 4, 1), @@ -3340,7 +3350,7 @@ static struct samsung_mux_clock gscl_mux_clks[] __initdata = { aclk_gscl_333_user_p, MUX_SEL_GSCL, 0, 1), }; -static struct samsung_gate_clock gscl_gate_clks[] __initdata = { +static const struct samsung_gate_clock gscl_gate_clks[] __initconst = { /* ENABLE_ACLK_GSCL */ GATE(CLK_ACLK_BTS_GSCL2, "aclk_bts_gscl2", "mout_aclk_gscl_333_user", ENABLE_ACLK_GSCL, 11, 0, 0), @@ -3356,9 +3366,11 @@ static struct samsung_gate_clock gscl_gate_clks[] __initdata = { GATE(CLK_ACLK_GSCLNP_111, "aclk_gsclnp_111", "mout_aclk_gscl_111_user", ENABLE_ACLK_GSCL, 6, CLK_IGNORE_UNUSED, 0), GATE(CLK_ACLK_GSCLRTND_333, "aclk_gsclrtnd_333", - "mout_aclk_gscl_333_user", ENABLE_ACLK_GSCL, 5, 0, 0), + "mout_aclk_gscl_333_user", ENABLE_ACLK_GSCL, 5, + CLK_IGNORE_UNUSED, 0), GATE(CLK_ACLK_GSCLBEND_333, "aclk_gsclbend_333", - "mout_aclk_gscl_333_user", ENABLE_ACLK_GSCL, 4, 0, 0), + "mout_aclk_gscl_333_user", ENABLE_ACLK_GSCL, 4, + CLK_IGNORE_UNUSED, 0), GATE(CLK_ACLK_GSD, "aclk_gsd", "mout_aclk_gscl_333_user", ENABLE_ACLK_GSCL, 3, 0, 0), GATE(CLK_ACLK_GSCL2, "aclk_gscl2", "mout_aclk_gscl_333_user", @@ -3412,7 +3424,7 @@ static struct samsung_gate_clock gscl_gate_clks[] __initdata = { ENABLE_PCLK_GSCL_SECURE_SMMU_GSCL2, 0, 0, 0), }; -static struct samsung_cmu_info gscl_cmu_info __initdata = { +static const struct samsung_cmu_info gscl_cmu_info __initconst = { .mux_clks = gscl_mux_clks, .nr_mux_clks = ARRAY_SIZE(gscl_mux_clks), .gate_clks = gscl_gate_clks, @@ -3465,7 +3477,7 @@ CLK_OF_DECLARE(exynos5433_cmu_gscl, "samsung,exynos5433-cmu-gscl", #define APOLLO_INTR_SPREAD_USE_STANDBYWFI 0x1084 #define APOLLO_INTR_SPREAD_BLOCKING_DURATION 0x1088 -static unsigned long apollo_clk_regs[] __initdata = { +static const unsigned long apollo_clk_regs[] __initconst = { APOLLO_PLL_LOCK, APOLLO_PLL_CON0, APOLLO_PLL_CON1, @@ -3500,15 +3512,16 @@ PNAME(mout_bus_pll_apollo_user_p) = { "oscclk", "sclk_bus_pll_apollo", }; PNAME(mout_apollo_p) = { "mout_apollo_pll", "mout_bus_pll_apollo_user", }; -static struct samsung_pll_clock apollo_pll_clks[] __initdata = { +static const struct samsung_pll_clock apollo_pll_clks[] __initconst = { PLL(pll_35xx, CLK_FOUT_APOLLO_PLL, "fout_apollo_pll", "oscclk", APOLLO_PLL_LOCK, APOLLO_PLL_CON0, exynos5443_pll_rates), }; -static struct samsung_mux_clock apollo_mux_clks[] __initdata = { +static const struct samsung_mux_clock apollo_mux_clks[] __initconst = { /* MUX_SEL_APOLLO0 */ MUX_F(CLK_MOUT_APOLLO_PLL, "mout_apollo_pll", mout_apollo_pll_p, - MUX_SEL_APOLLO0, 0, 1, CLK_SET_RATE_PARENT, 0), + MUX_SEL_APOLLO0, 0, 1, CLK_SET_RATE_PARENT | + CLK_RECALC_NEW_RATES, 0), /* MUX_SEL_APOLLO1 */ MUX(CLK_MOUT_BUS_PLL_APOLLO_USER, "mout_bus_pll_apollo_user", @@ -3519,7 +3532,7 @@ static struct samsung_mux_clock apollo_mux_clks[] __initdata = { 0, 1, CLK_SET_RATE_PARENT, 0), }; -static struct samsung_div_clock apollo_div_clks[] __initdata = { +static const struct samsung_div_clock apollo_div_clks[] __initconst = { /* DIV_APOLLO0 */ DIV_F(CLK_DIV_CNTCLK_APOLLO, "div_cntclk_apollo", "div_apollo2", DIV_APOLLO0, 24, 3, CLK_GET_RATE_NOCACHE, @@ -3550,7 +3563,7 @@ static struct samsung_div_clock apollo_div_clks[] __initdata = { CLK_DIVIDER_READ_ONLY), }; -static struct samsung_gate_clock apollo_gate_clks[] __initdata = { +static const struct samsung_gate_clock apollo_gate_clks[] __initconst = { /* ENABLE_ACLK_APOLLO */ GATE(CLK_ACLK_ASATBSLV_APOLLO_3_CSSYS, "aclk_asatbslv_apollo_3_cssys", "div_atclk_apollo", ENABLE_ACLK_APOLLO, @@ -3589,28 +3602,64 @@ static struct samsung_gate_clock apollo_gate_clks[] __initdata = { ENABLE_SCLK_APOLLO, 3, CLK_IGNORE_UNUSED, 0), GATE(CLK_SCLK_HPM_APOLLO, "sclk_hpm_apollo", "div_sclk_hpm_apollo", ENABLE_SCLK_APOLLO, 1, CLK_IGNORE_UNUSED, 0), - GATE(CLK_SCLK_APOLLO, "sclk_apollo", "div_apollo2", - ENABLE_SCLK_APOLLO, 0, - CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0), }; -static struct samsung_cmu_info apollo_cmu_info __initdata = { - .pll_clks = apollo_pll_clks, - .nr_pll_clks = ARRAY_SIZE(apollo_pll_clks), - .mux_clks = apollo_mux_clks, - .nr_mux_clks = ARRAY_SIZE(apollo_mux_clks), - .div_clks = apollo_div_clks, - .nr_div_clks = ARRAY_SIZE(apollo_div_clks), - .gate_clks = apollo_gate_clks, - .nr_gate_clks = ARRAY_SIZE(apollo_gate_clks), - .nr_clk_ids = APOLLO_NR_CLK, - .clk_regs = apollo_clk_regs, - .nr_clk_regs = ARRAY_SIZE(apollo_clk_regs), +#define E5433_APOLLO_DIV0(cntclk, pclk_dbg, atclk, pclk, aclk) \ + (((cntclk) << 24) | ((pclk_dbg) << 20) | ((atclk) << 16) | \ + ((pclk) << 12) | ((aclk) << 8)) + +#define E5433_APOLLO_DIV1(hpm, copy) \ + (((hpm) << 4) | ((copy) << 0)) + +static const struct exynos_cpuclk_cfg_data exynos5433_apolloclk_d[] __initconst = { + { 1300000, E5433_APOLLO_DIV0(3, 7, 7, 7, 2), E5433_APOLLO_DIV1(7, 1), }, + { 1200000, E5433_APOLLO_DIV0(3, 7, 7, 7, 2), E5433_APOLLO_DIV1(7, 1), }, + { 1100000, E5433_APOLLO_DIV0(3, 7, 7, 7, 2), E5433_APOLLO_DIV1(7, 1), }, + { 1000000, E5433_APOLLO_DIV0(3, 7, 7, 7, 2), E5433_APOLLO_DIV1(7, 1), }, + { 900000, E5433_APOLLO_DIV0(3, 7, 7, 7, 2), E5433_APOLLO_DIV1(7, 1), }, + { 800000, E5433_APOLLO_DIV0(3, 7, 7, 7, 2), E5433_APOLLO_DIV1(7, 1), }, + { 700000, E5433_APOLLO_DIV0(3, 7, 7, 7, 2), E5433_APOLLO_DIV1(7, 1), }, + { 600000, E5433_APOLLO_DIV0(3, 7, 7, 7, 1), E5433_APOLLO_DIV1(7, 1), }, + { 500000, E5433_APOLLO_DIV0(3, 7, 7, 7, 1), E5433_APOLLO_DIV1(7, 1), }, + { 400000, E5433_APOLLO_DIV0(3, 7, 7, 7, 1), E5433_APOLLO_DIV1(7, 1), }, + { 0 }, }; static void __init exynos5433_cmu_apollo_init(struct device_node *np) { - samsung_cmu_register_one(np, &apollo_cmu_info); + void __iomem *reg_base; + struct samsung_clk_provider *ctx; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + panic("%s: failed to map registers\n", __func__); + return; + } + + ctx = samsung_clk_init(np, reg_base, APOLLO_NR_CLK); + if (!ctx) { + panic("%s: unable to allocate ctx\n", __func__); + return; + } + + samsung_clk_register_pll(ctx, apollo_pll_clks, + ARRAY_SIZE(apollo_pll_clks), reg_base); + samsung_clk_register_mux(ctx, apollo_mux_clks, + ARRAY_SIZE(apollo_mux_clks)); + samsung_clk_register_div(ctx, apollo_div_clks, + ARRAY_SIZE(apollo_div_clks)); + samsung_clk_register_gate(ctx, apollo_gate_clks, + ARRAY_SIZE(apollo_gate_clks)); + + exynos_register_cpu_clock(ctx, CLK_SCLK_APOLLO, "apolloclk", + mout_apollo_p[0], mout_apollo_p[1], 0x200, + exynos5433_apolloclk_d, ARRAY_SIZE(exynos5433_apolloclk_d), + CLK_CPU_HAS_E5433_REGS_LAYOUT); + + samsung_clk_sleep_init(reg_base, apollo_clk_regs, + ARRAY_SIZE(apollo_clk_regs)); + + samsung_clk_of_add_provider(np, ctx); } CLK_OF_DECLARE(exynos5433_cmu_apollo, "samsung,exynos5433-cmu-apollo", exynos5433_cmu_apollo_init); @@ -3651,7 +3700,7 @@ CLK_OF_DECLARE(exynos5433_cmu_apollo, "samsung,exynos5433-cmu-apollo", #define ATLAS_INTR_SPREAD_USE_STANDBYWFI 0x1084 #define ATLAS_INTR_SPREAD_BLOCKING_DURATION 0x1088 -static unsigned long atlas_clk_regs[] __initdata = { +static const unsigned long atlas_clk_regs[] __initconst = { ATLAS_PLL_LOCK, ATLAS_PLL_CON0, ATLAS_PLL_CON1, @@ -3686,15 +3735,16 @@ PNAME(mout_bus_pll_atlas_user_p) = { "oscclk", "sclk_bus_pll_atlas", }; PNAME(mout_atlas_p) = { "mout_atlas_pll", "mout_bus_pll_atlas_user", }; -static struct samsung_pll_clock atlas_pll_clks[] __initdata = { +static const struct samsung_pll_clock atlas_pll_clks[] __initconst = { PLL(pll_35xx, CLK_FOUT_ATLAS_PLL, "fout_atlas_pll", "oscclk", ATLAS_PLL_LOCK, ATLAS_PLL_CON0, exynos5443_pll_rates), }; -static struct samsung_mux_clock atlas_mux_clks[] __initdata = { +static const struct samsung_mux_clock atlas_mux_clks[] __initconst = { /* MUX_SEL_ATLAS0 */ MUX_F(CLK_MOUT_ATLAS_PLL, "mout_atlas_pll", mout_atlas_pll_p, - MUX_SEL_ATLAS0, 0, 1, CLK_SET_RATE_PARENT, 0), + MUX_SEL_ATLAS0, 0, 1, CLK_SET_RATE_PARENT | + CLK_RECALC_NEW_RATES, 0), /* MUX_SEL_ATLAS1 */ MUX(CLK_MOUT_BUS_PLL_ATLAS_USER, "mout_bus_pll_atlas_user", @@ -3705,7 +3755,7 @@ static struct samsung_mux_clock atlas_mux_clks[] __initdata = { 0, 1, CLK_SET_RATE_PARENT, 0), }; -static struct samsung_div_clock atlas_div_clks[] __initdata = { +static const struct samsung_div_clock atlas_div_clks[] __initconst = { /* DIV_ATLAS0 */ DIV_F(CLK_DIV_CNTCLK_ATLAS, "div_cntclk_atlas", "div_atlas2", DIV_ATLAS0, 24, 3, CLK_GET_RATE_NOCACHE, @@ -3736,7 +3786,7 @@ static struct samsung_div_clock atlas_div_clks[] __initdata = { CLK_DIVIDER_READ_ONLY), }; -static struct samsung_gate_clock atlas_gate_clks[] __initdata = { +static const struct samsung_gate_clock atlas_gate_clks[] __initconst = { /* ENABLE_ACLK_ATLAS */ GATE(CLK_ACLK_ATB_AUD_CSSYS, "aclk_atb_aud_cssys", "div_atclk_atlas", ENABLE_ACLK_ATLAS, @@ -3801,28 +3851,69 @@ static struct samsung_gate_clock atlas_gate_clks[] __initdata = { ENABLE_SCLK_ATLAS, 2, CLK_IGNORE_UNUSED, 0), GATE(CLK_ATCLK, "atclk", "div_atclk_atlas", ENABLE_SCLK_ATLAS, 1, CLK_IGNORE_UNUSED, 0), - GATE(CLK_SCLK_ATLAS, "sclk_atlas", "div_atlas2", - ENABLE_SCLK_ATLAS, 0, - CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0), }; -static struct samsung_cmu_info atlas_cmu_info __initdata = { - .pll_clks = atlas_pll_clks, - .nr_pll_clks = ARRAY_SIZE(atlas_pll_clks), - .mux_clks = atlas_mux_clks, - .nr_mux_clks = ARRAY_SIZE(atlas_mux_clks), - .div_clks = atlas_div_clks, - .nr_div_clks = ARRAY_SIZE(atlas_div_clks), - .gate_clks = atlas_gate_clks, - .nr_gate_clks = ARRAY_SIZE(atlas_gate_clks), - .nr_clk_ids = ATLAS_NR_CLK, - .clk_regs = atlas_clk_regs, - .nr_clk_regs = ARRAY_SIZE(atlas_clk_regs), +#define E5433_ATLAS_DIV0(cntclk, pclk_dbg, atclk, pclk, aclk) \ + (((cntclk) << 24) | ((pclk_dbg) << 20) | ((atclk) << 16) | \ + ((pclk) << 12) | ((aclk) << 8)) + +#define E5433_ATLAS_DIV1(hpm, copy) \ + (((hpm) << 4) | ((copy) << 0)) + +static const struct exynos_cpuclk_cfg_data exynos5433_atlasclk_d[] __initconst = { + { 1900000, E5433_ATLAS_DIV0(7, 7, 7, 7, 4), E5433_ATLAS_DIV1(7, 1), }, + { 1800000, E5433_ATLAS_DIV0(7, 7, 7, 7, 4), E5433_ATLAS_DIV1(7, 1), }, + { 1700000, E5433_ATLAS_DIV0(7, 7, 7, 7, 4), E5433_ATLAS_DIV1(7, 1), }, + { 1600000, E5433_ATLAS_DIV0(7, 7, 7, 7, 4), E5433_ATLAS_DIV1(7, 1), }, + { 1500000, E5433_ATLAS_DIV0(7, 7, 7, 7, 3), E5433_ATLAS_DIV1(7, 1), }, + { 1400000, E5433_ATLAS_DIV0(7, 7, 7, 7, 3), E5433_ATLAS_DIV1(7, 1), }, + { 1300000, E5433_ATLAS_DIV0(7, 7, 7, 7, 3), E5433_ATLAS_DIV1(7, 1), }, + { 1200000, E5433_ATLAS_DIV0(7, 7, 7, 7, 3), E5433_ATLAS_DIV1(7, 1), }, + { 1100000, E5433_ATLAS_DIV0(7, 7, 7, 7, 3), E5433_ATLAS_DIV1(7, 1), }, + { 1000000, E5433_ATLAS_DIV0(7, 7, 7, 7, 3), E5433_ATLAS_DIV1(7, 1), }, + { 900000, E5433_ATLAS_DIV0(7, 7, 7, 7, 2), E5433_ATLAS_DIV1(7, 1), }, + { 800000, E5433_ATLAS_DIV0(7, 7, 7, 7, 2), E5433_ATLAS_DIV1(7, 1), }, + { 700000, E5433_ATLAS_DIV0(7, 7, 7, 7, 2), E5433_ATLAS_DIV1(7, 1), }, + { 600000, E5433_ATLAS_DIV0(7, 7, 7, 7, 2), E5433_ATLAS_DIV1(7, 1), }, + { 500000, E5433_ATLAS_DIV0(7, 7, 7, 7, 2), E5433_ATLAS_DIV1(7, 1), }, + { 0 }, }; static void __init exynos5433_cmu_atlas_init(struct device_node *np) { - samsung_cmu_register_one(np, &atlas_cmu_info); + void __iomem *reg_base; + struct samsung_clk_provider *ctx; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + panic("%s: failed to map registers\n", __func__); + return; + } + + ctx = samsung_clk_init(np, reg_base, ATLAS_NR_CLK); + if (!ctx) { + panic("%s: unable to allocate ctx\n", __func__); + return; + } + + samsung_clk_register_pll(ctx, atlas_pll_clks, + ARRAY_SIZE(atlas_pll_clks), reg_base); + samsung_clk_register_mux(ctx, atlas_mux_clks, + ARRAY_SIZE(atlas_mux_clks)); + samsung_clk_register_div(ctx, atlas_div_clks, + ARRAY_SIZE(atlas_div_clks)); + samsung_clk_register_gate(ctx, atlas_gate_clks, + ARRAY_SIZE(atlas_gate_clks)); + + exynos_register_cpu_clock(ctx, CLK_SCLK_ATLAS, "atlasclk", + mout_atlas_p[0], mout_atlas_p[1], 0x200, + exynos5433_atlasclk_d, ARRAY_SIZE(exynos5433_atlasclk_d), + CLK_CPU_HAS_E5433_REGS_LAYOUT); + + samsung_clk_sleep_init(reg_base, atlas_clk_regs, + ARRAY_SIZE(atlas_clk_regs)); + + samsung_clk_of_add_provider(np, ctx); } CLK_OF_DECLARE(exynos5433_cmu_atlas, "samsung,exynos5433-cmu-atlas", exynos5433_cmu_atlas_init); @@ -3853,7 +3944,7 @@ CLK_OF_DECLARE(exynos5433_cmu_atlas, "samsung,exynos5433-cmu-atlas", #define ENABLE_IP_MSCL_SECURE_SMMU_M2MSCALER1 0x0b0c #define ENABLE_IP_MSCL_SECURE_SMMU_JPEG 0x0b10 -static unsigned long mscl_clk_regs[] __initdata = { +static const unsigned long mscl_clk_regs[] __initconst = { MUX_SEL_MSCL0, MUX_SEL_MSCL1, MUX_ENABLE_MSCL0, @@ -3881,7 +3972,7 @@ PNAME(mout_aclk_mscl_400_user_p) = { "oscclk", "aclk_mscl_400", }; PNAME(mout_sclk_jpeg_p) = { "mout_sclk_jpeg_user", "mout_aclk_mscl_400_user", }; -static struct samsung_mux_clock mscl_mux_clks[] __initdata = { +static const struct samsung_mux_clock mscl_mux_clks[] __initconst = { /* MUX_SEL_MSCL0 */ MUX(CLK_MOUT_SCLK_JPEG_USER, "mout_sclk_jpeg_user", mout_sclk_jpeg_user_p, MUX_SEL_MSCL0, 4, 1), @@ -3893,13 +3984,13 @@ static struct samsung_mux_clock mscl_mux_clks[] __initdata = { MUX_SEL_MSCL1, 0, 1), }; -static struct samsung_div_clock mscl_div_clks[] __initdata = { +static const struct samsung_div_clock mscl_div_clks[] __initconst = { /* DIV_MSCL */ DIV(CLK_DIV_PCLK_MSCL, "div_pclk_mscl", "mout_aclk_mscl_400_user", DIV_MSCL, 0, 3), }; -static struct samsung_gate_clock mscl_gate_clks[] __initdata = { +static const struct samsung_gate_clock mscl_gate_clks[] __initconst = { /* ENABLE_ACLK_MSCL */ GATE(CLK_ACLK_BTS_JPEG, "aclk_bts_jpeg", "mout_aclk_mscl_400_user", ENABLE_ACLK_MSCL, 9, 0, 0), @@ -3977,7 +4068,7 @@ static struct samsung_gate_clock mscl_gate_clks[] __initdata = { CLK_IGNORE_UNUSED | CLK_SET_RATE_PARENT, 0), }; -static struct samsung_cmu_info mscl_cmu_info __initdata = { +static const struct samsung_cmu_info mscl_cmu_info __initconst = { .mux_clks = mscl_mux_clks, .nr_mux_clks = ARRAY_SIZE(mscl_mux_clks), .div_clks = mscl_div_clks, @@ -4012,7 +4103,7 @@ CLK_OF_DECLARE(exynos5433_cmu_mscl, "samsung,exynos5433-cmu-mscl", #define ENABLE_IP_MFC1 0x0b04 #define ENABLE_IP_MFC_SECURE_SMMU_MFC 0x0b08 -static unsigned long mfc_clk_regs[] __initdata = { +static const unsigned long mfc_clk_regs[] __initconst = { MUX_SEL_MFC, MUX_ENABLE_MFC, DIV_MFC, @@ -4027,19 +4118,19 @@ static unsigned long mfc_clk_regs[] __initdata = { PNAME(mout_aclk_mfc_400_user_p) = { "oscclk", "aclk_mfc_400", }; -static struct samsung_mux_clock mfc_mux_clks[] __initdata = { +static const struct samsung_mux_clock mfc_mux_clks[] __initconst = { /* MUX_SEL_MFC */ MUX(CLK_MOUT_ACLK_MFC_400_USER, "mout_aclk_mfc_400_user", mout_aclk_mfc_400_user_p, MUX_SEL_MFC, 0, 0), }; -static struct samsung_div_clock mfc_div_clks[] __initdata = { +static const struct samsung_div_clock mfc_div_clks[] __initconst = { /* DIV_MFC */ DIV(CLK_DIV_PCLK_MFC, "div_pclk_mfc", "mout_aclk_mfc_400_user", DIV_MFC, 0, 2), }; -static struct samsung_gate_clock mfc_gate_clks[] __initdata = { +static const struct samsung_gate_clock mfc_gate_clks[] __initconst = { /* ENABLE_ACLK_MFC */ GATE(CLK_ACLK_BTS_MFC_1, "aclk_bts_mfc_1", "mout_aclk_mfc_400_user", ENABLE_ACLK_MFC, 6, 0, 0), @@ -4085,7 +4176,7 @@ static struct samsung_gate_clock mfc_gate_clks[] __initdata = { 0, CLK_IGNORE_UNUSED, 0), }; -static struct samsung_cmu_info mfc_cmu_info __initdata = { +static const struct samsung_cmu_info mfc_cmu_info __initconst = { .mux_clks = mfc_mux_clks, .nr_mux_clks = ARRAY_SIZE(mfc_mux_clks), .div_clks = mfc_div_clks, @@ -4120,7 +4211,7 @@ CLK_OF_DECLARE(exynos5433_cmu_mfc, "samsung,exynos5433-cmu-mfc", #define ENABLE_IP_HEVC1 0x0b04 #define ENABLE_IP_HEVC_SECURE_SMMU_HEVC 0x0b08 -static unsigned long hevc_clk_regs[] __initdata = { +static const unsigned long hevc_clk_regs[] __initconst = { MUX_SEL_HEVC, MUX_ENABLE_HEVC, DIV_HEVC, @@ -4135,19 +4226,19 @@ static unsigned long hevc_clk_regs[] __initdata = { PNAME(mout_aclk_hevc_400_user_p) = { "oscclk", "aclk_hevc_400", }; -static struct samsung_mux_clock hevc_mux_clks[] __initdata = { +static const struct samsung_mux_clock hevc_mux_clks[] __initconst = { /* MUX_SEL_HEVC */ MUX(CLK_MOUT_ACLK_HEVC_400_USER, "mout_aclk_hevc_400_user", mout_aclk_hevc_400_user_p, MUX_SEL_HEVC, 0, 0), }; -static struct samsung_div_clock hevc_div_clks[] __initdata = { +static const struct samsung_div_clock hevc_div_clks[] __initconst = { /* DIV_HEVC */ DIV(CLK_DIV_PCLK_HEVC, "div_pclk_hevc", "mout_aclk_hevc_400_user", DIV_HEVC, 0, 2), }; -static struct samsung_gate_clock hevc_gate_clks[] __initdata = { +static const struct samsung_gate_clock hevc_gate_clks[] __initconst = { /* ENABLE_ACLK_HEVC */ GATE(CLK_ACLK_BTS_HEVC_1, "aclk_bts_hevc_1", "mout_aclk_hevc_400_user", ENABLE_ACLK_HEVC, 6, 0, 0), @@ -4195,7 +4286,7 @@ static struct samsung_gate_clock hevc_gate_clks[] __initdata = { 0, CLK_IGNORE_UNUSED, 0), }; -static struct samsung_cmu_info hevc_cmu_info __initdata = { +static const struct samsung_cmu_info hevc_cmu_info __initconst = { .mux_clks = hevc_mux_clks, .nr_mux_clks = ARRAY_SIZE(hevc_mux_clks), .div_clks = hevc_div_clks, @@ -4232,7 +4323,7 @@ CLK_OF_DECLARE(exynos5433_cmu_hevc, "samsung,exynos5433-cmu-hevc", #define ENABLE_IP_ISP2 0x0b08 #define ENABLE_IP_ISP3 0x0b0c -static unsigned long isp_clk_regs[] __initdata = { +static const unsigned long isp_clk_regs[] __initconst = { MUX_SEL_ISP, MUX_ENABLE_ISP, DIV_ISP, @@ -4250,7 +4341,7 @@ static unsigned long isp_clk_regs[] __initdata = { PNAME(mout_aclk_isp_dis_400_user_p) = { "oscclk", "aclk_isp_dis_400", }; PNAME(mout_aclk_isp_400_user_p) = { "oscclk", "aclk_isp_400", }; -static struct samsung_mux_clock isp_mux_clks[] __initdata = { +static const struct samsung_mux_clock isp_mux_clks[] __initconst = { /* MUX_SEL_ISP */ MUX(CLK_MOUT_ACLK_ISP_DIS_400_USER, "mout_aclk_isp_dis_400_user", mout_aclk_isp_dis_400_user_p, MUX_SEL_ISP, 4, 0), @@ -4258,7 +4349,7 @@ static struct samsung_mux_clock isp_mux_clks[] __initdata = { mout_aclk_isp_400_user_p, MUX_SEL_ISP, 0, 0), }; -static struct samsung_div_clock isp_div_clks[] __initdata = { +static const struct samsung_div_clock isp_div_clks[] __initconst = { /* DIV_ISP */ DIV(CLK_DIV_PCLK_ISP_DIS, "div_pclk_isp_dis", "mout_aclk_isp_dis_400_user", DIV_ISP, 12, 3), @@ -4270,7 +4361,7 @@ static struct samsung_div_clock isp_div_clks[] __initdata = { "mout_aclk_isp_400_user", DIV_ISP, 0, 3), }; -static struct samsung_gate_clock isp_gate_clks[] __initdata = { +static const struct samsung_gate_clock isp_gate_clks[] __initconst = { /* ENABLE_ACLK_ISP0 */ GATE(CLK_ACLK_ISP_D_GLUE, "aclk_isp_d_glue", "mout_aclk_isp_400_user", ENABLE_ACLK_ISP0, 6, CLK_IGNORE_UNUSED, 0), @@ -4448,7 +4539,7 @@ static struct samsung_gate_clock isp_gate_clks[] __initdata = { 0, CLK_IGNORE_UNUSED, 0), }; -static struct samsung_cmu_info isp_cmu_info __initdata = { +static const struct samsung_cmu_info isp_cmu_info __initconst = { .mux_clks = isp_mux_clks, .nr_mux_clks = ARRAY_SIZE(isp_mux_clks), .div_clks = isp_div_clks, @@ -4504,7 +4595,7 @@ CLK_OF_DECLARE(exynos5433_cmu_isp, "samsung,exynos5433-cmu-isp", #define ENABLE_IP_CAM02 0X0b08 #define ENABLE_IP_CAM03 0X0b0C -static unsigned long cam0_clk_regs[] __initdata = { +static const unsigned long cam0_clk_regs[] __initconst = { MUX_SEL_CAM00, MUX_SEL_CAM01, MUX_SEL_CAM02, @@ -4588,14 +4679,14 @@ PNAME(mout_sclk_pixelasync_lite_c_init_a_p) = { "mout_aclk_cam0_552_user", "mout_aclk_cam0_400_user", }; -static struct samsung_fixed_rate_clock cam0_fixed_clks[] __initdata = { +static const struct samsung_fixed_rate_clock cam0_fixed_clks[] __initconst = { FRATE(CLK_PHYCLK_RXBYTEECLKHS0_S4_PHY, "phyclk_rxbyteclkhs0_s4_phy", NULL, 0, 100000000), FRATE(CLK_PHYCLK_RXBYTEECLKHS0_S2A_PHY, "phyclk_rxbyteclkhs0_s2a_phy", NULL, 0, 100000000), }; -static struct samsung_mux_clock cam0_mux_clks[] __initdata = { +static const struct samsung_mux_clock cam0_mux_clks[] __initconst = { /* MUX_SEL_CAM00 */ MUX(CLK_MOUT_ACLK_CAM0_333_USER, "mout_aclk_cam0_333_user", mout_aclk_cam0_333_user_p, MUX_SEL_CAM00, 8, 1), @@ -4669,7 +4760,7 @@ static struct samsung_mux_clock cam0_mux_clks[] __initdata = { MUX_SEL_CAM04, 0, 1), }; -static struct samsung_div_clock cam0_div_clks[] __initdata = { +static const struct samsung_div_clock cam0_div_clks[] __initconst = { /* DIV_CAM00 */ DIV(CLK_DIV_PCLK_CAM0_50, "div_pclk_cam0_50", "div_aclk_cam0_200", DIV_CAM00, 8, 2), @@ -4716,7 +4807,7 @@ static struct samsung_div_clock cam0_div_clks[] __initdata = { "mout_sclk_pixelasync_lite_c_init_b", DIV_CAM03, 0, 3), }; -static struct samsung_gate_clock cam0_gate_clks[] __initdata = { +static const struct samsung_gate_clock cam0_gate_clks[] __initconst = { /* ENABLE_ACLK_CAM00 */ GATE(CLK_ACLK_CSIS1, "aclk_csis1", "div_aclk_csis1", ENABLE_ACLK_CAM00, 6, 0, 0), @@ -4923,7 +5014,7 @@ static struct samsung_gate_clock cam0_gate_clks[] __initdata = { ENABLE_SCLK_CAM0, 0, 0, 0), }; -static struct samsung_cmu_info cam0_cmu_info __initdata = { +static const struct samsung_cmu_info cam0_cmu_info __initconst = { .mux_clks = cam0_mux_clks, .nr_mux_clks = ARRAY_SIZE(cam0_mux_clks), .div_clks = cam0_div_clks, @@ -4970,7 +5061,7 @@ CLK_OF_DECLARE(exynos5433_cmu_cam0, "samsung,exynos5433-cmu-cam0", #define ENABLE_IP_CAM11 0X0b04 #define ENABLE_IP_CAM12 0X0b08 -static unsigned long cam1_clk_regs[] __initdata = { +static const unsigned long cam1_clk_regs[] __initconst = { MUX_SEL_CAM10, MUX_SEL_CAM11, MUX_SEL_CAM12, @@ -5016,12 +5107,12 @@ PNAME(mout_aclk_lite_c_b_p) = { "mout_aclk_lite_c_a", PNAME(mout_aclk_lite_c_a_p) = { "mout_aclk_cam1_552_user", "mout_aclk_cam1_400_user", }; -static struct samsung_fixed_rate_clock cam1_fixed_clks[] __initdata = { +static const struct samsung_fixed_rate_clock cam1_fixed_clks[] __initconst = { FRATE(CLK_PHYCLK_RXBYTEECLKHS0_S2B, "phyclk_rxbyteclkhs0_s2b_phy", NULL, 0, 100000000), }; -static struct samsung_mux_clock cam1_mux_clks[] __initdata = { +static const struct samsung_mux_clock cam1_mux_clks[] __initconst = { /* MUX_SEL_CAM10 */ MUX(CLK_MOUT_SCLK_ISP_UART_USER, "mout_sclk_isp_uart_user", mout_sclk_isp_uart_user_p, MUX_SEL_CAM10, 20, 1), @@ -5057,7 +5148,7 @@ static struct samsung_mux_clock cam1_mux_clks[] __initdata = { MUX_SEL_CAM12, 0, 1), }; -static struct samsung_div_clock cam1_div_clks[] __initdata = { +static const struct samsung_div_clock cam1_div_clks[] __initconst = { /* DIV_CAM10 */ DIV(CLK_DIV_SCLK_ISP_MPWM, "div_sclk_isp_mpwm", "div_pclk_cam1_83", DIV_CAM10, 16, 2), @@ -5081,7 +5172,7 @@ static struct samsung_div_clock cam1_div_clks[] __initdata = { DIV_CAM11, 0, 3), }; -static struct samsung_gate_clock cam1_gate_clks[] __initdata = { +static const struct samsung_gate_clock cam1_gate_clks[] __initconst = { /* ENABLE_ACLK_CAM10 */ GATE(CLK_ACLK_ISP_GIC, "aclk_isp_gic", "mout_aclk_cam1_333_user", ENABLE_ACLK_CAM10, 4, 0, 0), @@ -5296,7 +5387,7 @@ static struct samsung_gate_clock cam1_gate_clks[] __initdata = { ENABLE_SCLK_CAM1, 0, 0, 0), }; -static struct samsung_cmu_info cam1_cmu_info __initdata = { +static const struct samsung_cmu_info cam1_cmu_info __initconst = { .mux_clks = cam1_mux_clks, .nr_mux_clks = ARRAY_SIZE(cam1_mux_clks), .div_clks = cam1_div_clks, diff --git a/drivers/clk/samsung/clk-exynos5440.c b/drivers/clk/samsung/clk-exynos5440.c index c57cff1e1..a57d01b99 100644 --- a/drivers/clk/samsung/clk-exynos5440.c +++ b/drivers/clk/samsung/clk-exynos5440.c @@ -35,7 +35,7 @@ static struct samsung_fixed_rate_clock exynos5440_fixed_rate_ext_clks[] __initda }; /* fixed rate clocks */ -static struct samsung_fixed_rate_clock exynos5440_fixed_rate_clks[] __initdata = { +static const struct samsung_fixed_rate_clock exynos5440_fixed_rate_clks[] __initconst = { FRATE(0, "ppll", NULL, 0, 1000000000), FRATE(0, "usb_phy0", NULL, 0, 60000000), FRATE(0, "usb_phy1", NULL, 0, 60000000), @@ -44,26 +44,26 @@ static struct samsung_fixed_rate_clock exynos5440_fixed_rate_clks[] __initdata = }; /* fixed factor clocks */ -static struct samsung_fixed_factor_clock exynos5440_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock exynos5440_fixed_factor_clks[] __initconst = { FFACTOR(0, "div250", "ppll", 1, 4, 0), FFACTOR(0, "div200", "ppll", 1, 5, 0), FFACTOR(0, "div125", "div250", 1, 2, 0), }; /* mux clocks */ -static struct samsung_mux_clock exynos5440_mux_clks[] __initdata = { +static const struct samsung_mux_clock exynos5440_mux_clks[] __initconst = { MUX(0, "mout_spi", mout_spi_p, MISC_DOUT1, 5, 1), MUX_A(CLK_ARM_CLK, "arm_clk", mout_armclk_p, CPU_CLK_STATUS, 0, 1, "armclk"), }; /* divider clocks */ -static struct samsung_div_clock exynos5440_div_clks[] __initdata = { +static const struct samsung_div_clock exynos5440_div_clks[] __initconst = { DIV(CLK_SPI_BAUD, "div_spi", "mout_spi", MISC_DOUT1, 3, 2), }; /* gate clocks */ -static struct samsung_gate_clock exynos5440_gate_clks[] __initdata = { +static const struct samsung_gate_clock exynos5440_gate_clks[] __initconst = { GATE(CLK_PB0_250, "pb0_250", "div250", CLKEN_OV_VAL, 3, 0, 0), GATE(CLK_PR0_250, "pr0_250", "div250", CLKEN_OV_VAL, 4, 0, 0), GATE(CLK_PR1_250, "pr1_250", "div250", CLKEN_OV_VAL, 5, 0, 0), @@ -125,8 +125,6 @@ static void __init exynos5440_clk_init(struct device_node *np) } ctx = samsung_clk_init(np, reg_base, CLK_NR_CLKS); - if (!ctx) - panic("%s: unable to allocate context.\n", __func__); samsung_clk_of_register_fixed_ext(ctx, exynos5440_fixed_rate_ext_clks, ARRAY_SIZE(exynos5440_fixed_rate_ext_clks), ext_clk_match); diff --git a/drivers/clk/samsung/clk-exynos7.c b/drivers/clk/samsung/clk-exynos7.c index ad68d463b..5931a4140 100644 --- a/drivers/clk/samsung/clk-exynos7.c +++ b/drivers/clk/samsung/clk-exynos7.c @@ -36,7 +36,7 @@ #define ENABLE_ACLK_TOPC1 0x0804 #define ENABLE_SCLK_TOPC1 0x0A04 -static struct samsung_fixed_factor_clock topc_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock topc_fixed_factor_clks[] __initconst = { FFACTOR(0, "ffac_topc_bus0_pll_div2", "mout_topc_bus0_pll", 1, 2, 0), FFACTOR(0, "ffac_topc_bus0_pll_div4", "ffac_topc_bus0_pll_div2", 1, 2, 0), @@ -69,7 +69,7 @@ PNAME(mout_topc_mfc_pll_half_p) = { "mout_topc_mfc_pll", PNAME(mout_topc_bus0_pll_out_p) = {"mout_topc_bus0_pll", "ffac_topc_bus0_pll_div2"}; -static unsigned long topc_clk_regs[] __initdata = { +static const unsigned long topc_clk_regs[] __initconst = { CC_PLL_LOCK, BUS0_PLL_LOCK, BUS1_DPLL_LOCK, @@ -89,7 +89,7 @@ static unsigned long topc_clk_regs[] __initdata = { DIV_TOPC3, }; -static struct samsung_mux_clock topc_mux_clks[] __initdata = { +static const struct samsung_mux_clock topc_mux_clks[] __initconst = { MUX(0, "mout_topc_bus0_pll", mout_topc_bus0_pll_ctrl_p, MUX_SEL_TOPC0, 0, 1), MUX(0, "mout_topc_bus1_pll", mout_topc_bus1_pll_ctrl_p, @@ -118,7 +118,7 @@ static struct samsung_mux_clock topc_mux_clks[] __initdata = { MUX(0, "mout_aclk_peris_66", mout_topc_group2, MUX_SEL_TOPC3, 24, 2), }; -static struct samsung_div_clock topc_div_clks[] __initdata = { +static const struct samsung_div_clock topc_div_clks[] __initconst = { DIV(DOUT_ACLK_CCORE_133, "dout_aclk_ccore_133", "mout_aclk_ccore_133", DIV_TOPC0, 4, 4), @@ -139,14 +139,14 @@ static struct samsung_div_clock topc_div_clks[] __initdata = { DIV_TOPC3, 28, 4), }; -static struct samsung_pll_rate_table pll1460x_24mhz_tbl[] __initdata = { +static const struct samsung_pll_rate_table pll1460x_24mhz_tbl[] __initconst = { PLL_36XX_RATE(491520000, 20, 1, 0, 31457), {}, }; -static struct samsung_gate_clock topc_gate_clks[] __initdata = { +static const struct samsung_gate_clock topc_gate_clks[] __initconst = { GATE(ACLK_CCORE_133, "aclk_ccore_133", "dout_aclk_ccore_133", - ENABLE_ACLK_TOPC0, 4, 0, 0), + ENABLE_ACLK_TOPC0, 4, CLK_IS_CRITICAL, 0), GATE(ACLK_MSCL_532, "aclk_mscl_532", "dout_aclk_mscl_532", ENABLE_ACLK_TOPC1, 20, 0, 0), @@ -174,7 +174,7 @@ static struct samsung_gate_clock topc_gate_clks[] __initdata = { ENABLE_SCLK_TOPC1, 0, 0, 0), }; -static struct samsung_pll_clock topc_pll_clks[] __initdata = { +static const struct samsung_pll_clock topc_pll_clks[] __initconst = { PLL(pll_1451x, 0, "fout_bus0_pll", "fin_pll", BUS0_PLL_LOCK, BUS0_PLL_CON0, NULL), PLL(pll_1452x, 0, "fout_cc_pll", "fin_pll", CC_PLL_LOCK, @@ -187,7 +187,7 @@ static struct samsung_pll_clock topc_pll_clks[] __initdata = { AUD_PLL_CON0, pll1460x_24mhz_tbl), }; -static struct samsung_cmu_info topc_cmu_info __initdata = { +static const struct samsung_cmu_info topc_cmu_info __initconst = { .pll_clks = topc_pll_clks, .nr_pll_clks = ARRAY_SIZE(topc_pll_clks), .mux_clks = topc_mux_clks, @@ -256,7 +256,7 @@ PNAME(mout_top0_group3) = {"ioclk_audiocdclk0", PNAME(mout_top0_group4) = {"ioclk_audiocdclk1", "mout_top0_aud_pll_user", "mout_top0_bus0_pll_half", "mout_top0_bus1_pll_half"}; -static unsigned long top0_clk_regs[] __initdata = { +static const unsigned long top0_clk_regs[] __initconst = { MUX_SEL_TOP00, MUX_SEL_TOP01, MUX_SEL_TOP03, @@ -275,7 +275,7 @@ static unsigned long top0_clk_regs[] __initdata = { ENABLE_SCLK_TOP0_PERIC3, }; -static struct samsung_mux_clock top0_mux_clks[] __initdata = { +static const struct samsung_mux_clock top0_mux_clks[] __initconst = { MUX(0, "mout_top0_aud_pll_user", mout_top0_aud_pll_user_p, MUX_SEL_TOP00, 0, 1), MUX(0, "mout_top0_mfc_pll_user", mout_top0_mfc_pll_user_p, @@ -315,7 +315,7 @@ static struct samsung_mux_clock top0_mux_clks[] __initdata = { MUX(0, "mout_sclk_spi4", mout_top0_group1, MUX_SEL_TOP0_PERIC3, 20, 2), }; -static struct samsung_div_clock top0_div_clks[] __initdata = { +static const struct samsung_div_clock top0_div_clks[] __initconst = { DIV(DOUT_ACLK_PERIC1, "dout_aclk_peric1_66", "mout_aclk_peric1_66", DIV_TOP03, 12, 6), DIV(DOUT_ACLK_PERIC0, "dout_aclk_peric0_66", "mout_aclk_peric0_66", @@ -338,7 +338,7 @@ static struct samsung_div_clock top0_div_clks[] __initdata = { DIV(0, "dout_sclk_spi4", "mout_sclk_spi4", DIV_TOP0_PERIC3, 20, 12), }; -static struct samsung_gate_clock top0_gate_clks[] __initdata = { +static const struct samsung_gate_clock top0_gate_clks[] __initconst = { GATE(CLK_ACLK_PERIC0_66, "aclk_peric0_66", "dout_aclk_peric0_66", ENABLE_ACLK_TOP03, 20, CLK_SET_RATE_PARENT, 0), GATE(CLK_ACLK_PERIC1_66, "aclk_peric1_66", "dout_aclk_peric1_66", @@ -372,7 +372,7 @@ static struct samsung_gate_clock top0_gate_clks[] __initdata = { ENABLE_SCLK_TOP0_PERIC3, 20, CLK_SET_RATE_PARENT, 0), }; -static struct samsung_fixed_factor_clock top0_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock top0_fixed_factor_clks[] __initconst = { FFACTOR(0, "ffac_top0_bus0_pll_div2", "mout_top0_bus0_pll_user", 1, 2, 0), FFACTOR(0, "ffac_top0_bus1_pll_div2", "mout_top0_bus1_pll_user", @@ -381,7 +381,7 @@ static struct samsung_fixed_factor_clock top0_fixed_factor_clks[] __initdata = { FFACTOR(0, "ffac_top0_mfc_pll_div2", "mout_top0_mfc_pll_user", 1, 2, 0), }; -static struct samsung_cmu_info top0_cmu_info __initdata = { +static const struct samsung_cmu_info top0_cmu_info __initconst = { .mux_clks = top0_mux_clks, .nr_mux_clks = ARRAY_SIZE(top0_mux_clks), .div_clks = top0_div_clks, @@ -438,7 +438,7 @@ PNAME(mout_top1_group1) = {"mout_top1_bus0_pll_half", "mout_top1_bus1_pll_half", "mout_top1_cc_pll_half", "mout_top1_mfc_pll_half"}; -static unsigned long top1_clk_regs[] __initdata = { +static const unsigned long top1_clk_regs[] __initconst = { MUX_SEL_TOP10, MUX_SEL_TOP11, MUX_SEL_TOP13, @@ -455,7 +455,7 @@ static unsigned long top1_clk_regs[] __initdata = { ENABLE_SCLK_TOP1_FSYS11, }; -static struct samsung_mux_clock top1_mux_clks[] __initdata = { +static const struct samsung_mux_clock top1_mux_clks[] __initconst = { MUX(0, "mout_top1_mfc_pll_user", mout_top1_mfc_pll_user_p, MUX_SEL_TOP10, 4, 1), MUX(0, "mout_top1_cc_pll_user", mout_top1_cc_pll_user_p, @@ -494,7 +494,7 @@ static struct samsung_mux_clock top1_mux_clks[] __initdata = { MUX_SEL_TOP1_FSYS11, 24, 2), }; -static struct samsung_div_clock top1_div_clks[] __initdata = { +static const struct samsung_div_clock top1_div_clks[] __initconst = { DIV(DOUT_ACLK_FSYS1_200, "dout_aclk_fsys1_200", "mout_aclk_fsys1_200", DIV_TOP13, 24, 4), DIV(DOUT_ACLK_FSYS0_200, "dout_aclk_fsys0_200", "mout_aclk_fsys0_200", @@ -521,7 +521,7 @@ static struct samsung_div_clock top1_div_clks[] __initdata = { "mout_sclk_phy_fsys1_26m", DIV_TOP1_FSYS11, 24, 6), }; -static struct samsung_gate_clock top1_gate_clks[] __initdata = { +static const struct samsung_gate_clock top1_gate_clks[] __initconst = { GATE(CLK_SCLK_MMC2, "sclk_mmc2", "dout_sclk_mmc2", ENABLE_SCLK_TOP1_FSYS0, 16, CLK_SET_RATE_PARENT, 0), GATE(0, "sclk_usbdrd300", "dout_sclk_usbdrd300", @@ -539,7 +539,8 @@ static struct samsung_gate_clock top1_gate_clks[] __initdata = { ENABLE_SCLK_TOP1_FSYS11, 12, CLK_SET_RATE_PARENT, 0), GATE(CLK_ACLK_FSYS0_200, "aclk_fsys0_200", "dout_aclk_fsys0_200", - ENABLE_ACLK_TOP13, 28, CLK_SET_RATE_PARENT, 0), + ENABLE_ACLK_TOP13, 28, CLK_SET_RATE_PARENT | + CLK_IS_CRITICAL, 0), GATE(CLK_ACLK_FSYS1_200, "aclk_fsys1_200", "dout_aclk_fsys1_200", ENABLE_ACLK_TOP13, 24, CLK_SET_RATE_PARENT, 0), @@ -548,7 +549,7 @@ static struct samsung_gate_clock top1_gate_clks[] __initdata = { 24, CLK_SET_RATE_PARENT, 0), }; -static struct samsung_fixed_factor_clock top1_fixed_factor_clks[] __initdata = { +static const struct samsung_fixed_factor_clock top1_fixed_factor_clks[] __initconst = { FFACTOR(0, "ffac_top1_bus0_pll_div2", "mout_top1_bus0_pll_user", 1, 2, 0), FFACTOR(0, "ffac_top1_bus1_pll_div2", "mout_top1_bus1_pll_user", @@ -557,7 +558,7 @@ static struct samsung_fixed_factor_clock top1_fixed_factor_clks[] __initdata = { FFACTOR(0, "ffac_top1_mfc_pll_div2", "mout_top1_mfc_pll_user", 1, 2, 0), }; -static struct samsung_cmu_info top1_cmu_info __initdata = { +static const struct samsung_cmu_info top1_cmu_info __initconst = { .mux_clks = top1_mux_clks, .nr_mux_clks = ARRAY_SIZE(top1_mux_clks), .div_clks = top1_div_clks, @@ -591,22 +592,22 @@ CLK_OF_DECLARE(exynos7_clk_top1, "samsung,exynos7-clock-top1", */ PNAME(mout_aclk_ccore_133_user_p) = { "fin_pll", "aclk_ccore_133" }; -static unsigned long ccore_clk_regs[] __initdata = { +static const unsigned long ccore_clk_regs[] __initconst = { MUX_SEL_CCORE, ENABLE_PCLK_CCORE, }; -static struct samsung_mux_clock ccore_mux_clks[] __initdata = { +static const struct samsung_mux_clock ccore_mux_clks[] __initconst = { MUX(0, "mout_aclk_ccore_133_user", mout_aclk_ccore_133_user_p, MUX_SEL_CCORE, 1, 1), }; -static struct samsung_gate_clock ccore_gate_clks[] __initdata = { +static const struct samsung_gate_clock ccore_gate_clks[] __initconst = { GATE(PCLK_RTC, "pclk_rtc", "mout_aclk_ccore_133_user", ENABLE_PCLK_CCORE, 8, 0, 0), }; -static struct samsung_cmu_info ccore_cmu_info __initdata = { +static const struct samsung_cmu_info ccore_cmu_info __initconst = { .mux_clks = ccore_mux_clks, .nr_mux_clks = ARRAY_SIZE(ccore_mux_clks), .gate_clks = ccore_gate_clks, @@ -633,20 +634,20 @@ CLK_OF_DECLARE(exynos7_clk_ccore, "samsung,exynos7-clock-ccore", PNAME(mout_aclk_peric0_66_user_p) = { "fin_pll", "aclk_peric0_66" }; PNAME(mout_sclk_uart0_user_p) = { "fin_pll", "sclk_uart0" }; -static unsigned long peric0_clk_regs[] __initdata = { +static const unsigned long peric0_clk_regs[] __initconst = { MUX_SEL_PERIC0, ENABLE_PCLK_PERIC0, ENABLE_SCLK_PERIC0, }; -static struct samsung_mux_clock peric0_mux_clks[] __initdata = { +static const struct samsung_mux_clock peric0_mux_clks[] __initconst = { MUX(0, "mout_aclk_peric0_66_user", mout_aclk_peric0_66_user_p, MUX_SEL_PERIC0, 0, 1), MUX(0, "mout_sclk_uart0_user", mout_sclk_uart0_user_p, MUX_SEL_PERIC0, 16, 1), }; -static struct samsung_gate_clock peric0_gate_clks[] __initdata = { +static const struct samsung_gate_clock peric0_gate_clks[] __initconst = { GATE(PCLK_HSI2C0, "pclk_hsi2c0", "mout_aclk_peric0_66_user", ENABLE_PCLK_PERIC0, 8, 0, 0), GATE(PCLK_HSI2C1, "pclk_hsi2c1", "mout_aclk_peric0_66_user", @@ -673,7 +674,7 @@ static struct samsung_gate_clock peric0_gate_clks[] __initdata = { GATE(SCLK_PWM, "sclk_pwm", "fin_pll", ENABLE_SCLK_PERIC0, 21, 0, 0), }; -static struct samsung_cmu_info peric0_cmu_info __initdata = { +static const struct samsung_cmu_info peric0_cmu_info __initconst = { .mux_clks = peric0_mux_clks, .nr_mux_clks = ARRAY_SIZE(peric0_mux_clks), .gate_clks = peric0_gate_clks, @@ -709,7 +710,7 @@ PNAME(mout_sclk_spi2_user_p) = { "fin_pll", "sclk_spi2" }; PNAME(mout_sclk_spi3_user_p) = { "fin_pll", "sclk_spi3" }; PNAME(mout_sclk_spi4_user_p) = { "fin_pll", "sclk_spi4" }; -static unsigned long peric1_clk_regs[] __initdata = { +static const unsigned long peric1_clk_regs[] __initconst = { MUX_SEL_PERIC10, MUX_SEL_PERIC11, MUX_SEL_PERIC12, @@ -717,7 +718,7 @@ static unsigned long peric1_clk_regs[] __initdata = { ENABLE_SCLK_PERIC10, }; -static struct samsung_mux_clock peric1_mux_clks[] __initdata = { +static const struct samsung_mux_clock peric1_mux_clks[] __initconst = { MUX(0, "mout_aclk_peric1_66_user", mout_aclk_peric1_66_user_p, MUX_SEL_PERIC10, 0, 1), @@ -739,7 +740,7 @@ static struct samsung_mux_clock peric1_mux_clks[] __initdata = { MUX_SEL_PERIC11, 28, 1), }; -static struct samsung_gate_clock peric1_gate_clks[] __initdata = { +static const struct samsung_gate_clock peric1_gate_clks[] __initconst = { GATE(PCLK_HSI2C2, "pclk_hsi2c2", "mout_aclk_peric1_66_user", ENABLE_PCLK_PERIC1, 4, 0, 0), GATE(PCLK_HSI2C3, "pclk_hsi2c3", "mout_aclk_peric1_66_user", @@ -797,7 +798,7 @@ static struct samsung_gate_clock peric1_gate_clks[] __initdata = { ENABLE_SCLK_PERIC10, 19, CLK_SET_RATE_PARENT, 0), }; -static struct samsung_cmu_info peric1_cmu_info __initdata = { +static const struct samsung_cmu_info peric1_cmu_info __initconst = { .mux_clks = peric1_mux_clks, .nr_mux_clks = ARRAY_SIZE(peric1_mux_clks), .gate_clks = peric1_gate_clks, @@ -825,7 +826,7 @@ CLK_OF_DECLARE(exynos7_clk_peric1, "samsung,exynos7-clock-peric1", /* List of parent clocks for Muxes in CMU_PERIS */ PNAME(mout_aclk_peris_66_user_p) = { "fin_pll", "aclk_peris_66" }; -static unsigned long peris_clk_regs[] __initdata = { +static const unsigned long peris_clk_regs[] __initconst = { MUX_SEL_PERIS, ENABLE_PCLK_PERIS, ENABLE_PCLK_PERIS_SECURE_CHIPID, @@ -833,12 +834,12 @@ static unsigned long peris_clk_regs[] __initdata = { ENABLE_SCLK_PERIS_SECURE_CHIPID, }; -static struct samsung_mux_clock peris_mux_clks[] __initdata = { +static const struct samsung_mux_clock peris_mux_clks[] __initconst = { MUX(0, "mout_aclk_peris_66_user", mout_aclk_peris_66_user_p, MUX_SEL_PERIS, 0, 1), }; -static struct samsung_gate_clock peris_gate_clks[] __initdata = { +static const struct samsung_gate_clock peris_gate_clks[] __initconst = { GATE(PCLK_WDT, "pclk_wdt", "mout_aclk_peris_66_user", ENABLE_PCLK_PERIS, 6, 0, 0), GATE(PCLK_TMU, "pclk_tmu_apbif", "mout_aclk_peris_66_user", @@ -852,7 +853,7 @@ static struct samsung_gate_clock peris_gate_clks[] __initdata = { GATE(SCLK_TMU, "sclk_tmu", "fin_pll", ENABLE_SCLK_PERIS, 10, 0, 0), }; -static struct samsung_cmu_info peris_cmu_info __initdata = { +static const struct samsung_cmu_info peris_cmu_info __initconst = { .mux_clks = peris_mux_clks, .nr_mux_clks = ARRAY_SIZE(peris_mux_clks), .gate_clks = peris_gate_clks, @@ -893,12 +894,12 @@ PNAME(mout_phyclk_usbdrd300_udrd30_pipe_pclk_user_p) = { "fin_pll", "phyclk_usbdrd300_udrd30_pipe_pclk" }; /* fixed rate clocks used in the FSYS0 block */ -static struct samsung_fixed_rate_clock fixed_rate_clks_fsys0[] __initdata = { +static const struct samsung_fixed_rate_clock fixed_rate_clks_fsys0[] __initconst = { FRATE(0, "phyclk_usbdrd300_udrd30_phyclock", NULL, 0, 60000000), FRATE(0, "phyclk_usbdrd300_udrd30_pipe_pclk", NULL, 0, 125000000), }; -static unsigned long fsys0_clk_regs[] __initdata = { +static const unsigned long fsys0_clk_regs[] __initconst = { MUX_SEL_FSYS00, MUX_SEL_FSYS01, MUX_SEL_FSYS02, @@ -909,7 +910,7 @@ static unsigned long fsys0_clk_regs[] __initdata = { ENABLE_SCLK_FSYS04, }; -static struct samsung_mux_clock fsys0_mux_clks[] __initdata = { +static const struct samsung_mux_clock fsys0_mux_clks[] __initconst = { MUX(0, "mout_aclk_fsys0_200_user", mout_aclk_fsys0_200_user_p, MUX_SEL_FSYS00, 24, 1), @@ -926,7 +927,7 @@ static struct samsung_mux_clock fsys0_mux_clks[] __initdata = { MUX_SEL_FSYS02, 28, 1), }; -static struct samsung_gate_clock fsys0_gate_clks[] __initdata = { +static const struct samsung_gate_clock fsys0_gate_clks[] __initconst = { GATE(ACLK_PDMA1, "aclk_pdma1", "mout_aclk_fsys0_200_user", ENABLE_ACLK_FSYS00, 3, 0, 0), GATE(ACLK_PDMA0, "aclk_pdma0", "mout_aclk_fsys0_200_user", @@ -960,7 +961,7 @@ static struct samsung_gate_clock fsys0_gate_clks[] __initdata = { ENABLE_SCLK_FSYS04, 28, 0, 0), }; -static struct samsung_cmu_info fsys0_cmu_info __initdata = { +static const struct samsung_cmu_info fsys0_cmu_info __initconst = { .fixed_clks = fixed_rate_clks_fsys0, .nr_fixed_clks = ARRAY_SIZE(fixed_rate_clks_fsys0), .mux_clks = fsys0_mux_clks, @@ -1005,7 +1006,7 @@ PNAME(mout_phyclk_ufs20_rx0_user_p) = { "fin_pll", "phyclk_ufs20_rx0_symbol" }; PNAME(mout_phyclk_ufs20_rx1_user_p) = { "fin_pll", "phyclk_ufs20_rx1_symbol" }; /* fixed rate clocks used in the FSYS1 block */ -static struct samsung_fixed_rate_clock fixed_rate_clks_fsys1[] __initdata = { +static const struct samsung_fixed_rate_clock fixed_rate_clks_fsys1[] __initconst = { FRATE(PHYCLK_UFS20_TX0_SYMBOL, "phyclk_ufs20_tx0_symbol", NULL, 0, 300000000), FRATE(PHYCLK_UFS20_RX0_SYMBOL, "phyclk_ufs20_rx0_symbol", NULL, @@ -1014,7 +1015,7 @@ static struct samsung_fixed_rate_clock fixed_rate_clks_fsys1[] __initdata = { 0, 300000000), }; -static unsigned long fsys1_clk_regs[] __initdata = { +static const unsigned long fsys1_clk_regs[] __initconst = { MUX_SEL_FSYS10, MUX_SEL_FSYS11, MUX_SEL_FSYS12, @@ -1026,7 +1027,7 @@ static unsigned long fsys1_clk_regs[] __initdata = { ENABLE_SCLK_FSYS13, }; -static struct samsung_mux_clock fsys1_mux_clks[] __initdata = { +static const struct samsung_mux_clock fsys1_mux_clks[] __initconst = { MUX(MOUT_FSYS1_PHYCLK_SEL1, "mout_fsys1_phyclk_sel1", mout_fsys1_group_p, MUX_SEL_FSYS10, 16, 2), MUX(0, "mout_fsys1_phyclk_sel0", mout_fsys1_group_p, @@ -1049,12 +1050,12 @@ static struct samsung_mux_clock fsys1_mux_clks[] __initdata = { mout_phyclk_ufs20_tx0_user_p, MUX_SEL_FSYS12, 28, 1), }; -static struct samsung_div_clock fsys1_div_clks[] __initdata = { +static const struct samsung_div_clock fsys1_div_clks[] __initconst = { DIV(DOUT_PCLK_FSYS1, "dout_pclk_fsys1", "mout_aclk_fsys1_200_user", DIV_FSYS1, 0, 2), }; -static struct samsung_gate_clock fsys1_gate_clks[] __initdata = { +static const struct samsung_gate_clock fsys1_gate_clks[] __initconst = { GATE(SCLK_UFSUNIPRO20_USER, "sclk_ufsunipro20_user", "mout_sclk_ufsunipro20_user", ENABLE_SCLK_FSYS11, 20, 0, 0), @@ -1089,7 +1090,7 @@ static struct samsung_gate_clock fsys1_gate_clks[] __initdata = { ENABLE_SCLK_FSYS13, 24, CLK_IGNORE_UNUSED, 0), }; -static struct samsung_cmu_info fsys1_cmu_info __initdata = { +static const struct samsung_cmu_info fsys1_cmu_info __initconst = { .fixed_clks = fixed_rate_clks_fsys1, .nr_fixed_clks = ARRAY_SIZE(fixed_rate_clks_fsys1), .mux_clks = fsys1_mux_clks, @@ -1119,22 +1120,22 @@ CLK_OF_DECLARE(exynos7_clk_fsys1, "samsung,exynos7-clock-fsys1", /* List of parent clocks for Muxes in CMU_MSCL */ PNAME(mout_aclk_mscl_532_user_p) = { "fin_pll", "aclk_mscl_532" }; -static unsigned long mscl_clk_regs[] __initdata = { +static const unsigned long mscl_clk_regs[] __initconst = { MUX_SEL_MSCL, DIV_MSCL, ENABLE_ACLK_MSCL, ENABLE_PCLK_MSCL, }; -static struct samsung_mux_clock mscl_mux_clks[] __initdata = { +static const struct samsung_mux_clock mscl_mux_clks[] __initconst = { MUX(USERMUX_ACLK_MSCL_532, "usermux_aclk_mscl_532", mout_aclk_mscl_532_user_p, MUX_SEL_MSCL, 0, 1), }; -static struct samsung_div_clock mscl_div_clks[] __initdata = { +static const struct samsung_div_clock mscl_div_clks[] __initconst = { DIV(DOUT_PCLK_MSCL, "dout_pclk_mscl", "usermux_aclk_mscl_532", DIV_MSCL, 0, 3), }; -static struct samsung_gate_clock mscl_gate_clks[] __initdata = { +static const struct samsung_gate_clock mscl_gate_clks[] __initconst = { GATE(ACLK_MSCL_0, "aclk_mscl_0", "usermux_aclk_mscl_532", ENABLE_ACLK_MSCL, 31, 0, 0), @@ -1204,7 +1205,7 @@ static struct samsung_gate_clock mscl_gate_clks[] __initdata = { ENABLE_PCLK_MSCL, 20, 0, 0), }; -static struct samsung_cmu_info mscl_cmu_info __initdata = { +static const struct samsung_cmu_info mscl_cmu_info __initconst = { .mux_clks = mscl_mux_clks, .nr_mux_clks = ARRAY_SIZE(mscl_mux_clks), .div_clks = mscl_div_clks, @@ -1238,7 +1239,7 @@ CLK_OF_DECLARE(exynos7_clk_mscl, "samsung,exynos7-clock-mscl", PNAME(mout_aud_pll_user_p) = { "fin_pll", "fout_aud_pll" }; PNAME(mout_aud_group_p) = { "dout_aud_cdclk", "ioclk_audiocdclk0" }; -static unsigned long aud_clk_regs[] __initdata = { +static const unsigned long aud_clk_regs[] __initconst = { MUX_SEL_AUD, DIV_AUD0, DIV_AUD1, @@ -1247,13 +1248,13 @@ static unsigned long aud_clk_regs[] __initdata = { ENABLE_SCLK_AUD, }; -static struct samsung_mux_clock aud_mux_clks[] __initdata = { +static const struct samsung_mux_clock aud_mux_clks[] __initconst = { MUX(0, "mout_sclk_i2s", mout_aud_group_p, MUX_SEL_AUD, 12, 1), MUX(0, "mout_sclk_pcm", mout_aud_group_p, MUX_SEL_AUD, 16, 1), MUX(0, "mout_aud_pll_user", mout_aud_pll_user_p, MUX_SEL_AUD, 20, 1), }; -static struct samsung_div_clock aud_div_clks[] __initdata = { +static const struct samsung_div_clock aud_div_clks[] __initconst = { DIV(0, "dout_aud_ca5", "mout_aud_pll_user", DIV_AUD0, 0, 4), DIV(0, "dout_aclk_aud", "dout_aud_ca5", DIV_AUD0, 4, 4), DIV(0, "dout_aud_pclk_dbg", "dout_aud_ca5", DIV_AUD0, 8, 4), @@ -1265,7 +1266,7 @@ static struct samsung_div_clock aud_div_clks[] __initdata = { DIV(0, "dout_aud_cdclk", "mout_aud_pll_user", DIV_AUD1, 24, 4), }; -static struct samsung_gate_clock aud_gate_clks[] __initdata = { +static const struct samsung_gate_clock aud_gate_clks[] __initconst = { GATE(SCLK_PCM, "sclk_pcm", "dout_sclk_pcm", ENABLE_SCLK_AUD, 27, CLK_SET_RATE_PARENT, 0), GATE(SCLK_I2S, "sclk_i2s", "dout_sclk_i2s", @@ -1293,7 +1294,7 @@ static struct samsung_gate_clock aud_gate_clks[] __initdata = { GATE(ACLK_ADMA, "aclk_dmac", "dout_aclk_aud", ENABLE_ACLK_AUD, 31, 0, 0), }; -static struct samsung_cmu_info aud_cmu_info __initdata = { +static const struct samsung_cmu_info aud_cmu_info __initconst = { .mux_clks = aud_mux_clks, .nr_mux_clks = ARRAY_SIZE(aud_mux_clks), .div_clks = aud_div_clks, diff --git a/drivers/clk/samsung/clk-pll.c b/drivers/clk/samsung/clk-pll.c index b7dd39610..48139bd51 100644 --- a/drivers/clk/samsung/clk-pll.c +++ b/drivers/clk/samsung/clk-pll.c @@ -79,7 +79,7 @@ static unsigned long samsung_pll2126_recalc_rate(struct clk_hw *hw, u32 pll_con, mdiv, pdiv, sdiv; u64 fvco = parent_rate; - pll_con = __raw_readl(pll->con_reg); + pll_con = readl_relaxed(pll->con_reg); mdiv = (pll_con >> PLL2126_MDIV_SHIFT) & PLL2126_MDIV_MASK; pdiv = (pll_con >> PLL2126_PDIV_SHIFT) & PLL2126_PDIV_MASK; sdiv = (pll_con >> PLL2126_SDIV_SHIFT) & PLL2126_SDIV_MASK; @@ -112,7 +112,7 @@ static unsigned long samsung_pll3000_recalc_rate(struct clk_hw *hw, u32 pll_con, mdiv, pdiv, sdiv; u64 fvco = parent_rate; - pll_con = __raw_readl(pll->con_reg); + pll_con = readl_relaxed(pll->con_reg); mdiv = (pll_con >> PLL3000_MDIV_SHIFT) & PLL3000_MDIV_MASK; pdiv = (pll_con >> PLL3000_PDIV_SHIFT) & PLL3000_PDIV_MASK; sdiv = (pll_con >> PLL3000_SDIV_SHIFT) & PLL3000_SDIV_MASK; @@ -149,7 +149,7 @@ static unsigned long samsung_pll35xx_recalc_rate(struct clk_hw *hw, u32 mdiv, pdiv, sdiv, pll_con; u64 fvco = parent_rate; - pll_con = __raw_readl(pll->con_reg); + pll_con = readl_relaxed(pll->con_reg); mdiv = (pll_con >> PLL35XX_MDIV_SHIFT) & PLL35XX_MDIV_MASK; pdiv = (pll_con >> PLL35XX_PDIV_SHIFT) & PLL35XX_PDIV_MASK; sdiv = (pll_con >> PLL35XX_SDIV_SHIFT) & PLL35XX_SDIV_MASK; @@ -186,19 +186,19 @@ static int samsung_pll35xx_set_rate(struct clk_hw *hw, unsigned long drate, return -EINVAL; } - tmp = __raw_readl(pll->con_reg); + tmp = readl_relaxed(pll->con_reg); if (!(samsung_pll35xx_mp_change(rate, tmp))) { /* If only s change, change just s value only*/ tmp &= ~(PLL35XX_SDIV_MASK << PLL35XX_SDIV_SHIFT); tmp |= rate->sdiv << PLL35XX_SDIV_SHIFT; - __raw_writel(tmp, pll->con_reg); + writel_relaxed(tmp, pll->con_reg); return 0; } /* Set PLL lock time. */ - __raw_writel(rate->pdiv * PLL35XX_LOCK_FACTOR, + writel_relaxed(rate->pdiv * PLL35XX_LOCK_FACTOR, pll->lock_reg); /* Change PLL PMS values */ @@ -208,12 +208,12 @@ static int samsung_pll35xx_set_rate(struct clk_hw *hw, unsigned long drate, tmp |= (rate->mdiv << PLL35XX_MDIV_SHIFT) | (rate->pdiv << PLL35XX_PDIV_SHIFT) | (rate->sdiv << PLL35XX_SDIV_SHIFT); - __raw_writel(tmp, pll->con_reg); + writel_relaxed(tmp, pll->con_reg); /* wait_lock_time */ do { cpu_relax(); - tmp = __raw_readl(pll->con_reg); + tmp = readl_relaxed(pll->con_reg); } while (!(tmp & (PLL35XX_LOCK_STAT_MASK << PLL35XX_LOCK_STAT_SHIFT))); return 0; @@ -253,8 +253,8 @@ static unsigned long samsung_pll36xx_recalc_rate(struct clk_hw *hw, s16 kdiv; u64 fvco = parent_rate; - pll_con0 = __raw_readl(pll->con_reg); - pll_con1 = __raw_readl(pll->con_reg + 4); + pll_con0 = readl_relaxed(pll->con_reg); + pll_con1 = readl_relaxed(pll->con_reg + 4); mdiv = (pll_con0 >> PLL36XX_MDIV_SHIFT) & PLL36XX_MDIV_MASK; pdiv = (pll_con0 >> PLL36XX_PDIV_SHIFT) & PLL36XX_PDIV_MASK; sdiv = (pll_con0 >> PLL36XX_SDIV_SHIFT) & PLL36XX_SDIV_MASK; @@ -294,20 +294,20 @@ static int samsung_pll36xx_set_rate(struct clk_hw *hw, unsigned long drate, return -EINVAL; } - pll_con0 = __raw_readl(pll->con_reg); - pll_con1 = __raw_readl(pll->con_reg + 4); + pll_con0 = readl_relaxed(pll->con_reg); + pll_con1 = readl_relaxed(pll->con_reg + 4); if (!(samsung_pll36xx_mpk_change(rate, pll_con0, pll_con1))) { /* If only s change, change just s value only*/ pll_con0 &= ~(PLL36XX_SDIV_MASK << PLL36XX_SDIV_SHIFT); pll_con0 |= (rate->sdiv << PLL36XX_SDIV_SHIFT); - __raw_writel(pll_con0, pll->con_reg); + writel_relaxed(pll_con0, pll->con_reg); return 0; } /* Set PLL lock time. */ - __raw_writel(rate->pdiv * PLL36XX_LOCK_FACTOR, pll->lock_reg); + writel_relaxed(rate->pdiv * PLL36XX_LOCK_FACTOR, pll->lock_reg); /* Change PLL PMS values */ pll_con0 &= ~((PLL36XX_MDIV_MASK << PLL36XX_MDIV_SHIFT) | @@ -316,16 +316,16 @@ static int samsung_pll36xx_set_rate(struct clk_hw *hw, unsigned long drate, pll_con0 |= (rate->mdiv << PLL36XX_MDIV_SHIFT) | (rate->pdiv << PLL36XX_PDIV_SHIFT) | (rate->sdiv << PLL36XX_SDIV_SHIFT); - __raw_writel(pll_con0, pll->con_reg); + writel_relaxed(pll_con0, pll->con_reg); pll_con1 &= ~(PLL36XX_KDIV_MASK << PLL36XX_KDIV_SHIFT); pll_con1 |= rate->kdiv << PLL36XX_KDIV_SHIFT; - __raw_writel(pll_con1, pll->con_reg + 4); + writel_relaxed(pll_con1, pll->con_reg + 4); /* wait_lock_time */ do { cpu_relax(); - tmp = __raw_readl(pll->con_reg); + tmp = readl_relaxed(pll->con_reg); } while (!(tmp & (1 << PLL36XX_LOCK_STAT_SHIFT))); return 0; @@ -366,7 +366,7 @@ static unsigned long samsung_pll45xx_recalc_rate(struct clk_hw *hw, u32 mdiv, pdiv, sdiv, pll_con; u64 fvco = parent_rate; - pll_con = __raw_readl(pll->con_reg); + pll_con = readl_relaxed(pll->con_reg); mdiv = (pll_con >> PLL45XX_MDIV_SHIFT) & PLL45XX_MDIV_MASK; pdiv = (pll_con >> PLL45XX_PDIV_SHIFT) & PLL45XX_PDIV_MASK; sdiv = (pll_con >> PLL45XX_SDIV_SHIFT) & PLL45XX_SDIV_MASK; @@ -409,14 +409,14 @@ static int samsung_pll45xx_set_rate(struct clk_hw *hw, unsigned long drate, return -EINVAL; } - con0 = __raw_readl(pll->con_reg); - con1 = __raw_readl(pll->con_reg + 0x4); + con0 = readl_relaxed(pll->con_reg); + con1 = readl_relaxed(pll->con_reg + 0x4); if (!(samsung_pll45xx_mp_change(con0, con1, rate))) { /* If only s change, change just s value only*/ con0 &= ~(PLL45XX_SDIV_MASK << PLL45XX_SDIV_SHIFT); con0 |= rate->sdiv << PLL45XX_SDIV_SHIFT; - __raw_writel(con0, pll->con_reg); + writel_relaxed(con0, pll->con_reg); return 0; } @@ -430,29 +430,29 @@ static int samsung_pll45xx_set_rate(struct clk_hw *hw, unsigned long drate, (rate->sdiv << PLL45XX_SDIV_SHIFT); /* Set PLL AFC value. */ - con1 = __raw_readl(pll->con_reg + 0x4); + con1 = readl_relaxed(pll->con_reg + 0x4); con1 &= ~(PLL45XX_AFC_MASK << PLL45XX_AFC_SHIFT); con1 |= (rate->afc << PLL45XX_AFC_SHIFT); /* Set PLL lock time. */ switch (pll->type) { case pll_4502: - __raw_writel(rate->pdiv * PLL4502_LOCK_FACTOR, pll->lock_reg); + writel_relaxed(rate->pdiv * PLL4502_LOCK_FACTOR, pll->lock_reg); break; case pll_4508: - __raw_writel(rate->pdiv * PLL4508_LOCK_FACTOR, pll->lock_reg); + writel_relaxed(rate->pdiv * PLL4508_LOCK_FACTOR, pll->lock_reg); break; default: break; } /* Set new configuration. */ - __raw_writel(con1, pll->con_reg + 0x4); - __raw_writel(con0, pll->con_reg); + writel_relaxed(con1, pll->con_reg + 0x4); + writel_relaxed(con0, pll->con_reg); /* Wait for locking. */ start = ktime_get(); - while (!(__raw_readl(pll->con_reg) & PLL45XX_LOCKED)) { + while (!(readl_relaxed(pll->con_reg) & PLL45XX_LOCKED)) { ktime_t delta = ktime_sub(ktime_get(), start); if (ktime_to_ms(delta) > PLL_TIMEOUT_MS) { @@ -513,8 +513,8 @@ static unsigned long samsung_pll46xx_recalc_rate(struct clk_hw *hw, u32 mdiv, pdiv, sdiv, kdiv, pll_con0, pll_con1, shift; u64 fvco = parent_rate; - pll_con0 = __raw_readl(pll->con_reg); - pll_con1 = __raw_readl(pll->con_reg + 4); + pll_con0 = readl_relaxed(pll->con_reg); + pll_con1 = readl_relaxed(pll->con_reg + 4); mdiv = (pll_con0 >> PLL46XX_MDIV_SHIFT) & ((pll->type == pll_1460x) ? PLL1460X_MDIV_MASK : PLL46XX_MDIV_MASK); pdiv = (pll_con0 >> PLL46XX_PDIV_SHIFT) & PLL46XX_PDIV_MASK; @@ -560,14 +560,14 @@ static int samsung_pll46xx_set_rate(struct clk_hw *hw, unsigned long drate, return -EINVAL; } - con0 = __raw_readl(pll->con_reg); - con1 = __raw_readl(pll->con_reg + 0x4); + con0 = readl_relaxed(pll->con_reg); + con1 = readl_relaxed(pll->con_reg + 0x4); if (!(samsung_pll46xx_mpk_change(con0, con1, rate))) { /* If only s change, change just s value only*/ con0 &= ~(PLL46XX_SDIV_MASK << PLL46XX_SDIV_SHIFT); con0 |= rate->sdiv << PLL46XX_SDIV_SHIFT; - __raw_writel(con0, pll->con_reg); + writel_relaxed(con0, pll->con_reg); return 0; } @@ -596,7 +596,7 @@ static int samsung_pll46xx_set_rate(struct clk_hw *hw, unsigned long drate, (rate->sdiv << PLL46XX_SDIV_SHIFT); /* Set PLL K, MFR and MRR values. */ - con1 = __raw_readl(pll->con_reg + 0x4); + con1 = readl_relaxed(pll->con_reg + 0x4); con1 &= ~((PLL46XX_KDIV_MASK << PLL46XX_KDIV_SHIFT) | (PLL46XX_MFR_MASK << PLL46XX_MFR_SHIFT) | (PLL46XX_MRR_MASK << PLL46XX_MRR_SHIFT)); @@ -605,13 +605,13 @@ static int samsung_pll46xx_set_rate(struct clk_hw *hw, unsigned long drate, (rate->mrr << PLL46XX_MRR_SHIFT); /* Write configuration to PLL */ - __raw_writel(lock, pll->lock_reg); - __raw_writel(con0, pll->con_reg); - __raw_writel(con1, pll->con_reg + 0x4); + writel_relaxed(lock, pll->lock_reg); + writel_relaxed(con0, pll->con_reg); + writel_relaxed(con1, pll->con_reg + 0x4); /* Wait for locking. */ start = ktime_get(); - while (!(__raw_readl(pll->con_reg) & PLL46XX_LOCKED)) { + while (!(readl_relaxed(pll->con_reg) & PLL46XX_LOCKED)) { ktime_t delta = ktime_sub(ktime_get(), start); if (ktime_to_ms(delta) > PLL_TIMEOUT_MS) { @@ -656,7 +656,7 @@ static unsigned long samsung_pll6552_recalc_rate(struct clk_hw *hw, u32 mdiv, pdiv, sdiv, pll_con; u64 fvco = parent_rate; - pll_con = __raw_readl(pll->con_reg); + pll_con = readl_relaxed(pll->con_reg); if (pll->type == pll_6552_s3c2416) { mdiv = (pll_con >> PLL6552_MDIV_SHIFT_2416) & PLL6552_MDIV_MASK; pdiv = (pll_con >> PLL6552_PDIV_SHIFT_2416) & PLL6552_PDIV_MASK; @@ -696,8 +696,8 @@ static unsigned long samsung_pll6553_recalc_rate(struct clk_hw *hw, u32 mdiv, pdiv, sdiv, kdiv, pll_con0, pll_con1; u64 fvco = parent_rate; - pll_con0 = __raw_readl(pll->con_reg); - pll_con1 = __raw_readl(pll->con_reg + 0x4); + pll_con0 = readl_relaxed(pll->con_reg); + pll_con1 = readl_relaxed(pll->con_reg + 0x4); mdiv = (pll_con0 >> PLL6553_MDIV_SHIFT) & PLL6553_MDIV_MASK; pdiv = (pll_con0 >> PLL6553_PDIV_SHIFT) & PLL6553_PDIV_MASK; sdiv = (pll_con0 >> PLL6553_SDIV_SHIFT) & PLL6553_SDIV_MASK; @@ -734,7 +734,7 @@ static unsigned long samsung_s3c2410_pll_recalc_rate(struct clk_hw *hw, u32 pll_con, mdiv, pdiv, sdiv; u64 fvco = parent_rate; - pll_con = __raw_readl(pll->con_reg); + pll_con = readl_relaxed(pll->con_reg); mdiv = (pll_con >> PLLS3C2410_MDIV_SHIFT) & PLLS3C2410_MDIV_MASK; pdiv = (pll_con >> PLLS3C2410_PDIV_SHIFT) & PLLS3C2410_PDIV_MASK; sdiv = (pll_con >> PLLS3C2410_SDIV_SHIFT) & PLLS3C2410_SDIV_MASK; @@ -752,7 +752,7 @@ static unsigned long samsung_s3c2440_mpll_recalc_rate(struct clk_hw *hw, u32 pll_con, mdiv, pdiv, sdiv; u64 fvco = parent_rate; - pll_con = __raw_readl(pll->con_reg); + pll_con = readl_relaxed(pll->con_reg); mdiv = (pll_con >> PLLS3C2410_MDIV_SHIFT) & PLLS3C2410_MDIV_MASK; pdiv = (pll_con >> PLLS3C2410_PDIV_SHIFT) & PLLS3C2410_PDIV_MASK; sdiv = (pll_con >> PLLS3C2410_SDIV_SHIFT) & PLLS3C2410_SDIV_MASK; @@ -778,7 +778,7 @@ static int samsung_s3c2410_pll_set_rate(struct clk_hw *hw, unsigned long drate, return -EINVAL; } - tmp = __raw_readl(pll->con_reg); + tmp = readl_relaxed(pll->con_reg); /* Change PLL PMS values */ tmp &= ~((PLLS3C2410_MDIV_MASK << PLLS3C2410_MDIV_SHIFT) | @@ -787,7 +787,7 @@ static int samsung_s3c2410_pll_set_rate(struct clk_hw *hw, unsigned long drate, tmp |= (rate->mdiv << PLLS3C2410_MDIV_SHIFT) | (rate->pdiv << PLLS3C2410_PDIV_SHIFT) | (rate->sdiv << PLLS3C2410_SDIV_SHIFT); - __raw_writel(tmp, pll->con_reg); + writel_relaxed(tmp, pll->con_reg); /* Time to settle according to the manual */ udelay(300); @@ -798,7 +798,7 @@ static int samsung_s3c2410_pll_set_rate(struct clk_hw *hw, unsigned long drate, static int samsung_s3c2410_pll_enable(struct clk_hw *hw, int bit, bool enable) { struct samsung_clk_pll *pll = to_clk_pll(hw); - u32 pll_en = __raw_readl(pll->lock_reg + PLLS3C2410_ENABLE_REG_OFFSET); + u32 pll_en = readl_relaxed(pll->lock_reg + PLLS3C2410_ENABLE_REG_OFFSET); u32 pll_en_orig = pll_en; if (enable) @@ -806,7 +806,7 @@ static int samsung_s3c2410_pll_enable(struct clk_hw *hw, int bit, bool enable) else pll_en |= BIT(bit); - __raw_writel(pll_en, pll->lock_reg + PLLS3C2410_ENABLE_REG_OFFSET); + writel_relaxed(pll_en, pll->lock_reg + PLLS3C2410_ENABLE_REG_OFFSET); /* if we started the UPLL, then allow to settle */ if (enable && (pll_en_orig & BIT(bit))) @@ -905,7 +905,7 @@ static unsigned long samsung_pll2550x_recalc_rate(struct clk_hw *hw, u32 r, p, m, s, pll_stat; u64 fvco = parent_rate; - pll_stat = __raw_readl(pll->reg_base + pll->offset * 3); + pll_stat = readl_relaxed(pll->reg_base + pll->offset * 3); r = (pll_stat >> PLL2550X_R_SHIFT) & PLL2550X_R_MASK; if (!r) return 0; @@ -983,7 +983,7 @@ static unsigned long samsung_pll2550xx_recalc_rate(struct clk_hw *hw, u32 mdiv, pdiv, sdiv, pll_con; u64 fvco = parent_rate; - pll_con = __raw_readl(pll->con_reg); + pll_con = readl_relaxed(pll->con_reg); mdiv = (pll_con >> PLL2550XX_M_SHIFT) & PLL2550XX_M_MASK; pdiv = (pll_con >> PLL2550XX_P_SHIFT) & PLL2550XX_P_MASK; sdiv = (pll_con >> PLL2550XX_S_SHIFT) & PLL2550XX_S_MASK; @@ -1019,19 +1019,19 @@ static int samsung_pll2550xx_set_rate(struct clk_hw *hw, unsigned long drate, return -EINVAL; } - tmp = __raw_readl(pll->con_reg); + tmp = readl_relaxed(pll->con_reg); if (!(samsung_pll2550xx_mp_change(rate->mdiv, rate->pdiv, tmp))) { /* If only s change, change just s value only*/ tmp &= ~(PLL2550XX_S_MASK << PLL2550XX_S_SHIFT); tmp |= rate->sdiv << PLL2550XX_S_SHIFT; - __raw_writel(tmp, pll->con_reg); + writel_relaxed(tmp, pll->con_reg); return 0; } /* Set PLL lock time. */ - __raw_writel(rate->pdiv * PLL2550XX_LOCK_FACTOR, pll->lock_reg); + writel_relaxed(rate->pdiv * PLL2550XX_LOCK_FACTOR, pll->lock_reg); /* Change PLL PMS values */ tmp &= ~((PLL2550XX_M_MASK << PLL2550XX_M_SHIFT) | @@ -1040,12 +1040,12 @@ static int samsung_pll2550xx_set_rate(struct clk_hw *hw, unsigned long drate, tmp |= (rate->mdiv << PLL2550XX_M_SHIFT) | (rate->pdiv << PLL2550XX_P_SHIFT) | (rate->sdiv << PLL2550XX_S_SHIFT); - __raw_writel(tmp, pll->con_reg); + writel_relaxed(tmp, pll->con_reg); /* wait_lock_time */ do { cpu_relax(); - tmp = __raw_readl(pll->con_reg); + tmp = readl_relaxed(pll->con_reg); } while (!(tmp & (PLL2550XX_LOCK_STAT_MASK << PLL2550XX_LOCK_STAT_SHIFT))); @@ -1089,8 +1089,8 @@ static unsigned long samsung_pll2650xx_recalc_rate(struct clk_hw *hw, s16 kdiv; u64 fvco = parent_rate; - pll_con0 = __raw_readl(pll->con_reg); - pll_con2 = __raw_readl(pll->con_reg + 8); + pll_con0 = readl_relaxed(pll->con_reg); + pll_con2 = readl_relaxed(pll->con_reg + 8); mdiv = (pll_con0 >> PLL2650XX_MDIV_SHIFT) & PLL2650XX_MDIV_MASK; pdiv = (pll_con0 >> PLL2650XX_PDIV_SHIFT) & PLL2650XX_PDIV_MASK; sdiv = (pll_con0 >> PLL2650XX_SDIV_SHIFT) & PLL2650XX_SDIV_MASK; @@ -1117,8 +1117,8 @@ static int samsung_pll2650xx_set_rate(struct clk_hw *hw, unsigned long drate, return -EINVAL; } - pll_con0 = __raw_readl(pll->con_reg); - pll_con2 = __raw_readl(pll->con_reg + 8); + pll_con0 = readl_relaxed(pll->con_reg); + pll_con2 = readl_relaxed(pll->con_reg + 8); /* Change PLL PMS values */ pll_con0 &= ~(PLL2650XX_MDIV_MASK << PLL2650XX_MDIV_SHIFT | @@ -1135,13 +1135,13 @@ static int samsung_pll2650xx_set_rate(struct clk_hw *hw, unsigned long drate, << PLL2650XX_KDIV_SHIFT; /* Set PLL lock time. */ - __raw_writel(PLL2650XX_LOCK_FACTOR * rate->pdiv, pll->lock_reg); + writel_relaxed(PLL2650XX_LOCK_FACTOR * rate->pdiv, pll->lock_reg); - __raw_writel(pll_con0, pll->con_reg); - __raw_writel(pll_con2, pll->con_reg + 8); + writel_relaxed(pll_con0, pll->con_reg); + writel_relaxed(pll_con2, pll->con_reg + 8); do { - tmp = __raw_readl(pll->con_reg); + tmp = readl_relaxed(pll->con_reg); } while (!(tmp & (0x1 << PLL2650XX_PLL_LOCKTIME_SHIFT))); return 0; diff --git a/drivers/clk/samsung/clk-s3c2410-dclk.c b/drivers/clk/samsung/clk-s3c2410-dclk.c index ec6fb14d9..ae9a595c7 100644 --- a/drivers/clk/samsung/clk-s3c2410-dclk.c +++ b/drivers/clk/samsung/clk-s3c2410-dclk.c @@ -428,8 +428,9 @@ MODULE_DEVICE_TABLE(platform, s3c24xx_dclk_driver_ids); static struct platform_driver s3c24xx_dclk_driver = { .driver = { - .name = "s3c24xx-dclk", - .pm = &s3c24xx_dclk_pm_ops, + .name = "s3c24xx-dclk", + .pm = &s3c24xx_dclk_pm_ops, + .suppress_bind_attrs = true, }, .probe = s3c24xx_dclk_probe, .remove = s3c24xx_dclk_remove, diff --git a/drivers/clk/samsung/clk-s3c2410.c b/drivers/clk/samsung/clk-s3c2410.c index d7b011c1f..d7a1e772d 100644 --- a/drivers/clk/samsung/clk-s3c2410.c +++ b/drivers/clk/samsung/clk-s3c2410.c @@ -374,8 +374,6 @@ void __init s3c2410_common_clk_init(struct device_node *np, unsigned long xti_f, } ctx = samsung_clk_init(np, reg_base, NR_CLKS); - if (!ctx) - panic("%s: unable to allocate context.\n", __func__); /* Register external clocks only in non-dt cases */ if (!np) diff --git a/drivers/clk/samsung/clk-s3c2412.c b/drivers/clk/samsung/clk-s3c2412.c index effe3736e..ec873ee15 100644 --- a/drivers/clk/samsung/clk-s3c2412.c +++ b/drivers/clk/samsung/clk-s3c2412.c @@ -265,8 +265,6 @@ void __init s3c2412_common_clk_init(struct device_node *np, unsigned long xti_f, } ctx = samsung_clk_init(np, reg_base, NR_CLKS); - if (!ctx) - panic("%s: unable to allocate context.\n", __func__); /* Register external clocks only in non-dt cases */ if (!np) diff --git a/drivers/clk/samsung/clk-s3c2443.c b/drivers/clk/samsung/clk-s3c2443.c index 37562783b..5e24a17e1 100644 --- a/drivers/clk/samsung/clk-s3c2443.c +++ b/drivers/clk/samsung/clk-s3c2443.c @@ -400,8 +400,6 @@ void __init s3c2443_common_clk_init(struct device_node *np, unsigned long xti_f, } ctx = samsung_clk_init(np, reg_base, NR_CLKS); - if (!ctx) - panic("%s: unable to allocate context.\n", __func__); /* Register external clocks only in non-dt cases */ if (!np) diff --git a/drivers/clk/samsung/clk-s3c64xx.c b/drivers/clk/samsung/clk-s3c64xx.c index 60aa775bd..a48bd5f17 100644 --- a/drivers/clk/samsung/clk-s3c64xx.c +++ b/drivers/clk/samsung/clk-s3c64xx.c @@ -471,8 +471,6 @@ void __init s3c64xx_clk_init(struct device_node *np, unsigned long xtal_f, } ctx = samsung_clk_init(np, reg_base, NR_CLKS); - if (!ctx) - panic("%s: unable to allocate context.\n", __func__); /* Register external clocks. */ if (!np) diff --git a/drivers/clk/samsung/clk-s5pv210-audss.c b/drivers/clk/samsung/clk-s5pv210-audss.c index eefb84b22..c66ed2d14 100644 --- a/drivers/clk/samsung/clk-s5pv210-audss.c +++ b/drivers/clk/samsung/clk-s5pv210-audss.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include @@ -194,20 +194,6 @@ unregister: return ret; } -static int s5pv210_audss_clk_remove(struct platform_device *pdev) -{ - int i; - - of_clk_del_provider(pdev->dev.of_node); - - for (i = 0; i < clk_data.clk_num; i++) { - if (!IS_ERR(clk_table[i])) - clk_unregister(clk_table[i]); - } - - return 0; -} - static const struct of_device_id s5pv210_audss_clk_of_match[] = { { .compatible = "samsung,s5pv210-audss-clock", }, {}, @@ -216,10 +202,10 @@ static const struct of_device_id s5pv210_audss_clk_of_match[] = { static struct platform_driver s5pv210_audss_clk_driver = { .driver = { .name = "s5pv210-audss-clk", + .suppress_bind_attrs = true, .of_match_table = s5pv210_audss_clk_of_match, }, .probe = s5pv210_audss_clk_probe, - .remove = s5pv210_audss_clk_remove, }; static int __init s5pv210_audss_clk_init(void) @@ -227,14 +213,3 @@ static int __init s5pv210_audss_clk_init(void) return platform_driver_register(&s5pv210_audss_clk_driver); } core_initcall(s5pv210_audss_clk_init); - -static void __exit s5pv210_audss_clk_exit(void) -{ - platform_driver_unregister(&s5pv210_audss_clk_driver); -} -module_exit(s5pv210_audss_clk_exit); - -MODULE_AUTHOR("Tomasz Figa "); -MODULE_DESCRIPTION("S5PV210 Audio Subsystem Clock Controller"); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:s5pv210-audss-clk"); diff --git a/drivers/clk/samsung/clk-s5pv210.c b/drivers/clk/samsung/clk-s5pv210.c index 523022620..fd2725710 100644 --- a/drivers/clk/samsung/clk-s5pv210.c +++ b/drivers/clk/samsung/clk-s5pv210.c @@ -784,8 +784,6 @@ static void __init __s5pv210_clk_init(struct device_node *np, struct samsung_clk_provider *ctx; ctx = samsung_clk_init(np, reg_base, NR_CLKS); - if (!ctx) - panic("%s: unable to allocate context.\n", __func__); samsung_clk_register_mux(ctx, early_mux_clks, ARRAY_SIZE(early_mux_clks)); diff --git a/drivers/clk/samsung/clk.c b/drivers/clk/samsung/clk.c index f38a6c49f..b7d87d6db 100644 --- a/drivers/clk/samsung/clk.c +++ b/drivers/clk/samsung/clk.c @@ -346,9 +346,9 @@ static struct syscore_ops samsung_clk_syscore_ops = { .resume = samsung_clk_resume, }; -static void samsung_clk_sleep_init(void __iomem *reg_base, - const unsigned long *rdump, - unsigned long nr_rdump) +void samsung_clk_sleep_init(void __iomem *reg_base, + const unsigned long *rdump, + unsigned long nr_rdump) { struct samsung_clock_reg_cache *reg_cache; @@ -370,9 +370,9 @@ static void samsung_clk_sleep_init(void __iomem *reg_base, } #else -static void samsung_clk_sleep_init(void __iomem *reg_base, - const unsigned long *rdump, - unsigned long nr_rdump) {} +void samsung_clk_sleep_init(void __iomem *reg_base, + const unsigned long *rdump, + unsigned long nr_rdump) {} #endif /* @@ -381,7 +381,7 @@ static void samsung_clk_sleep_init(void __iomem *reg_base, */ struct samsung_clk_provider * __init samsung_cmu_register_one( struct device_node *np, - struct samsung_cmu_info *cmu) + const struct samsung_cmu_info *cmu) { void __iomem *reg_base; struct samsung_clk_provider *ctx; diff --git a/drivers/clk/samsung/clk.h b/drivers/clk/samsung/clk.h index aa872d2c5..da3bdebab 100644 --- a/drivers/clk/samsung/clk.h +++ b/drivers/clk/samsung/clk.h @@ -261,7 +261,7 @@ struct samsung_gate_clock { #define GATE_DA(_id, dname, cname, pname, o, b, f, gf, a) \ __GATE(_id, dname, cname, pname, o, b, f, gf, a) -#define PNAME(x) static const char *x[] __initdata +#define PNAME(x) static const char * const x[] __initconst /** * struct samsung_clk_reg_dump: register dump of clock controller registers. @@ -330,28 +330,28 @@ struct samsung_clock_reg_cache { struct samsung_cmu_info { /* list of pll clocks and respective count */ - struct samsung_pll_clock *pll_clks; + const struct samsung_pll_clock *pll_clks; unsigned int nr_pll_clks; /* list of mux clocks and respective count */ - struct samsung_mux_clock *mux_clks; + const struct samsung_mux_clock *mux_clks; unsigned int nr_mux_clks; /* list of div clocks and respective count */ - struct samsung_div_clock *div_clks; + const struct samsung_div_clock *div_clks; unsigned int nr_div_clks; /* list of gate clocks and respective count */ - struct samsung_gate_clock *gate_clks; + const struct samsung_gate_clock *gate_clks; unsigned int nr_gate_clks; /* list of fixed clocks and respective count */ - struct samsung_fixed_rate_clock *fixed_clks; + const struct samsung_fixed_rate_clock *fixed_clks; unsigned int nr_fixed_clks; /* list of fixed factor clocks and respective count */ - struct samsung_fixed_factor_clock *fixed_factor_clks; + const struct samsung_fixed_factor_clock *fixed_factor_clks; unsigned int nr_fixed_factor_clks; /* total number of clocks with IDs assigned*/ unsigned int nr_clk_ids; /* list and number of clocks registers */ - unsigned long *clk_regs; + const unsigned long *clk_regs; unsigned int nr_clk_regs; }; @@ -395,10 +395,14 @@ extern void __init samsung_clk_register_pll(struct samsung_clk_provider *ctx, extern struct samsung_clk_provider __init *samsung_cmu_register_one( struct device_node *, - struct samsung_cmu_info *); + const struct samsung_cmu_info *); extern unsigned long _get_rate(const char *clk_name); +extern void samsung_clk_sleep_init(void __iomem *reg_base, + const unsigned long *rdump, + unsigned long nr_rdump); + extern void samsung_clk_save(void __iomem *base, struct samsung_clk_reg_dump *rd, unsigned int num_regs); diff --git a/drivers/clk/st/clk-flexgen.c b/drivers/clk/st/clk-flexgen.c index 627267c7e..546bd79c8 100644 --- a/drivers/clk/st/clk-flexgen.c +++ b/drivers/clk/st/clk-flexgen.c @@ -267,7 +267,6 @@ static void __init st_of_flexgen_setup(struct device_node *np) const char **parents; int num_parents, i; spinlock_t *rlock = NULL; - unsigned long flex_flags = 0; int ret; pnode = of_get_parent(np); @@ -308,12 +307,15 @@ static void __init st_of_flexgen_setup(struct device_node *np) for (i = 0; i < clk_data->clk_num; i++) { struct clk *clk; const char *clk_name; + unsigned long flex_flags = 0; if (of_property_read_string_index(np, "clock-output-names", i, &clk_name)) { break; } + of_clk_detect_critical(np, i, &flex_flags); + /* * If we read an empty clock name then the output is unused */ diff --git a/drivers/clk/st/clkgen-fsyn.c b/drivers/clk/st/clkgen-fsyn.c index dec4eaaec..09afeb851 100644 --- a/drivers/clk/st/clkgen-fsyn.c +++ b/drivers/clk/st/clkgen-fsyn.c @@ -1027,7 +1027,7 @@ static const struct clk_ops st_quadfs_ops = { static struct clk * __init st_clk_register_quadfs_fsynth( const char *name, const char *parent_name, struct clkgen_quadfs_data *quadfs, void __iomem *reg, u32 chan, - spinlock_t *lock) + unsigned long flags, spinlock_t *lock) { struct st_clk_quadfs_fsynth *fs; struct clk *clk; @@ -1045,7 +1045,7 @@ static struct clk * __init st_clk_register_quadfs_fsynth( init.name = name; init.ops = &st_quadfs_ops; - init.flags = CLK_GET_RATE_NOCACHE | CLK_IS_BASIC; + init.flags = flags | CLK_GET_RATE_NOCACHE | CLK_IS_BASIC; init.parent_names = &parent_name; init.num_parents = 1; @@ -1115,6 +1115,7 @@ static void __init st_of_create_quadfs_fsynths( for (fschan = 0; fschan < QUADFS_MAX_CHAN; fschan++) { struct clk *clk; const char *clk_name; + unsigned long flags = 0; if (of_property_read_string_index(np, "clock-output-names", fschan, &clk_name)) { @@ -1127,8 +1128,11 @@ static void __init st_of_create_quadfs_fsynths( if (*clk_name == '\0') continue; + of_clk_detect_critical(np, fschan, &flags); + clk = st_clk_register_quadfs_fsynth(clk_name, pll_name, - quadfs, reg, fschan, lock); + quadfs, reg, fschan, + flags, lock); /* * If there was an error registering this clock output, clean diff --git a/drivers/clk/st/clkgen-pll.c b/drivers/clk/st/clkgen-pll.c index 38f6f3a90..0b5990e82 100644 --- a/drivers/clk/st/clkgen-pll.c +++ b/drivers/clk/st/clkgen-pll.c @@ -840,7 +840,7 @@ static const struct clk_ops stm_pll4600c28_ops = { static struct clk * __init clkgen_pll_register(const char *parent_name, struct clkgen_pll_data *pll_data, - void __iomem *reg, + void __iomem *reg, unsigned long pll_flags, const char *clk_name, spinlock_t *lock) { struct clkgen_pll *pll; @@ -854,7 +854,7 @@ static struct clk * __init clkgen_pll_register(const char *parent_name, init.name = clk_name; init.ops = pll_data->ops; - init.flags = CLK_IS_BASIC | CLK_GET_RATE_NOCACHE; + init.flags = pll_flags | CLK_IS_BASIC | CLK_GET_RATE_NOCACHE; init.parent_names = &parent_name; init.num_parents = 1; @@ -948,7 +948,7 @@ static void __init clkgena_c65_pll_setup(struct device_node *np) */ clk_data->clks[0] = clkgen_pll_register(parent_name, (struct clkgen_pll_data *) &st_pll1600c65_ax, - reg + CLKGENAx_PLL0_OFFSET, clk_name, NULL); + reg + CLKGENAx_PLL0_OFFSET, 0, clk_name, NULL); if (IS_ERR(clk_data->clks[0])) goto err; @@ -977,7 +977,7 @@ static void __init clkgena_c65_pll_setup(struct device_node *np) */ clk_data->clks[2] = clkgen_pll_register(parent_name, (struct clkgen_pll_data *) &st_pll800c65_ax, - reg + CLKGENAx_PLL1_OFFSET, clk_name, NULL); + reg + CLKGENAx_PLL1_OFFSET, 0, clk_name, NULL); if (IS_ERR(clk_data->clks[2])) goto err; @@ -995,7 +995,7 @@ CLK_OF_DECLARE(clkgena_c65_plls, static struct clk * __init clkgen_odf_register(const char *parent_name, void __iomem *reg, struct clkgen_pll_data *pll_data, - int odf, + unsigned long pll_flags, int odf, spinlock_t *odf_lock, const char *odf_name) { @@ -1004,7 +1004,7 @@ static struct clk * __init clkgen_odf_register(const char *parent_name, struct clk_gate *gate; struct clk_divider *div; - flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT; + flags = pll_flags | CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT; gate = kzalloc(sizeof(*gate), GFP_KERNEL); if (!gate) @@ -1099,6 +1099,7 @@ static void __init clkgen_c32_pll_setup(struct device_node *np) int num_odfs, odf; struct clk_onecell_data *clk_data; struct clkgen_pll_data *data; + unsigned long pll_flags = 0; match = of_match_node(c32_pll_of_match, np); if (!match) { @@ -1116,8 +1117,10 @@ static void __init clkgen_c32_pll_setup(struct device_node *np) if (!pll_base) return; - clk = clkgen_pll_register(parent_name, data, pll_base, np->name, - data->lock); + of_clk_detect_critical(np, 0, &pll_flags); + + clk = clkgen_pll_register(parent_name, data, pll_base, pll_flags, + np->name, data->lock); if (IS_ERR(clk)) return; @@ -1139,12 +1142,15 @@ static void __init clkgen_c32_pll_setup(struct device_node *np) for (odf = 0; odf < num_odfs; odf++) { struct clk *clk; const char *clk_name; + unsigned long odf_flags = 0; if (of_property_read_string_index(np, "clock-output-names", odf, &clk_name)) return; - clk = clkgen_odf_register(pll_name, pll_base, data, + of_clk_detect_critical(np, odf, &odf_flags); + + clk = clkgen_odf_register(pll_name, pll_base, data, odf_flags, odf, &clkgena_c32_odf_lock, clk_name); if (IS_ERR(clk)) goto err; @@ -1206,7 +1212,8 @@ static void __init clkgengpu_c32_pll_setup(struct device_node *np) /* * PLL 1200MHz output */ - clk = clkgen_pll_register(parent_name, data, reg, clk_name, data->lock); + clk = clkgen_pll_register(parent_name, data, reg, + 0, clk_name, data->lock); if (!IS_ERR(clk)) of_clk_add_provider(np, of_clk_src_simple_get, clk); diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig new file mode 100644 index 000000000..2afcbd39e --- /dev/null +++ b/drivers/clk/sunxi-ng/Kconfig @@ -0,0 +1,65 @@ +config SUNXI_CCU + bool "Clock support for Allwinner SoCs" + default ARCH_SUNXI + +if SUNXI_CCU + +# Base clock types + +config SUNXI_CCU_DIV + bool + select SUNXI_CCU_MUX + +config SUNXI_CCU_FRAC + bool + +config SUNXI_CCU_GATE + bool + +config SUNXI_CCU_MUX + bool + +config SUNXI_CCU_PHASE + bool + +# Multi-factor clocks + +config SUNXI_CCU_NK + bool + select SUNXI_CCU_GATE + +config SUNXI_CCU_NKM + bool + select RATIONAL + select SUNXI_CCU_GATE + +config SUNXI_CCU_NKMP + bool + select RATIONAL + select SUNXI_CCU_GATE + +config SUNXI_CCU_NM + bool + select RATIONAL + select SUNXI_CCU_FRAC + select SUNXI_CCU_GATE + +config SUNXI_CCU_MP + bool + select SUNXI_CCU_GATE + select SUNXI_CCU_MUX + +# SoC Drivers + +config SUN8I_H3_CCU + bool "Support for the Allwinner H3 CCU" + select SUNXI_CCU_DIV + select SUNXI_CCU_NK + select SUNXI_CCU_NKM + select SUNXI_CCU_NKMP + select SUNXI_CCU_NM + select SUNXI_CCU_MP + select SUNXI_CCU_PHASE + default MACH_SUN8I + +endif diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile new file mode 100644 index 000000000..633ce642f --- /dev/null +++ b/drivers/clk/sunxi-ng/Makefile @@ -0,0 +1,20 @@ +# Common objects +obj-$(CONFIG_SUNXI_CCU) += ccu_common.o +obj-$(CONFIG_SUNXI_CCU) += ccu_reset.o + +# Base clock types +obj-$(CONFIG_SUNXI_CCU_DIV) += ccu_div.o +obj-$(CONFIG_SUNXI_CCU_FRAC) += ccu_frac.o +obj-$(CONFIG_SUNXI_CCU_GATE) += ccu_gate.o +obj-$(CONFIG_SUNXI_CCU_MUX) += ccu_mux.o +obj-$(CONFIG_SUNXI_CCU_PHASE) += ccu_phase.o + +# Multi-factor clocks +obj-$(CONFIG_SUNXI_CCU_NK) += ccu_nk.o +obj-$(CONFIG_SUNXI_CCU_NKM) += ccu_nkm.o +obj-$(CONFIG_SUNXI_CCU_NKMP) += ccu_nkmp.o +obj-$(CONFIG_SUNXI_CCU_NM) += ccu_nm.o +obj-$(CONFIG_SUNXI_CCU_MP) += ccu_mp.o + +# SoC support +obj-$(CONFIG_SUN8I_H3_CCU) += ccu-sun8i-h3.o diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c new file mode 100644 index 000000000..267f99523 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c @@ -0,0 +1,826 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include + +#include "ccu_common.h" +#include "ccu_reset.h" + +#include "ccu_div.h" +#include "ccu_gate.h" +#include "ccu_mp.h" +#include "ccu_mult.h" +#include "ccu_nk.h" +#include "ccu_nkm.h" +#include "ccu_nkmp.h" +#include "ccu_nm.h" +#include "ccu_phase.h" + +#include "ccu-sun8i-h3.h" + +static SUNXI_CCU_NKMP_WITH_GATE_LOCK(pll_cpux_clk, "pll-cpux", + "osc24M", 0x000, + 8, 5, /* N */ + 4, 2, /* K */ + 0, 2, /* M */ + 16, 2, /* P */ + BIT(31), /* gate */ + BIT(28), /* lock */ + 0); + +/* + * The Audio PLL is supposed to have 4 outputs: 3 fixed factors from + * the base (2x, 4x and 8x), and one variable divider (the one true + * pll audio). + * + * We don't have any need for the variable divider for now, so we just + * hardcode it to match with the clock names + */ +#define SUN8I_H3_PLL_AUDIO_REG 0x008 + +static SUNXI_CCU_NM_WITH_GATE_LOCK(pll_audio_base_clk, "pll-audio-base", + "osc24M", 0x008, + 8, 7, /* N */ + 0, 5, /* M */ + BIT(31), /* gate */ + BIT(28), /* lock */ + 0); + +static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_video_clk, "pll-video", + "osc24M", 0x0010, + 8, 7, /* N */ + 0, 4, /* M */ + BIT(24), /* frac enable */ + BIT(25), /* frac select */ + 270000000, /* frac rate 0 */ + 297000000, /* frac rate 1 */ + BIT(31), /* gate */ + BIT(28), /* lock */ + 0); + +static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_ve_clk, "pll-ve", + "osc24M", 0x0018, + 8, 7, /* N */ + 0, 4, /* M */ + BIT(24), /* frac enable */ + BIT(25), /* frac select */ + 270000000, /* frac rate 0 */ + 297000000, /* frac rate 1 */ + BIT(31), /* gate */ + BIT(28), /* lock */ + 0); + +static SUNXI_CCU_NKM_WITH_GATE_LOCK(pll_ddr_clk, "pll-ddr", + "osc24M", 0x020, + 8, 5, /* N */ + 4, 2, /* K */ + 0, 2, /* M */ + BIT(31), /* gate */ + BIT(28), /* lock */ + 0); + +static SUNXI_CCU_NK_WITH_GATE_LOCK_POSTDIV(pll_periph0_clk, "pll-periph0", + "osc24M", 0x028, + 8, 5, /* N */ + 4, 2, /* K */ + BIT(31), /* gate */ + BIT(28), /* lock */ + 2, /* post-div */ + 0); + +static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_gpu_clk, "pll-gpu", + "osc24M", 0x0038, + 8, 7, /* N */ + 0, 4, /* M */ + BIT(24), /* frac enable */ + BIT(25), /* frac select */ + 270000000, /* frac rate 0 */ + 297000000, /* frac rate 1 */ + BIT(31), /* gate */ + BIT(28), /* lock */ + 0); + +static SUNXI_CCU_NK_WITH_GATE_LOCK_POSTDIV(pll_periph1_clk, "pll-periph1", + "osc24M", 0x044, + 8, 5, /* N */ + 4, 2, /* K */ + BIT(31), /* gate */ + BIT(28), /* lock */ + 2, /* post-div */ + 0); + +static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_de_clk, "pll-de", + "osc24M", 0x0048, + 8, 7, /* N */ + 0, 4, /* M */ + BIT(24), /* frac enable */ + BIT(25), /* frac select */ + 270000000, /* frac rate 0 */ + 297000000, /* frac rate 1 */ + BIT(31), /* gate */ + BIT(28), /* lock */ + 0); + +static const char * const cpux_parents[] = { "osc32k", "osc24M", + "pll-cpux" , "pll-cpux" }; +static SUNXI_CCU_MUX(cpux_clk, "cpux", cpux_parents, + 0x050, 16, 2, CLK_IS_CRITICAL); + +static SUNXI_CCU_M(axi_clk, "axi", "cpux", 0x050, 0, 2, 0); + +static const char * const ahb1_parents[] = { "osc32k", "osc24M", + "axi" , "pll-periph0" }; +static struct ccu_div ahb1_clk = { + .div = _SUNXI_CCU_DIV_FLAGS(4, 2, CLK_DIVIDER_POWER_OF_TWO), + + .mux = { + .shift = 12, + .width = 2, + + .variable_prediv = { + .index = 3, + .shift = 6, + .width = 2, + }, + }, + + .common = { + .reg = 0x054, + .features = CCU_FEATURE_VARIABLE_PREDIV, + .hw.init = CLK_HW_INIT_PARENTS("ahb1", + ahb1_parents, + &ccu_div_ops, + 0), + }, +}; + +static struct clk_div_table apb1_div_table[] = { + { .val = 0, .div = 2 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 4 }, + { .val = 3, .div = 8 }, + { /* Sentinel */ }, +}; +static SUNXI_CCU_DIV_TABLE(apb1_clk, "apb1", "ahb1", + 0x054, 8, 2, apb1_div_table, 0); + +static const char * const apb2_parents[] = { "osc32k", "osc24M", + "pll-periph0" , "pll-periph0" }; +static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", apb2_parents, 0x058, + 0, 5, /* M */ + 16, 2, /* P */ + 24, 2, /* mux */ + 0); + +static const char * const ahb2_parents[] = { "ahb1" , "pll-periph0" }; +static struct ccu_mux ahb2_clk = { + .mux = { + .shift = 0, + .width = 1, + + .fixed_prediv = { + .index = 1, + .div = 2, + }, + }, + + .common = { + .reg = 0x05c, + .features = CCU_FEATURE_FIXED_PREDIV, + .hw.init = CLK_HW_INIT_PARENTS("ahb2", + ahb2_parents, + &ccu_mux_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_ce_clk, "bus-ce", "ahb1", + 0x060, BIT(5), 0); +static SUNXI_CCU_GATE(bus_dma_clk, "bus-dma", "ahb1", + 0x060, BIT(6), 0); +static SUNXI_CCU_GATE(bus_mmc0_clk, "bus-mmc0", "ahb1", + 0x060, BIT(8), 0); +static SUNXI_CCU_GATE(bus_mmc1_clk, "bus-mmc1", "ahb1", + 0x060, BIT(9), 0); +static SUNXI_CCU_GATE(bus_mmc2_clk, "bus-mmc2", "ahb1", + 0x060, BIT(10), 0); +static SUNXI_CCU_GATE(bus_nand_clk, "bus-nand", "ahb1", + 0x060, BIT(13), 0); +static SUNXI_CCU_GATE(bus_dram_clk, "bus-dram", "ahb1", + 0x060, BIT(14), 0); +static SUNXI_CCU_GATE(bus_emac_clk, "bus-emac", "ahb2", + 0x060, BIT(17), 0); +static SUNXI_CCU_GATE(bus_ts_clk, "bus-ts", "ahb1", + 0x060, BIT(18), 0); +static SUNXI_CCU_GATE(bus_hstimer_clk, "bus-hstimer", "ahb1", + 0x060, BIT(19), 0); +static SUNXI_CCU_GATE(bus_spi0_clk, "bus-spi0", "ahb1", + 0x060, BIT(20), 0); +static SUNXI_CCU_GATE(bus_spi1_clk, "bus-spi1", "ahb1", + 0x060, BIT(21), 0); +static SUNXI_CCU_GATE(bus_otg_clk, "bus-otg", "ahb1", + 0x060, BIT(23), 0); +static SUNXI_CCU_GATE(bus_ehci0_clk, "bus-ehci0", "ahb1", + 0x060, BIT(24), 0); +static SUNXI_CCU_GATE(bus_ehci1_clk, "bus-ehci1", "ahb2", + 0x060, BIT(25), 0); +static SUNXI_CCU_GATE(bus_ehci2_clk, "bus-ehci2", "ahb2", + 0x060, BIT(26), 0); +static SUNXI_CCU_GATE(bus_ehci3_clk, "bus-ehci3", "ahb2", + 0x060, BIT(27), 0); +static SUNXI_CCU_GATE(bus_ohci0_clk, "bus-ohci0", "ahb1", + 0x060, BIT(28), 0); +static SUNXI_CCU_GATE(bus_ohci1_clk, "bus-ohci1", "ahb2", + 0x060, BIT(29), 0); +static SUNXI_CCU_GATE(bus_ohci2_clk, "bus-ohci2", "ahb2", + 0x060, BIT(30), 0); +static SUNXI_CCU_GATE(bus_ohci3_clk, "bus-ohci3", "ahb2", + 0x060, BIT(31), 0); + +static SUNXI_CCU_GATE(bus_ve_clk, "bus-ve", "ahb1", + 0x064, BIT(0), 0); +static SUNXI_CCU_GATE(bus_tcon0_clk, "bus-tcon0", "ahb1", + 0x064, BIT(3), 0); +static SUNXI_CCU_GATE(bus_tcon1_clk, "bus-tcon1", "ahb1", + 0x064, BIT(4), 0); +static SUNXI_CCU_GATE(bus_deinterlace_clk, "bus-deinterlace", "ahb1", + 0x064, BIT(5), 0); +static SUNXI_CCU_GATE(bus_csi_clk, "bus-csi", "ahb1", + 0x064, BIT(8), 0); +static SUNXI_CCU_GATE(bus_tve_clk, "bus-tve", "ahb1", + 0x064, BIT(9), 0); +static SUNXI_CCU_GATE(bus_hdmi_clk, "bus-hdmi", "ahb1", + 0x064, BIT(11), 0); +static SUNXI_CCU_GATE(bus_de_clk, "bus-de", "ahb1", + 0x064, BIT(12), 0); +static SUNXI_CCU_GATE(bus_gpu_clk, "bus-gpu", "ahb1", + 0x064, BIT(20), 0); +static SUNXI_CCU_GATE(bus_msgbox_clk, "bus-msgbox", "ahb1", + 0x064, BIT(21), 0); +static SUNXI_CCU_GATE(bus_spinlock_clk, "bus-spinlock", "ahb1", + 0x064, BIT(22), 0); + +static SUNXI_CCU_GATE(bus_codec_clk, "bus-codec", "apb1", + 0x068, BIT(0), 0); +static SUNXI_CCU_GATE(bus_spdif_clk, "bus-spdif", "apb1", + 0x068, BIT(1), 0); +static SUNXI_CCU_GATE(bus_pio_clk, "bus-pio", "apb1", + 0x068, BIT(5), 0); +static SUNXI_CCU_GATE(bus_ths_clk, "bus-ths", "apb1", + 0x068, BIT(8), 0); +static SUNXI_CCU_GATE(bus_i2s0_clk, "bus-i2s0", "apb1", + 0x068, BIT(12), 0); +static SUNXI_CCU_GATE(bus_i2s1_clk, "bus-i2s1", "apb1", + 0x068, BIT(13), 0); +static SUNXI_CCU_GATE(bus_i2s2_clk, "bus-i2s2", "apb1", + 0x068, BIT(14), 0); + +static SUNXI_CCU_GATE(bus_i2c0_clk, "bus-i2c0", "apb2", + 0x06c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_i2c1_clk, "bus-i2c1", "apb2", + 0x06c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_i2c2_clk, "bus-i2c2", "apb2", + 0x06c, BIT(2), 0); +static SUNXI_CCU_GATE(bus_uart0_clk, "bus-uart0", "apb2", + 0x06c, BIT(16), 0); +static SUNXI_CCU_GATE(bus_uart1_clk, "bus-uart1", "apb2", + 0x06c, BIT(17), 0); +static SUNXI_CCU_GATE(bus_uart2_clk, "bus-uart2", "apb2", + 0x06c, BIT(18), 0); +static SUNXI_CCU_GATE(bus_uart3_clk, "bus-uart3", "apb2", + 0x06c, BIT(19), 0); +static SUNXI_CCU_GATE(bus_scr_clk, "bus-scr", "apb2", + 0x06c, BIT(20), 0); + +static SUNXI_CCU_GATE(bus_ephy_clk, "bus-ephy", "ahb1", + 0x070, BIT(0), 0); +static SUNXI_CCU_GATE(bus_dbg_clk, "bus-dbg", "ahb1", + 0x070, BIT(7), 0); + +static struct clk_div_table ths_div_table[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 4 }, + { .val = 3, .div = 6 }, +}; +static SUNXI_CCU_DIV_TABLE_WITH_GATE(ths_clk, "ths", "osc24M", + 0x074, 0, 2, ths_div_table, BIT(31), 0); + +static const char * const mod0_default_parents[] = { "osc24M", "pll-periph0", + "pll-periph1" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(nand_clk, "nand", mod0_default_parents, 0x080, + 0, 4, /* M */ + 16, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(mmc0_clk, "mmc0", mod0_default_parents, 0x088, + 0, 4, /* M */ + 16, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_PHASE(mmc0_sample_clk, "mmc0_sample", "mmc0", + 0x088, 20, 3, 0); +static SUNXI_CCU_PHASE(mmc0_output_clk, "mmc0_output", "mmc0", + 0x088, 8, 3, 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(mmc1_clk, "mmc1", mod0_default_parents, 0x08c, + 0, 4, /* M */ + 16, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_PHASE(mmc1_sample_clk, "mmc1_sample", "mmc1", + 0x08c, 20, 3, 0); +static SUNXI_CCU_PHASE(mmc1_output_clk, "mmc1_output", "mmc1", + 0x08c, 8, 3, 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(mmc2_clk, "mmc2", mod0_default_parents, 0x090, + 0, 4, /* M */ + 16, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_PHASE(mmc2_sample_clk, "mmc2_sample", "mmc2", + 0x090, 20, 3, 0); +static SUNXI_CCU_PHASE(mmc2_output_clk, "mmc2_output", "mmc2", + 0x090, 8, 3, 0); + +static const char * const ts_parents[] = { "osc24M", "pll-periph0", }; +static SUNXI_CCU_MP_WITH_MUX_GATE(ts_clk, "ts", ts_parents, 0x098, + 0, 4, /* M */ + 16, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(ce_clk, "ce", mod0_default_parents, 0x09c, + 0, 4, /* M */ + 16, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(spi0_clk, "spi0", mod0_default_parents, 0x0a0, + 0, 4, /* M */ + 16, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4, + 0, 4, /* M */ + 16, 2, /* P */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static const char * const i2s_parents[] = { "pll-audio-8x", "pll-audio-4x", + "pll-audio-2x", "pll-audio" }; +static SUNXI_CCU_MUX_WITH_GATE(i2s0_clk, "i2s0", i2s_parents, + 0x0b0, 16, 2, BIT(31), 0); + +static SUNXI_CCU_MUX_WITH_GATE(i2s1_clk, "i2s1", i2s_parents, + 0x0b4, 16, 2, BIT(31), 0); + +static SUNXI_CCU_MUX_WITH_GATE(i2s2_clk, "i2s2", i2s_parents, + 0x0b8, 16, 2, BIT(31), 0); + +static SUNXI_CCU_M_WITH_GATE(spdif_clk, "spdif", "pll-audio", + 0x0c0, 0, 4, BIT(31), 0); + +static SUNXI_CCU_GATE(usb_phy0_clk, "usb-phy0", "osc24M", + 0x0cc, BIT(8), 0); +static SUNXI_CCU_GATE(usb_phy1_clk, "usb-phy1", "osc24M", + 0x0cc, BIT(9), 0); +static SUNXI_CCU_GATE(usb_phy2_clk, "usb-phy2", "osc24M", + 0x0cc, BIT(10), 0); +static SUNXI_CCU_GATE(usb_phy3_clk, "usb-phy3", "osc24M", + 0x0cc, BIT(11), 0); +static SUNXI_CCU_GATE(usb_ohci0_clk, "usb-ohci0", "osc24M", + 0x0cc, BIT(16), 0); +static SUNXI_CCU_GATE(usb_ohci1_clk, "usb-ohci1", "osc24M", + 0x0cc, BIT(17), 0); +static SUNXI_CCU_GATE(usb_ohci2_clk, "usb-ohci2", "osc24M", + 0x0cc, BIT(18), 0); +static SUNXI_CCU_GATE(usb_ohci3_clk, "usb-ohci3", "osc24M", + 0x0cc, BIT(19), 0); + +static const char * const dram_parents[] = { "pll-ddr", "pll-periph0-2x" }; +static SUNXI_CCU_M_WITH_MUX(dram_clk, "dram", dram_parents, + 0x0f4, 0, 4, 20, 2, CLK_IS_CRITICAL); + +static SUNXI_CCU_GATE(dram_ve_clk, "dram-ve", "dram", + 0x100, BIT(0), 0); +static SUNXI_CCU_GATE(dram_csi_clk, "dram-csi", "dram", + 0x100, BIT(1), 0); +static SUNXI_CCU_GATE(dram_deinterlace_clk, "dram-deinterlace", "dram", + 0x100, BIT(2), 0); +static SUNXI_CCU_GATE(dram_ts_clk, "dram-ts", "dram", + 0x100, BIT(3), 0); + +static const char * const de_parents[] = { "pll-periph0-2x", "pll-de" }; +static SUNXI_CCU_M_WITH_MUX_GATE(de_clk, "de", de_parents, + 0x104, 0, 4, 24, 3, BIT(31), 0); + +static const char * const tcon_parents[] = { "pll-video" }; +static SUNXI_CCU_M_WITH_MUX_GATE(tcon_clk, "tcon", tcon_parents, + 0x118, 0, 4, 24, 3, BIT(31), 0); + +static const char * const tve_parents[] = { "pll-de", "pll-periph1" }; +static SUNXI_CCU_M_WITH_MUX_GATE(tve_clk, "tve", tve_parents, + 0x120, 0, 4, 24, 3, BIT(31), 0); + +static const char * const deinterlace_parents[] = { "pll-periph0", "pll-periph1" }; +static SUNXI_CCU_M_WITH_MUX_GATE(deinterlace_clk, "deinterlace", deinterlace_parents, + 0x124, 0, 4, 24, 3, BIT(31), 0); + +static SUNXI_CCU_GATE(csi_misc_clk, "csi-misc", "osc24M", + 0x130, BIT(31), 0); + +static const char * const csi_sclk_parents[] = { "pll-periph0", "pll-periph1" }; +static SUNXI_CCU_M_WITH_MUX_GATE(csi_sclk_clk, "csi-sclk", csi_sclk_parents, + 0x134, 16, 4, 24, 3, BIT(31), 0); + +static const char * const csi_mclk_parents[] = { "osc24M", "pll-video", "pll-periph0" }; +static SUNXI_CCU_M_WITH_MUX_GATE(csi_mclk_clk, "csi-mclk", csi_mclk_parents, + 0x134, 0, 5, 8, 3, BIT(15), 0); + +static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve", + 0x13c, 16, 3, BIT(31), 0); + +static SUNXI_CCU_GATE(ac_dig_clk, "ac-dig", "pll-audio", + 0x140, BIT(31), 0); +static SUNXI_CCU_GATE(avs_clk, "avs", "osc24M", + 0x144, BIT(31), 0); + +static const char * const hdmi_parents[] = { "pll-video" }; +static SUNXI_CCU_M_WITH_MUX_GATE(hdmi_clk, "hdmi", hdmi_parents, + 0x150, 0, 4, 24, 2, BIT(31), 0); + +static SUNXI_CCU_GATE(hdmi_ddc_clk, "hdmi-ddc", "osc24M", + 0x154, BIT(31), 0); + +static const char * const mbus_parents[] = { "osc24M", "pll-periph0-2x", "pll-ddr" }; +static SUNXI_CCU_M_WITH_MUX_GATE(mbus_clk, "mbus", mbus_parents, + 0x15c, 0, 3, 24, 2, BIT(31), CLK_IS_CRITICAL); + +static SUNXI_CCU_M_WITH_GATE(gpu_clk, "gpu", "pll-gpu", + 0x1a0, 0, 3, BIT(31), 0); + +static struct ccu_common *sun8i_h3_ccu_clks[] = { + &pll_cpux_clk.common, + &pll_audio_base_clk.common, + &pll_video_clk.common, + &pll_ve_clk.common, + &pll_ddr_clk.common, + &pll_periph0_clk.common, + &pll_gpu_clk.common, + &pll_periph1_clk.common, + &pll_de_clk.common, + &cpux_clk.common, + &axi_clk.common, + &ahb1_clk.common, + &apb1_clk.common, + &apb2_clk.common, + &ahb2_clk.common, + &bus_ce_clk.common, + &bus_dma_clk.common, + &bus_mmc0_clk.common, + &bus_mmc1_clk.common, + &bus_mmc2_clk.common, + &bus_nand_clk.common, + &bus_dram_clk.common, + &bus_emac_clk.common, + &bus_ts_clk.common, + &bus_hstimer_clk.common, + &bus_spi0_clk.common, + &bus_spi1_clk.common, + &bus_otg_clk.common, + &bus_ehci0_clk.common, + &bus_ehci1_clk.common, + &bus_ehci2_clk.common, + &bus_ehci3_clk.common, + &bus_ohci0_clk.common, + &bus_ohci1_clk.common, + &bus_ohci2_clk.common, + &bus_ohci3_clk.common, + &bus_ve_clk.common, + &bus_tcon0_clk.common, + &bus_tcon1_clk.common, + &bus_deinterlace_clk.common, + &bus_csi_clk.common, + &bus_tve_clk.common, + &bus_hdmi_clk.common, + &bus_de_clk.common, + &bus_gpu_clk.common, + &bus_msgbox_clk.common, + &bus_spinlock_clk.common, + &bus_codec_clk.common, + &bus_spdif_clk.common, + &bus_pio_clk.common, + &bus_ths_clk.common, + &bus_i2s0_clk.common, + &bus_i2s1_clk.common, + &bus_i2s2_clk.common, + &bus_i2c0_clk.common, + &bus_i2c1_clk.common, + &bus_i2c2_clk.common, + &bus_uart0_clk.common, + &bus_uart1_clk.common, + &bus_uart2_clk.common, + &bus_uart3_clk.common, + &bus_scr_clk.common, + &bus_ephy_clk.common, + &bus_dbg_clk.common, + &ths_clk.common, + &nand_clk.common, + &mmc0_clk.common, + &mmc0_sample_clk.common, + &mmc0_output_clk.common, + &mmc1_clk.common, + &mmc1_sample_clk.common, + &mmc1_output_clk.common, + &mmc2_clk.common, + &mmc2_sample_clk.common, + &mmc2_output_clk.common, + &ts_clk.common, + &ce_clk.common, + &spi0_clk.common, + &spi1_clk.common, + &i2s0_clk.common, + &i2s1_clk.common, + &i2s2_clk.common, + &spdif_clk.common, + &usb_phy0_clk.common, + &usb_phy1_clk.common, + &usb_phy2_clk.common, + &usb_phy3_clk.common, + &usb_ohci0_clk.common, + &usb_ohci1_clk.common, + &usb_ohci2_clk.common, + &usb_ohci3_clk.common, + &dram_clk.common, + &dram_ve_clk.common, + &dram_csi_clk.common, + &dram_deinterlace_clk.common, + &dram_ts_clk.common, + &de_clk.common, + &tcon_clk.common, + &tve_clk.common, + &deinterlace_clk.common, + &csi_misc_clk.common, + &csi_sclk_clk.common, + &csi_mclk_clk.common, + &ve_clk.common, + &ac_dig_clk.common, + &avs_clk.common, + &hdmi_clk.common, + &hdmi_ddc_clk.common, + &mbus_clk.common, + &gpu_clk.common, +}; + +/* We hardcode the divider to 4 for now */ +static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio", + "pll-audio-base", 4, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x", + "pll-audio-base", 2, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x", + "pll-audio-base", 1, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x", + "pll-audio-base", 1, 2, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x", + "pll-periph0", 1, 2, 0); + +static struct clk_hw_onecell_data sun8i_h3_hw_clks = { + .hws = { + [CLK_PLL_CPUX] = &pll_cpux_clk.common.hw, + [CLK_PLL_AUDIO_BASE] = &pll_audio_base_clk.common.hw, + [CLK_PLL_AUDIO] = &pll_audio_clk.hw, + [CLK_PLL_AUDIO_2X] = &pll_audio_2x_clk.hw, + [CLK_PLL_AUDIO_4X] = &pll_audio_4x_clk.hw, + [CLK_PLL_AUDIO_8X] = &pll_audio_8x_clk.hw, + [CLK_PLL_VIDEO] = &pll_video_clk.common.hw, + [CLK_PLL_VE] = &pll_ve_clk.common.hw, + [CLK_PLL_DDR] = &pll_ddr_clk.common.hw, + [CLK_PLL_PERIPH0] = &pll_periph0_clk.common.hw, + [CLK_PLL_PERIPH0_2X] = &pll_periph0_2x_clk.hw, + [CLK_PLL_GPU] = &pll_gpu_clk.common.hw, + [CLK_PLL_PERIPH1] = &pll_periph1_clk.common.hw, + [CLK_PLL_DE] = &pll_de_clk.common.hw, + [CLK_CPUX] = &cpux_clk.common.hw, + [CLK_AXI] = &axi_clk.common.hw, + [CLK_AHB1] = &ahb1_clk.common.hw, + [CLK_APB1] = &apb1_clk.common.hw, + [CLK_APB2] = &apb2_clk.common.hw, + [CLK_AHB2] = &ahb2_clk.common.hw, + [CLK_BUS_CE] = &bus_ce_clk.common.hw, + [CLK_BUS_DMA] = &bus_dma_clk.common.hw, + [CLK_BUS_MMC0] = &bus_mmc0_clk.common.hw, + [CLK_BUS_MMC1] = &bus_mmc1_clk.common.hw, + [CLK_BUS_MMC2] = &bus_mmc2_clk.common.hw, + [CLK_BUS_NAND] = &bus_nand_clk.common.hw, + [CLK_BUS_DRAM] = &bus_dram_clk.common.hw, + [CLK_BUS_EMAC] = &bus_emac_clk.common.hw, + [CLK_BUS_TS] = &bus_ts_clk.common.hw, + [CLK_BUS_HSTIMER] = &bus_hstimer_clk.common.hw, + [CLK_BUS_SPI0] = &bus_spi0_clk.common.hw, + [CLK_BUS_SPI1] = &bus_spi1_clk.common.hw, + [CLK_BUS_OTG] = &bus_otg_clk.common.hw, + [CLK_BUS_EHCI0] = &bus_ehci0_clk.common.hw, + [CLK_BUS_EHCI1] = &bus_ehci1_clk.common.hw, + [CLK_BUS_EHCI2] = &bus_ehci2_clk.common.hw, + [CLK_BUS_EHCI3] = &bus_ehci3_clk.common.hw, + [CLK_BUS_OHCI0] = &bus_ohci0_clk.common.hw, + [CLK_BUS_OHCI1] = &bus_ohci1_clk.common.hw, + [CLK_BUS_OHCI2] = &bus_ohci2_clk.common.hw, + [CLK_BUS_OHCI3] = &bus_ohci3_clk.common.hw, + [CLK_BUS_VE] = &bus_ve_clk.common.hw, + [CLK_BUS_TCON0] = &bus_tcon0_clk.common.hw, + [CLK_BUS_TCON1] = &bus_tcon1_clk.common.hw, + [CLK_BUS_DEINTERLACE] = &bus_deinterlace_clk.common.hw, + [CLK_BUS_CSI] = &bus_csi_clk.common.hw, + [CLK_BUS_TVE] = &bus_tve_clk.common.hw, + [CLK_BUS_HDMI] = &bus_hdmi_clk.common.hw, + [CLK_BUS_DE] = &bus_de_clk.common.hw, + [CLK_BUS_GPU] = &bus_gpu_clk.common.hw, + [CLK_BUS_MSGBOX] = &bus_msgbox_clk.common.hw, + [CLK_BUS_SPINLOCK] = &bus_spinlock_clk.common.hw, + [CLK_BUS_CODEC] = &bus_codec_clk.common.hw, + [CLK_BUS_SPDIF] = &bus_spdif_clk.common.hw, + [CLK_BUS_PIO] = &bus_pio_clk.common.hw, + [CLK_BUS_THS] = &bus_ths_clk.common.hw, + [CLK_BUS_I2S0] = &bus_i2s0_clk.common.hw, + [CLK_BUS_I2S1] = &bus_i2s1_clk.common.hw, + [CLK_BUS_I2S2] = &bus_i2s2_clk.common.hw, + [CLK_BUS_I2C0] = &bus_i2c0_clk.common.hw, + [CLK_BUS_I2C1] = &bus_i2c1_clk.common.hw, + [CLK_BUS_I2C2] = &bus_i2c2_clk.common.hw, + [CLK_BUS_UART0] = &bus_uart0_clk.common.hw, + [CLK_BUS_UART1] = &bus_uart1_clk.common.hw, + [CLK_BUS_UART2] = &bus_uart2_clk.common.hw, + [CLK_BUS_UART3] = &bus_uart3_clk.common.hw, + [CLK_BUS_SCR] = &bus_scr_clk.common.hw, + [CLK_BUS_EPHY] = &bus_ephy_clk.common.hw, + [CLK_BUS_DBG] = &bus_dbg_clk.common.hw, + [CLK_THS] = &ths_clk.common.hw, + [CLK_NAND] = &nand_clk.common.hw, + [CLK_MMC0] = &mmc0_clk.common.hw, + [CLK_MMC0_SAMPLE] = &mmc0_sample_clk.common.hw, + [CLK_MMC0_OUTPUT] = &mmc0_output_clk.common.hw, + [CLK_MMC1] = &mmc1_clk.common.hw, + [CLK_MMC1_SAMPLE] = &mmc1_sample_clk.common.hw, + [CLK_MMC1_OUTPUT] = &mmc1_output_clk.common.hw, + [CLK_MMC2] = &mmc2_clk.common.hw, + [CLK_MMC2_SAMPLE] = &mmc2_sample_clk.common.hw, + [CLK_MMC2_OUTPUT] = &mmc2_output_clk.common.hw, + [CLK_TS] = &ts_clk.common.hw, + [CLK_CE] = &ce_clk.common.hw, + [CLK_SPI0] = &spi0_clk.common.hw, + [CLK_SPI1] = &spi1_clk.common.hw, + [CLK_I2S0] = &i2s0_clk.common.hw, + [CLK_I2S1] = &i2s1_clk.common.hw, + [CLK_I2S2] = &i2s2_clk.common.hw, + [CLK_SPDIF] = &spdif_clk.common.hw, + [CLK_USB_PHY0] = &usb_phy0_clk.common.hw, + [CLK_USB_PHY1] = &usb_phy1_clk.common.hw, + [CLK_USB_PHY2] = &usb_phy2_clk.common.hw, + [CLK_USB_PHY3] = &usb_phy3_clk.common.hw, + [CLK_USB_OHCI0] = &usb_ohci0_clk.common.hw, + [CLK_USB_OHCI1] = &usb_ohci1_clk.common.hw, + [CLK_USB_OHCI2] = &usb_ohci2_clk.common.hw, + [CLK_USB_OHCI3] = &usb_ohci3_clk.common.hw, + [CLK_DRAM] = &dram_clk.common.hw, + [CLK_DRAM_VE] = &dram_ve_clk.common.hw, + [CLK_DRAM_CSI] = &dram_csi_clk.common.hw, + [CLK_DRAM_DEINTERLACE] = &dram_deinterlace_clk.common.hw, + [CLK_DRAM_TS] = &dram_ts_clk.common.hw, + [CLK_DE] = &de_clk.common.hw, + [CLK_TCON0] = &tcon_clk.common.hw, + [CLK_TVE] = &tve_clk.common.hw, + [CLK_DEINTERLACE] = &deinterlace_clk.common.hw, + [CLK_CSI_MISC] = &csi_misc_clk.common.hw, + [CLK_CSI_SCLK] = &csi_sclk_clk.common.hw, + [CLK_CSI_MCLK] = &csi_mclk_clk.common.hw, + [CLK_VE] = &ve_clk.common.hw, + [CLK_AC_DIG] = &ac_dig_clk.common.hw, + [CLK_AVS] = &avs_clk.common.hw, + [CLK_HDMI] = &hdmi_clk.common.hw, + [CLK_HDMI_DDC] = &hdmi_ddc_clk.common.hw, + [CLK_MBUS] = &mbus_clk.common.hw, + [CLK_GPU] = &gpu_clk.common.hw, + }, + .num = CLK_NUMBER, +}; + +static struct ccu_reset_map sun8i_h3_ccu_resets[] = { + [RST_USB_PHY0] = { 0x0cc, BIT(0) }, + [RST_USB_PHY1] = { 0x0cc, BIT(1) }, + [RST_USB_PHY2] = { 0x0cc, BIT(2) }, + [RST_USB_PHY3] = { 0x0cc, BIT(3) }, + + [RST_MBUS] = { 0x0fc, BIT(31) }, + + [RST_BUS_CE] = { 0x2c0, BIT(5) }, + [RST_BUS_DMA] = { 0x2c0, BIT(6) }, + [RST_BUS_MMC0] = { 0x2c0, BIT(8) }, + [RST_BUS_MMC1] = { 0x2c0, BIT(9) }, + [RST_BUS_MMC2] = { 0x2c0, BIT(10) }, + [RST_BUS_NAND] = { 0x2c0, BIT(13) }, + [RST_BUS_DRAM] = { 0x2c0, BIT(14) }, + [RST_BUS_EMAC] = { 0x2c0, BIT(17) }, + [RST_BUS_TS] = { 0x2c0, BIT(18) }, + [RST_BUS_HSTIMER] = { 0x2c0, BIT(19) }, + [RST_BUS_SPI0] = { 0x2c0, BIT(20) }, + [RST_BUS_SPI1] = { 0x2c0, BIT(21) }, + [RST_BUS_OTG] = { 0x2c0, BIT(23) }, + [RST_BUS_EHCI0] = { 0x2c0, BIT(24) }, + [RST_BUS_EHCI1] = { 0x2c0, BIT(25) }, + [RST_BUS_EHCI2] = { 0x2c0, BIT(26) }, + [RST_BUS_EHCI3] = { 0x2c0, BIT(27) }, + [RST_BUS_OHCI0] = { 0x2c0, BIT(28) }, + [RST_BUS_OHCI1] = { 0x2c0, BIT(29) }, + [RST_BUS_OHCI2] = { 0x2c0, BIT(30) }, + [RST_BUS_OHCI3] = { 0x2c0, BIT(31) }, + + [RST_BUS_VE] = { 0x2c4, BIT(0) }, + [RST_BUS_TCON0] = { 0x2c4, BIT(3) }, + [RST_BUS_TCON1] = { 0x2c4, BIT(4) }, + [RST_BUS_DEINTERLACE] = { 0x2c4, BIT(5) }, + [RST_BUS_CSI] = { 0x2c4, BIT(8) }, + [RST_BUS_TVE] = { 0x2c4, BIT(9) }, + [RST_BUS_HDMI0] = { 0x2c4, BIT(10) }, + [RST_BUS_HDMI1] = { 0x2c4, BIT(11) }, + [RST_BUS_DE] = { 0x2c4, BIT(12) }, + [RST_BUS_GPU] = { 0x2c4, BIT(20) }, + [RST_BUS_MSGBOX] = { 0x2c4, BIT(21) }, + [RST_BUS_SPINLOCK] = { 0x2c4, BIT(22) }, + [RST_BUS_DBG] = { 0x2c4, BIT(31) }, + + [RST_BUS_EPHY] = { 0x2c8, BIT(2) }, + + [RST_BUS_CODEC] = { 0x2d0, BIT(0) }, + [RST_BUS_SPDIF] = { 0x2d0, BIT(1) }, + [RST_BUS_THS] = { 0x2d0, BIT(8) }, + [RST_BUS_I2S0] = { 0x2d0, BIT(12) }, + [RST_BUS_I2S1] = { 0x2d0, BIT(13) }, + [RST_BUS_I2S2] = { 0x2d0, BIT(14) }, + + [RST_BUS_I2C0] = { 0x2d8, BIT(0) }, + [RST_BUS_I2C1] = { 0x2d8, BIT(1) }, + [RST_BUS_I2C2] = { 0x2d8, BIT(2) }, + [RST_BUS_UART0] = { 0x2d8, BIT(16) }, + [RST_BUS_UART1] = { 0x2d8, BIT(17) }, + [RST_BUS_UART2] = { 0x2d8, BIT(18) }, + [RST_BUS_UART3] = { 0x2d8, BIT(19) }, + [RST_BUS_SCR] = { 0x2d8, BIT(20) }, +}; + +static const struct sunxi_ccu_desc sun8i_h3_ccu_desc = { + .ccu_clks = sun8i_h3_ccu_clks, + .num_ccu_clks = ARRAY_SIZE(sun8i_h3_ccu_clks), + + .hw_clks = &sun8i_h3_hw_clks, + + .resets = sun8i_h3_ccu_resets, + .num_resets = ARRAY_SIZE(sun8i_h3_ccu_resets), +}; + +static void __init sun8i_h3_ccu_setup(struct device_node *node) +{ + void __iomem *reg; + u32 val; + + reg = of_io_request_and_map(node, 0, of_node_full_name(node)); + if (IS_ERR(reg)) { + pr_err("%s: Could not map the clock registers\n", + of_node_full_name(node)); + return; + } + + /* Force the PLL-Audio-1x divider to 4 */ + val = readl(reg + SUN8I_H3_PLL_AUDIO_REG); + val &= ~GENMASK(19, 16); + writel(val | (3 << 16), reg + SUN8I_H3_PLL_AUDIO_REG); + + sunxi_ccu_probe(node, reg, &sun8i_h3_ccu_desc); +} +CLK_OF_DECLARE(sun8i_h3_ccu, "allwinner,sun8i-h3-ccu", + sun8i_h3_ccu_setup); diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.h b/drivers/clk/sunxi-ng/ccu-sun8i-h3.h new file mode 100644 index 000000000..78be712c7 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.h @@ -0,0 +1,62 @@ +/* + * Copyright 2016 Maxime Ripard + * + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_SUN8I_H3_H_ +#define _CCU_SUN8I_H3_H_ + +#include +#include + +#define CLK_PLL_CPUX 0 +#define CLK_PLL_AUDIO_BASE 1 +#define CLK_PLL_AUDIO 2 +#define CLK_PLL_AUDIO_2X 3 +#define CLK_PLL_AUDIO_4X 4 +#define CLK_PLL_AUDIO_8X 5 +#define CLK_PLL_VIDEO 6 +#define CLK_PLL_VE 7 +#define CLK_PLL_DDR 8 +#define CLK_PLL_PERIPH0 9 +#define CLK_PLL_PERIPH0_2X 10 +#define CLK_PLL_GPU 11 +#define CLK_PLL_PERIPH1 12 +#define CLK_PLL_DE 13 + +/* The CPUX clock is exported */ + +#define CLK_AXI 15 +#define CLK_AHB1 16 +#define CLK_APB1 17 +#define CLK_APB2 18 +#define CLK_AHB2 19 + +/* All the bus gates are exported */ + +/* The first bunch of module clocks are exported */ + +#define CLK_DRAM 96 + +/* All the DRAM gates are exported */ + +/* Some more module clocks are exported */ + +#define CLK_MBUS 113 + +/* And the GPU module clock is exported */ + +#define CLK_NUMBER (CLK_GPU + 1) + +#endif /* _CCU_SUN8I_H3_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_common.c b/drivers/clk/sunxi-ng/ccu_common.c new file mode 100644 index 000000000..51d4bac97 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_common.c @@ -0,0 +1,90 @@ +/* + * Copyright 2016 Maxime Ripard + * + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include + +#include "ccu_common.h" +#include "ccu_reset.h" + +static DEFINE_SPINLOCK(ccu_lock); + +void ccu_helper_wait_for_lock(struct ccu_common *common, u32 lock) +{ + u32 reg; + + if (!lock) + return; + + WARN_ON(readl_relaxed_poll_timeout(common->base + common->reg, reg, + reg & lock, 100, 70000)); +} + +int sunxi_ccu_probe(struct device_node *node, void __iomem *reg, + const struct sunxi_ccu_desc *desc) +{ + struct ccu_reset *reset; + int i, ret; + + for (i = 0; i < desc->num_ccu_clks; i++) { + struct ccu_common *cclk = desc->ccu_clks[i]; + + if (!cclk) + continue; + + cclk->base = reg; + cclk->lock = &ccu_lock; + } + + for (i = 0; i < desc->hw_clks->num ; i++) { + struct clk_hw *hw = desc->hw_clks->hws[i]; + + if (!hw) + continue; + + ret = clk_hw_register(NULL, hw); + if (ret) { + pr_err("Couldn't register clock %s\n", + clk_hw_get_name(hw)); + goto err_clk_unreg; + } + } + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, + desc->hw_clks); + if (ret) + goto err_clk_unreg; + + reset = kzalloc(sizeof(*reset), GFP_KERNEL); + reset->rcdev.of_node = node; + reset->rcdev.ops = &ccu_reset_ops; + reset->rcdev.owner = THIS_MODULE; + reset->rcdev.nr_resets = desc->num_resets; + reset->base = reg; + reset->lock = &ccu_lock; + reset->reset_map = desc->resets; + + ret = reset_controller_register(&reset->rcdev); + if (ret) + goto err_of_clk_unreg; + + return 0; + +err_of_clk_unreg: +err_clk_unreg: + return ret; +} diff --git a/drivers/clk/sunxi-ng/ccu_common.h b/drivers/clk/sunxi-ng/ccu_common.h new file mode 100644 index 000000000..b3d9abfbd --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_common.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include +#include + +#define CCU_FEATURE_FRACTIONAL BIT(0) +#define CCU_FEATURE_VARIABLE_PREDIV BIT(1) +#define CCU_FEATURE_FIXED_PREDIV BIT(2) +#define CCU_FEATURE_FIXED_POSTDIV BIT(3) + +struct device_node; + +#define CLK_HW_INIT(_name, _parent, _ops, _flags) \ + &(struct clk_init_data) { \ + .flags = _flags, \ + .name = _name, \ + .parent_names = (const char *[]) { _parent }, \ + .num_parents = 1, \ + .ops = _ops, \ + } + +#define CLK_HW_INIT_PARENTS(_name, _parents, _ops, _flags) \ + &(struct clk_init_data) { \ + .flags = _flags, \ + .name = _name, \ + .parent_names = _parents, \ + .num_parents = ARRAY_SIZE(_parents), \ + .ops = _ops, \ + } + +#define CLK_FIXED_FACTOR(_struct, _name, _parent, \ + _div, _mult, _flags) \ + struct clk_fixed_factor _struct = { \ + .div = _div, \ + .mult = _mult, \ + .hw.init = CLK_HW_INIT(_name, \ + _parent, \ + &clk_fixed_factor_ops, \ + _flags), \ + } + +struct ccu_common { + void __iomem *base; + u16 reg; + + unsigned long features; + spinlock_t *lock; + struct clk_hw hw; +}; + +static inline struct ccu_common *hw_to_ccu_common(struct clk_hw *hw) +{ + return container_of(hw, struct ccu_common, hw); +} + +struct sunxi_ccu_desc { + struct ccu_common **ccu_clks; + unsigned long num_ccu_clks; + + struct clk_hw_onecell_data *hw_clks; + + struct ccu_reset_map *resets; + unsigned long num_resets; +}; + +void ccu_helper_wait_for_lock(struct ccu_common *common, u32 lock); + +int sunxi_ccu_probe(struct device_node *node, void __iomem *reg, + const struct sunxi_ccu_desc *desc); + +#endif /* _COMMON_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_div.c b/drivers/clk/sunxi-ng/ccu_div.c new file mode 100644 index 000000000..8659b4cb6 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_div.c @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include + +#include "ccu_gate.h" +#include "ccu_div.h" + +static unsigned long ccu_div_round_rate(struct ccu_mux_internal *mux, + unsigned long parent_rate, + unsigned long rate, + void *data) +{ + struct ccu_div *cd = data; + unsigned long val; + + /* + * We can't use divider_round_rate that assumes that there's + * several parents, while we might be called to evaluate + * several different parents. + */ + val = divider_get_val(rate, parent_rate, cd->div.table, cd->div.width, + cd->div.flags); + + return divider_recalc_rate(&cd->common.hw, parent_rate, val, + cd->div.table, cd->div.flags); +} + +static void ccu_div_disable(struct clk_hw *hw) +{ + struct ccu_div *cd = hw_to_ccu_div(hw); + + return ccu_gate_helper_disable(&cd->common, cd->enable); +} + +static int ccu_div_enable(struct clk_hw *hw) +{ + struct ccu_div *cd = hw_to_ccu_div(hw); + + return ccu_gate_helper_enable(&cd->common, cd->enable); +} + +static int ccu_div_is_enabled(struct clk_hw *hw) +{ + struct ccu_div *cd = hw_to_ccu_div(hw); + + return ccu_gate_helper_is_enabled(&cd->common, cd->enable); +} + +static unsigned long ccu_div_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct ccu_div *cd = hw_to_ccu_div(hw); + unsigned long val; + u32 reg; + + reg = readl(cd->common.base + cd->common.reg); + val = reg >> cd->div.shift; + val &= (1 << cd->div.width) - 1; + + ccu_mux_helper_adjust_parent_for_prediv(&cd->common, &cd->mux, -1, + &parent_rate); + + return divider_recalc_rate(hw, parent_rate, val, cd->div.table, + cd->div.flags); +} + +static int ccu_div_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct ccu_div *cd = hw_to_ccu_div(hw); + + return ccu_mux_helper_determine_rate(&cd->common, &cd->mux, + req, ccu_div_round_rate, cd); +} + +static int ccu_div_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct ccu_div *cd = hw_to_ccu_div(hw); + unsigned long flags; + unsigned long val; + u32 reg; + + ccu_mux_helper_adjust_parent_for_prediv(&cd->common, &cd->mux, -1, + &parent_rate); + + val = divider_get_val(rate, parent_rate, cd->div.table, cd->div.width, + cd->div.flags); + + spin_lock_irqsave(cd->common.lock, flags); + + reg = readl(cd->common.base + cd->common.reg); + reg &= ~GENMASK(cd->div.width + cd->div.shift - 1, cd->div.shift); + + writel(reg | (val << cd->div.shift), + cd->common.base + cd->common.reg); + + spin_unlock_irqrestore(cd->common.lock, flags); + + return 0; +} + +static u8 ccu_div_get_parent(struct clk_hw *hw) +{ + struct ccu_div *cd = hw_to_ccu_div(hw); + + return ccu_mux_helper_get_parent(&cd->common, &cd->mux); +} + +static int ccu_div_set_parent(struct clk_hw *hw, u8 index) +{ + struct ccu_div *cd = hw_to_ccu_div(hw); + + return ccu_mux_helper_set_parent(&cd->common, &cd->mux, index); +} + +const struct clk_ops ccu_div_ops = { + .disable = ccu_div_disable, + .enable = ccu_div_enable, + .is_enabled = ccu_div_is_enabled, + + .get_parent = ccu_div_get_parent, + .set_parent = ccu_div_set_parent, + + .determine_rate = ccu_div_determine_rate, + .recalc_rate = ccu_div_recalc_rate, + .set_rate = ccu_div_set_rate, +}; diff --git a/drivers/clk/sunxi-ng/ccu_div.h b/drivers/clk/sunxi-ng/ccu_div.h new file mode 100644 index 000000000..653ade576 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_div.h @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_DIV_H_ +#define _CCU_DIV_H_ + +#include + +#include "ccu_common.h" +#include "ccu_mux.h" + +struct _ccu_div { + u8 shift; + u8 width; + + u32 flags; + + struct clk_div_table *table; +}; + +#define _SUNXI_CCU_DIV_TABLE_FLAGS(_shift, _width, _table, _flags) \ + { \ + .shift = _shift, \ + .width = _width, \ + .flags = _flags, \ + .table = _table, \ + } + +#define _SUNXI_CCU_DIV_FLAGS(_shift, _width, _flags) \ + _SUNXI_CCU_DIV_TABLE_FLAGS(_shift, _width, NULL, _flags) + +#define _SUNXI_CCU_DIV_TABLE(_shift, _width, _table) \ + _SUNXI_CCU_DIV_TABLE_FLAGS(_shift, _width, _table, 0) + +#define _SUNXI_CCU_DIV(_shift, _width) \ + _SUNXI_CCU_DIV_TABLE_FLAGS(_shift, _width, NULL, 0) + +struct ccu_div { + u32 enable; + + struct _ccu_div div; + struct ccu_mux_internal mux; + struct ccu_common common; +}; + +#define SUNXI_CCU_DIV_TABLE_WITH_GATE(_struct, _name, _parent, _reg, \ + _shift, _width, \ + _table, _gate, _flags) \ + struct ccu_div _struct = { \ + .div = _SUNXI_CCU_DIV_TABLE(_shift, _width, \ + _table), \ + .enable = _gate, \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT(_name, \ + _parent, \ + &ccu_div_ops, \ + _flags), \ + } \ + } + + +#define SUNXI_CCU_DIV_TABLE(_struct, _name, _parent, _reg, \ + _shift, _width, \ + _table, _flags) \ + SUNXI_CCU_DIV_TABLE_WITH_GATE(_struct, _name, _parent, _reg, \ + _shift, _width, _table, 0, \ + _flags) + +#define SUNXI_CCU_M_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, _muxshift, _muxwidth, \ + _gate, _flags) \ + struct ccu_div _struct = { \ + .enable = _gate, \ + .div = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .mux = SUNXI_CLK_MUX(_muxshift, _muxwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_PARENTS(_name, \ + _parents, \ + &ccu_div_ops, \ + _flags), \ + }, \ + } + +#define SUNXI_CCU_M_WITH_MUX(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, _muxshift, _muxwidth, \ + _flags) \ + SUNXI_CCU_M_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, _muxshift, _muxwidth, \ + 0, _flags) + + +#define SUNXI_CCU_M_WITH_GATE(_struct, _name, _parent, _reg, \ + _mshift, _mwidth, _gate, \ + _flags) \ + struct ccu_div _struct = { \ + .enable = _gate, \ + .div = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT(_name, \ + _parent, \ + &ccu_div_ops, \ + _flags), \ + }, \ + } + +#define SUNXI_CCU_M(_struct, _name, _parent, _reg, _mshift, _mwidth, \ + _flags) \ + SUNXI_CCU_M_WITH_GATE(_struct, _name, _parent, _reg, \ + _mshift, _mwidth, 0, _flags) + +static inline struct ccu_div *hw_to_ccu_div(struct clk_hw *hw) +{ + struct ccu_common *common = hw_to_ccu_common(hw); + + return container_of(common, struct ccu_div, common); +} + +extern const struct clk_ops ccu_div_ops; + +#endif /* _CCU_DIV_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_frac.c b/drivers/clk/sunxi-ng/ccu_frac.c new file mode 100644 index 000000000..5c4b10cd1 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_frac.c @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include +#include + +#include "ccu_frac.h" + +bool ccu_frac_helper_is_enabled(struct ccu_common *common, + struct _ccu_frac *cf) +{ + if (!(common->features & CCU_FEATURE_FRACTIONAL)) + return false; + + return !(readl(common->base + common->reg) & cf->enable); +} + +void ccu_frac_helper_enable(struct ccu_common *common, + struct _ccu_frac *cf) +{ + unsigned long flags; + u32 reg; + + if (!(common->features & CCU_FEATURE_FRACTIONAL)) + return; + + spin_lock_irqsave(common->lock, flags); + reg = readl(common->base + common->reg); + writel(reg & ~cf->enable, common->base + common->reg); + spin_unlock_irqrestore(common->lock, flags); +} + +void ccu_frac_helper_disable(struct ccu_common *common, + struct _ccu_frac *cf) +{ + unsigned long flags; + u32 reg; + + if (!(common->features & CCU_FEATURE_FRACTIONAL)) + return; + + spin_lock_irqsave(common->lock, flags); + reg = readl(common->base + common->reg); + writel(reg | cf->enable, common->base + common->reg); + spin_unlock_irqrestore(common->lock, flags); +} + +bool ccu_frac_helper_has_rate(struct ccu_common *common, + struct _ccu_frac *cf, + unsigned long rate) +{ + if (!(common->features & CCU_FEATURE_FRACTIONAL)) + return false; + + return (cf->rates[0] == rate) || (cf->rates[1] == rate); +} + +unsigned long ccu_frac_helper_read_rate(struct ccu_common *common, + struct _ccu_frac *cf) +{ + u32 reg; + + printk("%s: Read fractional\n", clk_hw_get_name(&common->hw)); + + if (!(common->features & CCU_FEATURE_FRACTIONAL)) + return 0; + + printk("%s: clock is fractional (rates %lu and %lu)\n", + clk_hw_get_name(&common->hw), cf->rates[0], cf->rates[1]); + + reg = readl(common->base + common->reg); + + printk("%s: clock reg is 0x%x (select is 0x%x)\n", + clk_hw_get_name(&common->hw), reg, cf->select); + + return (reg & cf->select) ? cf->rates[1] : cf->rates[0]; +} + +int ccu_frac_helper_set_rate(struct ccu_common *common, + struct _ccu_frac *cf, + unsigned long rate) +{ + unsigned long flags; + u32 reg, sel; + + if (!(common->features & CCU_FEATURE_FRACTIONAL)) + return -EINVAL; + + if (cf->rates[0] == rate) + sel = 0; + else if (cf->rates[1] == rate) + sel = cf->select; + else + return -EINVAL; + + spin_lock_irqsave(common->lock, flags); + reg = readl(common->base + common->reg); + reg &= ~cf->select; + writel(reg | sel, common->base + common->reg); + spin_unlock_irqrestore(common->lock, flags); + + return 0; +} diff --git a/drivers/clk/sunxi-ng/ccu_frac.h b/drivers/clk/sunxi-ng/ccu_frac.h new file mode 100644 index 000000000..e4c670b1c --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_frac.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_FRAC_H_ +#define _CCU_FRAC_H_ + +#include + +#include "ccu_common.h" + +struct _ccu_frac { + u32 enable; + u32 select; + + unsigned long rates[2]; +}; + +#define _SUNXI_CCU_FRAC(_enable, _select, _rate1, _rate2) \ + { \ + .enable = _enable, \ + .select = _select, \ + .rates = { _rate1, _rate2 }, \ + } + +bool ccu_frac_helper_is_enabled(struct ccu_common *common, + struct _ccu_frac *cf); +void ccu_frac_helper_enable(struct ccu_common *common, + struct _ccu_frac *cf); +void ccu_frac_helper_disable(struct ccu_common *common, + struct _ccu_frac *cf); + +bool ccu_frac_helper_has_rate(struct ccu_common *common, + struct _ccu_frac *cf, + unsigned long rate); + +unsigned long ccu_frac_helper_read_rate(struct ccu_common *common, + struct _ccu_frac *cf); + +int ccu_frac_helper_set_rate(struct ccu_common *common, + struct _ccu_frac *cf, + unsigned long rate); + +#endif /* _CCU_FRAC_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_gate.c b/drivers/clk/sunxi-ng/ccu_gate.c new file mode 100644 index 000000000..8a81f9d4a --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_gate.c @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include + +#include "ccu_gate.h" + +void ccu_gate_helper_disable(struct ccu_common *common, u32 gate) +{ + unsigned long flags; + u32 reg; + + if (!gate) + return; + + spin_lock_irqsave(common->lock, flags); + + reg = readl(common->base + common->reg); + writel(reg & ~gate, common->base + common->reg); + + spin_unlock_irqrestore(common->lock, flags); +} + +static void ccu_gate_disable(struct clk_hw *hw) +{ + struct ccu_gate *cg = hw_to_ccu_gate(hw); + + return ccu_gate_helper_disable(&cg->common, cg->enable); +} + +int ccu_gate_helper_enable(struct ccu_common *common, u32 gate) +{ + unsigned long flags; + u32 reg; + + if (!gate) + return 0; + + spin_lock_irqsave(common->lock, flags); + + reg = readl(common->base + common->reg); + writel(reg | gate, common->base + common->reg); + + spin_unlock_irqrestore(common->lock, flags); + + return 0; +} + +static int ccu_gate_enable(struct clk_hw *hw) +{ + struct ccu_gate *cg = hw_to_ccu_gate(hw); + + return ccu_gate_helper_enable(&cg->common, cg->enable); +} + +int ccu_gate_helper_is_enabled(struct ccu_common *common, u32 gate) +{ + if (!gate) + return 1; + + return readl(common->base + common->reg) & gate; +} + +static int ccu_gate_is_enabled(struct clk_hw *hw) +{ + struct ccu_gate *cg = hw_to_ccu_gate(hw); + + return ccu_gate_helper_is_enabled(&cg->common, cg->enable); +} + +const struct clk_ops ccu_gate_ops = { + .disable = ccu_gate_disable, + .enable = ccu_gate_enable, + .is_enabled = ccu_gate_is_enabled, +}; diff --git a/drivers/clk/sunxi-ng/ccu_gate.h b/drivers/clk/sunxi-ng/ccu_gate.h new file mode 100644 index 000000000..4466169bd --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_gate.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_GATE_H_ +#define _CCU_GATE_H_ + +#include + +#include "ccu_common.h" + +struct ccu_gate { + u32 enable; + + struct ccu_common common; +}; + +#define SUNXI_CCU_GATE(_struct, _name, _parent, _reg, _gate, _flags) \ + struct ccu_gate _struct = { \ + .enable = _gate, \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT(_name, \ + _parent, \ + &ccu_gate_ops, \ + _flags), \ + } \ + } + +static inline struct ccu_gate *hw_to_ccu_gate(struct clk_hw *hw) +{ + struct ccu_common *common = hw_to_ccu_common(hw); + + return container_of(common, struct ccu_gate, common); +} + +void ccu_gate_helper_disable(struct ccu_common *common, u32 gate); +int ccu_gate_helper_enable(struct ccu_common *common, u32 gate); +int ccu_gate_helper_is_enabled(struct ccu_common *common, u32 gate); + +extern const struct clk_ops ccu_gate_ops; + +#endif /* _CCU_GATE_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_mp.c b/drivers/clk/sunxi-ng/ccu_mp.c new file mode 100644 index 000000000..cbf33ef5f --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_mp.c @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include + +#include "ccu_gate.h" +#include "ccu_mp.h" + +static void ccu_mp_find_best(unsigned long parent, unsigned long rate, + unsigned int max_m, unsigned int max_p, + unsigned int *m, unsigned int *p) +{ + unsigned long best_rate = 0; + unsigned int best_m = 0, best_p = 0; + unsigned int _m, _p; + + for (_p = 0; _p <= max_p; _p++) { + for (_m = 1; _m <= max_m; _m++) { + unsigned long tmp_rate = (parent >> _p) / _m; + + if (tmp_rate > rate) + continue; + + if ((rate - tmp_rate) < (rate - best_rate)) { + best_rate = tmp_rate; + best_m = _m; + best_p = _p; + } + } + } + + *m = best_m; + *p = best_p; +} + +static unsigned long ccu_mp_round_rate(struct ccu_mux_internal *mux, + unsigned long parent_rate, + unsigned long rate, + void *data) +{ + struct ccu_mp *cmp = data; + unsigned int m, p; + + ccu_mp_find_best(parent_rate, rate, + 1 << cmp->m.width, (1 << cmp->p.width) - 1, + &m, &p); + + return (parent_rate >> p) / m; +} + +static void ccu_mp_disable(struct clk_hw *hw) +{ + struct ccu_mp *cmp = hw_to_ccu_mp(hw); + + return ccu_gate_helper_disable(&cmp->common, cmp->enable); +} + +static int ccu_mp_enable(struct clk_hw *hw) +{ + struct ccu_mp *cmp = hw_to_ccu_mp(hw); + + return ccu_gate_helper_enable(&cmp->common, cmp->enable); +} + +static int ccu_mp_is_enabled(struct clk_hw *hw) +{ + struct ccu_mp *cmp = hw_to_ccu_mp(hw); + + return ccu_gate_helper_is_enabled(&cmp->common, cmp->enable); +} + +static unsigned long ccu_mp_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct ccu_mp *cmp = hw_to_ccu_mp(hw); + unsigned int m, p; + u32 reg; + + reg = readl(cmp->common.base + cmp->common.reg); + + m = reg >> cmp->m.shift; + m &= (1 << cmp->m.width) - 1; + + p = reg >> cmp->p.shift; + p &= (1 << cmp->p.width) - 1; + + return (parent_rate >> p) / (m + 1); +} + +static int ccu_mp_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct ccu_mp *cmp = hw_to_ccu_mp(hw); + + return ccu_mux_helper_determine_rate(&cmp->common, &cmp->mux, + req, ccu_mp_round_rate, cmp); +} + +static int ccu_mp_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct ccu_mp *cmp = hw_to_ccu_mp(hw); + unsigned long flags; + unsigned int m, p; + u32 reg; + + ccu_mp_find_best(parent_rate, rate, + 1 << cmp->m.width, (1 << cmp->p.width) - 1, + &m, &p); + + + spin_lock_irqsave(cmp->common.lock, flags); + + reg = readl(cmp->common.base + cmp->common.reg); + reg &= ~GENMASK(cmp->m.width + cmp->m.shift - 1, cmp->m.shift); + reg &= ~GENMASK(cmp->p.width + cmp->p.shift - 1, cmp->p.shift); + + writel(reg | (p << cmp->p.shift) | ((m - 1) << cmp->m.shift), + cmp->common.base + cmp->common.reg); + + spin_unlock_irqrestore(cmp->common.lock, flags); + + return 0; +} + +static u8 ccu_mp_get_parent(struct clk_hw *hw) +{ + struct ccu_mp *cmp = hw_to_ccu_mp(hw); + + return ccu_mux_helper_get_parent(&cmp->common, &cmp->mux); +} + +static int ccu_mp_set_parent(struct clk_hw *hw, u8 index) +{ + struct ccu_mp *cmp = hw_to_ccu_mp(hw); + + return ccu_mux_helper_set_parent(&cmp->common, &cmp->mux, index); +} + +const struct clk_ops ccu_mp_ops = { + .disable = ccu_mp_disable, + .enable = ccu_mp_enable, + .is_enabled = ccu_mp_is_enabled, + + .get_parent = ccu_mp_get_parent, + .set_parent = ccu_mp_set_parent, + + .determine_rate = ccu_mp_determine_rate, + .recalc_rate = ccu_mp_recalc_rate, + .set_rate = ccu_mp_set_rate, +}; diff --git a/drivers/clk/sunxi-ng/ccu_mp.h b/drivers/clk/sunxi-ng/ccu_mp.h new file mode 100644 index 000000000..3cf12bf95 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_mp.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_MP_H_ +#define _CCU_MP_H_ + +#include + +#include "ccu_common.h" +#include "ccu_div.h" +#include "ccu_mult.h" +#include "ccu_mux.h" + +/* + * struct ccu_mp - Definition of an M-P clock + * + * Clocks based on the formula parent >> P / M + */ +struct ccu_mp { + u32 enable; + + struct _ccu_div m; + struct _ccu_div p; + struct ccu_mux_internal mux; + struct ccu_common common; +}; + +#define SUNXI_CCU_MP_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, \ + _pshift, _pwidth, \ + _muxshift, _muxwidth, \ + _gate, _flags) \ + struct ccu_mp _struct = { \ + .enable = _gate, \ + .m = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .p = _SUNXI_CCU_DIV(_pshift, _pwidth), \ + .mux = SUNXI_CLK_MUX(_muxshift, _muxwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_PARENTS(_name, \ + _parents, \ + &ccu_mp_ops, \ + _flags), \ + } \ + } + +#define SUNXI_CCU_MP_WITH_MUX(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, \ + _pshift, _pwidth, \ + _muxshift, _muxwidth, \ + _flags) \ + SUNXI_CCU_MP_WITH_MUX_GATE(_struct, _name, _parents, _reg, \ + _mshift, _mwidth, \ + _pshift, _pwidth, \ + _muxshift, _muxwidth, \ + 0, _flags) + +static inline struct ccu_mp *hw_to_ccu_mp(struct clk_hw *hw) +{ + struct ccu_common *common = hw_to_ccu_common(hw); + + return container_of(common, struct ccu_mp, common); +} + +extern const struct clk_ops ccu_mp_ops; + +#endif /* _CCU_MP_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_mult.h b/drivers/clk/sunxi-ng/ccu_mult.h new file mode 100644 index 000000000..609db6610 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_mult.h @@ -0,0 +1,15 @@ +#ifndef _CCU_MULT_H_ +#define _CCU_MULT_H_ + +struct _ccu_mult { + u8 shift; + u8 width; +}; + +#define _SUNXI_CCU_MULT(_shift, _width) \ + { \ + .shift = _shift, \ + .width = _width, \ + } + +#endif /* _CCU_MULT_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_mux.c b/drivers/clk/sunxi-ng/ccu_mux.c new file mode 100644 index 000000000..58fc36e7d --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_mux.c @@ -0,0 +1,187 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include + +#include "ccu_gate.h" +#include "ccu_mux.h" + +void ccu_mux_helper_adjust_parent_for_prediv(struct ccu_common *common, + struct ccu_mux_internal *cm, + int parent_index, + unsigned long *parent_rate) +{ + u8 prediv = 1; + u32 reg; + + if (!((common->features & CCU_FEATURE_FIXED_PREDIV) || + (common->features & CCU_FEATURE_VARIABLE_PREDIV))) + return; + + reg = readl(common->base + common->reg); + if (parent_index < 0) { + parent_index = reg >> cm->shift; + parent_index &= (1 << cm->width) - 1; + } + + if (common->features & CCU_FEATURE_FIXED_PREDIV) + if (parent_index == cm->fixed_prediv.index) + prediv = cm->fixed_prediv.div; + + if (common->features & CCU_FEATURE_VARIABLE_PREDIV) + if (parent_index == cm->variable_prediv.index) { + u8 div; + + div = reg >> cm->variable_prediv.shift; + div &= (1 << cm->variable_prediv.width) - 1; + prediv = div + 1; + } + + *parent_rate = *parent_rate / prediv; +} + +int ccu_mux_helper_determine_rate(struct ccu_common *common, + struct ccu_mux_internal *cm, + struct clk_rate_request *req, + unsigned long (*round)(struct ccu_mux_internal *, + unsigned long, + unsigned long, + void *), + void *data) +{ + unsigned long best_parent_rate = 0, best_rate = 0; + struct clk_hw *best_parent, *hw = &common->hw; + unsigned int i; + + for (i = 0; i < clk_hw_get_num_parents(hw); i++) { + unsigned long tmp_rate, parent_rate; + struct clk_hw *parent; + + parent = clk_hw_get_parent_by_index(hw, i); + if (!parent) + continue; + + parent_rate = clk_hw_get_rate(parent); + ccu_mux_helper_adjust_parent_for_prediv(common, cm, i, + &parent_rate); + + tmp_rate = round(cm, clk_hw_get_rate(parent), req->rate, data); + if (tmp_rate == req->rate) { + best_parent = parent; + best_parent_rate = parent_rate; + best_rate = tmp_rate; + goto out; + } + + if ((req->rate - tmp_rate) < (req->rate - best_rate)) { + best_rate = tmp_rate; + best_parent_rate = parent_rate; + best_parent = parent; + } + } + + if (best_rate == 0) + return -EINVAL; + +out: + req->best_parent_hw = best_parent; + req->best_parent_rate = best_parent_rate; + req->rate = best_rate; + return 0; +} + +u8 ccu_mux_helper_get_parent(struct ccu_common *common, + struct ccu_mux_internal *cm) +{ + u32 reg; + u8 parent; + + reg = readl(common->base + common->reg); + parent = reg >> cm->shift; + parent &= (1 << cm->width) - 1; + + return parent; +} + +int ccu_mux_helper_set_parent(struct ccu_common *common, + struct ccu_mux_internal *cm, + u8 index) +{ + unsigned long flags; + u32 reg; + + spin_lock_irqsave(common->lock, flags); + + reg = readl(common->base + common->reg); + reg &= ~GENMASK(cm->width + cm->shift - 1, cm->shift); + writel(reg | (index << cm->shift), common->base + common->reg); + + spin_unlock_irqrestore(common->lock, flags); + + return 0; +} + +static void ccu_mux_disable(struct clk_hw *hw) +{ + struct ccu_mux *cm = hw_to_ccu_mux(hw); + + return ccu_gate_helper_disable(&cm->common, cm->enable); +} + +static int ccu_mux_enable(struct clk_hw *hw) +{ + struct ccu_mux *cm = hw_to_ccu_mux(hw); + + return ccu_gate_helper_enable(&cm->common, cm->enable); +} + +static int ccu_mux_is_enabled(struct clk_hw *hw) +{ + struct ccu_mux *cm = hw_to_ccu_mux(hw); + + return ccu_gate_helper_is_enabled(&cm->common, cm->enable); +} + +static u8 ccu_mux_get_parent(struct clk_hw *hw) +{ + struct ccu_mux *cm = hw_to_ccu_mux(hw); + + return ccu_mux_helper_get_parent(&cm->common, &cm->mux); +} + +static int ccu_mux_set_parent(struct clk_hw *hw, u8 index) +{ + struct ccu_mux *cm = hw_to_ccu_mux(hw); + + return ccu_mux_helper_set_parent(&cm->common, &cm->mux, index); +} + +static unsigned long ccu_mux_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct ccu_mux *cm = hw_to_ccu_mux(hw); + + ccu_mux_helper_adjust_parent_for_prediv(&cm->common, &cm->mux, -1, + &parent_rate); + + return parent_rate; +} + +const struct clk_ops ccu_mux_ops = { + .disable = ccu_mux_disable, + .enable = ccu_mux_enable, + .is_enabled = ccu_mux_is_enabled, + + .get_parent = ccu_mux_get_parent, + .set_parent = ccu_mux_set_parent, + + .determine_rate = __clk_mux_determine_rate, + .recalc_rate = ccu_mux_recalc_rate, +}; diff --git a/drivers/clk/sunxi-ng/ccu_mux.h b/drivers/clk/sunxi-ng/ccu_mux.h new file mode 100644 index 000000000..945082631 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_mux.h @@ -0,0 +1,91 @@ +#ifndef _CCU_MUX_H_ +#define _CCU_MUX_H_ + +#include + +#include "ccu_common.h" + +struct ccu_mux_internal { + u8 shift; + u8 width; + + struct { + u8 index; + u8 div; + } fixed_prediv; + + struct { + u8 index; + u8 shift; + u8 width; + } variable_prediv; +}; + +#define SUNXI_CLK_MUX(_shift, _width) \ + { \ + .shift = _shift, \ + .width = _width, \ + } + +struct ccu_mux { + u16 reg; + u32 enable; + + struct ccu_mux_internal mux; + struct ccu_common common; +}; + +#define SUNXI_CCU_MUX(_struct, _name, _parents, _reg, _shift, _width, _flags) \ + struct ccu_mux _struct = { \ + .mux = SUNXI_CLK_MUX(_shift, _width), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_PARENTS(_name, \ + _parents, \ + &ccu_mux_ops, \ + _flags), \ + } \ + } + +#define SUNXI_CCU_MUX_WITH_GATE(_struct, _name, _parents, _reg, \ + _shift, _width, _gate, _flags) \ + struct ccu_mux _struct = { \ + .enable = _gate, \ + .mux = SUNXI_CLK_MUX(_shift, _width), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT_PARENTS(_name, \ + _parents, \ + &ccu_mux_ops, \ + _flags), \ + } \ + } + +static inline struct ccu_mux *hw_to_ccu_mux(struct clk_hw *hw) +{ + struct ccu_common *common = hw_to_ccu_common(hw); + + return container_of(common, struct ccu_mux, common); +} + +extern const struct clk_ops ccu_mux_ops; + +void ccu_mux_helper_adjust_parent_for_prediv(struct ccu_common *common, + struct ccu_mux_internal *cm, + int parent_index, + unsigned long *parent_rate); +int ccu_mux_helper_determine_rate(struct ccu_common *common, + struct ccu_mux_internal *cm, + struct clk_rate_request *req, + unsigned long (*round)(struct ccu_mux_internal *, + unsigned long, + unsigned long, + void *), + void *data); +u8 ccu_mux_helper_get_parent(struct ccu_common *common, + struct ccu_mux_internal *cm); +int ccu_mux_helper_set_parent(struct ccu_common *common, + struct ccu_mux_internal *cm, + u8 index); + +#endif /* _CCU_MUX_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_nk.c b/drivers/clk/sunxi-ng/ccu_nk.c new file mode 100644 index 000000000..d6fafb397 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_nk.c @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include +#include + +#include "ccu_gate.h" +#include "ccu_nk.h" + +static void ccu_nk_find_best(unsigned long parent, unsigned long rate, + unsigned int max_n, unsigned int max_k, + unsigned int *n, unsigned int *k) +{ + unsigned long best_rate = 0; + unsigned int best_k = 0, best_n = 0; + unsigned int _k, _n; + + for (_k = 1; _k <= max_k; _k++) { + for (_n = 1; _n <= max_n; _n++) { + unsigned long tmp_rate = parent * _n * _k; + + if (tmp_rate > rate) + continue; + + if ((rate - tmp_rate) < (rate - best_rate)) { + best_rate = tmp_rate; + best_k = _k; + best_n = _n; + } + } + } + + *k = best_k; + *n = best_n; +} + +static void ccu_nk_disable(struct clk_hw *hw) +{ + struct ccu_nk *nk = hw_to_ccu_nk(hw); + + return ccu_gate_helper_disable(&nk->common, nk->enable); +} + +static int ccu_nk_enable(struct clk_hw *hw) +{ + struct ccu_nk *nk = hw_to_ccu_nk(hw); + + return ccu_gate_helper_enable(&nk->common, nk->enable); +} + +static int ccu_nk_is_enabled(struct clk_hw *hw) +{ + struct ccu_nk *nk = hw_to_ccu_nk(hw); + + return ccu_gate_helper_is_enabled(&nk->common, nk->enable); +} + +static unsigned long ccu_nk_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct ccu_nk *nk = hw_to_ccu_nk(hw); + unsigned long rate, n, k; + u32 reg; + + reg = readl(nk->common.base + nk->common.reg); + + n = reg >> nk->n.shift; + n &= (1 << nk->n.width) - 1; + + k = reg >> nk->k.shift; + k &= (1 << nk->k.width) - 1; + + rate = parent_rate * (n + 1) * (k + 1); + + if (nk->common.features & CCU_FEATURE_FIXED_POSTDIV) + rate /= nk->fixed_post_div; + + return rate; +} + +static long ccu_nk_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + struct ccu_nk *nk = hw_to_ccu_nk(hw); + unsigned int n, k; + + if (nk->common.features & CCU_FEATURE_FIXED_POSTDIV) + rate *= nk->fixed_post_div; + + ccu_nk_find_best(*parent_rate, rate, + 1 << nk->n.width, 1 << nk->k.width, + &n, &k); + + rate = *parent_rate * n * k; + if (nk->common.features & CCU_FEATURE_FIXED_POSTDIV) + rate = rate / nk->fixed_post_div; + + return rate; +} + +static int ccu_nk_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct ccu_nk *nk = hw_to_ccu_nk(hw); + unsigned long flags; + unsigned int n, k; + u32 reg; + + if (nk->common.features & CCU_FEATURE_FIXED_POSTDIV) + rate = rate * nk->fixed_post_div; + + ccu_nk_find_best(parent_rate, rate, + 1 << nk->n.width, 1 << nk->k.width, + &n, &k); + + spin_lock_irqsave(nk->common.lock, flags); + + reg = readl(nk->common.base + nk->common.reg); + reg &= ~GENMASK(nk->n.width + nk->n.shift - 1, nk->n.shift); + reg &= ~GENMASK(nk->k.width + nk->k.shift - 1, nk->k.shift); + + writel(reg | ((k - 1) << nk->k.shift) | ((n - 1) << nk->n.shift), + nk->common.base + nk->common.reg); + + spin_unlock_irqrestore(nk->common.lock, flags); + + ccu_helper_wait_for_lock(&nk->common, nk->lock); + + return 0; +} + +const struct clk_ops ccu_nk_ops = { + .disable = ccu_nk_disable, + .enable = ccu_nk_enable, + .is_enabled = ccu_nk_is_enabled, + + .recalc_rate = ccu_nk_recalc_rate, + .round_rate = ccu_nk_round_rate, + .set_rate = ccu_nk_set_rate, +}; diff --git a/drivers/clk/sunxi-ng/ccu_nk.h b/drivers/clk/sunxi-ng/ccu_nk.h new file mode 100644 index 000000000..4b52da0c2 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_nk.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_NK_H_ +#define _CCU_NK_H_ + +#include + +#include "ccu_common.h" +#include "ccu_div.h" +#include "ccu_mult.h" + +/* + * struct ccu_nk - Definition of an N-K clock + * + * Clocks based on the formula parent * N * K + */ +struct ccu_nk { + u16 reg; + u32 enable; + u32 lock; + + struct _ccu_mult n; + struct _ccu_mult k; + + unsigned int fixed_post_div; + + struct ccu_common common; +}; + +#define SUNXI_CCU_NK_WITH_GATE_LOCK_POSTDIV(_struct, _name, _parent, _reg, \ + _nshift, _nwidth, \ + _kshift, _kwidth, \ + _gate, _lock, _postdiv, \ + _flags) \ + struct ccu_nk _struct = { \ + .enable = _gate, \ + .lock = _lock, \ + .k = _SUNXI_CCU_MULT(_kshift, _kwidth), \ + .n = _SUNXI_CCU_MULT(_nshift, _nwidth), \ + .fixed_post_div = _postdiv, \ + .common = { \ + .reg = _reg, \ + .features = CCU_FEATURE_FIXED_POSTDIV, \ + .hw.init = CLK_HW_INIT(_name, \ + _parent, \ + &ccu_nk_ops, \ + _flags), \ + }, \ + } + +static inline struct ccu_nk *hw_to_ccu_nk(struct clk_hw *hw) +{ + struct ccu_common *common = hw_to_ccu_common(hw); + + return container_of(common, struct ccu_nk, common); +} + +extern const struct clk_ops ccu_nk_ops; + +#endif /* _CCU_NK_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_nkm.c b/drivers/clk/sunxi-ng/ccu_nkm.c new file mode 100644 index 000000000..2071822b1 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_nkm.c @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include +#include + +#include "ccu_gate.h" +#include "ccu_nkm.h" + +struct _ccu_nkm { + unsigned long n, max_n; + unsigned long k, max_k; + unsigned long m, max_m; +}; + +static void ccu_nkm_find_best(unsigned long parent, unsigned long rate, + struct _ccu_nkm *nkm) +{ + unsigned long best_rate = 0; + unsigned long best_n = 0, best_k = 0, best_m = 0; + unsigned long _n, _k, _m; + + for (_k = 1; _k <= nkm->max_k; _k++) { + unsigned long tmp_rate; + + rational_best_approximation(rate / _k, parent, + nkm->max_n, nkm->max_m, &_n, &_m); + + tmp_rate = parent * _n * _k / _m; + + if (tmp_rate > rate) + continue; + + if ((rate - tmp_rate) < (rate - best_rate)) { + best_rate = tmp_rate; + best_n = _n; + best_k = _k; + best_m = _m; + } + } + + nkm->n = best_n; + nkm->k = best_k; + nkm->m = best_m; +} + +static void ccu_nkm_disable(struct clk_hw *hw) +{ + struct ccu_nkm *nkm = hw_to_ccu_nkm(hw); + + return ccu_gate_helper_disable(&nkm->common, nkm->enable); +} + +static int ccu_nkm_enable(struct clk_hw *hw) +{ + struct ccu_nkm *nkm = hw_to_ccu_nkm(hw); + + return ccu_gate_helper_enable(&nkm->common, nkm->enable); +} + +static int ccu_nkm_is_enabled(struct clk_hw *hw) +{ + struct ccu_nkm *nkm = hw_to_ccu_nkm(hw); + + return ccu_gate_helper_is_enabled(&nkm->common, nkm->enable); +} + +static unsigned long ccu_nkm_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct ccu_nkm *nkm = hw_to_ccu_nkm(hw); + unsigned long n, m, k; + u32 reg; + + reg = readl(nkm->common.base + nkm->common.reg); + + n = reg >> nkm->n.shift; + n &= (1 << nkm->n.width) - 1; + + k = reg >> nkm->k.shift; + k &= (1 << nkm->k.width) - 1; + + m = reg >> nkm->m.shift; + m &= (1 << nkm->m.width) - 1; + + return parent_rate * (n + 1) * (k + 1) / (m + 1); +} + +static long ccu_nkm_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + struct ccu_nkm *nkm = hw_to_ccu_nkm(hw); + struct _ccu_nkm _nkm; + + _nkm.max_n = 1 << nkm->n.width; + _nkm.max_k = 1 << nkm->k.width; + _nkm.max_m = 1 << nkm->m.width; + + ccu_nkm_find_best(*parent_rate, rate, &_nkm); + + return *parent_rate * _nkm.n * _nkm.k / _nkm.m; +} + +static int ccu_nkm_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct ccu_nkm *nkm = hw_to_ccu_nkm(hw); + struct _ccu_nkm _nkm; + unsigned long flags; + u32 reg; + + _nkm.max_n = 1 << nkm->n.width; + _nkm.max_k = 1 << nkm->k.width; + _nkm.max_m = 1 << nkm->m.width; + + ccu_nkm_find_best(parent_rate, rate, &_nkm); + + spin_lock_irqsave(nkm->common.lock, flags); + + reg = readl(nkm->common.base + nkm->common.reg); + reg &= ~GENMASK(nkm->n.width + nkm->n.shift - 1, nkm->n.shift); + reg &= ~GENMASK(nkm->k.width + nkm->k.shift - 1, nkm->k.shift); + reg &= ~GENMASK(nkm->m.width + nkm->m.shift - 1, nkm->m.shift); + + reg |= (_nkm.n - 1) << nkm->n.shift; + reg |= (_nkm.k - 1) << nkm->k.shift; + reg |= (_nkm.m - 1) << nkm->m.shift; + + writel(reg, nkm->common.base + nkm->common.reg); + + spin_unlock_irqrestore(nkm->common.lock, flags); + + ccu_helper_wait_for_lock(&nkm->common, nkm->lock); + + return 0; +} + +const struct clk_ops ccu_nkm_ops = { + .disable = ccu_nkm_disable, + .enable = ccu_nkm_enable, + .is_enabled = ccu_nkm_is_enabled, + + .recalc_rate = ccu_nkm_recalc_rate, + .round_rate = ccu_nkm_round_rate, + .set_rate = ccu_nkm_set_rate, +}; diff --git a/drivers/clk/sunxi-ng/ccu_nkm.h b/drivers/clk/sunxi-ng/ccu_nkm.h new file mode 100644 index 000000000..1936ac1c6 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_nkm.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_NKM_H_ +#define _CCU_NKM_H_ + +#include + +#include "ccu_common.h" +#include "ccu_div.h" +#include "ccu_mult.h" + +/* + * struct ccu_nkm - Definition of an N-K-M clock + * + * Clocks based on the formula parent * N * K / M + */ +struct ccu_nkm { + u32 enable; + u32 lock; + + struct _ccu_mult n; + struct _ccu_mult k; + struct _ccu_div m; + + struct ccu_common common; +}; + +#define SUNXI_CCU_NKM_WITH_GATE_LOCK(_struct, _name, _parent, _reg, \ + _nshift, _nwidth, \ + _kshift, _kwidth, \ + _mshift, _mwidth, \ + _gate, _lock, _flags) \ + struct ccu_nkm _struct = { \ + .enable = _gate, \ + .lock = _lock, \ + .k = _SUNXI_CCU_MULT(_kshift, _kwidth), \ + .n = _SUNXI_CCU_MULT(_nshift, _nwidth), \ + .m = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT(_name, \ + _parent, \ + &ccu_nkm_ops, \ + _flags), \ + }, \ + } + +static inline struct ccu_nkm *hw_to_ccu_nkm(struct clk_hw *hw) +{ + struct ccu_common *common = hw_to_ccu_common(hw); + + return container_of(common, struct ccu_nkm, common); +} + +extern const struct clk_ops ccu_nkm_ops; + +#endif /* _CCU_NKM_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_nkmp.c b/drivers/clk/sunxi-ng/ccu_nkmp.c new file mode 100644 index 000000000..9f2b98e19 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_nkmp.c @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include +#include + +#include "ccu_gate.h" +#include "ccu_nkmp.h" + +struct _ccu_nkmp { + unsigned long n, max_n; + unsigned long k, max_k; + unsigned long m, max_m; + unsigned long p, max_p; +}; + +static void ccu_nkmp_find_best(unsigned long parent, unsigned long rate, + struct _ccu_nkmp *nkmp) +{ + unsigned long best_rate = 0; + unsigned long best_n = 0, best_k = 0, best_m = 0, best_p = 0; + unsigned long _n, _k, _m, _p; + + for (_k = 1; _k <= nkmp->max_k; _k++) { + for (_p = 0; _p <= nkmp->max_p; _p++) { + unsigned long tmp_rate; + + rational_best_approximation(rate / _k, parent >> _p, + nkmp->max_n, nkmp->max_m, + &_n, &_m); + + tmp_rate = (parent * _n * _k >> _p) / _m; + + if (tmp_rate > rate) + continue; + + if ((rate - tmp_rate) < (rate - best_rate)) { + best_rate = tmp_rate; + best_n = _n; + best_k = _k; + best_m = _m; + best_p = _p; + } + } + } + + nkmp->n = best_n; + nkmp->k = best_k; + nkmp->m = best_m; + nkmp->p = best_p; +} + +static void ccu_nkmp_disable(struct clk_hw *hw) +{ + struct ccu_nkmp *nkmp = hw_to_ccu_nkmp(hw); + + return ccu_gate_helper_disable(&nkmp->common, nkmp->enable); +} + +static int ccu_nkmp_enable(struct clk_hw *hw) +{ + struct ccu_nkmp *nkmp = hw_to_ccu_nkmp(hw); + + return ccu_gate_helper_enable(&nkmp->common, nkmp->enable); +} + +static int ccu_nkmp_is_enabled(struct clk_hw *hw) +{ + struct ccu_nkmp *nkmp = hw_to_ccu_nkmp(hw); + + return ccu_gate_helper_is_enabled(&nkmp->common, nkmp->enable); +} + +static unsigned long ccu_nkmp_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct ccu_nkmp *nkmp = hw_to_ccu_nkmp(hw); + unsigned long n, m, k, p; + u32 reg; + + reg = readl(nkmp->common.base + nkmp->common.reg); + + n = reg >> nkmp->n.shift; + n &= (1 << nkmp->n.width) - 1; + + k = reg >> nkmp->k.shift; + k &= (1 << nkmp->k.width) - 1; + + m = reg >> nkmp->m.shift; + m &= (1 << nkmp->m.width) - 1; + + p = reg >> nkmp->p.shift; + p &= (1 << nkmp->p.width) - 1; + + return (parent_rate * (n + 1) * (k + 1) >> p) / (m + 1); +} + +static long ccu_nkmp_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + struct ccu_nkmp *nkmp = hw_to_ccu_nkmp(hw); + struct _ccu_nkmp _nkmp; + + _nkmp.max_n = 1 << nkmp->n.width; + _nkmp.max_k = 1 << nkmp->k.width; + _nkmp.max_m = 1 << nkmp->m.width; + _nkmp.max_p = (1 << nkmp->p.width) - 1; + + ccu_nkmp_find_best(*parent_rate, rate, + &_nkmp); + + return (*parent_rate * _nkmp.n * _nkmp.k >> _nkmp.p) / _nkmp.m; +} + +static int ccu_nkmp_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct ccu_nkmp *nkmp = hw_to_ccu_nkmp(hw); + struct _ccu_nkmp _nkmp; + unsigned long flags; + u32 reg; + + _nkmp.max_n = 1 << nkmp->n.width; + _nkmp.max_k = 1 << nkmp->k.width; + _nkmp.max_m = 1 << nkmp->m.width; + _nkmp.max_p = (1 << nkmp->p.width) - 1; + + ccu_nkmp_find_best(parent_rate, rate, &_nkmp); + + spin_lock_irqsave(nkmp->common.lock, flags); + + reg = readl(nkmp->common.base + nkmp->common.reg); + reg &= ~GENMASK(nkmp->n.width + nkmp->n.shift - 1, nkmp->n.shift); + reg &= ~GENMASK(nkmp->k.width + nkmp->k.shift - 1, nkmp->k.shift); + reg &= ~GENMASK(nkmp->m.width + nkmp->m.shift - 1, nkmp->m.shift); + reg &= ~GENMASK(nkmp->p.width + nkmp->p.shift - 1, nkmp->p.shift); + + reg |= (_nkmp.n - 1) << nkmp->n.shift; + reg |= (_nkmp.k - 1) << nkmp->k.shift; + reg |= (_nkmp.m - 1) << nkmp->m.shift; + reg |= _nkmp.p << nkmp->p.shift; + + writel(reg, nkmp->common.base + nkmp->common.reg); + + spin_unlock_irqrestore(nkmp->common.lock, flags); + + ccu_helper_wait_for_lock(&nkmp->common, nkmp->lock); + + return 0; +} + +const struct clk_ops ccu_nkmp_ops = { + .disable = ccu_nkmp_disable, + .enable = ccu_nkmp_enable, + .is_enabled = ccu_nkmp_is_enabled, + + .recalc_rate = ccu_nkmp_recalc_rate, + .round_rate = ccu_nkmp_round_rate, + .set_rate = ccu_nkmp_set_rate, +}; diff --git a/drivers/clk/sunxi-ng/ccu_nkmp.h b/drivers/clk/sunxi-ng/ccu_nkmp.h new file mode 100644 index 000000000..5adb0c92a --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_nkmp.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_NKMP_H_ +#define _CCU_NKMP_H_ + +#include + +#include "ccu_common.h" +#include "ccu_div.h" +#include "ccu_mult.h" + +/* + * struct ccu_nkmp - Definition of an N-K-M-P clock + * + * Clocks based on the formula parent * N * K >> P / M + */ +struct ccu_nkmp { + u32 enable; + u32 lock; + + struct _ccu_mult n; + struct _ccu_mult k; + struct _ccu_div m; + struct _ccu_div p; + + struct ccu_common common; +}; + +#define SUNXI_CCU_NKMP_WITH_GATE_LOCK(_struct, _name, _parent, _reg, \ + _nshift, _nwidth, \ + _kshift, _kwidth, \ + _mshift, _mwidth, \ + _pshift, _pwidth, \ + _gate, _lock, _flags) \ + struct ccu_nkmp _struct = { \ + .enable = _gate, \ + .lock = _lock, \ + .n = _SUNXI_CCU_MULT(_nshift, _nwidth), \ + .k = _SUNXI_CCU_MULT(_kshift, _kwidth), \ + .m = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .p = _SUNXI_CCU_DIV(_pshift, _pwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT(_name, \ + _parent, \ + &ccu_nkmp_ops, \ + _flags), \ + }, \ + } + +static inline struct ccu_nkmp *hw_to_ccu_nkmp(struct clk_hw *hw) +{ + struct ccu_common *common = hw_to_ccu_common(hw); + + return container_of(common, struct ccu_nkmp, common); +} + +extern const struct clk_ops ccu_nkmp_ops; + +#endif /* _CCU_NKMP_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_nm.c b/drivers/clk/sunxi-ng/ccu_nm.c new file mode 100644 index 000000000..e35ddd8ee --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_nm.c @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include +#include + +#include "ccu_frac.h" +#include "ccu_gate.h" +#include "ccu_nm.h" + +static void ccu_nm_disable(struct clk_hw *hw) +{ + struct ccu_nm *nm = hw_to_ccu_nm(hw); + + return ccu_gate_helper_disable(&nm->common, nm->enable); +} + +static int ccu_nm_enable(struct clk_hw *hw) +{ + struct ccu_nm *nm = hw_to_ccu_nm(hw); + + return ccu_gate_helper_enable(&nm->common, nm->enable); +} + +static int ccu_nm_is_enabled(struct clk_hw *hw) +{ + struct ccu_nm *nm = hw_to_ccu_nm(hw); + + return ccu_gate_helper_is_enabled(&nm->common, nm->enable); +} + +static unsigned long ccu_nm_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct ccu_nm *nm = hw_to_ccu_nm(hw); + unsigned long n, m; + u32 reg; + + if (ccu_frac_helper_is_enabled(&nm->common, &nm->frac)) + return ccu_frac_helper_read_rate(&nm->common, &nm->frac); + + reg = readl(nm->common.base + nm->common.reg); + + n = reg >> nm->n.shift; + n &= (1 << nm->n.width) - 1; + + m = reg >> nm->m.shift; + m &= (1 << nm->m.width) - 1; + + return parent_rate * (n + 1) / (m + 1); +} + +static long ccu_nm_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *parent_rate) +{ + struct ccu_nm *nm = hw_to_ccu_nm(hw); + unsigned long n, m; + + rational_best_approximation(rate, *parent_rate, + 1 << nm->n.width, 1 << nm->m.width, + &n, &m); + + return *parent_rate * n / m; +} + +static int ccu_nm_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct ccu_nm *nm = hw_to_ccu_nm(hw); + unsigned long flags; + unsigned long n, m; + u32 reg; + + if (ccu_frac_helper_has_rate(&nm->common, &nm->frac, rate)) + return ccu_frac_helper_set_rate(&nm->common, &nm->frac, rate); + else + ccu_frac_helper_disable(&nm->common, &nm->frac); + + rational_best_approximation(rate, parent_rate, + 1 << nm->n.width, 1 << nm->m.width, + &n, &m); + + spin_lock_irqsave(nm->common.lock, flags); + + reg = readl(nm->common.base + nm->common.reg); + reg &= ~GENMASK(nm->n.width + nm->n.shift - 1, nm->n.shift); + reg &= ~GENMASK(nm->m.width + nm->m.shift - 1, nm->m.shift); + + writel(reg | ((m - 1) << nm->m.shift) | ((n - 1) << nm->n.shift), + nm->common.base + nm->common.reg); + + spin_unlock_irqrestore(nm->common.lock, flags); + + ccu_helper_wait_for_lock(&nm->common, nm->lock); + + return 0; +} + +const struct clk_ops ccu_nm_ops = { + .disable = ccu_nm_disable, + .enable = ccu_nm_enable, + .is_enabled = ccu_nm_is_enabled, + + .recalc_rate = ccu_nm_recalc_rate, + .round_rate = ccu_nm_round_rate, + .set_rate = ccu_nm_set_rate, +}; diff --git a/drivers/clk/sunxi-ng/ccu_nm.h b/drivers/clk/sunxi-ng/ccu_nm.h new file mode 100644 index 000000000..0b7bcd33a --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_nm.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_NM_H_ +#define _CCU_NM_H_ + +#include + +#include "ccu_common.h" +#include "ccu_div.h" +#include "ccu_frac.h" +#include "ccu_mult.h" + +/* + * struct ccu_nm - Definition of an N-M clock + * + * Clocks based on the formula parent * N / M + */ +struct ccu_nm { + u32 enable; + u32 lock; + + struct _ccu_mult n; + struct _ccu_div m; + struct _ccu_frac frac; + + struct ccu_common common; +}; + +#define SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(_struct, _name, _parent, _reg, \ + _nshift, _nwidth, \ + _mshift, _mwidth, \ + _frac_en, _frac_sel, \ + _frac_rate_0, _frac_rate_1, \ + _gate, _lock, _flags) \ + struct ccu_nm _struct = { \ + .enable = _gate, \ + .lock = _lock, \ + .n = _SUNXI_CCU_MULT(_nshift, _nwidth), \ + .m = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .frac = _SUNXI_CCU_FRAC(_frac_en, _frac_sel, \ + _frac_rate_0, \ + _frac_rate_1), \ + .common = { \ + .reg = _reg, \ + .features = CCU_FEATURE_FRACTIONAL, \ + .hw.init = CLK_HW_INIT(_name, \ + _parent, \ + &ccu_nm_ops, \ + _flags), \ + }, \ + } + +#define SUNXI_CCU_NM_WITH_GATE_LOCK(_struct, _name, _parent, _reg, \ + _nshift, _nwidth, \ + _mshift, _mwidth, \ + _gate, _lock, _flags) \ + struct ccu_nm _struct = { \ + .enable = _gate, \ + .lock = _lock, \ + .n = _SUNXI_CCU_MULT(_nshift, _nwidth), \ + .m = _SUNXI_CCU_DIV(_mshift, _mwidth), \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT(_name, \ + _parent, \ + &ccu_nm_ops, \ + _flags), \ + }, \ + } + +static inline struct ccu_nm *hw_to_ccu_nm(struct clk_hw *hw) +{ + struct ccu_common *common = hw_to_ccu_common(hw); + + return container_of(common, struct ccu_nm, common); +} + +extern const struct clk_ops ccu_nm_ops; + +#endif /* _CCU_NM_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_phase.c b/drivers/clk/sunxi-ng/ccu_phase.c new file mode 100644 index 000000000..400c58ad7 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_phase.c @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include +#include + +#include "ccu_phase.h" + +static int ccu_phase_get_phase(struct clk_hw *hw) +{ + struct ccu_phase *phase = hw_to_ccu_phase(hw); + struct clk_hw *parent, *grandparent; + unsigned int parent_rate, grandparent_rate; + u16 step, parent_div; + u32 reg; + u8 delay; + + reg = readl(phase->common.base + phase->common.reg); + delay = (reg >> phase->shift); + delay &= (1 << phase->width) - 1; + + if (!delay) + return 180; + + /* Get our parent clock, it's the one that can adjust its rate */ + parent = clk_hw_get_parent(hw); + if (!parent) + return -EINVAL; + + /* And its rate */ + parent_rate = clk_hw_get_rate(parent); + if (!parent_rate) + return -EINVAL; + + /* Now, get our parent's parent (most likely some PLL) */ + grandparent = clk_hw_get_parent(parent); + if (!grandparent) + return -EINVAL; + + /* And its rate */ + grandparent_rate = clk_hw_get_rate(grandparent); + if (!grandparent_rate) + return -EINVAL; + + /* Get our parent clock divider */ + parent_div = grandparent_rate / parent_rate; + + step = DIV_ROUND_CLOSEST(360, parent_div); + return delay * step; +} + +static int ccu_phase_set_phase(struct clk_hw *hw, int degrees) +{ + struct ccu_phase *phase = hw_to_ccu_phase(hw); + struct clk_hw *parent, *grandparent; + unsigned int parent_rate, grandparent_rate; + unsigned long flags; + u32 reg; + u8 delay; + + /* Get our parent clock, it's the one that can adjust its rate */ + parent = clk_hw_get_parent(hw); + if (!parent) + return -EINVAL; + + /* And its rate */ + parent_rate = clk_hw_get_rate(parent); + if (!parent_rate) + return -EINVAL; + + /* Now, get our parent's parent (most likely some PLL) */ + grandparent = clk_hw_get_parent(parent); + if (!grandparent) + return -EINVAL; + + /* And its rate */ + grandparent_rate = clk_hw_get_rate(grandparent); + if (!grandparent_rate) + return -EINVAL; + + if (degrees != 180) { + u16 step, parent_div; + + /* Get our parent divider */ + parent_div = grandparent_rate / parent_rate; + + /* + * We can only outphase the clocks by multiple of the + * PLL's period. + * + * Since our parent clock is only a divider, and the + * formula to get the outphasing in degrees is deg = + * 360 * delta / period + * + * If we simplify this formula, we can see that the + * only thing that we're concerned about is the number + * of period we want to outphase our clock from, and + * the divider set by our parent clock. + */ + step = DIV_ROUND_CLOSEST(360, parent_div); + delay = DIV_ROUND_CLOSEST(degrees, step); + } else { + delay = 0; + } + + spin_lock_irqsave(phase->common.lock, flags); + reg = readl(phase->common.base + phase->common.reg); + reg &= ~GENMASK(phase->width + phase->shift - 1, phase->shift); + writel(reg | (delay << phase->shift), + phase->common.base + phase->common.reg); + spin_unlock_irqrestore(phase->common.lock, flags); + + return 0; +} + +const struct clk_ops ccu_phase_ops = { + .get_phase = ccu_phase_get_phase, + .set_phase = ccu_phase_set_phase, +}; diff --git a/drivers/clk/sunxi-ng/ccu_phase.h b/drivers/clk/sunxi-ng/ccu_phase.h new file mode 100644 index 000000000..75a091a4c --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_phase.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_PHASE_H_ +#define _CCU_PHASE_H_ + +#include + +#include "ccu_common.h" + +struct ccu_phase { + u8 shift; + u8 width; + + struct ccu_common common; +}; + +#define SUNXI_CCU_PHASE(_struct, _name, _parent, _reg, _shift, _width, _flags) \ + struct ccu_phase _struct = { \ + .shift = _shift, \ + .width = _width, \ + .common = { \ + .reg = _reg, \ + .hw.init = CLK_HW_INIT(_name, \ + _parent, \ + &ccu_phase_ops, \ + _flags), \ + } \ + } + +static inline struct ccu_phase *hw_to_ccu_phase(struct clk_hw *hw) +{ + struct ccu_common *common = hw_to_ccu_common(hw); + + return container_of(common, struct ccu_phase, common); +} + +extern const struct clk_ops ccu_phase_ops; + +#endif /* _CCU_PHASE_H_ */ diff --git a/drivers/clk/sunxi-ng/ccu_reset.c b/drivers/clk/sunxi-ng/ccu_reset.c new file mode 100644 index 000000000..6c31d4878 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_reset.c @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2016 Maxime Ripard + * Maxime Ripard + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +#include +#include + +#include "ccu_reset.h" + +static int ccu_reset_assert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct ccu_reset *ccu = rcdev_to_ccu_reset(rcdev); + const struct ccu_reset_map *map = &ccu->reset_map[id]; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(ccu->lock, flags); + + reg = readl(ccu->base + map->reg); + writel(reg & ~map->bit, ccu->base + map->reg); + + spin_unlock_irqrestore(ccu->lock, flags); + + return 0; +} + +static int ccu_reset_deassert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct ccu_reset *ccu = rcdev_to_ccu_reset(rcdev); + const struct ccu_reset_map *map = &ccu->reset_map[id]; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(ccu->lock, flags); + + reg = readl(ccu->base + map->reg); + writel(reg | map->bit, ccu->base + map->reg); + + spin_unlock_irqrestore(ccu->lock, flags); + + return 0; +} + +const struct reset_control_ops ccu_reset_ops = { + .assert = ccu_reset_assert, + .deassert = ccu_reset_deassert, +}; diff --git a/drivers/clk/sunxi-ng/ccu_reset.h b/drivers/clk/sunxi-ng/ccu_reset.h new file mode 100644 index 000000000..36a467921 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu_reset.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2016 Maxime Ripard. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CCU_RESET_H_ +#define _CCU_RESET_H_ + +#include + +struct ccu_reset_map { + u16 reg; + u32 bit; +}; + + +struct ccu_reset { + void __iomem *base; + struct ccu_reset_map *reset_map; + spinlock_t *lock; + + struct reset_controller_dev rcdev; +}; + +static inline struct ccu_reset *rcdev_to_ccu_reset(struct reset_controller_dev *rcdev) +{ + return container_of(rcdev, struct ccu_reset, rcdev); +} + +extern const struct reset_control_ops ccu_reset_ops; + +#endif /* _CCU_RESET_H_ */ diff --git a/drivers/clk/sunxi/clk-a10-pll2.c b/drivers/clk/sunxi/clk-a10-pll2.c index 0ee1f363e..d8eab90ae 100644 --- a/drivers/clk/sunxi/clk-a10-pll2.c +++ b/drivers/clk/sunxi/clk-a10-pll2.c @@ -73,7 +73,7 @@ static void __init sun4i_pll2_setup(struct device_node *node, SUN4I_PLL2_PRE_DIV_WIDTH, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &sun4i_a10_pll2_lock); - if (!prediv_clk) { + if (IS_ERR(prediv_clk)) { pr_err("Couldn't register the prediv clock\n"); goto err_free_array; } @@ -106,7 +106,7 @@ static void __init sun4i_pll2_setup(struct device_node *node, &mult->hw, &clk_multiplier_ops, &gate->hw, &clk_gate_ops, CLK_SET_RATE_PARENT); - if (!base_clk) { + if (IS_ERR(base_clk)) { pr_err("Couldn't register the base multiplier clock\n"); goto err_free_multiplier; } diff --git a/drivers/clk/sunxi/clk-factors.c b/drivers/clk/sunxi/clk-factors.c index ddefe9668..dfe5e3e32 100644 --- a/drivers/clk/sunxi/clk-factors.c +++ b/drivers/clk/sunxi/clk-factors.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/clk/sunxi/clk-sun6i-apb0-gates.c b/drivers/clk/sunxi/clk-sun6i-apb0-gates.c index 68021fa5e..09cdb9874 100644 --- a/drivers/clk/sunxi/clk-sun6i-apb0-gates.c +++ b/drivers/clk/sunxi/clk-sun6i-apb0-gates.c @@ -9,7 +9,7 @@ */ #include -#include +#include #include #include #include @@ -33,7 +33,6 @@ static const struct of_device_id sun6i_a31_apb0_gates_clk_dt_ids[] = { { .compatible = "allwinner,sun8i-a23-apb0-gates-clk", .data = &sun8i_a23_apb0_gates }, { /* sentinel */ } }; -MODULE_DEVICE_TABLE(of, sun6i_a31_apb0_gates_clk_dt_ids); static int sun6i_a31_apb0_gates_clk_probe(struct platform_device *pdev) { @@ -102,8 +101,4 @@ static struct platform_driver sun6i_a31_apb0_gates_clk_driver = { }, .probe = sun6i_a31_apb0_gates_clk_probe, }; -module_platform_driver(sun6i_a31_apb0_gates_clk_driver); - -MODULE_AUTHOR("Boris BREZILLON "); -MODULE_DESCRIPTION("Allwinner A31 APB0 gate clocks driver"); -MODULE_LICENSE("GPL v2"); +builtin_platform_driver(sun6i_a31_apb0_gates_clk_driver); diff --git a/drivers/clk/sunxi/clk-sun6i-apb0.c b/drivers/clk/sunxi/clk-sun6i-apb0.c index e703e1895..b9c8d3592 100644 --- a/drivers/clk/sunxi/clk-sun6i-apb0.c +++ b/drivers/clk/sunxi/clk-sun6i-apb0.c @@ -9,7 +9,7 @@ */ #include -#include +#include #include #include @@ -61,7 +61,6 @@ static const struct of_device_id sun6i_a31_apb0_clk_dt_ids[] = { { .compatible = "allwinner,sun6i-a31-apb0-clk" }, { /* sentinel */ } }; -MODULE_DEVICE_TABLE(of, sun6i_a31_apb0_clk_dt_ids); static struct platform_driver sun6i_a31_apb0_clk_driver = { .driver = { @@ -70,8 +69,4 @@ static struct platform_driver sun6i_a31_apb0_clk_driver = { }, .probe = sun6i_a31_apb0_clk_probe, }; -module_platform_driver(sun6i_a31_apb0_clk_driver); - -MODULE_AUTHOR("Boris BREZILLON "); -MODULE_DESCRIPTION("Allwinner A31 APB0 clock Driver"); -MODULE_LICENSE("GPL v2"); +builtin_platform_driver(sun6i_a31_apb0_clk_driver); diff --git a/drivers/clk/sunxi/clk-sun6i-ar100.c b/drivers/clk/sunxi/clk-sun6i-ar100.c index 84a187e55..64ca3e9e3 100644 --- a/drivers/clk/sunxi/clk-sun6i-ar100.c +++ b/drivers/clk/sunxi/clk-sun6i-ar100.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include @@ -91,32 +91,17 @@ static int sun6i_a31_ar100_clk_probe(struct platform_device *pdev) return 0; } -static int sun6i_a31_ar100_clk_remove(struct platform_device *pdev) -{ - struct device_node *np = pdev->dev.of_node; - struct clk *clk = platform_get_drvdata(pdev); - - sunxi_factors_unregister(np, clk); - - return 0; -} - static const struct of_device_id sun6i_a31_ar100_clk_dt_ids[] = { { .compatible = "allwinner,sun6i-a31-ar100-clk" }, { /* sentinel */ } }; -MODULE_DEVICE_TABLE(of, sun6i_a31_ar100_clk_dt_ids); static struct platform_driver sun6i_a31_ar100_clk_driver = { .driver = { .name = "sun6i-a31-ar100-clk", .of_match_table = sun6i_a31_ar100_clk_dt_ids, + .suppress_bind_attrs = true, }, .probe = sun6i_a31_ar100_clk_probe, - .remove = sun6i_a31_ar100_clk_remove, }; -module_platform_driver(sun6i_a31_ar100_clk_driver); - -MODULE_AUTHOR("Boris BREZILLON "); -MODULE_DESCRIPTION("Allwinner A31 AR100 clock Driver"); -MODULE_LICENSE("GPL v2"); +builtin_platform_driver(sun6i_a31_ar100_clk_driver); diff --git a/drivers/clk/sunxi/clk-sun8i-apb0.c b/drivers/clk/sunxi/clk-sun8i-apb0.c index 2ea61debf..a5666e1d0 100644 --- a/drivers/clk/sunxi/clk-sun8i-apb0.c +++ b/drivers/clk/sunxi/clk-sun8i-apb0.c @@ -15,7 +15,7 @@ */ #include -#include +#include #include #include #include @@ -108,7 +108,6 @@ static const struct of_device_id sun8i_a23_apb0_clk_dt_ids[] = { { .compatible = "allwinner,sun8i-a23-apb0-clk" }, { /* sentinel */ } }; -MODULE_DEVICE_TABLE(of, sun8i_a23_apb0_clk_dt_ids); static struct platform_driver sun8i_a23_apb0_clk_driver = { .driver = { @@ -117,8 +116,4 @@ static struct platform_driver sun8i_a23_apb0_clk_driver = { }, .probe = sun8i_a23_apb0_clk_probe, }; -module_platform_driver(sun8i_a23_apb0_clk_driver); - -MODULE_AUTHOR("Chen-Yu Tsai "); -MODULE_DESCRIPTION("Allwinner A23 APB0 clock Driver"); -MODULE_LICENSE("GPL v2"); +builtin_platform_driver(sun8i_a23_apb0_clk_driver); diff --git a/drivers/clk/sunxi/clk-sun8i-mbus.c b/drivers/clk/sunxi/clk-sun8i-mbus.c index 411d3033a..b200ebf15 100644 --- a/drivers/clk/sunxi/clk-sun8i-mbus.c +++ b/drivers/clk/sunxi/clk-sun8i-mbus.c @@ -48,7 +48,7 @@ static void __init sun8i_a23_mbus_setup(struct device_node *node) return; reg = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (!reg) { + if (IS_ERR(reg)) { pr_err("Could not get registers for sun8i-mbus-clk\n"); goto err_free_parents; } diff --git a/drivers/clk/sunxi/clk-sun9i-mmc.c b/drivers/clk/sunxi/clk-sun9i-mmc.c index 716737388..6041bdba2 100644 --- a/drivers/clk/sunxi/clk-sun9i-mmc.c +++ b/drivers/clk/sunxi/clk-sun9i-mmc.c @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include #include @@ -183,39 +183,17 @@ err_clk_register: return ret; } -static int sun9i_a80_mmc_config_clk_remove(struct platform_device *pdev) -{ - struct device_node *np = pdev->dev.of_node; - struct sun9i_mmc_clk_data *data = platform_get_drvdata(pdev); - struct clk_onecell_data *clk_data = &data->clk_data; - int i; - - reset_controller_unregister(&data->rcdev); - of_clk_del_provider(np); - for (i = 0; i < clk_data->clk_num; i++) - clk_unregister(clk_data->clks[i]); - - reset_control_assert(data->reset); - - return 0; -} - static const struct of_device_id sun9i_a80_mmc_config_clk_dt_ids[] = { { .compatible = "allwinner,sun9i-a80-mmc-config-clk" }, { /* sentinel */ } }; -MODULE_DEVICE_TABLE(of, sun9i_a80_mmc_config_clk_dt_ids); static struct platform_driver sun9i_a80_mmc_config_clk_driver = { .driver = { .name = "sun9i-a80-mmc-config-clk", + .suppress_bind_attrs = true, .of_match_table = sun9i_a80_mmc_config_clk_dt_ids, }, .probe = sun9i_a80_mmc_config_clk_probe, - .remove = sun9i_a80_mmc_config_clk_remove, }; -module_platform_driver(sun9i_a80_mmc_config_clk_driver); - -MODULE_AUTHOR("Chen-Yu Tsai "); -MODULE_DESCRIPTION("Allwinner A80 MMC clock/reset Driver"); -MODULE_LICENSE("GPL v2"); +builtin_platform_driver(sun9i_a80_mmc_config_clk_driver); diff --git a/drivers/clk/tegra/clk-id.h b/drivers/clk/tegra/clk-id.h index 36c974916..5738635c5 100644 --- a/drivers/clk/tegra/clk-id.h +++ b/drivers/clk/tegra/clk-id.h @@ -238,7 +238,6 @@ enum clk_id { tegra_clk_sor0, tegra_clk_sor0_lvds, tegra_clk_sor1, - tegra_clk_sor1_brick, tegra_clk_sor1_src, tegra_clk_spdif, tegra_clk_spdif_2x, diff --git a/drivers/clk/tegra/clk-pll.c b/drivers/clk/tegra/clk-pll.c index 4e194ecc8..b3855360d 100644 --- a/drivers/clk/tegra/clk-pll.c +++ b/drivers/clk/tegra/clk-pll.c @@ -191,6 +191,53 @@ #define PLLSS_REF_SRC_SEL_SHIFT 25 #define PLLSS_REF_SRC_SEL_MASK (3 << PLLSS_REF_SRC_SEL_SHIFT) +#define UTMIP_PLL_CFG1 0x484 +#define UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(x) (((x) & 0xfff) << 0) +#define UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(x) (((x) & 0x1f) << 27) +#define UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN BIT(12) +#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN BIT(14) +#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP BIT(15) +#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN BIT(16) +#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERUP BIT(17) + +#define UTMIP_PLL_CFG2 0x488 +#define UTMIP_PLL_CFG2_STABLE_COUNT(x) (((x) & 0xfff) << 6) +#define UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(x) (((x) & 0x3f) << 18) +#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN BIT(0) +#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERUP BIT(1) +#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN BIT(2) +#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERUP BIT(3) +#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN BIT(4) +#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERUP BIT(5) +#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERDOWN BIT(24) +#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERUP BIT(25) +#define UTMIP_PLL_CFG2_PHY_XTAL_CLOCKEN BIT(30) + +#define UTMIPLL_HW_PWRDN_CFG0 0x52c +#define UTMIPLL_HW_PWRDN_CFG0_IDDQ_SWCTL BIT(0) +#define UTMIPLL_HW_PWRDN_CFG0_IDDQ_OVERRIDE BIT(1) +#define UTMIPLL_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL BIT(2) +#define UTMIPLL_HW_PWRDN_CFG0_SEQ_IN_SWCTL BIT(4) +#define UTMIPLL_HW_PWRDN_CFG0_SEQ_RESET_INPUT_VALUE BIT(5) +#define UTMIPLL_HW_PWRDN_CFG0_USE_LOCKDET BIT(6) +#define UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE BIT(24) +#define UTMIPLL_HW_PWRDN_CFG0_SEQ_START_STATE BIT(25) + +#define PLLU_HW_PWRDN_CFG0 0x530 +#define PLLU_HW_PWRDN_CFG0_CLK_SWITCH_SWCTL BIT(0) +#define PLLU_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL BIT(2) +#define PLLU_HW_PWRDN_CFG0_USE_LOCKDET BIT(6) +#define PLLU_HW_PWRDN_CFG0_USE_SWITCH_DETECT BIT(7) +#define PLLU_HW_PWRDN_CFG0_SEQ_ENABLE BIT(24) +#define PLLU_HW_PWRDN_CFG0_IDDQ_PD_INCLUDE BIT(28) + +#define XUSB_PLL_CFG0 0x534 +#define XUSB_PLL_CFG0_UTMIPLL_LOCK_DLY 0x3ff +#define XUSB_PLL_CFG0_PLLU_LOCK_DLY (0x3ff << 14) + +#define PLLU_BASE_CLKENABLE_USB BIT(21) +#define PLLU_BASE_OVERRIDE BIT(24) + #define pll_readl(offset, p) readl_relaxed(p->clk_base + offset) #define pll_readl_base(p) pll_readl(p->params->base_reg, p) #define pll_readl_misc(p) pll_readl(p->params->misc_reg, p) @@ -973,6 +1020,133 @@ const struct clk_ops tegra_clk_plle_ops = { .enable = clk_plle_enable, }; +/* + * Structure defining the fields for USB UTMI clocks Parameters. + */ +struct utmi_clk_param { + /* Oscillator Frequency in Hz */ + u32 osc_frequency; + /* UTMIP PLL Enable Delay Count */ + u8 enable_delay_count; + /* UTMIP PLL Stable count */ + u8 stable_count; + /* UTMIP PLL Active delay count */ + u8 active_delay_count; + /* UTMIP PLL Xtal frequency count */ + u8 xtal_freq_count; +}; + +static const struct utmi_clk_param utmi_parameters[] = { + { + .osc_frequency = 13000000, .enable_delay_count = 0x02, + .stable_count = 0x33, .active_delay_count = 0x05, + .xtal_freq_count = 0x7f + }, { + .osc_frequency = 19200000, .enable_delay_count = 0x03, + .stable_count = 0x4b, .active_delay_count = 0x06, + .xtal_freq_count = 0xbb + }, { + .osc_frequency = 12000000, .enable_delay_count = 0x02, + .stable_count = 0x2f, .active_delay_count = 0x04, + .xtal_freq_count = 0x76 + }, { + .osc_frequency = 26000000, .enable_delay_count = 0x04, + .stable_count = 0x66, .active_delay_count = 0x09, + .xtal_freq_count = 0xfe + }, { + .osc_frequency = 16800000, .enable_delay_count = 0x03, + .stable_count = 0x41, .active_delay_count = 0x0a, + .xtal_freq_count = 0xa4 + }, { + .osc_frequency = 38400000, .enable_delay_count = 0x0, + .stable_count = 0x0, .active_delay_count = 0x6, + .xtal_freq_count = 0x80 + }, +}; + +static int clk_pllu_enable(struct clk_hw *hw) +{ + struct tegra_clk_pll *pll = to_clk_pll(hw); + struct clk_hw *pll_ref = clk_hw_get_parent(hw); + struct clk_hw *osc = clk_hw_get_parent(pll_ref); + const struct utmi_clk_param *params = NULL; + unsigned long flags = 0, input_rate; + unsigned int i; + int ret = 0; + u32 value; + + if (!osc) { + pr_err("%s: failed to get OSC clock\n", __func__); + return -EINVAL; + } + + input_rate = clk_hw_get_rate(osc); + + if (pll->lock) + spin_lock_irqsave(pll->lock, flags); + + _clk_pll_enable(hw); + + ret = clk_pll_wait_for_lock(pll); + if (ret < 0) + goto out; + + for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) { + if (input_rate == utmi_parameters[i].osc_frequency) { + params = &utmi_parameters[i]; + break; + } + } + + if (!params) { + pr_err("%s: unexpected input rate %lu Hz\n", __func__, + input_rate); + ret = -EINVAL; + goto out; + } + + value = pll_readl_base(pll); + value &= ~PLLU_BASE_OVERRIDE; + pll_writel_base(value, pll); + + value = readl_relaxed(pll->clk_base + UTMIP_PLL_CFG2); + /* Program UTMIP PLL stable and active counts */ + value &= ~UTMIP_PLL_CFG2_STABLE_COUNT(~0); + value |= UTMIP_PLL_CFG2_STABLE_COUNT(params->stable_count); + value &= ~UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(~0); + value |= UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(params->active_delay_count); + /* Remove power downs from UTMIP PLL control bits */ + value &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN; + value &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN; + value &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN; + writel_relaxed(value, pll->clk_base + UTMIP_PLL_CFG2); + + value = readl_relaxed(pll->clk_base + UTMIP_PLL_CFG1); + /* Program UTMIP PLL delay and oscillator frequency counts */ + value &= ~UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(~0); + value |= UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(params->enable_delay_count); + value &= ~UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(~0); + value |= UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(params->xtal_freq_count); + /* Remove power downs from UTMIP PLL control bits */ + value &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; + value &= ~UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN; + value &= ~UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN; + writel_relaxed(value, pll->clk_base + UTMIP_PLL_CFG1); + +out: + if (pll->lock) + spin_unlock_irqrestore(pll->lock, flags); + + return ret; +} + +static const struct clk_ops tegra_clk_pllu_ops = { + .is_enabled = clk_pll_is_enabled, + .enable = clk_pllu_enable, + .disable = clk_pll_disable, + .recalc_rate = clk_pll_recalc_rate, +}; + static int _pll_fixed_mdiv(struct tegra_clk_pll_params *pll_params, unsigned long parent_rate) { @@ -1505,6 +1679,112 @@ static void clk_plle_tegra114_disable(struct clk_hw *hw) if (pll->lock) spin_unlock_irqrestore(pll->lock, flags); } + +static int clk_pllu_tegra114_enable(struct clk_hw *hw) +{ + struct tegra_clk_pll *pll = to_clk_pll(hw); + const struct utmi_clk_param *params = NULL; + struct clk *osc = __clk_lookup("osc"); + unsigned long flags = 0, input_rate; + unsigned int i; + int ret = 0; + u32 value; + + if (!osc) { + pr_err("%s: failed to get OSC clock\n", __func__); + return -EINVAL; + } + + input_rate = clk_hw_get_rate(__clk_get_hw(osc)); + + if (pll->lock) + spin_lock_irqsave(pll->lock, flags); + + _clk_pll_enable(hw); + + ret = clk_pll_wait_for_lock(pll); + if (ret < 0) + goto out; + + for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) { + if (input_rate == utmi_parameters[i].osc_frequency) { + params = &utmi_parameters[i]; + break; + } + } + + if (!params) { + pr_err("%s: unexpected input rate %lu Hz\n", __func__, + input_rate); + ret = -EINVAL; + goto out; + } + + value = pll_readl_base(pll); + value &= ~PLLU_BASE_OVERRIDE; + pll_writel_base(value, pll); + + value = readl_relaxed(pll->clk_base + UTMIP_PLL_CFG2); + /* Program UTMIP PLL stable and active counts */ + value &= ~UTMIP_PLL_CFG2_STABLE_COUNT(~0); + value |= UTMIP_PLL_CFG2_STABLE_COUNT(params->stable_count); + value &= ~UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(~0); + value |= UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(params->active_delay_count); + /* Remove power downs from UTMIP PLL control bits */ + value &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN; + value &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN; + value &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN; + writel_relaxed(value, pll->clk_base + UTMIP_PLL_CFG2); + + value = readl_relaxed(pll->clk_base + UTMIP_PLL_CFG1); + /* Program UTMIP PLL delay and oscillator frequency counts */ + value &= ~UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(~0); + value |= UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(params->enable_delay_count); + value &= ~UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(~0); + value |= UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(params->xtal_freq_count); + /* Remove power downs from UTMIP PLL control bits */ + value &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; + value &= ~UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN; + value &= ~UTMIP_PLL_CFG1_FORCE_PLLU_POWERUP; + value &= ~UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN; + writel_relaxed(value, pll->clk_base + UTMIP_PLL_CFG1); + + /* Setup HW control of UTMIPLL */ + value = readl_relaxed(pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + value |= UTMIPLL_HW_PWRDN_CFG0_USE_LOCKDET; + value &= ~UTMIPLL_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL; + value |= UTMIPLL_HW_PWRDN_CFG0_SEQ_START_STATE; + writel_relaxed(value, pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + + value = readl_relaxed(pll->clk_base + UTMIP_PLL_CFG1); + value &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP; + value &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; + writel_relaxed(value, pll->clk_base + UTMIP_PLL_CFG1); + + udelay(1); + + /* + * Setup SW override of UTMIPLL assuming USB2.0 ports are assigned + * to USB2 + */ + value = readl_relaxed(pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + value |= UTMIPLL_HW_PWRDN_CFG0_IDDQ_SWCTL; + value &= ~UTMIPLL_HW_PWRDN_CFG0_IDDQ_OVERRIDE; + writel_relaxed(value, pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + + udelay(1); + + /* Enable HW control of UTMIPLL */ + value = readl_relaxed(pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + value |= UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE; + writel_relaxed(value, pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + +out: + if (pll->lock) + spin_unlock_irqrestore(pll->lock, flags); + + return ret; +} #endif static struct tegra_clk_pll *_tegra_init_pll(void __iomem *clk_base, @@ -1614,6 +1894,27 @@ struct clk *tegra_clk_register_plle(const char *name, const char *parent_name, return clk; } +struct clk *tegra_clk_register_pllu(const char *name, const char *parent_name, + void __iomem *clk_base, unsigned long flags, + struct tegra_clk_pll_params *pll_params, spinlock_t *lock) +{ + struct tegra_clk_pll *pll; + struct clk *clk; + + pll_params->flags |= TEGRA_PLLU; + + pll = _tegra_init_pll(clk_base, NULL, pll_params, lock); + if (IS_ERR(pll)) + return ERR_CAST(pll); + + clk = _tegra_clk_register_pll(pll, name, parent_name, flags, + &tegra_clk_pllu_ops); + if (IS_ERR(clk)) + kfree(pll); + + return clk; +} + #if defined(CONFIG_ARCH_TEGRA_114_SOC) || \ defined(CONFIG_ARCH_TEGRA_124_SOC) || \ defined(CONFIG_ARCH_TEGRA_132_SOC) || \ @@ -1652,6 +1953,12 @@ static const struct clk_ops tegra_clk_plle_tegra114_ops = { .recalc_rate = clk_pll_recalc_rate, }; +static const struct clk_ops tegra_clk_pllu_tegra114_ops = { + .is_enabled = clk_pll_is_enabled, + .enable = clk_pllu_tegra114_enable, + .disable = clk_pll_disable, + .recalc_rate = clk_pll_recalc_rate, +}; struct clk *tegra_clk_register_pllxc(const char *name, const char *parent_name, void __iomem *clk_base, void __iomem *pmc, @@ -1919,6 +2226,29 @@ struct clk *tegra_clk_register_plle_tegra114(const char *name, return clk; } + +struct clk * +tegra_clk_register_pllu_tegra114(const char *name, const char *parent_name, + void __iomem *clk_base, unsigned long flags, + struct tegra_clk_pll_params *pll_params, + spinlock_t *lock) +{ + struct tegra_clk_pll *pll; + struct clk *clk; + + pll_params->flags |= TEGRA_PLLU; + + pll = _tegra_init_pll(clk_base, NULL, pll_params, lock); + if (IS_ERR(pll)) + return ERR_CAST(pll); + + clk = _tegra_clk_register_pll(pll, name, parent_name, flags, + &tegra_clk_pllu_tegra114_ops); + if (IS_ERR(clk)) + kfree(pll); + + return clk; +} #endif #if defined(CONFIG_ARCH_TEGRA_124_SOC) || defined(CONFIG_ARCH_TEGRA_132_SOC) @@ -2187,6 +2517,152 @@ static int clk_plle_tegra210_is_enabled(struct clk_hw *hw) return val & PLLE_BASE_ENABLE ? 1 : 0; } +static int clk_pllu_tegra210_enable(struct clk_hw *hw) +{ + struct tegra_clk_pll *pll = to_clk_pll(hw); + struct clk_hw *pll_ref = clk_hw_get_parent(hw); + struct clk_hw *osc = clk_hw_get_parent(pll_ref); + const struct utmi_clk_param *params = NULL; + unsigned long flags = 0, input_rate; + unsigned int i; + int ret = 0; + u32 value; + + if (!osc) { + pr_err("%s: failed to get OSC clock\n", __func__); + return -EINVAL; + } + + input_rate = clk_hw_get_rate(osc); + + if (pll->lock) + spin_lock_irqsave(pll->lock, flags); + + _clk_pll_enable(hw); + + ret = clk_pll_wait_for_lock(pll); + if (ret < 0) + goto out; + + for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) { + if (input_rate == utmi_parameters[i].osc_frequency) { + params = &utmi_parameters[i]; + break; + } + } + + if (!params) { + pr_err("%s: unexpected input rate %lu Hz\n", __func__, + input_rate); + ret = -EINVAL; + goto out; + } + + value = pll_readl_base(pll); + value &= ~PLLU_BASE_OVERRIDE; + pll_writel_base(value, pll); + + /* Put PLLU under HW control */ + value = readl_relaxed(pll->clk_base + PLLU_HW_PWRDN_CFG0); + value |= PLLU_HW_PWRDN_CFG0_IDDQ_PD_INCLUDE | + PLLU_HW_PWRDN_CFG0_USE_SWITCH_DETECT | + PLLU_HW_PWRDN_CFG0_USE_LOCKDET; + value &= ~(PLLU_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL | + PLLU_HW_PWRDN_CFG0_CLK_SWITCH_SWCTL); + writel_relaxed(value, pll->clk_base + PLLU_HW_PWRDN_CFG0); + + value = readl_relaxed(pll->clk_base + XUSB_PLL_CFG0); + value &= ~XUSB_PLL_CFG0_PLLU_LOCK_DLY; + writel_relaxed(value, pll->clk_base + XUSB_PLL_CFG0); + + udelay(1); + + value = readl_relaxed(pll->clk_base + PLLU_HW_PWRDN_CFG0); + value |= PLLU_HW_PWRDN_CFG0_SEQ_ENABLE; + writel_relaxed(value, pll->clk_base + PLLU_HW_PWRDN_CFG0); + + udelay(1); + + /* Disable PLLU clock branch to UTMIPLL since it uses OSC */ + value = pll_readl_base(pll); + value &= ~PLLU_BASE_CLKENABLE_USB; + pll_writel_base(value, pll); + + value = readl_relaxed(pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + if (value & UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE) { + pr_debug("UTMIPLL already enabled\n"); + goto out; + } + + value &= ~UTMIPLL_HW_PWRDN_CFG0_IDDQ_OVERRIDE; + writel_relaxed(value, pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + + /* Program UTMIP PLL stable and active counts */ + value = readl_relaxed(pll->clk_base + UTMIP_PLL_CFG2); + value &= ~UTMIP_PLL_CFG2_STABLE_COUNT(~0); + value |= UTMIP_PLL_CFG2_STABLE_COUNT(params->stable_count); + value &= ~UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(~0); + value |= UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(params->active_delay_count); + value |= UTMIP_PLL_CFG2_PHY_XTAL_CLOCKEN; + writel_relaxed(value, pll->clk_base + UTMIP_PLL_CFG2); + + /* Program UTMIP PLL delay and oscillator frequency counts */ + value = readl_relaxed(pll->clk_base + UTMIP_PLL_CFG1); + value &= ~UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(~0); + value |= UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(params->enable_delay_count); + value &= ~UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(~0); + value |= UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(params->xtal_freq_count); + writel_relaxed(value, pll->clk_base + UTMIP_PLL_CFG1); + + /* Remove power downs from UTMIP PLL control bits */ + value = readl_relaxed(pll->clk_base + UTMIP_PLL_CFG1); + value &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; + value |= UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP; + writel(value, pll->clk_base + UTMIP_PLL_CFG1); + + udelay(1); + + /* Enable samplers for SNPS, XUSB_HOST, XUSB_DEV */ + value = readl_relaxed(pll->clk_base + UTMIP_PLL_CFG2); + value |= UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERUP; + value |= UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERUP; + value |= UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERUP; + value &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN; + value &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN; + value &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERDOWN; + writel_relaxed(value, pll->clk_base + UTMIP_PLL_CFG2); + + /* Setup HW control of UTMIPLL */ + value = readl_relaxed(pll->clk_base + UTMIP_PLL_CFG1); + value &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP; + value &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; + writel_relaxed(value, pll->clk_base + UTMIP_PLL_CFG1); + + value = readl_relaxed(pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + value |= UTMIPLL_HW_PWRDN_CFG0_USE_LOCKDET; + value &= ~UTMIPLL_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL; + writel_relaxed(value, pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + + udelay(1); + + value = readl_relaxed(pll->clk_base + XUSB_PLL_CFG0); + value &= ~XUSB_PLL_CFG0_UTMIPLL_LOCK_DLY; + writel_relaxed(value, pll->clk_base + XUSB_PLL_CFG0); + + udelay(1); + + /* Enable HW control of UTMIPLL */ + value = readl_relaxed(pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + value |= UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE; + writel_relaxed(value, pll->clk_base + UTMIPLL_HW_PWRDN_CFG0); + +out: + if (pll->lock) + spin_unlock_irqrestore(pll->lock, flags); + + return ret; +} + static const struct clk_ops tegra_clk_plle_tegra210_ops = { .is_enabled = clk_plle_tegra210_is_enabled, .enable = clk_plle_tegra210_enable, @@ -2194,6 +2670,13 @@ static const struct clk_ops tegra_clk_plle_tegra210_ops = { .recalc_rate = clk_pll_recalc_rate, }; +static const struct clk_ops tegra_clk_pllu_tegra210_ops = { + .is_enabled = clk_pll_is_enabled, + .enable = clk_pllu_tegra210_enable, + .disable = clk_pll_disable, + .recalc_rate = clk_pllre_recalc_rate, +}; + struct clk *tegra_clk_register_plle_tegra210(const char *name, const char *parent_name, void __iomem *clk_base, unsigned long flags, @@ -2434,4 +2917,26 @@ struct clk *tegra_clk_register_pllmb(const char *name, const char *parent_name, return clk; } + +struct clk *tegra_clk_register_pllu_tegra210(const char *name, + const char *parent_name, void __iomem *clk_base, + unsigned long flags, struct tegra_clk_pll_params *pll_params, + spinlock_t *lock) +{ + struct tegra_clk_pll *pll; + struct clk *clk; + + pll_params->flags |= TEGRA_PLLU; + + pll = _tegra_init_pll(clk_base, NULL, pll_params, lock); + if (IS_ERR(pll)) + return ERR_CAST(pll); + + clk = _tegra_clk_register_pll(pll, name, parent_name, flags, + &tegra_clk_pllu_tegra210_ops); + if (IS_ERR(clk)) + kfree(pll); + + return clk; +} #endif diff --git a/drivers/clk/tegra/clk-tegra-periph.c b/drivers/clk/tegra/clk-tegra-periph.c index 29d04c663..4ce4e7fb1 100644 --- a/drivers/clk/tegra/clk-tegra-periph.c +++ b/drivers/clk/tegra/clk-tegra-periph.c @@ -594,15 +594,17 @@ static u32 mux_pllp_plld_plld2_clkm_idx[] = { [0] = 0, [1] = 2, [2] = 5, [3] = 6 }; -static const char *mux_plldp_sor1_src[] = { - "pll_dp", "clk_sor1_src" -}; -#define mux_plldp_sor1_src_idx NULL - -static const char *mux_clkm_sor1_brick_sor1_src[] = { - "clk_m", "sor1_brick", "sor1_src", "sor1_brick" -}; -#define mux_clkm_sor1_brick_sor1_src_idx NULL +static const char *mux_sor_safe_sor1_brick_sor1_src[] = { + /* + * Bit 0 of the mux selects sor1_brick, irrespective of bit 1, so the + * sor1_brick parent appears twice in the list below. This is merely + * to support clk_get_parent() if firmware happened to set these bits + * to 0b11. While not an invalid setting, code should always set the + * bits to 0b01 to select sor1_brick. + */ + "sor_safe", "sor1_brick", "sor1_src", "sor1_brick" +}; +#define mux_sor_safe_sor1_brick_sor1_src_idx NULL static const char *mux_pllp_pllre_clkm[] = { "pll_p", "pll_re_out1", "clk_m" @@ -778,8 +780,7 @@ static struct tegra_periph_init_data periph_clks[] = { MUX8("nvjpg", mux_pllc2_c_c3_pllp_plla1_clkm, CLK_SOURCE_NVJPG, 195, 0, tegra_clk_nvjpg), MUX8("ape", mux_plla_pllc4_out0_pllc_pllc4_out1_pllp_pllc4_out2_clkm, CLK_SOURCE_APE, 198, TEGRA_PERIPH_ON_APB, tegra_clk_ape), MUX8_NOGATE_LOCK("sor1_src", mux_pllp_plld_plld2_clkm, CLK_SOURCE_SOR1, tegra_clk_sor1_src, &sor1_lock), - NODIV("sor1_brick", mux_plldp_sor1_src, CLK_SOURCE_SOR1, 14, MASK(1), 183, 0, tegra_clk_sor1_brick, &sor1_lock), - NODIV("sor1", mux_clkm_sor1_brick_sor1_src, CLK_SOURCE_SOR1, 15, MASK(1), 183, 0, tegra_clk_sor1, &sor1_lock), + NODIV("sor1", mux_sor_safe_sor1_brick_sor1_src, CLK_SOURCE_SOR1, 14, MASK(2), 183, 0, tegra_clk_sor1, &sor1_lock), MUX8("sdmmc_legacy", mux_pllp_out3_clkm_pllp_pllc4, CLK_SOURCE_SDMMC_LEGACY, 193, TEGRA_PERIPH_ON_APB | TEGRA_PERIPH_NO_RESET, tegra_clk_sdmmc_legacy), MUX8("qspi", mux_pllp_pllc_pllc_out1_pllc4_out2_pllc4_out1_clkm_pllc4_out0, CLK_SOURCE_QSPI, 211, TEGRA_PERIPH_ON_APB, tegra_clk_qspi), I2C("vii2c", mux_pllp_pllc_clkm, CLK_SOURCE_VI_I2C, 208, tegra_clk_vi_i2c), @@ -791,7 +792,7 @@ static struct tegra_periph_init_data periph_clks[] = { static struct tegra_periph_init_data gate_clks[] = { GATE("rtc", "clk_32k", 4, TEGRA_PERIPH_ON_APB | TEGRA_PERIPH_NO_RESET, tegra_clk_rtc, 0), - GATE("timer", "clk_m", 5, 0, tegra_clk_timer, 0), + GATE("timer", "clk_m", 5, 0, tegra_clk_timer, CLK_IS_CRITICAL), GATE("isp", "clk_m", 23, 0, tegra_clk_isp, 0), GATE("vcp", "clk_m", 29, 0, tegra_clk_vcp, 0), GATE("apbdma", "clk_m", 34, 0, tegra_clk_apbdma, 0), diff --git a/drivers/clk/tegra/clk-tegra114.c b/drivers/clk/tegra/clk-tegra114.c index b78054fac..933b5dd69 100644 --- a/drivers/clk/tegra/clk-tegra114.c +++ b/drivers/clk/tegra/clk-tegra114.c @@ -113,32 +113,6 @@ #define CCLKG_BURST_POLICY 0x368 -#define UTMIP_PLL_CFG2 0x488 -#define UTMIP_PLL_CFG2_STABLE_COUNT(x) (((x) & 0xffff) << 6) -#define UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(x) (((x) & 0x3f) << 18) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN BIT(0) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN BIT(2) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN BIT(4) - -#define UTMIP_PLL_CFG1 0x484 -#define UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(x) (((x) & 0x1f) << 6) -#define UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(x) (((x) & 0xfff) << 0) -#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERUP BIT(17) -#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN BIT(16) -#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP BIT(15) -#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN BIT(14) -#define UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN BIT(12) - -#define UTMIPLL_HW_PWRDN_CFG0 0x52c -#define UTMIPLL_HW_PWRDN_CFG0_SEQ_START_STATE BIT(25) -#define UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE BIT(24) -#define UTMIPLL_HW_PWRDN_CFG0_USE_LOCKDET BIT(6) -#define UTMIPLL_HW_PWRDN_CFG0_SEQ_RESET_INPUT_VALUE BIT(5) -#define UTMIPLL_HW_PWRDN_CFG0_SEQ_IN_SWCTL BIT(4) -#define UTMIPLL_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL BIT(2) -#define UTMIPLL_HW_PWRDN_CFG0_IDDQ_OVERRIDE BIT(1) -#define UTMIPLL_HW_PWRDN_CFG0_IDDQ_SWCTL BIT(0) - #define CLK_SOURCE_CSITE 0x1d4 #define CLK_SOURCE_EMC 0x19c @@ -454,7 +428,7 @@ static struct tegra_clk_pll_params pll_d_params = { .div_nmp = &pllp_nmp, .freq_table = pll_d_freq_table, .flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON | - TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE, + TEGRA_PLL_HAS_LOCK_ENABLE, }; static struct tegra_clk_pll_params pll_d2_params = { @@ -472,7 +446,7 @@ static struct tegra_clk_pll_params pll_d2_params = { .div_nmp = &pllp_nmp, .freq_table = pll_d_freq_table, .flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_LFCON | - TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE, + TEGRA_PLL_HAS_LOCK_ENABLE, }; static const struct pdiv_map pllu_p[] = { @@ -649,43 +623,6 @@ static unsigned long tegra114_input_freq[] = { #define MASK(x) (BIT(x) - 1) -struct utmi_clk_param { - /* Oscillator Frequency in KHz */ - u32 osc_frequency; - /* UTMIP PLL Enable Delay Count */ - u8 enable_delay_count; - /* UTMIP PLL Stable count */ - u8 stable_count; - /* UTMIP PLL Active delay count */ - u8 active_delay_count; - /* UTMIP PLL Xtal frequency count */ - u8 xtal_freq_count; -}; - -static const struct utmi_clk_param utmi_parameters[] = { - { - .osc_frequency = 13000000, .enable_delay_count = 0x02, - .stable_count = 0x33, .active_delay_count = 0x05, - .xtal_freq_count = 0x7f - }, { - .osc_frequency = 19200000, .enable_delay_count = 0x03, - .stable_count = 0x4b, .active_delay_count = 0x06, - .xtal_freq_count = 0xbb - }, { - .osc_frequency = 12000000, .enable_delay_count = 0x02, - .stable_count = 0x2f, .active_delay_count = 0x04, - .xtal_freq_count = 0x76 - }, { - .osc_frequency = 26000000, .enable_delay_count = 0x04, - .stable_count = 0x66, .active_delay_count = 0x09, - .xtal_freq_count = 0xfe - }, { - .osc_frequency = 16800000, .enable_delay_count = 0x03, - .stable_count = 0x41, .active_delay_count = 0x0a, - .xtal_freq_count = 0xa4 - }, -}; - /* peripheral mux definitions */ static const char *mux_plld_out0_plld2_out0[] = { @@ -986,92 +923,9 @@ static void __init tegra114_fixed_clk_init(void __iomem *clk_base) } -static __init void tegra114_utmi_param_configure(void __iomem *clk_base) -{ - unsigned int i; - u32 reg; - - for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) { - if (osc_freq == utmi_parameters[i].osc_frequency) - break; - } - - if (i >= ARRAY_SIZE(utmi_parameters)) { - pr_err("%s: Unexpected oscillator freq %lu\n", __func__, - osc_freq); - return; - } - - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG2); - - /* Program UTMIP PLL stable and active counts */ - /* [FIXME] arclk_rst.h says WRONG! This should be 1ms -> 0x50 Check! */ - reg &= ~UTMIP_PLL_CFG2_STABLE_COUNT(~0); - reg |= UTMIP_PLL_CFG2_STABLE_COUNT(utmi_parameters[i].stable_count); - - reg &= ~UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(~0); - - reg |= UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(utmi_parameters[i]. - active_delay_count); - - /* Remove power downs from UTMIP PLL control bits */ - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN; - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN; - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN; - - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG2); - - /* Program UTMIP PLL delay and oscillator frequency counts */ - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1); - reg &= ~UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(~0); - - reg |= UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(utmi_parameters[i]. - enable_delay_count); - - reg &= ~UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(~0); - reg |= UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(utmi_parameters[i]. - xtal_freq_count); - - /* Remove power downs from UTMIP PLL control bits */ - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLLU_POWERUP; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN; - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1); - - /* Setup HW control of UTMIPLL */ - reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0); - reg |= UTMIPLL_HW_PWRDN_CFG0_USE_LOCKDET; - reg &= ~UTMIPLL_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL; - reg |= UTMIPLL_HW_PWRDN_CFG0_SEQ_START_STATE; - writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0); - - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1); - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1); - - udelay(1); - - /* Setup SW override of UTMIPLL assuming USB2.0 - ports are assigned to USB2 */ - reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0); - reg |= UTMIPLL_HW_PWRDN_CFG0_IDDQ_SWCTL; - reg &= ~UTMIPLL_HW_PWRDN_CFG0_IDDQ_OVERRIDE; - writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0); - - udelay(1); - - /* Enable HW control UTMIPLL */ - reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0); - reg |= UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE; - writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0); -} - static void __init tegra114_pll_init(void __iomem *clk_base, void __iomem *pmc) { - u32 val; struct clk *clk; /* PLLC */ @@ -1118,16 +972,10 @@ static void __init tegra114_pll_init(void __iomem *clk_base, CLK_SET_RATE_PARENT, 1, 1); /* PLLU */ - val = readl(clk_base + pll_u_params.base_reg); - val &= ~BIT(24); /* disable PLLU_OVERRIDE */ - writel(val, clk_base + pll_u_params.base_reg); - - clk = tegra_clk_register_pll("pll_u", "pll_ref", clk_base, pmc, 0, - &pll_u_params, &pll_u_lock); + clk = tegra_clk_register_pllu_tegra114("pll_u", "pll_ref", clk_base, 0, + &pll_u_params, &pll_u_lock); clks[TEGRA114_CLK_PLL_U] = clk; - tegra114_utmi_param_configure(clk_base); - /* PLLU_480M */ clk = clk_register_gate(NULL, "pll_u_480M", "pll_u", CLK_SET_RATE_PARENT, clk_base + PLLU_BASE, diff --git a/drivers/clk/tegra/clk-tegra124.c b/drivers/clk/tegra/clk-tegra124.c index f4fbbf16a..a112d3d2b 100644 --- a/drivers/clk/tegra/clk-tegra124.c +++ b/drivers/clk/tegra/clk-tegra124.c @@ -99,32 +99,6 @@ #define CCLKG_BURST_POLICY 0x368 -#define UTMIP_PLL_CFG2 0x488 -#define UTMIP_PLL_CFG2_STABLE_COUNT(x) (((x) & 0xffff) << 6) -#define UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(x) (((x) & 0x3f) << 18) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN BIT(0) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN BIT(2) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN BIT(4) - -#define UTMIP_PLL_CFG1 0x484 -#define UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(x) (((x) & 0x1f) << 6) -#define UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(x) (((x) & 0xfff) << 0) -#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERUP BIT(17) -#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN BIT(16) -#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP BIT(15) -#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN BIT(14) -#define UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN BIT(12) - -#define UTMIPLL_HW_PWRDN_CFG0 0x52c -#define UTMIPLL_HW_PWRDN_CFG0_SEQ_START_STATE BIT(25) -#define UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE BIT(24) -#define UTMIPLL_HW_PWRDN_CFG0_USE_LOCKDET BIT(6) -#define UTMIPLL_HW_PWRDN_CFG0_SEQ_RESET_INPUT_VALUE BIT(5) -#define UTMIPLL_HW_PWRDN_CFG0_SEQ_IN_SWCTL BIT(4) -#define UTMIPLL_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL BIT(2) -#define UTMIPLL_HW_PWRDN_CFG0_IDDQ_OVERRIDE BIT(1) -#define UTMIPLL_HW_PWRDN_CFG0_IDDQ_SWCTL BIT(0) - /* Tegra CPU clock and reset control regs */ #define CLK_RST_CONTROLLER_CPU_CMPLX_STATUS 0x470 @@ -764,43 +738,6 @@ static struct tegra_clk_pll_params pll_u_params = { TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE, }; -struct utmi_clk_param { - /* Oscillator Frequency in KHz */ - u32 osc_frequency; - /* UTMIP PLL Enable Delay Count */ - u8 enable_delay_count; - /* UTMIP PLL Stable count */ - u8 stable_count; - /* UTMIP PLL Active delay count */ - u8 active_delay_count; - /* UTMIP PLL Xtal frequency count */ - u8 xtal_freq_count; -}; - -static const struct utmi_clk_param utmi_parameters[] = { - { - .osc_frequency = 13000000, .enable_delay_count = 0x02, - .stable_count = 0x33, .active_delay_count = 0x05, - .xtal_freq_count = 0x7f - }, { - .osc_frequency = 19200000, .enable_delay_count = 0x03, - .stable_count = 0x4b, .active_delay_count = 0x06, - .xtal_freq_count = 0xbb - }, { - .osc_frequency = 12000000, .enable_delay_count = 0x02, - .stable_count = 0x2f, .active_delay_count = 0x04, - .xtal_freq_count = 0x76 - }, { - .osc_frequency = 26000000, .enable_delay_count = 0x04, - .stable_count = 0x66, .active_delay_count = 0x09, - .xtal_freq_count = 0xfe - }, { - .osc_frequency = 16800000, .enable_delay_count = 0x03, - .stable_count = 0x41, .active_delay_count = 0x0a, - .xtal_freq_count = 0xa4 - }, -}; - static struct tegra_clk tegra124_clks[tegra_clk_max] __initdata = { [tegra_clk_ispb] = { .dt_id = TEGRA124_CLK_ISPB, .present = true }, [tegra_clk_rtc] = { .dt_id = TEGRA124_CLK_RTC, .present = true }, @@ -1063,88 +1000,6 @@ static struct tegra_devclk devclks[] __initdata = { static struct clk **clks; -static void tegra124_utmi_param_configure(void __iomem *clk_base) -{ - unsigned int i; - u32 reg; - - for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) { - if (osc_freq == utmi_parameters[i].osc_frequency) - break; - } - - if (i >= ARRAY_SIZE(utmi_parameters)) { - pr_err("%s: Unexpected oscillator freq %lu\n", __func__, - osc_freq); - return; - } - - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG2); - - /* Program UTMIP PLL stable and active counts */ - /* [FIXME] arclk_rst.h says WRONG! This should be 1ms -> 0x50 Check! */ - reg &= ~UTMIP_PLL_CFG2_STABLE_COUNT(~0); - reg |= UTMIP_PLL_CFG2_STABLE_COUNT(utmi_parameters[i].stable_count); - - reg &= ~UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(~0); - - reg |= UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(utmi_parameters[i]. - active_delay_count); - - /* Remove power downs from UTMIP PLL control bits */ - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN; - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN; - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN; - - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG2); - - /* Program UTMIP PLL delay and oscillator frequency counts */ - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1); - reg &= ~UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(~0); - - reg |= UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(utmi_parameters[i]. - enable_delay_count); - - reg &= ~UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(~0); - reg |= UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(utmi_parameters[i]. - xtal_freq_count); - - /* Remove power downs from UTMIP PLL control bits */ - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLLU_POWERUP; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN; - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1); - - /* Setup HW control of UTMIPLL */ - reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0); - reg |= UTMIPLL_HW_PWRDN_CFG0_USE_LOCKDET; - reg &= ~UTMIPLL_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL; - reg |= UTMIPLL_HW_PWRDN_CFG0_SEQ_START_STATE; - writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0); - - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1); - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1); - - udelay(1); - - /* Setup SW override of UTMIPLL assuming USB2.0 - ports are assigned to USB2 */ - reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0); - reg |= UTMIPLL_HW_PWRDN_CFG0_IDDQ_SWCTL; - reg &= ~UTMIPLL_HW_PWRDN_CFG0_IDDQ_OVERRIDE; - writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0); - - udelay(1); - - /* Enable HW control UTMIPLL */ - reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0); - reg |= UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE; - writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0); -} - static __init void tegra124_periph_clk_init(void __iomem *clk_base, void __iomem *pmc_base) { @@ -1195,7 +1050,6 @@ static __init void tegra124_periph_clk_init(void __iomem *clk_base, static void __init tegra124_pll_init(void __iomem *clk_base, void __iomem *pmc) { - u32 val; struct clk *clk; /* PLLC */ @@ -1256,17 +1110,11 @@ static void __init tegra124_pll_init(void __iomem *clk_base, clks[TEGRA124_CLK_PLL_M_UD] = clk; /* PLLU */ - val = readl(clk_base + pll_u_params.base_reg); - val &= ~BIT(24); /* disable PLLU_OVERRIDE */ - writel(val, clk_base + pll_u_params.base_reg); - - clk = tegra_clk_register_pll("pll_u", "pll_ref", clk_base, pmc, 0, - &pll_u_params, &pll_u_lock); + clk = tegra_clk_register_pllu_tegra114("pll_u", "pll_ref", clk_base, 0, + &pll_u_params, &pll_u_lock); clk_register_clkdev(clk, "pll_u", NULL); clks[TEGRA124_CLK_PLL_U] = clk; - tegra124_utmi_param_configure(clk_base); - /* PLLU_480M */ clk = clk_register_gate(NULL, "pll_u_480M", "pll_u", CLK_SET_RATE_PARENT, clk_base + PLLU_BASE, diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c index 456cf586d..2896d2e78 100644 --- a/drivers/clk/tegra/clk-tegra210.c +++ b/drivers/clk/tegra/clk-tegra210.c @@ -155,27 +155,6 @@ #define PMC_PLLM_WB0_OVERRIDE 0x1dc #define PMC_PLLM_WB0_OVERRIDE_2 0x2b0 -#define UTMIP_PLL_CFG2 0x488 -#define UTMIP_PLL_CFG2_STABLE_COUNT(x) (((x) & 0xfff) << 6) -#define UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(x) (((x) & 0x3f) << 18) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN BIT(0) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERUP BIT(1) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN BIT(2) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERUP BIT(3) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN BIT(4) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERUP BIT(5) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERDOWN BIT(24) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERUP BIT(25) - -#define UTMIP_PLL_CFG1 0x484 -#define UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(x) (((x) & 0x1f) << 27) -#define UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(x) (((x) & 0xfff) << 0) -#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERUP BIT(17) -#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN BIT(16) -#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP BIT(15) -#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN BIT(14) -#define UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN BIT(12) - #define SATA_PLL_CFG0 0x490 #define SATA_PLL_CFG0_PADPLL_RESET_SWCTL BIT(0) #define SATA_PLL_CFG0_PADPLL_USE_LOCKDET BIT(2) @@ -1366,9 +1345,9 @@ static u32 pll_expo_p_to_pdiv(u32 p, u32 *pdiv) static struct tegra_clk_pll_freq_table pll_x_freq_table[] = { /* 1 GHz */ - { 12000000, 1000000000, 166, 1, 1, 0 }, /* actual: 996.0 MHz */ - { 13000000, 1000000000, 153, 1, 1, 0 }, /* actual: 994.0 MHz */ - { 38400000, 1000000000, 156, 3, 1, 0 }, /* actual: 998.4 MHz */ + { 12000000, 1000000000, 166, 1, 2, 0 }, /* actual: 996.0 MHz */ + { 13000000, 1000000000, 153, 1, 2, 0 }, /* actual: 994.0 MHz */ + { 38400000, 1000000000, 156, 3, 2, 0 }, /* actual: 998.4 MHz */ { 0, 0, 0, 0, 0, 0 }, }; @@ -1417,9 +1396,9 @@ static struct div_nmp pllc_nmp = { }; static struct tegra_clk_pll_freq_table pll_cx_freq_table[] = { - { 12000000, 510000000, 85, 1, 1, 0 }, - { 13000000, 510000000, 78, 1, 1, 0 }, /* actual: 507.0 MHz */ - { 38400000, 510000000, 79, 3, 1, 0 }, /* actual: 505.6 MHz */ + { 12000000, 510000000, 85, 1, 2, 0 }, + { 13000000, 510000000, 78, 1, 2, 0 }, /* actual: 507.0 MHz */ + { 38400000, 510000000, 79, 3, 2, 0 }, /* actual: 505.6 MHz */ { 0, 0, 0, 0, 0, 0 }, }; @@ -1532,9 +1511,9 @@ static struct div_nmp pllss_nmp = { }; static struct tegra_clk_pll_freq_table pll_c4_vco_freq_table[] = { - { 12000000, 600000000, 50, 1, 0, 0 }, - { 13000000, 600000000, 46, 1, 0, 0 }, /* actual: 598.0 MHz */ - { 38400000, 600000000, 62, 4, 0, 0 }, /* actual: 595.2 MHz */ + { 12000000, 600000000, 50, 1, 1, 0 }, + { 13000000, 600000000, 46, 1, 1, 0 }, /* actual: 598.0 MHz */ + { 38400000, 600000000, 62, 4, 1, 0 }, /* actual: 595.2 MHz */ { 0, 0, 0, 0, 0, 0 }, }; @@ -1583,19 +1562,19 @@ static struct tegra_clk_pll_params pll_c4_vco_params = { }; static struct tegra_clk_pll_freq_table pll_m_freq_table[] = { - { 12000000, 800000000, 66, 1, 0, 0 }, /* actual: 792.0 MHz */ - { 13000000, 800000000, 61, 1, 0, 0 }, /* actual: 793.0 MHz */ - { 38400000, 297600000, 93, 4, 2, 0 }, - { 38400000, 400000000, 125, 4, 2, 0 }, - { 38400000, 532800000, 111, 4, 1, 0 }, - { 38400000, 665600000, 104, 3, 1, 0 }, - { 38400000, 800000000, 125, 3, 1, 0 }, - { 38400000, 931200000, 97, 4, 0, 0 }, - { 38400000, 1065600000, 111, 4, 0, 0 }, - { 38400000, 1200000000, 125, 4, 0, 0 }, - { 38400000, 1331200000, 104, 3, 0, 0 }, - { 38400000, 1459200000, 76, 2, 0, 0 }, - { 38400000, 1600000000, 125, 3, 0, 0 }, + { 12000000, 800000000, 66, 1, 1, 0 }, /* actual: 792.0 MHz */ + { 13000000, 800000000, 61, 1, 1, 0 }, /* actual: 793.0 MHz */ + { 38400000, 297600000, 93, 4, 3, 0 }, + { 38400000, 400000000, 125, 4, 3, 0 }, + { 38400000, 532800000, 111, 4, 2, 0 }, + { 38400000, 665600000, 104, 3, 2, 0 }, + { 38400000, 800000000, 125, 3, 2, 0 }, + { 38400000, 931200000, 97, 4, 1, 0 }, + { 38400000, 1065600000, 111, 4, 1, 0 }, + { 38400000, 1200000000, 125, 4, 1, 0 }, + { 38400000, 1331200000, 104, 3, 1, 0 }, + { 38400000, 1459200000, 76, 2, 1, 0 }, + { 38400000, 1600000000, 125, 3, 1, 0 }, { 0, 0, 0, 0, 0, 0 }, }; @@ -1705,9 +1684,9 @@ static struct tegra_clk_pll_params pll_e_params = { }; static struct tegra_clk_pll_freq_table pll_re_vco_freq_table[] = { - { 12000000, 672000000, 56, 1, 0, 0 }, - { 13000000, 672000000, 51, 1, 0, 0 }, /* actual: 663.0 MHz */ - { 38400000, 672000000, 70, 4, 0, 0 }, + { 12000000, 672000000, 56, 1, 1, 0 }, + { 13000000, 672000000, 51, 1, 1, 0 }, /* actual: 663.0 MHz */ + { 38400000, 672000000, 70, 4, 1, 0 }, { 0, 0, 0, 0, 0, 0 }, }; @@ -1754,8 +1733,8 @@ static struct div_nmp pllp_nmp = { }; static struct tegra_clk_pll_freq_table pll_p_freq_table[] = { - { 12000000, 408000000, 34, 1, 0, 0 }, - { 38400000, 408000000, 85, 8, 0, 0 }, /* cf = 4.8MHz, allowed exception */ + { 12000000, 408000000, 34, 1, 1, 0 }, + { 38400000, 408000000, 85, 8, 1, 0 }, /* cf = 4.8MHz, allowed exception */ { 0, 0, 0, 0, 0, 0 }, }; @@ -1820,14 +1799,14 @@ static struct div_nmp plla_nmp = { }; static struct tegra_clk_pll_freq_table pll_a_freq_table[] = { - { 12000000, 282240000, 47, 1, 1, 1, 0xf148 }, /* actual: 282240234 */ - { 12000000, 368640000, 61, 1, 1, 1, 0xfe15 }, /* actual: 368640381 */ - { 12000000, 240000000, 60, 1, 2, 1, 0 }, - { 13000000, 282240000, 43, 1, 1, 1, 0xfd7d }, /* actual: 282239807 */ - { 13000000, 368640000, 56, 1, 1, 1, 0x06d8 }, /* actual: 368640137 */ - { 13000000, 240000000, 55, 1, 2, 1, 0 }, /* actual: 238.3 MHz */ - { 38400000, 282240000, 44, 3, 1, 1, 0xf333 }, /* actual: 282239844 */ - { 38400000, 368640000, 57, 3, 1, 1, 0x0333 }, /* actual: 368639844 */ + { 12000000, 282240000, 47, 1, 2, 1, 0xf148 }, /* actual: 282240234 */ + { 12000000, 368640000, 61, 1, 2, 1, 0xfe15 }, /* actual: 368640381 */ + { 12000000, 240000000, 60, 1, 3, 1, 0 }, + { 13000000, 282240000, 43, 1, 2, 1, 0xfd7d }, /* actual: 282239807 */ + { 13000000, 368640000, 56, 1, 2, 1, 0x06d8 }, /* actual: 368640137 */ + { 13000000, 240000000, 55, 1, 3, 1, 0 }, /* actual: 238.3 MHz */ + { 38400000, 282240000, 44, 3, 2, 1, 0xf333 }, /* actual: 282239844 */ + { 38400000, 368640000, 57, 3, 2, 1, 0x0333 }, /* actual: 368639844 */ { 38400000, 240000000, 75, 3, 3, 1, 0 }, { 0, 0, 0, 0, 0, 0, 0 }, }; @@ -1873,9 +1852,9 @@ static struct div_nmp plld_nmp = { }; static struct tegra_clk_pll_freq_table pll_d_freq_table[] = { - { 12000000, 594000000, 99, 1, 1, 0, 0 }, - { 13000000, 594000000, 91, 1, 1, 0, 0xfc4f }, /* actual: 594000183 */ - { 38400000, 594000000, 30, 1, 1, 0, 0x0e00 }, + { 12000000, 594000000, 99, 1, 2, 0, 0 }, + { 13000000, 594000000, 91, 1, 2, 0, 0xfc4f }, /* actual: 594000183 */ + { 38400000, 594000000, 30, 1, 2, 0, 0x0e00 }, { 0, 0, 0, 0, 0, 0, 0 }, }; @@ -1911,9 +1890,9 @@ static struct tegra_clk_pll_params pll_d_params = { }; static struct tegra_clk_pll_freq_table tegra210_pll_d2_freq_table[] = { - { 12000000, 594000000, 99, 1, 1, 0, 0xf000 }, - { 13000000, 594000000, 91, 1, 1, 0, 0xfc4f }, /* actual: 594000183 */ - { 38400000, 594000000, 30, 1, 1, 0, 0x0e00 }, + { 12000000, 594000000, 99, 1, 2, 0, 0xf000 }, + { 13000000, 594000000, 91, 1, 2, 0, 0xfc4f }, /* actual: 594000183 */ + { 38400000, 594000000, 30, 1, 2, 0, 0x0e00 }, { 0, 0, 0, 0, 0, 0, 0 }, }; @@ -1935,8 +1914,9 @@ static struct tegra_clk_pll_params pll_d2_params = { .sdm_din_mask = PLLA_SDM_DIN_MASK, .sdm_ctrl_reg = PLLD2_MISC1, .sdm_ctrl_en_mask = PLLD2_SDM_EN_MASK, - .ssc_ctrl_reg = PLLD2_MISC1, - .ssc_ctrl_en_mask = PLLD2_SSC_EN_MASK, + /* disable spread-spectrum for pll_d2 */ + .ssc_ctrl_reg = 0, + .ssc_ctrl_en_mask = 0, .round_p_to_pdiv = pll_qlin_p_to_pdiv, .pdiv_tohw = pll_qlin_pdiv_to_hw, .div_nmp = &pllss_nmp, @@ -1955,9 +1935,9 @@ static struct tegra_clk_pll_params pll_d2_params = { }; static struct tegra_clk_pll_freq_table pll_dp_freq_table[] = { - { 12000000, 270000000, 90, 1, 3, 0, 0xf000 }, - { 13000000, 270000000, 83, 1, 3, 0, 0xf000 }, /* actual: 269.8 MHz */ - { 38400000, 270000000, 28, 1, 3, 0, 0xf400 }, + { 12000000, 270000000, 90, 1, 4, 0, 0xf000 }, + { 13000000, 270000000, 83, 1, 4, 0, 0xf000 }, /* actual: 269.8 MHz */ + { 38400000, 270000000, 28, 1, 4, 0, 0xf400 }, { 0, 0, 0, 0, 0, 0, 0 }, }; @@ -2007,9 +1987,9 @@ static struct div_nmp pllu_nmp = { }; static struct tegra_clk_pll_freq_table pll_u_freq_table[] = { - { 12000000, 480000000, 40, 1, 0, 0 }, - { 13000000, 480000000, 36, 1, 0, 0 }, /* actual: 468.0 MHz */ - { 38400000, 480000000, 25, 2, 0, 0 }, + { 12000000, 480000000, 40, 1, 1, 0 }, + { 13000000, 480000000, 36, 1, 1, 0 }, /* actual: 468.0 MHz */ + { 38400000, 480000000, 25, 2, 1, 0 }, { 0, 0, 0, 0, 0, 0 }, }; @@ -2037,47 +2017,6 @@ static struct tegra_clk_pll_params pll_u_vco_params = { .calc_rate = tegra210_pll_fixed_mdiv_cfg, }; -struct utmi_clk_param { - /* Oscillator Frequency in KHz */ - u32 osc_frequency; - /* UTMIP PLL Enable Delay Count */ - u8 enable_delay_count; - /* UTMIP PLL Stable count */ - u16 stable_count; - /* UTMIP PLL Active delay count */ - u8 active_delay_count; - /* UTMIP PLL Xtal frequency count */ - u16 xtal_freq_count; -}; - -static const struct utmi_clk_param utmi_parameters[] = { - { - .osc_frequency = 38400000, .enable_delay_count = 0x0, - .stable_count = 0x0, .active_delay_count = 0x6, - .xtal_freq_count = 0x80 - }, { - .osc_frequency = 13000000, .enable_delay_count = 0x02, - .stable_count = 0x33, .active_delay_count = 0x05, - .xtal_freq_count = 0x7f - }, { - .osc_frequency = 19200000, .enable_delay_count = 0x03, - .stable_count = 0x4b, .active_delay_count = 0x06, - .xtal_freq_count = 0xbb - }, { - .osc_frequency = 12000000, .enable_delay_count = 0x02, - .stable_count = 0x2f, .active_delay_count = 0x08, - .xtal_freq_count = 0x76 - }, { - .osc_frequency = 26000000, .enable_delay_count = 0x04, - .stable_count = 0x66, .active_delay_count = 0x09, - .xtal_freq_count = 0xfe - }, { - .osc_frequency = 16800000, .enable_delay_count = 0x03, - .stable_count = 0x41, .active_delay_count = 0x0a, - .xtal_freq_count = 0xa4 - }, -}; - static struct tegra_clk tegra210_clks[tegra_clk_max] __initdata = { [tegra_clk_ispb] = { .dt_id = TEGRA210_CLK_ISPB, .present = true }, [tegra_clk_rtc] = { .dt_id = TEGRA210_CLK_RTC, .present = true }, @@ -2154,6 +2093,8 @@ static struct tegra_clk tegra210_clks[tegra_clk_max] __initdata = { [tegra_clk_dpaux1] = { .dt_id = TEGRA210_CLK_DPAUX1, .present = true }, [tegra_clk_sor0] = { .dt_id = TEGRA210_CLK_SOR0, .present = true }, [tegra_clk_sor0_lvds] = { .dt_id = TEGRA210_CLK_SOR0_LVDS, .present = true }, + [tegra_clk_sor1] = { .dt_id = TEGRA210_CLK_SOR1, .present = true }, + [tegra_clk_sor1_src] = { .dt_id = TEGRA210_CLK_SOR1_SRC, .present = true }, [tegra_clk_gpu] = { .dt_id = TEGRA210_CLK_GPU, .present = true }, [tegra_clk_pll_g_ref] = { .dt_id = TEGRA210_CLK_PLL_G_REF, .present = true, }, [tegra_clk_uartb_8] = { .dt_id = TEGRA210_CLK_UARTB, .present = true }, @@ -2345,114 +2286,6 @@ static struct tegra_audio_clk_info tegra210_audio_plls[] = { static struct clk **clks; -static void tegra210_utmi_param_configure(void __iomem *clk_base) -{ - u32 reg; - int i; - - for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) { - if (osc_freq == utmi_parameters[i].osc_frequency) - break; - } - - if (i >= ARRAY_SIZE(utmi_parameters)) { - pr_err("%s: Unexpected oscillator freq %lu\n", __func__, - osc_freq); - return; - } - - reg = readl_relaxed(clk_base + PLLU_HW_PWRDN_CFG0); - reg |= PLLU_HW_PWRDN_CFG0_IDDQ_PD_INCLUDE | - PLLU_HW_PWRDN_CFG0_USE_SWITCH_DETECT | - PLLU_HW_PWRDN_CFG0_USE_LOCKDET; - reg &= ~(PLLU_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL | - PLLU_HW_PWRDN_CFG0_CLK_SWITCH_SWCTL); - writel_relaxed(reg, clk_base + PLLU_HW_PWRDN_CFG0); - - reg = readl_relaxed(clk_base + PLLU_HW_PWRDN_CFG0); - reg |= PLLU_HW_PWRDN_CFG0_SEQ_ENABLE; - writel_relaxed(reg, clk_base + PLLU_HW_PWRDN_CFG0); - udelay(1); - - reg = readl_relaxed(clk_base + PLLU_BASE); - reg &= ~PLLU_BASE_CLKENABLE_USB; - writel_relaxed(reg, clk_base + PLLU_BASE); - - reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0); - reg &= ~UTMIPLL_HW_PWRDN_CFG0_IDDQ_OVERRIDE; - writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0); - - udelay(10); - - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG2); - - /* Program UTMIP PLL stable and active counts */ - /* [FIXME] arclk_rst.h says WRONG! This should be 1ms -> 0x50 Check! */ - reg &= ~UTMIP_PLL_CFG2_STABLE_COUNT(~0); - reg |= UTMIP_PLL_CFG2_STABLE_COUNT(utmi_parameters[i].stable_count); - - reg &= ~UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(~0); - - reg |= UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(utmi_parameters[i]. - active_delay_count); - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG2); - - /* Program UTMIP PLL delay and oscillator frequency counts */ - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1); - reg &= ~UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(~0); - - reg |= UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(utmi_parameters[i]. - enable_delay_count); - - reg &= ~UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(~0); - reg |= UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(utmi_parameters[i]. - xtal_freq_count); - - reg |= UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN; - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1); - - /* Remove power downs from UTMIP PLL control bits */ - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1); - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; - reg |= UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP; - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1); - udelay(1); - - /* Enable samplers for SNPS, XUSB_HOST, XUSB_DEV */ - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG2); - reg |= UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERUP; - reg |= UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERUP; - reg |= UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERUP; - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN; - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN; - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_D_POWERDOWN; - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG2); - - /* Setup HW control of UTMIPLL */ - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1); - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP; - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1); - - reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0); - reg |= UTMIPLL_HW_PWRDN_CFG0_USE_LOCKDET; - reg &= ~UTMIPLL_HW_PWRDN_CFG0_CLK_ENABLE_SWCTL; - writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0); - - udelay(1); - - reg = readl_relaxed(clk_base + XUSB_PLL_CFG0); - reg &= ~XUSB_PLL_CFG0_UTMIPLL_LOCK_DLY; - writel_relaxed(reg, clk_base + XUSB_PLL_CFG0); - - udelay(1); - - /* Enable HW control UTMIPLL */ - reg = readl_relaxed(clk_base + UTMIPLL_HW_PWRDN_CFG0); - reg |= UTMIPLL_HW_PWRDN_CFG0_SEQ_ENABLE; - writel_relaxed(reg, clk_base + UTMIPLL_HW_PWRDN_CFG0); -} - static __init void tegra210_periph_clk_init(void __iomem *clk_base, void __iomem *pmc_base) { @@ -2463,18 +2296,18 @@ static __init void tegra210_periph_clk_init(void __iomem *clk_base, 1, 2); clks[TEGRA210_CLK_XUSB_SS_DIV2] = clk; - clk = tegra_clk_register_periph_fixed("dpaux", "pll_p", 0, clk_base, + clk = tegra_clk_register_periph_fixed("sor_safe", "pll_p", 0, clk_base, + 1, 17, 222); + clks[TEGRA210_CLK_SOR_SAFE] = clk; + + clk = tegra_clk_register_periph_fixed("dpaux", "sor_safe", 0, clk_base, 1, 17, 181); clks[TEGRA210_CLK_DPAUX] = clk; - clk = tegra_clk_register_periph_fixed("dpaux1", "pll_p", 0, clk_base, + clk = tegra_clk_register_periph_fixed("dpaux1", "sor_safe", 0, clk_base, 1, 17, 207); clks[TEGRA210_CLK_DPAUX1] = clk; - clk = tegra_clk_register_periph_fixed("sor_safe", "pll_p", 0, clk_base, - 1, 17, 222); - clks[TEGRA210_CLK_SOR_SAFE] = clk; - /* pll_d_dsi_out */ clk = clk_register_gate(NULL, "pll_d_dsi_out", "pll_d_out0", 0, clk_base + PLLD_MISC0, 21, 0, &pll_d_lock); @@ -2520,7 +2353,6 @@ static __init void tegra210_periph_clk_init(void __iomem *clk_base, static void __init tegra210_pll_init(void __iomem *clk_base, void __iomem *pmc) { - u32 val; struct clk *clk; /* PLLC */ @@ -2580,12 +2412,9 @@ static void __init tegra210_pll_init(void __iomem *clk_base, clks[TEGRA210_CLK_PLL_M_UD] = clk; /* PLLU_VCO */ - val = readl(clk_base + pll_u_vco_params.base_reg); - val &= ~PLLU_BASE_OVERRIDE; /* disable PLLU_OVERRIDE */ - writel(val, clk_base + pll_u_vco_params.base_reg); - - clk = tegra_clk_register_pllre("pll_u_vco", "pll_ref", clk_base, pmc, - 0, &pll_u_vco_params, &pll_u_lock, pll_ref_freq); + clk = tegra_clk_register_pllu_tegra210("pll_u_vco", "pll_ref", + clk_base, 0, &pll_u_vco_params, + &pll_u_lock); clk_register_clkdev(clk, "pll_u_vco", NULL); clks[TEGRA210_CLK_PLL_U] = clk; @@ -2618,8 +2447,6 @@ static void __init tegra210_pll_init(void __iomem *clk_base, clk_register_clkdev(clk, "pll_u_out2", NULL); clks[TEGRA210_CLK_PLL_U_OUT2] = clk; - tegra210_utmi_param_configure(clk_base); - /* PLLU_480M */ clk = clk_register_gate(NULL, "pll_u_480M", "pll_u_vco", CLK_SET_RATE_PARENT, clk_base + PLLU_BASE, diff --git a/drivers/clk/tegra/clk-tegra30.c b/drivers/clk/tegra/clk-tegra30.c index 9396f4930..8e2db5ead 100644 --- a/drivers/clk/tegra/clk-tegra30.c +++ b/drivers/clk/tegra/clk-tegra30.c @@ -118,20 +118,6 @@ #define AUDIO_SYNC_DOUBLER 0x49c -#define UTMIP_PLL_CFG2 0x488 -#define UTMIP_PLL_CFG2_STABLE_COUNT(x) (((x) & 0xffff) << 6) -#define UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(x) (((x) & 0x3f) << 18) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN BIT(0) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN BIT(2) -#define UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN BIT(4) - -#define UTMIP_PLL_CFG1 0x484 -#define UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(x) (((x) & 0x1f) << 6) -#define UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(x) (((x) & 0xfff) << 0) -#define UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN BIT(14) -#define UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN BIT(12) -#define UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN BIT(16) - /* Tegra CPU clock and reset control regs */ #define TEGRA_CLK_RST_CONTROLLER_CLK_CPU_CMPLX 0x4c #define TEGRA_CLK_RST_CONTROLLER_RST_CPU_CMPLX_SET 0x340 @@ -207,46 +193,6 @@ static DEFINE_SPINLOCK(emc_lock); static struct clk **clks; -/* - * Structure defining the fields for USB UTMI clocks Parameters. - */ -struct utmi_clk_param { - /* Oscillator Frequency in KHz */ - u32 osc_frequency; - /* UTMIP PLL Enable Delay Count */ - u8 enable_delay_count; - /* UTMIP PLL Stable count */ - u8 stable_count; - /* UTMIP PLL Active delay count */ - u8 active_delay_count; - /* UTMIP PLL Xtal frequency count */ - u8 xtal_freq_count; -}; - -static const struct utmi_clk_param utmi_parameters[] = { - { - .osc_frequency = 13000000, .enable_delay_count = 0x02, - .stable_count = 0x33, .active_delay_count = 0x05, - .xtal_freq_count = 0x7f - }, { - .osc_frequency = 19200000, .enable_delay_count = 0x03, - .stable_count = 0x4b, .active_delay_count = 0x06, - .xtal_freq_count = 0xbb - }, { - .osc_frequency = 12000000, .enable_delay_count = 0x02, - .stable_count = 0x2f, .active_delay_count = 0x04, - .xtal_freq_count = 0x76 - }, { - .osc_frequency = 26000000, .enable_delay_count = 0x04, - .stable_count = 0x66, .active_delay_count = 0x09, - .xtal_freq_count = 0xfe - }, { - .osc_frequency = 16800000, .enable_delay_count = 0x03, - .stable_count = 0x41, .active_delay_count = 0x0a, - .xtal_freq_count = 0xa4 - }, -}; - static struct tegra_clk_pll_freq_table pll_c_freq_table[] = { { 12000000, 1040000000, 520, 6, 1, 8 }, { 13000000, 1040000000, 480, 6, 1, 8 }, @@ -873,59 +819,6 @@ static struct tegra_clk tegra30_clks[tegra_clk_max] __initdata = { [tegra_clk_pll_a_out0] = { .dt_id = TEGRA30_CLK_PLL_A_OUT0, .present = true }, }; -static void tegra30_utmi_param_configure(void) -{ - unsigned int i; - u32 reg; - - for (i = 0; i < ARRAY_SIZE(utmi_parameters); i++) { - if (input_freq == utmi_parameters[i].osc_frequency) - break; - } - - if (i >= ARRAY_SIZE(utmi_parameters)) { - pr_err("%s: Unexpected input rate %lu\n", __func__, input_freq); - return; - } - - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG2); - - /* Program UTMIP PLL stable and active counts */ - reg &= ~UTMIP_PLL_CFG2_STABLE_COUNT(~0); - reg |= UTMIP_PLL_CFG2_STABLE_COUNT( - utmi_parameters[i].stable_count); - - reg &= ~UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(~0); - - reg |= UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT( - utmi_parameters[i].active_delay_count); - - /* Remove power downs from UTMIP PLL control bits */ - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_A_POWERDOWN; - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_B_POWERDOWN; - reg &= ~UTMIP_PLL_CFG2_FORCE_PD_SAMP_C_POWERDOWN; - - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG2); - - /* Program UTMIP PLL delay and oscillator frequency counts */ - reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1); - reg &= ~UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(~0); - - reg |= UTMIP_PLL_CFG1_ENABLE_DLY_COUNT( - utmi_parameters[i].enable_delay_count); - - reg &= ~UTMIP_PLL_CFG1_XTAL_FREQ_COUNT(~0); - reg |= UTMIP_PLL_CFG1_XTAL_FREQ_COUNT( - utmi_parameters[i].xtal_freq_count); - - /* Remove power downs from UTMIP PLL control bits */ - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ACTIVE_POWERDOWN; - reg &= ~UTMIP_PLL_CFG1_FORCE_PLLU_POWERDOWN; - - writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1); -} - static const char *pll_e_parents[] = { "pll_ref", "pll_p" }; static void __init tegra30_pll_init(void) @@ -972,12 +865,10 @@ static void __init tegra30_pll_init(void) clks[TEGRA30_CLK_PLL_X_OUT0] = clk; /* PLLU */ - clk = tegra_clk_register_pll("pll_u", "pll_ref", clk_base, pmc_base, 0, - &pll_u_params, NULL); + clk = tegra_clk_register_pllu("pll_u", "pll_ref", clk_base, 0, + &pll_u_params, NULL); clks[TEGRA30_CLK_PLL_U] = clk; - tegra30_utmi_param_configure(); - /* PLLD */ clk = tegra_clk_register_pll("pll_d", "pll_ref", clk_base, pmc_base, 0, &pll_d_params, &pll_d_lock); diff --git a/drivers/clk/tegra/clk.h b/drivers/clk/tegra/clk.h index 9421f0310..6ba82ecff 100644 --- a/drivers/clk/tegra/clk.h +++ b/drivers/clk/tegra/clk.h @@ -427,6 +427,23 @@ struct clk *tegra_clk_register_pllmb(const char *name, const char *parent_name, struct tegra_clk_pll_params *pll_params, spinlock_t *lock); +struct clk *tegra_clk_register_pllu(const char *name, const char *parent_name, + void __iomem *clk_base, unsigned long flags, + struct tegra_clk_pll_params *pll_params, + spinlock_t *lock); + +struct clk *tegra_clk_register_pllu_tegra114(const char *name, + const char *parent_name, + void __iomem *clk_base, unsigned long flags, + struct tegra_clk_pll_params *pll_params, + spinlock_t *lock); + +struct clk *tegra_clk_register_pllu_tegra210(const char *name, + const char *parent_name, + void __iomem *clk_base, unsigned long flags, + struct tegra_clk_pll_params *pll_params, + spinlock_t *lock); + /** * struct tegra_clk_pll_out - PLL divider down clock * diff --git a/drivers/clk/ti/clk-33xx.c b/drivers/clk/ti/clk-33xx.c index ef2ec64fe..0e47d95fa 100644 --- a/drivers/clk/ti/clk-33xx.c +++ b/drivers/clk/ti/clk-33xx.c @@ -108,6 +108,9 @@ static struct ti_dt_clk am33xx_clks[] = { DT_CLK("48300200.ehrpwm", "tbclk", "ehrpwm0_tbclk"), DT_CLK("48302200.ehrpwm", "tbclk", "ehrpwm1_tbclk"), DT_CLK("48304200.ehrpwm", "tbclk", "ehrpwm2_tbclk"), + DT_CLK("48300200.pwm", "tbclk", "ehrpwm0_tbclk"), + DT_CLK("48302200.pwm", "tbclk", "ehrpwm1_tbclk"), + DT_CLK("48304200.pwm", "tbclk", "ehrpwm2_tbclk"), { .node_name = NULL }, }; diff --git a/drivers/clk/ti/clk-43xx.c b/drivers/clk/ti/clk-43xx.c index 097fc90bf..e816a7500 100644 --- a/drivers/clk/ti/clk-43xx.c +++ b/drivers/clk/ti/clk-43xx.c @@ -58,6 +58,7 @@ static struct ti_dt_clk am43xx_clks[] = { DT_CLK(NULL, "smartreflex1_fck", "smartreflex1_fck"), DT_CLK(NULL, "sha0_fck", "sha0_fck"), DT_CLK(NULL, "aes0_fck", "aes0_fck"), + DT_CLK(NULL, "rng_fck", "rng_fck"), DT_CLK(NULL, "timer1_fck", "timer1_fck"), DT_CLK(NULL, "timer2_fck", "timer2_fck"), DT_CLK(NULL, "timer3_fck", "timer3_fck"), @@ -115,6 +116,12 @@ static struct ti_dt_clk am43xx_clks[] = { DT_CLK("48306200.ehrpwm", "tbclk", "ehrpwm3_tbclk"), DT_CLK("48308200.ehrpwm", "tbclk", "ehrpwm4_tbclk"), DT_CLK("4830a200.ehrpwm", "tbclk", "ehrpwm5_tbclk"), + DT_CLK("48300200.pwm", "tbclk", "ehrpwm0_tbclk"), + DT_CLK("48302200.pwm", "tbclk", "ehrpwm1_tbclk"), + DT_CLK("48304200.pwm", "tbclk", "ehrpwm2_tbclk"), + DT_CLK("48306200.pwm", "tbclk", "ehrpwm3_tbclk"), + DT_CLK("48308200.pwm", "tbclk", "ehrpwm4_tbclk"), + DT_CLK("4830a200.pwm", "tbclk", "ehrpwm5_tbclk"), { .node_name = NULL }, }; diff --git a/drivers/clk/ux500/u8500_of_clk.c b/drivers/clk/ux500/u8500_of_clk.c index 9a736d939..e960d686d 100644 --- a/drivers/clk/ux500/u8500_of_clk.c +++ b/drivers/clk/ux500/u8500_of_clk.c @@ -11,7 +11,6 @@ #include #include #include -#include #include "clk.h" #define PRCC_NUM_PERIPH_CLUSTERS 6 @@ -48,11 +47,6 @@ static struct clk *ux500_twocell_get(struct of_phandle_args *clkspec, return PRCC_SHOW(clk_data, base, bit); } -static const struct of_device_id u8500_clk_of_match[] = { - { .compatible = "stericsson,u8500-clks", }, - { }, -}; - /* CLKRST4 is missing making it hard to index things */ enum clkrst_index { CLKRST1_INDEX = 0, @@ -63,22 +57,15 @@ enum clkrst_index { CLKRST_MAX, }; -void u8500_clk_init(void) +static void u8500_clk_init(struct device_node *np) { struct prcmu_fw_version *fw_version; - struct device_node *np = NULL; struct device_node *child = NULL; const char *sgaclk_parent = NULL; struct clk *clk, *rtc_clk, *twd_clk; u32 bases[CLKRST_MAX]; int i; - if (of_have_populated_dt()) - np = of_find_matching_node(NULL, u8500_clk_of_match); - if (!np) { - pr_err("Either DT or U8500 Clock node not found\n"); - return; - } for (i = 0; i < ARRAY_SIZE(bases); i++) { struct resource r; @@ -573,3 +560,4 @@ void u8500_clk_init(void) of_clk_add_provider(child, of_clk_src_simple_get, twd_clk); } } +CLK_OF_DECLARE(u8500_clks, "stericsson,u8500-clks", u8500_clk_init); diff --git a/drivers/clk/ux500/u8540_clk.c b/drivers/clk/ux500/u8540_clk.c index 86549e59f..133859f0e 100644 --- a/drivers/clk/ux500/u8540_clk.c +++ b/drivers/clk/ux500/u8540_clk.c @@ -12,14 +12,8 @@ #include #include #include -#include #include "clk.h" -static const struct of_device_id u8540_clk_of_match[] = { - { .compatible = "stericsson,u8540-clks", }, - { } -}; - /* CLKRST4 is missing making it hard to index things */ enum clkrst_index { CLKRST1_INDEX = 0, @@ -30,19 +24,12 @@ enum clkrst_index { CLKRST_MAX, }; -void u8540_clk_init(void) +static void u8540_clk_init(struct device_node *np) { struct clk *clk; - struct device_node *np = NULL; u32 bases[CLKRST_MAX]; int i; - if (of_have_populated_dt()) - np = of_find_matching_node(NULL, u8540_clk_of_match); - if (!np) { - pr_err("Either DT or U8540 Clock node not found\n"); - return; - } for (i = 0; i < ARRAY_SIZE(bases); i++) { struct resource r; @@ -607,3 +594,4 @@ void u8540_clk_init(void) bases[CLKRST6_INDEX], BIT(0), CLK_SET_RATE_GATE); clk_register_clkdev(clk, NULL, "rng"); } +CLK_OF_DECLARE(u8540_clks, "stericsson,u8540-clks", u8540_clk_init); diff --git a/drivers/clk/ux500/u9540_clk.c b/drivers/clk/ux500/u9540_clk.c index 2138a4c8c..7b6bca49c 100644 --- a/drivers/clk/ux500/u9540_clk.c +++ b/drivers/clk/ux500/u9540_clk.c @@ -9,10 +9,10 @@ #include #include -#include #include "clk.h" -void u9540_clk_init(void) +static void u9540_clk_init(struct device_node *np) { /* register clocks here */ } +CLK_OF_DECLARE(u9540_clks, "stericsson,u9540-clks", u9540_clk_init); diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 47352d25c..567788664 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -27,6 +27,20 @@ config CLKBLD_I8253 config CLKSRC_MMIO bool +config BCM2835_TIMER + bool "BCM2835 timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables the support for the BCM2835 timer driver. + +config BCM_KONA_TIMER + bool "BCM mobile timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables the support for the BCM Kona mobile timer driver. + config DIGICOLOR_TIMER bool "Digicolor timer driver" if COMPILE_TEST depends on GENERIC_CLOCKEVENTS @@ -141,6 +155,72 @@ config CLKSRC_DBX500_PRCMU help Use the always on PRCMU Timer as clocksource +config CLPS711X_TIMER + bool "Cirrus logic timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Cirrus Logic PS711 timer. + +config ATLAS7_TIMER + bool "Atlas7 timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Atlas7 timer. + +config MOXART_TIMER + bool "Moxart timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Moxart timer. + +config MXS_TIMER + bool "Mxs timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + select STMP_DEVICE + help + Enables support for the Mxs timer. + +config PRIMA2_TIMER + bool "Prima2 timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Prima2 timer. + +config U300_TIMER + bool "U300 timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + depends on ARM + select CLKSRC_MMIO + help + Enables support for the U300 timer. + +config NSPIRE_TIMER + bool "NSpire timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Nspire timer. + +config KEYSTONE_TIMER + bool "Keystone timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + depends on ARM || ARM64 + select CLKSRC_MMIO + help + Enables support for the Keystone timer. + +config INTEGRATOR_AP_TIMER + bool "Integrator-ap timer driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_MMIO + help + Enables support for the Integrator-ap timer. + config CLKSRC_DBX500_PRCMU_SCHED_CLOCK bool "Clocksource PRCMU Timer sched_clock" depends on (CLKSRC_DBX500_PRCMU && !CLKSRC_NOMADIK_MTU_SCHED_CLOCK) @@ -208,14 +288,16 @@ config ARM_ARCH_TIMER select CLKSRC_ACPI if ACPI config ARM_ARCH_TIMER_EVTSTREAM - bool "Support for ARM architected timer event stream generation" + bool "Enable ARM architected timer event stream generation by default" default y if ARM_ARCH_TIMER depends on ARM_ARCH_TIMER help - This option enables support for event stream generation based on - the ARM architected timer. It is used for waking up CPUs executing - the wfe instruction at a frequency represented as a power-of-2 - divisor of the clock rate. + This option enables support by default for event stream generation + based on the ARM architected timer. It is used for waking up CPUs + executing the wfe instruction at a frequency represented as a + power-of-2 divisor of the clock rate. The behaviour can also be + overridden on the command line using the + clocksource.arm_arch_timer.evtstream parameter. The main use of the event stream is wfe-based timeouts of userspace locking implementations. It might also be useful for imposing timeout on wfe to safeguard against any programming errors in case an expected @@ -224,8 +306,9 @@ config ARM_ARCH_TIMER_EVTSTREAM hardware anomalies of missing events. config ARM_GLOBAL_TIMER - bool + bool "Support for the ARM global timer" if COMPILE_TEST select CLKSRC_OF if OF + depends on ARM help This options enables support for the ARM global timer unit @@ -243,7 +326,7 @@ config CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK Use ARM global timer clock source as sched_clock config ARMV7M_SYSTICK - bool + bool "Support for the ARMv7M system time" if COMPILE_TEST select CLKSRC_OF if OF select CLKSRC_MMIO help @@ -254,9 +337,12 @@ config ATMEL_PIT def_bool SOC_AT91SAM9 || SOC_SAMA5 config ATMEL_ST - bool + bool "Atmel ST timer support" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS select CLKSRC_OF select MFD_SYSCON + help + Support for the Atmel ST timer. config CLKSRC_METAG_GENERIC def_bool y if METAG @@ -270,7 +356,7 @@ config CLKSRC_EXYNOS_MCT Support for Multi Core Timer controller on Exynos SoCs. config CLKSRC_SAMSUNG_PWM - bool "PWM timer drvier for Samsung S3C, S5P" if COMPILE_TEST + bool "PWM timer driver for Samsung S3C, S5P" if COMPILE_TEST depends on GENERIC_CLOCKEVENTS depends on HAS_IOMEM help @@ -293,6 +379,14 @@ config VF_PIT_TIMER help Support for Period Interrupt Timer on Freescale Vybrid Family SoCs. +config OXNAS_RPS_TIMER + bool "Oxford Semiconductor OXNAS RPS Timers driver" if COMPILE_TEST + depends on GENERIC_CLOCKEVENTS + select CLKSRC_OF + select CLKSRC_MMIO + help + This enables support for the Oxford Semiconductor OXNAS RPS timers. + config SYS_SUPPORTS_SH_CMT bool @@ -361,8 +455,8 @@ config CLKSRC_QCOM Qualcomm SoCs. config CLKSRC_VERSATILE - bool "ARM Versatile (Express) reference platforms clock source" - depends on PLAT_VERSATILE && GENERIC_SCHED_CLOCK && !ARCH_USES_GETTIMEOFFSET + bool "ARM Versatile (Express) reference platforms clock source" if COMPILE_TEST + depends on GENERIC_SCHED_CLOCK && !ARCH_USES_GETTIMEOFFSET select CLKSRC_OF default y if MFD_VEXPRESS_SYSREG help diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index 473974f95..fd9d6df0b 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -19,21 +19,21 @@ obj-$(CONFIG_CLKSRC_NOMADIK_MTU) += nomadik-mtu.o obj-$(CONFIG_CLKSRC_DBX500_PRCMU) += clksrc-dbx500-prcmu.o obj-$(CONFIG_ARMADA_370_XP_TIMER) += time-armada-370-xp.o obj-$(CONFIG_ORION_TIMER) += time-orion.o -obj-$(CONFIG_ARCH_BCM2835) += bcm2835_timer.o -obj-$(CONFIG_ARCH_CLPS711X) += clps711x-timer.o -obj-$(CONFIG_ARCH_ATLAS7) += timer-atlas7.o -obj-$(CONFIG_ARCH_MOXART) += moxart_timer.o -obj-$(CONFIG_ARCH_MXS) += mxs_timer.o +obj-$(CONFIG_BCM2835_TIMER) += bcm2835_timer.o +obj-$(CONFIG_CLPS711X_TIMER) += clps711x-timer.o +obj-$(CONFIG_ATLAS7_TIMER) += timer-atlas7.o +obj-$(CONFIG_MOXART_TIMER) += moxart_timer.o +obj-$(CONFIG_MXS_TIMER) += mxs_timer.o obj-$(CONFIG_CLKSRC_PXA) += pxa_timer.o -obj-$(CONFIG_ARCH_PRIMA2) += timer-prima2.o -obj-$(CONFIG_ARCH_U300) += timer-u300.o +obj-$(CONFIG_PRIMA2_TIMER) += timer-prima2.o +obj-$(CONFIG_U300_TIMER) += timer-u300.o obj-$(CONFIG_SUN4I_TIMER) += sun4i_timer.o obj-$(CONFIG_SUN5I_HSTIMER) += timer-sun5i.o obj-$(CONFIG_MESON6_TIMER) += meson6_timer.o obj-$(CONFIG_TEGRA_TIMER) += tegra20_timer.o obj-$(CONFIG_VT8500_TIMER) += vt8500_timer.o -obj-$(CONFIG_ARCH_NSPIRE) += zevio-timer.o -obj-$(CONFIG_ARCH_BCM_MOBILE) += bcm_kona_timer.o +obj-$(CONFIG_NSPIRE_TIMER) += zevio-timer.o +obj-$(CONFIG_BCM_KONA_TIMER) += bcm_kona_timer.o obj-$(CONFIG_CADENCE_TTC_TIMER) += cadence_ttc_timer.o obj-$(CONFIG_CLKSRC_EFM32) += time-efm32.o obj-$(CONFIG_CLKSRC_STM32) += timer-stm32.o @@ -48,6 +48,7 @@ obj-$(CONFIG_MTK_TIMER) += mtk_timer.o obj-$(CONFIG_CLKSRC_PISTACHIO) += time-pistachio.o obj-$(CONFIG_CLKSRC_TI_32K) += timer-ti-32k.o obj-$(CONFIG_CLKSRC_NPS) += timer-nps.o +obj-$(CONFIG_OXNAS_RPS_TIMER) += timer-oxnas-rps.o obj-$(CONFIG_ARM_ARCH_TIMER) += arm_arch_timer.o obj-$(CONFIG_ARM_GLOBAL_TIMER) += arm_global_timer.o @@ -55,8 +56,8 @@ obj-$(CONFIG_ARMV7M_SYSTICK) += armv7m_systick.o obj-$(CONFIG_ARM_TIMER_SP804) += timer-sp804.o obj-$(CONFIG_CLKSRC_METAG_GENERIC) += metag_generic.o obj-$(CONFIG_ARCH_HAS_TICK_BROADCAST) += dummy_timer.o -obj-$(CONFIG_ARCH_KEYSTONE) += timer-keystone.o -obj-$(CONFIG_ARCH_INTEGRATOR_AP) += timer-integrator-ap.o +obj-$(CONFIG_KEYSTONE_TIMER) += timer-keystone.o +obj-$(CONFIG_INTEGRATOR_AP_TIMER) += timer-integrator-ap.o obj-$(CONFIG_CLKSRC_VERSATILE) += versatile.o obj-$(CONFIG_CLKSRC_MIPS_GIC) += mips-gic-timer.o obj-$(CONFIG_CLKSRC_TANGO_XTAL) += tango_xtal.o diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 4814446a0..57700541f 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -8,6 +8,9 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ + +#define pr_fmt(fmt) "arm_arch_timer: " fmt + #include #include #include @@ -79,6 +82,14 @@ static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI; static bool arch_timer_c3stop; static bool arch_timer_mem_use_virtual; +static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM); + +static int __init early_evtstrm_cfg(char *buf) +{ + return strtobool(buf, &evtstrm_enable); +} +early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg); + /* * Architected system timer support. */ @@ -362,17 +373,36 @@ static bool arch_timer_has_nonsecure_ppi(void) arch_timer_ppi[PHYS_NONSECURE_PPI]); } -static int arch_timer_setup(struct clock_event_device *clk) +static u32 check_ppi_trigger(int irq) +{ + u32 flags = irq_get_trigger_type(irq); + + if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) { + pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq); + pr_warn("WARNING: Please fix your firmware\n"); + flags = IRQF_TRIGGER_LOW; + } + + return flags; +} + +static int arch_timer_starting_cpu(unsigned int cpu) { + struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt); + u32 flags; + __arch_timer_setup(ARCH_CP15_TIMER, clk); - enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], 0); + flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]); + enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags); - if (arch_timer_has_nonsecure_ppi()) - enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0); + if (arch_timer_has_nonsecure_ppi()) { + flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]); + enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags); + } arch_counter_set_user_access(); - if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM)) + if (evtstrm_enable) arch_timer_configure_evtstream(); return 0; @@ -519,29 +549,14 @@ static void arch_timer_stop(struct clock_event_device *clk) clk->set_state_shutdown(clk); } -static int arch_timer_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) +static int arch_timer_dying_cpu(unsigned int cpu) { - /* - * Grab cpu pointer in each case to avoid spurious - * preemptible warnings - */ - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_STARTING: - arch_timer_setup(this_cpu_ptr(arch_timer_evt)); - break; - case CPU_DYING: - arch_timer_stop(this_cpu_ptr(arch_timer_evt)); - break; - } + struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt); - return NOTIFY_OK; + arch_timer_stop(clk); + return 0; } -static struct notifier_block arch_timer_cpu_nb = { - .notifier_call = arch_timer_cpu_notify, -}; - #ifdef CONFIG_CPU_PM static unsigned int saved_cntkctl; static int arch_timer_cpu_pm_notify(struct notifier_block *self, @@ -562,11 +577,21 @@ static int __init arch_timer_cpu_pm_init(void) { return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier); } + +static void __init arch_timer_cpu_pm_deinit(void) +{ + WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier)); +} + #else static int __init arch_timer_cpu_pm_init(void) { return 0; } + +static void __init arch_timer_cpu_pm_deinit(void) +{ +} #endif static int __init arch_timer_register(void) @@ -613,22 +638,23 @@ static int __init arch_timer_register(void) goto out_free; } - err = register_cpu_notifier(&arch_timer_cpu_nb); - if (err) - goto out_free_irq; - err = arch_timer_cpu_pm_init(); if (err) goto out_unreg_notify; - /* Immediately configure the timer on the boot CPU */ - arch_timer_setup(this_cpu_ptr(arch_timer_evt)); + /* Register and immediately configure the timer on the boot CPU */ + err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING, + "AP_ARM_ARCH_TIMER_STARTING", + arch_timer_starting_cpu, arch_timer_dying_cpu); + if (err) + goto out_unreg_cpupm; return 0; +out_unreg_cpupm: + arch_timer_cpu_pm_deinit(); + out_unreg_notify: - unregister_cpu_notifier(&arch_timer_cpu_nb); -out_free_irq: free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt); if (arch_timer_has_nonsecure_ppi()) free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], @@ -693,25 +719,26 @@ arch_timer_needs_probing(int type, const struct of_device_id *matches) return needs_probing; } -static void __init arch_timer_common_init(void) +static int __init arch_timer_common_init(void) { unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER; /* Wait until both nodes are probed if we have two timers */ if ((arch_timers_present & mask) != mask) { if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match)) - return; + return 0; if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match)) - return; + return 0; } arch_timer_banner(arch_timers_present); arch_counter_register(arch_timers_present); - arch_timer_arch_init(); + return arch_timer_arch_init(); } -static void __init arch_timer_init(void) +static int __init arch_timer_init(void) { + int ret; /* * If HYP mode is available, we know that the physical timer * has been configured to be accessible from PL1. Use it, so @@ -739,23 +766,30 @@ static void __init arch_timer_init(void) if (!has_ppi) { pr_warn("arch_timer: No interrupt available, giving up\n"); - return; + return -EINVAL; } } - arch_timer_register(); - arch_timer_common_init(); + ret = arch_timer_register(); + if (ret) + return ret; + + ret = arch_timer_common_init(); + if (ret) + return ret; arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI]; + + return 0; } -static void __init arch_timer_of_init(struct device_node *np) +static int __init arch_timer_of_init(struct device_node *np) { int i; if (arch_timers_present & ARCH_CP15_TIMER) { pr_warn("arch_timer: multiple nodes in dt, skipping\n"); - return; + return 0; } arch_timers_present |= ARCH_CP15_TIMER; @@ -774,23 +808,23 @@ static void __init arch_timer_of_init(struct device_node *np) of_property_read_bool(np, "arm,cpu-registers-not-fw-configured")) arch_timer_uses_ppi = PHYS_SECURE_PPI; - arch_timer_init(); + return arch_timer_init(); } CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init); CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init); -static void __init arch_timer_mem_init(struct device_node *np) +static int __init arch_timer_mem_init(struct device_node *np) { struct device_node *frame, *best_frame = NULL; void __iomem *cntctlbase, *base; - unsigned int irq; + unsigned int irq, ret = -EINVAL; u32 cnttidr; arch_timers_present |= ARCH_MEM_TIMER; cntctlbase = of_iomap(np, 0); if (!cntctlbase) { pr_err("arch_timer: Can't find CNTCTLBase\n"); - return; + return -ENXIO; } cnttidr = readl_relaxed(cntctlbase + CNTTIDR); @@ -830,6 +864,7 @@ static void __init arch_timer_mem_init(struct device_node *np) best_frame = of_node_get(frame); } + ret= -ENXIO; base = arch_counter_base = of_iomap(best_frame, 0); if (!base) { pr_err("arch_timer: Can't map frame's registers\n"); @@ -841,6 +876,7 @@ static void __init arch_timer_mem_init(struct device_node *np) else irq = irq_of_parse_and_map(best_frame, 0); + ret = -EINVAL; if (!irq) { pr_err("arch_timer: Frame missing %s irq", arch_timer_mem_use_virtual ? "virt" : "phys"); @@ -848,11 +884,15 @@ static void __init arch_timer_mem_init(struct device_node *np) } arch_timer_detect_rate(base, np); - arch_timer_mem_register(base, irq); - arch_timer_common_init(); + ret = arch_timer_mem_register(base, irq); + if (ret) + goto out; + + return arch_timer_common_init(); out: iounmap(cntctlbase); of_node_put(best_frame); + return ret; } CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem", arch_timer_mem_init); diff --git a/drivers/clocksource/arm_global_timer.c b/drivers/clocksource/arm_global_timer.c index 9df0d1699..8da03298f 100644 --- a/drivers/clocksource/arm_global_timer.c +++ b/drivers/clocksource/arm_global_timer.c @@ -165,9 +165,9 @@ static irqreturn_t gt_clockevent_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static int gt_clockevents_init(struct clock_event_device *clk) +static int gt_starting_cpu(unsigned int cpu) { - int cpu = smp_processor_id(); + struct clock_event_device *clk = this_cpu_ptr(gt_evt); clk->name = "arm_global_timer"; clk->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | @@ -186,10 +186,13 @@ static int gt_clockevents_init(struct clock_event_device *clk) return 0; } -static void gt_clockevents_stop(struct clock_event_device *clk) +static int gt_dying_cpu(unsigned int cpu) { + struct clock_event_device *clk = this_cpu_ptr(gt_evt); + gt_clockevent_shutdown(clk); disable_percpu_irq(clk->irq); + return 0; } static cycle_t gt_clocksource_read(struct clocksource *cs) @@ -238,7 +241,7 @@ static void __init gt_delay_timer_init(void) register_current_timer_delay(>_delay_timer); } -static void __init gt_clocksource_init(void) +static int __init gt_clocksource_init(void) { writel(0, gt_base + GT_CONTROL); writel(0, gt_base + GT_COUNTER0); @@ -249,28 +252,10 @@ static void __init gt_clocksource_init(void) #ifdef CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK sched_clock_register(gt_sched_clock_read, 64, gt_clk_rate); #endif - clocksource_register_hz(>_clocksource, gt_clk_rate); + return clocksource_register_hz(>_clocksource, gt_clk_rate); } -static int gt_cpu_notify(struct notifier_block *self, unsigned long action, - void *hcpu) -{ - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_STARTING: - gt_clockevents_init(this_cpu_ptr(gt_evt)); - break; - case CPU_DYING: - gt_clockevents_stop(this_cpu_ptr(gt_evt)); - break; - } - - return NOTIFY_OK; -} -static struct notifier_block gt_cpu_nb = { - .notifier_call = gt_cpu_notify, -}; - -static void __init global_timer_of_register(struct device_node *np) +static int __init global_timer_of_register(struct device_node *np) { struct clk *gt_clk; int err = 0; @@ -283,19 +268,19 @@ static void __init global_timer_of_register(struct device_node *np) if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9 && (read_cpuid_id() & 0xf0000f) < 0x200000) { pr_warn("global-timer: non support for this cpu version.\n"); - return; + return -ENOSYS; } gt_ppi = irq_of_parse_and_map(np, 0); if (!gt_ppi) { pr_warn("global-timer: unable to parse irq\n"); - return; + return -EINVAL; } gt_base = of_iomap(np, 0); if (!gt_base) { pr_warn("global-timer: invalid base address\n"); - return; + return -ENXIO; } gt_clk = of_clk_get(np, 0); @@ -325,18 +310,20 @@ static void __init global_timer_of_register(struct device_node *np) goto out_free; } - err = register_cpu_notifier(>_cpu_nb); - if (err) { - pr_warn("global-timer: unable to register cpu notifier.\n"); + /* Register and immediately configure the timer on the boot CPU */ + err = gt_clocksource_init(); + if (err) + goto out_irq; + + err = cpuhp_setup_state(CPUHP_AP_ARM_GLOBAL_TIMER_STARTING, + "AP_ARM_GLOBAL_TIMER_STARTING", + gt_starting_cpu, gt_dying_cpu); + if (err) goto out_irq; - } - /* Immediately configure the timer on the boot CPU */ - gt_clocksource_init(); - gt_clockevents_init(this_cpu_ptr(gt_evt)); gt_delay_timer_init(); - return; + return 0; out_irq: free_percpu_irq(gt_ppi, gt_evt); @@ -347,6 +334,8 @@ out_clk: out_unmap: iounmap(gt_base); WARN(err, "ARM Global timer register failed (%d)\n", err); + + return err; } /* Only tested on r2p2 and r3p0 */ diff --git a/drivers/clocksource/armv7m_systick.c b/drivers/clocksource/armv7m_systick.c index addfd2c64..a315491b7 100644 --- a/drivers/clocksource/armv7m_systick.c +++ b/drivers/clocksource/armv7m_systick.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,7 @@ #define SYSTICK_LOAD_RELOAD_MASK 0x00FFFFFF -static void __init system_timer_of_register(struct device_node *np) +static int __init system_timer_of_register(struct device_node *np) { struct clk *clk = NULL; void __iomem *base; @@ -31,22 +32,26 @@ static void __init system_timer_of_register(struct device_node *np) base = of_iomap(np, 0); if (!base) { pr_warn("system-timer: invalid base address\n"); - return; + return -ENXIO; } ret = of_property_read_u32(np, "clock-frequency", &rate); if (ret) { clk = of_clk_get(np, 0); - if (IS_ERR(clk)) + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); goto out_unmap; + } ret = clk_prepare_enable(clk); if (ret) goto out_clk_put; rate = clk_get_rate(clk); - if (!rate) + if (!rate) { + ret = -EINVAL; goto out_clk_disable; + } } writel_relaxed(SYSTICK_LOAD_RELOAD_MASK, base + SYST_RVR); @@ -64,7 +69,7 @@ static void __init system_timer_of_register(struct device_node *np) pr_info("ARM System timer initialized as clocksource\n"); - return; + return 0; out_clk_disable: clk_disable_unprepare(clk); @@ -73,6 +78,8 @@ out_clk_put: out_unmap: iounmap(base); pr_warn("ARM System timer register failed (%d)\n", ret); + + return ret; } CLOCKSOURCE_OF_DECLARE(arm_systick, "arm,armv7m-systick", diff --git a/drivers/clocksource/asm9260_timer.c b/drivers/clocksource/asm9260_timer.c index 217438d39..1ba871b7f 100644 --- a/drivers/clocksource/asm9260_timer.c +++ b/drivers/clocksource/asm9260_timer.c @@ -184,7 +184,7 @@ static irqreturn_t asm9260_timer_interrupt(int irq, void *dev_id) * Timer initialization * --------------------------------------------------------------------------- */ -static void __init asm9260_timer_init(struct device_node *np) +static int __init asm9260_timer_init(struct device_node *np) { int irq; struct clk *clk; @@ -192,20 +192,26 @@ static void __init asm9260_timer_init(struct device_node *np) unsigned long rate; priv.base = of_io_request_and_map(np, 0, np->name); - if (IS_ERR(priv.base)) - panic("%s: unable to map resource", np->name); + if (IS_ERR(priv.base)) { + pr_err("%s: unable to map resource", np->name); + return PTR_ERR(priv.base); + } clk = of_clk_get(np, 0); ret = clk_prepare_enable(clk); - if (ret) - panic("Failed to enable clk!\n"); + if (ret) { + pr_err("Failed to enable clk!\n"); + return ret; + } irq = irq_of_parse_and_map(np, 0); ret = request_irq(irq, asm9260_timer_interrupt, IRQF_TIMER, DRIVER_NAME, &event_dev); - if (ret) - panic("Failed to setup irq!\n"); + if (ret) { + pr_err("Failed to setup irq!\n"); + return ret; + } /* set all timers for count-up */ writel_relaxed(BM_DIR_DEFAULT, priv.base + HW_DIR); @@ -229,6 +235,8 @@ static void __init asm9260_timer_init(struct device_node *np) priv.ticks_per_jiffy = DIV_ROUND_CLOSEST(rate, HZ); event_dev.cpumask = cpumask_of(0); clockevents_config_and_register(&event_dev, rate, 0x2c00, 0xfffffffe); + + return 0; } CLOCKSOURCE_OF_DECLARE(asm9260_timer, "alphascale,asm9260-timer", asm9260_timer_init); diff --git a/drivers/clocksource/bcm2835_timer.c b/drivers/clocksource/bcm2835_timer.c index 6f2822928..e71acf231 100644 --- a/drivers/clocksource/bcm2835_timer.c +++ b/drivers/clocksource/bcm2835_timer.c @@ -80,19 +80,24 @@ static irqreturn_t bcm2835_time_interrupt(int irq, void *dev_id) } } -static void __init bcm2835_timer_init(struct device_node *node) +static int __init bcm2835_timer_init(struct device_node *node) { void __iomem *base; u32 freq; - int irq; + int irq, ret; struct bcm2835_timer *timer; base = of_iomap(node, 0); - if (!base) - panic("Can't remap registers"); + if (!base) { + pr_err("Can't remap registers"); + return -ENXIO; + } - if (of_property_read_u32(node, "clock-frequency", &freq)) - panic("Can't read clock-frequency"); + ret = of_property_read_u32(node, "clock-frequency", &freq); + if (ret) { + pr_err("Can't read clock-frequency"); + return ret; + } system_clock = base + REG_COUNTER_LO; sched_clock_register(bcm2835_sched_read, 32, freq); @@ -101,12 +106,16 @@ static void __init bcm2835_timer_init(struct device_node *node) freq, 300, 32, clocksource_mmio_readl_up); irq = irq_of_parse_and_map(node, DEFAULT_TIMER); - if (irq <= 0) - panic("Can't parse IRQ"); + if (irq <= 0) { + pr_err("Can't parse IRQ"); + return -EINVAL; + } timer = kzalloc(sizeof(*timer), GFP_KERNEL); - if (!timer) - panic("Can't allocate timer struct\n"); + if (!timer) { + pr_err("Can't allocate timer struct\n"); + return -ENOMEM; + } timer->control = base + REG_CONTROL; timer->compare = base + REG_COMPARE(DEFAULT_TIMER); @@ -121,12 +130,17 @@ static void __init bcm2835_timer_init(struct device_node *node) timer->act.dev_id = timer; timer->act.handler = bcm2835_time_interrupt; - if (setup_irq(irq, &timer->act)) - panic("Can't set up timer IRQ\n"); + ret = setup_irq(irq, &timer->act); + if (ret) { + pr_err("Can't set up timer IRQ\n"); + return ret; + } clockevents_config_and_register(&timer->evt, freq, 0xf, 0xffffffff); pr_info("bcm2835: system timer (irq = %d)\n", irq); + + return 0; } CLOCKSOURCE_OF_DECLARE(bcm2835, "brcm,bcm2835-system-timer", bcm2835_timer_init); diff --git a/drivers/clocksource/bcm_kona_timer.c b/drivers/clocksource/bcm_kona_timer.c index e717e87df..92f6e4dee 100644 --- a/drivers/clocksource/bcm_kona_timer.c +++ b/drivers/clocksource/bcm_kona_timer.c @@ -20,7 +20,6 @@ #include #include -#include #include #include @@ -67,10 +66,10 @@ static void kona_timer_disable_and_clear(void __iomem *base) } -static void +static int kona_timer_get_counter(void __iomem *timer_base, uint32_t *msw, uint32_t *lsw) { - int loop_limit = 4; + int loop_limit = 3; /* * Read 64-bit free running counter @@ -84,18 +83,19 @@ kona_timer_get_counter(void __iomem *timer_base, uint32_t *msw, uint32_t *lsw) * if new hi-word is equal to previously read hi-word then stop. */ - while (--loop_limit) { + do { *msw = readl(timer_base + KONA_GPTIMER_STCHI_OFFSET); *lsw = readl(timer_base + KONA_GPTIMER_STCLO_OFFSET); if (*msw == readl(timer_base + KONA_GPTIMER_STCHI_OFFSET)) break; - } + } while (--loop_limit); if (!loop_limit) { pr_err("bcm_kona_timer: getting counter failed.\n"); pr_err(" Timer will be impacted\n"); + return -ETIMEDOUT; } - return; + return 0; } static int kona_timer_set_next_event(unsigned long clc, @@ -113,8 +113,11 @@ static int kona_timer_set_next_event(unsigned long clc, uint32_t lsw, msw; uint32_t reg; + int ret; - kona_timer_get_counter(timers.tmr_regs, &msw, &lsw); + ret = kona_timer_get_counter(timers.tmr_regs, &msw, &lsw); + if (ret) + return ret; /* Load the "next" event tick value */ writel(lsw + clc, timers.tmr_regs + KONA_GPTIMER_STCM0_OFFSET); @@ -163,16 +166,11 @@ static struct irqaction kona_timer_irq = { .handler = kona_timer_interrupt, }; -static void __init kona_timer_init(struct device_node *node) +static int __init kona_timer_init(struct device_node *node) { u32 freq; struct clk *external_clk; - if (!of_device_is_available(node)) { - pr_info("Kona Timer v1 marked as disabled in device tree\n"); - return; - } - external_clk = of_clk_get_by_name(node, NULL); if (!IS_ERR(external_clk)) { @@ -182,7 +180,7 @@ static void __init kona_timer_init(struct device_node *node) arch_timer_rate = freq; } else { pr_err("Kona Timer v1 unable to determine clock-frequency"); - return; + return -EINVAL; } /* Setup IRQ numbers */ @@ -196,6 +194,8 @@ static void __init kona_timer_init(struct device_node *node) kona_timer_clockevents_init(); setup_irq(timers.tmr_irq, &kona_timer_irq); kona_timer_set_next_event((arch_timer_rate / HZ), NULL); + + return 0; } CLOCKSOURCE_OF_DECLARE(brcm_kona, "brcm,kona-timer", kona_timer_init); diff --git a/drivers/clocksource/cadence_ttc_timer.c b/drivers/clocksource/cadence_ttc_timer.c index 9be6018bd..fbfbdec13 100644 --- a/drivers/clocksource/cadence_ttc_timer.c +++ b/drivers/clocksource/cadence_ttc_timer.c @@ -322,22 +322,22 @@ static int ttc_rate_change_clocksource_cb(struct notifier_block *nb, return NOTIFY_DONE; } -static void __init ttc_setup_clocksource(struct clk *clk, void __iomem *base, +static int __init ttc_setup_clocksource(struct clk *clk, void __iomem *base, u32 timer_width) { struct ttc_timer_clocksource *ttccs; int err; ttccs = kzalloc(sizeof(*ttccs), GFP_KERNEL); - if (WARN_ON(!ttccs)) - return; + if (!ttccs) + return -ENOMEM; ttccs->ttc.clk = clk; err = clk_prepare_enable(ttccs->ttc.clk); - if (WARN_ON(err)) { + if (err) { kfree(ttccs); - return; + return err; } ttccs->ttc.freq = clk_get_rate(ttccs->ttc.clk); @@ -345,8 +345,10 @@ static void __init ttc_setup_clocksource(struct clk *clk, void __iomem *base, ttccs->ttc.clk_rate_change_nb.notifier_call = ttc_rate_change_clocksource_cb; ttccs->ttc.clk_rate_change_nb.next = NULL; - if (clk_notifier_register(ttccs->ttc.clk, - &ttccs->ttc.clk_rate_change_nb)) + + err = clk_notifier_register(ttccs->ttc.clk, + &ttccs->ttc.clk_rate_change_nb); + if (err) pr_warn("Unable to register clock notifier.\n"); ttccs->ttc.base_addr = base; @@ -368,14 +370,16 @@ static void __init ttc_setup_clocksource(struct clk *clk, void __iomem *base, ttccs->ttc.base_addr + TTC_CNT_CNTRL_OFFSET); err = clocksource_register_hz(&ttccs->cs, ttccs->ttc.freq / PRESCALE); - if (WARN_ON(err)) { + if (err) { kfree(ttccs); - return; + return err; } ttc_sched_clock_val_reg = base + TTC_COUNT_VAL_OFFSET; sched_clock_register(ttc_sched_clock_read, timer_width, ttccs->ttc.freq / PRESCALE); + + return 0; } static int ttc_rate_change_clockevent_cb(struct notifier_block *nb, @@ -401,30 +405,35 @@ static int ttc_rate_change_clockevent_cb(struct notifier_block *nb, } } -static void __init ttc_setup_clockevent(struct clk *clk, - void __iomem *base, u32 irq) +static int __init ttc_setup_clockevent(struct clk *clk, + void __iomem *base, u32 irq) { struct ttc_timer_clockevent *ttcce; int err; ttcce = kzalloc(sizeof(*ttcce), GFP_KERNEL); - if (WARN_ON(!ttcce)) - return; + if (!ttcce) + return -ENOMEM; ttcce->ttc.clk = clk; err = clk_prepare_enable(ttcce->ttc.clk); - if (WARN_ON(err)) { + if (err) { kfree(ttcce); - return; + return err; } ttcce->ttc.clk_rate_change_nb.notifier_call = ttc_rate_change_clockevent_cb; ttcce->ttc.clk_rate_change_nb.next = NULL; - if (clk_notifier_register(ttcce->ttc.clk, - &ttcce->ttc.clk_rate_change_nb)) + + err = clk_notifier_register(ttcce->ttc.clk, + &ttcce->ttc.clk_rate_change_nb); + if (err) { pr_warn("Unable to register clock notifier.\n"); + return err; + } + ttcce->ttc.freq = clk_get_rate(ttcce->ttc.clk); ttcce->ttc.base_addr = base; @@ -451,13 +460,15 @@ static void __init ttc_setup_clockevent(struct clk *clk, err = request_irq(irq, ttc_clock_event_interrupt, IRQF_TIMER, ttcce->ce.name, ttcce); - if (WARN_ON(err)) { + if (err) { kfree(ttcce); - return; + return err; } clockevents_config_and_register(&ttcce->ce, ttcce->ttc.freq / PRESCALE, 1, 0xfffe); + + return 0; } /** @@ -466,17 +477,17 @@ static void __init ttc_setup_clockevent(struct clk *clk, * Initializes the timer hardware and register the clock source and clock event * timers with Linux kernal timer framework */ -static void __init ttc_timer_init(struct device_node *timer) +static int __init ttc_timer_init(struct device_node *timer) { unsigned int irq; void __iomem *timer_baseaddr; struct clk *clk_cs, *clk_ce; static int initialized; - int clksel; + int clksel, ret; u32 timer_width = 16; if (initialized) - return; + return 0; initialized = 1; @@ -488,13 +499,13 @@ static void __init ttc_timer_init(struct device_node *timer) timer_baseaddr = of_iomap(timer, 0); if (!timer_baseaddr) { pr_err("ERROR: invalid timer base address\n"); - BUG(); + return -ENXIO; } irq = irq_of_parse_and_map(timer, 1); if (irq <= 0) { pr_err("ERROR: invalid interrupt number\n"); - BUG(); + return -EINVAL; } of_property_read_u32(timer, "timer-width", &timer_width); @@ -504,7 +515,7 @@ static void __init ttc_timer_init(struct device_node *timer) clk_cs = of_clk_get(timer, clksel); if (IS_ERR(clk_cs)) { pr_err("ERROR: timer input clock not found\n"); - BUG(); + return PTR_ERR(clk_cs); } clksel = readl_relaxed(timer_baseaddr + 4 + TTC_CLK_CNTRL_OFFSET); @@ -512,13 +523,20 @@ static void __init ttc_timer_init(struct device_node *timer) clk_ce = of_clk_get(timer, clksel); if (IS_ERR(clk_ce)) { pr_err("ERROR: timer input clock not found\n"); - BUG(); + return PTR_ERR(clk_ce); } - ttc_setup_clocksource(clk_cs, timer_baseaddr, timer_width); - ttc_setup_clockevent(clk_ce, timer_baseaddr + 4, irq); + ret = ttc_setup_clocksource(clk_cs, timer_baseaddr, timer_width); + if (ret) + return ret; + + ret = ttc_setup_clockevent(clk_ce, timer_baseaddr + 4, irq); + if (ret) + return ret; pr_info("%s #0 at %p, irq=%d\n", timer->name, timer_baseaddr, irq); + + return 0; } CLOCKSOURCE_OF_DECLARE(ttc, "cdns,ttc", ttc_timer_init); diff --git a/drivers/clocksource/clksrc-dbx500-prcmu.c b/drivers/clocksource/clksrc-dbx500-prcmu.c index dfad6eb99..77a365f57 100644 --- a/drivers/clocksource/clksrc-dbx500-prcmu.c +++ b/drivers/clocksource/clksrc-dbx500-prcmu.c @@ -64,7 +64,7 @@ static u64 notrace dbx500_prcmu_sched_clock_read(void) #endif -static void __init clksrc_dbx500_prcmu_init(struct device_node *node) +static int __init clksrc_dbx500_prcmu_init(struct device_node *node) { clksrc_dbx500_timer_base = of_iomap(node, 0); @@ -84,7 +84,7 @@ static void __init clksrc_dbx500_prcmu_init(struct device_node *node) #ifdef CONFIG_CLKSRC_DBX500_PRCMU_SCHED_CLOCK sched_clock_register(dbx500_prcmu_sched_clock_read, 32, RATE_32K); #endif - clocksource_register_hz(&clocksource_dbx500_prcmu, RATE_32K); + return clocksource_register_hz(&clocksource_dbx500_prcmu, RATE_32K); } CLOCKSOURCE_OF_DECLARE(dbx500_prcmu, "stericsson,db8500-prcmu-timer-4", clksrc_dbx500_prcmu_init); diff --git a/drivers/clocksource/clksrc-probe.c b/drivers/clocksource/clksrc-probe.c index 7cb6c923a..bc62be97f 100644 --- a/drivers/clocksource/clksrc-probe.c +++ b/drivers/clocksource/clksrc-probe.c @@ -28,15 +28,23 @@ void __init clocksource_probe(void) { struct device_node *np; const struct of_device_id *match; - of_init_fn_1 init_func; + of_init_fn_1_ret init_func_ret; unsigned clocksources = 0; + int ret; for_each_matching_node_and_match(np, __clksrc_of_table, &match) { if (!of_device_is_available(np)) continue; - init_func = match->data; - init_func(np); + init_func_ret = match->data; + + ret = init_func_ret(np); + if (ret) { + pr_err("Failed to initialize '%s': %d", + of_node_full_name(np), ret); + continue; + } + clocksources++; } diff --git a/drivers/clocksource/clksrc_st_lpc.c b/drivers/clocksource/clksrc_st_lpc.c index 65ec46744..03cc49217 100644 --- a/drivers/clocksource/clksrc_st_lpc.c +++ b/drivers/clocksource/clksrc_st_lpc.c @@ -92,7 +92,7 @@ static int __init st_clksrc_setup_clk(struct device_node *np) return 0; } -static void __init st_clksrc_of_register(struct device_node *np) +static int __init st_clksrc_of_register(struct device_node *np) { int ret; uint32_t mode; @@ -100,32 +100,36 @@ static void __init st_clksrc_of_register(struct device_node *np) ret = of_property_read_u32(np, "st,lpc-mode", &mode); if (ret) { pr_err("clksrc-st-lpc: An LPC mode must be provided\n"); - return; + return ret; } /* LPC can either run as a Clocksource or in RTC or WDT mode */ if (mode != ST_LPC_MODE_CLKSRC) - return; + return 0; ddata.base = of_iomap(np, 0); if (!ddata.base) { pr_err("clksrc-st-lpc: Unable to map iomem\n"); - return; + return -ENXIO; } - if (st_clksrc_setup_clk(np)) { + ret = st_clksrc_setup_clk(np); + if (ret) { iounmap(ddata.base); - return; + return ret; } - if (st_clksrc_init()) { + ret = st_clksrc_init(); + if (ret) { clk_disable_unprepare(ddata.clk); clk_put(ddata.clk); iounmap(ddata.base); - return; + return ret; } pr_info("clksrc-st-lpc: clocksource initialised - running @ %luHz\n", clk_get_rate(ddata.clk)); + + return ret; } CLOCKSOURCE_OF_DECLARE(ddata, "st,stih407-lpc", st_clksrc_of_register); diff --git a/drivers/clocksource/clps711x-timer.c b/drivers/clocksource/clps711x-timer.c index cdd86e352..24db6d605 100644 --- a/drivers/clocksource/clps711x-timer.c +++ b/drivers/clocksource/clps711x-timer.c @@ -104,7 +104,7 @@ void __init clps711x_clksrc_init(void __iomem *tc1_base, void __iomem *tc2_base, } #ifdef CONFIG_CLKSRC_OF -static void __init clps711x_timer_init(struct device_node *np) +static int __init clps711x_timer_init(struct device_node *np) { unsigned int irq = irq_of_parse_and_map(np, 0); struct clk *clock = of_clk_get(np, 0); @@ -112,14 +112,12 @@ static void __init clps711x_timer_init(struct device_node *np) switch (of_alias_get_id(np, "timer")) { case CLPS711X_CLKSRC_CLOCKSOURCE: - BUG_ON(_clps711x_clksrc_init(clock, base)); - break; + return _clps711x_clksrc_init(clock, base); case CLPS711X_CLKSRC_CLOCKEVENT: - BUG_ON(_clps711x_clkevt_init(clock, base, irq)); - break; + return _clps711x_clkevt_init(clock, base, irq); default: - break; + return -EINVAL; } } -CLOCKSOURCE_OF_DECLARE(clps711x, "cirrus,clps711x-timer", clps711x_timer_init); +CLOCKSOURCE_OF_DECLARE(clps711x, "cirrus,ep7209-timer", clps711x_timer_init); #endif diff --git a/drivers/clocksource/dummy_timer.c b/drivers/clocksource/dummy_timer.c index 776b6c86d..89f1c2edb 100644 --- a/drivers/clocksource/dummy_timer.c +++ b/drivers/clocksource/dummy_timer.c @@ -16,10 +16,9 @@ static DEFINE_PER_CPU(struct clock_event_device, dummy_timer_evt); -static void dummy_timer_setup(void) +static int dummy_timer_starting_cpu(unsigned int cpu) { - int cpu = smp_processor_id(); - struct clock_event_device *evt = raw_cpu_ptr(&dummy_timer_evt); + struct clock_event_device *evt = per_cpu_ptr(&dummy_timer_evt, cpu); evt->name = "dummy_timer"; evt->features = CLOCK_EVT_FEAT_PERIODIC | @@ -29,36 +28,13 @@ static void dummy_timer_setup(void) evt->cpumask = cpumask_of(cpu); clockevents_register_device(evt); + return 0; } -static int dummy_timer_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - if ((action & ~CPU_TASKS_FROZEN) == CPU_STARTING) - dummy_timer_setup(); - - return NOTIFY_OK; -} - -static struct notifier_block dummy_timer_cpu_nb = { - .notifier_call = dummy_timer_cpu_notify, -}; - static int __init dummy_timer_register(void) { - int err = 0; - - cpu_notifier_register_begin(); - err = __register_cpu_notifier(&dummy_timer_cpu_nb); - if (err) - goto out; - - /* We won't get a call on the boot CPU, so register immediately */ - if (num_possible_cpus() > 1) - dummy_timer_setup(); - -out: - cpu_notifier_register_done(); - return err; + return cpuhp_setup_state(CPUHP_AP_DUMMY_TIMER_STARTING, + "AP_DUMMY_TIMER_STARTING", + dummy_timer_starting_cpu, NULL); } early_initcall(dummy_timer_register); diff --git a/drivers/clocksource/dw_apb_timer_of.c b/drivers/clocksource/dw_apb_timer_of.c index 860843cef..aee6c0d39 100644 --- a/drivers/clocksource/dw_apb_timer_of.c +++ b/drivers/clocksource/dw_apb_timer_of.c @@ -143,7 +143,7 @@ static struct delay_timer dw_apb_delay_timer = { #endif static int num_called; -static void __init dw_apb_timer_init(struct device_node *timer) +static int __init dw_apb_timer_init(struct device_node *timer) { switch (num_called) { case 0: @@ -164,6 +164,8 @@ static void __init dw_apb_timer_init(struct device_node *timer) } num_called++; + + return 0; } CLOCKSOURCE_OF_DECLARE(pc3x2_timer, "picochip,pc3x2-timer", dw_apb_timer_init); CLOCKSOURCE_OF_DECLARE(apb_timer_osc, "snps,dw-apb-timer-osc", dw_apb_timer_init); diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index be09bc0b5..41840d02c 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -232,7 +232,7 @@ static cycles_t exynos4_read_current_timer(void) return exynos4_read_count_32(); } -static void __init exynos4_clocksource_init(void) +static int __init exynos4_clocksource_init(void) { exynos4_mct_frc_start(); @@ -244,6 +244,8 @@ static void __init exynos4_clocksource_init(void) panic("%s: can't register clocksource\n", mct_frc.name); sched_clock_register(exynos4_read_sched_clock, 32, clk_rate); + + return 0; } static void exynos4_mct_comp0_stop(void) @@ -335,12 +337,14 @@ static struct irqaction mct_comp_event_irq = { .dev_id = &mct_comp_device, }; -static void exynos4_clockevent_init(void) +static int exynos4_clockevent_init(void) { mct_comp_device.cpumask = cpumask_of(0); clockevents_config_and_register(&mct_comp_device, clk_rate, 0xf, 0xffffffff); setup_irq(mct_irqs[MCT_G0_IRQ], &mct_comp_event_irq); + + return 0; } static DEFINE_PER_CPU(struct mct_clock_event_device, percpu_mct_tick); @@ -439,10 +443,11 @@ static irqreturn_t exynos4_mct_tick_isr(int irq, void *dev_id) return IRQ_HANDLED; } -static int exynos4_local_timer_setup(struct mct_clock_event_device *mevt) +static int exynos4_mct_starting_cpu(unsigned int cpu) { + struct mct_clock_event_device *mevt = + per_cpu_ptr(&percpu_mct_tick, cpu); struct clock_event_device *evt = &mevt->evt; - unsigned int cpu = smp_processor_id(); mevt->base = EXYNOS4_MCT_L_BASE(cpu); snprintf(mevt->name, sizeof(mevt->name), "mct_tick%d", cpu); @@ -476,8 +481,10 @@ static int exynos4_local_timer_setup(struct mct_clock_event_device *mevt) return 0; } -static void exynos4_local_timer_stop(struct mct_clock_event_device *mevt) +static int exynos4_mct_dying_cpu(unsigned int cpu) { + struct mct_clock_event_device *mevt = + per_cpu_ptr(&percpu_mct_tick, cpu); struct clock_event_device *evt = &mevt->evt; evt->set_state_shutdown(evt); @@ -487,39 +494,12 @@ static void exynos4_local_timer_stop(struct mct_clock_event_device *mevt) } else { disable_percpu_irq(mct_irqs[MCT_L0_IRQ]); } + return 0; } -static int exynos4_mct_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - struct mct_clock_event_device *mevt; - - /* - * Grab cpu pointer in each case to avoid spurious - * preemptible warnings - */ - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_STARTING: - mevt = this_cpu_ptr(&percpu_mct_tick); - exynos4_local_timer_setup(mevt); - break; - case CPU_DYING: - mevt = this_cpu_ptr(&percpu_mct_tick); - exynos4_local_timer_stop(mevt); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block exynos4_mct_cpu_nb = { - .notifier_call = exynos4_mct_cpu_notify, -}; - -static void __init exynos4_timer_resources(struct device_node *np, void __iomem *base) +static int __init exynos4_timer_resources(struct device_node *np, void __iomem *base) { int err, cpu; - struct mct_clock_event_device *mevt = this_cpu_ptr(&percpu_mct_tick); struct clk *mct_clk, *tick_clk; tick_clk = np ? of_clk_get_by_name(np, "fin_pll") : @@ -566,21 +546,25 @@ static void __init exynos4_timer_resources(struct device_node *np, void __iomem } } - err = register_cpu_notifier(&exynos4_mct_cpu_nb); + /* Install hotplug callbacks which configure the timer on this CPU */ + err = cpuhp_setup_state(CPUHP_AP_EXYNOS4_MCT_TIMER_STARTING, + "AP_EXYNOS4_MCT_TIMER_STARTING", + exynos4_mct_starting_cpu, + exynos4_mct_dying_cpu); if (err) goto out_irq; - /* Immediately configure the timer on the boot CPU */ - exynos4_local_timer_setup(mevt); - return; + return 0; out_irq: free_percpu_irq(mct_irqs[MCT_L0_IRQ], &percpu_mct_tick); + return err; } -static void __init mct_init_dt(struct device_node *np, unsigned int int_type) +static int __init mct_init_dt(struct device_node *np, unsigned int int_type) { u32 nr_irqs, i; + int ret; mct_int_type = int_type; @@ -600,18 +584,24 @@ static void __init mct_init_dt(struct device_node *np, unsigned int int_type) for (i = MCT_L0_IRQ; i < nr_irqs; i++) mct_irqs[i] = irq_of_parse_and_map(np, i); - exynos4_timer_resources(np, of_iomap(np, 0)); - exynos4_clocksource_init(); - exynos4_clockevent_init(); + ret = exynos4_timer_resources(np, of_iomap(np, 0)); + if (ret) + return ret; + + ret = exynos4_clocksource_init(); + if (ret) + return ret; + + return exynos4_clockevent_init(); } -static void __init mct_init_spi(struct device_node *np) +static int __init mct_init_spi(struct device_node *np) { return mct_init_dt(np, MCT_INT_SPI); } -static void __init mct_init_ppi(struct device_node *np) +static int __init mct_init_ppi(struct device_node *np) { return mct_init_dt(np, MCT_INT_PPI); } diff --git a/drivers/clocksource/fsl_ftm_timer.c b/drivers/clocksource/fsl_ftm_timer.c index 517e1c762..738515b89 100644 --- a/drivers/clocksource/fsl_ftm_timer.c +++ b/drivers/clocksource/fsl_ftm_timer.c @@ -316,15 +316,16 @@ static int __init ftm_calc_closest_round_cyc(unsigned long freq) return 0; } -static void __init ftm_timer_init(struct device_node *np) +static int __init ftm_timer_init(struct device_node *np) { unsigned long freq; - int irq; + int ret, irq; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) - return; + return -ENOMEM; + ret = -ENXIO; priv->clkevt_base = of_iomap(np, 0); if (!priv->clkevt_base) { pr_err("ftm: unable to map event timer registers\n"); @@ -337,6 +338,7 @@ static void __init ftm_timer_init(struct device_node *np) goto err; } + ret = -EINVAL; irq = irq_of_parse_and_map(np, 0); if (irq <= 0) { pr_err("ftm: unable to get IRQ from DT, %d\n", irq); @@ -349,18 +351,22 @@ static void __init ftm_timer_init(struct device_node *np) if (!freq) goto err; - if (ftm_calc_closest_round_cyc(freq)) + ret = ftm_calc_closest_round_cyc(freq); + if (ret) goto err; - if (ftm_clocksource_init(freq)) + ret = ftm_clocksource_init(freq); + if (ret) goto err; - if (ftm_clockevent_init(freq, irq)) + ret = ftm_clockevent_init(freq, irq); + if (ret) goto err; - return; + return 0; err: kfree(priv); + return ret; } CLOCKSOURCE_OF_DECLARE(flextimer, "fsl,ftm-timer", ftm_timer_init); diff --git a/drivers/clocksource/h8300_timer16.c b/drivers/clocksource/h8300_timer16.c index 75c44079b..07d9d5be9 100644 --- a/drivers/clocksource/h8300_timer16.c +++ b/drivers/clocksource/h8300_timer16.c @@ -126,7 +126,7 @@ static struct timer16_priv timer16_priv = { #define REG_CH 0 #define REG_COMM 1 -static void __init h8300_16timer_init(struct device_node *node) +static int __init h8300_16timer_init(struct device_node *node) { void __iomem *base[2]; int ret, irq; @@ -136,9 +136,10 @@ static void __init h8300_16timer_init(struct device_node *node) clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("failed to get clock for clocksource\n"); - return; + return PTR_ERR(clk); } + ret = -ENXIO; base[REG_CH] = of_iomap(node, 0); if (!base[REG_CH]) { pr_err("failed to map registers for clocksource\n"); @@ -151,6 +152,7 @@ static void __init h8300_16timer_init(struct device_node *node) goto unmap_ch; } + ret = -EINVAL; irq = irq_of_parse_and_map(node, 0); if (!irq) { pr_err("failed to get irq for clockevent\n"); @@ -174,7 +176,7 @@ static void __init h8300_16timer_init(struct device_node *node) clocksource_register_hz(&timer16_priv.cs, clk_get_rate(clk) / 8); - return; + return 0; unmap_comm: iounmap(base[REG_COMM]); @@ -182,6 +184,8 @@ unmap_ch: iounmap(base[REG_CH]); free_clk: clk_put(clk); + return ret; } -CLOCKSOURCE_OF_DECLARE(h8300_16bit, "renesas,16bit-timer", h8300_16timer_init); +CLOCKSOURCE_OF_DECLARE(h8300_16bit, "renesas,16bit-timer", + h8300_16timer_init); diff --git a/drivers/clocksource/h8300_timer8.c b/drivers/clocksource/h8300_timer8.c index c151941e1..546bb180f 100644 --- a/drivers/clocksource/h8300_timer8.c +++ b/drivers/clocksource/h8300_timer8.c @@ -164,24 +164,26 @@ static struct timer8_priv timer8_priv = { }, }; -static void __init h8300_8timer_init(struct device_node *node) +static int __init h8300_8timer_init(struct device_node *node) { void __iomem *base; - int irq; + int irq, ret; struct clk *clk; clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("failed to get clock for clockevent\n"); - return; + return PTR_ERR(clk); } + ret = ENXIO; base = of_iomap(node, 0); if (!base) { pr_err("failed to map registers for clockevent\n"); goto free_clk; } + ret = -EINVAL; irq = irq_of_parse_and_map(node, 0); if (!irq) { pr_err("failed to get irq for clockevent\n"); @@ -205,11 +207,12 @@ static void __init h8300_8timer_init(struct device_node *node) clockevents_config_and_register(&timer8_priv.ced, timer8_priv.rate, 1, 0x0000ffff); - return; + return 0; unmap_reg: iounmap(base); free_clk: clk_put(clk); + return ret; } CLOCKSOURCE_OF_DECLARE(h8300_8bit, "renesas,8bit-timer", h8300_8timer_init); diff --git a/drivers/clocksource/h8300_tpu.c b/drivers/clocksource/h8300_tpu.c index d4c1a287c..7bdf1991c 100644 --- a/drivers/clocksource/h8300_tpu.c +++ b/drivers/clocksource/h8300_tpu.c @@ -119,15 +119,16 @@ static struct tpu_priv tpu_priv = { #define CH_L 0 #define CH_H 1 -static void __init h8300_tpu_init(struct device_node *node) +static int __init h8300_tpu_init(struct device_node *node) { void __iomem *base[2]; struct clk *clk; + int ret = -ENXIO; clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("failed to get clock for clocksource\n"); - return; + return PTR_ERR(clk); } base[CH_L] = of_iomap(node, CH_L); @@ -144,14 +145,13 @@ static void __init h8300_tpu_init(struct device_node *node) tpu_priv.mapbase1 = base[CH_L]; tpu_priv.mapbase2 = base[CH_H]; - clocksource_register_hz(&tpu_priv.cs, clk_get_rate(clk) / 64); - - return; + return clocksource_register_hz(&tpu_priv.cs, clk_get_rate(clk) / 64); unmap_L: iounmap(base[CH_H]); free_clk: clk_put(clk); + return ret; } CLOCKSOURCE_OF_DECLARE(h8300_tpu, "renesas,tpu", h8300_tpu_init); diff --git a/drivers/clocksource/meson6_timer.c b/drivers/clocksource/meson6_timer.c index 1fa22c4d2..52af591a9 100644 --- a/drivers/clocksource/meson6_timer.c +++ b/drivers/clocksource/meson6_timer.c @@ -126,18 +126,22 @@ static struct irqaction meson6_timer_irq = { .dev_id = &meson6_clockevent, }; -static void __init meson6_timer_init(struct device_node *node) +static int __init meson6_timer_init(struct device_node *node) { u32 val; int ret, irq; timer_base = of_io_request_and_map(node, 0, "meson6-timer"); - if (IS_ERR(timer_base)) - panic("Can't map registers"); + if (IS_ERR(timer_base)) { + pr_err("Can't map registers"); + return -ENXIO; + } irq = irq_of_parse_and_map(node, 0); - if (irq <= 0) - panic("Can't parse IRQ"); + if (irq <= 0) { + pr_err("Can't parse IRQ"); + return -EINVAL; + } /* Set 1us for timer E */ val = readl(timer_base + TIMER_ISA_MUX); @@ -158,14 +162,17 @@ static void __init meson6_timer_init(struct device_node *node) meson6_clkevt_time_stop(CED_ID); ret = setup_irq(irq, &meson6_timer_irq); - if (ret) + if (ret) { pr_warn("failed to setup irq %d\n", irq); + return ret; + } meson6_clockevent.cpumask = cpu_possible_mask; meson6_clockevent.irq = irq; clockevents_config_and_register(&meson6_clockevent, USEC_PER_SEC, 1, 0xfffe); + return 0; } CLOCKSOURCE_OF_DECLARE(meson6, "amlogic,meson6-timer", meson6_timer_init); diff --git a/drivers/clocksource/metag_generic.c b/drivers/clocksource/metag_generic.c index bcd5c0d60..a80ab3e44 100644 --- a/drivers/clocksource/metag_generic.c +++ b/drivers/clocksource/metag_generic.c @@ -90,7 +90,7 @@ unsigned long long sched_clock(void) return ticks << HARDWARE_TO_NS_SHIFT; } -static void arch_timer_setup(unsigned int cpu) +static int arch_timer_starting_cpu(unsigned int cpu) { unsigned int txdivtime; struct clock_event_device *clk = &per_cpu(local_clockevent, cpu); @@ -132,27 +132,9 @@ static void arch_timer_setup(unsigned int cpu) val = core_reg_read(TXUCT_ID, TXTIMER_REGNUM, thread0); __core_reg_set(TXTIMER, val); } + return 0; } -static int arch_timer_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - int cpu = (long)hcpu; - - switch (action) { - case CPU_STARTING: - case CPU_STARTING_FROZEN: - arch_timer_setup(cpu); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block arch_timer_cpu_nb = { - .notifier_call = arch_timer_cpu_notify, -}; - int __init metag_generic_timer_init(void) { /* @@ -170,11 +152,8 @@ int __init metag_generic_timer_init(void) setup_irq(tbisig_map(TBID_SIGNUM_TRT), &metag_timer_irq); - /* Configure timer on boot CPU */ - arch_timer_setup(smp_processor_id()); - - /* Hook cpu boot to configure other CPU's timers */ - register_cpu_notifier(&arch_timer_cpu_nb); - - return 0; + /* Hook cpu boot to configure the CPU's timers */ + return cpuhp_setup_state(CPUHP_AP_METAG_TIMER_STARTING, + "AP_METAG_TIMER_STARTING", + arch_timer_starting_cpu, NULL); } diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c index 89d3e4d79..b4b3ab5a1 100644 --- a/drivers/clocksource/mips-gic-timer.c +++ b/drivers/clocksource/mips-gic-timer.c @@ -49,10 +49,9 @@ struct irqaction gic_compare_irqaction = { .name = "timer", }; -static void gic_clockevent_cpu_init(struct clock_event_device *cd) +static void gic_clockevent_cpu_init(unsigned int cpu, + struct clock_event_device *cd) { - unsigned int cpu = smp_processor_id(); - cd->name = "MIPS GIC"; cd->features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP; @@ -79,19 +78,10 @@ static void gic_update_frequency(void *data) clockevents_update_freq(this_cpu_ptr(&gic_clockevent_device), rate); } -static int gic_cpu_notifier(struct notifier_block *nb, unsigned long action, - void *data) +static int gic_starting_cpu(unsigned int cpu) { - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_STARTING: - gic_clockevent_cpu_init(this_cpu_ptr(&gic_clockevent_device)); - break; - case CPU_DYING: - gic_clockevent_cpu_exit(this_cpu_ptr(&gic_clockevent_device)); - break; - } - - return NOTIFY_OK; + gic_clockevent_cpu_init(cpu, this_cpu_ptr(&gic_clockevent_device)); + return 0; } static int gic_clk_notifier(struct notifier_block *nb, unsigned long action, @@ -105,10 +95,11 @@ static int gic_clk_notifier(struct notifier_block *nb, unsigned long action, return NOTIFY_OK; } - -static struct notifier_block gic_cpu_nb = { - .notifier_call = gic_cpu_notifier, -}; +static int gic_dying_cpu(unsigned int cpu) +{ + gic_clockevent_cpu_exit(this_cpu_ptr(&gic_clockevent_device)); + return 0; +} static struct notifier_block gic_clk_nb = { .notifier_call = gic_clk_notifier, @@ -125,12 +116,9 @@ static int gic_clockevent_init(void) if (ret < 0) return ret; - ret = register_cpu_notifier(&gic_cpu_nb); - if (ret < 0) - pr_warn("GIC: Unable to register CPU notifier\n"); - - gic_clockevent_cpu_init(this_cpu_ptr(&gic_clockevent_device)); - + cpuhp_setup_state(CPUHP_AP_MIPS_GIC_TIMER_STARTING, + "AP_MIPS_GIC_TIMER_STARTING", gic_starting_cpu, + gic_dying_cpu); return 0; } @@ -146,7 +134,7 @@ static struct clocksource gic_clocksource = { .archdata = { .vdso_clock_mode = VDSO_CLOCK_GIC }, }; -static void __init __gic_clocksource_init(void) +static int __init __gic_clocksource_init(void) { int ret; @@ -159,6 +147,8 @@ static void __init __gic_clocksource_init(void) ret = clocksource_register_hz(&gic_clocksource, gic_frequency); if (ret < 0) pr_warn("GIC: Unable to register clocksource\n"); + + return ret; } void __init gic_clocksource_init(unsigned int frequency) @@ -174,36 +164,40 @@ void __init gic_clocksource_init(unsigned int frequency) gic_start_count(); } -static void __init gic_clocksource_of_init(struct device_node *node) +static int __init gic_clocksource_of_init(struct device_node *node) { struct clk *clk; int ret; - if (WARN_ON(!gic_present || !node->parent || - !of_device_is_compatible(node->parent, "mti,gic"))) - return; + if (!gic_present || !node->parent || + !of_device_is_compatible(node->parent, "mti,gic")) { + pr_warn("No DT definition for the mips gic driver"); + return -ENXIO; + } clk = of_clk_get(node, 0); if (!IS_ERR(clk)) { if (clk_prepare_enable(clk) < 0) { pr_err("GIC failed to enable clock\n"); clk_put(clk); - return; + return PTR_ERR(clk); } gic_frequency = clk_get_rate(clk); } else if (of_property_read_u32(node, "clock-frequency", &gic_frequency)) { pr_err("GIC frequency not specified.\n"); - return; + return -EINVAL;; } gic_timer_irq = irq_of_parse_and_map(node, 0); if (!gic_timer_irq) { pr_err("GIC timer IRQ not specified.\n"); - return; + return -EINVAL;; } - __gic_clocksource_init(); + ret = __gic_clocksource_init(); + if (ret) + return ret; ret = gic_clockevent_init(); if (!ret && !IS_ERR(clk)) { @@ -213,6 +207,8 @@ static void __init gic_clocksource_of_init(struct device_node *node) /* And finally start the counter */ gic_start_count(); + + return 0; } CLOCKSOURCE_OF_DECLARE(mips_gic_timer, "mti,gic-timer", gic_clocksource_of_init); diff --git a/drivers/clocksource/moxart_timer.c b/drivers/clocksource/moxart_timer.c index 19857af65..841454417 100644 --- a/drivers/clocksource/moxart_timer.c +++ b/drivers/clocksource/moxart_timer.c @@ -119,34 +119,45 @@ static struct irqaction moxart_timer_irq = { .dev_id = &moxart_clockevent, }; -static void __init moxart_timer_init(struct device_node *node) +static int __init moxart_timer_init(struct device_node *node) { int ret, irq; unsigned long pclk; struct clk *clk; base = of_iomap(node, 0); - if (!base) - panic("%s: of_iomap failed\n", node->full_name); + if (!base) { + pr_err("%s: of_iomap failed\n", node->full_name); + return -ENXIO; + } irq = irq_of_parse_and_map(node, 0); - if (irq <= 0) - panic("%s: irq_of_parse_and_map failed\n", node->full_name); + if (irq <= 0) { + pr_err("%s: irq_of_parse_and_map failed\n", node->full_name); + return -EINVAL; + } ret = setup_irq(irq, &moxart_timer_irq); - if (ret) - panic("%s: setup_irq failed\n", node->full_name); + if (ret) { + pr_err("%s: setup_irq failed\n", node->full_name); + return ret; + } clk = of_clk_get(node, 0); - if (IS_ERR(clk)) - panic("%s: of_clk_get failed\n", node->full_name); + if (IS_ERR(clk)) { + pr_err("%s: of_clk_get failed\n", node->full_name); + return PTR_ERR(clk); + } pclk = clk_get_rate(clk); - if (clocksource_mmio_init(base + TIMER2_BASE + REG_COUNT, - "moxart_timer", pclk, 200, 32, - clocksource_mmio_readl_down)) - panic("%s: clocksource_mmio_init failed\n", node->full_name); + ret = clocksource_mmio_init(base + TIMER2_BASE + REG_COUNT, + "moxart_timer", pclk, 200, 32, + clocksource_mmio_readl_down); + if (ret) { + pr_err("%s: clocksource_mmio_init failed\n", node->full_name); + return ret; + } clock_count_per_tick = DIV_ROUND_CLOSEST(pclk, HZ); @@ -164,5 +175,7 @@ static void __init moxart_timer_init(struct device_node *node) */ clockevents_config_and_register(&moxart_clockevent, pclk, 0x4, 0xfffffffe); + + return 0; } CLOCKSOURCE_OF_DECLARE(moxart, "moxa,moxart-timer", moxart_timer_init); diff --git a/drivers/clocksource/mps2-timer.c b/drivers/clocksource/mps2-timer.c index 3d33a5e23..3e4431ed9 100644 --- a/drivers/clocksource/mps2-timer.c +++ b/drivers/clocksource/mps2-timer.c @@ -250,7 +250,7 @@ out: return ret; } -static void __init mps2_timer_init(struct device_node *np) +static int __init mps2_timer_init(struct device_node *np) { static int has_clocksource, has_clockevent; int ret; @@ -259,7 +259,7 @@ static void __init mps2_timer_init(struct device_node *np) ret = mps2_clocksource_init(np); if (!ret) { has_clocksource = 1; - return; + return 0; } } @@ -267,9 +267,11 @@ static void __init mps2_timer_init(struct device_node *np) ret = mps2_clockevent_init(np); if (!ret) { has_clockevent = 1; - return; + return 0; } } + + return 0; } CLOCKSOURCE_OF_DECLARE(mps2_timer, "arm,mps2-timer", mps2_timer_init); diff --git a/drivers/clocksource/mtk_timer.c b/drivers/clocksource/mtk_timer.c index 7e583f8ea..90659493c 100644 --- a/drivers/clocksource/mtk_timer.c +++ b/drivers/clocksource/mtk_timer.c @@ -181,7 +181,7 @@ static void mtk_timer_enable_irq(struct mtk_clock_event_device *evt, u8 timer) evt->gpt_base + GPT_IRQ_EN_REG); } -static void __init mtk_timer_init(struct device_node *node) +static int __init mtk_timer_init(struct device_node *node) { struct mtk_clock_event_device *evt; struct resource res; @@ -190,7 +190,7 @@ static void __init mtk_timer_init(struct device_node *node) evt = kzalloc(sizeof(*evt), GFP_KERNEL); if (!evt) - return; + return -ENOMEM; evt->dev.name = "mtk_tick"; evt->dev.rating = 300; @@ -248,7 +248,7 @@ static void __init mtk_timer_init(struct device_node *node) mtk_timer_enable_irq(evt, GPT_CLK_EVT); - return; + return 0; err_clk_disable: clk_disable_unprepare(clk); @@ -262,5 +262,7 @@ err_mem: release_mem_region(res.start, resource_size(&res)); err_kzalloc: kfree(evt); + + return -EINVAL; } CLOCKSOURCE_OF_DECLARE(mtk_mt6577, "mediatek,mt6577-timer", mtk_timer_init); diff --git a/drivers/clocksource/mxs_timer.c b/drivers/clocksource/mxs_timer.c index f5ce2961c..0ba0a913b 100644 --- a/drivers/clocksource/mxs_timer.c +++ b/drivers/clocksource/mxs_timer.c @@ -31,8 +31,6 @@ #include #include -#include - /* * There are 2 versions of the timrot on Freescale MXS-based SoCs. * The v1 on MX23 only gets 16 bits counter, while v2 on MX28 @@ -226,10 +224,10 @@ static int __init mxs_clocksource_init(struct clk *timer_clk) return 0; } -static void __init mxs_timer_init(struct device_node *np) +static int __init mxs_timer_init(struct device_node *np) { struct clk *timer_clk; - int irq; + int irq, ret; mxs_timrot_base = of_iomap(np, 0); WARN_ON(!mxs_timrot_base); @@ -237,10 +235,12 @@ static void __init mxs_timer_init(struct device_node *np) timer_clk = of_clk_get(np, 0); if (IS_ERR(timer_clk)) { pr_err("%s: failed to get clk\n", __func__); - return; + return PTR_ERR(timer_clk); } - clk_prepare_enable(timer_clk); + ret = clk_prepare_enable(timer_clk); + if (ret) + return ret; /* * Initialize timers to a known state @@ -278,11 +278,19 @@ static void __init mxs_timer_init(struct device_node *np) mxs_timrot_base + HW_TIMROT_FIXED_COUNTn(1)); /* init and register the timer to the framework */ - mxs_clocksource_init(timer_clk); - mxs_clockevent_init(timer_clk); + ret = mxs_clocksource_init(timer_clk); + if (ret) + return ret; + + ret = mxs_clockevent_init(timer_clk); + if (ret) + return ret; /* Make irqs happen */ irq = irq_of_parse_and_map(np, 0); - setup_irq(irq, &mxs_timer_irq); + if (irq <= 0) + return -EINVAL; + + return setup_irq(irq, &mxs_timer_irq); } CLOCKSOURCE_OF_DECLARE(mxs, "fsl,timrot", mxs_timer_init); diff --git a/drivers/clocksource/nomadik-mtu.c b/drivers/clocksource/nomadik-mtu.c index bc8dd443c..3c124d1ca 100644 --- a/drivers/clocksource/nomadik-mtu.c +++ b/drivers/clocksource/nomadik-mtu.c @@ -193,10 +193,11 @@ static struct irqaction nmdk_timer_irq = { .dev_id = &nmdk_clkevt, }; -static void __init nmdk_timer_init(void __iomem *base, int irq, +static int __init nmdk_timer_init(void __iomem *base, int irq, struct clk *pclk, struct clk *clk) { unsigned long rate; + int ret; mtu_base = base; @@ -226,10 +227,12 @@ static void __init nmdk_timer_init(void __iomem *base, int irq, /* Timer 0 is the free running clocksource */ nmdk_clksrc_reset(); - if (clocksource_mmio_init(mtu_base + MTU_VAL(0), "mtu_0", - rate, 200, 32, clocksource_mmio_readl_down)) - pr_err("timer: failed to initialize clock source %s\n", - "mtu_0"); + ret = clocksource_mmio_init(mtu_base + MTU_VAL(0), "mtu_0", + rate, 200, 32, clocksource_mmio_readl_down); + if (ret) { + pr_err("timer: failed to initialize clock source %s\n", "mtu_0"); + return ret; + } #ifdef CONFIG_CLKSRC_NOMADIK_MTU_SCHED_CLOCK sched_clock_register(nomadik_read_sched_clock, 32, rate); @@ -244,9 +247,11 @@ static void __init nmdk_timer_init(void __iomem *base, int irq, mtu_delay_timer.read_current_timer = &nmdk_timer_read_current_timer; mtu_delay_timer.freq = rate; register_current_timer_delay(&mtu_delay_timer); + + return 0; } -static void __init nmdk_timer_of_init(struct device_node *node) +static int __init nmdk_timer_of_init(struct device_node *node) { struct clk *pclk; struct clk *clk; @@ -254,22 +259,30 @@ static void __init nmdk_timer_of_init(struct device_node *node) int irq; base = of_iomap(node, 0); - if (!base) - panic("Can't remap registers"); + if (!base) { + pr_err("Can't remap registers"); + return -ENXIO; + } pclk = of_clk_get_by_name(node, "apb_pclk"); - if (IS_ERR(pclk)) - panic("could not get apb_pclk"); + if (IS_ERR(pclk)) { + pr_err("could not get apb_pclk"); + return PTR_ERR(pclk); + } clk = of_clk_get_by_name(node, "timclk"); - if (IS_ERR(clk)) - panic("could not get timclk"); + if (IS_ERR(clk)) { + pr_err("could not get timclk"); + return PTR_ERR(clk); + } irq = irq_of_parse_and_map(node, 0); - if (irq <= 0) - panic("Can't parse IRQ"); + if (irq <= 0) { + pr_err("Can't parse IRQ"); + return -EINVAL; + } - nmdk_timer_init(base, irq, pclk, clk); + return nmdk_timer_init(base, irq, pclk, clk); } CLOCKSOURCE_OF_DECLARE(nomadik_mtu, "st,nomadik-mtu", nmdk_timer_of_init); diff --git a/drivers/clocksource/pxa_timer.c b/drivers/clocksource/pxa_timer.c index 45b6a4999..3e1cb512f 100644 --- a/drivers/clocksource/pxa_timer.c +++ b/drivers/clocksource/pxa_timer.c @@ -21,6 +21,8 @@ #include #include +#include + #include #define OSMR0 0x00 /* OS Timer 0 Match Register */ @@ -150,8 +152,10 @@ static struct irqaction pxa_ost0_irq = { .dev_id = &ckevt_pxa_osmr0, }; -static void __init pxa_timer_common_init(int irq, unsigned long clock_tick_rate) +static int __init pxa_timer_common_init(int irq, unsigned long clock_tick_rate) { + int ret; + timer_writel(0, OIER); timer_writel(OSSR_M0 | OSSR_M1 | OSSR_M2 | OSSR_M3, OSSR); @@ -159,39 +163,57 @@ static void __init pxa_timer_common_init(int irq, unsigned long clock_tick_rate) ckevt_pxa_osmr0.cpumask = cpumask_of(0); - setup_irq(irq, &pxa_ost0_irq); + ret = setup_irq(irq, &pxa_ost0_irq); + if (ret) { + pr_err("Failed to setup irq"); + return ret; + } + + ret = clocksource_mmio_init(timer_base + OSCR, "oscr0", clock_tick_rate, 200, + 32, clocksource_mmio_readl_up); + if (ret) { + pr_err("Failed to init clocksource"); + return ret; + } - clocksource_mmio_init(timer_base + OSCR, "oscr0", clock_tick_rate, 200, - 32, clocksource_mmio_readl_up); clockevents_config_and_register(&ckevt_pxa_osmr0, clock_tick_rate, MIN_OSCR_DELTA * 2, 0x7fffffff); + + return 0; } -static void __init pxa_timer_dt_init(struct device_node *np) +static int __init pxa_timer_dt_init(struct device_node *np) { struct clk *clk; - int irq; + int irq, ret; /* timer registers are shared with watchdog timer */ timer_base = of_iomap(np, 0); - if (!timer_base) - panic("%s: unable to map resource\n", np->name); + if (!timer_base) { + pr_err("%s: unable to map resource\n", np->name); + return -ENXIO; + } clk = of_clk_get(np, 0); if (IS_ERR(clk)) { pr_crit("%s: unable to get clk\n", np->name); - return; + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) { + pr_crit("Failed to prepare clock"); + return ret; } - clk_prepare_enable(clk); /* we are only interested in OS-timer0 irq */ irq = irq_of_parse_and_map(np, 0); if (irq <= 0) { pr_crit("%s: unable to parse OS-timer0 irq\n", np->name); - return; + return -EINVAL; } - pxa_timer_common_init(irq, clk_get_rate(clk)); + return pxa_timer_common_init(irq, clk_get_rate(clk)); } CLOCKSOURCE_OF_DECLARE(pxa_timer, "marvell,pxa-timer", pxa_timer_dt_init); diff --git a/drivers/clocksource/qcom-timer.c b/drivers/clocksource/qcom-timer.c index f8e09f923..3283cfa2a 100644 --- a/drivers/clocksource/qcom-timer.c +++ b/drivers/clocksource/qcom-timer.c @@ -105,9 +105,9 @@ static struct clocksource msm_clocksource = { static int msm_timer_irq; static int msm_timer_has_ppi; -static int msm_local_timer_setup(struct clock_event_device *evt) +static int msm_local_timer_starting_cpu(unsigned int cpu) { - int cpu = smp_processor_id(); + struct clock_event_device *evt = per_cpu_ptr(msm_evt, cpu); int err; evt->irq = msm_timer_irq; @@ -135,35 +135,15 @@ static int msm_local_timer_setup(struct clock_event_device *evt) return 0; } -static void msm_local_timer_stop(struct clock_event_device *evt) +static int msm_local_timer_dying_cpu(unsigned int cpu) { + struct clock_event_device *evt = per_cpu_ptr(msm_evt, cpu); + evt->set_state_shutdown(evt); disable_percpu_irq(evt->irq); + return 0; } -static int msm_timer_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - /* - * Grab cpu pointer in each case to avoid spurious - * preemptible warnings - */ - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_STARTING: - msm_local_timer_setup(this_cpu_ptr(msm_evt)); - break; - case CPU_DYING: - msm_local_timer_stop(this_cpu_ptr(msm_evt)); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block msm_timer_cpu_nb = { - .notifier_call = msm_timer_cpu_notify, -}; - static u64 notrace msm_sched_clock_read(void) { return msm_clocksource.read(&msm_clocksource); @@ -178,7 +158,7 @@ static struct delay_timer msm_delay_timer = { .read_current_timer = msm_read_current_timer, }; -static void __init msm_timer_init(u32 dgt_hz, int sched_bits, int irq, +static int __init msm_timer_init(u32 dgt_hz, int sched_bits, int irq, bool percpu) { struct clocksource *cs = &msm_clocksource; @@ -200,14 +180,15 @@ static void __init msm_timer_init(u32 dgt_hz, int sched_bits, int irq, if (res) { pr_err("request_percpu_irq failed\n"); } else { - res = register_cpu_notifier(&msm_timer_cpu_nb); + /* Install and invoke hotplug callbacks */ + res = cpuhp_setup_state(CPUHP_AP_QCOM_TIMER_STARTING, + "AP_QCOM_TIMER_STARTING", + msm_local_timer_starting_cpu, + msm_local_timer_dying_cpu); if (res) { free_percpu_irq(irq, msm_evt); goto err; } - - /* Immediately configure the timer on the boot CPU */ - msm_local_timer_setup(raw_cpu_ptr(msm_evt)); } err: @@ -218,12 +199,14 @@ err: sched_clock_register(msm_sched_clock_read, sched_bits, dgt_hz); msm_delay_timer.freq = dgt_hz; register_current_timer_delay(&msm_delay_timer); + + return res; } -static void __init msm_dt_timer_init(struct device_node *np) +static int __init msm_dt_timer_init(struct device_node *np) { u32 freq; - int irq; + int irq, ret; struct resource res; u32 percpu_offset; void __iomem *base; @@ -232,34 +215,35 @@ static void __init msm_dt_timer_init(struct device_node *np) base = of_iomap(np, 0); if (!base) { pr_err("Failed to map event base\n"); - return; + return -ENXIO; } /* We use GPT0 for the clockevent */ irq = irq_of_parse_and_map(np, 1); if (irq <= 0) { pr_err("Can't get irq\n"); - return; + return -EINVAL; } /* We use CPU0's DGT for the clocksource */ if (of_property_read_u32(np, "cpu-offset", &percpu_offset)) percpu_offset = 0; - if (of_address_to_resource(np, 0, &res)) { + ret = of_address_to_resource(np, 0, &res); + if (ret) { pr_err("Failed to parse DGT resource\n"); - return; + return ret; } cpu0_base = ioremap(res.start + percpu_offset, resource_size(&res)); if (!cpu0_base) { pr_err("Failed to map source base\n"); - return; + return -EINVAL; } if (of_property_read_u32(np, "clock-frequency", &freq)) { pr_err("Unknown frequency\n"); - return; + return -EINVAL; } event_base = base + 0x4; @@ -268,7 +252,7 @@ static void __init msm_dt_timer_init(struct device_node *np) freq /= 4; writel_relaxed(DGT_CLK_CTL_DIV_4, source_base + DGT_CLK_CTL); - msm_timer_init(freq, 32, irq, !!percpu_offset); + return msm_timer_init(freq, 32, irq, !!percpu_offset); } CLOCKSOURCE_OF_DECLARE(kpss_timer, "qcom,kpss-timer", msm_dt_timer_init); CLOCKSOURCE_OF_DECLARE(scss_timer, "qcom,scss-timer", msm_dt_timer_init); diff --git a/drivers/clocksource/rockchip_timer.c b/drivers/clocksource/rockchip_timer.c index b991b288c..23e267acb 100644 --- a/drivers/clocksource/rockchip_timer.c +++ b/drivers/clocksource/rockchip_timer.c @@ -19,7 +19,8 @@ #define TIMER_LOAD_COUNT0 0x00 #define TIMER_LOAD_COUNT1 0x04 -#define TIMER_CONTROL_REG 0x10 +#define TIMER_CONTROL_REG3288 0x10 +#define TIMER_CONTROL_REG3399 0x1c #define TIMER_INT_STATUS 0x18 #define TIMER_DISABLE 0x0 @@ -31,6 +32,7 @@ struct bc_timer { struct clock_event_device ce; void __iomem *base; + void __iomem *ctrl; u32 freq; }; @@ -46,15 +48,20 @@ static inline void __iomem *rk_base(struct clock_event_device *ce) return rk_timer(ce)->base; } +static inline void __iomem *rk_ctrl(struct clock_event_device *ce) +{ + return rk_timer(ce)->ctrl; +} + static inline void rk_timer_disable(struct clock_event_device *ce) { - writel_relaxed(TIMER_DISABLE, rk_base(ce) + TIMER_CONTROL_REG); + writel_relaxed(TIMER_DISABLE, rk_ctrl(ce)); } static inline void rk_timer_enable(struct clock_event_device *ce, u32 flags) { writel_relaxed(TIMER_ENABLE | TIMER_INT_UNMASK | flags, - rk_base(ce) + TIMER_CONTROL_REG); + rk_ctrl(ce)); } static void rk_timer_update_counter(unsigned long cycles, @@ -106,37 +113,42 @@ static irqreturn_t rk_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static void __init rk_timer_init(struct device_node *np) +static int __init rk_timer_init(struct device_node *np, u32 ctrl_reg) { struct clock_event_device *ce = &bc_timer.ce; struct clk *timer_clk; struct clk *pclk; - int ret, irq; + int ret = -EINVAL, irq; bc_timer.base = of_iomap(np, 0); if (!bc_timer.base) { pr_err("Failed to get base address for '%s'\n", TIMER_NAME); - return; + return -ENXIO; } + bc_timer.ctrl = bc_timer.base + ctrl_reg; pclk = of_clk_get_by_name(np, "pclk"); if (IS_ERR(pclk)) { + ret = PTR_ERR(pclk); pr_err("Failed to get pclk for '%s'\n", TIMER_NAME); goto out_unmap; } - if (clk_prepare_enable(pclk)) { + ret = clk_prepare_enable(pclk); + if (ret) { pr_err("Failed to enable pclk for '%s'\n", TIMER_NAME); goto out_unmap; } timer_clk = of_clk_get_by_name(np, "timer"); if (IS_ERR(timer_clk)) { + ret = PTR_ERR(timer_clk); pr_err("Failed to get timer clock for '%s'\n", TIMER_NAME); goto out_timer_clk; } - if (clk_prepare_enable(timer_clk)) { + ret = clk_prepare_enable(timer_clk); + if (ret) { pr_err("Failed to enable timer clock\n"); goto out_timer_clk; } @@ -145,17 +157,19 @@ static void __init rk_timer_init(struct device_node *np) irq = irq_of_parse_and_map(np, 0); if (!irq) { + ret = -EINVAL; pr_err("Failed to map interrupts for '%s'\n", TIMER_NAME); goto out_irq; } ce->name = TIMER_NAME; - ce->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; + ce->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_DYNIRQ; ce->set_next_event = rk_timer_set_next_event; ce->set_state_shutdown = rk_timer_shutdown; ce->set_state_periodic = rk_timer_set_periodic; ce->irq = irq; - ce->cpumask = cpumask_of(0); + ce->cpumask = cpu_possible_mask; ce->rating = 250; rk_timer_interrupt_clear(ce); @@ -169,7 +183,7 @@ static void __init rk_timer_init(struct device_node *np) clockevents_config_and_register(ce, bc_timer.freq, 1, UINT_MAX); - return; + return 0; out_irq: clk_disable_unprepare(timer_clk); @@ -177,6 +191,21 @@ out_timer_clk: clk_disable_unprepare(pclk); out_unmap: iounmap(bc_timer.base); + + return ret; +} + +static int __init rk3288_timer_init(struct device_node *np) +{ + return rk_timer_init(np, TIMER_CONTROL_REG3288); +} + +static int __init rk3399_timer_init(struct device_node *np) +{ + return rk_timer_init(np, TIMER_CONTROL_REG3399); } -CLOCKSOURCE_OF_DECLARE(rk_timer, "rockchip,rk3288-timer", rk_timer_init); +CLOCKSOURCE_OF_DECLARE(rk3288_timer, "rockchip,rk3288-timer", + rk3288_timer_init); +CLOCKSOURCE_OF_DECLARE(rk3399_timer, "rockchip,rk3399-timer", + rk3399_timer_init); diff --git a/drivers/clocksource/samsung_pwm_timer.c b/drivers/clocksource/samsung_pwm_timer.c index 9502bc4c3..54565bd00 100644 --- a/drivers/clocksource/samsung_pwm_timer.c +++ b/drivers/clocksource/samsung_pwm_timer.c @@ -130,9 +130,9 @@ static void samsung_time_stop(unsigned int channel) spin_lock_irqsave(&samsung_pwm_lock, flags); - tcon = __raw_readl(pwm.base + REG_TCON); + tcon = readl_relaxed(pwm.base + REG_TCON); tcon &= ~TCON_START(channel); - __raw_writel(tcon, pwm.base + REG_TCON); + writel_relaxed(tcon, pwm.base + REG_TCON); spin_unlock_irqrestore(&samsung_pwm_lock, flags); } @@ -148,14 +148,14 @@ static void samsung_time_setup(unsigned int channel, unsigned long tcnt) spin_lock_irqsave(&samsung_pwm_lock, flags); - tcon = __raw_readl(pwm.base + REG_TCON); + tcon = readl_relaxed(pwm.base + REG_TCON); tcon &= ~(TCON_START(tcon_chan) | TCON_AUTORELOAD(tcon_chan)); tcon |= TCON_MANUALUPDATE(tcon_chan); - __raw_writel(tcnt, pwm.base + REG_TCNTB(channel)); - __raw_writel(tcnt, pwm.base + REG_TCMPB(channel)); - __raw_writel(tcon, pwm.base + REG_TCON); + writel_relaxed(tcnt, pwm.base + REG_TCNTB(channel)); + writel_relaxed(tcnt, pwm.base + REG_TCMPB(channel)); + writel_relaxed(tcon, pwm.base + REG_TCON); spin_unlock_irqrestore(&samsung_pwm_lock, flags); } @@ -170,7 +170,7 @@ static void samsung_time_start(unsigned int channel, bool periodic) spin_lock_irqsave(&samsung_pwm_lock, flags); - tcon = __raw_readl(pwm.base + REG_TCON); + tcon = readl_relaxed(pwm.base + REG_TCON); tcon &= ~TCON_MANUALUPDATE(channel); tcon |= TCON_START(channel); @@ -180,7 +180,7 @@ static void samsung_time_start(unsigned int channel, bool periodic) else tcon &= ~TCON_AUTORELOAD(channel); - __raw_writel(tcon, pwm.base + REG_TCON); + writel_relaxed(tcon, pwm.base + REG_TCON); spin_unlock_irqrestore(&samsung_pwm_lock, flags); } @@ -333,11 +333,10 @@ static u64 notrace samsung_read_sched_clock(void) return samsung_clocksource_read(NULL); } -static void __init samsung_clocksource_init(void) +static int __init samsung_clocksource_init(void) { unsigned long pclk; unsigned long clock_rate; - int ret; pclk = clk_get_rate(pwm.timerclk); @@ -358,9 +357,7 @@ static void __init samsung_clocksource_init(void) pwm.variant.bits, clock_rate); samsung_clocksource.mask = CLOCKSOURCE_MASK(pwm.variant.bits); - ret = clocksource_register_hz(&samsung_clocksource, clock_rate); - if (ret) - panic("samsung_clocksource_timer: can't register clocksource\n"); + return clocksource_register_hz(&samsung_clocksource, clock_rate); } static void __init samsung_timer_resources(void) @@ -380,26 +377,31 @@ static void __init samsung_timer_resources(void) /* * PWM master driver */ -static void __init _samsung_pwm_clocksource_init(void) +static int __init _samsung_pwm_clocksource_init(void) { u8 mask; int channel; mask = ~pwm.variant.output_mask & ((1 << SAMSUNG_PWM_NUM) - 1); channel = fls(mask) - 1; - if (channel < 0) - panic("failed to find PWM channel for clocksource"); + if (channel < 0) { + pr_crit("failed to find PWM channel for clocksource"); + return -EINVAL; + } pwm.source_id = channel; mask &= ~(1 << channel); channel = fls(mask) - 1; - if (channel < 0) - panic("failed to find PWM channel for clock event"); + if (channel < 0) { + pr_crit("failed to find PWM channel for clock event"); + return -EINVAL; + } pwm.event_id = channel; samsung_timer_resources(); samsung_clockevent_init(); - samsung_clocksource_init(); + + return samsung_clocksource_init(); } void __init samsung_pwm_clocksource_init(void __iomem *base, @@ -417,8 +419,8 @@ void __init samsung_pwm_clocksource_init(void __iomem *base, } #ifdef CONFIG_CLKSRC_OF -static void __init samsung_pwm_alloc(struct device_node *np, - const struct samsung_pwm_variant *variant) +static int __init samsung_pwm_alloc(struct device_node *np, + const struct samsung_pwm_variant *variant) { struct property *prop; const __be32 *cur; @@ -441,14 +443,16 @@ static void __init samsung_pwm_alloc(struct device_node *np, pwm.base = of_iomap(np, 0); if (!pwm.base) { pr_err("%s: failed to map PWM registers\n", __func__); - return; + return -ENXIO; } pwm.timerclk = of_clk_get_by_name(np, "timers"); - if (IS_ERR(pwm.timerclk)) - panic("failed to get timers clock for timer"); + if (IS_ERR(pwm.timerclk)) { + pr_crit("failed to get timers clock for timer"); + return PTR_ERR(pwm.timerclk); + } - _samsung_pwm_clocksource_init(); + return _samsung_pwm_clocksource_init(); } static const struct samsung_pwm_variant s3c24xx_variant = { @@ -458,9 +462,9 @@ static const struct samsung_pwm_variant s3c24xx_variant = { .tclk_mask = (1 << 4), }; -static void __init s3c2410_pwm_clocksource_init(struct device_node *np) +static int __init s3c2410_pwm_clocksource_init(struct device_node *np) { - samsung_pwm_alloc(np, &s3c24xx_variant); + return samsung_pwm_alloc(np, &s3c24xx_variant); } CLOCKSOURCE_OF_DECLARE(s3c2410_pwm, "samsung,s3c2410-pwm", s3c2410_pwm_clocksource_init); @@ -471,9 +475,9 @@ static const struct samsung_pwm_variant s3c64xx_variant = { .tclk_mask = (1 << 7) | (1 << 6) | (1 << 5), }; -static void __init s3c64xx_pwm_clocksource_init(struct device_node *np) +static int __init s3c64xx_pwm_clocksource_init(struct device_node *np) { - samsung_pwm_alloc(np, &s3c64xx_variant); + return samsung_pwm_alloc(np, &s3c64xx_variant); } CLOCKSOURCE_OF_DECLARE(s3c6400_pwm, "samsung,s3c6400-pwm", s3c64xx_pwm_clocksource_init); @@ -484,9 +488,9 @@ static const struct samsung_pwm_variant s5p64x0_variant = { .tclk_mask = 0, }; -static void __init s5p64x0_pwm_clocksource_init(struct device_node *np) +static int __init s5p64x0_pwm_clocksource_init(struct device_node *np) { - samsung_pwm_alloc(np, &s5p64x0_variant); + return samsung_pwm_alloc(np, &s5p64x0_variant); } CLOCKSOURCE_OF_DECLARE(s5p6440_pwm, "samsung,s5p6440-pwm", s5p64x0_pwm_clocksource_init); @@ -497,9 +501,9 @@ static const struct samsung_pwm_variant s5p_variant = { .tclk_mask = (1 << 5), }; -static void __init s5p_pwm_clocksource_init(struct device_node *np) +static int __init s5p_pwm_clocksource_init(struct device_node *np) { - samsung_pwm_alloc(np, &s5p_variant); + return samsung_pwm_alloc(np, &s5p_variant); } CLOCKSOURCE_OF_DECLARE(s5pc100_pwm, "samsung,s5pc100-pwm", s5p_pwm_clocksource_init); #endif diff --git a/drivers/clocksource/sun4i_timer.c b/drivers/clocksource/sun4i_timer.c index e84877a2c..c83452cac 100644 --- a/drivers/clocksource/sun4i_timer.c +++ b/drivers/clocksource/sun4i_timer.c @@ -150,7 +150,7 @@ static u64 notrace sun4i_timer_sched_read(void) return ~readl(timer_base + TIMER_CNTVAL_REG(1)); } -static void __init sun4i_timer_init(struct device_node *node) +static int __init sun4i_timer_init(struct device_node *node) { unsigned long rate = 0; struct clk *clk; @@ -158,17 +158,28 @@ static void __init sun4i_timer_init(struct device_node *node) u32 val; timer_base = of_iomap(node, 0); - if (!timer_base) - panic("Can't map registers"); + if (!timer_base) { + pr_crit("Can't map registers"); + return -ENXIO; + } irq = irq_of_parse_and_map(node, 0); - if (irq <= 0) - panic("Can't parse IRQ"); + if (irq <= 0) { + pr_crit("Can't parse IRQ"); + return -EINVAL; + } clk = of_clk_get(node, 0); - if (IS_ERR(clk)) - panic("Can't get timer clock"); - clk_prepare_enable(clk); + if (IS_ERR(clk)) { + pr_crit("Can't get timer clock"); + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) { + pr_err("Failed to prepare clock"); + return ret; + } rate = clk_get_rate(clk); @@ -186,8 +197,12 @@ static void __init sun4i_timer_init(struct device_node *node) of_machine_is_compatible("allwinner,sun5i-a10s")) sched_clock_register(sun4i_timer_sched_read, 32, rate); - clocksource_mmio_init(timer_base + TIMER_CNTVAL_REG(1), node->name, - rate, 350, 32, clocksource_mmio_readl_down); + ret = clocksource_mmio_init(timer_base + TIMER_CNTVAL_REG(1), node->name, + rate, 350, 32, clocksource_mmio_readl_down); + if (ret) { + pr_err("Failed to register clocksource"); + return ret; + } ticks_per_jiffy = DIV_ROUND_UP(rate, HZ); @@ -207,12 +222,16 @@ static void __init sun4i_timer_init(struct device_node *node) TIMER_SYNC_TICKS, 0xffffffff); ret = setup_irq(irq, &sun4i_timer_irq); - if (ret) - pr_warn("failed to setup irq %d\n", irq); + if (ret) { + pr_err("failed to setup irq %d\n", irq); + return ret; + } /* Enable timer0 interrupt */ val = readl(timer_base + TIMER_IRQ_EN_REG); writel(val | TIMER_IRQ_EN(0), timer_base + TIMER_IRQ_EN_REG); + + return ret; } CLOCKSOURCE_OF_DECLARE(sun4i, "allwinner,sun4i-a10-timer", sun4i_timer_init); diff --git a/drivers/clocksource/tango_xtal.c b/drivers/clocksource/tango_xtal.c index c407c47a3..12fcef8cf 100644 --- a/drivers/clocksource/tango_xtal.c +++ b/drivers/clocksource/tango_xtal.c @@ -19,7 +19,7 @@ static u64 notrace read_sched_clock(void) return read_xtal_counter(); } -static void __init tango_clocksource_init(struct device_node *np) +static int __init tango_clocksource_init(struct device_node *np) { struct clk *clk; int xtal_freq, ret; @@ -27,13 +27,13 @@ static void __init tango_clocksource_init(struct device_node *np) xtal_in_cnt = of_iomap(np, 0); if (xtal_in_cnt == NULL) { pr_err("%s: invalid address\n", np->full_name); - return; + return -ENXIO; } clk = of_clk_get(np, 0); if (IS_ERR(clk)) { pr_err("%s: invalid clock\n", np->full_name); - return; + return PTR_ERR(clk); } xtal_freq = clk_get_rate(clk); @@ -44,11 +44,13 @@ static void __init tango_clocksource_init(struct device_node *np) 32, clocksource_mmio_readl_up); if (ret) { pr_err("%s: registration failed\n", np->full_name); - return; + return ret; } sched_clock_register(read_sched_clock, 32, xtal_freq); register_current_timer_delay(&delay_timer); + + return 0; } CLOCKSOURCE_OF_DECLARE(tango, "sigma,tick-counter", tango_clocksource_init); diff --git a/drivers/clocksource/tegra20_timer.c b/drivers/clocksource/tegra20_timer.c index 7b94ad2ab..f960891aa 100644 --- a/drivers/clocksource/tegra20_timer.c +++ b/drivers/clocksource/tegra20_timer.c @@ -165,7 +165,7 @@ static struct irqaction tegra_timer_irq = { .dev_id = &tegra_clockevent, }; -static void __init tegra20_init_timer(struct device_node *np) +static int __init tegra20_init_timer(struct device_node *np) { struct clk *clk; unsigned long rate; @@ -174,13 +174,13 @@ static void __init tegra20_init_timer(struct device_node *np) timer_reg_base = of_iomap(np, 0); if (!timer_reg_base) { pr_err("Can't map timer registers\n"); - BUG(); + return -ENXIO; } tegra_timer_irq.irq = irq_of_parse_and_map(np, 2); if (tegra_timer_irq.irq <= 0) { pr_err("Failed to map timer IRQ\n"); - BUG(); + return -EINVAL; } clk = of_clk_get(np, 0); @@ -211,10 +211,12 @@ static void __init tegra20_init_timer(struct device_node *np) sched_clock_register(tegra_read_sched_clock, 32, 1000000); - if (clocksource_mmio_init(timer_reg_base + TIMERUS_CNTR_1US, - "timer_us", 1000000, 300, 32, clocksource_mmio_readl_up)) { + ret = clocksource_mmio_init(timer_reg_base + TIMERUS_CNTR_1US, + "timer_us", 1000000, 300, 32, + clocksource_mmio_readl_up); + if (ret) { pr_err("Failed to register clocksource\n"); - BUG(); + return ret; } tegra_delay_timer.read_current_timer = @@ -225,24 +227,26 @@ static void __init tegra20_init_timer(struct device_node *np) ret = setup_irq(tegra_timer_irq.irq, &tegra_timer_irq); if (ret) { pr_err("Failed to register timer IRQ: %d\n", ret); - BUG(); + return ret; } tegra_clockevent.cpumask = cpu_all_mask; tegra_clockevent.irq = tegra_timer_irq.irq; clockevents_config_and_register(&tegra_clockevent, 1000000, 0x1, 0x1fffffff); + + return 0; } CLOCKSOURCE_OF_DECLARE(tegra20_timer, "nvidia,tegra20-timer", tegra20_init_timer); -static void __init tegra20_init_rtc(struct device_node *np) +static int __init tegra20_init_rtc(struct device_node *np) { struct clk *clk; rtc_base = of_iomap(np, 0); if (!rtc_base) { pr_err("Can't map RTC registers"); - BUG(); + return -ENXIO; } /* @@ -255,6 +259,6 @@ static void __init tegra20_init_rtc(struct device_node *np) else clk_prepare_enable(clk); - register_persistent_clock(NULL, tegra_read_persistent_clock64); + return register_persistent_clock(NULL, tegra_read_persistent_clock64); } CLOCKSOURCE_OF_DECLARE(tegra20_rtc, "nvidia,tegra20-rtc", tegra20_init_rtc); diff --git a/drivers/clocksource/time-armada-370-xp.c b/drivers/clocksource/time-armada-370-xp.c index d93ec3c4f..3c39e6f45 100644 --- a/drivers/clocksource/time-armada-370-xp.c +++ b/drivers/clocksource/time-armada-370-xp.c @@ -170,10 +170,10 @@ static irqreturn_t armada_370_xp_timer_interrupt(int irq, void *dev_id) /* * Setup the local clock events for a CPU. */ -static int armada_370_xp_timer_setup(struct clock_event_device *evt) +static int armada_370_xp_timer_starting_cpu(unsigned int cpu) { + struct clock_event_device *evt = per_cpu_ptr(armada_370_xp_evt, cpu); u32 clr = 0, set = 0; - int cpu = smp_processor_id(); if (timer25Mhz) set = TIMER0_25MHZ; @@ -200,35 +200,15 @@ static int armada_370_xp_timer_setup(struct clock_event_device *evt) return 0; } -static void armada_370_xp_timer_stop(struct clock_event_device *evt) +static int armada_370_xp_timer_dying_cpu(unsigned int cpu) { + struct clock_event_device *evt = per_cpu_ptr(armada_370_xp_evt, cpu); + evt->set_state_shutdown(evt); disable_percpu_irq(evt->irq); + return 0; } -static int armada_370_xp_timer_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - /* - * Grab cpu pointer in each case to avoid spurious - * preemptible warnings - */ - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_STARTING: - armada_370_xp_timer_setup(this_cpu_ptr(armada_370_xp_evt)); - break; - case CPU_DYING: - armada_370_xp_timer_stop(this_cpu_ptr(armada_370_xp_evt)); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block armada_370_xp_timer_cpu_nb = { - .notifier_call = armada_370_xp_timer_cpu_notify, -}; - static u32 timer0_ctrl_reg, timer0_local_ctrl_reg; static int armada_370_xp_timer_suspend(void) @@ -246,7 +226,7 @@ static void armada_370_xp_timer_resume(void) writel(timer0_local_ctrl_reg, local_base + TIMER_CTRL_OFF); } -struct syscore_ops armada_370_xp_timer_syscore_ops = { +static struct syscore_ops armada_370_xp_timer_syscore_ops = { .suspend = armada_370_xp_timer_suspend, .resume = armada_370_xp_timer_resume, }; @@ -260,14 +240,22 @@ static struct delay_timer armada_370_delay_timer = { .read_current_timer = armada_370_delay_timer_read, }; -static void __init armada_370_xp_timer_common_init(struct device_node *np) +static int __init armada_370_xp_timer_common_init(struct device_node *np) { u32 clr = 0, set = 0; int res; timer_base = of_iomap(np, 0); - WARN_ON(!timer_base); + if (!timer_base) { + pr_err("Failed to iomap"); + return -ENXIO; + } + local_base = of_iomap(np, 1); + if (!local_base) { + pr_err("Failed to iomap"); + return -ENXIO; + } if (timer25Mhz) { set = TIMER0_25MHZ; @@ -306,14 +294,17 @@ static void __init armada_370_xp_timer_common_init(struct device_node *np) */ sched_clock_register(armada_370_xp_read_sched_clock, 32, timer_clk); - clocksource_mmio_init(timer_base + TIMER0_VAL_OFF, - "armada_370_xp_clocksource", - timer_clk, 300, 32, clocksource_mmio_readl_down); - - register_cpu_notifier(&armada_370_xp_timer_cpu_nb); + res = clocksource_mmio_init(timer_base + TIMER0_VAL_OFF, + "armada_370_xp_clocksource", + timer_clk, 300, 32, clocksource_mmio_readl_down); + if (res) { + pr_err("Failed to initialize clocksource mmio"); + return res; + } armada_370_xp_evt = alloc_percpu(struct clock_event_device); - + if (!armada_370_xp_evt) + return -ENOMEM; /* * Setup clockevent timer (interrupt-driven). @@ -323,33 +314,56 @@ static void __init armada_370_xp_timer_common_init(struct device_node *np) "armada_370_xp_per_cpu_tick", armada_370_xp_evt); /* Immediately configure the timer on the boot CPU */ - if (!res) - armada_370_xp_timer_setup(this_cpu_ptr(armada_370_xp_evt)); + if (res) { + pr_err("Failed to request percpu irq"); + return res; + } + + res = cpuhp_setup_state(CPUHP_AP_ARMADA_TIMER_STARTING, + "AP_ARMADA_TIMER_STARTING", + armada_370_xp_timer_starting_cpu, + armada_370_xp_timer_dying_cpu); + if (res) { + pr_err("Failed to setup hotplug state and timer"); + return res; + } register_syscore_ops(&armada_370_xp_timer_syscore_ops); + + return 0; } -static void __init armada_xp_timer_init(struct device_node *np) +static int __init armada_xp_timer_init(struct device_node *np) { struct clk *clk = of_clk_get_by_name(np, "fixed"); + int ret; + + if (IS_ERR(clk)) { + pr_err("Failed to get clock"); + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) + return ret; - /* The 25Mhz fixed clock is mandatory, and must always be available */ - BUG_ON(IS_ERR(clk)); - clk_prepare_enable(clk); timer_clk = clk_get_rate(clk); - armada_370_xp_timer_common_init(np); + return armada_370_xp_timer_common_init(np); } CLOCKSOURCE_OF_DECLARE(armada_xp, "marvell,armada-xp-timer", armada_xp_timer_init); -static void __init armada_375_timer_init(struct device_node *np) +static int __init armada_375_timer_init(struct device_node *np) { struct clk *clk; + int ret; clk = of_clk_get_by_name(np, "fixed"); if (!IS_ERR(clk)) { - clk_prepare_enable(clk); + ret = clk_prepare_enable(clk); + if (ret) + return ret; timer_clk = clk_get_rate(clk); } else { @@ -360,27 +374,43 @@ static void __init armada_375_timer_init(struct device_node *np) clk = of_clk_get(np, 0); /* Must have at least a clock */ - BUG_ON(IS_ERR(clk)); - clk_prepare_enable(clk); + if (IS_ERR(clk)) { + pr_err("Failed to get clock"); + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) + return ret; + timer_clk = clk_get_rate(clk) / TIMER_DIVIDER; timer25Mhz = false; } - armada_370_xp_timer_common_init(np); + return armada_370_xp_timer_common_init(np); } CLOCKSOURCE_OF_DECLARE(armada_375, "marvell,armada-375-timer", armada_375_timer_init); -static void __init armada_370_timer_init(struct device_node *np) +static int __init armada_370_timer_init(struct device_node *np) { - struct clk *clk = of_clk_get(np, 0); + struct clk *clk; + int ret; + + clk = of_clk_get(np, 0); + if (IS_ERR(clk)) { + pr_err("Failed to get clock"); + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) + return ret; - BUG_ON(IS_ERR(clk)); - clk_prepare_enable(clk); timer_clk = clk_get_rate(clk) / TIMER_DIVIDER; timer25Mhz = false; - armada_370_xp_timer_common_init(np); + return armada_370_xp_timer_common_init(np); } CLOCKSOURCE_OF_DECLARE(armada_370, "marvell,armada-370-timer", armada_370_timer_init); diff --git a/drivers/clocksource/time-efm32.c b/drivers/clocksource/time-efm32.c index b06e4c2be..5ac344b38 100644 --- a/drivers/clocksource/time-efm32.c +++ b/drivers/clocksource/time-efm32.c @@ -233,10 +233,15 @@ static int __init efm32_clockevent_init(struct device_node *np) DIV_ROUND_CLOSEST(rate, 1024), 0xf, 0xffff); - setup_irq(irq, &efm32_clock_event_irq); + ret = setup_irq(irq, &efm32_clock_event_irq); + if (ret) { + pr_err("Failed setup irq"); + goto err_setup_irq; + } return 0; +err_setup_irq: err_get_irq: iounmap(base); @@ -255,16 +260,16 @@ err_clk_get: * This function asserts that we have exactly one clocksource and one * clock_event_device in the end. */ -static void __init efm32_timer_init(struct device_node *np) +static int __init efm32_timer_init(struct device_node *np) { static int has_clocksource, has_clockevent; - int ret; + int ret = 0; if (!has_clocksource) { ret = efm32_clocksource_init(np); if (!ret) { has_clocksource = 1; - return; + return 0; } } @@ -272,9 +277,11 @@ static void __init efm32_timer_init(struct device_node *np) ret = efm32_clockevent_init(np); if (!ret) { has_clockevent = 1; - return; + return 0; } } + + return ret; } CLOCKSOURCE_OF_DECLARE(efm32compat, "efm32,timer", efm32_timer_init); CLOCKSOURCE_OF_DECLARE(efm32, "energymicro,efm32-timer", efm32_timer_init); diff --git a/drivers/clocksource/time-lpc32xx.c b/drivers/clocksource/time-lpc32xx.c index daae61e8c..9649cfdb9 100644 --- a/drivers/clocksource/time-lpc32xx.c +++ b/drivers/clocksource/time-lpc32xx.c @@ -288,16 +288,16 @@ err_clk_enable: * This function asserts that we have exactly one clocksource and one * clock_event_device in the end. */ -static void __init lpc32xx_timer_init(struct device_node *np) +static int __init lpc32xx_timer_init(struct device_node *np) { static int has_clocksource, has_clockevent; - int ret; + int ret = 0; if (!has_clocksource) { ret = lpc32xx_clocksource_init(np); if (!ret) { has_clocksource = 1; - return; + return 0; } } @@ -305,8 +305,10 @@ static void __init lpc32xx_timer_init(struct device_node *np) ret = lpc32xx_clockevent_init(np); if (!ret) { has_clockevent = 1; - return; + return 0; } } + + return ret; } CLOCKSOURCE_OF_DECLARE(lpc32xx_timer, "nxp,lpc3220-timer", lpc32xx_timer_init); diff --git a/drivers/clocksource/time-orion.c b/drivers/clocksource/time-orion.c index 0ece7427b..a28f496e9 100644 --- a/drivers/clocksource/time-orion.c +++ b/drivers/clocksource/time-orion.c @@ -104,25 +104,36 @@ static struct irqaction orion_clkevt_irq = { .handler = orion_clkevt_irq_handler, }; -static void __init orion_timer_init(struct device_node *np) +static int __init orion_timer_init(struct device_node *np) { struct clk *clk; - int irq; + int irq, ret; /* timer registers are shared with watchdog timer */ timer_base = of_iomap(np, 0); - if (!timer_base) - panic("%s: unable to map resource\n", np->name); + if (!timer_base) { + pr_err("%s: unable to map resource\n", np->name); + return -ENXIO; + } clk = of_clk_get(np, 0); - if (IS_ERR(clk)) - panic("%s: unable to get clk\n", np->name); - clk_prepare_enable(clk); + if (IS_ERR(clk)) { + pr_err("%s: unable to get clk\n", np->name); + return PTR_ERR(clk); + } + + ret = clk_prepare_enable(clk); + if (ret) { + pr_err("Failed to prepare clock"); + return ret; + } /* we are only interested in timer1 irq */ irq = irq_of_parse_and_map(np, 1); - if (irq <= 0) - panic("%s: unable to parse timer1 irq\n", np->name); + if (irq <= 0) { + pr_err("%s: unable to parse timer1 irq\n", np->name); + return -EINVAL; + } /* setup timer0 as free-running clocksource */ writel(~0, timer_base + TIMER0_VAL); @@ -130,19 +141,30 @@ static void __init orion_timer_init(struct device_node *np) atomic_io_modify(timer_base + TIMER_CTRL, TIMER0_RELOAD_EN | TIMER0_EN, TIMER0_RELOAD_EN | TIMER0_EN); - clocksource_mmio_init(timer_base + TIMER0_VAL, "orion_clocksource", - clk_get_rate(clk), 300, 32, - clocksource_mmio_readl_down); + + ret = clocksource_mmio_init(timer_base + TIMER0_VAL, "orion_clocksource", + clk_get_rate(clk), 300, 32, + clocksource_mmio_readl_down); + if (ret) { + pr_err("Failed to initialize mmio timer"); + return ret; + } + sched_clock_register(orion_read_sched_clock, 32, clk_get_rate(clk)); /* setup timer1 as clockevent timer */ - if (setup_irq(irq, &orion_clkevt_irq)) - panic("%s: unable to setup irq\n", np->name); + ret = setup_irq(irq, &orion_clkevt_irq); + if (ret) { + pr_err("%s: unable to setup irq\n", np->name); + return ret; + } ticks_per_jiffy = (clk_get_rate(clk) + HZ/2) / HZ; orion_clkevt.cpumask = cpumask_of(0); orion_clkevt.irq = irq; clockevents_config_and_register(&orion_clkevt, clk_get_rate(clk), ORION_ONESHOT_MIN, ORION_ONESHOT_MAX); + + return 0; } CLOCKSOURCE_OF_DECLARE(orion_timer, "marvell,orion-timer", orion_timer_init); diff --git a/drivers/clocksource/time-pistachio.c b/drivers/clocksource/time-pistachio.c index 376e59bc5..a8e6c7df8 100644 --- a/drivers/clocksource/time-pistachio.c +++ b/drivers/clocksource/time-pistachio.c @@ -148,7 +148,7 @@ static struct pistachio_clocksource pcs_gpt = { }, }; -static void __init pistachio_clksrc_of_init(struct device_node *node) +static int __init pistachio_clksrc_of_init(struct device_node *node) { struct clk *sys_clk, *fast_clk; struct regmap *periph_regs; @@ -158,61 +158,61 @@ static void __init pistachio_clksrc_of_init(struct device_node *node) pcs_gpt.base = of_iomap(node, 0); if (!pcs_gpt.base) { pr_err("cannot iomap\n"); - return; + return -ENXIO; } periph_regs = syscon_regmap_lookup_by_phandle(node, "img,cr-periph"); if (IS_ERR(periph_regs)) { pr_err("cannot get peripheral regmap (%ld)\n", PTR_ERR(periph_regs)); - return; + return PTR_ERR(periph_regs); } /* Switch to using the fast counter clock */ ret = regmap_update_bits(periph_regs, PERIP_TIMER_CONTROL, 0xf, 0x0); if (ret) - return; + return ret; sys_clk = of_clk_get_by_name(node, "sys"); if (IS_ERR(sys_clk)) { pr_err("clock get failed (%ld)\n", PTR_ERR(sys_clk)); - return; + return PTR_ERR(sys_clk); } fast_clk = of_clk_get_by_name(node, "fast"); if (IS_ERR(fast_clk)) { pr_err("clock get failed (%lu)\n", PTR_ERR(fast_clk)); - return; + return PTR_ERR(fast_clk); } ret = clk_prepare_enable(sys_clk); if (ret < 0) { pr_err("failed to enable clock (%d)\n", ret); - return; + return ret; } ret = clk_prepare_enable(fast_clk); if (ret < 0) { pr_err("failed to enable clock (%d)\n", ret); clk_disable_unprepare(sys_clk); - return; + return ret; } rate = clk_get_rate(fast_clk); /* Disable irq's for clocksource usage */ - gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 0); - gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 1); - gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 2); - gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 3); + gpt_writel(pcs_gpt.base, 0, TIMER_IRQ_MASK, 0); + gpt_writel(pcs_gpt.base, 0, TIMER_IRQ_MASK, 1); + gpt_writel(pcs_gpt.base, 0, TIMER_IRQ_MASK, 2); + gpt_writel(pcs_gpt.base, 0, TIMER_IRQ_MASK, 3); /* Enable timer block */ writel(TIMER_ME_GLOBAL, pcs_gpt.base); raw_spin_lock_init(&pcs_gpt.lock); sched_clock_register(pistachio_read_sched_clock, 32, rate); - clocksource_register_hz(&pcs_gpt.cs, rate); + return clocksource_register_hz(&pcs_gpt.cs, rate); } CLOCKSOURCE_OF_DECLARE(pistachio_gptimer, "img,pistachio-gptimer", pistachio_clksrc_of_init); diff --git a/drivers/clocksource/timer-atlas7.c b/drivers/clocksource/timer-atlas7.c index 27fa13680..4334e0330 100644 --- a/drivers/clocksource/timer-atlas7.c +++ b/drivers/clocksource/timer-atlas7.c @@ -172,9 +172,9 @@ static struct irqaction sirfsoc_timer1_irq = { .handler = sirfsoc_timer_interrupt, }; -static int sirfsoc_local_timer_setup(struct clock_event_device *ce) +static int sirfsoc_local_timer_starting_cpu(unsigned int cpu) { - int cpu = smp_processor_id(); + struct clock_event_device *ce = per_cpu_ptr(sirfsoc_clockevent, cpu); struct irqaction *action; if (cpu == 0) @@ -203,54 +203,31 @@ static int sirfsoc_local_timer_setup(struct clock_event_device *ce) return 0; } -static void sirfsoc_local_timer_stop(struct clock_event_device *ce) +static int sirfsoc_local_timer_dying_cpu(unsigned int cpu) { - int cpu = smp_processor_id(); - sirfsoc_timer_count_disable(1); if (cpu == 0) remove_irq(sirfsoc_timer_irq.irq, &sirfsoc_timer_irq); else remove_irq(sirfsoc_timer1_irq.irq, &sirfsoc_timer1_irq); + return 0; } -static int sirfsoc_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - /* - * Grab cpu pointer in each case to avoid spurious - * preemptible warnings - */ - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_STARTING: - sirfsoc_local_timer_setup(this_cpu_ptr(sirfsoc_clockevent)); - break; - case CPU_DYING: - sirfsoc_local_timer_stop(this_cpu_ptr(sirfsoc_clockevent)); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block sirfsoc_cpu_nb = { - .notifier_call = sirfsoc_cpu_notify, -}; - -static void __init sirfsoc_clockevent_init(void) +static int __init sirfsoc_clockevent_init(void) { sirfsoc_clockevent = alloc_percpu(struct clock_event_device); BUG_ON(!sirfsoc_clockevent); - BUG_ON(register_cpu_notifier(&sirfsoc_cpu_nb)); - - /* Immediately configure the timer on the boot CPU */ - sirfsoc_local_timer_setup(this_cpu_ptr(sirfsoc_clockevent)); + /* Install and invoke hotplug callbacks */ + return cpuhp_setup_state(CPUHP_AP_MARCO_TIMER_STARTING, + "AP_MARCO_TIMER_STARTING", + sirfsoc_local_timer_starting_cpu, + sirfsoc_local_timer_dying_cpu); } /* initialize the kernel jiffy timer source */ -static void __init sirfsoc_atlas7_timer_init(struct device_node *np) +static int __init sirfsoc_atlas7_timer_init(struct device_node *np) { struct clk *clk; @@ -279,23 +256,29 @@ static void __init sirfsoc_atlas7_timer_init(struct device_node *np) BUG_ON(clocksource_register_hz(&sirfsoc_clocksource, atlas7_timer_rate)); - sirfsoc_clockevent_init(); + return sirfsoc_clockevent_init(); } -static void __init sirfsoc_of_timer_init(struct device_node *np) +static int __init sirfsoc_of_timer_init(struct device_node *np) { sirfsoc_timer_base = of_iomap(np, 0); - if (!sirfsoc_timer_base) - panic("unable to map timer cpu registers\n"); + if (!sirfsoc_timer_base) { + pr_err("unable to map timer cpu registers\n"); + return -ENXIO; + } sirfsoc_timer_irq.irq = irq_of_parse_and_map(np, 0); - if (!sirfsoc_timer_irq.irq) - panic("No irq passed for timer0 via DT\n"); + if (!sirfsoc_timer_irq.irq) { + pr_err("No irq passed for timer0 via DT\n"); + return -EINVAL; + } sirfsoc_timer1_irq.irq = irq_of_parse_and_map(np, 1); - if (!sirfsoc_timer1_irq.irq) - panic("No irq passed for timer1 via DT\n"); + if (!sirfsoc_timer1_irq.irq) { + pr_err("No irq passed for timer1 via DT\n"); + return -EINVAL; + } - sirfsoc_atlas7_timer_init(np); + return sirfsoc_atlas7_timer_init(np); } CLOCKSOURCE_OF_DECLARE(sirfsoc_atlas7_timer, "sirf,atlas7-tick", sirfsoc_of_timer_init); diff --git a/drivers/clocksource/timer-atmel-pit.c b/drivers/clocksource/timer-atmel-pit.c index d911c5dca..7f0f5b26d 100644 --- a/drivers/clocksource/timer-atmel-pit.c +++ b/drivers/clocksource/timer-atmel-pit.c @@ -177,7 +177,7 @@ static irqreturn_t at91sam926x_pit_interrupt(int irq, void *dev_id) /* * Set up both clocksource and clockevent support. */ -static void __init at91sam926x_pit_common_init(struct pit_data *data) +static int __init at91sam926x_pit_common_init(struct pit_data *data) { unsigned long pit_rate; unsigned bits; @@ -204,14 +204,21 @@ static void __init at91sam926x_pit_common_init(struct pit_data *data) data->clksrc.rating = 175; data->clksrc.read = read_pit_clk; data->clksrc.flags = CLOCK_SOURCE_IS_CONTINUOUS; - clocksource_register_hz(&data->clksrc, pit_rate); + + ret = clocksource_register_hz(&data->clksrc, pit_rate); + if (ret) { + pr_err("Failed to register clocksource"); + return ret; + } /* Set up irq handler */ ret = request_irq(data->irq, at91sam926x_pit_interrupt, IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL, "at91_tick", data); - if (ret) - panic(pr_fmt("Unable to setup IRQ\n")); + if (ret) { + pr_err("Unable to setup IRQ\n"); + return ret; + } /* Set up and register clockevents */ data->clkevt.name = "pit"; @@ -226,34 +233,49 @@ static void __init at91sam926x_pit_common_init(struct pit_data *data) data->clkevt.resume = at91sam926x_pit_resume; data->clkevt.suspend = at91sam926x_pit_suspend; clockevents_register_device(&data->clkevt); + + return 0; } -static void __init at91sam926x_pit_dt_init(struct device_node *node) +static int __init at91sam926x_pit_dt_init(struct device_node *node) { struct pit_data *data; + int ret; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) - panic(pr_fmt("Unable to allocate memory\n")); + return -ENOMEM; data->base = of_iomap(node, 0); - if (!data->base) - panic(pr_fmt("Could not map PIT address\n")); + if (!data->base) { + pr_err("Could not map PIT address\n"); + return -ENXIO; + } data->mck = of_clk_get(node, 0); if (IS_ERR(data->mck)) /* Fallback on clkdev for !CCF-based boards */ data->mck = clk_get(NULL, "mck"); - if (IS_ERR(data->mck)) - panic(pr_fmt("Unable to get mck clk\n")); + if (IS_ERR(data->mck)) { + pr_err("Unable to get mck clk\n"); + return PTR_ERR(data->mck); + } + + ret = clk_prepare_enable(data->mck); + if (ret) { + pr_err("Unable to enable mck\n"); + return ret; + } /* Get the interrupts property */ data->irq = irq_of_parse_and_map(node, 0); - if (!data->irq) - panic(pr_fmt("Unable to get IRQ from DT\n")); + if (!data->irq) { + pr_err("Unable to get IRQ from DT\n"); + return -EINVAL; + } - at91sam926x_pit_common_init(data); + return at91sam926x_pit_common_init(data); } CLOCKSOURCE_OF_DECLARE(at91sam926x_pit, "atmel,at91sam9260-pit", at91sam926x_pit_dt_init); diff --git a/drivers/clocksource/timer-atmel-st.c b/drivers/clocksource/timer-atmel-st.c index 29d21d68d..e90ab5b63 100644 --- a/drivers/clocksource/timer-atmel-st.c +++ b/drivers/clocksource/timer-atmel-st.c @@ -194,15 +194,17 @@ static struct clock_event_device clkevt = { /* * ST (system timer) module supports both clockevents and clocksource. */ -static void __init atmel_st_timer_init(struct device_node *node) +static int __init atmel_st_timer_init(struct device_node *node) { struct clk *sclk; unsigned int sclk_rate, val; int irq, ret; regmap_st = syscon_node_to_regmap(node); - if (IS_ERR(regmap_st)) - panic(pr_fmt("Unable to get regmap\n")); + if (IS_ERR(regmap_st)) { + pr_err("Unable to get regmap\n"); + return PTR_ERR(regmap_st); + } /* Disable all timer interrupts, and clear any pending ones */ regmap_write(regmap_st, AT91_ST_IDR, @@ -211,27 +213,37 @@ static void __init atmel_st_timer_init(struct device_node *node) /* Get the interrupts property */ irq = irq_of_parse_and_map(node, 0); - if (!irq) - panic(pr_fmt("Unable to get IRQ from DT\n")); + if (!irq) { + pr_err("Unable to get IRQ from DT\n"); + return -EINVAL; + } /* Make IRQs happen for the system timer */ ret = request_irq(irq, at91rm9200_timer_interrupt, IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL, "at91_tick", regmap_st); - if (ret) - panic(pr_fmt("Unable to setup IRQ\n")); + if (ret) { + pr_err("Unable to setup IRQ\n"); + return ret; + } sclk = of_clk_get(node, 0); - if (IS_ERR(sclk)) - panic(pr_fmt("Unable to get slow clock\n")); + if (IS_ERR(sclk)) { + pr_err("Unable to get slow clock\n"); + return PTR_ERR(sclk); + } - clk_prepare_enable(sclk); - if (ret) - panic(pr_fmt("Could not enable slow clock\n")); + ret = clk_prepare_enable(sclk); + if (ret) { + pr_err("Could not enable slow clock\n"); + return ret; + } sclk_rate = clk_get_rate(sclk); - if (!sclk_rate) - panic(pr_fmt("Invalid slow clock rate\n")); + if (!sclk_rate) { + pr_err("Invalid slow clock rate\n"); + return -EINVAL; + } timer_latch = (sclk_rate + HZ / 2) / HZ; /* The 32KiHz "Slow Clock" (tick every 30517.58 nanoseconds) is used @@ -246,7 +258,7 @@ static void __init atmel_st_timer_init(struct device_node *node) 2, AT91_ST_ALMV); /* register clocksource */ - clocksource_register_hz(&clk32k, sclk_rate); + return clocksource_register_hz(&clk32k, sclk_rate); } CLOCKSOURCE_OF_DECLARE(atmel_st_timer, "atmel,at91rm9200-st", atmel_st_timer_init); diff --git a/drivers/clocksource/timer-digicolor.c b/drivers/clocksource/timer-digicolor.c index a536eeb63..10318cc99 100644 --- a/drivers/clocksource/timer-digicolor.c +++ b/drivers/clocksource/timer-digicolor.c @@ -63,7 +63,7 @@ struct digicolor_timer { int timer_id; /* one of TIMER_* */ }; -struct digicolor_timer *dc_timer(struct clock_event_device *ce) +static struct digicolor_timer *dc_timer(struct clock_event_device *ce) { return container_of(ce, struct digicolor_timer, ce); } @@ -148,7 +148,7 @@ static u64 notrace digicolor_timer_sched_read(void) return ~readl(dc_timer_dev.base + COUNT(TIMER_B)); } -static void __init digicolor_timer_init(struct device_node *node) +static int __init digicolor_timer_init(struct device_node *node) { unsigned long rate; struct clk *clk; @@ -161,19 +161,19 @@ static void __init digicolor_timer_init(struct device_node *node) dc_timer_dev.base = of_iomap(node, 0); if (!dc_timer_dev.base) { pr_err("Can't map registers"); - return; + return -ENXIO; } irq = irq_of_parse_and_map(node, dc_timer_dev.timer_id); if (irq <= 0) { pr_err("Can't parse IRQ"); - return; + return -EINVAL; } clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("Can't get timer clock"); - return; + return PTR_ERR(clk); } clk_prepare_enable(clk); rate = clk_get_rate(clk); @@ -190,13 +190,17 @@ static void __init digicolor_timer_init(struct device_node *node) ret = request_irq(irq, digicolor_timer_interrupt, IRQF_TIMER | IRQF_IRQPOLL, "digicolor_timerC", &dc_timer_dev.ce); - if (ret) + if (ret) { pr_warn("request of timer irq %d failed (%d)\n", irq, ret); + return ret; + } dc_timer_dev.ce.cpumask = cpu_possible_mask; dc_timer_dev.ce.irq = irq; clockevents_config_and_register(&dc_timer_dev.ce, rate, 0, 0xffffffff); + + return 0; } CLOCKSOURCE_OF_DECLARE(conexant_digicolor, "cnxt,cx92755-timer", digicolor_timer_init); diff --git a/drivers/clocksource/timer-imx-gpt.c b/drivers/clocksource/timer-imx-gpt.c index 99ec96769..f595460bf 100644 --- a/drivers/clocksource/timer-imx-gpt.c +++ b/drivers/clocksource/timer-imx-gpt.c @@ -407,8 +407,10 @@ static const struct imx_gpt_data imx6dl_gpt_data = { .set_next_event = v2_set_next_event, }; -static void __init _mxc_timer_init(struct imx_timer *imxtm) +static int __init _mxc_timer_init(struct imx_timer *imxtm) { + int ret; + switch (imxtm->type) { case GPT_TYPE_IMX1: imxtm->gpt = &imx1_gpt_data; @@ -423,12 +425,12 @@ static void __init _mxc_timer_init(struct imx_timer *imxtm) imxtm->gpt = &imx6dl_gpt_data; break; default: - BUG(); + return -EINVAL; } if (IS_ERR(imxtm->clk_per)) { pr_err("i.MX timer: unable to get clk\n"); - return; + return PTR_ERR(imxtm->clk_per); } if (!IS_ERR(imxtm->clk_ipg)) @@ -446,8 +448,11 @@ static void __init _mxc_timer_init(struct imx_timer *imxtm) imxtm->gpt->gpt_setup_tctl(imxtm); /* init and register the timer to the framework */ - mxc_clocksource_init(imxtm); - mxc_clockevent_init(imxtm); + ret = mxc_clocksource_init(imxtm); + if (ret) + return ret; + + return mxc_clockevent_init(imxtm); } void __init mxc_timer_init(unsigned long pbase, int irq, enum imx_gpt_type type) @@ -469,21 +474,27 @@ void __init mxc_timer_init(unsigned long pbase, int irq, enum imx_gpt_type type) _mxc_timer_init(imxtm); } -static void __init mxc_timer_init_dt(struct device_node *np, enum imx_gpt_type type) +static int __init mxc_timer_init_dt(struct device_node *np, enum imx_gpt_type type) { struct imx_timer *imxtm; static int initialized; + int ret; /* Support one instance only */ if (initialized) - return; + return 0; imxtm = kzalloc(sizeof(*imxtm), GFP_KERNEL); - BUG_ON(!imxtm); + if (!imxtm) + return -ENOMEM; imxtm->base = of_iomap(np, 0); - WARN_ON(!imxtm->base); + if (!imxtm->base) + return -ENXIO; + imxtm->irq = irq_of_parse_and_map(np, 0); + if (imxtm->irq <= 0) + return -EINVAL; imxtm->clk_ipg = of_clk_get_by_name(np, "ipg"); @@ -494,22 +505,26 @@ static void __init mxc_timer_init_dt(struct device_node *np, enum imx_gpt_type imxtm->type = type; - _mxc_timer_init(imxtm); + ret = _mxc_timer_init(imxtm); + if (ret) + return ret; initialized = 1; + + return 0; } -static void __init imx1_timer_init_dt(struct device_node *np) +static int __init imx1_timer_init_dt(struct device_node *np) { - mxc_timer_init_dt(np, GPT_TYPE_IMX1); + return mxc_timer_init_dt(np, GPT_TYPE_IMX1); } -static void __init imx21_timer_init_dt(struct device_node *np) +static int __init imx21_timer_init_dt(struct device_node *np) { - mxc_timer_init_dt(np, GPT_TYPE_IMX21); + return mxc_timer_init_dt(np, GPT_TYPE_IMX21); } -static void __init imx31_timer_init_dt(struct device_node *np) +static int __init imx31_timer_init_dt(struct device_node *np) { enum imx_gpt_type type = GPT_TYPE_IMX31; @@ -522,12 +537,12 @@ static void __init imx31_timer_init_dt(struct device_node *np) if (of_machine_is_compatible("fsl,imx6dl")) type = GPT_TYPE_IMX6DL; - mxc_timer_init_dt(np, type); + return mxc_timer_init_dt(np, type); } -static void __init imx6dl_timer_init_dt(struct device_node *np) +static int __init imx6dl_timer_init_dt(struct device_node *np) { - mxc_timer_init_dt(np, GPT_TYPE_IMX6DL); + return mxc_timer_init_dt(np, GPT_TYPE_IMX6DL); } CLOCKSOURCE_OF_DECLARE(imx1_timer, "fsl,imx1-gpt", imx1_timer_init_dt); diff --git a/drivers/clocksource/timer-integrator-ap.c b/drivers/clocksource/timer-integrator-ap.c index 3f59ac218..df6e672af 100644 --- a/drivers/clocksource/timer-integrator-ap.c +++ b/drivers/clocksource/timer-integrator-ap.c @@ -36,11 +36,12 @@ static u64 notrace integrator_read_sched_clock(void) return -readl(sched_clk_base + TIMER_VALUE); } -static void integrator_clocksource_init(unsigned long inrate, - void __iomem *base) +static int integrator_clocksource_init(unsigned long inrate, + void __iomem *base) { u32 ctrl = TIMER_CTRL_ENABLE | TIMER_CTRL_PERIODIC; unsigned long rate = inrate; + int ret; if (rate >= 1500000) { rate /= 16; @@ -50,11 +51,15 @@ static void integrator_clocksource_init(unsigned long inrate, writel(0xffff, base + TIMER_LOAD); writel(ctrl, base + TIMER_CTRL); - clocksource_mmio_init(base + TIMER_VALUE, "timer2", - rate, 200, 16, clocksource_mmio_readl_down); + ret = clocksource_mmio_init(base + TIMER_VALUE, "timer2", + rate, 200, 16, clocksource_mmio_readl_down); + if (ret) + return ret; sched_clk_base = base; sched_clock_register(integrator_read_sched_clock, 16, rate); + + return 0; } static unsigned long timer_reload; @@ -138,11 +143,12 @@ static struct irqaction integrator_timer_irq = { .dev_id = &integrator_clockevent, }; -static void integrator_clockevent_init(unsigned long inrate, - void __iomem *base, int irq) +static int integrator_clockevent_init(unsigned long inrate, + void __iomem *base, int irq) { unsigned long rate = inrate; unsigned int ctrl = 0; + int ret; clkevt_base = base; /* Calculate and program a divisor */ @@ -156,14 +162,18 @@ static void integrator_clockevent_init(unsigned long inrate, timer_reload = rate / HZ; writel(ctrl, clkevt_base + TIMER_CTRL); - setup_irq(irq, &integrator_timer_irq); + ret = setup_irq(irq, &integrator_timer_irq); + if (ret) + return ret; + clockevents_config_and_register(&integrator_clockevent, rate, 1, 0xffffU); + return 0; } -static void __init integrator_ap_timer_init_of(struct device_node *node) +static int __init integrator_ap_timer_init_of(struct device_node *node) { const char *path; void __iomem *base; @@ -176,12 +186,12 @@ static void __init integrator_ap_timer_init_of(struct device_node *node) base = of_io_request_and_map(node, 0, "integrator-timer"); if (IS_ERR(base)) - return; + return PTR_ERR(base); clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("No clock for %s\n", node->name); - return; + return PTR_ERR(clk); } clk_prepare_enable(clk); rate = clk_get_rate(clk); @@ -189,30 +199,37 @@ static void __init integrator_ap_timer_init_of(struct device_node *node) err = of_property_read_string(of_aliases, "arm,timer-primary", &path); - if (WARN_ON(err)) - return; + if (err) { + pr_warn("Failed to read property"); + return err; + } + pri_node = of_find_node_by_path(path); + err = of_property_read_string(of_aliases, "arm,timer-secondary", &path); - if (WARN_ON(err)) - return; + if (err) { + pr_warn("Failed to read property"); + return err; + } + + sec_node = of_find_node_by_path(path); - if (node == pri_node) { + if (node == pri_node) /* The primary timer lacks IRQ, use as clocksource */ - integrator_clocksource_init(rate, base); - return; - } + return integrator_clocksource_init(rate, base); if (node == sec_node) { /* The secondary timer will drive the clock event */ irq = irq_of_parse_and_map(node, 0); - integrator_clockevent_init(rate, base, irq); - return; + return integrator_clockevent_init(rate, base, irq); } pr_info("Timer @%p unused\n", base); clk_disable_unprepare(clk); + + return 0; } CLOCKSOURCE_OF_DECLARE(integrator_ap_timer, "arm,integrator-timer", diff --git a/drivers/clocksource/timer-keystone.c b/drivers/clocksource/timer-keystone.c index 1cea08cf6..ab68a47ab 100644 --- a/drivers/clocksource/timer-keystone.c +++ b/drivers/clocksource/timer-keystone.c @@ -144,7 +144,7 @@ static int keystone_set_periodic(struct clock_event_device *evt) return 0; } -static void __init keystone_timer_init(struct device_node *np) +static int __init keystone_timer_init(struct device_node *np) { struct clock_event_device *event_dev = &timer.event_dev; unsigned long rate; @@ -154,20 +154,20 @@ static void __init keystone_timer_init(struct device_node *np) irq = irq_of_parse_and_map(np, 0); if (!irq) { pr_err("%s: failed to map interrupts\n", __func__); - return; + return -EINVAL; } timer.base = of_iomap(np, 0); if (!timer.base) { pr_err("%s: failed to map registers\n", __func__); - return; + return -ENXIO; } clk = of_clk_get(np, 0); if (IS_ERR(clk)) { pr_err("%s: failed to get clock\n", __func__); iounmap(timer.base); - return; + return PTR_ERR(clk); } error = clk_prepare_enable(clk); @@ -219,11 +219,12 @@ static void __init keystone_timer_init(struct device_node *np) clockevents_config_and_register(event_dev, rate, 1, ULONG_MAX); pr_info("keystone timer clock @%lu Hz\n", rate); - return; + return 0; err: clk_put(clk); iounmap(timer.base); + return error; } CLOCKSOURCE_OF_DECLARE(keystone_timer, "ti,keystone-timer", - keystone_timer_init); + keystone_timer_init); diff --git a/drivers/clocksource/timer-nps.c b/drivers/clocksource/timer-nps.c index d46108920..70c149af8 100644 --- a/drivers/clocksource/timer-nps.c +++ b/drivers/clocksource/timer-nps.c @@ -55,8 +55,8 @@ static cycle_t nps_clksrc_read(struct clocksource *clksrc) return (cycle_t)ioread32be(nps_msu_reg_low_addr[cluster]); } -static void __init nps_setup_clocksource(struct device_node *node, - struct clk *clk) +static int __init nps_setup_clocksource(struct device_node *node, + struct clk *clk) { int ret, cluster; @@ -68,7 +68,7 @@ static void __init nps_setup_clocksource(struct device_node *node, ret = clk_prepare_enable(clk); if (ret) { pr_err("Couldn't enable parent clock\n"); - return; + return ret; } nps_timer_rate = clk_get_rate(clk); @@ -79,19 +79,21 @@ static void __init nps_setup_clocksource(struct device_node *node, pr_err("Couldn't register clock source.\n"); clk_disable_unprepare(clk); } + + return ret; } -static void __init nps_timer_init(struct device_node *node) +static int __init nps_timer_init(struct device_node *node) { struct clk *clk; clk = of_clk_get(node, 0); if (IS_ERR(clk)) { pr_err("Can't get timer clock.\n"); - return; + return PTR_ERR(clk); } - nps_setup_clocksource(node, clk); + return nps_setup_clocksource(node, clk); } CLOCKSOURCE_OF_DECLARE(ezchip_nps400_clksrc, "ezchip,nps400-timer", diff --git a/drivers/clocksource/timer-oxnas-rps.c b/drivers/clocksource/timer-oxnas-rps.c new file mode 100644 index 000000000..bd887e2a8 --- /dev/null +++ b/drivers/clocksource/timer-oxnas-rps.c @@ -0,0 +1,297 @@ +/* + * drivers/clocksource/timer-oxnas-rps.c + * + * Copyright (C) 2009 Oxford Semiconductor Ltd + * Copyright (C) 2013 Ma Haijun + * Copyright (C) 2016 Neil Armstrong + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* TIMER1 used as tick + * TIMER2 used as clocksource + */ + +/* Registers definitions */ + +#define TIMER_LOAD_REG 0x0 +#define TIMER_CURR_REG 0x4 +#define TIMER_CTRL_REG 0x8 +#define TIMER_CLRINT_REG 0xC + +#define TIMER_BITS 24 + +#define TIMER_MAX_VAL (BIT(TIMER_BITS) - 1) + +#define TIMER_PERIODIC BIT(6) +#define TIMER_ENABLE BIT(7) + +#define TIMER_DIV1 (0) +#define TIMER_DIV16 (1 << 2) +#define TIMER_DIV256 (2 << 2) + +#define TIMER1_REG_OFFSET 0 +#define TIMER2_REG_OFFSET 0x20 + +/* Clockevent & Clocksource data */ + +struct oxnas_rps_timer { + struct clock_event_device clkevent; + void __iomem *clksrc_base; + void __iomem *clkevt_base; + unsigned long timer_period; + unsigned int timer_prescaler; + struct clk *clk; + int irq; +}; + +static irqreturn_t oxnas_rps_timer_irq(int irq, void *dev_id) +{ + struct oxnas_rps_timer *rps = dev_id; + + writel_relaxed(0, rps->clkevt_base + TIMER_CLRINT_REG); + + rps->clkevent.event_handler(&rps->clkevent); + + return IRQ_HANDLED; +} + +static void oxnas_rps_timer_config(struct oxnas_rps_timer *rps, + unsigned long period, + unsigned int periodic) +{ + uint32_t cfg = rps->timer_prescaler; + + if (period) + cfg |= TIMER_ENABLE; + + if (periodic) + cfg |= TIMER_PERIODIC; + + writel_relaxed(period, rps->clkevt_base + TIMER_LOAD_REG); + writel_relaxed(cfg, rps->clkevt_base + TIMER_CTRL_REG); +} + +static int oxnas_rps_timer_shutdown(struct clock_event_device *evt) +{ + struct oxnas_rps_timer *rps = + container_of(evt, struct oxnas_rps_timer, clkevent); + + oxnas_rps_timer_config(rps, 0, 0); + + return 0; +} + +static int oxnas_rps_timer_set_periodic(struct clock_event_device *evt) +{ + struct oxnas_rps_timer *rps = + container_of(evt, struct oxnas_rps_timer, clkevent); + + oxnas_rps_timer_config(rps, rps->timer_period, 1); + + return 0; +} + +static int oxnas_rps_timer_set_oneshot(struct clock_event_device *evt) +{ + struct oxnas_rps_timer *rps = + container_of(evt, struct oxnas_rps_timer, clkevent); + + oxnas_rps_timer_config(rps, rps->timer_period, 0); + + return 0; +} + +static int oxnas_rps_timer_next_event(unsigned long delta, + struct clock_event_device *evt) +{ + struct oxnas_rps_timer *rps = + container_of(evt, struct oxnas_rps_timer, clkevent); + + oxnas_rps_timer_config(rps, delta, 0); + + return 0; +} + +static int __init oxnas_rps_clockevent_init(struct oxnas_rps_timer *rps) +{ + ulong clk_rate = clk_get_rate(rps->clk); + ulong timer_rate; + + /* Start with prescaler 1 */ + rps->timer_prescaler = TIMER_DIV1; + rps->timer_period = DIV_ROUND_UP(clk_rate, HZ); + timer_rate = clk_rate; + + if (rps->timer_period > TIMER_MAX_VAL) { + rps->timer_prescaler = TIMER_DIV16; + timer_rate = clk_rate / 16; + rps->timer_period = DIV_ROUND_UP(timer_rate, HZ); + } + if (rps->timer_period > TIMER_MAX_VAL) { + rps->timer_prescaler = TIMER_DIV256; + timer_rate = clk_rate / 256; + rps->timer_period = DIV_ROUND_UP(timer_rate, HZ); + } + + rps->clkevent.name = "oxnas-rps"; + rps->clkevent.features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_DYNIRQ; + rps->clkevent.tick_resume = oxnas_rps_timer_shutdown; + rps->clkevent.set_state_shutdown = oxnas_rps_timer_shutdown; + rps->clkevent.set_state_periodic = oxnas_rps_timer_set_periodic; + rps->clkevent.set_state_oneshot = oxnas_rps_timer_set_oneshot; + rps->clkevent.set_next_event = oxnas_rps_timer_next_event; + rps->clkevent.rating = 200; + rps->clkevent.cpumask = cpu_possible_mask; + rps->clkevent.irq = rps->irq; + clockevents_config_and_register(&rps->clkevent, + timer_rate, + 1, + TIMER_MAX_VAL); + + pr_info("Registered clock event rate %luHz prescaler %x period %lu\n", + clk_rate, + rps->timer_prescaler, + rps->timer_period); + + return 0; +} + +/* Clocksource */ + +static void __iomem *timer_sched_base; + +static u64 notrace oxnas_rps_read_sched_clock(void) +{ + return ~readl_relaxed(timer_sched_base); +} + +static int __init oxnas_rps_clocksource_init(struct oxnas_rps_timer *rps) +{ + ulong clk_rate = clk_get_rate(rps->clk); + int ret; + + /* use prescale 16 */ + clk_rate = clk_rate / 16; + + writel_relaxed(TIMER_MAX_VAL, rps->clksrc_base + TIMER_LOAD_REG); + writel_relaxed(TIMER_PERIODIC | TIMER_ENABLE | TIMER_DIV16, + rps->clksrc_base + TIMER_CTRL_REG); + + timer_sched_base = rps->clksrc_base + TIMER_CURR_REG; + sched_clock_register(oxnas_rps_read_sched_clock, + TIMER_BITS, clk_rate); + ret = clocksource_mmio_init(timer_sched_base, + "oxnas_rps_clocksource_timer", + clk_rate, 250, TIMER_BITS, + clocksource_mmio_readl_down); + if (WARN_ON(ret)) { + pr_err("can't register clocksource\n"); + return ret; + } + + pr_info("Registered clocksource rate %luHz\n", clk_rate); + + return 0; +} + +static int __init oxnas_rps_timer_init(struct device_node *np) +{ + struct oxnas_rps_timer *rps; + void __iomem *base; + int ret; + + rps = kzalloc(sizeof(*rps), GFP_KERNEL); + if (!rps) + return -ENOMEM; + + rps->clk = of_clk_get(np, 0); + if (IS_ERR(rps->clk)) { + ret = PTR_ERR(rps->clk); + goto err_alloc; + } + + ret = clk_prepare_enable(rps->clk); + if (ret) + goto err_clk; + + base = of_iomap(np, 0); + if (!base) { + ret = -ENXIO; + goto err_clk_prepare; + } + + rps->irq = irq_of_parse_and_map(np, 0); + if (rps->irq < 0) { + ret = -EINVAL; + goto err_iomap; + } + + rps->clkevt_base = base + TIMER1_REG_OFFSET; + rps->clksrc_base = base + TIMER2_REG_OFFSET; + + /* Disable timers */ + writel_relaxed(0, rps->clkevt_base + TIMER_CTRL_REG); + writel_relaxed(0, rps->clksrc_base + TIMER_CTRL_REG); + writel_relaxed(0, rps->clkevt_base + TIMER_LOAD_REG); + writel_relaxed(0, rps->clksrc_base + TIMER_LOAD_REG); + writel_relaxed(0, rps->clkevt_base + TIMER_CLRINT_REG); + writel_relaxed(0, rps->clksrc_base + TIMER_CLRINT_REG); + + ret = request_irq(rps->irq, oxnas_rps_timer_irq, + IRQF_TIMER | IRQF_IRQPOLL, + "rps-timer", rps); + if (ret) + goto err_iomap; + + ret = oxnas_rps_clocksource_init(rps); + if (ret) + goto err_irqreq; + + ret = oxnas_rps_clockevent_init(rps); + if (ret) + goto err_irqreq; + + return 0; + +err_irqreq: + free_irq(rps->irq, rps); +err_iomap: + iounmap(base); +err_clk_prepare: + clk_disable_unprepare(rps->clk); +err_clk: + clk_put(rps->clk); +err_alloc: + kfree(rps); + + return ret; +} + +CLOCKSOURCE_OF_DECLARE(ox810se_rps, + "oxsemi,ox810se-rps-timer", oxnas_rps_timer_init); diff --git a/drivers/clocksource/timer-prima2.c b/drivers/clocksource/timer-prima2.c index 2854c663e..c32148ec7 100644 --- a/drivers/clocksource/timer-prima2.c +++ b/drivers/clocksource/timer-prima2.c @@ -19,7 +19,6 @@ #include #include #include -#include #define PRIMA2_CLOCK_FREQ 1000000 @@ -189,24 +188,36 @@ static void __init sirfsoc_clockevent_init(void) } /* initialize the kernel jiffy timer source */ -static void __init sirfsoc_prima2_timer_init(struct device_node *np) +static int __init sirfsoc_prima2_timer_init(struct device_node *np) { unsigned long rate; struct clk *clk; + int ret; clk = of_clk_get(np, 0); - BUG_ON(IS_ERR(clk)); + if (IS_ERR(clk)) { + pr_err("Failed to get clock"); + return PTR_ERR(clk); + } - BUG_ON(clk_prepare_enable(clk)); + ret = clk_prepare_enable(clk); + if (ret) { + pr_err("Failed to enable clock"); + return ret; + } rate = clk_get_rate(clk); - BUG_ON(rate < PRIMA2_CLOCK_FREQ); - BUG_ON(rate % PRIMA2_CLOCK_FREQ); + if (rate < PRIMA2_CLOCK_FREQ || rate % PRIMA2_CLOCK_FREQ) { + pr_err("Invalid clock rate"); + return -EINVAL; + } sirfsoc_timer_base = of_iomap(np, 0); - if (!sirfsoc_timer_base) - panic("unable to map timer cpu registers\n"); + if (!sirfsoc_timer_base) { + pr_err("unable to map timer cpu registers\n"); + return -ENXIO; + } sirfsoc_timer_irq.irq = irq_of_parse_and_map(np, 0); @@ -216,14 +227,23 @@ static void __init sirfsoc_prima2_timer_init(struct device_node *np) writel_relaxed(0, sirfsoc_timer_base + SIRFSOC_TIMER_COUNTER_HI); writel_relaxed(BIT(0), sirfsoc_timer_base + SIRFSOC_TIMER_STATUS); - BUG_ON(clocksource_register_hz(&sirfsoc_clocksource, - PRIMA2_CLOCK_FREQ)); + ret = clocksource_register_hz(&sirfsoc_clocksource, PRIMA2_CLOCK_FREQ); + if (ret) { + pr_err("Failed to register clocksource"); + return ret; + } sched_clock_register(sirfsoc_read_sched_clock, 64, PRIMA2_CLOCK_FREQ); - BUG_ON(setup_irq(sirfsoc_timer_irq.irq, &sirfsoc_timer_irq)); + ret = setup_irq(sirfsoc_timer_irq.irq, &sirfsoc_timer_irq); + if (ret) { + pr_err("Failed to setup irq"); + return ret; + } sirfsoc_clockevent_init(); + + return 0; } CLOCKSOURCE_OF_DECLARE(sirfsoc_prima2_timer, "sirf,prima2-tick", sirfsoc_prima2_timer_init); diff --git a/drivers/clocksource/timer-sp804.c b/drivers/clocksource/timer-sp804.c index 5f45b9ade..d07863388 100644 --- a/drivers/clocksource/timer-sp804.c +++ b/drivers/clocksource/timer-sp804.c @@ -77,7 +77,7 @@ void __init sp804_timer_disable(void __iomem *base) writel(0, base + TIMER_CTRL); } -void __init __sp804_clocksource_and_sched_clock_init(void __iomem *base, +int __init __sp804_clocksource_and_sched_clock_init(void __iomem *base, const char *name, struct clk *clk, int use_sched_clock) @@ -89,14 +89,13 @@ void __init __sp804_clocksource_and_sched_clock_init(void __iomem *base, if (IS_ERR(clk)) { pr_err("sp804: clock not found: %d\n", (int)PTR_ERR(clk)); - return; + return PTR_ERR(clk); } } rate = sp804_get_clock_rate(clk); - if (rate < 0) - return; + return -EINVAL; /* setup timer 0 as free-running clocksource */ writel(0, base + TIMER_CTRL); @@ -112,6 +111,8 @@ void __init __sp804_clocksource_and_sched_clock_init(void __iomem *base, sched_clock_base = base; sched_clock_register(sp804_read, 32, rate); } + + return 0; } @@ -186,7 +187,7 @@ static struct irqaction sp804_timer_irq = { .dev_id = &sp804_clockevent, }; -void __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struct clk *clk, const char *name) +int __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struct clk *clk, const char *name) { struct clock_event_device *evt = &sp804_clockevent; long rate; @@ -196,12 +197,12 @@ void __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struc if (IS_ERR(clk)) { pr_err("sp804: %s clock not found: %d\n", name, (int)PTR_ERR(clk)); - return; + return PTR_ERR(clk); } rate = sp804_get_clock_rate(clk); if (rate < 0) - return; + return -EINVAL; clkevt_base = base; clkevt_reload = DIV_ROUND_CLOSEST(rate, HZ); @@ -213,27 +214,31 @@ void __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struc setup_irq(irq, &sp804_timer_irq); clockevents_config_and_register(evt, rate, 0xf, 0xffffffff); + + return 0; } -static void __init sp804_of_init(struct device_node *np) +static int __init sp804_of_init(struct device_node *np) { static bool initialized = false; void __iomem *base; - int irq; + int irq, ret = -EINVAL; u32 irq_num = 0; struct clk *clk1, *clk2; const char *name = of_get_property(np, "compatible", NULL); base = of_iomap(np, 0); - if (WARN_ON(!base)) - return; + if (!base) + return -ENXIO; /* Ensure timers are disabled */ writel(0, base + TIMER_CTRL); writel(0, base + TIMER_2_BASE + TIMER_CTRL); - if (initialized || !of_device_is_available(np)) + if (initialized || !of_device_is_available(np)) { + ret = -EINVAL; goto err; + } clk1 = of_clk_get(np, 0); if (IS_ERR(clk1)) @@ -256,35 +261,53 @@ static void __init sp804_of_init(struct device_node *np) of_property_read_u32(np, "arm,sp804-has-irq", &irq_num); if (irq_num == 2) { - __sp804_clockevents_init(base + TIMER_2_BASE, irq, clk2, name); - __sp804_clocksource_and_sched_clock_init(base, name, clk1, 1); + + ret = __sp804_clockevents_init(base + TIMER_2_BASE, irq, clk2, name); + if (ret) + goto err; + + ret = __sp804_clocksource_and_sched_clock_init(base, name, clk1, 1); + if (ret) + goto err; } else { - __sp804_clockevents_init(base, irq, clk1 , name); - __sp804_clocksource_and_sched_clock_init(base + TIMER_2_BASE, - name, clk2, 1); + + ret = __sp804_clockevents_init(base, irq, clk1 , name); + if (ret) + goto err; + + ret =__sp804_clocksource_and_sched_clock_init(base + TIMER_2_BASE, + name, clk2, 1); + if (ret) + goto err; } initialized = true; - return; + return 0; err: iounmap(base); + return ret; } CLOCKSOURCE_OF_DECLARE(sp804, "arm,sp804", sp804_of_init); -static void __init integrator_cp_of_init(struct device_node *np) +static int __init integrator_cp_of_init(struct device_node *np) { static int init_count = 0; void __iomem *base; - int irq; + int irq, ret = -EINVAL; const char *name = of_get_property(np, "compatible", NULL); struct clk *clk; base = of_iomap(np, 0); - if (WARN_ON(!base)) - return; + if (!base) { + pr_err("Failed to iomap"); + return -ENXIO; + } + clk = of_clk_get(np, 0); - if (WARN_ON(IS_ERR(clk))) - return; + if (IS_ERR(clk)) { + pr_err("Failed to get clock"); + return PTR_ERR(clk); + } /* Ensure timer is disabled */ writel(0, base + TIMER_CTRL); @@ -292,19 +315,24 @@ static void __init integrator_cp_of_init(struct device_node *np) if (init_count == 2 || !of_device_is_available(np)) goto err; - if (!init_count) - __sp804_clocksource_and_sched_clock_init(base, name, clk, 0); - else { + if (!init_count) { + ret = __sp804_clocksource_and_sched_clock_init(base, name, clk, 0); + if (ret) + goto err; + } else { irq = irq_of_parse_and_map(np, 0); if (irq <= 0) goto err; - __sp804_clockevents_init(base, irq, clk, name); + ret = __sp804_clockevents_init(base, irq, clk, name); + if (ret) + goto err; } init_count++; - return; + return 0; err: iounmap(base); + return ret; } CLOCKSOURCE_OF_DECLARE(intcp, "arm,integrator-cp-timer", integrator_cp_of_init); diff --git a/drivers/clocksource/timer-stm32.c b/drivers/clocksource/timer-stm32.c index f3dcb7679..1b2574c4f 100644 --- a/drivers/clocksource/timer-stm32.c +++ b/drivers/clocksource/timer-stm32.c @@ -98,7 +98,7 @@ static struct stm32_clock_event_ddata clock_event_ddata = { }, }; -static void __init stm32_clockevent_init(struct device_node *np) +static int __init stm32_clockevent_init(struct device_node *np) { struct stm32_clock_event_ddata *data = &clock_event_ddata; struct clk *clk; @@ -130,12 +130,14 @@ static void __init stm32_clockevent_init(struct device_node *np) data->base = of_iomap(np, 0); if (!data->base) { + ret = -ENXIO; pr_err("failed to map registers for clockevent\n"); goto err_iomap; } irq = irq_of_parse_and_map(np, 0); if (!irq) { + ret = -EINVAL; pr_err("%s: failed to get irq.\n", np->full_name); goto err_get_irq; } @@ -173,7 +175,7 @@ static void __init stm32_clockevent_init(struct device_node *np) pr_info("%s: STM32 clockevent driver initialized (%d bits)\n", np->full_name, bits); - return; + return ret; err_get_irq: iounmap(data->base); @@ -182,7 +184,7 @@ err_iomap: err_clk_enable: clk_put(clk); err_clk_get: - return; + return ret; } CLOCKSOURCE_OF_DECLARE(stm32, "st,stm32-timer", stm32_clockevent_init); diff --git a/drivers/clocksource/timer-sun5i.c b/drivers/clocksource/timer-sun5i.c index 24c83f9ef..c184eb841 100644 --- a/drivers/clocksource/timer-sun5i.c +++ b/drivers/clocksource/timer-sun5i.c @@ -311,33 +311,42 @@ err_free: return ret; } -static void __init sun5i_timer_init(struct device_node *node) +static int __init sun5i_timer_init(struct device_node *node) { struct reset_control *rstc; void __iomem *timer_base; struct clk *clk; - int irq; + int irq, ret; timer_base = of_io_request_and_map(node, 0, of_node_full_name(node)); - if (IS_ERR(timer_base)) - panic("Can't map registers"); + if (IS_ERR(timer_base)) { + pr_err("Can't map registers"); + return PTR_ERR(timer_base);; + } irq = irq_of_parse_and_map(node, 0); - if (irq <= 0) - panic("Can't parse IRQ"); + if (irq <= 0) { + pr_err("Can't parse IRQ"); + return -EINVAL; + } clk = of_clk_get(node, 0); - if (IS_ERR(clk)) - panic("Can't get timer clock"); + if (IS_ERR(clk)) { + pr_err("Can't get timer clock"); + return PTR_ERR(clk); + } rstc = of_reset_control_get(node, NULL); if (!IS_ERR(rstc)) reset_control_deassert(rstc); - sun5i_setup_clocksource(node, timer_base, clk, irq); - sun5i_setup_clockevent(node, timer_base, clk, irq); + ret = sun5i_setup_clocksource(node, timer_base, clk, irq); + if (ret) + return ret; + + return sun5i_setup_clockevent(node, timer_base, clk, irq); } CLOCKSOURCE_OF_DECLARE(sun5i_a13, "allwinner,sun5i-a13-hstimer", - sun5i_timer_init); + sun5i_timer_init); CLOCKSOURCE_OF_DECLARE(sun7i_a20, "allwinner,sun7i-a20-hstimer", - sun5i_timer_init); + sun5i_timer_init); diff --git a/drivers/clocksource/timer-ti-32k.c b/drivers/clocksource/timer-ti-32k.c index 8518d9dfb..92b7e390f 100644 --- a/drivers/clocksource/timer-ti-32k.c +++ b/drivers/clocksource/timer-ti-32k.c @@ -88,14 +88,14 @@ static u64 notrace omap_32k_read_sched_clock(void) return ti_32k_read_cycles(&ti_32k_timer.cs); } -static void __init ti_32k_timer_init(struct device_node *np) +static int __init ti_32k_timer_init(struct device_node *np) { int ret; ti_32k_timer.base = of_iomap(np, 0); if (!ti_32k_timer.base) { pr_err("Can't ioremap 32k timer base\n"); - return; + return -ENXIO; } ti_32k_timer.counter = ti_32k_timer.base; @@ -116,11 +116,13 @@ static void __init ti_32k_timer_init(struct device_node *np) ret = clocksource_register_hz(&ti_32k_timer.cs, 32768); if (ret) { pr_err("32k_counter: can't register clocksource\n"); - return; + return ret; } sched_clock_register(omap_32k_read_sched_clock, 32, 32768); pr_info("OMAP clocksource: 32k_counter at 32768 Hz\n"); + + return 0; } CLOCKSOURCE_OF_DECLARE(ti_32k_timer, "ti,omap-counter32k", ti_32k_timer_init); diff --git a/drivers/clocksource/timer-u300.c b/drivers/clocksource/timer-u300.c index 1744b2438..704e40c6f 100644 --- a/drivers/clocksource/timer-u300.c +++ b/drivers/clocksource/timer-u300.c @@ -359,27 +359,37 @@ static struct delay_timer u300_delay_timer; /* * This sets up the system timers, clock source and clock event. */ -static void __init u300_timer_init_of(struct device_node *np) +static int __init u300_timer_init_of(struct device_node *np) { unsigned int irq; struct clk *clk; unsigned long rate; + int ret; u300_timer_base = of_iomap(np, 0); - if (!u300_timer_base) - panic("could not ioremap system timer\n"); + if (!u300_timer_base) { + pr_err("could not ioremap system timer\n"); + return -ENXIO; + } /* Get the IRQ for the GP1 timer */ irq = irq_of_parse_and_map(np, 2); - if (!irq) - panic("no IRQ for system timer\n"); + if (!irq) { + pr_err("no IRQ for system timer\n"); + return -EINVAL; + } pr_info("U300 GP1 timer @ base: %p, IRQ: %u\n", u300_timer_base, irq); /* Clock the interrupt controller */ clk = of_clk_get(np, 0); - BUG_ON(IS_ERR(clk)); - clk_prepare_enable(clk); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + ret = clk_prepare_enable(clk); + if (ret) + return ret; + rate = clk_get_rate(clk); u300_clockevent_data.ticks_per_jiffy = DIV_ROUND_CLOSEST(rate, HZ); @@ -410,7 +420,9 @@ static void __init u300_timer_init_of(struct device_node *np) u300_timer_base + U300_TIMER_APP_RGPT1); /* Set up the IRQ handler */ - setup_irq(irq, &u300_timer_irq); + ret = setup_irq(irq, &u300_timer_irq); + if (ret) + return ret; /* Reset the General Purpose timer 2 */ writel(U300_TIMER_APP_RGPT2_TIMER_RESET, @@ -428,9 +440,12 @@ static void __init u300_timer_init_of(struct device_node *np) u300_timer_base + U300_TIMER_APP_EGPT2); /* Use general purpose timer 2 as clock source */ - if (clocksource_mmio_init(u300_timer_base + U300_TIMER_APP_GPT2CC, - "GPT2", rate, 300, 32, clocksource_mmio_readl_up)) + ret = clocksource_mmio_init(u300_timer_base + U300_TIMER_APP_GPT2CC, + "GPT2", rate, 300, 32, clocksource_mmio_readl_up); + if (ret) { pr_err("timer: failed to initialize U300 clock source\n"); + return ret; + } /* Configure and register the clockevent */ clockevents_config_and_register(&u300_clockevent_data.cevd, rate, @@ -440,6 +455,7 @@ static void __init u300_timer_init_of(struct device_node *np) * TODO: init and register the rest of the timers too, they can be * used by hrtimers! */ + return 0; } CLOCKSOURCE_OF_DECLARE(u300_timer, "stericsson,u300-apptimer", diff --git a/drivers/clocksource/versatile.c b/drivers/clocksource/versatile.c index 0a26d3dde..220b490a8 100644 --- a/drivers/clocksource/versatile.c +++ b/drivers/clocksource/versatile.c @@ -25,16 +25,18 @@ static u64 notrace versatile_sys_24mhz_read(void) return readl(versatile_sys_24mhz); } -static void __init versatile_sched_clock_init(struct device_node *node) +static int __init versatile_sched_clock_init(struct device_node *node) { void __iomem *base = of_iomap(node, 0); if (!base) - return; + return -ENXIO; versatile_sys_24mhz = base + SYS_24MHZ; sched_clock_register(versatile_sys_24mhz_read, 32, 24000000); + + return 0; } CLOCKSOURCE_OF_DECLARE(vexpress, "arm,vexpress-sysreg", versatile_sched_clock_init); diff --git a/drivers/clocksource/vf_pit_timer.c b/drivers/clocksource/vf_pit_timer.c index a0e6c6853..55d8d8402 100644 --- a/drivers/clocksource/vf_pit_timer.c +++ b/drivers/clocksource/vf_pit_timer.c @@ -156,15 +156,18 @@ static int __init pit_clockevent_init(unsigned long rate, int irq) return 0; } -static void __init pit_timer_init(struct device_node *np) +static int __init pit_timer_init(struct device_node *np) { struct clk *pit_clk; void __iomem *timer_base; unsigned long clk_rate; - int irq; + int irq, ret; timer_base = of_iomap(np, 0); - BUG_ON(!timer_base); + if (!timer_base) { + pr_err("Failed to iomap"); + return -ENXIO; + } /* * PIT0 and PIT1 can be chained to build a 64-bit timer, @@ -175,12 +178,16 @@ static void __init pit_timer_init(struct device_node *np) clkevt_base = timer_base + PITn_OFFSET(3); irq = irq_of_parse_and_map(np, 0); - BUG_ON(irq <= 0); + if (irq <= 0) + return -EINVAL; pit_clk = of_clk_get(np, 0); - BUG_ON(IS_ERR(pit_clk)); + if (IS_ERR(pit_clk)) + return PTR_ERR(pit_clk); - BUG_ON(clk_prepare_enable(pit_clk)); + ret = clk_prepare_enable(pit_clk); + if (ret) + return ret; clk_rate = clk_get_rate(pit_clk); cycle_per_jiffy = clk_rate / (HZ); @@ -188,8 +195,10 @@ static void __init pit_timer_init(struct device_node *np) /* enable the pit module */ __raw_writel(~PITMCR_MDIS, timer_base + PITMCR); - BUG_ON(pit_clocksource_init(clk_rate)); + ret = pit_clocksource_init(clk_rate); + if (ret) + return ret; - pit_clockevent_init(clk_rate, irq); + return pit_clockevent_init(clk_rate, irq); } CLOCKSOURCE_OF_DECLARE(vf610, "fsl,vf610-pit", pit_timer_init); diff --git a/drivers/clocksource/vt8500_timer.c b/drivers/clocksource/vt8500_timer.c index ddb409274..b15069483 100644 --- a/drivers/clocksource/vt8500_timer.c +++ b/drivers/clocksource/vt8500_timer.c @@ -121,38 +121,48 @@ static struct irqaction irq = { .dev_id = &clockevent, }; -static void __init vt8500_timer_init(struct device_node *np) +static int __init vt8500_timer_init(struct device_node *np) { - int timer_irq; + int timer_irq, ret; regbase = of_iomap(np, 0); if (!regbase) { pr_err("%s: Missing iobase description in Device Tree\n", __func__); - return; + return -ENXIO; } + timer_irq = irq_of_parse_and_map(np, 0); if (!timer_irq) { pr_err("%s: Missing irq description in Device Tree\n", __func__); - return; + return -EINVAL; } writel(1, regbase + TIMER_CTRL_VAL); writel(0xf, regbase + TIMER_STATUS_VAL); writel(~0, regbase + TIMER_MATCH_VAL); - if (clocksource_register_hz(&clocksource, VT8500_TIMER_HZ)) + ret = clocksource_register_hz(&clocksource, VT8500_TIMER_HZ); + if (ret) { pr_err("%s: vt8500_timer_init: clocksource_register failed for %s\n", - __func__, clocksource.name); + __func__, clocksource.name); + return ret; + } clockevent.cpumask = cpumask_of(0); - if (setup_irq(timer_irq, &irq)) + ret = setup_irq(timer_irq, &irq); + if (ret) { pr_err("%s: setup_irq failed for %s\n", __func__, clockevent.name); + return ret; + } + clockevents_config_and_register(&clockevent, VT8500_TIMER_HZ, MIN_OSCR_DELTA * 2, 0xf0000000); + + return 0; } CLOCKSOURCE_OF_DECLARE(vt8500, "via,vt8500-timer", vt8500_timer_init); diff --git a/drivers/clocksource/zevio-timer.c b/drivers/clocksource/zevio-timer.c index ceaa6133f..9a53f5ef6 100644 --- a/drivers/clocksource/zevio-timer.c +++ b/drivers/clocksource/zevio-timer.c @@ -210,9 +210,9 @@ error_free: return ret; } -static void __init zevio_timer_init(struct device_node *node) +static int __init zevio_timer_init(struct device_node *node) { - BUG_ON(zevio_timer_add(node)); + return zevio_timer_add(node); } CLOCKSOURCE_OF_DECLARE(zevio_timer, "lsi,zevio-timer", zevio_timer_init); diff --git a/drivers/connector/cn_proc.c b/drivers/connector/cn_proc.c index b02f9c606..a782ce877 100644 --- a/drivers/connector/cn_proc.c +++ b/drivers/connector/cn_proc.c @@ -22,7 +22,6 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -#include #include #include #include @@ -390,5 +389,4 @@ static int __init cn_proc_init(void) } return 0; } - -module_init(cn_proc_init); +device_initcall(cn_proc_init); diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index b7445b6ae..74919aa81 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -31,23 +31,17 @@ config CPU_FREQ_BOOST_SW depends on THERMAL config CPU_FREQ_STAT - tristate "CPU frequency translation statistics" - default y + bool "CPU frequency transition statistics" help - This driver exports CPU frequency statistics information through sysfs - file system. - - To compile this driver as a module, choose M here: the - module will be called cpufreq_stats. + Export CPU frequency statistics information through sysfs. If in doubt, say N. config CPU_FREQ_STAT_DETAILS - bool "CPU frequency translation statistics details" + bool "CPU frequency transition statistics details" depends on CPU_FREQ_STAT help - This will show detail CPU frequency translation table in sysfs file - system. + Show detailed CPU frequency transition table in sysfs. If in doubt, say N. diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 32a15052f..297e9128f 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -468,20 +468,17 @@ unsigned int acpi_cpufreq_fast_switch(struct cpufreq_policy *policy, struct acpi_cpufreq_data *data = policy->driver_data; struct acpi_processor_performance *perf; struct cpufreq_frequency_table *entry; - unsigned int next_perf_state, next_freq, freq; + unsigned int next_perf_state, next_freq, index; /* * Find the closest frequency above target_freq. - * - * The table is sorted in the reverse order with respect to the - * frequency and all of the entries are valid (see the initialization). */ - entry = policy->freq_table; - do { - entry++; - freq = entry->frequency; - } while (freq >= target_freq && freq != CPUFREQ_TABLE_END); - entry--; + if (policy->cached_target_freq == target_freq) + index = policy->cached_resolved_idx; + else + index = cpufreq_table_find_index_dl(policy, target_freq); + + entry = &policy->freq_table[index]; next_freq = entry->frequency; next_perf_state = entry->driver_data; diff --git a/drivers/cpufreq/amd_freq_sensitivity.c b/drivers/cpufreq/amd_freq_sensitivity.c index 404360cad..042023bbb 100644 --- a/drivers/cpufreq/amd_freq_sensitivity.c +++ b/drivers/cpufreq/amd_freq_sensitivity.c @@ -48,9 +48,8 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy, struct policy_dbs_info *policy_dbs = policy->governor_data; struct dbs_data *od_data = policy_dbs->dbs_data; struct od_dbs_tuners *od_tuners = od_data->tuners; - struct od_policy_dbs_info *od_info = to_dbs_info(policy_dbs); - if (!od_info->freq_table) + if (!policy->freq_table) return freq_next; rdmsr_on_cpu(policy->cpu, MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, @@ -92,10 +91,9 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy, else { unsigned int index; - cpufreq_frequency_table_target(policy, - od_info->freq_table, policy->cur - 1, - CPUFREQ_RELATION_H, &index); - freq_next = od_info->freq_table[index].frequency; + index = cpufreq_table_find_index_h(policy, + policy->cur - 1); + freq_next = policy->freq_table[index].frequency; } data->freq_prev = freq_next; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 5617c7087..3dd4884c6 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -74,19 +74,12 @@ static inline bool has_target(void) } /* internal prototypes */ -static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event); static unsigned int __cpufreq_get(struct cpufreq_policy *policy); +static int cpufreq_init_governor(struct cpufreq_policy *policy); +static void cpufreq_exit_governor(struct cpufreq_policy *policy); static int cpufreq_start_governor(struct cpufreq_policy *policy); - -static inline void cpufreq_exit_governor(struct cpufreq_policy *policy) -{ - (void)cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); -} - -static inline void cpufreq_stop_governor(struct cpufreq_policy *policy) -{ - (void)cpufreq_governor(policy, CPUFREQ_GOV_STOP); -} +static void cpufreq_stop_governor(struct cpufreq_policy *policy); +static void cpufreq_governor_limits(struct cpufreq_policy *policy); /** * Two notifier lists: the "policy" list is involved in the @@ -133,15 +126,6 @@ struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) } EXPORT_SYMBOL_GPL(get_governor_parent_kobj); -struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu) -{ - struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); - - return policy && !policy_is_inactive(policy) ? - policy->freq_table : NULL; -} -EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table); - static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) { u64 idle_time; @@ -354,6 +338,7 @@ static void __cpufreq_notify_transition(struct cpufreq_policy *policy, pr_debug("FREQ: %lu - CPU: %lu\n", (unsigned long)freqs->new, (unsigned long)freqs->cpu); trace_cpu_frequency(freqs->new, freqs->cpu); + cpufreq_stats_record_transition(policy, freqs->new); srcu_notifier_call_chain(&cpufreq_transition_notifier_list, CPUFREQ_POSTCHANGE, freqs); if (likely(policy) && likely(policy->cpu == freqs->cpu)) @@ -507,6 +492,38 @@ void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) } EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); +/** + * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported + * one. + * @target_freq: target frequency to resolve. + * + * The target to driver frequency mapping is cached in the policy. + * + * Return: Lowest driver-supported frequency greater than or equal to the + * given target_freq, subject to policy (min/max) and driver limitations. + */ +unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + target_freq = clamp_val(target_freq, policy->min, policy->max); + policy->cached_target_freq = target_freq; + + if (cpufreq_driver->target_index) { + int idx; + + idx = cpufreq_frequency_table_target(policy, target_freq, + CPUFREQ_RELATION_L); + policy->cached_resolved_idx = idx; + return policy->freq_table[idx].frequency; + } + + if (cpufreq_driver->resolve_freq) + return cpufreq_driver->resolve_freq(policy, target_freq); + + return target_freq; +} +EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq); + /********************************************************************* * SYSFS INTERFACE * *********************************************************************/ @@ -1115,6 +1132,7 @@ static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify) CPUFREQ_REMOVE_POLICY, policy); down_write(&policy->rwsem); + cpufreq_stats_free_table(policy); cpufreq_remove_dev_symlink(policy); kobj = &policy->kobj; cmp = &policy->kobj_unregister; @@ -1265,13 +1283,12 @@ static int cpufreq_online(unsigned int cpu) } } - blocking_notifier_call_chain(&cpufreq_policy_notifier_list, - CPUFREQ_START, policy); - if (new_policy) { ret = cpufreq_add_dev_interface(policy); if (ret) goto out_exit_policy; + + cpufreq_stats_create_table(policy); blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_CREATE_POLICY, policy); @@ -1280,6 +1297,9 @@ static int cpufreq_online(unsigned int cpu) write_unlock_irqrestore(&cpufreq_driver_lock, flags); } + blocking_notifier_call_chain(&cpufreq_policy_notifier_list, + CPUFREQ_START, policy); + ret = cpufreq_init_policy(policy); if (ret) { pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", @@ -1556,9 +1576,6 @@ static unsigned int cpufreq_update_current_freq(struct cpufreq_policy *policy) { unsigned int new_freq; - if (cpufreq_suspended) - return 0; - new_freq = cpufreq_driver->get(policy->cpu); if (!new_freq) return 0; @@ -1864,14 +1881,17 @@ static int __target_intermediate(struct cpufreq_policy *policy, return ret; } -static int __target_index(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *freq_table, int index) +static int __target_index(struct cpufreq_policy *policy, int index) { struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; unsigned int intermediate_freq = 0; + unsigned int newfreq = policy->freq_table[index].frequency; int retval = -EINVAL; bool notify; + if (newfreq == policy->cur) + return 0; + notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); if (notify) { /* Handle switching to intermediate frequency */ @@ -1886,7 +1906,7 @@ static int __target_index(struct cpufreq_policy *policy, freqs.old = freqs.new; } - freqs.new = freq_table[index].frequency; + freqs.new = newfreq; pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", __func__, policy->cpu, freqs.old, freqs.new); @@ -1923,17 +1943,13 @@ int __cpufreq_driver_target(struct cpufreq_policy *policy, unsigned int relation) { unsigned int old_target_freq = target_freq; - struct cpufreq_frequency_table *freq_table; - int index, retval; + int index; if (cpufreq_disabled()) return -ENODEV; /* Make sure that target_freq is within supported range */ - if (target_freq > policy->max) - target_freq = policy->max; - if (target_freq < policy->min) - target_freq = policy->min; + target_freq = clamp_val(target_freq, policy->min, policy->max); pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", policy->cpu, target_freq, relation, old_target_freq); @@ -1956,23 +1972,9 @@ int __cpufreq_driver_target(struct cpufreq_policy *policy, if (!cpufreq_driver->target_index) return -EINVAL; - freq_table = cpufreq_frequency_get_table(policy->cpu); - if (unlikely(!freq_table)) { - pr_err("%s: Unable to find freq_table\n", __func__); - return -EINVAL; - } - - retval = cpufreq_frequency_table_target(policy, freq_table, target_freq, - relation, &index); - if (unlikely(retval)) { - pr_err("%s: Unable to find matching freq\n", __func__); - return retval; - } - - if (freq_table[index].frequency == policy->cur) - return 0; + index = cpufreq_frequency_table_target(policy, target_freq, relation); - return __target_index(policy, freq_table, index); + return __target_index(policy, index); } EXPORT_SYMBOL_GPL(__cpufreq_driver_target); @@ -1997,7 +1999,7 @@ __weak struct cpufreq_governor *cpufreq_fallback_governor(void) return NULL; } -static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event) +static int cpufreq_init_governor(struct cpufreq_policy *policy) { int ret; @@ -2025,36 +2027,82 @@ static int cpufreq_governor(struct cpufreq_policy *policy, unsigned int event) } } - if (event == CPUFREQ_GOV_POLICY_INIT) - if (!try_module_get(policy->governor->owner)) - return -EINVAL; - - pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event); + if (!try_module_get(policy->governor->owner)) + return -EINVAL; - ret = policy->governor->governor(policy, event); + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); - if (event == CPUFREQ_GOV_POLICY_INIT) { - if (ret) + if (policy->governor->init) { + ret = policy->governor->init(policy); + if (ret) { module_put(policy->governor->owner); - else - policy->governor->initialized++; - } else if (event == CPUFREQ_GOV_POLICY_EXIT) { - policy->governor->initialized--; - module_put(policy->governor->owner); + return ret; + } } - return ret; + return 0; +} + +static void cpufreq_exit_governor(struct cpufreq_policy *policy) +{ + if (cpufreq_suspended || !policy->governor) + return; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + + if (policy->governor->exit) + policy->governor->exit(policy); + + module_put(policy->governor->owner); } static int cpufreq_start_governor(struct cpufreq_policy *policy) { int ret; + if (cpufreq_suspended) + return 0; + + if (!policy->governor) + return -EINVAL; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + if (cpufreq_driver->get && !cpufreq_driver->setpolicy) cpufreq_update_current_freq(policy); - ret = cpufreq_governor(policy, CPUFREQ_GOV_START); - return ret ? ret : cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); + if (policy->governor->start) { + ret = policy->governor->start(policy); + if (ret) + return ret; + } + + if (policy->governor->limits) + policy->governor->limits(policy); + + return 0; +} + +static void cpufreq_stop_governor(struct cpufreq_policy *policy) +{ + if (cpufreq_suspended || !policy->governor) + return; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + + if (policy->governor->stop) + policy->governor->stop(policy); +} + +static void cpufreq_governor_limits(struct cpufreq_policy *policy) +{ + if (cpufreq_suspended || !policy->governor) + return; + + pr_debug("%s: for CPU %u\n", __func__, policy->cpu); + + if (policy->governor->limits) + policy->governor->limits(policy); } int cpufreq_register_governor(struct cpufreq_governor *governor) @@ -2069,7 +2117,6 @@ int cpufreq_register_governor(struct cpufreq_governor *governor) mutex_lock(&cpufreq_governor_mutex); - governor->initialized = 0; err = -EBUSY; if (!find_governor(governor->name)) { err = 0; @@ -2184,6 +2231,8 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, policy->min = new_policy->min; policy->max = new_policy->max; + policy->cached_target_freq = UINT_MAX; + pr_debug("new min and max freqs are %u - %u kHz\n", policy->min, policy->max); @@ -2195,7 +2244,8 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, if (new_policy->governor == policy->governor) { pr_debug("cpufreq: governor limits update\n"); - return cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); + cpufreq_governor_limits(policy); + return 0; } pr_debug("governor switch\n"); @@ -2210,7 +2260,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, /* start new governor */ policy->governor = new_policy->governor; - ret = cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT); + ret = cpufreq_init_governor(policy); if (!ret) { ret = cpufreq_start_governor(policy); if (!ret) { @@ -2224,7 +2274,7 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, pr_debug("starting governor %s failed\n", policy->governor->name); if (old_gov) { policy->governor = old_gov; - if (cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) + if (cpufreq_init_governor(policy)) policy->governor = NULL; else cpufreq_start_governor(policy); @@ -2309,26 +2359,25 @@ static struct notifier_block __refdata cpufreq_cpu_notifier = { *********************************************************************/ static int cpufreq_boost_set_sw(int state) { - struct cpufreq_frequency_table *freq_table; struct cpufreq_policy *policy; int ret = -EINVAL; for_each_active_policy(policy) { - freq_table = cpufreq_frequency_get_table(policy->cpu); - if (freq_table) { - ret = cpufreq_frequency_table_cpuinfo(policy, - freq_table); - if (ret) { - pr_err("%s: Policy frequency update failed\n", - __func__); - break; - } - - down_write(&policy->rwsem); - policy->user_policy.max = policy->max; - cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); - up_write(&policy->rwsem); + if (!policy->freq_table) + continue; + + ret = cpufreq_frequency_table_cpuinfo(policy, + policy->freq_table); + if (ret) { + pr_err("%s: Policy frequency update failed\n", + __func__); + break; } + + down_write(&policy->rwsem); + policy->user_policy.max = policy->max; + cpufreq_governor_limits(policy); + up_write(&policy->rwsem); } return ret; diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index 7306830c5..a70367f53 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -17,7 +17,6 @@ struct cs_policy_dbs_info { struct policy_dbs_info policy_dbs; unsigned int down_skip; - unsigned int requested_freq; }; static inline struct cs_policy_dbs_info *to_dbs_info(struct policy_dbs_info *policy_dbs) @@ -81,19 +80,17 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy) /* Check for frequency increase */ if (load > dbs_data->up_threshold) { + unsigned int requested_freq = policy->cur; + dbs_info->down_skip = 0; /* if we are already at full speed then break out early */ - if (dbs_info->requested_freq == policy->max) + if (requested_freq == policy->max) goto out; - dbs_info->requested_freq += get_freq_target(cs_tuners, policy); - - if (dbs_info->requested_freq > policy->max) - dbs_info->requested_freq = policy->max; + requested_freq += get_freq_target(cs_tuners, policy); - __cpufreq_driver_target(policy, dbs_info->requested_freq, - CPUFREQ_RELATION_H); + __cpufreq_driver_target(policy, requested_freq, CPUFREQ_RELATION_H); goto out; } @@ -104,36 +101,27 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy) /* Check for frequency decrease */ if (load < cs_tuners->down_threshold) { - unsigned int freq_target; + unsigned int freq_target, requested_freq = policy->cur; /* * if we cannot reduce the frequency anymore, break out early */ - if (policy->cur == policy->min) + if (requested_freq == policy->min) goto out; freq_target = get_freq_target(cs_tuners, policy); - if (dbs_info->requested_freq > freq_target) - dbs_info->requested_freq -= freq_target; + if (requested_freq > freq_target) + requested_freq -= freq_target; else - dbs_info->requested_freq = policy->min; + requested_freq = policy->min; - __cpufreq_driver_target(policy, dbs_info->requested_freq, - CPUFREQ_RELATION_L); + __cpufreq_driver_target(policy, requested_freq, CPUFREQ_RELATION_L); } out: return dbs_data->sampling_rate; } -static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val, - void *data); - -static struct notifier_block cs_cpufreq_notifier_block = { - .notifier_call = dbs_cpufreq_notifier, -}; - /************************** sysfs interface ************************/ -static struct dbs_governor cs_dbs_gov; static ssize_t store_sampling_down_factor(struct gov_attr_set *attr_set, const char *buf, size_t count) @@ -274,15 +262,13 @@ static void cs_free(struct policy_dbs_info *policy_dbs) kfree(to_dbs_info(policy_dbs)); } -static int cs_init(struct dbs_data *dbs_data, bool notify) +static int cs_init(struct dbs_data *dbs_data) { struct cs_dbs_tuners *tuners; tuners = kzalloc(sizeof(*tuners), GFP_KERNEL); - if (!tuners) { - pr_err("%s: kzalloc failed\n", __func__); + if (!tuners) return -ENOMEM; - } tuners->down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD; tuners->freq_step = DEF_FREQUENCY_STEP; @@ -294,19 +280,11 @@ static int cs_init(struct dbs_data *dbs_data, bool notify) dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * jiffies_to_usecs(10); - if (notify) - cpufreq_register_notifier(&cs_cpufreq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - return 0; } -static void cs_exit(struct dbs_data *dbs_data, bool notify) +static void cs_exit(struct dbs_data *dbs_data) { - if (notify) - cpufreq_unregister_notifier(&cs_cpufreq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - kfree(dbs_data->tuners); } @@ -315,16 +293,10 @@ static void cs_start(struct cpufreq_policy *policy) struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); dbs_info->down_skip = 0; - dbs_info->requested_freq = policy->cur; } -static struct dbs_governor cs_dbs_gov = { - .gov = { - .name = "conservative", - .governor = cpufreq_governor_dbs, - .max_transition_latency = TRANSITION_LATENCY_LIMIT, - .owner = THIS_MODULE, - }, +static struct dbs_governor cs_governor = { + .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"), .kobj_type = { .default_attrs = cs_attributes }, .gov_dbs_timer = cs_dbs_timer, .alloc = cs_alloc, @@ -334,33 +306,7 @@ static struct dbs_governor cs_dbs_gov = { .start = cs_start, }; -#define CPU_FREQ_GOV_CONSERVATIVE (&cs_dbs_gov.gov) - -static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val, - void *data) -{ - struct cpufreq_freqs *freq = data; - struct cpufreq_policy *policy = cpufreq_cpu_get_raw(freq->cpu); - struct cs_policy_dbs_info *dbs_info; - - if (!policy) - return 0; - - /* policy isn't governed by conservative governor */ - if (policy->governor != CPU_FREQ_GOV_CONSERVATIVE) - return 0; - - dbs_info = to_dbs_info(policy->governor_data); - /* - * we only care if our internally tracked freq moves outside the 'valid' - * ranges of frequency available to us otherwise we do not change it - */ - if (dbs_info->requested_freq > policy->max - || dbs_info->requested_freq < policy->min) - dbs_info->requested_freq = freq->new; - - return 0; -} +#define CPU_FREQ_GOV_CONSERVATIVE (&cs_governor.gov) static int __init cpufreq_gov_dbs_init(void) { diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index be498d56d..e415349ab 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -336,17 +336,6 @@ static inline void gov_clear_update_util(struct cpufreq_policy *policy) synchronize_sched(); } -static void gov_cancel_work(struct cpufreq_policy *policy) -{ - struct policy_dbs_info *policy_dbs = policy->governor_data; - - gov_clear_update_util(policy_dbs->policy); - irq_work_sync(&policy_dbs->irq_work); - cancel_work_sync(&policy_dbs->work); - atomic_set(&policy_dbs->work_count, 0); - policy_dbs->work_in_progress = false; -} - static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *policy, struct dbs_governor *gov) { @@ -389,7 +378,7 @@ static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs, gov->free(policy_dbs); } -static int cpufreq_governor_init(struct cpufreq_policy *policy) +int cpufreq_dbs_governor_init(struct cpufreq_policy *policy) { struct dbs_governor *gov = dbs_governor_of(policy); struct dbs_data *dbs_data; @@ -429,7 +418,7 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy) gov_attr_set_init(&dbs_data->attr_set, &policy_dbs->list); - ret = gov->init(dbs_data, !policy->governor->initialized); + ret = gov->init(dbs_data); if (ret) goto free_policy_dbs_info; @@ -458,13 +447,13 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy) goto out; /* Failure, so roll back. */ - pr_err("cpufreq: Governor initialization failed (dbs_data kobject init error %d)\n", ret); + pr_err("initialization failed (dbs_data kobject init error %d)\n", ret); policy->governor_data = NULL; if (!have_governor_per_policy()) gov->gdbs_data = NULL; - gov->exit(dbs_data, !policy->governor->initialized); + gov->exit(dbs_data); kfree(dbs_data); free_policy_dbs_info: @@ -474,8 +463,9 @@ out: mutex_unlock(&gov_dbs_data_mutex); return ret; } +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_init); -static int cpufreq_governor_exit(struct cpufreq_policy *policy) +void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy) { struct dbs_governor *gov = dbs_governor_of(policy); struct policy_dbs_info *policy_dbs = policy->governor_data; @@ -493,17 +483,17 @@ static int cpufreq_governor_exit(struct cpufreq_policy *policy) if (!have_governor_per_policy()) gov->gdbs_data = NULL; - gov->exit(dbs_data, policy->governor->initialized == 1); + gov->exit(dbs_data); kfree(dbs_data); } free_policy_dbs_info(policy_dbs, gov); mutex_unlock(&gov_dbs_data_mutex); - return 0; } +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_exit); -static int cpufreq_governor_start(struct cpufreq_policy *policy) +int cpufreq_dbs_governor_start(struct cpufreq_policy *policy) { struct dbs_governor *gov = dbs_governor_of(policy); struct policy_dbs_info *policy_dbs = policy->governor_data; @@ -539,47 +529,28 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy) gov_set_update_util(policy_dbs, sampling_rate); return 0; } +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_start); -static int cpufreq_governor_stop(struct cpufreq_policy *policy) +void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy) { - gov_cancel_work(policy); - return 0; + struct policy_dbs_info *policy_dbs = policy->governor_data; + + gov_clear_update_util(policy_dbs->policy); + irq_work_sync(&policy_dbs->irq_work); + cancel_work_sync(&policy_dbs->work); + atomic_set(&policy_dbs->work_count, 0); + policy_dbs->work_in_progress = false; } +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_stop); -static int cpufreq_governor_limits(struct cpufreq_policy *policy) +void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy) { struct policy_dbs_info *policy_dbs = policy->governor_data; mutex_lock(&policy_dbs->timer_mutex); - - if (policy->max < policy->cur) - __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); - else if (policy->min > policy->cur) - __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); - + cpufreq_policy_apply_limits(policy); gov_update_sample_delay(policy_dbs, 0); mutex_unlock(&policy_dbs->timer_mutex); - - return 0; -} - -int cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event) -{ - if (event == CPUFREQ_GOV_POLICY_INIT) { - return cpufreq_governor_init(policy); - } else if (policy->governor_data) { - switch (event) { - case CPUFREQ_GOV_POLICY_EXIT: - return cpufreq_governor_exit(policy); - case CPUFREQ_GOV_START: - return cpufreq_governor_start(policy); - case CPUFREQ_GOV_STOP: - return cpufreq_governor_stop(policy); - case CPUFREQ_GOV_LIMITS: - return cpufreq_governor_limits(policy); - } - } - return -EINVAL; } -EXPORT_SYMBOL_GPL(cpufreq_governor_dbs); +EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_limits); diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h index 34eb214b6..ef1037e9c 100644 --- a/drivers/cpufreq/cpufreq_governor.h +++ b/drivers/cpufreq/cpufreq_governor.h @@ -138,8 +138,8 @@ struct dbs_governor { unsigned int (*gov_dbs_timer)(struct cpufreq_policy *policy); struct policy_dbs_info *(*alloc)(void); void (*free)(struct policy_dbs_info *policy_dbs); - int (*init)(struct dbs_data *dbs_data, bool notify); - void (*exit)(struct dbs_data *dbs_data, bool notify); + int (*init)(struct dbs_data *dbs_data); + void (*exit)(struct dbs_data *dbs_data); void (*start)(struct cpufreq_policy *policy); }; @@ -148,6 +148,25 @@ static inline struct dbs_governor *dbs_governor_of(struct cpufreq_policy *policy return container_of(policy->governor, struct dbs_governor, gov); } +/* Governor callback routines */ +int cpufreq_dbs_governor_init(struct cpufreq_policy *policy); +void cpufreq_dbs_governor_exit(struct cpufreq_policy *policy); +int cpufreq_dbs_governor_start(struct cpufreq_policy *policy); +void cpufreq_dbs_governor_stop(struct cpufreq_policy *policy); +void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy); + +#define CPUFREQ_DBS_GOVERNOR_INITIALIZER(_name_) \ + { \ + .name = _name_, \ + .max_transition_latency = TRANSITION_LATENCY_LIMIT, \ + .owner = THIS_MODULE, \ + .init = cpufreq_dbs_governor_init, \ + .exit = cpufreq_dbs_governor_exit, \ + .start = cpufreq_dbs_governor_start, \ + .stop = cpufreq_dbs_governor_stop, \ + .limits = cpufreq_dbs_governor_limits, \ + } + /* Governor specific operations */ struct od_ops { unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy, @@ -155,7 +174,6 @@ struct od_ops { }; unsigned int dbs_update(struct cpufreq_policy *policy); -int cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event); void od_register_powersave_bias_handler(unsigned int (*f) (struct cpufreq_policy *, unsigned int, unsigned int), unsigned int powersave_bias); diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index b596c8410..58d6fe616 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -74,34 +74,30 @@ static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy, { unsigned int freq_req, freq_reduc, freq_avg; unsigned int freq_hi, freq_lo; - unsigned int index = 0; + unsigned int index; unsigned int delay_hi_us; struct policy_dbs_info *policy_dbs = policy->governor_data; struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); struct dbs_data *dbs_data = policy_dbs->dbs_data; struct od_dbs_tuners *od_tuners = dbs_data->tuners; + struct cpufreq_frequency_table *freq_table = policy->freq_table; - if (!dbs_info->freq_table) { + if (!freq_table) { dbs_info->freq_lo = 0; dbs_info->freq_lo_delay_us = 0; return freq_next; } - cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next, - relation, &index); - freq_req = dbs_info->freq_table[index].frequency; + index = cpufreq_frequency_table_target(policy, freq_next, relation); + freq_req = freq_table[index].frequency; freq_reduc = freq_req * od_tuners->powersave_bias / 1000; freq_avg = freq_req - freq_reduc; /* Find freq bounds for freq_avg in freq_table */ - index = 0; - cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg, - CPUFREQ_RELATION_H, &index); - freq_lo = dbs_info->freq_table[index].frequency; - index = 0; - cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg, - CPUFREQ_RELATION_L, &index); - freq_hi = dbs_info->freq_table[index].frequency; + index = cpufreq_table_find_index_h(policy, freq_avg); + freq_lo = freq_table[index].frequency; + index = cpufreq_table_find_index_l(policy, freq_avg); + freq_hi = freq_table[index].frequency; /* Find out how long we have to be in hi and lo freqs */ if (freq_hi == freq_lo) { @@ -122,7 +118,6 @@ static void ondemand_powersave_bias_init(struct cpufreq_policy *policy) { struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); - dbs_info->freq_table = cpufreq_frequency_get_table(policy->cpu); dbs_info->freq_lo = 0; } @@ -370,17 +365,15 @@ static void od_free(struct policy_dbs_info *policy_dbs) kfree(to_dbs_info(policy_dbs)); } -static int od_init(struct dbs_data *dbs_data, bool notify) +static int od_init(struct dbs_data *dbs_data) { struct od_dbs_tuners *tuners; u64 idle_time; int cpu; tuners = kzalloc(sizeof(*tuners), GFP_KERNEL); - if (!tuners) { - pr_err("%s: kzalloc failed\n", __func__); + if (!tuners) return -ENOMEM; - } cpu = get_cpu(); idle_time = get_cpu_idle_time_us(cpu, NULL); @@ -411,7 +404,7 @@ static int od_init(struct dbs_data *dbs_data, bool notify) return 0; } -static void od_exit(struct dbs_data *dbs_data, bool notify) +static void od_exit(struct dbs_data *dbs_data) { kfree(dbs_data->tuners); } @@ -429,12 +422,7 @@ static struct od_ops od_ops = { }; static struct dbs_governor od_dbs_gov = { - .gov = { - .name = "ondemand", - .governor = cpufreq_governor_dbs, - .max_transition_latency = TRANSITION_LATENCY_LIMIT, - .owner = THIS_MODULE, - }, + .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("ondemand"), .kobj_type = { .default_attrs = od_attributes }, .gov_dbs_timer = od_dbs_timer, .alloc = od_alloc, diff --git a/drivers/cpufreq/cpufreq_ondemand.h b/drivers/cpufreq/cpufreq_ondemand.h index f0121db3c..640ea4e97 100644 --- a/drivers/cpufreq/cpufreq_ondemand.h +++ b/drivers/cpufreq/cpufreq_ondemand.h @@ -13,7 +13,6 @@ struct od_policy_dbs_info { struct policy_dbs_info policy_dbs; - struct cpufreq_frequency_table *freq_table; unsigned int freq_lo; unsigned int freq_lo_delay_us; unsigned int freq_hi_delay_us; diff --git a/drivers/cpufreq/cpufreq_performance.c b/drivers/cpufreq/cpufreq_performance.c index af9f4b96f..dafb679ad 100644 --- a/drivers/cpufreq/cpufreq_performance.c +++ b/drivers/cpufreq/cpufreq_performance.c @@ -16,27 +16,16 @@ #include #include -static int cpufreq_governor_performance(struct cpufreq_policy *policy, - unsigned int event) +static void cpufreq_gov_performance_limits(struct cpufreq_policy *policy) { - switch (event) { - case CPUFREQ_GOV_START: - case CPUFREQ_GOV_LIMITS: - pr_debug("setting to %u kHz because of event %u\n", - policy->max, event); - __cpufreq_driver_target(policy, policy->max, - CPUFREQ_RELATION_H); - break; - default: - break; - } - return 0; + pr_debug("setting to %u kHz\n", policy->max); + __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); } static struct cpufreq_governor cpufreq_gov_performance = { .name = "performance", - .governor = cpufreq_governor_performance, .owner = THIS_MODULE, + .limits = cpufreq_gov_performance_limits, }; static int __init cpufreq_gov_performance_init(void) diff --git a/drivers/cpufreq/cpufreq_powersave.c b/drivers/cpufreq/cpufreq_powersave.c index b8b400232..78a651038 100644 --- a/drivers/cpufreq/cpufreq_powersave.c +++ b/drivers/cpufreq/cpufreq_powersave.c @@ -16,26 +16,15 @@ #include #include -static int cpufreq_governor_powersave(struct cpufreq_policy *policy, - unsigned int event) +static void cpufreq_gov_powersave_limits(struct cpufreq_policy *policy) { - switch (event) { - case CPUFREQ_GOV_START: - case CPUFREQ_GOV_LIMITS: - pr_debug("setting to %u kHz because of event %u\n", - policy->min, event); - __cpufreq_driver_target(policy, policy->min, - CPUFREQ_RELATION_L); - break; - default: - break; - } - return 0; + pr_debug("setting to %u kHz\n", policy->min); + __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); } static struct cpufreq_governor cpufreq_gov_powersave = { .name = "powersave", - .governor = cpufreq_governor_powersave, + .limits = cpufreq_gov_powersave_limits, .owner = THIS_MODULE, }; diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 5e370a30a..06d3abdff 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -15,7 +15,7 @@ #include #include -static spinlock_t cpufreq_stats_lock; +static DEFINE_SPINLOCK(cpufreq_stats_lock); struct cpufreq_stats { unsigned int total_trans; @@ -52,6 +52,9 @@ static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf) ssize_t len = 0; int i; + if (policy->fast_switch_enabled) + return 0; + cpufreq_stats_update(stats); for (i = 0; i < stats->state_num; i++) { len += sprintf(buf + len, "%u %llu\n", stats->freq_table[i], @@ -68,6 +71,9 @@ static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf) ssize_t len = 0; int i, j; + if (policy->fast_switch_enabled) + return 0; + len += snprintf(buf + len, PAGE_SIZE - len, " From : To\n"); len += snprintf(buf + len, PAGE_SIZE - len, " : "); for (i = 0; i < stats->state_num; i++) { @@ -130,7 +136,7 @@ static int freq_table_get_index(struct cpufreq_stats *stats, unsigned int freq) return -1; } -static void __cpufreq_stats_free_table(struct cpufreq_policy *policy) +void cpufreq_stats_free_table(struct cpufreq_policy *policy) { struct cpufreq_stats *stats = policy->stats; @@ -146,39 +152,25 @@ static void __cpufreq_stats_free_table(struct cpufreq_policy *policy) policy->stats = NULL; } -static void cpufreq_stats_free_table(unsigned int cpu) -{ - struct cpufreq_policy *policy; - - policy = cpufreq_cpu_get(cpu); - if (!policy) - return; - - __cpufreq_stats_free_table(policy); - - cpufreq_cpu_put(policy); -} - -static int __cpufreq_stats_create_table(struct cpufreq_policy *policy) +void cpufreq_stats_create_table(struct cpufreq_policy *policy) { unsigned int i = 0, count = 0, ret = -ENOMEM; struct cpufreq_stats *stats; unsigned int alloc_size; - unsigned int cpu = policy->cpu; struct cpufreq_frequency_table *pos, *table; /* We need cpufreq table for creating stats table */ - table = cpufreq_frequency_get_table(cpu); + table = policy->freq_table; if (unlikely(!table)) - return 0; + return; /* stats already initialized */ if (policy->stats) - return -EEXIST; + return; stats = kzalloc(sizeof(*stats), GFP_KERNEL); if (!stats) - return -ENOMEM; + return; /* Find total allocation size */ cpufreq_for_each_valid_entry(pos, table) @@ -215,80 +207,32 @@ static int __cpufreq_stats_create_table(struct cpufreq_policy *policy) policy->stats = stats; ret = sysfs_create_group(&policy->kobj, &stats_attr_group); if (!ret) - return 0; + return; /* We failed, release resources */ policy->stats = NULL; kfree(stats->time_in_state); free_stat: kfree(stats); - - return ret; -} - -static void cpufreq_stats_create_table(unsigned int cpu) -{ - struct cpufreq_policy *policy; - - /* - * "likely(!policy)" because normally cpufreq_stats will be registered - * before cpufreq driver - */ - policy = cpufreq_cpu_get(cpu); - if (likely(!policy)) - return; - - __cpufreq_stats_create_table(policy); - - cpufreq_cpu_put(policy); } -static int cpufreq_stat_notifier_policy(struct notifier_block *nb, - unsigned long val, void *data) +void cpufreq_stats_record_transition(struct cpufreq_policy *policy, + unsigned int new_freq) { - int ret = 0; - struct cpufreq_policy *policy = data; - - if (val == CPUFREQ_CREATE_POLICY) - ret = __cpufreq_stats_create_table(policy); - else if (val == CPUFREQ_REMOVE_POLICY) - __cpufreq_stats_free_table(policy); - - return ret; -} - -static int cpufreq_stat_notifier_trans(struct notifier_block *nb, - unsigned long val, void *data) -{ - struct cpufreq_freqs *freq = data; - struct cpufreq_policy *policy = cpufreq_cpu_get(freq->cpu); - struct cpufreq_stats *stats; + struct cpufreq_stats *stats = policy->stats; int old_index, new_index; - if (!policy) { - pr_err("%s: No policy found\n", __func__); - return 0; - } - - if (val != CPUFREQ_POSTCHANGE) - goto put_policy; - - if (!policy->stats) { + if (!stats) { pr_debug("%s: No stats found\n", __func__); - goto put_policy; + return; } - stats = policy->stats; - old_index = stats->last_index; - new_index = freq_table_get_index(stats, freq->new); + new_index = freq_table_get_index(stats, new_freq); /* We can't do stats->time_in_state[-1]= .. */ - if (old_index == -1 || new_index == -1) - goto put_policy; - - if (old_index == new_index) - goto put_policy; + if (old_index == -1 || new_index == -1 || old_index == new_index) + return; cpufreq_stats_update(stats); @@ -297,61 +241,4 @@ static int cpufreq_stat_notifier_trans(struct notifier_block *nb, stats->trans_table[old_index * stats->max_state + new_index]++; #endif stats->total_trans++; - -put_policy: - cpufreq_cpu_put(policy); - return 0; } - -static struct notifier_block notifier_policy_block = { - .notifier_call = cpufreq_stat_notifier_policy -}; - -static struct notifier_block notifier_trans_block = { - .notifier_call = cpufreq_stat_notifier_trans -}; - -static int __init cpufreq_stats_init(void) -{ - int ret; - unsigned int cpu; - - spin_lock_init(&cpufreq_stats_lock); - ret = cpufreq_register_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - if (ret) - return ret; - - for_each_online_cpu(cpu) - cpufreq_stats_create_table(cpu); - - ret = cpufreq_register_notifier(¬ifier_trans_block, - CPUFREQ_TRANSITION_NOTIFIER); - if (ret) { - cpufreq_unregister_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - for_each_online_cpu(cpu) - cpufreq_stats_free_table(cpu); - return ret; - } - - return 0; -} -static void __exit cpufreq_stats_exit(void) -{ - unsigned int cpu; - - cpufreq_unregister_notifier(¬ifier_policy_block, - CPUFREQ_POLICY_NOTIFIER); - cpufreq_unregister_notifier(¬ifier_trans_block, - CPUFREQ_TRANSITION_NOTIFIER); - for_each_online_cpu(cpu) - cpufreq_stats_free_table(cpu); -} - -MODULE_AUTHOR("Zou Nan hai "); -MODULE_DESCRIPTION("Export cpufreq stats via sysfs"); -MODULE_LICENSE("GPL"); - -module_init(cpufreq_stats_init); -module_exit(cpufreq_stats_exit); diff --git a/drivers/cpufreq/cpufreq_userspace.c b/drivers/cpufreq/cpufreq_userspace.c index 9f3dec9a3..bd897e3e1 100644 --- a/drivers/cpufreq/cpufreq_userspace.c +++ b/drivers/cpufreq/cpufreq_userspace.c @@ -65,66 +65,66 @@ static int cpufreq_userspace_policy_init(struct cpufreq_policy *policy) return 0; } -static int cpufreq_governor_userspace(struct cpufreq_policy *policy, - unsigned int event) +static void cpufreq_userspace_policy_exit(struct cpufreq_policy *policy) +{ + mutex_lock(&userspace_mutex); + kfree(policy->governor_data); + policy->governor_data = NULL; + mutex_unlock(&userspace_mutex); +} + +static int cpufreq_userspace_policy_start(struct cpufreq_policy *policy) { unsigned int *setspeed = policy->governor_data; - unsigned int cpu = policy->cpu; - int rc = 0; - if (event == CPUFREQ_GOV_POLICY_INIT) - return cpufreq_userspace_policy_init(policy); + BUG_ON(!policy->cur); + pr_debug("started managing cpu %u\n", policy->cpu); - if (!setspeed) - return -EINVAL; - - switch (event) { - case CPUFREQ_GOV_POLICY_EXIT: - mutex_lock(&userspace_mutex); - policy->governor_data = NULL; - kfree(setspeed); - mutex_unlock(&userspace_mutex); - break; - case CPUFREQ_GOV_START: - BUG_ON(!policy->cur); - pr_debug("started managing cpu %u\n", cpu); - - mutex_lock(&userspace_mutex); - per_cpu(cpu_is_managed, cpu) = 1; - *setspeed = policy->cur; - mutex_unlock(&userspace_mutex); - break; - case CPUFREQ_GOV_STOP: - pr_debug("managing cpu %u stopped\n", cpu); - - mutex_lock(&userspace_mutex); - per_cpu(cpu_is_managed, cpu) = 0; - *setspeed = 0; - mutex_unlock(&userspace_mutex); - break; - case CPUFREQ_GOV_LIMITS: - mutex_lock(&userspace_mutex); - pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz, last set to %u kHz\n", - cpu, policy->min, policy->max, policy->cur, *setspeed); - - if (policy->max < *setspeed) - __cpufreq_driver_target(policy, policy->max, - CPUFREQ_RELATION_H); - else if (policy->min > *setspeed) - __cpufreq_driver_target(policy, policy->min, - CPUFREQ_RELATION_L); - else - __cpufreq_driver_target(policy, *setspeed, - CPUFREQ_RELATION_L); - mutex_unlock(&userspace_mutex); - break; - } - return rc; + mutex_lock(&userspace_mutex); + per_cpu(cpu_is_managed, policy->cpu) = 1; + *setspeed = policy->cur; + mutex_unlock(&userspace_mutex); + return 0; +} + +static void cpufreq_userspace_policy_stop(struct cpufreq_policy *policy) +{ + unsigned int *setspeed = policy->governor_data; + + pr_debug("managing cpu %u stopped\n", policy->cpu); + + mutex_lock(&userspace_mutex); + per_cpu(cpu_is_managed, policy->cpu) = 0; + *setspeed = 0; + mutex_unlock(&userspace_mutex); +} + +static void cpufreq_userspace_policy_limits(struct cpufreq_policy *policy) +{ + unsigned int *setspeed = policy->governor_data; + + mutex_lock(&userspace_mutex); + + pr_debug("limit event for cpu %u: %u - %u kHz, currently %u kHz, last set to %u kHz\n", + policy->cpu, policy->min, policy->max, policy->cur, *setspeed); + + if (policy->max < *setspeed) + __cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H); + else if (policy->min > *setspeed) + __cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L); + else + __cpufreq_driver_target(policy, *setspeed, CPUFREQ_RELATION_L); + + mutex_unlock(&userspace_mutex); } static struct cpufreq_governor cpufreq_gov_userspace = { .name = "userspace", - .governor = cpufreq_governor_userspace, + .init = cpufreq_userspace_policy_init, + .exit = cpufreq_userspace_policy_exit, + .start = cpufreq_userspace_policy_start, + .stop = cpufreq_userspace_policy_stop, + .limits = cpufreq_userspace_policy_limits, .store_setspeed = cpufreq_set, .show_setspeed = show_speed, .owner = THIS_MODULE, diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c index 7e336d20c..b95a87280 100644 --- a/drivers/cpufreq/davinci-cpufreq.c +++ b/drivers/cpufreq/davinci-cpufreq.c @@ -38,26 +38,6 @@ struct davinci_cpufreq { }; static struct davinci_cpufreq cpufreq; -static int davinci_verify_speed(struct cpufreq_policy *policy) -{ - struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data; - struct cpufreq_frequency_table *freq_table = pdata->freq_table; - struct clk *armclk = cpufreq.armclk; - - if (freq_table) - return cpufreq_frequency_table_verify(policy, freq_table); - - if (policy->cpu) - return -EINVAL; - - cpufreq_verify_within_cpu_limits(policy); - policy->min = clk_round_rate(armclk, policy->min * 1000) / 1000; - policy->max = clk_round_rate(armclk, policy->max * 1000) / 1000; - cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, - policy->cpuinfo.max_freq); - return 0; -} - static int davinci_target(struct cpufreq_policy *policy, unsigned int idx) { struct davinci_cpufreq_config *pdata = cpufreq.dev->platform_data; @@ -121,7 +101,7 @@ static int davinci_cpu_init(struct cpufreq_policy *policy) static struct cpufreq_driver davinci_driver = { .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, - .verify = davinci_verify_speed, + .verify = cpufreq_generic_frequency_table_verify, .target_index = davinci_target, .get = cpufreq_generic_get, .init = davinci_cpu_init, diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index a8f1daffc..3bbbf9e69 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -63,8 +63,6 @@ int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, else return 0; } -EXPORT_SYMBOL_GPL(cpufreq_frequency_table_cpuinfo); - int cpufreq_frequency_table_verify(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table) @@ -108,20 +106,16 @@ EXPORT_SYMBOL_GPL(cpufreq_frequency_table_verify); */ int cpufreq_generic_frequency_table_verify(struct cpufreq_policy *policy) { - struct cpufreq_frequency_table *table = - cpufreq_frequency_get_table(policy->cpu); - if (!table) + if (!policy->freq_table) return -ENODEV; - return cpufreq_frequency_table_verify(policy, table); + return cpufreq_frequency_table_verify(policy, policy->freq_table); } EXPORT_SYMBOL_GPL(cpufreq_generic_frequency_table_verify); -int cpufreq_frequency_table_target(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *table, - unsigned int target_freq, - unsigned int relation, - unsigned int *index) +int cpufreq_table_index_unsorted(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int relation) { struct cpufreq_frequency_table optimal = { .driver_data = ~0, @@ -132,7 +126,9 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy, .frequency = 0, }; struct cpufreq_frequency_table *pos; + struct cpufreq_frequency_table *table = policy->freq_table; unsigned int freq, diff, i = 0; + int index; pr_debug("request for target %u kHz (relation: %u) for cpu %u\n", target_freq, relation, policy->cpu); @@ -196,25 +192,26 @@ int cpufreq_frequency_table_target(struct cpufreq_policy *policy, } } if (optimal.driver_data > i) { - if (suboptimal.driver_data > i) - return -EINVAL; - *index = suboptimal.driver_data; - } else - *index = optimal.driver_data; + if (suboptimal.driver_data > i) { + WARN(1, "Invalid frequency table: %d\n", policy->cpu); + return 0; + } - pr_debug("target index is %u, freq is:%u kHz\n", *index, - table[*index].frequency); + index = suboptimal.driver_data; + } else + index = optimal.driver_data; - return 0; + pr_debug("target index is %u, freq is:%u kHz\n", index, + table[index].frequency); + return index; } -EXPORT_SYMBOL_GPL(cpufreq_frequency_table_target); +EXPORT_SYMBOL_GPL(cpufreq_table_index_unsorted); int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy, unsigned int freq) { - struct cpufreq_frequency_table *pos, *table; + struct cpufreq_frequency_table *pos, *table = policy->freq_table; - table = cpufreq_frequency_get_table(policy->cpu); if (unlikely(!table)) { pr_debug("%s: Unable to find frequency table\n", __func__); return -ENOENT; @@ -300,15 +297,72 @@ struct freq_attr *cpufreq_generic_attr[] = { }; EXPORT_SYMBOL_GPL(cpufreq_generic_attr); +static int set_freq_table_sorted(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *pos, *table = policy->freq_table; + struct cpufreq_frequency_table *prev = NULL; + int ascending = 0; + + policy->freq_table_sorted = CPUFREQ_TABLE_UNSORTED; + + cpufreq_for_each_valid_entry(pos, table) { + if (!prev) { + prev = pos; + continue; + } + + if (pos->frequency == prev->frequency) { + pr_warn("Duplicate freq-table entries: %u\n", + pos->frequency); + return -EINVAL; + } + + /* Frequency increased from prev to pos */ + if (pos->frequency > prev->frequency) { + /* But frequency was decreasing earlier */ + if (ascending < 0) { + pr_debug("Freq table is unsorted\n"); + return 0; + } + + ascending++; + } else { + /* Frequency decreased from prev to pos */ + + /* But frequency was increasing earlier */ + if (ascending > 0) { + pr_debug("Freq table is unsorted\n"); + return 0; + } + + ascending--; + } + + prev = pos; + } + + if (ascending > 0) + policy->freq_table_sorted = CPUFREQ_TABLE_SORTED_ASCENDING; + else + policy->freq_table_sorted = CPUFREQ_TABLE_SORTED_DESCENDING; + + pr_debug("Freq table is sorted in %s order\n", + ascending > 0 ? "ascending" : "descending"); + + return 0; +} + int cpufreq_table_validate_and_show(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table) { - int ret = cpufreq_frequency_table_cpuinfo(policy, table); + int ret; - if (!ret) - policy->freq_table = table; + ret = cpufreq_frequency_table_cpuinfo(policy, table); + if (ret) + return ret; - return ret; + policy->freq_table = table; + return set_freq_table_sorted(policy); } EXPORT_SYMBOL_GPL(cpufreq_table_validate_and_show); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 1b159171f..be9eade14 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -35,6 +35,7 @@ #include #include #include +#include #define ATOM_RATIOS 0x66a #define ATOM_VIDS 0x66b @@ -96,7 +97,6 @@ static inline u64 div_ext_fp(u64 x, u64 y) * read from MPERF MSR between last and current sample * @tsc: Difference of time stamp counter between last and * current sample - * @freq: Effective frequency calculated from APERF/MPERF * @time: Current time from scheduler * * This structure is used in the cpudata structure to store performance sample @@ -108,7 +108,6 @@ struct sample { u64 aperf; u64 mperf; u64 tsc; - int freq; u64 time; }; @@ -281,9 +280,9 @@ struct cpu_defaults { static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu); static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu); -static struct pstate_adjust_policy pid_params; -static struct pstate_funcs pstate_funcs; -static int hwp_active; +static struct pstate_adjust_policy pid_params __read_mostly; +static struct pstate_funcs pstate_funcs __read_mostly; +static int hwp_active __read_mostly; #ifdef CONFIG_ACPI static bool acpi_ppc; @@ -807,7 +806,8 @@ static void __init intel_pstate_sysfs_expose_params(void) static void intel_pstate_hwp_enable(struct cpudata *cpudata) { /* First disable HWP notification interrupt as we don't process them */ - wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); + if (static_cpu_has(X86_FEATURE_HWP_NOTIFY)) + wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); } @@ -972,7 +972,7 @@ static int core_get_turbo_pstate(void) u64 value; int nont, ret; - rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value); + rdmsrl(MSR_TURBO_RATIO_LIMIT, value); nont = core_get_max_pstate(); ret = (value) & 255; if (ret <= nont) @@ -1001,7 +1001,7 @@ static int knl_get_turbo_pstate(void) u64 value; int nont, ret; - rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value); + rdmsrl(MSR_TURBO_RATIO_LIMIT, value); nont = core_get_max_pstate(); ret = (((value) >> 8) & 0xFF); if (ret <= nont) @@ -1091,6 +1091,26 @@ static struct cpu_defaults knl_params = { }, }; +static struct cpu_defaults bxt_params = { + .pid_policy = { + .sample_rate_ms = 10, + .deadband = 0, + .setpoint = 60, + .p_gain_pct = 14, + .d_gain_pct = 0, + .i_gain_pct = 4, + }, + .funcs = { + .get_max = core_get_max_pstate, + .get_max_physical = core_get_max_pstate_physical, + .get_min = core_get_min_pstate, + .get_turbo = core_get_turbo_pstate, + .get_scaling = core_get_scaling, + .get_val = core_get_val, + .get_target_pstate = get_target_pstate_use_cpu_load, + }, +}; + static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) { int max_perf = cpu->pstate.turbo_pstate; @@ -1113,17 +1133,12 @@ static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf); } -static inline void intel_pstate_record_pstate(struct cpudata *cpu, int pstate) -{ - trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); - cpu->pstate.current_pstate = pstate; -} - static void intel_pstate_set_min_pstate(struct cpudata *cpu) { int pstate = cpu->pstate.min_pstate; - intel_pstate_record_pstate(cpu, pstate); + trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); + cpu->pstate.current_pstate = pstate; /* * Generally, there is no guarantee that this code will always run on * the CPU being updated, so force the register update to run on the @@ -1283,10 +1298,11 @@ static inline void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) intel_pstate_get_min_max(cpu, &min_perf, &max_perf); pstate = clamp_t(int, pstate, min_perf, max_perf); + trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); if (pstate == cpu->pstate.current_pstate) return; - intel_pstate_record_pstate(cpu, pstate); + cpu->pstate.current_pstate = pstate; wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate)); } @@ -1334,29 +1350,32 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time, (unsigned long)&policy } static const struct x86_cpu_id intel_pstate_cpu_ids[] = { - ICPU(0x2a, core_params), - ICPU(0x2d, core_params), - ICPU(0x37, silvermont_params), - ICPU(0x3a, core_params), - ICPU(0x3c, core_params), - ICPU(0x3d, core_params), - ICPU(0x3e, core_params), - ICPU(0x3f, core_params), - ICPU(0x45, core_params), - ICPU(0x46, core_params), - ICPU(0x47, core_params), - ICPU(0x4c, airmont_params), - ICPU(0x4e, core_params), - ICPU(0x4f, core_params), - ICPU(0x5e, core_params), - ICPU(0x56, core_params), - ICPU(0x57, knl_params), + ICPU(INTEL_FAM6_SANDYBRIDGE, core_params), + ICPU(INTEL_FAM6_SANDYBRIDGE_X, core_params), + ICPU(INTEL_FAM6_ATOM_SILVERMONT1, silvermont_params), + ICPU(INTEL_FAM6_IVYBRIDGE, core_params), + ICPU(INTEL_FAM6_HASWELL_CORE, core_params), + ICPU(INTEL_FAM6_BROADWELL_CORE, core_params), + ICPU(INTEL_FAM6_IVYBRIDGE_X, core_params), + ICPU(INTEL_FAM6_HASWELL_X, core_params), + ICPU(INTEL_FAM6_HASWELL_ULT, core_params), + ICPU(INTEL_FAM6_HASWELL_GT3E, core_params), + ICPU(INTEL_FAM6_BROADWELL_GT3E, core_params), + ICPU(INTEL_FAM6_ATOM_AIRMONT, airmont_params), + ICPU(INTEL_FAM6_SKYLAKE_MOBILE, core_params), + ICPU(INTEL_FAM6_BROADWELL_X, core_params), + ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_params), + ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), + ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_params), + ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_params), {} }; MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); -static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] = { - ICPU(0x56, core_params), +static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { + ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), + ICPU(INTEL_FAM6_BROADWELL_X, core_params), + ICPU(INTEL_FAM6_SKYLAKE_X, core_params), {} }; @@ -1575,12 +1594,12 @@ static struct cpufreq_driver intel_pstate_driver = { .name = "intel_pstate", }; -static int __initdata no_load; -static int __initdata no_hwp; -static int __initdata hwp_only; -static unsigned int force_load; +static int no_load __initdata; +static int no_hwp __initdata; +static int hwp_only __initdata; +static unsigned int force_load __initdata; -static int intel_pstate_msrs_not_valid(void) +static int __init intel_pstate_msrs_not_valid(void) { if (!pstate_funcs.get_max() || !pstate_funcs.get_min() || @@ -1590,7 +1609,7 @@ static int intel_pstate_msrs_not_valid(void) return 0; } -static void copy_pid_params(struct pstate_adjust_policy *policy) +static void __init copy_pid_params(struct pstate_adjust_policy *policy) { pid_params.sample_rate_ms = policy->sample_rate_ms; pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC; @@ -1601,7 +1620,7 @@ static void copy_pid_params(struct pstate_adjust_policy *policy) pid_params.setpoint = policy->setpoint; } -static void copy_cpu_funcs(struct pstate_funcs *funcs) +static void __init copy_cpu_funcs(struct pstate_funcs *funcs) { pstate_funcs.get_max = funcs->get_max; pstate_funcs.get_max_physical = funcs->get_max_physical; @@ -1616,7 +1635,7 @@ static void copy_cpu_funcs(struct pstate_funcs *funcs) #ifdef CONFIG_ACPI -static bool intel_pstate_no_acpi_pss(void) +static bool __init intel_pstate_no_acpi_pss(void) { int i; @@ -1645,7 +1664,7 @@ static bool intel_pstate_no_acpi_pss(void) return true; } -static bool intel_pstate_has_acpi_ppc(void) +static bool __init intel_pstate_has_acpi_ppc(void) { int i; @@ -1673,7 +1692,7 @@ struct hw_vendor_info { }; /* Hardware vendor-specific info that has its own power management modes */ -static struct hw_vendor_info vendor_info[] = { +static struct hw_vendor_info vendor_info[] __initdata = { {1, "HP ", "ProLiant", PSS}, {1, "ORACLE", "X4-2 ", PPC}, {1, "ORACLE", "X4-2L ", PPC}, @@ -1692,7 +1711,7 @@ static struct hw_vendor_info vendor_info[] = { {0, "", ""}, }; -static bool intel_pstate_platform_pwr_mgmt_exists(void) +static bool __init intel_pstate_platform_pwr_mgmt_exists(void) { struct acpi_table_header hdr; struct hw_vendor_info *v_info; diff --git a/drivers/cpufreq/mvebu-cpufreq.c b/drivers/cpufreq/mvebu-cpufreq.c index e920889b9..ed915ee85 100644 --- a/drivers/cpufreq/mvebu-cpufreq.c +++ b/drivers/cpufreq/mvebu-cpufreq.c @@ -70,7 +70,7 @@ static int __init armada_xp_pmsu_cpufreq_init(void) continue; } - clk = clk_get(cpu_dev, 0); + clk = clk_get(cpu_dev, NULL); if (IS_ERR(clk)) { pr_err("Cannot get clock for CPU %d\n", cpu); return PTR_ERR(clk); diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c index 54c45368e..d3ffde806 100644 --- a/drivers/cpufreq/powernv-cpufreq.c +++ b/drivers/cpufreq/powernv-cpufreq.c @@ -64,12 +64,14 @@ /** * struct global_pstate_info - Per policy data structure to maintain history of * global pstates - * @highest_lpstate: The local pstate from which we are ramping down + * @highest_lpstate_idx: The local pstate index from which we are + * ramping down * @elapsed_time: Time in ms spent in ramping down from - * highest_lpstate + * highest_lpstate_idx * @last_sampled_time: Time from boot in ms when global pstates were * last set - * @last_lpstate,last_gpstate: Last set values for local and global pstates + * @last_lpstate_idx, Last set value of local pstate and global + * last_gpstate_idx pstate in terms of cpufreq table index * @timer: Is used for ramping down if cpu goes idle for * a long time with global pstate held high * @gpstate_lock: A spinlock to maintain synchronization between @@ -77,11 +79,11 @@ * governer's target_index calls */ struct global_pstate_info { - int highest_lpstate; + int highest_lpstate_idx; unsigned int elapsed_time; unsigned int last_sampled_time; - int last_lpstate; - int last_gpstate; + int last_lpstate_idx; + int last_gpstate_idx; spinlock_t gpstate_lock; struct timer_list timer; }; @@ -124,29 +126,66 @@ static int nr_chips; static DEFINE_PER_CPU(struct chip *, chip_info); /* - * Note: The set of pstates consists of contiguous integers, the - * smallest of which is indicated by powernv_pstate_info.min, the - * largest of which is indicated by powernv_pstate_info.max. + * Note: + * The set of pstates consists of contiguous integers. + * powernv_pstate_info stores the index of the frequency table for + * max, min and nominal frequencies. It also stores number of + * available frequencies. * - * The nominal pstate is the highest non-turbo pstate in this - * platform. This is indicated by powernv_pstate_info.nominal. + * powernv_pstate_info.nominal indicates the index to the highest + * non-turbo frequency. */ static struct powernv_pstate_info { - int min; - int max; - int nominal; - int nr_pstates; + unsigned int min; + unsigned int max; + unsigned int nominal; + unsigned int nr_pstates; } powernv_pstate_info; +/* Use following macros for conversions between pstate_id and index */ +static inline int idx_to_pstate(unsigned int i) +{ + if (unlikely(i >= powernv_pstate_info.nr_pstates)) { + pr_warn_once("index %u is out of bound\n", i); + return powernv_freqs[powernv_pstate_info.nominal].driver_data; + } + + return powernv_freqs[i].driver_data; +} + +static inline unsigned int pstate_to_idx(int pstate) +{ + int min = powernv_freqs[powernv_pstate_info.min].driver_data; + int max = powernv_freqs[powernv_pstate_info.max].driver_data; + + if (min > 0) { + if (unlikely((pstate < max) || (pstate > min))) { + pr_warn_once("pstate %d is out of bound\n", pstate); + return powernv_pstate_info.nominal; + } + } else { + if (unlikely((pstate > max) || (pstate < min))) { + pr_warn_once("pstate %d is out of bound\n", pstate); + return powernv_pstate_info.nominal; + } + } + /* + * abs() is deliberately used so that is works with + * both monotonically increasing and decreasing + * pstate values + */ + return abs(pstate - idx_to_pstate(powernv_pstate_info.max)); +} + static inline void reset_gpstates(struct cpufreq_policy *policy) { struct global_pstate_info *gpstates = policy->driver_data; - gpstates->highest_lpstate = 0; + gpstates->highest_lpstate_idx = 0; gpstates->elapsed_time = 0; gpstates->last_sampled_time = 0; - gpstates->last_lpstate = 0; - gpstates->last_gpstate = 0; + gpstates->last_lpstate_idx = 0; + gpstates->last_gpstate_idx = 0; } /* @@ -156,9 +195,10 @@ static inline void reset_gpstates(struct cpufreq_policy *policy) static int init_powernv_pstates(void) { struct device_node *power_mgt; - int i, pstate_min, pstate_max, pstate_nominal, nr_pstates = 0; + int i, nr_pstates = 0; const __be32 *pstate_ids, *pstate_freqs; u32 len_ids, len_freqs; + u32 pstate_min, pstate_max, pstate_nominal; power_mgt = of_find_node_by_path("/ibm,opal/power-mgt"); if (!power_mgt) { @@ -208,6 +248,7 @@ static int init_powernv_pstates(void) return -ENODEV; } + powernv_pstate_info.nr_pstates = nr_pstates; pr_debug("NR PStates %d\n", nr_pstates); for (i = 0; i < nr_pstates; i++) { u32 id = be32_to_cpu(pstate_ids[i]); @@ -216,15 +257,17 @@ static int init_powernv_pstates(void) pr_debug("PState id %d freq %d MHz\n", id, freq); powernv_freqs[i].frequency = freq * 1000; /* kHz */ powernv_freqs[i].driver_data = id; + + if (id == pstate_max) + powernv_pstate_info.max = i; + else if (id == pstate_nominal) + powernv_pstate_info.nominal = i; + else if (id == pstate_min) + powernv_pstate_info.min = i; } + /* End of list marker entry */ powernv_freqs[i].frequency = CPUFREQ_TABLE_END; - - powernv_pstate_info.min = pstate_min; - powernv_pstate_info.max = pstate_max; - powernv_pstate_info.nominal = pstate_nominal; - powernv_pstate_info.nr_pstates = nr_pstates; - return 0; } @@ -233,12 +276,12 @@ static unsigned int pstate_id_to_freq(int pstate_id) { int i; - i = powernv_pstate_info.max - pstate_id; + i = pstate_to_idx(pstate_id); if (i >= powernv_pstate_info.nr_pstates || i < 0) { pr_warn("PState id %d outside of PState table, " "reporting nominal id %d instead\n", - pstate_id, powernv_pstate_info.nominal); - i = powernv_pstate_info.max - powernv_pstate_info.nominal; + pstate_id, idx_to_pstate(powernv_pstate_info.nominal)); + i = powernv_pstate_info.nominal; } return powernv_freqs[i].frequency; @@ -252,7 +295,7 @@ static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy, char *buf) { return sprintf(buf, "%u\n", - pstate_id_to_freq(powernv_pstate_info.nominal)); + powernv_freqs[powernv_pstate_info.nominal].frequency); } struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq = @@ -426,7 +469,7 @@ static void set_pstate(void *data) */ static inline unsigned int get_nominal_index(void) { - return powernv_pstate_info.max - powernv_pstate_info.nominal; + return powernv_pstate_info.nominal; } static void powernv_cpufreq_throttle_check(void *data) @@ -435,20 +478,22 @@ static void powernv_cpufreq_throttle_check(void *data) unsigned int cpu = smp_processor_id(); unsigned long pmsr; int pmsr_pmax; + unsigned int pmsr_pmax_idx; pmsr = get_pmspr(SPRN_PMSR); chip = this_cpu_read(chip_info); /* Check for Pmax Capping */ pmsr_pmax = (s8)PMSR_MAX(pmsr); - if (pmsr_pmax != powernv_pstate_info.max) { + pmsr_pmax_idx = pstate_to_idx(pmsr_pmax); + if (pmsr_pmax_idx != powernv_pstate_info.max) { if (chip->throttled) goto next; chip->throttled = true; - if (pmsr_pmax < powernv_pstate_info.nominal) { - pr_warn_once("CPU %d on Chip %u has Pmax reduced below nominal frequency (%d < %d)\n", + if (pmsr_pmax_idx > powernv_pstate_info.nominal) { + pr_warn_once("CPU %d on Chip %u has Pmax(%d) reduced below nominal frequency(%d)\n", cpu, chip->id, pmsr_pmax, - powernv_pstate_info.nominal); + idx_to_pstate(powernv_pstate_info.nominal)); chip->throttle_sub_turbo++; } else { chip->throttle_turbo++; @@ -484,34 +529,35 @@ next: /** * calc_global_pstate - Calculate global pstate - * @elapsed_time: Elapsed time in milliseconds - * @local_pstate: New local pstate - * @highest_lpstate: pstate from which its ramping down + * @elapsed_time: Elapsed time in milliseconds + * @local_pstate_idx: New local pstate + * @highest_lpstate_idx: pstate from which its ramping down * * Finds the appropriate global pstate based on the pstate from which its * ramping down and the time elapsed in ramping down. It follows a quadratic * equation which ensures that it reaches ramping down to pmin in 5sec. */ static inline int calc_global_pstate(unsigned int elapsed_time, - int highest_lpstate, int local_pstate) + int highest_lpstate_idx, + int local_pstate_idx) { - int pstate_diff; + int index_diff; /* * Using ramp_down_percent we get the percentage of rampdown * that we are expecting to be dropping. Difference between - * highest_lpstate and powernv_pstate_info.min will give a absolute + * highest_lpstate_idx and powernv_pstate_info.min will give a absolute * number of how many pstates we will drop eventually by the end of * 5 seconds, then just scale it get the number pstates to be dropped. */ - pstate_diff = ((int)ramp_down_percent(elapsed_time) * - (highest_lpstate - powernv_pstate_info.min)) / 100; + index_diff = ((int)ramp_down_percent(elapsed_time) * + (powernv_pstate_info.min - highest_lpstate_idx)) / 100; /* Ensure that global pstate is >= to local pstate */ - if (highest_lpstate - pstate_diff < local_pstate) - return local_pstate; + if (highest_lpstate_idx + index_diff >= local_pstate_idx) + return local_pstate_idx; else - return highest_lpstate - pstate_diff; + return highest_lpstate_idx + index_diff; } static inline void queue_gpstate_timer(struct global_pstate_info *gpstates) @@ -530,8 +576,7 @@ static inline void queue_gpstate_timer(struct global_pstate_info *gpstates) else timer_interval = GPSTATE_TIMER_INTERVAL; - mod_timer_pinned(&gpstates->timer, jiffies + - msecs_to_jiffies(timer_interval)); + mod_timer(&gpstates->timer, jiffies + msecs_to_jiffies(timer_interval)); } /** @@ -547,7 +592,7 @@ void gpstate_timer_handler(unsigned long data) { struct cpufreq_policy *policy = (struct cpufreq_policy *)data; struct global_pstate_info *gpstates = policy->driver_data; - int gpstate_id; + int gpstate_idx; unsigned int time_diff = jiffies_to_msecs(jiffies) - gpstates->last_sampled_time; struct powernv_smp_call_data freq_data; @@ -557,29 +602,29 @@ void gpstate_timer_handler(unsigned long data) gpstates->last_sampled_time += time_diff; gpstates->elapsed_time += time_diff; - freq_data.pstate_id = gpstates->last_lpstate; + freq_data.pstate_id = idx_to_pstate(gpstates->last_lpstate_idx); - if ((gpstates->last_gpstate == freq_data.pstate_id) || + if ((gpstates->last_gpstate_idx == gpstates->last_lpstate_idx) || (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME)) { - gpstate_id = freq_data.pstate_id; + gpstate_idx = pstate_to_idx(freq_data.pstate_id); reset_gpstates(policy); - gpstates->highest_lpstate = freq_data.pstate_id; + gpstates->highest_lpstate_idx = gpstate_idx; } else { - gpstate_id = calc_global_pstate(gpstates->elapsed_time, - gpstates->highest_lpstate, - freq_data.pstate_id); + gpstate_idx = calc_global_pstate(gpstates->elapsed_time, + gpstates->highest_lpstate_idx, + gpstates->last_lpstate_idx); } /* * If local pstate is equal to global pstate, rampdown is over * So timer is not required to be queued. */ - if (gpstate_id != freq_data.pstate_id) + if (gpstate_idx != gpstates->last_lpstate_idx) queue_gpstate_timer(gpstates); - freq_data.gpstate_id = gpstate_id; - gpstates->last_gpstate = freq_data.gpstate_id; - gpstates->last_lpstate = freq_data.pstate_id; + freq_data.gpstate_id = idx_to_pstate(gpstate_idx); + gpstates->last_gpstate_idx = pstate_to_idx(freq_data.gpstate_id); + gpstates->last_lpstate_idx = pstate_to_idx(freq_data.pstate_id); spin_unlock(&gpstates->gpstate_lock); @@ -596,7 +641,7 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, unsigned int new_index) { struct powernv_smp_call_data freq_data; - unsigned int cur_msec, gpstate_id; + unsigned int cur_msec, gpstate_idx; struct global_pstate_info *gpstates = policy->driver_data; if (unlikely(rebooting) && new_index != get_nominal_index()) @@ -608,15 +653,15 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, cur_msec = jiffies_to_msecs(get_jiffies_64()); spin_lock(&gpstates->gpstate_lock); - freq_data.pstate_id = powernv_freqs[new_index].driver_data; + freq_data.pstate_id = idx_to_pstate(new_index); if (!gpstates->last_sampled_time) { - gpstate_id = freq_data.pstate_id; - gpstates->highest_lpstate = freq_data.pstate_id; + gpstate_idx = new_index; + gpstates->highest_lpstate_idx = new_index; goto gpstates_done; } - if (gpstates->last_gpstate > freq_data.pstate_id) { + if (gpstates->last_gpstate_idx < new_index) { gpstates->elapsed_time += cur_msec - gpstates->last_sampled_time; @@ -627,34 +672,34 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, */ if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) { reset_gpstates(policy); - gpstates->highest_lpstate = freq_data.pstate_id; - gpstate_id = freq_data.pstate_id; + gpstates->highest_lpstate_idx = new_index; + gpstate_idx = new_index; } else { /* Elaspsed_time is less than 5 seconds, continue to rampdown */ - gpstate_id = calc_global_pstate(gpstates->elapsed_time, - gpstates->highest_lpstate, - freq_data.pstate_id); + gpstate_idx = calc_global_pstate(gpstates->elapsed_time, + gpstates->highest_lpstate_idx, + new_index); } } else { reset_gpstates(policy); - gpstates->highest_lpstate = freq_data.pstate_id; - gpstate_id = freq_data.pstate_id; + gpstates->highest_lpstate_idx = new_index; + gpstate_idx = new_index; } /* * If local pstate is equal to global pstate, rampdown is over * So timer is not required to be queued. */ - if (gpstate_id != freq_data.pstate_id) + if (gpstate_idx != new_index) queue_gpstate_timer(gpstates); else del_timer_sync(&gpstates->timer); gpstates_done: - freq_data.gpstate_id = gpstate_id; + freq_data.gpstate_id = idx_to_pstate(gpstate_idx); gpstates->last_sampled_time = cur_msec; - gpstates->last_gpstate = freq_data.gpstate_id; - gpstates->last_lpstate = freq_data.pstate_id; + gpstates->last_gpstate_idx = gpstate_idx; + gpstates->last_lpstate_idx = new_index; spin_unlock(&gpstates->gpstate_lock); @@ -699,7 +744,7 @@ static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy) policy->driver_data = gpstates; /* initialize timer */ - init_timer_deferrable(&gpstates->timer); + init_timer_pinned_deferrable(&gpstates->timer); gpstates->timer.data = (unsigned long)policy; gpstates->timer.function = gpstate_timer_handler; gpstates->timer.expires = jiffies + @@ -760,9 +805,7 @@ void powernv_cpufreq_work_fn(struct work_struct *work) struct cpufreq_policy policy; cpufreq_get_policy(&policy, cpu); - cpufreq_frequency_table_target(&policy, policy.freq_table, - policy.cur, - CPUFREQ_RELATION_C, &index); + index = cpufreq_table_find_index_c(&policy, policy.cur); powernv_cpufreq_target_index(&policy, index); cpumask_andnot(&mask, &mask, policy.cpus); } @@ -848,8 +891,8 @@ static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy) struct powernv_smp_call_data freq_data; struct global_pstate_info *gpstates = policy->driver_data; - freq_data.pstate_id = powernv_pstate_info.min; - freq_data.gpstate_id = powernv_pstate_info.min; + freq_data.pstate_id = idx_to_pstate(powernv_pstate_info.min); + freq_data.gpstate_id = idx_to_pstate(powernv_pstate_info.min); smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1); del_timer_sync(&gpstates->timer); } diff --git a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c index 7c4cd5c63..dc112481a 100644 --- a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c +++ b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c @@ -94,7 +94,7 @@ static int pmi_notifier(struct notifier_block *nb, unsigned long event, void *data) { struct cpufreq_policy *policy = data; - struct cpufreq_frequency_table *cbe_freqs; + struct cpufreq_frequency_table *cbe_freqs = policy->freq_table; u8 node; /* Should this really be called for CPUFREQ_ADJUST and CPUFREQ_NOTIFY @@ -103,7 +103,6 @@ static int pmi_notifier(struct notifier_block *nb, if (event == CPUFREQ_START) return 0; - cbe_freqs = cpufreq_frequency_get_table(policy->cpu); node = cbe_cpu_to_node(policy->cpu); pr_debug("got notified, event=%lu, node=%u\n", event, node); diff --git a/drivers/cpufreq/s3c24xx-cpufreq.c b/drivers/cpufreq/s3c24xx-cpufreq.c index ae8eaed77..7b596fa38 100644 --- a/drivers/cpufreq/s3c24xx-cpufreq.c +++ b/drivers/cpufreq/s3c24xx-cpufreq.c @@ -293,12 +293,8 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy, __func__, policy, target_freq, relation); if (ftab) { - if (cpufreq_frequency_table_target(policy, ftab, - target_freq, relation, - &index)) { - s3c_freq_dbg("%s: table failed\n", __func__); - return -EINVAL; - } + index = cpufreq_frequency_table_target(policy, target_freq, + relation); s3c_freq_dbg("%s: adjust %d to entry %d (%u)\n", __func__, target_freq, index, ftab[index].frequency); @@ -315,7 +311,6 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy, pll = NULL; } else { struct cpufreq_policy tmp_policy; - int ret; /* we keep the cpu pll table in Hz, to ensure we get an * accurate value for the PLL output. */ @@ -323,20 +318,14 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy, tmp_policy.min = policy->min * 1000; tmp_policy.max = policy->max * 1000; tmp_policy.cpu = policy->cpu; + tmp_policy.freq_table = pll_reg; - /* cpufreq_frequency_table_target uses a pointer to 'index' - * which is the number of the table entry, not the value of + /* cpufreq_frequency_table_target returns the index + * of the table entry, not the value of * the table entry's index field. */ - ret = cpufreq_frequency_table_target(&tmp_policy, pll_reg, - target_freq, relation, - &index); - - if (ret < 0) { - pr_err("%s: no PLL available\n", __func__); - goto err_notpossible; - } - + index = cpufreq_frequency_table_target(&tmp_policy, target_freq, + relation); pll = pll_reg + index; s3c_freq_dbg("%s: target %u => %u\n", @@ -346,10 +335,6 @@ static int s3c_cpufreq_target(struct cpufreq_policy *policy, } return s3c_cpufreq_settarget(policy, target_freq, pll); - - err_notpossible: - pr_err("no compatible settings for %d\n", target_freq); - return -EINVAL; } struct clk *s3c_cpufreq_clk_get(struct device *dev, const char *name) @@ -571,11 +556,7 @@ static int s3c_cpufreq_build_freq(void) { int size, ret; - if (!cpu_cur.info->calc_freqtable) - return -EINVAL; - kfree(ftab); - ftab = NULL; size = cpu_cur.info->calc_freqtable(&cpu_cur, NULL, 0); size++; diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c index 06d85917b..f82074eea 100644 --- a/drivers/cpufreq/s5pv210-cpufreq.c +++ b/drivers/cpufreq/s5pv210-cpufreq.c @@ -220,7 +220,7 @@ static void s5pv210_set_refresh(enum s5pv210_dmc_port ch, unsigned long freq) tmp1 /= tmp; - __raw_writel(tmp1, reg); + writel_relaxed(tmp1, reg); } static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index) @@ -246,12 +246,7 @@ static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index) new_freq = s5pv210_freq_table[index].frequency; /* Finding current running level index */ - if (cpufreq_frequency_table_target(policy, s5pv210_freq_table, - old_freq, CPUFREQ_RELATION_H, - &priv_index)) { - ret = -EINVAL; - goto exit; - } + priv_index = cpufreq_table_find_index_h(policy, old_freq); arm_volt = dvs_conf[index].arm_volt; int_volt = dvs_conf[index].int_volt; @@ -301,29 +296,29 @@ static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index) * 1. Temporary Change divider for MFC and G3D * SCLKA2M(200/1=200)->(200/4=50)Mhz */ - reg = __raw_readl(S5P_CLK_DIV2); + reg = readl_relaxed(S5P_CLK_DIV2); reg &= ~(S5P_CLKDIV2_G3D_MASK | S5P_CLKDIV2_MFC_MASK); reg |= (3 << S5P_CLKDIV2_G3D_SHIFT) | (3 << S5P_CLKDIV2_MFC_SHIFT); - __raw_writel(reg, S5P_CLK_DIV2); + writel_relaxed(reg, S5P_CLK_DIV2); /* For MFC, G3D dividing */ do { - reg = __raw_readl(S5P_CLKDIV_STAT0); + reg = readl_relaxed(S5P_CLKDIV_STAT0); } while (reg & ((1 << 16) | (1 << 17))); /* * 2. Change SCLKA2M(200Mhz)to SCLKMPLL in MFC_MUX, G3D MUX * (200/4=50)->(667/4=166)Mhz */ - reg = __raw_readl(S5P_CLK_SRC2); + reg = readl_relaxed(S5P_CLK_SRC2); reg &= ~(S5P_CLKSRC2_G3D_MASK | S5P_CLKSRC2_MFC_MASK); reg |= (1 << S5P_CLKSRC2_G3D_SHIFT) | (1 << S5P_CLKSRC2_MFC_SHIFT); - __raw_writel(reg, S5P_CLK_SRC2); + writel_relaxed(reg, S5P_CLK_SRC2); do { - reg = __raw_readl(S5P_CLKMUX_STAT1); + reg = readl_relaxed(S5P_CLKMUX_STAT1); } while (reg & ((1 << 7) | (1 << 3))); /* @@ -335,19 +330,19 @@ static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index) s5pv210_set_refresh(DMC1, 133000); /* 4. SCLKAPLL -> SCLKMPLL */ - reg = __raw_readl(S5P_CLK_SRC0); + reg = readl_relaxed(S5P_CLK_SRC0); reg &= ~(S5P_CLKSRC0_MUX200_MASK); reg |= (0x1 << S5P_CLKSRC0_MUX200_SHIFT); - __raw_writel(reg, S5P_CLK_SRC0); + writel_relaxed(reg, S5P_CLK_SRC0); do { - reg = __raw_readl(S5P_CLKMUX_STAT0); + reg = readl_relaxed(S5P_CLKMUX_STAT0); } while (reg & (0x1 << 18)); } /* Change divider */ - reg = __raw_readl(S5P_CLK_DIV0); + reg = readl_relaxed(S5P_CLK_DIV0); reg &= ~(S5P_CLKDIV0_APLL_MASK | S5P_CLKDIV0_A2M_MASK | S5P_CLKDIV0_HCLK200_MASK | S5P_CLKDIV0_PCLK100_MASK | @@ -363,25 +358,25 @@ static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index) (clkdiv_val[index][6] << S5P_CLKDIV0_HCLK133_SHIFT) | (clkdiv_val[index][7] << S5P_CLKDIV0_PCLK66_SHIFT)); - __raw_writel(reg, S5P_CLK_DIV0); + writel_relaxed(reg, S5P_CLK_DIV0); do { - reg = __raw_readl(S5P_CLKDIV_STAT0); + reg = readl_relaxed(S5P_CLKDIV_STAT0); } while (reg & 0xff); /* ARM MCS value changed */ - reg = __raw_readl(S5P_ARM_MCS_CON); + reg = readl_relaxed(S5P_ARM_MCS_CON); reg &= ~0x3; if (index >= L3) reg |= 0x3; else reg |= 0x1; - __raw_writel(reg, S5P_ARM_MCS_CON); + writel_relaxed(reg, S5P_ARM_MCS_CON); if (pll_changing) { /* 5. Set Lock time = 30us*24Mhz = 0x2cf */ - __raw_writel(0x2cf, S5P_APLL_LOCK); + writel_relaxed(0x2cf, S5P_APLL_LOCK); /* * 6. Turn on APLL @@ -389,12 +384,12 @@ static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index) * 6-2. Wait untile the PLL is locked */ if (index == L0) - __raw_writel(APLL_VAL_1000, S5P_APLL_CON); + writel_relaxed(APLL_VAL_1000, S5P_APLL_CON); else - __raw_writel(APLL_VAL_800, S5P_APLL_CON); + writel_relaxed(APLL_VAL_800, S5P_APLL_CON); do { - reg = __raw_readl(S5P_APLL_CON); + reg = readl_relaxed(S5P_APLL_CON); } while (!(reg & (0x1 << 29))); /* @@ -402,39 +397,39 @@ static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index) * to SCLKA2M(200Mhz) in MFC_MUX and G3D MUX * (667/4=166)->(200/4=50)Mhz */ - reg = __raw_readl(S5P_CLK_SRC2); + reg = readl_relaxed(S5P_CLK_SRC2); reg &= ~(S5P_CLKSRC2_G3D_MASK | S5P_CLKSRC2_MFC_MASK); reg |= (0 << S5P_CLKSRC2_G3D_SHIFT) | (0 << S5P_CLKSRC2_MFC_SHIFT); - __raw_writel(reg, S5P_CLK_SRC2); + writel_relaxed(reg, S5P_CLK_SRC2); do { - reg = __raw_readl(S5P_CLKMUX_STAT1); + reg = readl_relaxed(S5P_CLKMUX_STAT1); } while (reg & ((1 << 7) | (1 << 3))); /* * 8. Change divider for MFC and G3D * (200/4=50)->(200/1=200)Mhz */ - reg = __raw_readl(S5P_CLK_DIV2); + reg = readl_relaxed(S5P_CLK_DIV2); reg &= ~(S5P_CLKDIV2_G3D_MASK | S5P_CLKDIV2_MFC_MASK); reg |= (clkdiv_val[index][10] << S5P_CLKDIV2_G3D_SHIFT) | (clkdiv_val[index][9] << S5P_CLKDIV2_MFC_SHIFT); - __raw_writel(reg, S5P_CLK_DIV2); + writel_relaxed(reg, S5P_CLK_DIV2); /* For MFC, G3D dividing */ do { - reg = __raw_readl(S5P_CLKDIV_STAT0); + reg = readl_relaxed(S5P_CLKDIV_STAT0); } while (reg & ((1 << 16) | (1 << 17))); /* 9. Change MPLL to APLL in MSYS_MUX */ - reg = __raw_readl(S5P_CLK_SRC0); + reg = readl_relaxed(S5P_CLK_SRC0); reg &= ~(S5P_CLKSRC0_MUX200_MASK); reg |= (0x0 << S5P_CLKSRC0_MUX200_SHIFT); - __raw_writel(reg, S5P_CLK_SRC0); + writel_relaxed(reg, S5P_CLK_SRC0); do { - reg = __raw_readl(S5P_CLKMUX_STAT0); + reg = readl_relaxed(S5P_CLKMUX_STAT0); } while (reg & (0x1 << 18)); /* @@ -451,13 +446,13 @@ static int s5pv210_target(struct cpufreq_policy *policy, unsigned int index) * and memory refresh parameter should be changed */ if (bus_speed_changing) { - reg = __raw_readl(S5P_CLK_DIV6); + reg = readl_relaxed(S5P_CLK_DIV6); reg &= ~S5P_CLKDIV6_ONEDRAM_MASK; reg |= (clkdiv_val[index][8] << S5P_CLKDIV6_ONEDRAM_SHIFT); - __raw_writel(reg, S5P_CLK_DIV6); + writel_relaxed(reg, S5P_CLK_DIV6); do { - reg = __raw_readl(S5P_CLKDIV_STAT1); + reg = readl_relaxed(S5P_CLKDIV_STAT1); } while (reg & (1 << 15)); /* Reconfigure DRAM refresh counter value */ @@ -497,7 +492,7 @@ static int check_mem_type(void __iomem *dmc_reg) { unsigned long val; - val = __raw_readl(dmc_reg + 0x4); + val = readl_relaxed(dmc_reg + 0x4); val = (val & (0xf << 8)); return val >> 8; @@ -542,10 +537,10 @@ static int s5pv210_cpu_init(struct cpufreq_policy *policy) } /* Find current refresh counter and frequency each DMC */ - s5pv210_dram_conf[0].refresh = (__raw_readl(dmc_base[0] + 0x30) * 1000); + s5pv210_dram_conf[0].refresh = (readl_relaxed(dmc_base[0] + 0x30) * 1000); s5pv210_dram_conf[0].freq = clk_get_rate(dmc0_clk); - s5pv210_dram_conf[1].refresh = (__raw_readl(dmc_base[1] + 0x30) * 1000); + s5pv210_dram_conf[1].refresh = (readl_relaxed(dmc_base[1] + 0x30) * 1000); s5pv210_dram_conf[1].freq = clk_get_rate(dmc1_clk); policy->suspend_freq = SLEEP_FREQ; diff --git a/drivers/cpuidle/cpuidle-arm.c b/drivers/cpuidle/cpuidle-arm.c index e342565e8..f440d385e 100644 --- a/drivers/cpuidle/cpuidle-arm.c +++ b/drivers/cpuidle/cpuidle-arm.c @@ -36,26 +36,12 @@ static int arm_enter_idle_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, int idx) { - int ret; - - if (!idx) { - cpu_do_idle(); - return idx; - } - - ret = cpu_pm_enter(); - if (!ret) { - /* - * Pass idle state index to cpu_suspend which in turn will - * call the CPU ops suspend protocol with idle index as a - * parameter. - */ - ret = arm_cpuidle_suspend(idx); - - cpu_pm_exit(); - } - - return ret ? -1 : idx; + /* + * Pass idle state index to arm_cpuidle_suspend which in turn + * will call the CPU ops suspend protocol with idle index as a + * parameter. + */ + return CPU_PM_CPU_IDLE_ENTER(arm_cpuidle_suspend, idx); } static struct cpuidle_driver arm_idle_driver = { @@ -135,6 +121,7 @@ static int __init arm_idle_init(void) dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) { pr_err("Failed to allocate cpuidle device\n"); + ret = -ENOMEM; goto out_fail; } dev->cpu = cpu; diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c index e12dc30d8..f7ca891b5 100644 --- a/drivers/cpuidle/cpuidle-powernv.c +++ b/drivers/cpuidle/cpuidle-powernv.c @@ -20,7 +20,7 @@ #include #include -#define MAX_POWERNV_IDLE_STATES 8 +#define POWERNV_THRESHOLD_LATENCY_NS 200000 struct cpuidle_driver powernv_idle_driver = { .name = "powernv_idle", @@ -29,6 +29,9 @@ struct cpuidle_driver powernv_idle_driver = { static int max_idle_state; static struct cpuidle_state *cpuidle_state_table; + +static u64 stop_psscr_table[CPUIDLE_STATE_MAX]; + static u64 snooze_timeout; static bool snooze_timeout_en; @@ -93,16 +96,27 @@ static int fastsleep_loop(struct cpuidle_device *dev, return index; } #endif + +static int stop_loop(struct cpuidle_device *dev, + struct cpuidle_driver *drv, + int index) +{ + ppc64_runlatch_off(); + power9_idle_stop(stop_psscr_table[index]); + ppc64_runlatch_on(); + return index; +} + /* * States for dedicated partition case. */ -static struct cpuidle_state powernv_states[MAX_POWERNV_IDLE_STATES] = { +static struct cpuidle_state powernv_states[CPUIDLE_STATE_MAX] = { { /* Snooze */ .name = "snooze", .desc = "snooze", .exit_latency = 0, .target_residency = 0, - .enter = &snooze_loop }, + .enter = snooze_loop }, }; static int powernv_cpuidle_add_cpu_notifier(struct notifier_block *n, @@ -168,7 +182,11 @@ static int powernv_add_idle_states(void) struct device_node *power_mgt; int nr_idle_states = 1; /* Snooze */ int dt_idle_states; - u32 *latency_ns, *residency_ns, *flags; + u32 latency_ns[CPUIDLE_STATE_MAX]; + u32 residency_ns[CPUIDLE_STATE_MAX]; + u32 flags[CPUIDLE_STATE_MAX]; + u64 psscr_val[CPUIDLE_STATE_MAX]; + const char *names[CPUIDLE_STATE_MAX]; int i, rc; /* Currently we have snooze statically defined */ @@ -186,26 +204,55 @@ static int powernv_add_idle_states(void) goto out; } - flags = kzalloc(sizeof(*flags) * dt_idle_states, GFP_KERNEL); + /* + * Since snooze is used as first idle state, max idle states allowed is + * CPUIDLE_STATE_MAX -1 + */ + if (dt_idle_states > CPUIDLE_STATE_MAX - 1) { + pr_warn("cpuidle-powernv: discovered idle states more than allowed"); + dt_idle_states = CPUIDLE_STATE_MAX - 1; + } + if (of_property_read_u32_array(power_mgt, "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { pr_warn("cpuidle-powernv : missing ibm,cpu-idle-state-flags in DT\n"); - goto out_free_flags; + goto out; } - latency_ns = kzalloc(sizeof(*latency_ns) * dt_idle_states, GFP_KERNEL); - rc = of_property_read_u32_array(power_mgt, - "ibm,cpu-idle-state-latencies-ns", latency_ns, dt_idle_states); - if (rc) { + if (of_property_read_u32_array(power_mgt, + "ibm,cpu-idle-state-latencies-ns", latency_ns, + dt_idle_states)) { pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n"); - goto out_free_latency; + goto out; + } + if (of_property_read_string_array(power_mgt, + "ibm,cpu-idle-state-names", names, dt_idle_states) < 0) { + pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n"); + goto out; } - residency_ns = kzalloc(sizeof(*residency_ns) * dt_idle_states, GFP_KERNEL); + /* + * If the idle states use stop instruction, probe for psscr values + * which are necessary to specify required stop level. + */ + if (flags[0] & (OPAL_PM_STOP_INST_FAST | OPAL_PM_STOP_INST_DEEP)) + if (of_property_read_u64_array(power_mgt, + "ibm,cpu-idle-state-psscr", psscr_val, dt_idle_states)) { + pr_warn("cpuidle-powernv: missing ibm,cpu-idle-states-psscr in DT\n"); + goto out; + } + rc = of_property_read_u32_array(power_mgt, "ibm,cpu-idle-state-residency-ns", residency_ns, dt_idle_states); for (i = 0; i < dt_idle_states; i++) { + /* + * If an idle state has exit latency beyond + * POWERNV_THRESHOLD_LATENCY_NS then don't use it + * in cpu-idle. + */ + if (latency_ns[i] > POWERNV_THRESHOLD_LATENCY_NS) + continue; /* * Cpuidle accepts exit_latency and target_residency in us. @@ -217,7 +264,17 @@ static int powernv_add_idle_states(void) strcpy(powernv_states[nr_idle_states].desc, "Nap"); powernv_states[nr_idle_states].flags = 0; powernv_states[nr_idle_states].target_residency = 100; - powernv_states[nr_idle_states].enter = &nap_loop; + powernv_states[nr_idle_states].enter = nap_loop; + } else if ((flags[i] & OPAL_PM_STOP_INST_FAST) && + !(flags[i] & OPAL_PM_TIMEBASE_STOP)) { + strncpy(powernv_states[nr_idle_states].name, + names[i], CPUIDLE_NAME_LEN); + strncpy(powernv_states[nr_idle_states].desc, + names[i], CPUIDLE_NAME_LEN); + powernv_states[nr_idle_states].flags = 0; + + powernv_states[nr_idle_states].enter = stop_loop; + stop_psscr_table[nr_idle_states] = psscr_val[i]; } /* @@ -232,7 +289,17 @@ static int powernv_add_idle_states(void) strcpy(powernv_states[nr_idle_states].desc, "FastSleep"); powernv_states[nr_idle_states].flags = CPUIDLE_FLAG_TIMER_STOP; powernv_states[nr_idle_states].target_residency = 300000; - powernv_states[nr_idle_states].enter = &fastsleep_loop; + powernv_states[nr_idle_states].enter = fastsleep_loop; + } else if ((flags[i] & OPAL_PM_STOP_INST_DEEP) && + (flags[i] & OPAL_PM_TIMEBASE_STOP)) { + strncpy(powernv_states[nr_idle_states].name, + names[i], CPUIDLE_NAME_LEN); + strncpy(powernv_states[nr_idle_states].desc, + names[i], CPUIDLE_NAME_LEN); + + powernv_states[nr_idle_states].flags = CPUIDLE_FLAG_TIMER_STOP; + powernv_states[nr_idle_states].enter = stop_loop; + stop_psscr_table[nr_idle_states] = psscr_val[i]; } #endif powernv_states[nr_idle_states].exit_latency = @@ -245,12 +312,6 @@ static int powernv_add_idle_states(void) nr_idle_states++; } - - kfree(residency_ns); -out_free_latency: - kfree(latency_ns); -out_free_flags: - kfree(flags); out: return nr_idle_states; } diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index d77ba2f12..1af94e2d1 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -159,6 +159,19 @@ config CRYPTO_GHASH_S390 It is available as of z196. +config CRYPTO_CRC32_S390 + tristate "CRC-32 algorithms" + depends on S390 + select CRYPTO_HASH + select CRC32 + help + Select this option if you want to use hardware accelerated + implementations of CRC algorithms. With this option, you + can optimize the computation of CRC-32 (IEEE 802.3 Ethernet) + and CRC-32C (Castagnoli). + + It is available with IBM z13 or later. + config CRYPTO_DEV_MV_CESA tristate "Marvell's Cryptographic Engine" depends on PLAT_ORION diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c index 95b73968c..10db7df36 100644 --- a/drivers/crypto/bfin_crc.c +++ b/drivers/crypto/bfin_crc.c @@ -588,11 +588,6 @@ static int bfin_crypto_crc_probe(struct platform_device *pdev) crypto_init_queue(&crc->queue, CRC_CCRYPTO_QUEUE_LENGTH); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res == NULL) { - dev_err(&pdev->dev, "Cannot get IORESOURCE_MEM\n"); - return -ENOENT; - } - crc->regs = devm_ioremap_resource(dev, res); if (IS_ERR((void *)crc->regs)) { dev_err(&pdev->dev, "Cannot map CRC IO\n"); diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index 5652a5341..64bf3024b 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -1,6 +1,6 @@ config CRYPTO_DEV_FSL_CAAM tristate "Freescale CAAM-Multicore driver backend" - depends on FSL_SOC || ARCH_MXC + depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE help Enables the driver module for Freescale's Cryptographic Accelerator and Assurance Module (CAAM), also known as the SEC version 4 (SEC4). @@ -99,6 +99,18 @@ config CRYPTO_DEV_FSL_CAAM_AHASH_API To compile this as a module, choose M here: the module will be called caamhash. +config CRYPTO_DEV_FSL_CAAM_PKC_API + tristate "Register public key cryptography implementations with Crypto API" + depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR + default y + select CRYPTO_RSA + help + Selecting this will allow SEC Public key support for RSA. + Supported cryptographic primitives: encryption, decryption, + signature and verification. + To compile this as a module, choose M here: the module + will be called caam_pkc. + config CRYPTO_DEV_FSL_CAAM_RNG_API tristate "Register caam device for hwrng API" depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR @@ -116,10 +128,6 @@ config CRYPTO_DEV_FSL_CAAM_IMX def_bool SOC_IMX6 || SOC_IMX7D depends on CRYPTO_DEV_FSL_CAAM -config CRYPTO_DEV_FSL_CAAM_LE - def_bool CRYPTO_DEV_FSL_CAAM_IMX || SOC_LS1021A - depends on CRYPTO_DEV_FSL_CAAM - config CRYPTO_DEV_FSL_CAAM_DEBUG bool "Enable debug output in CAAM driver" depends on CRYPTO_DEV_FSL_CAAM diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index 550758a33..08bf5515a 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -2,7 +2,7 @@ # Makefile for the CAAM backend and dependent components # ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG), y) - EXTRA_CFLAGS := -DDEBUG + ccflags-y := -DDEBUG endif obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o @@ -10,6 +10,8 @@ obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o caam-objs := ctrl.o caam_jr-objs := jr.o key_gen.o error.o +caam_pkc-y := caampkc.o pkc_desc.o diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index e9703f9d1..36365b3ef 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -847,7 +847,7 @@ static int ahash_update_ctx(struct ahash_request *req) *next_buflen, 0); } else { (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= - SEC4_SG_LEN_FIN; + cpu_to_caam32(SEC4_SG_LEN_FIN); } state->current_buf = !state->current_buf; @@ -949,7 +949,8 @@ static int ahash_final_ctx(struct ahash_request *req) state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, buf, state->buf_dma, buflen, last_buflen); - (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= SEC4_SG_LEN_FIN; + (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= + cpu_to_caam32(SEC4_SG_LEN_FIN); edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c new file mode 100644 index 000000000..851015e65 --- /dev/null +++ b/drivers/crypto/caam/caampkc.c @@ -0,0 +1,607 @@ +/* + * caam - Freescale FSL CAAM support for Public Key Cryptography + * + * Copyright 2016 Freescale Semiconductor, Inc. + * + * There is no Shared Descriptor for PKC so that the Job Descriptor must carry + * all the desired key parameters, input and output pointers. + */ +#include "compat.h" +#include "regs.h" +#include "intern.h" +#include "jr.h" +#include "error.h" +#include "desc_constr.h" +#include "sg_sw_sec4.h" +#include "caampkc.h" + +#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + sizeof(struct rsa_pub_pdb)) +#define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \ + sizeof(struct rsa_priv_f1_pdb)) + +static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); + + if (edesc->sec4_sg_bytes) + dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes, + DMA_TO_DEVICE); +} + +static void rsa_pub_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_pub_pdb *pdb = &edesc->pdb.pub; + + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->e_dma, key->e_sz, DMA_TO_DEVICE); +} + +static void rsa_priv_f1_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_priv_f1_pdb *pdb = &edesc->pdb.priv_f1; + + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); +} + +/* RSA Job Completion handler */ +static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context) +{ + struct akcipher_request *req = context; + struct rsa_edesc *edesc; + + if (err) + caam_jr_strstatus(dev, err); + + edesc = container_of(desc, struct rsa_edesc, hw_desc[0]); + + rsa_pub_unmap(dev, edesc, req); + rsa_io_unmap(dev, edesc, req); + kfree(edesc); + + akcipher_request_complete(req, err); +} + +static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err, + void *context) +{ + struct akcipher_request *req = context; + struct rsa_edesc *edesc; + + if (err) + caam_jr_strstatus(dev, err); + + edesc = container_of(desc, struct rsa_edesc, hw_desc[0]); + + rsa_priv_f1_unmap(dev, edesc, req); + rsa_io_unmap(dev, edesc, req); + kfree(edesc); + + akcipher_request_complete(req, err); +} + +static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, + size_t desclen) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *dev = ctx->dev; + struct rsa_edesc *edesc; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + int sgc; + int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; + int src_nents, dst_nents; + + src_nents = sg_nents_for_len(req->src, req->src_len); + dst_nents = sg_nents_for_len(req->dst, req->dst_len); + + if (src_nents > 1) + sec4_sg_len = src_nents; + if (dst_nents > 1) + sec4_sg_len += dst_nents; + + sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); + + /* allocate space for base edesc, hw desc commands and link tables */ + edesc = kzalloc(sizeof(*edesc) + desclen + sec4_sg_bytes, + GFP_DMA | flags); + if (!edesc) + return ERR_PTR(-ENOMEM); + + sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + if (unlikely(!sgc)) { + dev_err(dev, "unable to map source\n"); + goto src_fail; + } + + sgc = dma_map_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE); + if (unlikely(!sgc)) { + dev_err(dev, "unable to map destination\n"); + goto dst_fail; + } + + edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen; + + sec4_sg_index = 0; + if (src_nents > 1) { + sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); + sec4_sg_index += src_nents; + } + if (dst_nents > 1) + sg_to_sec4_sg_last(req->dst, dst_nents, + edesc->sec4_sg + sec4_sg_index, 0); + + /* Save nents for later use in Job Descriptor */ + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + + if (!sec4_sg_bytes) + return edesc; + + edesc->sec4_sg_dma = dma_map_single(dev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(dev, edesc->sec4_sg_dma)) { + dev_err(dev, "unable to map S/G table\n"); + goto sec4_sg_fail; + } + + edesc->sec4_sg_bytes = sec4_sg_bytes; + + return edesc; + +sec4_sg_fail: + dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE); +dst_fail: + dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); +src_fail: + kfree(edesc); + return ERR_PTR(-ENOMEM); +} + +static int set_rsa_pub_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_pub_pdb *pdb = &edesc->pdb.pub; + int sec4_sg_index = 0; + + pdb->n_dma = dma_map_single(dev, key->n, key->n_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->n_dma)) { + dev_err(dev, "Unable to map RSA modulus memory\n"); + return -ENOMEM; + } + + pdb->e_dma = dma_map_single(dev, key->e, key->e_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->e_dma)) { + dev_err(dev, "Unable to map RSA public exponent memory\n"); + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + return -ENOMEM; + } + + if (edesc->src_nents > 1) { + pdb->sgf |= RSA_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->src_nents; + } else { + pdb->f_dma = sg_dma_address(req->src); + } + + if (edesc->dst_nents > 1) { + pdb->sgf |= RSA_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->g_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= (key->e_sz << RSA_PDB_E_SHIFT) | key->n_sz; + pdb->f_len = req->src_len; + + return 0; +} + +static int set_rsa_priv_f1_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_priv_f1_pdb *pdb = &edesc->pdb.priv_f1; + int sec4_sg_index = 0; + + pdb->n_dma = dma_map_single(dev, key->n, key->n_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->n_dma)) { + dev_err(dev, "Unable to map modulus memory\n"); + return -ENOMEM; + } + + pdb->d_dma = dma_map_single(dev, key->d, key->d_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->d_dma)) { + dev_err(dev, "Unable to map RSA private exponent memory\n"); + dma_unmap_single(dev, pdb->n_dma, key->n_sz, DMA_TO_DEVICE); + return -ENOMEM; + } + + if (edesc->src_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->src_nents; + } else { + pdb->g_dma = sg_dma_address(req->src); + } + + if (edesc->dst_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->f_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= (key->d_sz << RSA_PDB_D_SHIFT) | key->n_sz; + + return 0; +} + +static int caam_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + if (unlikely(!key->n || !key->e)) + return -EINVAL; + + if (req->dst_len < key->n_sz) { + req->dst_len = key->n_sz; + dev_err(jrdev, "Output buffer length less than parameter n\n"); + return -EOVERFLOW; + } + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PUB_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Encrypt Protocol Data Block */ + ret = set_rsa_pub_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_pub_desc(edesc->hw_desc, &edesc->pdb.pub); + + ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_pub_done, req); + if (!ret) + return -EINPROGRESS; + + rsa_pub_unmap(jrdev, edesc, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static int caam_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + if (unlikely(!key->n || !key->d)) + return -EINVAL; + + if (req->dst_len < key->n_sz) { + req->dst_len = key->n_sz; + dev_err(jrdev, "Output buffer length less than parameter n\n"); + return -EOVERFLOW; + } + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F1_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Decrypt Protocol Data Block - Private Key Form #1 */ + ret = set_rsa_priv_f1_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_priv_f1_desc(edesc->hw_desc, &edesc->pdb.priv_f1); + + ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f1_done, req); + if (!ret) + return -EINPROGRESS; + + rsa_priv_f1_unmap(jrdev, edesc, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static void caam_rsa_free_key(struct caam_rsa_key *key) +{ + kzfree(key->d); + kfree(key->e); + kfree(key->n); + key->d = NULL; + key->e = NULL; + key->n = NULL; + key->d_sz = 0; + key->e_sz = 0; + key->n_sz = 0; +} + +/** + * caam_read_raw_data - Read a raw byte stream as a positive integer. + * The function skips buffer's leading zeros, copies the remained data + * to a buffer allocated in the GFP_DMA | GFP_KERNEL zone and returns + * the address of the new buffer. + * + * @buf : The data to read + * @nbytes: The amount of data to read + */ +static inline u8 *caam_read_raw_data(const u8 *buf, size_t *nbytes) +{ + u8 *val; + + while (!*buf && *nbytes) { + buf++; + (*nbytes)--; + } + + val = kzalloc(*nbytes, GFP_DMA | GFP_KERNEL); + if (!val) + return NULL; + + memcpy(val, buf, *nbytes); + + return val; +} + +static int caam_rsa_check_key_length(unsigned int len) +{ + if (len > 4096) + return -EINVAL; + return 0; +} + +static int caam_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rsa_key raw_key = {0}; + struct caam_rsa_key *rsa_key = &ctx->key; + int ret; + + /* Free the old RSA key if any */ + caam_rsa_free_key(rsa_key); + + ret = rsa_parse_pub_key(&raw_key, key, keylen); + if (ret) + return ret; + + /* Copy key in DMA zone */ + rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA | GFP_KERNEL); + if (!rsa_key->e) + goto err; + + /* + * Skip leading zeros and copy the positive integer to a buffer + * allocated in the GFP_DMA | GFP_KERNEL zone. The decryption descriptor + * expects a positive integer for the RSA modulus and uses its length as + * decryption output length. + */ + rsa_key->n = caam_read_raw_data(raw_key.n, &raw_key.n_sz); + if (!rsa_key->n) + goto err; + + if (caam_rsa_check_key_length(raw_key.n_sz << 3)) { + caam_rsa_free_key(rsa_key); + return -EINVAL; + } + + rsa_key->e_sz = raw_key.e_sz; + rsa_key->n_sz = raw_key.n_sz; + + memcpy(rsa_key->e, raw_key.e, raw_key.e_sz); + + return 0; +err: + caam_rsa_free_key(rsa_key); + return -ENOMEM; +} + +static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rsa_key raw_key = {0}; + struct caam_rsa_key *rsa_key = &ctx->key; + int ret; + + /* Free the old RSA key if any */ + caam_rsa_free_key(rsa_key); + + ret = rsa_parse_priv_key(&raw_key, key, keylen); + if (ret) + return ret; + + /* Copy key in DMA zone */ + rsa_key->d = kzalloc(raw_key.d_sz, GFP_DMA | GFP_KERNEL); + if (!rsa_key->d) + goto err; + + rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA | GFP_KERNEL); + if (!rsa_key->e) + goto err; + + /* + * Skip leading zeros and copy the positive integer to a buffer + * allocated in the GFP_DMA | GFP_KERNEL zone. The decryption descriptor + * expects a positive integer for the RSA modulus and uses its length as + * decryption output length. + */ + rsa_key->n = caam_read_raw_data(raw_key.n, &raw_key.n_sz); + if (!rsa_key->n) + goto err; + + if (caam_rsa_check_key_length(raw_key.n_sz << 3)) { + caam_rsa_free_key(rsa_key); + return -EINVAL; + } + + rsa_key->d_sz = raw_key.d_sz; + rsa_key->e_sz = raw_key.e_sz; + rsa_key->n_sz = raw_key.n_sz; + + memcpy(rsa_key->d, raw_key.d, raw_key.d_sz); + memcpy(rsa_key->e, raw_key.e, raw_key.e_sz); + + return 0; + +err: + caam_rsa_free_key(rsa_key); + return -ENOMEM; +} + +static int caam_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + + return (key->n) ? key->n_sz : -EINVAL; +} + +/* Per session pkc's driver context creation function */ +static int caam_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + + ctx->dev = caam_jr_alloc(); + + if (IS_ERR(ctx->dev)) { + dev_err(ctx->dev, "Job Ring Device allocation for transform failed\n"); + return PTR_ERR(ctx->dev); + } + + return 0; +} + +/* Per session pkc's driver context cleanup function */ +static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + + caam_rsa_free_key(key); + caam_jr_free(ctx->dev); +} + +static struct akcipher_alg caam_rsa = { + .encrypt = caam_rsa_enc, + .decrypt = caam_rsa_dec, + .sign = caam_rsa_dec, + .verify = caam_rsa_enc, + .set_pub_key = caam_rsa_set_pub_key, + .set_priv_key = caam_rsa_set_priv_key, + .max_size = caam_rsa_max_size, + .init = caam_rsa_init_tfm, + .exit = caam_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "rsa-caam", + .cra_priority = 3000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct caam_rsa_ctx), + }, +}; + +/* Public Key Cryptography module initialization handler */ +static int __init caam_pkc_init(void) +{ + struct device_node *dev_node; + struct platform_device *pdev; + struct device *ctrldev; + struct caam_drv_private *priv; + u32 cha_inst, pk_inst; + int err; + + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); + if (!dev_node) { + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); + if (!dev_node) + return -ENODEV; + } + + pdev = of_find_device_by_node(dev_node); + if (!pdev) { + of_node_put(dev_node); + return -ENODEV; + } + + ctrldev = &pdev->dev; + priv = dev_get_drvdata(ctrldev); + of_node_put(dev_node); + + /* + * If priv is NULL, it's probably because the caam driver wasn't + * properly initialized (e.g. RNG4 init failed). Thus, bail out here. + */ + if (!priv) + return -ENODEV; + + /* Determine public key hardware accelerator presence. */ + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); + pk_inst = (cha_inst & CHA_ID_LS_PK_MASK) >> CHA_ID_LS_PK_SHIFT; + + /* Do not register algorithms if PKHA is not present. */ + if (!pk_inst) + return -ENODEV; + + err = crypto_register_akcipher(&caam_rsa); + if (err) + dev_warn(ctrldev, "%s alg registration failed\n", + caam_rsa.base.cra_driver_name); + else + dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n"); + + return err; +} + +static void __exit caam_pkc_exit(void) +{ + crypto_unregister_akcipher(&caam_rsa); +} + +module_init(caam_pkc_init); +module_exit(caam_pkc_exit); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("FSL CAAM support for PKC functions of crypto API"); +MODULE_AUTHOR("Freescale Semiconductor"); diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h new file mode 100644 index 000000000..f595d159b --- /dev/null +++ b/drivers/crypto/caam/caampkc.h @@ -0,0 +1,70 @@ +/* + * caam - Freescale FSL CAAM support for Public Key Cryptography descriptors + * + * Copyright 2016 Freescale Semiconductor, Inc. + * + * There is no Shared Descriptor for PKC so that the Job Descriptor must carry + * all the desired key parameters, input and output pointers. + */ + +#ifndef _PKC_DESC_H_ +#define _PKC_DESC_H_ +#include "compat.h" +#include "pdb.h" + +/** + * caam_rsa_key - CAAM RSA key structure. Keys are allocated in DMA zone. + * @n : RSA modulus raw byte stream + * @e : RSA public exponent raw byte stream + * @d : RSA private exponent raw byte stream + * @n_sz : length in bytes of RSA modulus n + * @e_sz : length in bytes of RSA public exponent + * @d_sz : length in bytes of RSA private exponent + */ +struct caam_rsa_key { + u8 *n; + u8 *e; + u8 *d; + size_t n_sz; + size_t e_sz; + size_t d_sz; +}; + +/** + * caam_rsa_ctx - per session context. + * @key : RSA key in DMA zone + * @dev : device structure + */ +struct caam_rsa_ctx { + struct caam_rsa_key key; + struct device *dev; +}; + +/** + * rsa_edesc - s/w-extended rsa descriptor + * @src_nents : number of segments in input scatterlist + * @dst_nents : number of segments in output scatterlist + * @sec4_sg_bytes : length of h/w link table + * @sec4_sg_dma : dma address of h/w link table + * @sec4_sg : pointer to h/w link table + * @pdb : specific RSA Protocol Data Block (PDB) + * @hw_desc : descriptor followed by link tables if any + */ +struct rsa_edesc { + int src_nents; + int dst_nents; + int sec4_sg_bytes; + dma_addr_t sec4_sg_dma; + struct sec4_sg_entry *sec4_sg; + union { + struct rsa_pub_pdb pub; + struct rsa_priv_f1_pdb priv_f1; + } pdb; + u32 hw_desc[]; +}; + +/* Descriptor construction primitives. */ +void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb); +void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb); + +#endif diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index b6955ecdf..7149cd249 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -35,8 +35,11 @@ #include #include #include +#include #include #include #include +#include +#include #endif /* !defined(CAAM_COMPAT_H) */ diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 5ad5f3009..0ec112ee5 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -15,6 +15,9 @@ #include "desc_constr.h" #include "error.h" +bool caam_little_end; +EXPORT_SYMBOL(caam_little_end); + /* * i.MX targets tend to have clock control subsystems that can * enable/disable clocking to our device. @@ -106,7 +109,7 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, if (ctrlpriv->virt_en == 1) { - setbits32(&ctrl->deco_rsr, DECORSR_JR0); + clrsetbits_32(&ctrl->deco_rsr, 0, DECORSR_JR0); while (!(rd_reg32(&ctrl->deco_rsr) & DECORSR_VALID) && --timeout) @@ -115,7 +118,7 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, timeout = 100000; } - setbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE); + clrsetbits_32(&ctrl->deco_rq, 0, DECORR_RQD0ENABLE); while (!(rd_reg32(&ctrl->deco_rq) & DECORR_DEN0) && --timeout) @@ -123,12 +126,12 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, if (!timeout) { dev_err(ctrldev, "failed to acquire DECO 0\n"); - clrbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE); + clrsetbits_32(&ctrl->deco_rq, DECORR_RQD0ENABLE, 0); return -ENODEV; } for (i = 0; i < desc_len(desc); i++) - wr_reg32(&deco->descbuf[i], *(desc + i)); + wr_reg32(&deco->descbuf[i], caam32_to_cpu(*(desc + i))); flags = DECO_JQCR_WHL; /* @@ -139,7 +142,7 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, flags |= DECO_JQCR_FOUR; /* Instruct the DECO to execute it */ - setbits32(&deco->jr_ctl_hi, flags); + clrsetbits_32(&deco->jr_ctl_hi, 0, flags); timeout = 10000000; do { @@ -158,10 +161,10 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, DECO_OP_STATUS_HI_ERR_MASK; if (ctrlpriv->virt_en == 1) - clrbits32(&ctrl->deco_rsr, DECORSR_JR0); + clrsetbits_32(&ctrl->deco_rsr, DECORSR_JR0, 0); /* Mark the DECO as free */ - clrbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE); + clrsetbits_32(&ctrl->deco_rq, DECORR_RQD0ENABLE, 0); if (!timeout) return -EAGAIN; @@ -349,7 +352,7 @@ static void kick_trng(struct platform_device *pdev, int ent_delay) r4tst = &ctrl->r4tst[0]; /* put RNG4 into program mode */ - setbits32(&r4tst->rtmctl, RTMCTL_PRGM); + clrsetbits_32(&r4tst->rtmctl, 0, RTMCTL_PRGM); /* * Performance-wise, it does not make sense to @@ -363,7 +366,7 @@ static void kick_trng(struct platform_device *pdev, int ent_delay) >> RTSDCTL_ENT_DLY_SHIFT; if (ent_delay <= val) { /* put RNG4 into run mode */ - clrbits32(&r4tst->rtmctl, RTMCTL_PRGM); + clrsetbits_32(&r4tst->rtmctl, RTMCTL_PRGM, 0); return; } @@ -381,9 +384,9 @@ static void kick_trng(struct platform_device *pdev, int ent_delay) * select raw sampling in both entropy shifter * and statistical checker */ - setbits32(&val, RTMCTL_SAMP_MODE_RAW_ES_SC); + clrsetbits_32(&val, 0, RTMCTL_SAMP_MODE_RAW_ES_SC); /* put RNG4 into run mode */ - clrbits32(&val, RTMCTL_PRGM); + clrsetbits_32(&val, RTMCTL_PRGM, 0); /* write back the control register */ wr_reg32(&r4tst->rtmctl, val); } @@ -406,6 +409,23 @@ int caam_get_era(void) } EXPORT_SYMBOL(caam_get_era); +#ifdef CONFIG_DEBUG_FS +static int caam_debugfs_u64_get(void *data, u64 *val) +{ + *val = caam64_to_cpu(*(u64 *)data); + return 0; +} + +static int caam_debugfs_u32_get(void *data, u64 *val) +{ + *val = caam32_to_cpu(*(u32 *)data); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); +#endif + /* Probe routine for CAAM top (controller) level */ static int caam_probe(struct platform_device *pdev) { @@ -504,6 +524,10 @@ static int caam_probe(struct platform_device *pdev) ret = -ENOMEM; goto disable_caam_emi_slow; } + + caam_little_end = !(bool)(rd_reg32(&ctrl->perfmon.status) & + (CSTA_PLEND | CSTA_ALT_PLEND)); + /* Finding the page size for using the CTPR_MS register */ comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms); pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT; @@ -559,9 +583,9 @@ static int caam_probe(struct platform_device *pdev) } if (ctrlpriv->virt_en == 1) - setbits32(&ctrl->jrstart, JRSTART_JR0_START | - JRSTART_JR1_START | JRSTART_JR2_START | - JRSTART_JR3_START); + clrsetbits_32(&ctrl->jrstart, 0, JRSTART_JR0_START | + JRSTART_JR1_START | JRSTART_JR2_START | + JRSTART_JR3_START); if (sizeof(dma_addr_t) == sizeof(u64)) if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) @@ -693,7 +717,7 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->rng4_sh_init = ~ctrlpriv->rng4_sh_init & RDSTA_IFMASK; /* Enable RDB bit so that RNG works faster */ - setbits32(&ctrl->scfgr, SCFGR_RDBENABLE); + clrsetbits_32(&ctrl->scfgr, 0, SCFGR_RDBENABLE); } /* NOTE: RTIC detection ought to go here, around Si time */ @@ -719,48 +743,59 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); /* Controller-level - performance monitor counters */ + ctrlpriv->ctl_rq_dequeued = - debugfs_create_u64("rq_dequeued", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->req_dequeued); + debugfs_create_file("rq_dequeued", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->req_dequeued, + &caam_fops_u64_ro); ctrlpriv->ctl_ob_enc_req = - debugfs_create_u64("ob_rq_encrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_req); + debugfs_create_file("ob_rq_encrypted", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_enc_req, + &caam_fops_u64_ro); ctrlpriv->ctl_ib_dec_req = - debugfs_create_u64("ib_rq_decrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_req); + debugfs_create_file("ib_rq_decrypted", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_dec_req, + &caam_fops_u64_ro); ctrlpriv->ctl_ob_enc_bytes = - debugfs_create_u64("ob_bytes_encrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_bytes); + debugfs_create_file("ob_bytes_encrypted", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_enc_bytes, + &caam_fops_u64_ro); ctrlpriv->ctl_ob_prot_bytes = - debugfs_create_u64("ob_bytes_protected", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_prot_bytes); + debugfs_create_file("ob_bytes_protected", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_prot_bytes, + &caam_fops_u64_ro); ctrlpriv->ctl_ib_dec_bytes = - debugfs_create_u64("ib_bytes_decrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_bytes); + debugfs_create_file("ib_bytes_decrypted", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_dec_bytes, + &caam_fops_u64_ro); ctrlpriv->ctl_ib_valid_bytes = - debugfs_create_u64("ib_bytes_validated", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_valid_bytes); + debugfs_create_file("ib_bytes_validated", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_valid_bytes, + &caam_fops_u64_ro); /* Controller level - global status values */ ctrlpriv->ctl_faultaddr = - debugfs_create_u64("fault_addr", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultaddr); + debugfs_create_file("fault_addr", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->faultaddr, + &caam_fops_u32_ro); ctrlpriv->ctl_faultdetail = - debugfs_create_u32("fault_detail", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultdetail); + debugfs_create_file("fault_detail", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->faultdetail, + &caam_fops_u32_ro); ctrlpriv->ctl_faultstatus = - debugfs_create_u32("fault_status", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->status); + debugfs_create_file("fault_status", + S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->status, + &caam_fops_u32_ro); /* Internal covering keys (useful in non-secure mode only) */ ctrlpriv->ctl_kek_wrap.data = &ctrlpriv->ctrl->kek[0]; diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index 1e93c6af2..26427c11a 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -20,19 +20,18 @@ #define SEC4_SG_BPID_MASK 0x000000ff #define SEC4_SG_BPID_SHIFT 16 #define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */ -#define SEC4_SG_OFFS_MASK 0x00001fff +#define SEC4_SG_OFFSET_MASK 0x00001fff struct sec4_sg_entry { -#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX +#if !defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) && \ + defined(CONFIG_CRYPTO_DEV_FSL_CAAM_IMX) u32 rsvd1; dma_addr_t ptr; #else u64 ptr; #endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_IMX */ u32 len; - u8 rsvd2; - u8 buf_pool_id; - u16 offset; + u32 bpid_offset; }; /* Max size of any CAAM descriptor in 32-bit words, inclusive of header */ @@ -454,6 +453,8 @@ struct sec4_sg_entry { #define OP_PCLID_PUBLICKEYPAIR (0x14 << OP_PCLID_SHIFT) #define OP_PCLID_DSASIGN (0x15 << OP_PCLID_SHIFT) #define OP_PCLID_DSAVERIFY (0x16 << OP_PCLID_SHIFT) +#define OP_PCLID_RSAENC_PUBKEY (0x18 << OP_PCLID_SHIFT) +#define OP_PCLID_RSADEC_PRVKEY (0x19 << OP_PCLID_SHIFT) /* Assuming OP_TYPE = OP_TYPE_DECAP_PROTOCOL/ENCAP_PROTOCOL */ #define OP_PCLID_IPSEC (0x01 << OP_PCLID_SHIFT) diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 98d07de24..d3869b95e 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -5,6 +5,7 @@ */ #include "desc.h" +#include "regs.h" #define IMMEDIATE (1 << 23) #define CAAM_CMD_SZ sizeof(u32) @@ -30,9 +31,11 @@ LDST_SRCDST_WORD_DECOCTRL | \ (LDOFF_ENABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT)) +extern bool caam_little_end; + static inline int desc_len(u32 *desc) { - return *desc & HDR_DESCLEN_MASK; + return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK; } static inline int desc_bytes(void *desc) @@ -52,7 +55,7 @@ static inline void *sh_desc_pdb(u32 *desc) static inline void init_desc(u32 *desc, u32 options) { - *desc = (options | HDR_ONE) + 1; + *desc = cpu_to_caam32((options | HDR_ONE) + 1); } static inline void init_sh_desc(u32 *desc, u32 options) @@ -74,13 +77,21 @@ static inline void init_job_desc(u32 *desc, u32 options) init_desc(desc, CMD_DESC_HDR | options); } +static inline void init_job_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes) +{ + u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ; + + init_job_desc(desc, (((pdb_len + 1) << HDR_START_IDX_SHIFT)) | options); +} + static inline void append_ptr(u32 *desc, dma_addr_t ptr) { dma_addr_t *offset = (dma_addr_t *)desc_end(desc); - *offset = ptr; + *offset = cpu_to_caam_dma(ptr); - (*desc) += CAAM_PTR_SZ / CAAM_CMD_SZ; + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + + CAAM_PTR_SZ / CAAM_CMD_SZ); } static inline void init_job_desc_shared(u32 *desc, dma_addr_t ptr, int len, @@ -99,16 +110,17 @@ static inline void append_data(u32 *desc, void *data, int len) if (len) /* avoid sparse warning: memcpy with byte count of 0 */ memcpy(offset, data, len); - (*desc) += (len + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ; + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + + (len + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ); } static inline void append_cmd(u32 *desc, u32 command) { u32 *cmd = desc_end(desc); - *cmd = command; + *cmd = cpu_to_caam32(command); - (*desc)++; + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + 1); } #define append_u32 append_cmd @@ -117,16 +129,22 @@ static inline void append_u64(u32 *desc, u64 data) { u32 *offset = desc_end(desc); - *offset = upper_32_bits(data); - *(++offset) = lower_32_bits(data); + /* Only 32-bit alignment is guaranteed in descriptor buffer */ + if (caam_little_end) { + *offset = cpu_to_caam32(lower_32_bits(data)); + *(++offset) = cpu_to_caam32(upper_32_bits(data)); + } else { + *offset = cpu_to_caam32(upper_32_bits(data)); + *(++offset) = cpu_to_caam32(lower_32_bits(data)); + } - (*desc) += 2; + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + 2); } /* Write command without affecting header, and return pointer to next word */ static inline u32 *write_cmd(u32 *desc, u32 command) { - *desc = command; + *desc = cpu_to_caam32(command); return desc + 1; } @@ -168,14 +186,17 @@ APPEND_CMD_RET(move, MOVE) static inline void set_jump_tgt_here(u32 *desc, u32 *jump_cmd) { - *jump_cmd = *jump_cmd | (desc_len(desc) - (jump_cmd - desc)); + *jump_cmd = cpu_to_caam32(caam32_to_cpu(*jump_cmd) | + (desc_len(desc) - (jump_cmd - desc))); } static inline void set_move_tgt_here(u32 *desc, u32 *move_cmd) { - *move_cmd &= ~MOVE_OFFSET_MASK; - *move_cmd = *move_cmd | ((desc_len(desc) << (MOVE_OFFSET_SHIFT + 2)) & - MOVE_OFFSET_MASK); + u32 val = caam32_to_cpu(*move_cmd); + + val &= ~MOVE_OFFSET_MASK; + val |= (desc_len(desc) << (MOVE_OFFSET_SHIFT + 2)) & MOVE_OFFSET_MASK; + *move_cmd = cpu_to_caam32(val); } #define APPEND_CMD(cmd, op) \ diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 5ef4be22e..a81f551ac 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -31,7 +31,7 @@ static int caam_reset_hw_jr(struct device *dev) * mask interrupts since we are going to poll * for reset completion status */ - setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); + clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK); /* initiate flush (required prior to reset) */ wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); @@ -57,7 +57,7 @@ static int caam_reset_hw_jr(struct device *dev) } /* unmask interrupts */ - clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); + clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0); return 0; } @@ -147,7 +147,7 @@ static irqreturn_t caam_jr_interrupt(int irq, void *st_dev) } /* mask valid interrupts */ - setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); + clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK); /* Have valid interrupt at this point, just ACK and trigger */ wr_reg32(&jrp->rregs->jrintstatus, irqstate); @@ -182,7 +182,7 @@ static void caam_jr_dequeue(unsigned long devarg) sw_idx = (tail + i) & (JOBR_DEPTH - 1); if (jrp->outring[hw_idx].desc == - jrp->entinfo[sw_idx].desc_addr_dma) + caam_dma_to_cpu(jrp->entinfo[sw_idx].desc_addr_dma)) break; /* found */ } /* we should never fail to find a matching descriptor */ @@ -200,7 +200,7 @@ static void caam_jr_dequeue(unsigned long devarg) usercall = jrp->entinfo[sw_idx].callbk; userarg = jrp->entinfo[sw_idx].cbkarg; userdesc = jrp->entinfo[sw_idx].desc_addr_virt; - userstatus = jrp->outring[hw_idx].jrstatus; + userstatus = caam32_to_cpu(jrp->outring[hw_idx].jrstatus); /* * Make sure all information from the job has been obtained @@ -236,7 +236,7 @@ static void caam_jr_dequeue(unsigned long devarg) } /* reenable / unmask IRQs */ - clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); + clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0); } /** @@ -330,7 +330,7 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, int head, tail, desc_size; dma_addr_t desc_dma; - desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32); + desc_size = (caam32_to_cpu(*desc) & HDR_JD_LENGTH_MASK) * sizeof(u32); desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE); if (dma_mapping_error(dev, desc_dma)) { dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n"); @@ -356,7 +356,7 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, head_entry->cbkarg = areq; head_entry->desc_addr_dma = desc_dma; - jrp->inpring[jrp->inp_ring_write_index] = desc_dma; + jrp->inpring[jrp->inp_ring_write_index] = cpu_to_caam_dma(desc_dma); /* * Guarantee that the descriptor's DMA address has been written to @@ -444,9 +444,9 @@ static int caam_jr_init(struct device *dev) spin_lock_init(&jrp->outlock); /* Select interrupt coalescing parameters */ - setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC | - (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) | - (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); + clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC | + (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) | + (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); return 0; diff --git a/drivers/crypto/caam/pdb.h b/drivers/crypto/caam/pdb.h index 3a87c0cf8..aaa00dd1c 100644 --- a/drivers/crypto/caam/pdb.h +++ b/drivers/crypto/caam/pdb.h @@ -1,18 +1,19 @@ /* * CAAM Protocol Data Block (PDB) definition header file * - * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2008-2016 Freescale Semiconductor, Inc. * */ #ifndef CAAM_PDB_H #define CAAM_PDB_H +#include "compat.h" /* * PDB- IPSec ESP Header Modification Options */ -#define PDBHMO_ESP_DECAP_SHIFT 12 -#define PDBHMO_ESP_ENCAP_SHIFT 4 +#define PDBHMO_ESP_DECAP_SHIFT 28 +#define PDBHMO_ESP_ENCAP_SHIFT 28 /* * Encap and Decap - Decrement TTL (Hop Limit) - Based on the value of the * Options Byte IP version (IPvsn) field: @@ -32,12 +33,23 @@ */ #define PDBHMO_ESP_DFBIT (0x04 << PDBHMO_ESP_ENCAP_SHIFT) +#define PDBNH_ESP_ENCAP_SHIFT 16 +#define PDBNH_ESP_ENCAP_MASK (0xff << PDBNH_ESP_ENCAP_SHIFT) + +#define PDBHDRLEN_ESP_DECAP_SHIFT 16 +#define PDBHDRLEN_MASK (0x0fff << PDBHDRLEN_ESP_DECAP_SHIFT) + +#define PDB_NH_OFFSET_SHIFT 8 +#define PDB_NH_OFFSET_MASK (0xff << PDB_NH_OFFSET_SHIFT) + /* * PDB - IPSec ESP Encap/Decap Options */ #define PDBOPTS_ESP_ARSNONE 0x00 /* no antireplay window */ #define PDBOPTS_ESP_ARS32 0x40 /* 32-entry antireplay window */ +#define PDBOPTS_ESP_ARS128 0x80 /* 128-entry antireplay window */ #define PDBOPTS_ESP_ARS64 0xc0 /* 64-entry antireplay window */ +#define PDBOPTS_ESP_ARS_MASK 0xc0 /* antireplay window mask */ #define PDBOPTS_ESP_IVSRC 0x20 /* IV comes from internal random gen */ #define PDBOPTS_ESP_ESN 0x10 /* extended sequence included */ #define PDBOPTS_ESP_OUTFMT 0x08 /* output only decapsulation (decap) */ @@ -54,35 +66,73 @@ /* * General IPSec encap/decap PDB definitions */ + +/** + * ipsec_encap_cbc - PDB part for IPsec CBC encapsulation + * @iv: 16-byte array initialization vector + */ struct ipsec_encap_cbc { - u32 iv[4]; + u8 iv[16]; }; +/** + * ipsec_encap_ctr - PDB part for IPsec CTR encapsulation + * @ctr_nonce: 4-byte array nonce + * @ctr_initial: initial count constant + * @iv: initialization vector + */ struct ipsec_encap_ctr { - u32 ctr_nonce; + u8 ctr_nonce[4]; u32 ctr_initial; - u32 iv[2]; + u64 iv; }; +/** + * ipsec_encap_ccm - PDB part for IPsec CCM encapsulation + * @salt: 3-byte array salt (lower 24 bits) + * @ccm_opt: CCM algorithm options - MSB-LSB description: + * b0_flags (8b) - CCM B0; use 0x5B for 8-byte ICV, 0x6B for 12-byte ICV, + * 0x7B for 16-byte ICV (cf. RFC4309, RFC3610) + * ctr_flags (8b) - counter flags; constant equal to 0x3 + * ctr_initial (16b) - initial count constant + * @iv: initialization vector + */ struct ipsec_encap_ccm { - u32 salt; /* lower 24 bits */ - u8 b0_flags; - u8 ctr_flags; - u16 ctr_initial; - u32 iv[2]; + u8 salt[4]; + u32 ccm_opt; + u64 iv; }; +/** + * ipsec_encap_gcm - PDB part for IPsec GCM encapsulation + * @salt: 3-byte array salt (lower 24 bits) + * @rsvd: reserved, do not use + * @iv: initialization vector + */ struct ipsec_encap_gcm { - u32 salt; /* lower 24 bits */ + u8 salt[4]; u32 rsvd1; - u32 iv[2]; + u64 iv; }; +/** + * ipsec_encap_pdb - PDB for IPsec encapsulation + * @options: MSB-LSB description + * hmo (header manipulation options) - 4b + * reserved - 4b + * next header - 8b + * next header offset - 8b + * option flags (depend on selected algorithm) - 8b + * @seq_num_ext_hi: (optional) IPsec Extended Sequence Number (ESN) + * @seq_num: IPsec sequence number + * @spi: IPsec SPI (Security Parameters Index) + * @ip_hdr_len: optional IP Header length (in bytes) + * reserved - 16b + * Opt. IP Hdr Len - 16b + * @ip_hdr: optional IP Header content + */ struct ipsec_encap_pdb { - u8 hmo_rsvd; - u8 ip_nh; - u8 ip_nh_offset; - u8 options; + u32 options; u32 seq_num_ext_hi; u32 seq_num; union { @@ -92,36 +142,65 @@ struct ipsec_encap_pdb { struct ipsec_encap_gcm gcm; }; u32 spi; - u16 rsvd1; - u16 ip_hdr_len; - u32 ip_hdr[0]; /* optional IP Header content */ + u32 ip_hdr_len; + u32 ip_hdr[0]; }; +/** + * ipsec_decap_cbc - PDB part for IPsec CBC decapsulation + * @rsvd: reserved, do not use + */ struct ipsec_decap_cbc { u32 rsvd[2]; }; +/** + * ipsec_decap_ctr - PDB part for IPsec CTR decapsulation + * @ctr_nonce: 4-byte array nonce + * @ctr_initial: initial count constant + */ struct ipsec_decap_ctr { - u32 salt; + u8 ctr_nonce[4]; u32 ctr_initial; }; +/** + * ipsec_decap_ccm - PDB part for IPsec CCM decapsulation + * @salt: 3-byte salt (lower 24 bits) + * @ccm_opt: CCM algorithm options - MSB-LSB description: + * b0_flags (8b) - CCM B0; use 0x5B for 8-byte ICV, 0x6B for 12-byte ICV, + * 0x7B for 16-byte ICV (cf. RFC4309, RFC3610) + * ctr_flags (8b) - counter flags; constant equal to 0x3 + * ctr_initial (16b) - initial count constant + */ struct ipsec_decap_ccm { - u32 salt; - u8 iv_flags; - u8 ctr_flags; - u16 ctr_initial; + u8 salt[4]; + u32 ccm_opt; }; +/** + * ipsec_decap_gcm - PDB part for IPsec GCN decapsulation + * @salt: 4-byte salt + * @rsvd: reserved, do not use + */ struct ipsec_decap_gcm { - u32 salt; + u8 salt[4]; u32 resvd; }; +/** + * ipsec_decap_pdb - PDB for IPsec decapsulation + * @options: MSB-LSB description + * hmo (header manipulation options) - 4b + * IP header length - 12b + * next header offset - 8b + * option flags (depend on selected algorithm) - 8b + * @seq_num_ext_hi: (optional) IPsec Extended Sequence Number (ESN) + * @seq_num: IPsec sequence number + * @anti_replay: Anti-replay window; size depends on ARS (option flags) + */ struct ipsec_decap_pdb { - u16 hmo_ip_hdr_len; - u8 ip_nh_offset; - u8 options; + u32 options; union { struct ipsec_decap_cbc cbc; struct ipsec_decap_ctr ctr; @@ -130,8 +209,7 @@ struct ipsec_decap_pdb { }; u32 seq_num_ext_hi; u32 seq_num; - u32 anti_replay[2]; - u32 end_index[0]; + __be32 anti_replay[4]; }; /* @@ -399,4 +477,52 @@ struct dsa_verify_pdb { u8 *ab; /* only used if ECC processing */ }; +/* RSA Protocol Data Block */ +#define RSA_PDB_SGF_SHIFT 28 +#define RSA_PDB_E_SHIFT 12 +#define RSA_PDB_E_MASK (0xFFF << RSA_PDB_E_SHIFT) +#define RSA_PDB_D_SHIFT 12 +#define RSA_PDB_D_MASK (0xFFF << RSA_PDB_D_SHIFT) + +#define RSA_PDB_SGF_F (0x8 << RSA_PDB_SGF_SHIFT) +#define RSA_PDB_SGF_G (0x4 << RSA_PDB_SGF_SHIFT) +#define RSA_PRIV_PDB_SGF_F (0x4 << RSA_PDB_SGF_SHIFT) +#define RSA_PRIV_PDB_SGF_G (0x8 << RSA_PDB_SGF_SHIFT) + +#define RSA_PRIV_KEY_FRM_1 0 + +/** + * RSA Encrypt Protocol Data Block + * @sgf: scatter-gather field + * @f_dma: dma address of input data + * @g_dma: dma address of encrypted output data + * @n_dma: dma address of RSA modulus + * @e_dma: dma address of RSA public exponent + * @f_len: length in octets of the input data + */ +struct rsa_pub_pdb { + u32 sgf; + dma_addr_t f_dma; + dma_addr_t g_dma; + dma_addr_t n_dma; + dma_addr_t e_dma; + u32 f_len; +} __packed; + +/** + * RSA Decrypt PDB - Private Key Form #1 + * @sgf: scatter-gather field + * @g_dma: dma address of encrypted input data + * @f_dma: dma address of output data + * @n_dma: dma address of RSA modulus + * @d_dma: dma address of RSA private exponent + */ +struct rsa_priv_f1_pdb { + u32 sgf; + dma_addr_t g_dma; + dma_addr_t f_dma; + dma_addr_t n_dma; + dma_addr_t d_dma; +} __packed; + #endif diff --git a/drivers/crypto/caam/pkc_desc.c b/drivers/crypto/caam/pkc_desc.c new file mode 100644 index 000000000..4e4183e61 --- /dev/null +++ b/drivers/crypto/caam/pkc_desc.c @@ -0,0 +1,36 @@ +/* + * caam - Freescale FSL CAAM support for Public Key Cryptography descriptors + * + * Copyright 2016 Freescale Semiconductor, Inc. + * + * There is no Shared Descriptor for PKC so that the Job Descriptor must carry + * all the desired key parameters, input and output pointers. + */ +#include "caampkc.h" +#include "desc_constr.h" + +/* Descriptor for RSA Public operation */ +void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, sizeof(*pdb)); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->n_dma); + append_ptr(desc, pdb->e_dma); + append_cmd(desc, pdb->f_len); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSAENC_PUBKEY); +} + +/* Descriptor for RSA Private operation - Private Key Form #1 */ +void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, sizeof(*pdb)); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->n_dma); + append_ptr(desc, pdb->d_dma); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | + RSA_PRIV_KEY_FRM_1); +} diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index 0ba9c4059..b3c5016f6 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -8,6 +8,7 @@ #define REGS_H #include +#include #include /* @@ -65,46 +66,56 @@ * */ -#ifdef CONFIG_ARM -/* These are common macros for Power, put here for ARM */ -#define setbits32(_addr, _v) writel((readl(_addr) | (_v)), (_addr)) -#define clrbits32(_addr, _v) writel((readl(_addr) & ~(_v)), (_addr)) +extern bool caam_little_end; -#define out_arch(type, endian, a, v) __raw_write##type(cpu_to_##endian(v), a) -#define in_arch(type, endian, a) endian##_to_cpu(__raw_read##type(a)) +#define caam_to_cpu(len) \ +static inline u##len caam##len ## _to_cpu(u##len val) \ +{ \ + if (caam_little_end) \ + return le##len ## _to_cpu(val); \ + else \ + return be##len ## _to_cpu(val); \ +} -#define out_le32(a, v) out_arch(l, le32, a, v) -#define in_le32(a) in_arch(l, le32, a) +#define cpu_to_caam(len) \ +static inline u##len cpu_to_caam##len(u##len val) \ +{ \ + if (caam_little_end) \ + return cpu_to_le##len(val); \ + else \ + return cpu_to_be##len(val); \ +} -#define out_be32(a, v) out_arch(l, be32, a, v) -#define in_be32(a) in_arch(l, be32, a) +caam_to_cpu(16) +caam_to_cpu(32) +caam_to_cpu(64) +cpu_to_caam(16) +cpu_to_caam(32) +cpu_to_caam(64) -#define clrsetbits(type, addr, clear, set) \ - out_##type((addr), (in_##type(addr) & ~(clear)) | (set)) +static inline void wr_reg32(void __iomem *reg, u32 data) +{ + if (caam_little_end) + iowrite32(data, reg); + else + iowrite32be(data, reg); +} -#define clrsetbits_be32(addr, clear, set) clrsetbits(be32, addr, clear, set) -#define clrsetbits_le32(addr, clear, set) clrsetbits(le32, addr, clear, set) -#endif +static inline u32 rd_reg32(void __iomem *reg) +{ + if (caam_little_end) + return ioread32(reg); -#ifdef __BIG_ENDIAN -#define wr_reg32(reg, data) out_be32(reg, data) -#define rd_reg32(reg) in_be32(reg) -#define clrsetbits_32(addr, clear, set) clrsetbits_be32(addr, clear, set) -#ifdef CONFIG_64BIT -#define wr_reg64(reg, data) out_be64(reg, data) -#define rd_reg64(reg) in_be64(reg) -#endif -#else -#ifdef __LITTLE_ENDIAN -#define wr_reg32(reg, data) __raw_writel(data, reg) -#define rd_reg32(reg) __raw_readl(reg) -#define clrsetbits_32(addr, clear, set) clrsetbits_le32(addr, clear, set) -#ifdef CONFIG_64BIT -#define wr_reg64(reg, data) __raw_writeq(data, reg) -#define rd_reg64(reg) __raw_readq(reg) -#endif -#endif -#endif + return ioread32be(reg); +} + +static inline void clrsetbits_32(void __iomem *reg, u32 clear, u32 set) +{ + if (caam_little_end) + iowrite32((ioread32(reg) & ~clear) | set, reg); + else + iowrite32be((ioread32be(reg) & ~clear) | set, reg); +} /* * The only users of these wr/rd_reg64 functions is the Job Ring (JR). @@ -123,29 +134,67 @@ * base + 0x0000 : least-significant 32 bits * base + 0x0004 : most-significant 32 bits */ +#ifdef CONFIG_64BIT +static inline void wr_reg64(void __iomem *reg, u64 data) +{ + if (caam_little_end) + iowrite64(data, reg); + else + iowrite64be(data, reg); +} -#ifndef CONFIG_64BIT -#if !defined(CONFIG_CRYPTO_DEV_FSL_CAAM_LE) || \ - defined(CONFIG_CRYPTO_DEV_FSL_CAAM_IMX) -#define REG64_MS32(reg) ((u32 __iomem *)(reg)) -#define REG64_LS32(reg) ((u32 __iomem *)(reg) + 1) -#else -#define REG64_MS32(reg) ((u32 __iomem *)(reg) + 1) -#define REG64_LS32(reg) ((u32 __iomem *)(reg)) -#endif - -static inline void wr_reg64(u64 __iomem *reg, u64 data) +static inline u64 rd_reg64(void __iomem *reg) { - wr_reg32(REG64_MS32(reg), data >> 32); - wr_reg32(REG64_LS32(reg), data); + if (caam_little_end) + return ioread64(reg); + else + return ioread64be(reg); } -static inline u64 rd_reg64(u64 __iomem *reg) +#else /* CONFIG_64BIT */ +static inline void wr_reg64(void __iomem *reg, u64 data) { - return ((u64)rd_reg32(REG64_MS32(reg)) << 32 | - (u64)rd_reg32(REG64_LS32(reg))); +#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX + if (caam_little_end) { + wr_reg32((u32 __iomem *)(reg) + 1, data >> 32); + wr_reg32((u32 __iomem *)(reg), data); + } else +#endif + { + wr_reg32((u32 __iomem *)(reg), data >> 32); + wr_reg32((u32 __iomem *)(reg) + 1, data); + } } + +static inline u64 rd_reg64(void __iomem *reg) +{ +#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX + if (caam_little_end) + return ((u64)rd_reg32((u32 __iomem *)(reg) + 1) << 32 | + (u64)rd_reg32((u32 __iomem *)(reg))); + else #endif + return ((u64)rd_reg32((u32 __iomem *)(reg)) << 32 | + (u64)rd_reg32((u32 __iomem *)(reg) + 1)); +} +#endif /* CONFIG_64BIT */ + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT +#ifdef CONFIG_SOC_IMX7D +#define cpu_to_caam_dma(value) \ + (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | \ + (u64)cpu_to_caam32(upper_32_bits(value))) +#define caam_dma_to_cpu(value) \ + (((u64)caam32_to_cpu(lower_32_bits(value)) << 32) | \ + (u64)caam32_to_cpu(upper_32_bits(value))) +#else +#define cpu_to_caam_dma(value) cpu_to_caam64(value) +#define caam_dma_to_cpu(value) caam64_to_cpu(value) +#endif /* CONFIG_SOC_IMX7D */ +#else +#define cpu_to_caam_dma(value) cpu_to_caam32(value) +#define caam_dma_to_cpu(value) caam32_to_cpu(value) +#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */ /* * jr_outentry @@ -249,6 +298,8 @@ struct caam_perfmon { u32 faultliodn; /* FALR - Fault Address LIODN */ u32 faultdetail; /* FADR - Fault Addr Detail */ u32 rsvd2; +#define CSTA_PLEND BIT(10) +#define CSTA_ALT_PLEND BIT(18) u32 status; /* CSTA - CAAM Status */ u64 rsvd3; diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index 12ec6616e..19dc64fed 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -5,18 +5,19 @@ * */ +#include "regs.h" + struct sec4_sg_entry; /* * convert single dma address to h/w link table format */ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, - dma_addr_t dma, u32 len, u32 offset) + dma_addr_t dma, u32 len, u16 offset) { - sec4_sg_ptr->ptr = dma; - sec4_sg_ptr->len = len; - sec4_sg_ptr->buf_pool_id = 0; - sec4_sg_ptr->offset = offset; + sec4_sg_ptr->ptr = cpu_to_caam_dma(dma); + sec4_sg_ptr->len = cpu_to_caam32(len); + sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK); #ifdef DEBUG print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr, @@ -30,7 +31,7 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, */ static inline struct sec4_sg_entry * sg_to_sec4_sg(struct scatterlist *sg, int sg_count, - struct sec4_sg_entry *sec4_sg_ptr, u32 offset) + struct sec4_sg_entry *sec4_sg_ptr, u16 offset) { while (sg_count) { dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), @@ -48,10 +49,10 @@ sg_to_sec4_sg(struct scatterlist *sg, int sg_count, */ static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count, struct sec4_sg_entry *sec4_sg_ptr, - u32 offset) + u16 offset) { sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset); - sec4_sg_ptr->len |= SEC4_SG_LEN_FIN; + sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); } static inline struct sec4_sg_entry *sg_to_sec4_sg_len( diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c index 0d0d4529e..58a4244b4 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes-xts.c +++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c @@ -14,9 +14,8 @@ #include #include #include -#include -#include #include +#include #include #include "ccp-crypto.h" @@ -110,15 +109,12 @@ static int ccp_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, ctx->u.aes.key_len = key_len / 2; sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); - return crypto_ablkcipher_setkey(ctx->u.aes.tfm_ablkcipher, key, - key_len); + return crypto_skcipher_setkey(ctx->u.aes.tfm_skcipher, key, key_len); } static int ccp_aes_xts_crypt(struct ablkcipher_request *req, unsigned int encrypt) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); unsigned int unit; @@ -146,14 +142,19 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req, if ((unit_size == CCP_XTS_AES_UNIT_SIZE__LAST) || (ctx->u.aes.key_len != AES_KEYSIZE_128)) { + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->u.aes.tfm_skcipher); + /* Use the fallback to process the request for any * unsupported unit sizes or key sizes */ - ablkcipher_request_set_tfm(req, ctx->u.aes.tfm_ablkcipher); - ret = (encrypt) ? crypto_ablkcipher_encrypt(req) : - crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); - + skcipher_request_set_tfm(subreq, ctx->u.aes.tfm_skcipher); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + ret = encrypt ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); return ret; } @@ -192,23 +193,21 @@ static int ccp_aes_xts_decrypt(struct ablkcipher_request *req) static int ccp_aes_xts_cra_init(struct crypto_tfm *tfm) { struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_ablkcipher *fallback_tfm; + struct crypto_skcipher *fallback_tfm; ctx->complete = ccp_aes_xts_complete; ctx->u.aes.key_len = 0; - fallback_tfm = crypto_alloc_ablkcipher(crypto_tfm_alg_name(tfm), 0, - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK); + fallback_tfm = crypto_alloc_skcipher("xts(aes)", 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback_tfm)) { - pr_warn("could not load fallback driver %s\n", - crypto_tfm_alg_name(tfm)); + pr_warn("could not load fallback driver xts(aes)\n"); return PTR_ERR(fallback_tfm); } - ctx->u.aes.tfm_ablkcipher = fallback_tfm; + ctx->u.aes.tfm_skcipher = fallback_tfm; - tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx) + - fallback_tfm->base.crt_ablkcipher.reqsize; + tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx); return 0; } @@ -217,9 +216,7 @@ static void ccp_aes_xts_cra_exit(struct crypto_tfm *tfm) { struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->u.aes.tfm_ablkcipher) - crypto_free_ablkcipher(ctx->u.aes.tfm_ablkcipher); - ctx->u.aes.tfm_ablkcipher = NULL; + crypto_free_skcipher(ctx->u.aes.tfm_skcipher); } static int ccp_register_aes_xts_alg(struct list_head *head, diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index a326ec20b..8335b32e8 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -69,7 +68,7 @@ static inline struct ccp_crypto_ahash_alg * /***** AES related defines *****/ struct ccp_aes_ctx { /* Fallback cipher for XTS with unsupported unit sizes */ - struct crypto_ablkcipher *tfm_ablkcipher; + struct crypto_skcipher *tfm_skcipher; /* Cipher used to generate CMAC K1/K2 keys */ struct crypto_cipher *tfm_cipher; diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index e8ef9fd24..d64af8625 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -31,22 +31,42 @@ #include "cesa.h" +/* Limit of the crypto queue before reaching the backlog */ +#define CESA_CRYPTO_DEFAULT_MAX_QLEN 128 + static int allhwsupport = !IS_ENABLED(CONFIG_CRYPTO_DEV_MV_CESA); module_param_named(allhwsupport, allhwsupport, int, 0444); MODULE_PARM_DESC(allhwsupport, "Enable support for all hardware (even it if overlaps with the mv_cesa driver)"); struct mv_cesa_dev *cesa_dev; -static void mv_cesa_dequeue_req_unlocked(struct mv_cesa_engine *engine) +struct crypto_async_request * +mv_cesa_dequeue_req_locked(struct mv_cesa_engine *engine, + struct crypto_async_request **backlog) { - struct crypto_async_request *req, *backlog; + struct crypto_async_request *req; + + *backlog = crypto_get_backlog(&engine->queue); + req = crypto_dequeue_request(&engine->queue); + + if (!req) + return NULL; + + return req; +} + +static void mv_cesa_rearm_engine(struct mv_cesa_engine *engine) +{ + struct crypto_async_request *req = NULL, *backlog = NULL; struct mv_cesa_ctx *ctx; - spin_lock_bh(&cesa_dev->lock); - backlog = crypto_get_backlog(&cesa_dev->queue); - req = crypto_dequeue_request(&cesa_dev->queue); - engine->req = req; - spin_unlock_bh(&cesa_dev->lock); + + spin_lock_bh(&engine->lock); + if (!engine->req) { + req = mv_cesa_dequeue_req_locked(engine, &backlog); + engine->req = req; + } + spin_unlock_bh(&engine->lock); if (!req) return; @@ -55,8 +75,47 @@ static void mv_cesa_dequeue_req_unlocked(struct mv_cesa_engine *engine) backlog->complete(backlog, -EINPROGRESS); ctx = crypto_tfm_ctx(req->tfm); - ctx->ops->prepare(req, engine); ctx->ops->step(req); + + return; +} + +static int mv_cesa_std_process(struct mv_cesa_engine *engine, u32 status) +{ + struct crypto_async_request *req; + struct mv_cesa_ctx *ctx; + int res; + + req = engine->req; + ctx = crypto_tfm_ctx(req->tfm); + res = ctx->ops->process(req, status); + + if (res == 0) { + ctx->ops->complete(req); + mv_cesa_engine_enqueue_complete_request(engine, req); + } else if (res == -EINPROGRESS) { + ctx->ops->step(req); + } + + return res; +} + +static int mv_cesa_int_process(struct mv_cesa_engine *engine, u32 status) +{ + if (engine->chain.first && engine->chain.last) + return mv_cesa_tdma_process(engine, status); + + return mv_cesa_std_process(engine, status); +} + +static inline void +mv_cesa_complete_req(struct mv_cesa_ctx *ctx, struct crypto_async_request *req, + int res) +{ + ctx->ops->cleanup(req); + local_bh_disable(); + req->complete(req, res); + local_bh_enable(); } static irqreturn_t mv_cesa_int(int irq, void *priv) @@ -83,49 +142,55 @@ static irqreturn_t mv_cesa_int(int irq, void *priv) writel(~status, engine->regs + CESA_SA_FPGA_INT_STATUS); writel(~status, engine->regs + CESA_SA_INT_STATUS); + /* Process fetched requests */ + res = mv_cesa_int_process(engine, status & mask); ret = IRQ_HANDLED; + spin_lock_bh(&engine->lock); req = engine->req; + if (res != -EINPROGRESS) + engine->req = NULL; spin_unlock_bh(&engine->lock); - if (req) { - ctx = crypto_tfm_ctx(req->tfm); - res = ctx->ops->process(req, status & mask); - if (res != -EINPROGRESS) { - spin_lock_bh(&engine->lock); - engine->req = NULL; - mv_cesa_dequeue_req_unlocked(engine); - spin_unlock_bh(&engine->lock); - ctx->ops->cleanup(req); - local_bh_disable(); - req->complete(req, res); - local_bh_enable(); - } else { - ctx->ops->step(req); - } + + ctx = crypto_tfm_ctx(req->tfm); + + if (res && res != -EINPROGRESS) + mv_cesa_complete_req(ctx, req, res); + + /* Launch the next pending request */ + mv_cesa_rearm_engine(engine); + + /* Iterate over the complete queue */ + while (true) { + req = mv_cesa_engine_dequeue_complete_request(engine); + if (!req) + break; + + mv_cesa_complete_req(ctx, req, 0); } } return ret; } -int mv_cesa_queue_req(struct crypto_async_request *req) +int mv_cesa_queue_req(struct crypto_async_request *req, + struct mv_cesa_req *creq) { int ret; - int i; + struct mv_cesa_engine *engine = creq->engine; - spin_lock_bh(&cesa_dev->lock); - ret = crypto_enqueue_request(&cesa_dev->queue, req); - spin_unlock_bh(&cesa_dev->lock); + spin_lock_bh(&engine->lock); + ret = crypto_enqueue_request(&engine->queue, req); + if ((mv_cesa_req_get_type(creq) == CESA_DMA_REQ) && + (ret == -EINPROGRESS || + (ret == -EBUSY && req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) + mv_cesa_tdma_chain(engine, creq); + spin_unlock_bh(&engine->lock); if (ret != -EINPROGRESS) return ret; - for (i = 0; i < cesa_dev->caps->nengines; i++) { - spin_lock_bh(&cesa_dev->engines[i].lock); - if (!cesa_dev->engines[i].req) - mv_cesa_dequeue_req_unlocked(&cesa_dev->engines[i]); - spin_unlock_bh(&cesa_dev->engines[i].lock); - } + mv_cesa_rearm_engine(engine); return -EINPROGRESS; } @@ -309,6 +374,10 @@ static int mv_cesa_dev_dma_init(struct mv_cesa_dev *cesa) if (!dma->padding_pool) return -ENOMEM; + dma->iv_pool = dmam_pool_create("cesa_iv", dev, 16, 1, 0); + if (!dma->iv_pool) + return -ENOMEM; + cesa->dma = dma; return 0; @@ -416,7 +485,7 @@ static int mv_cesa_probe(struct platform_device *pdev) return -ENOMEM; spin_lock_init(&cesa->lock); - crypto_init_queue(&cesa->queue, 50); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); cesa->regs = devm_ioremap_resource(dev, res); if (IS_ERR(cesa->regs)) @@ -489,6 +558,10 @@ static int mv_cesa_probe(struct platform_device *pdev) engine); if (ret) goto err_cleanup; + + crypto_init_queue(&engine->queue, CESA_CRYPTO_DEFAULT_MAX_QLEN); + atomic_set(&engine->load, 0); + INIT_LIST_HEAD(&engine->complete_queue); } cesa_dev = cesa; diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index 74071e45a..e423d33de 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -271,10 +271,13 @@ struct mv_cesa_op_ctx { /* TDMA descriptor flags */ #define CESA_TDMA_DST_IN_SRAM BIT(31) #define CESA_TDMA_SRC_IN_SRAM BIT(30) -#define CESA_TDMA_TYPE_MSK GENMASK(29, 0) +#define CESA_TDMA_END_OF_REQ BIT(29) +#define CESA_TDMA_BREAK_CHAIN BIT(28) +#define CESA_TDMA_TYPE_MSK GENMASK(27, 0) #define CESA_TDMA_DUMMY 0 #define CESA_TDMA_DATA 1 #define CESA_TDMA_OP 2 +#define CESA_TDMA_IV 3 /** * struct mv_cesa_tdma_desc - TDMA descriptor @@ -390,6 +393,7 @@ struct mv_cesa_dev_dma { struct dma_pool *op_pool; struct dma_pool *cache_pool; struct dma_pool *padding_pool; + struct dma_pool *iv_pool; }; /** @@ -398,7 +402,6 @@ struct mv_cesa_dev_dma { * @regs: device registers * @sram_size: usable SRAM size * @lock: device lock - * @queue: crypto request queue * @engines: array of engines * @dma: dma pools * @@ -410,7 +413,6 @@ struct mv_cesa_dev { struct device *dev; unsigned int sram_size; spinlock_t lock; - struct crypto_queue queue; struct mv_cesa_engine *engines; struct mv_cesa_dev_dma *dma; }; @@ -429,6 +431,11 @@ struct mv_cesa_dev { * @int_mask: interrupt mask cache * @pool: memory pool pointing to the memory region reserved in * SRAM + * @queue: fifo of the pending crypto requests + * @load: engine load counter, useful for load balancing + * @chain: list of the current tdma descriptors being processed + * by this engine. + * @complete_queue: fifo of the processed requests by the engine * * Structure storing CESA engine information. */ @@ -444,23 +451,27 @@ struct mv_cesa_engine { size_t max_req_len; u32 int_mask; struct gen_pool *pool; + struct crypto_queue queue; + atomic_t load; + struct mv_cesa_tdma_chain chain; + struct list_head complete_queue; }; /** * struct mv_cesa_req_ops - CESA request operations - * @prepare: prepare a request to be executed on the specified engine * @process: process a request chunk result (should return 0 if the * operation, -EINPROGRESS if it needs more steps or an error * code) * @step: launch the crypto operation on the next chunk * @cleanup: cleanup the crypto request (release associated data) + * @complete: complete the request, i.e copy result or context from sram when + * needed. */ struct mv_cesa_req_ops { - void (*prepare)(struct crypto_async_request *req, - struct mv_cesa_engine *engine); int (*process)(struct crypto_async_request *req, u32 status); void (*step)(struct crypto_async_request *req); void (*cleanup)(struct crypto_async_request *req); + void (*complete)(struct crypto_async_request *req); }; /** @@ -507,21 +518,11 @@ enum mv_cesa_req_type { /** * struct mv_cesa_req - CESA request - * @type: request type * @engine: engine associated with this request + * @chain: list of tdma descriptors associated with this request */ struct mv_cesa_req { - enum mv_cesa_req_type type; struct mv_cesa_engine *engine; -}; - -/** - * struct mv_cesa_tdma_req - CESA TDMA request - * @base: base information - * @chain: TDMA chain - */ -struct mv_cesa_tdma_req { - struct mv_cesa_req base; struct mv_cesa_tdma_chain chain; }; @@ -538,13 +539,11 @@ struct mv_cesa_sg_std_iter { /** * struct mv_cesa_ablkcipher_std_req - cipher standard request - * @base: base information * @op: operation context * @offset: current operation offset * @size: size of the crypto operation */ struct mv_cesa_ablkcipher_std_req { - struct mv_cesa_req base; struct mv_cesa_op_ctx op; unsigned int offset; unsigned int size; @@ -558,34 +557,27 @@ struct mv_cesa_ablkcipher_std_req { * @dst_nents: number of entries in the dest sg list */ struct mv_cesa_ablkcipher_req { - union { - struct mv_cesa_req base; - struct mv_cesa_tdma_req dma; - struct mv_cesa_ablkcipher_std_req std; - } req; + struct mv_cesa_req base; + struct mv_cesa_ablkcipher_std_req std; int src_nents; int dst_nents; }; /** * struct mv_cesa_ahash_std_req - standard hash request - * @base: base information * @offset: current operation offset */ struct mv_cesa_ahash_std_req { - struct mv_cesa_req base; unsigned int offset; }; /** * struct mv_cesa_ahash_dma_req - DMA hash request - * @base: base information * @padding: padding buffer * @padding_dma: DMA address of the padding buffer * @cache_dma: DMA address of the cache buffer */ struct mv_cesa_ahash_dma_req { - struct mv_cesa_tdma_req base; u8 *padding; dma_addr_t padding_dma; u8 *cache; @@ -604,8 +596,8 @@ struct mv_cesa_ahash_dma_req { * @state: hash state */ struct mv_cesa_ahash_req { + struct mv_cesa_req base; union { - struct mv_cesa_req base; struct mv_cesa_ahash_dma_req dma; struct mv_cesa_ahash_std_req std; } req; @@ -623,6 +615,35 @@ struct mv_cesa_ahash_req { extern struct mv_cesa_dev *cesa_dev; + +static inline void +mv_cesa_engine_enqueue_complete_request(struct mv_cesa_engine *engine, + struct crypto_async_request *req) +{ + list_add_tail(&req->list, &engine->complete_queue); +} + +static inline struct crypto_async_request * +mv_cesa_engine_dequeue_complete_request(struct mv_cesa_engine *engine) +{ + struct crypto_async_request *req; + + req = list_first_entry_or_null(&engine->complete_queue, + struct crypto_async_request, + list); + if (req) + list_del(&req->list); + + return req; +} + + +static inline enum mv_cesa_req_type +mv_cesa_req_get_type(struct mv_cesa_req *req) +{ + return req->chain.first ? CESA_DMA_REQ : CESA_STD_REQ; +} + static inline void mv_cesa_update_op_cfg(struct mv_cesa_op_ctx *op, u32 cfg, u32 mask) { @@ -695,7 +716,32 @@ static inline bool mv_cesa_mac_op_is_first_frag(const struct mv_cesa_op_ctx *op) CESA_SA_DESC_CFG_FIRST_FRAG; } -int mv_cesa_queue_req(struct crypto_async_request *req); +int mv_cesa_queue_req(struct crypto_async_request *req, + struct mv_cesa_req *creq); + +struct crypto_async_request * +mv_cesa_dequeue_req_locked(struct mv_cesa_engine *engine, + struct crypto_async_request **backlog); + +static inline struct mv_cesa_engine *mv_cesa_select_engine(int weight) +{ + int i; + u32 min_load = U32_MAX; + struct mv_cesa_engine *selected = NULL; + + for (i = 0; i < cesa_dev->caps->nengines; i++) { + struct mv_cesa_engine *engine = cesa_dev->engines + i; + u32 load = atomic_read(&engine->load); + if (load < min_load) { + min_load = load; + selected = engine; + } + } + + atomic_add(weight, &selected->load); + + return selected; +} /* * Helper function that indicates whether a crypto request needs to be @@ -765,9 +811,9 @@ static inline bool mv_cesa_req_dma_iter_next_op(struct mv_cesa_dma_iter *iter) return iter->op_len; } -void mv_cesa_dma_step(struct mv_cesa_tdma_req *dreq); +void mv_cesa_dma_step(struct mv_cesa_req *dreq); -static inline int mv_cesa_dma_process(struct mv_cesa_tdma_req *dreq, +static inline int mv_cesa_dma_process(struct mv_cesa_req *dreq, u32 status) { if (!(status & CESA_SA_INT_ACC0_IDMA_DONE)) @@ -779,10 +825,13 @@ static inline int mv_cesa_dma_process(struct mv_cesa_tdma_req *dreq, return 0; } -void mv_cesa_dma_prepare(struct mv_cesa_tdma_req *dreq, +void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, struct mv_cesa_engine *engine); +void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq); +void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, + struct mv_cesa_req *dreq); +int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status); -void mv_cesa_dma_cleanup(struct mv_cesa_tdma_req *dreq); static inline void mv_cesa_tdma_desc_iter_init(struct mv_cesa_tdma_chain *chain) @@ -790,6 +839,9 @@ mv_cesa_tdma_desc_iter_init(struct mv_cesa_tdma_chain *chain) memset(chain, 0, sizeof(*chain)); } +int mv_cesa_dma_add_iv_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, + u32 size, u32 flags, gfp_t gfp_flags); + struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, const struct mv_cesa_op_ctx *op_templ, bool skip_ctx, diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c index dcf1fceb9..d19dc9614 100644 --- a/drivers/crypto/marvell/cipher.c +++ b/drivers/crypto/marvell/cipher.c @@ -70,25 +70,28 @@ mv_cesa_ablkcipher_dma_cleanup(struct ablkcipher_request *req) dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_BIDIRECTIONAL); } - mv_cesa_dma_cleanup(&creq->req.dma); + mv_cesa_dma_cleanup(&creq->base); } static inline void mv_cesa_ablkcipher_cleanup(struct ablkcipher_request *req) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - if (creq->req.base.type == CESA_DMA_REQ) + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ablkcipher_dma_cleanup(req); } static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; + struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_engine *engine = creq->base.engine; size_t len = min_t(size_t, req->nbytes - sreq->offset, CESA_SA_SRAM_PAYLOAD_SIZE); + mv_cesa_adjust_op(engine, &sreq->op); + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); + len = sg_pcopy_to_buffer(req->src, creq->src_nents, engine->sram + CESA_SA_DATA_SRAM_OFFSET, len, sreq->offset); @@ -106,6 +109,8 @@ static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); + BUG_ON(readl(engine->regs + CESA_SA_CMD) & + CESA_SA_CMD_EN_CESA_SA_ACCL0); writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } @@ -113,8 +118,8 @@ static int mv_cesa_ablkcipher_std_process(struct ablkcipher_request *req, u32 status) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; + struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_engine *engine = creq->base.engine; size_t len; len = sg_pcopy_from_buffer(req->dst, creq->dst_nents, @@ -133,23 +138,12 @@ static int mv_cesa_ablkcipher_process(struct crypto_async_request *req, { struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; - int ret; - - if (creq->req.base.type == CESA_DMA_REQ) - ret = mv_cesa_dma_process(&creq->req.dma, status); - else - ret = mv_cesa_ablkcipher_std_process(ablkreq, status); - - if (ret) - return ret; + struct mv_cesa_req *basereq = &creq->base; - memcpy_fromio(ablkreq->info, - engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, - crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq))); + if (mv_cesa_req_get_type(basereq) == CESA_STD_REQ) + return mv_cesa_ablkcipher_std_process(ablkreq, status); - return 0; + return mv_cesa_dma_process(basereq, status); } static void mv_cesa_ablkcipher_step(struct crypto_async_request *req) @@ -157,8 +151,8 @@ static void mv_cesa_ablkcipher_step(struct crypto_async_request *req) struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); - if (creq->req.base.type == CESA_DMA_REQ) - mv_cesa_dma_step(&creq->req.dma); + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_dma_step(&creq->base); else mv_cesa_ablkcipher_std_step(ablkreq); } @@ -167,22 +161,19 @@ static inline void mv_cesa_ablkcipher_dma_prepare(struct ablkcipher_request *req) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_tdma_req *dreq = &creq->req.dma; + struct mv_cesa_req *basereq = &creq->base; - mv_cesa_dma_prepare(dreq, dreq->base.engine); + mv_cesa_dma_prepare(basereq, basereq->engine); } static inline void mv_cesa_ablkcipher_std_prepare(struct ablkcipher_request *req) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; + struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; sreq->size = 0; sreq->offset = 0; - mv_cesa_adjust_op(engine, &sreq->op); - memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); } static inline void mv_cesa_ablkcipher_prepare(struct crypto_async_request *req, @@ -190,9 +181,9 @@ static inline void mv_cesa_ablkcipher_prepare(struct crypto_async_request *req, { struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); - creq->req.base.engine = engine; + creq->base.engine = engine; - if (creq->req.base.type == CESA_DMA_REQ) + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ablkcipher_dma_prepare(ablkreq); else mv_cesa_ablkcipher_std_prepare(ablkreq); @@ -206,11 +197,34 @@ mv_cesa_ablkcipher_req_cleanup(struct crypto_async_request *req) mv_cesa_ablkcipher_cleanup(ablkreq); } +static void +mv_cesa_ablkcipher_complete(struct crypto_async_request *req) +{ + struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct mv_cesa_engine *engine = creq->base.engine; + unsigned int ivsize; + + atomic_sub(ablkreq->nbytes, &engine->load); + ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq)); + + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) { + struct mv_cesa_req *basereq; + + basereq = &creq->base; + memcpy(ablkreq->info, basereq->chain.last->data, ivsize); + } else { + memcpy_fromio(ablkreq->info, + engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, + ivsize); + } +} + static const struct mv_cesa_req_ops mv_cesa_ablkcipher_req_ops = { .step = mv_cesa_ablkcipher_step, .process = mv_cesa_ablkcipher_process, - .prepare = mv_cesa_ablkcipher_prepare, .cleanup = mv_cesa_ablkcipher_req_cleanup, + .complete = mv_cesa_ablkcipher_complete, }; static int mv_cesa_ablkcipher_cra_init(struct crypto_tfm *tfm) @@ -295,15 +309,14 @@ static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; - struct mv_cesa_tdma_req *dreq = &creq->req.dma; + struct mv_cesa_req *basereq = &creq->base; struct mv_cesa_ablkcipher_dma_iter iter; - struct mv_cesa_tdma_chain chain; bool skip_ctx = false; int ret; + unsigned int ivsize; - dreq->base.type = CESA_DMA_REQ; - dreq->chain.first = NULL; - dreq->chain.last = NULL; + basereq->chain.first = NULL; + basereq->chain.last = NULL; if (req->src != req->dst) { ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, @@ -324,13 +337,13 @@ static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, return -ENOMEM; } - mv_cesa_tdma_desc_iter_init(&chain); + mv_cesa_tdma_desc_iter_init(&basereq->chain); mv_cesa_ablkcipher_req_iter_init(&iter, req); do { struct mv_cesa_op_ctx *op; - op = mv_cesa_dma_add_op(&chain, op_templ, skip_ctx, flags); + op = mv_cesa_dma_add_op(&basereq->chain, op_templ, skip_ctx, flags); if (IS_ERR(op)) { ret = PTR_ERR(op); goto err_free_tdma; @@ -340,30 +353,38 @@ static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, mv_cesa_set_crypt_op_len(op, iter.base.op_len); /* Add input transfers */ - ret = mv_cesa_dma_add_op_transfers(&chain, &iter.base, + ret = mv_cesa_dma_add_op_transfers(&basereq->chain, &iter.base, &iter.src, flags); if (ret) goto err_free_tdma; /* Add dummy desc to launch the crypto operation */ - ret = mv_cesa_dma_add_dummy_launch(&chain, flags); + ret = mv_cesa_dma_add_dummy_launch(&basereq->chain, flags); if (ret) goto err_free_tdma; /* Add output transfers */ - ret = mv_cesa_dma_add_op_transfers(&chain, &iter.base, + ret = mv_cesa_dma_add_op_transfers(&basereq->chain, &iter.base, &iter.dst, flags); if (ret) goto err_free_tdma; } while (mv_cesa_ablkcipher_req_iter_next_op(&iter)); - dreq->chain = chain; + /* Add output data for IV */ + ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req)); + ret = mv_cesa_dma_add_iv_op(&basereq->chain, CESA_SA_CRYPT_IV_SRAM_OFFSET, + ivsize, CESA_TDMA_SRC_IN_SRAM, flags); + + if (ret) + goto err_free_tdma; + + basereq->chain.last->flags |= CESA_TDMA_END_OF_REQ; return 0; err_free_tdma: - mv_cesa_dma_cleanup(dreq); + mv_cesa_dma_cleanup(basereq); if (req->dst != req->src) dma_unmap_sg(cesa_dev->dev, req->dst, creq->dst_nents, DMA_FROM_DEVICE); @@ -380,11 +401,13 @@ mv_cesa_ablkcipher_std_req_init(struct ablkcipher_request *req, const struct mv_cesa_op_ctx *op_templ) { struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; + struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_req *basereq = &creq->base; - sreq->base.type = CESA_STD_REQ; sreq->op = *op_templ; sreq->skip_ctx = false; + basereq->chain.first = NULL; + basereq->chain.last = NULL; return 0; } @@ -414,7 +437,6 @@ static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req, mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_OP_CRYPT_ONLY, CESA_SA_DESC_CFG_OP_MSK); - /* TODO: add a threshold for DMA usage */ if (cesa_dev->caps->has_tdma) ret = mv_cesa_ablkcipher_dma_req_init(req, tmpl); else @@ -423,28 +445,41 @@ static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req, return ret; } -static int mv_cesa_des_op(struct ablkcipher_request *req, - struct mv_cesa_op_ctx *tmpl) +static int mv_cesa_ablkcipher_queue_req(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) { - struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(req->base.tfm); int ret; - - mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_DES, - CESA_SA_DESC_CFG_CRYPTM_MSK); - - memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES_KEY_SIZE); + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_engine *engine; ret = mv_cesa_ablkcipher_req_init(req, tmpl); if (ret) return ret; - ret = mv_cesa_queue_req(&req->base); + engine = mv_cesa_select_engine(req->nbytes); + mv_cesa_ablkcipher_prepare(&req->base, engine); + + ret = mv_cesa_queue_req(&req->base, &creq->base); + if (mv_cesa_req_needs_cleanup(&req->base, ret)) mv_cesa_ablkcipher_cleanup(req); return ret; } +static int mv_cesa_des_op(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_DES, + CESA_SA_DESC_CFG_CRYPTM_MSK); + + memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES_KEY_SIZE); + + return mv_cesa_ablkcipher_queue_req(req, tmpl); +} + static int mv_cesa_ecb_des_encrypt(struct ablkcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -547,22 +582,13 @@ static int mv_cesa_des3_op(struct ablkcipher_request *req, struct mv_cesa_op_ctx *tmpl) { struct mv_cesa_des3_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - int ret; mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_3DES, CESA_SA_DESC_CFG_CRYPTM_MSK); memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES3_EDE_KEY_SIZE); - ret = mv_cesa_ablkcipher_req_init(req, tmpl); - if (ret) - return ret; - - ret = mv_cesa_queue_req(&req->base); - if (mv_cesa_req_needs_cleanup(&req->base, ret)) - mv_cesa_ablkcipher_cleanup(req); - - return ret; + return mv_cesa_ablkcipher_queue_req(req, tmpl); } static int mv_cesa_ecb_des3_ede_encrypt(struct ablkcipher_request *req) @@ -673,7 +699,7 @@ static int mv_cesa_aes_op(struct ablkcipher_request *req, struct mv_cesa_op_ctx *tmpl) { struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - int ret, i; + int i; u32 *key; u32 cfg; @@ -696,15 +722,7 @@ static int mv_cesa_aes_op(struct ablkcipher_request *req, CESA_SA_DESC_CFG_CRYPTM_MSK | CESA_SA_DESC_CFG_AES_LEN_MSK); - ret = mv_cesa_ablkcipher_req_init(req, tmpl); - if (ret) - return ret; - - ret = mv_cesa_queue_req(&req->base); - if (mv_cesa_req_needs_cleanup(&req->base, ret)) - mv_cesa_ablkcipher_cleanup(req); - - return ret; + return mv_cesa_ablkcipher_queue_req(req, tmpl); } static int mv_cesa_ecb_aes_encrypt(struct ablkcipher_request *req) diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 7a5058da9..82e0f4e6e 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -103,14 +103,14 @@ static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req) dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); mv_cesa_ahash_dma_free_cache(&creq->req.dma); - mv_cesa_dma_cleanup(&creq->req.dma.base); + mv_cesa_dma_cleanup(&creq->base); } static inline void mv_cesa_ahash_cleanup(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - if (creq->req.base.type == CESA_DMA_REQ) + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ahash_dma_cleanup(req); } @@ -118,7 +118,7 @@ static void mv_cesa_ahash_last_cleanup(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - if (creq->req.base.type == CESA_DMA_REQ) + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ahash_dma_last_cleanup(req); } @@ -157,11 +157,23 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_ahash_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; + struct mv_cesa_engine *engine = creq->base.engine; struct mv_cesa_op_ctx *op; unsigned int new_cache_ptr = 0; u32 frag_mode; size_t len; + unsigned int digsize; + int i; + + mv_cesa_adjust_op(engine, &creq->op_tmpl); + memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); + + digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + for (i = 0; i < digsize / 4; i++) + writel_relaxed(creq->state[i], engine->regs + CESA_IVDIG(i)); + + mv_cesa_adjust_op(engine, &creq->op_tmpl); + memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); if (creq->cache_ptr) memcpy_toio(engine->sram + CESA_SA_DATA_SRAM_OFFSET, @@ -237,6 +249,8 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); + BUG_ON(readl(engine->regs + CESA_SA_CMD) & + CESA_SA_CMD_EN_CESA_SA_ACCL0); writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } @@ -254,20 +268,17 @@ static int mv_cesa_ahash_std_process(struct ahash_request *req, u32 status) static inline void mv_cesa_ahash_dma_prepare(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - struct mv_cesa_tdma_req *dreq = &creq->req.dma.base; + struct mv_cesa_req *basereq = &creq->base; - mv_cesa_dma_prepare(dreq, dreq->base.engine); + mv_cesa_dma_prepare(basereq, basereq->engine); } static void mv_cesa_ahash_std_prepare(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_ahash_std_req *sreq = &creq->req.std; - struct mv_cesa_engine *engine = sreq->base.engine; sreq->offset = 0; - mv_cesa_adjust_op(engine, &creq->op_tmpl); - memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); } static void mv_cesa_ahash_step(struct crypto_async_request *req) @@ -275,8 +286,8 @@ static void mv_cesa_ahash_step(struct crypto_async_request *req) struct ahash_request *ahashreq = ahash_request_cast(req); struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); - if (creq->req.base.type == CESA_DMA_REQ) - mv_cesa_dma_step(&creq->req.dma.base); + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + mv_cesa_dma_step(&creq->base); else mv_cesa_ahash_std_step(ahashreq); } @@ -285,28 +296,25 @@ static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status) { struct ahash_request *ahashreq = ahash_request_cast(req); struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); - struct mv_cesa_engine *engine = creq->req.base.engine; - unsigned int digsize; - int ret, i; - if (creq->req.base.type == CESA_DMA_REQ) - ret = mv_cesa_dma_process(&creq->req.dma.base, status); - else - ret = mv_cesa_ahash_std_process(ahashreq, status); + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) + return mv_cesa_dma_process(&creq->base, status); - if (ret == -EINPROGRESS) - return ret; + return mv_cesa_ahash_std_process(ahashreq, status); +} + +static void mv_cesa_ahash_complete(struct crypto_async_request *req) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + struct mv_cesa_engine *engine = creq->base.engine; + unsigned int digsize; + int i; digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); for (i = 0; i < digsize / 4; i++) creq->state[i] = readl_relaxed(engine->regs + CESA_IVDIG(i)); - if (creq->cache_ptr) - sg_pcopy_to_buffer(ahashreq->src, creq->src_nents, - creq->cache, - creq->cache_ptr, - ahashreq->nbytes - creq->cache_ptr); - if (creq->last_req) { /* * Hardware's MD5 digest is in little endian format, but @@ -325,7 +333,7 @@ static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status) } } - return ret; + atomic_sub(ahashreq->nbytes, &engine->load); } static void mv_cesa_ahash_prepare(struct crypto_async_request *req, @@ -333,19 +341,13 @@ static void mv_cesa_ahash_prepare(struct crypto_async_request *req, { struct ahash_request *ahashreq = ahash_request_cast(req); struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); - unsigned int digsize; - int i; - creq->req.base.engine = engine; + creq->base.engine = engine; - if (creq->req.base.type == CESA_DMA_REQ) + if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_ahash_dma_prepare(ahashreq); else mv_cesa_ahash_std_prepare(ahashreq); - - digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); - for (i = 0; i < digsize / 4; i++) - writel_relaxed(creq->state[i], engine->regs + CESA_IVDIG(i)); } static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req) @@ -357,13 +359,19 @@ static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req) mv_cesa_ahash_last_cleanup(ahashreq); mv_cesa_ahash_cleanup(ahashreq); + + if (creq->cache_ptr) + sg_pcopy_to_buffer(ahashreq->src, creq->src_nents, + creq->cache, + creq->cache_ptr, + ahashreq->nbytes - creq->cache_ptr); } static const struct mv_cesa_req_ops mv_cesa_ahash_req_ops = { .step = mv_cesa_ahash_step, .process = mv_cesa_ahash_process, - .prepare = mv_cesa_ahash_prepare, .cleanup = mv_cesa_ahash_req_cleanup, + .complete = mv_cesa_ahash_complete, }; static int mv_cesa_ahash_init(struct ahash_request *req, @@ -553,15 +561,14 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; - struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; - struct mv_cesa_tdma_req *dreq = &ahashdreq->base; + struct mv_cesa_req *basereq = &creq->base; struct mv_cesa_ahash_dma_iter iter; struct mv_cesa_op_ctx *op = NULL; unsigned int frag_len; int ret; - dreq->chain.first = NULL; - dreq->chain.last = NULL; + basereq->chain.first = NULL; + basereq->chain.last = NULL; if (creq->src_nents) { ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, @@ -572,14 +579,14 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) } } - mv_cesa_tdma_desc_iter_init(&dreq->chain); + mv_cesa_tdma_desc_iter_init(&basereq->chain); mv_cesa_ahash_req_iter_init(&iter, req); /* * Add the cache (left-over data from a previous block) first. * This will never overflow the SRAM size. */ - ret = mv_cesa_ahash_dma_add_cache(&dreq->chain, &iter, creq, flags); + ret = mv_cesa_ahash_dma_add_cache(&basereq->chain, &iter, creq, flags); if (ret) goto err_free_tdma; @@ -590,7 +597,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) * data. We intentionally do not add the final op block. */ while (true) { - ret = mv_cesa_dma_add_op_transfers(&dreq->chain, + ret = mv_cesa_dma_add_op_transfers(&basereq->chain, &iter.base, &iter.src, flags); if (ret) @@ -601,7 +608,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (!mv_cesa_ahash_req_iter_next_op(&iter)) break; - op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl, + op = mv_cesa_dma_add_frag(&basereq->chain, &creq->op_tmpl, frag_len, flags); if (IS_ERR(op)) { ret = PTR_ERR(op); @@ -619,10 +626,10 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) * operation, which depends whether this is the final request. */ if (creq->last_req) - op = mv_cesa_ahash_dma_last_req(&dreq->chain, &iter, creq, + op = mv_cesa_ahash_dma_last_req(&basereq->chain, &iter, creq, frag_len, flags); else if (frag_len) - op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl, + op = mv_cesa_dma_add_frag(&basereq->chain, &creq->op_tmpl, frag_len, flags); if (IS_ERR(op)) { @@ -632,7 +639,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (op) { /* Add dummy desc to wait for crypto operation end */ - ret = mv_cesa_dma_add_dummy_end(&dreq->chain, flags); + ret = mv_cesa_dma_add_dummy_end(&basereq->chain, flags); if (ret) goto err_free_tdma; } @@ -643,10 +650,13 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) else creq->cache_ptr = 0; + basereq->chain.last->flags |= (CESA_TDMA_END_OF_REQ | + CESA_TDMA_BREAK_CHAIN); + return 0; err_free_tdma: - mv_cesa_dma_cleanup(dreq); + mv_cesa_dma_cleanup(basereq); dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); err: @@ -660,11 +670,6 @@ static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached) struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); int ret; - if (cesa_dev->caps->has_tdma) - creq->req.base.type = CESA_DMA_REQ; - else - creq->req.base.type = CESA_STD_REQ; - creq->src_nents = sg_nents_for_len(req->src, req->nbytes); if (creq->src_nents < 0) { dev_err(cesa_dev->dev, "Invalid number of src SG"); @@ -678,19 +683,19 @@ static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached) if (*cached) return 0; - if (creq->req.base.type == CESA_DMA_REQ) + if (cesa_dev->caps->has_tdma) ret = mv_cesa_ahash_dma_req_init(req); return ret; } -static int mv_cesa_ahash_update(struct ahash_request *req) +static int mv_cesa_ahash_queue_req(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_engine *engine; bool cached = false; int ret; - creq->len += req->nbytes; ret = mv_cesa_ahash_req_init(req, &cached); if (ret) return ret; @@ -698,61 +703,48 @@ static int mv_cesa_ahash_update(struct ahash_request *req) if (cached) return 0; - ret = mv_cesa_queue_req(&req->base); + engine = mv_cesa_select_engine(req->nbytes); + mv_cesa_ahash_prepare(&req->base, engine); + + ret = mv_cesa_queue_req(&req->base, &creq->base); + if (mv_cesa_req_needs_cleanup(&req->base, ret)) mv_cesa_ahash_cleanup(req); return ret; } +static int mv_cesa_ahash_update(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + creq->len += req->nbytes; + + return mv_cesa_ahash_queue_req(req); +} + static int mv_cesa_ahash_final(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl; - bool cached = false; - int ret; mv_cesa_set_mac_op_total_len(tmpl, creq->len); creq->last_req = true; req->nbytes = 0; - ret = mv_cesa_ahash_req_init(req, &cached); - if (ret) - return ret; - - if (cached) - return 0; - - ret = mv_cesa_queue_req(&req->base); - if (mv_cesa_req_needs_cleanup(&req->base, ret)) - mv_cesa_ahash_cleanup(req); - - return ret; + return mv_cesa_ahash_queue_req(req); } static int mv_cesa_ahash_finup(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl; - bool cached = false; - int ret; creq->len += req->nbytes; mv_cesa_set_mac_op_total_len(tmpl, creq->len); creq->last_req = true; - ret = mv_cesa_ahash_req_init(req, &cached); - if (ret) - return ret; - - if (cached) - return 0; - - ret = mv_cesa_queue_req(&req->base); - if (mv_cesa_req_needs_cleanup(&req->base, ret)) - mv_cesa_ahash_cleanup(req); - - return ret; + return mv_cesa_ahash_queue_req(req); } static int mv_cesa_ahash_export(struct ahash_request *req, void *hash, diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c index 0ad8f1ecf..86a065bcc 100644 --- a/drivers/crypto/marvell/tdma.c +++ b/drivers/crypto/marvell/tdma.c @@ -37,9 +37,9 @@ bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *iter, return true; } -void mv_cesa_dma_step(struct mv_cesa_tdma_req *dreq) +void mv_cesa_dma_step(struct mv_cesa_req *dreq) { - struct mv_cesa_engine *engine = dreq->base.engine; + struct mv_cesa_engine *engine = dreq->engine; writel_relaxed(0, engine->regs + CESA_SA_CFG); @@ -53,19 +53,25 @@ void mv_cesa_dma_step(struct mv_cesa_tdma_req *dreq) engine->regs + CESA_SA_CFG); writel_relaxed(dreq->chain.first->cur_dma, engine->regs + CESA_TDMA_NEXT_ADDR); + BUG_ON(readl(engine->regs + CESA_SA_CMD) & + CESA_SA_CMD_EN_CESA_SA_ACCL0); writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } -void mv_cesa_dma_cleanup(struct mv_cesa_tdma_req *dreq) +void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq) { struct mv_cesa_tdma_desc *tdma; for (tdma = dreq->chain.first; tdma;) { struct mv_cesa_tdma_desc *old_tdma = tdma; + u32 type = tdma->flags & CESA_TDMA_TYPE_MSK; - if (tdma->flags & CESA_TDMA_OP) + if (type == CESA_TDMA_OP) dma_pool_free(cesa_dev->dma->op_pool, tdma->op, le32_to_cpu(tdma->src)); + else if (type == CESA_TDMA_IV) + dma_pool_free(cesa_dev->dma->iv_pool, tdma->data, + le32_to_cpu(tdma->dst)); tdma = tdma->next; dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma, @@ -76,7 +82,7 @@ void mv_cesa_dma_cleanup(struct mv_cesa_tdma_req *dreq) dreq->chain.last = NULL; } -void mv_cesa_dma_prepare(struct mv_cesa_tdma_req *dreq, +void mv_cesa_dma_prepare(struct mv_cesa_req *dreq, struct mv_cesa_engine *engine) { struct mv_cesa_tdma_desc *tdma; @@ -88,11 +94,97 @@ void mv_cesa_dma_prepare(struct mv_cesa_tdma_req *dreq, if (tdma->flags & CESA_TDMA_SRC_IN_SRAM) tdma->src = cpu_to_le32(tdma->src + engine->sram_dma); - if (tdma->flags & CESA_TDMA_OP) + if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP) mv_cesa_adjust_op(engine, tdma->op); } } +void mv_cesa_tdma_chain(struct mv_cesa_engine *engine, + struct mv_cesa_req *dreq) +{ + if (engine->chain.first == NULL && engine->chain.last == NULL) { + engine->chain.first = dreq->chain.first; + engine->chain.last = dreq->chain.last; + } else { + struct mv_cesa_tdma_desc *last; + + last = engine->chain.last; + last->next = dreq->chain.first; + engine->chain.last = dreq->chain.last; + + if (!(last->flags & CESA_TDMA_BREAK_CHAIN)) + last->next_dma = dreq->chain.first->cur_dma; + } +} + +int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) +{ + struct crypto_async_request *req = NULL; + struct mv_cesa_tdma_desc *tdma = NULL, *next = NULL; + dma_addr_t tdma_cur; + int res = 0; + + tdma_cur = readl(engine->regs + CESA_TDMA_CUR); + + for (tdma = engine->chain.first; tdma; tdma = next) { + spin_lock_bh(&engine->lock); + next = tdma->next; + spin_unlock_bh(&engine->lock); + + if (tdma->flags & CESA_TDMA_END_OF_REQ) { + struct crypto_async_request *backlog = NULL; + struct mv_cesa_ctx *ctx; + u32 current_status; + + spin_lock_bh(&engine->lock); + /* + * if req is NULL, this means we're processing the + * request in engine->req. + */ + if (!req) + req = engine->req; + else + req = mv_cesa_dequeue_req_locked(engine, + &backlog); + + /* Re-chaining to the next request */ + engine->chain.first = tdma->next; + tdma->next = NULL; + + /* If this is the last request, clear the chain */ + if (engine->chain.first == NULL) + engine->chain.last = NULL; + spin_unlock_bh(&engine->lock); + + ctx = crypto_tfm_ctx(req->tfm); + current_status = (tdma->cur_dma == tdma_cur) ? + status : CESA_SA_INT_ACC0_IDMA_DONE; + res = ctx->ops->process(req, current_status); + ctx->ops->complete(req); + + if (res == 0) + mv_cesa_engine_enqueue_complete_request(engine, + req); + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + } + + if (res || tdma->cur_dma == tdma_cur) + break; + } + + /* Save the last request in error to engine->req, so that the core + * knows which request was fautly */ + if (res) { + spin_lock_bh(&engine->lock); + engine->req = req; + spin_unlock_bh(&engine->lock); + } + + return res; +} + static struct mv_cesa_tdma_desc * mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags) { @@ -117,6 +209,32 @@ mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags) return new_tdma; } +int mv_cesa_dma_add_iv_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src, + u32 size, u32 flags, gfp_t gfp_flags) +{ + + struct mv_cesa_tdma_desc *tdma; + u8 *iv; + dma_addr_t dma_handle; + + tdma = mv_cesa_dma_add_desc(chain, gfp_flags); + if (IS_ERR(tdma)) + return PTR_ERR(tdma); + + iv = dma_pool_alloc(cesa_dev->dma->iv_pool, gfp_flags, &dma_handle); + if (!iv) + return -ENOMEM; + + tdma->byte_cnt = cpu_to_le32(size | BIT(31)); + tdma->src = src; + tdma->dst = cpu_to_le32(dma_handle); + tdma->data = iv; + + flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); + tdma->flags = flags | CESA_TDMA_IV; + return 0; +} + struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, const struct mv_cesa_op_ctx *op_templ, bool skip_ctx, diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index 59ed54e46..625ee50fd 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -11,7 +11,6 @@ * http://www.gnu.org/copyleft/gpl.html */ -#include #include #include #include @@ -25,6 +24,7 @@ #include #include #include +#include #define DCP_MAX_CHANS 4 #define DCP_BUF_SZ PAGE_SIZE @@ -84,7 +84,7 @@ struct dcp_async_ctx { unsigned int hot:1; /* Crypto-specific context */ - struct crypto_ablkcipher *fallback; + struct crypto_skcipher *fallback; unsigned int key_len; uint8_t key[AES_KEYSIZE_128]; }; @@ -374,20 +374,22 @@ static int dcp_chan_thread_aes(void *data) static int mxs_dcp_block_fallback(struct ablkcipher_request *req, int enc) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); - struct dcp_async_ctx *ctx = crypto_ablkcipher_ctx( - crypto_ablkcipher_reqtfm(req)); + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct dcp_async_ctx *ctx = crypto_ablkcipher_ctx(tfm); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); int ret; - ablkcipher_request_set_tfm(req, ctx->fallback); + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); if (enc) - ret = crypto_ablkcipher_encrypt(req); + ret = crypto_skcipher_encrypt(subreq); else - ret = crypto_ablkcipher_decrypt(req); + ret = crypto_skcipher_decrypt(subreq); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + skcipher_request_zero(subreq); return ret; } @@ -453,28 +455,22 @@ static int mxs_dcp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, return 0; } - /* Check if the key size is supported by kernel at all. */ - if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) { - tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; - return -EINVAL; - } - /* * If the requested AES key size is not supported by the hardware, * but is supported by in-kernel software implementation, we use * software fallback. */ - actx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - actx->fallback->base.crt_flags |= - tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK; + crypto_skcipher_clear_flags(actx->fallback, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(actx->fallback, + tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); - ret = crypto_ablkcipher_setkey(actx->fallback, key, len); + ret = crypto_skcipher_setkey(actx->fallback, key, len); if (!ret) return 0; tfm->base.crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm->base.crt_flags |= - actx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK; + tfm->base.crt_flags |= crypto_skcipher_get_flags(actx->fallback) & + CRYPTO_TFM_RES_MASK; return ret; } @@ -484,9 +480,9 @@ static int mxs_dcp_aes_fallback_init(struct crypto_tfm *tfm) const char *name = crypto_tfm_alg_name(tfm); const uint32_t flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK; struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm); - struct crypto_ablkcipher *blk; + struct crypto_skcipher *blk; - blk = crypto_alloc_ablkcipher(name, 0, flags); + blk = crypto_alloc_skcipher(name, 0, flags); if (IS_ERR(blk)) return PTR_ERR(blk); @@ -499,8 +495,7 @@ static void mxs_dcp_aes_fallback_exit(struct crypto_tfm *tfm) { struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm); - crypto_free_ablkcipher(actx->fallback); - actx->fallback = NULL; + crypto_free_skcipher(actx->fallback); } /* diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index ce174d3b8..4ab53a604 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -528,8 +528,6 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) omap_aes_dma_stop(dd); - dmaengine_terminate_all(dd->dma_lch_in); - dmaengine_terminate_all(dd->dma_lch_out); return 0; } @@ -580,10 +578,12 @@ static int omap_aes_copy_sgs(struct omap_aes_dev *dd) sg_init_table(&dd->in_sgl, 1); sg_set_buf(&dd->in_sgl, buf_in, total); dd->in_sg = &dd->in_sgl; + dd->in_sg_len = 1; sg_init_table(&dd->out_sgl, 1); sg_set_buf(&dd->out_sgl, buf_out, total); dd->out_sg = &dd->out_sgl; + dd->out_sg_len = 1; return 0; } @@ -604,7 +604,6 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, crypto_ablkcipher_reqtfm(req)); struct omap_aes_dev *dd = omap_aes_find_dev(ctx); struct omap_aes_reqctx *rctx; - int len; if (!dd) return -ENODEV; @@ -616,6 +615,14 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, dd->in_sg = req->src; dd->out_sg = req->dst; + dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total); + if (dd->in_sg_len < 0) + return dd->in_sg_len; + + dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total); + if (dd->out_sg_len < 0) + return dd->out_sg_len; + if (omap_aes_check_aligned(dd->in_sg, dd->total) || omap_aes_check_aligned(dd->out_sg, dd->total)) { if (omap_aes_copy_sgs(dd)) @@ -625,11 +632,6 @@ static int omap_aes_prepare_req(struct crypto_engine *engine, dd->sgs_copied = 0; } - len = ALIGN(dd->total, AES_BLOCK_SIZE); - dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, len); - dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, len); - BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0); - rctx = ablkcipher_request_ctx(req); ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); rctx->mode &= FLAGS_MODE_MASK; @@ -1185,17 +1187,19 @@ static int omap_aes_probe(struct platform_device *pdev) spin_unlock(&list_lock); for (i = 0; i < dd->pdata->algs_info_size; i++) { - for (j = 0; j < dd->pdata->algs_info[i].size; j++) { - algp = &dd->pdata->algs_info[i].algs_list[j]; + if (!dd->pdata->algs_info[i].registered) { + for (j = 0; j < dd->pdata->algs_info[i].size; j++) { + algp = &dd->pdata->algs_info[i].algs_list[j]; - pr_debug("reg alg: %s\n", algp->cra_name); - INIT_LIST_HEAD(&algp->cra_list); + pr_debug("reg alg: %s\n", algp->cra_name); + INIT_LIST_HEAD(&algp->cra_list); - err = crypto_register_alg(algp); - if (err) - goto err_algs; + err = crypto_register_alg(algp); + if (err) + goto err_algs; - dd->pdata->algs_info[i].registered++; + dd->pdata->algs_info[i].registered++; + } } } diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 3eedb0311..5691434ff 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -560,10 +560,12 @@ static int omap_des_copy_sgs(struct omap_des_dev *dd) sg_init_table(&dd->in_sgl, 1); sg_set_buf(&dd->in_sgl, buf_in, dd->total); dd->in_sg = &dd->in_sgl; + dd->in_sg_len = 1; sg_init_table(&dd->out_sgl, 1); sg_set_buf(&dd->out_sgl, buf_out, dd->total); dd->out_sg = &dd->out_sgl; + dd->out_sg_len = 1; return 0; } @@ -595,6 +597,14 @@ static int omap_des_prepare_req(struct crypto_engine *engine, dd->in_sg = req->src; dd->out_sg = req->dst; + dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total); + if (dd->in_sg_len < 0) + return dd->in_sg_len; + + dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total); + if (dd->out_sg_len < 0) + return dd->out_sg_len; + if (omap_des_copy_needed(dd->in_sg) || omap_des_copy_needed(dd->out_sg)) { if (omap_des_copy_sgs(dd)) @@ -604,10 +614,6 @@ static int omap_des_prepare_req(struct crypto_engine *engine, dd->sgs_copied = 0; } - dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total); - dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, dd->total); - BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0); - rctx = ablkcipher_request_ctx(req); ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); rctx->mode &= FLAGS_MODE_MASK; diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 63464e86f..7fe4eef12 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -100,6 +100,8 @@ #define DEFAULT_TIMEOUT_INTERVAL HZ +#define DEFAULT_AUTOSUSPEND_DELAY 1000 + /* mostly device flags */ #define FLAGS_BUSY 0 #define FLAGS_FINAL 1 @@ -173,7 +175,7 @@ struct omap_sham_ctx { struct omap_sham_hmac_ctx base[0]; }; -#define OMAP_SHAM_QUEUE_LENGTH 1 +#define OMAP_SHAM_QUEUE_LENGTH 10 struct omap_sham_algs_info { struct ahash_alg *algs_list; @@ -813,7 +815,6 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd) { struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); - dmaengine_terminate_all(dd->dma_lch); if (ctx->flags & BIT(FLAGS_SG)) { dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); @@ -999,7 +1000,8 @@ static void omap_sham_finish_req(struct ahash_request *req, int err) dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY)); - pm_runtime_put(dd->dev); + pm_runtime_mark_last_busy(dd->dev); + pm_runtime_put_autosuspend(dd->dev); if (req->base.complete) req->base.complete(&req->base, err); @@ -1093,7 +1095,7 @@ static int omap_sham_update(struct ahash_request *req) ctx->offset = 0; if (ctx->flags & BIT(FLAGS_FINUP)) { - if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) { + if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 240) { /* * OMAP HW accel works only with buffers >= 9 * will switch to bypass in final() @@ -1149,9 +1151,13 @@ static int omap_sham_final(struct ahash_request *req) if (ctx->flags & BIT(FLAGS_ERROR)) return 0; /* uncompleted hash is not needed */ - /* OMAP HW accel works only with buffers >= 9 */ - /* HMAC is always >= 9 because ipad == block size */ - if ((ctx->digcnt + ctx->bufcnt) < 9) + /* + * OMAP HW accel works only with buffers >= 9. + * HMAC is always >= 9 because ipad == block size. + * If buffersize is less than 240, we use fallback SW encoding, + * as using DMA + HW in this case doesn't provide any benefit. + */ + if ((ctx->digcnt + ctx->bufcnt) < 240) return omap_sham_final_shash(req); else if (ctx->bufcnt) return omap_sham_enqueue(req, OP_FINAL); @@ -1328,7 +1334,7 @@ static struct ahash_alg algs_sha1_md5[] = { .halg.base = { .cra_name = "sha1", .cra_driver_name = "omap-sha1", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | @@ -1351,7 +1357,7 @@ static struct ahash_alg algs_sha1_md5[] = { .halg.base = { .cra_name = "md5", .cra_driver_name = "omap-md5", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | @@ -1375,7 +1381,7 @@ static struct ahash_alg algs_sha1_md5[] = { .halg.base = { .cra_name = "hmac(sha1)", .cra_driver_name = "omap-hmac-sha1", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | @@ -1400,7 +1406,7 @@ static struct ahash_alg algs_sha1_md5[] = { .halg.base = { .cra_name = "hmac(md5)", .cra_driver_name = "omap-hmac-md5", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | @@ -1428,7 +1434,7 @@ static struct ahash_alg algs_sha224_sha256[] = { .halg.base = { .cra_name = "sha224", .cra_driver_name = "omap-sha224", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1450,7 +1456,7 @@ static struct ahash_alg algs_sha224_sha256[] = { .halg.base = { .cra_name = "sha256", .cra_driver_name = "omap-sha256", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1473,7 +1479,7 @@ static struct ahash_alg algs_sha224_sha256[] = { .halg.base = { .cra_name = "hmac(sha224)", .cra_driver_name = "omap-hmac-sha224", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1497,7 +1503,7 @@ static struct ahash_alg algs_sha224_sha256[] = { .halg.base = { .cra_name = "hmac(sha256)", .cra_driver_name = "omap-hmac-sha256", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1523,7 +1529,7 @@ static struct ahash_alg algs_sha384_sha512[] = { .halg.base = { .cra_name = "sha384", .cra_driver_name = "omap-sha384", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1545,7 +1551,7 @@ static struct ahash_alg algs_sha384_sha512[] = { .halg.base = { .cra_name = "sha512", .cra_driver_name = "omap-sha512", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1568,7 +1574,7 @@ static struct ahash_alg algs_sha384_sha512[] = { .halg.base = { .cra_name = "hmac(sha384)", .cra_driver_name = "omap-hmac-sha384", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1592,7 +1598,7 @@ static struct ahash_alg algs_sha384_sha512[] = { .halg.base = { .cra_name = "hmac(sha512)", .cra_driver_name = "omap-hmac-sha512", - .cra_priority = 100, + .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, @@ -1946,6 +1952,9 @@ static int omap_sham_probe(struct platform_device *pdev) dd->flags |= dd->pdata->flags; + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); + pm_runtime_enable(dev); pm_runtime_irq_safe(dev); diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index 3b1c7ecf0..475760988 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -171,7 +171,7 @@ struct spacc_ablk_ctx { * The fallback cipher. If the operation can't be done in hardware, * fallback to a software version. */ - struct crypto_ablkcipher *sw_cipher; + struct crypto_skcipher *sw_cipher; }; /* AEAD cipher context. */ @@ -789,33 +789,35 @@ static int spacc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, * request for any other size (192 bits) then we need to do a software * fallback. */ - if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256 && - ctx->sw_cipher) { + if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256) { + if (!ctx->sw_cipher) + return -EINVAL; + /* * Set the fallback transform to use the same request flags as * the hardware transform. */ - ctx->sw_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - ctx->sw_cipher->base.crt_flags |= - cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK; + crypto_skcipher_clear_flags(ctx->sw_cipher, + CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->sw_cipher, + cipher->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + err = crypto_skcipher_setkey(ctx->sw_cipher, key, len); + + tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm->crt_flags |= + crypto_skcipher_get_flags(ctx->sw_cipher) & + CRYPTO_TFM_RES_MASK; - err = crypto_ablkcipher_setkey(ctx->sw_cipher, key, len); if (err) goto sw_setkey_failed; - } else if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256 && - !ctx->sw_cipher) - err = -EINVAL; + } memcpy(ctx->key, key, len); ctx->key_len = len; sw_setkey_failed: - if (err && ctx->sw_cipher) { - tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm->crt_flags |= - ctx->sw_cipher->base.crt_flags & CRYPTO_TFM_RES_MASK; - } - return err; } @@ -910,20 +912,21 @@ static int spacc_ablk_do_fallback(struct ablkcipher_request *req, struct crypto_tfm *old_tfm = crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(old_tfm); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->sw_cipher); int err; - if (!ctx->sw_cipher) - return -EINVAL; - /* * Change the request to use the software fallback transform, and once * the ciphering has completed, put the old transform back into the * request. */ - ablkcipher_request_set_tfm(req, ctx->sw_cipher); - err = is_encrypt ? crypto_ablkcipher_encrypt(req) : - crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(old_tfm)); + skcipher_request_set_tfm(subreq, ctx->sw_cipher); + skcipher_request_set_callback(subreq, req->base.flags, NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + err = is_encrypt ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); return err; } @@ -1015,12 +1018,13 @@ static int spacc_ablk_cra_init(struct crypto_tfm *tfm) ctx->generic.flags = spacc_alg->type; ctx->generic.engine = engine; if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { - ctx->sw_cipher = crypto_alloc_ablkcipher(alg->cra_name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + ctx->sw_cipher = crypto_alloc_skcipher( + alg->cra_name, 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->sw_cipher)) { dev_warn(engine->dev, "failed to allocate fallback for %s\n", alg->cra_name); - ctx->sw_cipher = NULL; + return PTR_ERR(ctx->sw_cipher); } } ctx->generic.key_offs = spacc_alg->key_offs; @@ -1035,9 +1039,7 @@ static void spacc_ablk_cra_exit(struct crypto_tfm *tfm) { struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->sw_cipher) - crypto_free_ablkcipher(ctx->sw_cipher); - ctx->sw_cipher = NULL; + crypto_free_skcipher(ctx->sw_cipher); } static int spacc_ablk_encrypt(struct ablkcipher_request *req) diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index 85b44e577..ce3cae40f 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -4,12 +4,13 @@ config CRYPTO_DEV_QAT select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER select CRYPTO_AKCIPHER + select CRYPTO_DH select CRYPTO_HMAC + select CRYPTO_RSA select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 select FW_LOADER - select ASN1 config CRYPTO_DEV_QAT_DH895xCC tristate "Support for Intel(R) DH895xCC" diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index c5bd5a9ab..6bc68bc00 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -229,6 +229,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->reset_device = adf_reset_flr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; } diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 879e04cae..618cec360 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -239,6 +239,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->reset_device = adf_reset_flr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; } diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 5fc3dbb9a..92fb6ffdc 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -1,12 +1,3 @@ -$(obj)/qat_rsapubkey-asn1.o: $(obj)/qat_rsapubkey-asn1.c \ - $(obj)/qat_rsapubkey-asn1.h -$(obj)/qat_rsaprivkey-asn1.o: $(obj)/qat_rsaprivkey-asn1.c \ - $(obj)/qat_rsaprivkey-asn1.h -$(obj)/qat_asym_algs.o: $(obj)/qat_rsapubkey-asn1.h $(obj)/qat_rsaprivkey-asn1.h - -clean-files += qat_rsapubkey-asn1.c qat_rsapubkey-asn1.h -clean-files += qat_rsaprivkey-asn1.c qat_rsaprivkey-asn1.h - obj-$(CONFIG_CRYPTO_DEV_QAT) += intel_qat.o intel_qat-objs := adf_cfg.o \ adf_isr.o \ @@ -20,8 +11,6 @@ intel_qat-objs := adf_cfg.o \ adf_hw_arbiter.o \ qat_crypto.o \ qat_algs.o \ - qat_rsapubkey-asn1.o \ - qat_rsaprivkey-asn1.o \ qat_asym_algs.o \ qat_uclo.o \ qat_hal.o diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 5a07208ce..e88225365 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -176,6 +176,7 @@ struct adf_hw_device_data { void (*disable_iov)(struct adf_accel_dev *accel_dev); void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); + void (*reset_device)(struct adf_accel_dev *accel_dev); const char *fw_name; const char *fw_mmp_name; uint32_t fuses; diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index b40d9c8da..2839fccdd 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -82,18 +82,12 @@ struct adf_reset_dev_data { struct work_struct reset_work; }; -void adf_dev_restore(struct adf_accel_dev *accel_dev) +void adf_reset_sbr(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); struct pci_dev *parent = pdev->bus->self; uint16_t bridge_ctl = 0; - if (accel_dev->is_vf) - return; - - dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n", - accel_dev->accel_id); - if (!parent) parent = pdev; @@ -101,6 +95,8 @@ void adf_dev_restore(struct adf_accel_dev *accel_dev) dev_info(&GET_DEV(accel_dev), "Transaction still in progress. Proceeding\n"); + dev_info(&GET_DEV(accel_dev), "Secondary bus reset\n"); + pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl); bridge_ctl |= PCI_BRIDGE_CTL_BUS_RESET; pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); @@ -108,8 +104,40 @@ void adf_dev_restore(struct adf_accel_dev *accel_dev) bridge_ctl &= ~PCI_BRIDGE_CTL_BUS_RESET; pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); msleep(100); - pci_restore_state(pdev); - pci_save_state(pdev); +} +EXPORT_SYMBOL_GPL(adf_reset_sbr); + +void adf_reset_flr(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + u16 control = 0; + int pos = 0; + + dev_info(&GET_DEV(accel_dev), "Function level reset\n"); + pos = pci_pcie_cap(pdev); + if (!pos) { + dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); + return; + } + pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &control); + control |= PCI_EXP_DEVCTL_BCR_FLR; + pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, control); + msleep(100); +} +EXPORT_SYMBOL_GPL(adf_reset_flr); + +void adf_dev_restore(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + if (hw_device->reset_device) { + dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n", + accel_dev->accel_id); + hw_device->reset_device(accel_dev); + pci_restore_state(pdev); + pci_save_state(pdev); + } } static void adf_device_reset_worker(struct work_struct *work) @@ -243,7 +271,8 @@ EXPORT_SYMBOL_GPL(adf_disable_aer); int adf_init_aer(void) { - device_reset_wq = create_workqueue("qat_device_reset_wq"); + device_reset_wq = alloc_workqueue("qat_device_reset_wq", + WQ_MEM_RECLAIM, 0); return !device_reset_wq ? -EFAULT : 0; } diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 75faa39bc..980e07475 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -141,6 +141,8 @@ int adf_ae_stop(struct adf_accel_dev *accel_dev); int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf); void adf_disable_aer(struct adf_accel_dev *accel_dev); +void adf_reset_sbr(struct adf_accel_dev *accel_dev); +void adf_reset_flr(struct adf_accel_dev *accel_dev); void adf_dev_restore(struct adf_accel_dev *accel_dev); int adf_init_aer(void); void adf_exit_aer(void); diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 4a526e2f1..9320ae1d0 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -292,7 +292,7 @@ EXPORT_SYMBOL_GPL(adf_sriov_configure); int __init adf_init_pf_wq(void) { /* Workqueue for PF2VF responses */ - pf2vf_resp_wq = create_workqueue("qat_pf2vf_resp_wq"); + pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq", WQ_MEM_RECLAIM, 0); return !pf2vf_resp_wq ? -ENOMEM : 0; } diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index aa689cabe..bf99e11a3 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -321,7 +321,7 @@ EXPORT_SYMBOL_GPL(adf_vf_isr_resource_alloc); int __init adf_init_vf_wq(void) { - adf_vf_stop_wq = create_workqueue("adf_vf_stop_wq"); + adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0); return !adf_vf_stop_wq ? -EFAULT : 0; } diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 4c9deef6a..20f35df8a 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -947,13 +947,13 @@ static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, return 0; out_free_all: - memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd)); - dma_free_coherent(dev, sizeof(*ctx->enc_cd), + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + dma_free_coherent(dev, sizeof(*ctx->dec_cd), ctx->dec_cd, ctx->dec_cd_paddr); ctx->dec_cd = NULL; out_free_enc: - memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd)); - dma_free_coherent(dev, sizeof(*ctx->dec_cd), + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + dma_free_coherent(dev, sizeof(*ctx->enc_cd), ctx->enc_cd, ctx->enc_cd_paddr); ctx->enc_cd = NULL; return -ENOMEM; diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 05f49d4f9..0d35dca2e 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -49,11 +49,12 @@ #include #include #include +#include +#include +#include #include #include #include -#include "qat_rsapubkey-asn1.h" -#include "qat_rsaprivkey-asn1.h" #include "icp_qat_fw_pke.h" #include "adf_accel_devices.h" #include "adf_transport.h" @@ -75,6 +76,14 @@ struct qat_rsa_input_params { dma_addr_t d; dma_addr_t n; } dec; + struct { + dma_addr_t c; + dma_addr_t p; + dma_addr_t q; + dma_addr_t dp; + dma_addr_t dq; + dma_addr_t qinv; + } dec_crt; u64 in_tab[8]; }; } __packed __aligned(64); @@ -95,71 +104,480 @@ struct qat_rsa_ctx { char *n; char *e; char *d; + char *p; + char *q; + char *dp; + char *dq; + char *qinv; dma_addr_t dma_n; dma_addr_t dma_e; dma_addr_t dma_d; + dma_addr_t dma_p; + dma_addr_t dma_q; + dma_addr_t dma_dp; + dma_addr_t dma_dq; + dma_addr_t dma_qinv; unsigned int key_sz; + bool crt_mode; + struct qat_crypto_instance *inst; +} __packed __aligned(64); + +struct qat_dh_input_params { + union { + struct { + dma_addr_t b; + dma_addr_t xa; + dma_addr_t p; + } in; + struct { + dma_addr_t xa; + dma_addr_t p; + } in_g2; + u64 in_tab[8]; + }; +} __packed __aligned(64); + +struct qat_dh_output_params { + union { + dma_addr_t r; + u64 out_tab[8]; + }; +} __packed __aligned(64); + +struct qat_dh_ctx { + char *g; + char *xa; + char *p; + dma_addr_t dma_g; + dma_addr_t dma_xa; + dma_addr_t dma_p; + unsigned int p_size; + bool g2; struct qat_crypto_instance *inst; } __packed __aligned(64); -struct qat_rsa_request { - struct qat_rsa_input_params in; - struct qat_rsa_output_params out; +struct qat_asym_request { + union { + struct qat_rsa_input_params rsa; + struct qat_dh_input_params dh; + } in; + union { + struct qat_rsa_output_params rsa; + struct qat_dh_output_params dh; + } out; dma_addr_t phy_in; dma_addr_t phy_out; char *src_align; char *dst_align; struct icp_qat_fw_pke_request req; - struct qat_rsa_ctx *ctx; + union { + struct qat_rsa_ctx *rsa; + struct qat_dh_ctx *dh; + } ctx; + union { + struct akcipher_request *rsa; + struct kpp_request *dh; + } areq; int err; + void (*cb)(struct icp_qat_fw_pke_resp *resp); } __aligned(64); -static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) +static void qat_dh_cb(struct icp_qat_fw_pke_resp *resp) { - struct akcipher_request *areq = (void *)(__force long)resp->opaque; - struct qat_rsa_request *req = PTR_ALIGN(akcipher_request_ctx(areq), 64); - struct device *dev = &GET_DEV(req->ctx->inst->accel_dev); + struct qat_asym_request *req = (void *)(__force long)resp->opaque; + struct kpp_request *areq = req->areq.dh; + struct device *dev = &GET_DEV(req->ctx.dh->inst->accel_dev); int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( resp->pke_resp_hdr.comn_resp_flags); err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; - if (req->src_align) - dma_free_coherent(dev, req->ctx->key_sz, req->src_align, - req->in.enc.m); - else - dma_unmap_single(dev, req->in.enc.m, req->ctx->key_sz, - DMA_TO_DEVICE); + if (areq->src) { + if (req->src_align) + dma_free_coherent(dev, req->ctx.dh->p_size, + req->src_align, req->in.dh.in.b); + else + dma_unmap_single(dev, req->in.dh.in.b, + req->ctx.dh->p_size, DMA_TO_DEVICE); + } - areq->dst_len = req->ctx->key_sz; + areq->dst_len = req->ctx.dh->p_size; if (req->dst_align) { - char *ptr = req->dst_align; + scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, + areq->dst_len, 1); - while (!(*ptr) && areq->dst_len) { - areq->dst_len--; - ptr++; - } + dma_free_coherent(dev, req->ctx.dh->p_size, req->dst_align, + req->out.dh.r); + } else { + dma_unmap_single(dev, req->out.dh.r, req->ctx.dh->p_size, + DMA_FROM_DEVICE); + } - if (areq->dst_len != req->ctx->key_sz) - memmove(req->dst_align, ptr, areq->dst_len); + dma_unmap_single(dev, req->phy_in, sizeof(struct qat_dh_input_params), + DMA_TO_DEVICE); + dma_unmap_single(dev, req->phy_out, + sizeof(struct qat_dh_output_params), + DMA_TO_DEVICE); - scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, - areq->dst_len, 1); + kpp_request_complete(areq, err); +} + +#define PKE_DH_1536 0x390c1a49 +#define PKE_DH_G2_1536 0x2e0b1a3e +#define PKE_DH_2048 0x4d0c1a60 +#define PKE_DH_G2_2048 0x3e0b1a55 +#define PKE_DH_3072 0x510c1a77 +#define PKE_DH_G2_3072 0x3a0b1a6c +#define PKE_DH_4096 0x690c1a8e +#define PKE_DH_G2_4096 0x4a0b1a83 + +static unsigned long qat_dh_fn_id(unsigned int len, bool g2) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 1536: + return g2 ? PKE_DH_G2_1536 : PKE_DH_1536; + case 2048: + return g2 ? PKE_DH_G2_2048 : PKE_DH_2048; + case 3072: + return g2 ? PKE_DH_G2_3072 : PKE_DH_3072; + case 4096: + return g2 ? PKE_DH_G2_4096 : PKE_DH_4096; + default: + return 0; + }; +} + +static inline struct qat_dh_ctx *qat_dh_get_params(struct crypto_kpp *tfm) +{ + return kpp_tfm_ctx(tfm); +} + +static int qat_dh_compute_value(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + struct qat_asym_request *qat_req = + PTR_ALIGN(kpp_request_ctx(req), 64); + struct icp_qat_fw_pke_request *msg = &qat_req->req; + int ret, ctr = 0; + int n_input_params = 0; + + if (unlikely(!ctx->xa)) + return -EINVAL; + + if (req->dst_len < ctx->p_size) { + req->dst_len = ctx->p_size; + return -EOVERFLOW; + } + memset(msg, '\0', sizeof(*msg)); + ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, + ICP_QAT_FW_COMN_REQ_FLAG_SET); + + msg->pke_hdr.cd_pars.func_id = qat_dh_fn_id(ctx->p_size, + !req->src && ctx->g2); + if (unlikely(!msg->pke_hdr.cd_pars.func_id)) + return -EINVAL; + + qat_req->cb = qat_dh_cb; + qat_req->ctx.dh = ctx; + qat_req->areq.dh = req; + msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; + msg->pke_hdr.comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, + QAT_COMN_CD_FLD_TYPE_64BIT_ADR); - dma_free_coherent(dev, req->ctx->key_sz, req->dst_align, - req->out.enc.c); + /* + * If no source is provided use g as base + */ + if (req->src) { + qat_req->in.dh.in.xa = ctx->dma_xa; + qat_req->in.dh.in.p = ctx->dma_p; + n_input_params = 3; } else { - char *ptr = sg_virt(areq->dst); + if (ctx->g2) { + qat_req->in.dh.in_g2.xa = ctx->dma_xa; + qat_req->in.dh.in_g2.p = ctx->dma_p; + n_input_params = 2; + } else { + qat_req->in.dh.in.b = ctx->dma_g; + qat_req->in.dh.in.xa = ctx->dma_xa; + qat_req->in.dh.in.p = ctx->dma_p; + n_input_params = 3; + } + } - while (!(*ptr) && areq->dst_len) { - areq->dst_len--; - ptr++; + ret = -ENOMEM; + if (req->src) { + /* + * src can be of any size in valid range, but HW expects it to + * be the same as modulo p so in case it is different we need + * to allocate a new buf and copy src data. + * In other case we just need to map the user provided buffer. + * Also need to make sure that it is in contiguous buffer. + */ + if (sg_is_last(req->src) && req->src_len == ctx->p_size) { + qat_req->src_align = NULL; + qat_req->in.dh.in.b = dma_map_single(dev, + sg_virt(req->src), + req->src_len, + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, + qat_req->in.dh.in.b))) + return ret; + + } else { + int shift = ctx->p_size - req->src_len; + + qat_req->src_align = dma_zalloc_coherent(dev, + ctx->p_size, + &qat_req->in.dh.in.b, + GFP_KERNEL); + if (unlikely(!qat_req->src_align)) + return ret; + + scatterwalk_map_and_copy(qat_req->src_align + shift, + req->src, 0, req->src_len, 0); } + } + /* + * dst can be of any size in valid range, but HW expects it to be the + * same as modulo m so in case it is different we need to allocate a + * new buf and copy src data. + * In other case we just need to map the user provided buffer. + * Also need to make sure that it is in contiguous buffer. + */ + if (sg_is_last(req->dst) && req->dst_len == ctx->p_size) { + qat_req->dst_align = NULL; + qat_req->out.dh.r = dma_map_single(dev, sg_virt(req->dst), + req->dst_len, + DMA_FROM_DEVICE); - if (sg_virt(areq->dst) != ptr && areq->dst_len) - memmove(sg_virt(areq->dst), ptr, areq->dst_len); + if (unlikely(dma_mapping_error(dev, qat_req->out.dh.r))) + goto unmap_src; + + } else { + qat_req->dst_align = dma_zalloc_coherent(dev, ctx->p_size, + &qat_req->out.dh.r, + GFP_KERNEL); + if (unlikely(!qat_req->dst_align)) + goto unmap_src; + } - dma_unmap_single(dev, req->out.enc.c, req->ctx->key_sz, + qat_req->in.dh.in_tab[n_input_params] = 0; + qat_req->out.dh.out_tab[1] = 0; + /* Mapping in.in.b or in.in_g2.xa is the same */ + qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh.in.b, + sizeof(struct qat_dh_input_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) + goto unmap_dst; + + qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh.r, + sizeof(struct qat_dh_output_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) + goto unmap_in_params; + + msg->pke_mid.src_data_addr = qat_req->phy_in; + msg->pke_mid.dest_data_addr = qat_req->phy_out; + msg->pke_mid.opaque = (uint64_t)(__force long)qat_req; + msg->input_param_count = n_input_params; + msg->output_param_count = 1; + + do { + ret = adf_send_message(ctx->inst->pke_tx, (uint32_t *)msg); + } while (ret == -EBUSY && ctr++ < 100); + + if (!ret) + return -EINPROGRESS; + + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_dh_output_params), + DMA_TO_DEVICE); +unmap_in_params: + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_dh_input_params), + DMA_TO_DEVICE); +unmap_dst: + if (qat_req->dst_align) + dma_free_coherent(dev, ctx->p_size, qat_req->dst_align, + qat_req->out.dh.r); + else + if (!dma_mapping_error(dev, qat_req->out.dh.r)) + dma_unmap_single(dev, qat_req->out.dh.r, ctx->p_size, + DMA_FROM_DEVICE); +unmap_src: + if (req->src) { + if (qat_req->src_align) + dma_free_coherent(dev, ctx->p_size, qat_req->src_align, + qat_req->in.dh.in.b); + else + if (!dma_mapping_error(dev, qat_req->in.dh.in.b)) + dma_unmap_single(dev, qat_req->in.dh.in.b, + ctx->p_size, + DMA_TO_DEVICE); + } + return ret; +} + +static int qat_dh_check_params_length(unsigned int p_len) +{ + switch (p_len) { + case 1536: + case 2048: + case 3072: + case 4096: + return 0; + } + return -EINVAL; +} + +static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params) +{ + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + + if (unlikely(!params->p || !params->g)) + return -EINVAL; + + if (qat_dh_check_params_length(params->p_size << 3)) + return -EINVAL; + + ctx->p_size = params->p_size; + ctx->p = dma_zalloc_coherent(dev, ctx->p_size, &ctx->dma_p, GFP_KERNEL); + if (!ctx->p) + return -ENOMEM; + memcpy(ctx->p, params->p, ctx->p_size); + + /* If g equals 2 don't copy it */ + if (params->g_size == 1 && *(char *)params->g == 0x02) { + ctx->g2 = true; + return 0; + } + + ctx->g = dma_zalloc_coherent(dev, ctx->p_size, &ctx->dma_g, GFP_KERNEL); + if (!ctx->g) { + dma_free_coherent(dev, ctx->p_size, ctx->p, ctx->dma_p); + ctx->p = NULL; + return -ENOMEM; + } + memcpy(ctx->g + (ctx->p_size - params->g_size), params->g, + params->g_size); + + return 0; +} + +static void qat_dh_clear_ctx(struct device *dev, struct qat_dh_ctx *ctx) +{ + if (ctx->g) { + dma_free_coherent(dev, ctx->p_size, ctx->g, ctx->dma_g); + ctx->g = NULL; + } + if (ctx->xa) { + dma_free_coherent(dev, ctx->p_size, ctx->xa, ctx->dma_xa); + ctx->xa = NULL; + } + if (ctx->p) { + dma_free_coherent(dev, ctx->p_size, ctx->p, ctx->dma_p); + ctx->p = NULL; + } + ctx->p_size = 0; + ctx->g2 = false; +} + +static int qat_dh_set_secret(struct crypto_kpp *tfm, void *buf, + unsigned int len) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + struct dh params; + int ret; + + if (crypto_dh_decode_key(buf, len, ¶ms) < 0) + return -EINVAL; + + /* Free old secret if any */ + qat_dh_clear_ctx(dev, ctx); + + ret = qat_dh_set_params(ctx, ¶ms); + if (ret < 0) + return ret; + + ctx->xa = dma_zalloc_coherent(dev, ctx->p_size, &ctx->dma_xa, + GFP_KERNEL); + if (!ctx->xa) { + qat_dh_clear_ctx(dev, ctx); + return -ENOMEM; + } + memcpy(ctx->xa + (ctx->p_size - params.key_size), params.key, + params.key_size); + + return 0; +} + +static int qat_dh_max_size(struct crypto_kpp *tfm) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + + return ctx->p ? ctx->p_size : -EINVAL; +} + +static int qat_dh_init_tfm(struct crypto_kpp *tfm) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct qat_crypto_instance *inst = + qat_crypto_get_instance_node(get_current_node()); + + if (!inst) + return -EINVAL; + + ctx->p_size = 0; + ctx->g2 = false; + ctx->inst = inst; + return 0; +} + +static void qat_dh_exit_tfm(struct crypto_kpp *tfm) +{ + struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + + qat_dh_clear_ctx(dev, ctx); + qat_crypto_put_instance(ctx->inst); +} + +static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) +{ + struct qat_asym_request *req = (void *)(__force long)resp->opaque; + struct akcipher_request *areq = req->areq.rsa; + struct device *dev = &GET_DEV(req->ctx.rsa->inst->accel_dev); + int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( + resp->pke_resp_hdr.comn_resp_flags); + + err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; + + if (req->src_align) + dma_free_coherent(dev, req->ctx.rsa->key_sz, req->src_align, + req->in.rsa.enc.m); + else + dma_unmap_single(dev, req->in.rsa.enc.m, req->ctx.rsa->key_sz, + DMA_TO_DEVICE); + + areq->dst_len = req->ctx.rsa->key_sz; + if (req->dst_align) { + scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, + areq->dst_len, 1); + + dma_free_coherent(dev, req->ctx.rsa->key_sz, req->dst_align, + req->out.rsa.enc.c); + } else { + dma_unmap_single(dev, req->out.rsa.enc.c, req->ctx.rsa->key_sz, DMA_FROM_DEVICE); } @@ -175,8 +593,9 @@ static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) void qat_alg_asym_callback(void *_resp) { struct icp_qat_fw_pke_resp *resp = _resp; + struct qat_asym_request *areq = (void *)(__force long)resp->opaque; - qat_rsa_cb(resp); + areq->cb(resp); } #define PKE_RSA_EP_512 0x1c161b21 @@ -237,13 +656,42 @@ static unsigned long qat_rsa_dec_fn_id(unsigned int len) }; } +#define PKE_RSA_DP2_512 0x1c131b57 +#define PKE_RSA_DP2_1024 0x26131c2d +#define PKE_RSA_DP2_1536 0x45111d12 +#define PKE_RSA_DP2_2048 0x59121dfa +#define PKE_RSA_DP2_3072 0x81121ed9 +#define PKE_RSA_DP2_4096 0xb1111fb2 + +static unsigned long qat_rsa_dec_fn_id_crt(unsigned int len) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 512: + return PKE_RSA_DP2_512; + case 1024: + return PKE_RSA_DP2_1024; + case 1536: + return PKE_RSA_DP2_1536; + case 2048: + return PKE_RSA_DP2_2048; + case 3072: + return PKE_RSA_DP2_3072; + case 4096: + return PKE_RSA_DP2_4096; + default: + return 0; + }; +} + static int qat_rsa_enc(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); - struct qat_rsa_request *qat_req = + struct qat_asym_request *qat_req = PTR_ALIGN(akcipher_request_ctx(req), 64); struct icp_qat_fw_pke_request *msg = &qat_req->req; int ret, ctr = 0; @@ -262,14 +710,16 @@ static int qat_rsa_enc(struct akcipher_request *req) if (unlikely(!msg->pke_hdr.cd_pars.func_id)) return -EINVAL; - qat_req->ctx = ctx; + qat_req->cb = qat_rsa_cb; + qat_req->ctx.rsa = ctx; + qat_req->areq.rsa = req; msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; msg->pke_hdr.comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, QAT_COMN_CD_FLD_TYPE_64BIT_ADR); - qat_req->in.enc.e = ctx->dma_e; - qat_req->in.enc.n = ctx->dma_n; + qat_req->in.rsa.enc.e = ctx->dma_e; + qat_req->in.rsa.enc.n = ctx->dma_n; ret = -ENOMEM; /* @@ -281,16 +731,16 @@ static int qat_rsa_enc(struct akcipher_request *req) */ if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { qat_req->src_align = NULL; - qat_req->in.enc.m = dma_map_single(dev, sg_virt(req->src), + qat_req->in.rsa.enc.m = dma_map_single(dev, sg_virt(req->src), req->src_len, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->in.enc.m))) + if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.enc.m))) return ret; } else { int shift = ctx->key_sz - req->src_len; qat_req->src_align = dma_zalloc_coherent(dev, ctx->key_sz, - &qat_req->in.enc.m, + &qat_req->in.rsa.enc.m, GFP_KERNEL); if (unlikely(!qat_req->src_align)) return ret; @@ -300,30 +750,30 @@ static int qat_rsa_enc(struct akcipher_request *req) } if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { qat_req->dst_align = NULL; - qat_req->out.enc.c = dma_map_single(dev, sg_virt(req->dst), - req->dst_len, - DMA_FROM_DEVICE); + qat_req->out.rsa.enc.c = dma_map_single(dev, sg_virt(req->dst), + req->dst_len, + DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->out.enc.c))) + if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.enc.c))) goto unmap_src; } else { qat_req->dst_align = dma_zalloc_coherent(dev, ctx->key_sz, - &qat_req->out.enc.c, + &qat_req->out.rsa.enc.c, GFP_KERNEL); if (unlikely(!qat_req->dst_align)) goto unmap_src; } - qat_req->in.in_tab[3] = 0; - qat_req->out.out_tab[1] = 0; - qat_req->phy_in = dma_map_single(dev, &qat_req->in.enc.m, + qat_req->in.rsa.in_tab[3] = 0; + qat_req->out.rsa.out_tab[1] = 0; + qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.enc.m, sizeof(struct qat_rsa_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; - qat_req->phy_out = dma_map_single(dev, &qat_req->out.enc.c, + qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.enc.c, sizeof(struct qat_rsa_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) @@ -331,7 +781,7 @@ static int qat_rsa_enc(struct akcipher_request *req) msg->pke_mid.src_data_addr = qat_req->phy_in; msg->pke_mid.dest_data_addr = qat_req->phy_out; - msg->pke_mid.opaque = (uint64_t)(__force long)req; + msg->pke_mid.opaque = (uint64_t)(__force long)qat_req; msg->input_param_count = 3; msg->output_param_count = 1; do { @@ -353,19 +803,19 @@ unmap_in_params: unmap_dst: if (qat_req->dst_align) dma_free_coherent(dev, ctx->key_sz, qat_req->dst_align, - qat_req->out.enc.c); + qat_req->out.rsa.enc.c); else - if (!dma_mapping_error(dev, qat_req->out.enc.c)) - dma_unmap_single(dev, qat_req->out.enc.c, ctx->key_sz, - DMA_FROM_DEVICE); + if (!dma_mapping_error(dev, qat_req->out.rsa.enc.c)) + dma_unmap_single(dev, qat_req->out.rsa.enc.c, + ctx->key_sz, DMA_FROM_DEVICE); unmap_src: if (qat_req->src_align) dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, - qat_req->in.enc.m); + qat_req->in.rsa.enc.m); else - if (!dma_mapping_error(dev, qat_req->in.enc.m)) - dma_unmap_single(dev, qat_req->in.enc.m, ctx->key_sz, - DMA_TO_DEVICE); + if (!dma_mapping_error(dev, qat_req->in.rsa.enc.m)) + dma_unmap_single(dev, qat_req->in.rsa.enc.m, + ctx->key_sz, DMA_TO_DEVICE); return ret; } @@ -375,7 +825,7 @@ static int qat_rsa_dec(struct akcipher_request *req) struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); - struct qat_rsa_request *qat_req = + struct qat_asym_request *qat_req = PTR_ALIGN(akcipher_request_ctx(req), 64); struct icp_qat_fw_pke_request *msg = &qat_req->req; int ret, ctr = 0; @@ -390,18 +840,30 @@ static int qat_rsa_dec(struct akcipher_request *req) memset(msg, '\0', sizeof(*msg)); ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, ICP_QAT_FW_COMN_REQ_FLAG_SET); - msg->pke_hdr.cd_pars.func_id = qat_rsa_dec_fn_id(ctx->key_sz); + msg->pke_hdr.cd_pars.func_id = ctx->crt_mode ? + qat_rsa_dec_fn_id_crt(ctx->key_sz) : + qat_rsa_dec_fn_id(ctx->key_sz); if (unlikely(!msg->pke_hdr.cd_pars.func_id)) return -EINVAL; - qat_req->ctx = ctx; + qat_req->cb = qat_rsa_cb; + qat_req->ctx.rsa = ctx; + qat_req->areq.rsa = req; msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; msg->pke_hdr.comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, QAT_COMN_CD_FLD_TYPE_64BIT_ADR); - qat_req->in.dec.d = ctx->dma_d; - qat_req->in.dec.n = ctx->dma_n; + if (ctx->crt_mode) { + qat_req->in.rsa.dec_crt.p = ctx->dma_p; + qat_req->in.rsa.dec_crt.q = ctx->dma_q; + qat_req->in.rsa.dec_crt.dp = ctx->dma_dp; + qat_req->in.rsa.dec_crt.dq = ctx->dma_dq; + qat_req->in.rsa.dec_crt.qinv = ctx->dma_qinv; + } else { + qat_req->in.rsa.dec.d = ctx->dma_d; + qat_req->in.rsa.dec.n = ctx->dma_n; + } ret = -ENOMEM; /* @@ -413,16 +875,16 @@ static int qat_rsa_dec(struct akcipher_request *req) */ if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { qat_req->src_align = NULL; - qat_req->in.dec.c = dma_map_single(dev, sg_virt(req->src), + qat_req->in.rsa.dec.c = dma_map_single(dev, sg_virt(req->src), req->dst_len, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->in.dec.c))) + if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.dec.c))) return ret; } else { int shift = ctx->key_sz - req->src_len; qat_req->src_align = dma_zalloc_coherent(dev, ctx->key_sz, - &qat_req->in.dec.c, + &qat_req->in.rsa.dec.c, GFP_KERNEL); if (unlikely(!qat_req->src_align)) return ret; @@ -432,31 +894,34 @@ static int qat_rsa_dec(struct akcipher_request *req) } if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { qat_req->dst_align = NULL; - qat_req->out.dec.m = dma_map_single(dev, sg_virt(req->dst), + qat_req->out.rsa.dec.m = dma_map_single(dev, sg_virt(req->dst), req->dst_len, DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, qat_req->out.dec.m))) + if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.dec.m))) goto unmap_src; } else { qat_req->dst_align = dma_zalloc_coherent(dev, ctx->key_sz, - &qat_req->out.dec.m, + &qat_req->out.rsa.dec.m, GFP_KERNEL); if (unlikely(!qat_req->dst_align)) goto unmap_src; } - qat_req->in.in_tab[3] = 0; - qat_req->out.out_tab[1] = 0; - qat_req->phy_in = dma_map_single(dev, &qat_req->in.dec.c, + if (ctx->crt_mode) + qat_req->in.rsa.in_tab[6] = 0; + else + qat_req->in.rsa.in_tab[3] = 0; + qat_req->out.rsa.out_tab[1] = 0; + qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.dec.c, sizeof(struct qat_rsa_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; - qat_req->phy_out = dma_map_single(dev, &qat_req->out.dec.m, + qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.dec.m, sizeof(struct qat_rsa_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) @@ -464,8 +929,12 @@ static int qat_rsa_dec(struct akcipher_request *req) msg->pke_mid.src_data_addr = qat_req->phy_in; msg->pke_mid.dest_data_addr = qat_req->phy_out; - msg->pke_mid.opaque = (uint64_t)(__force long)req; - msg->input_param_count = 3; + msg->pke_mid.opaque = (uint64_t)(__force long)qat_req; + if (ctx->crt_mode) + msg->input_param_count = 6; + else + msg->input_param_count = 3; + msg->output_param_count = 1; do { ret = adf_send_message(ctx->inst->pke_tx, (uint32_t *)msg); @@ -486,26 +955,24 @@ unmap_in_params: unmap_dst: if (qat_req->dst_align) dma_free_coherent(dev, ctx->key_sz, qat_req->dst_align, - qat_req->out.dec.m); + qat_req->out.rsa.dec.m); else - if (!dma_mapping_error(dev, qat_req->out.dec.m)) - dma_unmap_single(dev, qat_req->out.dec.m, ctx->key_sz, - DMA_FROM_DEVICE); + if (!dma_mapping_error(dev, qat_req->out.rsa.dec.m)) + dma_unmap_single(dev, qat_req->out.rsa.dec.m, + ctx->key_sz, DMA_FROM_DEVICE); unmap_src: if (qat_req->src_align) dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, - qat_req->in.dec.c); + qat_req->in.rsa.dec.c); else - if (!dma_mapping_error(dev, qat_req->in.dec.c)) - dma_unmap_single(dev, qat_req->in.dec.c, ctx->key_sz, - DMA_TO_DEVICE); + if (!dma_mapping_error(dev, qat_req->in.rsa.dec.c)) + dma_unmap_single(dev, qat_req->in.rsa.dec.c, + ctx->key_sz, DMA_TO_DEVICE); return ret; } -int qat_rsa_get_n(void *context, size_t hdrlen, unsigned char tag, - const void *value, size_t vlen) +int qat_rsa_set_n(struct qat_rsa_ctx *ctx, const char *value, size_t vlen) { - struct qat_rsa_ctx *ctx = context; struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); const char *ptr = value; @@ -518,11 +985,6 @@ int qat_rsa_get_n(void *context, size_t hdrlen, unsigned char tag, ctx->key_sz = vlen; ret = -EINVAL; - /* In FIPS mode only allow key size 2K & 3K */ - if (fips_enabled && (ctx->key_sz != 256 && ctx->key_sz != 384)) { - pr_err("QAT: RSA: key size not allowed in FIPS mode\n"); - goto err; - } /* invalid key size provided */ if (!qat_rsa_enc_fn_id(ctx->key_sz)) goto err; @@ -540,10 +1002,8 @@ err: return ret; } -int qat_rsa_get_e(void *context, size_t hdrlen, unsigned char tag, - const void *value, size_t vlen) +int qat_rsa_set_e(struct qat_rsa_ctx *ctx, const char *value, size_t vlen) { - struct qat_rsa_ctx *ctx = context; struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); const char *ptr = value; @@ -559,18 +1019,15 @@ int qat_rsa_get_e(void *context, size_t hdrlen, unsigned char tag, } ctx->e = dma_zalloc_coherent(dev, ctx->key_sz, &ctx->dma_e, GFP_KERNEL); - if (!ctx->e) { - ctx->e = NULL; + if (!ctx->e) return -ENOMEM; - } + memcpy(ctx->e + (ctx->key_sz - vlen), ptr, vlen); return 0; } -int qat_rsa_get_d(void *context, size_t hdrlen, unsigned char tag, - const void *value, size_t vlen) +int qat_rsa_set_d(struct qat_rsa_ctx *ctx, const char *value, size_t vlen) { - struct qat_rsa_ctx *ctx = context; struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); const char *ptr = value; @@ -585,12 +1042,6 @@ int qat_rsa_get_d(void *context, size_t hdrlen, unsigned char tag, if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) goto err; - /* In FIPS mode only allow key size 2K & 3K */ - if (fips_enabled && (vlen != 256 && vlen != 384)) { - pr_err("QAT: RSA: key size not allowed in FIPS mode\n"); - goto err; - } - ret = -ENOMEM; ctx->d = dma_zalloc_coherent(dev, ctx->key_sz, &ctx->dma_d, GFP_KERNEL); if (!ctx->d) @@ -603,12 +1054,106 @@ err: return ret; } -static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, - unsigned int keylen, bool private) +static void qat_rsa_drop_leading_zeros(const char **ptr, unsigned int *len) { - struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct device *dev = &GET_DEV(ctx->inst->accel_dev); - int ret; + while (!**ptr && *len) { + (*ptr)++; + (*len)--; + } +} + +static void qat_rsa_setkey_crt(struct qat_rsa_ctx *ctx, struct rsa_key *rsa_key) +{ + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr; + unsigned int len; + unsigned int half_key_sz = ctx->key_sz / 2; + + /* p */ + ptr = rsa_key->p; + len = rsa_key->p_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto err; + ctx->p = dma_zalloc_coherent(dev, half_key_sz, &ctx->dma_p, GFP_KERNEL); + if (!ctx->p) + goto err; + memcpy(ctx->p + (half_key_sz - len), ptr, len); + + /* q */ + ptr = rsa_key->q; + len = rsa_key->q_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_p; + ctx->q = dma_zalloc_coherent(dev, half_key_sz, &ctx->dma_q, GFP_KERNEL); + if (!ctx->q) + goto free_p; + memcpy(ctx->q + (half_key_sz - len), ptr, len); + + /* dp */ + ptr = rsa_key->dp; + len = rsa_key->dp_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_q; + ctx->dp = dma_zalloc_coherent(dev, half_key_sz, &ctx->dma_dp, + GFP_KERNEL); + if (!ctx->dp) + goto free_q; + memcpy(ctx->dp + (half_key_sz - len), ptr, len); + + /* dq */ + ptr = rsa_key->dq; + len = rsa_key->dq_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_dp; + ctx->dq = dma_zalloc_coherent(dev, half_key_sz, &ctx->dma_dq, + GFP_KERNEL); + if (!ctx->dq) + goto free_dp; + memcpy(ctx->dq + (half_key_sz - len), ptr, len); + + /* qinv */ + ptr = rsa_key->qinv; + len = rsa_key->qinv_sz; + qat_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_dq; + ctx->qinv = dma_zalloc_coherent(dev, half_key_sz, &ctx->dma_qinv, + GFP_KERNEL); + if (!ctx->qinv) + goto free_dq; + memcpy(ctx->qinv + (half_key_sz - len), ptr, len); + + ctx->crt_mode = true; + return; + +free_dq: + memset(ctx->dq, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq); + ctx->dq = NULL; +free_dp: + memset(ctx->dp, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp); + ctx->dp = NULL; +free_q: + memset(ctx->q, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q); + ctx->q = NULL; +free_p: + memset(ctx->p, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p); + ctx->p = NULL; +err: + ctx->crt_mode = false; +} + +static void qat_rsa_clear_ctx(struct device *dev, struct qat_rsa_ctx *ctx) +{ + unsigned int half_key_sz = ctx->key_sz / 2; /* Free the old key if any */ if (ctx->n) @@ -619,19 +1164,68 @@ static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, memset(ctx->d, '\0', ctx->key_sz); dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); } + if (ctx->p) { + memset(ctx->p, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->p, ctx->dma_p); + } + if (ctx->q) { + memset(ctx->q, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->q, ctx->dma_q); + } + if (ctx->dp) { + memset(ctx->dp, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dp, ctx->dma_dp); + } + if (ctx->dq) { + memset(ctx->dq, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->dq, ctx->dma_dq); + } + if (ctx->qinv) { + memset(ctx->qinv, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz, ctx->qinv, ctx->dma_qinv); + } ctx->n = NULL; ctx->e = NULL; ctx->d = NULL; + ctx->p = NULL; + ctx->q = NULL; + ctx->dp = NULL; + ctx->dq = NULL; + ctx->qinv = NULL; + ctx->crt_mode = false; + ctx->key_sz = 0; +} + +static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, bool private) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + struct rsa_key rsa_key; + int ret; + + qat_rsa_clear_ctx(dev, ctx); if (private) - ret = asn1_ber_decoder(&qat_rsaprivkey_decoder, ctx, key, - keylen); + ret = rsa_parse_priv_key(&rsa_key, key, keylen); else - ret = asn1_ber_decoder(&qat_rsapubkey_decoder, ctx, key, - keylen); + ret = rsa_parse_pub_key(&rsa_key, key, keylen); + if (ret < 0) + goto free; + + ret = qat_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz); if (ret < 0) goto free; + ret = qat_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); + if (ret < 0) + goto free; + if (private) { + ret = qat_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); + if (ret < 0) + goto free; + qat_rsa_setkey_crt(ctx, &rsa_key); + } if (!ctx->n || !ctx->e) { /* invalid key provided */ @@ -646,20 +1240,7 @@ static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, return 0; free: - if (ctx->d) { - memset(ctx->d, '\0', ctx->key_sz); - dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); - ctx->d = NULL; - } - if (ctx->e) { - dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); - ctx->e = NULL; - } - if (ctx->n) { - dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); - ctx->n = NULL; - ctx->key_sz = 0; - } + qat_rsa_clear_ctx(dev, ctx); return ret; } @@ -725,7 +1306,7 @@ static struct akcipher_alg rsa = { .max_size = qat_rsa_max_size, .init = qat_rsa_init_tfm, .exit = qat_rsa_exit_tfm, - .reqsize = sizeof(struct qat_rsa_request) + 64, + .reqsize = sizeof(struct qat_asym_request) + 64, .base = { .cra_name = "rsa", .cra_driver_name = "qat-rsa", @@ -735,6 +1316,23 @@ static struct akcipher_alg rsa = { }, }; +static struct kpp_alg dh = { + .set_secret = qat_dh_set_secret, + .generate_public_key = qat_dh_compute_value, + .compute_shared_secret = qat_dh_compute_value, + .max_size = qat_dh_max_size, + .init = qat_dh_init_tfm, + .exit = qat_dh_exit_tfm, + .reqsize = sizeof(struct qat_asym_request) + 64, + .base = { + .cra_name = "dh", + .cra_driver_name = "qat-dh", + .cra_priority = 1000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct qat_dh_ctx), + }, +}; + int qat_asym_algs_register(void) { int ret = 0; @@ -743,7 +1341,11 @@ int qat_asym_algs_register(void) if (++active_devs == 1) { rsa.base.cra_flags = 0; ret = crypto_register_akcipher(&rsa); + if (ret) + goto unlock; + ret = crypto_register_kpp(&dh); } +unlock: mutex_unlock(&algs_lock); return ret; } @@ -751,7 +1353,9 @@ int qat_asym_algs_register(void) void qat_asym_algs_unregister(void) { mutex_lock(&algs_lock); - if (--active_devs == 0) + if (--active_devs == 0) { crypto_unregister_akcipher(&rsa); + crypto_unregister_kpp(&dh); + } mutex_unlock(&algs_lock); } diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 6e1d5e185..1dfcab317 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -252,6 +252,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->reset_device = adf_reset_sbr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; } diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c index dbcbbe242..b04b42f48 100644 --- a/drivers/crypto/qce/ablkcipher.c +++ b/drivers/crypto/qce/ablkcipher.c @@ -15,8 +15,8 @@ #include #include #include -#include #include +#include #include "cipher.h" @@ -189,7 +189,7 @@ static int qce_ablkcipher_setkey(struct crypto_ablkcipher *ablk, const u8 *key, memcpy(ctx->enc_key, key, keylen); return 0; fallback: - ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen); + ret = crypto_skcipher_setkey(ctx->fallback, key, keylen); if (!ret) ctx->enc_keylen = keylen; return ret; @@ -212,10 +212,16 @@ static int qce_ablkcipher_crypt(struct ablkcipher_request *req, int encrypt) if (IS_AES(rctx->flags) && ctx->enc_keylen != AES_KEYSIZE_128 && ctx->enc_keylen != AES_KEYSIZE_256) { - ablkcipher_request_set_tfm(req, ctx->fallback); - ret = encrypt ? crypto_ablkcipher_encrypt(req) : - crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + ret = encrypt ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); return ret; } @@ -239,10 +245,9 @@ static int qce_ablkcipher_init(struct crypto_tfm *tfm) memset(ctx, 0, sizeof(*ctx)); tfm->crt_ablkcipher.reqsize = sizeof(struct qce_cipher_reqctx); - ctx->fallback = crypto_alloc_ablkcipher(crypto_tfm_alg_name(tfm), - CRYPTO_ALG_TYPE_ABLKCIPHER, - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK); + ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(tfm), 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->fallback)) return PTR_ERR(ctx->fallback); @@ -253,7 +258,7 @@ static void qce_ablkcipher_exit(struct crypto_tfm *tfm) { struct qce_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_ablkcipher(ctx->fallback); + crypto_free_skcipher(ctx->fallback); } struct qce_ablkcipher_def { diff --git a/drivers/crypto/qce/cipher.h b/drivers/crypto/qce/cipher.h index 5c6a5f863..2b0278bb6 100644 --- a/drivers/crypto/qce/cipher.h +++ b/drivers/crypto/qce/cipher.h @@ -22,7 +22,7 @@ struct qce_cipher_ctx { u8 enc_key[QCE_MAX_KEY_SIZE]; unsigned int enc_keylen; - struct crypto_ablkcipher *fallback; + struct crypto_skcipher *fallback; }; /** diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 2b3a0cfe3..dce1af0ce 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -155,43 +155,43 @@ * expansion of its usage. */ struct samsung_aes_variant { - unsigned int aes_offset; + unsigned int aes_offset; }; struct s5p_aes_reqctx { - unsigned long mode; + unsigned long mode; }; struct s5p_aes_ctx { - struct s5p_aes_dev *dev; + struct s5p_aes_dev *dev; - uint8_t aes_key[AES_MAX_KEY_SIZE]; - uint8_t nonce[CTR_RFC3686_NONCE_SIZE]; - int keylen; + uint8_t aes_key[AES_MAX_KEY_SIZE]; + uint8_t nonce[CTR_RFC3686_NONCE_SIZE]; + int keylen; }; struct s5p_aes_dev { - struct device *dev; - struct clk *clk; - void __iomem *ioaddr; - void __iomem *aes_ioaddr; - int irq_fc; + struct device *dev; + struct clk *clk; + void __iomem *ioaddr; + void __iomem *aes_ioaddr; + int irq_fc; - struct ablkcipher_request *req; - struct s5p_aes_ctx *ctx; - struct scatterlist *sg_src; - struct scatterlist *sg_dst; + struct ablkcipher_request *req; + struct s5p_aes_ctx *ctx; + struct scatterlist *sg_src; + struct scatterlist *sg_dst; /* In case of unaligned access: */ - struct scatterlist *sg_src_cpy; - struct scatterlist *sg_dst_cpy; + struct scatterlist *sg_src_cpy; + struct scatterlist *sg_dst_cpy; - struct tasklet_struct tasklet; - struct crypto_queue queue; - bool busy; - spinlock_t lock; + struct tasklet_struct tasklet; + struct crypto_queue queue; + bool busy; + spinlock_t lock; - struct samsung_aes_variant *variant; + struct samsung_aes_variant *variant; }; static struct s5p_aes_dev *s5p_dev; @@ -421,11 +421,11 @@ static bool s5p_aes_rx(struct s5p_aes_dev *dev) static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) { struct platform_device *pdev = dev_id; - struct s5p_aes_dev *dev = platform_get_drvdata(pdev); - uint32_t status; - unsigned long flags; - bool set_dma_tx = false; - bool set_dma_rx = false; + struct s5p_aes_dev *dev = platform_get_drvdata(pdev); + bool set_dma_tx = false; + bool set_dma_rx = false; + unsigned long flags; + uint32_t status; spin_lock_irqsave(&dev->lock, flags); @@ -538,10 +538,10 @@ static int s5p_set_outdata_start(struct s5p_aes_dev *dev, static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode) { - struct ablkcipher_request *req = dev->req; - uint32_t aes_control; - int err; - unsigned long flags; + struct ablkcipher_request *req = dev->req; + uint32_t aes_control; + unsigned long flags; + int err; aes_control = SSS_AES_KEY_CHANGE_MODE; if (mode & FLAGS_AES_DECRYPT) @@ -653,10 +653,10 @@ exit: static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); - struct s5p_aes_reqctx *reqctx = ablkcipher_request_ctx(req); - struct s5p_aes_dev *dev = ctx->dev; + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct s5p_aes_reqctx *reqctx = ablkcipher_request_ctx(req); + struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct s5p_aes_dev *dev = ctx->dev; if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { dev_err(dev->dev, "request size is not exact amount of AES blocks\n"); @@ -671,7 +671,7 @@ static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode) static int s5p_aes_setkey(struct crypto_ablkcipher *cipher, const uint8_t *key, unsigned int keylen) { - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm); if (keylen != AES_KEYSIZE_128 && @@ -763,11 +763,11 @@ static struct crypto_alg algs[] = { static int s5p_aes_probe(struct platform_device *pdev) { - int i, j, err = -ENODEV; - struct s5p_aes_dev *pdata; - struct device *dev = &pdev->dev; - struct resource *res; + struct device *dev = &pdev->dev; + int i, j, err = -ENODEV; struct samsung_aes_variant *variant; + struct s5p_aes_dev *pdata; + struct resource *res; if (s5p_dev) return -EEXIST; diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index c3f3d89e4..0c49956ee 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -14,10 +14,9 @@ * Based on omap-aes.c and tegra-aes.c */ -#include #include -#include #include +#include #include #include @@ -150,10 +149,7 @@ struct sahara_ctx { /* AES-specific context */ int keylen; u8 key[AES_KEYSIZE_128]; - struct crypto_ablkcipher *fallback; - - /* SHA-specific context */ - struct crypto_shash *shash_fallback; + struct crypto_skcipher *fallback; }; struct sahara_aes_reqctx { @@ -620,25 +616,21 @@ static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, return 0; } - if (keylen != AES_KEYSIZE_128 && - keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) + if (keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) return -EINVAL; /* * The requested key size is not supported by HW, do a fallback. */ - ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - ctx->fallback->base.crt_flags |= - (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); - ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen); - if (ret) { - struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm); + ret = crypto_skcipher_setkey(ctx->fallback, key, keylen); - tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm_aux->crt_flags |= - (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK); - } + tfm->base.crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm->base.crt_flags |= crypto_skcipher_get_flags(ctx->fallback) & + CRYPTO_TFM_RES_MASK; return ret; } @@ -670,16 +662,20 @@ static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode) static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct sahara_ctx *ctx = crypto_ablkcipher_ctx( crypto_ablkcipher_reqtfm(req)); int err; if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_encrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + err = crypto_skcipher_encrypt(subreq); + skcipher_request_zero(subreq); return err; } @@ -688,16 +684,20 @@ static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req) static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct sahara_ctx *ctx = crypto_ablkcipher_ctx( crypto_ablkcipher_reqtfm(req)); int err; if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + err = crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); return err; } @@ -706,16 +706,20 @@ static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req) static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct sahara_ctx *ctx = crypto_ablkcipher_ctx( crypto_ablkcipher_reqtfm(req)); int err; if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_encrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + err = crypto_skcipher_encrypt(subreq); + skcipher_request_zero(subreq); return err; } @@ -724,16 +728,20 @@ static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req) static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req) { - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); struct sahara_ctx *ctx = crypto_ablkcipher_ctx( crypto_ablkcipher_reqtfm(req)); int err; if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->nbytes, req->info); + err = crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); return err; } @@ -745,8 +753,9 @@ static int sahara_aes_cra_init(struct crypto_tfm *tfm) const char *name = crypto_tfm_alg_name(tfm); struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->fallback = crypto_alloc_ablkcipher(name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + ctx->fallback = crypto_alloc_skcipher(name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->fallback)) { pr_err("Error allocating fallback algo %s\n", name); return PTR_ERR(ctx->fallback); @@ -761,9 +770,7 @@ static void sahara_aes_cra_exit(struct crypto_tfm *tfm) { struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->fallback) - crypto_free_ablkcipher(ctx->fallback); - ctx->fallback = NULL; + crypto_free_skcipher(ctx->fallback); } static u32 sahara_sha_init_hdr(struct sahara_dev *dev, @@ -1180,15 +1187,6 @@ static int sahara_sha_import(struct ahash_request *req, const void *in) static int sahara_sha_cra_init(struct crypto_tfm *tfm) { - const char *name = crypto_tfm_alg_name(tfm); - struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->shash_fallback = crypto_alloc_shash(name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->shash_fallback)) { - pr_err("Error allocating fallback algo %s\n", name); - return PTR_ERR(ctx->shash_fallback); - } crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct sahara_sha_reqctx) + SHA_BUFFER_LEN + SHA256_BLOCK_SIZE); @@ -1196,14 +1194,6 @@ static int sahara_sha_cra_init(struct crypto_tfm *tfm) return 0; } -static void sahara_sha_cra_exit(struct crypto_tfm *tfm) -{ - struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto_free_shash(ctx->shash_fallback); - ctx->shash_fallback = NULL; -} - static struct crypto_alg aes_algs[] = { { .cra_name = "ecb(aes)", @@ -1272,7 +1262,6 @@ static struct ahash_alg sha_v3_algs[] = { .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = sahara_sha_cra_init, - .cra_exit = sahara_sha_cra_exit, } }, }; @@ -1300,7 +1289,6 @@ static struct ahash_alg sha_v4_algs[] = { .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = sahara_sha_cra_init, - .cra_exit = sahara_sha_cra_exit, } }, }; diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index b7ee8d301..0418a2f41 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -91,10 +91,17 @@ static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, return be16_to_cpu(ptr->len); } -static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1) +static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, + bool is_sec1) { if (!is_sec1) - ptr->j_extent = 0; + ptr->j_extent = val; +} + +static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1) +{ + if (!is_sec1) + ptr->j_extent |= val; } /* @@ -111,7 +118,7 @@ static void map_single_talitos_ptr(struct device *dev, to_talitos_ptr_len(ptr, len, is_sec1); to_talitos_ptr(ptr, dma_addr, is_sec1); - to_talitos_ptr_extent_clear(ptr, is_sec1); + to_talitos_ptr_ext_set(ptr, 0, is_sec1); } /* @@ -804,6 +811,11 @@ static void talitos_unregister_rng(struct device *dev) * crypto alg */ #define TALITOS_CRA_PRIORITY 3000 +/* + * Defines a priority for doing AEAD with descriptors type + * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP + */ +#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) #define TALITOS_MAX_KEY_SIZE 96 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ @@ -904,35 +916,59 @@ struct talitos_edesc { static void talitos_sg_unmap(struct device *dev, struct talitos_edesc *edesc, struct scatterlist *src, - struct scatterlist *dst) + struct scatterlist *dst, + unsigned int len, unsigned int offset) { + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); unsigned int src_nents = edesc->src_nents ? : 1; unsigned int dst_nents = edesc->dst_nents ? : 1; + if (is_sec1 && dst && dst_nents > 1) { + dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset, + len, DMA_FROM_DEVICE); + sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len, + offset); + } if (src != dst) { - dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + if (src_nents == 1 || !is_sec1) + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); - if (dst) { + if (dst && (dst_nents == 1 || !is_sec1)) dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); - } - } else + } else if (src_nents == 1 || !is_sec1) { dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); + } } static void ipsec_esp_unmap(struct device *dev, struct talitos_edesc *edesc, struct aead_request *areq) { - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE); + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct talitos_ctx *ctx = crypto_aead_ctx(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + + if (edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) + unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], + DMA_FROM_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); - talitos_sg_unmap(dev, edesc, areq->src, areq->dst); + talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, + areq->assoclen); if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); + + if (!(edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)) { + unsigned int dst_nents = edesc->dst_nents ? : 1; + + sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, + areq->assoclen + areq->cryptlen - ivsize); + } } /* @@ -942,6 +978,8 @@ static void ipsec_esp_encrypt_done(struct device *dev, struct talitos_desc *desc, void *context, int err) { + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); struct aead_request *areq = context; struct crypto_aead *authenc = crypto_aead_reqtfm(areq); unsigned int authsize = crypto_aead_authsize(authenc); @@ -955,8 +993,11 @@ static void ipsec_esp_encrypt_done(struct device *dev, /* copy the generated ICV to dst */ if (edesc->icv_ool) { - icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; + if (is_sec1) + icvdata = edesc->buf + areq->assoclen + areq->cryptlen; + else + icvdata = &edesc->link_tbl[edesc->src_nents + + edesc->dst_nents + 2]; sg = sg_last(areq->dst, edesc->dst_nents); memcpy((char *)sg_virt(sg) + sg->length - authsize, icvdata, authsize); @@ -977,6 +1018,8 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, struct talitos_edesc *edesc; struct scatterlist *sg; char *oicv, *icv; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); edesc = container_of(desc, struct talitos_edesc, desc); @@ -988,7 +1031,12 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, icv = (char *)sg_virt(sg) + sg->length - authsize; if (edesc->dma_len) { - oicv = (char *)&edesc->link_tbl[edesc->src_nents + + if (is_sec1) + oicv = (char *)&edesc->dma_link_tbl + + req->assoclen + req->cryptlen; + else + oicv = (char *) + &edesc->link_tbl[edesc->src_nents + edesc->dst_nents + 2]; if (edesc->icv_ool) icv = oicv + authsize; @@ -1050,8 +1098,8 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, to_talitos_ptr(link_tbl_ptr + count, sg_dma_address(sg) + offset, 0); - link_tbl_ptr[count].len = cpu_to_be16(len); - link_tbl_ptr[count].j_extent = 0; + to_talitos_ptr_len(link_tbl_ptr + count, len, 0); + to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); count++; cryptlen -= len; offset = 0; @@ -1062,17 +1110,43 @@ next: /* tag end of link table */ if (count > 0) - link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN; + to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, + DESC_PTR_LNKTBL_RETURN, 0); return count; } -static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count, - int cryptlen, - struct talitos_ptr *link_tbl_ptr) +int talitos_sg_map(struct device *dev, struct scatterlist *src, + unsigned int len, struct talitos_edesc *edesc, + struct talitos_ptr *ptr, + int sg_count, unsigned int offset, int tbl_off) { - return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen, - link_tbl_ptr); + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + + to_talitos_ptr_len(ptr, len, is_sec1); + to_talitos_ptr_ext_set(ptr, 0, is_sec1); + + if (sg_count == 1) { + to_talitos_ptr(ptr, sg_dma_address(src) + offset, is_sec1); + return sg_count; + } + if (is_sec1) { + to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, is_sec1); + return sg_count; + } + sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, + &edesc->link_tbl[tbl_off]); + if (sg_count == 1) { + /* Only one segment now, so no link tbl needed*/ + copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); + return sg_count; + } + to_talitos_ptr(ptr, edesc->dma_link_tbl + + tbl_off * sizeof(struct talitos_ptr), is_sec1); + to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); + + return sg_count; } /* @@ -1093,42 +1167,52 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, int tbl_off = 0; int sg_count, ret; int sg_link_tbl_len; + bool sync_needed = false; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); /* hmac key */ map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, DMA_TO_DEVICE); - sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1, - (areq->src == areq->dst) ? DMA_BIDIRECTIONAL - : DMA_TO_DEVICE); - /* hmac data */ - desc->ptr[1].len = cpu_to_be16(areq->assoclen); - if (sg_count > 1 && - (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0, - areq->assoclen, - &edesc->link_tbl[tbl_off])) > 1) { - to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off * - sizeof(struct talitos_ptr), 0); - desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP; + sg_count = edesc->src_nents ?: 1; + if (is_sec1 && sg_count > 1) + sg_copy_to_buffer(areq->src, sg_count, edesc->buf, + areq->assoclen + cryptlen); + else + sg_count = dma_map_sg(dev, areq->src, sg_count, + (areq->src == areq->dst) ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE); - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - edesc->dma_len, DMA_BIDIRECTIONAL); + /* hmac data */ + ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc, + &desc->ptr[1], sg_count, 0, tbl_off); + if (ret > 1) { tbl_off += ret; - } else { - to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0); - desc->ptr[1].j_extent = 0; + sync_needed = true; } /* cipher iv */ - to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0); - desc->ptr[2].len = cpu_to_be16(ivsize); - desc->ptr[2].j_extent = 0; + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { + to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, is_sec1); + to_talitos_ptr_len(&desc->ptr[2], ivsize, is_sec1); + to_talitos_ptr_ext_set(&desc->ptr[2], 0, is_sec1); + } else { + to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, is_sec1); + to_talitos_ptr_len(&desc->ptr[3], ivsize, is_sec1); + to_talitos_ptr_ext_set(&desc->ptr[3], 0, is_sec1); + } /* cipher key */ - map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, - (char *)&ctx->key + ctx->authkeylen, - DMA_TO_DEVICE); + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, + (char *)&ctx->key + ctx->authkeylen, + DMA_TO_DEVICE); + else + map_single_talitos_ptr(dev, &desc->ptr[2], ctx->enckeylen, + (char *)&ctx->key + ctx->authkeylen, + DMA_TO_DEVICE); /* * cipher in @@ -1136,78 +1220,82 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, * extent is bytes of HMAC postpended to ciphertext, * typically 12 for ipsec */ - desc->ptr[4].len = cpu_to_be16(cryptlen); - desc->ptr[4].j_extent = authsize; + to_talitos_ptr_len(&desc->ptr[4], cryptlen, is_sec1); + to_talitos_ptr_ext_set(&desc->ptr[4], 0, is_sec1); sg_link_tbl_len = cryptlen; - if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) - sg_link_tbl_len += authsize; - if (sg_count == 1) { - to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src) + - areq->assoclen, 0); - } else if ((ret = sg_to_link_tbl_offset(areq->src, sg_count, - areq->assoclen, sg_link_tbl_len, - &edesc->link_tbl[tbl_off])) > - 1) { - desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; - to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl + - tbl_off * - sizeof(struct talitos_ptr), 0); - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - edesc->dma_len, - DMA_BIDIRECTIONAL); - tbl_off += ret; - } else { - copy_talitos_ptr(&desc->ptr[4], &edesc->link_tbl[tbl_off], 0); + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { + to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1); + + if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) + sg_link_tbl_len += authsize; } - /* cipher out */ - desc->ptr[5].len = cpu_to_be16(cryptlen); - desc->ptr[5].j_extent = authsize; + sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc, + &desc->ptr[4], sg_count, areq->assoclen, + tbl_off); - if (areq->src != areq->dst) - sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1, - DMA_FROM_DEVICE); + if (sg_count > 1) { + tbl_off += sg_count; + sync_needed = true; + } - edesc->icv_ool = false; + /* cipher out */ + if (areq->src != areq->dst) { + sg_count = edesc->dst_nents ? : 1; + if (!is_sec1 || sg_count == 1) + dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); + } - if (sg_count == 1) { - to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst) + - areq->assoclen, 0); - } else if ((sg_count = - sg_to_link_tbl_offset(areq->dst, sg_count, - areq->assoclen, cryptlen, - &edesc->link_tbl[tbl_off])) > 1) { - struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; - - to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl + - tbl_off * sizeof(struct talitos_ptr), 0); - - /* Add an entry to the link table for ICV data */ - tbl_ptr += sg_count - 1; - tbl_ptr->j_extent = 0; - tbl_ptr++; - tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; - tbl_ptr->len = cpu_to_be16(authsize); - - /* icv data follows link tables */ - to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + - (edesc->src_nents + edesc->dst_nents + - 2) * sizeof(struct talitos_ptr) + - authsize, 0); - desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP; - dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, - edesc->dma_len, DMA_BIDIRECTIONAL); + sg_count = talitos_sg_map(dev, areq->dst, cryptlen, edesc, + &desc->ptr[5], sg_count, areq->assoclen, + tbl_off); + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); + + if (sg_count > 1) { edesc->icv_ool = true; + sync_needed = true; + + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { + struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; + int offset = (edesc->src_nents + edesc->dst_nents + 2) * + sizeof(struct talitos_ptr) + authsize; + + /* Add an entry to the link table for ICV data */ + tbl_ptr += sg_count - 1; + to_talitos_ptr_ext_set(tbl_ptr, 0, is_sec1); + tbl_ptr++; + to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN, + is_sec1); + to_talitos_ptr_len(tbl_ptr, authsize, is_sec1); + + /* icv data follows link tables */ + to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset, + is_sec1); + } } else { - copy_talitos_ptr(&desc->ptr[5], &edesc->link_tbl[tbl_off], 0); + edesc->icv_ool = false; + } + + /* ICV data */ + if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) { + to_talitos_ptr_len(&desc->ptr[6], authsize, is_sec1); + to_talitos_ptr(&desc->ptr[6], edesc->dma_link_tbl + + areq->assoclen + cryptlen, is_sec1); } /* iv out */ - map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, - DMA_FROM_DEVICE); + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, + DMA_FROM_DEVICE); + + if (sync_needed) + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, + DMA_BIDIRECTIONAL); ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { @@ -1233,7 +1321,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, bool encrypt) { struct talitos_edesc *edesc; - int src_nents, dst_nents, alloc_len, dma_len; + int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len; dma_addr_t iv_dma = 0; gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1251,8 +1339,8 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); if (!dst || dst == src) { - src_nents = sg_nents_for_len(src, - assoclen + cryptlen + authsize); + src_len = assoclen + cryptlen + authsize; + src_nents = sg_nents_for_len(src, src_len); if (src_nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); err = ERR_PTR(-EINVAL); @@ -1260,17 +1348,18 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, } src_nents = (src_nents == 1) ? 0 : src_nents; dst_nents = dst ? src_nents : 0; + dst_len = 0; } else { /* dst && dst != src*/ - src_nents = sg_nents_for_len(src, assoclen + cryptlen + - (encrypt ? 0 : authsize)); + src_len = assoclen + cryptlen + (encrypt ? 0 : authsize); + src_nents = sg_nents_for_len(src, src_len); if (src_nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); err = ERR_PTR(-EINVAL); goto error_sg; } src_nents = (src_nents == 1) ? 0 : src_nents; - dst_nents = sg_nents_for_len(dst, assoclen + cryptlen + - (encrypt ? authsize : 0)); + dst_len = assoclen + cryptlen + (encrypt ? authsize : 0); + dst_nents = sg_nents_for_len(dst, dst_len); if (dst_nents < 0) { dev_err(dev, "Invalid number of dst SG.\n"); err = ERR_PTR(-EINVAL); @@ -1287,8 +1376,8 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, alloc_len = sizeof(struct talitos_edesc); if (src_nents || dst_nents) { if (is_sec1) - dma_len = (src_nents ? cryptlen : 0) + - (dst_nents ? cryptlen : 0); + dma_len = (src_nents ? src_len : 0) + + (dst_nents ? dst_len : 0); else dma_len = (src_nents + dst_nents + 2) * sizeof(struct talitos_ptr) + authsize * 2; @@ -1412,40 +1501,13 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, return 0; } -static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src, - struct scatterlist *dst, unsigned int len, - struct talitos_edesc *edesc) -{ - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); - - if (is_sec1) { - if (!edesc->src_nents) { - dma_unmap_sg(dev, src, 1, - dst != src ? DMA_TO_DEVICE - : DMA_BIDIRECTIONAL); - } - if (dst && edesc->dst_nents) { - dma_sync_single_for_device(dev, - edesc->dma_link_tbl + len, - len, DMA_FROM_DEVICE); - sg_copy_from_buffer(dst, edesc->dst_nents ? : 1, - edesc->buf + len, len); - } else if (dst && dst != src) { - dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE); - } - } else { - talitos_sg_unmap(dev, edesc, src, dst); - } -} - static void common_nonsnoop_unmap(struct device *dev, struct talitos_edesc *edesc, struct ablkcipher_request *areq) { unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); - unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc); + talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); @@ -1470,100 +1532,6 @@ static void ablkcipher_done(struct device *dev, areq->base.complete(&areq->base, err); } -int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src, - unsigned int len, struct talitos_edesc *edesc, - enum dma_data_direction dir, struct talitos_ptr *ptr) -{ - int sg_count; - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); - - to_talitos_ptr_len(ptr, len, is_sec1); - - if (is_sec1) { - sg_count = edesc->src_nents ? : 1; - - if (sg_count == 1) { - dma_map_sg(dev, src, 1, dir); - to_talitos_ptr(ptr, sg_dma_address(src), is_sec1); - } else { - sg_copy_to_buffer(src, sg_count, edesc->buf, len); - to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1); - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - len, DMA_TO_DEVICE); - } - } else { - to_talitos_ptr_extent_clear(ptr, is_sec1); - - sg_count = dma_map_sg(dev, src, edesc->src_nents ? : 1, dir); - - if (sg_count == 1) { - to_talitos_ptr(ptr, sg_dma_address(src), is_sec1); - } else { - sg_count = sg_to_link_tbl(src, sg_count, len, - &edesc->link_tbl[0]); - if (sg_count > 1) { - to_talitos_ptr(ptr, edesc->dma_link_tbl, 0); - ptr->j_extent |= DESC_PTR_LNKTBL_JUMP; - dma_sync_single_for_device(dev, - edesc->dma_link_tbl, - edesc->dma_len, - DMA_BIDIRECTIONAL); - } else { - /* Only one segment now, so no link tbl needed*/ - to_talitos_ptr(ptr, sg_dma_address(src), - is_sec1); - } - } - } - return sg_count; -} - -void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst, - unsigned int len, struct talitos_edesc *edesc, - enum dma_data_direction dir, - struct talitos_ptr *ptr, int sg_count) -{ - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); - - if (dir != DMA_NONE) - sg_count = dma_map_sg(dev, dst, edesc->dst_nents ? : 1, dir); - - to_talitos_ptr_len(ptr, len, is_sec1); - - if (is_sec1) { - if (sg_count == 1) { - if (dir != DMA_NONE) - dma_map_sg(dev, dst, 1, dir); - to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1); - } else { - to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1); - dma_sync_single_for_device(dev, - edesc->dma_link_tbl + len, - len, DMA_FROM_DEVICE); - } - } else { - to_talitos_ptr_extent_clear(ptr, is_sec1); - - if (sg_count == 1) { - to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1); - } else { - struct talitos_ptr *link_tbl_ptr = - &edesc->link_tbl[edesc->src_nents + 1]; - - to_talitos_ptr(ptr, edesc->dma_link_tbl + - (edesc->src_nents + 1) * - sizeof(struct talitos_ptr), 0); - ptr->j_extent |= DESC_PTR_LNKTBL_JUMP; - sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr); - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - edesc->dma_len, - DMA_BIDIRECTIONAL); - } - } -} - static int common_nonsnoop(struct talitos_edesc *edesc, struct ablkcipher_request *areq, void (*callback) (struct device *dev, @@ -1577,6 +1545,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, unsigned int cryptlen = areq->nbytes; unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); int sg_count, ret; + bool sync_needed = false; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); @@ -1586,25 +1555,39 @@ static int common_nonsnoop(struct talitos_edesc *edesc, /* cipher iv */ to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1); to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1); - to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1); + to_talitos_ptr_ext_set(&desc->ptr[1], 0, is_sec1); /* cipher key */ map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, (char *)&ctx->key, DMA_TO_DEVICE); + sg_count = edesc->src_nents ?: 1; + if (is_sec1 && sg_count > 1) + sg_copy_to_buffer(areq->src, sg_count, edesc->buf, + cryptlen); + else + sg_count = dma_map_sg(dev, areq->src, sg_count, + (areq->src == areq->dst) ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE); /* * cipher in */ - sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc, - (areq->src == areq->dst) ? - DMA_BIDIRECTIONAL : DMA_TO_DEVICE, - &desc->ptr[3]); + sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc, + &desc->ptr[3], sg_count, 0, 0); + if (sg_count > 1) + sync_needed = true; /* cipher out */ - map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc, - (areq->src == areq->dst) ? DMA_NONE - : DMA_FROM_DEVICE, - &desc->ptr[4], sg_count); + if (areq->src != areq->dst) { + sg_count = edesc->dst_nents ? : 1; + if (!is_sec1 || sg_count == 1) + dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); + } + + ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4], + sg_count, 0, (edesc->src_nents + 1)); + if (ret > 1) + sync_needed = true; /* iv out */ map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, @@ -1613,6 +1596,10 @@ static int common_nonsnoop(struct talitos_edesc *edesc, /* last DWORD empty */ desc->ptr[6] = zero_entry; + if (sync_needed) + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, DMA_BIDIRECTIONAL); + ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { common_nonsnoop_unmap(dev, edesc, areq); @@ -1676,7 +1663,7 @@ static void common_nonsnoop_hash_unmap(struct device *dev, unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); - unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc); + talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); /* When using hashctx-in, must unmap it. */ if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) @@ -1747,8 +1734,10 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; int ret; + bool sync_needed = false; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); + int sg_count; /* first DWORD empty */ desc->ptr[0] = zero_entry; @@ -1773,11 +1762,19 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, else desc->ptr[2] = zero_entry; + sg_count = edesc->src_nents ?: 1; + if (is_sec1 && sg_count > 1) + sg_copy_to_buffer(areq->src, sg_count, edesc->buf, length); + else + sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, + DMA_TO_DEVICE); /* * data in */ - map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc, - DMA_TO_DEVICE, &desc->ptr[3]); + sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, + &desc->ptr[3], sg_count, 0, 0); + if (sg_count > 1) + sync_needed = true; /* fifth DWORD empty */ desc->ptr[4] = zero_entry; @@ -1798,6 +1795,10 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); + if (sync_needed) + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, DMA_BIDIRECTIONAL); + ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { common_nonsnoop_hash_unmap(dev, edesc, areq); @@ -2124,6 +2125,7 @@ static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, struct talitos_alg_template { u32 type; + u32 priority; union { struct crypto_alg crypto; struct ahash_alg hash; @@ -2154,6 +2156,27 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA1_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA1_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2176,6 +2199,29 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA1_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA1_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2196,6 +2242,27 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA224_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA224_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2218,6 +2285,29 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA224_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA224_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2238,6 +2328,27 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA256_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA256_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2260,6 +2371,29 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA256_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA256_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2364,6 +2498,27 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_MD5_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_MD5_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.aead = { .base = { @@ -2385,6 +2540,28 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_MD5_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_MD5_HMAC, + }, /* ABLKCIPHER algorithms. */ { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .alg.crypto = { @@ -2901,7 +3078,10 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, } alg->cra_module = THIS_MODULE; - alg->cra_priority = TALITOS_CRA_PRIORITY; + if (t_alg->algt.priority) + alg->cra_priority = t_alg->algt.priority; + else + alg->cra_priority = TALITOS_CRA_PRIORITY; alg->cra_alignmask = 0; alg->cra_ctxsize = sizeof(struct talitos_ctx); alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; diff --git a/drivers/crypto/ux500/cryp/Makefile b/drivers/crypto/ux500/cryp/Makefile index e5d362a6f..b497ae3dd 100644 --- a/drivers/crypto/ux500/cryp/Makefile +++ b/drivers/crypto/ux500/cryp/Makefile @@ -4,9 +4,9 @@ # * License terms: GNU General Public License (GPL) version 2 */ ifdef CONFIG_CRYPTO_DEV_UX500_DEBUG -CFLAGS_cryp_core.o := -DDEBUG -O0 -CFLAGS_cryp.o := -DDEBUG -O0 -CFLAGS_cryp_irq.o := -DDEBUG -O0 +CFLAGS_cryp_core.o := -DDEBUG +CFLAGS_cryp.o := -DDEBUG +CFLAGS_cryp_irq.o := -DDEBUG endif obj-$(CONFIG_CRYPTO_DEV_UX500_CRYP) += ux500_cryp.o diff --git a/drivers/crypto/ux500/hash/Makefile b/drivers/crypto/ux500/hash/Makefile index b2f90d9ba..784d9c0a8 100644 --- a/drivers/crypto/ux500/hash/Makefile +++ b/drivers/crypto/ux500/hash/Makefile @@ -4,7 +4,7 @@ # License terms: GNU General Public License (GPL) version 2 # ifdef CONFIG_CRYPTO_DEV_UX500_DEBUG -CFLAGS_hash_core.o := -DDEBUG -O0 +CFLAGS_hash_core.o := -DDEBUG endif obj-$(CONFIG_CRYPTO_DEV_UX500_HASH) += ux500_hash.o diff --git a/drivers/crypto/vmx/.gitignore b/drivers/crypto/vmx/.gitignore new file mode 100644 index 000000000..af4a7ce47 --- /dev/null +++ b/drivers/crypto/vmx/.gitignore @@ -0,0 +1,2 @@ +aesp8-ppc.S +ghashp8-ppc.S diff --git a/drivers/crypto/vmx/Kconfig b/drivers/crypto/vmx/Kconfig index 89d8208d9..a83ead109 100644 --- a/drivers/crypto/vmx/Kconfig +++ b/drivers/crypto/vmx/Kconfig @@ -1,7 +1,7 @@ config CRYPTO_DEV_VMX_ENCRYPT tristate "Encryption acceleration support on P8 CPU" depends on CRYPTO_DEV_VMX - default y + default m help Support for VMX cryptographic acceleration instructions on Power8 CPU. This module supports acceleration for AES and GHASH in hardware. If you diff --git a/drivers/crypto/vmx/Makefile b/drivers/crypto/vmx/Makefile index d28ab96a2..de6e241b0 100644 --- a/drivers/crypto/vmx/Makefile +++ b/drivers/crypto/vmx/Makefile @@ -1,5 +1,5 @@ obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o -vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o ghash.o +vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) TARGET := linux-ppc64le diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c new file mode 100644 index 000000000..24353ec33 --- /dev/null +++ b/drivers/crypto/vmx/aes_xts.c @@ -0,0 +1,190 @@ +/** + * AES XTS routines supporting VMX In-core instructions on Power 8 + * + * Copyright (C) 2015 International Business Machines Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundations; version 2 only. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY of FITNESS FOR A PARTICUPAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Author: Leonidas S. Barbosa + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aesp8-ppc.h" + +struct p8_aes_xts_ctx { + struct crypto_blkcipher *fallback; + struct aes_key enc_key; + struct aes_key dec_key; + struct aes_key tweak_key; +}; + +static int p8_aes_xts_init(struct crypto_tfm *tfm) +{ + const char *alg; + struct crypto_blkcipher *fallback; + struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + + if (!(alg = crypto_tfm_alg_name(tfm))) { + printk(KERN_ERR "Failed to get algorithm name.\n"); + return -ENOENT; + } + + fallback = + crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + printk(KERN_ERR + "Failed to allocate transformation for '%s': %ld\n", + alg, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + printk(KERN_INFO "Using '%s' as fallback implementation.\n", + crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback)); + + crypto_blkcipher_set_flags( + fallback, + crypto_blkcipher_get_flags((struct crypto_blkcipher *)tfm)); + ctx->fallback = fallback; + + return 0; +} + +static void p8_aes_xts_exit(struct crypto_tfm *tfm) +{ + struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->fallback) { + crypto_free_blkcipher(ctx->fallback); + ctx->fallback = NULL; + } +} + +static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) +{ + int ret; + struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + + ret = xts_check_key(tfm, key, keylen); + if (ret) + return ret; + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + ret = aes_p8_set_encrypt_key(key + keylen/2, (keylen/2) * 8, &ctx->tweak_key); + ret += aes_p8_set_encrypt_key(key, (keylen/2) * 8, &ctx->enc_key); + ret += aes_p8_set_decrypt_key(key, (keylen/2) * 8, &ctx->dec_key); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen); + return ret; +} + +static int p8_aes_xts_crypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, + unsigned int nbytes, int enc) +{ + int ret; + u8 tweak[AES_BLOCK_SIZE]; + u8 *iv; + struct blkcipher_walk walk; + struct p8_aes_xts_ctx *ctx = + crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); + struct blkcipher_desc fallback_desc = { + .tfm = ctx->fallback, + .info = desc->info, + .flags = desc->flags + }; + + if (in_interrupt()) { + ret = enc ? crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes) : + crypto_blkcipher_decrypt(&fallback_desc, dst, src, nbytes); + } else { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + + blkcipher_walk_init(&walk, dst, src, nbytes); + + ret = blkcipher_walk_virt(desc, &walk); + iv = walk.iv; + memset(tweak, 0, AES_BLOCK_SIZE); + aes_p8_encrypt(iv, tweak, &ctx->tweak_key); + + while ((nbytes = walk.nbytes)) { + if (enc) + aes_p8_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr, + nbytes & AES_BLOCK_MASK, &ctx->enc_key, NULL, tweak); + else + aes_p8_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr, + nbytes & AES_BLOCK_MASK, &ctx->dec_key, NULL, tweak); + + nbytes &= AES_BLOCK_SIZE - 1; + ret = blkcipher_walk_done(desc, &walk, nbytes); + } + + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + } + return ret; +} + +static int p8_aes_xts_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return p8_aes_xts_crypt(desc, dst, src, nbytes, 1); +} + +static int p8_aes_xts_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return p8_aes_xts_crypt(desc, dst, src, nbytes, 0); +} + +struct crypto_alg p8_aes_xts_alg = { + .cra_name = "xts(aes)", + .cra_driver_name = "p8_aes_xts", + .cra_module = THIS_MODULE, + .cra_priority = 2000, + .cra_type = &crypto_blkcipher_type, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, + .cra_alignmask = 0, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct p8_aes_xts_ctx), + .cra_init = p8_aes_xts_init, + .cra_exit = p8_aes_xts_exit, + .cra_blkcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .setkey = p8_aes_xts_setkey, + .encrypt = p8_aes_xts_encrypt, + .decrypt = p8_aes_xts_decrypt, + } +}; diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h index 4cd34ee54..01972e16a 100644 --- a/drivers/crypto/vmx/aesp8-ppc.h +++ b/drivers/crypto/vmx/aesp8-ppc.h @@ -19,3 +19,7 @@ void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len, void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out, size_t len, const struct aes_key *key, const u8 *iv); +void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len, + const struct aes_key *key1, const struct aes_key *key2, u8 *iv); +void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len, + const struct aes_key *key1, const struct aes_key *key2, u8 *iv); diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl index 228053921..0b4a293b8 100644 --- a/drivers/crypto/vmx/aesp8-ppc.pl +++ b/drivers/crypto/vmx/aesp8-ppc.pl @@ -1,4 +1,11 @@ -#!/usr/bin/env perl +#! /usr/bin/env perl +# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the OpenSSL license (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + # # ==================================================================== # Written by Andy Polyakov for the OpenSSL @@ -20,6 +27,19 @@ # instructions are interleaved. It's reckoned that eventual # misalignment penalties at page boundaries are in average lower # than additional overhead in pure AltiVec approach. +# +# May 2016 +# +# Add XTS subroutine, 9x on little- and 12x improvement on big-endian +# systems were measured. +# +###################################################################### +# Current large-block performance in cycles per byte processed with +# 128-bit key (less is better). +# +# CBC en-/decrypt CTR XTS +# POWER8[le] 3.96/0.72 0.74 1.1 +# POWER8[be] 3.75/0.65 0.66 1.0 $flavour = shift; @@ -1875,6 +1895,1845 @@ Lctr32_enc8x_done: ___ }} }}} +######################################################################### +{{{ # XTS procedures # +# int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len, # +# const AES_KEY *key1, const AES_KEY *key2, # +# [const] unsigned char iv[16]); # +# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which # +# input tweak value is assumed to be encrypted already, and last tweak # +# value, one suitable for consecutive call on same chunk of data, is # +# written back to original buffer. In addition, in "tweak chaining" # +# mode only complete input blocks are processed. # + +my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) = map("r$_",(3..10)); +my ($rndkey0,$rndkey1,$inout) = map("v$_",(0..2)); +my ($output,$inptail,$inpperm,$leperm,$keyperm) = map("v$_",(3..7)); +my ($tweak,$seven,$eighty7,$tmp,$tweak1) = map("v$_",(8..12)); +my $taillen = $key2; + + ($inp,$idx) = ($idx,$inp); # reassign + +$code.=<<___; +.globl .${prefix}_xts_encrypt + mr $inp,r3 # reassign + li r3,-1 + ${UCMP}i $len,16 + bltlr- + + lis r0,0xfff0 + mfspr r12,256 # save vrsave + li r11,0 + mtspr 256,r0 + + vspltisb $seven,0x07 # 0x070707..07 + le?lvsl $leperm,r11,r11 + le?vspltisb $tmp,0x0f + le?vxor $leperm,$leperm,$seven + + li $idx,15 + lvx $tweak,0,$ivp # load [unaligned] iv + lvsl $inpperm,0,$ivp + lvx $inptail,$idx,$ivp + le?vxor $inpperm,$inpperm,$tmp + vperm $tweak,$tweak,$inptail,$inpperm + + neg r11,$inp + lvsr $inpperm,0,r11 # prepare for unaligned load + lvx $inout,0,$inp + addi $inp,$inp,15 # 15 is not typo + le?vxor $inpperm,$inpperm,$tmp + + ${UCMP}i $key2,0 # key2==NULL? + beq Lxts_enc_no_key2 + + ?lvsl $keyperm,0,$key2 # prepare for unaligned key + lwz $rounds,240($key2) + srwi $rounds,$rounds,1 + subi $rounds,$rounds,1 + li $idx,16 + + lvx $rndkey0,0,$key2 + lvx $rndkey1,$idx,$key2 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $tweak,$tweak,$rndkey0 + lvx $rndkey0,$idx,$key2 + addi $idx,$idx,16 + mtctr $rounds + +Ltweak_xts_enc: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $tweak,$tweak,$rndkey1 + lvx $rndkey1,$idx,$key2 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipher $tweak,$tweak,$rndkey0 + lvx $rndkey0,$idx,$key2 + addi $idx,$idx,16 + bdnz Ltweak_xts_enc + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $tweak,$tweak,$rndkey1 + lvx $rndkey1,$idx,$key2 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipherlast $tweak,$tweak,$rndkey0 + + li $ivp,0 # don't chain the tweak + b Lxts_enc + +Lxts_enc_no_key2: + li $idx,-16 + and $len,$len,$idx # in "tweak chaining" + # mode only complete + # blocks are processed +Lxts_enc: + lvx $inptail,0,$inp + addi $inp,$inp,16 + + ?lvsl $keyperm,0,$key1 # prepare for unaligned key + lwz $rounds,240($key1) + srwi $rounds,$rounds,1 + subi $rounds,$rounds,1 + li $idx,16 + + vslb $eighty7,$seven,$seven # 0x808080..80 + vor $eighty7,$eighty7,$seven # 0x878787..87 + vspltisb $tmp,1 # 0x010101..01 + vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01 + + ${UCMP}i $len,96 + bge _aesp8_xts_encrypt6x + + andi. $taillen,$len,15 + subic r0,$len,32 + subi $taillen,$taillen,16 + subfe r0,r0,r0 + and r0,r0,$taillen + add $inp,$inp,r0 + + lvx $rndkey0,0,$key1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + vperm $inout,$inout,$inptail,$inpperm + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$inout,$tweak + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + mtctr $rounds + b Loop_xts_enc + +.align 5 +Loop_xts_enc: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipher $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + bdnz Loop_xts_enc + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + li $idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $rndkey0,$rndkey0,$tweak + vcipherlast $output,$inout,$rndkey0 + + le?vperm $tmp,$output,$output,$leperm + be?nop + le?stvx_u $tmp,0,$out + be?stvx_u $output,0,$out + addi $out,$out,16 + + subic. $len,$len,16 + beq Lxts_enc_done + + vmr $inout,$inptail + lvx $inptail,0,$inp + addi $inp,$inp,16 + lvx $rndkey0,0,$key1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + + subic r0,$len,32 + subfe r0,r0,r0 + and r0,r0,$taillen + add $inp,$inp,r0 + + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $tweak,$tweak,$tmp + + vperm $inout,$inout,$inptail,$inpperm + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$inout,$tweak + vxor $output,$output,$rndkey0 # just in case $len<16 + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + + mtctr $rounds + ${UCMP}i $len,16 + bge Loop_xts_enc + + vxor $output,$output,$tweak + lvsr $inpperm,0,$len # $inpperm is no longer needed + vxor $inptail,$inptail,$inptail # $inptail is no longer needed + vspltisb $tmp,-1 + vperm $inptail,$inptail,$tmp,$inpperm + vsel $inout,$inout,$output,$inptail + + subi r11,$out,17 + subi $out,$out,16 + mtctr $len + li $len,16 +Loop_xts_enc_steal: + lbzu r0,1(r11) + stb r0,16(r11) + bdnz Loop_xts_enc_steal + + mtctr $rounds + b Loop_xts_enc # one more time... + +Lxts_enc_done: + ${UCMP}i $ivp,0 + beq Lxts_enc_ret + + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $tweak,$tweak,$tmp + + le?vperm $tweak,$tweak,$tweak,$leperm + stvx_u $tweak,0,$ivp + +Lxts_enc_ret: + mtspr 256,r12 # restore vrsave + li r3,0 + blr + .long 0 + .byte 0,12,0x04,0,0x80,6,6,0 + .long 0 +.size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt + +.globl .${prefix}_xts_decrypt + mr $inp,r3 # reassign + li r3,-1 + ${UCMP}i $len,16 + bltlr- + + lis r0,0xfff8 + mfspr r12,256 # save vrsave + li r11,0 + mtspr 256,r0 + + andi. r0,$len,15 + neg r0,r0 + andi. r0,r0,16 + sub $len,$len,r0 + + vspltisb $seven,0x07 # 0x070707..07 + le?lvsl $leperm,r11,r11 + le?vspltisb $tmp,0x0f + le?vxor $leperm,$leperm,$seven + + li $idx,15 + lvx $tweak,0,$ivp # load [unaligned] iv + lvsl $inpperm,0,$ivp + lvx $inptail,$idx,$ivp + le?vxor $inpperm,$inpperm,$tmp + vperm $tweak,$tweak,$inptail,$inpperm + + neg r11,$inp + lvsr $inpperm,0,r11 # prepare for unaligned load + lvx $inout,0,$inp + addi $inp,$inp,15 # 15 is not typo + le?vxor $inpperm,$inpperm,$tmp + + ${UCMP}i $key2,0 # key2==NULL? + beq Lxts_dec_no_key2 + + ?lvsl $keyperm,0,$key2 # prepare for unaligned key + lwz $rounds,240($key2) + srwi $rounds,$rounds,1 + subi $rounds,$rounds,1 + li $idx,16 + + lvx $rndkey0,0,$key2 + lvx $rndkey1,$idx,$key2 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $tweak,$tweak,$rndkey0 + lvx $rndkey0,$idx,$key2 + addi $idx,$idx,16 + mtctr $rounds + +Ltweak_xts_dec: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $tweak,$tweak,$rndkey1 + lvx $rndkey1,$idx,$key2 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipher $tweak,$tweak,$rndkey0 + lvx $rndkey0,$idx,$key2 + addi $idx,$idx,16 + bdnz Ltweak_xts_dec + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vcipher $tweak,$tweak,$rndkey1 + lvx $rndkey1,$idx,$key2 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vcipherlast $tweak,$tweak,$rndkey0 + + li $ivp,0 # don't chain the tweak + b Lxts_dec + +Lxts_dec_no_key2: + neg $idx,$len + andi. $idx,$idx,15 + add $len,$len,$idx # in "tweak chaining" + # mode only complete + # blocks are processed +Lxts_dec: + lvx $inptail,0,$inp + addi $inp,$inp,16 + + ?lvsl $keyperm,0,$key1 # prepare for unaligned key + lwz $rounds,240($key1) + srwi $rounds,$rounds,1 + subi $rounds,$rounds,1 + li $idx,16 + + vslb $eighty7,$seven,$seven # 0x808080..80 + vor $eighty7,$eighty7,$seven # 0x878787..87 + vspltisb $tmp,1 # 0x010101..01 + vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01 + + ${UCMP}i $len,96 + bge _aesp8_xts_decrypt6x + + lvx $rndkey0,0,$key1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + vperm $inout,$inout,$inptail,$inpperm + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$inout,$tweak + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + mtctr $rounds + + ${UCMP}i $len,16 + blt Ltail_xts_dec + be?b Loop_xts_dec + +.align 5 +Loop_xts_dec: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vncipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vncipher $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + bdnz Loop_xts_dec + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vncipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + li $idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $rndkey0,$rndkey0,$tweak + vncipherlast $output,$inout,$rndkey0 + + le?vperm $tmp,$output,$output,$leperm + be?nop + le?stvx_u $tmp,0,$out + be?stvx_u $output,0,$out + addi $out,$out,16 + + subic. $len,$len,16 + beq Lxts_dec_done + + vmr $inout,$inptail + lvx $inptail,0,$inp + addi $inp,$inp,16 + lvx $rndkey0,0,$key1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $tweak,$tweak,$tmp + + vperm $inout,$inout,$inptail,$inpperm + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $inout,$inout,$tweak + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + + mtctr $rounds + ${UCMP}i $len,16 + bge Loop_xts_dec + +Ltail_xts_dec: + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak1,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $tweak1,$tweak1,$tmp + + subi $inp,$inp,16 + add $inp,$inp,$len + + vxor $inout,$inout,$tweak # :-( + vxor $inout,$inout,$tweak1 # :-) + +Loop_xts_dec_short: + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vncipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vncipher $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + bdnz Loop_xts_dec_short + + ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm + vncipher $inout,$inout,$rndkey1 + lvx $rndkey1,$idx,$key1 + li $idx,16 + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + vxor $rndkey0,$rndkey0,$tweak1 + vncipherlast $output,$inout,$rndkey0 + + le?vperm $tmp,$output,$output,$leperm + be?nop + le?stvx_u $tmp,0,$out + be?stvx_u $output,0,$out + + vmr $inout,$inptail + lvx $inptail,0,$inp + #addi $inp,$inp,16 + lvx $rndkey0,0,$key1 + lvx $rndkey1,$idx,$key1 + addi $idx,$idx,16 + vperm $inout,$inout,$inptail,$inpperm + ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm + + lvsr $inpperm,0,$len # $inpperm is no longer needed + vxor $inptail,$inptail,$inptail # $inptail is no longer needed + vspltisb $tmp,-1 + vperm $inptail,$inptail,$tmp,$inpperm + vsel $inout,$inout,$output,$inptail + + vxor $rndkey0,$rndkey0,$tweak + vxor $inout,$inout,$rndkey0 + lvx $rndkey0,$idx,$key1 + addi $idx,$idx,16 + + subi r11,$out,1 + mtctr $len + li $len,16 +Loop_xts_dec_steal: + lbzu r0,1(r11) + stb r0,16(r11) + bdnz Loop_xts_dec_steal + + mtctr $rounds + b Loop_xts_dec # one more time... + +Lxts_dec_done: + ${UCMP}i $ivp,0 + beq Lxts_dec_ret + + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $tweak,$tweak,$tmp + + le?vperm $tweak,$tweak,$tweak,$leperm + stvx_u $tweak,0,$ivp + +Lxts_dec_ret: + mtspr 256,r12 # restore vrsave + li r3,0 + blr + .long 0 + .byte 0,12,0x04,0,0x80,6,6,0 + .long 0 +.size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt +___ +######################################################################### +{{ # Optimized XTS procedures # +my $key_=$key2; +my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31)); + $x00=0 if ($flavour =~ /osx/); +my ($in0, $in1, $in2, $in3, $in4, $in5 )=map("v$_",(0..5)); +my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16)); +my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22)); +my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys + # v26-v31 last 6 round keys +my ($keyperm)=($out0); # aliases with "caller", redundant assignment +my $taillen=$x70; + +$code.=<<___; +.align 5 +_aesp8_xts_encrypt6x: + $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) + mflr r11 + li r7,`$FRAME+8*16+15` + li r3,`$FRAME+8*16+31` + $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp) + stvx v20,r7,$sp # ABI says so + addi r7,r7,32 + stvx v21,r3,$sp + addi r3,r3,32 + stvx v22,r7,$sp + addi r7,r7,32 + stvx v23,r3,$sp + addi r3,r3,32 + stvx v24,r7,$sp + addi r7,r7,32 + stvx v25,r3,$sp + addi r3,r3,32 + stvx v26,r7,$sp + addi r7,r7,32 + stvx v27,r3,$sp + addi r3,r3,32 + stvx v28,r7,$sp + addi r7,r7,32 + stvx v29,r3,$sp + addi r3,r3,32 + stvx v30,r7,$sp + stvx v31,r3,$sp + li r0,-1 + stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave + li $x10,0x10 + $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) + li $x20,0x20 + $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) + li $x30,0x30 + $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) + li $x40,0x40 + $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) + li $x50,0x50 + $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) + li $x60,0x60 + $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) + li $x70,0x70 + mtspr 256,r0 + + subi $rounds,$rounds,3 # -4 in total + + lvx $rndkey0,$x00,$key1 # load key schedule + lvx v30,$x10,$key1 + addi $key1,$key1,0x20 + lvx v31,$x00,$key1 + ?vperm $rndkey0,$rndkey0,v30,$keyperm + addi $key_,$sp,$FRAME+15 + mtctr $rounds + +Load_xts_enc_key: + ?vperm v24,v30,v31,$keyperm + lvx v30,$x10,$key1 + addi $key1,$key1,0x20 + stvx v24,$x00,$key_ # off-load round[1] + ?vperm v25,v31,v30,$keyperm + lvx v31,$x00,$key1 + stvx v25,$x10,$key_ # off-load round[2] + addi $key_,$key_,0x20 + bdnz Load_xts_enc_key + + lvx v26,$x10,$key1 + ?vperm v24,v30,v31,$keyperm + lvx v27,$x20,$key1 + stvx v24,$x00,$key_ # off-load round[3] + ?vperm v25,v31,v26,$keyperm + lvx v28,$x30,$key1 + stvx v25,$x10,$key_ # off-load round[4] + addi $key_,$sp,$FRAME+15 # rewind $key_ + ?vperm v26,v26,v27,$keyperm + lvx v29,$x40,$key1 + ?vperm v27,v27,v28,$keyperm + lvx v30,$x50,$key1 + ?vperm v28,v28,v29,$keyperm + lvx v31,$x60,$key1 + ?vperm v29,v29,v30,$keyperm + lvx $twk5,$x70,$key1 # borrow $twk5 + ?vperm v30,v30,v31,$keyperm + lvx v24,$x00,$key_ # pre-load round[1] + ?vperm v31,v31,$twk5,$keyperm + lvx v25,$x10,$key_ # pre-load round[2] + + vperm $in0,$inout,$inptail,$inpperm + subi $inp,$inp,31 # undo "caller" + vxor $twk0,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $out0,$in0,$twk0 + vxor $tweak,$tweak,$tmp + + lvx_u $in1,$x10,$inp + vxor $twk1,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in1,$in1,$in1,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out1,$in1,$twk1 + vxor $tweak,$tweak,$tmp + + lvx_u $in2,$x20,$inp + andi. $taillen,$len,15 + vxor $twk2,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in2,$in2,$in2,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out2,$in2,$twk2 + vxor $tweak,$tweak,$tmp + + lvx_u $in3,$x30,$inp + sub $len,$len,$taillen + vxor $twk3,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in3,$in3,$in3,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out3,$in3,$twk3 + vxor $tweak,$tweak,$tmp + + lvx_u $in4,$x40,$inp + subi $len,$len,0x60 + vxor $twk4,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in4,$in4,$in4,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out4,$in4,$twk4 + vxor $tweak,$tweak,$tmp + + lvx_u $in5,$x50,$inp + addi $inp,$inp,0x60 + vxor $twk5,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in5,$in5,$in5,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out5,$in5,$twk5 + vxor $tweak,$tweak,$tmp + + vxor v31,v31,$rndkey0 + mtctr $rounds + b Loop_xts_enc6x + +.align 5 +Loop_xts_enc6x: + vcipher $out0,$out0,v24 + vcipher $out1,$out1,v24 + vcipher $out2,$out2,v24 + vcipher $out3,$out3,v24 + vcipher $out4,$out4,v24 + vcipher $out5,$out5,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vcipher $out0,$out0,v25 + vcipher $out1,$out1,v25 + vcipher $out2,$out2,v25 + vcipher $out3,$out3,v25 + vcipher $out4,$out4,v25 + vcipher $out5,$out5,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_xts_enc6x + + subic $len,$len,96 # $len-=96 + vxor $in0,$twk0,v31 # xor with last round key + vcipher $out0,$out0,v24 + vcipher $out1,$out1,v24 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk0,$tweak,$rndkey0 + vaddubm $tweak,$tweak,$tweak + vcipher $out2,$out2,v24 + vcipher $out3,$out3,v24 + vsldoi $tmp,$tmp,$tmp,15 + vcipher $out4,$out4,v24 + vcipher $out5,$out5,v24 + + subfe. r0,r0,r0 # borrow?-1:0 + vand $tmp,$tmp,$eighty7 + vcipher $out0,$out0,v25 + vcipher $out1,$out1,v25 + vxor $tweak,$tweak,$tmp + vcipher $out2,$out2,v25 + vcipher $out3,$out3,v25 + vxor $in1,$twk1,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk1,$tweak,$rndkey0 + vcipher $out4,$out4,v25 + vcipher $out5,$out5,v25 + + and r0,r0,$len + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vcipher $out0,$out0,v26 + vcipher $out1,$out1,v26 + vand $tmp,$tmp,$eighty7 + vcipher $out2,$out2,v26 + vcipher $out3,$out3,v26 + vxor $tweak,$tweak,$tmp + vcipher $out4,$out4,v26 + vcipher $out5,$out5,v26 + + add $inp,$inp,r0 # $inp is adjusted in such + # way that at exit from the + # loop inX-in5 are loaded + # with last "words" + vxor $in2,$twk2,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk2,$tweak,$rndkey0 + vaddubm $tweak,$tweak,$tweak + vcipher $out0,$out0,v27 + vcipher $out1,$out1,v27 + vsldoi $tmp,$tmp,$tmp,15 + vcipher $out2,$out2,v27 + vcipher $out3,$out3,v27 + vand $tmp,$tmp,$eighty7 + vcipher $out4,$out4,v27 + vcipher $out5,$out5,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vxor $tweak,$tweak,$tmp + vcipher $out0,$out0,v28 + vcipher $out1,$out1,v28 + vxor $in3,$twk3,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk3,$tweak,$rndkey0 + vcipher $out2,$out2,v28 + vcipher $out3,$out3,v28 + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vcipher $out4,$out4,v28 + vcipher $out5,$out5,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + vand $tmp,$tmp,$eighty7 + + vcipher $out0,$out0,v29 + vcipher $out1,$out1,v29 + vxor $tweak,$tweak,$tmp + vcipher $out2,$out2,v29 + vcipher $out3,$out3,v29 + vxor $in4,$twk4,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk4,$tweak,$rndkey0 + vcipher $out4,$out4,v29 + vcipher $out5,$out5,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + + vcipher $out0,$out0,v30 + vcipher $out1,$out1,v30 + vand $tmp,$tmp,$eighty7 + vcipher $out2,$out2,v30 + vcipher $out3,$out3,v30 + vxor $tweak,$tweak,$tmp + vcipher $out4,$out4,v30 + vcipher $out5,$out5,v30 + vxor $in5,$twk5,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk5,$tweak,$rndkey0 + + vcipherlast $out0,$out0,$in0 + lvx_u $in0,$x00,$inp # load next input block + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vcipherlast $out1,$out1,$in1 + lvx_u $in1,$x10,$inp + vcipherlast $out2,$out2,$in2 + le?vperm $in0,$in0,$in0,$leperm + lvx_u $in2,$x20,$inp + vand $tmp,$tmp,$eighty7 + vcipherlast $out3,$out3,$in3 + le?vperm $in1,$in1,$in1,$leperm + lvx_u $in3,$x30,$inp + vcipherlast $out4,$out4,$in4 + le?vperm $in2,$in2,$in2,$leperm + lvx_u $in4,$x40,$inp + vxor $tweak,$tweak,$tmp + vcipherlast $tmp,$out5,$in5 # last block might be needed + # in stealing mode + le?vperm $in3,$in3,$in3,$leperm + lvx_u $in5,$x50,$inp + addi $inp,$inp,0x60 + le?vperm $in4,$in4,$in4,$leperm + le?vperm $in5,$in5,$in5,$leperm + + le?vperm $out0,$out0,$out0,$leperm + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk0 + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + vxor $out1,$in1,$twk1 + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + vxor $out2,$in2,$twk2 + le?vperm $out4,$out4,$out4,$leperm + stvx_u $out3,$x30,$out + vxor $out3,$in3,$twk3 + le?vperm $out5,$tmp,$tmp,$leperm + stvx_u $out4,$x40,$out + vxor $out4,$in4,$twk4 + le?stvx_u $out5,$x50,$out + be?stvx_u $tmp, $x50,$out + vxor $out5,$in5,$twk5 + addi $out,$out,0x60 + + mtctr $rounds + beq Loop_xts_enc6x # did $len-=96 borrow? + + addic. $len,$len,0x60 + beq Lxts_enc6x_zero + cmpwi $len,0x20 + blt Lxts_enc6x_one + nop + beq Lxts_enc6x_two + cmpwi $len,0x40 + blt Lxts_enc6x_three + nop + beq Lxts_enc6x_four + +Lxts_enc6x_five: + vxor $out0,$in1,$twk0 + vxor $out1,$in2,$twk1 + vxor $out2,$in3,$twk2 + vxor $out3,$in4,$twk3 + vxor $out4,$in5,$twk4 + + bl _aesp8_xts_enc5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk5 # unused tweak + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + vxor $tmp,$out4,$twk5 # last block prep for stealing + le?vperm $out4,$out4,$out4,$leperm + stvx_u $out3,$x30,$out + stvx_u $out4,$x40,$out + addi $out,$out,0x50 + bne Lxts_enc6x_steal + b Lxts_enc6x_done + +.align 4 +Lxts_enc6x_four: + vxor $out0,$in2,$twk0 + vxor $out1,$in3,$twk1 + vxor $out2,$in4,$twk2 + vxor $out3,$in5,$twk3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_enc5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk4 # unused tweak + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + vxor $tmp,$out3,$twk4 # last block prep for stealing + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + stvx_u $out3,$x30,$out + addi $out,$out,0x40 + bne Lxts_enc6x_steal + b Lxts_enc6x_done + +.align 4 +Lxts_enc6x_three: + vxor $out0,$in3,$twk0 + vxor $out1,$in4,$twk1 + vxor $out2,$in5,$twk2 + vxor $out3,$out3,$out3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_enc5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk3 # unused tweak + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $tmp,$out2,$twk3 # last block prep for stealing + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + stvx_u $out2,$x20,$out + addi $out,$out,0x30 + bne Lxts_enc6x_steal + b Lxts_enc6x_done + +.align 4 +Lxts_enc6x_two: + vxor $out0,$in4,$twk0 + vxor $out1,$in5,$twk1 + vxor $out2,$out2,$out2 + vxor $out3,$out3,$out3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_enc5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk2 # unused tweak + vxor $tmp,$out1,$twk2 # last block prep for stealing + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + stvx_u $out1,$x10,$out + addi $out,$out,0x20 + bne Lxts_enc6x_steal + b Lxts_enc6x_done + +.align 4 +Lxts_enc6x_one: + vxor $out0,$in5,$twk0 + nop +Loop_xts_enc1x: + vcipher $out0,$out0,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vcipher $out0,$out0,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_xts_enc1x + + add $inp,$inp,$taillen + cmpwi $taillen,0 + vcipher $out0,$out0,v24 + + subi $inp,$inp,16 + vcipher $out0,$out0,v25 + + lvsr $inpperm,0,$taillen + vcipher $out0,$out0,v26 + + lvx_u $in0,0,$inp + vcipher $out0,$out0,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vcipher $out0,$out0,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + + vcipher $out0,$out0,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vxor $twk0,$twk0,v31 + + le?vperm $in0,$in0,$in0,$leperm + vcipher $out0,$out0,v30 + + vperm $in0,$in0,$in0,$inpperm + vcipherlast $out0,$out0,$twk0 + + vmr $twk0,$twk1 # unused tweak + vxor $tmp,$out0,$twk1 # last block prep for stealing + le?vperm $out0,$out0,$out0,$leperm + stvx_u $out0,$x00,$out # store output + addi $out,$out,0x10 + bne Lxts_enc6x_steal + b Lxts_enc6x_done + +.align 4 +Lxts_enc6x_zero: + cmpwi $taillen,0 + beq Lxts_enc6x_done + + add $inp,$inp,$taillen + subi $inp,$inp,16 + lvx_u $in0,0,$inp + lvsr $inpperm,0,$taillen # $in5 is no more + le?vperm $in0,$in0,$in0,$leperm + vperm $in0,$in0,$in0,$inpperm + vxor $tmp,$tmp,$twk0 +Lxts_enc6x_steal: + vxor $in0,$in0,$twk0 + vxor $out0,$out0,$out0 + vspltisb $out1,-1 + vperm $out0,$out0,$out1,$inpperm + vsel $out0,$in0,$tmp,$out0 # $tmp is last block, remember? + + subi r30,$out,17 + subi $out,$out,16 + mtctr $taillen +Loop_xts_enc6x_steal: + lbzu r0,1(r30) + stb r0,16(r30) + bdnz Loop_xts_enc6x_steal + + li $taillen,0 + mtctr $rounds + b Loop_xts_enc1x # one more time... + +.align 4 +Lxts_enc6x_done: + ${UCMP}i $ivp,0 + beq Lxts_enc6x_ret + + vxor $tweak,$twk0,$rndkey0 + le?vperm $tweak,$tweak,$tweak,$leperm + stvx_u $tweak,0,$ivp + +Lxts_enc6x_ret: + mtlr r11 + li r10,`$FRAME+15` + li r11,`$FRAME+31` + stvx $seven,r10,$sp # wipe copies of round keys + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + + mtspr 256,$vrsave + lvx v20,r10,$sp # ABI says so + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) + $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) + $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) + $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) + $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) + $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) + addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` + blr + .long 0 + .byte 0,12,0x04,1,0x80,6,6,0 + .long 0 + +.align 5 +_aesp8_xts_enc5x: + vcipher $out0,$out0,v24 + vcipher $out1,$out1,v24 + vcipher $out2,$out2,v24 + vcipher $out3,$out3,v24 + vcipher $out4,$out4,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vcipher $out0,$out0,v25 + vcipher $out1,$out1,v25 + vcipher $out2,$out2,v25 + vcipher $out3,$out3,v25 + vcipher $out4,$out4,v25 + lvx v25,$x10,$key_ # round[4] + bdnz _aesp8_xts_enc5x + + add $inp,$inp,$taillen + cmpwi $taillen,0 + vcipher $out0,$out0,v24 + vcipher $out1,$out1,v24 + vcipher $out2,$out2,v24 + vcipher $out3,$out3,v24 + vcipher $out4,$out4,v24 + + subi $inp,$inp,16 + vcipher $out0,$out0,v25 + vcipher $out1,$out1,v25 + vcipher $out2,$out2,v25 + vcipher $out3,$out3,v25 + vcipher $out4,$out4,v25 + vxor $twk0,$twk0,v31 + + vcipher $out0,$out0,v26 + lvsr $inpperm,r0,$taillen # $in5 is no more + vcipher $out1,$out1,v26 + vcipher $out2,$out2,v26 + vcipher $out3,$out3,v26 + vcipher $out4,$out4,v26 + vxor $in1,$twk1,v31 + + vcipher $out0,$out0,v27 + lvx_u $in0,0,$inp + vcipher $out1,$out1,v27 + vcipher $out2,$out2,v27 + vcipher $out3,$out3,v27 + vcipher $out4,$out4,v27 + vxor $in2,$twk2,v31 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vcipher $out0,$out0,v28 + vcipher $out1,$out1,v28 + vcipher $out2,$out2,v28 + vcipher $out3,$out3,v28 + vcipher $out4,$out4,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + vxor $in3,$twk3,v31 + + vcipher $out0,$out0,v29 + le?vperm $in0,$in0,$in0,$leperm + vcipher $out1,$out1,v29 + vcipher $out2,$out2,v29 + vcipher $out3,$out3,v29 + vcipher $out4,$out4,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vxor $in4,$twk4,v31 + + vcipher $out0,$out0,v30 + vperm $in0,$in0,$in0,$inpperm + vcipher $out1,$out1,v30 + vcipher $out2,$out2,v30 + vcipher $out3,$out3,v30 + vcipher $out4,$out4,v30 + + vcipherlast $out0,$out0,$twk0 + vcipherlast $out1,$out1,$in1 + vcipherlast $out2,$out2,$in2 + vcipherlast $out3,$out3,$in3 + vcipherlast $out4,$out4,$in4 + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 + +.align 5 +_aesp8_xts_decrypt6x: + $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp) + mflr r11 + li r7,`$FRAME+8*16+15` + li r3,`$FRAME+8*16+31` + $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp) + stvx v20,r7,$sp # ABI says so + addi r7,r7,32 + stvx v21,r3,$sp + addi r3,r3,32 + stvx v22,r7,$sp + addi r7,r7,32 + stvx v23,r3,$sp + addi r3,r3,32 + stvx v24,r7,$sp + addi r7,r7,32 + stvx v25,r3,$sp + addi r3,r3,32 + stvx v26,r7,$sp + addi r7,r7,32 + stvx v27,r3,$sp + addi r3,r3,32 + stvx v28,r7,$sp + addi r7,r7,32 + stvx v29,r3,$sp + addi r3,r3,32 + stvx v30,r7,$sp + stvx v31,r3,$sp + li r0,-1 + stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave + li $x10,0x10 + $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp) + li $x20,0x20 + $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp) + li $x30,0x30 + $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp) + li $x40,0x40 + $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp) + li $x50,0x50 + $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp) + li $x60,0x60 + $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp) + li $x70,0x70 + mtspr 256,r0 + + subi $rounds,$rounds,3 # -4 in total + + lvx $rndkey0,$x00,$key1 # load key schedule + lvx v30,$x10,$key1 + addi $key1,$key1,0x20 + lvx v31,$x00,$key1 + ?vperm $rndkey0,$rndkey0,v30,$keyperm + addi $key_,$sp,$FRAME+15 + mtctr $rounds + +Load_xts_dec_key: + ?vperm v24,v30,v31,$keyperm + lvx v30,$x10,$key1 + addi $key1,$key1,0x20 + stvx v24,$x00,$key_ # off-load round[1] + ?vperm v25,v31,v30,$keyperm + lvx v31,$x00,$key1 + stvx v25,$x10,$key_ # off-load round[2] + addi $key_,$key_,0x20 + bdnz Load_xts_dec_key + + lvx v26,$x10,$key1 + ?vperm v24,v30,v31,$keyperm + lvx v27,$x20,$key1 + stvx v24,$x00,$key_ # off-load round[3] + ?vperm v25,v31,v26,$keyperm + lvx v28,$x30,$key1 + stvx v25,$x10,$key_ # off-load round[4] + addi $key_,$sp,$FRAME+15 # rewind $key_ + ?vperm v26,v26,v27,$keyperm + lvx v29,$x40,$key1 + ?vperm v27,v27,v28,$keyperm + lvx v30,$x50,$key1 + ?vperm v28,v28,v29,$keyperm + lvx v31,$x60,$key1 + ?vperm v29,v29,v30,$keyperm + lvx $twk5,$x70,$key1 # borrow $twk5 + ?vperm v30,v30,v31,$keyperm + lvx v24,$x00,$key_ # pre-load round[1] + ?vperm v31,v31,$twk5,$keyperm + lvx v25,$x10,$key_ # pre-load round[2] + + vperm $in0,$inout,$inptail,$inpperm + subi $inp,$inp,31 # undo "caller" + vxor $twk0,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vand $tmp,$tmp,$eighty7 + vxor $out0,$in0,$twk0 + vxor $tweak,$tweak,$tmp + + lvx_u $in1,$x10,$inp + vxor $twk1,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in1,$in1,$in1,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out1,$in1,$twk1 + vxor $tweak,$tweak,$tmp + + lvx_u $in2,$x20,$inp + andi. $taillen,$len,15 + vxor $twk2,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in2,$in2,$in2,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out2,$in2,$twk2 + vxor $tweak,$tweak,$tmp + + lvx_u $in3,$x30,$inp + sub $len,$len,$taillen + vxor $twk3,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in3,$in3,$in3,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out3,$in3,$twk3 + vxor $tweak,$tweak,$tmp + + lvx_u $in4,$x40,$inp + subi $len,$len,0x60 + vxor $twk4,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in4,$in4,$in4,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out4,$in4,$twk4 + vxor $tweak,$tweak,$tmp + + lvx_u $in5,$x50,$inp + addi $inp,$inp,0x60 + vxor $twk5,$tweak,$rndkey0 + vsrab $tmp,$tweak,$seven # next tweak value + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + le?vperm $in5,$in5,$in5,$leperm + vand $tmp,$tmp,$eighty7 + vxor $out5,$in5,$twk5 + vxor $tweak,$tweak,$tmp + + vxor v31,v31,$rndkey0 + mtctr $rounds + b Loop_xts_dec6x + +.align 5 +Loop_xts_dec6x: + vncipher $out0,$out0,v24 + vncipher $out1,$out1,v24 + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vncipher $out4,$out4,v24 + vncipher $out5,$out5,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vncipher $out0,$out0,v25 + vncipher $out1,$out1,v25 + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vncipher $out4,$out4,v25 + vncipher $out5,$out5,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_xts_dec6x + + subic $len,$len,96 # $len-=96 + vxor $in0,$twk0,v31 # xor with last round key + vncipher $out0,$out0,v24 + vncipher $out1,$out1,v24 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk0,$tweak,$rndkey0 + vaddubm $tweak,$tweak,$tweak + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vsldoi $tmp,$tmp,$tmp,15 + vncipher $out4,$out4,v24 + vncipher $out5,$out5,v24 + + subfe. r0,r0,r0 # borrow?-1:0 + vand $tmp,$tmp,$eighty7 + vncipher $out0,$out0,v25 + vncipher $out1,$out1,v25 + vxor $tweak,$tweak,$tmp + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vxor $in1,$twk1,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk1,$tweak,$rndkey0 + vncipher $out4,$out4,v25 + vncipher $out5,$out5,v25 + + and r0,r0,$len + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vncipher $out0,$out0,v26 + vncipher $out1,$out1,v26 + vand $tmp,$tmp,$eighty7 + vncipher $out2,$out2,v26 + vncipher $out3,$out3,v26 + vxor $tweak,$tweak,$tmp + vncipher $out4,$out4,v26 + vncipher $out5,$out5,v26 + + add $inp,$inp,r0 # $inp is adjusted in such + # way that at exit from the + # loop inX-in5 are loaded + # with last "words" + vxor $in2,$twk2,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk2,$tweak,$rndkey0 + vaddubm $tweak,$tweak,$tweak + vncipher $out0,$out0,v27 + vncipher $out1,$out1,v27 + vsldoi $tmp,$tmp,$tmp,15 + vncipher $out2,$out2,v27 + vncipher $out3,$out3,v27 + vand $tmp,$tmp,$eighty7 + vncipher $out4,$out4,v27 + vncipher $out5,$out5,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vxor $tweak,$tweak,$tmp + vncipher $out0,$out0,v28 + vncipher $out1,$out1,v28 + vxor $in3,$twk3,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk3,$tweak,$rndkey0 + vncipher $out2,$out2,v28 + vncipher $out3,$out3,v28 + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vncipher $out4,$out4,v28 + vncipher $out5,$out5,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + vand $tmp,$tmp,$eighty7 + + vncipher $out0,$out0,v29 + vncipher $out1,$out1,v29 + vxor $tweak,$tweak,$tmp + vncipher $out2,$out2,v29 + vncipher $out3,$out3,v29 + vxor $in4,$twk4,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk4,$tweak,$rndkey0 + vncipher $out4,$out4,v29 + vncipher $out5,$out5,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + + vncipher $out0,$out0,v30 + vncipher $out1,$out1,v30 + vand $tmp,$tmp,$eighty7 + vncipher $out2,$out2,v30 + vncipher $out3,$out3,v30 + vxor $tweak,$tweak,$tmp + vncipher $out4,$out4,v30 + vncipher $out5,$out5,v30 + vxor $in5,$twk5,v31 + vsrab $tmp,$tweak,$seven # next tweak value + vxor $twk5,$tweak,$rndkey0 + + vncipherlast $out0,$out0,$in0 + lvx_u $in0,$x00,$inp # load next input block + vaddubm $tweak,$tweak,$tweak + vsldoi $tmp,$tmp,$tmp,15 + vncipherlast $out1,$out1,$in1 + lvx_u $in1,$x10,$inp + vncipherlast $out2,$out2,$in2 + le?vperm $in0,$in0,$in0,$leperm + lvx_u $in2,$x20,$inp + vand $tmp,$tmp,$eighty7 + vncipherlast $out3,$out3,$in3 + le?vperm $in1,$in1,$in1,$leperm + lvx_u $in3,$x30,$inp + vncipherlast $out4,$out4,$in4 + le?vperm $in2,$in2,$in2,$leperm + lvx_u $in4,$x40,$inp + vxor $tweak,$tweak,$tmp + vncipherlast $out5,$out5,$in5 + le?vperm $in3,$in3,$in3,$leperm + lvx_u $in5,$x50,$inp + addi $inp,$inp,0x60 + le?vperm $in4,$in4,$in4,$leperm + le?vperm $in5,$in5,$in5,$leperm + + le?vperm $out0,$out0,$out0,$leperm + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk0 + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + vxor $out1,$in1,$twk1 + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + vxor $out2,$in2,$twk2 + le?vperm $out4,$out4,$out4,$leperm + stvx_u $out3,$x30,$out + vxor $out3,$in3,$twk3 + le?vperm $out5,$out5,$out5,$leperm + stvx_u $out4,$x40,$out + vxor $out4,$in4,$twk4 + stvx_u $out5,$x50,$out + vxor $out5,$in5,$twk5 + addi $out,$out,0x60 + + mtctr $rounds + beq Loop_xts_dec6x # did $len-=96 borrow? + + addic. $len,$len,0x60 + beq Lxts_dec6x_zero + cmpwi $len,0x20 + blt Lxts_dec6x_one + nop + beq Lxts_dec6x_two + cmpwi $len,0x40 + blt Lxts_dec6x_three + nop + beq Lxts_dec6x_four + +Lxts_dec6x_five: + vxor $out0,$in1,$twk0 + vxor $out1,$in2,$twk1 + vxor $out2,$in3,$twk2 + vxor $out3,$in4,$twk3 + vxor $out4,$in5,$twk4 + + bl _aesp8_xts_dec5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk5 # unused tweak + vxor $twk1,$tweak,$rndkey0 + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk1 + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + le?vperm $out4,$out4,$out4,$leperm + stvx_u $out3,$x30,$out + stvx_u $out4,$x40,$out + addi $out,$out,0x50 + bne Lxts_dec6x_steal + b Lxts_dec6x_done + +.align 4 +Lxts_dec6x_four: + vxor $out0,$in2,$twk0 + vxor $out1,$in3,$twk1 + vxor $out2,$in4,$twk2 + vxor $out3,$in5,$twk3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_dec5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk4 # unused tweak + vmr $twk1,$twk5 + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk5 + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + le?vperm $out3,$out3,$out3,$leperm + stvx_u $out2,$x20,$out + stvx_u $out3,$x30,$out + addi $out,$out,0x40 + bne Lxts_dec6x_steal + b Lxts_dec6x_done + +.align 4 +Lxts_dec6x_three: + vxor $out0,$in3,$twk0 + vxor $out1,$in4,$twk1 + vxor $out2,$in5,$twk2 + vxor $out3,$out3,$out3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_dec5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk3 # unused tweak + vmr $twk1,$twk4 + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk4 + le?vperm $out2,$out2,$out2,$leperm + stvx_u $out1,$x10,$out + stvx_u $out2,$x20,$out + addi $out,$out,0x30 + bne Lxts_dec6x_steal + b Lxts_dec6x_done + +.align 4 +Lxts_dec6x_two: + vxor $out0,$in4,$twk0 + vxor $out1,$in5,$twk1 + vxor $out2,$out2,$out2 + vxor $out3,$out3,$out3 + vxor $out4,$out4,$out4 + + bl _aesp8_xts_dec5x + + le?vperm $out0,$out0,$out0,$leperm + vmr $twk0,$twk2 # unused tweak + vmr $twk1,$twk3 + le?vperm $out1,$out1,$out1,$leperm + stvx_u $out0,$x00,$out # store output + vxor $out0,$in0,$twk3 + stvx_u $out1,$x10,$out + addi $out,$out,0x20 + bne Lxts_dec6x_steal + b Lxts_dec6x_done + +.align 4 +Lxts_dec6x_one: + vxor $out0,$in5,$twk0 + nop +Loop_xts_dec1x: + vncipher $out0,$out0,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vncipher $out0,$out0,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Loop_xts_dec1x + + subi r0,$taillen,1 + vncipher $out0,$out0,v24 + + andi. r0,r0,16 + cmpwi $taillen,0 + vncipher $out0,$out0,v25 + + sub $inp,$inp,r0 + vncipher $out0,$out0,v26 + + lvx_u $in0,0,$inp + vncipher $out0,$out0,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vncipher $out0,$out0,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + + vncipher $out0,$out0,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vxor $twk0,$twk0,v31 + + le?vperm $in0,$in0,$in0,$leperm + vncipher $out0,$out0,v30 + + mtctr $rounds + vncipherlast $out0,$out0,$twk0 + + vmr $twk0,$twk1 # unused tweak + vmr $twk1,$twk2 + le?vperm $out0,$out0,$out0,$leperm + stvx_u $out0,$x00,$out # store output + addi $out,$out,0x10 + vxor $out0,$in0,$twk2 + bne Lxts_dec6x_steal + b Lxts_dec6x_done + +.align 4 +Lxts_dec6x_zero: + cmpwi $taillen,0 + beq Lxts_dec6x_done + + lvx_u $in0,0,$inp + le?vperm $in0,$in0,$in0,$leperm + vxor $out0,$in0,$twk1 +Lxts_dec6x_steal: + vncipher $out0,$out0,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vncipher $out0,$out0,v25 + lvx v25,$x10,$key_ # round[4] + bdnz Lxts_dec6x_steal + + add $inp,$inp,$taillen + vncipher $out0,$out0,v24 + + cmpwi $taillen,0 + vncipher $out0,$out0,v25 + + lvx_u $in0,0,$inp + vncipher $out0,$out0,v26 + + lvsr $inpperm,0,$taillen # $in5 is no more + vncipher $out0,$out0,v27 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vncipher $out0,$out0,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + + vncipher $out0,$out0,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vxor $twk1,$twk1,v31 + + le?vperm $in0,$in0,$in0,$leperm + vncipher $out0,$out0,v30 + + vperm $in0,$in0,$in0,$inpperm + vncipherlast $tmp,$out0,$twk1 + + le?vperm $out0,$tmp,$tmp,$leperm + le?stvx_u $out0,0,$out + be?stvx_u $tmp,0,$out + + vxor $out0,$out0,$out0 + vspltisb $out1,-1 + vperm $out0,$out0,$out1,$inpperm + vsel $out0,$in0,$tmp,$out0 + vxor $out0,$out0,$twk0 + + subi r30,$out,1 + mtctr $taillen +Loop_xts_dec6x_steal: + lbzu r0,1(r30) + stb r0,16(r30) + bdnz Loop_xts_dec6x_steal + + li $taillen,0 + mtctr $rounds + b Loop_xts_dec1x # one more time... + +.align 4 +Lxts_dec6x_done: + ${UCMP}i $ivp,0 + beq Lxts_dec6x_ret + + vxor $tweak,$twk0,$rndkey0 + le?vperm $tweak,$tweak,$tweak,$leperm + stvx_u $tweak,0,$ivp + +Lxts_dec6x_ret: + mtlr r11 + li r10,`$FRAME+15` + li r11,`$FRAME+31` + stvx $seven,r10,$sp # wipe copies of round keys + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + stvx $seven,r10,$sp + addi r10,r10,32 + stvx $seven,r11,$sp + addi r11,r11,32 + + mtspr 256,$vrsave + lvx v20,r10,$sp # ABI says so + addi r10,r10,32 + lvx v21,r11,$sp + addi r11,r11,32 + lvx v22,r10,$sp + addi r10,r10,32 + lvx v23,r11,$sp + addi r11,r11,32 + lvx v24,r10,$sp + addi r10,r10,32 + lvx v25,r11,$sp + addi r11,r11,32 + lvx v26,r10,$sp + addi r10,r10,32 + lvx v27,r11,$sp + addi r11,r11,32 + lvx v28,r10,$sp + addi r10,r10,32 + lvx v29,r11,$sp + addi r11,r11,32 + lvx v30,r10,$sp + lvx v31,r11,$sp + $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp) + $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp) + $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp) + $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp) + $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp) + $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp) + addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T` + blr + .long 0 + .byte 0,12,0x04,1,0x80,6,6,0 + .long 0 + +.align 5 +_aesp8_xts_dec5x: + vncipher $out0,$out0,v24 + vncipher $out1,$out1,v24 + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vncipher $out4,$out4,v24 + lvx v24,$x20,$key_ # round[3] + addi $key_,$key_,0x20 + + vncipher $out0,$out0,v25 + vncipher $out1,$out1,v25 + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vncipher $out4,$out4,v25 + lvx v25,$x10,$key_ # round[4] + bdnz _aesp8_xts_dec5x + + subi r0,$taillen,1 + vncipher $out0,$out0,v24 + vncipher $out1,$out1,v24 + vncipher $out2,$out2,v24 + vncipher $out3,$out3,v24 + vncipher $out4,$out4,v24 + + andi. r0,r0,16 + cmpwi $taillen,0 + vncipher $out0,$out0,v25 + vncipher $out1,$out1,v25 + vncipher $out2,$out2,v25 + vncipher $out3,$out3,v25 + vncipher $out4,$out4,v25 + vxor $twk0,$twk0,v31 + + sub $inp,$inp,r0 + vncipher $out0,$out0,v26 + vncipher $out1,$out1,v26 + vncipher $out2,$out2,v26 + vncipher $out3,$out3,v26 + vncipher $out4,$out4,v26 + vxor $in1,$twk1,v31 + + vncipher $out0,$out0,v27 + lvx_u $in0,0,$inp + vncipher $out1,$out1,v27 + vncipher $out2,$out2,v27 + vncipher $out3,$out3,v27 + vncipher $out4,$out4,v27 + vxor $in2,$twk2,v31 + + addi $key_,$sp,$FRAME+15 # rewind $key_ + vncipher $out0,$out0,v28 + vncipher $out1,$out1,v28 + vncipher $out2,$out2,v28 + vncipher $out3,$out3,v28 + vncipher $out4,$out4,v28 + lvx v24,$x00,$key_ # re-pre-load round[1] + vxor $in3,$twk3,v31 + + vncipher $out0,$out0,v29 + le?vperm $in0,$in0,$in0,$leperm + vncipher $out1,$out1,v29 + vncipher $out2,$out2,v29 + vncipher $out3,$out3,v29 + vncipher $out4,$out4,v29 + lvx v25,$x10,$key_ # re-pre-load round[2] + vxor $in4,$twk4,v31 + + vncipher $out0,$out0,v30 + vncipher $out1,$out1,v30 + vncipher $out2,$out2,v30 + vncipher $out3,$out3,v30 + vncipher $out4,$out4,v30 + + vncipherlast $out0,$out0,$twk0 + vncipherlast $out1,$out1,$in1 + vncipherlast $out2,$out2,$in2 + vncipherlast $out3,$out3,$in3 + vncipherlast $out4,$out4,$in4 + mtctr $rounds + blr + .long 0 + .byte 0,12,0x14,0,0,0,0,0 +___ +}} }}} + my $consts=1; foreach(split("\n",$code)) { s/\`([^\`]*)\`/eval($1)/geo; @@ -1898,7 +3757,7 @@ foreach(split("\n",$code)) { if ($flavour =~ /le$/o) { SWITCH: for($conv) { /\?inv/ && do { @bytes=map($_^0xf,@bytes); last; }; - /\?rev/ && do { @bytes=reverse(@bytes); last; }; + /\?rev/ && do { @bytes=reverse(@bytes); last; }; } } diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c index e163d5770..31a98dc6f 100644 --- a/drivers/crypto/vmx/vmx.c +++ b/drivers/crypto/vmx/vmx.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -31,10 +32,12 @@ extern struct shash_alg p8_ghash_alg; extern struct crypto_alg p8_aes_alg; extern struct crypto_alg p8_aes_cbc_alg; extern struct crypto_alg p8_aes_ctr_alg; +extern struct crypto_alg p8_aes_xts_alg; static struct crypto_alg *algs[] = { &p8_aes_alg, &p8_aes_cbc_alg, &p8_aes_ctr_alg, + &p8_aes_xts_alg, NULL, }; @@ -43,9 +46,6 @@ int __init p8_init(void) int ret = 0; struct crypto_alg **alg_it; - if (!(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) - return -ENODEV; - for (alg_it = algs; *alg_it; alg_it++) { ret = crypto_register_alg(*alg_it); printk(KERN_INFO "crypto_register_alg '%s' = %d\n", @@ -78,7 +78,7 @@ void __exit p8_exit(void) crypto_unregister_shash(&p8_ghash_alg); } -module_init(p8_init); +module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, p8_init); module_exit(p8_exit); MODULE_AUTHOR("Marcelo Cerri"); diff --git a/drivers/dax/dax.c b/drivers/dax/dax.c index b891a129b..29f600f2c 100644 --- a/drivers/dax/dax.c +++ b/drivers/dax/dax.c @@ -211,11 +211,9 @@ int devm_create_dax_dev(struct dax_region *dax_region, struct resource *res, } dax_dev->dev = dev; - rc = devm_add_action(dax_region->dev, unregister_dax_dev, dev); - if (rc) { - unregister_dax_dev(dev); + rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_dev, dev); + if (rc) return rc; - } return 0; @@ -461,7 +459,7 @@ static int __dax_dev_pmd_fault(struct dax_dev *dax_dev, } pgoff = linear_page_index(vma, pmd_addr); - phys = pgoff_to_phys(dax_dev, pgoff, PAGE_SIZE); + phys = pgoff_to_phys(dax_dev, pgoff, PMD_SIZE); if (phys == -1) { dev_dbg(dev, "%s: phys_to_pgoff(%#lx) failed\n", __func__, pgoff); diff --git a/drivers/dax/pmem.c b/drivers/dax/pmem.c index 82e6743c4..1f01e98c8 100644 --- a/drivers/dax/pmem.c +++ b/drivers/dax/pmem.c @@ -102,21 +102,19 @@ static int dax_pmem_probe(struct device *dev) if (rc) return rc; - rc = devm_add_action(dev, dax_pmem_percpu_exit, &dax_pmem->ref); - if (rc) { - dax_pmem_percpu_exit(&dax_pmem->ref); + rc = devm_add_action_or_reset(dev, dax_pmem_percpu_exit, + &dax_pmem->ref); + if (rc) return rc; - } addr = devm_memremap_pages(dev, &res, &dax_pmem->ref, altmap); if (IS_ERR(addr)) return PTR_ERR(addr); - rc = devm_add_action(dev, dax_pmem_percpu_kill, &dax_pmem->ref); - if (rc) { - dax_pmem_percpu_kill(&dax_pmem->ref); + rc = devm_add_action_or_reset(dev, dax_pmem_percpu_kill, + &dax_pmem->ref); + if (rc) return rc; - } /* adjust the dax_region resource to the start of data */ res.start += le64_to_cpu(pfn_sb->dataoff); diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig index 78dac0e9d..a5be56ec5 100644 --- a/drivers/devfreq/Kconfig +++ b/drivers/devfreq/Kconfig @@ -75,7 +75,7 @@ config DEVFREQ_GOV_PASSIVE comment "DEVFREQ Drivers" config ARM_EXYNOS_BUS_DEVFREQ - bool "ARM EXYNOS Generic Memory Bus DEVFREQ Driver" + tristate "ARM EXYNOS Generic Memory Bus DEVFREQ Driver" depends on ARCH_EXYNOS select DEVFREQ_GOV_SIMPLE_ONDEMAND select DEVFREQ_GOV_PASSIVE diff --git a/drivers/devfreq/devfreq-event.c b/drivers/devfreq/devfreq-event.c index 39b048eda..9aea2c7ec 100644 --- a/drivers/devfreq/devfreq-event.c +++ b/drivers/devfreq/devfreq-event.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -481,13 +481,3 @@ static int __init devfreq_event_init(void) return 0; } subsys_initcall(devfreq_event_init); - -static void __exit devfreq_event_exit(void) -{ - class_destroy(devfreq_event_class); -} -module_exit(devfreq_event_exit); - -MODULE_AUTHOR("Chanwoo Choi "); -MODULE_DESCRIPTION("DEVFREQ-Event class support"); -MODULE_LICENSE("GPL"); diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index e92418fac..478006b77 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -707,10 +707,12 @@ struct devfreq *devfreq_get_devfreq_by_phandle(struct device *dev, int index) if (devfreq->dev.parent && devfreq->dev.parent->of_node == node) { mutex_unlock(&devfreq_list_lock); + of_node_put(node); return devfreq; } } mutex_unlock(&devfreq_list_lock); + of_node_put(node); return ERR_PTR(-EPROBE_DEFER); } @@ -1199,13 +1201,6 @@ static int __init devfreq_init(void) } subsys_initcall(devfreq_init); -static void __exit devfreq_exit(void) -{ - class_destroy(devfreq_class); - destroy_workqueue(devfreq_wq); -} -module_exit(devfreq_exit); - /* * The followings are helper functions for devfreq user device drivers with * OPP framework. @@ -1471,7 +1466,3 @@ void devm_devfreq_unregister_notifier(struct device *dev, devm_devfreq_dev_match, devfreq)); } EXPORT_SYMBOL(devm_devfreq_unregister_notifier); - -MODULE_AUTHOR("MyungJoo Ham "); -MODULE_DESCRIPTION("devfreq class support"); -MODULE_LICENSE("GPL"); diff --git a/drivers/devfreq/event/Kconfig b/drivers/devfreq/event/Kconfig index 1e8b4f469..eb6f74a2b 100644 --- a/drivers/devfreq/event/Kconfig +++ b/drivers/devfreq/event/Kconfig @@ -14,7 +14,7 @@ menuconfig PM_DEVFREQ_EVENT if PM_DEVFREQ_EVENT config DEVFREQ_EVENT_EXYNOS_NOCP - bool "EXYNOS NoC (Network On Chip) Probe DEVFREQ event Driver" + tristate "EXYNOS NoC (Network On Chip) Probe DEVFREQ event Driver" depends on ARCH_EXYNOS select PM_OPP help @@ -22,7 +22,7 @@ config DEVFREQ_EVENT_EXYNOS_NOCP (Network on Chip) Probe counters to measure the bandwidth of AXI bus. config DEVFREQ_EVENT_EXYNOS_PPMU - bool "EXYNOS PPMU (Platform Performance Monitoring Unit) DEVFREQ event Driver" + tristate "EXYNOS PPMU (Platform Performance Monitoring Unit) DEVFREQ event Driver" depends on ARCH_EXYNOS select PM_OPP help diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c index f312485f1..845bf25fb 100644 --- a/drivers/devfreq/event/exynos-ppmu.c +++ b/drivers/devfreq/event/exynos-ppmu.c @@ -482,7 +482,8 @@ static int exynos_ppmu_probe(struct platform_device *pdev) if (!info->edev) { dev_err(&pdev->dev, "failed to allocate memory devfreq-event devices\n"); - return -ENOMEM; + ret = -ENOMEM; + goto err; } edev = info->edev; platform_set_drvdata(pdev, info); diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c index 2363d0a18..29866f7e6 100644 --- a/drivers/devfreq/exynos-bus.c +++ b/drivers/devfreq/exynos-bus.c @@ -383,7 +383,7 @@ err_clk: static int exynos_bus_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; + struct device_node *np = dev->of_node, *node; struct devfreq_dev_profile *profile; struct devfreq_simple_ondemand_data *ondemand_data; struct devfreq_passive_data *passive_data; @@ -407,7 +407,7 @@ static int exynos_bus_probe(struct platform_device *pdev) /* Parse the device-tree to get the resource information */ ret = exynos_bus_parse_of(np, bus); if (ret < 0) - goto err; + return ret; profile = devm_kzalloc(dev, sizeof(*profile), GFP_KERNEL); if (!profile) { @@ -415,10 +415,13 @@ static int exynos_bus_probe(struct platform_device *pdev) goto err; } - if (of_parse_phandle(dev->of_node, "devfreq", 0)) + node = of_parse_phandle(dev->of_node, "devfreq", 0); + if (node) { + of_node_put(node); goto passive; - else + } else { ret = exynos_bus_parent_parse_of(np, bus); + } if (ret < 0) goto err; diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig index 9824bc4ad..25bcfa0b4 100644 --- a/drivers/dma-buf/Kconfig +++ b/drivers/dma-buf/Kconfig @@ -1,11 +1,20 @@ menu "DMABUF options" config SYNC_FILE - bool "sync_file support for fences" + bool "Explicit Synchronization Framework" default n select ANON_INODES select DMA_SHARED_BUFFER ---help--- - This option enables the fence framework synchronization to export - sync_files to userspace that can represent one or more fences. + The Sync File Framework adds explicit syncronization via + userspace. It enables send/receive 'struct fence' objects to/from + userspace via Sync File fds for synchronization between drivers via + userspace components. It has been ported from Android. + + The first and main user for this is graphics in which a fence is + associated with a buffer. When a job is submitted to the GPU a fence + is attached to the buffer and is transferred via userspace, using Sync + Files fds, to the DRM driver for example. More details at + Documentation/sync_file.txt. + endmenu diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile index 4a424eca7..f353db213 100644 --- a/drivers/dma-buf/Makefile +++ b/drivers/dma-buf/Makefile @@ -1,2 +1,2 @@ -obj-y := dma-buf.o fence.o reservation.o seqno-fence.o +obj-y := dma-buf.o fence.o reservation.o seqno-fence.o fence-array.o obj-$(CONFIG_SYNC_FILE) += sync_file.o diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 6355ab38d..ddaee60ae 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -334,6 +334,7 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) struct reservation_object *resv = exp_info->resv; struct file *file; size_t alloc_size = sizeof(struct dma_buf); + int ret; if (!exp_info->resv) alloc_size += sizeof(struct reservation_object); @@ -357,8 +358,8 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) dmabuf = kzalloc(alloc_size, GFP_KERNEL); if (!dmabuf) { - module_put(exp_info->owner); - return ERR_PTR(-ENOMEM); + ret = -ENOMEM; + goto err_module; } dmabuf->priv = exp_info->priv; @@ -379,8 +380,8 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) file = anon_inode_getfile("dmabuf", &dma_buf_fops, dmabuf, exp_info->flags); if (IS_ERR(file)) { - kfree(dmabuf); - return ERR_CAST(file); + ret = PTR_ERR(file); + goto err_dmabuf; } file->f_mode |= FMODE_LSEEK; @@ -394,6 +395,12 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) mutex_unlock(&db_list.lock); return dmabuf; + +err_dmabuf: + kfree(dmabuf); +err_module: + module_put(exp_info->owner); + return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(dma_buf_export); @@ -824,7 +831,7 @@ void dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) EXPORT_SYMBOL_GPL(dma_buf_vunmap); #ifdef CONFIG_DEBUG_FS -static int dma_buf_describe(struct seq_file *s) +static int dma_buf_debug_show(struct seq_file *s, void *unused) { int ret; struct dma_buf *buf_obj; @@ -879,17 +886,9 @@ static int dma_buf_describe(struct seq_file *s) return 0; } -static int dma_buf_show(struct seq_file *s, void *unused) -{ - void (*func)(struct seq_file *) = s->private; - - func(s); - return 0; -} - static int dma_buf_debug_open(struct inode *inode, struct file *file) { - return single_open(file, dma_buf_show, inode->i_private); + return single_open(file, dma_buf_debug_show, NULL); } static const struct file_operations dma_buf_debug_fops = { @@ -903,20 +902,23 @@ static struct dentry *dma_buf_debugfs_dir; static int dma_buf_init_debugfs(void) { + struct dentry *d; int err = 0; - dma_buf_debugfs_dir = debugfs_create_dir("dma_buf", NULL); + d = debugfs_create_dir("dma_buf", NULL); + if (IS_ERR(d)) + return PTR_ERR(d); - if (IS_ERR(dma_buf_debugfs_dir)) { - err = PTR_ERR(dma_buf_debugfs_dir); - dma_buf_debugfs_dir = NULL; - return err; - } - - err = dma_buf_debugfs_create_file("bufinfo", dma_buf_describe); + dma_buf_debugfs_dir = d; - if (err) + d = debugfs_create_file("bufinfo", S_IRUGO, dma_buf_debugfs_dir, + NULL, &dma_buf_debug_fops); + if (IS_ERR(d)) { pr_debug("dma_buf: debugfs: failed to create node bufinfo\n"); + debugfs_remove_recursive(dma_buf_debugfs_dir); + dma_buf_debugfs_dir = NULL; + err = PTR_ERR(d); + } return err; } @@ -926,17 +928,6 @@ static void dma_buf_uninit_debugfs(void) if (dma_buf_debugfs_dir) debugfs_remove_recursive(dma_buf_debugfs_dir); } - -int dma_buf_debugfs_create_file(const char *name, - int (*write)(struct seq_file *)) -{ - struct dentry *d; - - d = debugfs_create_file(name, S_IRUGO, dma_buf_debugfs_dir, - write, &dma_buf_debug_fops); - - return PTR_ERR_OR_ZERO(d); -} #else static inline int dma_buf_init_debugfs(void) { diff --git a/drivers/dma-buf/fence-array.c b/drivers/dma-buf/fence-array.c new file mode 100644 index 000000000..a8731c853 --- /dev/null +++ b/drivers/dma-buf/fence-array.c @@ -0,0 +1,144 @@ +/* + * fence-array: aggregate fences to be waited together + * + * Copyright (C) 2016 Collabora Ltd + * Copyright (C) 2016 Advanced Micro Devices, Inc. + * Authors: + * Gustavo Padovan + * Christian König + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include +#include + +static void fence_array_cb_func(struct fence *f, struct fence_cb *cb); + +static const char *fence_array_get_driver_name(struct fence *fence) +{ + return "fence_array"; +} + +static const char *fence_array_get_timeline_name(struct fence *fence) +{ + return "unbound"; +} + +static void fence_array_cb_func(struct fence *f, struct fence_cb *cb) +{ + struct fence_array_cb *array_cb = + container_of(cb, struct fence_array_cb, cb); + struct fence_array *array = array_cb->array; + + if (atomic_dec_and_test(&array->num_pending)) + fence_signal(&array->base); + fence_put(&array->base); +} + +static bool fence_array_enable_signaling(struct fence *fence) +{ + struct fence_array *array = to_fence_array(fence); + struct fence_array_cb *cb = (void *)(&array[1]); + unsigned i; + + for (i = 0; i < array->num_fences; ++i) { + cb[i].array = array; + /* + * As we may report that the fence is signaled before all + * callbacks are complete, we need to take an additional + * reference count on the array so that we do not free it too + * early. The core fence handling will only hold the reference + * until we signal the array as complete (but that is now + * insufficient). + */ + fence_get(&array->base); + if (fence_add_callback(array->fences[i], &cb[i].cb, + fence_array_cb_func)) { + fence_put(&array->base); + if (atomic_dec_and_test(&array->num_pending)) + return false; + } + } + + return true; +} + +static bool fence_array_signaled(struct fence *fence) +{ + struct fence_array *array = to_fence_array(fence); + + return atomic_read(&array->num_pending) <= 0; +} + +static void fence_array_release(struct fence *fence) +{ + struct fence_array *array = to_fence_array(fence); + unsigned i; + + for (i = 0; i < array->num_fences; ++i) + fence_put(array->fences[i]); + + kfree(array->fences); + fence_free(fence); +} + +const struct fence_ops fence_array_ops = { + .get_driver_name = fence_array_get_driver_name, + .get_timeline_name = fence_array_get_timeline_name, + .enable_signaling = fence_array_enable_signaling, + .signaled = fence_array_signaled, + .wait = fence_default_wait, + .release = fence_array_release, +}; + +/** + * fence_array_create - Create a custom fence array + * @num_fences: [in] number of fences to add in the array + * @fences: [in] array containing the fences + * @context: [in] fence context to use + * @seqno: [in] sequence number to use + * @signal_on_any [in] signal on any fence in the array + * + * Allocate a fence_array object and initialize the base fence with fence_init(). + * In case of error it returns NULL. + * + * The caller should allocte the fences array with num_fences size + * and fill it with the fences it wants to add to the object. Ownership of this + * array is take and fence_put() is used on each fence on release. + * + * If @signal_on_any is true the fence array signals if any fence in the array + * signals, otherwise it signals when all fences in the array signal. + */ +struct fence_array *fence_array_create(int num_fences, struct fence **fences, + u64 context, unsigned seqno, + bool signal_on_any) +{ + struct fence_array *array; + size_t size = sizeof(*array); + + /* Allocate the callback structures behind the array. */ + size += num_fences * sizeof(struct fence_array_cb); + array = kzalloc(size, GFP_KERNEL); + if (!array) + return NULL; + + spin_lock_init(&array->lock); + fence_init(&array->base, &fence_array_ops, &array->lock, + context, seqno); + + array->num_fences = num_fences; + atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences); + array->fences = fences; + + return array; +} +EXPORT_SYMBOL(fence_array_create); diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/fence.c index 7b05dbe9b..4d51f9e83 100644 --- a/drivers/dma-buf/fence.c +++ b/drivers/dma-buf/fence.c @@ -35,7 +35,7 @@ EXPORT_TRACEPOINT_SYMBOL(fence_emit); * context or not. One device can have multiple separate contexts, * and they're used if some engine can run independently of another. */ -static atomic_t fence_context_counter = ATOMIC_INIT(0); +static atomic64_t fence_context_counter = ATOMIC64_INIT(0); /** * fence_context_alloc - allocate an array of fence contexts @@ -44,10 +44,10 @@ static atomic_t fence_context_counter = ATOMIC_INIT(0); * This function will return the first index of the number of fences allocated. * The fence context is used for setting fence->context to a unique number. */ -unsigned fence_context_alloc(unsigned num) +u64 fence_context_alloc(unsigned num) { BUG_ON(!num); - return atomic_add_return(num, &fence_context_counter) - num; + return atomic64_add_return(num, &fence_context_counter) - num; } EXPORT_SYMBOL(fence_context_alloc); @@ -513,7 +513,7 @@ EXPORT_SYMBOL(fence_wait_any_timeout); */ void fence_init(struct fence *fence, const struct fence_ops *ops, - spinlock_t *lock, unsigned context, unsigned seqno) + spinlock_t *lock, u64 context, unsigned seqno) { BUG_ON(!lock); BUG_ON(!ops || !ops->wait || !ops->enable_signaling || diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c index f08cf2d83..9aaa608df 100644 --- a/drivers/dma-buf/sync_file.c +++ b/drivers/dma-buf/sync_file.c @@ -82,7 +82,7 @@ struct sync_file *sync_file_create(struct fence *fence) sync_file->num_fences = 1; atomic_set(&sync_file->status, 1); - snprintf(sync_file->name, sizeof(sync_file->name), "%s-%s%d-%d", + snprintf(sync_file->name, sizeof(sync_file->name), "%s-%s%llu-%d", fence->ops->get_driver_name(fence), fence->ops->get_timeline_name(fence), fence->context, fence->seqno); diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 8c98779a1..739f797b4 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -339,6 +339,20 @@ config MV_XOR ---help--- Enable support for the Marvell XOR engine. +config MV_XOR_V2 + bool "Marvell XOR engine version 2 support " + depends on ARM64 + select DMA_ENGINE + select DMA_ENGINE_RAID + select ASYNC_TX_ENABLE_CHANNEL_SWITCH + select GENERIC_MSI_IRQ_DOMAIN + ---help--- + Enable support for the Marvell version 2 XOR engine. + + This engine provides acceleration for copy, XOR and RAID6 + operations, and is available on Marvell Armada 7K and 8K + platforms. + config MXS_DMA bool "MXS DMA support" depends on SOC_IMX23 || SOC_IMX28 || SOC_IMX6Q || SOC_IMX6UL @@ -519,19 +533,31 @@ config XGENE_DMA help Enable support for the APM X-Gene SoC DMA engine. -config XILINX_VDMA - tristate "Xilinx AXI VDMA Engine" +config XILINX_DMA + tristate "Xilinx AXI DMAS Engine" depends on (ARCH_ZYNQ || MICROBLAZE || ARM64) select DMA_ENGINE help Enable support for Xilinx AXI VDMA Soft IP. - This engine provides high-bandwidth direct memory access + AXI VDMA engine provides high-bandwidth direct memory access between memory and AXI4-Stream video type target peripherals including peripherals which support AXI4- Stream Video Protocol. It has two stream interfaces/ channels, Memory Mapped to Stream (MM2S) and Stream to Memory Mapped (S2MM) for the data transfers. + AXI CDMA engine provides high-bandwidth direct memory access + between a memory-mapped source address and a memory-mapped + destination address. + AXI DMA engine provides high-bandwidth one dimensional direct + memory access between memory and AXI4-Stream target peripherals. + +config XILINX_ZYNQMP_DMA + tristate "Xilinx ZynqMP DMA Engine" + depends on (ARCH_ZYNQ || MICROBLAZE || ARM64) + select DMA_ENGINE + help + Enable support for Xilinx ZynqMP DMA controller. config ZX_DMA tristate "ZTE ZX296702 DMA support" diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 614f28b0b..e4dc9cac7 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -45,6 +45,7 @@ obj-$(CONFIG_MMP_TDMA) += mmp_tdma.o obj-$(CONFIG_MOXART_DMA) += moxart-dma.o obj-$(CONFIG_MPC512X_DMA) += mpc512x_dma.o obj-$(CONFIG_MV_XOR) += mv_xor.o +obj-$(CONFIG_MV_XOR_V2) += mv_xor_v2.o obj-$(CONFIG_MXS_DMA) += mxs-dma.o obj-$(CONFIG_MX3_IPU) += ipu/ obj-$(CONFIG_NBPFAXI_DMA) += nbpfaxi.o diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index 81db1c481..939a7c31f 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -1443,8 +1443,6 @@ static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); if (!dsg) { pl08x_free_txd(pl08x, txd); - dev_err(&pl08x->adev->dev, "%s no memory for pl080 sg\n", - __func__); return NULL; } list_add_tail(&dsg->node, &txd->dsg_list); @@ -1901,11 +1899,8 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, */ for (i = 0; i < channels; i++) { chan = kzalloc(sizeof(*chan), GFP_KERNEL); - if (!chan) { - dev_err(&pl08x->adev->dev, - "%s no memory for channel\n", __func__); + if (!chan) return -ENOMEM; - } chan->host = pl08x; chan->state = PL08X_CHAN_IDLE; @@ -2360,9 +2355,6 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)), GFP_KERNEL); if (!pl08x->phy_chans) { - dev_err(&adev->dev, "%s failed to allocate " - "physical channel holders\n", - __func__); ret = -ENOMEM; goto out_no_phychans; } diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index 75bd6621d..832cbd647 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -456,7 +456,7 @@ static struct at_xdmac_desc *at_xdmac_alloc_desc(struct dma_chan *chan, return desc; } -void at_xdmac_init_used_desc(struct at_xdmac_desc *desc) +static void at_xdmac_init_used_desc(struct at_xdmac_desc *desc) { memset(&desc->lld, 0, sizeof(desc->lld)); INIT_LIST_HEAD(&desc->descs_list); @@ -1195,14 +1195,14 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, desc->lld.mbr_cfg = chan_cc; dev_dbg(chan2dev(chan), - "%s: lld: mbr_da=%pad, mbr_ds=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", - __func__, &desc->lld.mbr_da, &desc->lld.mbr_ds, desc->lld.mbr_ubc, + "%s: lld: mbr_da=%pad, mbr_ds=0x%08x, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n", + __func__, &desc->lld.mbr_da, desc->lld.mbr_ds, desc->lld.mbr_ubc, desc->lld.mbr_cfg); return desc; } -struct dma_async_tx_descriptor * +static struct dma_async_tx_descriptor * at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, size_t len, unsigned long flags) { @@ -2067,7 +2067,7 @@ err_dma_unregister: err_clk_disable: clk_disable_unprepare(atxdmac->clk); err_free_irq: - free_irq(atxdmac->irq, atxdmac->dma.dev); + free_irq(atxdmac->irq, atxdmac); return ret; } @@ -2081,7 +2081,7 @@ static int at_xdmac_remove(struct platform_device *pdev) dma_async_device_unregister(&atxdmac->dma); clk_disable_unprepare(atxdmac->clk); - free_irq(atxdmac->irq, atxdmac->dma.dev); + free_irq(atxdmac->irq, atxdmac); for (i = 0; i < atxdmac->dma.chancnt; i++) { struct at_xdmac_chan *atchan = &atxdmac->chan[i]; diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c index 6149b27c3..e18dc596c 100644 --- a/drivers/dma/bcm2835-dma.c +++ b/drivers/dma/bcm2835-dma.c @@ -393,11 +393,12 @@ static void bcm2835_dma_fill_cb_chain_with_sg( unsigned int sg_len) { struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); - size_t max_len = bcm2835_dma_max_frame_length(c); - unsigned int i, len; + size_t len, max_len; + unsigned int i; dma_addr_t addr; struct scatterlist *sgent; + max_len = bcm2835_dma_max_frame_length(c); for_each_sg(sgl, sgent, sg_len, i) { for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent); len > 0; @@ -613,7 +614,7 @@ static void bcm2835_dma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&c->vc.lock, flags); } -struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy( +static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy( struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, size_t len, unsigned long flags) { diff --git a/drivers/dma/bestcomm/bestcomm.c b/drivers/dma/bestcomm/bestcomm.c index 180fedb41..7ce843723 100644 --- a/drivers/dma/bestcomm/bestcomm.c +++ b/drivers/dma/bestcomm/bestcomm.c @@ -397,8 +397,6 @@ static int mpc52xx_bcom_probe(struct platform_device *op) /* Get a clean struct */ bcom_eng = kzalloc(sizeof(struct bcom_engine), GFP_KERNEL); if (!bcom_eng) { - printk(KERN_ERR DRIVER_NAME ": " - "Can't allocate state structure\n"); rv = -ENOMEM; goto error_sramclean; } diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c index c340ca9bd..e4acd63e4 100644 --- a/drivers/dma/coh901318.c +++ b/drivers/dma/coh901318.c @@ -266,7 +266,7 @@ static int dma_memcpy_channels[] = { COH901318_CX_CTRL_DDMA_LEGACY | \ COH901318_CX_CTRL_PRDD_SOURCE) -const struct coh_dma_channel chan_config[U300_DMA_CHANNELS] = { +static const struct coh_dma_channel chan_config[U300_DMA_CHANNELS] = { { .number = U300_DMA_MSL_TX_0, .name = "MSL TX 0", @@ -1280,6 +1280,7 @@ struct coh901318_desc { struct coh901318_base { struct device *dev; void __iomem *virtbase; + unsigned int irq; struct coh901318_pool pool; struct powersave pm; struct dma_device dma_slave; @@ -1364,7 +1365,6 @@ static int coh901318_debugfs_read(struct file *file, char __user *buf, } static const struct file_operations coh901318_debugfs_status_operations = { - .owner = THIS_MODULE, .open = simple_open, .read = coh901318_debugfs_read, .llseek = default_llseek, @@ -2422,7 +2422,7 @@ coh901318_tx_status(struct dma_chan *chan, dma_cookie_t cookie, enum dma_status ret; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE) + if (ret == DMA_COMPLETE || !txstate) return ret; dma_set_residue(txstate, coh901318_get_bytes_left(chan)); @@ -2680,6 +2680,8 @@ static int __init coh901318_probe(struct platform_device *pdev) if (err) return err; + base->irq = irq; + err = coh901318_pool_create(&base->pool, &pdev->dev, sizeof(struct coh901318_lli), 32); @@ -2755,11 +2757,31 @@ static int __init coh901318_probe(struct platform_device *pdev) coh901318_pool_destroy(&base->pool); return err; } +static void coh901318_base_remove(struct coh901318_base *base, const int *pick_chans) +{ + int chans_i; + int i = 0; + struct coh901318_chan *cohc; + + for (chans_i = 0; pick_chans[chans_i] != -1; chans_i += 2) { + for (i = pick_chans[chans_i]; i <= pick_chans[chans_i+1]; i++) { + cohc = &base->chans[i]; + + tasklet_kill(&cohc->tasklet); + } + } + +} static int coh901318_remove(struct platform_device *pdev) { struct coh901318_base *base = platform_get_drvdata(pdev); + devm_free_irq(&pdev->dev, base->irq, base); + + coh901318_base_remove(base, dma_slave_channels); + coh901318_base_remove(base, dma_memcpy_channels); + of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&base->dma_memcpy); dma_async_device_unregister(&base->dma_slave); @@ -2780,13 +2802,13 @@ static struct platform_driver coh901318_driver = { }, }; -int __init coh901318_init(void) +static int __init coh901318_init(void) { return platform_driver_probe(&coh901318_driver, coh901318_probe); } subsys_initcall(coh901318_init); -void __exit coh901318_exit(void) +static void __exit coh901318_exit(void) { platform_driver_unregister(&coh901318_driver); } diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c index ceedafbd2..4b2317426 100644 --- a/drivers/dma/cppi41.c +++ b/drivers/dma/cppi41.c @@ -497,16 +497,13 @@ static struct dma_async_tx_descriptor *cppi41_dma_prep_slave_sg( struct cppi41_desc *d; struct scatterlist *sg; unsigned int i; - unsigned int num; - num = 0; d = c->desc; for_each_sg(sgl, sg, sg_len, i) { u32 addr; u32 len; /* We need to use more than one desc once musb supports sg */ - BUG_ON(num > 0); addr = lower_32_bits(sg_dma_address(sg)); len = sg_dma_len(sg); diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index c34680943..7f0b9aa15 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -270,6 +270,9 @@ static irqreturn_t axi_dmac_interrupt_handler(int irq, void *devid) unsigned int pending; pending = axi_dmac_read(dmac, AXI_DMAC_REG_IRQ_PENDING); + if (!pending) + return IRQ_NONE; + axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_PENDING, pending); spin_lock(&dmac->chan.vchan.lock); @@ -579,7 +582,9 @@ static int axi_dmac_probe(struct platform_device *pdev) return -ENOMEM; dmac->irq = platform_get_irq(pdev, 0); - if (dmac->irq <= 0) + if (dmac->irq < 0) + return dmac->irq; + if (dmac->irq == 0) return -EINVAL; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -683,6 +688,7 @@ static const struct of_device_id axi_dmac_of_match_table[] = { { .compatible = "adi,axi-dmac-1.00.a" }, { }, }; +MODULE_DEVICE_TABLE(of, axi_dmac_of_match_table); static struct platform_driver axi_dmac_driver = { .driver = { diff --git a/drivers/dma/dma-jz4740.c b/drivers/dma/dma-jz4740.c index 7638b24ce..9689b36c0 100644 --- a/drivers/dma/dma-jz4740.c +++ b/drivers/dma/dma-jz4740.c @@ -573,12 +573,26 @@ err_unregister: return ret; } +static void jz4740_cleanup_vchan(struct dma_device *dmadev) +{ + struct jz4740_dmaengine_chan *chan, *_chan; + + list_for_each_entry_safe(chan, _chan, + &dmadev->channels, vchan.chan.device_node) { + list_del(&chan->vchan.chan.device_node); + tasklet_kill(&chan->vchan.task); + } +} + + static int jz4740_dma_remove(struct platform_device *pdev) { struct jz4740_dma_dev *dmadev = platform_get_drvdata(pdev); int irq = platform_get_irq(pdev, 0); free_irq(irq, dmadev); + + jz4740_cleanup_vchan(&dmadev->ddev); dma_async_device_unregister(&dmadev->ddev); clk_disable_unprepare(dmadev->clk); diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index b8576fd6b..1245db543 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -51,6 +51,16 @@ module_param(iterations, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(iterations, "Iterations before stopping test (default: infinite)"); +static unsigned int sg_buffers = 1; +module_param(sg_buffers, uint, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(sg_buffers, + "Number of scatter gather buffers (default: 1)"); + +static unsigned int dmatest = 1; +module_param(dmatest, uint, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(dmatest, + "dmatest 0-memcpy 1-slave_sg (default: 1)"); + static unsigned int xor_sources = 3; module_param(xor_sources, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(xor_sources, @@ -431,6 +441,8 @@ static int dmatest_func(void *data) dev = chan->device; if (thread->type == DMA_MEMCPY) src_cnt = dst_cnt = 1; + else if (thread->type == DMA_SG) + src_cnt = dst_cnt = sg_buffers; else if (thread->type == DMA_XOR) { /* force odd to ensure dst = src */ src_cnt = min_odd(params->xor_sources | 1, dev->max_xor); @@ -485,6 +497,8 @@ static int dmatest_func(void *data) dma_addr_t *dsts; unsigned int src_off, dst_off, len; u8 align = 0; + struct scatterlist tx_sg[src_cnt]; + struct scatterlist rx_sg[src_cnt]; total_tests++; @@ -577,10 +591,22 @@ static int dmatest_func(void *data) um->bidi_cnt++; } + sg_init_table(tx_sg, src_cnt); + sg_init_table(rx_sg, src_cnt); + for (i = 0; i < src_cnt; i++) { + sg_dma_address(&rx_sg[i]) = srcs[i]; + sg_dma_address(&tx_sg[i]) = dsts[i] + dst_off; + sg_dma_len(&tx_sg[i]) = len; + sg_dma_len(&rx_sg[i]) = len; + } + if (thread->type == DMA_MEMCPY) tx = dev->device_prep_dma_memcpy(chan, dsts[0] + dst_off, srcs[0], len, flags); + else if (thread->type == DMA_SG) + tx = dev->device_prep_dma_sg(chan, tx_sg, src_cnt, + rx_sg, src_cnt, flags); else if (thread->type == DMA_XOR) tx = dev->device_prep_dma_xor(chan, dsts[0] + dst_off, @@ -748,6 +774,8 @@ static int dmatest_add_threads(struct dmatest_info *info, if (type == DMA_MEMCPY) op = "copy"; + else if (type == DMA_SG) + op = "sg"; else if (type == DMA_XOR) op = "xor"; else if (type == DMA_PQ) @@ -802,9 +830,19 @@ static int dmatest_add_channel(struct dmatest_info *info, INIT_LIST_HEAD(&dtc->threads); if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { - cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY); - thread_count += cnt > 0 ? cnt : 0; + if (dmatest == 0) { + cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY); + thread_count += cnt > 0 ? cnt : 0; + } } + + if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) { + if (dmatest == 1) { + cnt = dmatest_add_threads(info, dtc, DMA_SG); + thread_count += cnt > 0 ? cnt : 0; + } + } + if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { cnt = dmatest_add_threads(info, dtc, DMA_XOR); thread_count += cnt > 0 ? cnt : 0; @@ -877,6 +915,7 @@ static void run_threaded_test(struct dmatest_info *info) request_channels(info, DMA_MEMCPY); request_channels(info, DMA_XOR); + request_channels(info, DMA_SG); request_channels(info, DMA_PQ); } diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index 8181ed131..3d277fa76 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -239,6 +239,9 @@ struct edma_cc { bool chmap_exist; enum dma_event_q default_queue; + unsigned int ccint; + unsigned int ccerrint; + /* * The slot_inuse bit for each PaRAM slot is clear unless the slot is * in use by Linux or if it is allocated to be used by DSP. @@ -1069,10 +1072,8 @@ static struct dma_async_tx_descriptor *edma_prep_slave_sg( edesc = kzalloc(sizeof(*edesc) + sg_len * sizeof(edesc->pset[0]), GFP_ATOMIC); - if (!edesc) { - dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__); + if (!edesc) return NULL; - } edesc->pset_nr = sg_len; edesc->residue = 0; @@ -1114,14 +1115,17 @@ static struct dma_async_tx_descriptor *edma_prep_slave_sg( edesc->absync = ret; edesc->residue += sg_dma_len(sg); - /* If this is the last in a current SG set of transactions, - enable interrupts so that next set is processed */ - if (!((i+1) % MAX_NR_SG)) - edesc->pset[i].param.opt |= TCINTEN; - - /* If this is the last set, enable completion interrupt flag */ if (i == sg_len - 1) + /* Enable completion interrupt */ edesc->pset[i].param.opt |= TCINTEN; + else if (!((i+1) % MAX_NR_SG)) + /* + * Enable early completion interrupt for the + * intermediateset. In this case the driver will be + * notified when the paRAM set is submitted to TC. This + * will allow more time to set up the next set of slots. + */ + edesc->pset[i].param.opt |= (TCINTEN | TCCMODE); } edesc->residue_stat = edesc->residue; @@ -1173,10 +1177,8 @@ static struct dma_async_tx_descriptor *edma_prep_dma_memcpy( edesc = kzalloc(sizeof(*edesc) + nslots * sizeof(edesc->pset[0]), GFP_ATOMIC); - if (!edesc) { - dev_dbg(dev, "Failed to allocate a descriptor\n"); + if (!edesc) return NULL; - } edesc->pset_nr = nslots; edesc->residue = edesc->residue_stat = len; @@ -1298,10 +1300,8 @@ static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( edesc = kzalloc(sizeof(*edesc) + nslots * sizeof(edesc->pset[0]), GFP_ATOMIC); - if (!edesc) { - dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__); + if (!edesc) return NULL; - } edesc->cyclic = 1; edesc->pset_nr = nslots; @@ -2207,10 +2207,8 @@ static int edma_probe(struct platform_device *pdev) return ret; ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL); - if (!ecc) { - dev_err(dev, "Can't allocate controller\n"); + if (!ecc) return -ENOMEM; - } ecc->dev = dev; ecc->id = pdev->id; @@ -2288,6 +2286,7 @@ static int edma_probe(struct platform_device *pdev) dev_err(dev, "CCINT (%d) failed --> %d\n", irq, ret); return ret; } + ecc->ccint = irq; } irq = platform_get_irq_byname(pdev, "edma3_ccerrint"); @@ -2303,6 +2302,7 @@ static int edma_probe(struct platform_device *pdev) dev_err(dev, "CCERRINT (%d) failed --> %d\n", irq, ret); return ret; } + ecc->ccerrint = irq; } ecc->dummy_slot = edma_alloc_slot(ecc, EDMA_SLOT_ANY); @@ -2393,11 +2393,27 @@ err_reg1: return ret; } +static void edma_cleanupp_vchan(struct dma_device *dmadev) +{ + struct edma_chan *echan, *_echan; + + list_for_each_entry_safe(echan, _echan, + &dmadev->channels, vchan.chan.device_node) { + list_del(&echan->vchan.chan.device_node); + tasklet_kill(&echan->vchan.task); + } +} + static int edma_remove(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct edma_cc *ecc = dev_get_drvdata(dev); + devm_free_irq(dev, ecc->ccint, ecc); + devm_free_irq(dev, ecc->ccerrint, ecc); + + edma_cleanupp_vchan(&ecc->dma_slave); + if (dev->of_node) of_dma_controller_free(dev->of_node); dma_async_device_unregister(&ecc->dma_slave); diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c index be2e62b87..6775f2c74 100644 --- a/drivers/dma/fsl-edma.c +++ b/drivers/dma/fsl-edma.c @@ -852,6 +852,25 @@ fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma return 0; } +static void fsl_edma_irq_exit( + struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) +{ + if (fsl_edma->txirq == fsl_edma->errirq) { + devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma); + } else { + devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma); + devm_free_irq(&pdev->dev, fsl_edma->errirq, fsl_edma); + } +} + +static void fsl_disable_clocks(struct fsl_edma_engine *fsl_edma) +{ + int i; + + for (i = 0; i < DMAMUX_NR; i++) + clk_disable_unprepare(fsl_edma->muxclk[i]); +} + static int fsl_edma_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -897,6 +916,10 @@ static int fsl_edma_probe(struct platform_device *pdev) ret = clk_prepare_enable(fsl_edma->muxclk[i]); if (ret) { + /* disable only clks which were enabled on error */ + for (; i >= 0; i--) + clk_disable_unprepare(fsl_edma->muxclk[i]); + dev_err(&pdev->dev, "DMAMUX clk block failed.\n"); return ret; } @@ -951,14 +974,18 @@ static int fsl_edma_probe(struct platform_device *pdev) ret = dma_async_device_register(&fsl_edma->dma_dev); if (ret) { - dev_err(&pdev->dev, "Can't register Freescale eDMA engine.\n"); + dev_err(&pdev->dev, + "Can't register Freescale eDMA engine. (%d)\n", ret); + fsl_disable_clocks(fsl_edma); return ret; } ret = of_dma_controller_register(np, fsl_edma_xlate, fsl_edma); if (ret) { - dev_err(&pdev->dev, "Can't register Freescale eDMA of_dma.\n"); + dev_err(&pdev->dev, + "Can't register Freescale eDMA of_dma. (%d)\n", ret); dma_async_device_unregister(&fsl_edma->dma_dev); + fsl_disable_clocks(fsl_edma); return ret; } @@ -968,17 +995,27 @@ static int fsl_edma_probe(struct platform_device *pdev) return 0; } +static void fsl_edma_cleanup_vchan(struct dma_device *dmadev) +{ + struct fsl_edma_chan *chan, *_chan; + + list_for_each_entry_safe(chan, _chan, + &dmadev->channels, vchan.chan.device_node) { + list_del(&chan->vchan.chan.device_node); + tasklet_kill(&chan->vchan.task); + } +} + static int fsl_edma_remove(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev); - int i; + fsl_edma_irq_exit(pdev, fsl_edma); + fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); of_dma_controller_free(np); dma_async_device_unregister(&fsl_edma->dma_dev); - - for (i = 0; i < DMAMUX_NR; i++) - clk_disable_unprepare(fsl_edma->muxclk[i]); + fsl_disable_clocks(fsl_edma); return 0; } diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index 4d9470f16..de2a2a2b1 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -337,7 +337,7 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq( re_chan = container_of(chan, struct fsl_re_chan, chan); if (len > FSL_RE_MAX_DATA_LEN) { - dev_err(re_chan->dev, "genq tx length %lu, max length %d\n", + dev_err(re_chan->dev, "genq tx length %zu, max length %d\n", len, FSL_RE_MAX_DATA_LEN); return NULL; } @@ -424,7 +424,7 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq( re_chan = container_of(chan, struct fsl_re_chan, chan); if (len > FSL_RE_MAX_DATA_LEN) { - dev_err(re_chan->dev, "pq tx length is %lu, max length is %d\n", + dev_err(re_chan->dev, "pq tx length is %zu, max length is %d\n", len, FSL_RE_MAX_DATA_LEN); return NULL; } @@ -545,7 +545,7 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_memcpy( re_chan = container_of(chan, struct fsl_re_chan, chan); if (len > FSL_RE_MAX_DATA_LEN) { - dev_err(re_chan->dev, "cp tx length is %lu, max length is %d\n", + dev_err(re_chan->dev, "cp tx length is %zu, max length is %d\n", len, FSL_RE_MAX_DATA_LEN); return NULL; } @@ -836,6 +836,7 @@ static int fsl_re_probe(struct platform_device *ofdev) rc = of_property_read_u32(np, "reg", &off); if (rc) { dev_err(dev, "Reg property not found in JQ node\n"); + of_node_put(np); return -ENODEV; } /* Find out the Job Rings present under each JQ */ @@ -856,6 +857,8 @@ static int fsl_re_probe(struct platform_device *ofdev) static void fsl_re_remove_chan(struct fsl_re_chan *chan) { + tasklet_kill(&chan->irqtask); + dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr, chan->inb_phys_addr); @@ -890,7 +893,6 @@ static struct of_device_id fsl_re_ids[] = { static struct platform_driver fsl_re_driver = { .driver = { .name = "fsl-raideng", - .owner = THIS_MODULE, .of_match_table = fsl_re_ids, }, .probe = fsl_re_probe, diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index a8828ed63..911b7177e 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -1234,7 +1234,6 @@ static int fsl_dma_chan_probe(struct fsldma_device *fdev, /* alloc channel */ chan = kzalloc(sizeof(*chan), GFP_KERNEL); if (!chan) { - dev_err(fdev->dev, "no free memory for DMA channels!\n"); err = -ENOMEM; goto out_return; } @@ -1340,7 +1339,6 @@ static int fsldma_of_probe(struct platform_device *op) fdev = kzalloc(sizeof(*fdev), GFP_KERNEL); if (!fdev) { - dev_err(&op->dev, "No enough memory for 'priv'\n"); err = -ENOMEM; goto out_return; } diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c index f8c5cd533..c5f21efd6 100644 --- a/drivers/dma/hsu/hsu.c +++ b/drivers/dma/hsu/hsu.c @@ -126,28 +126,33 @@ static void hsu_dma_start_transfer(struct hsu_dma_chan *hsuc) hsu_dma_start_channel(hsuc); } -static u32 hsu_dma_chan_get_sr(struct hsu_dma_chan *hsuc) -{ - unsigned long flags; - u32 sr; - - spin_lock_irqsave(&hsuc->vchan.lock, flags); - sr = hsu_chan_readl(hsuc, HSU_CH_SR); - spin_unlock_irqrestore(&hsuc->vchan.lock, flags); - - return sr & ~(HSU_CH_SR_DESCE_ANY | HSU_CH_SR_CDESC_ANY); -} - -irqreturn_t hsu_dma_irq(struct hsu_dma_chip *chip, unsigned short nr) +/* + * hsu_dma_get_status() - get DMA channel status + * @chip: HSUART DMA chip + * @nr: DMA channel number + * @status: pointer for DMA Channel Status Register value + * + * Description: + * The function reads and clears the DMA Channel Status Register, checks + * if it was a timeout interrupt and returns a corresponding value. + * + * Caller should provide a valid pointer for the DMA Channel Status + * Register value that will be returned in @status. + * + * Return: + * 1 for DMA timeout status, 0 for other DMA status, or error code for + * invalid parameters or no interrupt pending. + */ +int hsu_dma_get_status(struct hsu_dma_chip *chip, unsigned short nr, + u32 *status) { struct hsu_dma_chan *hsuc; - struct hsu_dma_desc *desc; unsigned long flags; u32 sr; /* Sanity check */ if (nr >= chip->hsu->nr_channels) - return IRQ_NONE; + return -EINVAL; hsuc = &chip->hsu->chan[nr]; @@ -155,22 +160,65 @@ irqreturn_t hsu_dma_irq(struct hsu_dma_chip *chip, unsigned short nr) * No matter what situation, need read clear the IRQ status * There is a bug, see Errata 5, HSD 2900918 */ - sr = hsu_dma_chan_get_sr(hsuc); + spin_lock_irqsave(&hsuc->vchan.lock, flags); + sr = hsu_chan_readl(hsuc, HSU_CH_SR); + spin_unlock_irqrestore(&hsuc->vchan.lock, flags); + + /* Check if any interrupt is pending */ + sr &= ~(HSU_CH_SR_DESCE_ANY | HSU_CH_SR_CDESC_ANY); if (!sr) - return IRQ_NONE; + return -EIO; /* Timeout IRQ, need wait some time, see Errata 2 */ if (sr & HSU_CH_SR_DESCTO_ANY) udelay(2); + /* + * At this point, at least one of Descriptor Time Out, Channel Error + * or Descriptor Done bits must be set. Clear the Descriptor Time Out + * bits and if sr is still non-zero, it must be channel error or + * descriptor done which are higher priority than timeout and handled + * in hsu_dma_do_irq(). Else, it must be a timeout. + */ sr &= ~HSU_CH_SR_DESCTO_ANY; - if (!sr) - return IRQ_HANDLED; + + *status = sr; + + return sr ? 0 : 1; +} +EXPORT_SYMBOL_GPL(hsu_dma_get_status); + +/* + * hsu_dma_do_irq() - DMA interrupt handler + * @chip: HSUART DMA chip + * @nr: DMA channel number + * @status: Channel Status Register value + * + * Description: + * This function handles Channel Error and Descriptor Done interrupts. + * This function should be called after determining that the DMA interrupt + * is not a normal timeout interrupt, ie. hsu_dma_get_status() returned 0. + * + * Return: + * IRQ_NONE for invalid channel number, IRQ_HANDLED otherwise. + */ +irqreturn_t hsu_dma_do_irq(struct hsu_dma_chip *chip, unsigned short nr, + u32 status) +{ + struct hsu_dma_chan *hsuc; + struct hsu_dma_desc *desc; + unsigned long flags; + + /* Sanity check */ + if (nr >= chip->hsu->nr_channels) + return IRQ_NONE; + + hsuc = &chip->hsu->chan[nr]; spin_lock_irqsave(&hsuc->vchan.lock, flags); desc = hsuc->desc; if (desc) { - if (sr & HSU_CH_SR_CHE) { + if (status & HSU_CH_SR_CHE) { desc->status = DMA_ERROR; } else if (desc->active < desc->nents) { hsu_dma_start_channel(hsuc); @@ -184,7 +232,7 @@ irqreturn_t hsu_dma_irq(struct hsu_dma_chip *chip, unsigned short nr) return IRQ_HANDLED; } -EXPORT_SYMBOL_GPL(hsu_dma_irq); +EXPORT_SYMBOL_GPL(hsu_dma_do_irq); static struct hsu_dma_desc *hsu_dma_alloc_desc(unsigned int nents) { diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c index e2db76bd5..991605853 100644 --- a/drivers/dma/hsu/pci.c +++ b/drivers/dma/hsu/pci.c @@ -27,13 +27,20 @@ static irqreturn_t hsu_pci_irq(int irq, void *dev) { struct hsu_dma_chip *chip = dev; u32 dmaisr; + u32 status; unsigned short i; irqreturn_t ret = IRQ_NONE; + int err; dmaisr = readl(chip->regs + HSU_PCI_DMAISR); for (i = 0; i < chip->hsu->nr_channels; i++) { - if (dmaisr & 0x1) - ret |= hsu_dma_irq(chip, i); + if (dmaisr & 0x1) { + err = hsu_dma_get_status(chip, i, &status); + if (err > 0) + ret |= IRQ_HANDLED; + else if (err == 0) + ret |= hsu_dma_do_irq(chip, i, status); + } dmaisr >>= 1; } diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c index a4c53be48..624f1e1e9 100644 --- a/drivers/dma/img-mdc-dma.c +++ b/drivers/dma/img-mdc-dma.c @@ -861,7 +861,6 @@ static int mdc_dma_probe(struct platform_device *pdev) { struct mdc_dma *mdma; struct resource *res; - const struct of_device_id *match; unsigned int i; u32 val; int ret; @@ -871,8 +870,7 @@ static int mdc_dma_probe(struct platform_device *pdev) return -ENOMEM; platform_set_drvdata(pdev, mdma); - match = of_match_device(mdc_dma_of_match, &pdev->dev); - mdma->soc = match->data; + mdma->soc = of_device_get_match_data(&pdev->dev); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); mdma->regs = devm_ioremap_resource(&pdev->dev, res); diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c index 48d85f8b9..a960608c0 100644 --- a/drivers/dma/imx-dma.c +++ b/drivers/dma/imx-dma.c @@ -167,6 +167,7 @@ struct imxdma_channel { u32 ccr_to_device; bool enabled_2d; int slot_2d; + unsigned int irq; }; enum imx_dma_type { @@ -186,6 +187,9 @@ struct imxdma_engine { struct imx_dma_2d_config slots_2d[IMX_DMA_2D_SLOTS]; struct imxdma_channel channel[IMX_DMA_CHANNELS]; enum imx_dma_type devtype; + unsigned int irq; + unsigned int irq_err; + }; struct imxdma_filter_data { @@ -1048,7 +1052,7 @@ static struct dma_chan *imxdma_xlate(struct of_phandle_args *dma_spec, } static int __init imxdma_probe(struct platform_device *pdev) - { +{ struct imxdma_engine *imxdma; struct resource *res; const struct of_device_id *of_id; @@ -1100,6 +1104,7 @@ static int __init imxdma_probe(struct platform_device *pdev) dev_warn(imxdma->dev, "Can't register IRQ for DMA\n"); goto disable_dma_ahb_clk; } + imxdma->irq = irq; irq_err = platform_get_irq(pdev, 1); if (irq_err < 0) { @@ -1113,6 +1118,7 @@ static int __init imxdma_probe(struct platform_device *pdev) dev_warn(imxdma->dev, "Can't register ERRIRQ for DMA\n"); goto disable_dma_ahb_clk; } + imxdma->irq_err = irq_err; } /* enable DMA module */ @@ -1150,6 +1156,8 @@ static int __init imxdma_probe(struct platform_device *pdev) irq + i, i); goto disable_dma_ahb_clk; } + + imxdmac->irq = irq + i; init_timer(&imxdmac->watchdog); imxdmac->watchdog.function = &imxdma_watchdog; imxdmac->watchdog.data = (unsigned long)imxdmac; @@ -1217,10 +1225,31 @@ disable_dma_ipg_clk: return ret; } +static void imxdma_free_irq(struct platform_device *pdev, struct imxdma_engine *imxdma) +{ + int i; + + if (is_imx1_dma(imxdma)) { + disable_irq(imxdma->irq); + disable_irq(imxdma->irq_err); + } + + for (i = 0; i < IMX_DMA_CHANNELS; i++) { + struct imxdma_channel *imxdmac = &imxdma->channel[i]; + + if (!is_imx1_dma(imxdma)) + disable_irq(imxdmac->irq); + + tasklet_kill(&imxdmac->dma_tasklet); + } +} + static int imxdma_remove(struct platform_device *pdev) { struct imxdma_engine *imxdma = platform_get_drvdata(pdev); + imxdma_free_irq(pdev, imxdma); + dma_async_device_unregister(&imxdma->dma_device); if (pdev->dev.of_node) diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index ff8f98e25..4db2316c8 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -385,6 +386,7 @@ struct sdma_engine { const struct sdma_driver_data *drvdata; u32 spba_start_addr; u32 spba_end_addr; + unsigned int irq; }; static struct sdma_driver_data sdma_imx31 = { @@ -571,28 +573,20 @@ static void sdma_enable_channel(struct sdma_engine *sdma, int channel) static int sdma_run_channel0(struct sdma_engine *sdma) { int ret; - unsigned long timeout = 500; + u32 reg; sdma_enable_channel(sdma, 0); - while (!(ret = readl_relaxed(sdma->regs + SDMA_H_INTR) & 1)) { - if (timeout-- <= 0) - break; - udelay(1); - } - - if (ret) { - /* Clear the interrupt status */ - writel_relaxed(ret, sdma->regs + SDMA_H_INTR); - } else { + ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP, + reg, !(reg & 1), 1, 500); + if (ret) dev_err(sdma->dev, "Timeout waiting for CH0 ready\n"); - } /* Set bits of CONFIG register with dynamic context switching */ if (readl(sdma->regs + SDMA_H_CONFIG) == 0) writel_relaxed(SDMA_H_CONFIG_CSM, sdma->regs + SDMA_H_CONFIG); - return ret ? 0 : -ETIMEDOUT; + return ret; } static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, @@ -727,9 +721,9 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) unsigned long stat; stat = readl_relaxed(sdma->regs + SDMA_H_INTR); - /* not interested in channel 0 interrupts */ - stat &= ~1; writel_relaxed(stat, sdma->regs + SDMA_H_INTR); + /* channel 0 is special and not handled here, see run_channel0() */ + stat &= ~1; while (stat) { int channel = fls(stat) - 1; @@ -758,7 +752,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, * These are needed once we start to support transfers between * two peripherals or memory-to-memory transfers */ - int per_2_per = 0, emi_2_emi = 0; + int per_2_per = 0; sdmac->pc_from_device = 0; sdmac->pc_to_device = 0; @@ -766,7 +760,6 @@ static void sdma_get_pc(struct sdma_channel *sdmac, switch (peripheral_type) { case IMX_DMATYPE_MEMORY: - emi_2_emi = sdma->script_addrs->ap_2_ap_addr; break; case IMX_DMATYPE_DSP: emi_2_per = sdma->script_addrs->bp_2_ap_addr; @@ -999,8 +992,6 @@ static int sdma_config_channel(struct dma_chan *chan) } else __set_bit(sdmac->event_id0, sdmac->event_mask); - /* Watermark Level */ - sdmac->watermark_level |= sdmac->watermark_level; /* Address */ sdmac->shp_addr = sdmac->per_address; sdmac->per_addr = sdmac->per_address2; @@ -1715,6 +1706,8 @@ static int sdma_probe(struct platform_device *pdev) if (ret) return ret; + sdma->irq = irq; + sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL); if (!sdma->script_addrs) return -ENOMEM; @@ -1840,6 +1833,7 @@ static int sdma_remove(struct platform_device *pdev) struct sdma_engine *sdma = platform_get_drvdata(pdev); int i; + devm_free_irq(&pdev->dev, sdma->irq, sdma); dma_async_device_unregister(&sdma->dma_device); kfree(sdma->script_addrs); /* Kill the tasklet */ diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index d406056e8..7145f7716 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -1212,7 +1212,7 @@ static void ioat_shutdown(struct pci_dev *pdev) ioat_disable_interrupts(ioat_dma); } -void ioat_resume(struct ioatdma_device *ioat_dma) +static void ioat_resume(struct ioatdma_device *ioat_dma) { struct ioatdma_chan *ioat_chan; u32 chanerr; diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c index 1ba2fd738..39de89801 100644 --- a/drivers/dma/k3dma.c +++ b/drivers/dma/k3dma.c @@ -102,6 +102,7 @@ struct k3_dma_dev { struct clk *clk; u32 dma_channels; u32 dma_requests; + unsigned int irq; }; #define to_k3_dma(dmadev) container_of(dmadev, struct k3_dma_dev, slave) @@ -425,10 +426,9 @@ static struct dma_async_tx_descriptor *k3_dma_prep_memcpy( num = DIV_ROUND_UP(len, DMA_MAX_SIZE); ds = kzalloc(sizeof(*ds) + num * sizeof(ds->desc_hw[0]), GFP_ATOMIC); - if (!ds) { - dev_dbg(chan->device->dev, "vchan %p: kzalloc fail\n", &c->vc); + if (!ds) return NULL; - } + ds->desc_hw_lli = __virt_to_phys((unsigned long)&ds->desc_hw[0]); ds->size = len; ds->desc_num = num; @@ -481,10 +481,9 @@ static struct dma_async_tx_descriptor *k3_dma_prep_slave_sg( } ds = kzalloc(sizeof(*ds) + num * sizeof(ds->desc_hw[0]), GFP_ATOMIC); - if (!ds) { - dev_dbg(chan->device->dev, "vchan %p: kzalloc fail\n", &c->vc); + if (!ds) return NULL; - } + ds->desc_hw_lli = __virt_to_phys((unsigned long)&ds->desc_hw[0]); ds->desc_num = num; num = 0; @@ -705,6 +704,8 @@ static int k3_dma_probe(struct platform_device *op) if (ret) return ret; + d->irq = irq; + /* init phy channel */ d->phy = devm_kzalloc(&op->dev, d->dma_channels * sizeof(struct k3_dma_phy), GFP_KERNEL); @@ -759,7 +760,7 @@ static int k3_dma_probe(struct platform_device *op) ret = dma_async_device_register(&d->slave); if (ret) - return ret; + goto dma_async_register_fail; ret = of_dma_controller_register((&op->dev)->of_node, k3_of_dma_simple_xlate, d); @@ -776,6 +777,8 @@ static int k3_dma_probe(struct platform_device *op) of_dma_register_fail: dma_async_device_unregister(&d->slave); +dma_async_register_fail: + clk_disable_unprepare(d->clk); return ret; } @@ -787,6 +790,8 @@ static int k3_dma_remove(struct platform_device *op) dma_async_device_unregister(&d->slave); of_dma_controller_free((&op->dev)->of_node); + devm_free_irq(&op->dev, d->irq, d); + list_for_each_entry_safe(c, cn, &d->slave.channels, vc.chan.device_node) { list_del(&c->vc.chan.device_node); tasklet_kill(&c->vc.task); diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index 56f1fd68b..f4b25fb0d 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -931,6 +931,25 @@ static void dma_do_tasklet(unsigned long data) static int mmp_pdma_remove(struct platform_device *op) { struct mmp_pdma_device *pdev = platform_get_drvdata(op); + struct mmp_pdma_phy *phy; + int i, irq = 0, irq_num = 0; + + + for (i = 0; i < pdev->dma_channels; i++) { + if (platform_get_irq(op, i) > 0) + irq_num++; + } + + if (irq_num != pdev->dma_channels) { + irq = platform_get_irq(op, 0); + devm_free_irq(&op->dev, irq, pdev); + } else { + for (i = 0; i < pdev->dma_channels; i++) { + phy = &pdev->phy[i]; + irq = platform_get_irq(op, i); + devm_free_irq(&op->dev, irq, phy); + } + } dma_async_device_unregister(&pdev->device); return 0; diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c index 3df042260..b3441f57a 100644 --- a/drivers/dma/mmp_tdma.c +++ b/drivers/dma/mmp_tdma.c @@ -404,7 +404,7 @@ static void mmp_tdma_free_chan_resources(struct dma_chan *chan) return; } -struct mmp_tdma_desc *mmp_tdma_alloc_descriptor(struct mmp_tdma_chan *tdmac) +static struct mmp_tdma_desc *mmp_tdma_alloc_descriptor(struct mmp_tdma_chan *tdmac) { struct gen_pool *gpool; int size = tdmac->desc_num * sizeof(struct mmp_tdma_desc); @@ -551,10 +551,9 @@ static int mmp_tdma_chan_init(struct mmp_tdma_device *tdev, /* alloc channel */ tdmac = devm_kzalloc(tdev->dev, sizeof(*tdmac), GFP_KERNEL); - if (!tdmac) { - dev_err(tdev->dev, "no free memory for DMA channels!\n"); + if (!tdmac) return -ENOMEM; - } + if (irq) tdmac->irq = irq; tdmac->dev = tdev->dev; @@ -593,7 +592,7 @@ static bool mmp_tdma_filter_fn(struct dma_chan *chan, void *fn_param) return true; } -struct dma_chan *mmp_tdma_xlate(struct of_phandle_args *dma_spec, +static struct dma_chan *mmp_tdma_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct mmp_tdma_device *tdev = ofdma->of_dma_data; diff --git a/drivers/dma/moxart-dma.c b/drivers/dma/moxart-dma.c index 631c4435e..a6e642792 100644 --- a/drivers/dma/moxart-dma.c +++ b/drivers/dma/moxart-dma.c @@ -148,6 +148,7 @@ struct moxart_chan { struct moxart_dmadev { struct dma_device dma_slave; struct moxart_chan slave_chans[APB_DMA_MAX_CHANNEL]; + unsigned int irq; }; struct moxart_filter_data { @@ -574,10 +575,8 @@ static int moxart_probe(struct platform_device *pdev) struct moxart_dmadev *mdc; mdc = devm_kzalloc(dev, sizeof(*mdc), GFP_KERNEL); - if (!mdc) { - dev_err(dev, "can't allocate DMA container\n"); + if (!mdc) return -ENOMEM; - } irq = irq_of_parse_and_map(node, 0); if (irq == NO_IRQ) { @@ -617,6 +616,7 @@ static int moxart_probe(struct platform_device *pdev) dev_err(dev, "devm_request_irq failed\n"); return ret; } + mdc->irq = irq; ret = dma_async_device_register(&mdc->dma_slave); if (ret) { @@ -640,6 +640,8 @@ static int moxart_remove(struct platform_device *pdev) { struct moxart_dmadev *m = platform_get_drvdata(pdev); + devm_free_irq(&pdev->dev, m->irq, m); + dma_async_device_unregister(&m->dma_slave); if (pdev->dev.of_node) diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c index ccadafa51..fa86592c7 100644 --- a/drivers/dma/mpc512x_dma.c +++ b/drivers/dma/mpc512x_dma.c @@ -1110,6 +1110,7 @@ static int mpc_dma_remove(struct platform_device *op) } free_irq(mdma->irq, mdma); irq_dispose_mapping(mdma->irq); + tasklet_kill(&mdma->tasklet); return 0; } diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index d0446a759..f4c9f98ec 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -1057,7 +1057,7 @@ mv_xor_channel_add(struct mv_xor_device *xordev, err_free_irq: free_irq(mv_chan->irq, mv_chan); - err_free_dma: +err_free_dma: dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE, mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); return ERR_PTR(ret); diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c new file mode 100644 index 000000000..a28a01fcb --- /dev/null +++ b/drivers/dma/mv_xor_v2.c @@ -0,0 +1,878 @@ +/* + * Copyright (C) 2015-2016 Marvell International Ltd. + + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dmaengine.h" + +/* DMA Engine Registers */ +#define MV_XOR_V2_DMA_DESQ_BALR_OFF 0x000 +#define MV_XOR_V2_DMA_DESQ_BAHR_OFF 0x004 +#define MV_XOR_V2_DMA_DESQ_SIZE_OFF 0x008 +#define MV_XOR_V2_DMA_DESQ_DONE_OFF 0x00C +#define MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK 0x7FFF +#define MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT 0 +#define MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_MASK 0x1FFF +#define MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_SHIFT 16 +#define MV_XOR_V2_DMA_DESQ_ARATTR_OFF 0x010 +#define MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK 0x3F3F +#define MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE 0x202 +#define MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE 0x3C3C +#define MV_XOR_V2_DMA_IMSG_CDAT_OFF 0x014 +#define MV_XOR_V2_DMA_IMSG_THRD_OFF 0x018 +#define MV_XOR_V2_DMA_IMSG_THRD_MASK 0x7FFF +#define MV_XOR_V2_DMA_IMSG_THRD_SHIFT 0x0 +#define MV_XOR_V2_DMA_DESQ_AWATTR_OFF 0x01C + /* Same flags as MV_XOR_V2_DMA_DESQ_ARATTR_OFF */ +#define MV_XOR_V2_DMA_DESQ_ALLOC_OFF 0x04C +#define MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_MASK 0xFFFF +#define MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_SHIFT 16 +#define MV_XOR_V2_DMA_IMSG_BALR_OFF 0x050 +#define MV_XOR_V2_DMA_IMSG_BAHR_OFF 0x054 +#define MV_XOR_V2_DMA_DESQ_CTRL_OFF 0x100 +#define MV_XOR_V2_DMA_DESQ_CTRL_32B 1 +#define MV_XOR_V2_DMA_DESQ_CTRL_128B 7 +#define MV_XOR_V2_DMA_DESQ_STOP_OFF 0x800 +#define MV_XOR_V2_DMA_DESQ_DEALLOC_OFF 0x804 +#define MV_XOR_V2_DMA_DESQ_ADD_OFF 0x808 + +/* XOR Global registers */ +#define MV_XOR_V2_GLOB_BW_CTRL 0x4 +#define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_SHIFT 0 +#define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_VAL 64 +#define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_SHIFT 8 +#define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_VAL 8 +#define MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_SHIFT 12 +#define MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_VAL 4 +#define MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_SHIFT 16 +#define MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_VAL 4 +#define MV_XOR_V2_GLOB_PAUSE 0x014 +#define MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL 0x8 +#define MV_XOR_V2_GLOB_SYS_INT_CAUSE 0x200 +#define MV_XOR_V2_GLOB_SYS_INT_MASK 0x204 +#define MV_XOR_V2_GLOB_MEM_INT_CAUSE 0x220 +#define MV_XOR_V2_GLOB_MEM_INT_MASK 0x224 + +#define MV_XOR_V2_MIN_DESC_SIZE 32 +#define MV_XOR_V2_EXT_DESC_SIZE 128 + +#define MV_XOR_V2_DESC_RESERVED_SIZE 12 +#define MV_XOR_V2_DESC_BUFF_D_ADDR_SIZE 12 + +#define MV_XOR_V2_CMD_LINE_NUM_MAX_D_BUF 8 + +/* + * Descriptors queue size. With 32 bytes descriptors, up to 2^14 + * descriptors are allowed, with 128 bytes descriptors, up to 2^12 + * descriptors are allowed. This driver uses 128 bytes descriptors, + * but experimentation has shown that a set of 1024 descriptors is + * sufficient to reach a good level of performance. + */ +#define MV_XOR_V2_DESC_NUM 1024 + +/** + * struct mv_xor_v2_descriptor - DMA HW descriptor + * @desc_id: used by S/W and is not affected by H/W. + * @flags: error and status flags + * @crc32_result: CRC32 calculation result + * @desc_ctrl: operation mode and control flags + * @buff_size: amount of bytes to be processed + * @fill_pattern_src_addr: Fill-Pattern or Source-Address and + * AW-Attributes + * @data_buff_addr: Source (and might be RAID6 destination) + * addresses of data buffers in RAID5 and RAID6 + * @reserved: reserved + */ +struct mv_xor_v2_descriptor { + u16 desc_id; + u16 flags; + u32 crc32_result; + u32 desc_ctrl; + + /* Definitions for desc_ctrl */ +#define DESC_NUM_ACTIVE_D_BUF_SHIFT 22 +#define DESC_OP_MODE_SHIFT 28 +#define DESC_OP_MODE_NOP 0 /* Idle operation */ +#define DESC_OP_MODE_MEMCPY 1 /* Pure-DMA operation */ +#define DESC_OP_MODE_MEMSET 2 /* Mem-Fill operation */ +#define DESC_OP_MODE_MEMINIT 3 /* Mem-Init operation */ +#define DESC_OP_MODE_MEM_COMPARE 4 /* Mem-Compare operation */ +#define DESC_OP_MODE_CRC32 5 /* CRC32 calculation */ +#define DESC_OP_MODE_XOR 6 /* RAID5 (XOR) operation */ +#define DESC_OP_MODE_RAID6 7 /* RAID6 P&Q-generation */ +#define DESC_OP_MODE_RAID6_REC 8 /* RAID6 Recovery */ +#define DESC_Q_BUFFER_ENABLE BIT(16) +#define DESC_P_BUFFER_ENABLE BIT(17) +#define DESC_IOD BIT(27) + + u32 buff_size; + u32 fill_pattern_src_addr[4]; + u32 data_buff_addr[MV_XOR_V2_DESC_BUFF_D_ADDR_SIZE]; + u32 reserved[MV_XOR_V2_DESC_RESERVED_SIZE]; +}; + +/** + * struct mv_xor_v2_device - implements a xor device + * @lock: lock for the engine + * @dma_base: memory mapped DMA register base + * @glob_base: memory mapped global register base + * @irq_tasklet: + * @free_sw_desc: linked list of free SW descriptors + * @dmadev: dma device + * @dmachan: dma channel + * @hw_desq: HW descriptors queue + * @hw_desq_virt: virtual address of DESCQ + * @sw_desq: SW descriptors queue + * @desc_size: HW descriptor size + * @npendings: number of pending descriptors (for which tx_submit has + * been called, but not yet issue_pending) + */ +struct mv_xor_v2_device { + spinlock_t lock; + void __iomem *dma_base; + void __iomem *glob_base; + struct clk *clk; + struct tasklet_struct irq_tasklet; + struct list_head free_sw_desc; + struct dma_device dmadev; + struct dma_chan dmachan; + dma_addr_t hw_desq; + struct mv_xor_v2_descriptor *hw_desq_virt; + struct mv_xor_v2_sw_desc *sw_desq; + int desc_size; + unsigned int npendings; +}; + +/** + * struct mv_xor_v2_sw_desc - implements a xor SW descriptor + * @idx: descriptor index + * @async_tx: support for the async_tx api + * @hw_desc: assosiated HW descriptor + * @free_list: node of the free SW descriprots list +*/ +struct mv_xor_v2_sw_desc { + int idx; + struct dma_async_tx_descriptor async_tx; + struct mv_xor_v2_descriptor hw_desc; + struct list_head free_list; +}; + +/* + * Fill the data buffers to a HW descriptor + */ +static void mv_xor_v2_set_data_buffers(struct mv_xor_v2_device *xor_dev, + struct mv_xor_v2_descriptor *desc, + dma_addr_t src, int index) +{ + int arr_index = ((index >> 1) * 3); + + /* + * Fill the buffer's addresses to the descriptor. + * + * The format of the buffers address for 2 sequential buffers + * X and X + 1: + * + * First word: Buffer-DX-Address-Low[31:0] + * Second word: Buffer-DX+1-Address-Low[31:0] + * Third word: DX+1-Buffer-Address-High[47:32] [31:16] + * DX-Buffer-Address-High[47:32] [15:0] + */ + if ((index & 0x1) == 0) { + desc->data_buff_addr[arr_index] = lower_32_bits(src); + + desc->data_buff_addr[arr_index + 2] &= ~0xFFFF; + desc->data_buff_addr[arr_index + 2] |= + upper_32_bits(src) & 0xFFFF; + } else { + desc->data_buff_addr[arr_index + 1] = + lower_32_bits(src); + + desc->data_buff_addr[arr_index + 2] &= ~0xFFFF0000; + desc->data_buff_addr[arr_index + 2] |= + (upper_32_bits(src) & 0xFFFF) << 16; + } +} + +/* + * Return the next available index in the DESQ. + */ +static int mv_xor_v2_get_desq_write_ptr(struct mv_xor_v2_device *xor_dev) +{ + /* read the index for the next available descriptor in the DESQ */ + u32 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ALLOC_OFF); + + return ((reg >> MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_SHIFT) + & MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_MASK); +} + +/* + * notify the engine of new descriptors, and update the available index. + */ +static void mv_xor_v2_add_desc_to_desq(struct mv_xor_v2_device *xor_dev, + int num_of_desc) +{ + /* write the number of new descriptors in the DESQ. */ + writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ADD_OFF); +} + +/* + * free HW descriptors + */ +static void mv_xor_v2_free_desc_from_desq(struct mv_xor_v2_device *xor_dev, + int num_of_desc) +{ + /* write the number of new descriptors in the DESQ. */ + writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DEALLOC_OFF); +} + +/* + * Set descriptor size + * Return the HW descriptor size in bytes + */ +static int mv_xor_v2_set_desc_size(struct mv_xor_v2_device *xor_dev) +{ + writel(MV_XOR_V2_DMA_DESQ_CTRL_128B, + xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_CTRL_OFF); + + return MV_XOR_V2_EXT_DESC_SIZE; +} + +/* + * Set the IMSG threshold + */ +static inline +void mv_xor_v2_set_imsg_thrd(struct mv_xor_v2_device *xor_dev, int thrd_val) +{ + u32 reg; + + reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF); + + reg &= (~MV_XOR_V2_DMA_IMSG_THRD_MASK << MV_XOR_V2_DMA_IMSG_THRD_SHIFT); + reg |= (thrd_val << MV_XOR_V2_DMA_IMSG_THRD_SHIFT); + + writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF); +} + +static irqreturn_t mv_xor_v2_interrupt_handler(int irq, void *data) +{ + struct mv_xor_v2_device *xor_dev = data; + unsigned int ndescs; + u32 reg; + + reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF); + + ndescs = ((reg >> MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT) & + MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK); + + /* No descriptors to process */ + if (!ndescs) + return IRQ_NONE; + + /* + * Update IMSG threshold, to disable new IMSG interrupts until + * end of the tasklet + */ + mv_xor_v2_set_imsg_thrd(xor_dev, MV_XOR_V2_DESC_NUM); + + /* schedule a tasklet to handle descriptors callbacks */ + tasklet_schedule(&xor_dev->irq_tasklet); + + return IRQ_HANDLED; +} + +/* + * submit a descriptor to the DMA engine + */ +static dma_cookie_t +mv_xor_v2_tx_submit(struct dma_async_tx_descriptor *tx) +{ + int desq_ptr; + void *dest_hw_desc; + dma_cookie_t cookie; + struct mv_xor_v2_sw_desc *sw_desc = + container_of(tx, struct mv_xor_v2_sw_desc, async_tx); + struct mv_xor_v2_device *xor_dev = + container_of(tx->chan, struct mv_xor_v2_device, dmachan); + + dev_dbg(xor_dev->dmadev.dev, + "%s sw_desc %p: async_tx %p\n", + __func__, sw_desc, &sw_desc->async_tx); + + /* assign coookie */ + spin_lock_bh(&xor_dev->lock); + cookie = dma_cookie_assign(tx); + + /* get the next available slot in the DESQ */ + desq_ptr = mv_xor_v2_get_desq_write_ptr(xor_dev); + + /* copy the HW descriptor from the SW descriptor to the DESQ */ + dest_hw_desc = xor_dev->hw_desq_virt + desq_ptr; + + memcpy(dest_hw_desc, &sw_desc->hw_desc, xor_dev->desc_size); + + xor_dev->npendings++; + + spin_unlock_bh(&xor_dev->lock); + + return cookie; +} + +/* + * Prepare a SW descriptor + */ +static struct mv_xor_v2_sw_desc * +mv_xor_v2_prep_sw_desc(struct mv_xor_v2_device *xor_dev) +{ + struct mv_xor_v2_sw_desc *sw_desc; + + /* Lock the channel */ + spin_lock_bh(&xor_dev->lock); + + if (list_empty(&xor_dev->free_sw_desc)) { + spin_unlock_bh(&xor_dev->lock); + /* schedule tasklet to free some descriptors */ + tasklet_schedule(&xor_dev->irq_tasklet); + return NULL; + } + + /* get a free SW descriptor from the SW DESQ */ + sw_desc = list_first_entry(&xor_dev->free_sw_desc, + struct mv_xor_v2_sw_desc, free_list); + list_del(&sw_desc->free_list); + + /* Release the channel */ + spin_unlock_bh(&xor_dev->lock); + + /* set the async tx descriptor */ + dma_async_tx_descriptor_init(&sw_desc->async_tx, &xor_dev->dmachan); + sw_desc->async_tx.tx_submit = mv_xor_v2_tx_submit; + async_tx_ack(&sw_desc->async_tx); + + return sw_desc; +} + +/* + * Prepare a HW descriptor for a memcpy operation + */ +static struct dma_async_tx_descriptor * +mv_xor_v2_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct mv_xor_v2_sw_desc *sw_desc; + struct mv_xor_v2_descriptor *hw_descriptor; + struct mv_xor_v2_device *xor_dev; + + xor_dev = container_of(chan, struct mv_xor_v2_device, dmachan); + + dev_dbg(xor_dev->dmadev.dev, + "%s len: %zu src %pad dest %pad flags: %ld\n", + __func__, len, &src, &dest, flags); + + sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); + + sw_desc->async_tx.flags = flags; + + /* set the HW descriptor */ + hw_descriptor = &sw_desc->hw_desc; + + /* save the SW descriptor ID to restore when operation is done */ + hw_descriptor->desc_id = sw_desc->idx; + + /* Set the MEMCPY control word */ + hw_descriptor->desc_ctrl = + DESC_OP_MODE_MEMCPY << DESC_OP_MODE_SHIFT; + + if (flags & DMA_PREP_INTERRUPT) + hw_descriptor->desc_ctrl |= DESC_IOD; + + /* Set source address */ + hw_descriptor->fill_pattern_src_addr[0] = lower_32_bits(src); + hw_descriptor->fill_pattern_src_addr[1] = + upper_32_bits(src) & 0xFFFF; + + /* Set Destination address */ + hw_descriptor->fill_pattern_src_addr[2] = lower_32_bits(dest); + hw_descriptor->fill_pattern_src_addr[3] = + upper_32_bits(dest) & 0xFFFF; + + /* Set buffers size */ + hw_descriptor->buff_size = len; + + /* return the async tx descriptor */ + return &sw_desc->async_tx; +} + +/* + * Prepare a HW descriptor for a XOR operation + */ +static struct dma_async_tx_descriptor * +mv_xor_v2_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, + unsigned int src_cnt, size_t len, unsigned long flags) +{ + struct mv_xor_v2_sw_desc *sw_desc; + struct mv_xor_v2_descriptor *hw_descriptor; + struct mv_xor_v2_device *xor_dev = + container_of(chan, struct mv_xor_v2_device, dmachan); + int i; + + if (src_cnt > MV_XOR_V2_CMD_LINE_NUM_MAX_D_BUF || src_cnt < 1) + return NULL; + + dev_dbg(xor_dev->dmadev.dev, + "%s src_cnt: %d len: %zu dest %pad flags: %ld\n", + __func__, src_cnt, len, &dest, flags); + + sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); + + sw_desc->async_tx.flags = flags; + + /* set the HW descriptor */ + hw_descriptor = &sw_desc->hw_desc; + + /* save the SW descriptor ID to restore when operation is done */ + hw_descriptor->desc_id = sw_desc->idx; + + /* Set the XOR control word */ + hw_descriptor->desc_ctrl = + DESC_OP_MODE_XOR << DESC_OP_MODE_SHIFT; + hw_descriptor->desc_ctrl |= DESC_P_BUFFER_ENABLE; + + if (flags & DMA_PREP_INTERRUPT) + hw_descriptor->desc_ctrl |= DESC_IOD; + + /* Set the data buffers */ + for (i = 0; i < src_cnt; i++) + mv_xor_v2_set_data_buffers(xor_dev, hw_descriptor, src[i], i); + + hw_descriptor->desc_ctrl |= + src_cnt << DESC_NUM_ACTIVE_D_BUF_SHIFT; + + /* Set Destination address */ + hw_descriptor->fill_pattern_src_addr[2] = lower_32_bits(dest); + hw_descriptor->fill_pattern_src_addr[3] = + upper_32_bits(dest) & 0xFFFF; + + /* Set buffers size */ + hw_descriptor->buff_size = len; + + /* return the async tx descriptor */ + return &sw_desc->async_tx; +} + +/* + * Prepare a HW descriptor for interrupt operation. + */ +static struct dma_async_tx_descriptor * +mv_xor_v2_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) +{ + struct mv_xor_v2_sw_desc *sw_desc; + struct mv_xor_v2_descriptor *hw_descriptor; + struct mv_xor_v2_device *xor_dev = + container_of(chan, struct mv_xor_v2_device, dmachan); + + sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); + + /* set the HW descriptor */ + hw_descriptor = &sw_desc->hw_desc; + + /* save the SW descriptor ID to restore when operation is done */ + hw_descriptor->desc_id = sw_desc->idx; + + /* Set the INTERRUPT control word */ + hw_descriptor->desc_ctrl = + DESC_OP_MODE_NOP << DESC_OP_MODE_SHIFT; + hw_descriptor->desc_ctrl |= DESC_IOD; + + /* return the async tx descriptor */ + return &sw_desc->async_tx; +} + +/* + * push pending transactions to hardware + */ +static void mv_xor_v2_issue_pending(struct dma_chan *chan) +{ + struct mv_xor_v2_device *xor_dev = + container_of(chan, struct mv_xor_v2_device, dmachan); + + spin_lock_bh(&xor_dev->lock); + + /* + * update the engine with the number of descriptors to + * process + */ + mv_xor_v2_add_desc_to_desq(xor_dev, xor_dev->npendings); + xor_dev->npendings = 0; + + /* Activate the channel */ + writel(0, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF); + + spin_unlock_bh(&xor_dev->lock); +} + +static inline +int mv_xor_v2_get_pending_params(struct mv_xor_v2_device *xor_dev, + int *pending_ptr) +{ + u32 reg; + + reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF); + + /* get the next pending descriptor index */ + *pending_ptr = ((reg >> MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_SHIFT) & + MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_MASK); + + /* get the number of descriptors pending handle */ + return ((reg >> MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT) & + MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK); +} + +/* + * handle the descriptors after HW process + */ +static void mv_xor_v2_tasklet(unsigned long data) +{ + struct mv_xor_v2_device *xor_dev = (struct mv_xor_v2_device *) data; + int pending_ptr, num_of_pending, i; + struct mv_xor_v2_descriptor *next_pending_hw_desc = NULL; + struct mv_xor_v2_sw_desc *next_pending_sw_desc = NULL; + + dev_dbg(xor_dev->dmadev.dev, "%s %d\n", __func__, __LINE__); + + /* get the pending descriptors parameters */ + num_of_pending = mv_xor_v2_get_pending_params(xor_dev, &pending_ptr); + + /* next HW descriptor */ + next_pending_hw_desc = xor_dev->hw_desq_virt + pending_ptr; + + /* loop over free descriptors */ + for (i = 0; i < num_of_pending; i++) { + + if (pending_ptr > MV_XOR_V2_DESC_NUM) + pending_ptr = 0; + + if (next_pending_sw_desc != NULL) + next_pending_hw_desc++; + + /* get the SW descriptor related to the HW descriptor */ + next_pending_sw_desc = + &xor_dev->sw_desq[next_pending_hw_desc->desc_id]; + + /* call the callback */ + if (next_pending_sw_desc->async_tx.cookie > 0) { + /* + * update the channel's completed cookie - no + * lock is required the IMSG threshold provide + * the locking + */ + dma_cookie_complete(&next_pending_sw_desc->async_tx); + + if (next_pending_sw_desc->async_tx.callback) + next_pending_sw_desc->async_tx.callback( + next_pending_sw_desc->async_tx.callback_param); + + dma_descriptor_unmap(&next_pending_sw_desc->async_tx); + } + + dma_run_dependencies(&next_pending_sw_desc->async_tx); + + /* Lock the channel */ + spin_lock_bh(&xor_dev->lock); + + /* add the SW descriptor to the free descriptors list */ + list_add(&next_pending_sw_desc->free_list, + &xor_dev->free_sw_desc); + + /* Release the channel */ + spin_unlock_bh(&xor_dev->lock); + + /* increment the next descriptor */ + pending_ptr++; + } + + if (num_of_pending != 0) { + /* free the descriptores */ + mv_xor_v2_free_desc_from_desq(xor_dev, num_of_pending); + } + + /* Update IMSG threshold, to enable new IMSG interrupts */ + mv_xor_v2_set_imsg_thrd(xor_dev, 0); +} + +/* + * Set DMA Interrupt-message (IMSG) parameters + */ +static void mv_xor_v2_set_msi_msg(struct msi_desc *desc, struct msi_msg *msg) +{ + struct mv_xor_v2_device *xor_dev = dev_get_drvdata(desc->dev); + + writel(msg->address_lo, + xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BALR_OFF); + writel(msg->address_hi & 0xFFFF, + xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BAHR_OFF); + writel(msg->data, + xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_CDAT_OFF); +} + +static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev) +{ + u32 reg; + + /* write the DESQ size to the DMA engine */ + writel(MV_XOR_V2_DESC_NUM, + xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_SIZE_OFF); + + /* write the DESQ address to the DMA enngine*/ + writel(xor_dev->hw_desq & 0xFFFFFFFF, + xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BALR_OFF); + writel((xor_dev->hw_desq & 0xFFFF00000000) >> 32, + xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BAHR_OFF); + + /* enable the DMA engine */ + writel(0, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF); + + /* + * This is a temporary solution, until we activate the + * SMMU. Set the attributes for reading & writing data buffers + * & descriptors to: + * + * - OuterShareable - Snoops will be performed on CPU caches + * - Enable cacheable - Bufferable, Modifiable, Other Allocate + * and Allocate + */ + reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF); + reg &= ~MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK; + reg |= MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE | + MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE; + writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF); + + reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF); + reg &= ~MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK; + reg |= MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE | + MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE; + writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF); + + /* BW CTRL - set values to optimize the XOR performance: + * + * - Set WrBurstLen & RdBurstLen - the unit will issue + * maximum of 256B write/read transactions. + * - Limit the number of outstanding write & read data + * (OBB/IBB) requests to the maximal value. + */ + reg = ((MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_VAL << + MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_SHIFT) | + (MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_VAL << + MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_SHIFT) | + (MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_VAL << + MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_SHIFT) | + (MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_VAL << + MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_SHIFT)); + writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_BW_CTRL); + + /* Disable the AXI timer feature */ + reg = readl(xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE); + reg |= MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL; + writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE); + + return 0; +} + +static int mv_xor_v2_probe(struct platform_device *pdev) +{ + struct mv_xor_v2_device *xor_dev; + struct resource *res; + int i, ret = 0; + struct dma_device *dma_dev; + struct mv_xor_v2_sw_desc *sw_desc; + struct msi_desc *msi_desc; + + BUILD_BUG_ON(sizeof(struct mv_xor_v2_descriptor) != + MV_XOR_V2_EXT_DESC_SIZE); + + xor_dev = devm_kzalloc(&pdev->dev, sizeof(*xor_dev), GFP_KERNEL); + if (!xor_dev) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + xor_dev->dma_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(xor_dev->dma_base)) + return PTR_ERR(xor_dev->dma_base); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + xor_dev->glob_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(xor_dev->glob_base)) + return PTR_ERR(xor_dev->glob_base); + + platform_set_drvdata(pdev, xor_dev); + + xor_dev->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(xor_dev->clk) && PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) + return -EPROBE_DEFER; + if (!IS_ERR(xor_dev->clk)) { + ret = clk_prepare_enable(xor_dev->clk); + if (ret) + return ret; + } + + ret = platform_msi_domain_alloc_irqs(&pdev->dev, 1, + mv_xor_v2_set_msi_msg); + if (ret) + goto disable_clk; + + msi_desc = first_msi_entry(&pdev->dev); + if (!msi_desc) + goto free_msi_irqs; + + ret = devm_request_irq(&pdev->dev, msi_desc->irq, + mv_xor_v2_interrupt_handler, 0, + dev_name(&pdev->dev), xor_dev); + if (ret) + goto free_msi_irqs; + + tasklet_init(&xor_dev->irq_tasklet, mv_xor_v2_tasklet, + (unsigned long) xor_dev); + + xor_dev->desc_size = mv_xor_v2_set_desc_size(xor_dev); + + dma_cookie_init(&xor_dev->dmachan); + + /* + * allocate coherent memory for hardware descriptors + * note: writecombine gives slightly better performance, but + * requires that we explicitly flush the writes + */ + xor_dev->hw_desq_virt = + dma_alloc_coherent(&pdev->dev, + xor_dev->desc_size * MV_XOR_V2_DESC_NUM, + &xor_dev->hw_desq, GFP_KERNEL); + if (!xor_dev->hw_desq_virt) { + ret = -ENOMEM; + goto free_msi_irqs; + } + + /* alloc memory for the SW descriptors */ + xor_dev->sw_desq = devm_kzalloc(&pdev->dev, sizeof(*sw_desc) * + MV_XOR_V2_DESC_NUM, GFP_KERNEL); + if (!xor_dev->sw_desq) { + ret = -ENOMEM; + goto free_hw_desq; + } + + spin_lock_init(&xor_dev->lock); + + /* init the free SW descriptors list */ + INIT_LIST_HEAD(&xor_dev->free_sw_desc); + + /* add all SW descriptors to the free list */ + for (i = 0; i < MV_XOR_V2_DESC_NUM; i++) { + xor_dev->sw_desq[i].idx = i; + list_add(&xor_dev->sw_desq[i].free_list, + &xor_dev->free_sw_desc); + } + + dma_dev = &xor_dev->dmadev; + + /* set DMA capabilities */ + dma_cap_zero(dma_dev->cap_mask); + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_cap_set(DMA_XOR, dma_dev->cap_mask); + dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); + + /* init dma link list */ + INIT_LIST_HEAD(&dma_dev->channels); + + /* set base routines */ + dma_dev->device_tx_status = dma_cookie_status; + dma_dev->device_issue_pending = mv_xor_v2_issue_pending; + dma_dev->dev = &pdev->dev; + + dma_dev->device_prep_dma_memcpy = mv_xor_v2_prep_dma_memcpy; + dma_dev->device_prep_dma_interrupt = mv_xor_v2_prep_dma_interrupt; + dma_dev->max_xor = 8; + dma_dev->device_prep_dma_xor = mv_xor_v2_prep_dma_xor; + + xor_dev->dmachan.device = dma_dev; + + list_add_tail(&xor_dev->dmachan.device_node, + &dma_dev->channels); + + mv_xor_v2_descq_init(xor_dev); + + ret = dma_async_device_register(dma_dev); + if (ret) + goto free_hw_desq; + + dev_notice(&pdev->dev, "Marvell Version 2 XOR driver\n"); + + return 0; + +free_hw_desq: + dma_free_coherent(&pdev->dev, + xor_dev->desc_size * MV_XOR_V2_DESC_NUM, + xor_dev->hw_desq_virt, xor_dev->hw_desq); +free_msi_irqs: + platform_msi_domain_free_irqs(&pdev->dev); +disable_clk: + if (!IS_ERR(xor_dev->clk)) + clk_disable_unprepare(xor_dev->clk); + return ret; +} + +static int mv_xor_v2_remove(struct platform_device *pdev) +{ + struct mv_xor_v2_device *xor_dev = platform_get_drvdata(pdev); + + dma_async_device_unregister(&xor_dev->dmadev); + + dma_free_coherent(&pdev->dev, + xor_dev->desc_size * MV_XOR_V2_DESC_NUM, + xor_dev->hw_desq_virt, xor_dev->hw_desq); + + platform_msi_domain_free_irqs(&pdev->dev); + + clk_disable_unprepare(xor_dev->clk); + + return 0; +} + +#ifdef CONFIG_OF +static const struct of_device_id mv_xor_v2_dt_ids[] = { + { .compatible = "marvell,xor-v2", }, + {}, +}; +MODULE_DEVICE_TABLE(of, mv_xor_v2_dt_ids); +#endif + +static struct platform_driver mv_xor_v2_driver = { + .probe = mv_xor_v2_probe, + .remove = mv_xor_v2_remove, + .driver = { + .name = "mv_xor_v2", + .of_match_table = of_match_ptr(mv_xor_v2_dt_ids), + }, +}; + +module_platform_driver(mv_xor_v2_driver); + +MODULE_DESCRIPTION("DMA engine driver for Marvell's Version 2 of XOR engine"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c index 2b5a198ac..08c45c185 100644 --- a/drivers/dma/nbpfaxi.c +++ b/drivers/dma/nbpfaxi.c @@ -227,6 +227,7 @@ struct nbpf_device { void __iomem *base; struct clk *clk; const struct nbpf_config *config; + unsigned int eirq; struct nbpf_channel chan[]; }; @@ -1300,10 +1301,9 @@ static int nbpf_probe(struct platform_device *pdev) nbpf = devm_kzalloc(dev, sizeof(*nbpf) + num_channels * sizeof(nbpf->chan[0]), GFP_KERNEL); - if (!nbpf) { - dev_err(dev, "Memory allocation failed\n"); + if (!nbpf) return -ENOMEM; - } + dma_dev = &nbpf->dma_dev; dma_dev->dev = dev; @@ -1376,6 +1376,7 @@ static int nbpf_probe(struct platform_device *pdev) IRQF_SHARED, "dma error", nbpf); if (ret < 0) return ret; + nbpf->eirq = eirq; INIT_LIST_HEAD(&dma_dev->channels); @@ -1447,6 +1448,17 @@ e_clk_off: static int nbpf_remove(struct platform_device *pdev) { struct nbpf_device *nbpf = platform_get_drvdata(pdev); + int i; + + devm_free_irq(&pdev->dev, nbpf->eirq, nbpf); + + for (i = 0; i < nbpf->config->num_channels; i++) { + struct nbpf_channel *chan = nbpf->chan + i; + + devm_free_irq(&pdev->dev, chan->irq, chan); + + tasklet_kill(&chan->tasklet); + } of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&nbpf->dma_dev); diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 1e984e18c..d99ca2b51 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -59,6 +59,8 @@ struct omap_sg { dma_addr_t addr; uint32_t en; /* number of elements (24-bit) */ uint32_t fn; /* number of frames (16-bit) */ + int32_t fi; /* for double indexing */ + int16_t ei; /* for double indexing */ }; struct omap_desc { @@ -66,7 +68,8 @@ struct omap_desc { enum dma_transfer_direction dir; dma_addr_t dev_addr; - int16_t fi; /* for OMAP_DMA_SYNC_PACKET */ + int32_t fi; /* for OMAP_DMA_SYNC_PACKET / double indexing */ + int16_t ei; /* for double indexing */ uint8_t es; /* CSDP_DATA_TYPE_xxx */ uint32_t ccr; /* CCR value */ uint16_t clnk_ctrl; /* CLNK_CTRL value */ @@ -379,8 +382,8 @@ static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d, } omap_dma_chan_write(c, cxsa, sg->addr); - omap_dma_chan_write(c, cxei, 0); - omap_dma_chan_write(c, cxfi, 0); + omap_dma_chan_write(c, cxei, sg->ei); + omap_dma_chan_write(c, cxfi, sg->fi); omap_dma_chan_write(c, CEN, sg->en); omap_dma_chan_write(c, CFN, sg->fn); @@ -425,7 +428,7 @@ static void omap_dma_start_desc(struct omap_chan *c) } omap_dma_chan_write(c, cxsa, d->dev_addr); - omap_dma_chan_write(c, cxei, 0); + omap_dma_chan_write(c, cxei, d->ei); omap_dma_chan_write(c, cxfi, d->fi); omap_dma_chan_write(c, CSDP, d->csdp); omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl); @@ -971,6 +974,89 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy( return vchan_tx_prep(&c->vc, &d->vd, tx_flags); } +static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved( + struct dma_chan *chan, struct dma_interleaved_template *xt, + unsigned long flags) +{ + struct omap_chan *c = to_omap_dma_chan(chan); + struct omap_desc *d; + struct omap_sg *sg; + uint8_t data_type; + size_t src_icg, dst_icg; + + /* Slave mode is not supported */ + if (is_slave_direction(xt->dir)) + return NULL; + + if (xt->frame_size != 1 || xt->numf == 0) + return NULL; + + d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC); + if (!d) + return NULL; + + data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size)); + if (data_type > CSDP_DATA_TYPE_32) + data_type = CSDP_DATA_TYPE_32; + + sg = &d->sg[0]; + d->dir = DMA_MEM_TO_MEM; + d->dev_addr = xt->src_start; + d->es = data_type; + sg->en = xt->sgl[0].size / BIT(data_type); + sg->fn = xt->numf; + sg->addr = xt->dst_start; + d->sglen = 1; + d->ccr = c->ccr; + + src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); + dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); + if (src_icg) { + d->ccr |= CCR_SRC_AMODE_DBLIDX; + d->ei = 1; + d->fi = src_icg; + } else if (xt->src_inc) { + d->ccr |= CCR_SRC_AMODE_POSTINC; + d->fi = 0; + } else { + dev_err(chan->device->dev, + "%s: SRC constant addressing is not supported\n", + __func__); + kfree(d); + return NULL; + } + + if (dst_icg) { + d->ccr |= CCR_DST_AMODE_DBLIDX; + sg->ei = 1; + sg->fi = dst_icg; + } else if (xt->dst_inc) { + d->ccr |= CCR_DST_AMODE_POSTINC; + sg->fi = 0; + } else { + dev_err(chan->device->dev, + "%s: DST constant addressing is not supported\n", + __func__); + kfree(d); + return NULL; + } + + d->cicr = CICR_DROP_IE | CICR_FRAME_IE; + + d->csdp = data_type; + + if (dma_omap1()) { + d->cicr |= CICR_TOUT_IE; + d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF; + } else { + d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED; + d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; + d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; + } + + return vchan_tx_prep(&c->vc, &d->vd, flags); +} + static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) { struct omap_chan *c = to_omap_dma_chan(chan); @@ -1116,6 +1202,7 @@ static int omap_dma_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask); + dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask); od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources; od->ddev.device_free_chan_resources = omap_dma_free_chan_resources; od->ddev.device_tx_status = omap_dma_tx_status; @@ -1123,6 +1210,7 @@ static int omap_dma_probe(struct platform_device *pdev) od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg; od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic; od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy; + od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved; od->ddev.device_config = omap_dma_slave_config; od->ddev.device_pause = omap_dma_pause; od->ddev.device_resume = omap_dma_resume; @@ -1204,10 +1292,14 @@ static int omap_dma_probe(struct platform_device *pdev) static int omap_dma_remove(struct platform_device *pdev) { struct omap_dmadev *od = platform_get_drvdata(pdev); + int irq; if (pdev->dev.of_node) of_dma_controller_free(pdev->dev.of_node); + irq = platform_get_irq(pdev, 1); + devm_free_irq(&pdev->dev, irq, od); + dma_async_device_unregister(&od->ddev); if (!od->legacy) { diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 372b4359d..4fc3ffbd5 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -2828,10 +2828,8 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) /* Allocate a new DMAC and its Channels */ pl330 = devm_kzalloc(&adev->dev, sizeof(*pl330), GFP_KERNEL); - if (!pl330) { - dev_err(&adev->dev, "unable to allocate mem\n"); + if (!pl330) return -ENOMEM; - } pd = &pl330->ddma; pd->dev = &adev->dev; @@ -2890,7 +2888,6 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) pl330->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL); if (!pl330->peripherals) { ret = -ENOMEM; - dev_err(&adev->dev, "unable to allocate pl330->peripherals\n"); goto probe_err2; } @@ -3005,12 +3002,18 @@ static int pl330_remove(struct amba_device *adev) { struct pl330_dmac *pl330 = amba_get_drvdata(adev); struct dma_pl330_chan *pch, *_p; + int i, irq; pm_runtime_get_noresume(pl330->ddma.dev); if (adev->dev.of_node) of_dma_controller_free(adev->dev.of_node); + for (i = 0; i < AMBA_NR_IRQS; i++) { + irq = adev->irq[i]; + devm_free_irq(&adev->dev, irq, pl330); + } + dma_async_device_unregister(&pl330->ddma); /* Idle the DMAC */ diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index 9217f893b..da3688b94 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -4084,7 +4084,6 @@ static int ppc440spe_adma_probe(struct platform_device *ofdev) /* create a device */ adev = kzalloc(sizeof(*adev), GFP_KERNEL); if (!adev) { - dev_err(&ofdev->dev, "failed to allocate device\n"); initcode = PPC_ADMA_INIT_ALLOC; ret = -ENOMEM; goto err_adev_alloc; @@ -4145,7 +4144,6 @@ static int ppc440spe_adma_probe(struct platform_device *ofdev) /* create a channel */ chan = kzalloc(sizeof(*chan), GFP_KERNEL); if (!chan) { - dev_err(&ofdev->dev, "can't allocate channel structure\n"); initcode = PPC_ADMA_INIT_CHANNEL; ret = -ENOMEM; goto err_chan_alloc; diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index e756a30cc..3f56f9ca4 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "dmaengine.h" @@ -118,6 +119,8 @@ struct pxad_chan { struct pxad_phy *phy; struct dma_pool *desc_pool; /* Descriptors pool */ dma_cookie_t bus_error; + + wait_queue_head_t wq_state; }; struct pxad_device { @@ -318,7 +321,6 @@ static int dbg_open_##name(struct inode *inode, struct file *file) \ return single_open(file, dbg_show_##name, inode->i_private); \ } \ static const struct file_operations dbg_fops_##name = { \ - .owner = THIS_MODULE, \ .open = dbg_open_##name, \ .llseek = seq_lseek, \ .read = seq_read, \ @@ -572,6 +574,7 @@ static void pxad_launch_chan(struct pxad_chan *chan, */ phy_writel(chan->phy, desc->first, DDADR); phy_enable(chan->phy, chan->misaligned); + wake_up(&chan->wq_state); } static void set_updater_desc(struct pxad_desc_sw *sw_desc, @@ -635,7 +638,7 @@ static bool pxad_try_hotchain(struct virt_dma_chan *vc, vd_last_issued = list_entry(vc->desc_issued.prev, struct virt_dma_desc, node); pxad_desc_chain(vd_last_issued, vd); - if (is_chan_running(chan) || is_desc_completed(vd_last_issued)) + if (is_chan_running(chan) || is_desc_completed(vd)) return true; } @@ -668,6 +671,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id) struct virt_dma_desc *vd, *tmp; unsigned int dcsr; unsigned long flags; + bool vd_completed; dma_cookie_t last_started = 0; BUG_ON(!chan); @@ -678,15 +682,17 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id) spin_lock_irqsave(&chan->vc.lock, flags); list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) { + vd_completed = is_desc_completed(vd); dev_dbg(&chan->vc.chan.dev->device, - "%s(): checking txd %p[%x]: completed=%d\n", - __func__, vd, vd->tx.cookie, is_desc_completed(vd)); + "%s(): checking txd %p[%x]: completed=%d dcsr=0x%x\n", + __func__, vd, vd->tx.cookie, vd_completed, + dcsr); last_started = vd->tx.cookie; if (to_pxad_sw_desc(vd)->cyclic) { vchan_cyclic_callback(vd); break; } - if (is_desc_completed(vd)) { + if (vd_completed) { list_del(&vd->node); vchan_cookie_complete(vd); } else { @@ -717,6 +723,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id) } } spin_unlock_irqrestore(&chan->vc.lock, flags); + wake_up(&chan->wq_state); return IRQ_HANDLED; } @@ -1268,6 +1275,14 @@ static enum dma_status pxad_tx_status(struct dma_chan *dchan, return ret; } +static void pxad_synchronize(struct dma_chan *dchan) +{ + struct pxad_chan *chan = to_pxad_chan(dchan); + + wait_event(chan->wq_state, !is_chan_running(chan)); + vchan_synchronize(&chan->vc); +} + static void pxad_free_channels(struct dma_device *dmadev) { struct pxad_chan *c, *cn; @@ -1372,6 +1387,7 @@ static int pxad_init_dmadev(struct platform_device *op, pdev->slave.device_tx_status = pxad_tx_status; pdev->slave.device_issue_pending = pxad_issue_pending; pdev->slave.device_config = pxad_config; + pdev->slave.device_synchronize = pxad_synchronize; pdev->slave.device_terminate_all = pxad_terminate_all; if (op->dev.coherent_dma_mask) @@ -1389,6 +1405,7 @@ static int pxad_init_dmadev(struct platform_device *op, return -ENOMEM; c->vc.desc_free = pxad_free_desc; vchan_init(&c->vc, &pdev->slave); + init_waitqueue_head(&c->wq_state); } return dma_async_device_register(&pdev->slave); diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 969b48176..03c4eb3fd 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -48,6 +48,7 @@ #include #include #include +#include #include "../dmaengine.h" #include "../virt-dma.h" @@ -58,6 +59,8 @@ struct bam_desc_hw { __le16 flags; }; +#define BAM_DMA_AUTOSUSPEND_DELAY 100 + #define DESC_FLAG_INT BIT(15) #define DESC_FLAG_EOT BIT(14) #define DESC_FLAG_EOB BIT(13) @@ -527,12 +530,17 @@ static void bam_free_chan(struct dma_chan *chan) struct bam_device *bdev = bchan->bdev; u32 val; unsigned long flags; + int ret; + + ret = pm_runtime_get_sync(bdev->dev); + if (ret < 0) + return; vchan_free_chan_resources(to_virt_chan(chan)); if (bchan->curr_txd) { dev_err(bchan->bdev->dev, "Cannot free busy channel\n"); - return; + goto err; } spin_lock_irqsave(&bchan->vc.lock, flags); @@ -550,6 +558,10 @@ static void bam_free_chan(struct dma_chan *chan) /* disable irq */ writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN)); + +err: + pm_runtime_mark_last_busy(bdev->dev); + pm_runtime_put_autosuspend(bdev->dev); } /** @@ -696,11 +708,18 @@ static int bam_pause(struct dma_chan *chan) struct bam_chan *bchan = to_bam_chan(chan); struct bam_device *bdev = bchan->bdev; unsigned long flag; + int ret; + + ret = pm_runtime_get_sync(bdev->dev); + if (ret < 0) + return ret; spin_lock_irqsave(&bchan->vc.lock, flag); writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT)); bchan->paused = 1; spin_unlock_irqrestore(&bchan->vc.lock, flag); + pm_runtime_mark_last_busy(bdev->dev); + pm_runtime_put_autosuspend(bdev->dev); return 0; } @@ -715,11 +734,18 @@ static int bam_resume(struct dma_chan *chan) struct bam_chan *bchan = to_bam_chan(chan); struct bam_device *bdev = bchan->bdev; unsigned long flag; + int ret; + + ret = pm_runtime_get_sync(bdev->dev); + if (ret < 0) + return ret; spin_lock_irqsave(&bchan->vc.lock, flag); writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT)); bchan->paused = 0; spin_unlock_irqrestore(&bchan->vc.lock, flag); + pm_runtime_mark_last_busy(bdev->dev); + pm_runtime_put_autosuspend(bdev->dev); return 0; } @@ -795,6 +821,7 @@ static irqreturn_t bam_dma_irq(int irq, void *data) { struct bam_device *bdev = data; u32 clr_mask = 0, srcs = 0; + int ret; srcs |= process_channel_irqs(bdev); @@ -802,6 +829,10 @@ static irqreturn_t bam_dma_irq(int irq, void *data) if (srcs & P_IRQ) tasklet_schedule(&bdev->task); + ret = pm_runtime_get_sync(bdev->dev); + if (ret < 0) + return ret; + if (srcs & BAM_IRQ) { clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS)); @@ -814,6 +845,9 @@ static irqreturn_t bam_dma_irq(int irq, void *data) writel_relaxed(clr_mask, bam_addr(bdev, 0, BAM_IRQ_CLR)); } + pm_runtime_mark_last_busy(bdev->dev); + pm_runtime_put_autosuspend(bdev->dev); + return IRQ_HANDLED; } @@ -893,6 +927,7 @@ static void bam_start_dma(struct bam_chan *bchan) struct bam_desc_hw *desc; struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt, sizeof(struct bam_desc_hw)); + int ret; lockdep_assert_held(&bchan->vc.lock); @@ -904,6 +939,10 @@ static void bam_start_dma(struct bam_chan *bchan) async_desc = container_of(vd, struct bam_async_desc, vd); bchan->curr_txd = async_desc; + ret = pm_runtime_get_sync(bdev->dev); + if (ret < 0) + return; + /* on first use, initialize the channel hardware */ if (!bchan->initialized) bam_chan_init_hw(bchan, async_desc->dir); @@ -946,6 +985,9 @@ static void bam_start_dma(struct bam_chan *bchan) wmb(); writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw), bam_addr(bdev, bchan->id, BAM_P_EVNT_REG)); + + pm_runtime_mark_last_busy(bdev->dev); + pm_runtime_put_autosuspend(bdev->dev); } /** @@ -970,6 +1012,7 @@ static void dma_tasklet(unsigned long data) bam_start_dma(bchan); spin_unlock_irqrestore(&bchan->vc.lock, flags); } + } /** @@ -1213,6 +1256,13 @@ static int bam_dma_probe(struct platform_device *pdev) if (ret) goto err_unregister_dma; + pm_runtime_irq_safe(&pdev->dev); + pm_runtime_set_autosuspend_delay(&pdev->dev, BAM_DMA_AUTOSUSPEND_DELAY); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_mark_last_busy(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + return 0; err_unregister_dma: @@ -1233,6 +1283,8 @@ static int bam_dma_remove(struct platform_device *pdev) struct bam_device *bdev = platform_get_drvdata(pdev); u32 i; + pm_runtime_force_suspend(&pdev->dev); + of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&bdev->common); @@ -1260,11 +1312,66 @@ static int bam_dma_remove(struct platform_device *pdev) return 0; } +static int __maybe_unused bam_dma_runtime_suspend(struct device *dev) +{ + struct bam_device *bdev = dev_get_drvdata(dev); + + clk_disable(bdev->bamclk); + + return 0; +} + +static int __maybe_unused bam_dma_runtime_resume(struct device *dev) +{ + struct bam_device *bdev = dev_get_drvdata(dev); + int ret; + + ret = clk_enable(bdev->bamclk); + if (ret < 0) { + dev_err(dev, "clk_enable failed: %d\n", ret); + return ret; + } + + return 0; +} + +static int __maybe_unused bam_dma_suspend(struct device *dev) +{ + struct bam_device *bdev = dev_get_drvdata(dev); + + pm_runtime_force_suspend(dev); + + clk_unprepare(bdev->bamclk); + + return 0; +} + +static int __maybe_unused bam_dma_resume(struct device *dev) +{ + struct bam_device *bdev = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare(bdev->bamclk); + if (ret) + return ret; + + pm_runtime_force_resume(dev); + + return 0; +} + +static const struct dev_pm_ops bam_dma_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(bam_dma_suspend, bam_dma_resume) + SET_RUNTIME_PM_OPS(bam_dma_runtime_suspend, bam_dma_runtime_resume, + NULL) +}; + static struct platform_driver bam_dma_driver = { .probe = bam_dma_probe, .remove = bam_dma_remove, .driver = { .name = "bam-dma-engine", + .pm = &bam_dma_pm_ops, .of_match_table = bam_of_match, }, }; diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index 41b5c6dee..b2374cd91 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -708,6 +708,7 @@ static int hidma_remove(struct platform_device *pdev) pm_runtime_get_sync(dmadev->ddev.dev); dma_async_device_unregister(&dmadev->ddev); devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev); + tasklet_kill(&dmadev->task); hidma_debug_uninit(dmadev); hidma_ll_uninit(dmadev->lldev); hidma_free(dmadev); diff --git a/drivers/dma/qcom/hidma_ll.c b/drivers/dma/qcom/hidma_ll.c index f39290015..ad20dfb64 100644 --- a/drivers/dma/qcom/hidma_ll.c +++ b/drivers/dma/qcom/hidma_ll.c @@ -831,6 +831,7 @@ int hidma_ll_uninit(struct hidma_lldev *lldev) required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres; tasklet_kill(&lldev->task); + tasklet_kill(&lldev->rst_task); memset(lldev->trepool, 0, required_bytes); lldev->trepool = NULL; lldev->pending_tre_count = 0; diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c index c0e365321..82f36e466 100644 --- a/drivers/dma/qcom/hidma_mgmt.c +++ b/drivers/dma/qcom/hidma_mgmt.c @@ -371,8 +371,8 @@ static int __init hidma_mgmt_of_populate_channels(struct device_node *np) pdevinfo.size_data = 0; pdevinfo.dma_mask = DMA_BIT_MASK(64); new_pdev = platform_device_register_full(&pdevinfo); - if (!new_pdev) { - ret = -ENODEV; + if (IS_ERR(new_pdev)) { + ret = PTR_ERR(new_pdev); goto out; } of_dma_configure(&new_pdev->dev, child); @@ -392,8 +392,7 @@ static int __init hidma_mgmt_init(void) #if defined(CONFIG_OF) && defined(CONFIG_OF_IRQ) struct device_node *child; - for (child = of_find_matching_node(NULL, hidma_mgmt_match); child; - child = of_find_matching_node(child, hidma_mgmt_match)) { + for_each_matching_node(child, hidma_mgmt_match) { /* device tree based firmware here */ hidma_mgmt_of_populate_channels(child); of_node_put(child); diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c index 17ccdfd28..ce6707558 100644 --- a/drivers/dma/s3c24xx-dma.c +++ b/drivers/dma/s3c24xx-dma.c @@ -768,16 +768,12 @@ static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan, spin_lock_irqsave(&s3cchan->vc.lock, flags); ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE) { - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - return ret; - } /* * There's no point calculating the residue if there's * no txstate to store the value. */ - if (!txstate) { + if (ret == DMA_COMPLETE || !txstate) { spin_unlock_irqrestore(&s3cchan->vc.lock, flags); return ret; } @@ -1105,11 +1101,8 @@ static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma, */ for (i = 0; i < channels; i++) { chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL); - if (!chan) { - dev_err(dmadev->dev, - "%s no memory for channel\n", __func__); + if (!chan) return -ENOMEM; - } chan->id = i; chan->host = s3cdma; @@ -1143,8 +1136,10 @@ static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev) struct s3c24xx_dma_chan *next; list_for_each_entry_safe(chan, - next, &dmadev->channels, vc.chan.device_node) + next, &dmadev->channels, vc.chan.device_node) { list_del(&chan->vc.chan.device_node); + tasklet_kill(&chan->vc.task); + } } /* s3c2410, s3c2440 and s3c2442 have a 0x40 stride without separate clocks */ @@ -1366,6 +1361,18 @@ err_memcpy: return ret; } +static void s3c24xx_dma_free_irq(struct platform_device *pdev, + struct s3c24xx_dma_engine *s3cdma) +{ + int i; + + for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) { + struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; + + devm_free_irq(&pdev->dev, phy->irq, phy); + } +} + static int s3c24xx_dma_remove(struct platform_device *pdev) { const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev); @@ -1376,6 +1383,8 @@ static int s3c24xx_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&s3cdma->slave); dma_async_device_unregister(&s3cdma->memcpy); + s3c24xx_dma_free_irq(pdev, s3cdma); + s3c24xx_dma_free_virtual_channels(&s3cdma->slave); s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy); diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index dfb179262..0dd953884 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -311,7 +311,7 @@ static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan) { u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR); - return (chcr & (RCAR_DMACHCR_DE | RCAR_DMACHCR_TE)) == RCAR_DMACHCR_DE; + return !!(chcr & (RCAR_DMACHCR_DE | RCAR_DMACHCR_TE)); } static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan) @@ -510,7 +510,7 @@ static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan, spin_lock_irqsave(&chan->lock, flags); list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free); - list_add_tail(&desc->node, &chan->desc.free); + list_add(&desc->node, &chan->desc.free); spin_unlock_irqrestore(&chan->lock, flags); } @@ -990,6 +990,8 @@ static void rcar_dmac_free_chan_resources(struct dma_chan *chan) list_splice_init(&rchan->desc.done, &list); list_splice_init(&rchan->desc.wait, &list); + rchan->desc.running = NULL; + list_for_each_entry(desc, &list, node) rcar_dmac_realloc_hwdesc(rchan, desc, 0); @@ -1143,19 +1145,46 @@ static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan, struct rcar_dmac_desc *desc = chan->desc.running; struct rcar_dmac_xfer_chunk *running = NULL; struct rcar_dmac_xfer_chunk *chunk; + enum dma_status status; unsigned int residue = 0; unsigned int dptr = 0; if (!desc) return 0; + /* + * If the cookie corresponds to a descriptor that has been completed + * there is no residue. The same check has already been performed by the + * caller but without holding the channel lock, so the descriptor could + * now be complete. + */ + status = dma_cookie_status(&chan->chan, cookie, NULL); + if (status == DMA_COMPLETE) + return 0; + /* * If the cookie doesn't correspond to the currently running transfer * then the descriptor hasn't been processed yet, and the residue is * equal to the full descriptor size. */ - if (cookie != desc->async_tx.cookie) - return desc->size; + if (cookie != desc->async_tx.cookie) { + list_for_each_entry(desc, &chan->desc.pending, node) { + if (cookie == desc->async_tx.cookie) + return desc->size; + } + list_for_each_entry(desc, &chan->desc.active, node) { + if (cookie == desc->async_tx.cookie) + return desc->size; + } + + /* + * No descriptor found for the cookie, there's thus no residue. + * This shouldn't happen if the calling driver passes a correct + * cookie value. + */ + WARN(1, "No descriptor for cookie!"); + return 0; + } /* * In descriptor mode the descriptor running pointer is not maintained @@ -1202,6 +1231,10 @@ static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan, residue = rcar_dmac_chan_get_residue(rchan, cookie); spin_unlock_irqrestore(&rchan->lock, flags); + /* if there's no residue, the cookie is complete */ + if (!residue) + return DMA_COMPLETE; + dma_set_residue(txstate, residue); return status; diff --git a/drivers/dma/sh/shdmac.c b/drivers/dma/sh/shdmac.c index 80d864024..c94ffab0d 100644 --- a/drivers/dma/sh/shdmac.c +++ b/drivers/dma/sh/shdmac.c @@ -532,11 +532,8 @@ static int sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id, sh_chan = devm_kzalloc(sdev->dma_dev.dev, sizeof(struct sh_dmae_chan), GFP_KERNEL); - if (!sh_chan) { - dev_err(sdev->dma_dev.dev, - "No free memory for allocating dma channels!\n"); + if (!sh_chan) return -ENOMEM; - } schan = &sh_chan->shdma_chan; schan->max_xfer_len = SH_DMA_TCR_MAX + 1; @@ -732,10 +729,8 @@ static int sh_dmae_probe(struct platform_device *pdev) shdev = devm_kzalloc(&pdev->dev, sizeof(struct sh_dmae_device), GFP_KERNEL); - if (!shdev) { - dev_err(&pdev->dev, "Not enough memory\n"); + if (!shdev) return -ENOMEM; - } dma_dev = &shdev->shdma_dev.dma_dev; diff --git a/drivers/dma/sh/sudmac.c b/drivers/dma/sh/sudmac.c index 6da2eaa6c..69b9564dc 100644 --- a/drivers/dma/sh/sudmac.c +++ b/drivers/dma/sh/sudmac.c @@ -245,11 +245,8 @@ static int sudmac_chan_probe(struct sudmac_device *su_dev, int id, int irq, int err; sc = devm_kzalloc(&pdev->dev, sizeof(struct sudmac_chan), GFP_KERNEL); - if (!sc) { - dev_err(sdev->dma_dev.dev, - "No free memory for allocating dma channels!\n"); + if (!sc) return -ENOMEM; - } schan = &sc->shdma_chan; schan->max_xfer_len = 64 * 1024 * 1024 - 1; @@ -349,10 +346,8 @@ static int sudmac_probe(struct platform_device *pdev) err = -ENOMEM; su_dev = devm_kzalloc(&pdev->dev, sizeof(struct sudmac_device), GFP_KERNEL); - if (!su_dev) { - dev_err(&pdev->dev, "Not enough memory\n"); + if (!su_dev) return err; - } dma_dev = &su_dev->shdma_dev.dma_dev; diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c index e48350e65..d8bc3f2a7 100644 --- a/drivers/dma/sirf-dma.c +++ b/drivers/dma/sirf-dma.c @@ -854,10 +854,9 @@ static int sirfsoc_dma_probe(struct platform_device *op) int ret, i; sdma = devm_kzalloc(dev, sizeof(*sdma), GFP_KERNEL); - if (!sdma) { - dev_err(dev, "Memory exhausted!\n"); + if (!sdma) return -ENOMEM; - } + data = (struct sirfsoc_dmadata *) (of_match_device(op->dev.driver->of_match_table, &op->dev)->data); @@ -981,6 +980,7 @@ static int sirfsoc_dma_remove(struct platform_device *op) of_dma_controller_free(op->dev.of_node); dma_async_device_unregister(&sdma->dma); free_irq(sdma->irq, sdma); + tasklet_kill(&sdma->tasklet); irq_dispose_mapping(sdma->irq); pm_runtime_disable(&op->dev); if (!pm_runtime_status_suspended(&op->dev)) @@ -1126,17 +1126,17 @@ static const struct dev_pm_ops sirfsoc_dma_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(sirfsoc_dma_pm_suspend, sirfsoc_dma_pm_resume) }; -struct sirfsoc_dmadata sirfsoc_dmadata_a6 = { +static struct sirfsoc_dmadata sirfsoc_dmadata_a6 = { .exec = sirfsoc_dma_execute_hw_a6, .type = SIRFSOC_DMA_VER_A6, }; -struct sirfsoc_dmadata sirfsoc_dmadata_a7v1 = { +static struct sirfsoc_dmadata sirfsoc_dmadata_a7v1 = { .exec = sirfsoc_dma_execute_hw_a7v1, .type = SIRFSOC_DMA_VER_A7V1, }; -struct sirfsoc_dmadata sirfsoc_dmadata_a7v2 = { +static struct sirfsoc_dmadata sirfsoc_dmadata_a7v2 = { .exec = sirfsoc_dma_execute_hw_a7v2, .type = SIRFSOC_DMA_VER_A7V2, }; diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 6fb830746..8b18e44a0 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -2588,7 +2588,7 @@ static enum dma_status d40_tx_status(struct dma_chan *chan, } ret = dma_cookie_status(chan, cookie, txstate); - if (ret != DMA_COMPLETE) + if (ret != DMA_COMPLETE && txstate) dma_set_residue(txstate, stedma40_residue(chan)); if (d40_is_paused(d40c)) @@ -3237,10 +3237,8 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev) (num_phy_chans + num_log_chans + num_memcpy_chans) * sizeof(struct d40_chan), GFP_KERNEL); - if (base == NULL) { - d40_err(&pdev->dev, "Out of memory\n"); + if (base == NULL) goto failure; - } base->rev = rev; base->clk = clk; diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c index 27b818dee..13b42dd99 100644 --- a/drivers/dma/ste_dma40_ll.c +++ b/drivers/dma/ste_dma40_ll.c @@ -10,7 +10,7 @@ #include "ste_dma40_ll.h" -u8 d40_width_to_bits(enum dma_slave_buswidth width) +static u8 d40_width_to_bits(enum dma_slave_buswidth width) { if (width == DMA_SLAVE_BUSWIDTH_1_BYTE) return STEDMA40_ESIZE_8_BIT; diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index 5065ca43f..3835fcde3 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -865,7 +865,7 @@ static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan, size_t bytes = 0; ret = dma_cookie_status(chan, cookie, state); - if (ret == DMA_COMPLETE) + if (ret == DMA_COMPLETE || !state) return ret; spin_lock_irqsave(&vchan->vc.lock, flags); diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index 01e316f73..6ab9eb985 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -300,10 +300,8 @@ static struct tegra_dma_desc *tegra_dma_desc_get( /* Allocate DMA desc */ dma_desc = kzalloc(sizeof(*dma_desc), GFP_NOWAIT); - if (!dma_desc) { - dev_err(tdc2dev(tdc), "dma_desc alloc failed\n"); + if (!dma_desc) return NULL; - } dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan); dma_desc->txd.tx_submit = tegra_dma_tx_submit; @@ -340,8 +338,7 @@ static struct tegra_dma_sg_req *tegra_dma_sg_req_get( spin_unlock_irqrestore(&tdc->lock, flags); sg_req = kzalloc(sizeof(struct tegra_dma_sg_req), GFP_NOWAIT); - if (!sg_req) - dev_err(tdc2dev(tdc), "sg_req alloc failed\n"); + return sg_req; } @@ -484,7 +481,7 @@ static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, * load new configuration. */ tegra_dma_pause(tdc, false); - status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); + status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); /* * If interrupt is pending then do nothing as the ISR will handle @@ -822,13 +819,8 @@ static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, /* Check on wait_ack desc status */ list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) { if (dma_desc->txd.cookie == cookie) { - residual = dma_desc->bytes_requested - - (dma_desc->bytes_transferred % - dma_desc->bytes_requested); - dma_set_residue(txstate, residual); ret = dma_desc->dma_status; - spin_unlock_irqrestore(&tdc->lock, flags); - return ret; + goto found; } } @@ -836,17 +828,22 @@ static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, list_for_each_entry(sg_req, &tdc->pending_sg_req, node) { dma_desc = sg_req->dma_desc; if (dma_desc->txd.cookie == cookie) { - residual = dma_desc->bytes_requested - - (dma_desc->bytes_transferred % - dma_desc->bytes_requested); - dma_set_residue(txstate, residual); ret = dma_desc->dma_status; - spin_unlock_irqrestore(&tdc->lock, flags); - return ret; + goto found; } } - dev_dbg(tdc2dev(tdc), "cookie %d does not found\n", cookie); + dev_dbg(tdc2dev(tdc), "cookie %d not found\n", cookie); + dma_desc = NULL; + +found: + if (dma_desc && txstate) { + residual = dma_desc->bytes_requested - + (dma_desc->bytes_transferred % + dma_desc->bytes_requested); + dma_set_residue(txstate, residual); + } + spin_unlock_irqrestore(&tdc->lock, flags); return ret; } @@ -905,7 +902,6 @@ static int get_transfer_param(struct tegra_dma_channel *tdc, unsigned long *apb_seq, unsigned long *csr, unsigned int *burst_size, enum dma_slave_buswidth *slave_bw) { - switch (direction) { case DMA_MEM_TO_DEV: *apb_addr = tdc->dma_sconfig.dst_addr; @@ -948,8 +944,8 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma_desc *dma_desc; - unsigned int i; - struct scatterlist *sg; + unsigned int i; + struct scatterlist *sg; unsigned long csr, ahb_seq, apb_ptr, apb_seq; struct list_head req_list; struct tegra_dma_sg_req *sg_req = NULL; @@ -1062,7 +1058,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma_desc *dma_desc = NULL; - struct tegra_dma_sg_req *sg_req = NULL; + struct tegra_dma_sg_req *sg_req = NULL; unsigned long csr, ahb_seq, apb_ptr, apb_seq; int len; size_t remain_len; @@ -1204,7 +1200,6 @@ static void tegra_dma_free_chan_resources(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma *tdma = tdc->tdma; - struct tegra_dma_desc *dma_desc; struct tegra_dma_sg_req *sg_req; struct list_head dma_desc_list; @@ -1305,7 +1300,7 @@ static const struct tegra_dma_chip_data tegra148_dma_chip_data = { static int tegra_dma_probe(struct platform_device *pdev) { - struct resource *res; + struct resource *res; struct tegra_dma *tdma; int ret; int i; @@ -1319,10 +1314,8 @@ static int tegra_dma_probe(struct platform_device *pdev) tdma = devm_kzalloc(&pdev->dev, sizeof(*tdma) + cdata->nr_channels * sizeof(struct tegra_dma_channel), GFP_KERNEL); - if (!tdma) { - dev_err(&pdev->dev, "Error: memory allocation failed\n"); + if (!tdma) return -ENOMEM; - } tdma->dev = &pdev->dev; tdma->chip_data = cdata; diff --git a/drivers/dma/ti-dma-crossbar.c b/drivers/dma/ti-dma-crossbar.c index e107779b1..5ae294b25 100644 --- a/drivers/dma/ti-dma-crossbar.c +++ b/drivers/dma/ti-dma-crossbar.c @@ -452,7 +452,7 @@ static struct platform_driver ti_dma_xbar_driver = { .probe = ti_dma_xbar_probe, }; -int omap_dmaxbar_init(void) +static int omap_dmaxbar_init(void) { return platform_driver_register(&ti_dma_xbar_driver); } diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c index 559cd4073..e82745aa4 100644 --- a/drivers/dma/timb_dma.c +++ b/drivers/dma/timb_dma.c @@ -337,18 +337,14 @@ static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan) int err; td_desc = kzalloc(sizeof(struct timb_dma_desc), GFP_KERNEL); - if (!td_desc) { - dev_err(chan2dev(chan), "Failed to alloc descriptor\n"); + if (!td_desc) goto out; - } td_desc->desc_list_len = td_chan->desc_elems * TIMB_DMA_DESC_SIZE; td_desc->desc_list = kzalloc(td_desc->desc_list_len, GFP_KERNEL); - if (!td_desc->desc_list) { - dev_err(chan2dev(chan), "Failed to alloc descriptor\n"); + if (!td_desc->desc_list) goto err; - } dma_async_tx_descriptor_init(&td_desc->txd, chan); td_desc->txd.tx_submit = td_tx_submit; diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c index 8849318b3..7632290e7 100644 --- a/drivers/dma/txx9dmac.c +++ b/drivers/dma/txx9dmac.c @@ -1165,9 +1165,12 @@ static int txx9dmac_chan_remove(struct platform_device *pdev) { struct txx9dmac_chan *dc = platform_get_drvdata(pdev); + dma_async_device_unregister(&dc->dma); - if (dc->irq >= 0) + if (dc->irq >= 0) { + devm_free_irq(&pdev->dev, dc->irq, dc); tasklet_kill(&dc->tasklet); + } dc->ddev->chan[pdev->id % TXX9_DMA_MAX_NR_CHANNELS] = NULL; return 0; } @@ -1228,8 +1231,10 @@ static int txx9dmac_remove(struct platform_device *pdev) struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); txx9dmac_off(ddev); - if (ddev->irq >= 0) + if (ddev->irq >= 0) { + devm_free_irq(&pdev->dev, ddev->irq, ddev); tasklet_kill(&ddev->tasklet); + } return 0; } diff --git a/drivers/dma/xilinx/Makefile b/drivers/dma/xilinx/Makefile index 3c4e9f2fe..9e91f8f5b 100644 --- a/drivers/dma/xilinx/Makefile +++ b/drivers/dma/xilinx/Makefile @@ -1 +1,2 @@ -obj-$(CONFIG_XILINX_VDMA) += xilinx_vdma.o +obj-$(CONFIG_XILINX_DMA) += xilinx_dma.o +obj-$(CONFIG_XILINX_ZYNQMP_DMA) += zynqmp_dma.o diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c new file mode 100644 index 000000000..4e223d094 --- /dev/null +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -0,0 +1,2689 @@ +/* + * DMA driver for Xilinx Video DMA Engine + * + * Copyright (C) 2010-2014 Xilinx, Inc. All rights reserved. + * + * Based on the Freescale DMA driver. + * + * Description: + * The AXI Video Direct Memory Access (AXI VDMA) core is a soft Xilinx IP + * core that provides high-bandwidth direct memory access between memory + * and AXI4-Stream type video target peripherals. The core provides efficient + * two dimensional DMA operations with independent asynchronous read (S2MM) + * and write (MM2S) channel operation. It can be configured to have either + * one channel or two channels. If configured as two channels, one is to + * transmit to the video device (MM2S) and another is to receive from the + * video device (S2MM). Initialization, status, interrupt and management + * registers are accessed through an AXI4-Lite slave interface. + * + * The AXI Direct Memory Access (AXI DMA) core is a soft Xilinx IP core that + * provides high-bandwidth one dimensional direct memory access between memory + * and AXI4-Stream target peripherals. It supports one receive and one + * transmit channel, both of them optional at synthesis time. + * + * The AXI CDMA, is a soft IP, which provides high-bandwidth Direct Memory + * Access (DMA) between a memory-mapped source address and a memory-mapped + * destination address. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../dmaengine.h" + +/* Register/Descriptor Offsets */ +#define XILINX_DMA_MM2S_CTRL_OFFSET 0x0000 +#define XILINX_DMA_S2MM_CTRL_OFFSET 0x0030 +#define XILINX_VDMA_MM2S_DESC_OFFSET 0x0050 +#define XILINX_VDMA_S2MM_DESC_OFFSET 0x00a0 + +/* Control Registers */ +#define XILINX_DMA_REG_DMACR 0x0000 +#define XILINX_DMA_DMACR_DELAY_MAX 0xff +#define XILINX_DMA_DMACR_DELAY_SHIFT 24 +#define XILINX_DMA_DMACR_FRAME_COUNT_MAX 0xff +#define XILINX_DMA_DMACR_FRAME_COUNT_SHIFT 16 +#define XILINX_DMA_DMACR_ERR_IRQ BIT(14) +#define XILINX_DMA_DMACR_DLY_CNT_IRQ BIT(13) +#define XILINX_DMA_DMACR_FRM_CNT_IRQ BIT(12) +#define XILINX_DMA_DMACR_MASTER_SHIFT 8 +#define XILINX_DMA_DMACR_FSYNCSRC_SHIFT 5 +#define XILINX_DMA_DMACR_FRAMECNT_EN BIT(4) +#define XILINX_DMA_DMACR_GENLOCK_EN BIT(3) +#define XILINX_DMA_DMACR_RESET BIT(2) +#define XILINX_DMA_DMACR_CIRC_EN BIT(1) +#define XILINX_DMA_DMACR_RUNSTOP BIT(0) +#define XILINX_DMA_DMACR_FSYNCSRC_MASK GENMASK(6, 5) + +#define XILINX_DMA_REG_DMASR 0x0004 +#define XILINX_DMA_DMASR_EOL_LATE_ERR BIT(15) +#define XILINX_DMA_DMASR_ERR_IRQ BIT(14) +#define XILINX_DMA_DMASR_DLY_CNT_IRQ BIT(13) +#define XILINX_DMA_DMASR_FRM_CNT_IRQ BIT(12) +#define XILINX_DMA_DMASR_SOF_LATE_ERR BIT(11) +#define XILINX_DMA_DMASR_SG_DEC_ERR BIT(10) +#define XILINX_DMA_DMASR_SG_SLV_ERR BIT(9) +#define XILINX_DMA_DMASR_EOF_EARLY_ERR BIT(8) +#define XILINX_DMA_DMASR_SOF_EARLY_ERR BIT(7) +#define XILINX_DMA_DMASR_DMA_DEC_ERR BIT(6) +#define XILINX_DMA_DMASR_DMA_SLAVE_ERR BIT(5) +#define XILINX_DMA_DMASR_DMA_INT_ERR BIT(4) +#define XILINX_DMA_DMASR_IDLE BIT(1) +#define XILINX_DMA_DMASR_HALTED BIT(0) +#define XILINX_DMA_DMASR_DELAY_MASK GENMASK(31, 24) +#define XILINX_DMA_DMASR_FRAME_COUNT_MASK GENMASK(23, 16) + +#define XILINX_DMA_REG_CURDESC 0x0008 +#define XILINX_DMA_REG_TAILDESC 0x0010 +#define XILINX_DMA_REG_REG_INDEX 0x0014 +#define XILINX_DMA_REG_FRMSTORE 0x0018 +#define XILINX_DMA_REG_THRESHOLD 0x001c +#define XILINX_DMA_REG_FRMPTR_STS 0x0024 +#define XILINX_DMA_REG_PARK_PTR 0x0028 +#define XILINX_DMA_PARK_PTR_WR_REF_SHIFT 8 +#define XILINX_DMA_PARK_PTR_RD_REF_SHIFT 0 +#define XILINX_DMA_REG_VDMA_VERSION 0x002c + +/* Register Direct Mode Registers */ +#define XILINX_DMA_REG_VSIZE 0x0000 +#define XILINX_DMA_REG_HSIZE 0x0004 + +#define XILINX_DMA_REG_FRMDLY_STRIDE 0x0008 +#define XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT 24 +#define XILINX_DMA_FRMDLY_STRIDE_STRIDE_SHIFT 0 + +#define XILINX_VDMA_REG_START_ADDRESS(n) (0x000c + 4 * (n)) +#define XILINX_VDMA_REG_START_ADDRESS_64(n) (0x000c + 8 * (n)) + +/* HW specific definitions */ +#define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x20 + +#define XILINX_DMA_DMAXR_ALL_IRQ_MASK \ + (XILINX_DMA_DMASR_FRM_CNT_IRQ | \ + XILINX_DMA_DMASR_DLY_CNT_IRQ | \ + XILINX_DMA_DMASR_ERR_IRQ) + +#define XILINX_DMA_DMASR_ALL_ERR_MASK \ + (XILINX_DMA_DMASR_EOL_LATE_ERR | \ + XILINX_DMA_DMASR_SOF_LATE_ERR | \ + XILINX_DMA_DMASR_SG_DEC_ERR | \ + XILINX_DMA_DMASR_SG_SLV_ERR | \ + XILINX_DMA_DMASR_EOF_EARLY_ERR | \ + XILINX_DMA_DMASR_SOF_EARLY_ERR | \ + XILINX_DMA_DMASR_DMA_DEC_ERR | \ + XILINX_DMA_DMASR_DMA_SLAVE_ERR | \ + XILINX_DMA_DMASR_DMA_INT_ERR) + +/* + * Recoverable errors are DMA Internal error, SOF Early, EOF Early + * and SOF Late. They are only recoverable when C_FLUSH_ON_FSYNC + * is enabled in the h/w system. + */ +#define XILINX_DMA_DMASR_ERR_RECOVER_MASK \ + (XILINX_DMA_DMASR_SOF_LATE_ERR | \ + XILINX_DMA_DMASR_EOF_EARLY_ERR | \ + XILINX_DMA_DMASR_SOF_EARLY_ERR | \ + XILINX_DMA_DMASR_DMA_INT_ERR) + +/* Axi VDMA Flush on Fsync bits */ +#define XILINX_DMA_FLUSH_S2MM 3 +#define XILINX_DMA_FLUSH_MM2S 2 +#define XILINX_DMA_FLUSH_BOTH 1 + +/* Delay loop counter to prevent hardware failure */ +#define XILINX_DMA_LOOP_COUNT 1000000 + +/* AXI DMA Specific Registers/Offsets */ +#define XILINX_DMA_REG_SRCDSTADDR 0x18 +#define XILINX_DMA_REG_BTT 0x28 + +/* AXI DMA Specific Masks/Bit fields */ +#define XILINX_DMA_MAX_TRANS_LEN GENMASK(22, 0) +#define XILINX_DMA_CR_COALESCE_MAX GENMASK(23, 16) +#define XILINX_DMA_CR_CYCLIC_BD_EN_MASK BIT(4) +#define XILINX_DMA_CR_COALESCE_SHIFT 16 +#define XILINX_DMA_BD_SOP BIT(27) +#define XILINX_DMA_BD_EOP BIT(26) +#define XILINX_DMA_COALESCE_MAX 255 +#define XILINX_DMA_NUM_APP_WORDS 5 + +/* Multi-Channel DMA Descriptor offsets*/ +#define XILINX_DMA_MCRX_CDESC(x) (0x40 + (x-1) * 0x20) +#define XILINX_DMA_MCRX_TDESC(x) (0x48 + (x-1) * 0x20) + +/* Multi-Channel DMA Masks/Shifts */ +#define XILINX_DMA_BD_HSIZE_MASK GENMASK(15, 0) +#define XILINX_DMA_BD_STRIDE_MASK GENMASK(15, 0) +#define XILINX_DMA_BD_VSIZE_MASK GENMASK(31, 19) +#define XILINX_DMA_BD_TDEST_MASK GENMASK(4, 0) +#define XILINX_DMA_BD_STRIDE_SHIFT 0 +#define XILINX_DMA_BD_VSIZE_SHIFT 19 + +/* AXI CDMA Specific Registers/Offsets */ +#define XILINX_CDMA_REG_SRCADDR 0x18 +#define XILINX_CDMA_REG_DSTADDR 0x20 + +/* AXI CDMA Specific Masks */ +#define XILINX_CDMA_CR_SGMODE BIT(3) + +/** + * struct xilinx_vdma_desc_hw - Hardware Descriptor + * @next_desc: Next Descriptor Pointer @0x00 + * @pad1: Reserved @0x04 + * @buf_addr: Buffer address @0x08 + * @buf_addr_msb: MSB of Buffer address @0x0C + * @vsize: Vertical Size @0x10 + * @hsize: Horizontal Size @0x14 + * @stride: Number of bytes between the first + * pixels of each horizontal line @0x18 + */ +struct xilinx_vdma_desc_hw { + u32 next_desc; + u32 pad1; + u32 buf_addr; + u32 buf_addr_msb; + u32 vsize; + u32 hsize; + u32 stride; +} __aligned(64); + +/** + * struct xilinx_axidma_desc_hw - Hardware Descriptor for AXI DMA + * @next_desc: Next Descriptor Pointer @0x00 + * @next_desc_msb: MSB of Next Descriptor Pointer @0x04 + * @buf_addr: Buffer address @0x08 + * @buf_addr_msb: MSB of Buffer address @0x0C + * @pad1: Reserved @0x10 + * @pad2: Reserved @0x14 + * @control: Control field @0x18 + * @status: Status field @0x1C + * @app: APP Fields @0x20 - 0x30 + */ +struct xilinx_axidma_desc_hw { + u32 next_desc; + u32 next_desc_msb; + u32 buf_addr; + u32 buf_addr_msb; + u32 mcdma_control; + u32 vsize_stride; + u32 control; + u32 status; + u32 app[XILINX_DMA_NUM_APP_WORDS]; +} __aligned(64); + +/** + * struct xilinx_cdma_desc_hw - Hardware Descriptor + * @next_desc: Next Descriptor Pointer @0x00 + * @next_descmsb: Next Descriptor Pointer MSB @0x04 + * @src_addr: Source address @0x08 + * @src_addrmsb: Source address MSB @0x0C + * @dest_addr: Destination address @0x10 + * @dest_addrmsb: Destination address MSB @0x14 + * @control: Control field @0x18 + * @status: Status field @0x1C + */ +struct xilinx_cdma_desc_hw { + u32 next_desc; + u32 next_desc_msb; + u32 src_addr; + u32 src_addr_msb; + u32 dest_addr; + u32 dest_addr_msb; + u32 control; + u32 status; +} __aligned(64); + +/** + * struct xilinx_vdma_tx_segment - Descriptor segment + * @hw: Hardware descriptor + * @node: Node in the descriptor segments list + * @phys: Physical address of segment + */ +struct xilinx_vdma_tx_segment { + struct xilinx_vdma_desc_hw hw; + struct list_head node; + dma_addr_t phys; +} __aligned(64); + +/** + * struct xilinx_axidma_tx_segment - Descriptor segment + * @hw: Hardware descriptor + * @node: Node in the descriptor segments list + * @phys: Physical address of segment + */ +struct xilinx_axidma_tx_segment { + struct xilinx_axidma_desc_hw hw; + struct list_head node; + dma_addr_t phys; +} __aligned(64); + +/** + * struct xilinx_cdma_tx_segment - Descriptor segment + * @hw: Hardware descriptor + * @node: Node in the descriptor segments list + * @phys: Physical address of segment + */ +struct xilinx_cdma_tx_segment { + struct xilinx_cdma_desc_hw hw; + struct list_head node; + dma_addr_t phys; +} __aligned(64); + +/** + * struct xilinx_dma_tx_descriptor - Per Transaction structure + * @async_tx: Async transaction descriptor + * @segments: TX segments list + * @node: Node in the channel descriptors list + * @cyclic: Check for cyclic transfers. + */ +struct xilinx_dma_tx_descriptor { + struct dma_async_tx_descriptor async_tx; + struct list_head segments; + struct list_head node; + bool cyclic; +}; + +/** + * struct xilinx_dma_chan - Driver specific DMA channel structure + * @xdev: Driver specific device structure + * @ctrl_offset: Control registers offset + * @desc_offset: TX descriptor registers offset + * @lock: Descriptor operation lock + * @pending_list: Descriptors waiting + * @active_list: Descriptors ready to submit + * @done_list: Complete descriptors + * @common: DMA common channel + * @desc_pool: Descriptors pool + * @dev: The dma device + * @irq: Channel IRQ + * @id: Channel ID + * @direction: Transfer direction + * @num_frms: Number of frames + * @has_sg: Support scatter transfers + * @cyclic: Check for cyclic transfers. + * @genlock: Support genlock mode + * @err: Channel has errors + * @tasklet: Cleanup work after irq + * @config: Device configuration info + * @flush_on_fsync: Flush on Frame sync + * @desc_pendingcount: Descriptor pending count + * @ext_addr: Indicates 64 bit addressing is supported by dma channel + * @desc_submitcount: Descriptor h/w submitted count + * @residue: Residue for AXI DMA + * @seg_v: Statically allocated segments base + * @cyclic_seg_v: Statically allocated segment base for cyclic transfers + * @start_transfer: Differentiate b/w DMA IP's transfer + */ +struct xilinx_dma_chan { + struct xilinx_dma_device *xdev; + u32 ctrl_offset; + u32 desc_offset; + spinlock_t lock; + struct list_head pending_list; + struct list_head active_list; + struct list_head done_list; + struct dma_chan common; + struct dma_pool *desc_pool; + struct device *dev; + int irq; + int id; + enum dma_transfer_direction direction; + int num_frms; + bool has_sg; + bool cyclic; + bool genlock; + bool err; + struct tasklet_struct tasklet; + struct xilinx_vdma_config config; + bool flush_on_fsync; + u32 desc_pendingcount; + bool ext_addr; + u32 desc_submitcount; + u32 residue; + struct xilinx_axidma_tx_segment *seg_v; + struct xilinx_axidma_tx_segment *cyclic_seg_v; + void (*start_transfer)(struct xilinx_dma_chan *chan); + u16 tdest; +}; + +struct xilinx_dma_config { + enum xdma_ip_type dmatype; + int (*clk_init)(struct platform_device *pdev, struct clk **axi_clk, + struct clk **tx_clk, struct clk **txs_clk, + struct clk **rx_clk, struct clk **rxs_clk); +}; + +/** + * struct xilinx_dma_device - DMA device structure + * @regs: I/O mapped base address + * @dev: Device Structure + * @common: DMA device structure + * @chan: Driver specific DMA channel + * @has_sg: Specifies whether Scatter-Gather is present or not + * @mcdma: Specifies whether Multi-Channel is present or not + * @flush_on_fsync: Flush on frame sync + * @ext_addr: Indicates 64 bit addressing is supported by dma device + * @pdev: Platform device structure pointer + * @dma_config: DMA config structure + * @axi_clk: DMA Axi4-lite interace clock + * @tx_clk: DMA mm2s clock + * @txs_clk: DMA mm2s stream clock + * @rx_clk: DMA s2mm clock + * @rxs_clk: DMA s2mm stream clock + * @nr_channels: Number of channels DMA device supports + * @chan_id: DMA channel identifier + */ +struct xilinx_dma_device { + void __iomem *regs; + struct device *dev; + struct dma_device common; + struct xilinx_dma_chan *chan[XILINX_DMA_MAX_CHANS_PER_DEVICE]; + bool has_sg; + bool mcdma; + u32 flush_on_fsync; + bool ext_addr; + struct platform_device *pdev; + const struct xilinx_dma_config *dma_config; + struct clk *axi_clk; + struct clk *tx_clk; + struct clk *txs_clk; + struct clk *rx_clk; + struct clk *rxs_clk; + u32 nr_channels; + u32 chan_id; +}; + +/* Macros */ +#define to_xilinx_chan(chan) \ + container_of(chan, struct xilinx_dma_chan, common) +#define to_dma_tx_descriptor(tx) \ + container_of(tx, struct xilinx_dma_tx_descriptor, async_tx) +#define xilinx_dma_poll_timeout(chan, reg, val, cond, delay_us, timeout_us) \ + readl_poll_timeout(chan->xdev->regs + chan->ctrl_offset + reg, val, \ + cond, delay_us, timeout_us) + +/* IO accessors */ +static inline u32 dma_read(struct xilinx_dma_chan *chan, u32 reg) +{ + return ioread32(chan->xdev->regs + reg); +} + +static inline void dma_write(struct xilinx_dma_chan *chan, u32 reg, u32 value) +{ + iowrite32(value, chan->xdev->regs + reg); +} + +static inline void vdma_desc_write(struct xilinx_dma_chan *chan, u32 reg, + u32 value) +{ + dma_write(chan, chan->desc_offset + reg, value); +} + +static inline u32 dma_ctrl_read(struct xilinx_dma_chan *chan, u32 reg) +{ + return dma_read(chan, chan->ctrl_offset + reg); +} + +static inline void dma_ctrl_write(struct xilinx_dma_chan *chan, u32 reg, + u32 value) +{ + dma_write(chan, chan->ctrl_offset + reg, value); +} + +static inline void dma_ctrl_clr(struct xilinx_dma_chan *chan, u32 reg, + u32 clr) +{ + dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) & ~clr); +} + +static inline void dma_ctrl_set(struct xilinx_dma_chan *chan, u32 reg, + u32 set) +{ + dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) | set); +} + +/** + * vdma_desc_write_64 - 64-bit descriptor write + * @chan: Driver specific VDMA channel + * @reg: Register to write + * @value_lsb: lower address of the descriptor. + * @value_msb: upper address of the descriptor. + * + * Since vdma driver is trying to write to a register offset which is not a + * multiple of 64 bits(ex : 0x5c), we are writing as two separate 32 bits + * instead of a single 64 bit register write. + */ +static inline void vdma_desc_write_64(struct xilinx_dma_chan *chan, u32 reg, + u32 value_lsb, u32 value_msb) +{ + /* Write the lsb 32 bits*/ + writel(value_lsb, chan->xdev->regs + chan->desc_offset + reg); + + /* Write the msb 32 bits */ + writel(value_msb, chan->xdev->regs + chan->desc_offset + reg + 4); +} + +static inline void dma_writeq(struct xilinx_dma_chan *chan, u32 reg, u64 value) +{ + lo_hi_writeq(value, chan->xdev->regs + chan->ctrl_offset + reg); +} + +static inline void xilinx_write(struct xilinx_dma_chan *chan, u32 reg, + dma_addr_t addr) +{ + if (chan->ext_addr) + dma_writeq(chan, reg, addr); + else + dma_ctrl_write(chan, reg, addr); +} + +static inline void xilinx_axidma_buf(struct xilinx_dma_chan *chan, + struct xilinx_axidma_desc_hw *hw, + dma_addr_t buf_addr, size_t sg_used, + size_t period_len) +{ + if (chan->ext_addr) { + hw->buf_addr = lower_32_bits(buf_addr + sg_used + period_len); + hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used + + period_len); + } else { + hw->buf_addr = buf_addr + sg_used + period_len; + } +} + +/* ----------------------------------------------------------------------------- + * Descriptors and segments alloc and free + */ + +/** + * xilinx_vdma_alloc_tx_segment - Allocate transaction segment + * @chan: Driver specific DMA channel + * + * Return: The allocated segment on success and NULL on failure. + */ +static struct xilinx_vdma_tx_segment * +xilinx_vdma_alloc_tx_segment(struct xilinx_dma_chan *chan) +{ + struct xilinx_vdma_tx_segment *segment; + dma_addr_t phys; + + segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys); + if (!segment) + return NULL; + + segment->phys = phys; + + return segment; +} + +/** + * xilinx_cdma_alloc_tx_segment - Allocate transaction segment + * @chan: Driver specific DMA channel + * + * Return: The allocated segment on success and NULL on failure. + */ +static struct xilinx_cdma_tx_segment * +xilinx_cdma_alloc_tx_segment(struct xilinx_dma_chan *chan) +{ + struct xilinx_cdma_tx_segment *segment; + dma_addr_t phys; + + segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys); + if (!segment) + return NULL; + + segment->phys = phys; + + return segment; +} + +/** + * xilinx_axidma_alloc_tx_segment - Allocate transaction segment + * @chan: Driver specific DMA channel + * + * Return: The allocated segment on success and NULL on failure. + */ +static struct xilinx_axidma_tx_segment * +xilinx_axidma_alloc_tx_segment(struct xilinx_dma_chan *chan) +{ + struct xilinx_axidma_tx_segment *segment; + dma_addr_t phys; + + segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys); + if (!segment) + return NULL; + + segment->phys = phys; + + return segment; +} + +/** + * xilinx_dma_free_tx_segment - Free transaction segment + * @chan: Driver specific DMA channel + * @segment: DMA transaction segment + */ +static void xilinx_dma_free_tx_segment(struct xilinx_dma_chan *chan, + struct xilinx_axidma_tx_segment *segment) +{ + dma_pool_free(chan->desc_pool, segment, segment->phys); +} + +/** + * xilinx_cdma_free_tx_segment - Free transaction segment + * @chan: Driver specific DMA channel + * @segment: DMA transaction segment + */ +static void xilinx_cdma_free_tx_segment(struct xilinx_dma_chan *chan, + struct xilinx_cdma_tx_segment *segment) +{ + dma_pool_free(chan->desc_pool, segment, segment->phys); +} + +/** + * xilinx_vdma_free_tx_segment - Free transaction segment + * @chan: Driver specific DMA channel + * @segment: DMA transaction segment + */ +static void xilinx_vdma_free_tx_segment(struct xilinx_dma_chan *chan, + struct xilinx_vdma_tx_segment *segment) +{ + dma_pool_free(chan->desc_pool, segment, segment->phys); +} + +/** + * xilinx_dma_tx_descriptor - Allocate transaction descriptor + * @chan: Driver specific DMA channel + * + * Return: The allocated descriptor on success and NULL on failure. + */ +static struct xilinx_dma_tx_descriptor * +xilinx_dma_alloc_tx_descriptor(struct xilinx_dma_chan *chan) +{ + struct xilinx_dma_tx_descriptor *desc; + + desc = kzalloc(sizeof(*desc), GFP_KERNEL); + if (!desc) + return NULL; + + INIT_LIST_HEAD(&desc->segments); + + return desc; +} + +/** + * xilinx_dma_free_tx_descriptor - Free transaction descriptor + * @chan: Driver specific DMA channel + * @desc: DMA transaction descriptor + */ +static void +xilinx_dma_free_tx_descriptor(struct xilinx_dma_chan *chan, + struct xilinx_dma_tx_descriptor *desc) +{ + struct xilinx_vdma_tx_segment *segment, *next; + struct xilinx_cdma_tx_segment *cdma_segment, *cdma_next; + struct xilinx_axidma_tx_segment *axidma_segment, *axidma_next; + + if (!desc) + return; + + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { + list_for_each_entry_safe(segment, next, &desc->segments, node) { + list_del(&segment->node); + xilinx_vdma_free_tx_segment(chan, segment); + } + } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { + list_for_each_entry_safe(cdma_segment, cdma_next, + &desc->segments, node) { + list_del(&cdma_segment->node); + xilinx_cdma_free_tx_segment(chan, cdma_segment); + } + } else { + list_for_each_entry_safe(axidma_segment, axidma_next, + &desc->segments, node) { + list_del(&axidma_segment->node); + xilinx_dma_free_tx_segment(chan, axidma_segment); + } + } + + kfree(desc); +} + +/* Required functions */ + +/** + * xilinx_dma_free_desc_list - Free descriptors list + * @chan: Driver specific DMA channel + * @list: List to parse and delete the descriptor + */ +static void xilinx_dma_free_desc_list(struct xilinx_dma_chan *chan, + struct list_head *list) +{ + struct xilinx_dma_tx_descriptor *desc, *next; + + list_for_each_entry_safe(desc, next, list, node) { + list_del(&desc->node); + xilinx_dma_free_tx_descriptor(chan, desc); + } +} + +/** + * xilinx_dma_free_descriptors - Free channel descriptors + * @chan: Driver specific DMA channel + */ +static void xilinx_dma_free_descriptors(struct xilinx_dma_chan *chan) +{ + unsigned long flags; + + spin_lock_irqsave(&chan->lock, flags); + + xilinx_dma_free_desc_list(chan, &chan->pending_list); + xilinx_dma_free_desc_list(chan, &chan->done_list); + xilinx_dma_free_desc_list(chan, &chan->active_list); + + spin_unlock_irqrestore(&chan->lock, flags); +} + +/** + * xilinx_dma_free_chan_resources - Free channel resources + * @dchan: DMA channel + */ +static void xilinx_dma_free_chan_resources(struct dma_chan *dchan) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + + dev_dbg(chan->dev, "Free all channel resources.\n"); + + xilinx_dma_free_descriptors(chan); + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + xilinx_dma_free_tx_segment(chan, chan->cyclic_seg_v); + xilinx_dma_free_tx_segment(chan, chan->seg_v); + } + dma_pool_destroy(chan->desc_pool); + chan->desc_pool = NULL; +} + +/** + * xilinx_dma_chan_handle_cyclic - Cyclic dma callback + * @chan: Driver specific dma channel + * @desc: dma transaction descriptor + * @flags: flags for spin lock + */ +static void xilinx_dma_chan_handle_cyclic(struct xilinx_dma_chan *chan, + struct xilinx_dma_tx_descriptor *desc, + unsigned long *flags) +{ + dma_async_tx_callback callback; + void *callback_param; + + callback = desc->async_tx.callback; + callback_param = desc->async_tx.callback_param; + if (callback) { + spin_unlock_irqrestore(&chan->lock, *flags); + callback(callback_param); + spin_lock_irqsave(&chan->lock, *flags); + } +} + +/** + * xilinx_dma_chan_desc_cleanup - Clean channel descriptors + * @chan: Driver specific DMA channel + */ +static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan) +{ + struct xilinx_dma_tx_descriptor *desc, *next; + unsigned long flags; + + spin_lock_irqsave(&chan->lock, flags); + + list_for_each_entry_safe(desc, next, &chan->done_list, node) { + dma_async_tx_callback callback; + void *callback_param; + + if (desc->cyclic) { + xilinx_dma_chan_handle_cyclic(chan, desc, &flags); + break; + } + + /* Remove from the list of running transactions */ + list_del(&desc->node); + + /* Run the link descriptor callback function */ + callback = desc->async_tx.callback; + callback_param = desc->async_tx.callback_param; + if (callback) { + spin_unlock_irqrestore(&chan->lock, flags); + callback(callback_param); + spin_lock_irqsave(&chan->lock, flags); + } + + /* Run any dependencies, then free the descriptor */ + dma_run_dependencies(&desc->async_tx); + xilinx_dma_free_tx_descriptor(chan, desc); + } + + spin_unlock_irqrestore(&chan->lock, flags); +} + +/** + * xilinx_dma_do_tasklet - Schedule completion tasklet + * @data: Pointer to the Xilinx DMA channel structure + */ +static void xilinx_dma_do_tasklet(unsigned long data) +{ + struct xilinx_dma_chan *chan = (struct xilinx_dma_chan *)data; + + xilinx_dma_chan_desc_cleanup(chan); +} + +/** + * xilinx_dma_alloc_chan_resources - Allocate channel resources + * @dchan: DMA channel + * + * Return: '0' on success and failure value on error + */ +static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + + /* Has this channel already been allocated? */ + if (chan->desc_pool) + return 0; + + /* + * We need the descriptor to be aligned to 64bytes + * for meeting Xilinx VDMA specification requirement. + */ + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + chan->desc_pool = dma_pool_create("xilinx_dma_desc_pool", + chan->dev, + sizeof(struct xilinx_axidma_tx_segment), + __alignof__(struct xilinx_axidma_tx_segment), + 0); + } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { + chan->desc_pool = dma_pool_create("xilinx_cdma_desc_pool", + chan->dev, + sizeof(struct xilinx_cdma_tx_segment), + __alignof__(struct xilinx_cdma_tx_segment), + 0); + } else { + chan->desc_pool = dma_pool_create("xilinx_vdma_desc_pool", + chan->dev, + sizeof(struct xilinx_vdma_tx_segment), + __alignof__(struct xilinx_vdma_tx_segment), + 0); + } + + if (!chan->desc_pool) { + dev_err(chan->dev, + "unable to allocate channel %d descriptor pool\n", + chan->id); + return -ENOMEM; + } + + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + /* + * For AXI DMA case after submitting a pending_list, keep + * an extra segment allocated so that the "next descriptor" + * pointer on the tail descriptor always points to a + * valid descriptor, even when paused after reaching taildesc. + * This way, it is possible to issue additional + * transfers without halting and restarting the channel. + */ + chan->seg_v = xilinx_axidma_alloc_tx_segment(chan); + + /* + * For cyclic DMA mode we need to program the tail Descriptor + * register with a value which is not a part of the BD chain + * so allocating a desc segment during channel allocation for + * programming tail descriptor. + */ + chan->cyclic_seg_v = xilinx_axidma_alloc_tx_segment(chan); + } + + dma_cookie_init(dchan); + + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + /* For AXI DMA resetting once channel will reset the + * other channel as well so enable the interrupts here. + */ + dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, + XILINX_DMA_DMAXR_ALL_IRQ_MASK); + } + + if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg) + dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, + XILINX_CDMA_CR_SGMODE); + + return 0; +} + +/** + * xilinx_dma_tx_status - Get DMA transaction status + * @dchan: DMA channel + * @cookie: Transaction identifier + * @txstate: Transaction state + * + * Return: DMA transaction status + */ +static enum dma_status xilinx_dma_tx_status(struct dma_chan *dchan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dma_tx_descriptor *desc; + struct xilinx_axidma_tx_segment *segment; + struct xilinx_axidma_desc_hw *hw; + enum dma_status ret; + unsigned long flags; + u32 residue = 0; + + ret = dma_cookie_status(dchan, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + spin_lock_irqsave(&chan->lock, flags); + + desc = list_last_entry(&chan->active_list, + struct xilinx_dma_tx_descriptor, node); + if (chan->has_sg) { + list_for_each_entry(segment, &desc->segments, node) { + hw = &segment->hw; + residue += (hw->control - hw->status) & + XILINX_DMA_MAX_TRANS_LEN; + } + } + spin_unlock_irqrestore(&chan->lock, flags); + + chan->residue = residue; + dma_set_residue(txstate, chan->residue); + } + + return ret; +} + +/** + * xilinx_dma_is_running - Check if DMA channel is running + * @chan: Driver specific DMA channel + * + * Return: '1' if running, '0' if not. + */ +static bool xilinx_dma_is_running(struct xilinx_dma_chan *chan) +{ + return !(dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) & + XILINX_DMA_DMASR_HALTED) && + (dma_ctrl_read(chan, XILINX_DMA_REG_DMACR) & + XILINX_DMA_DMACR_RUNSTOP); +} + +/** + * xilinx_dma_is_idle - Check if DMA channel is idle + * @chan: Driver specific DMA channel + * + * Return: '1' if idle, '0' if not. + */ +static bool xilinx_dma_is_idle(struct xilinx_dma_chan *chan) +{ + return dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) & + XILINX_DMA_DMASR_IDLE; +} + +/** + * xilinx_dma_halt - Halt DMA channel + * @chan: Driver specific DMA channel + */ +static void xilinx_dma_halt(struct xilinx_dma_chan *chan) +{ + int err; + u32 val; + + dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP); + + /* Wait for the hardware to halt */ + err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, + (val & XILINX_DMA_DMASR_HALTED), 0, + XILINX_DMA_LOOP_COUNT); + + if (err) { + dev_err(chan->dev, "Cannot stop channel %p: %x\n", + chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); + chan->err = true; + } +} + +/** + * xilinx_dma_start - Start DMA channel + * @chan: Driver specific DMA channel + */ +static void xilinx_dma_start(struct xilinx_dma_chan *chan) +{ + int err; + u32 val; + + dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP); + + /* Wait for the hardware to start */ + err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, + !(val & XILINX_DMA_DMASR_HALTED), 0, + XILINX_DMA_LOOP_COUNT); + + if (err) { + dev_err(chan->dev, "Cannot start channel %p: %x\n", + chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); + + chan->err = true; + } +} + +/** + * xilinx_vdma_start_transfer - Starts VDMA transfer + * @chan: Driver specific channel struct pointer + */ +static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) +{ + struct xilinx_vdma_config *config = &chan->config; + struct xilinx_dma_tx_descriptor *desc, *tail_desc; + u32 reg; + struct xilinx_vdma_tx_segment *tail_segment; + + /* This function was invoked with lock held */ + if (chan->err) + return; + + if (list_empty(&chan->pending_list)) + return; + + desc = list_first_entry(&chan->pending_list, + struct xilinx_dma_tx_descriptor, node); + tail_desc = list_last_entry(&chan->pending_list, + struct xilinx_dma_tx_descriptor, node); + + tail_segment = list_last_entry(&tail_desc->segments, + struct xilinx_vdma_tx_segment, node); + + /* If it is SG mode and hardware is busy, cannot submit */ + if (chan->has_sg && xilinx_dma_is_running(chan) && + !xilinx_dma_is_idle(chan)) { + dev_dbg(chan->dev, "DMA controller still busy\n"); + return; + } + + /* + * If hardware is idle, then all descriptors on the running lists are + * done, start new transfers + */ + if (chan->has_sg) + dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC, + desc->async_tx.phys); + + /* Configure the hardware using info in the config structure */ + reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); + + if (config->frm_cnt_en) + reg |= XILINX_DMA_DMACR_FRAMECNT_EN; + else + reg &= ~XILINX_DMA_DMACR_FRAMECNT_EN; + + /* Configure channel to allow number frame buffers */ + dma_ctrl_write(chan, XILINX_DMA_REG_FRMSTORE, + chan->desc_pendingcount); + + /* + * With SG, start with circular mode, so that BDs can be fetched. + * In direct register mode, if not parking, enable circular mode + */ + if (chan->has_sg || !config->park) + reg |= XILINX_DMA_DMACR_CIRC_EN; + + if (config->park) + reg &= ~XILINX_DMA_DMACR_CIRC_EN; + + dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); + + if (config->park && (config->park_frm >= 0) && + (config->park_frm < chan->num_frms)) { + if (chan->direction == DMA_MEM_TO_DEV) + dma_write(chan, XILINX_DMA_REG_PARK_PTR, + config->park_frm << + XILINX_DMA_PARK_PTR_RD_REF_SHIFT); + else + dma_write(chan, XILINX_DMA_REG_PARK_PTR, + config->park_frm << + XILINX_DMA_PARK_PTR_WR_REF_SHIFT); + } + + /* Start the hardware */ + xilinx_dma_start(chan); + + if (chan->err) + return; + + /* Start the transfer */ + if (chan->has_sg) { + dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC, + tail_segment->phys); + } else { + struct xilinx_vdma_tx_segment *segment, *last = NULL; + int i = 0; + + if (chan->desc_submitcount < chan->num_frms) + i = chan->desc_submitcount; + + list_for_each_entry(segment, &desc->segments, node) { + if (chan->ext_addr) + vdma_desc_write_64(chan, + XILINX_VDMA_REG_START_ADDRESS_64(i++), + segment->hw.buf_addr, + segment->hw.buf_addr_msb); + else + vdma_desc_write(chan, + XILINX_VDMA_REG_START_ADDRESS(i++), + segment->hw.buf_addr); + + last = segment; + } + + if (!last) + return; + + /* HW expects these parameters to be same for one transaction */ + vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize); + vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE, + last->hw.stride); + vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize); + } + + if (!chan->has_sg) { + list_del(&desc->node); + list_add_tail(&desc->node, &chan->active_list); + chan->desc_submitcount++; + chan->desc_pendingcount--; + if (chan->desc_submitcount == chan->num_frms) + chan->desc_submitcount = 0; + } else { + list_splice_tail_init(&chan->pending_list, &chan->active_list); + chan->desc_pendingcount = 0; + } +} + +/** + * xilinx_cdma_start_transfer - Starts cdma transfer + * @chan: Driver specific channel struct pointer + */ +static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan) +{ + struct xilinx_dma_tx_descriptor *head_desc, *tail_desc; + struct xilinx_cdma_tx_segment *tail_segment; + u32 ctrl_reg = dma_read(chan, XILINX_DMA_REG_DMACR); + + if (chan->err) + return; + + if (list_empty(&chan->pending_list)) + return; + + head_desc = list_first_entry(&chan->pending_list, + struct xilinx_dma_tx_descriptor, node); + tail_desc = list_last_entry(&chan->pending_list, + struct xilinx_dma_tx_descriptor, node); + tail_segment = list_last_entry(&tail_desc->segments, + struct xilinx_cdma_tx_segment, node); + + if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) { + ctrl_reg &= ~XILINX_DMA_CR_COALESCE_MAX; + ctrl_reg |= chan->desc_pendingcount << + XILINX_DMA_CR_COALESCE_SHIFT; + dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, ctrl_reg); + } + + if (chan->has_sg) { + xilinx_write(chan, XILINX_DMA_REG_CURDESC, + head_desc->async_tx.phys); + + /* Update tail ptr register which will start the transfer */ + xilinx_write(chan, XILINX_DMA_REG_TAILDESC, + tail_segment->phys); + } else { + /* In simple mode */ + struct xilinx_cdma_tx_segment *segment; + struct xilinx_cdma_desc_hw *hw; + + segment = list_first_entry(&head_desc->segments, + struct xilinx_cdma_tx_segment, + node); + + hw = &segment->hw; + + xilinx_write(chan, XILINX_CDMA_REG_SRCADDR, hw->src_addr); + xilinx_write(chan, XILINX_CDMA_REG_DSTADDR, hw->dest_addr); + + /* Start the transfer */ + dma_ctrl_write(chan, XILINX_DMA_REG_BTT, + hw->control & XILINX_DMA_MAX_TRANS_LEN); + } + + list_splice_tail_init(&chan->pending_list, &chan->active_list); + chan->desc_pendingcount = 0; +} + +/** + * xilinx_dma_start_transfer - Starts DMA transfer + * @chan: Driver specific channel struct pointer + */ +static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) +{ + struct xilinx_dma_tx_descriptor *head_desc, *tail_desc; + struct xilinx_axidma_tx_segment *tail_segment, *old_head, *new_head; + u32 reg; + + if (chan->err) + return; + + if (list_empty(&chan->pending_list)) + return; + + /* If it is SG mode and hardware is busy, cannot submit */ + if (chan->has_sg && xilinx_dma_is_running(chan) && + !xilinx_dma_is_idle(chan)) { + dev_dbg(chan->dev, "DMA controller still busy\n"); + return; + } + + head_desc = list_first_entry(&chan->pending_list, + struct xilinx_dma_tx_descriptor, node); + tail_desc = list_last_entry(&chan->pending_list, + struct xilinx_dma_tx_descriptor, node); + tail_segment = list_last_entry(&tail_desc->segments, + struct xilinx_axidma_tx_segment, node); + + if (chan->has_sg && !chan->xdev->mcdma) { + old_head = list_first_entry(&head_desc->segments, + struct xilinx_axidma_tx_segment, node); + new_head = chan->seg_v; + /* Copy Buffer Descriptor fields. */ + new_head->hw = old_head->hw; + + /* Swap and save new reserve */ + list_replace_init(&old_head->node, &new_head->node); + chan->seg_v = old_head; + + tail_segment->hw.next_desc = chan->seg_v->phys; + head_desc->async_tx.phys = new_head->phys; + } + + reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); + + if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) { + reg &= ~XILINX_DMA_CR_COALESCE_MAX; + reg |= chan->desc_pendingcount << + XILINX_DMA_CR_COALESCE_SHIFT; + dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); + } + + if (chan->has_sg && !chan->xdev->mcdma) + xilinx_write(chan, XILINX_DMA_REG_CURDESC, + head_desc->async_tx.phys); + + if (chan->has_sg && chan->xdev->mcdma) { + if (chan->direction == DMA_MEM_TO_DEV) { + dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC, + head_desc->async_tx.phys); + } else { + if (!chan->tdest) { + dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC, + head_desc->async_tx.phys); + } else { + dma_ctrl_write(chan, + XILINX_DMA_MCRX_CDESC(chan->tdest), + head_desc->async_tx.phys); + } + } + } + + xilinx_dma_start(chan); + + if (chan->err) + return; + + /* Start the transfer */ + if (chan->has_sg && !chan->xdev->mcdma) { + if (chan->cyclic) + xilinx_write(chan, XILINX_DMA_REG_TAILDESC, + chan->cyclic_seg_v->phys); + else + xilinx_write(chan, XILINX_DMA_REG_TAILDESC, + tail_segment->phys); + } else if (chan->has_sg && chan->xdev->mcdma) { + if (chan->direction == DMA_MEM_TO_DEV) { + dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC, + tail_segment->phys); + } else { + if (!chan->tdest) { + dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC, + tail_segment->phys); + } else { + dma_ctrl_write(chan, + XILINX_DMA_MCRX_TDESC(chan->tdest), + tail_segment->phys); + } + } + } else { + struct xilinx_axidma_tx_segment *segment; + struct xilinx_axidma_desc_hw *hw; + + segment = list_first_entry(&head_desc->segments, + struct xilinx_axidma_tx_segment, + node); + hw = &segment->hw; + + xilinx_write(chan, XILINX_DMA_REG_SRCDSTADDR, hw->buf_addr); + + /* Start the transfer */ + dma_ctrl_write(chan, XILINX_DMA_REG_BTT, + hw->control & XILINX_DMA_MAX_TRANS_LEN); + } + + list_splice_tail_init(&chan->pending_list, &chan->active_list); + chan->desc_pendingcount = 0; +} + +/** + * xilinx_dma_issue_pending - Issue pending transactions + * @dchan: DMA channel + */ +static void xilinx_dma_issue_pending(struct dma_chan *dchan) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + unsigned long flags; + + spin_lock_irqsave(&chan->lock, flags); + chan->start_transfer(chan); + spin_unlock_irqrestore(&chan->lock, flags); +} + +/** + * xilinx_dma_complete_descriptor - Mark the active descriptor as complete + * @chan : xilinx DMA channel + * + * CONTEXT: hardirq + */ +static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan) +{ + struct xilinx_dma_tx_descriptor *desc, *next; + + /* This function was invoked with lock held */ + if (list_empty(&chan->active_list)) + return; + + list_for_each_entry_safe(desc, next, &chan->active_list, node) { + list_del(&desc->node); + if (!desc->cyclic) + dma_cookie_complete(&desc->async_tx); + list_add_tail(&desc->node, &chan->done_list); + } +} + +/** + * xilinx_dma_reset - Reset DMA channel + * @chan: Driver specific DMA channel + * + * Return: '0' on success and failure value on error + */ +static int xilinx_dma_reset(struct xilinx_dma_chan *chan) +{ + int err; + u32 tmp; + + dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RESET); + + /* Wait for the hardware to finish reset */ + err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp, + !(tmp & XILINX_DMA_DMACR_RESET), 0, + XILINX_DMA_LOOP_COUNT); + + if (err) { + dev_err(chan->dev, "reset timeout, cr %x, sr %x\n", + dma_ctrl_read(chan, XILINX_DMA_REG_DMACR), + dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); + return -ETIMEDOUT; + } + + chan->err = false; + + return err; +} + +/** + * xilinx_dma_chan_reset - Reset DMA channel and enable interrupts + * @chan: Driver specific DMA channel + * + * Return: '0' on success and failure value on error + */ +static int xilinx_dma_chan_reset(struct xilinx_dma_chan *chan) +{ + int err; + + /* Reset VDMA */ + err = xilinx_dma_reset(chan); + if (err) + return err; + + /* Enable interrupts */ + dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, + XILINX_DMA_DMAXR_ALL_IRQ_MASK); + + return 0; +} + +/** + * xilinx_dma_irq_handler - DMA Interrupt handler + * @irq: IRQ number + * @data: Pointer to the Xilinx DMA channel structure + * + * Return: IRQ_HANDLED/IRQ_NONE + */ +static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) +{ + struct xilinx_dma_chan *chan = data; + u32 status; + + /* Read the status and ack the interrupts. */ + status = dma_ctrl_read(chan, XILINX_DMA_REG_DMASR); + if (!(status & XILINX_DMA_DMAXR_ALL_IRQ_MASK)) + return IRQ_NONE; + + dma_ctrl_write(chan, XILINX_DMA_REG_DMASR, + status & XILINX_DMA_DMAXR_ALL_IRQ_MASK); + + if (status & XILINX_DMA_DMASR_ERR_IRQ) { + /* + * An error occurred. If C_FLUSH_ON_FSYNC is enabled and the + * error is recoverable, ignore it. Otherwise flag the error. + * + * Only recoverable errors can be cleared in the DMASR register, + * make sure not to write to other error bits to 1. + */ + u32 errors = status & XILINX_DMA_DMASR_ALL_ERR_MASK; + + dma_ctrl_write(chan, XILINX_DMA_REG_DMASR, + errors & XILINX_DMA_DMASR_ERR_RECOVER_MASK); + + if (!chan->flush_on_fsync || + (errors & ~XILINX_DMA_DMASR_ERR_RECOVER_MASK)) { + dev_err(chan->dev, + "Channel %p has errors %x, cdr %x tdr %x\n", + chan, errors, + dma_ctrl_read(chan, XILINX_DMA_REG_CURDESC), + dma_ctrl_read(chan, XILINX_DMA_REG_TAILDESC)); + chan->err = true; + } + } + + if (status & XILINX_DMA_DMASR_DLY_CNT_IRQ) { + /* + * Device takes too long to do the transfer when user requires + * responsiveness. + */ + dev_dbg(chan->dev, "Inter-packet latency too long\n"); + } + + if (status & XILINX_DMA_DMASR_FRM_CNT_IRQ) { + spin_lock(&chan->lock); + xilinx_dma_complete_descriptor(chan); + chan->start_transfer(chan); + spin_unlock(&chan->lock); + } + + tasklet_schedule(&chan->tasklet); + return IRQ_HANDLED; +} + +/** + * append_desc_queue - Queuing descriptor + * @chan: Driver specific dma channel + * @desc: dma transaction descriptor + */ +static void append_desc_queue(struct xilinx_dma_chan *chan, + struct xilinx_dma_tx_descriptor *desc) +{ + struct xilinx_vdma_tx_segment *tail_segment; + struct xilinx_dma_tx_descriptor *tail_desc; + struct xilinx_axidma_tx_segment *axidma_tail_segment; + struct xilinx_cdma_tx_segment *cdma_tail_segment; + + if (list_empty(&chan->pending_list)) + goto append; + + /* + * Add the hardware descriptor to the chain of hardware descriptors + * that already exists in memory. + */ + tail_desc = list_last_entry(&chan->pending_list, + struct xilinx_dma_tx_descriptor, node); + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { + tail_segment = list_last_entry(&tail_desc->segments, + struct xilinx_vdma_tx_segment, + node); + tail_segment->hw.next_desc = (u32)desc->async_tx.phys; + } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { + cdma_tail_segment = list_last_entry(&tail_desc->segments, + struct xilinx_cdma_tx_segment, + node); + cdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys; + } else { + axidma_tail_segment = list_last_entry(&tail_desc->segments, + struct xilinx_axidma_tx_segment, + node); + axidma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys; + } + + /* + * Add the software descriptor and all children to the list + * of pending transactions + */ +append: + list_add_tail(&desc->node, &chan->pending_list); + chan->desc_pendingcount++; + + if (chan->has_sg && (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) + && unlikely(chan->desc_pendingcount > chan->num_frms)) { + dev_dbg(chan->dev, "desc pendingcount is too high\n"); + chan->desc_pendingcount = chan->num_frms; + } +} + +/** + * xilinx_dma_tx_submit - Submit DMA transaction + * @tx: Async transaction descriptor + * + * Return: cookie value on success and failure value on error + */ +static dma_cookie_t xilinx_dma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx); + struct xilinx_dma_chan *chan = to_xilinx_chan(tx->chan); + dma_cookie_t cookie; + unsigned long flags; + int err; + + if (chan->cyclic) { + xilinx_dma_free_tx_descriptor(chan, desc); + return -EBUSY; + } + + if (chan->err) { + /* + * If reset fails, need to hard reset the system. + * Channel is no longer functional + */ + err = xilinx_dma_chan_reset(chan); + if (err < 0) + return err; + } + + spin_lock_irqsave(&chan->lock, flags); + + cookie = dma_cookie_assign(tx); + + /* Put this transaction onto the tail of the pending queue */ + append_desc_queue(chan, desc); + + if (desc->cyclic) + chan->cyclic = true; + + spin_unlock_irqrestore(&chan->lock, flags); + + return cookie; +} + +/** + * xilinx_vdma_dma_prep_interleaved - prepare a descriptor for a + * DMA_SLAVE transaction + * @dchan: DMA channel + * @xt: Interleaved template pointer + * @flags: transfer ack flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, + struct dma_interleaved_template *xt, + unsigned long flags) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dma_tx_descriptor *desc; + struct xilinx_vdma_tx_segment *segment, *prev = NULL; + struct xilinx_vdma_desc_hw *hw; + + if (!is_slave_direction(xt->dir)) + return NULL; + + if (!xt->numf || !xt->sgl[0].size) + return NULL; + + if (xt->frame_size != 1) + return NULL; + + /* Allocate a transaction descriptor. */ + desc = xilinx_dma_alloc_tx_descriptor(chan); + if (!desc) + return NULL; + + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = xilinx_dma_tx_submit; + async_tx_ack(&desc->async_tx); + + /* Allocate the link descriptor from DMA pool */ + segment = xilinx_vdma_alloc_tx_segment(chan); + if (!segment) + goto error; + + /* Fill in the hardware descriptor */ + hw = &segment->hw; + hw->vsize = xt->numf; + hw->hsize = xt->sgl[0].size; + hw->stride = (xt->sgl[0].icg + xt->sgl[0].size) << + XILINX_DMA_FRMDLY_STRIDE_STRIDE_SHIFT; + hw->stride |= chan->config.frm_dly << + XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT; + + if (xt->dir != DMA_MEM_TO_DEV) { + if (chan->ext_addr) { + hw->buf_addr = lower_32_bits(xt->dst_start); + hw->buf_addr_msb = upper_32_bits(xt->dst_start); + } else { + hw->buf_addr = xt->dst_start; + } + } else { + if (chan->ext_addr) { + hw->buf_addr = lower_32_bits(xt->src_start); + hw->buf_addr_msb = upper_32_bits(xt->src_start); + } else { + hw->buf_addr = xt->src_start; + } + } + + /* Insert the segment into the descriptor segments list. */ + list_add_tail(&segment->node, &desc->segments); + + prev = segment; + + /* Link the last hardware descriptor with the first. */ + segment = list_first_entry(&desc->segments, + struct xilinx_vdma_tx_segment, node); + desc->async_tx.phys = segment->phys; + + return &desc->async_tx; + +error: + xilinx_dma_free_tx_descriptor(chan, desc); + return NULL; +} + +/** + * xilinx_cdma_prep_memcpy - prepare descriptors for a memcpy transaction + * @dchan: DMA channel + * @dma_dst: destination address + * @dma_src: source address + * @len: transfer length + * @flags: transfer ack flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +xilinx_cdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst, + dma_addr_t dma_src, size_t len, unsigned long flags) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dma_tx_descriptor *desc; + struct xilinx_cdma_tx_segment *segment, *prev; + struct xilinx_cdma_desc_hw *hw; + + if (!len || len > XILINX_DMA_MAX_TRANS_LEN) + return NULL; + + desc = xilinx_dma_alloc_tx_descriptor(chan); + if (!desc) + return NULL; + + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = xilinx_dma_tx_submit; + + /* Allocate the link descriptor from DMA pool */ + segment = xilinx_cdma_alloc_tx_segment(chan); + if (!segment) + goto error; + + hw = &segment->hw; + hw->control = len; + hw->src_addr = dma_src; + hw->dest_addr = dma_dst; + if (chan->ext_addr) { + hw->src_addr_msb = upper_32_bits(dma_src); + hw->dest_addr_msb = upper_32_bits(dma_dst); + } + + /* Fill the previous next descriptor with current */ + prev = list_last_entry(&desc->segments, + struct xilinx_cdma_tx_segment, node); + prev->hw.next_desc = segment->phys; + + /* Insert the segment into the descriptor segments list. */ + list_add_tail(&segment->node, &desc->segments); + + prev = segment; + + /* Link the last hardware descriptor with the first. */ + segment = list_first_entry(&desc->segments, + struct xilinx_cdma_tx_segment, node); + desc->async_tx.phys = segment->phys; + prev->hw.next_desc = segment->phys; + + return &desc->async_tx; + +error: + xilinx_dma_free_tx_descriptor(chan, desc); + return NULL; +} + +/** + * xilinx_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction + * @dchan: DMA channel + * @sgl: scatterlist to transfer to/from + * @sg_len: number of entries in @scatterlist + * @direction: DMA direction + * @flags: transfer ack flags + * @context: APP words of the descriptor + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg( + struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len, + enum dma_transfer_direction direction, unsigned long flags, + void *context) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dma_tx_descriptor *desc; + struct xilinx_axidma_tx_segment *segment = NULL, *prev = NULL; + u32 *app_w = (u32 *)context; + struct scatterlist *sg; + size_t copy; + size_t sg_used; + unsigned int i; + + if (!is_slave_direction(direction)) + return NULL; + + /* Allocate a transaction descriptor. */ + desc = xilinx_dma_alloc_tx_descriptor(chan); + if (!desc) + return NULL; + + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = xilinx_dma_tx_submit; + + /* Build transactions using information in the scatter gather list */ + for_each_sg(sgl, sg, sg_len, i) { + sg_used = 0; + + /* Loop until the entire scatterlist entry is used */ + while (sg_used < sg_dma_len(sg)) { + struct xilinx_axidma_desc_hw *hw; + + /* Get a free segment */ + segment = xilinx_axidma_alloc_tx_segment(chan); + if (!segment) + goto error; + + /* + * Calculate the maximum number of bytes to transfer, + * making sure it is less than the hw limit + */ + copy = min_t(size_t, sg_dma_len(sg) - sg_used, + XILINX_DMA_MAX_TRANS_LEN); + hw = &segment->hw; + + /* Fill in the descriptor */ + xilinx_axidma_buf(chan, hw, sg_dma_address(sg), + sg_used, 0); + + hw->control = copy; + + if (chan->direction == DMA_MEM_TO_DEV) { + if (app_w) + memcpy(hw->app, app_w, sizeof(u32) * + XILINX_DMA_NUM_APP_WORDS); + } + + if (prev) + prev->hw.next_desc = segment->phys; + + prev = segment; + sg_used += copy; + + /* + * Insert the segment into the descriptor segments + * list. + */ + list_add_tail(&segment->node, &desc->segments); + } + } + + segment = list_first_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + desc->async_tx.phys = segment->phys; + prev->hw.next_desc = segment->phys; + + /* For the last DMA_MEM_TO_DEV transfer, set EOP */ + if (chan->direction == DMA_MEM_TO_DEV) { + segment->hw.control |= XILINX_DMA_BD_SOP; + segment = list_last_entry(&desc->segments, + struct xilinx_axidma_tx_segment, + node); + segment->hw.control |= XILINX_DMA_BD_EOP; + } + + return &desc->async_tx; + +error: + xilinx_dma_free_tx_descriptor(chan, desc); + return NULL; +} + +/** + * xilinx_dma_prep_dma_cyclic - prepare descriptors for a DMA_SLAVE transaction + * @chan: DMA channel + * @sgl: scatterlist to transfer to/from + * @sg_len: number of entries in @scatterlist + * @direction: DMA direction + * @flags: transfer ack flags + */ +static struct dma_async_tx_descriptor *xilinx_dma_prep_dma_cyclic( + struct dma_chan *dchan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dma_tx_descriptor *desc; + struct xilinx_axidma_tx_segment *segment, *head_segment, *prev = NULL; + size_t copy, sg_used; + unsigned int num_periods; + int i; + u32 reg; + + if (!period_len) + return NULL; + + num_periods = buf_len / period_len; + + if (!num_periods) + return NULL; + + if (!is_slave_direction(direction)) + return NULL; + + /* Allocate a transaction descriptor. */ + desc = xilinx_dma_alloc_tx_descriptor(chan); + if (!desc) + return NULL; + + chan->direction = direction; + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = xilinx_dma_tx_submit; + + for (i = 0; i < num_periods; ++i) { + sg_used = 0; + + while (sg_used < period_len) { + struct xilinx_axidma_desc_hw *hw; + + /* Get a free segment */ + segment = xilinx_axidma_alloc_tx_segment(chan); + if (!segment) + goto error; + + /* + * Calculate the maximum number of bytes to transfer, + * making sure it is less than the hw limit + */ + copy = min_t(size_t, period_len - sg_used, + XILINX_DMA_MAX_TRANS_LEN); + hw = &segment->hw; + xilinx_axidma_buf(chan, hw, buf_addr, sg_used, + period_len * i); + hw->control = copy; + + if (prev) + prev->hw.next_desc = segment->phys; + + prev = segment; + sg_used += copy; + + /* + * Insert the segment into the descriptor segments + * list. + */ + list_add_tail(&segment->node, &desc->segments); + } + } + + head_segment = list_first_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + desc->async_tx.phys = head_segment->phys; + + desc->cyclic = true; + reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); + reg |= XILINX_DMA_CR_CYCLIC_BD_EN_MASK; + dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); + + segment = list_last_entry(&desc->segments, + struct xilinx_axidma_tx_segment, + node); + segment->hw.next_desc = (u32) head_segment->phys; + + /* For the last DMA_MEM_TO_DEV transfer, set EOP */ + if (direction == DMA_MEM_TO_DEV) { + head_segment->hw.control |= XILINX_DMA_BD_SOP; + segment->hw.control |= XILINX_DMA_BD_EOP; + } + + return &desc->async_tx; + +error: + xilinx_dma_free_tx_descriptor(chan, desc); + return NULL; +} + +/** + * xilinx_dma_prep_interleaved - prepare a descriptor for a + * DMA_SLAVE transaction + * @dchan: DMA channel + * @xt: Interleaved template pointer + * @flags: transfer ack flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +xilinx_dma_prep_interleaved(struct dma_chan *dchan, + struct dma_interleaved_template *xt, + unsigned long flags) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dma_tx_descriptor *desc; + struct xilinx_axidma_tx_segment *segment; + struct xilinx_axidma_desc_hw *hw; + + if (!is_slave_direction(xt->dir)) + return NULL; + + if (!xt->numf || !xt->sgl[0].size) + return NULL; + + if (xt->frame_size != 1) + return NULL; + + /* Allocate a transaction descriptor. */ + desc = xilinx_dma_alloc_tx_descriptor(chan); + if (!desc) + return NULL; + + chan->direction = xt->dir; + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = xilinx_dma_tx_submit; + + /* Get a free segment */ + segment = xilinx_axidma_alloc_tx_segment(chan); + if (!segment) + goto error; + + hw = &segment->hw; + + /* Fill in the descriptor */ + if (xt->dir != DMA_MEM_TO_DEV) + hw->buf_addr = xt->dst_start; + else + hw->buf_addr = xt->src_start; + + hw->mcdma_control = chan->tdest & XILINX_DMA_BD_TDEST_MASK; + hw->vsize_stride = (xt->numf << XILINX_DMA_BD_VSIZE_SHIFT) & + XILINX_DMA_BD_VSIZE_MASK; + hw->vsize_stride |= (xt->sgl[0].icg + xt->sgl[0].size) & + XILINX_DMA_BD_STRIDE_MASK; + hw->control = xt->sgl[0].size & XILINX_DMA_BD_HSIZE_MASK; + + /* + * Insert the segment into the descriptor segments + * list. + */ + list_add_tail(&segment->node, &desc->segments); + + + segment = list_first_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + desc->async_tx.phys = segment->phys; + + /* For the last DMA_MEM_TO_DEV transfer, set EOP */ + if (xt->dir == DMA_MEM_TO_DEV) { + segment->hw.control |= XILINX_DMA_BD_SOP; + segment = list_last_entry(&desc->segments, + struct xilinx_axidma_tx_segment, + node); + segment->hw.control |= XILINX_DMA_BD_EOP; + } + + return &desc->async_tx; + +error: + xilinx_dma_free_tx_descriptor(chan, desc); + return NULL; +} + +/** + * xilinx_dma_terminate_all - Halt the channel and free descriptors + * @chan: Driver specific DMA Channel pointer + */ +static int xilinx_dma_terminate_all(struct dma_chan *dchan) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + u32 reg; + + if (chan->cyclic) + xilinx_dma_chan_reset(chan); + + /* Halt the DMA engine */ + xilinx_dma_halt(chan); + + /* Remove and free all of the descriptors in the lists */ + xilinx_dma_free_descriptors(chan); + + if (chan->cyclic) { + reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); + reg &= ~XILINX_DMA_CR_CYCLIC_BD_EN_MASK; + dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); + chan->cyclic = false; + } + + return 0; +} + +/** + * xilinx_dma_channel_set_config - Configure VDMA channel + * Run-time configuration for Axi VDMA, supports: + * . halt the channel + * . configure interrupt coalescing and inter-packet delay threshold + * . start/stop parking + * . enable genlock + * + * @dchan: DMA channel + * @cfg: VDMA device configuration pointer + * + * Return: '0' on success and failure value on error + */ +int xilinx_vdma_channel_set_config(struct dma_chan *dchan, + struct xilinx_vdma_config *cfg) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + u32 dmacr; + + if (cfg->reset) + return xilinx_dma_chan_reset(chan); + + dmacr = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR); + + chan->config.frm_dly = cfg->frm_dly; + chan->config.park = cfg->park; + + /* genlock settings */ + chan->config.gen_lock = cfg->gen_lock; + chan->config.master = cfg->master; + + if (cfg->gen_lock && chan->genlock) { + dmacr |= XILINX_DMA_DMACR_GENLOCK_EN; + dmacr |= cfg->master << XILINX_DMA_DMACR_MASTER_SHIFT; + } + + chan->config.frm_cnt_en = cfg->frm_cnt_en; + if (cfg->park) + chan->config.park_frm = cfg->park_frm; + else + chan->config.park_frm = -1; + + chan->config.coalesc = cfg->coalesc; + chan->config.delay = cfg->delay; + + if (cfg->coalesc <= XILINX_DMA_DMACR_FRAME_COUNT_MAX) { + dmacr |= cfg->coalesc << XILINX_DMA_DMACR_FRAME_COUNT_SHIFT; + chan->config.coalesc = cfg->coalesc; + } + + if (cfg->delay <= XILINX_DMA_DMACR_DELAY_MAX) { + dmacr |= cfg->delay << XILINX_DMA_DMACR_DELAY_SHIFT; + chan->config.delay = cfg->delay; + } + + /* FSync Source selection */ + dmacr &= ~XILINX_DMA_DMACR_FSYNCSRC_MASK; + dmacr |= cfg->ext_fsync << XILINX_DMA_DMACR_FSYNCSRC_SHIFT; + + dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, dmacr); + + return 0; +} +EXPORT_SYMBOL(xilinx_vdma_channel_set_config); + +/* ----------------------------------------------------------------------------- + * Probe and remove + */ + +/** + * xilinx_dma_chan_remove - Per Channel remove function + * @chan: Driver specific DMA channel + */ +static void xilinx_dma_chan_remove(struct xilinx_dma_chan *chan) +{ + /* Disable all interrupts */ + dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, + XILINX_DMA_DMAXR_ALL_IRQ_MASK); + + if (chan->irq > 0) + free_irq(chan->irq, chan); + + tasklet_kill(&chan->tasklet); + + list_del(&chan->common.device_node); +} + +static int axidma_clk_init(struct platform_device *pdev, struct clk **axi_clk, + struct clk **tx_clk, struct clk **rx_clk, + struct clk **sg_clk, struct clk **tmp_clk) +{ + int err; + + *tmp_clk = NULL; + + *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); + if (IS_ERR(*axi_clk)) { + err = PTR_ERR(*axi_clk); + dev_err(&pdev->dev, "failed to get axi_aclk (%u)\n", err); + return err; + } + + *tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk"); + if (IS_ERR(*tx_clk)) + *tx_clk = NULL; + + *rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk"); + if (IS_ERR(*rx_clk)) + *rx_clk = NULL; + + *sg_clk = devm_clk_get(&pdev->dev, "m_axi_sg_aclk"); + if (IS_ERR(*sg_clk)) + *sg_clk = NULL; + + err = clk_prepare_enable(*axi_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable axi_clk (%u)\n", err); + return err; + } + + err = clk_prepare_enable(*tx_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); + goto err_disable_axiclk; + } + + err = clk_prepare_enable(*rx_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err); + goto err_disable_txclk; + } + + err = clk_prepare_enable(*sg_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable sg_clk (%u)\n", err); + goto err_disable_rxclk; + } + + return 0; + +err_disable_rxclk: + clk_disable_unprepare(*rx_clk); +err_disable_txclk: + clk_disable_unprepare(*tx_clk); +err_disable_axiclk: + clk_disable_unprepare(*axi_clk); + + return err; +} + +static int axicdma_clk_init(struct platform_device *pdev, struct clk **axi_clk, + struct clk **dev_clk, struct clk **tmp_clk, + struct clk **tmp1_clk, struct clk **tmp2_clk) +{ + int err; + + *tmp_clk = NULL; + *tmp1_clk = NULL; + *tmp2_clk = NULL; + + *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); + if (IS_ERR(*axi_clk)) { + err = PTR_ERR(*axi_clk); + dev_err(&pdev->dev, "failed to get axi_clk (%u)\n", err); + return err; + } + + *dev_clk = devm_clk_get(&pdev->dev, "m_axi_aclk"); + if (IS_ERR(*dev_clk)) { + err = PTR_ERR(*dev_clk); + dev_err(&pdev->dev, "failed to get dev_clk (%u)\n", err); + return err; + } + + err = clk_prepare_enable(*axi_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable axi_clk (%u)\n", err); + return err; + } + + err = clk_prepare_enable(*dev_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable dev_clk (%u)\n", err); + goto err_disable_axiclk; + } + + return 0; + +err_disable_axiclk: + clk_disable_unprepare(*axi_clk); + + return err; +} + +static int axivdma_clk_init(struct platform_device *pdev, struct clk **axi_clk, + struct clk **tx_clk, struct clk **txs_clk, + struct clk **rx_clk, struct clk **rxs_clk) +{ + int err; + + *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); + if (IS_ERR(*axi_clk)) { + err = PTR_ERR(*axi_clk); + dev_err(&pdev->dev, "failed to get axi_aclk (%u)\n", err); + return err; + } + + *tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk"); + if (IS_ERR(*tx_clk)) + *tx_clk = NULL; + + *txs_clk = devm_clk_get(&pdev->dev, "m_axis_mm2s_aclk"); + if (IS_ERR(*txs_clk)) + *txs_clk = NULL; + + *rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk"); + if (IS_ERR(*rx_clk)) + *rx_clk = NULL; + + *rxs_clk = devm_clk_get(&pdev->dev, "s_axis_s2mm_aclk"); + if (IS_ERR(*rxs_clk)) + *rxs_clk = NULL; + + err = clk_prepare_enable(*axi_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable axi_clk (%u)\n", err); + return err; + } + + err = clk_prepare_enable(*tx_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); + goto err_disable_axiclk; + } + + err = clk_prepare_enable(*txs_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable txs_clk (%u)\n", err); + goto err_disable_txclk; + } + + err = clk_prepare_enable(*rx_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err); + goto err_disable_txsclk; + } + + err = clk_prepare_enable(*rxs_clk); + if (err) { + dev_err(&pdev->dev, "failed to enable rxs_clk (%u)\n", err); + goto err_disable_rxclk; + } + + return 0; + +err_disable_rxclk: + clk_disable_unprepare(*rx_clk); +err_disable_txsclk: + clk_disable_unprepare(*txs_clk); +err_disable_txclk: + clk_disable_unprepare(*tx_clk); +err_disable_axiclk: + clk_disable_unprepare(*axi_clk); + + return err; +} + +static void xdma_disable_allclks(struct xilinx_dma_device *xdev) +{ + clk_disable_unprepare(xdev->rxs_clk); + clk_disable_unprepare(xdev->rx_clk); + clk_disable_unprepare(xdev->txs_clk); + clk_disable_unprepare(xdev->tx_clk); + clk_disable_unprepare(xdev->axi_clk); +} + +/** + * xilinx_dma_chan_probe - Per Channel Probing + * It get channel features from the device tree entry and + * initialize special channel handling routines + * + * @xdev: Driver specific device structure + * @node: Device node + * + * Return: '0' on success and failure value on error + */ +static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, + struct device_node *node, int chan_id) +{ + struct xilinx_dma_chan *chan; + bool has_dre = false; + u32 value, width; + int err; + + /* Allocate and initialize the channel structure */ + chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL); + if (!chan) + return -ENOMEM; + + chan->dev = xdev->dev; + chan->xdev = xdev; + chan->has_sg = xdev->has_sg; + chan->desc_pendingcount = 0x0; + chan->ext_addr = xdev->ext_addr; + + spin_lock_init(&chan->lock); + INIT_LIST_HEAD(&chan->pending_list); + INIT_LIST_HEAD(&chan->done_list); + INIT_LIST_HEAD(&chan->active_list); + + /* Retrieve the channel properties from the device tree */ + has_dre = of_property_read_bool(node, "xlnx,include-dre"); + + chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode"); + + err = of_property_read_u32(node, "xlnx,datawidth", &value); + if (err) { + dev_err(xdev->dev, "missing xlnx,datawidth property\n"); + return err; + } + width = value >> 3; /* Convert bits to bytes */ + + /* If data width is greater than 8 bytes, DRE is not in hw */ + if (width > 8) + has_dre = false; + + if (!has_dre) + xdev->common.copy_align = fls(width - 1); + + if (of_device_is_compatible(node, "xlnx,axi-vdma-mm2s-channel") || + of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") || + of_device_is_compatible(node, "xlnx,axi-cdma-channel")) { + chan->direction = DMA_MEM_TO_DEV; + chan->id = chan_id; + chan->tdest = chan_id; + + chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET; + if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { + chan->desc_offset = XILINX_VDMA_MM2S_DESC_OFFSET; + + if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH || + xdev->flush_on_fsync == XILINX_DMA_FLUSH_MM2S) + chan->flush_on_fsync = true; + } + } else if (of_device_is_compatible(node, + "xlnx,axi-vdma-s2mm-channel") || + of_device_is_compatible(node, + "xlnx,axi-dma-s2mm-channel")) { + chan->direction = DMA_DEV_TO_MEM; + chan->id = chan_id; + chan->tdest = chan_id - xdev->nr_channels; + + chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET; + if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { + chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET; + + if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH || + xdev->flush_on_fsync == XILINX_DMA_FLUSH_S2MM) + chan->flush_on_fsync = true; + } + } else { + dev_err(xdev->dev, "Invalid channel compatible node\n"); + return -EINVAL; + } + + /* Request the interrupt */ + chan->irq = irq_of_parse_and_map(node, 0); + err = request_irq(chan->irq, xilinx_dma_irq_handler, IRQF_SHARED, + "xilinx-dma-controller", chan); + if (err) { + dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq); + return err; + } + + if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) + chan->start_transfer = xilinx_dma_start_transfer; + else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) + chan->start_transfer = xilinx_cdma_start_transfer; + else + chan->start_transfer = xilinx_vdma_start_transfer; + + /* Initialize the tasklet */ + tasklet_init(&chan->tasklet, xilinx_dma_do_tasklet, + (unsigned long)chan); + + /* + * Initialize the DMA channel and add it to the DMA engine channels + * list. + */ + chan->common.device = &xdev->common; + + list_add_tail(&chan->common.device_node, &xdev->common.channels); + xdev->chan[chan->id] = chan; + + /* Reset the channel */ + err = xilinx_dma_chan_reset(chan); + if (err < 0) { + dev_err(xdev->dev, "Reset channel failed\n"); + return err; + } + + return 0; +} + +/** + * xilinx_dma_child_probe - Per child node probe + * It get number of dma-channels per child node from + * device-tree and initializes all the channels. + * + * @xdev: Driver specific device structure + * @node: Device node + * + * Return: 0 always. + */ +static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, + struct device_node *node) { + int ret, i, nr_channels = 1; + + ret = of_property_read_u32(node, "dma-channels", &nr_channels); + if ((ret < 0) && xdev->mcdma) + dev_warn(xdev->dev, "missing dma-channels property\n"); + + for (i = 0; i < nr_channels; i++) + xilinx_dma_chan_probe(xdev, node, xdev->chan_id++); + + xdev->nr_channels += nr_channels; + + return 0; +} + +/** + * of_dma_xilinx_xlate - Translation function + * @dma_spec: Pointer to DMA specifier as found in the device tree + * @ofdma: Pointer to DMA controller data + * + * Return: DMA channel pointer on success and NULL on error + */ +static struct dma_chan *of_dma_xilinx_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct xilinx_dma_device *xdev = ofdma->of_dma_data; + int chan_id = dma_spec->args[0]; + + if (chan_id >= xdev->nr_channels || !xdev->chan[chan_id]) + return NULL; + + return dma_get_slave_channel(&xdev->chan[chan_id]->common); +} + +static const struct xilinx_dma_config axidma_config = { + .dmatype = XDMA_TYPE_AXIDMA, + .clk_init = axidma_clk_init, +}; + +static const struct xilinx_dma_config axicdma_config = { + .dmatype = XDMA_TYPE_CDMA, + .clk_init = axicdma_clk_init, +}; + +static const struct xilinx_dma_config axivdma_config = { + .dmatype = XDMA_TYPE_VDMA, + .clk_init = axivdma_clk_init, +}; + +static const struct of_device_id xilinx_dma_of_ids[] = { + { .compatible = "xlnx,axi-dma-1.00.a", .data = &axidma_config }, + { .compatible = "xlnx,axi-cdma-1.00.a", .data = &axicdma_config }, + { .compatible = "xlnx,axi-vdma-1.00.a", .data = &axivdma_config }, + {} +}; +MODULE_DEVICE_TABLE(of, xilinx_dma_of_ids); + +/** + * xilinx_dma_probe - Driver probe function + * @pdev: Pointer to the platform_device structure + * + * Return: '0' on success and failure value on error + */ +static int xilinx_dma_probe(struct platform_device *pdev) +{ + int (*clk_init)(struct platform_device *, struct clk **, struct clk **, + struct clk **, struct clk **, struct clk **) + = axivdma_clk_init; + struct device_node *node = pdev->dev.of_node; + struct xilinx_dma_device *xdev; + struct device_node *child, *np = pdev->dev.of_node; + struct resource *io; + u32 num_frames, addr_width; + int i, err; + + /* Allocate and initialize the DMA engine structure */ + xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL); + if (!xdev) + return -ENOMEM; + + xdev->dev = &pdev->dev; + if (np) { + const struct of_device_id *match; + + match = of_match_node(xilinx_dma_of_ids, np); + if (match && match->data) { + xdev->dma_config = match->data; + clk_init = xdev->dma_config->clk_init; + } + } + + err = clk_init(pdev, &xdev->axi_clk, &xdev->tx_clk, &xdev->txs_clk, + &xdev->rx_clk, &xdev->rxs_clk); + if (err) + return err; + + /* Request and map I/O memory */ + io = platform_get_resource(pdev, IORESOURCE_MEM, 0); + xdev->regs = devm_ioremap_resource(&pdev->dev, io); + if (IS_ERR(xdev->regs)) + return PTR_ERR(xdev->regs); + + /* Retrieve the DMA engine properties from the device tree */ + xdev->has_sg = of_property_read_bool(node, "xlnx,include-sg"); + if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) + xdev->mcdma = of_property_read_bool(node, "xlnx,mcdma"); + + if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { + err = of_property_read_u32(node, "xlnx,num-fstores", + &num_frames); + if (err < 0) { + dev_err(xdev->dev, + "missing xlnx,num-fstores property\n"); + return err; + } + + err = of_property_read_u32(node, "xlnx,flush-fsync", + &xdev->flush_on_fsync); + if (err < 0) + dev_warn(xdev->dev, + "missing xlnx,flush-fsync property\n"); + } + + err = of_property_read_u32(node, "xlnx,addrwidth", &addr_width); + if (err < 0) + dev_warn(xdev->dev, "missing xlnx,addrwidth property\n"); + + if (addr_width > 32) + xdev->ext_addr = true; + else + xdev->ext_addr = false; + + /* Set the dma mask bits */ + dma_set_mask(xdev->dev, DMA_BIT_MASK(addr_width)); + + /* Initialize the DMA engine */ + xdev->common.dev = &pdev->dev; + + INIT_LIST_HEAD(&xdev->common.channels); + if (!(xdev->dma_config->dmatype == XDMA_TYPE_CDMA)) { + dma_cap_set(DMA_SLAVE, xdev->common.cap_mask); + dma_cap_set(DMA_PRIVATE, xdev->common.cap_mask); + } + + xdev->common.device_alloc_chan_resources = + xilinx_dma_alloc_chan_resources; + xdev->common.device_free_chan_resources = + xilinx_dma_free_chan_resources; + xdev->common.device_terminate_all = xilinx_dma_terminate_all; + xdev->common.device_tx_status = xilinx_dma_tx_status; + xdev->common.device_issue_pending = xilinx_dma_issue_pending; + if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask); + xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg; + xdev->common.device_prep_dma_cyclic = + xilinx_dma_prep_dma_cyclic; + xdev->common.device_prep_interleaved_dma = + xilinx_dma_prep_interleaved; + /* Residue calculation is supported by only AXI DMA */ + xdev->common.residue_granularity = + DMA_RESIDUE_GRANULARITY_SEGMENT; + } else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { + dma_cap_set(DMA_MEMCPY, xdev->common.cap_mask); + xdev->common.device_prep_dma_memcpy = xilinx_cdma_prep_memcpy; + } else { + xdev->common.device_prep_interleaved_dma = + xilinx_vdma_dma_prep_interleaved; + } + + platform_set_drvdata(pdev, xdev); + + /* Initialize the channels */ + for_each_child_of_node(node, child) { + err = xilinx_dma_child_probe(xdev, child); + if (err < 0) + goto disable_clks; + } + + if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { + for (i = 0; i < xdev->nr_channels; i++) + if (xdev->chan[i]) + xdev->chan[i]->num_frms = num_frames; + } + + /* Register the DMA engine with the core */ + dma_async_device_register(&xdev->common); + + err = of_dma_controller_register(node, of_dma_xilinx_xlate, + xdev); + if (err < 0) { + dev_err(&pdev->dev, "Unable to register DMA to DT\n"); + dma_async_device_unregister(&xdev->common); + goto error; + } + + dev_info(&pdev->dev, "Xilinx AXI VDMA Engine Driver Probed!!\n"); + + return 0; + +disable_clks: + xdma_disable_allclks(xdev); +error: + for (i = 0; i < xdev->nr_channels; i++) + if (xdev->chan[i]) + xilinx_dma_chan_remove(xdev->chan[i]); + + return err; +} + +/** + * xilinx_dma_remove - Driver remove function + * @pdev: Pointer to the platform_device structure + * + * Return: Always '0' + */ +static int xilinx_dma_remove(struct platform_device *pdev) +{ + struct xilinx_dma_device *xdev = platform_get_drvdata(pdev); + int i; + + of_dma_controller_free(pdev->dev.of_node); + + dma_async_device_unregister(&xdev->common); + + for (i = 0; i < xdev->nr_channels; i++) + if (xdev->chan[i]) + xilinx_dma_chan_remove(xdev->chan[i]); + + xdma_disable_allclks(xdev); + + return 0; +} + +static struct platform_driver xilinx_vdma_driver = { + .driver = { + .name = "xilinx-vdma", + .of_match_table = xilinx_dma_of_ids, + }, + .probe = xilinx_dma_probe, + .remove = xilinx_dma_remove, +}; + +module_platform_driver(xilinx_vdma_driver); + +MODULE_AUTHOR("Xilinx, Inc."); +MODULE_DESCRIPTION("Xilinx VDMA driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c new file mode 100644 index 000000000..6d221e5c7 --- /dev/null +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -0,0 +1,1151 @@ +/* + * DMA driver for Xilinx ZynqMP DMA Engine + * + * Copyright (C) 2016 Xilinx, Inc. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../dmaengine.h" + +/* Register Offsets */ +#define ZYNQMP_DMA_ISR 0x100 +#define ZYNQMP_DMA_IMR 0x104 +#define ZYNQMP_DMA_IER 0x108 +#define ZYNQMP_DMA_IDS 0x10C +#define ZYNQMP_DMA_CTRL0 0x110 +#define ZYNQMP_DMA_CTRL1 0x114 +#define ZYNQMP_DMA_DATA_ATTR 0x120 +#define ZYNQMP_DMA_DSCR_ATTR 0x124 +#define ZYNQMP_DMA_SRC_DSCR_WRD0 0x128 +#define ZYNQMP_DMA_SRC_DSCR_WRD1 0x12C +#define ZYNQMP_DMA_SRC_DSCR_WRD2 0x130 +#define ZYNQMP_DMA_SRC_DSCR_WRD3 0x134 +#define ZYNQMP_DMA_DST_DSCR_WRD0 0x138 +#define ZYNQMP_DMA_DST_DSCR_WRD1 0x13C +#define ZYNQMP_DMA_DST_DSCR_WRD2 0x140 +#define ZYNQMP_DMA_DST_DSCR_WRD3 0x144 +#define ZYNQMP_DMA_SRC_START_LSB 0x158 +#define ZYNQMP_DMA_SRC_START_MSB 0x15C +#define ZYNQMP_DMA_DST_START_LSB 0x160 +#define ZYNQMP_DMA_DST_START_MSB 0x164 +#define ZYNQMP_DMA_RATE_CTRL 0x18C +#define ZYNQMP_DMA_IRQ_SRC_ACCT 0x190 +#define ZYNQMP_DMA_IRQ_DST_ACCT 0x194 +#define ZYNQMP_DMA_CTRL2 0x200 + +/* Interrupt registers bit field definitions */ +#define ZYNQMP_DMA_DONE BIT(10) +#define ZYNQMP_DMA_AXI_WR_DATA BIT(9) +#define ZYNQMP_DMA_AXI_RD_DATA BIT(8) +#define ZYNQMP_DMA_AXI_RD_DST_DSCR BIT(7) +#define ZYNQMP_DMA_AXI_RD_SRC_DSCR BIT(6) +#define ZYNQMP_DMA_IRQ_DST_ACCT_ERR BIT(5) +#define ZYNQMP_DMA_IRQ_SRC_ACCT_ERR BIT(4) +#define ZYNQMP_DMA_BYTE_CNT_OVRFL BIT(3) +#define ZYNQMP_DMA_DST_DSCR_DONE BIT(2) +#define ZYNQMP_DMA_INV_APB BIT(0) + +/* Control 0 register bit field definitions */ +#define ZYNQMP_DMA_OVR_FETCH BIT(7) +#define ZYNQMP_DMA_POINT_TYPE_SG BIT(6) +#define ZYNQMP_DMA_RATE_CTRL_EN BIT(3) + +/* Control 1 register bit field definitions */ +#define ZYNQMP_DMA_SRC_ISSUE GENMASK(4, 0) + +/* Data Attribute register bit field definitions */ +#define ZYNQMP_DMA_ARBURST GENMASK(27, 26) +#define ZYNQMP_DMA_ARCACHE GENMASK(25, 22) +#define ZYNQMP_DMA_ARCACHE_OFST 22 +#define ZYNQMP_DMA_ARQOS GENMASK(21, 18) +#define ZYNQMP_DMA_ARQOS_OFST 18 +#define ZYNQMP_DMA_ARLEN GENMASK(17, 14) +#define ZYNQMP_DMA_ARLEN_OFST 14 +#define ZYNQMP_DMA_AWBURST GENMASK(13, 12) +#define ZYNQMP_DMA_AWCACHE GENMASK(11, 8) +#define ZYNQMP_DMA_AWCACHE_OFST 8 +#define ZYNQMP_DMA_AWQOS GENMASK(7, 4) +#define ZYNQMP_DMA_AWQOS_OFST 4 +#define ZYNQMP_DMA_AWLEN GENMASK(3, 0) +#define ZYNQMP_DMA_AWLEN_OFST 0 + +/* Descriptor Attribute register bit field definitions */ +#define ZYNQMP_DMA_AXCOHRNT BIT(8) +#define ZYNQMP_DMA_AXCACHE GENMASK(7, 4) +#define ZYNQMP_DMA_AXCACHE_OFST 4 +#define ZYNQMP_DMA_AXQOS GENMASK(3, 0) +#define ZYNQMP_DMA_AXQOS_OFST 0 + +/* Control register 2 bit field definitions */ +#define ZYNQMP_DMA_ENABLE BIT(0) + +/* Buffer Descriptor definitions */ +#define ZYNQMP_DMA_DESC_CTRL_STOP 0x10 +#define ZYNQMP_DMA_DESC_CTRL_COMP_INT 0x4 +#define ZYNQMP_DMA_DESC_CTRL_SIZE_256 0x2 +#define ZYNQMP_DMA_DESC_CTRL_COHRNT 0x1 + +/* Interrupt Mask specific definitions */ +#define ZYNQMP_DMA_INT_ERR (ZYNQMP_DMA_AXI_RD_DATA | \ + ZYNQMP_DMA_AXI_WR_DATA | \ + ZYNQMP_DMA_AXI_RD_DST_DSCR | \ + ZYNQMP_DMA_AXI_RD_SRC_DSCR | \ + ZYNQMP_DMA_INV_APB) +#define ZYNQMP_DMA_INT_OVRFL (ZYNQMP_DMA_BYTE_CNT_OVRFL | \ + ZYNQMP_DMA_IRQ_SRC_ACCT_ERR | \ + ZYNQMP_DMA_IRQ_DST_ACCT_ERR) +#define ZYNQMP_DMA_INT_DONE (ZYNQMP_DMA_DONE | ZYNQMP_DMA_DST_DSCR_DONE) +#define ZYNQMP_DMA_INT_EN_DEFAULT_MASK (ZYNQMP_DMA_INT_DONE | \ + ZYNQMP_DMA_INT_ERR | \ + ZYNQMP_DMA_INT_OVRFL | \ + ZYNQMP_DMA_DST_DSCR_DONE) + +/* Max number of descriptors per channel */ +#define ZYNQMP_DMA_NUM_DESCS 32 + +/* Max transfer size per descriptor */ +#define ZYNQMP_DMA_MAX_TRANS_LEN 0x40000000 + +/* Reset values for data attributes */ +#define ZYNQMP_DMA_AXCACHE_VAL 0xF +#define ZYNQMP_DMA_ARLEN_RST_VAL 0xF +#define ZYNQMP_DMA_AWLEN_RST_VAL 0xF + +#define ZYNQMP_DMA_SRC_ISSUE_RST_VAL 0x1F + +#define ZYNQMP_DMA_IDS_DEFAULT_MASK 0xFFF + +/* Bus width in bits */ +#define ZYNQMP_DMA_BUS_WIDTH_64 64 +#define ZYNQMP_DMA_BUS_WIDTH_128 128 + +#define ZYNQMP_DMA_DESC_SIZE(chan) (chan->desc_size) + +#define to_chan(chan) container_of(chan, struct zynqmp_dma_chan, \ + common) +#define tx_to_desc(tx) container_of(tx, struct zynqmp_dma_desc_sw, \ + async_tx) + +/** + * struct zynqmp_dma_desc_ll - Hw linked list descriptor + * @addr: Buffer address + * @size: Size of the buffer + * @ctrl: Control word + * @nxtdscraddr: Next descriptor base address + * @rsvd: Reserved field and for Hw internal use. + */ +struct zynqmp_dma_desc_ll { + u64 addr; + u32 size; + u32 ctrl; + u64 nxtdscraddr; + u64 rsvd; +}; __aligned(64) + +/** + * struct zynqmp_dma_desc_sw - Per Transaction structure + * @src: Source address for simple mode dma + * @dst: Destination address for simple mode dma + * @len: Transfer length for simple mode dma + * @node: Node in the channel descriptor list + * @tx_list: List head for the current transfer + * @async_tx: Async transaction descriptor + * @src_v: Virtual address of the src descriptor + * @src_p: Physical address of the src descriptor + * @dst_v: Virtual address of the dst descriptor + * @dst_p: Physical address of the dst descriptor + */ +struct zynqmp_dma_desc_sw { + u64 src; + u64 dst; + u32 len; + struct list_head node; + struct list_head tx_list; + struct dma_async_tx_descriptor async_tx; + struct zynqmp_dma_desc_ll *src_v; + dma_addr_t src_p; + struct zynqmp_dma_desc_ll *dst_v; + dma_addr_t dst_p; +}; + +/** + * struct zynqmp_dma_chan - Driver specific DMA channel structure + * @zdev: Driver specific device structure + * @regs: Control registers offset + * @lock: Descriptor operation lock + * @pending_list: Descriptors waiting + * @free_list: Descriptors free + * @active_list: Descriptors active + * @sw_desc_pool: SW descriptor pool + * @done_list: Complete descriptors + * @common: DMA common channel + * @desc_pool_v: Statically allocated descriptor base + * @desc_pool_p: Physical allocated descriptor base + * @desc_free_cnt: Descriptor available count + * @dev: The dma device + * @irq: Channel IRQ + * @is_dmacoherent: Tells whether dma operations are coherent or not + * @tasklet: Cleanup work after irq + * @idle : Channel status; + * @desc_size: Size of the low level descriptor + * @err: Channel has errors + * @bus_width: Bus width + * @src_burst_len: Source burst length + * @dst_burst_len: Dest burst length + * @clk_main: Pointer to main clock + * @clk_apb: Pointer to apb clock + */ +struct zynqmp_dma_chan { + struct zynqmp_dma_device *zdev; + void __iomem *regs; + spinlock_t lock; + struct list_head pending_list; + struct list_head free_list; + struct list_head active_list; + struct zynqmp_dma_desc_sw *sw_desc_pool; + struct list_head done_list; + struct dma_chan common; + void *desc_pool_v; + dma_addr_t desc_pool_p; + u32 desc_free_cnt; + struct device *dev; + int irq; + bool is_dmacoherent; + struct tasklet_struct tasklet; + bool idle; + u32 desc_size; + bool err; + u32 bus_width; + u32 src_burst_len; + u32 dst_burst_len; + struct clk *clk_main; + struct clk *clk_apb; +}; + +/** + * struct zynqmp_dma_device - DMA device structure + * @dev: Device Structure + * @common: DMA device structure + * @chan: Driver specific DMA channel + */ +struct zynqmp_dma_device { + struct device *dev; + struct dma_device common; + struct zynqmp_dma_chan *chan; +}; + +static inline void zynqmp_dma_writeq(struct zynqmp_dma_chan *chan, u32 reg, + u64 value) +{ + lo_hi_writeq(value, chan->regs + reg); +} + +/** + * zynqmp_dma_update_desc_to_ctrlr - Updates descriptor to the controller + * @chan: ZynqMP DMA DMA channel pointer + * @desc: Transaction descriptor pointer + */ +static void zynqmp_dma_update_desc_to_ctrlr(struct zynqmp_dma_chan *chan, + struct zynqmp_dma_desc_sw *desc) +{ + dma_addr_t addr; + + addr = desc->src_p; + zynqmp_dma_writeq(chan, ZYNQMP_DMA_SRC_START_LSB, addr); + addr = desc->dst_p; + zynqmp_dma_writeq(chan, ZYNQMP_DMA_DST_START_LSB, addr); +} + +/** + * zynqmp_dma_desc_config_eod - Mark the descriptor as end descriptor + * @chan: ZynqMP DMA channel pointer + * @desc: Hw descriptor pointer + */ +static void zynqmp_dma_desc_config_eod(struct zynqmp_dma_chan *chan, + void *desc) +{ + struct zynqmp_dma_desc_ll *hw = (struct zynqmp_dma_desc_ll *)desc; + + hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_STOP; + hw++; + hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_COMP_INT | ZYNQMP_DMA_DESC_CTRL_STOP; +} + +/** + * zynqmp_dma_config_sg_ll_desc - Configure the linked list descriptor + * @chan: ZynqMP DMA channel pointer + * @sdesc: Hw descriptor pointer + * @src: Source buffer address + * @dst: Destination buffer address + * @len: Transfer length + * @prev: Previous hw descriptor pointer + */ +static void zynqmp_dma_config_sg_ll_desc(struct zynqmp_dma_chan *chan, + struct zynqmp_dma_desc_ll *sdesc, + dma_addr_t src, dma_addr_t dst, size_t len, + struct zynqmp_dma_desc_ll *prev) +{ + struct zynqmp_dma_desc_ll *ddesc = sdesc + 1; + + sdesc->size = ddesc->size = len; + sdesc->addr = src; + ddesc->addr = dst; + + sdesc->ctrl = ddesc->ctrl = ZYNQMP_DMA_DESC_CTRL_SIZE_256; + if (chan->is_dmacoherent) { + sdesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT; + ddesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT; + } + + if (prev) { + dma_addr_t addr = chan->desc_pool_p + + ((uintptr_t)sdesc - (uintptr_t)chan->desc_pool_v); + ddesc = prev + 1; + prev->nxtdscraddr = addr; + ddesc->nxtdscraddr = addr + ZYNQMP_DMA_DESC_SIZE(chan); + } +} + +/** + * zynqmp_dma_init - Initialize the channel + * @chan: ZynqMP DMA channel pointer + */ +static void zynqmp_dma_init(struct zynqmp_dma_chan *chan) +{ + u32 val; + + writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS); + val = readl(chan->regs + ZYNQMP_DMA_ISR); + writel(val, chan->regs + ZYNQMP_DMA_ISR); + + if (chan->is_dmacoherent) { + val = ZYNQMP_DMA_AXCOHRNT; + val = (val & ~ZYNQMP_DMA_AXCACHE) | + (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_AXCACHE_OFST); + writel(val, chan->regs + ZYNQMP_DMA_DSCR_ATTR); + } + + val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR); + if (chan->is_dmacoherent) { + val = (val & ~ZYNQMP_DMA_ARCACHE) | + (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_ARCACHE_OFST); + val = (val & ~ZYNQMP_DMA_AWCACHE) | + (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_AWCACHE_OFST); + } + writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR); + + /* Clearing the interrupt account rgisters */ + val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT); + val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); + + chan->idle = true; +} + +/** + * zynqmp_dma_tx_submit - Submit DMA transaction + * @tx: Async transaction descriptor pointer + * + * Return: cookie value + */ +static dma_cookie_t zynqmp_dma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct zynqmp_dma_chan *chan = to_chan(tx->chan); + struct zynqmp_dma_desc_sw *desc, *new; + dma_cookie_t cookie; + + new = tx_to_desc(tx); + spin_lock_bh(&chan->lock); + cookie = dma_cookie_assign(tx); + + if (!list_empty(&chan->pending_list)) { + desc = list_last_entry(&chan->pending_list, + struct zynqmp_dma_desc_sw, node); + if (!list_empty(&desc->tx_list)) + desc = list_last_entry(&desc->tx_list, + struct zynqmp_dma_desc_sw, node); + desc->src_v->nxtdscraddr = new->src_p; + desc->src_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP; + desc->dst_v->nxtdscraddr = new->dst_p; + desc->dst_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP; + } + + list_add_tail(&new->node, &chan->pending_list); + spin_unlock_bh(&chan->lock); + + return cookie; +} + +/** + * zynqmp_dma_get_descriptor - Get the sw descriptor from the pool + * @chan: ZynqMP DMA channel pointer + * + * Return: The sw descriptor + */ +static struct zynqmp_dma_desc_sw * +zynqmp_dma_get_descriptor(struct zynqmp_dma_chan *chan) +{ + struct zynqmp_dma_desc_sw *desc; + + spin_lock_bh(&chan->lock); + desc = list_first_entry(&chan->free_list, + struct zynqmp_dma_desc_sw, node); + list_del(&desc->node); + spin_unlock_bh(&chan->lock); + + INIT_LIST_HEAD(&desc->tx_list); + /* Clear the src and dst descriptor memory */ + memset((void *)desc->src_v, 0, ZYNQMP_DMA_DESC_SIZE(chan)); + memset((void *)desc->dst_v, 0, ZYNQMP_DMA_DESC_SIZE(chan)); + + return desc; +} + +/** + * zynqmp_dma_free_descriptor - Issue pending transactions + * @chan: ZynqMP DMA channel pointer + * @sdesc: Transaction descriptor pointer + */ +static void zynqmp_dma_free_descriptor(struct zynqmp_dma_chan *chan, + struct zynqmp_dma_desc_sw *sdesc) +{ + struct zynqmp_dma_desc_sw *child, *next; + + chan->desc_free_cnt++; + list_add_tail(&sdesc->node, &chan->free_list); + list_for_each_entry_safe(child, next, &sdesc->tx_list, node) { + chan->desc_free_cnt++; + list_move_tail(&child->node, &chan->free_list); + } +} + +/** + * zynqmp_dma_free_desc_list - Free descriptors list + * @chan: ZynqMP DMA channel pointer + * @list: List to parse and delete the descriptor + */ +static void zynqmp_dma_free_desc_list(struct zynqmp_dma_chan *chan, + struct list_head *list) +{ + struct zynqmp_dma_desc_sw *desc, *next; + + list_for_each_entry_safe(desc, next, list, node) + zynqmp_dma_free_descriptor(chan, desc); +} + +/** + * zynqmp_dma_alloc_chan_resources - Allocate channel resources + * @dchan: DMA channel + * + * Return: Number of descriptors on success and failure value on error + */ +static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan) +{ + struct zynqmp_dma_chan *chan = to_chan(dchan); + struct zynqmp_dma_desc_sw *desc; + int i; + + chan->sw_desc_pool = kzalloc(sizeof(*desc) * ZYNQMP_DMA_NUM_DESCS, + GFP_KERNEL); + if (!chan->sw_desc_pool) + return -ENOMEM; + + chan->idle = true; + chan->desc_free_cnt = ZYNQMP_DMA_NUM_DESCS; + + INIT_LIST_HEAD(&chan->free_list); + + for (i = 0; i < ZYNQMP_DMA_NUM_DESCS; i++) { + desc = chan->sw_desc_pool + i; + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = zynqmp_dma_tx_submit; + list_add_tail(&desc->node, &chan->free_list); + } + + chan->desc_pool_v = dma_zalloc_coherent(chan->dev, + (2 * chan->desc_size * ZYNQMP_DMA_NUM_DESCS), + &chan->desc_pool_p, GFP_KERNEL); + if (!chan->desc_pool_v) + return -ENOMEM; + + for (i = 0; i < ZYNQMP_DMA_NUM_DESCS; i++) { + desc = chan->sw_desc_pool + i; + desc->src_v = (struct zynqmp_dma_desc_ll *) (chan->desc_pool_v + + (i * ZYNQMP_DMA_DESC_SIZE(chan) * 2)); + desc->dst_v = (struct zynqmp_dma_desc_ll *) (desc->src_v + 1); + desc->src_p = chan->desc_pool_p + + (i * ZYNQMP_DMA_DESC_SIZE(chan) * 2); + desc->dst_p = desc->src_p + ZYNQMP_DMA_DESC_SIZE(chan); + } + + return ZYNQMP_DMA_NUM_DESCS; +} + +/** + * zynqmp_dma_start - Start DMA channel + * @chan: ZynqMP DMA channel pointer + */ +static void zynqmp_dma_start(struct zynqmp_dma_chan *chan) +{ + writel(ZYNQMP_DMA_INT_EN_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IER); + chan->idle = false; + writel(ZYNQMP_DMA_ENABLE, chan->regs + ZYNQMP_DMA_CTRL2); +} + +/** + * zynqmp_dma_handle_ovfl_int - Process the overflow interrupt + * @chan: ZynqMP DMA channel pointer + * @status: Interrupt status value + */ +static void zynqmp_dma_handle_ovfl_int(struct zynqmp_dma_chan *chan, u32 status) +{ + u32 val; + + if (status & ZYNQMP_DMA_IRQ_DST_ACCT_ERR) + val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); + if (status & ZYNQMP_DMA_IRQ_SRC_ACCT_ERR) + val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT); +} + +static void zynqmp_dma_config(struct zynqmp_dma_chan *chan) +{ + u32 val; + + val = readl(chan->regs + ZYNQMP_DMA_CTRL0); + val |= ZYNQMP_DMA_POINT_TYPE_SG; + writel(val, chan->regs + ZYNQMP_DMA_CTRL0); + + val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR); + val = (val & ~ZYNQMP_DMA_ARLEN) | + (chan->src_burst_len << ZYNQMP_DMA_ARLEN_OFST); + val = (val & ~ZYNQMP_DMA_AWLEN) | + (chan->dst_burst_len << ZYNQMP_DMA_AWLEN_OFST); + writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR); +} + +/** + * zynqmp_dma_device_config - Zynqmp dma device configuration + * @dchan: DMA channel + * @config: DMA device config + */ +static int zynqmp_dma_device_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + struct zynqmp_dma_chan *chan = to_chan(dchan); + + chan->src_burst_len = config->src_maxburst; + chan->dst_burst_len = config->dst_maxburst; + + return 0; +} + +/** + * zynqmp_dma_start_transfer - Initiate the new transfer + * @chan: ZynqMP DMA channel pointer + */ +static void zynqmp_dma_start_transfer(struct zynqmp_dma_chan *chan) +{ + struct zynqmp_dma_desc_sw *desc; + + if (!chan->idle) + return; + + zynqmp_dma_config(chan); + + desc = list_first_entry_or_null(&chan->pending_list, + struct zynqmp_dma_desc_sw, node); + if (!desc) + return; + + list_splice_tail_init(&chan->pending_list, &chan->active_list); + zynqmp_dma_update_desc_to_ctrlr(chan, desc); + zynqmp_dma_start(chan); +} + + +/** + * zynqmp_dma_chan_desc_cleanup - Cleanup the completed descriptors + * @chan: ZynqMP DMA channel + */ +static void zynqmp_dma_chan_desc_cleanup(struct zynqmp_dma_chan *chan) +{ + struct zynqmp_dma_desc_sw *desc, *next; + + list_for_each_entry_safe(desc, next, &chan->done_list, node) { + dma_async_tx_callback callback; + void *callback_param; + + list_del(&desc->node); + + callback = desc->async_tx.callback; + callback_param = desc->async_tx.callback_param; + if (callback) { + spin_unlock(&chan->lock); + callback(callback_param); + spin_lock(&chan->lock); + } + + /* Run any dependencies, then free the descriptor */ + zynqmp_dma_free_descriptor(chan, desc); + } +} + +/** + * zynqmp_dma_complete_descriptor - Mark the active descriptor as complete + * @chan: ZynqMP DMA channel pointer + */ +static void zynqmp_dma_complete_descriptor(struct zynqmp_dma_chan *chan) +{ + struct zynqmp_dma_desc_sw *desc; + + desc = list_first_entry_or_null(&chan->active_list, + struct zynqmp_dma_desc_sw, node); + if (!desc) + return; + list_del(&desc->node); + dma_cookie_complete(&desc->async_tx); + list_add_tail(&desc->node, &chan->done_list); +} + +/** + * zynqmp_dma_issue_pending - Issue pending transactions + * @dchan: DMA channel pointer + */ +static void zynqmp_dma_issue_pending(struct dma_chan *dchan) +{ + struct zynqmp_dma_chan *chan = to_chan(dchan); + + spin_lock_bh(&chan->lock); + zynqmp_dma_start_transfer(chan); + spin_unlock_bh(&chan->lock); +} + +/** + * zynqmp_dma_free_descriptors - Free channel descriptors + * @dchan: DMA channel pointer + */ +static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan) +{ + zynqmp_dma_free_desc_list(chan, &chan->active_list); + zynqmp_dma_free_desc_list(chan, &chan->pending_list); + zynqmp_dma_free_desc_list(chan, &chan->done_list); +} + +/** + * zynqmp_dma_free_chan_resources - Free channel resources + * @dchan: DMA channel pointer + */ +static void zynqmp_dma_free_chan_resources(struct dma_chan *dchan) +{ + struct zynqmp_dma_chan *chan = to_chan(dchan); + + spin_lock_bh(&chan->lock); + zynqmp_dma_free_descriptors(chan); + spin_unlock_bh(&chan->lock); + dma_free_coherent(chan->dev, + (2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS), + chan->desc_pool_v, chan->desc_pool_p); + kfree(chan->sw_desc_pool); +} + +/** + * zynqmp_dma_reset - Reset the channel + * @chan: ZynqMP DMA channel pointer + */ +static void zynqmp_dma_reset(struct zynqmp_dma_chan *chan) +{ + writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS); + + zynqmp_dma_complete_descriptor(chan); + zynqmp_dma_chan_desc_cleanup(chan); + zynqmp_dma_free_descriptors(chan); + zynqmp_dma_init(chan); +} + +/** + * zynqmp_dma_irq_handler - ZynqMP DMA Interrupt handler + * @irq: IRQ number + * @data: Pointer to the ZynqMP DMA channel structure + * + * Return: IRQ_HANDLED/IRQ_NONE + */ +static irqreturn_t zynqmp_dma_irq_handler(int irq, void *data) +{ + struct zynqmp_dma_chan *chan = (struct zynqmp_dma_chan *)data; + u32 isr, imr, status; + irqreturn_t ret = IRQ_NONE; + + isr = readl(chan->regs + ZYNQMP_DMA_ISR); + imr = readl(chan->regs + ZYNQMP_DMA_IMR); + status = isr & ~imr; + + writel(isr, chan->regs + ZYNQMP_DMA_ISR); + if (status & ZYNQMP_DMA_INT_DONE) { + tasklet_schedule(&chan->tasklet); + ret = IRQ_HANDLED; + } + + if (status & ZYNQMP_DMA_DONE) + chan->idle = true; + + if (status & ZYNQMP_DMA_INT_ERR) { + chan->err = true; + tasklet_schedule(&chan->tasklet); + dev_err(chan->dev, "Channel %p has errors\n", chan); + ret = IRQ_HANDLED; + } + + if (status & ZYNQMP_DMA_INT_OVRFL) { + zynqmp_dma_handle_ovfl_int(chan, status); + dev_info(chan->dev, "Channel %p overflow interrupt\n", chan); + ret = IRQ_HANDLED; + } + + return ret; +} + +/** + * zynqmp_dma_do_tasklet - Schedule completion tasklet + * @data: Pointer to the ZynqMP DMA channel structure + */ +static void zynqmp_dma_do_tasklet(unsigned long data) +{ + struct zynqmp_dma_chan *chan = (struct zynqmp_dma_chan *)data; + u32 count; + + spin_lock(&chan->lock); + + if (chan->err) { + zynqmp_dma_reset(chan); + chan->err = false; + goto unlock; + } + + count = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); + + while (count) { + zynqmp_dma_complete_descriptor(chan); + zynqmp_dma_chan_desc_cleanup(chan); + count--; + } + + if (chan->idle) + zynqmp_dma_start_transfer(chan); + +unlock: + spin_unlock(&chan->lock); +} + +/** + * zynqmp_dma_device_terminate_all - Aborts all transfers on a channel + * @dchan: DMA channel pointer + * + * Return: Always '0' + */ +static int zynqmp_dma_device_terminate_all(struct dma_chan *dchan) +{ + struct zynqmp_dma_chan *chan = to_chan(dchan); + + spin_lock_bh(&chan->lock); + writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS); + zynqmp_dma_free_descriptors(chan); + spin_unlock_bh(&chan->lock); + + return 0; +} + +/** + * zynqmp_dma_prep_memcpy - prepare descriptors for memcpy transaction + * @dchan: DMA channel + * @dma_dst: Destination buffer address + * @dma_src: Source buffer address + * @len: Transfer length + * @flags: transfer ack flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy( + struct dma_chan *dchan, dma_addr_t dma_dst, + dma_addr_t dma_src, size_t len, ulong flags) +{ + struct zynqmp_dma_chan *chan; + struct zynqmp_dma_desc_sw *new, *first = NULL; + void *desc = NULL, *prev = NULL; + size_t copy; + u32 desc_cnt; + + chan = to_chan(dchan); + + if (len > ZYNQMP_DMA_MAX_TRANS_LEN) + return NULL; + + desc_cnt = DIV_ROUND_UP(len, ZYNQMP_DMA_MAX_TRANS_LEN); + + spin_lock_bh(&chan->lock); + if (desc_cnt > chan->desc_free_cnt) { + spin_unlock_bh(&chan->lock); + dev_dbg(chan->dev, "chan %p descs are not available\n", chan); + return NULL; + } + chan->desc_free_cnt = chan->desc_free_cnt - desc_cnt; + spin_unlock_bh(&chan->lock); + + do { + /* Allocate and populate the descriptor */ + new = zynqmp_dma_get_descriptor(chan); + + copy = min_t(size_t, len, ZYNQMP_DMA_MAX_TRANS_LEN); + desc = (struct zynqmp_dma_desc_ll *)new->src_v; + zynqmp_dma_config_sg_ll_desc(chan, desc, dma_src, + dma_dst, copy, prev); + prev = desc; + len -= copy; + dma_src += copy; + dma_dst += copy; + if (!first) + first = new; + else + list_add_tail(&new->node, &first->tx_list); + } while (len); + + zynqmp_dma_desc_config_eod(chan, desc); + async_tx_ack(&first->async_tx); + first->async_tx.flags = flags; + return &first->async_tx; +} + +/** + * zynqmp_dma_prep_slave_sg - prepare descriptors for a memory sg transaction + * @dchan: DMA channel + * @dst_sg: Destination scatter list + * @dst_sg_len: Number of entries in destination scatter list + * @src_sg: Source scatter list + * @src_sg_len: Number of entries in source scatter list + * @flags: transfer ack flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor *zynqmp_dma_prep_sg( + struct dma_chan *dchan, struct scatterlist *dst_sg, + unsigned int dst_sg_len, struct scatterlist *src_sg, + unsigned int src_sg_len, unsigned long flags) +{ + struct zynqmp_dma_desc_sw *new, *first = NULL; + struct zynqmp_dma_chan *chan = to_chan(dchan); + void *desc = NULL, *prev = NULL; + size_t len, dst_avail, src_avail; + dma_addr_t dma_dst, dma_src; + u32 desc_cnt = 0, i; + struct scatterlist *sg; + + for_each_sg(src_sg, sg, src_sg_len, i) + desc_cnt += DIV_ROUND_UP(sg_dma_len(sg), + ZYNQMP_DMA_MAX_TRANS_LEN); + + spin_lock_bh(&chan->lock); + if (desc_cnt > chan->desc_free_cnt) { + spin_unlock_bh(&chan->lock); + dev_dbg(chan->dev, "chan %p descs are not available\n", chan); + return NULL; + } + chan->desc_free_cnt = chan->desc_free_cnt - desc_cnt; + spin_unlock_bh(&chan->lock); + + dst_avail = sg_dma_len(dst_sg); + src_avail = sg_dma_len(src_sg); + + /* Run until we are out of scatterlist entries */ + while (true) { + /* Allocate and populate the descriptor */ + new = zynqmp_dma_get_descriptor(chan); + desc = (struct zynqmp_dma_desc_ll *)new->src_v; + len = min_t(size_t, src_avail, dst_avail); + len = min_t(size_t, len, ZYNQMP_DMA_MAX_TRANS_LEN); + if (len == 0) + goto fetch; + dma_dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) - + dst_avail; + dma_src = sg_dma_address(src_sg) + sg_dma_len(src_sg) - + src_avail; + + zynqmp_dma_config_sg_ll_desc(chan, desc, dma_src, dma_dst, + len, prev); + prev = desc; + dst_avail -= len; + src_avail -= len; + + if (!first) + first = new; + else + list_add_tail(&new->node, &first->tx_list); +fetch: + /* Fetch the next dst scatterlist entry */ + if (dst_avail == 0) { + if (dst_sg_len == 0) + break; + dst_sg = sg_next(dst_sg); + if (dst_sg == NULL) + break; + dst_sg_len--; + dst_avail = sg_dma_len(dst_sg); + } + /* Fetch the next src scatterlist entry */ + if (src_avail == 0) { + if (src_sg_len == 0) + break; + src_sg = sg_next(src_sg); + if (src_sg == NULL) + break; + src_sg_len--; + src_avail = sg_dma_len(src_sg); + } + } + + zynqmp_dma_desc_config_eod(chan, desc); + first->async_tx.flags = flags; + return &first->async_tx; +} + +/** + * zynqmp_dma_chan_remove - Channel remove function + * @chan: ZynqMP DMA channel pointer + */ +static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan) +{ + if (!chan) + return; + + devm_free_irq(chan->zdev->dev, chan->irq, chan); + tasklet_kill(&chan->tasklet); + list_del(&chan->common.device_node); + clk_disable_unprepare(chan->clk_apb); + clk_disable_unprepare(chan->clk_main); +} + +/** + * zynqmp_dma_chan_probe - Per Channel Probing + * @zdev: Driver specific device structure + * @pdev: Pointer to the platform_device structure + * + * Return: '0' on success and failure value on error + */ +static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, + struct platform_device *pdev) +{ + struct zynqmp_dma_chan *chan; + struct resource *res; + struct device_node *node = pdev->dev.of_node; + int err; + + chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL); + if (!chan) + return -ENOMEM; + chan->dev = zdev->dev; + chan->zdev = zdev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + chan->regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(chan->regs)) + return PTR_ERR(chan->regs); + + chan->bus_width = ZYNQMP_DMA_BUS_WIDTH_64; + chan->dst_burst_len = ZYNQMP_DMA_AWLEN_RST_VAL; + chan->src_burst_len = ZYNQMP_DMA_ARLEN_RST_VAL; + err = of_property_read_u32(node, "xlnx,bus-width", &chan->bus_width); + if (err < 0) { + dev_err(&pdev->dev, "missing xlnx,bus-width property\n"); + return err; + } + + if (chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_64 && + chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_128) { + dev_err(zdev->dev, "invalid bus-width value"); + return -EINVAL; + } + + chan->is_dmacoherent = of_property_read_bool(node, "dma-coherent"); + zdev->chan = chan; + tasklet_init(&chan->tasklet, zynqmp_dma_do_tasklet, (ulong)chan); + spin_lock_init(&chan->lock); + INIT_LIST_HEAD(&chan->active_list); + INIT_LIST_HEAD(&chan->pending_list); + INIT_LIST_HEAD(&chan->done_list); + INIT_LIST_HEAD(&chan->free_list); + + dma_cookie_init(&chan->common); + chan->common.device = &zdev->common; + list_add_tail(&chan->common.device_node, &zdev->common.channels); + + zynqmp_dma_init(chan); + chan->irq = platform_get_irq(pdev, 0); + if (chan->irq < 0) + return -ENXIO; + err = devm_request_irq(&pdev->dev, chan->irq, zynqmp_dma_irq_handler, 0, + "zynqmp-dma", chan); + if (err) + return err; + chan->clk_main = devm_clk_get(&pdev->dev, "clk_main"); + if (IS_ERR(chan->clk_main)) { + dev_err(&pdev->dev, "main clock not found.\n"); + return PTR_ERR(chan->clk_main); + } + + chan->clk_apb = devm_clk_get(&pdev->dev, "clk_apb"); + if (IS_ERR(chan->clk_apb)) { + dev_err(&pdev->dev, "apb clock not found.\n"); + return PTR_ERR(chan->clk_apb); + } + + err = clk_prepare_enable(chan->clk_main); + if (err) { + dev_err(&pdev->dev, "Unable to enable main clock.\n"); + return err; + } + + err = clk_prepare_enable(chan->clk_apb); + if (err) { + clk_disable_unprepare(chan->clk_main); + dev_err(&pdev->dev, "Unable to enable apb clock.\n"); + return err; + } + + chan->desc_size = sizeof(struct zynqmp_dma_desc_ll); + chan->idle = true; + return 0; +} + +/** + * of_zynqmp_dma_xlate - Translation function + * @dma_spec: Pointer to DMA specifier as found in the device tree + * @ofdma: Pointer to DMA controller data + * + * Return: DMA channel pointer on success and NULL on error + */ +static struct dma_chan *of_zynqmp_dma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct zynqmp_dma_device *zdev = ofdma->of_dma_data; + + return dma_get_slave_channel(&zdev->chan->common); +} + +/** + * zynqmp_dma_probe - Driver probe function + * @pdev: Pointer to the platform_device structure + * + * Return: '0' on success and failure value on error + */ +static int zynqmp_dma_probe(struct platform_device *pdev) +{ + struct zynqmp_dma_device *zdev; + struct dma_device *p; + int ret; + + zdev = devm_kzalloc(&pdev->dev, sizeof(*zdev), GFP_KERNEL); + if (!zdev) + return -ENOMEM; + + zdev->dev = &pdev->dev; + INIT_LIST_HEAD(&zdev->common.channels); + + dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); + dma_cap_set(DMA_SG, zdev->common.cap_mask); + dma_cap_set(DMA_MEMCPY, zdev->common.cap_mask); + + p = &zdev->common; + p->device_prep_dma_sg = zynqmp_dma_prep_sg; + p->device_prep_dma_memcpy = zynqmp_dma_prep_memcpy; + p->device_terminate_all = zynqmp_dma_device_terminate_all; + p->device_issue_pending = zynqmp_dma_issue_pending; + p->device_alloc_chan_resources = zynqmp_dma_alloc_chan_resources; + p->device_free_chan_resources = zynqmp_dma_free_chan_resources; + p->device_tx_status = dma_cookie_status; + p->device_config = zynqmp_dma_device_config; + p->dev = &pdev->dev; + + platform_set_drvdata(pdev, zdev); + + ret = zynqmp_dma_chan_probe(zdev, pdev); + if (ret) { + dev_err(&pdev->dev, "Probing channel failed\n"); + goto free_chan_resources; + } + + p->dst_addr_widths = BIT(zdev->chan->bus_width / 8); + p->src_addr_widths = BIT(zdev->chan->bus_width / 8); + + dma_async_device_register(&zdev->common); + + ret = of_dma_controller_register(pdev->dev.of_node, + of_zynqmp_dma_xlate, zdev); + if (ret) { + dev_err(&pdev->dev, "Unable to register DMA to DT\n"); + dma_async_device_unregister(&zdev->common); + goto free_chan_resources; + } + + dev_info(&pdev->dev, "ZynqMP DMA driver Probe success\n"); + + return 0; + +free_chan_resources: + zynqmp_dma_chan_remove(zdev->chan); + return ret; +} + +/** + * zynqmp_dma_remove - Driver remove function + * @pdev: Pointer to the platform_device structure + * + * Return: Always '0' + */ +static int zynqmp_dma_remove(struct platform_device *pdev) +{ + struct zynqmp_dma_device *zdev = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&zdev->common); + + zynqmp_dma_chan_remove(zdev->chan); + + return 0; +} + +static const struct of_device_id zynqmp_dma_of_match[] = { + { .compatible = "xlnx,zynqmp-dma-1.0", }, + {} +}; +MODULE_DEVICE_TABLE(of, zynqmp_dma_of_match); + +static struct platform_driver zynqmp_dma_driver = { + .driver = { + .name = "xilinx-zynqmp-dma", + .of_match_table = zynqmp_dma_of_match, + }, + .probe = zynqmp_dma_probe, + .remove = zynqmp_dma_remove, +}; + +module_platform_driver(zynqmp_dma_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Xilinx, Inc."); +MODULE_DESCRIPTION("Xilinx ZynqMP DMA driver"); diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index 6ca7474ba..dff1a4a6d 100644 --- a/drivers/edac/Kconfig +++ b/drivers/edac/Kconfig @@ -251,6 +251,14 @@ config EDAC_SBRIDGE Support for error detection and correction the Intel Sandy Bridge, Ivy Bridge and Haswell Integrated Memory Controllers. +config EDAC_SKX + tristate "Intel Skylake server Integrated MC" + depends on EDAC_MM_EDAC && PCI && X86_64 && X86_MCE_INTEL + depends on PCI_MMCONFIG + help + Support for error detection and correction the Intel + Skylake server Integrated Memory Controllers. + config EDAC_MPC85XX tristate "Freescale MPC83xx / MPC85xx" depends on EDAC_MM_EDAC && FSL_SOC @@ -391,6 +399,13 @@ config EDAC_ALTERA_OCRAM Support for error detection and correction on the Altera On-Chip RAM Memory for Altera SoCs. +config EDAC_ALTERA_ETHERNET + bool "Altera Ethernet FIFO ECC" + depends on EDAC_ALTERA=y + help + Support for error detection and correction on the + Altera Ethernet FIFO Memory for Altera SoCs. + config EDAC_SYNOPSYS tristate "Synopsys DDR Memory Controller" depends on EDAC_MM_EDAC && ARCH_ZYNQ diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile index f9e4a3e0e..986049925 100644 --- a/drivers/edac/Makefile +++ b/drivers/edac/Makefile @@ -31,6 +31,7 @@ obj-$(CONFIG_EDAC_I5400) += i5400_edac.o obj-$(CONFIG_EDAC_I7300) += i7300_edac.o obj-$(CONFIG_EDAC_I7CORE) += i7core_edac.o obj-$(CONFIG_EDAC_SBRIDGE) += sb_edac.o +obj-$(CONFIG_EDAC_SKX) += skx_edac.o obj-$(CONFIG_EDAC_E7XXX) += e7xxx_edac.o obj-$(CONFIG_EDAC_E752X) += e752x_edac.o obj-$(CONFIG_EDAC_I82443BXGX) += i82443bxgx_edac.o diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 5b4d223d6..2398d0701 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -19,12 +19,15 @@ #include #include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -548,10 +551,10 @@ module_platform_driver(altr_edac_driver); * trigger testing are different for each memory. */ -const struct edac_device_prv_data ocramecc_data; -const struct edac_device_prv_data l2ecc_data; -const struct edac_device_prv_data a10_ocramecc_data; -const struct edac_device_prv_data a10_l2ecc_data; +static const struct edac_device_prv_data ocramecc_data; +static const struct edac_device_prv_data l2ecc_data; +static const struct edac_device_prv_data a10_ocramecc_data; +static const struct edac_device_prv_data a10_l2ecc_data; static irqreturn_t altr_edac_device_handler(int irq, void *dev_id) { @@ -686,11 +689,9 @@ static void altr_create_edacdev_dbgfs(struct edac_device_ctl_info *edac_dci, static const struct of_device_id altr_edac_device_of_match[] = { #ifdef CONFIG_EDAC_ALTERA_L2C { .compatible = "altr,socfpga-l2-ecc", .data = &l2ecc_data }, - { .compatible = "altr,socfpga-a10-l2-ecc", .data = &a10_l2ecc_data }, #endif #ifdef CONFIG_EDAC_ALTERA_OCRAM { .compatible = "altr,socfpga-ocram-ecc", .data = &ocramecc_data }, - { .compatible = "altr,socfpga-a10-ocram-ecc", .data = &a10_ocramecc_data }, #endif {}, }; @@ -825,16 +826,16 @@ static struct platform_driver altr_edac_device_driver = { }; module_platform_driver(altr_edac_device_driver); -/*********************** OCRAM EDAC Device Functions *********************/ +/******************* Arria10 Device ECC Shared Functions *****************/ -#ifdef CONFIG_EDAC_ALTERA_OCRAM /* * Test for memory's ECC dependencies upon entry because platform specific * startup should have initialized the memory and enabled the ECC. * Can't turn on ECC here because accessing un-initialized memory will * cause CE/UE errors possibly causing an ABORT. */ -static int altr_check_ecc_deps(struct altr_edac_device_dev *device) +static int __maybe_unused +altr_check_ecc_deps(struct altr_edac_device_dev *device) { void __iomem *base = device->base; const struct edac_device_prv_data *prv = device->data; @@ -848,6 +849,227 @@ static int altr_check_ecc_deps(struct altr_edac_device_dev *device) return -ENODEV; } +static irqreturn_t __maybe_unused altr_edac_a10_ecc_irq(int irq, void *dev_id) +{ + struct altr_edac_device_dev *dci = dev_id; + void __iomem *base = dci->base; + + if (irq == dci->sb_irq) { + writel(ALTR_A10_ECC_SERRPENA, + base + ALTR_A10_ECC_INTSTAT_OFST); + edac_device_handle_ce(dci->edac_dev, 0, 0, dci->edac_dev_name); + + return IRQ_HANDLED; + } else if (irq == dci->db_irq) { + writel(ALTR_A10_ECC_DERRPENA, + base + ALTR_A10_ECC_INTSTAT_OFST); + edac_device_handle_ue(dci->edac_dev, 0, 0, dci->edac_dev_name); + if (dci->data->panic) + panic("\nEDAC:ECC_DEVICE[Uncorrectable errors]\n"); + + return IRQ_HANDLED; + } + + WARN_ON(1); + + return IRQ_NONE; +} + +/******************* Arria10 Memory Buffer Functions *********************/ + +static inline int a10_get_irq_mask(struct device_node *np) +{ + int irq; + const u32 *handle = of_get_property(np, "interrupts", NULL); + + if (!handle) + return -ENODEV; + irq = be32_to_cpup(handle); + return irq; +} + +static inline void ecc_set_bits(u32 bit_mask, void __iomem *ioaddr) +{ + u32 value = readl(ioaddr); + + value |= bit_mask; + writel(value, ioaddr); +} + +static inline void ecc_clear_bits(u32 bit_mask, void __iomem *ioaddr) +{ + u32 value = readl(ioaddr); + + value &= ~bit_mask; + writel(value, ioaddr); +} + +static inline int ecc_test_bits(u32 bit_mask, void __iomem *ioaddr) +{ + u32 value = readl(ioaddr); + + return (value & bit_mask) ? 1 : 0; +} + +/* + * This function uses the memory initialization block in the Arria10 ECC + * controller to initialize/clear the entire memory data and ECC data. + */ +static int __maybe_unused altr_init_memory_port(void __iomem *ioaddr, int port) +{ + int limit = ALTR_A10_ECC_INIT_WATCHDOG_10US; + u32 init_mask, stat_mask, clear_mask; + int ret = 0; + + if (port) { + init_mask = ALTR_A10_ECC_INITB; + stat_mask = ALTR_A10_ECC_INITCOMPLETEB; + clear_mask = ALTR_A10_ECC_ERRPENB_MASK; + } else { + init_mask = ALTR_A10_ECC_INITA; + stat_mask = ALTR_A10_ECC_INITCOMPLETEA; + clear_mask = ALTR_A10_ECC_ERRPENA_MASK; + } + + ecc_set_bits(init_mask, (ioaddr + ALTR_A10_ECC_CTRL_OFST)); + while (limit--) { + if (ecc_test_bits(stat_mask, + (ioaddr + ALTR_A10_ECC_INITSTAT_OFST))) + break; + udelay(1); + } + if (limit < 0) + ret = -EBUSY; + + /* Clear any pending ECC interrupts */ + writel(clear_mask, (ioaddr + ALTR_A10_ECC_INTSTAT_OFST)); + + return ret; +} + +static __init int __maybe_unused +altr_init_a10_ecc_block(struct device_node *np, u32 irq_mask, + u32 ecc_ctrl_en_mask, bool dual_port) +{ + int ret = 0; + void __iomem *ecc_block_base; + struct regmap *ecc_mgr_map; + char *ecc_name; + struct device_node *np_eccmgr; + + ecc_name = (char *)np->name; + + /* Get the ECC Manager - parent of the device EDACs */ + np_eccmgr = of_get_parent(np); + ecc_mgr_map = syscon_regmap_lookup_by_phandle(np_eccmgr, + "altr,sysmgr-syscon"); + of_node_put(np_eccmgr); + if (IS_ERR(ecc_mgr_map)) { + edac_printk(KERN_ERR, EDAC_DEVICE, + "Unable to get syscon altr,sysmgr-syscon\n"); + return -ENODEV; + } + + /* Map the ECC Block */ + ecc_block_base = of_iomap(np, 0); + if (!ecc_block_base) { + edac_printk(KERN_ERR, EDAC_DEVICE, + "Unable to map %s ECC block\n", ecc_name); + return -ENODEV; + } + + /* Disable ECC */ + regmap_write(ecc_mgr_map, A10_SYSMGR_ECC_INTMASK_SET_OFST, irq_mask); + writel(ALTR_A10_ECC_SERRINTEN, + (ecc_block_base + ALTR_A10_ECC_ERRINTENR_OFST)); + ecc_clear_bits(ecc_ctrl_en_mask, + (ecc_block_base + ALTR_A10_ECC_CTRL_OFST)); + /* Ensure all writes complete */ + wmb(); + /* Use HW initialization block to initialize memory for ECC */ + ret = altr_init_memory_port(ecc_block_base, 0); + if (ret) { + edac_printk(KERN_ERR, EDAC_DEVICE, + "ECC: cannot init %s PORTA memory\n", ecc_name); + goto out; + } + + if (dual_port) { + ret = altr_init_memory_port(ecc_block_base, 1); + if (ret) { + edac_printk(KERN_ERR, EDAC_DEVICE, + "ECC: cannot init %s PORTB memory\n", + ecc_name); + goto out; + } + } + + /* Interrupt mode set to every SBERR */ + regmap_write(ecc_mgr_map, ALTR_A10_ECC_INTMODE_OFST, + ALTR_A10_ECC_INTMODE); + /* Enable ECC */ + ecc_set_bits(ecc_ctrl_en_mask, (ecc_block_base + + ALTR_A10_ECC_CTRL_OFST)); + writel(ALTR_A10_ECC_SERRINTEN, + (ecc_block_base + ALTR_A10_ECC_ERRINTENS_OFST)); + regmap_write(ecc_mgr_map, A10_SYSMGR_ECC_INTMASK_CLR_OFST, irq_mask); + /* Ensure all writes complete */ + wmb(); +out: + iounmap(ecc_block_base); + return ret; +} + +static int validate_parent_available(struct device_node *np); +static const struct of_device_id altr_edac_a10_device_of_match[]; +static int __init __maybe_unused altr_init_a10_ecc_device_type(char *compat) +{ + int irq; + struct device_node *child, *np = of_find_compatible_node(NULL, NULL, + "altr,socfpga-a10-ecc-manager"); + if (!np) { + edac_printk(KERN_ERR, EDAC_DEVICE, "ECC Manager not found\n"); + return -ENODEV; + } + + for_each_child_of_node(np, child) { + const struct of_device_id *pdev_id; + const struct edac_device_prv_data *prv; + + if (!of_device_is_available(child)) + continue; + if (!of_device_is_compatible(child, compat)) + continue; + + if (validate_parent_available(child)) + continue; + + irq = a10_get_irq_mask(child); + if (irq < 0) + continue; + + /* Get matching node and check for valid result */ + pdev_id = of_match_node(altr_edac_a10_device_of_match, child); + if (IS_ERR_OR_NULL(pdev_id)) + continue; + + /* Validate private data pointer before dereferencing */ + prv = pdev_id->data; + if (!prv) + continue; + + altr_init_a10_ecc_block(child, BIT(irq), + prv->ecc_enable_mask, 0); + } + + of_node_put(np); + return 0; +} + +/*********************** OCRAM EDAC Device Functions *********************/ + +#ifdef CONFIG_EDAC_ALTERA_OCRAM + static void *ocram_alloc_mem(size_t size, void **other) { struct device_node *np; @@ -882,25 +1104,7 @@ static void ocram_free_mem(void *p, size_t size, void *other) gen_pool_free((struct gen_pool *)other, (u32)p, size); } -static irqreturn_t altr_edac_a10_ecc_irq(struct altr_edac_device_dev *dci, - bool sberr) -{ - void __iomem *base = dci->base; - - if (sberr) { - writel(ALTR_A10_ECC_SERRPENA, - base + ALTR_A10_ECC_INTSTAT_OFST); - edac_device_handle_ce(dci->edac_dev, 0, 0, dci->edac_dev_name); - } else { - writel(ALTR_A10_ECC_DERRPENA, - base + ALTR_A10_ECC_INTSTAT_OFST); - edac_device_handle_ue(dci->edac_dev, 0, 0, dci->edac_dev_name); - panic("\nEDAC:ECC_DEVICE[Uncorrectable errors]\n"); - } - return IRQ_HANDLED; -} - -const struct edac_device_prv_data ocramecc_data = { +static const struct edac_device_prv_data ocramecc_data = { .setup = altr_check_ecc_deps, .ce_clear_mask = (ALTR_OCR_ECC_EN | ALTR_OCR_ECC_SERR), .ue_clear_mask = (ALTR_OCR_ECC_EN | ALTR_OCR_ECC_DERR), @@ -916,7 +1120,7 @@ const struct edac_device_prv_data ocramecc_data = { .inject_fops = &altr_edac_device_inject_fops, }; -const struct edac_device_prv_data a10_ocramecc_data = { +static const struct edac_device_prv_data a10_ocramecc_data = { .setup = altr_check_ecc_deps, .ce_clear_mask = ALTR_A10_ECC_SERRPENA, .ue_clear_mask = ALTR_A10_ECC_DERRPENA, @@ -929,6 +1133,12 @@ const struct edac_device_prv_data a10_ocramecc_data = { .set_err_ofst = ALTR_A10_ECC_INTTEST_OFST, .ecc_irq_handler = altr_edac_a10_ecc_irq, .inject_fops = &altr_edac_a10_device_inject_fops, + /* + * OCRAM panic on uncorrectable error because sleep/resume + * functions and FPGA contents are stored in OCRAM. Prefer + * a kernel panic over executing/loading corrupted data. + */ + .panic = true, }; #endif /* CONFIG_EDAC_ALTERA_OCRAM */ @@ -988,25 +1198,33 @@ static int altr_l2_check_deps(struct altr_edac_device_dev *device) return -ENODEV; } -static irqreturn_t altr_edac_a10_l2_irq(struct altr_edac_device_dev *dci, - bool sberr) +static irqreturn_t altr_edac_a10_l2_irq(int irq, void *dev_id) { - if (sberr) { + struct altr_edac_device_dev *dci = dev_id; + + if (irq == dci->sb_irq) { regmap_write(dci->edac->ecc_mgr_map, A10_SYSGMR_MPU_CLEAR_L2_ECC_OFST, A10_SYSGMR_MPU_CLEAR_L2_ECC_SB); edac_device_handle_ce(dci->edac_dev, 0, 0, dci->edac_dev_name); - } else { + + return IRQ_HANDLED; + } else if (irq == dci->db_irq) { regmap_write(dci->edac->ecc_mgr_map, A10_SYSGMR_MPU_CLEAR_L2_ECC_OFST, A10_SYSGMR_MPU_CLEAR_L2_ECC_MB); edac_device_handle_ue(dci->edac_dev, 0, 0, dci->edac_dev_name); panic("\nEDAC:ECC_DEVICE[Uncorrectable errors]\n"); + + return IRQ_HANDLED; } - return IRQ_HANDLED; + + WARN_ON(1); + + return IRQ_NONE; } -const struct edac_device_prv_data l2ecc_data = { +static const struct edac_device_prv_data l2ecc_data = { .setup = altr_l2_check_deps, .ce_clear_mask = 0, .ue_clear_mask = 0, @@ -1021,7 +1239,7 @@ const struct edac_device_prv_data l2ecc_data = { .inject_fops = &altr_edac_device_inject_fops, }; -const struct edac_device_prv_data a10_l2ecc_data = { +static const struct edac_device_prv_data a10_l2ecc_data = { .setup = altr_l2_check_deps, .ce_clear_mask = ALTR_A10_L2_ECC_SERR_CLR, .ue_clear_mask = ALTR_A10_L2_ECC_MERR_CLR, @@ -1040,7 +1258,49 @@ const struct edac_device_prv_data a10_l2ecc_data = { #endif /* CONFIG_EDAC_ALTERA_L2C */ +/********************* Ethernet Device Functions ********************/ + +#ifdef CONFIG_EDAC_ALTERA_ETHERNET + +static const struct edac_device_prv_data a10_enetecc_data = { + .setup = altr_check_ecc_deps, + .ce_clear_mask = ALTR_A10_ECC_SERRPENA, + .ue_clear_mask = ALTR_A10_ECC_DERRPENA, + .dbgfs_name = "altr_trigger", + .ecc_enable_mask = ALTR_A10_COMMON_ECC_EN_CTL, + .ecc_en_ofst = ALTR_A10_ECC_CTRL_OFST, + .ce_set_mask = ALTR_A10_ECC_TSERRA, + .ue_set_mask = ALTR_A10_ECC_TDERRA, + .set_err_ofst = ALTR_A10_ECC_INTTEST_OFST, + .ecc_irq_handler = altr_edac_a10_ecc_irq, + .inject_fops = &altr_edac_a10_device_inject_fops, +}; + +static int __init socfpga_init_ethernet_ecc(void) +{ + return altr_init_a10_ecc_device_type("altr,socfpga-eth-mac-ecc"); +} + +early_initcall(socfpga_init_ethernet_ecc); + +#endif /* CONFIG_EDAC_ALTERA_ETHERNET */ + /********************* Arria10 EDAC Device Functions *************************/ +static const struct of_device_id altr_edac_a10_device_of_match[] = { +#ifdef CONFIG_EDAC_ALTERA_L2C + { .compatible = "altr,socfpga-a10-l2-ecc", .data = &a10_l2ecc_data }, +#endif +#ifdef CONFIG_EDAC_ALTERA_OCRAM + { .compatible = "altr,socfpga-a10-ocram-ecc", + .data = &a10_ocramecc_data }, +#endif +#ifdef CONFIG_EDAC_ALTERA_ETHERNET + { .compatible = "altr,socfpga-eth-mac-ecc", + .data = &a10_enetecc_data }, +#endif + {}, +}; +MODULE_DEVICE_TABLE(of, altr_edac_a10_device_of_match); /* * The Arria10 EDAC Device Functions differ from the Cyclone5/Arria5 @@ -1075,28 +1335,42 @@ static ssize_t altr_edac_a10_device_trig(struct file *file, return count; } -static irqreturn_t altr_edac_a10_irq_handler(int irq, void *dev_id) +static void altr_edac_a10_irq_handler(struct irq_desc *desc) { - irqreturn_t rc = IRQ_NONE; - struct altr_arria10_edac *edac = dev_id; - struct altr_edac_device_dev *dci; - int irq_status; - bool sberr = (irq == edac->sb_irq) ? 1 : 0; - int sm_offset = sberr ? A10_SYSMGR_ECC_INTSTAT_SERR_OFST : - A10_SYSMGR_ECC_INTSTAT_DERR_OFST; + int dberr, bit, sm_offset, irq_status; + struct altr_arria10_edac *edac = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + int irq = irq_desc_get_irq(desc); + + dberr = (irq == edac->db_irq) ? 1 : 0; + sm_offset = dberr ? A10_SYSMGR_ECC_INTSTAT_DERR_OFST : + A10_SYSMGR_ECC_INTSTAT_SERR_OFST; + + chained_irq_enter(chip, desc); regmap_read(edac->ecc_mgr_map, sm_offset, &irq_status); - if ((irq != edac->sb_irq) && (irq != edac->db_irq)) { - WARN_ON(1); - } else { - list_for_each_entry(dci, &edac->a10_ecc_devices, next) { - if (irq_status & dci->data->irq_status_mask) - rc = dci->data->ecc_irq_handler(dci, sberr); - } + for_each_set_bit(bit, (unsigned long *)&irq_status, 32) { + irq = irq_linear_revmap(edac->domain, dberr * 32 + bit); + if (irq) + generic_handle_irq(irq); } - return rc; + chained_irq_exit(chip, desc); +} + +static int validate_parent_available(struct device_node *np) +{ + struct device_node *parent; + int ret = 0; + + /* Ensure parent device is enabled if parent node exists */ + parent = of_parse_phandle(np, "altr,ecc-parent", 0); + if (parent && !of_device_is_available(parent)) + ret = -ENODEV; + + of_node_put(parent); + return ret; } static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, @@ -1111,7 +1385,7 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, const struct edac_device_prv_data *prv; /* Get matching node and check for valid result */ const struct of_device_id *pdev_id = - of_match_node(altr_edac_device_of_match, np); + of_match_node(altr_edac_a10_device_of_match, np); if (IS_ERR_OR_NULL(pdev_id)) return -ENODEV; @@ -1120,6 +1394,9 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, if (IS_ERR_OR_NULL(prv)) return -ENODEV; + if (validate_parent_available(np)) + return -ENODEV; + if (!devres_open_group(edac->dev, altr_edac_a10_device_add, GFP_KERNEL)) return -ENOMEM; @@ -1168,6 +1445,34 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } + altdev->sb_irq = irq_of_parse_and_map(np, 0); + if (!altdev->sb_irq) { + edac_printk(KERN_ERR, EDAC_DEVICE, "Error allocating SBIRQ\n"); + rc = -ENODEV; + goto err_release_group1; + } + rc = devm_request_irq(edac->dev, altdev->sb_irq, + prv->ecc_irq_handler, + IRQF_SHARED, ecc_name, altdev); + if (rc) { + edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); + goto err_release_group1; + } + + altdev->db_irq = irq_of_parse_and_map(np, 1); + if (!altdev->db_irq) { + edac_printk(KERN_ERR, EDAC_DEVICE, "Error allocating DBIRQ\n"); + rc = -ENODEV; + goto err_release_group1; + } + rc = devm_request_irq(edac->dev, altdev->db_irq, + prv->ecc_irq_handler, + IRQF_SHARED, ecc_name, altdev); + if (rc) { + edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); + goto err_release_group1; + } + rc = edac_device_add_device(dci); if (rc) { dev_err(edac->dev, "edac_device_add_device failed\n"); @@ -1186,7 +1491,6 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, err_release_group1: edac_device_free_ctl_info(dci); err_release_group: - edac_printk(KERN_ALERT, EDAC_DEVICE, "%s: %d\n", __func__, __LINE__); devres_release_group(edac->dev, NULL); edac_printk(KERN_ERR, EDAC_DEVICE, "%s:Error setting up EDAC device: %d\n", ecc_name, rc); @@ -1194,11 +1498,43 @@ err_release_group: return rc; } +static void a10_eccmgr_irq_mask(struct irq_data *d) +{ + struct altr_arria10_edac *edac = irq_data_get_irq_chip_data(d); + + regmap_write(edac->ecc_mgr_map, A10_SYSMGR_ECC_INTMASK_SET_OFST, + BIT(d->hwirq)); +} + +static void a10_eccmgr_irq_unmask(struct irq_data *d) +{ + struct altr_arria10_edac *edac = irq_data_get_irq_chip_data(d); + + regmap_write(edac->ecc_mgr_map, A10_SYSMGR_ECC_INTMASK_CLR_OFST, + BIT(d->hwirq)); +} + +static int a10_eccmgr_irqdomain_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) +{ + struct altr_arria10_edac *edac = d->host_data; + + irq_set_chip_and_handler(irq, &edac->irq_chip, handle_simple_irq); + irq_set_chip_data(irq, edac); + irq_set_noprobe(irq); + + return 0; +} + +struct irq_domain_ops a10_eccmgr_ic_ops = { + .map = a10_eccmgr_irqdomain_map, + .xlate = irq_domain_xlate_twocell, +}; + static int altr_edac_a10_probe(struct platform_device *pdev) { struct altr_arria10_edac *edac; struct device_node *child; - int rc; edac = devm_kzalloc(&pdev->dev, sizeof(*edac), GFP_KERNEL); if (!edac) @@ -1216,32 +1552,50 @@ static int altr_edac_a10_probe(struct platform_device *pdev) return PTR_ERR(edac->ecc_mgr_map); } + edac->irq_chip.name = pdev->dev.of_node->name; + edac->irq_chip.irq_mask = a10_eccmgr_irq_mask; + edac->irq_chip.irq_unmask = a10_eccmgr_irq_unmask; + edac->domain = irq_domain_add_linear(pdev->dev.of_node, 64, + &a10_eccmgr_ic_ops, edac); + if (!edac->domain) { + dev_err(&pdev->dev, "Error adding IRQ domain\n"); + return -ENOMEM; + } + edac->sb_irq = platform_get_irq(pdev, 0); - rc = devm_request_irq(&pdev->dev, edac->sb_irq, - altr_edac_a10_irq_handler, - IRQF_SHARED, dev_name(&pdev->dev), edac); - if (rc) { - edac_printk(KERN_ERR, EDAC_DEVICE, "No SBERR IRQ resource\n"); - return rc; + if (edac->sb_irq < 0) { + dev_err(&pdev->dev, "No SBERR IRQ resource\n"); + return edac->sb_irq; } + irq_set_chained_handler_and_data(edac->sb_irq, + altr_edac_a10_irq_handler, + edac); + edac->db_irq = platform_get_irq(pdev, 1); - rc = devm_request_irq(&pdev->dev, edac->db_irq, - altr_edac_a10_irq_handler, - IRQF_SHARED, dev_name(&pdev->dev), edac); - if (rc) { - edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); - return rc; + if (edac->db_irq < 0) { + dev_err(&pdev->dev, "No DBERR IRQ resource\n"); + return edac->db_irq; } + irq_set_chained_handler_and_data(edac->db_irq, + altr_edac_a10_irq_handler, + edac); for_each_child_of_node(pdev->dev.of_node, child) { if (!of_device_is_available(child)) continue; if (of_device_is_compatible(child, "altr,socfpga-a10-l2-ecc")) altr_edac_a10_device_add(edac, child); - else if (of_device_is_compatible(child, - "altr,socfpga-a10-ocram-ecc")) + else if ((of_device_is_compatible(child, + "altr,socfpga-a10-ocram-ecc")) || + (of_device_is_compatible(child, + "altr,socfpga-eth-mac-ecc"))) altr_edac_a10_device_add(edac, child); + else if (of_device_is_compatible(child, + "altr,sdram-edac-a10")) + of_platform_populate(pdev->dev.of_node, + altr_sdram_ctrl_of_match, + NULL, &pdev->dev); } return 0; diff --git a/drivers/edac/altera_edac.h b/drivers/edac/altera_edac.h index 42090f36b..687d8e754 100644 --- a/drivers/edac/altera_edac.h +++ b/drivers/edac/altera_edac.h @@ -230,8 +230,13 @@ struct altr_sdram_mc_data { #define ALTR_A10_ECC_INITCOMPLETEB BIT(8) #define ALTR_A10_ECC_ERRINTEN_OFST 0x10 +#define ALTR_A10_ECC_ERRINTENS_OFST 0x14 +#define ALTR_A10_ECC_ERRINTENR_OFST 0x18 #define ALTR_A10_ECC_SERRINTEN BIT(0) +#define ALTR_A10_ECC_INTMODE_OFST 0x1C +#define ALTR_A10_ECC_INTMODE BIT(0) + #define ALTR_A10_ECC_INTSTAT_OFST 0x20 #define ALTR_A10_ECC_SERRPENA BIT(0) #define ALTR_A10_ECC_DERRPENA BIT(8) @@ -280,6 +285,12 @@ struct altr_sdram_mc_data { /* Arria 10 OCRAM ECC Management Group Defines */ #define ALTR_A10_OCRAM_ECC_EN_CTL (BIT(1) | BIT(0)) +/* Arria 10 Ethernet ECC Management Group Defines */ +#define ALTR_A10_COMMON_ECC_EN_CTL BIT(0) + +/* A10 ECC Controller memory initialization timeout */ +#define ALTR_A10_ECC_INIT_WATCHDOG_10US 10000 + struct altr_edac_device_dev; struct edac_device_prv_data { @@ -295,10 +306,10 @@ struct edac_device_prv_data { int ce_set_mask; int ue_set_mask; int set_err_ofst; - irqreturn_t (*ecc_irq_handler)(struct altr_edac_device_dev *dci, - bool sb); + irqreturn_t (*ecc_irq_handler)(int irq, void *dev_id); int trig_alloc_sz; const struct file_operations *inject_fops; + bool panic; }; struct altr_edac_device_dev { @@ -320,6 +331,8 @@ struct altr_arria10_edac { struct regmap *ecc_mgr_map; int sb_irq; int db_irq; + struct irq_domain *domain; + struct irq_chip irq_chip; struct list_head a10_ecc_devices; }; diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 46784eb2e..8c0ec2128 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -2966,11 +2966,11 @@ static int __init amd64_edac_init(void) int err = -ENODEV; int i; - opstate_init(); - if (amd_cache_northbridges() < 0) goto err_ret; + opstate_init(); + err = -ENOMEM; ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL); if (!ecc_stngs) diff --git a/drivers/edac/skx_edac.c b/drivers/edac/skx_edac.c new file mode 100644 index 000000000..0ff4878c2 --- /dev/null +++ b/drivers/edac/skx_edac.c @@ -0,0 +1,1121 @@ +/* + * EDAC driver for Intel(R) Xeon(R) Skylake processors + * Copyright (c) 2016, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "edac_core.h" + +#define SKX_REVISION " Ver: 1.0 " + +/* + * Debug macros + */ +#define skx_printk(level, fmt, arg...) \ + edac_printk(level, "skx", fmt, ##arg) + +#define skx_mc_printk(mci, level, fmt, arg...) \ + edac_mc_chipset_printk(mci, level, "skx", fmt, ##arg) + +/* + * Get a bit field at register value , from bit to bit + */ +#define GET_BITFIELD(v, lo, hi) \ + (((v) & GENMASK_ULL((hi), (lo))) >> (lo)) + +static LIST_HEAD(skx_edac_list); + +static u64 skx_tolm, skx_tohm; + +#define NUM_IMC 2 /* memory controllers per socket */ +#define NUM_CHANNELS 3 /* channels per memory controller */ +#define NUM_DIMMS 2 /* Max DIMMS per channel */ + +#define MASK26 0x3FFFFFF /* Mask for 2^26 */ +#define MASK29 0x1FFFFFFF /* Mask for 2^29 */ + +/* + * Each cpu socket contains some pci devices that provide global + * information, and also some that are local to each of the two + * memory controllers on the die. + */ +struct skx_dev { + struct list_head list; + u8 bus[4]; + struct pci_dev *sad_all; + struct pci_dev *util_all; + u32 mcroute; + struct skx_imc { + struct mem_ctl_info *mci; + u8 mc; /* system wide mc# */ + u8 lmc; /* socket relative mc# */ + u8 src_id, node_id; + struct skx_channel { + struct pci_dev *cdev; + struct skx_dimm { + u8 close_pg; + u8 bank_xor_enable; + u8 fine_grain_bank; + u8 rowbits; + u8 colbits; + } dimms[NUM_DIMMS]; + } chan[NUM_CHANNELS]; + } imc[NUM_IMC]; +}; +static int skx_num_sockets; + +struct skx_pvt { + struct skx_imc *imc; +}; + +struct decoded_addr { + struct skx_dev *dev; + u64 addr; + int socket; + int imc; + int channel; + u64 chan_addr; + int sktways; + int chanways; + int dimm; + int rank; + int channel_rank; + u64 rank_address; + int row; + int column; + int bank_address; + int bank_group; +}; + +static struct skx_dev *get_skx_dev(u8 bus, u8 idx) +{ + struct skx_dev *d; + + list_for_each_entry(d, &skx_edac_list, list) { + if (d->bus[idx] == bus) + return d; + } + + return NULL; +} + +enum munittype { + CHAN0, CHAN1, CHAN2, SAD_ALL, UTIL_ALL, SAD +}; + +struct munit { + u16 did; + u16 devfn[NUM_IMC]; + u8 busidx; + u8 per_socket; + enum munittype mtype; +}; + +/* + * List of PCI device ids that we need together with some device + * number and function numbers to tell which memory controller the + * device belongs to. + */ +static const struct munit skx_all_munits[] = { + { 0x2054, { }, 1, 1, SAD_ALL }, + { 0x2055, { }, 1, 1, UTIL_ALL }, + { 0x2040, { PCI_DEVFN(10, 0), PCI_DEVFN(12, 0) }, 2, 2, CHAN0 }, + { 0x2044, { PCI_DEVFN(10, 4), PCI_DEVFN(12, 4) }, 2, 2, CHAN1 }, + { 0x2048, { PCI_DEVFN(11, 0), PCI_DEVFN(13, 0) }, 2, 2, CHAN2 }, + { 0x208e, { }, 1, 0, SAD }, + { } +}; + +/* + * We use the per-socket device 0x2016 to count how many sockets are present, + * and to detemine which PCI buses are associated with each socket. Allocate + * and build the full list of all the skx_dev structures that we need here. + */ +static int get_all_bus_mappings(void) +{ + struct pci_dev *pdev, *prev; + struct skx_dev *d; + u32 reg; + int ndev = 0; + + prev = NULL; + for (;;) { + pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x2016, prev); + if (!pdev) + break; + ndev++; + d = kzalloc(sizeof(*d), GFP_KERNEL); + if (!d) { + pci_dev_put(pdev); + return -ENOMEM; + } + pci_read_config_dword(pdev, 0xCC, ®); + d->bus[0] = GET_BITFIELD(reg, 0, 7); + d->bus[1] = GET_BITFIELD(reg, 8, 15); + d->bus[2] = GET_BITFIELD(reg, 16, 23); + d->bus[3] = GET_BITFIELD(reg, 24, 31); + edac_dbg(2, "busses: %x, %x, %x, %x\n", + d->bus[0], d->bus[1], d->bus[2], d->bus[3]); + list_add_tail(&d->list, &skx_edac_list); + skx_num_sockets++; + prev = pdev; + } + + return ndev; +} + +static int get_all_munits(const struct munit *m) +{ + struct pci_dev *pdev, *prev; + struct skx_dev *d; + u32 reg; + int i = 0, ndev = 0; + + prev = NULL; + for (;;) { + pdev = pci_get_device(PCI_VENDOR_ID_INTEL, m->did, prev); + if (!pdev) + break; + ndev++; + if (m->per_socket == NUM_IMC) { + for (i = 0; i < NUM_IMC; i++) + if (m->devfn[i] == pdev->devfn) + break; + if (i == NUM_IMC) + goto fail; + } + d = get_skx_dev(pdev->bus->number, m->busidx); + if (!d) + goto fail; + + /* Be sure that the device is enabled */ + if (unlikely(pci_enable_device(pdev) < 0)) { + skx_printk(KERN_ERR, + "Couldn't enable %04x:%04x\n", PCI_VENDOR_ID_INTEL, m->did); + goto fail; + } + + switch (m->mtype) { + case CHAN0: case CHAN1: case CHAN2: + pci_dev_get(pdev); + d->imc[i].chan[m->mtype].cdev = pdev; + break; + case SAD_ALL: + pci_dev_get(pdev); + d->sad_all = pdev; + break; + case UTIL_ALL: + pci_dev_get(pdev); + d->util_all = pdev; + break; + case SAD: + /* + * one of these devices per core, including cores + * that don't exist on this SKU. Ignore any that + * read a route table of zero, make sure all the + * non-zero values match. + */ + pci_read_config_dword(pdev, 0xB4, ®); + if (reg != 0) { + if (d->mcroute == 0) + d->mcroute = reg; + else if (d->mcroute != reg) { + skx_printk(KERN_ERR, + "mcroute mismatch\n"); + goto fail; + } + } + ndev--; + break; + } + + prev = pdev; + } + + return ndev; +fail: + pci_dev_put(pdev); + return -ENODEV; +} + +const struct x86_cpu_id skx_cpuids[] = { + { X86_VENDOR_INTEL, 6, 0x55, 0, 0 }, /* Skylake */ + { } +}; +MODULE_DEVICE_TABLE(x86cpu, skx_cpuids); + +static u8 get_src_id(struct skx_dev *d) +{ + u32 reg; + + pci_read_config_dword(d->util_all, 0xF0, ®); + + return GET_BITFIELD(reg, 12, 14); +} + +static u8 skx_get_node_id(struct skx_dev *d) +{ + u32 reg; + + pci_read_config_dword(d->util_all, 0xF4, ®); + + return GET_BITFIELD(reg, 0, 2); +} + +static int get_dimm_attr(u32 reg, int lobit, int hibit, int add, int minval, + int maxval, char *name) +{ + u32 val = GET_BITFIELD(reg, lobit, hibit); + + if (val < minval || val > maxval) { + edac_dbg(2, "bad %s = %d (raw=%x)\n", name, val, reg); + return -EINVAL; + } + return val + add; +} + +#define IS_DIMM_PRESENT(mtr) GET_BITFIELD((mtr), 15, 15) + +#define numrank(reg) get_dimm_attr((reg), 12, 13, 0, 1, 2, "ranks") +#define numrow(reg) get_dimm_attr((reg), 2, 4, 12, 1, 6, "rows") +#define numcol(reg) get_dimm_attr((reg), 0, 1, 10, 0, 2, "cols") + +static int get_width(u32 mtr) +{ + switch (GET_BITFIELD(mtr, 8, 9)) { + case 0: + return DEV_X4; + case 1: + return DEV_X8; + case 2: + return DEV_X16; + } + return DEV_UNKNOWN; +} + +static int skx_get_hi_lo(void) +{ + struct pci_dev *pdev; + u32 reg; + + pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x2034, NULL); + if (!pdev) { + edac_dbg(0, "Can't get tolm/tohm\n"); + return -ENODEV; + } + + pci_read_config_dword(pdev, 0xD0, ®); + skx_tolm = reg; + pci_read_config_dword(pdev, 0xD4, ®); + skx_tohm = reg; + pci_read_config_dword(pdev, 0xD8, ®); + skx_tohm |= (u64)reg << 32; + + pci_dev_put(pdev); + edac_dbg(2, "tolm=%llx tohm=%llx\n", skx_tolm, skx_tohm); + + return 0; +} + +static int get_dimm_info(u32 mtr, u32 amap, struct dimm_info *dimm, + struct skx_imc *imc, int chan, int dimmno) +{ + int banks = 16, ranks, rows, cols, npages; + u64 size; + + if (!IS_DIMM_PRESENT(mtr)) + return 0; + ranks = numrank(mtr); + rows = numrow(mtr); + cols = numcol(mtr); + + /* + * Compute size in 8-byte (2^3) words, then shift to MiB (2^20) + */ + size = ((1ull << (rows + cols + ranks)) * banks) >> (20 - 3); + npages = MiB_TO_PAGES(size); + + edac_dbg(0, "mc#%d: channel %d, dimm %d, %lld Mb (%d pages) bank: %d, rank: %d, row: %#x, col: %#x\n", + imc->mc, chan, dimmno, size, npages, + banks, ranks, rows, cols); + + imc->chan[chan].dimms[dimmno].close_pg = GET_BITFIELD(mtr, 0, 0); + imc->chan[chan].dimms[dimmno].bank_xor_enable = GET_BITFIELD(mtr, 9, 9); + imc->chan[chan].dimms[dimmno].fine_grain_bank = GET_BITFIELD(amap, 0, 0); + imc->chan[chan].dimms[dimmno].rowbits = rows; + imc->chan[chan].dimms[dimmno].colbits = cols; + + dimm->nr_pages = npages; + dimm->grain = 32; + dimm->dtype = get_width(mtr); + dimm->mtype = MEM_DDR4; + dimm->edac_mode = EDAC_SECDED; /* likely better than this */ + snprintf(dimm->label, sizeof(dimm->label), "CPU_SrcID#%u_MC#%u_Chan#%u_DIMM#%u", + imc->src_id, imc->lmc, chan, dimmno); + + return 1; +} + +#define SKX_GET_MTMTR(dev, reg) \ + pci_read_config_dword((dev), 0x87c, ®) + +static bool skx_check_ecc(struct pci_dev *pdev) +{ + u32 mtmtr; + + SKX_GET_MTMTR(pdev, mtmtr); + + return !!GET_BITFIELD(mtmtr, 2, 2); +} + +static int skx_get_dimm_config(struct mem_ctl_info *mci) +{ + struct skx_pvt *pvt = mci->pvt_info; + struct skx_imc *imc = pvt->imc; + struct dimm_info *dimm; + int i, j; + u32 mtr, amap; + int ndimms; + + for (i = 0; i < NUM_CHANNELS; i++) { + ndimms = 0; + pci_read_config_dword(imc->chan[i].cdev, 0x8C, &amap); + for (j = 0; j < NUM_DIMMS; j++) { + dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, + mci->n_layers, i, j, 0); + pci_read_config_dword(imc->chan[i].cdev, + 0x80 + 4*j, &mtr); + ndimms += get_dimm_info(mtr, amap, dimm, imc, i, j); + } + if (ndimms && !skx_check_ecc(imc->chan[0].cdev)) { + skx_printk(KERN_ERR, "ECC is disabled on imc %d\n", imc->mc); + return -ENODEV; + } + } + + return 0; +} + +static void skx_unregister_mci(struct skx_imc *imc) +{ + struct mem_ctl_info *mci = imc->mci; + + if (!mci) + return; + + edac_dbg(0, "MC%d: mci = %p\n", imc->mc, mci); + + /* Remove MC sysfs nodes */ + edac_mc_del_mc(mci->pdev); + + edac_dbg(1, "%s: free mci struct\n", mci->ctl_name); + kfree(mci->ctl_name); + edac_mc_free(mci); +} + +static int skx_register_mci(struct skx_imc *imc) +{ + struct mem_ctl_info *mci; + struct edac_mc_layer layers[2]; + struct pci_dev *pdev = imc->chan[0].cdev; + struct skx_pvt *pvt; + int rc; + + /* allocate a new MC control structure */ + layers[0].type = EDAC_MC_LAYER_CHANNEL; + layers[0].size = NUM_CHANNELS; + layers[0].is_virt_csrow = false; + layers[1].type = EDAC_MC_LAYER_SLOT; + layers[1].size = NUM_DIMMS; + layers[1].is_virt_csrow = true; + mci = edac_mc_alloc(imc->mc, ARRAY_SIZE(layers), layers, + sizeof(struct skx_pvt)); + + if (unlikely(!mci)) + return -ENOMEM; + + edac_dbg(0, "MC#%d: mci = %p\n", imc->mc, mci); + + /* Associate skx_dev and mci for future usage */ + imc->mci = mci; + pvt = mci->pvt_info; + pvt->imc = imc; + + mci->ctl_name = kasprintf(GFP_KERNEL, "Skylake Socket#%d IMC#%d", + imc->node_id, imc->lmc); + mci->mtype_cap = MEM_FLAG_DDR4; + mci->edac_ctl_cap = EDAC_FLAG_NONE; + mci->edac_cap = EDAC_FLAG_NONE; + mci->mod_name = "skx_edac.c"; + mci->dev_name = pci_name(imc->chan[0].cdev); + mci->mod_ver = SKX_REVISION; + mci->ctl_page_to_phys = NULL; + + rc = skx_get_dimm_config(mci); + if (rc < 0) + goto fail; + + /* record ptr to the generic device */ + mci->pdev = &pdev->dev; + + /* add this new MC control structure to EDAC's list of MCs */ + if (unlikely(edac_mc_add_mc(mci))) { + edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); + rc = -EINVAL; + goto fail; + } + + return 0; + +fail: + kfree(mci->ctl_name); + edac_mc_free(mci); + imc->mci = NULL; + return rc; +} + +#define SKX_MAX_SAD 24 + +#define SKX_GET_SAD(d, i, reg) \ + pci_read_config_dword((d)->sad_all, 0x60 + 8 * (i), ®) +#define SKX_GET_ILV(d, i, reg) \ + pci_read_config_dword((d)->sad_all, 0x64 + 8 * (i), ®) + +#define SKX_SAD_MOD3MODE(sad) GET_BITFIELD((sad), 30, 31) +#define SKX_SAD_MOD3(sad) GET_BITFIELD((sad), 27, 27) +#define SKX_SAD_LIMIT(sad) (((u64)GET_BITFIELD((sad), 7, 26) << 26) | MASK26) +#define SKX_SAD_MOD3ASMOD2(sad) GET_BITFIELD((sad), 5, 6) +#define SKX_SAD_ATTR(sad) GET_BITFIELD((sad), 3, 4) +#define SKX_SAD_INTERLEAVE(sad) GET_BITFIELD((sad), 1, 2) +#define SKX_SAD_ENABLE(sad) GET_BITFIELD((sad), 0, 0) + +#define SKX_ILV_REMOTE(tgt) (((tgt) & 8) == 0) +#define SKX_ILV_TARGET(tgt) ((tgt) & 7) + +static bool skx_sad_decode(struct decoded_addr *res) +{ + struct skx_dev *d = list_first_entry(&skx_edac_list, typeof(*d), list); + u64 addr = res->addr; + int i, idx, tgt, lchan, shift; + u32 sad, ilv; + u64 limit, prev_limit; + int remote = 0; + + /* Simple sanity check for I/O space or out of range */ + if (addr >= skx_tohm || (addr >= skx_tolm && addr < BIT_ULL(32))) { + edac_dbg(0, "Address %llx out of range\n", addr); + return false; + } + +restart: + prev_limit = 0; + for (i = 0; i < SKX_MAX_SAD; i++) { + SKX_GET_SAD(d, i, sad); + limit = SKX_SAD_LIMIT(sad); + if (SKX_SAD_ENABLE(sad)) { + if (addr >= prev_limit && addr <= limit) + goto sad_found; + } + prev_limit = limit + 1; + } + edac_dbg(0, "No SAD entry for %llx\n", addr); + return false; + +sad_found: + SKX_GET_ILV(d, i, ilv); + + switch (SKX_SAD_INTERLEAVE(sad)) { + case 0: + idx = GET_BITFIELD(addr, 6, 8); + break; + case 1: + idx = GET_BITFIELD(addr, 8, 10); + break; + case 2: + idx = GET_BITFIELD(addr, 12, 14); + break; + case 3: + idx = GET_BITFIELD(addr, 30, 32); + break; + } + + tgt = GET_BITFIELD(ilv, 4 * idx, 4 * idx + 3); + + /* If point to another node, find it and start over */ + if (SKX_ILV_REMOTE(tgt)) { + if (remote) { + edac_dbg(0, "Double remote!\n"); + return false; + } + remote = 1; + list_for_each_entry(d, &skx_edac_list, list) { + if (d->imc[0].src_id == SKX_ILV_TARGET(tgt)) + goto restart; + } + edac_dbg(0, "Can't find node %d\n", SKX_ILV_TARGET(tgt)); + return false; + } + + if (SKX_SAD_MOD3(sad) == 0) + lchan = SKX_ILV_TARGET(tgt); + else { + switch (SKX_SAD_MOD3MODE(sad)) { + case 0: + shift = 6; + break; + case 1: + shift = 8; + break; + case 2: + shift = 12; + break; + default: + edac_dbg(0, "illegal mod3mode\n"); + return false; + } + switch (SKX_SAD_MOD3ASMOD2(sad)) { + case 0: + lchan = (addr >> shift) % 3; + break; + case 1: + lchan = (addr >> shift) % 2; + break; + case 2: + lchan = (addr >> shift) % 2; + lchan = (lchan << 1) | ~lchan; + break; + case 3: + lchan = ((addr >> shift) % 2) << 1; + break; + } + lchan = (lchan << 1) | (SKX_ILV_TARGET(tgt) & 1); + } + + res->dev = d; + res->socket = d->imc[0].src_id; + res->imc = GET_BITFIELD(d->mcroute, lchan * 3, lchan * 3 + 2); + res->channel = GET_BITFIELD(d->mcroute, lchan * 2 + 18, lchan * 2 + 19); + + edac_dbg(2, "%llx: socket=%d imc=%d channel=%d\n", + res->addr, res->socket, res->imc, res->channel); + return true; +} + +#define SKX_MAX_TAD 8 + +#define SKX_GET_TADBASE(d, mc, i, reg) \ + pci_read_config_dword((d)->imc[mc].chan[0].cdev, 0x850 + 4 * (i), ®) +#define SKX_GET_TADWAYNESS(d, mc, i, reg) \ + pci_read_config_dword((d)->imc[mc].chan[0].cdev, 0x880 + 4 * (i), ®) +#define SKX_GET_TADCHNILVOFFSET(d, mc, ch, i, reg) \ + pci_read_config_dword((d)->imc[mc].chan[ch].cdev, 0x90 + 4 * (i), ®) + +#define SKX_TAD_BASE(b) ((u64)GET_BITFIELD((b), 12, 31) << 26) +#define SKX_TAD_SKT_GRAN(b) GET_BITFIELD((b), 4, 5) +#define SKX_TAD_CHN_GRAN(b) GET_BITFIELD((b), 6, 7) +#define SKX_TAD_LIMIT(b) (((u64)GET_BITFIELD((b), 12, 31) << 26) | MASK26) +#define SKX_TAD_OFFSET(b) ((u64)GET_BITFIELD((b), 4, 23) << 26) +#define SKX_TAD_SKTWAYS(b) (1 << GET_BITFIELD((b), 10, 11)) +#define SKX_TAD_CHNWAYS(b) (GET_BITFIELD((b), 8, 9) + 1) + +/* which bit used for both socket and channel interleave */ +static int skx_granularity[] = { 6, 8, 12, 30 }; + +static u64 skx_do_interleave(u64 addr, int shift, int ways, u64 lowbits) +{ + addr >>= shift; + addr /= ways; + addr <<= shift; + + return addr | (lowbits & ((1ull << shift) - 1)); +} + +static bool skx_tad_decode(struct decoded_addr *res) +{ + int i; + u32 base, wayness, chnilvoffset; + int skt_interleave_bit, chn_interleave_bit; + u64 channel_addr; + + for (i = 0; i < SKX_MAX_TAD; i++) { + SKX_GET_TADBASE(res->dev, res->imc, i, base); + SKX_GET_TADWAYNESS(res->dev, res->imc, i, wayness); + if (SKX_TAD_BASE(base) <= res->addr && res->addr <= SKX_TAD_LIMIT(wayness)) + goto tad_found; + } + edac_dbg(0, "No TAD entry for %llx\n", res->addr); + return false; + +tad_found: + res->sktways = SKX_TAD_SKTWAYS(wayness); + res->chanways = SKX_TAD_CHNWAYS(wayness); + skt_interleave_bit = skx_granularity[SKX_TAD_SKT_GRAN(base)]; + chn_interleave_bit = skx_granularity[SKX_TAD_CHN_GRAN(base)]; + + SKX_GET_TADCHNILVOFFSET(res->dev, res->imc, res->channel, i, chnilvoffset); + channel_addr = res->addr - SKX_TAD_OFFSET(chnilvoffset); + + if (res->chanways == 3 && skt_interleave_bit > chn_interleave_bit) { + /* Must handle channel first, then socket */ + channel_addr = skx_do_interleave(channel_addr, chn_interleave_bit, + res->chanways, channel_addr); + channel_addr = skx_do_interleave(channel_addr, skt_interleave_bit, + res->sktways, channel_addr); + } else { + /* Handle socket then channel. Preserve low bits from original address */ + channel_addr = skx_do_interleave(channel_addr, skt_interleave_bit, + res->sktways, res->addr); + channel_addr = skx_do_interleave(channel_addr, chn_interleave_bit, + res->chanways, res->addr); + } + + res->chan_addr = channel_addr; + + edac_dbg(2, "%llx: chan_addr=%llx sktways=%d chanways=%d\n", + res->addr, res->chan_addr, res->sktways, res->chanways); + return true; +} + +#define SKX_MAX_RIR 4 + +#define SKX_GET_RIRWAYNESS(d, mc, ch, i, reg) \ + pci_read_config_dword((d)->imc[mc].chan[ch].cdev, \ + 0x108 + 4 * (i), ®) +#define SKX_GET_RIRILV(d, mc, ch, idx, i, reg) \ + pci_read_config_dword((d)->imc[mc].chan[ch].cdev, \ + 0x120 + 16 * idx + 4 * (i), ®) + +#define SKX_RIR_VALID(b) GET_BITFIELD((b), 31, 31) +#define SKX_RIR_LIMIT(b) (((u64)GET_BITFIELD((b), 1, 11) << 29) | MASK29) +#define SKX_RIR_WAYS(b) (1 << GET_BITFIELD((b), 28, 29)) +#define SKX_RIR_CHAN_RANK(b) GET_BITFIELD((b), 16, 19) +#define SKX_RIR_OFFSET(b) ((u64)(GET_BITFIELD((b), 2, 15) << 26)) + +static bool skx_rir_decode(struct decoded_addr *res) +{ + int i, idx, chan_rank; + int shift; + u32 rirway, rirlv; + u64 rank_addr, prev_limit = 0, limit; + + if (res->dev->imc[res->imc].chan[res->channel].dimms[0].close_pg) + shift = 6; + else + shift = 13; + + for (i = 0; i < SKX_MAX_RIR; i++) { + SKX_GET_RIRWAYNESS(res->dev, res->imc, res->channel, i, rirway); + limit = SKX_RIR_LIMIT(rirway); + if (SKX_RIR_VALID(rirway)) { + if (prev_limit <= res->chan_addr && + res->chan_addr <= limit) + goto rir_found; + } + prev_limit = limit; + } + edac_dbg(0, "No RIR entry for %llx\n", res->addr); + return false; + +rir_found: + rank_addr = res->chan_addr >> shift; + rank_addr /= SKX_RIR_WAYS(rirway); + rank_addr <<= shift; + rank_addr |= res->chan_addr & GENMASK_ULL(shift - 1, 0); + + res->rank_address = rank_addr; + idx = (res->chan_addr >> shift) % SKX_RIR_WAYS(rirway); + + SKX_GET_RIRILV(res->dev, res->imc, res->channel, idx, i, rirlv); + res->rank_address = rank_addr - SKX_RIR_OFFSET(rirlv); + chan_rank = SKX_RIR_CHAN_RANK(rirlv); + res->channel_rank = chan_rank; + res->dimm = chan_rank / 4; + res->rank = chan_rank % 4; + + edac_dbg(2, "%llx: dimm=%d rank=%d chan_rank=%d rank_addr=%llx\n", + res->addr, res->dimm, res->rank, + res->channel_rank, res->rank_address); + return true; +} + +static u8 skx_close_row[] = { + 15, 16, 17, 18, 20, 21, 22, 28, 10, 11, 12, 13, 29, 30, 31, 32, 33 +}; +static u8 skx_close_column[] = { + 3, 4, 5, 14, 19, 23, 24, 25, 26, 27 +}; +static u8 skx_open_row[] = { + 14, 15, 16, 20, 28, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31, 32, 33 +}; +static u8 skx_open_column[] = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +}; +static u8 skx_open_fine_column[] = { + 3, 4, 5, 7, 8, 9, 10, 11, 12, 13 +}; + +static int skx_bits(u64 addr, int nbits, u8 *bits) +{ + int i, res = 0; + + for (i = 0; i < nbits; i++) + res |= ((addr >> bits[i]) & 1) << i; + return res; +} + +static int skx_bank_bits(u64 addr, int b0, int b1, int do_xor, int x0, int x1) +{ + int ret = GET_BITFIELD(addr, b0, b0) | (GET_BITFIELD(addr, b1, b1) << 1); + + if (do_xor) + ret ^= GET_BITFIELD(addr, x0, x0) | (GET_BITFIELD(addr, x1, x1) << 1); + + return ret; +} + +static bool skx_mad_decode(struct decoded_addr *r) +{ + struct skx_dimm *dimm = &r->dev->imc[r->imc].chan[r->channel].dimms[r->dimm]; + int bg0 = dimm->fine_grain_bank ? 6 : 13; + + if (dimm->close_pg) { + r->row = skx_bits(r->rank_address, dimm->rowbits, skx_close_row); + r->column = skx_bits(r->rank_address, dimm->colbits, skx_close_column); + r->column |= 0x400; /* C10 is autoprecharge, always set */ + r->bank_address = skx_bank_bits(r->rank_address, 8, 9, dimm->bank_xor_enable, 22, 28); + r->bank_group = skx_bank_bits(r->rank_address, 6, 7, dimm->bank_xor_enable, 20, 21); + } else { + r->row = skx_bits(r->rank_address, dimm->rowbits, skx_open_row); + if (dimm->fine_grain_bank) + r->column = skx_bits(r->rank_address, dimm->colbits, skx_open_fine_column); + else + r->column = skx_bits(r->rank_address, dimm->colbits, skx_open_column); + r->bank_address = skx_bank_bits(r->rank_address, 18, 19, dimm->bank_xor_enable, 22, 23); + r->bank_group = skx_bank_bits(r->rank_address, bg0, 17, dimm->bank_xor_enable, 20, 21); + } + r->row &= (1u << dimm->rowbits) - 1; + + edac_dbg(2, "%llx: row=%x col=%x bank_addr=%d bank_group=%d\n", + r->addr, r->row, r->column, r->bank_address, + r->bank_group); + return true; +} + +static bool skx_decode(struct decoded_addr *res) +{ + + return skx_sad_decode(res) && skx_tad_decode(res) && + skx_rir_decode(res) && skx_mad_decode(res); +} + +#ifdef CONFIG_EDAC_DEBUG +/* + * Debug feature. Make /sys/kernel/debug/skx_edac_test/addr. + * Write an address to this file to exercise the address decode + * logic in this driver. + */ +static struct dentry *skx_test; +static u64 skx_fake_addr; + +static int debugfs_u64_set(void *data, u64 val) +{ + struct decoded_addr res; + + res.addr = val; + skx_decode(&res); + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); + +static struct dentry *mydebugfs_create(const char *name, umode_t mode, + struct dentry *parent, u64 *value) +{ + return debugfs_create_file(name, mode, parent, value, &fops_u64_wo); +} + +static void setup_skx_debug(void) +{ + skx_test = debugfs_create_dir("skx_edac_test", NULL); + mydebugfs_create("addr", S_IWUSR, skx_test, &skx_fake_addr); +} + +static void teardown_skx_debug(void) +{ + debugfs_remove_recursive(skx_test); +} +#else +static void setup_skx_debug(void) +{ +} + +static void teardown_skx_debug(void) +{ +} +#endif /*CONFIG_EDAC_DEBUG*/ + +static void skx_mce_output_error(struct mem_ctl_info *mci, + const struct mce *m, + struct decoded_addr *res) +{ + enum hw_event_mc_err_type tp_event; + char *type, *optype, msg[256]; + bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0); + bool overflow = GET_BITFIELD(m->status, 62, 62); + bool uncorrected_error = GET_BITFIELD(m->status, 61, 61); + bool recoverable; + u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52); + u32 mscod = GET_BITFIELD(m->status, 16, 31); + u32 errcode = GET_BITFIELD(m->status, 0, 15); + u32 optypenum = GET_BITFIELD(m->status, 4, 6); + + recoverable = GET_BITFIELD(m->status, 56, 56); + + if (uncorrected_error) { + if (ripv) { + type = "FATAL"; + tp_event = HW_EVENT_ERR_FATAL; + } else { + type = "NON_FATAL"; + tp_event = HW_EVENT_ERR_UNCORRECTED; + } + } else { + type = "CORRECTED"; + tp_event = HW_EVENT_ERR_CORRECTED; + } + + /* + * According with Table 15-9 of the Intel Architecture spec vol 3A, + * memory errors should fit in this mask: + * 000f 0000 1mmm cccc (binary) + * where: + * f = Correction Report Filtering Bit. If 1, subsequent errors + * won't be shown + * mmm = error type + * cccc = channel + * If the mask doesn't match, report an error to the parsing logic + */ + if (!((errcode & 0xef80) == 0x80)) { + optype = "Can't parse: it is not a mem"; + } else { + switch (optypenum) { + case 0: + optype = "generic undef request error"; + break; + case 1: + optype = "memory read error"; + break; + case 2: + optype = "memory write error"; + break; + case 3: + optype = "addr/cmd error"; + break; + case 4: + optype = "memory scrubbing error"; + break; + default: + optype = "reserved"; + break; + } + } + + snprintf(msg, sizeof(msg), + "%s%s err_code:%04x:%04x socket:%d imc:%d rank:%d bg:%d ba:%d row:%x col:%x", + overflow ? " OVERFLOW" : "", + (uncorrected_error && recoverable) ? " recoverable" : "", + mscod, errcode, + res->socket, res->imc, res->rank, + res->bank_group, res->bank_address, res->row, res->column); + + edac_dbg(0, "%s\n", msg); + + /* Call the helper to output message */ + edac_mc_handle_error(tp_event, mci, core_err_cnt, + m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0, + res->channel, res->dimm, -1, + optype, msg); +} + +static int skx_mce_check_error(struct notifier_block *nb, unsigned long val, + void *data) +{ + struct mce *mce = (struct mce *)data; + struct decoded_addr res; + struct mem_ctl_info *mci; + char *type; + + if (get_edac_report_status() == EDAC_REPORTING_DISABLED) + return NOTIFY_DONE; + + /* ignore unless this is memory related with an address */ + if ((mce->status & 0xefff) >> 7 != 1 || !(mce->status & MCI_STATUS_ADDRV)) + return NOTIFY_DONE; + + res.addr = mce->addr; + if (!skx_decode(&res)) + return NOTIFY_DONE; + mci = res.dev->imc[res.imc].mci; + + if (mce->mcgstatus & MCG_STATUS_MCIP) + type = "Exception"; + else + type = "Event"; + + skx_mc_printk(mci, KERN_DEBUG, "HANDLING MCE MEMORY ERROR\n"); + + skx_mc_printk(mci, KERN_DEBUG, "CPU %d: Machine Check %s: %Lx " + "Bank %d: %016Lx\n", mce->extcpu, type, + mce->mcgstatus, mce->bank, mce->status); + skx_mc_printk(mci, KERN_DEBUG, "TSC %llx ", mce->tsc); + skx_mc_printk(mci, KERN_DEBUG, "ADDR %llx ", mce->addr); + skx_mc_printk(mci, KERN_DEBUG, "MISC %llx ", mce->misc); + + skx_mc_printk(mci, KERN_DEBUG, "PROCESSOR %u:%x TIME %llu SOCKET " + "%u APIC %x\n", mce->cpuvendor, mce->cpuid, + mce->time, mce->socketid, mce->apicid); + + skx_mce_output_error(mci, mce, &res); + + return NOTIFY_DONE; +} + +static struct notifier_block skx_mce_dec = { + .notifier_call = skx_mce_check_error, +}; + +static void skx_remove(void) +{ + int i, j; + struct skx_dev *d, *tmp; + + edac_dbg(0, "\n"); + + list_for_each_entry_safe(d, tmp, &skx_edac_list, list) { + list_del(&d->list); + for (i = 0; i < NUM_IMC; i++) { + skx_unregister_mci(&d->imc[i]); + for (j = 0; j < NUM_CHANNELS; j++) + pci_dev_put(d->imc[i].chan[j].cdev); + } + pci_dev_put(d->util_all); + pci_dev_put(d->sad_all); + + kfree(d); + } +} + +/* + * skx_init: + * make sure we are running on the correct cpu model + * search for all the devices we need + * check which DIMMs are present. + */ +int __init skx_init(void) +{ + const struct x86_cpu_id *id; + const struct munit *m; + int rc = 0, i; + u8 mc = 0, src_id, node_id; + struct skx_dev *d; + + edac_dbg(2, "\n"); + + id = x86_match_cpu(skx_cpuids); + if (!id) + return -ENODEV; + + rc = skx_get_hi_lo(); + if (rc) + return rc; + + rc = get_all_bus_mappings(); + if (rc < 0) + goto fail; + if (rc == 0) { + edac_dbg(2, "No memory controllers found\n"); + return -ENODEV; + } + + for (m = skx_all_munits; m->did; m++) { + rc = get_all_munits(m); + if (rc < 0) + goto fail; + if (rc != m->per_socket * skx_num_sockets) { + edac_dbg(2, "Expected %d, got %d of %x\n", + m->per_socket * skx_num_sockets, rc, m->did); + rc = -ENODEV; + goto fail; + } + } + + list_for_each_entry(d, &skx_edac_list, list) { + src_id = get_src_id(d); + node_id = skx_get_node_id(d); + edac_dbg(2, "src_id=%d node_id=%d\n", src_id, node_id); + for (i = 0; i < NUM_IMC; i++) { + d->imc[i].mc = mc++; + d->imc[i].lmc = i; + d->imc[i].src_id = src_id; + d->imc[i].node_id = node_id; + rc = skx_register_mci(&d->imc[i]); + if (rc < 0) + goto fail; + } + } + + /* Ensure that the OPSTATE is set correctly for POLL or NMI */ + opstate_init(); + + setup_skx_debug(); + + mce_register_decode_chain(&skx_mce_dec); + + return 0; +fail: + skx_remove(); + return rc; +} + +static void __exit skx_exit(void) +{ + edac_dbg(2, "\n"); + mce_unregister_decode_chain(&skx_mce_dec); + skx_remove(); + teardown_skx_debug(); +} + +module_init(skx_init); +module_exit(skx_exit); + +module_param(edac_op_state, int, 0444); +MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Tony Luck"); +MODULE_DESCRIPTION("MC Driver for Intel Skylake server processors"); diff --git a/drivers/extcon/Makefile b/drivers/extcon/Makefile index 2a0e4f45d..972c813c3 100644 --- a/drivers/extcon/Makefile +++ b/drivers/extcon/Makefile @@ -2,7 +2,8 @@ # Makefile for external connector class (extcon) devices # -obj-$(CONFIG_EXTCON) += extcon.o +obj-$(CONFIG_EXTCON) += extcon-core.o +extcon-core-objs += extcon.o devres.o obj-$(CONFIG_EXTCON_ADC_JACK) += extcon-adc-jack.o obj-$(CONFIG_EXTCON_ARIZONA) += extcon-arizona.o obj-$(CONFIG_EXTCON_AXP288) += extcon-axp288.o diff --git a/drivers/extcon/devres.c b/drivers/extcon/devres.c new file mode 100644 index 000000000..e686acd1c --- /dev/null +++ b/drivers/extcon/devres.c @@ -0,0 +1,216 @@ +/* + * drivers/extcon/devres.c - EXTCON device's resource management + * + * Copyright (C) 2016 Samsung Electronics + * Author: Chanwoo Choi + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include + +static int devm_extcon_dev_match(struct device *dev, void *res, void *data) +{ + struct extcon_dev **r = res; + + if (WARN_ON(!r || !*r)) + return 0; + + return *r == data; +} + +static void devm_extcon_dev_release(struct device *dev, void *res) +{ + extcon_dev_free(*(struct extcon_dev **)res); +} + + +static void devm_extcon_dev_unreg(struct device *dev, void *res) +{ + extcon_dev_unregister(*(struct extcon_dev **)res); +} + +struct extcon_dev_notifier_devres { + struct extcon_dev *edev; + unsigned int id; + struct notifier_block *nb; +}; + +static void devm_extcon_dev_notifier_unreg(struct device *dev, void *res) +{ + struct extcon_dev_notifier_devres *this = res; + + extcon_unregister_notifier(this->edev, this->id, this->nb); +} + +/** + * devm_extcon_dev_allocate - Allocate managed extcon device + * @dev: device owning the extcon device being created + * @supported_cable: Array of supported extcon ending with EXTCON_NONE. + * If supported_cable is NULL, cable name related APIs + * are disabled. + * + * This function manages automatically the memory of extcon device using device + * resource management and simplify the control of freeing the memory of extcon + * device. + * + * Returns the pointer memory of allocated extcon_dev if success + * or ERR_PTR(err) if fail + */ +struct extcon_dev *devm_extcon_dev_allocate(struct device *dev, + const unsigned int *supported_cable) +{ + struct extcon_dev **ptr, *edev; + + ptr = devres_alloc(devm_extcon_dev_release, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return ERR_PTR(-ENOMEM); + + edev = extcon_dev_allocate(supported_cable); + if (IS_ERR(edev)) { + devres_free(ptr); + return edev; + } + + edev->dev.parent = dev; + + *ptr = edev; + devres_add(dev, ptr); + + return edev; +} +EXPORT_SYMBOL_GPL(devm_extcon_dev_allocate); + +/** + * devm_extcon_dev_free() - Resource-managed extcon_dev_unregister() + * @dev: device the extcon belongs to + * @edev: the extcon device to unregister + * + * Free the memory that is allocated with devm_extcon_dev_allocate() + * function. + */ +void devm_extcon_dev_free(struct device *dev, struct extcon_dev *edev) +{ + WARN_ON(devres_release(dev, devm_extcon_dev_release, + devm_extcon_dev_match, edev)); +} +EXPORT_SYMBOL_GPL(devm_extcon_dev_free); + +/** + * devm_extcon_dev_register() - Resource-managed extcon_dev_register() + * @dev: device to allocate extcon device + * @edev: the new extcon device to register + * + * Managed extcon_dev_register() function. If extcon device is attached with + * this function, that extcon device is automatically unregistered on driver + * detach. Internally this function calls extcon_dev_register() function. + * To get more information, refer that function. + * + * If extcon device is registered with this function and the device needs to be + * unregistered separately, devm_extcon_dev_unregister() should be used. + * + * Returns 0 if success or negaive error number if failure. + */ +int devm_extcon_dev_register(struct device *dev, struct extcon_dev *edev) +{ + struct extcon_dev **ptr; + int ret; + + ptr = devres_alloc(devm_extcon_dev_unreg, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + ret = extcon_dev_register(edev); + if (ret) { + devres_free(ptr); + return ret; + } + + *ptr = edev; + devres_add(dev, ptr); + + return 0; +} +EXPORT_SYMBOL_GPL(devm_extcon_dev_register); + +/** + * devm_extcon_dev_unregister() - Resource-managed extcon_dev_unregister() + * @dev: device the extcon belongs to + * @edev: the extcon device to unregister + * + * Unregister extcon device that is registered with devm_extcon_dev_register() + * function. + */ +void devm_extcon_dev_unregister(struct device *dev, struct extcon_dev *edev) +{ + WARN_ON(devres_release(dev, devm_extcon_dev_unreg, + devm_extcon_dev_match, edev)); +} +EXPORT_SYMBOL_GPL(devm_extcon_dev_unregister); + +/** + * devm_extcon_register_notifier() - Resource-managed extcon_register_notifier() + * @dev: device to allocate extcon device + * @edev: the extcon device that has the external connecotr. + * @id: the unique id of each external connector in extcon enumeration. + * @nb: a notifier block to be registered. + * + * This function manages automatically the notifier of extcon device using + * device resource management and simplify the control of unregistering + * the notifier of extcon device. + * + * Note that the second parameter given to the callback of nb (val) is + * "old_state", not the current state. The current state can be retrieved + * by looking at the third pameter (edev pointer)'s state value. + * + * Returns 0 if success or negaive error number if failure. + */ +int devm_extcon_register_notifier(struct device *dev, struct extcon_dev *edev, + unsigned int id, struct notifier_block *nb) +{ + struct extcon_dev_notifier_devres *ptr; + int ret; + + ptr = devres_alloc(devm_extcon_dev_notifier_unreg, sizeof(*ptr), + GFP_KERNEL); + if (!ptr) + return -ENOMEM; + + ret = extcon_register_notifier(edev, id, nb); + if (ret) { + devres_free(ptr); + return ret; + } + + ptr->edev = edev; + ptr->id = id; + ptr->nb = nb; + devres_add(dev, ptr); + + return 0; +} +EXPORT_SYMBOL(devm_extcon_register_notifier); + +/** + * devm_extcon_unregister_notifier() + - Resource-managed extcon_unregister_notifier() + * @dev: device to allocate extcon device + * @edev: the extcon device that has the external connecotr. + * @id: the unique id of each external connector in extcon enumeration. + * @nb: a notifier block to be registered. + */ +void devm_extcon_unregister_notifier(struct device *dev, + struct extcon_dev *edev, unsigned int id, + struct notifier_block *nb) +{ + WARN_ON(devres_release(dev, devm_extcon_dev_notifier_unreg, + devm_extcon_dev_match, edev)); +} +EXPORT_SYMBOL(devm_extcon_unregister_notifier); diff --git a/drivers/extcon/extcon-adc-jack.c b/drivers/extcon/extcon-adc-jack.c index 7fc0ae191..44e48aa78 100644 --- a/drivers/extcon/extcon-adc-jack.c +++ b/drivers/extcon/extcon-adc-jack.c @@ -38,6 +38,7 @@ * @chan: iio channel being queried. */ struct adc_jack_data { + struct device *dev; struct extcon_dev *edev; const unsigned int **cable_names; @@ -49,6 +50,7 @@ struct adc_jack_data { struct delayed_work handler; struct iio_channel *chan; + bool wakeup_source; }; static void adc_jack_handler(struct work_struct *work) @@ -105,6 +107,7 @@ static int adc_jack_probe(struct platform_device *pdev) return -EINVAL; } + data->dev = &pdev->dev; data->edev = devm_extcon_dev_allocate(&pdev->dev, pdata->cable_names); if (IS_ERR(data->edev)) { dev_err(&pdev->dev, "failed to allocate extcon device\n"); @@ -128,6 +131,7 @@ static int adc_jack_probe(struct platform_device *pdev) return PTR_ERR(data->chan); data->handling_delay = msecs_to_jiffies(pdata->handling_delay_ms); + data->wakeup_source = pdata->wakeup_source; INIT_DEFERRABLE_WORK(&data->handler, adc_jack_handler); @@ -151,6 +155,9 @@ static int adc_jack_probe(struct platform_device *pdev) return err; } + if (data->wakeup_source) + device_init_wakeup(&pdev->dev, 1); + return 0; } @@ -165,11 +172,38 @@ static int adc_jack_remove(struct platform_device *pdev) return 0; } +#ifdef CONFIG_PM_SLEEP +static int adc_jack_suspend(struct device *dev) +{ + struct adc_jack_data *data = dev_get_drvdata(dev); + + cancel_delayed_work_sync(&data->handler); + if (device_may_wakeup(data->dev)) + enable_irq_wake(data->irq); + + return 0; +} + +static int adc_jack_resume(struct device *dev) +{ + struct adc_jack_data *data = dev_get_drvdata(dev); + + if (device_may_wakeup(data->dev)) + disable_irq_wake(data->irq); + + return 0; +} +#endif /* CONFIG_PM_SLEEP */ + +static SIMPLE_DEV_PM_OPS(adc_jack_pm_ops, + adc_jack_suspend, adc_jack_resume); + static struct platform_driver adc_jack_driver = { .probe = adc_jack_probe, .remove = adc_jack_remove, .driver = { .name = "adc-jack", + .pm = &adc_jack_pm_ops, }, }; diff --git a/drivers/extcon/extcon-usb-gpio.c b/drivers/extcon/extcon-usb-gpio.c index 2b2fecffb..2512660dc 100644 --- a/drivers/extcon/extcon-usb-gpio.c +++ b/drivers/extcon/extcon-usb-gpio.c @@ -24,8 +24,10 @@ #include #include #include +#include #include #include +#include #define USB_GPIO_DEBOUNCE_MS 20 /* ms */ @@ -91,7 +93,7 @@ static int usb_extcon_probe(struct platform_device *pdev) struct usb_extcon_info *info; int ret; - if (!np) + if (!np && !ACPI_HANDLE(dev)) return -EINVAL; info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL); @@ -141,7 +143,8 @@ static int usb_extcon_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, info); - device_init_wakeup(dev, 1); + device_init_wakeup(dev, true); + dev_pm_set_wake_irq(dev, info->id_irq); /* Perform initial detection */ usb_extcon_detect_cable(&info->wq_detcable.work); @@ -155,6 +158,9 @@ static int usb_extcon_remove(struct platform_device *pdev) cancel_delayed_work_sync(&info->wq_detcable); + dev_pm_clear_wake_irq(&pdev->dev); + device_init_wakeup(&pdev->dev, false); + return 0; } @@ -164,12 +170,6 @@ static int usb_extcon_suspend(struct device *dev) struct usb_extcon_info *info = dev_get_drvdata(dev); int ret = 0; - if (device_may_wakeup(dev)) { - ret = enable_irq_wake(info->id_irq); - if (ret) - return ret; - } - /* * We don't want to process any IRQs after this point * as GPIOs used behind I2C subsystem might not be @@ -185,13 +185,10 @@ static int usb_extcon_resume(struct device *dev) struct usb_extcon_info *info = dev_get_drvdata(dev); int ret = 0; - if (device_may_wakeup(dev)) { - ret = disable_irq_wake(info->id_irq); - if (ret) - return ret; - } - enable_irq(info->id_irq); + if (!device_may_wakeup(dev)) + queue_delayed_work(system_power_efficient_wq, + &info->wq_detcable, 0); return ret; } @@ -206,6 +203,12 @@ static const struct of_device_id usb_extcon_dt_match[] = { }; MODULE_DEVICE_TABLE(of, usb_extcon_dt_match); +static const struct platform_device_id usb_extcon_platform_ids[] = { + { .name = "extcon-usb-gpio", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(platform, usb_extcon_platform_ids); + static struct platform_driver usb_extcon_driver = { .probe = usb_extcon_probe, .remove = usb_extcon_remove, @@ -214,6 +217,7 @@ static struct platform_driver usb_extcon_driver = { .pm = &usb_extcon_pm_ops, .of_match_table = usb_extcon_dt_match, }, + .id_table = usb_extcon_platform_ids, }; module_platform_driver(usb_extcon_driver); diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c index 21a123cad..8682efc0f 100644 --- a/drivers/extcon/extcon.c +++ b/drivers/extcon/extcon.c @@ -77,6 +77,26 @@ static const char *extcon_name[] = { NULL, }; +/** + * struct extcon_cable - An internal data for each cable of extcon device. + * @edev: The extcon device + * @cable_index: Index of this cable in the edev + * @attr_g: Attribute group for the cable + * @attr_name: "name" sysfs entry + * @attr_state: "state" sysfs entry + * @attrs: Array pointing to attr_name and attr_state for attr_g + */ +struct extcon_cable { + struct extcon_dev *edev; + int cable_index; + + struct attribute_group attr_g; + struct device_attribute attr_name; + struct device_attribute attr_state; + + struct attribute *attrs[3]; /* to be fed to attr_g.attrs */ +}; + static struct class *extcon_class; #if defined(CONFIG_ANDROID) static struct class_compat *switch_class; @@ -127,38 +147,6 @@ static int find_cable_index_by_id(struct extcon_dev *edev, const unsigned int id return -EINVAL; } -static int find_cable_id_by_name(struct extcon_dev *edev, const char *name) -{ - int id = -EINVAL; - int i = 0; - - /* Find the id of extcon cable */ - while (extcon_name[i]) { - if (!strncmp(extcon_name[i], name, CABLE_NAME_MAX)) { - id = i; - break; - } - i++; - } - - return id; -} - -static int find_cable_index_by_name(struct extcon_dev *edev, const char *name) -{ - int id; - - if (edev->max_supported == 0) - return -EINVAL; - - /* Find the the number of extcon cable */ - id = find_cable_id_by_name(edev, name); - if (id < 0) - return id; - - return find_cable_index_by_id(edev, id); -} - static bool is_extcon_changed(u32 prev, u32 new, int idx, bool *attached) { if (((prev >> idx) & 0x1) != ((new >> idx) & 0x1)) { @@ -373,25 +361,6 @@ int extcon_get_cable_state_(struct extcon_dev *edev, const unsigned int id) } EXPORT_SYMBOL_GPL(extcon_get_cable_state_); -/** - * extcon_get_cable_state() - Get the status of a specific cable. - * @edev: the extcon device that has the cable. - * @cable_name: cable name. - * - * Note that this is slower than extcon_get_cable_state_. - */ -int extcon_get_cable_state(struct extcon_dev *edev, const char *cable_name) -{ - int id; - - id = find_cable_id_by_name(edev, cable_name); - if (id < 0) - return id; - - return extcon_get_cable_state_(edev, id); -} -EXPORT_SYMBOL_GPL(extcon_get_cable_state); - /** * extcon_set_cable_state_() - Set the status of a specific cable. * @edev: the extcon device that has the cable. @@ -421,28 +390,6 @@ int extcon_set_cable_state_(struct extcon_dev *edev, unsigned int id, } EXPORT_SYMBOL_GPL(extcon_set_cable_state_); -/** - * extcon_set_cable_state() - Set the status of a specific cable. - * @edev: the extcon device that has the cable. - * @cable_name: cable name. - * @cable_state: the new cable status. The default semantics is - * true: attached / false: detached. - * - * Note that this is slower than extcon_set_cable_state_. - */ -int extcon_set_cable_state(struct extcon_dev *edev, - const char *cable_name, bool cable_state) -{ - int id; - - id = find_cable_id_by_name(edev, cable_name); - if (id < 0) - return id; - - return extcon_set_cable_state_(edev, id, cable_state); -} -EXPORT_SYMBOL_GPL(extcon_set_cable_state); - /** * extcon_get_extcon_dev() - Get the extcon device instance from the name * @extcon_name: The extcon name provided with extcon_dev_register() @@ -466,105 +413,6 @@ out: } EXPORT_SYMBOL_GPL(extcon_get_extcon_dev); -/** - * extcon_register_interest() - Register a notifier for a state change of a - * specific cable, not an entier set of cables of a - * extcon device. - * @obj: an empty extcon_specific_cable_nb object to be returned. - * @extcon_name: the name of extcon device. - * if NULL, extcon_register_interest will register - * every cable with the target cable_name given. - * @cable_name: the target cable name. - * @nb: the notifier block to get notified. - * - * Provide an empty extcon_specific_cable_nb. extcon_register_interest() sets - * the struct for you. - * - * extcon_register_interest is a helper function for those who want to get - * notification for a single specific cable's status change. If a user wants - * to get notification for any changes of all cables of a extcon device, - * he/she should use the general extcon_register_notifier(). - * - * Note that the second parameter given to the callback of nb (val) is - * "old_state", not the current state. The current state can be retrieved - * by looking at the third pameter (edev pointer)'s state value. - */ -int extcon_register_interest(struct extcon_specific_cable_nb *obj, - const char *extcon_name, const char *cable_name, - struct notifier_block *nb) -{ - unsigned long flags; - int ret; - - if (!obj || !cable_name || !nb) - return -EINVAL; - - if (extcon_name) { - obj->edev = extcon_get_extcon_dev(extcon_name); - if (!obj->edev) - return -ENODEV; - - obj->cable_index = find_cable_index_by_name(obj->edev, - cable_name); - if (obj->cable_index < 0) - return obj->cable_index; - - obj->user_nb = nb; - - spin_lock_irqsave(&obj->edev->lock, flags); - ret = raw_notifier_chain_register( - &obj->edev->nh[obj->cable_index], - obj->user_nb); - spin_unlock_irqrestore(&obj->edev->lock, flags); - } else { - struct class_dev_iter iter; - struct extcon_dev *extd; - struct device *dev; - - if (!extcon_class) - return -ENODEV; - class_dev_iter_init(&iter, extcon_class, NULL, NULL); - while ((dev = class_dev_iter_next(&iter))) { - extd = dev_get_drvdata(dev); - - if (find_cable_index_by_name(extd, cable_name) < 0) - continue; - - class_dev_iter_exit(&iter); - return extcon_register_interest(obj, extd->name, - cable_name, nb); - } - - ret = -ENODEV; - } - - return ret; -} -EXPORT_SYMBOL_GPL(extcon_register_interest); - -/** - * extcon_unregister_interest() - Unregister the notifier registered by - * extcon_register_interest(). - * @obj: the extcon_specific_cable_nb object returned by - * extcon_register_interest(). - */ -int extcon_unregister_interest(struct extcon_specific_cable_nb *obj) -{ - unsigned long flags; - int ret; - - if (!obj) - return -EINVAL; - - spin_lock_irqsave(&obj->edev->lock, flags); - ret = raw_notifier_chain_unregister( - &obj->edev->nh[obj->cable_index], obj->user_nb); - spin_unlock_irqrestore(&obj->edev->lock, flags); - - return ret; -} -EXPORT_SYMBOL_GPL(extcon_unregister_interest); - /** * extcon_register_notifier() - Register a notifiee to get notified by * any attach status changes from the extcon. @@ -582,14 +430,35 @@ int extcon_register_notifier(struct extcon_dev *edev, unsigned int id, unsigned long flags; int ret, idx; - if (!edev || !nb) + if (!nb) return -EINVAL; - idx = find_cable_index_by_id(edev, id); + if (edev) { + idx = find_cable_index_by_id(edev, id); + if (idx < 0) + return idx; - spin_lock_irqsave(&edev->lock, flags); - ret = raw_notifier_chain_register(&edev->nh[idx], nb); - spin_unlock_irqrestore(&edev->lock, flags); + spin_lock_irqsave(&edev->lock, flags); + ret = raw_notifier_chain_register(&edev->nh[idx], nb); + spin_unlock_irqrestore(&edev->lock, flags); + } else { + struct extcon_dev *extd; + + mutex_lock(&extcon_dev_list_lock); + list_for_each_entry(extd, &extcon_dev_list, entry) { + idx = find_cable_index_by_id(extd, id); + if (idx >= 0) + break; + } + mutex_unlock(&extcon_dev_list_lock); + + if (idx >= 0) { + edev = extd; + return extcon_register_notifier(extd, id, nb); + } else { + ret = -ENODEV; + } + } return ret; } @@ -611,6 +480,8 @@ int extcon_unregister_notifier(struct extcon_dev *edev, unsigned int id, return -EINVAL; idx = find_cable_index_by_id(edev, id); + if (idx < 0) + return idx; spin_lock_irqsave(&edev->lock, flags); ret = raw_notifier_chain_unregister(&edev->nh[idx], nb); @@ -693,66 +564,6 @@ void extcon_dev_free(struct extcon_dev *edev) } EXPORT_SYMBOL_GPL(extcon_dev_free); -static int devm_extcon_dev_match(struct device *dev, void *res, void *data) -{ - struct extcon_dev **r = res; - - if (WARN_ON(!r || !*r)) - return 0; - - return *r == data; -} - -static void devm_extcon_dev_release(struct device *dev, void *res) -{ - extcon_dev_free(*(struct extcon_dev **)res); -} - -/** - * devm_extcon_dev_allocate - Allocate managed extcon device - * @dev: device owning the extcon device being created - * @supported_cable: Array of supported extcon ending with EXTCON_NONE. - * If supported_cable is NULL, cable name related APIs - * are disabled. - * - * This function manages automatically the memory of extcon device using device - * resource management and simplify the control of freeing the memory of extcon - * device. - * - * Returns the pointer memory of allocated extcon_dev if success - * or ERR_PTR(err) if fail - */ -struct extcon_dev *devm_extcon_dev_allocate(struct device *dev, - const unsigned int *supported_cable) -{ - struct extcon_dev **ptr, *edev; - - ptr = devres_alloc(devm_extcon_dev_release, sizeof(*ptr), GFP_KERNEL); - if (!ptr) - return ERR_PTR(-ENOMEM); - - edev = extcon_dev_allocate(supported_cable); - if (IS_ERR(edev)) { - devres_free(ptr); - return edev; - } - - edev->dev.parent = dev; - - *ptr = edev; - devres_add(dev, ptr); - - return edev; -} -EXPORT_SYMBOL_GPL(devm_extcon_dev_allocate); - -void devm_extcon_dev_free(struct device *dev, struct extcon_dev *edev) -{ - WARN_ON(devres_release(dev, devm_extcon_dev_release, - devm_extcon_dev_match, edev)); -} -EXPORT_SYMBOL_GPL(devm_extcon_dev_free); - /** * extcon_dev_register() - Register a new extcon device * @edev : the new extcon device (should be allocated before calling) @@ -1018,63 +829,6 @@ void extcon_dev_unregister(struct extcon_dev *edev) } EXPORT_SYMBOL_GPL(extcon_dev_unregister); -static void devm_extcon_dev_unreg(struct device *dev, void *res) -{ - extcon_dev_unregister(*(struct extcon_dev **)res); -} - -/** - * devm_extcon_dev_register() - Resource-managed extcon_dev_register() - * @dev: device to allocate extcon device - * @edev: the new extcon device to register - * - * Managed extcon_dev_register() function. If extcon device is attached with - * this function, that extcon device is automatically unregistered on driver - * detach. Internally this function calls extcon_dev_register() function. - * To get more information, refer that function. - * - * If extcon device is registered with this function and the device needs to be - * unregistered separately, devm_extcon_dev_unregister() should be used. - * - * Returns 0 if success or negaive error number if failure. - */ -int devm_extcon_dev_register(struct device *dev, struct extcon_dev *edev) -{ - struct extcon_dev **ptr; - int ret; - - ptr = devres_alloc(devm_extcon_dev_unreg, sizeof(*ptr), GFP_KERNEL); - if (!ptr) - return -ENOMEM; - - ret = extcon_dev_register(edev); - if (ret) { - devres_free(ptr); - return ret; - } - - *ptr = edev; - devres_add(dev, ptr); - - return 0; -} -EXPORT_SYMBOL_GPL(devm_extcon_dev_register); - -/** - * devm_extcon_dev_unregister() - Resource-managed extcon_dev_unregister() - * @dev: device the extcon belongs to - * @edev: the extcon device to unregister - * - * Unregister extcon device that is registered with devm_extcon_dev_register() - * function. - */ -void devm_extcon_dev_unregister(struct device *dev, struct extcon_dev *edev) -{ - WARN_ON(devres_release(dev, devm_extcon_dev_unreg, - devm_extcon_dev_match, edev)); -} -EXPORT_SYMBOL_GPL(devm_extcon_dev_unregister); - #ifdef CONFIG_OF /* * extcon_get_edev_by_phandle - Get the extcon device from devicetree @@ -1107,10 +861,12 @@ struct extcon_dev *extcon_get_edev_by_phandle(struct device *dev, int index) list_for_each_entry(edev, &extcon_dev_list, entry) { if (edev->dev.parent && edev->dev.parent->of_node == node) { mutex_unlock(&extcon_dev_list_lock); + of_node_put(node); return edev; } } mutex_unlock(&extcon_dev_list_lock); + of_node_put(node); return ERR_PTR(-EPROBE_DEFER); } diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index 6664f1108..0e22f2414 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -10,7 +10,7 @@ config ARM_PSCI_FW config ARM_SCPI_PROTOCOL tristate "ARM System Control and Power Interface (SCPI) Message Protocol" - depends on ARM_MHU + depends on MAILBOX help System Control and Power Interface (SCPI) Message Protocol is defined for the purpose of communication between the Application @@ -27,6 +27,15 @@ config ARM_SCPI_PROTOCOL This protocol library provides interface for all the client drivers making use of the features offered by the SCP. +config ARM_SCPI_POWER_DOMAIN + tristate "SCPI power domain driver" + depends on ARM_SCPI_PROTOCOL || (COMPILE_TEST && OF) + default y + select PM_GENERIC_DOMAINS if PM + help + This enables support for the SCPI power domains which can be + enabled or disabled via the SCP firmware + config EDD tristate "BIOS Enhanced Disk Drive calls determine boot disk" depends on X86 @@ -184,6 +193,7 @@ config FW_CFG_SYSFS_CMDLINE config QCOM_SCM bool depends on ARM || ARM64 + select RESET_CONTROLLER config QCOM_SCM_32 def_bool y diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 474bada56..44a59dcfc 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -3,6 +3,7 @@ # obj-$(CONFIG_ARM_PSCI_FW) += psci.o obj-$(CONFIG_ARM_SCPI_PROTOCOL) += arm_scpi.o +obj-$(CONFIG_ARM_SCPI_POWER_DOMAIN) += scpi_pm_domain.o obj-$(CONFIG_DMI) += dmi_scan.o obj-$(CONFIG_DMI_SYSFS) += dmi-sysfs.o obj-$(CONFIG_EDD) += edd.o diff --git a/drivers/firmware/arm_scpi.c b/drivers/firmware/arm_scpi.c index 7e3e595c9..ce2bc2a38 100644 --- a/drivers/firmware/arm_scpi.c +++ b/drivers/firmware/arm_scpi.c @@ -210,10 +210,6 @@ struct dvfs_info { } opps[MAX_DVFS_OPPS]; } __packed; -struct dvfs_get { - u8 index; -} __packed; - struct dvfs_set { u8 domain; u8 index; @@ -235,6 +231,11 @@ struct sensor_value { __le32 hi_val; } __packed; +struct dev_pstate_set { + u16 dev_id; + u8 pstate; +} __packed; + static struct scpi_drvinfo *scpi_info; static int scpi_linux_errmap[SCPI_ERR_MAX] = { @@ -431,11 +432,11 @@ static int scpi_clk_set_val(u16 clk_id, unsigned long rate) static int scpi_dvfs_get_idx(u8 domain) { int ret; - struct dvfs_get dvfs; + u8 dvfs_idx; ret = scpi_send_message(SCPI_CMD_GET_DVFS, &domain, sizeof(domain), - &dvfs, sizeof(dvfs)); - return ret ? ret : dvfs.index; + &dvfs_idx, sizeof(dvfs_idx)); + return ret ? ret : dvfs_idx; } static int scpi_dvfs_set_idx(u8 domain, u8 index) @@ -526,7 +527,7 @@ static int scpi_sensor_get_info(u16 sensor_id, struct scpi_sensor_info *info) return ret; } -int scpi_sensor_get_value(u16 sensor, u64 *val) +static int scpi_sensor_get_value(u16 sensor, u64 *val) { __le16 id = cpu_to_le16(sensor); struct sensor_value buf; @@ -541,6 +542,29 @@ int scpi_sensor_get_value(u16 sensor, u64 *val) return ret; } +static int scpi_device_get_power_state(u16 dev_id) +{ + int ret; + u8 pstate; + __le16 id = cpu_to_le16(dev_id); + + ret = scpi_send_message(SCPI_CMD_GET_DEVICE_PWR_STATE, &id, + sizeof(id), &pstate, sizeof(pstate)); + return ret ? ret : pstate; +} + +static int scpi_device_set_power_state(u16 dev_id, u8 pstate) +{ + int stat; + struct dev_pstate_set dev_set = { + .dev_id = cpu_to_le16(dev_id), + .pstate = pstate, + }; + + return scpi_send_message(SCPI_CMD_SET_DEVICE_PWR_STATE, &dev_set, + sizeof(dev_set), &stat, sizeof(stat)); +} + static struct scpi_ops scpi_ops = { .get_version = scpi_get_version, .clk_get_range = scpi_clk_get_range, @@ -552,6 +576,8 @@ static struct scpi_ops scpi_ops = { .sensor_get_capability = scpi_sensor_get_capability, .sensor_get_info = scpi_sensor_get_info, .sensor_get_value = scpi_sensor_get_value, + .device_get_power_state = scpi_device_get_power_state, + .device_set_power_state = scpi_device_set_power_state, }; struct scpi_ops *get_scpi_ops(void) @@ -683,9 +709,10 @@ static int scpi_probe(struct platform_device *pdev) struct mbox_client *cl = &pchan->cl; struct device_node *shmem = of_parse_phandle(np, "shmem", idx); - if (of_address_to_resource(shmem, 0, &res)) { + ret = of_address_to_resource(shmem, 0, &res); + of_node_put(shmem); + if (ret) { dev_err(dev, "failed to get SCPI payload mem resource\n"); - ret = -EINVAL; goto err; } diff --git a/drivers/firmware/broadcom/bcm47xx_sprom.c b/drivers/firmware/broadcom/bcm47xx_sprom.c index b6eb875d4..62aa3cf09 100644 --- a/drivers/firmware/broadcom/bcm47xx_sprom.c +++ b/drivers/firmware/broadcom/bcm47xx_sprom.c @@ -669,7 +669,7 @@ static int bcm47xx_get_sprom_bcma(struct bcma_bus *bus, struct ssb_sprom *out) case BCMA_HOSTTYPE_PCI: memset(out, 0, sizeof(struct ssb_sprom)); /* On BCM47XX all PCI buses share the same domain */ - if (config_enabled(CONFIG_BCM47XX)) + if (IS_ENABLED(CONFIG_BCM47XX)) snprintf(buf, sizeof(buf), "pci/%u/%u/", bus->host_pci->bus->number + 1, PCI_SLOT(bus->host_pci->devfn)); diff --git a/drivers/firmware/dmi-id.c b/drivers/firmware/dmi-id.c index 94a58a082..44c01390d 100644 --- a/drivers/firmware/dmi-id.c +++ b/drivers/firmware/dmi-id.c @@ -229,14 +229,14 @@ static int __init dmi_id_init(void) ret = device_register(dmi_dev); if (ret) - goto fail_free_dmi_dev; + goto fail_put_dmi_dev; return 0; -fail_free_dmi_dev: - kfree(dmi_dev); -fail_class_unregister: +fail_put_dmi_dev: + put_device(dmi_dev); +fail_class_unregister: class_unregister(&dmi_class); return ret; diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c index 17ccf0a87..c394b81fe 100644 --- a/drivers/firmware/efi/arm-runtime.c +++ b/drivers/firmware/efi/arm-runtime.c @@ -107,6 +107,11 @@ static int __init arm_enable_runtime_services(void) return 0; } + if (efi_enabled(EFI_RUNTIME_SERVICES)) { + pr_info("EFI runtime services access via paravirt.\n"); + return 0; + } + pr_info("Remapping and enabling EFI services.\n"); mapsize = efi.memmap.map_end - efi.memmap.map; diff --git a/drivers/firmware/efi/efi-pstore.c b/drivers/firmware/efi/efi-pstore.c index eac76a79a..30a24d09e 100644 --- a/drivers/firmware/efi/efi-pstore.c +++ b/drivers/firmware/efi/efi-pstore.c @@ -34,6 +34,7 @@ struct pstore_read_data { int *count; struct timespec *timespec; bool *compressed; + ssize_t *ecc_notice_size; char **buf; }; @@ -69,6 +70,7 @@ static int efi_pstore_read_func(struct efivar_entry *entry, void *data) *cb_data->compressed = true; else *cb_data->compressed = false; + *cb_data->ecc_notice_size = 0; } else if (sscanf(name, "dump-type%u-%u-%d-%lu", cb_data->type, &part, &cnt, &time) == 4) { *cb_data->id = generic_id(time, part, cnt); @@ -76,6 +78,7 @@ static int efi_pstore_read_func(struct efivar_entry *entry, void *data) cb_data->timespec->tv_sec = time; cb_data->timespec->tv_nsec = 0; *cb_data->compressed = false; + *cb_data->ecc_notice_size = 0; } else if (sscanf(name, "dump-type%u-%u-%lu", cb_data->type, &part, &time) == 3) { /* @@ -88,6 +91,7 @@ static int efi_pstore_read_func(struct efivar_entry *entry, void *data) cb_data->timespec->tv_sec = time; cb_data->timespec->tv_nsec = 0; *cb_data->compressed = false; + *cb_data->ecc_notice_size = 0; } else return 0; @@ -210,6 +214,7 @@ static int efi_pstore_sysfs_entry_iter(void *data, struct efivar_entry **pos) static ssize_t efi_pstore_read(u64 *id, enum pstore_type_id *type, int *count, struct timespec *timespec, char **buf, bool *compressed, + ssize_t *ecc_notice_size, struct pstore_info *psi) { struct pstore_read_data data; @@ -220,6 +225,7 @@ static ssize_t efi_pstore_read(u64 *id, enum pstore_type_id *type, data.count = count; data.timespec = timespec; data.compressed = compressed; + data.ecc_notice_size = ecc_notice_size; data.buf = buf; *data.buf = kzalloc(EFIVARS_DATA_SIZE_MAX, GFP_KERNEL); @@ -393,6 +399,13 @@ static __init int efivars_pstore_init(void) static __exit void efivars_pstore_exit(void) { + if (!efi_pstore_info.bufsize) + return; + + pstore_unregister(&efi_pstore_info); + kfree(efi_pstore_info.buf); + efi_pstore_info.buf = NULL; + efi_pstore_info.bufsize = 0; } module_init(efivars_pstore_init); diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 05509f3aa..7dd2e2d37 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -24,6 +24,9 @@ #include #include #include +#include +#include +#include #include @@ -195,6 +198,96 @@ static void generic_ops_unregister(void) efivars_unregister(&generic_efivars); } +#if IS_ENABLED(CONFIG_ACPI) +#define EFIVAR_SSDT_NAME_MAX 16 +static char efivar_ssdt[EFIVAR_SSDT_NAME_MAX] __initdata; +static int __init efivar_ssdt_setup(char *str) +{ + if (strlen(str) < sizeof(efivar_ssdt)) + memcpy(efivar_ssdt, str, strlen(str)); + else + pr_warn("efivar_ssdt: name too long: %s\n", str); + return 0; +} +__setup("efivar_ssdt=", efivar_ssdt_setup); + +static __init int efivar_ssdt_iter(efi_char16_t *name, efi_guid_t vendor, + unsigned long name_size, void *data) +{ + struct efivar_entry *entry; + struct list_head *list = data; + char utf8_name[EFIVAR_SSDT_NAME_MAX]; + int limit = min_t(unsigned long, EFIVAR_SSDT_NAME_MAX, name_size); + + ucs2_as_utf8(utf8_name, name, limit - 1); + if (strncmp(utf8_name, efivar_ssdt, limit) != 0) + return 0; + + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return 0; + + memcpy(entry->var.VariableName, name, name_size); + memcpy(&entry->var.VendorGuid, &vendor, sizeof(efi_guid_t)); + + efivar_entry_add(entry, list); + + return 0; +} + +static __init int efivar_ssdt_load(void) +{ + LIST_HEAD(entries); + struct efivar_entry *entry, *aux; + unsigned long size; + void *data; + int ret; + + ret = efivar_init(efivar_ssdt_iter, &entries, true, &entries); + + list_for_each_entry_safe(entry, aux, &entries, list) { + pr_info("loading SSDT from variable %s-%pUl\n", efivar_ssdt, + &entry->var.VendorGuid); + + list_del(&entry->list); + + ret = efivar_entry_size(entry, &size); + if (ret) { + pr_err("failed to get var size\n"); + goto free_entry; + } + + data = kmalloc(size, GFP_KERNEL); + if (!data) + goto free_entry; + + ret = efivar_entry_get(entry, NULL, &size, data); + if (ret) { + pr_err("failed to get var data\n"); + goto free_data; + } + + ret = acpi_load_table(data); + if (ret) { + pr_err("failed to load table: %d\n", ret); + goto free_data; + } + + goto free_entry; + +free_data: + kfree(data); + +free_entry: + kfree(entry); + } + + return ret; +} +#else +static inline int efivar_ssdt_load(void) { return 0; } +#endif + /* * We register the efi subsystem with the firmware subsystem and the * efivars subsystem with the efi subsystem, if the system was booted with @@ -218,6 +311,9 @@ static int __init efisubsys_init(void) if (error) goto err_put; + if (efi_enabled(EFI_RUNTIME_SERVICES)) + efivar_ssdt_load(); + error = sysfs_create_group(efi_kobj, &efi_subsys_attr_group); if (error) { pr_err("efi: Sysfs attribute export failed with error %d.\n", @@ -472,12 +568,14 @@ device_initcall(efi_load_efivars); FIELD_SIZEOF(struct efi_fdt_params, field) \ } -static __initdata struct { +struct params { const char name[32]; const char propname[32]; int offset; int size; -} dt_params[] = { +}; + +static __initdata struct params fdt_params[] = { UEFI_PARAM("System Table", "linux,uefi-system-table", system_table), UEFI_PARAM("MemMap Address", "linux,uefi-mmap-start", mmap), UEFI_PARAM("MemMap Size", "linux,uefi-mmap-size", mmap_size), @@ -485,44 +583,94 @@ static __initdata struct { UEFI_PARAM("MemMap Desc. Version", "linux,uefi-mmap-desc-ver", desc_ver) }; +static __initdata struct params xen_fdt_params[] = { + UEFI_PARAM("System Table", "xen,uefi-system-table", system_table), + UEFI_PARAM("MemMap Address", "xen,uefi-mmap-start", mmap), + UEFI_PARAM("MemMap Size", "xen,uefi-mmap-size", mmap_size), + UEFI_PARAM("MemMap Desc. Size", "xen,uefi-mmap-desc-size", desc_size), + UEFI_PARAM("MemMap Desc. Version", "xen,uefi-mmap-desc-ver", desc_ver) +}; + +#define EFI_FDT_PARAMS_SIZE ARRAY_SIZE(fdt_params) + +static __initdata struct { + const char *uname; + const char *subnode; + struct params *params; +} dt_params[] = { + { "hypervisor", "uefi", xen_fdt_params }, + { "chosen", NULL, fdt_params }, +}; + struct param_info { int found; void *params; + const char *missing; }; -static int __init fdt_find_uefi_params(unsigned long node, const char *uname, - int depth, void *data) +static int __init __find_uefi_params(unsigned long node, + struct param_info *info, + struct params *params) { - struct param_info *info = data; const void *prop; void *dest; u64 val; int i, len; - if (depth != 1 || strcmp(uname, "chosen") != 0) - return 0; - - for (i = 0; i < ARRAY_SIZE(dt_params); i++) { - prop = of_get_flat_dt_prop(node, dt_params[i].propname, &len); - if (!prop) + for (i = 0; i < EFI_FDT_PARAMS_SIZE; i++) { + prop = of_get_flat_dt_prop(node, params[i].propname, &len); + if (!prop) { + info->missing = params[i].name; return 0; - dest = info->params + dt_params[i].offset; + } + + dest = info->params + params[i].offset; info->found++; val = of_read_number(prop, len / sizeof(u32)); - if (dt_params[i].size == sizeof(u32)) + if (params[i].size == sizeof(u32)) *(u32 *)dest = val; else *(u64 *)dest = val; if (efi_enabled(EFI_DBG)) - pr_info(" %s: 0x%0*llx\n", dt_params[i].name, - dt_params[i].size * 2, val); + pr_info(" %s: 0x%0*llx\n", params[i].name, + params[i].size * 2, val); } + return 1; } +static int __init fdt_find_uefi_params(unsigned long node, const char *uname, + int depth, void *data) +{ + struct param_info *info = data; + int i; + + for (i = 0; i < ARRAY_SIZE(dt_params); i++) { + const char *subnode = dt_params[i].subnode; + + if (depth != 1 || strcmp(uname, dt_params[i].uname) != 0) { + info->missing = dt_params[i].params[0].name; + continue; + } + + if (subnode) { + int err = of_get_flat_dt_subnode_by_name(node, subnode); + + if (err < 0) + return 0; + + node = err; + } + + return __find_uefi_params(node, info, dt_params[i].params); + } + + return 0; +} + int __init efi_get_fdt_params(struct efi_fdt_params *params) { struct param_info info; @@ -538,7 +686,7 @@ int __init efi_get_fdt_params(struct efi_fdt_params *params) pr_info("UEFI not found.\n"); else if (!ret) pr_err("Can't find '%s' in device tree!\n", - dt_params[info.found].name); + info.missing); return ret; } diff --git a/drivers/firmware/efi/efibc.c b/drivers/firmware/efi/efibc.c index 8dd0c7085..503bbe2a9 100644 --- a/drivers/firmware/efi/efibc.c +++ b/drivers/firmware/efi/efibc.c @@ -37,13 +37,13 @@ static int efibc_set_variable(const char *name, const char *value) size_t size = (strlen(value) + 1) * sizeof(efi_char16_t); if (size > sizeof(entry->var.Data)) { - pr_err("value is too large"); + pr_err("value is too large (%zu bytes) for '%s' EFI variable\n", size, name); return -EINVAL; } entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) { - pr_err("failed to allocate efivar entry"); + pr_err("failed to allocate efivar entry for '%s' EFI variable\n", name); return -ENOMEM; } diff --git a/drivers/firmware/efi/runtime-wrappers.c b/drivers/firmware/efi/runtime-wrappers.c index 23bef6bb7..41958774c 100644 --- a/drivers/firmware/efi/runtime-wrappers.c +++ b/drivers/firmware/efi/runtime-wrappers.c @@ -22,7 +22,16 @@ #include #include -static void efi_call_virt_check_flags(unsigned long flags, const char *call) +/* + * Wrap around the new efi_call_virt_generic() macros so that the + * code doesn't get too cluttered: + */ +#define efi_call_virt(f, args...) \ + efi_call_virt_pointer(efi.systab->runtime, f, args) +#define __efi_call_virt(f, args...) \ + __efi_call_virt_pointer(efi.systab->runtime, f, args) + +void efi_call_virt_check_flags(unsigned long flags, const char *call) { unsigned long cur_flags, mismatch; @@ -38,48 +47,6 @@ static void efi_call_virt_check_flags(unsigned long flags, const char *call) local_irq_restore(flags); } -/* - * Arch code can implement the following three template macros, avoiding - * reptition for the void/non-void return cases of {__,}efi_call_virt: - * - * * arch_efi_call_virt_setup - * - * Sets up the environment for the call (e.g. switching page tables, - * allowing kernel-mode use of floating point, if required). - * - * * arch_efi_call_virt - * - * Performs the call. The last expression in the macro must be the call - * itself, allowing the logic to be shared by the void and non-void - * cases. - * - * * arch_efi_call_virt_teardown - * - * Restores the usual kernel environment once the call has returned. - */ - -#define efi_call_virt(f, args...) \ -({ \ - efi_status_t __s; \ - unsigned long flags; \ - arch_efi_call_virt_setup(); \ - local_save_flags(flags); \ - __s = arch_efi_call_virt(f, args); \ - efi_call_virt_check_flags(flags, __stringify(f)); \ - arch_efi_call_virt_teardown(); \ - __s; \ -}) - -#define __efi_call_virt(f, args...) \ -({ \ - unsigned long flags; \ - arch_efi_call_virt_setup(); \ - local_save_flags(flags); \ - arch_efi_call_virt(f, args); \ - efi_call_virt_check_flags(flags, __stringify(f)); \ - arch_efi_call_virt_teardown(); \ -}) - /* * According to section 7.1 of the UEFI spec, Runtime Services are not fully * reentrant, and there are particular combinations of calls that need to be diff --git a/drivers/firmware/psci.c b/drivers/firmware/psci.c index 03e045827..8263429e2 100644 --- a/drivers/firmware/psci.c +++ b/drivers/firmware/psci.c @@ -13,6 +13,7 @@ #define pr_fmt(fmt) "psci: " fmt +#include #include #include #include @@ -256,13 +257,6 @@ static int psci_dt_cpu_init_idle(struct device_node *cpu_node, int cpu) u32 *psci_states; struct device_node *state_node; - /* - * If the PSCI cpu_suspend function hook has not been initialized - * idle states must not be enabled, so bail out - */ - if (!psci_ops.cpu_suspend) - return -EOPNOTSUPP; - /* Count idle states */ while ((state_node = of_parse_phandle(cpu_node, "cpu-idle-states", count))) { @@ -310,11 +304,69 @@ free_mem: return ret; } +#ifdef CONFIG_ACPI +#include + +static int __maybe_unused psci_acpi_cpu_init_idle(unsigned int cpu) +{ + int i, count; + u32 *psci_states; + struct acpi_lpi_state *lpi; + struct acpi_processor *pr = per_cpu(processors, cpu); + + if (unlikely(!pr || !pr->flags.has_lpi)) + return -EINVAL; + + count = pr->power.count - 1; + if (count <= 0) + return -ENODEV; + + psci_states = kcalloc(count, sizeof(*psci_states), GFP_KERNEL); + if (!psci_states) + return -ENOMEM; + + for (i = 0; i < count; i++) { + u32 state; + + lpi = &pr->power.lpi_states[i + 1]; + /* + * Only bits[31:0] represent a PSCI power_state while + * bits[63:32] must be 0x0 as per ARM ACPI FFH Specification + */ + state = lpi->address; + if (!psci_power_state_is_valid(state)) { + pr_warn("Invalid PSCI power state %#x\n", state); + kfree(psci_states); + return -EINVAL; + } + psci_states[i] = state; + } + /* Idle states parsed correctly, initialize per-cpu pointer */ + per_cpu(psci_power_state, cpu) = psci_states; + return 0; +} +#else +static int __maybe_unused psci_acpi_cpu_init_idle(unsigned int cpu) +{ + return -EINVAL; +} +#endif + int psci_cpu_init_idle(unsigned int cpu) { struct device_node *cpu_node; int ret; + /* + * If the PSCI cpu_suspend function hook has not been initialized + * idle states must not be enabled, so bail out + */ + if (!psci_ops.cpu_suspend) + return -EOPNOTSUPP; + + if (!acpi_disabled) + return psci_acpi_cpu_init_idle(cpu); + cpu_node = of_get_cpu_node(cpu, NULL); if (!cpu_node) return -ENODEV; diff --git a/drivers/firmware/qcom_scm-32.c b/drivers/firmware/qcom_scm-32.c index 0883292f6..c6aeedbdc 100644 --- a/drivers/firmware/qcom_scm-32.c +++ b/drivers/firmware/qcom_scm-32.c @@ -23,8 +23,7 @@ #include #include #include - -#include +#include #include "qcom_scm.h" @@ -96,44 +95,6 @@ struct qcom_scm_response { __le32 is_complete; }; -/** - * alloc_qcom_scm_command() - Allocate an SCM command - * @cmd_size: size of the command buffer - * @resp_size: size of the response buffer - * - * Allocate an SCM command, including enough room for the command - * and response headers as well as the command and response buffers. - * - * Returns a valid &qcom_scm_command on success or %NULL if the allocation fails. - */ -static struct qcom_scm_command *alloc_qcom_scm_command(size_t cmd_size, size_t resp_size) -{ - struct qcom_scm_command *cmd; - size_t len = sizeof(*cmd) + sizeof(struct qcom_scm_response) + cmd_size + - resp_size; - u32 offset; - - cmd = kzalloc(PAGE_ALIGN(len), GFP_KERNEL); - if (cmd) { - cmd->len = cpu_to_le32(len); - offset = offsetof(struct qcom_scm_command, buf); - cmd->buf_offset = cpu_to_le32(offset); - cmd->resp_hdr_offset = cpu_to_le32(offset + cmd_size); - } - return cmd; -} - -/** - * free_qcom_scm_command() - Free an SCM command - * @cmd: command to free - * - * Free an SCM command. - */ -static inline void free_qcom_scm_command(struct qcom_scm_command *cmd) -{ - kfree(cmd); -} - /** * qcom_scm_command_to_response() - Get a pointer to a qcom_scm_response * @cmd: command @@ -168,23 +129,6 @@ static inline void *qcom_scm_get_response_buffer(const struct qcom_scm_response return (void *)rsp + le32_to_cpu(rsp->buf_offset); } -static int qcom_scm_remap_error(int err) -{ - pr_err("qcom_scm_call failed with error code %d\n", err); - switch (err) { - case QCOM_SCM_ERROR: - return -EIO; - case QCOM_SCM_EINVAL_ADDR: - case QCOM_SCM_EINVAL_ARG: - return -EINVAL; - case QCOM_SCM_EOPNOTSUPP: - return -EOPNOTSUPP; - case QCOM_SCM_ENOMEM: - return -ENOMEM; - } - return -EINVAL; -} - static u32 smc(u32 cmd_addr) { int context_id; @@ -209,45 +153,9 @@ static u32 smc(u32 cmd_addr) return r0; } -static int __qcom_scm_call(const struct qcom_scm_command *cmd) -{ - int ret; - u32 cmd_addr = virt_to_phys(cmd); - - /* - * Flush the command buffer so that the secure world sees - * the correct data. - */ - secure_flush_area(cmd, cmd->len); - - ret = smc(cmd_addr); - if (ret < 0) - ret = qcom_scm_remap_error(ret); - - return ret; -} - -static void qcom_scm_inv_range(unsigned long start, unsigned long end) -{ - u32 cacheline_size, ctr; - - asm volatile("mrc p15, 0, %0, c0, c0, 1" : "=r" (ctr)); - cacheline_size = 4 << ((ctr >> 16) & 0xf); - - start = round_down(start, cacheline_size); - end = round_up(end, cacheline_size); - outer_inv_range(start, end); - while (start < end) { - asm ("mcr p15, 0, %0, c7, c6, 1" : : "r" (start) - : "memory"); - start += cacheline_size; - } - dsb(); - isb(); -} - /** * qcom_scm_call() - Send an SCM command + * @dev: struct device * @svc_id: service identifier * @cmd_id: command identifier * @cmd_buf: command buffer @@ -264,42 +172,59 @@ static void qcom_scm_inv_range(unsigned long start, unsigned long end) * and response buffers is taken care of by qcom_scm_call; however, callers are * responsible for any other cached buffers passed over to the secure world. */ -static int qcom_scm_call(u32 svc_id, u32 cmd_id, const void *cmd_buf, - size_t cmd_len, void *resp_buf, size_t resp_len) +static int qcom_scm_call(struct device *dev, u32 svc_id, u32 cmd_id, + const void *cmd_buf, size_t cmd_len, void *resp_buf, + size_t resp_len) { int ret; struct qcom_scm_command *cmd; struct qcom_scm_response *rsp; - unsigned long start, end; + size_t alloc_len = sizeof(*cmd) + cmd_len + sizeof(*rsp) + resp_len; + dma_addr_t cmd_phys; - cmd = alloc_qcom_scm_command(cmd_len, resp_len); + cmd = kzalloc(PAGE_ALIGN(alloc_len), GFP_KERNEL); if (!cmd) return -ENOMEM; + cmd->len = cpu_to_le32(alloc_len); + cmd->buf_offset = cpu_to_le32(sizeof(*cmd)); + cmd->resp_hdr_offset = cpu_to_le32(sizeof(*cmd) + cmd_len); + cmd->id = cpu_to_le32((svc_id << 10) | cmd_id); if (cmd_buf) memcpy(qcom_scm_get_command_buffer(cmd), cmd_buf, cmd_len); + rsp = qcom_scm_command_to_response(cmd); + + cmd_phys = dma_map_single(dev, cmd, alloc_len, DMA_TO_DEVICE); + if (dma_mapping_error(dev, cmd_phys)) { + kfree(cmd); + return -ENOMEM; + } + mutex_lock(&qcom_scm_lock); - ret = __qcom_scm_call(cmd); + ret = smc(cmd_phys); + if (ret < 0) + ret = qcom_scm_remap_error(ret); mutex_unlock(&qcom_scm_lock); if (ret) goto out; - rsp = qcom_scm_command_to_response(cmd); - start = (unsigned long)rsp; - do { - qcom_scm_inv_range(start, start + sizeof(*rsp)); + dma_sync_single_for_cpu(dev, cmd_phys + sizeof(*cmd) + cmd_len, + sizeof(*rsp), DMA_FROM_DEVICE); } while (!rsp->is_complete); - end = (unsigned long)qcom_scm_get_response_buffer(rsp) + resp_len; - qcom_scm_inv_range(start, end); - - if (resp_buf) - memcpy(resp_buf, qcom_scm_get_response_buffer(rsp), resp_len); + if (resp_buf) { + dma_sync_single_for_cpu(dev, cmd_phys + sizeof(*cmd) + cmd_len + + le32_to_cpu(rsp->buf_offset), + resp_len, DMA_FROM_DEVICE); + memcpy(resp_buf, qcom_scm_get_response_buffer(rsp), + resp_len); + } out: - free_qcom_scm_command(cmd); + dma_unmap_single(dev, cmd_phys, alloc_len, DMA_TO_DEVICE); + kfree(cmd); return ret; } @@ -342,6 +267,41 @@ static s32 qcom_scm_call_atomic1(u32 svc, u32 cmd, u32 arg1) return r0; } +/** + * qcom_scm_call_atomic2() - Send an atomic SCM command with two arguments + * @svc_id: service identifier + * @cmd_id: command identifier + * @arg1: first argument + * @arg2: second argument + * + * This shall only be used with commands that are guaranteed to be + * uninterruptable, atomic and SMP safe. + */ +static s32 qcom_scm_call_atomic2(u32 svc, u32 cmd, u32 arg1, u32 arg2) +{ + int context_id; + + register u32 r0 asm("r0") = SCM_ATOMIC(svc, cmd, 2); + register u32 r1 asm("r1") = (u32)&context_id; + register u32 r2 asm("r2") = arg1; + register u32 r3 asm("r3") = arg2; + + asm volatile( + __asmeq("%0", "r0") + __asmeq("%1", "r0") + __asmeq("%2", "r1") + __asmeq("%3", "r2") + __asmeq("%4", "r3") +#ifdef REQUIRES_SEC + ".arch_extension sec\n" +#endif + "smc #0 @ switch to secure world\n" + : "=r" (r0) + : "r" (r0), "r" (r1), "r" (r2), "r" (r3) + ); + return r0; +} + u32 qcom_scm_get_version(void) { int context_id; @@ -378,22 +338,6 @@ u32 qcom_scm_get_version(void) } EXPORT_SYMBOL(qcom_scm_get_version); -/* - * Set the cold/warm boot address for one of the CPU cores. - */ -static int qcom_scm_set_boot_addr(u32 addr, int flags) -{ - struct { - __le32 flags; - __le32 addr; - } cmd; - - cmd.addr = cpu_to_le32(addr); - cmd.flags = cpu_to_le32(flags); - return qcom_scm_call(QCOM_SCM_SVC_BOOT, QCOM_SCM_BOOT_ADDR, - &cmd, sizeof(cmd), NULL, 0); -} - /** * qcom_scm_set_cold_boot_addr() - Set the cold boot address for cpus * @entry: Entry point function for the cpus @@ -423,7 +367,8 @@ int __qcom_scm_set_cold_boot_addr(void *entry, const cpumask_t *cpus) set_cpu_present(cpu, false); } - return qcom_scm_set_boot_addr(virt_to_phys(entry), flags); + return qcom_scm_call_atomic2(QCOM_SCM_SVC_BOOT, QCOM_SCM_BOOT_ADDR, + flags, virt_to_phys(entry)); } /** @@ -434,11 +379,16 @@ int __qcom_scm_set_cold_boot_addr(void *entry, const cpumask_t *cpus) * Set the Linux entry point for the SCM to transfer control to when coming * out of a power down. CPU power down may be executed on cpuidle or hotplug. */ -int __qcom_scm_set_warm_boot_addr(void *entry, const cpumask_t *cpus) +int __qcom_scm_set_warm_boot_addr(struct device *dev, void *entry, + const cpumask_t *cpus) { int ret; int flags = 0; int cpu; + struct { + __le32 flags; + __le32 addr; + } cmd; /* * Reassign only if we are switching from hotplug entry point @@ -454,7 +404,10 @@ int __qcom_scm_set_warm_boot_addr(void *entry, const cpumask_t *cpus) if (!flags) return 0; - ret = qcom_scm_set_boot_addr(virt_to_phys(entry), flags); + cmd.addr = cpu_to_le32(virt_to_phys(entry)); + cmd.flags = cpu_to_le32(flags); + ret = qcom_scm_call(dev, QCOM_SCM_SVC_BOOT, QCOM_SCM_BOOT_ADDR, + &cmd, sizeof(cmd), NULL, 0); if (!ret) { for_each_cpu(cpu, cpus) qcom_scm_wb[cpu].entry = entry; @@ -477,25 +430,133 @@ void __qcom_scm_cpu_power_down(u32 flags) flags & QCOM_SCM_FLUSH_FLAG_MASK); } -int __qcom_scm_is_call_available(u32 svc_id, u32 cmd_id) +int __qcom_scm_is_call_available(struct device *dev, u32 svc_id, u32 cmd_id) { int ret; __le32 svc_cmd = cpu_to_le32((svc_id << 10) | cmd_id); __le32 ret_val = 0; - ret = qcom_scm_call(QCOM_SCM_SVC_INFO, QCOM_IS_CALL_AVAIL_CMD, &svc_cmd, - sizeof(svc_cmd), &ret_val, sizeof(ret_val)); + ret = qcom_scm_call(dev, QCOM_SCM_SVC_INFO, QCOM_IS_CALL_AVAIL_CMD, + &svc_cmd, sizeof(svc_cmd), &ret_val, + sizeof(ret_val)); if (ret) return ret; return le32_to_cpu(ret_val); } -int __qcom_scm_hdcp_req(struct qcom_scm_hdcp_req *req, u32 req_cnt, u32 *resp) +int __qcom_scm_hdcp_req(struct device *dev, struct qcom_scm_hdcp_req *req, + u32 req_cnt, u32 *resp) { if (req_cnt > QCOM_SCM_HDCP_MAX_REQ_CNT) return -ERANGE; - return qcom_scm_call(QCOM_SCM_SVC_HDCP, QCOM_SCM_CMD_HDCP, + return qcom_scm_call(dev, QCOM_SCM_SVC_HDCP, QCOM_SCM_CMD_HDCP, req, req_cnt * sizeof(*req), resp, sizeof(*resp)); } + +void __qcom_scm_init(void) +{ +} + +bool __qcom_scm_pas_supported(struct device *dev, u32 peripheral) +{ + __le32 out; + __le32 in; + int ret; + + in = cpu_to_le32(peripheral); + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, + QCOM_SCM_PAS_IS_SUPPORTED_CMD, + &in, sizeof(in), + &out, sizeof(out)); + + return ret ? false : !!out; +} + +int __qcom_scm_pas_init_image(struct device *dev, u32 peripheral, + dma_addr_t metadata_phys) +{ + __le32 scm_ret; + int ret; + struct { + __le32 proc; + __le32 image_addr; + } request; + + request.proc = cpu_to_le32(peripheral); + request.image_addr = cpu_to_le32(metadata_phys); + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, + QCOM_SCM_PAS_INIT_IMAGE_CMD, + &request, sizeof(request), + &scm_ret, sizeof(scm_ret)); + + return ret ? : le32_to_cpu(scm_ret); +} + +int __qcom_scm_pas_mem_setup(struct device *dev, u32 peripheral, + phys_addr_t addr, phys_addr_t size) +{ + __le32 scm_ret; + int ret; + struct { + __le32 proc; + __le32 addr; + __le32 len; + } request; + + request.proc = cpu_to_le32(peripheral); + request.addr = cpu_to_le32(addr); + request.len = cpu_to_le32(size); + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, + QCOM_SCM_PAS_MEM_SETUP_CMD, + &request, sizeof(request), + &scm_ret, sizeof(scm_ret)); + + return ret ? : le32_to_cpu(scm_ret); +} + +int __qcom_scm_pas_auth_and_reset(struct device *dev, u32 peripheral) +{ + __le32 out; + __le32 in; + int ret; + + in = cpu_to_le32(peripheral); + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, + QCOM_SCM_PAS_AUTH_AND_RESET_CMD, + &in, sizeof(in), + &out, sizeof(out)); + + return ret ? : le32_to_cpu(out); +} + +int __qcom_scm_pas_shutdown(struct device *dev, u32 peripheral) +{ + __le32 out; + __le32 in; + int ret; + + in = cpu_to_le32(peripheral); + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, + QCOM_SCM_PAS_SHUTDOWN_CMD, + &in, sizeof(in), + &out, sizeof(out)); + + return ret ? : le32_to_cpu(out); +} + +int __qcom_scm_pas_mss_reset(struct device *dev, bool reset) +{ + __le32 out; + __le32 in = cpu_to_le32(reset); + int ret; + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, QCOM_SCM_PAS_MSS_RESET, + &in, sizeof(in), + &out, sizeof(out)); + + return ret ? : le32_to_cpu(out); +} diff --git a/drivers/firmware/qcom_scm-64.c b/drivers/firmware/qcom_scm-64.c index bb6555f6d..4a0f5ead4 100644 --- a/drivers/firmware/qcom_scm-64.c +++ b/drivers/firmware/qcom_scm-64.c @@ -12,7 +12,150 @@ #include #include +#include +#include +#include +#include #include +#include +#include + +#include "qcom_scm.h" + +#define QCOM_SCM_FNID(s, c) ((((s) & 0xFF) << 8) | ((c) & 0xFF)) + +#define MAX_QCOM_SCM_ARGS 10 +#define MAX_QCOM_SCM_RETS 3 + +enum qcom_scm_arg_types { + QCOM_SCM_VAL, + QCOM_SCM_RO, + QCOM_SCM_RW, + QCOM_SCM_BUFVAL, +}; + +#define QCOM_SCM_ARGS_IMPL(num, a, b, c, d, e, f, g, h, i, j, ...) (\ + (((a) & 0x3) << 4) | \ + (((b) & 0x3) << 6) | \ + (((c) & 0x3) << 8) | \ + (((d) & 0x3) << 10) | \ + (((e) & 0x3) << 12) | \ + (((f) & 0x3) << 14) | \ + (((g) & 0x3) << 16) | \ + (((h) & 0x3) << 18) | \ + (((i) & 0x3) << 20) | \ + (((j) & 0x3) << 22) | \ + ((num) & 0xf)) + +#define QCOM_SCM_ARGS(...) QCOM_SCM_ARGS_IMPL(__VA_ARGS__, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + +/** + * struct qcom_scm_desc + * @arginfo: Metadata describing the arguments in args[] + * @args: The array of arguments for the secure syscall + * @res: The values returned by the secure syscall + */ +struct qcom_scm_desc { + u32 arginfo; + u64 args[MAX_QCOM_SCM_ARGS]; +}; + +static u64 qcom_smccc_convention = -1; +static DEFINE_MUTEX(qcom_scm_lock); + +#define QCOM_SCM_EBUSY_WAIT_MS 30 +#define QCOM_SCM_EBUSY_MAX_RETRY 20 + +#define N_EXT_QCOM_SCM_ARGS 7 +#define FIRST_EXT_ARG_IDX 3 +#define N_REGISTER_ARGS (MAX_QCOM_SCM_ARGS - N_EXT_QCOM_SCM_ARGS + 1) + +/** + * qcom_scm_call() - Invoke a syscall in the secure world + * @dev: device + * @svc_id: service identifier + * @cmd_id: command identifier + * @desc: Descriptor structure containing arguments and return values + * + * Sends a command to the SCM and waits for the command to finish processing. + * This should *only* be called in pre-emptible context. +*/ +static int qcom_scm_call(struct device *dev, u32 svc_id, u32 cmd_id, + const struct qcom_scm_desc *desc, + struct arm_smccc_res *res) +{ + int arglen = desc->arginfo & 0xf; + int retry_count = 0, i; + u32 fn_id = QCOM_SCM_FNID(svc_id, cmd_id); + u64 cmd, x5 = desc->args[FIRST_EXT_ARG_IDX]; + dma_addr_t args_phys = 0; + void *args_virt = NULL; + size_t alloc_len; + + if (unlikely(arglen > N_REGISTER_ARGS)) { + alloc_len = N_EXT_QCOM_SCM_ARGS * sizeof(u64); + args_virt = kzalloc(PAGE_ALIGN(alloc_len), GFP_KERNEL); + + if (!args_virt) + return -ENOMEM; + + if (qcom_smccc_convention == ARM_SMCCC_SMC_32) { + __le32 *args = args_virt; + + for (i = 0; i < N_EXT_QCOM_SCM_ARGS; i++) + args[i] = cpu_to_le32(desc->args[i + + FIRST_EXT_ARG_IDX]); + } else { + __le64 *args = args_virt; + + for (i = 0; i < N_EXT_QCOM_SCM_ARGS; i++) + args[i] = cpu_to_le64(desc->args[i + + FIRST_EXT_ARG_IDX]); + } + + args_phys = dma_map_single(dev, args_virt, alloc_len, + DMA_TO_DEVICE); + + if (dma_mapping_error(dev, args_phys)) { + kfree(args_virt); + return -ENOMEM; + } + + x5 = args_phys; + } + + do { + mutex_lock(&qcom_scm_lock); + + cmd = ARM_SMCCC_CALL_VAL(ARM_SMCCC_STD_CALL, + qcom_smccc_convention, + ARM_SMCCC_OWNER_SIP, fn_id); + + do { + arm_smccc_smc(cmd, desc->arginfo, desc->args[0], + desc->args[1], desc->args[2], x5, 0, 0, + res); + } while (res->a0 == QCOM_SCM_INTERRUPTED); + + mutex_unlock(&qcom_scm_lock); + + if (res->a0 == QCOM_SCM_V2_EBUSY) { + if (retry_count++ > QCOM_SCM_EBUSY_MAX_RETRY) + break; + msleep(QCOM_SCM_EBUSY_WAIT_MS); + } + } while (res->a0 == QCOM_SCM_V2_EBUSY); + + if (args_virt) { + dma_unmap_single(dev, args_phys, alloc_len, DMA_TO_DEVICE); + kfree(args_virt); + } + + if (res->a0 < 0) + return qcom_scm_remap_error(res->a0); + + return 0; +} /** * qcom_scm_set_cold_boot_addr() - Set the cold boot address for cpus @@ -29,13 +172,15 @@ int __qcom_scm_set_cold_boot_addr(void *entry, const cpumask_t *cpus) /** * qcom_scm_set_warm_boot_addr() - Set the warm boot address for cpus + * @dev: Device pointer * @entry: Entry point function for the cpus * @cpus: The cpumask of cpus that will use the entry point * * Set the Linux entry point for the SCM to transfer control to when coming * out of a power down. CPU power down may be executed on cpuidle or hotplug. */ -int __qcom_scm_set_warm_boot_addr(void *entry, const cpumask_t *cpus) +int __qcom_scm_set_warm_boot_addr(struct device *dev, void *entry, + const cpumask_t *cpus) { return -ENOTSUPP; } @@ -52,12 +197,164 @@ void __qcom_scm_cpu_power_down(u32 flags) { } -int __qcom_scm_is_call_available(u32 svc_id, u32 cmd_id) +int __qcom_scm_is_call_available(struct device *dev, u32 svc_id, u32 cmd_id) { - return -ENOTSUPP; + int ret; + struct qcom_scm_desc desc = {0}; + struct arm_smccc_res res; + + desc.arginfo = QCOM_SCM_ARGS(1); + desc.args[0] = QCOM_SCM_FNID(svc_id, cmd_id) | + (ARM_SMCCC_OWNER_SIP << ARM_SMCCC_OWNER_SHIFT); + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_INFO, QCOM_IS_CALL_AVAIL_CMD, + &desc, &res); + + return ret ? : res.a1; } -int __qcom_scm_hdcp_req(struct qcom_scm_hdcp_req *req, u32 req_cnt, u32 *resp) +int __qcom_scm_hdcp_req(struct device *dev, struct qcom_scm_hdcp_req *req, + u32 req_cnt, u32 *resp) { - return -ENOTSUPP; + int ret; + struct qcom_scm_desc desc = {0}; + struct arm_smccc_res res; + + if (req_cnt > QCOM_SCM_HDCP_MAX_REQ_CNT) + return -ERANGE; + + desc.args[0] = req[0].addr; + desc.args[1] = req[0].val; + desc.args[2] = req[1].addr; + desc.args[3] = req[1].val; + desc.args[4] = req[2].addr; + desc.args[5] = req[2].val; + desc.args[6] = req[3].addr; + desc.args[7] = req[3].val; + desc.args[8] = req[4].addr; + desc.args[9] = req[4].val; + desc.arginfo = QCOM_SCM_ARGS(10); + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_HDCP, QCOM_SCM_CMD_HDCP, &desc, + &res); + *resp = res.a1; + + return ret; +} + +void __qcom_scm_init(void) +{ + u64 cmd; + struct arm_smccc_res res; + u32 function = QCOM_SCM_FNID(QCOM_SCM_SVC_INFO, QCOM_IS_CALL_AVAIL_CMD); + + /* First try a SMC64 call */ + cmd = ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, + ARM_SMCCC_OWNER_SIP, function); + + arm_smccc_smc(cmd, QCOM_SCM_ARGS(1), cmd & (~BIT(ARM_SMCCC_TYPE_SHIFT)), + 0, 0, 0, 0, 0, &res); + + if (!res.a0 && res.a1) + qcom_smccc_convention = ARM_SMCCC_SMC_64; + else + qcom_smccc_convention = ARM_SMCCC_SMC_32; +} + +bool __qcom_scm_pas_supported(struct device *dev, u32 peripheral) +{ + int ret; + struct qcom_scm_desc desc = {0}; + struct arm_smccc_res res; + + desc.args[0] = peripheral; + desc.arginfo = QCOM_SCM_ARGS(1); + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, + QCOM_SCM_PAS_IS_SUPPORTED_CMD, + &desc, &res); + + return ret ? false : !!res.a1; +} + +int __qcom_scm_pas_init_image(struct device *dev, u32 peripheral, + dma_addr_t metadata_phys) +{ + int ret; + struct qcom_scm_desc desc = {0}; + struct arm_smccc_res res; + + desc.args[0] = peripheral; + desc.args[1] = metadata_phys; + desc.arginfo = QCOM_SCM_ARGS(2, QCOM_SCM_VAL, QCOM_SCM_RW); + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, QCOM_SCM_PAS_INIT_IMAGE_CMD, + &desc, &res); + + return ret ? : res.a1; +} + +int __qcom_scm_pas_mem_setup(struct device *dev, u32 peripheral, + phys_addr_t addr, phys_addr_t size) +{ + int ret; + struct qcom_scm_desc desc = {0}; + struct arm_smccc_res res; + + desc.args[0] = peripheral; + desc.args[1] = addr; + desc.args[2] = size; + desc.arginfo = QCOM_SCM_ARGS(3); + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, QCOM_SCM_PAS_MEM_SETUP_CMD, + &desc, &res); + + return ret ? : res.a1; +} + +int __qcom_scm_pas_auth_and_reset(struct device *dev, u32 peripheral) +{ + int ret; + struct qcom_scm_desc desc = {0}; + struct arm_smccc_res res; + + desc.args[0] = peripheral; + desc.arginfo = QCOM_SCM_ARGS(1); + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, + QCOM_SCM_PAS_AUTH_AND_RESET_CMD, + &desc, &res); + + return ret ? : res.a1; +} + +int __qcom_scm_pas_shutdown(struct device *dev, u32 peripheral) +{ + int ret; + struct qcom_scm_desc desc = {0}; + struct arm_smccc_res res; + + desc.args[0] = peripheral; + desc.arginfo = QCOM_SCM_ARGS(1); + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, QCOM_SCM_PAS_SHUTDOWN_CMD, + &desc, &res); + + return ret ? : res.a1; +} + +int __qcom_scm_pas_mss_reset(struct device *dev, bool reset) +{ + struct qcom_scm_desc desc = {0}; + struct arm_smccc_res res; + int ret; + + desc.args[0] = reset; + desc.args[1] = 0; + desc.arginfo = QCOM_SCM_ARGS(2); + + ret = qcom_scm_call(dev, QCOM_SCM_SVC_PIL, QCOM_SCM_PAS_MSS_RESET, &desc, + &res); + + return ret ? : res.a1; } diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c index 45c008d68..e64a501ad 100644 --- a/drivers/firmware/qcom_scm.c +++ b/drivers/firmware/qcom_scm.c @@ -10,19 +10,64 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. */ - +#include +#include #include #include +#include #include #include +#include +#include +#include +#include #include "qcom_scm.h" +struct qcom_scm { + struct device *dev; + struct clk *core_clk; + struct clk *iface_clk; + struct clk *bus_clk; + struct reset_controller_dev reset; +}; + +static struct qcom_scm *__scm; + +static int qcom_scm_clk_enable(void) +{ + int ret; + + ret = clk_prepare_enable(__scm->core_clk); + if (ret) + goto bail; + + ret = clk_prepare_enable(__scm->iface_clk); + if (ret) + goto disable_core; + + ret = clk_prepare_enable(__scm->bus_clk); + if (ret) + goto disable_iface; + + return 0; + +disable_iface: + clk_disable_unprepare(__scm->iface_clk); +disable_core: + clk_disable_unprepare(__scm->core_clk); +bail: + return ret; +} + +static void qcom_scm_clk_disable(void) +{ + clk_disable_unprepare(__scm->core_clk); + clk_disable_unprepare(__scm->iface_clk); + clk_disable_unprepare(__scm->bus_clk); +} + /** * qcom_scm_set_cold_boot_addr() - Set the cold boot address for cpus * @entry: Entry point function for the cpus @@ -47,7 +92,7 @@ EXPORT_SYMBOL(qcom_scm_set_cold_boot_addr); */ int qcom_scm_set_warm_boot_addr(void *entry, const cpumask_t *cpus) { - return __qcom_scm_set_warm_boot_addr(entry, cpus); + return __qcom_scm_set_warm_boot_addr(__scm->dev, entry, cpus); } EXPORT_SYMBOL(qcom_scm_set_warm_boot_addr); @@ -72,12 +117,17 @@ EXPORT_SYMBOL(qcom_scm_cpu_power_down); */ bool qcom_scm_hdcp_available(void) { - int ret; + int ret = qcom_scm_clk_enable(); - ret = __qcom_scm_is_call_available(QCOM_SCM_SVC_HDCP, - QCOM_SCM_CMD_HDCP); + if (ret) + return ret; - return (ret > 0) ? true : false; + ret = __qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_HDCP, + QCOM_SCM_CMD_HDCP); + + qcom_scm_clk_disable(); + + return ret > 0 ? true : false; } EXPORT_SYMBOL(qcom_scm_hdcp_available); @@ -91,6 +141,287 @@ EXPORT_SYMBOL(qcom_scm_hdcp_available); */ int qcom_scm_hdcp_req(struct qcom_scm_hdcp_req *req, u32 req_cnt, u32 *resp) { - return __qcom_scm_hdcp_req(req, req_cnt, resp); + int ret = qcom_scm_clk_enable(); + + if (ret) + return ret; + + ret = __qcom_scm_hdcp_req(__scm->dev, req, req_cnt, resp); + qcom_scm_clk_disable(); + return ret; } EXPORT_SYMBOL(qcom_scm_hdcp_req); + +/** + * qcom_scm_pas_supported() - Check if the peripheral authentication service is + * available for the given peripherial + * @peripheral: peripheral id + * + * Returns true if PAS is supported for this peripheral, otherwise false. + */ +bool qcom_scm_pas_supported(u32 peripheral) +{ + int ret; + + ret = __qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_PIL, + QCOM_SCM_PAS_IS_SUPPORTED_CMD); + if (ret <= 0) + return false; + + return __qcom_scm_pas_supported(__scm->dev, peripheral); +} +EXPORT_SYMBOL(qcom_scm_pas_supported); + +/** + * qcom_scm_pas_init_image() - Initialize peripheral authentication service + * state machine for a given peripheral, using the + * metadata + * @peripheral: peripheral id + * @metadata: pointer to memory containing ELF header, program header table + * and optional blob of data used for authenticating the metadata + * and the rest of the firmware + * @size: size of the metadata + * + * Returns 0 on success. + */ +int qcom_scm_pas_init_image(u32 peripheral, const void *metadata, size_t size) +{ + dma_addr_t mdata_phys; + void *mdata_buf; + int ret; + + /* + * During the scm call memory protection will be enabled for the meta + * data blob, so make sure it's physically contiguous, 4K aligned and + * non-cachable to avoid XPU violations. + */ + mdata_buf = dma_alloc_coherent(__scm->dev, size, &mdata_phys, + GFP_KERNEL); + if (!mdata_buf) { + dev_err(__scm->dev, "Allocation of metadata buffer failed.\n"); + return -ENOMEM; + } + memcpy(mdata_buf, metadata, size); + + ret = qcom_scm_clk_enable(); + if (ret) + goto free_metadata; + + ret = __qcom_scm_pas_init_image(__scm->dev, peripheral, mdata_phys); + + qcom_scm_clk_disable(); + +free_metadata: + dma_free_coherent(__scm->dev, size, mdata_buf, mdata_phys); + + return ret; +} +EXPORT_SYMBOL(qcom_scm_pas_init_image); + +/** + * qcom_scm_pas_mem_setup() - Prepare the memory related to a given peripheral + * for firmware loading + * @peripheral: peripheral id + * @addr: start address of memory area to prepare + * @size: size of the memory area to prepare + * + * Returns 0 on success. + */ +int qcom_scm_pas_mem_setup(u32 peripheral, phys_addr_t addr, phys_addr_t size) +{ + int ret; + + ret = qcom_scm_clk_enable(); + if (ret) + return ret; + + ret = __qcom_scm_pas_mem_setup(__scm->dev, peripheral, addr, size); + qcom_scm_clk_disable(); + + return ret; +} +EXPORT_SYMBOL(qcom_scm_pas_mem_setup); + +/** + * qcom_scm_pas_auth_and_reset() - Authenticate the given peripheral firmware + * and reset the remote processor + * @peripheral: peripheral id + * + * Return 0 on success. + */ +int qcom_scm_pas_auth_and_reset(u32 peripheral) +{ + int ret; + + ret = qcom_scm_clk_enable(); + if (ret) + return ret; + + ret = __qcom_scm_pas_auth_and_reset(__scm->dev, peripheral); + qcom_scm_clk_disable(); + + return ret; +} +EXPORT_SYMBOL(qcom_scm_pas_auth_and_reset); + +/** + * qcom_scm_pas_shutdown() - Shut down the remote processor + * @peripheral: peripheral id + * + * Returns 0 on success. + */ +int qcom_scm_pas_shutdown(u32 peripheral) +{ + int ret; + + ret = qcom_scm_clk_enable(); + if (ret) + return ret; + + ret = __qcom_scm_pas_shutdown(__scm->dev, peripheral); + qcom_scm_clk_disable(); + + return ret; +} +EXPORT_SYMBOL(qcom_scm_pas_shutdown); + +static int qcom_scm_pas_reset_assert(struct reset_controller_dev *rcdev, + unsigned long idx) +{ + if (idx != 0) + return -EINVAL; + + return __qcom_scm_pas_mss_reset(__scm->dev, 1); +} + +static int qcom_scm_pas_reset_deassert(struct reset_controller_dev *rcdev, + unsigned long idx) +{ + if (idx != 0) + return -EINVAL; + + return __qcom_scm_pas_mss_reset(__scm->dev, 0); +} + +static const struct reset_control_ops qcom_scm_pas_reset_ops = { + .assert = qcom_scm_pas_reset_assert, + .deassert = qcom_scm_pas_reset_deassert, +}; + +/** + * qcom_scm_is_available() - Checks if SCM is available + */ +bool qcom_scm_is_available(void) +{ + return !!__scm; +} +EXPORT_SYMBOL(qcom_scm_is_available); + +static int qcom_scm_probe(struct platform_device *pdev) +{ + struct qcom_scm *scm; + int ret; + + scm = devm_kzalloc(&pdev->dev, sizeof(*scm), GFP_KERNEL); + if (!scm) + return -ENOMEM; + + scm->core_clk = devm_clk_get(&pdev->dev, "core"); + if (IS_ERR(scm->core_clk)) { + if (PTR_ERR(scm->core_clk) == -EPROBE_DEFER) + return PTR_ERR(scm->core_clk); + + scm->core_clk = NULL; + } + + if (of_device_is_compatible(pdev->dev.of_node, "qcom,scm")) { + scm->iface_clk = devm_clk_get(&pdev->dev, "iface"); + if (IS_ERR(scm->iface_clk)) { + if (PTR_ERR(scm->iface_clk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "failed to acquire iface clk\n"); + return PTR_ERR(scm->iface_clk); + } + + scm->bus_clk = devm_clk_get(&pdev->dev, "bus"); + if (IS_ERR(scm->bus_clk)) { + if (PTR_ERR(scm->bus_clk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "failed to acquire bus clk\n"); + return PTR_ERR(scm->bus_clk); + } + } + + scm->reset.ops = &qcom_scm_pas_reset_ops; + scm->reset.nr_resets = 1; + scm->reset.of_node = pdev->dev.of_node; + reset_controller_register(&scm->reset); + + /* vote for max clk rate for highest performance */ + ret = clk_set_rate(scm->core_clk, INT_MAX); + if (ret) + return ret; + + __scm = scm; + __scm->dev = &pdev->dev; + + __qcom_scm_init(); + + return 0; +} + +static const struct of_device_id qcom_scm_dt_match[] = { + { .compatible = "qcom,scm-apq8064",}, + { .compatible = "qcom,scm-msm8660",}, + { .compatible = "qcom,scm-msm8960",}, + { .compatible = "qcom,scm",}, + {} +}; + +MODULE_DEVICE_TABLE(of, qcom_scm_dt_match); + +static struct platform_driver qcom_scm_driver = { + .driver = { + .name = "qcom_scm", + .of_match_table = qcom_scm_dt_match, + }, + .probe = qcom_scm_probe, +}; + +static int __init qcom_scm_init(void) +{ + struct device_node *np, *fw_np; + int ret; + + fw_np = of_find_node_by_name(NULL, "firmware"); + + if (!fw_np) + return -ENODEV; + + np = of_find_matching_node(fw_np, qcom_scm_dt_match); + + if (!np) { + of_node_put(fw_np); + return -ENODEV; + } + + of_node_put(np); + + ret = of_platform_populate(fw_np, qcom_scm_dt_match, NULL, NULL); + + of_node_put(fw_np); + + if (ret) + return ret; + + return platform_driver_register(&qcom_scm_driver); +} + +subsys_initcall(qcom_scm_init); + +static void __exit qcom_scm_exit(void) +{ + platform_driver_unregister(&qcom_scm_driver); +} +module_exit(qcom_scm_exit); + +MODULE_DESCRIPTION("Qualcomm SCM driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/firmware/qcom_scm.h b/drivers/firmware/qcom_scm.h index 2cce75c08..3584b00fe 100644 --- a/drivers/firmware/qcom_scm.h +++ b/drivers/firmware/qcom_scm.h @@ -19,7 +19,8 @@ #define QCOM_SCM_FLAG_HLOS 0x01 #define QCOM_SCM_FLAG_COLDBOOT_MC 0x02 #define QCOM_SCM_FLAG_WARMBOOT_MC 0x04 -extern int __qcom_scm_set_warm_boot_addr(void *entry, const cpumask_t *cpus); +extern int __qcom_scm_set_warm_boot_addr(struct device *dev, void *entry, + const cpumask_t *cpus); extern int __qcom_scm_set_cold_boot_addr(void *entry, const cpumask_t *cpus); #define QCOM_SCM_CMD_TERMINATE_PC 0x2 @@ -29,14 +30,34 @@ extern void __qcom_scm_cpu_power_down(u32 flags); #define QCOM_SCM_SVC_INFO 0x6 #define QCOM_IS_CALL_AVAIL_CMD 0x1 -extern int __qcom_scm_is_call_available(u32 svc_id, u32 cmd_id); +extern int __qcom_scm_is_call_available(struct device *dev, u32 svc_id, + u32 cmd_id); #define QCOM_SCM_SVC_HDCP 0x11 #define QCOM_SCM_CMD_HDCP 0x01 -extern int __qcom_scm_hdcp_req(struct qcom_scm_hdcp_req *req, u32 req_cnt, - u32 *resp); +extern int __qcom_scm_hdcp_req(struct device *dev, + struct qcom_scm_hdcp_req *req, u32 req_cnt, u32 *resp); + +extern void __qcom_scm_init(void); + +#define QCOM_SCM_SVC_PIL 0x2 +#define QCOM_SCM_PAS_INIT_IMAGE_CMD 0x1 +#define QCOM_SCM_PAS_MEM_SETUP_CMD 0x2 +#define QCOM_SCM_PAS_AUTH_AND_RESET_CMD 0x5 +#define QCOM_SCM_PAS_SHUTDOWN_CMD 0x6 +#define QCOM_SCM_PAS_IS_SUPPORTED_CMD 0x7 +#define QCOM_SCM_PAS_MSS_RESET 0xa +extern bool __qcom_scm_pas_supported(struct device *dev, u32 peripheral); +extern int __qcom_scm_pas_init_image(struct device *dev, u32 peripheral, + dma_addr_t metadata_phys); +extern int __qcom_scm_pas_mem_setup(struct device *dev, u32 peripheral, + phys_addr_t addr, phys_addr_t size); +extern int __qcom_scm_pas_auth_and_reset(struct device *dev, u32 peripheral); +extern int __qcom_scm_pas_shutdown(struct device *dev, u32 peripheral); +extern int __qcom_scm_pas_mss_reset(struct device *dev, bool reset); /* common error codes */ +#define QCOM_SCM_V2_EBUSY -12 #define QCOM_SCM_ENOMEM -5 #define QCOM_SCM_EOPNOTSUPP -4 #define QCOM_SCM_EINVAL_ADDR -3 @@ -44,4 +65,22 @@ extern int __qcom_scm_hdcp_req(struct qcom_scm_hdcp_req *req, u32 req_cnt, #define QCOM_SCM_ERROR -1 #define QCOM_SCM_INTERRUPTED 1 +static inline int qcom_scm_remap_error(int err) +{ + switch (err) { + case QCOM_SCM_ERROR: + return -EIO; + case QCOM_SCM_EINVAL_ADDR: + case QCOM_SCM_EINVAL_ARG: + return -EINVAL; + case QCOM_SCM_EOPNOTSUPP: + return -EOPNOTSUPP; + case QCOM_SCM_ENOMEM: + return -ENOMEM; + case QCOM_SCM_V2_EBUSY: + return -EBUSY; + } + return -EINVAL; +} + #endif diff --git a/drivers/firmware/scpi_pm_domain.c b/drivers/firmware/scpi_pm_domain.c new file mode 100644 index 000000000..f395dec27 --- /dev/null +++ b/drivers/firmware/scpi_pm_domain.c @@ -0,0 +1,163 @@ +/* + * SCPI Generic power domain support. + * + * Copyright (C) 2016 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include + +struct scpi_pm_domain { + struct generic_pm_domain genpd; + struct scpi_ops *ops; + u32 domain; + char name[30]; +}; + +/* + * These device power state values are not well-defined in the specification. + * In case, different implementations use different values, we can make these + * specific to compatibles rather than getting these values from device tree. + */ +enum scpi_power_domain_state { + SCPI_PD_STATE_ON = 0, + SCPI_PD_STATE_OFF = 3, +}; + +#define to_scpi_pd(gpd) container_of(gpd, struct scpi_pm_domain, genpd) + +static int scpi_pd_power(struct scpi_pm_domain *pd, bool power_on) +{ + int ret; + enum scpi_power_domain_state state; + + if (power_on) + state = SCPI_PD_STATE_ON; + else + state = SCPI_PD_STATE_OFF; + + ret = pd->ops->device_set_power_state(pd->domain, state); + if (ret) + return ret; + + return !(state == pd->ops->device_get_power_state(pd->domain)); +} + +static int scpi_pd_power_on(struct generic_pm_domain *domain) +{ + struct scpi_pm_domain *pd = to_scpi_pd(domain); + + return scpi_pd_power(pd, true); +} + +static int scpi_pd_power_off(struct generic_pm_domain *domain) +{ + struct scpi_pm_domain *pd = to_scpi_pd(domain); + + return scpi_pd_power(pd, false); +} + +static int scpi_pm_domain_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct scpi_pm_domain *scpi_pd; + struct genpd_onecell_data *scpi_pd_data; + struct generic_pm_domain **domains; + struct scpi_ops *scpi_ops; + int ret, num_domains, i; + + scpi_ops = get_scpi_ops(); + if (!scpi_ops) + return -EPROBE_DEFER; + + if (!np) { + dev_err(dev, "device tree node not found\n"); + return -ENODEV; + } + + if (!scpi_ops->device_set_power_state || + !scpi_ops->device_get_power_state) { + dev_err(dev, "power domains not supported in the firmware\n"); + return -ENODEV; + } + + ret = of_property_read_u32(np, "num-domains", &num_domains); + if (ret) { + dev_err(dev, "number of domains not found\n"); + return -EINVAL; + } + + scpi_pd = devm_kcalloc(dev, num_domains, sizeof(*scpi_pd), GFP_KERNEL); + if (!scpi_pd) + return -ENOMEM; + + scpi_pd_data = devm_kzalloc(dev, sizeof(*scpi_pd_data), GFP_KERNEL); + if (!scpi_pd_data) + return -ENOMEM; + + domains = devm_kcalloc(dev, num_domains, sizeof(*domains), GFP_KERNEL); + if (!domains) + return -ENOMEM; + + for (i = 0; i < num_domains; i++, scpi_pd++) { + domains[i] = &scpi_pd->genpd; + + scpi_pd->domain = i; + scpi_pd->ops = scpi_ops; + sprintf(scpi_pd->name, "%s.%d", np->name, i); + scpi_pd->genpd.name = scpi_pd->name; + scpi_pd->genpd.power_off = scpi_pd_power_off; + scpi_pd->genpd.power_on = scpi_pd_power_on; + + /* + * Treat all power domains as off at boot. + * + * The SCP firmware itself may have switched on some domains, + * but for reference counting purpose, keep it this way. + */ + pm_genpd_init(&scpi_pd->genpd, NULL, true); + } + + scpi_pd_data->domains = domains; + scpi_pd_data->num_domains = num_domains; + + of_genpd_add_provider_onecell(np, scpi_pd_data); + + return 0; +} + +static const struct of_device_id scpi_power_domain_ids[] = { + { .compatible = "arm,scpi-power-domains", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, scpi_power_domain_ids); + +static struct platform_driver scpi_power_domain_driver = { + .driver = { + .name = "scpi_power_domain", + .of_match_table = scpi_power_domain_ids, + }, + .probe = scpi_pm_domain_probe, +}; +module_platform_driver(scpi_power_domain_driver); + +MODULE_AUTHOR("Sudeep Holla "); +MODULE_DESCRIPTION("ARM SCPI power domain driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig index c9b9fdf6c..d61410299 100644 --- a/drivers/fpga/Kconfig +++ b/drivers/fpga/Kconfig @@ -21,6 +21,7 @@ config FPGA_MGR_SOCFPGA config FPGA_MGR_ZYNQ_FPGA tristate "Xilinx Zynq FPGA" + depends on HAS_DMA help FPGA manager driver support for Xilinx Zynq FPGAs. diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 5d457ff61..24caedb00 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -189,7 +189,7 @@ config GPIO_EP93XX config GPIO_ETRAXFS bool "Axis ETRAX FS General I/O" depends on CRIS || COMPILE_TEST - depends on OF + depends on OF_GPIO select GPIO_GENERIC select GPIOLIB_IRQCHIP help @@ -215,7 +215,7 @@ config GPIO_GENERIC_PLATFORM config GPIO_GRGPIO tristate "Aeroflex Gaisler GRGPIO support" - depends on OF + depends on OF_GPIO select GPIO_GENERIC select IRQ_DOMAIN help @@ -251,7 +251,7 @@ config GPIO_LOONGSON driver for GPIO functionality on Loongson-2F/3A/3B processors. config GPIO_LPC18XX - bool "NXP LPC18XX/43XX GPIO support" + tristate "NXP LPC18XX/43XX GPIO support" default y if ARCH_LPC18XX depends on OF_GPIO && (ARCH_LPC18XX || COMPILE_TEST) help @@ -313,7 +313,7 @@ config GPIO_MPC8XXX config GPIO_MVEBU def_bool y depends on PLAT_ORION - depends on OF + depends on OF_GPIO select GENERIC_IRQ_CHIP config GPIO_MXC @@ -406,7 +406,7 @@ config GPIO_TEGRA bool "NVIDIA Tegra GPIO support" default ARCH_TEGRA depends on ARCH_TEGRA || COMPILE_TEST - depends on OF + depends on OF_GPIO help Say yes here to support GPIO pins on NVIDIA Tegra SoCs. @@ -875,6 +875,15 @@ config GPIO_LP3943 LP3943 can be used as a GPIO expander which provides up to 16 GPIOs. Open drain outputs are required for this usage. +config GPIO_MAX77620 + tristate "GPIO support for PMIC MAX77620 and MAX20024" + depends on MFD_MAX77620 + help + GPIO driver for MAX77620 and MAX20024 PMIC from Maxim Semiconductor. + MAX77620 PMIC has 8 pins that can be configured as GPIOs. The + driver also provides interrupt support for each of the gpios. + Say yes here to enable the max77620 to be used as gpio controller. + config GPIO_MSIC bool "Intel MSIC mixed signal gpio support" depends on MFD_INTEL_MSIC @@ -1030,11 +1039,18 @@ config GPIO_BT8XX If unsure, say N. config GPIO_INTEL_MID - bool "Intel Mid GPIO support" - depends on X86 + bool "Intel MID GPIO support" + depends on X86_INTEL_MID + select GPIOLIB_IRQCHIP + help + Say Y here to support Intel MID GPIO. + +config GPIO_MERRIFIELD + tristate "Intel Merrifield GPIO support" + depends on X86_INTEL_MID select GPIOLIB_IRQCHIP help - Say Y here to support Intel Mid GPIO. + Say Y here to support Intel Merrifield GPIO. config GPIO_ML_IOH tristate "OKI SEMICONDUCTOR ML7213 IOH GPIO support" @@ -1084,7 +1100,7 @@ menu "SPI GPIO expanders" config GPIO_74X164 tristate "74x164 serial-in/parallel-out 8-bits shift register" - depends on OF + depends on OF_GPIO help Driver for 74x164 compatible serial-in/parallel-out 8-outputs shift registers. This driver can be used to provide access @@ -1115,6 +1131,7 @@ menu "SPI or I2C GPIO expanders" config GPIO_MCP23S08 tristate "Microchip MCP23xxx I/O expander" + depends on OF_GPIO select GPIOLIB_IRQCHIP help SPI/I2C driver for Microchip MCP23S08/MCP23S17/MCP23008/MCP23017 diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index 991598ea3..2a035ed8f 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -61,8 +61,10 @@ obj-$(CONFIG_GPIO_MAX730X) += gpio-max730x.o obj-$(CONFIG_GPIO_MAX7300) += gpio-max7300.o obj-$(CONFIG_GPIO_MAX7301) += gpio-max7301.o obj-$(CONFIG_GPIO_MAX732X) += gpio-max732x.o +obj-$(CONFIG_GPIO_MAX77620) += gpio-max77620.o obj-$(CONFIG_GPIO_MB86S7X) += gpio-mb86s7x.o obj-$(CONFIG_GPIO_MENZ127) += gpio-menz127.o +obj-$(CONFIG_GPIO_MERRIFIELD) += gpio-merrifield.o obj-$(CONFIG_GPIO_MC33880) += gpio-mc33880.o obj-$(CONFIG_GPIO_MC9S08DZ60) += gpio-mc9s08dz60.o obj-$(CONFIG_GPIO_MCP23S08) += gpio-mcp23s08.o diff --git a/drivers/gpio/gpio-74x164.c b/drivers/gpio/gpio-74x164.c index 80f9ddf13..a6607faf2 100644 --- a/drivers/gpio/gpio-74x164.c +++ b/drivers/gpio/gpio-74x164.c @@ -35,13 +35,8 @@ struct gen_74x164_chip { static int __gen_74x164_write_config(struct gen_74x164_chip *chip) { - struct spi_transfer xfer = { - .tx_buf = chip->buffer, - .len = chip->registers, - }; - - return spi_sync_transfer(to_spi_device(chip->gpio_chip.parent), - &xfer, 1); + return spi_write(to_spi_device(chip->gpio_chip.parent), chip->buffer, + chip->registers); } static int gen_74x164_get_value(struct gpio_chip *gc, unsigned offset) diff --git a/drivers/gpio/gpio-clps711x.c b/drivers/gpio/gpio-clps711x.c index 5a690256a..52fd63f02 100644 --- a/drivers/gpio/gpio-clps711x.c +++ b/drivers/gpio/gpio-clps711x.c @@ -20,8 +20,12 @@ static int clps711x_gpio_probe(struct platform_device *pdev) void __iomem *dat, *dir; struct gpio_chip *gc; struct resource *res; - int err, id = np ? of_alias_get_id(np, "gpio") : pdev->id; + int err, id; + if (!np) + return -ENODEV; + + id = of_alias_get_id(np, "gpio"); if ((id < 0) || (id > 4)) return -ENODEV; @@ -63,7 +67,7 @@ static int clps711x_gpio_probe(struct platform_device *pdev) break; } - gc->base = id * 8; + gc->base = -1; gc->owner = THIS_MODULE; platform_set_drvdata(pdev, gc); @@ -71,7 +75,7 @@ static int clps711x_gpio_probe(struct platform_device *pdev) } static const struct of_device_id __maybe_unused clps711x_gpio_ids[] = { - { .compatible = "cirrus,clps711x-gpio" }, + { .compatible = "cirrus,ep7209-gpio" }, { } }; MODULE_DEVICE_TABLE(of, clps711x_gpio_ids); diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c index 34779bb37..6193f62c0 100644 --- a/drivers/gpio/gpio-dwapb.c +++ b/drivers/gpio/gpio-dwapb.c @@ -486,6 +486,7 @@ dwapb_gpio_get_pdata(struct device *dev) pp->idx >= DWAPB_MAX_PORTS) { dev_err(dev, "missing/invalid port index for port%d\n", i); + fwnode_handle_put(fwnode); return ERR_PTR(-EINVAL); } diff --git a/drivers/gpio/gpio-f7188x.c b/drivers/gpio/gpio-f7188x.c index 05aa538c3..600be8418 100644 --- a/drivers/gpio/gpio-f7188x.c +++ b/drivers/gpio/gpio-f7188x.c @@ -125,6 +125,7 @@ static inline void superio_exit(int base) * GPIO chip. */ +static int f7188x_gpio_get_direction(struct gpio_chip *chip, unsigned offset); static int f7188x_gpio_direction_in(struct gpio_chip *chip, unsigned offset); static int f7188x_gpio_get(struct gpio_chip *chip, unsigned offset); static int f7188x_gpio_direction_out(struct gpio_chip *chip, @@ -139,6 +140,7 @@ static int f7188x_gpio_set_single_ended(struct gpio_chip *gc, .chip = { \ .label = DRVNAME, \ .owner = THIS_MODULE, \ + .get_direction = f7188x_gpio_get_direction, \ .direction_input = f7188x_gpio_direction_in, \ .get = f7188x_gpio_get, \ .direction_output = f7188x_gpio_direction_out, \ @@ -209,6 +211,26 @@ static struct f7188x_gpio_bank f81866_gpio_bank[] = { F7188X_GPIO_BANK(80, 8, 0x88), }; +static int f7188x_gpio_get_direction(struct gpio_chip *chip, unsigned offset) +{ + int err; + struct f7188x_gpio_bank *bank = + container_of(chip, struct f7188x_gpio_bank, chip); + struct f7188x_sio *sio = bank->data->sio; + u8 dir; + + err = superio_enter(sio->addr); + if (err) + return err; + superio_select(sio->addr, SIO_LD_GPIO); + + dir = superio_inb(sio->addr, gpio_dir(bank->regbase)); + + superio_exit(sio->addr); + + return !(dir & 1 << offset); +} + static int f7188x_gpio_direction_in(struct gpio_chip *chip, unsigned offset) { int err; diff --git a/drivers/gpio/gpio-intel-mid.c b/drivers/gpio/gpio-intel-mid.c index c0f7cce23..164de64b1 100644 --- a/drivers/gpio/gpio-intel-mid.c +++ b/drivers/gpio/gpio-intel-mid.c @@ -1,7 +1,7 @@ /* * Intel MID GPIO driver * - * Copyright (c) 2008-2014 Intel Corporation. + * Copyright (c) 2008-2014,2016 Intel Corporation. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -19,18 +19,18 @@ * Clovertrail platform Cloverview chip. */ -#include -#include -#include -#include #include -#include -#include #include +#include #include #include -#include +#include +#include +#include +#include #include +#include +#include #define INTEL_MID_IRQ_TYPE_EDGE (1 << 0) #define INTEL_MID_IRQ_TYPE_LEVEL (1 << 1) diff --git a/drivers/gpio/gpio-lynxpoint.c b/drivers/gpio/gpio-lynxpoint.c index 9df015e85..fbd393b46 100644 --- a/drivers/gpio/gpio-lynxpoint.c +++ b/drivers/gpio/gpio-lynxpoint.c @@ -383,7 +383,6 @@ static int lp_gpio_probe(struct platform_device *pdev) handle_simple_irq, IRQ_TYPE_NONE); if (ret) { dev_err(dev, "failed to add irqchip\n"); - gpiochip_remove(gc); return ret; } diff --git a/drivers/gpio/gpio-max77620.c b/drivers/gpio/gpio-max77620.c new file mode 100644 index 000000000..b46b436cb --- /dev/null +++ b/drivers/gpio/gpio-max77620.c @@ -0,0 +1,315 @@ +/* + * MAXIM MAX77620 GPIO driver + * + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include + +#define GPIO_REG_ADDR(offset) (MAX77620_REG_GPIO0 + offset) + +struct max77620_gpio { + struct gpio_chip gpio_chip; + struct regmap *rmap; + struct device *dev; + int gpio_irq; + int irq_base; + int gpio_base; +}; + +static const struct regmap_irq max77620_gpio_irqs[] = { + [0] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE0, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 0, + }, + [1] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE1, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 1, + }, + [2] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE2, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 2, + }, + [3] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE3, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 3, + }, + [4] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE4, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 4, + }, + [5] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE5, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 5, + }, + [6] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE6, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 6, + }, + [7] = { + .mask = MAX77620_IRQ_LVL2_GPIO_EDGE7, + .type_rising_mask = MAX77620_CNFG_GPIO_INT_RISING, + .type_falling_mask = MAX77620_CNFG_GPIO_INT_FALLING, + .reg_offset = 0, + .type_reg_offset = 7, + }, +}; + +static struct regmap_irq_chip max77620_gpio_irq_chip = { + .name = "max77620-gpio", + .irqs = max77620_gpio_irqs, + .num_irqs = ARRAY_SIZE(max77620_gpio_irqs), + .num_regs = 1, + .num_type_reg = 8, + .irq_reg_stride = 1, + .type_reg_stride = 1, + .status_base = MAX77620_REG_IRQ_LVL2_GPIO, + .type_base = MAX77620_REG_GPIO0, +}; + +static int max77620_gpio_dir_input(struct gpio_chip *gc, unsigned int offset) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + int ret; + + ret = regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_DIR_MASK, + MAX77620_CNFG_GPIO_DIR_INPUT); + if (ret < 0) + dev_err(mgpio->dev, "CNFG_GPIOx dir update failed: %d\n", ret); + + return ret; +} + +static int max77620_gpio_get(struct gpio_chip *gc, unsigned int offset) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + unsigned int val; + int ret; + + ret = regmap_read(mgpio->rmap, GPIO_REG_ADDR(offset), &val); + if (ret < 0) { + dev_err(mgpio->dev, "CNFG_GPIOx read failed: %d\n", ret); + return ret; + } + + if (val & MAX77620_CNFG_GPIO_DIR_MASK) + return !!(val & MAX77620_CNFG_GPIO_INPUT_VAL_MASK); + else + return !!(val & MAX77620_CNFG_GPIO_OUTPUT_VAL_MASK); +} + +static int max77620_gpio_dir_output(struct gpio_chip *gc, unsigned int offset, + int value) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + u8 val; + int ret; + + val = (value) ? MAX77620_CNFG_GPIO_OUTPUT_VAL_HIGH : + MAX77620_CNFG_GPIO_OUTPUT_VAL_LOW; + + ret = regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_OUTPUT_VAL_MASK, val); + if (ret < 0) { + dev_err(mgpio->dev, "CNFG_GPIOx val update failed: %d\n", ret); + return ret; + } + + ret = regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_DIR_MASK, + MAX77620_CNFG_GPIO_DIR_OUTPUT); + if (ret < 0) + dev_err(mgpio->dev, "CNFG_GPIOx dir update failed: %d\n", ret); + + return ret; +} + +static int max77620_gpio_set_debounce(struct gpio_chip *gc, + unsigned int offset, + unsigned int debounce) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + u8 val; + int ret; + + switch (debounce) { + case 0: + val = MAX77620_CNFG_GPIO_DBNC_None; + break; + case 1 ... 8: + val = MAX77620_CNFG_GPIO_DBNC_8ms; + break; + case 9 ... 16: + val = MAX77620_CNFG_GPIO_DBNC_16ms; + break; + case 17 ... 32: + val = MAX77620_CNFG_GPIO_DBNC_32ms; + break; + default: + dev_err(mgpio->dev, "Illegal value %u\n", debounce); + return -EINVAL; + } + + ret = regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_DBNC_MASK, val); + if (ret < 0) + dev_err(mgpio->dev, "CNFG_GPIOx_DBNC update failed: %d\n", ret); + + return ret; +} + +static void max77620_gpio_set(struct gpio_chip *gc, unsigned int offset, + int value) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + u8 val; + int ret; + + val = (value) ? MAX77620_CNFG_GPIO_OUTPUT_VAL_HIGH : + MAX77620_CNFG_GPIO_OUTPUT_VAL_LOW; + + ret = regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_OUTPUT_VAL_MASK, val); + if (ret < 0) + dev_err(mgpio->dev, "CNFG_GPIO_OUT update failed: %d\n", ret); +} + +static int max77620_gpio_set_single_ended(struct gpio_chip *gc, + unsigned int offset, + enum single_ended_mode mode) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + + switch (mode) { + case LINE_MODE_OPEN_DRAIN: + return regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_DRV_MASK, + MAX77620_CNFG_GPIO_DRV_OPENDRAIN); + case LINE_MODE_PUSH_PULL: + return regmap_update_bits(mgpio->rmap, GPIO_REG_ADDR(offset), + MAX77620_CNFG_GPIO_DRV_MASK, + MAX77620_CNFG_GPIO_DRV_PUSHPULL); + default: + break; + } + + return -ENOTSUPP; +} + +static int max77620_gpio_to_irq(struct gpio_chip *gc, unsigned int offset) +{ + struct max77620_gpio *mgpio = gpiochip_get_data(gc); + struct max77620_chip *chip = dev_get_drvdata(mgpio->dev->parent); + + return regmap_irq_get_virq(chip->gpio_irq_data, offset); +} + +static int max77620_gpio_probe(struct platform_device *pdev) +{ + struct max77620_chip *chip = dev_get_drvdata(pdev->dev.parent); + struct max77620_gpio *mgpio; + int gpio_irq; + int ret; + + gpio_irq = platform_get_irq(pdev, 0); + if (gpio_irq <= 0) { + dev_err(&pdev->dev, "GPIO irq not available %d\n", gpio_irq); + return -ENODEV; + } + + mgpio = devm_kzalloc(&pdev->dev, sizeof(*mgpio), GFP_KERNEL); + if (!mgpio) + return -ENOMEM; + + mgpio->rmap = chip->rmap; + mgpio->dev = &pdev->dev; + mgpio->gpio_irq = gpio_irq; + + mgpio->gpio_chip.label = pdev->name; + mgpio->gpio_chip.parent = &pdev->dev; + mgpio->gpio_chip.direction_input = max77620_gpio_dir_input; + mgpio->gpio_chip.get = max77620_gpio_get; + mgpio->gpio_chip.direction_output = max77620_gpio_dir_output; + mgpio->gpio_chip.set_debounce = max77620_gpio_set_debounce; + mgpio->gpio_chip.set = max77620_gpio_set; + mgpio->gpio_chip.set_single_ended = max77620_gpio_set_single_ended; + mgpio->gpio_chip.to_irq = max77620_gpio_to_irq; + mgpio->gpio_chip.ngpio = MAX77620_GPIO_NR; + mgpio->gpio_chip.can_sleep = 1; + mgpio->gpio_chip.base = -1; + mgpio->irq_base = -1; +#ifdef CONFIG_OF_GPIO + mgpio->gpio_chip.of_node = pdev->dev.parent->of_node; +#endif + + platform_set_drvdata(pdev, mgpio); + + ret = devm_gpiochip_add_data(&pdev->dev, &mgpio->gpio_chip, mgpio); + if (ret < 0) { + dev_err(&pdev->dev, "gpio_init: Failed to add max77620_gpio\n"); + return ret; + } + + mgpio->gpio_base = mgpio->gpio_chip.base; + ret = devm_regmap_add_irq_chip(&pdev->dev, chip->rmap, mgpio->gpio_irq, + IRQF_ONESHOT, mgpio->irq_base, + &max77620_gpio_irq_chip, + &chip->gpio_irq_data); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to add gpio irq_chip %d\n", ret); + return ret; + } + + return 0; +} + +static const struct platform_device_id max77620_gpio_devtype[] = { + { .name = "max77620-gpio", }, + {}, +}; +MODULE_DEVICE_TABLE(platform, max77620_gpio_devtype); + +static struct platform_driver max77620_gpio_driver = { + .driver.name = "max77620-gpio", + .probe = max77620_gpio_probe, + .id_table = max77620_gpio_devtype, +}; + +module_platform_driver(max77620_gpio_driver); + +MODULE_DESCRIPTION("GPIO interface for MAX77620 and MAX20024 PMIC"); +MODULE_AUTHOR("Laxman Dewangan "); +MODULE_AUTHOR("Chaitanya Bandi "); +MODULE_ALIAS("platform:max77620-gpio"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpio/gpio-mcp23s08.c b/drivers/gpio/gpio-mcp23s08.c index ac22efc18..99d37b56c 100644 --- a/drivers/gpio/gpio-mcp23s08.c +++ b/drivers/gpio/gpio-mcp23s08.c @@ -564,7 +564,7 @@ static int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev, mcp->chip.direction_output = mcp23s08_direction_output; mcp->chip.set = mcp23s08_set; mcp->chip.dbg_show = mcp23s08_dbg_show; -#ifdef CONFIG_OF +#ifdef CONFIG_OF_GPIO mcp->chip.of_gpio_n_cells = 2; mcp->chip.of_node = dev->of_node; #endif diff --git a/drivers/gpio/gpio-menz127.c b/drivers/gpio/gpio-menz127.c index cc103aff4..a1210e330 100644 --- a/drivers/gpio/gpio-menz127.c +++ b/drivers/gpio/gpio-menz127.c @@ -187,7 +187,6 @@ MODULE_DEVICE_TABLE(mcb, men_z127_ids); static struct mcb_driver men_z127_driver = { .driver = { .name = "z127-gpio", - .owner = THIS_MODULE, }, .probe = men_z127_probe, .remove = men_z127_remove, diff --git a/drivers/gpio/gpio-merrifield.c b/drivers/gpio/gpio-merrifield.c new file mode 100644 index 000000000..45b51278b --- /dev/null +++ b/drivers/gpio/gpio-merrifield.c @@ -0,0 +1,444 @@ +/* + * Intel Merrifield SoC GPIO driver + * + * Copyright (c) 2016 Intel Corporation. + * Author: Andy Shevchenko + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define GCCR 0x000 /* controller configuration */ +#define GPLR 0x004 /* pin level r/o */ +#define GPDR 0x01c /* pin direction */ +#define GPSR 0x034 /* pin set w/o */ +#define GPCR 0x04c /* pin clear w/o */ +#define GRER 0x064 /* rising edge detect */ +#define GFER 0x07c /* falling edge detect */ +#define GFBR 0x094 /* glitch filter bypass */ +#define GIMR 0x0ac /* interrupt mask */ +#define GISR 0x0c4 /* interrupt source */ +#define GITR 0x300 /* input type */ +#define GLPR 0x318 /* level input polarity */ +#define GWMR 0x400 /* wake mask */ +#define GWSR 0x418 /* wake source */ +#define GSIR 0xc00 /* secure input */ + +/* Intel Merrifield has 192 GPIO pins */ +#define MRFLD_NGPIO 192 + +struct mrfld_gpio_pinrange { + unsigned int gpio_base; + unsigned int pin_base; + unsigned int npins; +}; + +#define GPIO_PINRANGE(gstart, gend, pstart) \ + { \ + .gpio_base = (gstart), \ + .pin_base = (pstart), \ + .npins = (gend) - (gstart) + 1, \ + } + +struct mrfld_gpio { + struct gpio_chip chip; + void __iomem *reg_base; + raw_spinlock_t lock; + struct device *dev; +}; + +static const struct mrfld_gpio_pinrange mrfld_gpio_ranges[] = { + GPIO_PINRANGE(0, 11, 146), + GPIO_PINRANGE(12, 13, 144), + GPIO_PINRANGE(14, 15, 35), + GPIO_PINRANGE(16, 16, 164), + GPIO_PINRANGE(17, 18, 105), + GPIO_PINRANGE(19, 22, 101), + GPIO_PINRANGE(23, 30, 107), + GPIO_PINRANGE(32, 43, 67), + GPIO_PINRANGE(44, 63, 195), + GPIO_PINRANGE(64, 67, 140), + GPIO_PINRANGE(68, 69, 165), + GPIO_PINRANGE(70, 71, 65), + GPIO_PINRANGE(72, 76, 228), + GPIO_PINRANGE(77, 86, 37), + GPIO_PINRANGE(87, 87, 48), + GPIO_PINRANGE(88, 88, 47), + GPIO_PINRANGE(89, 96, 49), + GPIO_PINRANGE(97, 97, 34), + GPIO_PINRANGE(102, 119, 83), + GPIO_PINRANGE(120, 123, 79), + GPIO_PINRANGE(124, 135, 115), + GPIO_PINRANGE(137, 142, 158), + GPIO_PINRANGE(154, 163, 24), + GPIO_PINRANGE(164, 176, 215), + GPIO_PINRANGE(177, 189, 127), + GPIO_PINRANGE(190, 191, 178), +}; + +static void __iomem *gpio_reg(struct gpio_chip *chip, unsigned int offset, + unsigned int reg_type_offset) +{ + struct mrfld_gpio *priv = gpiochip_get_data(chip); + u8 reg = offset / 32; + + return priv->reg_base + reg_type_offset + reg * 4; +} + +static int mrfld_gpio_get(struct gpio_chip *chip, unsigned int offset) +{ + void __iomem *gplr = gpio_reg(chip, offset, GPLR); + + return !!(readl(gplr) & BIT(offset % 32)); +} + +static void mrfld_gpio_set(struct gpio_chip *chip, unsigned int offset, + int value) +{ + struct mrfld_gpio *priv = gpiochip_get_data(chip); + void __iomem *gpsr, *gpcr; + unsigned long flags; + + raw_spin_lock_irqsave(&priv->lock, flags); + + if (value) { + gpsr = gpio_reg(chip, offset, GPSR); + writel(BIT(offset % 32), gpsr); + } else { + gpcr = gpio_reg(chip, offset, GPCR); + writel(BIT(offset % 32), gpcr); + } + + raw_spin_unlock_irqrestore(&priv->lock, flags); +} + +static int mrfld_gpio_direction_input(struct gpio_chip *chip, + unsigned int offset) +{ + struct mrfld_gpio *priv = gpiochip_get_data(chip); + void __iomem *gpdr = gpio_reg(chip, offset, GPDR); + unsigned long flags; + u32 value; + + raw_spin_lock_irqsave(&priv->lock, flags); + + value = readl(gpdr); + value &= ~BIT(offset % 32); + writel(value, gpdr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + +static int mrfld_gpio_direction_output(struct gpio_chip *chip, + unsigned int offset, int value) +{ + struct mrfld_gpio *priv = gpiochip_get_data(chip); + void __iomem *gpdr = gpio_reg(chip, offset, GPDR); + unsigned long flags; + + mrfld_gpio_set(chip, offset, value); + + raw_spin_lock_irqsave(&priv->lock, flags); + + value = readl(gpdr); + value |= BIT(offset % 32); + writel(value, gpdr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + +static void mrfld_irq_ack(struct irq_data *d) +{ + struct mrfld_gpio *priv = irq_data_get_irq_chip_data(d); + u32 gpio = irqd_to_hwirq(d); + void __iomem *gisr = gpio_reg(&priv->chip, gpio, GISR); + unsigned long flags; + + raw_spin_lock_irqsave(&priv->lock, flags); + + writel(BIT(gpio % 32), gisr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); +} + +static void mrfld_irq_unmask_mask(struct irq_data *d, bool unmask) +{ + struct mrfld_gpio *priv = irq_data_get_irq_chip_data(d); + u32 gpio = irqd_to_hwirq(d); + void __iomem *gimr = gpio_reg(&priv->chip, gpio, GIMR); + unsigned long flags; + u32 value; + + raw_spin_lock_irqsave(&priv->lock, flags); + + if (unmask) + value = readl(gimr) | BIT(gpio % 32); + else + value = readl(gimr) & ~BIT(gpio % 32); + writel(value, gimr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); +} + +static void mrfld_irq_mask(struct irq_data *d) +{ + mrfld_irq_unmask_mask(d, false); +} + +static void mrfld_irq_unmask(struct irq_data *d) +{ + mrfld_irq_unmask_mask(d, true); +} + +static int mrfld_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct mrfld_gpio *priv = gpiochip_get_data(gc); + u32 gpio = irqd_to_hwirq(d); + void __iomem *grer = gpio_reg(&priv->chip, gpio, GRER); + void __iomem *gfer = gpio_reg(&priv->chip, gpio, GFER); + void __iomem *gitr = gpio_reg(&priv->chip, gpio, GITR); + void __iomem *glpr = gpio_reg(&priv->chip, gpio, GLPR); + unsigned long flags; + u32 value; + + raw_spin_lock_irqsave(&priv->lock, flags); + + if (type & IRQ_TYPE_EDGE_RISING) + value = readl(grer) | BIT(gpio % 32); + else + value = readl(grer) & ~BIT(gpio % 32); + writel(value, grer); + + if (type & IRQ_TYPE_EDGE_FALLING) + value = readl(gfer) | BIT(gpio % 32); + else + value = readl(gfer) & ~BIT(gpio % 32); + writel(value, gfer); + + /* + * To prevent glitches from triggering an unintended level interrupt, + * configure GLPR register first and then configure GITR. + */ + if (type & IRQ_TYPE_LEVEL_LOW) + value = readl(glpr) | BIT(gpio % 32); + else + value = readl(glpr) & ~BIT(gpio % 32); + writel(value, glpr); + + if (type & IRQ_TYPE_LEVEL_MASK) { + value = readl(gitr) | BIT(gpio % 32); + writel(value, gitr); + + irq_set_handler_locked(d, handle_level_irq); + } else if (type & IRQ_TYPE_EDGE_BOTH) { + value = readl(gitr) & ~BIT(gpio % 32); + writel(value, gitr); + + irq_set_handler_locked(d, handle_edge_irq); + } + + raw_spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + +static int mrfld_irq_set_wake(struct irq_data *d, unsigned int on) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct mrfld_gpio *priv = gpiochip_get_data(gc); + u32 gpio = irqd_to_hwirq(d); + void __iomem *gwmr = gpio_reg(&priv->chip, gpio, GWMR); + void __iomem *gwsr = gpio_reg(&priv->chip, gpio, GWSR); + unsigned long flags; + u32 value; + + raw_spin_lock_irqsave(&priv->lock, flags); + + /* Clear the existing wake status */ + writel(BIT(gpio % 32), gwsr); + + if (on) + value = readl(gwmr) | BIT(gpio % 32); + else + value = readl(gwmr) & ~BIT(gpio % 32); + writel(value, gwmr); + + raw_spin_unlock_irqrestore(&priv->lock, flags); + + dev_dbg(priv->dev, "%sable wake for gpio %u\n", on ? "en" : "dis", gpio); + return 0; +} + +static struct irq_chip mrfld_irqchip = { + .name = "gpio-merrifield", + .irq_ack = mrfld_irq_ack, + .irq_mask = mrfld_irq_mask, + .irq_unmask = mrfld_irq_unmask, + .irq_set_type = mrfld_irq_set_type, + .irq_set_wake = mrfld_irq_set_wake, +}; + +static void mrfld_irq_handler(struct irq_desc *desc) +{ + struct gpio_chip *gc = irq_desc_get_handler_data(desc); + struct mrfld_gpio *priv = gpiochip_get_data(gc); + struct irq_chip *irqchip = irq_desc_get_chip(desc); + unsigned long base, gpio; + + chained_irq_enter(irqchip, desc); + + /* Check GPIO controller to check which pin triggered the interrupt */ + for (base = 0; base < priv->chip.ngpio; base += 32) { + void __iomem *gisr = gpio_reg(&priv->chip, base, GISR); + void __iomem *gimr = gpio_reg(&priv->chip, base, GIMR); + unsigned long pending, enabled; + + pending = readl(gisr); + enabled = readl(gimr); + + /* Only interrupts that are enabled */ + pending &= enabled; + + for_each_set_bit(gpio, &pending, 32) { + unsigned int irq; + + irq = irq_find_mapping(gc->irqdomain, base + gpio); + generic_handle_irq(irq); + } + } + + chained_irq_exit(irqchip, desc); +} + +static void mrfld_irq_init_hw(struct mrfld_gpio *priv) +{ + void __iomem *reg; + unsigned int base; + + for (base = 0; base < priv->chip.ngpio; base += 32) { + /* Clear the rising-edge detect register */ + reg = gpio_reg(&priv->chip, base, GRER); + writel(0, reg); + /* Clear the falling-edge detect register */ + reg = gpio_reg(&priv->chip, base, GFER); + writel(0, reg); + } +} + +static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + const struct mrfld_gpio_pinrange *range; + struct mrfld_gpio *priv; + u32 gpio_base, irq_base; + void __iomem *base; + unsigned int i; + int retval; + + retval = pcim_enable_device(pdev); + if (retval) + return retval; + + retval = pcim_iomap_regions(pdev, BIT(1) | BIT(0), pci_name(pdev)); + if (retval) { + dev_err(&pdev->dev, "I/O memory mapping error\n"); + return retval; + } + + base = pcim_iomap_table(pdev)[1]; + + irq_base = readl(base); + gpio_base = readl(sizeof(u32) + base); + + /* Release the IO mapping, since we already get the info from BAR1 */ + pcim_iounmap_regions(pdev, BIT(1)); + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) { + dev_err(&pdev->dev, "can't allocate chip data\n"); + return -ENOMEM; + } + + priv->dev = &pdev->dev; + priv->reg_base = pcim_iomap_table(pdev)[0]; + + priv->chip.label = dev_name(&pdev->dev); + priv->chip.parent = &pdev->dev; + priv->chip.request = gpiochip_generic_request; + priv->chip.free = gpiochip_generic_free; + priv->chip.direction_input = mrfld_gpio_direction_input; + priv->chip.direction_output = mrfld_gpio_direction_output; + priv->chip.get = mrfld_gpio_get; + priv->chip.set = mrfld_gpio_set; + priv->chip.base = gpio_base; + priv->chip.ngpio = MRFLD_NGPIO; + priv->chip.can_sleep = false; + + raw_spin_lock_init(&priv->lock); + + pci_set_drvdata(pdev, priv); + retval = devm_gpiochip_add_data(&pdev->dev, &priv->chip, priv); + if (retval) { + dev_err(&pdev->dev, "gpiochip_add error %d\n", retval); + return retval; + } + + for (i = 0; i < ARRAY_SIZE(mrfld_gpio_ranges); i++) { + range = &mrfld_gpio_ranges[i]; + retval = gpiochip_add_pin_range(&priv->chip, + "pinctrl-merrifield", + range->gpio_base, + range->pin_base, + range->npins); + if (retval) { + dev_err(&pdev->dev, "failed to add GPIO pin range\n"); + return retval; + } + } + + retval = gpiochip_irqchip_add(&priv->chip, &mrfld_irqchip, irq_base, + handle_simple_irq, IRQ_TYPE_NONE); + if (retval) { + dev_err(&pdev->dev, "could not connect irqchip to gpiochip\n"); + return retval; + } + + mrfld_irq_init_hw(priv); + + gpiochip_set_chained_irqchip(&priv->chip, &mrfld_irqchip, pdev->irq, + mrfld_irq_handler); + + return 0; +} + +static const struct pci_device_id mrfld_gpio_ids[] = { + { PCI_VDEVICE(INTEL, 0x1199) }, + { } +}; +MODULE_DEVICE_TABLE(pci, mrfld_gpio_ids); + +static struct pci_driver mrfld_gpio_driver = { + .name = "gpio-merrifield", + .id_table = mrfld_gpio_ids, + .probe = mrfld_gpio_probe, +}; + +module_pci_driver(mrfld_gpio_driver); + +MODULE_AUTHOR("Andy Shevchenko "); +MODULE_DESCRIPTION("Intel Merrifield SoC GPIO driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpio/gpio-mmio.c b/drivers/gpio/gpio-mmio.c index 6c1cb3b8c..6ec144bae 100644 --- a/drivers/gpio/gpio-mmio.c +++ b/drivers/gpio/gpio-mmio.c @@ -61,6 +61,8 @@ o ` ~~~~\___/~~~~ ` controller in FPGA is ,.` #include #include #include +#include +#include static void bgpio_write8(void __iomem *reg, unsigned long data) { @@ -569,6 +571,41 @@ static void __iomem *bgpio_map(struct platform_device *pdev, return devm_ioremap_resource(&pdev->dev, r); } +#ifdef CONFIG_OF +static const struct of_device_id bgpio_of_match[] = { + { .compatible = "wd,mbl-gpio" }, + { } +}; +MODULE_DEVICE_TABLE(of, bgpio_of_match); + +static struct bgpio_pdata *bgpio_parse_dt(struct platform_device *pdev, + unsigned long *flags) +{ + struct bgpio_pdata *pdata; + + if (!of_match_device(bgpio_of_match, &pdev->dev)) + return NULL; + + pdata = devm_kzalloc(&pdev->dev, sizeof(struct bgpio_pdata), + GFP_KERNEL); + if (!pdata) + return ERR_PTR(-ENOMEM); + + pdata->base = -1; + + if (of_property_read_bool(pdev->dev.of_node, "no-output")) + *flags |= BGPIOF_NO_OUTPUT; + + return pdata; +} +#else +static struct bgpio_pdata *bgpio_parse_dt(struct platform_device *pdev, + unsigned long *flags) +{ + return NULL; +} +#endif /* CONFIG_OF */ + static int bgpio_pdev_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -579,10 +616,19 @@ static int bgpio_pdev_probe(struct platform_device *pdev) void __iomem *dirout; void __iomem *dirin; unsigned long sz; - unsigned long flags = pdev->id_entry->driver_data; + unsigned long flags = 0; int err; struct gpio_chip *gc; - struct bgpio_pdata *pdata = dev_get_platdata(dev); + struct bgpio_pdata *pdata; + + pdata = bgpio_parse_dt(pdev, &flags); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + + if (!pdata) { + pdata = dev_get_platdata(dev); + flags = pdev->id_entry->driver_data; + } r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dat"); if (!r) @@ -646,6 +692,7 @@ MODULE_DEVICE_TABLE(platform, bgpio_id_table); static struct platform_driver bgpio_driver = { .driver = { .name = "basic-mmio-gpio", + .of_match_table = of_match_ptr(bgpio_of_match), }, .id_table = bgpio_id_table, .probe = bgpio_pdev_probe, diff --git a/drivers/gpio/gpio-palmas.c b/drivers/gpio/gpio-palmas.c index e248707ca..839474430 100644 --- a/drivers/gpio/gpio-palmas.c +++ b/drivers/gpio/gpio-palmas.c @@ -208,7 +208,6 @@ static int palmas_gpio_probe(struct platform_device *pdev) static struct platform_driver palmas_gpio_driver = { .driver.name = "palmas-gpio", - .driver.owner = THIS_MODULE, .driver.of_match_table = of_palmas_gpio_match, .probe = palmas_gpio_probe, }; diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index 3745de659..02f2a5621 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -44,7 +44,7 @@ #define PCA_GPIO_MASK 0x00FF #define PCA_INT 0x0100 -#define PCA_PCAL 0x0200 +#define PCA_PCAL 0x0200 #define PCA953X_TYPE 0x1000 #define PCA957X_TYPE 0x2000 #define PCA_TYPE_MASK 0xF000 @@ -67,6 +67,8 @@ static const struct i2c_device_id pca953x_id[] = { { "pca9575", 16 | PCA957X_TYPE | PCA_INT, }, { "pca9698", 40 | PCA953X_TYPE, }, + { "pcal9555a", 16 | PCA953X_TYPE | PCA_INT | PCA_PCAL, }, + { "max7310", 8 | PCA953X_TYPE, }, { "max7312", 16 | PCA953X_TYPE | PCA_INT, }, { "max7313", 16 | PCA953X_TYPE | PCA_INT, }, @@ -135,7 +137,7 @@ static int pca953x_read_single(struct pca953x_chip *chip, int reg, u32 *val, static int pca953x_write_single(struct pca953x_chip *chip, int reg, u32 val, int off) { - int ret = 0; + int ret; int bank_shift = fls((chip->gpio_chip.ngpio - 1) / BANK_SZ); int offset = off / BANK_SZ; @@ -163,10 +165,13 @@ static int pca953x_write_regs(struct pca953x_chip *chip, int reg, u8 *val) NBANK(chip), val); } else { switch (chip->chip_type) { - case PCA953X_TYPE: - ret = i2c_smbus_write_word_data(chip->client, - reg << 1, cpu_to_le16(get_unaligned((u16 *)val))); + case PCA953X_TYPE: { + __le16 word = cpu_to_le16(get_unaligned((u16 *)val)); + + ret = i2c_smbus_write_word_data(chip->client, reg << 1, + (__force u16)word); break; + } case PCA957X_TYPE: ret = i2c_smbus_write_byte_data(chip->client, reg << 1, val[0]); @@ -235,7 +240,6 @@ static int pca953x_gpio_direction_input(struct gpio_chip *gc, unsigned off) goto exit; chip->reg_direction[off / BANK_SZ] = reg_val; - ret = 0; exit: mutex_unlock(&chip->i2c_lock); return ret; @@ -286,7 +290,6 @@ static int pca953x_gpio_direction_output(struct gpio_chip *gc, goto exit; chip->reg_direction[off / BANK_SZ] = reg_val; - ret = 0; exit: mutex_unlock(&chip->i2c_lock); return ret; @@ -351,7 +354,6 @@ exit: mutex_unlock(&chip->i2c_lock); } - static void pca953x_gpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, unsigned long *bits) { @@ -820,7 +822,7 @@ static int pca953x_remove(struct i2c_client *client) { struct pca953x_platform_data *pdata = dev_get_platdata(&client->dev); struct pca953x_chip *chip = i2c_get_clientdata(client); - int ret = 0; + int ret; if (pdata && pdata->teardown) { ret = pdata->teardown(client, chip->gpio_chip.base, @@ -861,6 +863,7 @@ static const struct of_device_id pca953x_dt_ids[] = { { .compatible = "maxim,max7315", .data = OF_953X( 8, PCA_INT), }, { .compatible = "ti,pca6107", .data = OF_953X( 8, PCA_INT), }, + { .compatible = "ti,pca9536", .data = OF_953X( 4, 0), }, { .compatible = "ti,tca6408", .data = OF_953X( 8, PCA_INT), }, { .compatible = "ti,tca6416", .data = OF_953X(16, PCA_INT), }, { .compatible = "ti,tca6424", .data = OF_953X(24, PCA_INT), }, diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c index 169c09aa3..d168410e2 100644 --- a/drivers/gpio/gpio-pcf857x.c +++ b/drivers/gpio/gpio-pcf857x.c @@ -440,6 +440,14 @@ static int pcf857x_remove(struct i2c_client *client) return status; } +static void pcf857x_shutdown(struct i2c_client *client) +{ + struct pcf857x *gpio = i2c_get_clientdata(client); + + /* Drive all the I/O lines high */ + gpio->write(gpio->client, BIT(gpio->chip.ngpio) - 1); +} + static struct i2c_driver pcf857x_driver = { .driver = { .name = "pcf857x", @@ -447,6 +455,7 @@ static struct i2c_driver pcf857x_driver = { }, .probe = pcf857x_probe, .remove = pcf857x_remove, + .shutdown = pcf857x_shutdown, .id_table = pcf857x_id, }; diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c index 681c93fb9..b96e0b466 100644 --- a/drivers/gpio/gpio-rcar.c +++ b/drivers/gpio/gpio-rcar.c @@ -334,6 +334,9 @@ static const struct of_device_id gpio_rcar_of_table[] = { }, { .compatible = "renesas,gpio-r8a7791", .data = &gpio_rcar_info_gen2, + }, { + .compatible = "renesas,gpio-r8a7792", + .data = &gpio_rcar_info_gen2, }, { .compatible = "renesas,gpio-r8a7793", .data = &gpio_rcar_info_gen2, diff --git a/drivers/gpio/gpio-rdc321x.c b/drivers/gpio/gpio-rdc321x.c index ec945b90f..cbf0f9e64 100644 --- a/drivers/gpio/gpio-rdc321x.c +++ b/drivers/gpio/gpio-rdc321x.c @@ -200,7 +200,6 @@ static int rdc321x_gpio_probe(struct platform_device *pdev) static struct platform_driver rdc321x_gpio_driver = { .driver.name = "rdc321x-gpio", - .driver.owner = THIS_MODULE, .probe = rdc321x_gpio_probe, }; diff --git a/drivers/gpio/gpio-sa1100.c b/drivers/gpio/gpio-sa1100.c index 0c99e8fb9..8d8ee0ebf 100644 --- a/drivers/gpio/gpio-sa1100.c +++ b/drivers/gpio/gpio-sa1100.c @@ -155,7 +155,7 @@ static int sa1100_gpio_irqdomain_map(struct irq_domain *d, { irq_set_chip_and_handler(irq, &sa1100_gpio_irq_chip, handle_edge_irq); - irq_set_noprobe(irq); + irq_set_probe(irq); return 0; } diff --git a/drivers/gpio/gpio-sch311x.c b/drivers/gpio/gpio-sch311x.c index a03b38ee2..b96990c26 100644 --- a/drivers/gpio/gpio-sch311x.c +++ b/drivers/gpio/gpio-sch311x.c @@ -296,7 +296,6 @@ static int sch311x_gpio_remove(struct platform_device *pdev) static struct platform_driver sch311x_gpio_driver = { .driver.name = DRV_NAME, - .driver.owner = THIS_MODULE, .probe = sch311x_gpio_probe, .remove = sch311x_gpio_remove, }; diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c index 6f7af28b8..f675132de 100644 --- a/drivers/gpio/gpio-stmpe.c +++ b/drivers/gpio/gpio-stmpe.c @@ -68,6 +68,22 @@ static void stmpe_gpio_set(struct gpio_chip *chip, unsigned offset, int val) stmpe_reg_write(stmpe, reg, mask); } +static int stmpe_gpio_get_direction(struct gpio_chip *chip, + unsigned offset) +{ + struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(chip); + struct stmpe *stmpe = stmpe_gpio->stmpe; + u8 reg = stmpe->regs[STMPE_IDX_GPDR_LSB] - (offset / 8); + u8 mask = 1 << (offset % 8); + int ret; + + ret = stmpe_reg_read(stmpe, reg); + if (ret < 0) + return ret; + + return !(ret & mask); +} + static int stmpe_gpio_direction_output(struct gpio_chip *chip, unsigned offset, int val) { @@ -106,6 +122,7 @@ static int stmpe_gpio_request(struct gpio_chip *chip, unsigned offset) static struct gpio_chip template_chip = { .label = "stmpe", .owner = THIS_MODULE, + .get_direction = stmpe_gpio_get_direction, .direction_input = stmpe_gpio_direction_input, .get = stmpe_gpio_get, .direction_output = stmpe_gpio_direction_output, @@ -416,7 +433,6 @@ static struct platform_driver stmpe_gpio_driver = { .driver = { .suppress_bind_attrs = true, .name = "stmpe-gpio", - .owner = THIS_MODULE, }, .probe = stmpe_gpio_probe, }; diff --git a/drivers/gpio/gpio-syscon.c b/drivers/gpio/gpio-syscon.c index 24b6d643e..537cec758 100644 --- a/drivers/gpio/gpio-syscon.c +++ b/drivers/gpio/gpio-syscon.c @@ -129,7 +129,7 @@ static int syscon_gpio_dir_out(struct gpio_chip *chip, unsigned offset, int val) static const struct syscon_gpio_data clps711x_mctrl_gpio = { /* ARM CLPS711X SYSFLG1 Bits 8-10 */ - .compatible = "cirrus,clps711x-syscon1", + .compatible = "cirrus,ep7209-syscon1", .flags = GPIO_SYSCON_FEAT_IN, .bit_count = 3, .dat_bit_offset = 0x40 * 8 + 8, @@ -168,7 +168,7 @@ static const struct syscon_gpio_data keystone_dsp_gpio = { static const struct of_device_id syscon_gpio_ids[] = { { - .compatible = "cirrus,clps711x-mctrl-gpio", + .compatible = "cirrus,ep7209-mctrl-gpio", .data = &clps711x_mctrl_gpio, }, { diff --git a/drivers/gpio/gpio-tc3589x.c b/drivers/gpio/gpio-tc3589x.c index 2e35ed3ab..8b3659352 100644 --- a/drivers/gpio/gpio-tc3589x.c +++ b/drivers/gpio/gpio-tc3589x.c @@ -343,7 +343,6 @@ static int tc3589x_gpio_probe(struct platform_device *pdev) static struct platform_driver tc3589x_gpio_driver = { .driver.name = "tc3589x-gpio", - .driver.owner = THIS_MODULE, .probe = tc3589x_gpio_probe, }; diff --git a/drivers/gpio/gpio-tps65218.c b/drivers/gpio/gpio-tps65218.c index 0eaeac8de..1c09a19ae 100644 --- a/drivers/gpio/gpio-tps65218.c +++ b/drivers/gpio/gpio-tps65218.c @@ -230,6 +230,12 @@ static const struct of_device_id tps65218_dt_match[] = { }; MODULE_DEVICE_TABLE(of, tps65218_dt_match); +static const struct platform_device_id tps65218_gpio_id_table[] = { + { "tps65218-gpio", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(platform, tps65218_gpio_id_table); + static struct platform_driver tps65218_gpio_driver = { .driver = { .name = "tps65218-gpio", @@ -237,6 +243,7 @@ static struct platform_driver tps65218_gpio_driver = { }, .probe = tps65218_gpio_probe, .remove = tps65218_gpio_remove, + .id_table = tps65218_gpio_id_table, }; module_platform_driver(tps65218_gpio_driver); diff --git a/drivers/gpio/gpio-tps6586x.c b/drivers/gpio/gpio-tps6586x.c index 6b15e68a3..042b9a207 100644 --- a/drivers/gpio/gpio-tps6586x.c +++ b/drivers/gpio/gpio-tps6586x.c @@ -131,7 +131,6 @@ static int tps6586x_gpio_probe(struct platform_device *pdev) static struct platform_driver tps6586x_gpio_driver = { .driver.name = "tps6586x-gpio", - .driver.owner = THIS_MODULE, .probe = tps6586x_gpio_probe, }; diff --git a/drivers/gpio/gpio-tps65910.c b/drivers/gpio/gpio-tps65910.c index 0ae6a5a54..e63d7dabf 100644 --- a/drivers/gpio/gpio-tps65910.c +++ b/drivers/gpio/gpio-tps65910.c @@ -184,7 +184,6 @@ skip_init: static struct platform_driver tps65910_gpio_driver = { .driver.name = "tps65910-gpio", - .driver.owner = THIS_MODULE, .probe = tps65910_gpio_probe, }; diff --git a/drivers/gpio/gpio-viperboard.c b/drivers/gpio/gpio-viperboard.c index dec47aafd..e6d1328dd 100644 --- a/drivers/gpio/gpio-viperboard.c +++ b/drivers/gpio/gpio-viperboard.c @@ -440,7 +440,6 @@ static int vprbrd_gpio_probe(struct platform_device *pdev) static struct platform_driver vprbrd_gpio_driver = { .driver.name = "viperboard-gpio", - .driver.owner = THIS_MODULE, .probe = vprbrd_gpio_probe, }; diff --git a/drivers/gpio/gpio-wm831x.c b/drivers/gpio/gpio-wm831x.c index 41ec78340..21f97bcd0 100644 --- a/drivers/gpio/gpio-wm831x.c +++ b/drivers/gpio/gpio-wm831x.c @@ -296,7 +296,6 @@ static int wm831x_gpio_probe(struct platform_device *pdev) static struct platform_driver wm831x_gpio_driver = { .driver.name = "wm831x-gpio", - .driver.owner = THIS_MODULE, .probe = wm831x_gpio_probe, }; diff --git a/drivers/gpio/gpio-wm8350.c b/drivers/gpio/gpio-wm8350.c index 07d45a3b2..e9765707d 100644 --- a/drivers/gpio/gpio-wm8350.c +++ b/drivers/gpio/gpio-wm8350.c @@ -139,7 +139,6 @@ static int wm8350_gpio_probe(struct platform_device *pdev) static struct platform_driver wm8350_gpio_driver = { .driver.name = "wm8350-gpio", - .driver.owner = THIS_MODULE, .probe = wm8350_gpio_probe, }; diff --git a/drivers/gpio/gpio-wm8994.c b/drivers/gpio/gpio-wm8994.c index 744af388c..2457aac85 100644 --- a/drivers/gpio/gpio-wm8994.c +++ b/drivers/gpio/gpio-wm8994.c @@ -299,7 +299,6 @@ static int wm8994_gpio_probe(struct platform_device *pdev) static struct platform_driver wm8994_gpio_driver = { .driver.name = "wm8994-gpio", - .driver.owner = THIS_MODULE, .probe = wm8994_gpio_probe, }; diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c index d0fbb7f99..14b2a6233 100644 --- a/drivers/gpio/gpio-xilinx.c +++ b/drivers/gpio/gpio-xilinx.c @@ -132,6 +132,53 @@ static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val) spin_unlock_irqrestore(&chip->gpio_lock[index], flags); } +/** + * xgpio_set_multiple - Write the specified signals of the GPIO device. + * @gc: Pointer to gpio_chip device structure. + * @mask: Mask of the GPIOS to modify. + * @bits: Value to be wrote on each GPIO + * + * This function writes the specified values into the specified signals of the + * GPIO devices. + */ +static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, + unsigned long *bits) +{ + unsigned long flags; + struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); + struct xgpio_instance *chip = gpiochip_get_data(gc); + int index = xgpio_index(chip, 0); + int offset, i; + + spin_lock_irqsave(&chip->gpio_lock[index], flags); + + /* Write to GPIO signals */ + for (i = 0; i < gc->ngpio; i++) { + if (*mask == 0) + break; + if (index != xgpio_index(chip, i)) { + xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET + + xgpio_regoffset(chip, i), + chip->gpio_state[index]); + spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + index = xgpio_index(chip, i); + spin_lock_irqsave(&chip->gpio_lock[index], flags); + } + if (__test_and_clear_bit(i, mask)) { + offset = xgpio_offset(chip, i); + if (test_bit(i, bits)) + chip->gpio_state[index] |= BIT(offset); + else + chip->gpio_state[index] &= ~BIT(offset); + } + } + + xgpio_writereg(mm_gc->regs + XGPIO_DATA_OFFSET + + xgpio_regoffset(chip, i), chip->gpio_state[index]); + + spin_unlock_irqrestore(&chip->gpio_lock[index], flags); +} + /** * xgpio_dir_in - Set the direction of the specified GPIO signal as input. * @gc: Pointer to gpio_chip device structure. @@ -306,6 +353,7 @@ static int xgpio_probe(struct platform_device *pdev) chip->mmchip.gc.direction_output = xgpio_dir_out; chip->mmchip.gc.get = xgpio_get; chip->mmchip.gc.set = xgpio_set; + chip->mmchip.gc.set_multiple = xgpio_set_multiple; chip->mmchip.save_regs = xgpio_save_regs; diff --git a/drivers/gpio/gpio-xlp.c b/drivers/gpio/gpio-xlp.c index 1a33a19d9..4620d050e 100644 --- a/drivers/gpio/gpio-xlp.c +++ b/drivers/gpio/gpio-xlp.c @@ -19,6 +19,7 @@ #include #include #include +#include /* * XLP GPIO has multiple 32 bit registers for each feature where each register @@ -299,7 +300,6 @@ static int xlp_gpio_probe(struct platform_device *pdev) struct gpio_chip *gc; struct resource *iores; struct xlp_gpio_priv *priv; - const struct of_device_id *of_id; void __iomem *gpio_base; int irq_base, irq, err; int ngpio; @@ -321,13 +321,26 @@ static int xlp_gpio_probe(struct platform_device *pdev) if (irq < 0) return irq; - of_id = of_match_device(xlp_gpio_of_ids, &pdev->dev); - if (!of_id) { - dev_err(&pdev->dev, "Failed to get soc type!\n"); - return -ENODEV; - } + if (pdev->dev.of_node) { + const struct of_device_id *of_id; - soc_type = (uintptr_t) of_id->data; + of_id = of_match_device(xlp_gpio_of_ids, &pdev->dev); + if (!of_id) { + dev_err(&pdev->dev, "Unable to match OF ID\n"); + return -ENODEV; + } + soc_type = (uintptr_t) of_id->data; + } else { + const struct acpi_device_id *acpi_id; + + acpi_id = acpi_match_device(pdev->dev.driver->acpi_match_table, + &pdev->dev); + if (!acpi_id || !acpi_id->driver_data) { + dev_err(&pdev->dev, "Unable to match ACPI ID\n"); + return -ENODEV; + } + soc_type = (uintptr_t) acpi_id->driver_data; + } switch (soc_type) { case XLP_GPIO_VARIANT_XLP832: @@ -388,14 +401,16 @@ static int xlp_gpio_probe(struct platform_device *pdev) gc->get = xlp_gpio_get; spin_lock_init(&priv->lock); - /* XLP has fixed IRQ range for GPIO interrupts */ - if (soc_type == GPIO_VARIANT_VULCAN) - irq_base = irq_alloc_descs(-1, 0, gc->ngpio, 0); - else + + /* XLP(MIPS) has fixed range for GPIO IRQs, Vulcan(ARM64) does not */ + if (soc_type != GPIO_VARIANT_VULCAN) { irq_base = irq_alloc_descs(-1, XLP_GPIO_IRQ_BASE, gc->ngpio, 0); - if (irq_base < 0) { - dev_err(&pdev->dev, "Failed to allocate IRQ numbers\n"); - return irq_base; + if (irq_base < 0) { + dev_err(&pdev->dev, "Failed to allocate IRQ numbers\n"); + return irq_base; + } + } else { + irq_base = 0; } err = gpiochip_add_data(gc, priv); @@ -423,10 +438,19 @@ out_free_desc: return err; } +#ifdef CONFIG_ACPI +static const struct acpi_device_id xlp_gpio_acpi_match[] = { + { "BRCM9006", GPIO_VARIANT_VULCAN }, + {}, +}; +MODULE_DEVICE_TABLE(acpi, xlp_gpio_acpi_match); +#endif + static struct platform_driver xlp_gpio_driver = { .driver = { .name = "xlp-gpio", .of_match_table = xlp_gpio_of_ids, + .acpi_match_table = ACPI_PTR(xlp_gpio_acpi_match), }, .probe = xlp_gpio_probe, }; diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 2dc52585e..af514618d 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -836,6 +836,7 @@ void acpi_gpiochip_add(struct gpio_chip *chip) } acpi_gpiochip_request_regions(acpi_gpio); + acpi_walk_dep_device_list(handle); } void acpi_gpiochip_remove(struct gpio_chip *chip) diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 4aabddb38..a28feb3ed 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -27,38 +26,30 @@ #include "gpiolib.h" -/* Private data structure for of_gpiochip_find_and_xlate */ -struct gg_data { - enum of_gpio_flags *flags; - struct of_phandle_args gpiospec; +static int of_gpiochip_match_node(struct gpio_chip *chip, void *data) +{ + return chip->gpiodev->dev.of_node == data; +} - struct gpio_desc *out_gpio; -}; +static struct gpio_chip *of_find_gpiochip_by_node(struct device_node *np) +{ + return gpiochip_find(np, of_gpiochip_match_node); +} -/* Private function for resolving node pointer to gpio_chip */ -static int of_gpiochip_find_and_xlate(struct gpio_chip *gc, void *data) +static struct gpio_desc *of_xlate_and_get_gpiod_flags(struct gpio_chip *chip, + struct of_phandle_args *gpiospec, + enum of_gpio_flags *flags) { - struct gg_data *gg_data = data; int ret; - if ((gc->of_node != gg_data->gpiospec.np) || - (gc->of_gpio_n_cells != gg_data->gpiospec.args_count) || - (!gc->of_xlate)) - return false; - - ret = gc->of_xlate(gc, &gg_data->gpiospec, gg_data->flags); - if (ret < 0) { - /* We've found a gpio chip, but the translation failed. - * Store translation error in out_gpio. - * Return false to keep looking, as more than one gpio chip - * could be registered per of-node. - */ - gg_data->out_gpio = ERR_PTR(ret); - return false; - } - - gg_data->out_gpio = gpiochip_get_desc(gc, ret); - return true; + if (chip->of_gpio_n_cells != gpiospec->args_count) + return ERR_PTR(-EINVAL); + + ret = chip->of_xlate(chip, gpiospec, flags); + if (ret < 0) + return ERR_PTR(ret); + + return gpiochip_get_desc(chip, ret); } /** @@ -75,34 +66,37 @@ static int of_gpiochip_find_and_xlate(struct gpio_chip *gc, void *data) struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np, const char *propname, int index, enum of_gpio_flags *flags) { - /* Return -EPROBE_DEFER to support probe() functions to be called - * later when the GPIO actually becomes available - */ - struct gg_data gg_data = { - .flags = flags, - .out_gpio = ERR_PTR(-EPROBE_DEFER) - }; + struct of_phandle_args gpiospec; + struct gpio_chip *chip; + struct gpio_desc *desc; int ret; - /* .of_xlate might decide to not fill in the flags, so clear it. */ - if (flags) - *flags = 0; - ret = of_parse_phandle_with_args(np, propname, "#gpio-cells", index, - &gg_data.gpiospec); + &gpiospec); if (ret) { pr_debug("%s: can't parse '%s' property of node '%s[%d]'\n", __func__, propname, np->full_name, index); return ERR_PTR(ret); } - gpiochip_find(&gg_data, of_gpiochip_find_and_xlate); + chip = of_find_gpiochip_by_node(gpiospec.np); + if (!chip) { + desc = ERR_PTR(-EPROBE_DEFER); + goto out; + } + + desc = of_xlate_and_get_gpiod_flags(chip, &gpiospec, flags); + if (IS_ERR(desc)) + goto out; - of_node_put(gg_data.gpiospec.np); pr_debug("%s: parsed '%s' property of node '%s[%d]' - status (%d)\n", __func__, propname, np->full_name, index, - PTR_ERR_OR_ZERO(gg_data.out_gpio)); - return gg_data.out_gpio; + PTR_ERR_OR_ZERO(desc)); + +out: + of_node_put(gpiospec.np); + + return desc; } int of_get_named_gpio_flags(struct device_node *np, const char *list_name, @@ -122,6 +116,7 @@ EXPORT_SYMBOL(of_get_named_gpio_flags); /** * of_parse_own_gpio() - Get a GPIO hog descriptor, names and flags for GPIO API * @np: device node to get GPIO from + * @chip: GPIO chip whose hog is parsed * @name: GPIO line name * @lflags: gpio_lookup_flags - returned from of_find_gpio() or * of_parse_own_gpio() @@ -131,19 +126,19 @@ EXPORT_SYMBOL(of_get_named_gpio_flags); * value on the error condition. */ static struct gpio_desc *of_parse_own_gpio(struct device_node *np, + struct gpio_chip *chip, const char **name, enum gpio_lookup_flags *lflags, enum gpiod_flags *dflags) { struct device_node *chip_np; enum of_gpio_flags xlate_flags; - struct gg_data gg_data = { - .flags = &xlate_flags, - }; + struct of_phandle_args gpiospec; + struct gpio_desc *desc; u32 tmp; - int i, ret; + int ret; - chip_np = np->parent; + chip_np = chip->of_node; if (!chip_np) return ERR_PTR(-EINVAL); @@ -155,25 +150,16 @@ static struct gpio_desc *of_parse_own_gpio(struct device_node *np, if (ret) return ERR_PTR(ret); - if (tmp > MAX_PHANDLE_ARGS) - return ERR_PTR(-EINVAL); + gpiospec.np = chip_np; + gpiospec.args_count = tmp; - gg_data.gpiospec.args_count = tmp; - gg_data.gpiospec.np = chip_np; - for (i = 0; i < tmp; i++) { - ret = of_property_read_u32_index(np, "gpios", i, - &gg_data.gpiospec.args[i]); - if (ret) - return ERR_PTR(ret); - } + ret = of_property_read_u32_array(np, "gpios", gpiospec.args, tmp); + if (ret) + return ERR_PTR(ret); - gpiochip_find(&gg_data, of_gpiochip_find_and_xlate); - if (!gg_data.out_gpio) { - if (np->parent == np) - return ERR_PTR(-ENXIO); - else - return ERR_PTR(-EINVAL); - } + desc = of_xlate_and_get_gpiod_flags(chip, &gpiospec, &xlate_flags); + if (IS_ERR(desc)) + return desc; if (xlate_flags & OF_GPIO_ACTIVE_LOW) *lflags |= GPIO_ACTIVE_LOW; @@ -186,14 +172,14 @@ static struct gpio_desc *of_parse_own_gpio(struct device_node *np, *dflags |= GPIOD_OUT_HIGH; else { pr_warn("GPIO line %d (%s): no hogging state specified, bailing out\n", - desc_to_gpio(gg_data.out_gpio), np->name); + desc_to_gpio(desc), np->name); return ERR_PTR(-EINVAL); } if (name && of_property_read_string(np, "line-name", name)) *name = np->name; - return gg_data.out_gpio; + return desc; } /** @@ -262,7 +248,7 @@ static int of_gpiochip_scan_gpios(struct gpio_chip *chip) if (!of_property_read_bool(np, "gpio-hog")) continue; - desc = of_parse_own_gpio(np, &name, &lflags, &dflags); + desc = of_parse_own_gpio(np, chip, &name, &lflags, &dflags); if (IS_ERR(desc)) continue; @@ -410,6 +396,7 @@ static int of_gpiochip_add_pin_range(struct gpio_chip *chip) break; pctldev = of_pinctrl_get(pinspec.np); + of_node_put(pinspec.np); if (!pctldev) return -EPROBE_DEFER; @@ -487,6 +474,9 @@ int of_gpiochip_add(struct gpio_chip *chip) chip->of_xlate = of_gpio_simple_xlate; } + if (chip->of_gpio_n_cells > MAX_PHANDLE_ARGS) + return -EINVAL; + status = of_gpiochip_add_pin_range(chip); if (status) return status; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index be74bd370..53ff25ac6 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -16,11 +16,14 @@ #include #include #include -#include #include #include #include #include +#include +#include +#include +#include #include #include "gpiolib.h" @@ -310,6 +313,497 @@ static int gpiochip_set_desc_names(struct gpio_chip *gc) return 0; } +/* + * GPIO line handle management + */ + +/** + * struct linehandle_state - contains the state of a userspace handle + * @gdev: the GPIO device the handle pertains to + * @label: consumer label used to tag descriptors + * @descs: the GPIO descriptors held by this handle + * @numdescs: the number of descriptors held in the descs array + */ +struct linehandle_state { + struct gpio_device *gdev; + const char *label; + struct gpio_desc *descs[GPIOHANDLES_MAX]; + u32 numdescs; +}; + +static long linehandle_ioctl(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + struct linehandle_state *lh = filep->private_data; + void __user *ip = (void __user *)arg; + struct gpiohandle_data ghd; + int i; + + if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) { + int val; + + /* TODO: check if descriptors are really input */ + for (i = 0; i < lh->numdescs; i++) { + val = gpiod_get_value_cansleep(lh->descs[i]); + if (val < 0) + return val; + ghd.values[i] = val; + } + + if (copy_to_user(ip, &ghd, sizeof(ghd))) + return -EFAULT; + + return 0; + } else if (cmd == GPIOHANDLE_SET_LINE_VALUES_IOCTL) { + int vals[GPIOHANDLES_MAX]; + + /* TODO: check if descriptors are really output */ + if (copy_from_user(&ghd, ip, sizeof(ghd))) + return -EFAULT; + + /* Clamp all values to [0,1] */ + for (i = 0; i < lh->numdescs; i++) + vals[i] = !!ghd.values[i]; + + /* Reuse the array setting function */ + gpiod_set_array_value_complex(false, + true, + lh->numdescs, + lh->descs, + vals); + return 0; + } + return -EINVAL; +} + +#ifdef CONFIG_COMPAT +static long linehandle_ioctl_compat(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + return linehandle_ioctl(filep, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +static int linehandle_release(struct inode *inode, struct file *filep) +{ + struct linehandle_state *lh = filep->private_data; + struct gpio_device *gdev = lh->gdev; + int i; + + for (i = 0; i < lh->numdescs; i++) + gpiod_free(lh->descs[i]); + kfree(lh->label); + kfree(lh); + put_device(&gdev->dev); + return 0; +} + +static const struct file_operations linehandle_fileops = { + .release = linehandle_release, + .owner = THIS_MODULE, + .llseek = noop_llseek, + .unlocked_ioctl = linehandle_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = linehandle_ioctl_compat, +#endif +}; + +static int linehandle_create(struct gpio_device *gdev, void __user *ip) +{ + struct gpiohandle_request handlereq; + struct linehandle_state *lh; + int fd, i, ret; + + if (copy_from_user(&handlereq, ip, sizeof(handlereq))) + return -EFAULT; + if ((handlereq.lines == 0) || (handlereq.lines > GPIOHANDLES_MAX)) + return -EINVAL; + + lh = kzalloc(sizeof(*lh), GFP_KERNEL); + if (!lh) + return -ENOMEM; + lh->gdev = gdev; + get_device(&gdev->dev); + + /* Make sure this is terminated */ + handlereq.consumer_label[sizeof(handlereq.consumer_label)-1] = '\0'; + if (strlen(handlereq.consumer_label)) { + lh->label = kstrdup(handlereq.consumer_label, + GFP_KERNEL); + if (!lh->label) { + ret = -ENOMEM; + goto out_free_lh; + } + } + + /* Request each GPIO */ + for (i = 0; i < handlereq.lines; i++) { + u32 offset = handlereq.lineoffsets[i]; + u32 lflags = handlereq.flags; + struct gpio_desc *desc; + + desc = &gdev->descs[offset]; + ret = gpiod_request(desc, lh->label); + if (ret) + goto out_free_descs; + lh->descs[i] = desc; + + if (lflags & GPIOHANDLE_REQUEST_ACTIVE_LOW) + set_bit(FLAG_ACTIVE_LOW, &desc->flags); + if (lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN) + set_bit(FLAG_OPEN_DRAIN, &desc->flags); + if (lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE) + set_bit(FLAG_OPEN_SOURCE, &desc->flags); + + /* + * Lines have to be requested explicitly for input + * or output, else the line will be treated "as is". + */ + if (lflags & GPIOHANDLE_REQUEST_OUTPUT) { + int val = !!handlereq.default_values[i]; + + ret = gpiod_direction_output(desc, val); + if (ret) + goto out_free_descs; + } else if (lflags & GPIOHANDLE_REQUEST_INPUT) { + ret = gpiod_direction_input(desc); + if (ret) + goto out_free_descs; + } + dev_dbg(&gdev->dev, "registered chardev handle for line %d\n", + offset); + } + /* Let i point at the last handle */ + i--; + lh->numdescs = handlereq.lines; + + fd = anon_inode_getfd("gpio-linehandle", + &linehandle_fileops, + lh, + O_RDONLY | O_CLOEXEC); + if (fd < 0) { + ret = fd; + goto out_free_descs; + } + + handlereq.fd = fd; + if (copy_to_user(ip, &handlereq, sizeof(handlereq))) { + ret = -EFAULT; + goto out_free_descs; + } + + dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n", + lh->numdescs); + + return 0; + +out_free_descs: + for (; i >= 0; i--) + gpiod_free(lh->descs[i]); + kfree(lh->label); +out_free_lh: + kfree(lh); + put_device(&gdev->dev); + return ret; +} + +/* + * GPIO line event management + */ + +/** + * struct lineevent_state - contains the state of a userspace event + * @gdev: the GPIO device the event pertains to + * @label: consumer label used to tag descriptors + * @desc: the GPIO descriptor held by this event + * @eflags: the event flags this line was requested with + * @irq: the interrupt that trigger in response to events on this GPIO + * @wait: wait queue that handles blocking reads of events + * @events: KFIFO for the GPIO events + * @read_lock: mutex lock to protect reads from colliding with adding + * new events to the FIFO + */ +struct lineevent_state { + struct gpio_device *gdev; + const char *label; + struct gpio_desc *desc; + u32 eflags; + int irq; + wait_queue_head_t wait; + DECLARE_KFIFO(events, struct gpioevent_data, 16); + struct mutex read_lock; +}; + +static unsigned int lineevent_poll(struct file *filep, + struct poll_table_struct *wait) +{ + struct lineevent_state *le = filep->private_data; + unsigned int events = 0; + + poll_wait(filep, &le->wait, wait); + + if (!kfifo_is_empty(&le->events)) + events = POLLIN | POLLRDNORM; + + return events; +} + + +static ssize_t lineevent_read(struct file *filep, + char __user *buf, + size_t count, + loff_t *f_ps) +{ + struct lineevent_state *le = filep->private_data; + unsigned int copied; + int ret; + + if (count < sizeof(struct gpioevent_data)) + return -EINVAL; + + do { + if (kfifo_is_empty(&le->events)) { + if (filep->f_flags & O_NONBLOCK) + return -EAGAIN; + + ret = wait_event_interruptible(le->wait, + !kfifo_is_empty(&le->events)); + if (ret) + return ret; + } + + if (mutex_lock_interruptible(&le->read_lock)) + return -ERESTARTSYS; + ret = kfifo_to_user(&le->events, buf, count, &copied); + mutex_unlock(&le->read_lock); + + if (ret) + return ret; + + /* + * If we couldn't read anything from the fifo (a different + * thread might have been faster) we either return -EAGAIN if + * the file descriptor is non-blocking, otherwise we go back to + * sleep and wait for more data to arrive. + */ + if (copied == 0 && (filep->f_flags & O_NONBLOCK)) + return -EAGAIN; + + } while (copied == 0); + + return copied; +} + +static int lineevent_release(struct inode *inode, struct file *filep) +{ + struct lineevent_state *le = filep->private_data; + struct gpio_device *gdev = le->gdev; + + free_irq(le->irq, le); + gpiod_free(le->desc); + kfree(le->label); + kfree(le); + put_device(&gdev->dev); + return 0; +} + +static long lineevent_ioctl(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + struct lineevent_state *le = filep->private_data; + void __user *ip = (void __user *)arg; + struct gpiohandle_data ghd; + + /* + * We can get the value for an event line but not set it, + * because it is input by definition. + */ + if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) { + int val; + + val = gpiod_get_value_cansleep(le->desc); + if (val < 0) + return val; + ghd.values[0] = val; + + if (copy_to_user(ip, &ghd, sizeof(ghd))) + return -EFAULT; + + return 0; + } + return -EINVAL; +} + +#ifdef CONFIG_COMPAT +static long lineevent_ioctl_compat(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + return lineevent_ioctl(filep, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +static const struct file_operations lineevent_fileops = { + .release = lineevent_release, + .read = lineevent_read, + .poll = lineevent_poll, + .owner = THIS_MODULE, + .llseek = noop_llseek, + .unlocked_ioctl = lineevent_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = lineevent_ioctl_compat, +#endif +}; + +static irqreturn_t lineevent_irq_thread(int irq, void *p) +{ + struct lineevent_state *le = p; + struct gpioevent_data ge; + int ret; + + ge.timestamp = ktime_get_real_ns(); + + if (le->eflags & GPIOEVENT_REQUEST_BOTH_EDGES) { + int level = gpiod_get_value_cansleep(le->desc); + + if (level) + /* Emit low-to-high event */ + ge.id = GPIOEVENT_EVENT_RISING_EDGE; + else + /* Emit high-to-low event */ + ge.id = GPIOEVENT_EVENT_FALLING_EDGE; + } else if (le->eflags & GPIOEVENT_REQUEST_RISING_EDGE) { + /* Emit low-to-high event */ + ge.id = GPIOEVENT_EVENT_RISING_EDGE; + } else if (le->eflags & GPIOEVENT_REQUEST_FALLING_EDGE) { + /* Emit high-to-low event */ + ge.id = GPIOEVENT_EVENT_FALLING_EDGE; + } else { + return IRQ_NONE; + } + + ret = kfifo_put(&le->events, ge); + if (ret != 0) + wake_up_poll(&le->wait, POLLIN); + + return IRQ_HANDLED; +} + +static int lineevent_create(struct gpio_device *gdev, void __user *ip) +{ + struct gpioevent_request eventreq; + struct lineevent_state *le; + struct gpio_desc *desc; + u32 offset; + u32 lflags; + u32 eflags; + int fd; + int ret; + int irqflags = 0; + + if (copy_from_user(&eventreq, ip, sizeof(eventreq))) + return -EFAULT; + + le = kzalloc(sizeof(*le), GFP_KERNEL); + if (!le) + return -ENOMEM; + le->gdev = gdev; + get_device(&gdev->dev); + + /* Make sure this is terminated */ + eventreq.consumer_label[sizeof(eventreq.consumer_label)-1] = '\0'; + if (strlen(eventreq.consumer_label)) { + le->label = kstrdup(eventreq.consumer_label, + GFP_KERNEL); + if (!le->label) { + ret = -ENOMEM; + goto out_free_le; + } + } + + offset = eventreq.lineoffset; + lflags = eventreq.handleflags; + eflags = eventreq.eventflags; + + /* This is just wrong: we don't look for events on output lines */ + if (lflags & GPIOHANDLE_REQUEST_OUTPUT) { + ret = -EINVAL; + goto out_free_label; + } + + desc = &gdev->descs[offset]; + ret = gpiod_request(desc, le->label); + if (ret) + goto out_free_desc; + le->desc = desc; + le->eflags = eflags; + + if (lflags & GPIOHANDLE_REQUEST_ACTIVE_LOW) + set_bit(FLAG_ACTIVE_LOW, &desc->flags); + if (lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN) + set_bit(FLAG_OPEN_DRAIN, &desc->flags); + if (lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE) + set_bit(FLAG_OPEN_SOURCE, &desc->flags); + + ret = gpiod_direction_input(desc); + if (ret) + goto out_free_desc; + + le->irq = gpiod_to_irq(desc); + if (le->irq <= 0) { + ret = -ENODEV; + goto out_free_desc; + } + + if (eflags & GPIOEVENT_REQUEST_RISING_EDGE) + irqflags |= IRQF_TRIGGER_RISING; + if (eflags & GPIOEVENT_REQUEST_FALLING_EDGE) + irqflags |= IRQF_TRIGGER_FALLING; + irqflags |= IRQF_ONESHOT; + irqflags |= IRQF_SHARED; + + INIT_KFIFO(le->events); + init_waitqueue_head(&le->wait); + mutex_init(&le->read_lock); + + /* Request a thread to read the events */ + ret = request_threaded_irq(le->irq, + NULL, + lineevent_irq_thread, + irqflags, + le->label, + le); + if (ret) + goto out_free_desc; + + fd = anon_inode_getfd("gpio-event", + &lineevent_fileops, + le, + O_RDONLY | O_CLOEXEC); + if (fd < 0) { + ret = fd; + goto out_free_irq; + } + + eventreq.fd = fd; + if (copy_to_user(ip, &eventreq, sizeof(eventreq))) { + ret = -EFAULT; + goto out_free_irq; + } + + return 0; + +out_free_irq: + free_irq(le->irq, le); +out_free_desc: + gpiod_free(le->desc); +out_free_label: + kfree(le->label); +out_free_le: + kfree(le); + put_device(&gdev->dev); + return ret; +} + /** * gpio_ioctl() - ioctl handler for the GPIO chardev */ @@ -385,6 +879,10 @@ static long gpio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) return -EFAULT; return 0; + } else if (cmd == GPIO_GET_LINEHANDLE_IOCTL) { + return linehandle_create(gdev, ip); + } else if (cmd == GPIO_GET_LINEEVENT_IOCTL) { + return lineevent_create(gdev, ip); } return -EINVAL; } @@ -548,13 +1046,14 @@ int gpiochip_add_data(struct gpio_chip *chip, void *data) if (chip->parent) { gdev->dev.parent = chip->parent; gdev->dev.of_node = chip->parent->of_node; - } else { + } + #ifdef CONFIG_OF_GPIO /* If the gpiochip has an assigned OF node this takes precedence */ - if (chip->of_node) - gdev->dev.of_node = chip->of_node; + if (chip->of_node) + gdev->dev.of_node = chip->of_node; #endif - } + gdev->id = ida_simple_get(&gpio_ida, 0, 0, GFP_KERNEL); if (gdev->id < 0) { status = gdev->id; @@ -2333,7 +2832,7 @@ static struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, desc = of_get_named_gpiod_flags(dev->of_node, prop_name, idx, &of_flags); - if (!IS_ERR(desc) || (PTR_ERR(desc) == -EPROBE_DEFER)) + if (!IS_ERR(desc) || (PTR_ERR(desc) != -ENOENT)) break; } diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index be43afb08..0238bf8bc 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -8,7 +8,7 @@ drm-y := drm_auth.o drm_bufs.o drm_cache.o \ drm_lock.o drm_memory.o drm_drv.o drm_vm.o \ drm_scatter.o drm_pci.o \ drm_platform.o drm_sysfs.o drm_hashtab.o drm_mm.o \ - drm_crtc.o drm_modes.o drm_edid.o \ + drm_crtc.o drm_fourcc.o drm_modes.o drm_edid.o \ drm_info.o drm_debugfs.o drm_encoder_slave.o \ drm_trace_points.o drm_global.o drm_prime.o \ drm_rect.o drm_vma_manager.o drm_flip_work.o \ @@ -23,7 +23,8 @@ drm-$(CONFIG_AGP) += drm_agpsupport.o drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_probe_helper.o \ drm_plane_helper.o drm_dp_mst_topology.o drm_atomic_helper.o \ - drm_kms_helper_common.o drm_dp_dual_mode_helper.o + drm_kms_helper_common.o drm_dp_dual_mode_helper.o \ + drm_simple_kms_helper.o drm_blend.o drm_kms_helper-$(CONFIG_DRM_LOAD_EDID_FIRMWARE) += drm_edid_load.o drm_kms_helper-$(CONFIG_DRM_FBDEV_EMULATION) += drm_fb_helper.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 56475b1f1..700c56baf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -85,8 +85,12 @@ extern int amdgpu_vm_debug; extern int amdgpu_sched_jobs; extern int amdgpu_sched_hw_submission; extern int amdgpu_powerplay; +extern int amdgpu_powercontainment; extern unsigned amdgpu_pcie_gen_cap; extern unsigned amdgpu_pcie_lane_cap; +extern unsigned amdgpu_cg_mask; +extern unsigned amdgpu_pg_mask; +extern char *amdgpu_disable_cu; #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000 #define AMDGPU_MAX_USEC_TIMEOUT 100000 /* 100 ms */ @@ -183,6 +187,10 @@ int amdgpu_set_clockgating_state(struct amdgpu_device *adev, int amdgpu_set_powergating_state(struct amdgpu_device *adev, enum amd_ip_block_type block_type, enum amd_powergating_state state); +int amdgpu_wait_for_idle(struct amdgpu_device *adev, + enum amd_ip_block_type block_type); +bool amdgpu_is_idle(struct amdgpu_device *adev, + enum amd_ip_block_type block_type); struct amdgpu_ip_block_version { enum amd_ip_block_type type; @@ -298,13 +306,16 @@ struct amdgpu_ring_funcs { uint32_t oa_base, uint32_t oa_size); /* testing functions */ int (*test_ring)(struct amdgpu_ring *ring); - int (*test_ib)(struct amdgpu_ring *ring); + int (*test_ib)(struct amdgpu_ring *ring, long timeout); /* insert NOP packets */ void (*insert_nop)(struct amdgpu_ring *ring, uint32_t count); /* pad the indirect buffer to the necessary number of dw */ void (*pad_ib)(struct amdgpu_ring *ring, struct amdgpu_ib *ib); unsigned (*init_cond_exec)(struct amdgpu_ring *ring); void (*patch_cond_exec)(struct amdgpu_ring *ring, unsigned offset); + /* note usage for clock and power gating */ + void (*begin_use)(struct amdgpu_ring *ring); + void (*end_use)(struct amdgpu_ring *ring); }; /* @@ -596,11 +607,9 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync, struct reservation_object *resv, void *owner); -bool amdgpu_sync_is_idle(struct amdgpu_sync *sync); -int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src, - struct fence *fence); +struct fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync, + struct amdgpu_ring *ring); struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync); -int amdgpu_sync_wait(struct amdgpu_sync *sync); void amdgpu_sync_free(struct amdgpu_sync *sync); int amdgpu_sync_init(void); void amdgpu_sync_fini(void); @@ -756,12 +765,11 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size, struct amdgpu_job **job); +void amdgpu_job_free_resources(struct amdgpu_job *job); void amdgpu_job_free(struct amdgpu_job *job); -void amdgpu_job_free_func(struct kref *refcount); int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring, struct amd_sched_entity *entity, void *owner, struct fence **f); -void amdgpu_job_timeout_func(struct work_struct *work); struct amdgpu_ring { struct amdgpu_device *adev; @@ -769,12 +777,9 @@ struct amdgpu_ring { struct amdgpu_fence_driver fence_drv; struct amd_gpu_scheduler sched; - spinlock_t fence_lock; struct amdgpu_bo *ring_obj; volatile uint32_t *ring; unsigned rptr_offs; - u64 next_rptr_gpu_addr; - volatile u32 *next_rptr_cpu_addr; unsigned wptr; unsigned wptr_old; unsigned ring_size; @@ -793,14 +798,16 @@ struct amdgpu_ring { u32 doorbell_index; bool use_doorbell; unsigned wptr_offs; - unsigned next_rptr_offs; unsigned fence_offs; uint64_t current_ctx; enum amdgpu_ring_type type; char name[16]; unsigned cond_exe_offs; - u64 cond_exe_gpu_addr; - volatile u32 *cond_exe_cpu_addr; + u64 cond_exe_gpu_addr; + volatile u32 *cond_exe_cpu_addr; +#if defined(CONFIG_DEBUG_FS) + struct dentry *ent; +#endif }; /* @@ -863,6 +870,7 @@ struct amdgpu_vm { struct amdgpu_bo *page_directory; unsigned max_pde_used; struct fence *page_directory_fence; + uint64_t last_eviction_counter; /* array of page tables, one for each page directory entry */ struct amdgpu_vm_pt *page_tables; @@ -885,13 +893,14 @@ struct amdgpu_vm_id { struct fence *first; struct amdgpu_sync active; struct fence *last_flush; - struct amdgpu_ring *last_user; atomic64_t owner; uint64_t pd_gpu_addr; /* last flushed PD/PT update */ struct fence *flushed_updates; + uint32_t current_gpu_reset_count; + uint32_t gds_base; uint32_t gds_size; uint32_t gws_base; @@ -907,6 +916,10 @@ struct amdgpu_vm_manager { struct list_head ids_lru; struct amdgpu_vm_id ids[AMDGPU_NUM_VM]; + /* Handling of VM fences */ + u64 fence_context; + unsigned seqno[AMDGPU_MAX_RINGS]; + uint32_t max_pfn; /* vram base address for page table entry */ u64 vram_base_offset; @@ -928,17 +941,14 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm); void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, struct list_head *validated, struct amdgpu_bo_list_entry *entry); -void amdgpu_vm_get_pt_bos(struct amdgpu_vm *vm, struct list_head *duplicates); +void amdgpu_vm_get_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct list_head *duplicates); void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev, struct amdgpu_vm *vm); int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_sync *sync, struct fence *fence, - unsigned *vm_id, uint64_t *vm_pd_addr); -int amdgpu_vm_flush(struct amdgpu_ring *ring, - unsigned vm_id, uint64_t pd_addr, - uint32_t gds_base, uint32_t gds_size, - uint32_t gws_base, uint32_t gws_size, - uint32_t oa_base, uint32_t oa_size); + struct amdgpu_job *job); +int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job); void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id); uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr); int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, @@ -1144,6 +1154,12 @@ struct amdgpu_cu_info { uint32_t bitmap[4][4]; }; +struct amdgpu_gfx_funcs { + /* get the gpu clock counter */ + uint64_t (*get_gpu_clock_counter)(struct amdgpu_device *adev); + void (*select_se_sh)(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance); +}; + struct amdgpu_gfx { struct mutex gpu_clock_mutex; struct amdgpu_gca_config config; @@ -1180,6 +1196,7 @@ struct amdgpu_gfx { /* ce ram size*/ unsigned ce_ram_size; struct amdgpu_cu_info cu_info; + const struct amdgpu_gfx_funcs *funcs; }; int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm, @@ -1197,10 +1214,6 @@ void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count); void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib); void amdgpu_ring_commit(struct amdgpu_ring *ring); void amdgpu_ring_undo(struct amdgpu_ring *ring); -unsigned amdgpu_ring_backup(struct amdgpu_ring *ring, - uint32_t **data); -int amdgpu_ring_restore(struct amdgpu_ring *ring, - unsigned size, uint32_t *data); int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned ring_size, u32 nop, u32 align_mask, struct amdgpu_irq_src *irq_src, unsigned irq_type, @@ -1252,6 +1265,7 @@ struct amdgpu_job { uint32_t num_ibs; void *owner; uint64_t ctx; + bool vm_needs_flush; unsigned vm_id; uint64_t vm_pd_addr; uint32_t gds_base, gds_size; @@ -1259,8 +1273,7 @@ struct amdgpu_job { uint32_t oa_base, oa_size; /* user fence handling */ - struct amdgpu_bo *uf_bo; - uint32_t uf_offset; + uint64_t uf_addr; uint64_t uf_sequence; }; @@ -1562,6 +1575,12 @@ struct amdgpu_dpm_funcs { u32 (*get_fan_control_mode)(struct amdgpu_device *adev); int (*set_fan_speed_percent)(struct amdgpu_device *adev, u32 speed); int (*get_fan_speed_percent)(struct amdgpu_device *adev, u32 *speed); + int (*force_clock_level)(struct amdgpu_device *adev, enum pp_clock_type type, uint32_t mask); + int (*print_clock_levels)(struct amdgpu_device *adev, enum pp_clock_type type, char *buf); + int (*get_sclk_od)(struct amdgpu_device *adev); + int (*set_sclk_od)(struct amdgpu_device *adev, uint32_t value); + int (*get_mclk_od)(struct amdgpu_device *adev); + int (*set_mclk_od)(struct amdgpu_device *adev, uint32_t value); }; struct amdgpu_dpm { @@ -1664,6 +1683,7 @@ struct amdgpu_uvd { struct amdgpu_ring ring; struct amdgpu_irq_src irq; bool address_64_bit; + bool use_ctx_buf; struct amd_sched_entity entity; }; @@ -1685,6 +1705,7 @@ struct amdgpu_vce { struct drm_file *filp[AMDGPU_MAX_VCE_HANDLES]; uint32_t img_size[AMDGPU_MAX_VCE_HANDLES]; struct delayed_work idle_work; + struct mutex idle_mutex; const struct firmware *fw; /* VCE firmware */ struct amdgpu_ring ring[AMDGPU_MAX_VCE_RINGS]; struct amdgpu_irq_src irq; @@ -1769,6 +1790,8 @@ int amdgpu_debugfs_init(struct drm_minor *minor); void amdgpu_debugfs_cleanup(struct drm_minor *minor); #endif +int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev); + /* * amdgpu smumgr functions */ @@ -1813,12 +1836,8 @@ struct amdgpu_asic_funcs { u32 sh_num, u32 reg_offset, u32 *value); void (*set_vga_state)(struct amdgpu_device *adev, bool state); int (*reset)(struct amdgpu_device *adev); - /* wait for mc_idle */ - int (*wait_for_mc_idle)(struct amdgpu_device *adev); /* get the reference clock */ u32 (*get_xclk)(struct amdgpu_device *adev); - /* get the gpu clock counter */ - uint64_t (*get_gpu_clock_counter)(struct amdgpu_device *adev); /* MM block clocks */ int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32 dclk); int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32 ecclk); @@ -2005,6 +2024,10 @@ struct amdgpu_device { spinlock_t didt_idx_lock; amdgpu_rreg_t didt_rreg; amdgpu_wreg_t didt_wreg; + /* protects concurrent gc_cac register access */ + spinlock_t gc_cac_idx_lock; + amdgpu_rreg_t gc_cac_rreg; + amdgpu_wreg_t gc_cac_wreg; /* protects concurrent ENDPOINT (audio) register access */ spinlock_t audio_endpt_idx_lock; amdgpu_block_rreg_t audio_endpt_rreg; @@ -2030,6 +2053,7 @@ struct amdgpu_device { atomic64_t vram_vis_usage; atomic64_t gtt_usage; atomic64_t num_bytes_moved; + atomic64_t num_evictions; atomic_t gpu_reset_counter; /* display */ @@ -2040,7 +2064,7 @@ struct amdgpu_device { struct amdgpu_irq_src hpd_irq; /* rings */ - unsigned fence_context; + u64 fence_context; unsigned num_rings; struct amdgpu_ring *rings[AMDGPU_MAX_RINGS]; bool ib_pool_ready; @@ -2133,6 +2157,8 @@ void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v); #define WREG32_UVD_CTX(reg, v) adev->uvd_ctx_wreg(adev, (reg), (v)) #define RREG32_DIDT(reg) adev->didt_rreg(adev, (reg)) #define WREG32_DIDT(reg, v) adev->didt_wreg(adev, (reg), (v)) +#define RREG32_GC_CAC(reg) adev->gc_cac_rreg(adev, (reg)) +#define WREG32_GC_CAC(reg, v) adev->gc_cac_wreg(adev, (reg), (v)) #define RREG32_AUDIO_ENDPT(block, reg) adev->audio_endpt_rreg(adev, (block), (reg)) #define WREG32_AUDIO_ENDPT(block, reg, v) adev->audio_endpt_wreg(adev, (block), (reg), (v)) #define WREG32_P(reg, val, mask) \ @@ -2208,12 +2234,10 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring) */ #define amdgpu_asic_set_vga_state(adev, state) (adev)->asic_funcs->set_vga_state((adev), (state)) #define amdgpu_asic_reset(adev) (adev)->asic_funcs->reset((adev)) -#define amdgpu_asic_wait_for_mc_idle(adev) (adev)->asic_funcs->wait_for_mc_idle((adev)) #define amdgpu_asic_get_xclk(adev) (adev)->asic_funcs->get_xclk((adev)) #define amdgpu_asic_set_uvd_clocks(adev, v, d) (adev)->asic_funcs->set_uvd_clocks((adev), (v), (d)) #define amdgpu_asic_set_vce_clocks(adev, ev, ec) (adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec)) #define amdgpu_asic_get_virtual_caps(adev) ((adev)->asic_funcs->get_virtual_caps((adev))) -#define amdgpu_asic_get_gpu_clock_counter(adev) (adev)->asic_funcs->get_gpu_clock_counter((adev)) #define amdgpu_asic_read_disabled_bios(adev) (adev)->asic_funcs->read_disabled_bios((adev)) #define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l)) #define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v))) @@ -2224,7 +2248,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring) #define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags))) #define amdgpu_ring_parse_cs(r, p, ib) ((r)->funcs->parse_cs((p), (ib))) #define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r)) -#define amdgpu_ring_test_ib(r) (r)->funcs->test_ib((r)) +#define amdgpu_ring_test_ib(r, t) (r)->funcs->test_ib((r), (t)) #define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r)) #define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r)) #define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r)) @@ -2266,6 +2290,8 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring) #define amdgpu_dpm_print_power_state(adev, ps) (adev)->pm.funcs->print_power_state((adev), (ps)) #define amdgpu_dpm_vblank_too_short(adev) (adev)->pm.funcs->vblank_too_short((adev)) #define amdgpu_dpm_enable_bapm(adev, e) (adev)->pm.funcs->enable_bapm((adev), (e)) +#define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev)) +#define amdgpu_gfx_select_se_sh(adev, se, sh, instance) (adev)->gfx.funcs->select_se_sh((adev), (se), (sh), (instance)) #define amdgpu_dpm_get_temperature(adev) \ ((adev)->pp_enabled ? \ @@ -2344,6 +2370,18 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring) #define amdgpu_dpm_force_clock_level(adev, type, level) \ (adev)->powerplay.pp_funcs->force_clock_level((adev)->powerplay.pp_handle, type, level) +#define amdgpu_dpm_get_sclk_od(adev) \ + (adev)->powerplay.pp_funcs->get_sclk_od((adev)->powerplay.pp_handle) + +#define amdgpu_dpm_set_sclk_od(adev, value) \ + (adev)->powerplay.pp_funcs->set_sclk_od((adev)->powerplay.pp_handle, value) + +#define amdgpu_dpm_get_mclk_od(adev) \ + ((adev)->powerplay.pp_funcs->get_mclk_od((adev)->powerplay.pp_handle)) + +#define amdgpu_dpm_set_mclk_od(adev, value) \ + ((adev)->powerplay.pp_funcs->set_mclk_od((adev)->powerplay.pp_handle, value)) + #define amdgpu_dpm_dispatch_task(adev, event_id, input, output) \ (adev)->powerplay.pp_funcs->dispatch_tasks((adev)->powerplay.pp_handle, (event_id), (input), (output)) @@ -2385,9 +2423,13 @@ bool amdgpu_device_is_px(struct drm_device *dev); #if defined(CONFIG_VGA_SWITCHEROO) void amdgpu_register_atpx_handler(void); void amdgpu_unregister_atpx_handler(void); +bool amdgpu_has_atpx_dgpu_power_cntl(void); +bool amdgpu_is_atpx_hybrid(void); #else static inline void amdgpu_register_atpx_handler(void) {} static inline void amdgpu_unregister_atpx_handler(void) {} +static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; } +static inline bool amdgpu_is_atpx_hybrid(void) { return false; } #endif /* diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c index 252edba16..892d60fb2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c @@ -421,29 +421,6 @@ static int acp_suspend(void *handle) static int acp_resume(void *handle) { - int i, ret; - struct acp_pm_domain *apd; - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - /* return early if no ACP */ - if (!adev->acp.acp_genpd) - return 0; - - /* SMU block will power on ACP irrespective of ACP runtime status. - * Power off explicitly based on genpd ACP runtime status so that ACP - * hw and ACP-genpd status are in sync. - * 'suspend_power_off' represents "Power status before system suspend" - */ - if (adev->acp.acp_genpd->gpd.suspend_power_off == true) { - apd = container_of(&adev->acp.acp_genpd->gpd, - struct acp_pm_domain, gpd); - - for (i = 4; i >= 0 ; i--) { - ret = acp_suspend_tile(apd->cgs_dev, ACP_TILE_P1 + i); - if (ret) - pr_err("ACP tile %d tile suspend failed\n", i); - } - } return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 32809f749..d080d0807 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -240,8 +240,8 @@ uint64_t get_gpu_clock_counter(struct kgd_dev *kgd) { struct amdgpu_device *rdev = (struct amdgpu_device *)kgd; - if (rdev->asic_funcs->get_gpu_clock_counter) - return rdev->asic_funcs->get_gpu_clock_counter(rdev); + if (rdev->gfx.funcs->get_gpu_clock_counter) + return rdev->gfx.funcs->get_gpu_clock_counter(rdev); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c index 1b4c069f7..10b5ddf2c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c @@ -28,6 +28,7 @@ struct amdgpu_atpx_functions { struct amdgpu_atpx { acpi_handle handle; struct amdgpu_atpx_functions functions; + bool is_hybrid; }; static struct amdgpu_atpx_priv { @@ -64,6 +65,14 @@ bool amdgpu_has_atpx(void) { return amdgpu_atpx_priv.atpx_detected; } +bool amdgpu_has_atpx_dgpu_power_cntl(void) { + return amdgpu_atpx_priv.atpx.functions.power_cntl; +} + +bool amdgpu_is_atpx_hybrid(void) { + return amdgpu_atpx_priv.atpx.is_hybrid; +} + /** * amdgpu_atpx_call - call an ATPX method * @@ -143,18 +152,12 @@ static void amdgpu_atpx_parse_functions(struct amdgpu_atpx_functions *f, u32 mas */ static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx) { - /* make sure required functions are enabled */ - /* dGPU power control is required */ - if (atpx->functions.power_cntl == false) { - printk("ATPX dGPU power cntl not present, forcing\n"); - atpx->functions.power_cntl = true; - } + u32 valid_bits = 0; if (atpx->functions.px_params) { union acpi_object *info; struct atpx_px_params output; size_t size; - u32 valid_bits; info = amdgpu_atpx_call(atpx->handle, ATPX_FUNCTION_GET_PX_PARAMETERS, NULL); if (!info) @@ -173,19 +176,34 @@ static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx) memcpy(&output, info->buffer.pointer, size); valid_bits = output.flags & output.valid_flags; - /* if separate mux flag is set, mux controls are required */ - if (valid_bits & ATPX_SEPARATE_MUX_FOR_I2C) { - atpx->functions.i2c_mux_cntl = true; - atpx->functions.disp_mux_cntl = true; - } - /* if any outputs are muxed, mux controls are required */ - if (valid_bits & (ATPX_CRT1_RGB_SIGNAL_MUXED | - ATPX_TV_SIGNAL_MUXED | - ATPX_DFP_SIGNAL_MUXED)) - atpx->functions.disp_mux_cntl = true; kfree(info); } + + /* if separate mux flag is set, mux controls are required */ + if (valid_bits & ATPX_SEPARATE_MUX_FOR_I2C) { + atpx->functions.i2c_mux_cntl = true; + atpx->functions.disp_mux_cntl = true; + } + /* if any outputs are muxed, mux controls are required */ + if (valid_bits & (ATPX_CRT1_RGB_SIGNAL_MUXED | + ATPX_TV_SIGNAL_MUXED | + ATPX_DFP_SIGNAL_MUXED)) + atpx->functions.disp_mux_cntl = true; + + + /* some bioses set these bits rather than flagging power_cntl as supported */ + if (valid_bits & (ATPX_DYNAMIC_PX_SUPPORTED | + ATPX_DYNAMIC_DGPU_POWER_OFF_SUPPORTED)) + atpx->functions.power_cntl = true; + + atpx->is_hybrid = false; + if (valid_bits & ATPX_MS_HYBRID_GFX_SUPPORTED) { + printk("ATPX Hybrid Graphics\n"); + atpx->functions.power_cntl = false; + atpx->is_hybrid = true; + } + return 0; } @@ -512,7 +530,6 @@ static int amdgpu_atpx_get_client_id(struct pci_dev *pdev) static const struct vga_switcheroo_handler amdgpu_atpx_handler = { .switchto = amdgpu_atpx_switchto, .power_state = amdgpu_atpx_power_state, - .init = amdgpu_atpx_init, .get_client_id = amdgpu_atpx_get_client_id, }; @@ -547,6 +564,7 @@ static bool amdgpu_atpx_detect(void) printk(KERN_INFO "vga_switcheroo: detected switching method %s handle\n", acpi_method_name); amdgpu_atpx_priv.atpx_detected = true; + amdgpu_atpx_init(); return true; } return false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c index 99ca75baa..2b6afe123 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c @@ -352,22 +352,22 @@ bool amdgpu_get_bios(struct amdgpu_device *adev) uint16_t tmp, bios_header_start; r = amdgpu_atrm_get_bios(adev); - if (r == false) + if (!r) r = amdgpu_acpi_vfct_bios(adev); - if (r == false) + if (!r) r = igp_read_bios_from_vram(adev); - if (r == false) + if (!r) r = amdgpu_read_bios(adev); - if (r == false) { + if (!r) { r = amdgpu_read_bios_from_rom(adev); } - if (r == false) { + if (!r) { r = amdgpu_read_disabled_bios(adev); } - if (r == false) { + if (!r) { r = amdgpu_read_platform_bios(adev); } - if (r == false || adev->bios == NULL) { + if (!r || adev->bios == NULL) { DRM_ERROR("Unable to locate a BIOS ROM\n"); adev->bios = NULL; return false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c index 823bf5e0b..651115dcc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c @@ -94,6 +94,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev, unsigned last_entry = 0, first_userptr = num_entries; unsigned i; int r; + unsigned long total_size = 0; array = drm_malloc_ab(num_entries, sizeof(struct amdgpu_bo_list_entry)); if (!array) @@ -140,6 +141,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev, if (entry->robj->prefered_domains == AMDGPU_GEM_DOMAIN_OA) oa_obj = entry->robj; + total_size += amdgpu_bo_size(entry->robj); trace_amdgpu_bo_list_set(list, entry->robj); } @@ -155,6 +157,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev, list->array = array; list->num_entries = num_entries; + trace_amdgpu_cs_bo_status(list->num_entries, total_size); return 0; error_free: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c index 6f9dcfddc..16e248e3f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c @@ -312,6 +312,8 @@ static uint32_t amdgpu_cgs_read_ind_register(struct cgs_device *cgs_device, return RREG32_UVD_CTX(index); case CGS_IND_REG__DIDT: return RREG32_DIDT(index); + case CGS_IND_REG_GC_CAC: + return RREG32_GC_CAC(index); case CGS_IND_REG__AUDIO_ENDPT: DRM_ERROR("audio endpt register access not implemented.\n"); return 0; @@ -336,6 +338,8 @@ static void amdgpu_cgs_write_ind_register(struct cgs_device *cgs_device, return WREG32_UVD_CTX(index, value); case CGS_IND_REG__DIDT: return WREG32_DIDT(index, value); + case CGS_IND_REG_GC_CAC: + return WREG32_GC_CAC(index, value); case CGS_IND_REG__AUDIO_ENDPT: DRM_ERROR("audio endpt register access not implemented.\n"); return; @@ -748,6 +752,9 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device, if (!adev->pm.fw) { switch (adev->asic_type) { + case CHIP_TOPAZ: + strcpy(fw_name, "/*(DEBLOBBED)*/"); + break; case CHIP_TONGA: strcpy(fw_name, "/*(DEBLOBBED)*/"); break; @@ -787,6 +794,7 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device, } hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data; + amdgpu_ucode_print_smc_hdr(&hdr->header); adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version); ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes); ucode_start_address = le32_to_cpu(hdr->ucode_start_addr); @@ -795,13 +803,14 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device, info->version = adev->pm.fw_version; info->image_size = ucode_size; + info->ucode_start_address = ucode_start_address; info->kptr = (void *)src; } return 0; } static int amdgpu_cgs_query_system_info(struct cgs_device *cgs_device, - struct cgs_system_info *sys_info) + struct cgs_system_info *sys_info) { CGS_FUNC_ADEV; @@ -821,6 +830,12 @@ static int amdgpu_cgs_query_system_info(struct cgs_device *cgs_device, case CGS_SYSTEM_INFO_PCIE_MLW: sys_info->value = adev->pm.pcie_mlw_mask; break; + case CGS_SYSTEM_INFO_PCIE_DEV: + sys_info->value = adev->pdev->device; + break; + case CGS_SYSTEM_INFO_PCIE_REV: + sys_info->value = adev->pdev->revision; + break; case CGS_SYSTEM_INFO_CG_FLAGS: sys_info->value = adev->cg_flags; break; @@ -830,6 +845,9 @@ static int amdgpu_cgs_query_system_info(struct cgs_device *cgs_device, case CGS_SYSTEM_INFO_GFX_CU_INFO: sys_info->value = adev->gfx.cu_info.number; break; + case CGS_SYSTEM_INFO_GFX_SE_INFO: + sys_info->value = adev->gfx.config.max_shader_engines; + break; default: return -ENODEV; } @@ -903,14 +921,12 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, acpi_handle handle; struct acpi_object_list input; struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; - union acpi_object *params = NULL; - union acpi_object *obj = NULL; + union acpi_object *params, *obj; uint8_t name[5] = {'\0'}; - struct cgs_acpi_method_argument *argument = NULL; + struct cgs_acpi_method_argument *argument; uint32_t i, count; acpi_status status; - int result = 0; - uint32_t func_no = 0xFFFFFFFF; + int result; handle = ACPI_HANDLE(&adev->pdev->dev); if (!handle) @@ -927,7 +943,6 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, if (info->pinput_argument == NULL) return -EINVAL; argument = info->pinput_argument; - func_no = argument->value; for (i = 0; i < info->input_count; i++) { if (((argument->type == ACPI_TYPE_STRING) || (argument->type == ACPI_TYPE_BUFFER)) && @@ -972,11 +987,11 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, params->integer.value = argument->value; break; case ACPI_TYPE_STRING: - params->string.length = argument->method_length; + params->string.length = argument->data_length; params->string.pointer = argument->pointer; break; case ACPI_TYPE_BUFFER: - params->buffer.length = argument->method_length; + params->buffer.length = argument->data_length; params->buffer.pointer = argument->pointer; break; default: @@ -996,7 +1011,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, if (ACPI_FAILURE(status)) { result = -EIO; - goto error; + goto free_input; } /* return the output info */ @@ -1006,7 +1021,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, if ((obj->type != ACPI_TYPE_PACKAGE) || (obj->package.count != count)) { result = -EIO; - goto error; + goto free_obj; } params = obj->package.elements; } else @@ -1014,13 +1029,13 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, if (params == NULL) { result = -EIO; - goto error; + goto free_obj; } for (i = 0; i < count; i++) { if (argument->type != params->type) { result = -EIO; - goto error; + goto free_obj; } switch (params->type) { case ACPI_TYPE_INTEGER: @@ -1030,7 +1045,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, if ((params->string.length != argument->data_length) || (params->string.pointer == NULL)) { result = -EIO; - goto error; + goto free_obj; } strncpy(argument->pointer, params->string.pointer, @@ -1039,7 +1054,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, case ACPI_TYPE_BUFFER: if (params->buffer.pointer == NULL) { result = -EIO; - goto error; + goto free_obj; } memcpy(argument->pointer, params->buffer.pointer, @@ -1052,9 +1067,10 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, params++; } -error: - if (obj != NULL) - kfree(obj); + result = 0; +free_obj: + kfree(obj); +free_input: kfree((void *)input.pointer); return result; } @@ -1066,7 +1082,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device, } #endif -int amdgpu_cgs_call_acpi_method(struct cgs_device *cgs_device, +static int amdgpu_cgs_call_acpi_method(struct cgs_device *cgs_device, uint32_t acpi_method, uint32_t acpi_function, void *pinput, void *poutput, @@ -1079,17 +1095,14 @@ int amdgpu_cgs_call_acpi_method(struct cgs_device *cgs_device, struct cgs_acpi_method_info info = {0}; acpi_input[0].type = CGS_ACPI_TYPE_INTEGER; - acpi_input[0].method_length = sizeof(uint32_t); acpi_input[0].data_length = sizeof(uint32_t); acpi_input[0].value = acpi_function; acpi_input[1].type = CGS_ACPI_TYPE_BUFFER; - acpi_input[1].method_length = CGS_ACPI_MAX_BUFFER_SIZE; acpi_input[1].data_length = input_size; acpi_input[1].pointer = pinput; acpi_output.type = CGS_ACPI_TYPE_BUFFER; - acpi_output.method_length = CGS_ACPI_MAX_BUFFER_SIZE; acpi_output.data_length = output_size; acpi_output.pointer = poutput; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 9bc8f1d99..0307ff588 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -216,11 +216,8 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) if (ret) goto free_all_kdata; - if (p->uf_entry.robj) { - p->job->uf_bo = amdgpu_bo_ref(p->uf_entry.robj); - p->job->uf_offset = uf_offset; - } - + if (p->uf_entry.robj) + p->job->uf_addr = uf_offset; kfree(chunk_array); return 0; @@ -459,7 +456,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, list_splice(&need_pages, &p->validated); } - amdgpu_vm_get_pt_bos(&fpriv->vm, &duplicates); + amdgpu_vm_get_pt_bos(p->adev, &fpriv->vm, &duplicates); p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev); p->bytes_moved = 0; @@ -472,6 +469,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, if (r) goto error_validate; + fpriv->vm.last_eviction_counter = + atomic64_read(&p->adev->num_evictions); + if (p->bo_list) { struct amdgpu_bo *gds = p->bo_list->gds_obj; struct amdgpu_bo *gws = p->bo_list->gws_obj; @@ -499,6 +499,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, } } + if (p->uf_entry.robj) + p->job->uf_addr += amdgpu_bo_gpu_offset(p->uf_entry.robj); + error_validate: if (r) { amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm); @@ -653,18 +656,21 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev, /* Only for UVD/VCE VM emulation */ if (ring->funcs->parse_cs) { + p->job->vm = NULL; for (i = 0; i < p->job->num_ibs; i++) { r = amdgpu_ring_parse_cs(ring, p, i); if (r) return r; } - } + } else { + p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->page_directory); - r = amdgpu_bo_vm_update_pte(p, vm); - if (!r) - amdgpu_cs_sync_rings(p); + r = amdgpu_bo_vm_update_pte(p, vm); + if (r) + return r; + } - return r; + return amdgpu_cs_sync_rings(p); } static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r) @@ -761,7 +767,7 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, } /* UVD & VCE fw doesn't support user fences */ - if (parser->job->uf_bo && ( + if (parser->job->uf_addr && ( parser->job->ring->type == AMDGPU_RING_TYPE_UVD || parser->job->ring->type == AMDGPU_RING_TYPE_VCE)) return -EINVAL; @@ -830,17 +836,13 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, { struct amdgpu_ring *ring = p->job->ring; struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity; - struct fence *fence; struct amdgpu_job *job; int r; job = p->job; p->job = NULL; - r = amd_sched_job_init(&job->base, &ring->sched, - entity, amdgpu_job_timeout_func, - amdgpu_job_free_func, - p->filp, &fence); + r = amd_sched_job_init(&job->base, &ring->sched, entity, p->filp); if (r) { amdgpu_job_free(job); return r; @@ -848,9 +850,10 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, job->owner = p->filp; job->ctx = entity->fence_context; - p->fence = fence_get(fence); - cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, fence); + p->fence = fence_get(&job->base.s_fence->finished); + cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence); job->uf_sequence = cs->out.handle; + amdgpu_job_free_resources(job); trace_amdgpu_cs_ioctl(job); amd_sched_entity_push_job(&job->base); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index b7f5650d8..39c01b942 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -25,6 +25,7 @@ * Alex Deucher * Jerome Glisse */ +#include #include #include #include @@ -35,6 +36,7 @@ #include #include #include "amdgpu.h" +#include "amdgpu_trace.h" #include "amdgpu_i2c.h" #include "atom.h" #include "amdgpu_atombios.h" @@ -79,24 +81,27 @@ bool amdgpu_device_is_px(struct drm_device *dev) uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, bool always_indirect) { + uint32_t ret; + if ((reg * 4) < adev->rmmio_size && !always_indirect) - return readl(((void __iomem *)adev->rmmio) + (reg * 4)); + ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); else { unsigned long flags; - uint32_t ret; spin_lock_irqsave(&adev->mmio_idx_lock, flags); writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4)); ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4)); spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); - - return ret; } + trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret); + return ret; } void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, bool always_indirect) { + trace_amdgpu_mm_wreg(adev->pdev->device, reg, v); + if ((reg * 4) < adev->rmmio_size && !always_indirect) writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); else { @@ -1070,11 +1075,14 @@ int amdgpu_set_clockgating_state(struct amdgpu_device *adev, int i, r = 0; for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_block_status[i].valid) + continue; if (adev->ip_blocks[i].type == block_type) { r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev, state); if (r) return r; + break; } } return r; @@ -1087,16 +1095,53 @@ int amdgpu_set_powergating_state(struct amdgpu_device *adev, int i, r = 0; for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_block_status[i].valid) + continue; if (adev->ip_blocks[i].type == block_type) { r = adev->ip_blocks[i].funcs->set_powergating_state((void *)adev, state); if (r) return r; + break; } } return r; } +int amdgpu_wait_for_idle(struct amdgpu_device *adev, + enum amd_ip_block_type block_type) +{ + int i, r; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_block_status[i].valid) + continue; + if (adev->ip_blocks[i].type == block_type) { + r = adev->ip_blocks[i].funcs->wait_for_idle((void *)adev); + if (r) + return r; + break; + } + } + return 0; + +} + +bool amdgpu_is_idle(struct amdgpu_device *adev, + enum amd_ip_block_type block_type) +{ + int i; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_block_status[i].valid) + continue; + if (adev->ip_blocks[i].type == block_type) + return adev->ip_blocks[i].funcs->is_idle((void *)adev); + } + return true; + +} + const struct amdgpu_ip_block_version * amdgpu_get_ip_block( struct amdgpu_device *adev, enum amd_ip_block_type type) @@ -1209,6 +1254,9 @@ static int amdgpu_early_init(struct amdgpu_device *adev) } } + adev->cg_flags &= amdgpu_cg_mask; + adev->pg_flags &= amdgpu_pg_mask; + return 0; } @@ -1440,9 +1488,12 @@ int amdgpu_device_init(struct amdgpu_device *adev, adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; adev->didt_rreg = &amdgpu_invalid_rreg; adev->didt_wreg = &amdgpu_invalid_wreg; + adev->gc_cac_rreg = &amdgpu_invalid_rreg; + adev->gc_cac_wreg = &amdgpu_invalid_wreg; adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; + DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n", amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); @@ -1467,6 +1518,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, spin_lock_init(&adev->pcie_idx_lock); spin_lock_init(&adev->uvd_ctx_idx_lock); spin_lock_init(&adev->didt_idx_lock); + spin_lock_init(&adev->gc_cac_idx_lock); spin_lock_init(&adev->audio_endpt_idx_lock); adev->rmmio_base = pci_resource_start(adev->pdev, 5); @@ -1511,17 +1563,20 @@ int amdgpu_device_init(struct amdgpu_device *adev, vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); /* Read BIOS */ - if (!amdgpu_get_bios(adev)) - return -EINVAL; + if (!amdgpu_get_bios(adev)) { + r = -EINVAL; + goto failed; + } /* Must be an ATOMBIOS */ if (!adev->is_atom_bios) { dev_err(adev->dev, "Expecting atombios for GPU\n"); - return -EINVAL; + r = -EINVAL; + goto failed; } r = amdgpu_atombios_init(adev); if (r) { dev_err(adev->dev, "amdgpu_atombios_init failed\n"); - return r; + goto failed; } /* See if the asic supports SR-IOV */ @@ -1538,7 +1593,8 @@ int amdgpu_device_init(struct amdgpu_device *adev, !(adev->virtualization.caps & AMDGPU_VIRT_CAPS_SRIOV_EN))) { if (!adev->bios) { dev_err(adev->dev, "Card not posted and no BIOS - ignoring\n"); - return -EINVAL; + r = -EINVAL; + goto failed; } DRM_INFO("GPU not posted. posting now...\n"); amdgpu_atom_asic_init(adev->mode_info.atom_context); @@ -1548,7 +1604,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, r = amdgpu_atombios_get_clock_info(adev); if (r) { dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); - return r; + goto failed; } /* init i2c buses */ amdgpu_atombios_i2c_init(adev); @@ -1557,7 +1613,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, r = amdgpu_fence_driver_init(adev); if (r) { dev_err(adev->dev, "amdgpu_fence_driver_init failed\n"); - return r; + goto failed; } /* init the mode config */ @@ -1567,7 +1623,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, if (r) { dev_err(adev->dev, "amdgpu_init failed\n"); amdgpu_fini(adev); - return r; + goto failed; } adev->accel_working = true; @@ -1577,7 +1633,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, r = amdgpu_ib_pool_init(adev); if (r) { dev_err(adev->dev, "IB initialization failed (%d).\n", r); - return r; + goto failed; } r = amdgpu_ib_ring_tests(adev); @@ -1594,6 +1650,12 @@ int amdgpu_device_init(struct amdgpu_device *adev, DRM_ERROR("registering register debugfs failed (%d).\n", r); } + r = amdgpu_debugfs_firmware_init(adev); + if (r) { + DRM_ERROR("registering firmware debugfs failed (%d).\n", r); + return r; + } + if ((amdgpu_testing & 1)) { if (adev->accel_working) amdgpu_test_moves(adev); @@ -1619,10 +1681,15 @@ int amdgpu_device_init(struct amdgpu_device *adev, r = amdgpu_late_init(adev); if (r) { dev_err(adev->dev, "amdgpu_late_init failed\n"); - return r; + goto failed; } return 0; + +failed: + if (runtime) + vga_switcheroo_fini_domain_pm_ops(adev->dev); + return r; } static void amdgpu_debugfs_remove_files(struct amdgpu_device *adev); @@ -1641,6 +1708,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev) DRM_INFO("amdgpu: finishing device.\n"); adev->shutdown = true; + drm_crtc_force_disable_all(adev->ddev); /* evict vram memory */ amdgpu_bo_evict_vram(adev); amdgpu_ib_pool_fini(adev); @@ -1656,6 +1724,8 @@ void amdgpu_device_fini(struct amdgpu_device *adev) kfree(adev->bios); adev->bios = NULL; vga_switcheroo_unregister_client(adev->pdev); + if (adev->flags & AMD_IS_PX) + vga_switcheroo_fini_domain_pm_ops(adev->dev); vga_client_register(adev->pdev, NULL, NULL, NULL); if (adev->rio_mem) pci_iounmap(adev->pdev, adev->rio_mem); @@ -1877,11 +1947,6 @@ int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon) */ int amdgpu_gpu_reset(struct amdgpu_device *adev) { - unsigned ring_sizes[AMDGPU_MAX_RINGS]; - uint32_t *ring_data[AMDGPU_MAX_RINGS]; - - bool saved = false; - int i, r; int resched; @@ -1890,22 +1955,30 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev) /* block TTM */ resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); - r = amdgpu_suspend(adev); - + /* block scheduler */ for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { struct amdgpu_ring *ring = adev->rings[i]; + if (!ring) continue; - - ring_sizes[i] = amdgpu_ring_backup(ring, &ring_data[i]); - if (ring_sizes[i]) { - saved = true; - dev_info(adev->dev, "Saved %d dwords of commands " - "on ring %d.\n", ring_sizes[i], i); - } + kthread_park(ring->sched.thread); + amd_sched_hw_job_reset(&ring->sched); } + /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ + amdgpu_fence_driver_force_completion(adev); + + /* save scratch */ + amdgpu_atombios_scratch_regs_save(adev); + r = amdgpu_suspend(adev); retry: + /* Disable fb access */ + if (adev->mode_info.num_crtc) { + struct amdgpu_mode_mc_save save; + amdgpu_display_stop_mc_access(adev, &save); + amdgpu_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GMC); + } + r = amdgpu_asic_reset(adev); /* post card */ amdgpu_atom_asic_init(adev->mode_info.atom_context); @@ -1914,32 +1987,29 @@ retry: dev_info(adev->dev, "GPU reset succeeded, trying to resume\n"); r = amdgpu_resume(adev); } - + /* restore scratch */ + amdgpu_atombios_scratch_regs_restore(adev); if (!r) { + r = amdgpu_ib_ring_tests(adev); + if (r) { + dev_err(adev->dev, "ib ring test failed (%d).\n", r); + r = amdgpu_suspend(adev); + goto retry; + } + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { struct amdgpu_ring *ring = adev->rings[i]; if (!ring) continue; - - amdgpu_ring_restore(ring, ring_sizes[i], ring_data[i]); - ring_sizes[i] = 0; - ring_data[i] = NULL; - } - - r = amdgpu_ib_ring_tests(adev); - if (r) { - dev_err(adev->dev, "ib ring test failed (%d).\n", r); - if (saved) { - saved = false; - r = amdgpu_suspend(adev); - goto retry; - } + amd_sched_job_recovery(&ring->sched); + kthread_unpark(ring->sched.thread); } } else { - amdgpu_fence_driver_force_completion(adev); + dev_err(adev->dev, "asic resume failed (%d).\n", r); for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - if (adev->rings[i]) - kfree(ring_data[i]); + if (adev->rings[i]) { + kthread_unpark(adev->rings[i]->sched.thread); + } } } @@ -1950,13 +2020,11 @@ retry: /* bad news, how to tell it to userspace ? */ dev_info(adev->dev, "GPU reset failed\n"); } + amdgpu_irq_gpu_reset_resume_helper(adev); return r; } -#define AMDGPU_DEFAULT_PCIE_GEN_MASK 0x30007 /* gen: chipset 1/2, asic 1/2/3 */ -#define AMDGPU_DEFAULT_PCIE_MLW_MASK 0x2f0000 /* 1/2/4/8/16 lanes */ - void amdgpu_get_pcie_info(struct amdgpu_device *adev) { u32 mask; @@ -2110,20 +2178,43 @@ static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf, struct amdgpu_device *adev = f->f_inode->i_private; ssize_t result = 0; int r; + bool use_bank; + unsigned instance_bank, sh_bank, se_bank; if (size & 0x3 || *pos & 0x3) return -EINVAL; + if (*pos & (1ULL << 62)) { + se_bank = (*pos >> 24) & 0x3FF; + sh_bank = (*pos >> 34) & 0x3FF; + instance_bank = (*pos >> 44) & 0x3FF; + use_bank = 1; + *pos &= 0xFFFFFF; + } else { + use_bank = 0; + } + + if (use_bank) { + if (sh_bank >= adev->gfx.config.max_sh_per_se || + se_bank >= adev->gfx.config.max_shader_engines) + return -EINVAL; + mutex_lock(&adev->grbm_idx_mutex); + amdgpu_gfx_select_se_sh(adev, se_bank, + sh_bank, instance_bank); + } + while (size) { uint32_t value; if (*pos > adev->rmmio_size) - return result; + goto end; value = RREG32(*pos >> 2); r = put_user(value, (uint32_t *)buf); - if (r) - return r; + if (r) { + result = r; + goto end; + } result += 4; buf += 4; @@ -2131,6 +2222,12 @@ static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf, size -= 4; } +end: + if (use_bank) { + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + mutex_unlock(&adev->grbm_idx_mutex); + } + return result; } @@ -2330,6 +2427,68 @@ static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user * return result; } +static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = f->f_inode->i_private; + ssize_t result = 0; + int r; + uint32_t *config, no_regs = 0; + + if (size & 0x3 || *pos & 0x3) + return -EINVAL; + + config = kmalloc(256 * sizeof(*config), GFP_KERNEL); + if (!config) + return -ENOMEM; + + /* version, increment each time something is added */ + config[no_regs++] = 0; + config[no_regs++] = adev->gfx.config.max_shader_engines; + config[no_regs++] = adev->gfx.config.max_tile_pipes; + config[no_regs++] = adev->gfx.config.max_cu_per_sh; + config[no_regs++] = adev->gfx.config.max_sh_per_se; + config[no_regs++] = adev->gfx.config.max_backends_per_se; + config[no_regs++] = adev->gfx.config.max_texture_channel_caches; + config[no_regs++] = adev->gfx.config.max_gprs; + config[no_regs++] = adev->gfx.config.max_gs_threads; + config[no_regs++] = adev->gfx.config.max_hw_contexts; + config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend; + config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend; + config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size; + config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size; + config[no_regs++] = adev->gfx.config.num_tile_pipes; + config[no_regs++] = adev->gfx.config.backend_enable_mask; + config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes; + config[no_regs++] = adev->gfx.config.mem_row_size_in_kb; + config[no_regs++] = adev->gfx.config.shader_engine_tile_size; + config[no_regs++] = adev->gfx.config.num_gpus; + config[no_regs++] = adev->gfx.config.multi_gpu_tile_size; + config[no_regs++] = adev->gfx.config.mc_arb_ramcfg; + config[no_regs++] = adev->gfx.config.gb_addr_config; + config[no_regs++] = adev->gfx.config.num_rbs; + + while (size && (*pos < no_regs * 4)) { + uint32_t value; + + value = config[*pos >> 2]; + r = put_user(value, (uint32_t *)buf); + if (r) { + kfree(config); + return r; + } + + result += 4; + buf += 4; + *pos += 4; + size -= 4; + } + + kfree(config); + return result; +} + + static const struct file_operations amdgpu_debugfs_regs_fops = { .owner = THIS_MODULE, .read = amdgpu_debugfs_regs_read, @@ -2355,11 +2514,18 @@ static const struct file_operations amdgpu_debugfs_regs_smc_fops = { .llseek = default_llseek }; +static const struct file_operations amdgpu_debugfs_gca_config_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_gca_config_read, + .llseek = default_llseek +}; + static const struct file_operations *debugfs_regs[] = { &amdgpu_debugfs_regs_fops, &amdgpu_debugfs_regs_didt_fops, &amdgpu_debugfs_regs_pcie_fops, &amdgpu_debugfs_regs_smc_fops, + &amdgpu_debugfs_gca_config_fops, }; static const char *debugfs_regs_names[] = { @@ -2367,6 +2533,7 @@ static const char *debugfs_regs_names[] = { "amdgpu_regs_didt", "amdgpu_regs_pcie", "amdgpu_regs_smc", + "amdgpu_gca_config", }; static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c index b0832da2e..76f960283 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c @@ -122,7 +122,7 @@ static void amdgpu_flip_work_func(struct work_struct *__work) spin_unlock_irqrestore(&crtc->dev->event_lock, flags); usleep_range(min_udelay, 2 * min_udelay); spin_lock_irqsave(&crtc->dev->event_lock, flags); - }; + } if (!repcnt) DRM_DEBUG_DRIVER("Delay problem on crtc %d: min_udelay %d, " @@ -220,19 +220,17 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc, r = amdgpu_bo_pin_restricted(new_rbo, AMDGPU_GEM_DOMAIN_VRAM, 0, 0, &base); if (unlikely(r != 0)) { - amdgpu_bo_unreserve(new_rbo); r = -EINVAL; DRM_ERROR("failed to pin new rbo buffer before flip\n"); - goto cleanup; + goto unreserve; } r = reservation_object_get_fences_rcu(new_rbo->tbo.resv, &work->excl, &work->shared_count, &work->shared); if (unlikely(r != 0)) { - amdgpu_bo_unreserve(new_rbo); DRM_ERROR("failed to get fences for buffer\n"); - goto cleanup; + goto unpin; } amdgpu_bo_get_tiling_flags(new_rbo, &tiling_flags); @@ -240,7 +238,7 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc, work->base = base; - r = drm_vblank_get(crtc->dev, amdgpu_crtc->crtc_id); + r = drm_crtc_vblank_get(crtc); if (r) { DRM_ERROR("failed to get vblank before flip\n"); goto pflip_cleanup; @@ -268,16 +266,18 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc, return 0; vblank_cleanup: - drm_vblank_put(crtc->dev, amdgpu_crtc->crtc_id); + drm_crtc_vblank_put(crtc); pflip_cleanup: if (unlikely(amdgpu_bo_reserve(new_rbo, false) != 0)) { DRM_ERROR("failed to reserve new rbo in error path\n"); goto cleanup; } +unpin: if (unlikely(amdgpu_bo_unpin(new_rbo) != 0)) { DRM_ERROR("failed to unpin new rbo in error path\n"); } +unreserve: amdgpu_bo_unreserve(new_rbo); cleanup: @@ -516,9 +516,7 @@ static void amdgpu_user_framebuffer_destroy(struct drm_framebuffer *fb) { struct amdgpu_framebuffer *amdgpu_fb = to_amdgpu_framebuffer(fb); - if (amdgpu_fb->obj) { - drm_gem_object_unreference_unlocked(amdgpu_fb->obj); - } + drm_gem_object_unreference_unlocked(amdgpu_fb->obj); drm_framebuffer_cleanup(fb); kfree(amdgpu_fb); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index f888c015f..9aa533cf4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -52,9 +52,10 @@ * - 3.1.0 - allow reading more status registers (GRBM, SRBM, SDMA, CP) * - 3.2.0 - GFX8: Uses EOP_TC_WB_ACTION_EN, so UMDs don't have to do the same * at the end of IBs. + * - 3.3.0 - Add VM support for UVD on supported hardware. */ #define KMS_DRIVER_MAJOR 3 -#define KMS_DRIVER_MINOR 2 +#define KMS_DRIVER_MINOR 3 #define KMS_DRIVER_PATCHLEVEL 0 int amdgpu_vram_limit = 0; @@ -82,8 +83,12 @@ int amdgpu_exp_hw_support = 0; int amdgpu_sched_jobs = 32; int amdgpu_sched_hw_submission = 2; int amdgpu_powerplay = -1; +int amdgpu_powercontainment = 1; unsigned amdgpu_pcie_gen_cap = 0; unsigned amdgpu_pcie_lane_cap = 0; +unsigned amdgpu_cg_mask = 0xffffffff; +unsigned amdgpu_pg_mask = 0xffffffff; +char *amdgpu_disable_cu = NULL; MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes"); module_param_named(vramlimit, amdgpu_vram_limit, int, 0600); @@ -160,6 +165,9 @@ module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444); #ifdef CONFIG_DRM_AMD_POWERPLAY MODULE_PARM_DESC(powerplay, "Powerplay component (1 = enable, 0 = disable, -1 = auto (default))"); module_param_named(powerplay, amdgpu_powerplay, int, 0444); + +MODULE_PARM_DESC(powercontainment, "Power Containment (1 = enable (default), 0 = disable)"); +module_param_named(powercontainment, amdgpu_powercontainment, int, 0444); #endif MODULE_PARM_DESC(pcie_gen_cap, "PCIE Gen Caps (0: autodetect (default))"); @@ -168,6 +176,15 @@ module_param_named(pcie_gen_cap, amdgpu_pcie_gen_cap, uint, 0444); MODULE_PARM_DESC(pcie_lane_cap, "PCIE Lane Caps (0: autodetect (default))"); module_param_named(pcie_lane_cap, amdgpu_pcie_lane_cap, uint, 0444); +MODULE_PARM_DESC(cg_mask, "Clockgating flags mask (0 = disable clock gating)"); +module_param_named(cg_mask, amdgpu_cg_mask, uint, 0444); + +MODULE_PARM_DESC(pg_mask, "Powergating flags mask (0 = disable power gating)"); +module_param_named(pg_mask, amdgpu_pg_mask, uint, 0444); + +MODULE_PARM_DESC(disable_cu, "Disable CUs (se.sh.cu,...)"); +module_param_named(disable_cu, amdgpu_disable_cu, charp, 0444); + static const struct pci_device_id pciidlist[] = { #ifdef CONFIG_DRM_AMDGPU_CIK /* Kaveri */ @@ -413,7 +430,10 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev) pci_save_state(pdev); pci_disable_device(pdev); pci_ignore_hotplug(pdev); - pci_set_power_state(pdev, PCI_D3cold); + if (amdgpu_is_atpx_hybrid()) + pci_set_power_state(pdev, PCI_D3cold); + else if (!amdgpu_has_atpx_dgpu_power_cntl()) + pci_set_power_state(pdev, PCI_D3hot); drm_dev->switch_power_state = DRM_SWITCH_POWER_DYNAMIC_OFF; return 0; @@ -430,7 +450,9 @@ static int amdgpu_pmops_runtime_resume(struct device *dev) drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - pci_set_power_state(pdev, PCI_D0); + if (amdgpu_is_atpx_hybrid() || + !amdgpu_has_atpx_dgpu_power_cntl()) + pci_set_power_state(pdev, PCI_D0); pci_restore_state(pdev); ret = pci_enable_device(pdev); if (ret) @@ -515,7 +537,7 @@ static struct drm_driver kms_driver = { .driver_features = DRIVER_USE_AGP | DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | - DRIVER_PRIME | DRIVER_RENDER, + DRIVER_PRIME | DRIVER_RENDER | DRIVER_MODESET, .dev_priv_size = 0, .load = amdgpu_driver_load_kms, .open = amdgpu_driver_open_kms, @@ -590,7 +612,6 @@ static int __init amdgpu_init(void) DRM_INFO("amdgpu kernel modesetting enabled.\n"); driver = &kms_driver; pdriver = &amdgpu_kms_pci_driver; - driver->driver_features |= DRIVER_MODESET; driver->num_ioctls = amdgpu_max_kms_ioctl; amdgpu_register_atpx_handler(); /* let modprobe override vga console setting */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index d1558768c..0b109aebf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -204,16 +204,25 @@ void amdgpu_fence_process(struct amdgpu_ring *ring) if (seq != ring->fence_drv.sync_seq) amdgpu_fence_schedule_fallback(ring); - while (last_seq != seq) { + if (unlikely(seq == last_seq)) + return; + + last_seq &= drv->num_fences_mask; + seq &= drv->num_fences_mask; + + do { struct fence *fence, **ptr; - ptr = &drv->fences[++last_seq & drv->num_fences_mask]; + ++last_seq; + last_seq &= drv->num_fences_mask; + ptr = &drv->fences[last_seq]; /* There is always exactly one thread signaling this fence slot */ fence = rcu_dereference_protected(*ptr, 1); RCU_INIT_POINTER(*ptr, NULL); - BUG_ON(!fence); + if (!fence) + continue; r = fence_signal(fence); if (!r) @@ -222,7 +231,7 @@ void amdgpu_fence_process(struct amdgpu_ring *ring) BUG(); fence_put(fence); - } + } while (last_seq != seq); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 8fab64860..88fbed238 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -503,7 +503,7 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev, if (r) goto error_print; - amdgpu_vm_get_pt_bos(bo_va->vm, &duplicates); + amdgpu_vm_get_pt_bos(adev, bo_va->vm, &duplicates); list_for_each_entry(entry, &list, head) { domain = amdgpu_mem_type_to_domain(entry->bo->mem.mem_type); /* if anything is swapped out don't swap it in here, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index 9f95da4f0..a074edd95 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -70,3 +70,47 @@ void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg) } } } + +/** + * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter + * + * @mask: array in which the per-shader array disable masks will be stored + * @max_se: number of SEs + * @max_sh: number of SHs + * + * The bitmask of CUs to be disabled in the shader array determined by se and + * sh is stored in mask[se * max_sh + sh]. + */ +void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_sh) +{ + unsigned se, sh, cu; + const char *p; + + memset(mask, 0, sizeof(*mask) * max_se * max_sh); + + if (!amdgpu_disable_cu || !*amdgpu_disable_cu) + return; + + p = amdgpu_disable_cu; + for (;;) { + char *next; + int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu); + if (ret < 3) { + DRM_ERROR("amdgpu: could not parse disable_cu\n"); + return; + } + + if (se < max_se && sh < max_sh && cu < 16) { + DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu); + mask[se * max_sh + sh] |= 1u << cu; + } else { + DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n", + se, sh, cu); + } + + next = strchr(p, ','); + if (!next) + break; + p = next + 1; + } +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index dc06cbda7..51321e154 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -27,4 +27,6 @@ int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg); void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg); +unsigned amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_sh); + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 194cfc1a8..ec1282af2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -33,6 +33,8 @@ #include "amdgpu.h" #include "atom.h" +#define AMDGPU_IB_TEST_TIMEOUT msecs_to_jiffies(1000) + /* * IB * IBs (Indirect Buffers) and areas of GPU accessible memory where @@ -122,7 +124,6 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, bool skip_preamble, need_ctx_switch; unsigned patch_offset = ~0; struct amdgpu_vm *vm; - struct fence *hwf; uint64_t ctx; unsigned i; @@ -160,10 +161,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, patch_offset = amdgpu_ring_init_cond_exec(ring); if (vm) { - r = amdgpu_vm_flush(ring, job->vm_id, job->vm_pd_addr, - job->gds_base, job->gds_size, - job->gws_base, job->gws_size, - job->oa_base, job->oa_size); + r = amdgpu_vm_flush(ring, job); if (r) { amdgpu_ring_undo(ring); return r; @@ -193,7 +191,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, if (ring->funcs->emit_hdp_invalidate) amdgpu_ring_emit_hdp_invalidate(ring); - r = amdgpu_fence_emit(ring, &hwf); + r = amdgpu_fence_emit(ring, f); if (r) { dev_err(adev->dev, "failed to emit fence (%d)\n", r); if (job && job->vm_id) @@ -203,17 +201,11 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, } /* wrap the last IB with fence */ - if (job && job->uf_bo) { - uint64_t addr = amdgpu_bo_gpu_offset(job->uf_bo); - - addr += job->uf_offset; - amdgpu_ring_emit_fence(ring, addr, job->uf_sequence, + if (job && job->uf_addr) { + amdgpu_ring_emit_fence(ring, job->uf_addr, job->uf_sequence, AMDGPU_FENCE_FLAG_64BIT); } - if (f) - *f = fence_get(hwf); - if (patch_offset != ~0 && ring->funcs->patch_cond_exec) amdgpu_ring_patch_cond_exec(ring, patch_offset); @@ -296,7 +288,7 @@ int amdgpu_ib_ring_tests(struct amdgpu_device *adev) if (!ring || !ring->ready) continue; - r = amdgpu_ring_test_ib(ring); + r = amdgpu_ring_test_ib(ring, AMDGPU_IB_TEST_TIMEOUT); if (r) { ring->ready = false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 835a3fa8d..278708f5a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -383,6 +383,18 @@ int amdgpu_irq_update(struct amdgpu_device *adev, return r; } +void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev) +{ + int i, j; + for (i = 0; i < AMDGPU_MAX_IRQ_SRC_ID; i++) { + struct amdgpu_irq_src *src = adev->irq.sources[i]; + if (!src) + continue; + for (j = 0; j < src->num_types; j++) + amdgpu_irq_update(adev, src, j); + } +} + /** * amdgpu_irq_get - enable interrupt * diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h index e124b59f3..7ef09352e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h @@ -94,6 +94,7 @@ int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src, unsigned type); bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src, unsigned type); +void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev); int amdgpu_irq_add_domain(struct amdgpu_device *adev); void amdgpu_irq_remove_domain(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index f0dafa514..6674d40eb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -28,21 +28,15 @@ #include "amdgpu.h" #include "amdgpu_trace.h" -static void amdgpu_job_free_handler(struct work_struct *ws) +static void amdgpu_job_timedout(struct amd_sched_job *s_job) { - struct amdgpu_job *job = container_of(ws, struct amdgpu_job, base.work_free_job); - amd_sched_job_put(&job->base); -} + struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base); -void amdgpu_job_timeout_func(struct work_struct *work) -{ - struct amdgpu_job *job = container_of(work, struct amdgpu_job, base.work_tdr.work); DRM_ERROR("ring %s timeout, last signaled seq=%u, last emitted seq=%u\n", - job->base.sched->name, - (uint32_t)atomic_read(&job->ring->fence_drv.last_seq), - job->ring->fence_drv.sync_seq); - - amd_sched_job_put(&job->base); + job->base.sched->name, + atomic_read(&job->ring->fence_drv.last_seq), + job->ring->fence_drv.sync_seq); + amdgpu_gpu_reset(job->adev); } int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, @@ -63,7 +57,6 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, (*job)->vm = vm; (*job)->ibs = (void *)&(*job)[1]; (*job)->num_ibs = num_ibs; - INIT_WORK(&(*job)->base.work_free_job, amdgpu_job_free_handler); amdgpu_sync_create(&(*job)->sync); @@ -86,27 +79,33 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size, return r; } -void amdgpu_job_free(struct amdgpu_job *job) +void amdgpu_job_free_resources(struct amdgpu_job *job) { - unsigned i; struct fence *f; + unsigned i; + /* use sched fence if available */ - f = (job->base.s_fence)? &job->base.s_fence->base : job->fence; + f = job->base.s_fence ? &job->base.s_fence->finished : job->fence; for (i = 0; i < job->num_ibs; ++i) - amdgpu_sa_bo_free(job->adev, &job->ibs[i].sa_bo, f); - fence_put(job->fence); + amdgpu_ib_free(job->adev, &job->ibs[i], f); +} - amdgpu_bo_unref(&job->uf_bo); - amdgpu_sync_free(&job->sync); +void amdgpu_job_free_cb(struct amd_sched_job *s_job) +{ + struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base); - if (!job->base.use_sched) - kfree(job); + fence_put(job->fence); + amdgpu_sync_free(&job->sync); + kfree(job); } -void amdgpu_job_free_func(struct kref *refcount) +void amdgpu_job_free(struct amdgpu_job *job) { - struct amdgpu_job *job = container_of(refcount, struct amdgpu_job, base.refcount); + amdgpu_job_free_resources(job); + + fence_put(job->fence); + amdgpu_sync_free(&job->sync); kfree(job); } @@ -114,22 +113,20 @@ int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring, struct amd_sched_entity *entity, void *owner, struct fence **f) { - struct fence *fence; int r; job->ring = ring; if (!f) return -EINVAL; - r = amd_sched_job_init(&job->base, &ring->sched, - entity, amdgpu_job_timeout_func, - amdgpu_job_free_func, owner, &fence); + r = amd_sched_job_init(&job->base, &ring->sched, entity, owner); if (r) return r; job->owner = owner; job->ctx = entity->fence_context; - *f = fence_get(fence); + *f = fence_get(&job->base.s_fence->finished); + amdgpu_job_free_resources(job); amd_sched_entity_push_job(&job->base); return 0; @@ -147,8 +144,8 @@ static struct fence *amdgpu_job_dependency(struct amd_sched_job *sched_job) int r; r = amdgpu_vm_grab_id(vm, ring, &job->sync, - &job->base.s_fence->base, - &job->vm_id, &job->vm_pd_addr); + &job->base.s_fence->finished, + job); if (r) DRM_ERROR("Error getting VM ID (%d)\n", r); @@ -170,29 +167,24 @@ static struct fence *amdgpu_job_run(struct amd_sched_job *sched_job) } job = to_amdgpu_job(sched_job); - r = amdgpu_sync_wait(&job->sync); - if (r) { - DRM_ERROR("failed to sync wait (%d)\n", r); - return NULL; - } + BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL)); trace_amdgpu_sched_run_job(job); r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job->sync.last_vm_update, job, &fence); - if (r) { + if (r) DRM_ERROR("Error scheduling IBs (%d)\n", r); - goto err; - } -err: - job->fence = fence; - amdgpu_job_free(job); + /* if gpu reset, hw fence will be replaced here */ + fence_put(job->fence); + job->fence = fence_get(fence); + amdgpu_job_free_resources(job); return fence; } const struct amd_sched_backend_ops amdgpu_sched_ops = { .dependency = amdgpu_job_dependency, .run_job = amdgpu_job_run, - .begin_job = amd_sched_job_begin, - .finish_job = amd_sched_job_finish, + .timedout_job = amdgpu_job_timedout, + .free_job = amdgpu_job_free_cb }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index d851ea150..d942654a1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -60,7 +60,10 @@ int amdgpu_driver_unload_kms(struct drm_device *dev) if (adev->rmmio == NULL) goto done_free; - pm_runtime_get_sync(dev->dev); + if (amdgpu_device_is_px(dev)) { + pm_runtime_get_sync(dev->dev); + pm_runtime_forbid(dev->dev); + } amdgpu_amdkfd_device_fini(adev); @@ -135,13 +138,75 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags) } out: - if (r) + if (r) { + /* balance pm_runtime_get_sync in amdgpu_driver_unload_kms */ + if (adev->rmmio && amdgpu_device_is_px(dev)) + pm_runtime_put_noidle(dev->dev); amdgpu_driver_unload_kms(dev); - + } return r; } +static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info, + struct drm_amdgpu_query_fw *query_fw, + struct amdgpu_device *adev) +{ + switch (query_fw->fw_type) { + case AMDGPU_INFO_FW_VCE: + fw_info->ver = adev->vce.fw_version; + fw_info->feature = adev->vce.fb_version; + break; + case AMDGPU_INFO_FW_UVD: + fw_info->ver = adev->uvd.fw_version; + fw_info->feature = 0; + break; + case AMDGPU_INFO_FW_GMC: + fw_info->ver = adev->mc.fw_version; + fw_info->feature = 0; + break; + case AMDGPU_INFO_FW_GFX_ME: + fw_info->ver = adev->gfx.me_fw_version; + fw_info->feature = adev->gfx.me_feature_version; + break; + case AMDGPU_INFO_FW_GFX_PFP: + fw_info->ver = adev->gfx.pfp_fw_version; + fw_info->feature = adev->gfx.pfp_feature_version; + break; + case AMDGPU_INFO_FW_GFX_CE: + fw_info->ver = adev->gfx.ce_fw_version; + fw_info->feature = adev->gfx.ce_feature_version; + break; + case AMDGPU_INFO_FW_GFX_RLC: + fw_info->ver = adev->gfx.rlc_fw_version; + fw_info->feature = adev->gfx.rlc_feature_version; + break; + case AMDGPU_INFO_FW_GFX_MEC: + if (query_fw->index == 0) { + fw_info->ver = adev->gfx.mec_fw_version; + fw_info->feature = adev->gfx.mec_feature_version; + } else if (query_fw->index == 1) { + fw_info->ver = adev->gfx.mec2_fw_version; + fw_info->feature = adev->gfx.mec2_feature_version; + } else + return -EINVAL; + break; + case AMDGPU_INFO_FW_SMC: + fw_info->ver = adev->pm.fw_version; + fw_info->feature = 0; + break; + case AMDGPU_INFO_FW_SDMA: + if (query_fw->index >= adev->sdma.num_instances) + return -EINVAL; + fw_info->ver = adev->sdma.instance[query_fw->index].fw_version; + fw_info->feature = adev->sdma.instance[query_fw->index].feature_version; + break; + default: + return -EINVAL; + } + return 0; +} + /* * Userspace get information ioctl */ @@ -288,67 +353,20 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0; } case AMDGPU_INFO_TIMESTAMP: - ui64 = amdgpu_asic_get_gpu_clock_counter(adev); + ui64 = amdgpu_gfx_get_gpu_clock_counter(adev); return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0; case AMDGPU_INFO_FW_VERSION: { struct drm_amdgpu_info_firmware fw_info; + int ret; /* We only support one instance of each IP block right now. */ if (info->query_fw.ip_instance != 0) return -EINVAL; - switch (info->query_fw.fw_type) { - case AMDGPU_INFO_FW_VCE: - fw_info.ver = adev->vce.fw_version; - fw_info.feature = adev->vce.fb_version; - break; - case AMDGPU_INFO_FW_UVD: - fw_info.ver = adev->uvd.fw_version; - fw_info.feature = 0; - break; - case AMDGPU_INFO_FW_GMC: - fw_info.ver = adev->mc.fw_version; - fw_info.feature = 0; - break; - case AMDGPU_INFO_FW_GFX_ME: - fw_info.ver = adev->gfx.me_fw_version; - fw_info.feature = adev->gfx.me_feature_version; - break; - case AMDGPU_INFO_FW_GFX_PFP: - fw_info.ver = adev->gfx.pfp_fw_version; - fw_info.feature = adev->gfx.pfp_feature_version; - break; - case AMDGPU_INFO_FW_GFX_CE: - fw_info.ver = adev->gfx.ce_fw_version; - fw_info.feature = adev->gfx.ce_feature_version; - break; - case AMDGPU_INFO_FW_GFX_RLC: - fw_info.ver = adev->gfx.rlc_fw_version; - fw_info.feature = adev->gfx.rlc_feature_version; - break; - case AMDGPU_INFO_FW_GFX_MEC: - if (info->query_fw.index == 0) { - fw_info.ver = adev->gfx.mec_fw_version; - fw_info.feature = adev->gfx.mec_feature_version; - } else if (info->query_fw.index == 1) { - fw_info.ver = adev->gfx.mec2_fw_version; - fw_info.feature = adev->gfx.mec2_feature_version; - } else - return -EINVAL; - break; - case AMDGPU_INFO_FW_SMC: - fw_info.ver = adev->pm.fw_version; - fw_info.feature = 0; - break; - case AMDGPU_INFO_FW_SDMA: - if (info->query_fw.index >= adev->sdma.num_instances) - return -EINVAL; - fw_info.ver = adev->sdma.instance[info->query_fw.index].fw_version; - fw_info.feature = adev->sdma.instance[info->query_fw.index].feature_version; - break; - default: - return -EINVAL; - } + ret = amdgpu_firmware_info(&fw_info, &info->query_fw, adev); + if (ret) + return ret; + return copy_to_user(out, &fw_info, min((size_t)size, sizeof(fw_info))) ? -EFAULT : 0; } @@ -566,6 +584,9 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr); + amdgpu_uvd_free_handles(adev, file_priv); + amdgpu_vce_free_handles(adev, file_priv); + amdgpu_vm_fini(adev, &fpriv->vm); idr_for_each_entry(&fpriv->bo_list_handles, list, handle) @@ -590,10 +611,6 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, void amdgpu_driver_preclose_kms(struct drm_device *dev, struct drm_file *file_priv) { - struct amdgpu_device *adev = dev->dev_private; - - amdgpu_uvd_free_handles(adev, file_priv); - amdgpu_vce_free_handles(adev, file_priv); } /* @@ -756,3 +773,130 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = { DRM_IOCTL_DEF_DRV(AMDGPU_GEM_USERPTR, amdgpu_gem_userptr_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), }; const int amdgpu_max_kms_ioctl = ARRAY_SIZE(amdgpu_ioctls_kms); + +/* + * Debugfs info + */ +#if defined(CONFIG_DEBUG_FS) + +static int amdgpu_debugfs_firmware_info(struct seq_file *m, void *data) +{ + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct amdgpu_device *adev = dev->dev_private; + struct drm_amdgpu_info_firmware fw_info; + struct drm_amdgpu_query_fw query_fw; + int ret, i; + + /* VCE */ + query_fw.fw_type = AMDGPU_INFO_FW_VCE; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "VCE feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* UVD */ + query_fw.fw_type = AMDGPU_INFO_FW_UVD; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "UVD feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* GMC */ + query_fw.fw_type = AMDGPU_INFO_FW_GMC; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "MC feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* ME */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_ME; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "ME feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* PFP */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_PFP; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "PFP feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* CE */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_CE; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "CE feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* RLC */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "RLC feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* MEC */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_MEC; + query_fw.index = 0; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "MEC feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* MEC2 */ + if (adev->asic_type == CHIP_KAVERI || + (adev->asic_type > CHIP_TOPAZ && adev->asic_type != CHIP_STONEY)) { + query_fw.index = 1; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "MEC2 feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + } + + /* SMC */ + query_fw.fw_type = AMDGPU_INFO_FW_SMC; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "SMC feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* SDMA */ + query_fw.fw_type = AMDGPU_INFO_FW_SDMA; + for (i = 0; i < adev->sdma.num_instances; i++) { + query_fw.index = i; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "SDMA%d feature version: %u, firmware version: 0x%08x\n", + i, fw_info.feature, fw_info.ver); + } + + return 0; +} + +static const struct drm_info_list amdgpu_firmware_info_list[] = { + {"amdgpu_firmware_info", amdgpu_debugfs_firmware_info, 0, NULL}, +}; +#endif + +int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + return amdgpu_debugfs_add_files(adev, amdgpu_firmware_info_list, + ARRAY_SIZE(amdgpu_firmware_info_list)); +#else + return 0; +#endif +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 7ecea83ce..6f0873c75 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -589,6 +589,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem) { struct amdgpu_bo *rbo; + struct ttm_mem_reg *old_mem = &bo->mem; if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) return; @@ -602,6 +603,8 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, /* move_notify is called before move happens */ amdgpu_update_memory_usage(rbo->adev, &bo->mem, new_mem); + + trace_amdgpu_ttm_bo_move(rbo, new_mem->mem_type, old_mem->mem_type); } int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c index 0e13d80d2..5cc7052e3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c @@ -305,7 +305,7 @@ static ssize_t amdgpu_get_pp_table(struct device *dev, struct drm_device *ddev = dev_get_drvdata(dev); struct amdgpu_device *adev = ddev->dev_private; char *table = NULL; - int size, i; + int size; if (adev->pp_enabled) size = amdgpu_dpm_get_pp_table(adev, &table); @@ -315,10 +315,7 @@ static ssize_t amdgpu_get_pp_table(struct device *dev, if (size >= PAGE_SIZE) size = PAGE_SIZE - 1; - for (i = 0; i < size; i++) { - sprintf(buf + i, "%02x", table[i]); - } - sprintf(buf + i, "\n"); + memcpy(buf, table, size); return size; } @@ -347,6 +344,8 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev, if (adev->pp_enabled) size = amdgpu_dpm_print_clock_levels(adev, PP_SCLK, buf); + else if (adev->pm.funcs->print_clock_levels) + size = adev->pm.funcs->print_clock_levels(adev, PP_SCLK, buf); return size; } @@ -363,7 +362,9 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev, uint32_t i, mask = 0; char sub_str[2]; - for (i = 0; i < strlen(buf) - 1; i++) { + for (i = 0; i < strlen(buf); i++) { + if (*(buf + i) == '\n') + continue; sub_str[0] = *(buf + i); sub_str[1] = '\0'; ret = kstrtol(sub_str, 0, &level); @@ -377,6 +378,8 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev, if (adev->pp_enabled) amdgpu_dpm_force_clock_level(adev, PP_SCLK, mask); + else if (adev->pm.funcs->force_clock_level) + adev->pm.funcs->force_clock_level(adev, PP_SCLK, mask); fail: return count; } @@ -391,6 +394,8 @@ static ssize_t amdgpu_get_pp_dpm_mclk(struct device *dev, if (adev->pp_enabled) size = amdgpu_dpm_print_clock_levels(adev, PP_MCLK, buf); + else if (adev->pm.funcs->print_clock_levels) + size = adev->pm.funcs->print_clock_levels(adev, PP_MCLK, buf); return size; } @@ -407,7 +412,9 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev, uint32_t i, mask = 0; char sub_str[2]; - for (i = 0; i < strlen(buf) - 1; i++) { + for (i = 0; i < strlen(buf); i++) { + if (*(buf + i) == '\n') + continue; sub_str[0] = *(buf + i); sub_str[1] = '\0'; ret = kstrtol(sub_str, 0, &level); @@ -421,6 +428,8 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev, if (adev->pp_enabled) amdgpu_dpm_force_clock_level(adev, PP_MCLK, mask); + else if (adev->pm.funcs->force_clock_level) + adev->pm.funcs->force_clock_level(adev, PP_MCLK, mask); fail: return count; } @@ -435,6 +444,8 @@ static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev, if (adev->pp_enabled) size = amdgpu_dpm_print_clock_levels(adev, PP_PCIE, buf); + else if (adev->pm.funcs->print_clock_levels) + size = adev->pm.funcs->print_clock_levels(adev, PP_PCIE, buf); return size; } @@ -451,7 +462,9 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev, uint32_t i, mask = 0; char sub_str[2]; - for (i = 0; i < strlen(buf) - 1; i++) { + for (i = 0; i < strlen(buf); i++) { + if (*(buf + i) == '\n') + continue; sub_str[0] = *(buf + i); sub_str[1] = '\0'; ret = kstrtol(sub_str, 0, &level); @@ -465,6 +478,100 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev, if (adev->pp_enabled) amdgpu_dpm_force_clock_level(adev, PP_PCIE, mask); + else if (adev->pm.funcs->force_clock_level) + adev->pm.funcs->force_clock_level(adev, PP_PCIE, mask); +fail: + return count; +} + +static ssize_t amdgpu_get_pp_sclk_od(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint32_t value = 0; + + if (adev->pp_enabled) + value = amdgpu_dpm_get_sclk_od(adev); + else if (adev->pm.funcs->get_sclk_od) + value = adev->pm.funcs->get_sclk_od(adev); + + return snprintf(buf, PAGE_SIZE, "%d\n", value); +} + +static ssize_t amdgpu_set_pp_sclk_od(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + long int value; + + ret = kstrtol(buf, 0, &value); + + if (ret) { + count = -EINVAL; + goto fail; + } + + if (adev->pp_enabled) { + amdgpu_dpm_set_sclk_od(adev, (uint32_t)value); + amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_READJUST_POWER_STATE, NULL, NULL); + } else if (adev->pm.funcs->set_sclk_od) { + adev->pm.funcs->set_sclk_od(adev, (uint32_t)value); + adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps; + amdgpu_pm_compute_clocks(adev); + } + +fail: + return count; +} + +static ssize_t amdgpu_get_pp_mclk_od(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + uint32_t value = 0; + + if (adev->pp_enabled) + value = amdgpu_dpm_get_mclk_od(adev); + else if (adev->pm.funcs->get_mclk_od) + value = adev->pm.funcs->get_mclk_od(adev); + + return snprintf(buf, PAGE_SIZE, "%d\n", value); +} + +static ssize_t amdgpu_set_pp_mclk_od(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = ddev->dev_private; + int ret; + long int value; + + ret = kstrtol(buf, 0, &value); + + if (ret) { + count = -EINVAL; + goto fail; + } + + if (adev->pp_enabled) { + amdgpu_dpm_set_mclk_od(adev, (uint32_t)value); + amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_READJUST_POWER_STATE, NULL, NULL); + } else if (adev->pm.funcs->set_mclk_od) { + adev->pm.funcs->set_mclk_od(adev, (uint32_t)value); + adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps; + amdgpu_pm_compute_clocks(adev); + } + fail: return count; } @@ -490,6 +597,12 @@ static DEVICE_ATTR(pp_dpm_mclk, S_IRUGO | S_IWUSR, static DEVICE_ATTR(pp_dpm_pcie, S_IRUGO | S_IWUSR, amdgpu_get_pp_dpm_pcie, amdgpu_set_pp_dpm_pcie); +static DEVICE_ATTR(pp_sclk_od, S_IRUGO | S_IWUSR, + amdgpu_get_pp_sclk_od, + amdgpu_set_pp_sclk_od); +static DEVICE_ATTR(pp_mclk_od, S_IRUGO | S_IWUSR, + amdgpu_get_pp_mclk_od, + amdgpu_set_pp_mclk_od); static ssize_t amdgpu_hwmon_show_temp(struct device *dev, struct device_attribute *attr, @@ -1108,22 +1221,34 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev) DRM_ERROR("failed to create device file pp_table\n"); return ret; } - ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk); - if (ret) { - DRM_ERROR("failed to create device file pp_dpm_sclk\n"); - return ret; - } - ret = device_create_file(adev->dev, &dev_attr_pp_dpm_mclk); - if (ret) { - DRM_ERROR("failed to create device file pp_dpm_mclk\n"); - return ret; - } - ret = device_create_file(adev->dev, &dev_attr_pp_dpm_pcie); - if (ret) { - DRM_ERROR("failed to create device file pp_dpm_pcie\n"); - return ret; - } } + + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_sclk\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_mclk); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_mclk\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_dpm_pcie); + if (ret) { + DRM_ERROR("failed to create device file pp_dpm_pcie\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_sclk_od); + if (ret) { + DRM_ERROR("failed to create device file pp_sclk_od\n"); + return ret; + } + ret = device_create_file(adev->dev, &dev_attr_pp_mclk_od); + if (ret) { + DRM_ERROR("failed to create device file pp_mclk_od\n"); + return ret; + } + ret = amdgpu_debugfs_pm_init(adev); if (ret) { DRM_ERROR("Failed to register debugfs file for dpm!\n"); @@ -1146,10 +1271,12 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev) device_remove_file(adev->dev, &dev_attr_pp_cur_state); device_remove_file(adev->dev, &dev_attr_pp_force_state); device_remove_file(adev->dev, &dev_attr_pp_table); - device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk); - device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk); - device_remove_file(adev->dev, &dev_attr_pp_dpm_pcie); } + device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk); + device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk); + device_remove_file(adev->dev, &dev_attr_pp_dpm_pcie); + device_remove_file(adev->dev, &dev_attr_pp_sclk_od); + device_remove_file(adev->dev, &dev_attr_pp_mclk_od); } void amdgpu_pm_compute_clocks(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c index 82256558e..c5738a22b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c @@ -52,6 +52,7 @@ static int amdgpu_powerplay_init(struct amdgpu_device *adev) pp_init->chip_family = adev->family; pp_init->chip_id = adev->asic_type; pp_init->device = amdgpu_cgs_create_device(adev); + pp_init->powercontainment_enabled = amdgpu_powercontainment; ret = amd_powerplay_init(pp_init, amd_pp); kfree(pp_init); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index 870f94942..85aeb0a80 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -28,6 +28,7 @@ */ #include #include +#include #include #include #include "amdgpu.h" @@ -48,6 +49,7 @@ */ static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring); +static void amdgpu_debugfs_ring_fini(struct amdgpu_ring *ring); /** * amdgpu_ring_alloc - allocate space on the ring buffer @@ -73,6 +75,10 @@ int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw) ring->count_dw = ndw; ring->wptr_old = ring->wptr; + + if (ring->funcs->begin_use) + ring->funcs->begin_use(ring); + return 0; } @@ -125,6 +131,9 @@ void amdgpu_ring_commit(struct amdgpu_ring *ring) mb(); amdgpu_ring_set_wptr(ring); + + if (ring->funcs->end_use) + ring->funcs->end_use(ring); } /** @@ -137,78 +146,9 @@ void amdgpu_ring_commit(struct amdgpu_ring *ring) void amdgpu_ring_undo(struct amdgpu_ring *ring) { ring->wptr = ring->wptr_old; -} - -/** - * amdgpu_ring_backup - Back up the content of a ring - * - * @ring: the ring we want to back up - * - * Saves all unprocessed commits from a ring, returns the number of dwords saved. - */ -unsigned amdgpu_ring_backup(struct amdgpu_ring *ring, - uint32_t **data) -{ - unsigned size, ptr, i; - - *data = NULL; - - if (ring->ring_obj == NULL) - return 0; - - /* it doesn't make sense to save anything if all fences are signaled */ - if (!amdgpu_fence_count_emitted(ring)) - return 0; - - ptr = le32_to_cpu(*ring->next_rptr_cpu_addr); - - size = ring->wptr + (ring->ring_size / 4); - size -= ptr; - size &= ring->ptr_mask; - if (size == 0) - return 0; - - /* and then save the content of the ring */ - *data = kmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); - if (!*data) - return 0; - for (i = 0; i < size; ++i) { - (*data)[i] = ring->ring[ptr++]; - ptr &= ring->ptr_mask; - } - - return size; -} - -/** - * amdgpu_ring_restore - append saved commands to the ring again - * - * @ring: ring to append commands to - * @size: number of dwords we want to write - * @data: saved commands - * - * Allocates space on the ring and restore the previously saved commands. - */ -int amdgpu_ring_restore(struct amdgpu_ring *ring, - unsigned size, uint32_t *data) -{ - int i, r; - - if (!size || !data) - return 0; - - /* restore the saved ring content */ - r = amdgpu_ring_alloc(ring, size); - if (r) - return r; - - for (i = 0; i < size; ++i) { - amdgpu_ring_write(ring, data[i]); - } - amdgpu_ring_commit(ring); - kfree(data); - return 0; + if (ring->funcs->end_use) + ring->funcs->end_use(ring); } /** @@ -260,14 +200,6 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, return r; } - r = amdgpu_wb_get(adev, &ring->next_rptr_offs); - if (r) { - dev_err(adev->dev, "(%d) ring next_rptr wb alloc failed\n", r); - return r; - } - ring->next_rptr_gpu_addr = adev->wb.gpu_addr + ring->next_rptr_offs * 4; - ring->next_rptr_cpu_addr = &adev->wb.wb[ring->next_rptr_offs]; - r = amdgpu_wb_get(adev, &ring->cond_exe_offs); if (r) { dev_err(adev->dev, "(%d) ring cond_exec_polling wb alloc failed\n", r); @@ -276,7 +208,6 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, ring->cond_exe_gpu_addr = adev->wb.gpu_addr + (ring->cond_exe_offs * 4); ring->cond_exe_cpu_addr = &adev->wb.wb[ring->cond_exe_offs]; - spin_lock_init(&ring->fence_lock); r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type); if (r) { dev_err(adev->dev, "failed initializing fences (%d).\n", r); @@ -310,6 +241,9 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, } r = amdgpu_bo_kmap(ring->ring_obj, (void **)&ring->ring); + + memset((void *)ring->ring, 0, ring->ring_size); + amdgpu_bo_unreserve(ring->ring_obj); if (r) { dev_err(adev->dev, "(%d) ring map failed\n", r); @@ -347,7 +281,6 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring) amdgpu_wb_free(ring->adev, ring->fence_offs); amdgpu_wb_free(ring->adev, ring->rptr_offs); amdgpu_wb_free(ring->adev, ring->wptr_offs); - amdgpu_wb_free(ring->adev, ring->next_rptr_offs); if (ring_obj) { r = amdgpu_bo_reserve(ring_obj, false); @@ -358,6 +291,7 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring) } amdgpu_bo_unref(&ring_obj); } + amdgpu_debugfs_ring_fini(ring); } /* @@ -365,57 +299,62 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring) */ #if defined(CONFIG_DEBUG_FS) -static int amdgpu_debugfs_ring_info(struct seq_file *m, void *data) +/* Layout of file is 12 bytes consisting of + * - rptr + * - wptr + * - driver's copy of wptr + * + * followed by n-words of ring data + */ +static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf, + size_t size, loff_t *pos) { - struct drm_info_node *node = (struct drm_info_node *) m->private; - struct drm_device *dev = node->minor->dev; - struct amdgpu_device *adev = dev->dev_private; - int roffset = (unsigned long)node->info_ent->data; - struct amdgpu_ring *ring = (void *)(((uint8_t*)adev) + roffset); - uint32_t rptr, wptr, rptr_next; - unsigned i; - - wptr = amdgpu_ring_get_wptr(ring); - seq_printf(m, "wptr: 0x%08x [%5d]\n", wptr, wptr); - - rptr = amdgpu_ring_get_rptr(ring); - rptr_next = le32_to_cpu(*ring->next_rptr_cpu_addr); - - seq_printf(m, "rptr: 0x%08x [%5d]\n", rptr, rptr); - - seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n", - ring->wptr, ring->wptr); - - if (!ring->ready) - return 0; - - /* print 8 dw before current rptr as often it's the last executed - * packet that is the root issue - */ - i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask; - while (i != rptr) { - seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]); - if (i == rptr) - seq_puts(m, " *"); - if (i == rptr_next) - seq_puts(m, " #"); - seq_puts(m, "\n"); - i = (i + 1) & ring->ptr_mask; + struct amdgpu_ring *ring = (struct amdgpu_ring*)f->f_inode->i_private; + int r, i; + uint32_t value, result, early[3]; + + if (*pos & 3 || size & 3) + return -EINVAL; + + result = 0; + + if (*pos < 12) { + early[0] = amdgpu_ring_get_rptr(ring); + early[1] = amdgpu_ring_get_wptr(ring); + early[2] = ring->wptr; + for (i = *pos / 4; i < 3 && size; i++) { + r = put_user(early[i], (uint32_t *)buf); + if (r) + return r; + buf += 4; + result += 4; + size -= 4; + *pos += 4; + } } - while (i != wptr) { - seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]); - if (i == rptr) - seq_puts(m, " *"); - if (i == rptr_next) - seq_puts(m, " #"); - seq_puts(m, "\n"); - i = (i + 1) & ring->ptr_mask; + + while (size) { + if (*pos >= (ring->ring_size + 12)) + return result; + + value = ring->ring[(*pos - 12)/4]; + r = put_user(value, (uint32_t*)buf); + if (r) + return r; + buf += 4; + result += 4; + size -= 4; + *pos += 4; } - return 0; + + return result; } -static struct drm_info_list amdgpu_debugfs_ring_info_list[AMDGPU_MAX_RINGS]; -static char amdgpu_debugfs_ring_names[AMDGPU_MAX_RINGS][32]; +static const struct file_operations amdgpu_debugfs_ring_fops = { + .owner = THIS_MODULE, + .read = amdgpu_debugfs_ring_read, + .llseek = default_llseek +}; #endif @@ -423,28 +362,27 @@ static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring) { #if defined(CONFIG_DEBUG_FS) - unsigned offset = (uint8_t*)ring - (uint8_t*)adev; - unsigned i; - struct drm_info_list *info; - char *name; - - for (i = 0; i < ARRAY_SIZE(amdgpu_debugfs_ring_info_list); ++i) { - info = &amdgpu_debugfs_ring_info_list[i]; - if (!info->data) - break; - } + struct drm_minor *minor = adev->ddev->primary; + struct dentry *ent, *root = minor->debugfs_root; + char name[32]; - if (i == ARRAY_SIZE(amdgpu_debugfs_ring_info_list)) - return -ENOSPC; - - name = &amdgpu_debugfs_ring_names[i][0]; sprintf(name, "amdgpu_ring_%s", ring->name); - info->name = name; - info->show = amdgpu_debugfs_ring_info; - info->driver_features = 0; - info->data = (void*)(uintptr_t)offset; - return amdgpu_debugfs_add_files(adev, info, 1); + ent = debugfs_create_file(name, + S_IFREG | S_IRUGO, root, + ring, &amdgpu_debugfs_ring_fops); + if (IS_ERR(ent)) + return PTR_ERR(ent); + + i_size_write(ent->d_inode, ring->ring_size + 12); + ring->ent = ent; #endif return 0; } + +static void amdgpu_debugfs_ring_fini(struct amdgpu_ring *ring) +{ +#if defined(CONFIG_DEBUG_FS) + debugfs_remove(ring->ent); +#endif +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c index 48618ee32..d8af37a84 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c @@ -428,7 +428,7 @@ void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, soffset, eoffset, eoffset - soffset); if (i->fence) - seq_printf(m, " protected by 0x%08x on context %d", + seq_printf(m, " protected by 0x%08x on context %llu", i->fence->seqno, i->fence->context); seq_printf(m, "\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index 34a92808b..5c8d3022f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -223,13 +223,16 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, } /** - * amdgpu_sync_is_idle - test if all fences are signaled + * amdgpu_sync_peek_fence - get the next fence not signaled yet * * @sync: the sync object + * @ring: optional ring to use for test * - * Returns true if all fences in the sync object are signaled. + * Returns the next fence not signaled yet without removing it from the sync + * object. */ -bool amdgpu_sync_is_idle(struct amdgpu_sync *sync) +struct fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync, + struct amdgpu_ring *ring) { struct amdgpu_sync_entry *e; struct hlist_node *tmp; @@ -237,6 +240,19 @@ bool amdgpu_sync_is_idle(struct amdgpu_sync *sync) hash_for_each_safe(sync->fences, i, tmp, e, node) { struct fence *f = e->fence; + struct amd_sched_fence *s_fence = to_amd_sched_fence(f); + + if (ring && s_fence) { + /* For fences from the same ring it is sufficient + * when they are scheduled. + */ + if (s_fence->sched == &ring->sched) { + if (fence_is_signaled(&s_fence->scheduled)) + continue; + + return &s_fence->scheduled; + } + } if (fence_is_signaled(f)) { hash_del(&e->node); @@ -245,58 +261,19 @@ bool amdgpu_sync_is_idle(struct amdgpu_sync *sync) continue; } - return false; + return f; } - return true; + return NULL; } /** - * amdgpu_sync_cycle_fences - move fences from one sync object into another + * amdgpu_sync_get_fence - get the next fence from the sync object * - * @dst: the destination sync object - * @src: the source sync object - * @fence: fence to add to source + * @sync: sync object to use * - * Remove all fences from source and put them into destination and add - * fence as new one into source. + * Get and removes the next fence from the sync object not signaled yet. */ -int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src, - struct fence *fence) -{ - struct amdgpu_sync_entry *e, *newone; - struct hlist_node *tmp; - int i; - - /* Allocate the new entry before moving the old ones */ - newone = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL); - if (!newone) - return -ENOMEM; - - hash_for_each_safe(src->fences, i, tmp, e, node) { - struct fence *f = e->fence; - - hash_del(&e->node); - if (fence_is_signaled(f)) { - fence_put(f); - kmem_cache_free(amdgpu_sync_slab, e); - continue; - } - - if (amdgpu_sync_add_later(dst, f)) { - kmem_cache_free(amdgpu_sync_slab, e); - continue; - } - - hash_add(dst->fences, &e->node, f->context); - } - - hash_add(src->fences, &newone->node, fence->context); - newone->fence = fence_get(fence); - - return 0; -} - struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync) { struct amdgpu_sync_entry *e; @@ -319,25 +296,6 @@ struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync) return NULL; } -int amdgpu_sync_wait(struct amdgpu_sync *sync) -{ - struct amdgpu_sync_entry *e; - struct hlist_node *tmp; - int i, r; - - hash_for_each_safe(sync->fences, i, tmp, e, node) { - r = fence_wait(e->fence, false); - if (r) - return r; - - hash_del(&e->node); - fence_put(e->fence); - kmem_cache_free(amdgpu_sync_slab, e); - } - - return 0; -} - /** * amdgpu_sync_free - free the sync object * diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h index 26a5f4acf..0d8d65eb4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h @@ -11,19 +11,68 @@ #define TRACE_SYSTEM amdgpu #define TRACE_INCLUDE_FILE amdgpu_trace +TRACE_EVENT(amdgpu_mm_rreg, + TP_PROTO(unsigned did, uint32_t reg, uint32_t value), + TP_ARGS(did, reg, value), + TP_STRUCT__entry( + __field(unsigned, did) + __field(uint32_t, reg) + __field(uint32_t, value) + ), + TP_fast_assign( + __entry->did = did; + __entry->reg = reg; + __entry->value = value; + ), + TP_printk("0x%04lx, 0x%04lx, 0x%08lx", + (unsigned long)__entry->did, + (unsigned long)__entry->reg, + (unsigned long)__entry->value) +); + +TRACE_EVENT(amdgpu_mm_wreg, + TP_PROTO(unsigned did, uint32_t reg, uint32_t value), + TP_ARGS(did, reg, value), + TP_STRUCT__entry( + __field(unsigned, did) + __field(uint32_t, reg) + __field(uint32_t, value) + ), + TP_fast_assign( + __entry->did = did; + __entry->reg = reg; + __entry->value = value; + ), + TP_printk("0x%04lx, 0x%04lx, 0x%08lx", + (unsigned long)__entry->did, + (unsigned long)__entry->reg, + (unsigned long)__entry->value) +); + TRACE_EVENT(amdgpu_bo_create, TP_PROTO(struct amdgpu_bo *bo), TP_ARGS(bo), TP_STRUCT__entry( __field(struct amdgpu_bo *, bo) __field(u32, pages) + __field(u32, type) + __field(u32, prefer) + __field(u32, allow) + __field(u32, visible) ), TP_fast_assign( __entry->bo = bo; __entry->pages = bo->tbo.num_pages; + __entry->type = bo->tbo.mem.mem_type; + __entry->prefer = bo->prefered_domains; + __entry->allow = bo->allowed_domains; + __entry->visible = bo->flags; ), - TP_printk("bo=%p, pages=%u", __entry->bo, __entry->pages) + + TP_printk("bo=%p,pages=%u,type=%d,prefered=%d,allowed=%d,visible=%d", + __entry->bo, __entry->pages, __entry->type, + __entry->prefer, __entry->allow, __entry->visible) ); TRACE_EVENT(amdgpu_cs, @@ -64,7 +113,7 @@ TRACE_EVENT(amdgpu_cs_ioctl, __entry->adev = job->adev; __entry->sched_job = &job->base; __entry->ib = job->ibs; - __entry->fence = &job->base.s_fence->base; + __entry->fence = &job->base.s_fence->finished; __entry->ring_name = job->ring->name; __entry->num_ibs = job->num_ibs; ), @@ -89,7 +138,7 @@ TRACE_EVENT(amdgpu_sched_run_job, __entry->adev = job->adev; __entry->sched_job = &job->base; __entry->ib = job->ibs; - __entry->fence = &job->base.s_fence->base; + __entry->fence = &job->base.s_fence->finished; __entry->ring_name = job->ring->name; __entry->num_ibs = job->num_ibs; ), @@ -100,24 +149,26 @@ TRACE_EVENT(amdgpu_sched_run_job, TRACE_EVENT(amdgpu_vm_grab_id, - TP_PROTO(struct amdgpu_vm *vm, int ring, unsigned vmid, - uint64_t pd_addr), - TP_ARGS(vm, ring, vmid, pd_addr), + TP_PROTO(struct amdgpu_vm *vm, int ring, struct amdgpu_job *job), + TP_ARGS(vm, ring, job), TP_STRUCT__entry( __field(struct amdgpu_vm *, vm) __field(u32, ring) __field(u32, vmid) __field(u64, pd_addr) + __field(u32, needs_flush) ), TP_fast_assign( __entry->vm = vm; __entry->ring = ring; - __entry->vmid = vmid; - __entry->pd_addr = pd_addr; + __entry->vmid = job->vm_id; + __entry->pd_addr = job->vm_pd_addr; + __entry->needs_flush = job->vm_needs_flush; ), - TP_printk("vm=%p, ring=%u, id=%u, pd_addr=%010Lx", __entry->vm, - __entry->ring, __entry->vmid, __entry->pd_addr) + TP_printk("vm=%p, ring=%u, id=%u, pd_addr=%010Lx needs_flush=%u", + __entry->vm, __entry->ring, __entry->vmid, + __entry->pd_addr, __entry->needs_flush) ); TRACE_EVENT(amdgpu_vm_bo_map, @@ -244,13 +295,55 @@ TRACE_EVENT(amdgpu_bo_list_set, TP_STRUCT__entry( __field(struct amdgpu_bo_list *, list) __field(struct amdgpu_bo *, bo) + __field(u64, bo_size) ), TP_fast_assign( __entry->list = list; __entry->bo = bo; + __entry->bo_size = amdgpu_bo_size(bo); ), - TP_printk("list=%p, bo=%p", __entry->list, __entry->bo) + TP_printk("list=%p, bo=%p, bo_size = %Ld", + __entry->list, + __entry->bo, + __entry->bo_size) +); + +TRACE_EVENT(amdgpu_cs_bo_status, + TP_PROTO(uint64_t total_bo, uint64_t total_size), + TP_ARGS(total_bo, total_size), + TP_STRUCT__entry( + __field(u64, total_bo) + __field(u64, total_size) + ), + + TP_fast_assign( + __entry->total_bo = total_bo; + __entry->total_size = total_size; + ), + TP_printk("total bo size = %Ld, total bo count = %Ld", + __entry->total_bo, __entry->total_size) +); + +TRACE_EVENT(amdgpu_ttm_bo_move, + TP_PROTO(struct amdgpu_bo* bo, uint32_t new_placement, uint32_t old_placement), + TP_ARGS(bo, new_placement, old_placement), + TP_STRUCT__entry( + __field(struct amdgpu_bo *, bo) + __field(u64, bo_size) + __field(u32, new_placement) + __field(u32, old_placement) + ), + + TP_fast_assign( + __entry->bo = bo; + __entry->bo_size = amdgpu_bo_size(bo); + __entry->new_placement = new_placement; + __entry->old_placement = old_placement; + ), + TP_printk("bo=%p from:%d to %d with size = %Ld", + __entry->bo, __entry->old_placement, + __entry->new_placement, __entry->bo_size) ); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 46c5297f6..716f2afeb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -286,9 +286,10 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo, r = amdgpu_copy_buffer(ring, old_start, new_start, new_mem->num_pages * PAGE_SIZE, /* bytes */ bo->resv, &fence); - /* FIXME: handle copy error */ - r = ttm_bo_move_accel_cleanup(bo, fence, - evict, no_wait_gpu, new_mem); + if (r) + return r; + + r = ttm_bo_pipeline_move(bo, fence, evict, new_mem); fence_put(fence); return r; } @@ -334,7 +335,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, if (unlikely(r)) { goto out_cleanup; } - r = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem); + r = ttm_bo_move_ttm(bo, true, interruptible, no_wait_gpu, new_mem); out_cleanup: ttm_bo_mem_put(bo, &tmp_mem); return r; @@ -367,7 +368,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, if (unlikely(r)) { return r; } - r = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem); + r = ttm_bo_move_ttm(bo, true, interruptible, no_wait_gpu, &tmp_mem); if (unlikely(r)) { goto out_cleanup; } @@ -396,6 +397,11 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, return -EINVAL; adev = amdgpu_get_adev(bo->bdev); + + /* remember the eviction */ + if (evict) + atomic64_inc(&adev->num_evictions); + if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { amdgpu_move_null(bo, new_mem); return 0; @@ -429,7 +435,8 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, if (r) { memcpy: - r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); + r = ttm_bo_move_memcpy(bo, evict, interruptible, + no_wait_gpu, new_mem); if (r) { return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 3959055eb..91aeed825 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -40,9 +40,16 @@ #include "uvd/uvd_4_2_d.h" /* 1 second timeout */ -#define UVD_IDLE_TIMEOUT_MS 1000 +#define UVD_IDLE_TIMEOUT msecs_to_jiffies(1000) + +/* Firmware versions for VI */ +#define FW_1_65_10 ((1 << 24) | (65 << 16) | (10 << 8)) +#define FW_1_87_11 ((1 << 24) | (87 << 16) | (11 << 8)) +#define FW_1_87_12 ((1 << 24) | (87 << 16) | (12 << 8)) +#define FW_1_37_15 ((1 << 24) | (37 << 16) | (15 << 8)) + /* Polaris10/11 firmware version */ -#define FW_1_66_16 ((1 << 24) | (66 << 16) | (16 << 8)) +#define FW_1_66_16 ((1 << 24) | (66 << 16) | (16 << 8)) /* Firmware Names */ #ifdef CONFIG_DRM_AMDGPU_CIK @@ -83,7 +90,6 @@ struct amdgpu_uvd_cs_ctx { #endif /*(DEBLOBBED)*/ -static void amdgpu_uvd_note_usage(struct amdgpu_device *adev); static void amdgpu_uvd_idle_work_handler(struct work_struct *work); int amdgpu_uvd_sw_init(struct amdgpu_device *adev) @@ -237,6 +243,23 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) if (!amdgpu_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0)) adev->uvd.address_64_bit = true; + switch (adev->asic_type) { + case CHIP_TONGA: + adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_65_10; + break; + case CHIP_CARRIZO: + adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_87_11; + break; + case CHIP_FIJI: + adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_87_12; + break; + case CHIP_STONEY: + adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_37_15; + break; + default: + adev->uvd.use_ctx_buf = adev->asic_type >= CHIP_POLARIS10; + } + return 0; } @@ -337,8 +360,6 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp) if (handle != 0 && adev->uvd.filp[i] == filp) { struct fence *fence; - amdgpu_uvd_note_usage(adev); - r = amdgpu_uvd_get_destroy_msg(ring, handle, false, &fence); if (r) { @@ -429,7 +450,7 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, unsigned fs_in_mb = width_in_mb * height_in_mb; unsigned image_size, tmp, min_dpb_size, num_dpb_buffer; - unsigned min_ctx_size = 0; + unsigned min_ctx_size = ~0; image_size = width * height; image_size += image_size / 2; @@ -548,7 +569,7 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, /* reference picture buffer */ min_dpb_size = image_size * num_dpb_buffer; - if (adev->asic_type < CHIP_POLARIS10){ + if (!adev->uvd.use_ctx_buf){ /* macroblock context buffer */ min_dpb_size += width_in_mb * height_in_mb * num_dpb_buffer * 192; @@ -653,7 +674,7 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx, } DRM_ERROR("No more free UVD handles!\n"); - return -EINVAL; + return -ENOSPC; case 1: /* it's a decode msg, calc buffer sizes */ @@ -904,8 +925,6 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx) return -EINVAL; } - amdgpu_uvd_note_usage(ctx.parser->adev); - return 0; } @@ -959,7 +978,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo, if (direct) { r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f); - job->fence = f; + job->fence = fence_get(f); if (r) goto err_free; @@ -1097,24 +1116,18 @@ static void amdgpu_uvd_idle_work_handler(struct work_struct *work) if (fences == 0 && handles == 0) { if (adev->pm.dpm_enabled) { amdgpu_dpm_enable_uvd(adev, false); - /* just work around for uvd clock remain high even - * when uvd dpm disabled on Polaris10 */ - if (adev->asic_type == CHIP_POLARIS10) - amdgpu_asic_set_uvd_clocks(adev, 0, 0); } else { amdgpu_asic_set_uvd_clocks(adev, 0, 0); } } else { - schedule_delayed_work(&adev->uvd.idle_work, - msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS)); + schedule_delayed_work(&adev->uvd.idle_work, UVD_IDLE_TIMEOUT); } } -static void amdgpu_uvd_note_usage(struct amdgpu_device *adev) +void amdgpu_uvd_ring_begin_use(struct amdgpu_ring *ring) { + struct amdgpu_device *adev = ring->adev; bool set_clocks = !cancel_delayed_work_sync(&adev->uvd.idle_work); - set_clocks &= schedule_delayed_work(&adev->uvd.idle_work, - msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS)); if (set_clocks) { if (adev->pm.dpm_enabled) { @@ -1124,3 +1137,49 @@ static void amdgpu_uvd_note_usage(struct amdgpu_device *adev) } } } + +void amdgpu_uvd_ring_end_use(struct amdgpu_ring *ring) +{ + schedule_delayed_work(&ring->adev->uvd.idle_work, UVD_IDLE_TIMEOUT); +} + +/** + * amdgpu_uvd_ring_test_ib - test ib execution + * + * @ring: amdgpu_ring pointer + * + * Test if we can successfully execute an IB + */ +int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout) +{ + struct fence *fence; + long r; + + r = amdgpu_uvd_get_create_msg(ring, 1, NULL); + if (r) { + DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r); + goto error; + } + + r = amdgpu_uvd_get_destroy_msg(ring, 1, true, &fence); + if (r) { + DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r); + goto error; + } + + r = fence_wait_timeout(fence, false, timeout); + if (r == 0) { + DRM_ERROR("amdgpu: IB test timed out.\n"); + r = -ETIMEDOUT; + } else if (r < 0) { + DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); + } else { + DRM_INFO("ib test on ring %d succeeded\n", ring->idx); + r = 0; + } + + fence_put(fence); + +error: + return r; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h index 9a3b44908..c85000960 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h @@ -35,5 +35,8 @@ int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp); int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx); +void amdgpu_uvd_ring_begin_use(struct amdgpu_ring *ring); +void amdgpu_uvd_ring_end_use(struct amdgpu_ring *ring); +int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index 0a08cf930..7f8c44308 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -36,7 +36,7 @@ #include "cikd.h" /* 1 second timeout */ -#define VCE_IDLE_TIMEOUT_MS 1000 +#define VCE_IDLE_TIMEOUT msecs_to_jiffies(1000) /* Firmware Names */ #ifdef CONFIG_DRM_AMDGPU_CIK @@ -76,8 +76,6 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size) unsigned ucode_version, version_major, version_minor, binary_id; int i, r; - INIT_DELAYED_WORK(&adev->vce.idle_work, amdgpu_vce_idle_work_handler); - switch (adev->asic_type) { #ifdef CONFIG_DRM_AMDGPU_CIK case CHIP_BONAIRE: @@ -188,6 +186,9 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size) adev->vce.filp[i] = NULL; } + INIT_DELAYED_WORK(&adev->vce.idle_work, amdgpu_vce_idle_work_handler); + mutex_init(&adev->vce.idle_mutex); + return 0; } @@ -211,6 +212,7 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev) amdgpu_ring_fini(&adev->vce.ring[1]); release_firmware(adev->vce.fw); + mutex_destroy(&adev->vce.idle_mutex); return 0; } @@ -301,37 +303,44 @@ static void amdgpu_vce_idle_work_handler(struct work_struct *work) amdgpu_asic_set_vce_clocks(adev, 0, 0); } } else { - schedule_delayed_work(&adev->vce.idle_work, - msecs_to_jiffies(VCE_IDLE_TIMEOUT_MS)); + schedule_delayed_work(&adev->vce.idle_work, VCE_IDLE_TIMEOUT); } } /** - * amdgpu_vce_note_usage - power up VCE + * amdgpu_vce_ring_begin_use - power up VCE * - * @adev: amdgpu_device pointer + * @ring: amdgpu ring * * Make sure VCE is powerd up when we want to use it */ -static void amdgpu_vce_note_usage(struct amdgpu_device *adev) +void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring) { - bool streams_changed = false; - bool set_clocks = !cancel_delayed_work_sync(&adev->vce.idle_work); - set_clocks &= schedule_delayed_work(&adev->vce.idle_work, - msecs_to_jiffies(VCE_IDLE_TIMEOUT_MS)); - - if (adev->pm.dpm_enabled) { - /* XXX figure out if the streams changed */ - streams_changed = false; - } + struct amdgpu_device *adev = ring->adev; + bool set_clocks; - if (set_clocks || streams_changed) { + mutex_lock(&adev->vce.idle_mutex); + set_clocks = !cancel_delayed_work_sync(&adev->vce.idle_work); + if (set_clocks) { if (adev->pm.dpm_enabled) { amdgpu_dpm_enable_vce(adev, true); } else { amdgpu_asic_set_vce_clocks(adev, 53300, 40000); } } + mutex_unlock(&adev->vce.idle_mutex); +} + +/** + * amdgpu_vce_ring_end_use - power VCE down + * + * @ring: amdgpu ring + * + * Schedule work to power VCE down again + */ +void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring) +{ + schedule_delayed_work(&ring->adev->vce.idle_work, VCE_IDLE_TIMEOUT); } /** @@ -348,11 +357,10 @@ void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp) int i, r; for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) { uint32_t handle = atomic_read(&adev->vce.handles[i]); + if (!handle || adev->vce.filp[i] != filp) continue; - amdgpu_vce_note_usage(adev); - r = amdgpu_vce_get_destroy_msg(ring, handle, false, NULL); if (r) DRM_ERROR("Error destroying VCE handle (%d)!\n", r); @@ -428,7 +436,7 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, ib->ptr[i] = 0x0; r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f); - job->fence = f; + job->fence = fence_get(f); if (r) goto err; @@ -460,7 +468,6 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, struct amdgpu_job *job; struct amdgpu_ib *ib; struct fence *f = NULL; - uint64_t dummy; int i, r; r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job); @@ -468,7 +475,6 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, return r; ib = &job->ibs[0]; - dummy = ib->gpu_addr + 1024; /* stitch together an VCE destroy msg */ ib->length_dw = 0; @@ -476,11 +482,14 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */ ib->ptr[ib->length_dw++] = handle; - ib->ptr[ib->length_dw++] = 0x00000014; /* len */ - ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */ - ib->ptr[ib->length_dw++] = upper_32_bits(dummy); - ib->ptr[ib->length_dw++] = dummy; - ib->ptr[ib->length_dw++] = 0x00000001; + ib->ptr[ib->length_dw++] = 0x00000020; /* len */ + ib->ptr[ib->length_dw++] = 0x00000002; /* task info */ + ib->ptr[ib->length_dw++] = 0xffffffff; /* next task info, set to 0xffffffff if no */ + ib->ptr[ib->length_dw++] = 0x00000001; /* destroy session */ + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0xffffffff; /* feedback is not needed, set to 0xffffffff and firmware will not output feedback */ + ib->ptr[ib->length_dw++] = 0x00000000; ib->ptr[ib->length_dw++] = 0x00000008; /* len */ ib->ptr[ib->length_dw++] = 0x02000001; /* destroy cmd */ @@ -490,7 +499,7 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, if (direct) { r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f); - job->fence = f; + job->fence = fence_get(f); if (r) goto err; @@ -571,12 +580,10 @@ static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx, * we we don't have another free session index. */ static int amdgpu_vce_validate_handle(struct amdgpu_cs_parser *p, - uint32_t handle, bool *allocated) + uint32_t handle, uint32_t *allocated) { unsigned i; - *allocated = false; - /* validate the handle */ for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) { if (atomic_read(&p->adev->vce.handles[i]) == handle) { @@ -593,7 +600,7 @@ static int amdgpu_vce_validate_handle(struct amdgpu_cs_parser *p, if (!atomic_cmpxchg(&p->adev->vce.handles[i], 0, handle)) { p->adev->vce.filp[i] = p->filp; p->adev->vce.img_size[i] = 0; - *allocated = true; + *allocated |= 1 << i; return i; } } @@ -613,15 +620,13 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx) struct amdgpu_ib *ib = &p->job->ibs[ib_idx]; unsigned fb_idx = 0, bs_idx = 0; int session_idx = -1; - bool destroyed = false; - bool created = false; - bool allocated = false; + uint32_t destroyed = 0; + uint32_t created = 0; + uint32_t allocated = 0; uint32_t tmp, handle = 0; uint32_t *size = &tmp; int i, r = 0, idx = 0; - amdgpu_vce_note_usage(p->adev); - while (idx < ib->length_dw) { uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx); uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1); @@ -632,30 +637,30 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx) goto out; } - if (destroyed) { - DRM_ERROR("No other command allowed after destroy!\n"); - r = -EINVAL; - goto out; - } - switch (cmd) { - case 0x00000001: // session + case 0x00000001: /* session */ handle = amdgpu_get_ib_value(p, ib_idx, idx + 2); session_idx = amdgpu_vce_validate_handle(p, handle, &allocated); - if (session_idx < 0) - return session_idx; + if (session_idx < 0) { + r = session_idx; + goto out; + } size = &p->adev->vce.img_size[session_idx]; break; - case 0x00000002: // task info + case 0x00000002: /* task info */ fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6); bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7); break; - case 0x01000001: // create - created = true; - if (!allocated) { + case 0x01000001: /* create */ + created |= 1 << session_idx; + if (destroyed & (1 << session_idx)) { + destroyed &= ~(1 << session_idx); + allocated |= 1 << session_idx; + + } else if (!(allocated & (1 << session_idx))) { DRM_ERROR("Handle already in use!\n"); r = -EINVAL; goto out; @@ -666,16 +671,16 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx) 8 * 3 / 2; break; - case 0x04000001: // config extension - case 0x04000002: // pic control - case 0x04000005: // rate control - case 0x04000007: // motion estimation - case 0x04000008: // rdo - case 0x04000009: // vui - case 0x05000002: // auxiliary buffer + case 0x04000001: /* config extension */ + case 0x04000002: /* pic control */ + case 0x04000005: /* rate control */ + case 0x04000007: /* motion estimation */ + case 0x04000008: /* rdo */ + case 0x04000009: /* vui */ + case 0x05000002: /* auxiliary buffer */ break; - case 0x03000001: // encode + case 0x03000001: /* encode */ r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 10, idx + 9, *size, 0); if (r) @@ -687,18 +692,18 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx) goto out; break; - case 0x02000001: // destroy - destroyed = true; + case 0x02000001: /* destroy */ + destroyed |= 1 << session_idx; break; - case 0x05000001: // context buffer + case 0x05000001: /* context buffer */ r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2, *size * 2, 0); if (r) goto out; break; - case 0x05000004: // video bitstream buffer + case 0x05000004: /* video bitstream buffer */ tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4); r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2, tmp, bs_idx); @@ -706,7 +711,7 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx) goto out; break; - case 0x05000005: // feedback buffer + case 0x05000005: /* feedback buffer */ r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2, 4096, fb_idx); if (r) @@ -728,21 +733,24 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx) idx += len / 4; } - if (allocated && !created) { + if (allocated & ~created) { DRM_ERROR("New session without create command!\n"); r = -ENOENT; } out: - if ((!r && destroyed) || (r && allocated)) { - /* - * IB contains a destroy msg or we have allocated an - * handle and got an error, anyway free the handle - */ - for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) - atomic_cmpxchg(&p->adev->vce.handles[i], handle, 0); + if (!r) { + /* No error, free all destroyed handle slots */ + tmp = destroyed; + } else { + /* Error during parsing, free all allocated handle slots */ + tmp = allocated; } + for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) + if (tmp & (1 << i)) + atomic_set(&p->adev->vce.handles[i], 0); + return r; } @@ -828,10 +836,10 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring) * @ring: the engine to test on * */ -int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring) +int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout) { struct fence *fence = NULL; - int r; + long r; /* skip vce ring1 ib test for now, since it's not reliable */ if (ring == &ring->adev->vce.ring[1]) @@ -839,21 +847,25 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring) r = amdgpu_vce_get_create_msg(ring, 1, NULL); if (r) { - DRM_ERROR("amdgpu: failed to get create msg (%d).\n", r); + DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r); goto error; } r = amdgpu_vce_get_destroy_msg(ring, 1, true, &fence); if (r) { - DRM_ERROR("amdgpu: failed to get destroy ib (%d).\n", r); + DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r); goto error; } - r = fence_wait(fence, false); - if (r) { - DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); + r = fence_wait_timeout(fence, false, timeout); + if (r == 0) { + DRM_ERROR("amdgpu: IB test timed out.\n"); + r = -ETIMEDOUT; + } else if (r < 0) { + DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); } else { DRM_INFO("ib test on ring %d succeeded\n", ring->idx); + r = 0; } error: fence_put(fence); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h index f40cf761c..63f83d0d9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h @@ -39,6 +39,8 @@ void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib, void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, unsigned flags); int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring); -int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring); +int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout); +void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring); +void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 9f36ed30b..80120fa40 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -25,6 +25,7 @@ * Alex Deucher * Jerome Glisse */ +#include #include #include #include "amdgpu.h" @@ -114,16 +115,26 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, /** * amdgpu_vm_get_bos - add the vm BOs to a duplicates list * + * @adev: amdgpu device pointer * @vm: vm providing the BOs * @duplicates: head of duplicates list * * Add the page directory to the BO duplicates list * for command submission. */ -void amdgpu_vm_get_pt_bos(struct amdgpu_vm *vm, struct list_head *duplicates) +void amdgpu_vm_get_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, + struct list_head *duplicates) { + uint64_t num_evictions; unsigned i; + /* We only need to validate the page tables + * if they aren't already valid. + */ + num_evictions = atomic64_read(&adev->num_evictions); + if (num_evictions == vm->last_eviction_counter) + return; + /* add the vm page table to the list */ for (i = 0; i <= vm->max_pde_used; ++i) { struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry; @@ -162,6 +173,13 @@ void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev, spin_unlock(&glob->lru_lock); } +static bool amdgpu_vm_is_gpu_reset(struct amdgpu_device *adev, + struct amdgpu_vm_id *id) +{ + return id->current_gpu_reset_count != + atomic_read(&adev->gpu_reset_counter) ? true : false; +} + /** * amdgpu_vm_grab_id - allocate the next free VMID * @@ -174,18 +192,67 @@ void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev, */ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_sync *sync, struct fence *fence, - unsigned *vm_id, uint64_t *vm_pd_addr) + struct amdgpu_job *job) { - uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory); struct amdgpu_device *adev = ring->adev; + uint64_t fence_context = adev->fence_context + ring->idx; struct fence *updates = sync->last_vm_update; - struct amdgpu_vm_id *id; - unsigned i = ring->idx; - int r; + struct amdgpu_vm_id *id, *idle; + struct fence **fences; + unsigned i; + int r = 0; + + fences = kmalloc_array(sizeof(void *), adev->vm_manager.num_ids, + GFP_KERNEL); + if (!fences) + return -ENOMEM; mutex_lock(&adev->vm_manager.lock); + /* Check if we have an idle VMID */ + i = 0; + list_for_each_entry(idle, &adev->vm_manager.ids_lru, list) { + fences[i] = amdgpu_sync_peek_fence(&idle->active, ring); + if (!fences[i]) + break; + ++i; + } + + /* If we can't find a idle VMID to use, wait till one becomes available */ + if (&idle->list == &adev->vm_manager.ids_lru) { + u64 fence_context = adev->vm_manager.fence_context + ring->idx; + unsigned seqno = ++adev->vm_manager.seqno[ring->idx]; + struct fence_array *array; + unsigned j; + + for (j = 0; j < i; ++j) + fence_get(fences[j]); + + array = fence_array_create(i, fences, fence_context, + seqno, true); + if (!array) { + for (j = 0; j < i; ++j) + fence_put(fences[j]); + kfree(fences); + r = -ENOMEM; + goto error; + } + + + r = amdgpu_sync_fence(ring->adev, sync, &array->base); + fence_put(&array->base); + if (r) + goto error; + + mutex_unlock(&adev->vm_manager.lock); + return 0; + + } + kfree(fences); + + job->vm_needs_flush = true; /* Check if we can use a VMID already assigned to this VM */ + i = ring->idx; do { struct fence *flushed; @@ -196,67 +263,52 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, /* Check all the prerequisites to using this VMID */ if (!id) continue; + if (amdgpu_vm_is_gpu_reset(adev, id)) + continue; if (atomic64_read(&id->owner) != vm->client_id) continue; - if (pd_addr != id->pd_gpu_addr) + if (job->vm_pd_addr != id->pd_gpu_addr) continue; - if (id->last_user != ring && - (!id->last_flush || !fence_is_signaled(id->last_flush))) + if (!id->last_flush) continue; - flushed = id->flushed_updates; - if (updates && (!flushed || fence_is_later(updates, flushed))) + if (id->last_flush->context != fence_context && + !fence_is_signaled(id->last_flush)) continue; - /* Good we can use this VMID */ - if (id->last_user == ring) { - r = amdgpu_sync_fence(ring->adev, sync, - id->first); - if (r) - goto error; - } + flushed = id->flushed_updates; + if (updates && + (!flushed || fence_is_later(updates, flushed))) + continue; - /* And remember this submission as user of the VMID */ + /* Good we can use this VMID. Remember this submission as + * user of the VMID. + */ r = amdgpu_sync_fence(ring->adev, &id->active, fence); if (r) goto error; + id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter); list_move_tail(&id->list, &adev->vm_manager.ids_lru); vm->ids[ring->idx] = id; - *vm_id = id - adev->vm_manager.ids; - *vm_pd_addr = AMDGPU_VM_NO_FLUSH; - trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr); + job->vm_id = id - adev->vm_manager.ids; + job->vm_needs_flush = false; + trace_amdgpu_vm_grab_id(vm, ring->idx, job); mutex_unlock(&adev->vm_manager.lock); return 0; } while (i != ring->idx); - id = list_first_entry(&adev->vm_manager.ids_lru, - struct amdgpu_vm_id, - list); - - if (!amdgpu_sync_is_idle(&id->active)) { - struct list_head *head = &adev->vm_manager.ids_lru; - struct amdgpu_vm_id *tmp; - - list_for_each_entry_safe(id, tmp, &adev->vm_manager.ids_lru, - list) { - if (amdgpu_sync_is_idle(&id->active)) { - list_move(&id->list, head); - head = &id->list; - } - } - id = list_first_entry(&adev->vm_manager.ids_lru, - struct amdgpu_vm_id, - list); - } + /* Still no ID to use? Then use the idle one found earlier */ + id = idle; - r = amdgpu_sync_cycle_fences(sync, &id->active, fence); + /* Remember this submission as user of the VMID */ + r = amdgpu_sync_fence(ring->adev, &id->active, fence); if (r) goto error; @@ -269,22 +321,46 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, fence_put(id->flushed_updates); id->flushed_updates = fence_get(updates); - id->pd_gpu_addr = pd_addr; - + id->pd_gpu_addr = job->vm_pd_addr; + id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter); list_move_tail(&id->list, &adev->vm_manager.ids_lru); - id->last_user = ring; atomic64_set(&id->owner, vm->client_id); vm->ids[ring->idx] = id; - *vm_id = id - adev->vm_manager.ids; - *vm_pd_addr = pd_addr; - trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr); + job->vm_id = id - adev->vm_manager.ids; + trace_amdgpu_vm_grab_id(vm, ring->idx, job); error: mutex_unlock(&adev->vm_manager.lock); return r; } +static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + const struct amdgpu_ip_block_version *ip_block; + + if (ring->type != AMDGPU_RING_TYPE_COMPUTE) + /* only compute rings */ + return false; + + ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); + if (!ip_block) + return false; + + if (ip_block->major <= 7) { + /* gfx7 has no workaround */ + return true; + } else if (ip_block->major == 8) { + if (adev->gfx.mec_fw_version >= 673) + /* gfx8 is fixed in MEC firmware 673 */ + return false; + else + return true; + } + return false; +} + /** * amdgpu_vm_flush - hardware flush the vm * @@ -294,59 +370,52 @@ error: * * Emit a VM flush when it is necessary. */ -int amdgpu_vm_flush(struct amdgpu_ring *ring, - unsigned vm_id, uint64_t pd_addr, - uint32_t gds_base, uint32_t gds_size, - uint32_t gws_base, uint32_t gws_size, - uint32_t oa_base, uint32_t oa_size) +int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job) { struct amdgpu_device *adev = ring->adev; - struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id]; + struct amdgpu_vm_id *id = &adev->vm_manager.ids[job->vm_id]; bool gds_switch_needed = ring->funcs->emit_gds_switch && ( - id->gds_base != gds_base || - id->gds_size != gds_size || - id->gws_base != gws_base || - id->gws_size != gws_size || - id->oa_base != oa_base || - id->oa_size != oa_size); + id->gds_base != job->gds_base || + id->gds_size != job->gds_size || + id->gws_base != job->gws_base || + id->gws_size != job->gws_size || + id->oa_base != job->oa_base || + id->oa_size != job->oa_size); int r; if (ring->funcs->emit_pipeline_sync && ( - pd_addr != AMDGPU_VM_NO_FLUSH || gds_switch_needed || - ring->type == AMDGPU_RING_TYPE_COMPUTE)) + job->vm_needs_flush || gds_switch_needed || + amdgpu_vm_ring_has_compute_vm_bug(ring))) amdgpu_ring_emit_pipeline_sync(ring); - if (ring->funcs->emit_vm_flush && - pd_addr != AMDGPU_VM_NO_FLUSH) { + if (ring->funcs->emit_vm_flush && (job->vm_needs_flush || + amdgpu_vm_is_gpu_reset(adev, id))) { struct fence *fence; - trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id); - amdgpu_ring_emit_vm_flush(ring, vm_id, pd_addr); + trace_amdgpu_vm_flush(job->vm_pd_addr, ring->idx, job->vm_id); + amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr); + + r = amdgpu_fence_emit(ring, &fence); + if (r) + return r; mutex_lock(&adev->vm_manager.lock); - if ((id->pd_gpu_addr == pd_addr) && (id->last_user == ring)) { - r = amdgpu_fence_emit(ring, &fence); - if (r) { - mutex_unlock(&adev->vm_manager.lock); - return r; - } - fence_put(id->last_flush); - id->last_flush = fence; - } + fence_put(id->last_flush); + id->last_flush = fence; mutex_unlock(&adev->vm_manager.lock); } if (gds_switch_needed) { - id->gds_base = gds_base; - id->gds_size = gds_size; - id->gws_base = gws_base; - id->gws_size = gws_size; - id->oa_base = oa_base; - id->oa_size = oa_size; - amdgpu_ring_emit_gds_switch(ring, vm_id, - gds_base, gds_size, - gws_base, gws_size, - oa_base, oa_size); + id->gds_base = job->gds_base; + id->gds_size = job->gds_size; + id->gws_base = job->gws_base; + id->gws_size = job->gws_size; + id->oa_base = job->oa_base; + id->oa_size = job->oa_size; + amdgpu_ring_emit_gds_switch(ring, job->vm_id, + job->gds_base, job->gds_size, + job->gws_base, job->gws_size, + job->oa_base, job->oa_size); } return 0; @@ -723,7 +792,7 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev, * @vm: requested vm * @start: start of GPU address range * @end: end of GPU address range - * @dst: destination address to map to + * @dst: destination address to map to, the next dst inside the function * @flags: mapping flags * * Update the page tables in the range @start - @end. @@ -737,49 +806,75 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev, { const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1; - uint64_t last_pe_start = ~0, last_pe_end = ~0, last_dst = ~0; - uint64_t addr; + uint64_t cur_pe_start, cur_pe_end, cur_dst; + uint64_t addr; /* next GPU address to be updated */ + uint64_t pt_idx; + struct amdgpu_bo *pt; + unsigned nptes; /* next number of ptes to be updated */ + uint64_t next_pe_start; + + /* initialize the variables */ + addr = start; + pt_idx = addr >> amdgpu_vm_block_size; + pt = vm->page_tables[pt_idx].entry.robj; + + if ((addr & ~mask) == (end & ~mask)) + nptes = end - addr; + else + nptes = AMDGPU_VM_PTE_COUNT - (addr & mask); + + cur_pe_start = amdgpu_bo_gpu_offset(pt); + cur_pe_start += (addr & mask) * 8; + cur_pe_end = cur_pe_start + 8 * nptes; + cur_dst = dst; + + /* for next ptb*/ + addr += nptes; + dst += nptes * AMDGPU_GPU_PAGE_SIZE; /* walk over the address space and update the page tables */ - for (addr = start; addr < end; ) { - uint64_t pt_idx = addr >> amdgpu_vm_block_size; - struct amdgpu_bo *pt = vm->page_tables[pt_idx].entry.robj; - unsigned nptes; - uint64_t pe_start; + while (addr < end) { + pt_idx = addr >> amdgpu_vm_block_size; + pt = vm->page_tables[pt_idx].entry.robj; if ((addr & ~mask) == (end & ~mask)) nptes = end - addr; else nptes = AMDGPU_VM_PTE_COUNT - (addr & mask); - pe_start = amdgpu_bo_gpu_offset(pt); - pe_start += (addr & mask) * 8; - - if (last_pe_end != pe_start) { + next_pe_start = amdgpu_bo_gpu_offset(pt); + next_pe_start += (addr & mask) * 8; + if (cur_pe_end == next_pe_start) { + /* The next ptb is consecutive to current ptb. + * Don't call amdgpu_vm_frag_ptes now. + * Will update two ptbs together in future. + */ + cur_pe_end += 8 * nptes; + } else { amdgpu_vm_frag_ptes(adev, vm_update_params, - last_pe_start, last_pe_end, - last_dst, flags); + cur_pe_start, cur_pe_end, + cur_dst, flags); - last_pe_start = pe_start; - last_pe_end = pe_start + 8 * nptes; - last_dst = dst; - } else { - last_pe_end += 8 * nptes; + cur_pe_start = next_pe_start; + cur_pe_end = next_pe_start + 8 * nptes; + cur_dst = dst; } + /* for next ptb*/ addr += nptes; dst += nptes * AMDGPU_GPU_PAGE_SIZE; } - amdgpu_vm_frag_ptes(adev, vm_update_params, last_pe_start, - last_pe_end, last_dst, flags); + amdgpu_vm_frag_ptes(adev, vm_update_params, cur_pe_start, + cur_pe_end, cur_dst, flags); } /** * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table * * @adev: amdgpu_device pointer + * @exclusive: fence we need to sync to * @src: address where to copy page table entries from * @pages_addr: DMA addresses to use for mapping * @vm: requested vm @@ -793,6 +888,7 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev, * Returns 0 for success, -EINVAL for failure. */ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, + struct fence *exclusive, uint64_t src, dma_addr_t *pages_addr, struct amdgpu_vm *vm, @@ -853,6 +949,10 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, vm_update_params.ib = &job->ibs[0]; + r = amdgpu_sync_fence(adev, &job->sync, exclusive); + if (r) + goto error_free; + r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv, owner); if (r) @@ -889,6 +989,7 @@ error_free: * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks * * @adev: amdgpu_device pointer + * @exclusive: fence we need to sync to * @gtt_flags: flags as they are used for GTT * @pages_addr: DMA addresses to use for mapping * @vm: requested vm @@ -902,6 +1003,7 @@ error_free: * Returns 0 for success, -EINVAL for failure. */ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, + struct fence *exclusive, uint32_t gtt_flags, dma_addr_t *pages_addr, struct amdgpu_vm *vm, @@ -932,7 +1034,8 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, addr += mapping->offset; if (!pages_addr || src) - return amdgpu_vm_bo_update_mapping(adev, src, pages_addr, vm, + return amdgpu_vm_bo_update_mapping(adev, exclusive, + src, pages_addr, vm, start, mapping->it.last, flags, addr, fence); @@ -940,7 +1043,8 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, uint64_t last; last = min((uint64_t)mapping->it.last, start + max_size - 1); - r = amdgpu_vm_bo_update_mapping(adev, src, pages_addr, vm, + r = amdgpu_vm_bo_update_mapping(adev, exclusive, + src, pages_addr, vm, start, last, flags, addr, fence); if (r) @@ -973,6 +1077,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va_mapping *mapping; dma_addr_t *pages_addr = NULL; uint32_t gtt_flags, flags; + struct fence *exclusive; uint64_t addr; int r; @@ -994,8 +1099,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, default: break; } + + exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv); } else { addr = 0; + exclusive = NULL; } flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem); @@ -1007,7 +1115,8 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, spin_unlock(&vm->status_lock); list_for_each_entry(mapping, &bo_va->invalids, list) { - r = amdgpu_vm_bo_split_mapping(adev, gtt_flags, pages_addr, vm, + r = amdgpu_vm_bo_split_mapping(adev, exclusive, + gtt_flags, pages_addr, vm, mapping, flags, addr, &bo_va->last_pt_update); if (r) @@ -1054,7 +1163,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev, struct amdgpu_bo_va_mapping, list); list_del(&mapping->list); - r = amdgpu_vm_bo_split_mapping(adev, 0, NULL, vm, mapping, + r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, NULL, vm, mapping, 0, 0, NULL); kfree(mapping); if (r) @@ -1426,7 +1535,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) r = amd_sched_entity_init(&ring->sched, &vm->entity, rq, amdgpu_sched_jobs); if (r) - return r; + goto err; vm->page_directory_fence = NULL; @@ -1445,6 +1554,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) amdgpu_bo_unreserve(vm->page_directory); if (r) goto error_free_page_directory; + vm->last_eviction_counter = atomic64_read(&adev->num_evictions); return 0; @@ -1455,6 +1565,9 @@ error_free_page_directory: error_free_sched_entity: amd_sched_entity_fini(&ring->sched, &vm->entity); +err: + drm_free_large(vm->page_tables); + return r; } @@ -1516,6 +1629,10 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev) &adev->vm_manager.ids_lru); } + adev->vm_manager.fence_context = fence_context_alloc(AMDGPU_MAX_RINGS); + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) + adev->vm_manager.seqno[i] = 0; + atomic_set(&adev->vm_manager.vm_pte_next_ring, 0); atomic64_set(&adev->vm_manager.client_counter, 0); } diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c index 040dd26ff..43934248b 100644 --- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c @@ -83,12 +83,14 @@ static const struct ci_pt_defaults defaults_bonaire_xt = { 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 } }; +#if 0 static const struct ci_pt_defaults defaults_bonaire_pro = { 1, 0xF, 0xFD, 0x19, 5, 45, 0, 0x65062, { 0x8C, 0x23F, 0x244, 0xA6, 0x83, 0x85, 0x86, 0x86, 0x83, 0xDB, 0xDB, 0xDA, 0x67, 0x60, 0x5F }, { 0x187, 0x193, 0x193, 0x1C7, 0x1D1, 0x1D1, 0x210, 0x219, 0x219, 0x266, 0x26C, 0x26C, 0x2C9, 0x2CB, 0x2CB } }; +#endif static const struct ci_pt_defaults defaults_saturn_xt = { @@ -97,12 +99,14 @@ static const struct ci_pt_defaults defaults_saturn_xt = { 0x187, 0x187, 0x187, 0x1C7, 0x1C7, 0x1C7, 0x210, 0x210, 0x210, 0x266, 0x266, 0x266, 0x2C9, 0x2C9, 0x2C9 } }; +#if 0 static const struct ci_pt_defaults defaults_saturn_pro = { 1, 0xF, 0xFD, 0x19, 5, 55, 0, 0x30000, { 0x96, 0x21D, 0x23B, 0xA1, 0x85, 0x87, 0x83, 0x84, 0x81, 0xE6, 0xE6, 0xE6, 0x71, 0x6A, 0x6A }, { 0x193, 0x19E, 0x19E, 0x1D2, 0x1DC, 0x1DC, 0x21A, 0x223, 0x223, 0x26E, 0x27E, 0x274, 0x2CF, 0x2D2, 0x2D2 } }; +#endif static const struct ci_pt_config_reg didt_config_ci[] = { @@ -735,19 +739,19 @@ static int ci_enable_didt(struct amdgpu_device *adev, bool enable) if (pi->caps_sq_ramping || pi->caps_db_ramping || pi->caps_td_ramping || pi->caps_tcp_ramping) { - gfx_v7_0_enter_rlc_safe_mode(adev); + adev->gfx.rlc.funcs->enter_safe_mode(adev); if (enable) { ret = ci_program_pt_config_registers(adev, didt_config_ci); if (ret) { - gfx_v7_0_exit_rlc_safe_mode(adev); + adev->gfx.rlc.funcs->exit_safe_mode(adev); return ret; } } ci_do_enable_didt(adev, enable); - gfx_v7_0_exit_rlc_safe_mode(adev); + adev->gfx.rlc.funcs->exit_safe_mode(adev); } return 0; @@ -3029,7 +3033,7 @@ static int ci_populate_single_memory_level(struct amdgpu_device *adev, if (pi->mclk_stutter_mode_threshold && (memory_clock <= pi->mclk_stutter_mode_threshold) && - (pi->uvd_enabled == false) && + (!pi->uvd_enabled) && (RREG32(mmDPG_PIPE_STUTTER_CONTROL) & DPG_PIPE_STUTTER_CONTROL__STUTTER_ENABLE_MASK) && (adev->pm.dpm.new_active_crtc_count <= 2)) memory_level->StutterEnable = true; @@ -3635,6 +3639,10 @@ static int ci_setup_default_dpm_tables(struct amdgpu_device *adev) ci_setup_default_pcie_tables(adev); + /* save a copy of the default DPM table */ + memcpy(&(pi->golden_dpm_table), &(pi->dpm_table), + sizeof(struct ci_dpm_table)); + return 0; } @@ -5753,13 +5761,22 @@ static int ci_dpm_init_microcode(struct amdgpu_device *adev) switch (adev->asic_type) { case CHIP_BONAIRE: - chip_name = "bonaire"; + if ((adev->pdev->revision == 0x80) || + (adev->pdev->revision == 0x81) || + (adev->pdev->device == 0x665f)) + chip_name = "bonaire_k"; + else + chip_name = "bonaire"; break; case CHIP_HAWAII: - chip_name = "hawaii"; + if (adev->pdev->revision == 0x80) + chip_name = "hawaii_k"; + else + chip_name = "hawaii"; break; case CHIP_KAVERI: case CHIP_KABINI: + case CHIP_MULLINS: default: BUG(); } @@ -6403,6 +6420,186 @@ static int ci_dpm_set_powergating_state(void *handle, return 0; } +static int ci_dpm_print_clock_levels(struct amdgpu_device *adev, + enum pp_clock_type type, char *buf) +{ + struct ci_power_info *pi = ci_get_pi(adev); + struct ci_single_dpm_table *sclk_table = &pi->dpm_table.sclk_table; + struct ci_single_dpm_table *mclk_table = &pi->dpm_table.mclk_table; + struct ci_single_dpm_table *pcie_table = &pi->dpm_table.pcie_speed_table; + + int i, now, size = 0; + uint32_t clock, pcie_speed; + + switch (type) { + case PP_SCLK: + amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_API_GetSclkFrequency); + clock = RREG32(mmSMC_MSG_ARG_0); + + for (i = 0; i < sclk_table->count; i++) { + if (clock > sclk_table->dpm_levels[i].value) + continue; + break; + } + now = i; + + for (i = 0; i < sclk_table->count; i++) + size += sprintf(buf + size, "%d: %uMhz %s\n", + i, sclk_table->dpm_levels[i].value / 100, + (i == now) ? "*" : ""); + break; + case PP_MCLK: + amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_API_GetMclkFrequency); + clock = RREG32(mmSMC_MSG_ARG_0); + + for (i = 0; i < mclk_table->count; i++) { + if (clock > mclk_table->dpm_levels[i].value) + continue; + break; + } + now = i; + + for (i = 0; i < mclk_table->count; i++) + size += sprintf(buf + size, "%d: %uMhz %s\n", + i, mclk_table->dpm_levels[i].value / 100, + (i == now) ? "*" : ""); + break; + case PP_PCIE: + pcie_speed = ci_get_current_pcie_speed(adev); + for (i = 0; i < pcie_table->count; i++) { + if (pcie_speed != pcie_table->dpm_levels[i].value) + continue; + break; + } + now = i; + + for (i = 0; i < pcie_table->count; i++) + size += sprintf(buf + size, "%d: %s %s\n", i, + (pcie_table->dpm_levels[i].value == 0) ? "2.5GB, x1" : + (pcie_table->dpm_levels[i].value == 1) ? "5.0GB, x16" : + (pcie_table->dpm_levels[i].value == 2) ? "8.0GB, x16" : "", + (i == now) ? "*" : ""); + break; + default: + break; + } + + return size; +} + +static int ci_dpm_force_clock_level(struct amdgpu_device *adev, + enum pp_clock_type type, uint32_t mask) +{ + struct ci_power_info *pi = ci_get_pi(adev); + + if (adev->pm.dpm.forced_level + != AMDGPU_DPM_FORCED_LEVEL_MANUAL) + return -EINVAL; + + switch (type) { + case PP_SCLK: + if (!pi->sclk_dpm_key_disabled) + amdgpu_ci_send_msg_to_smc_with_parameter(adev, + PPSMC_MSG_SCLKDPM_SetEnabledMask, + pi->dpm_level_enable_mask.sclk_dpm_enable_mask & mask); + break; + + case PP_MCLK: + if (!pi->mclk_dpm_key_disabled) + amdgpu_ci_send_msg_to_smc_with_parameter(adev, + PPSMC_MSG_MCLKDPM_SetEnabledMask, + pi->dpm_level_enable_mask.mclk_dpm_enable_mask & mask); + break; + + case PP_PCIE: + { + uint32_t tmp = mask & pi->dpm_level_enable_mask.pcie_dpm_enable_mask; + uint32_t level = 0; + + while (tmp >>= 1) + level++; + + if (!pi->pcie_dpm_key_disabled) + amdgpu_ci_send_msg_to_smc_with_parameter(adev, + PPSMC_MSG_PCIeDPM_ForceLevel, + level); + break; + } + default: + break; + } + + return 0; +} + +static int ci_dpm_get_sclk_od(struct amdgpu_device *adev) +{ + struct ci_power_info *pi = ci_get_pi(adev); + struct ci_single_dpm_table *sclk_table = &(pi->dpm_table.sclk_table); + struct ci_single_dpm_table *golden_sclk_table = + &(pi->golden_dpm_table.sclk_table); + int value; + + value = (sclk_table->dpm_levels[sclk_table->count - 1].value - + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) * + 100 / + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value; + + return value; +} + +static int ci_dpm_set_sclk_od(struct amdgpu_device *adev, uint32_t value) +{ + struct ci_power_info *pi = ci_get_pi(adev); + struct ci_ps *ps = ci_get_ps(adev->pm.dpm.requested_ps); + struct ci_single_dpm_table *golden_sclk_table = + &(pi->golden_dpm_table.sclk_table); + + if (value > 20) + value = 20; + + ps->performance_levels[ps->performance_level_count - 1].sclk = + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value * + value / 100 + + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value; + + return 0; +} + +static int ci_dpm_get_mclk_od(struct amdgpu_device *adev) +{ + struct ci_power_info *pi = ci_get_pi(adev); + struct ci_single_dpm_table *mclk_table = &(pi->dpm_table.mclk_table); + struct ci_single_dpm_table *golden_mclk_table = + &(pi->golden_dpm_table.mclk_table); + int value; + + value = (mclk_table->dpm_levels[mclk_table->count - 1].value - + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) * + 100 / + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value; + + return value; +} + +static int ci_dpm_set_mclk_od(struct amdgpu_device *adev, uint32_t value) +{ + struct ci_power_info *pi = ci_get_pi(adev); + struct ci_ps *ps = ci_get_ps(adev->pm.dpm.requested_ps); + struct ci_single_dpm_table *golden_mclk_table = + &(pi->golden_dpm_table.mclk_table); + + if (value > 20) + value = 20; + + ps->performance_levels[ps->performance_level_count - 1].mclk = + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value * + value / 100 + + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value; + + return 0; +} + const struct amd_ip_funcs ci_dpm_ip_funcs = { .name = "ci_dpm", .early_init = ci_dpm_early_init, @@ -6437,6 +6634,12 @@ static const struct amdgpu_dpm_funcs ci_dpm_funcs = { .get_fan_control_mode = &ci_dpm_get_fan_control_mode, .set_fan_speed_percent = &ci_dpm_set_fan_speed_percent, .get_fan_speed_percent = &ci_dpm_get_fan_speed_percent, + .print_clock_levels = ci_dpm_print_clock_levels, + .force_clock_level = ci_dpm_force_clock_level, + .get_sclk_od = ci_dpm_get_sclk_od, + .set_sclk_od = ci_dpm_set_sclk_od, + .get_mclk_od = ci_dpm_get_mclk_od, + .set_mclk_od = ci_dpm_set_mclk_od, }; static void ci_dpm_set_dpm_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.h b/drivers/gpu/drm/amd/amdgpu/ci_dpm.h index faccc30c9..91be2996a 100644 --- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.h +++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.h @@ -193,6 +193,7 @@ struct ci_pt_defaults { struct ci_power_info { struct ci_dpm_table dpm_table; + struct ci_dpm_table golden_dpm_table; u32 voltage_control; u32 mvdd_control; u32 vddci_control; diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c index 910431808..4efc901f6 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik.c +++ b/drivers/gpu/drm/amd/amdgpu/cik.c @@ -879,7 +879,7 @@ static void cik_vga_set_state(struct amdgpu_device *adev, bool state) uint32_t tmp; tmp = RREG32(mmCONFIG_CNTL); - if (state == false) + if (!state) tmp |= CONFIG_CNTL__VGA_DIS_MASK; else tmp &= ~CONFIG_CNTL__VGA_DIS_MASK; @@ -1035,12 +1035,12 @@ static uint32_t cik_read_indexed_register(struct amdgpu_device *adev, mutex_lock(&adev->grbm_idx_mutex); if (se_num != 0xffffffff || sh_num != 0xffffffff) - gfx_v7_0_select_se_sh(adev, se_num, sh_num); + amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff); val = RREG32(reg_offset); if (se_num != 0xffffffff || sh_num != 0xffffffff) - gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); return val; } @@ -1158,10 +1158,11 @@ static void kv_restore_regs_for_reset(struct amdgpu_device *adev, WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute); } -static void cik_gpu_pci_config_reset(struct amdgpu_device *adev) +static int cik_gpu_pci_config_reset(struct amdgpu_device *adev) { struct kv_reset_save_regs kv_save = { 0 }; u32 i; + int r = -EINVAL; dev_info(adev->dev, "GPU pci config reset\n"); @@ -1177,14 +1178,20 @@ static void cik_gpu_pci_config_reset(struct amdgpu_device *adev) /* wait for asic to come out of reset */ for (i = 0; i < adev->usec_timeout; i++) { - if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) + if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) { + /* enable BM */ + pci_set_master(adev->pdev); + r = 0; break; + } udelay(1); } /* does asic init need to be run first??? */ if (adev->flags & AMD_IS_APU) kv_restore_regs_for_reset(adev, &kv_save); + + return r; } static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung) @@ -1210,13 +1217,14 @@ static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hu */ static int cik_asic_reset(struct amdgpu_device *adev) { + int r; cik_set_bios_scratch_engine_hung(adev, true); - cik_gpu_pci_config_reset(adev); + r = cik_gpu_pci_config_reset(adev); cik_set_bios_scratch_engine_hung(adev, false); - return 0; + return r; } static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock, @@ -2014,9 +2022,6 @@ static const struct amdgpu_asic_funcs cik_asic_funcs = .set_uvd_clocks = &cik_set_uvd_clocks, .set_vce_clocks = &cik_set_vce_clocks, .get_virtual_caps = &cik_get_virtual_caps, - /* these should be moved to their own ip modules */ - .get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter, - .wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle, }; static int cik_common_early_init(void *handle) diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index cb429bfdc..af8b6bdf0 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -216,17 +216,6 @@ static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring, unsigned vm_id, bool ctx_switch) { u32 extra_bits = vm_id & 0xf; - u32 next_rptr = ring->wptr + 5; - - while ((next_rptr & 7) != 4) - next_rptr++; - - next_rptr += 4; - amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0)); - amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); - amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); - amdgpu_ring_write(ring, 1); /* number of DWs to follow */ - amdgpu_ring_write(ring, next_rptr); /* IB packet must end on a 8 DW boundary */ cik_sdma_ring_insert_nop(ring, (12 - (ring->wptr & 7)) % 8); @@ -357,7 +346,7 @@ static void cik_sdma_enable(struct amdgpu_device *adev, bool enable) u32 me_cntl; int i; - if (enable == false) { + if (!enable) { cik_sdma_gfx_stop(adev); cik_sdma_rlc_stop(adev); } @@ -620,20 +609,19 @@ static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring) * Test a simple IB in the DMA ring (CIK). * Returns 0 on success, error on failure. */ -static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) +static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; struct fence *f = NULL; - unsigned i; unsigned index; - int r; u32 tmp = 0; u64 gpu_addr; + long r; r = amdgpu_wb_get(adev, &index); if (r) { - dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r); + dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r); return r; } @@ -643,11 +631,12 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) memset(&ib, 0, sizeof(ib)); r = amdgpu_ib_get(adev, NULL, 256, &ib); if (r) { - DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); + DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r); goto err0; } - ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0); + ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, + SDMA_WRITE_SUB_OPCODE_LINEAR, 0); ib.ptr[1] = lower_32_bits(gpu_addr); ib.ptr[2] = upper_32_bits(gpu_addr); ib.ptr[3] = 1; @@ -657,28 +646,25 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) if (r) goto err1; - r = fence_wait(f, false); - if (r) { - DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); + r = fence_wait_timeout(f, false, timeout); + if (r == 0) { + DRM_ERROR("amdgpu: IB test timed out\n"); + r = -ETIMEDOUT; goto err1; - } - for (i = 0; i < adev->usec_timeout; i++) { - tmp = le32_to_cpu(adev->wb.wb[index]); - if (tmp == 0xDEADBEEF) - break; - DRM_UDELAY(1); - } - if (i < adev->usec_timeout) { - DRM_INFO("ib test on ring %d succeeded in %u usecs\n", - ring->idx, i); + } else if (r < 0) { + DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); goto err1; + } + tmp = le32_to_cpu(adev->wb.wb[index]); + if (tmp == 0xDEADBEEF) { + DRM_INFO("ib test on ring %d succeeded\n", ring->idx); + r = 0; } else { DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); r = -EINVAL; } err1: - fence_put(f); amdgpu_ib_free(adev, &ib, NULL); fence_put(f); err0: diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c index 933e425a8..2a11413ed 100644 --- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c @@ -425,7 +425,7 @@ static int cz_dpm_init(struct amdgpu_device *adev) pi->mgcg_cgtt_local1 = 0x0; pi->clock_slow_down_step = 25000; pi->skip_clock_slow_down = 1; - pi->enable_nb_ps_policy = 0; + pi->enable_nb_ps_policy = false; pi->caps_power_containment = true; pi->caps_cac = true; pi->didt_enabled = false; @@ -2219,6 +2219,7 @@ static void cz_dpm_powergate_vce(struct amdgpu_device *adev, bool gate) } } } else { /*pi->caps_vce_pg*/ + pi->vce_power_gated = gate; cz_update_vce_dpm(adev); cz_enable_vce_dpm(adev, !gate); } diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index 8227344d2..c1b04e9aa 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -2667,19 +2667,21 @@ static void dce_v10_0_cursor_reset(struct drm_crtc *crtc) } } -static void dce_v10_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, - u16 *blue, uint32_t start, uint32_t size) +static int dce_v10_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, + u16 *blue, uint32_t size) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - int end = (start + size > 256) ? 256 : start + size, i; + int i; /* userspace palettes are always correct as is */ - for (i = start; i < end; i++) { + for (i = 0; i < size; i++) { amdgpu_crtc->lut_r[i] = red[i] >> 6; amdgpu_crtc->lut_g[i] = green[i] >> 6; amdgpu_crtc->lut_b[i] = blue[i] >> 6; } dce_v10_0_crtc_load_lut(crtc); + + return 0; } static void dce_v10_0_crtc_destroy(struct drm_crtc *crtc) @@ -2717,13 +2719,13 @@ static void dce_v10_0_crtc_dpms(struct drm_crtc *crtc, int mode) type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id); amdgpu_irq_update(adev, &adev->crtc_irq, type); amdgpu_irq_update(adev, &adev->pageflip_irq, type); - drm_vblank_on(dev, amdgpu_crtc->crtc_id); + drm_crtc_vblank_on(crtc); dce_v10_0_crtc_load_lut(crtc); break; case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: case DRM_MODE_DPMS_OFF: - drm_vblank_off(dev, amdgpu_crtc->crtc_id); + drm_crtc_vblank_off(crtc); if (amdgpu_crtc->enabled) { dce_v10_0_vga_enable(crtc, true); amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE); @@ -3372,7 +3374,7 @@ static int dce_v10_0_pageflip_irq(struct amdgpu_device *adev, spin_unlock_irqrestore(&adev->ddev->event_lock, flags); - drm_vblank_put(adev->ddev, amdgpu_crtc->crtc_id); + drm_crtc_vblank_put(&amdgpu_crtc->base); schedule_work(&works->unpin_work); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index af26ec0bc..d4bf13390 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -307,11 +307,10 @@ static void dce_v11_0_page_flip(struct amdgpu_device *adev, struct amdgpu_crtc *amdgpu_crtc = adev->mode_info.crtcs[crtc_id]; u32 tmp; - /* flip at hsync for async, default is vsync */ - /* use UPDATE_IMMEDIATE_EN instead for async? */ + /* flip immediate for async, default is vsync */ tmp = RREG32(mmGRPH_FLIP_CONTROL + amdgpu_crtc->crtc_offset); tmp = REG_SET_FIELD(tmp, GRPH_FLIP_CONTROL, - GRPH_SURFACE_UPDATE_H_RETRACE_EN, async ? 1 : 0); + GRPH_SURFACE_UPDATE_IMMEDIATE_EN, async ? 1 : 0); WREG32(mmGRPH_FLIP_CONTROL + amdgpu_crtc->crtc_offset, tmp); /* update the scanout addresses */ WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset, @@ -2678,19 +2677,21 @@ static void dce_v11_0_cursor_reset(struct drm_crtc *crtc) } } -static void dce_v11_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, - u16 *blue, uint32_t start, uint32_t size) +static int dce_v11_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, + u16 *blue, uint32_t size) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - int end = (start + size > 256) ? 256 : start + size, i; + int i; /* userspace palettes are always correct as is */ - for (i = start; i < end; i++) { + for (i = 0; i < size; i++) { amdgpu_crtc->lut_r[i] = red[i] >> 6; amdgpu_crtc->lut_g[i] = green[i] >> 6; amdgpu_crtc->lut_b[i] = blue[i] >> 6; } dce_v11_0_crtc_load_lut(crtc); + + return 0; } static void dce_v11_0_crtc_destroy(struct drm_crtc *crtc) @@ -2728,13 +2729,13 @@ static void dce_v11_0_crtc_dpms(struct drm_crtc *crtc, int mode) type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id); amdgpu_irq_update(adev, &adev->crtc_irq, type); amdgpu_irq_update(adev, &adev->pageflip_irq, type); - drm_vblank_on(dev, amdgpu_crtc->crtc_id); + drm_crtc_vblank_on(crtc); dce_v11_0_crtc_load_lut(crtc); break; case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: case DRM_MODE_DPMS_OFF: - drm_vblank_off(dev, amdgpu_crtc->crtc_id); + drm_crtc_vblank_off(crtc); if (amdgpu_crtc->enabled) { dce_v11_0_vga_enable(crtc, true); amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE); @@ -3433,7 +3434,7 @@ static int dce_v11_0_pageflip_irq(struct amdgpu_device *adev, spin_unlock_irqrestore(&adev->ddev->event_lock, flags); - drm_vblank_put(adev->ddev, amdgpu_crtc->crtc_id); + drm_crtc_vblank_put(&amdgpu_crtc->base); schedule_work(&works->unpin_work); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index 3fb65e41a..4fdfab1e9 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -526,36 +526,16 @@ static void dce_v8_0_stop_mc_access(struct amdgpu_device *adev, crtc_enabled = REG_GET_FIELD(RREG32(mmCRTC_CONTROL + crtc_offsets[i]), CRTC_CONTROL, CRTC_MASTER_EN); if (crtc_enabled) { -#if 0 - u32 frame_count; - int j; - +#if 1 save->crtc_enabled[i] = true; tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]); if (REG_GET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN) == 0) { - amdgpu_display_vblank_wait(adev, i); - WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1); + /*it is correct only for RGB ; black is 0*/ + WREG32(mmCRTC_BLANK_DATA_COLOR + crtc_offsets[i], 0); tmp = REG_SET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 1); WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp); - WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0); - } - /* wait for the next frame */ - frame_count = amdgpu_display_vblank_get_counter(adev, i); - for (j = 0; j < adev->usec_timeout; j++) { - if (amdgpu_display_vblank_get_counter(adev, i) != frame_count) - break; - udelay(1); - } - tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]); - if (REG_GET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK) == 0) { - tmp = REG_SET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK, 1); - WREG32(mmGRPH_UPDATE + crtc_offsets[i], tmp); - } - tmp = RREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i]); - if (REG_GET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK) == 0) { - tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK, 1); - WREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i], tmp); } + mdelay(20); #else /* XXX this is a hack to avoid strange behavior with EFI on certain systems */ WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1); @@ -575,55 +555,22 @@ static void dce_v8_0_stop_mc_access(struct amdgpu_device *adev, static void dce_v8_0_resume_mc_access(struct amdgpu_device *adev, struct amdgpu_mode_mc_save *save) { - u32 tmp, frame_count; - int i, j; + u32 tmp; + int i; /* update crtc base addresses */ for (i = 0; i < adev->mode_info.num_crtc; i++) { WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], upper_32_bits(adev->mc.vram_start)); - WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], - upper_32_bits(adev->mc.vram_start)); WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i], (u32)adev->mc.vram_start); - WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i], - (u32)adev->mc.vram_start); if (save->crtc_enabled[i]) { - tmp = RREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i]); - if (REG_GET_FIELD(tmp, MASTER_UPDATE_MODE, MASTER_UPDATE_MODE) != 3) { - tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_MODE, MASTER_UPDATE_MODE, 3); - WREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i], tmp); - } - tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]); - if (REG_GET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK)) { - tmp = REG_SET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK, 0); - WREG32(mmGRPH_UPDATE + crtc_offsets[i], tmp); - } - tmp = RREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i]); - if (REG_GET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK)) { - tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK, 0); - WREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i], tmp); - } - for (j = 0; j < adev->usec_timeout; j++) { - tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]); - if (REG_GET_FIELD(tmp, GRPH_UPDATE, GRPH_SURFACE_UPDATE_PENDING) == 0) - break; - udelay(1); - } tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]); tmp = REG_SET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 0); - WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1); WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp); - WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0); - /* wait for the next frame */ - frame_count = amdgpu_display_vblank_get_counter(adev, i); - for (j = 0; j < adev->usec_timeout; j++) { - if (amdgpu_display_vblank_get_counter(adev, i) != frame_count) - break; - udelay(1); - } } + mdelay(20); } WREG32(mmVGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(adev->mc.vram_start)); @@ -2574,19 +2521,21 @@ static void dce_v8_0_cursor_reset(struct drm_crtc *crtc) } } -static void dce_v8_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, - u16 *blue, uint32_t start, uint32_t size) +static int dce_v8_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, + u16 *blue, uint32_t size) { struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); - int end = (start + size > 256) ? 256 : start + size, i; + int i; /* userspace palettes are always correct as is */ - for (i = start; i < end; i++) { + for (i = 0; i < size; i++) { amdgpu_crtc->lut_r[i] = red[i] >> 6; amdgpu_crtc->lut_g[i] = green[i] >> 6; amdgpu_crtc->lut_b[i] = blue[i] >> 6; } dce_v8_0_crtc_load_lut(crtc); + + return 0; } static void dce_v8_0_crtc_destroy(struct drm_crtc *crtc) @@ -2624,13 +2573,13 @@ static void dce_v8_0_crtc_dpms(struct drm_crtc *crtc, int mode) type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id); amdgpu_irq_update(adev, &adev->crtc_irq, type); amdgpu_irq_update(adev, &adev->pageflip_irq, type); - drm_vblank_on(dev, amdgpu_crtc->crtc_id); + drm_crtc_vblank_on(crtc); dce_v8_0_crtc_load_lut(crtc); break; case DRM_MODE_DPMS_STANDBY: case DRM_MODE_DPMS_SUSPEND: case DRM_MODE_DPMS_OFF: - drm_vblank_off(dev, amdgpu_crtc->crtc_id); + drm_crtc_vblank_off(crtc); if (amdgpu_crtc->enabled) { dce_v8_0_vga_enable(crtc, true); amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE); @@ -3376,7 +3325,7 @@ static int dce_v8_0_pageflip_irq(struct amdgpu_device *adev, spin_unlock_irqrestore(&adev->ddev->event_lock, flags); - drm_vblank_put(adev->ddev, amdgpu_crtc->crtc_id); + drm_crtc_vblank_put(&amdgpu_crtc->base); schedule_work(&works->unpin_work); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_smc.c b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c index b336c918d..b3e19ba4c 100644 --- a/drivers/gpu/drm/amd/amdgpu/fiji_smc.c +++ b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c @@ -173,7 +173,7 @@ static int fiji_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg) { if (!fiji_is_smc_ram_running(adev)) { - return -EINVAL;; + return -EINVAL; } if (wait_smu_response(adev)) { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 6d34dcd60..754566f51 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -1554,9 +1554,15 @@ static void gfx_v7_0_tiling_mode_table_init(struct amdgpu_device *adev) * registers are instanced per SE or SH. 0xffffffff means * broadcast to all SEs or SHs (CIK). */ -void gfx_v7_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num) +static void gfx_v7_0_select_se_sh(struct amdgpu_device *adev, + u32 se_num, u32 sh_num, u32 instance) { - u32 data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK; + u32 data; + + if (instance == 0xffffffff) + data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); + else + data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | @@ -1630,13 +1636,13 @@ static void gfx_v7_0_setup_rb(struct amdgpu_device *adev) mutex_lock(&adev->grbm_idx_mutex); for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { - gfx_v7_0_select_se_sh(adev, i, j); + gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff); data = gfx_v7_0_get_rb_active_bitmap(adev); active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * rb_bitmap_width_per_sh); } } - gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); adev->gfx.config.backend_enable_mask = active_rbs; @@ -1717,7 +1723,7 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev) * making sure that the following register writes will be broadcasted * to all the shaders */ - gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ @@ -2021,17 +2027,6 @@ static void gfx_v7_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, unsigned vm_id, bool ctx_switch) { u32 header, control = 0; - u32 next_rptr = ring->wptr + 5; - - if (ctx_switch) - next_rptr += 2; - - next_rptr += 4; - amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); - amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); - amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); - amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); - amdgpu_ring_write(ring, next_rptr); /* insert SWITCH_BUFFER packet before first IB in the ring frame */ if (ctx_switch) { @@ -2060,22 +2055,9 @@ static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring, struct amdgpu_ib *ib, unsigned vm_id, bool ctx_switch) { - u32 header, control = 0; - u32 next_rptr = ring->wptr + 5; - - control |= INDIRECT_BUFFER_VALID; - next_rptr += 4; - amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); - amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); - amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); - amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); - amdgpu_ring_write(ring, next_rptr); - - header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); + u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vm_id << 24); - control |= ib->length_dw | (vm_id << 24); - - amdgpu_ring_write(ring, header); + amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -2094,26 +2076,25 @@ static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring, * Provides a basic gfx ring test to verify that IBs are working. * Returns 0 on success, error on failure. */ -static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring) +static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; struct fence *f = NULL; uint32_t scratch; uint32_t tmp = 0; - unsigned i; - int r; + long r; r = amdgpu_gfx_scratch_get(adev, &scratch); if (r) { - DRM_ERROR("amdgpu: failed to get scratch reg (%d).\n", r); + DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r); return r; } WREG32(scratch, 0xCAFEDEAD); memset(&ib, 0, sizeof(ib)); r = amdgpu_ib_get(adev, NULL, 256, &ib); if (r) { - DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); + DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r); goto err1; } ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); @@ -2125,21 +2106,19 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring) if (r) goto err2; - r = fence_wait(f, false); - if (r) { - DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); + r = fence_wait_timeout(f, false, timeout); + if (r == 0) { + DRM_ERROR("amdgpu: IB test timed out\n"); + r = -ETIMEDOUT; goto err2; - } - for (i = 0; i < adev->usec_timeout; i++) { - tmp = RREG32(scratch); - if (tmp == 0xDEADBEEF) - break; - DRM_UDELAY(1); - } - if (i < adev->usec_timeout) { - DRM_INFO("ib test on ring %d succeeded in %u usecs\n", - ring->idx, i); + } else if (r < 0) { + DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); goto err2; + } + tmp = RREG32(scratch); + if (tmp == 0xDEADBEEF) { + DRM_INFO("ib test on ring %d succeeded\n", ring->idx); + r = 0; } else { DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n", scratch, tmp); @@ -2147,7 +2126,6 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring) } err2: - fence_put(f); amdgpu_ib_free(adev, &ib, NULL); fence_put(f); err1: @@ -3198,7 +3176,8 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev) } } adev->gfx.rlc.cs_data = ci_cs_data; - adev->gfx.rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4; + adev->gfx.rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */ + adev->gfx.rlc.cp_table_size += 64 * 1024; /* GDS */ src_ptr = adev->gfx.rlc.reg_list; dws = adev->gfx.rlc.reg_list_size; @@ -3356,7 +3335,7 @@ static void gfx_v7_0_wait_for_rlc_serdes(struct amdgpu_device *adev) mutex_lock(&adev->grbm_idx_mutex); for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { - gfx_v7_0_select_se_sh(adev, i, j); + gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff); for (k = 0; k < adev->usec_timeout; k++) { if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0) break; @@ -3364,7 +3343,7 @@ static void gfx_v7_0_wait_for_rlc_serdes(struct amdgpu_device *adev) } } } - gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | @@ -3411,7 +3390,7 @@ static u32 gfx_v7_0_halt_rlc(struct amdgpu_device *adev) return orig; } -void gfx_v7_0_enter_rlc_safe_mode(struct amdgpu_device *adev) +static void gfx_v7_0_enter_rlc_safe_mode(struct amdgpu_device *adev) { u32 tmp, i, mask; @@ -3433,7 +3412,7 @@ void gfx_v7_0_enter_rlc_safe_mode(struct amdgpu_device *adev) } } -void gfx_v7_0_exit_rlc_safe_mode(struct amdgpu_device *adev) +static void gfx_v7_0_exit_rlc_safe_mode(struct amdgpu_device *adev) { u32 tmp; @@ -3448,7 +3427,7 @@ void gfx_v7_0_exit_rlc_safe_mode(struct amdgpu_device *adev) * * Halt the RLC ME (MicroEngine) (CIK). */ -void gfx_v7_0_rlc_stop(struct amdgpu_device *adev) +static void gfx_v7_0_rlc_stop(struct amdgpu_device *adev) { WREG32(mmRLC_CNTL, 0); @@ -3524,7 +3503,7 @@ static int gfx_v7_0_rlc_resume(struct amdgpu_device *adev) WREG32(mmRLC_LB_CNTR_MAX, 0x00008000); mutex_lock(&adev->grbm_idx_mutex); - gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); WREG32(mmRLC_LB_INIT_CU_MASK, 0xffffffff); WREG32(mmRLC_LB_PARAMS, 0x00600408); WREG32(mmRLC_LB_CNTL, 0x80000004); @@ -3564,7 +3543,7 @@ static void gfx_v7_0_enable_cgcg(struct amdgpu_device *adev, bool enable) tmp = gfx_v7_0_halt_rlc(adev); mutex_lock(&adev->grbm_idx_mutex); - gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); tmp2 = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK | @@ -3615,7 +3594,7 @@ static void gfx_v7_0_enable_mgcg(struct amdgpu_device *adev, bool enable) tmp = gfx_v7_0_halt_rlc(adev); mutex_lock(&adev->grbm_idx_mutex); - gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); data = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK | @@ -3666,7 +3645,7 @@ static void gfx_v7_0_enable_mgcg(struct amdgpu_device *adev, bool enable) tmp = gfx_v7_0_halt_rlc(adev); mutex_lock(&adev->grbm_idx_mutex); - gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); data = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK | RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_1_MASK; @@ -3844,6 +3823,20 @@ static void gfx_v7_0_enable_gfx_cgpg(struct amdgpu_device *adev, } } +static void gfx_v7_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, + u32 bitmap) +{ + u32 data; + + if (!bitmap) + return; + + data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; + data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; + + WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data); +} + static u32 gfx_v7_0_get_cu_active_bitmap(struct amdgpu_device *adev) { u32 data, mask; @@ -4100,7 +4093,7 @@ static void gfx_v7_0_fini_pg(struct amdgpu_device *adev) * Fetches a GPU clock counter snapshot (SI). * Returns the 64 bit clock counter snapshot. */ -uint64_t gfx_v7_0_get_gpu_clock_counter(struct amdgpu_device *adev) +static uint64_t gfx_v7_0_get_gpu_clock_counter(struct amdgpu_device *adev) { uint64_t clock; @@ -4160,12 +4153,24 @@ static void gfx_v7_0_ring_emit_gds_switch(struct amdgpu_ring *ring, amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); } +static const struct amdgpu_gfx_funcs gfx_v7_0_gfx_funcs = { + .get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter, + .select_se_sh = &gfx_v7_0_select_se_sh, +}; + +static const struct amdgpu_rlc_funcs gfx_v7_0_rlc_funcs = { + .enter_safe_mode = gfx_v7_0_enter_rlc_safe_mode, + .exit_safe_mode = gfx_v7_0_exit_rlc_safe_mode +}; + static int gfx_v7_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; adev->gfx.num_gfx_rings = GFX7_NUM_GFX_RINGS; adev->gfx.num_compute_rings = GFX7_NUM_COMPUTE_RINGS; + adev->gfx.funcs = &gfx_v7_0_gfx_funcs; + adev->gfx.rlc.funcs = &gfx_v7_0_rlc_funcs; gfx_v7_0_set_ring_funcs(adev); gfx_v7_0_set_irq_funcs(adev); gfx_v7_0_set_gds_init(adev); @@ -5009,16 +5014,22 @@ static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev) int i, j, k, counter, active_cu_number = 0; u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; + unsigned disable_masks[4 * 2]; memset(cu_info, 0, sizeof(*cu_info)); + amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2); + mutex_lock(&adev->grbm_idx_mutex); for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { mask = 1; ao_bitmap = 0; counter = 0; - gfx_v7_0_select_se_sh(adev, i, j); + gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff); + if (i < 4 && j < 2) + gfx_v7_0_set_user_cu_inactive_bitmap( + adev, disable_masks[i * 2 + j]); bitmap = gfx_v7_0_get_cu_active_bitmap(adev); cu_info->bitmap[i][j] = bitmap; @@ -5034,7 +5045,7 @@ static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev) ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); } } - gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); cu_info->number = active_cu_number; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h index e747aa935..94e3ea147 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h @@ -26,11 +26,4 @@ extern const struct amd_ip_funcs gfx_v7_0_ip_funcs; -/* XXX these shouldn't be exported */ -void gfx_v7_0_enter_rlc_safe_mode(struct amdgpu_device *adev); -void gfx_v7_0_exit_rlc_safe_mode(struct amdgpu_device *adev); -void gfx_v7_0_rlc_stop(struct amdgpu_device *adev); -uint64_t gfx_v7_0_get_gpu_clock_counter(struct amdgpu_device *adev); -void gfx_v7_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num); - #endif diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 1127b2b62..1c8f2754f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -225,7 +225,8 @@ static const u32 tonga_mgcg_cgcg_init[] = static const u32 golden_settings_polaris11_a11[] = { - mmCB_HW_CONTROL, 0xfffdf3cf, 0x00006208, + mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208, + mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000, mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, mmDB_DEBUG2, 0xf00fffff, 0x00000400, mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, @@ -234,7 +235,7 @@ static const u32 golden_settings_polaris11_a11[] = mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000, mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, - mmSQ_CONFIG, 0x07f80000, 0x07180000, + mmSQ_CONFIG, 0x07f80000, 0x01180000, mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, mmTCC_CTRL, 0x00100000, 0xf31fff7f, mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3, @@ -256,8 +257,8 @@ static const u32 polaris11_golden_common_all[] = static const u32 golden_settings_polaris10_a11[] = { mmATC_MISC_CG, 0x000c0fc0, 0x000c0200, - mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208, - mmCB_HW_CONTROL_2, 0, 0x0f000000, + mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208, + mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000, mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, mmDB_DEBUG2, 0xf00fffff, 0x00000400, mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, @@ -364,6 +365,7 @@ static const u32 golden_settings_iceland_a11[] = mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002, mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000, + mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, mmTCC_CTRL, 0x00100000, 0xf31fff7f, @@ -460,8 +462,10 @@ static const u32 cz_golden_settings_a11[] = mmGB_GPU_ID, 0x0000000f, 0x00000000, mmPA_SC_ENHANCE, 0xffffffff, 0x00000001, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, + mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, mmTA_CNTL_AUX, 0x000f000f, 0x00010000, + mmTCC_CTRL, 0x00100000, 0xf31fff7f, mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302 @@ -742,26 +746,25 @@ static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring) return r; } -static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring) +static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; struct fence *f = NULL; uint32_t scratch; uint32_t tmp = 0; - unsigned i; - int r; + long r; r = amdgpu_gfx_scratch_get(adev, &scratch); if (r) { - DRM_ERROR("amdgpu: failed to get scratch reg (%d).\n", r); + DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r); return r; } WREG32(scratch, 0xCAFEDEAD); memset(&ib, 0, sizeof(ib)); r = amdgpu_ib_get(adev, NULL, 256, &ib); if (r) { - DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); + DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r); goto err1; } ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); @@ -773,28 +776,25 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring) if (r) goto err2; - r = fence_wait(f, false); - if (r) { - DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); + r = fence_wait_timeout(f, false, timeout); + if (r == 0) { + DRM_ERROR("amdgpu: IB test timed out.\n"); + r = -ETIMEDOUT; goto err2; - } - for (i = 0; i < adev->usec_timeout; i++) { - tmp = RREG32(scratch); - if (tmp == 0xDEADBEEF) - break; - DRM_UDELAY(1); - } - if (i < adev->usec_timeout) { - DRM_INFO("ib test on ring %d succeeded in %u usecs\n", - ring->idx, i); + } else if (r < 0) { + DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); goto err2; + } + tmp = RREG32(scratch); + if (tmp == 0xDEADBEEF) { + DRM_INFO("ib test on ring %d succeeded\n", ring->idx); + r = 0; } else { DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n", scratch, tmp); r = -EINVAL; } err2: - fence_put(f); amdgpu_ib_free(adev, &ib, NULL); fence_put(f); err1: @@ -1115,6 +1115,71 @@ static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev, buffer[count++] = cpu_to_le32(0); } +static void cz_init_cp_jump_table(struct amdgpu_device *adev) +{ + const __le32 *fw_data; + volatile u32 *dst_ptr; + int me, i, max_me = 4; + u32 bo_offset = 0; + u32 table_offset, table_size; + + if (adev->asic_type == CHIP_CARRIZO) + max_me = 5; + + /* write the cp table buffer */ + dst_ptr = adev->gfx.rlc.cp_table_ptr; + for (me = 0; me < max_me; me++) { + if (me == 0) { + const struct gfx_firmware_header_v1_0 *hdr = + (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; + fw_data = (const __le32 *) + (adev->gfx.ce_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 1) { + const struct gfx_firmware_header_v1_0 *hdr = + (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; + fw_data = (const __le32 *) + (adev->gfx.pfp_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 2) { + const struct gfx_firmware_header_v1_0 *hdr = + (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; + fw_data = (const __le32 *) + (adev->gfx.me_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 3) { + const struct gfx_firmware_header_v1_0 *hdr = + (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; + fw_data = (const __le32 *) + (adev->gfx.mec_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } else if (me == 4) { + const struct gfx_firmware_header_v1_0 *hdr = + (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data; + fw_data = (const __le32 *) + (adev->gfx.mec2_fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + table_offset = le32_to_cpu(hdr->jt_offset); + table_size = le32_to_cpu(hdr->jt_size); + } + + for (i = 0; i < table_size; i ++) { + dst_ptr[bo_offset + i] = + cpu_to_le32(le32_to_cpu(fw_data[table_offset + i])); + } + + bo_offset += table_size; + } +} + static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev) { int r; @@ -1130,6 +1195,18 @@ static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev) amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); adev->gfx.rlc.clear_state_obj = NULL; } + + /* jump table block */ + if (adev->gfx.rlc.cp_table_obj) { + r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false); + if (unlikely(r != 0)) + dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r); + amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj); + amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj); + + amdgpu_bo_unref(&adev->gfx.rlc.cp_table_obj); + adev->gfx.rlc.cp_table_obj = NULL; + } } static int gfx_v8_0_rlc_init(struct amdgpu_device *adev) @@ -1186,6 +1263,46 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev) amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj); } + if ((adev->asic_type == CHIP_CARRIZO) || + (adev->asic_type == CHIP_STONEY)) { + adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ + if (adev->gfx.rlc.cp_table_obj == NULL) { + r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true, + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, NULL, + &adev->gfx.rlc.cp_table_obj); + if (r) { + dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r); + return r; + } + } + + r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false); + if (unlikely(r != 0)) { + dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r); + return r; + } + r = amdgpu_bo_pin(adev->gfx.rlc.cp_table_obj, AMDGPU_GEM_DOMAIN_VRAM, + &adev->gfx.rlc.cp_table_gpu_addr); + if (r) { + amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj); + dev_warn(adev->dev, "(%d) pin RLC cp_table bo failed\n", r); + return r; + } + r = amdgpu_bo_kmap(adev->gfx.rlc.cp_table_obj, (void **)&adev->gfx.rlc.cp_table_ptr); + if (r) { + dev_warn(adev->dev, "(%d) map RLC cp table bo failed\n", r); + return r; + } + + cz_init_cp_jump_table(adev); + + amdgpu_bo_kunmap(adev->gfx.rlc.cp_table_obj); + amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj); + + } + return 0; } @@ -1567,7 +1684,6 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) RREG32(sec_ded_counter_registers[i]); fail: - fence_put(f); amdgpu_ib_free(adev, &ib, NULL); fence_put(f); @@ -3294,9 +3410,15 @@ static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev) } } -void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num) +static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, + u32 se_num, u32 sh_num, u32 instance) { - u32 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); + u32 data; + + if (instance == 0xffffffff) + data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); + else + data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) { data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); @@ -3346,13 +3468,13 @@ static void gfx_v8_0_setup_rb(struct amdgpu_device *adev) mutex_lock(&adev->grbm_idx_mutex); for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { - gfx_v8_0_select_se_sh(adev, i, j); + gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff); data = gfx_v8_0_get_rb_active_bitmap(adev); active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * rb_bitmap_width_per_sh); } } - gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); adev->gfx.config.backend_enable_mask = active_rbs; @@ -3456,7 +3578,7 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev) * making sure that the following register writes will be broadcasted * to all the shaders */ - gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); WREG32(mmPA_SC_FIFO_SIZE, (adev->gfx.config.sc_prim_fifo_size_frontend << @@ -3479,7 +3601,7 @@ static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev) mutex_lock(&adev->grbm_idx_mutex); for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { - gfx_v8_0_select_se_sh(adev, i, j); + gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff); for (k = 0; k < adev->usec_timeout; k++) { if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0) break; @@ -3487,7 +3609,7 @@ static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev) } } } - gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | @@ -3648,13 +3770,13 @@ static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev) WREG32(mmRLC_SRM_CNTL, data); } -static void polaris11_init_power_gating(struct amdgpu_device *adev) +static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev) { uint32_t data; if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | - AMD_PG_SUPPORT_GFX_SMG | - AMD_PG_SUPPORT_GFX_DMG)) { + AMD_PG_SUPPORT_GFX_SMG | + AMD_PG_SUPPORT_GFX_DMG)) { data = RREG32(mmCP_RB_WPTR_POLL_CNTL); data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); @@ -3679,6 +3801,53 @@ static void polaris11_init_power_gating(struct amdgpu_device *adev) } } +static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, + bool enable) +{ + u32 data, orig; + + orig = data = RREG32(mmRLC_PG_CNTL); + + if (enable) + data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK; + else + data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK; + + if (orig != data) + WREG32(mmRLC_PG_CNTL, data); +} + +static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, + bool enable) +{ + u32 data, orig; + + orig = data = RREG32(mmRLC_PG_CNTL); + + if (enable) + data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK; + else + data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK; + + if (orig != data) + WREG32(mmRLC_PG_CNTL, data); +} + +static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable) +{ + u32 data, orig; + + orig = data = RREG32(mmRLC_PG_CNTL); + + if (enable) + data &= ~RLC_PG_CNTL__CP_PG_DISABLE_MASK; + else + data |= RLC_PG_CNTL__CP_PG_DISABLE_MASK; + + if (orig != data) + WREG32(mmRLC_PG_CNTL, data); +} + static void gfx_v8_0_init_pg(struct amdgpu_device *adev) { if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | @@ -3691,8 +3860,25 @@ static void gfx_v8_0_init_pg(struct amdgpu_device *adev) gfx_v8_0_init_save_restore_list(adev); gfx_v8_0_enable_save_restore_machine(adev); - if (adev->asic_type == CHIP_POLARIS11) - polaris11_init_power_gating(adev); + if ((adev->asic_type == CHIP_CARRIZO) || + (adev->asic_type == CHIP_STONEY)) { + WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8); + gfx_v8_0_init_power_gating(adev); + WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask); + if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { + cz_enable_sck_slow_down_on_power_up(adev, true); + cz_enable_sck_slow_down_on_power_down(adev, true); + } else { + cz_enable_sck_slow_down_on_power_up(adev, false); + cz_enable_sck_slow_down_on_power_down(adev, false); + } + if (adev->pg_flags & AMD_PG_SUPPORT_CP) + cz_enable_cp_power_gating(adev, true); + else + cz_enable_cp_power_gating(adev, false); + } else if (adev->asic_type == CHIP_POLARIS11) { + gfx_v8_0_init_power_gating(adev); + } } } @@ -4931,7 +5117,7 @@ static int gfx_v8_0_soft_reset(void *handle) * Fetches a GPU clock counter snapshot. * Returns the 64 bit clock counter snapshot. */ -uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev) +static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev) { uint64_t clock; @@ -4991,12 +5177,18 @@ static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring, amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); } +static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = { + .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter, + .select_se_sh = &gfx_v8_0_select_se_sh, +}; + static int gfx_v8_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS; adev->gfx.num_compute_rings = GFX8_NUM_COMPUTE_RINGS; + adev->gfx.funcs = &gfx_v8_0_gfx_funcs; gfx_v8_0_set_ring_funcs(adev); gfx_v8_0_set_irq_funcs(adev); gfx_v8_0_set_gds_init(adev); @@ -5029,51 +5221,43 @@ static int gfx_v8_0_late_init(void *handle) return 0; } -static void polaris11_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, - bool enable) +static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, + bool enable) { uint32_t data, temp; - /* Send msg to SMU via Powerplay */ - amdgpu_set_powergating_state(adev, - AMD_IP_BLOCK_TYPE_SMC, - enable ? AMD_PG_STATE_GATE : AMD_PG_STATE_UNGATE); + if (adev->asic_type == CHIP_POLARIS11) + /* Send msg to SMU via Powerplay */ + amdgpu_set_powergating_state(adev, + AMD_IP_BLOCK_TYPE_SMC, + enable ? + AMD_PG_STATE_GATE : AMD_PG_STATE_UNGATE); - if (enable) { - /* Enable static MGPG */ - temp = data = RREG32(mmRLC_PG_CNTL); + temp = data = RREG32(mmRLC_PG_CNTL); + /* Enable static MGPG */ + if (enable) data |= RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK; - - if (temp != data) - WREG32(mmRLC_PG_CNTL, data); - } else { - temp = data = RREG32(mmRLC_PG_CNTL); + else data &= ~RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK; - if (temp != data) - WREG32(mmRLC_PG_CNTL, data); - } + if (temp != data) + WREG32(mmRLC_PG_CNTL, data); } -static void polaris11_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, - bool enable) +static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, + bool enable) { uint32_t data, temp; - if (enable) { - /* Enable dynamic MGPG */ - temp = data = RREG32(mmRLC_PG_CNTL); + temp = data = RREG32(mmRLC_PG_CNTL); + /* Enable dynamic MGPG */ + if (enable) data |= RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK; - - if (temp != data) - WREG32(mmRLC_PG_CNTL, data); - } else { - temp = data = RREG32(mmRLC_PG_CNTL); + else data &= ~RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK; - if (temp != data) - WREG32(mmRLC_PG_CNTL, data); - } + if (temp != data) + WREG32(mmRLC_PG_CNTL, data); } static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev, @@ -5081,19 +5265,63 @@ static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *ade { uint32_t data, temp; - if (enable) { - /* Enable quick PG */ - temp = data = RREG32(mmRLC_PG_CNTL); - data |= 0x100000; + temp = data = RREG32(mmRLC_PG_CNTL); + /* Enable quick PG */ + if (enable) + data |= RLC_PG_CNTL__QUICK_PG_ENABLE_MASK; + else + data &= ~RLC_PG_CNTL__QUICK_PG_ENABLE_MASK; - if (temp != data) - WREG32(mmRLC_PG_CNTL, data); - } else { - temp = data = RREG32(mmRLC_PG_CNTL); - data &= ~0x100000; + if (temp != data) + WREG32(mmRLC_PG_CNTL, data); +} - if (temp != data) - WREG32(mmRLC_PG_CNTL, data); +static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev, + bool enable) +{ + u32 data, orig; + + orig = data = RREG32(mmRLC_PG_CNTL); + + if (enable) + data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; + else + data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; + + if (orig != data) + WREG32(mmRLC_PG_CNTL, data); +} + +static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev, + bool enable) +{ + u32 data, orig; + + orig = data = RREG32(mmRLC_PG_CNTL); + + if (enable) + data |= RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK; + else + data &= ~RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK; + + if (orig != data) + WREG32(mmRLC_PG_CNTL, data); + + /* Read any GFX register to wake up GFX. */ + if (!enable) + data = RREG32(mmDB_RENDER_CONTROL); +} + +static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev, + bool enable) +{ + if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { + cz_enable_gfx_cg_power_gating(adev, true); + if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) + cz_enable_gfx_pipeline_power_gating(adev, true); + } else { + cz_enable_gfx_cg_power_gating(adev, false); + cz_enable_gfx_pipeline_power_gating(adev, false); } } @@ -5101,21 +5329,42 @@ static int gfx_v8_0_set_powergating_state(void *handle, enum amd_powergating_state state) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + bool enable = (state == AMD_PG_STATE_GATE) ? true : false; if (!(adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) return 0; switch (adev->asic_type) { + case CHIP_CARRIZO: + case CHIP_STONEY: + if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) + cz_update_gfx_cg_power_gating(adev, enable); + + if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) + gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true); + else + gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false); + + if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) + gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true); + else + gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false); + break; case CHIP_POLARIS11: - if (adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) - polaris11_enable_gfx_static_mg_power_gating(adev, - state == AMD_PG_STATE_GATE ? true : false); - else if (adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) - polaris11_enable_gfx_dynamic_mg_power_gating(adev, - state == AMD_PG_STATE_GATE ? true : false); + if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) + gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true); + else + gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false); + + if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) + gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true); + else + gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false); + + if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable) + polaris11_enable_gfx_quick_mg_power_gating(adev, true); else - polaris11_enable_gfx_quick_mg_power_gating(adev, - state == AMD_PG_STATE_GATE ? true : false); + polaris11_enable_gfx_quick_mg_power_gating(adev, false); break; default: break; @@ -5129,7 +5378,7 @@ static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev, { uint32_t data; - gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); @@ -5517,6 +5766,8 @@ static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev WREG32(mmRLC_CGCG_CGLS_CTRL, data); } + gfx_v8_0_wait_for_rlc_serdes(adev); + adev->gfx.rlc.funcs->exit_safe_mode(adev); } static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev, @@ -5642,17 +5893,6 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, unsigned vm_id, bool ctx_switch) { u32 header, control = 0; - u32 next_rptr = ring->wptr + 5; - - if (ctx_switch) - next_rptr += 2; - - next_rptr += 4; - amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); - amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); - amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); - amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); - amdgpu_ring_write(ring, next_rptr); /* insert SWITCH_BUFFER packet before first IB in the ring frame */ if (ctx_switch) { @@ -5681,23 +5921,9 @@ static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring, struct amdgpu_ib *ib, unsigned vm_id, bool ctx_switch) { - u32 header, control = 0; - u32 next_rptr = ring->wptr + 5; - - control |= INDIRECT_BUFFER_VALID; - - next_rptr += 4; - amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); - amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); - amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); - amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); - amdgpu_ring_write(ring, next_rptr); + u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vm_id << 24); - header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); - - control |= ib->length_dw | (vm_id << 24); - - amdgpu_ring_write(ring, header); + amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -6150,9 +6376,9 @@ static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev) { switch (adev->asic_type) { case CHIP_TOPAZ: - case CHIP_STONEY: adev->gfx.rlc.funcs = &iceland_rlc_funcs; break; + case CHIP_STONEY: case CHIP_CARRIZO: adev->gfx.rlc.funcs = &cz_rlc_funcs; break; @@ -6190,6 +6416,20 @@ static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev) } } +static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, + u32 bitmap) +{ + u32 data; + + if (!bitmap) + return; + + data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; + data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; + + WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data); +} + static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev) { u32 data, mask; @@ -6210,16 +6450,22 @@ static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev) int i, j, k, counter, active_cu_number = 0; u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; + unsigned disable_masks[4 * 2]; memset(cu_info, 0, sizeof(*cu_info)); + amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2); + mutex_lock(&adev->grbm_idx_mutex); for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { mask = 1; ao_bitmap = 0; counter = 0; - gfx_v8_0_select_se_sh(adev, i, j); + gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff); + if (i < 4 && j < 2) + gfx_v8_0_set_user_cu_inactive_bitmap( + adev, disable_masks[i * 2 + j]); bitmap = gfx_v8_0_get_cu_active_bitmap(adev); cu_info->bitmap[i][j] = bitmap; @@ -6235,7 +6481,7 @@ static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev) ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); } } - gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); cu_info->number = active_cu_number; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h index 16a49f53a..bc82c7943 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h @@ -26,7 +26,6 @@ extern const struct amd_ip_funcs gfx_v8_0_ip_funcs; -uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev); void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index e05402728..c2b554c5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -39,6 +39,7 @@ static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev); static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev); +static int gmc_v7_0_wait_for_idle(void *handle); /*(DEBLOBBED)*/ @@ -71,39 +72,15 @@ static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev) } } -/** - * gmc7_mc_wait_for_idle - wait for MC idle callback. - * - * @adev: amdgpu_device pointer - * - * Wait for the MC (memory controller) to be idle. - * (evergreen+). - * Returns 0 if the MC is idle, -1 if not. - */ -int gmc_v7_0_mc_wait_for_idle(struct amdgpu_device *adev) -{ - unsigned i; - u32 tmp; - - for (i = 0; i < adev->usec_timeout; i++) { - /* read MC_STATUS */ - tmp = RREG32(mmSRBM_STATUS) & 0x1F00; - if (!tmp) - return 0; - udelay(1); - } - return -1; -} - -void gmc_v7_0_mc_stop(struct amdgpu_device *adev, - struct amdgpu_mode_mc_save *save) +static void gmc_v7_0_mc_stop(struct amdgpu_device *adev, + struct amdgpu_mode_mc_save *save) { u32 blackout; if (adev->mode_info.num_crtc) amdgpu_display_stop_mc_access(adev, save); - amdgpu_asic_wait_for_mc_idle(adev); + gmc_v7_0_wait_for_idle((void *)adev); blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL); if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) { @@ -118,8 +95,8 @@ void gmc_v7_0_mc_stop(struct amdgpu_device *adev, udelay(100); } -void gmc_v7_0_mc_resume(struct amdgpu_device *adev, - struct amdgpu_mode_mc_save *save) +static void gmc_v7_0_mc_resume(struct amdgpu_device *adev, + struct amdgpu_mode_mc_save *save) { u32 tmp; @@ -310,7 +287,7 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev) amdgpu_display_set_vga_render_state(adev, false); gmc_v7_0_mc_stop(adev, &save); - if (amdgpu_asic_wait_for_mc_idle(adev)) { + if (gmc_v7_0_wait_for_idle((void *)adev)) { dev_warn(adev->dev, "Wait for MC idle timedout !\n"); } /* Update configuration */ @@ -330,7 +307,7 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev) WREG32(mmMC_VM_AGP_BASE, 0); WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF); WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF); - if (amdgpu_asic_wait_for_mc_idle(adev)) { + if (gmc_v7_0_wait_for_idle((void *)adev)) { dev_warn(adev->dev, "Wait for MC idle timedout !\n"); } gmc_v7_0_mc_resume(adev, &save); @@ -1136,7 +1113,7 @@ static int gmc_v7_0_soft_reset(void *handle) if (srbm_soft_reset) { gmc_v7_0_mc_stop(adev, &save); - if (gmc_v7_0_wait_for_idle(adev)) { + if (gmc_v7_0_wait_for_idle((void *)adev)) { dev_warn(adev->dev, "Wait for GMC idle timed out !\n"); } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h index 36fcbbc46..0b386b5d2 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h @@ -26,11 +26,4 @@ extern const struct amd_ip_funcs gmc_v7_0_ip_funcs; -/* XXX these shouldn't be exported */ -void gmc_v7_0_mc_stop(struct amdgpu_device *adev, - struct amdgpu_mode_mc_save *save); -void gmc_v7_0_mc_resume(struct amdgpu_device *adev, - struct amdgpu_mode_mc_save *save); -int gmc_v7_0_mc_wait_for_idle(struct amdgpu_device *adev); - #endif diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index a7b6de8a1..34695e2e3 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -41,6 +41,7 @@ static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev); static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev); +static int gmc_v8_0_wait_for_idle(void *handle); /*(DEBLOBBED)*/ @@ -100,6 +101,11 @@ static const u32 stoney_mgcg_cgcg_init[] = mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 }; +static const u32 golden_settings_stoney_common[] = +{ + mmMC_HUB_RDREQ_UVD, MC_HUB_RDREQ_UVD__PRESCALE_MASK, 0x00000004, + mmMC_RD_GRP_OTH, MC_RD_GRP_OTH__UVD_MASK, 0x00600000 +}; static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev) { @@ -139,50 +145,24 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev) amdgpu_program_register_sequence(adev, stoney_mgcg_cgcg_init, (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init)); + amdgpu_program_register_sequence(adev, + golden_settings_stoney_common, + (const u32)ARRAY_SIZE(golden_settings_stoney_common)); break; default: break; } } -/** - * gmc8_mc_wait_for_idle - wait for MC idle callback. - * - * @adev: amdgpu_device pointer - * - * Wait for the MC (memory controller) to be idle. - * (evergreen+). - * Returns 0 if the MC is idle, -1 if not. - */ -int gmc_v8_0_mc_wait_for_idle(struct amdgpu_device *adev) -{ - unsigned i; - u32 tmp; - - for (i = 0; i < adev->usec_timeout; i++) { - /* read MC_STATUS */ - tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__VMC_BUSY_MASK | - SRBM_STATUS__MCB_BUSY_MASK | - SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK | - SRBM_STATUS__MCC_BUSY_MASK | - SRBM_STATUS__MCD_BUSY_MASK | - SRBM_STATUS__VMC1_BUSY_MASK); - if (!tmp) - return 0; - udelay(1); - } - return -1; -} - -void gmc_v8_0_mc_stop(struct amdgpu_device *adev, - struct amdgpu_mode_mc_save *save) +static void gmc_v8_0_mc_stop(struct amdgpu_device *adev, + struct amdgpu_mode_mc_save *save) { u32 blackout; if (adev->mode_info.num_crtc) amdgpu_display_stop_mc_access(adev, save); - amdgpu_asic_wait_for_mc_idle(adev); + gmc_v8_0_wait_for_idle(adev); blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL); if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) { @@ -197,8 +177,8 @@ void gmc_v8_0_mc_stop(struct amdgpu_device *adev, udelay(100); } -void gmc_v8_0_mc_resume(struct amdgpu_device *adev, - struct amdgpu_mode_mc_save *save) +static void gmc_v8_0_mc_resume(struct amdgpu_device *adev, + struct amdgpu_mode_mc_save *save) { u32 tmp; @@ -391,7 +371,7 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev) amdgpu_display_set_vga_render_state(adev, false); gmc_v8_0_mc_stop(adev, &save); - if (amdgpu_asic_wait_for_mc_idle(adev)) { + if (gmc_v8_0_wait_for_idle((void *)adev)) { dev_warn(adev->dev, "Wait for MC idle timedout !\n"); } /* Update configuration */ @@ -411,7 +391,7 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev) WREG32(mmMC_VM_AGP_BASE, 0); WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF); WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF); - if (amdgpu_asic_wait_for_mc_idle(adev)) { + if (gmc_v8_0_wait_for_idle((void *)adev)) { dev_warn(adev->dev, "Wait for MC idle timedout !\n"); } gmc_v8_0_mc_resume(adev, &save); @@ -1138,7 +1118,7 @@ static int gmc_v8_0_soft_reset(void *handle) if (srbm_soft_reset) { gmc_v8_0_mc_stop(adev, &save); - if (gmc_v8_0_wait_for_idle(adev)) { + if (gmc_v8_0_wait_for_idle((void *)adev)) { dev_warn(adev->dev, "Wait for GMC idle timed out !\n"); } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h index 973436086..fc5001a81 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h @@ -26,11 +26,4 @@ extern const struct amd_ip_funcs gmc_v8_0_ip_funcs; -/* XXX these shouldn't be exported */ -void gmc_v8_0_mc_stop(struct amdgpu_device *adev, - struct amdgpu_mode_mc_save *save); -void gmc_v8_0_mc_resume(struct amdgpu_device *adev, - struct amdgpu_mode_mc_save *save); -int gmc_v8_0_mc_wait_for_idle(struct amdgpu_device *adev); - #endif diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c b/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c index 571e37566..90b4bcc72 100644 --- a/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c @@ -24,7 +24,7 @@ #include #include "drmP.h" #include "amdgpu.h" -#include "iceland_smumgr.h" +#include "iceland_smum.h" /*(DEBLOBBED)*/ diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_smc.c b/drivers/gpu/drm/amd/amdgpu/iceland_smc.c index 52ee08193..211839913 100644 --- a/drivers/gpu/drm/amd/amdgpu/iceland_smc.c +++ b/drivers/gpu/drm/amd/amdgpu/iceland_smc.c @@ -25,7 +25,7 @@ #include "drmP.h" #include "amdgpu.h" #include "ppsmc.h" -#include "iceland_smumgr.h" +#include "iceland_smum.h" #include "smu_ucode_xfer_vi.h" #include "amdgpu_ucode.h" @@ -211,7 +211,7 @@ static int iceland_send_msg_to_smc_without_waiting(struct amdgpu_device *adev, PPSMC_Msg msg) { if (!iceland_is_smc_ram_running(adev)) - return -EINVAL;; + return -EINVAL; if (wait_smu_response(adev)) { DRM_ERROR("Failed to send previous message\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_smum.h b/drivers/gpu/drm/amd/amdgpu/iceland_smum.h new file mode 100644 index 000000000..5983e3150 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/iceland_smum.h @@ -0,0 +1,41 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef ICELAND_SMUM_H +#define ICELAND_SMUM_H + +#include "ppsmc.h" + +extern int iceland_smu_init(struct amdgpu_device *adev); +extern int iceland_smu_fini(struct amdgpu_device *adev); +extern int iceland_smu_start(struct amdgpu_device *adev); + +struct iceland_smu_private_data +{ + uint8_t *header; + uint8_t *mec_image; + uint32_t header_addr_high; + uint32_t header_addr_low; +}; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c index a789a863d..a845e883f 100644 --- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c @@ -191,6 +191,7 @@ static void sumo_construct_vid_mapping_table(struct amdgpu_device *adev, vid_mapping_table->num_entries = i; } +#if 0 static const struct kv_lcac_config_values sx_local_cac_cfg_kv[] = { { 0, 4, 1 }, @@ -289,6 +290,7 @@ static const struct kv_lcac_config_reg cpl_cac_config_reg[] = { { 0xc0400d80, 0x003e0000, 17, 0x3fc00000, 22, 0x0001fffe, 1, 0x00000001, 0 } }; +#endif static const struct kv_pt_config_reg didt_config_kv[] = { @@ -507,19 +509,19 @@ static int kv_enable_didt(struct amdgpu_device *adev, bool enable) pi->caps_db_ramping || pi->caps_td_ramping || pi->caps_tcp_ramping) { - gfx_v7_0_enter_rlc_safe_mode(adev); + adev->gfx.rlc.funcs->enter_safe_mode(adev); if (enable) { ret = kv_program_pt_config_registers(adev, didt_config_kv); if (ret) { - gfx_v7_0_exit_rlc_safe_mode(adev); + adev->gfx.rlc.funcs->exit_safe_mode(adev); return ret; } } kv_do_enable_didt(adev, enable); - gfx_v7_0_exit_rlc_safe_mode(adev); + adev->gfx.rlc.funcs->exit_safe_mode(adev); } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/ppsmc.h b/drivers/gpu/drm/amd/amdgpu/ppsmc.h index 7837f2ecc..8463245f4 100644 --- a/drivers/gpu/drm/amd/amdgpu/ppsmc.h +++ b/drivers/gpu/drm/amd/amdgpu/ppsmc.h @@ -90,7 +90,9 @@ typedef uint8_t PPSMC_Result; #define PPSMC_StartFanControl ((uint8_t)0x5B) #define PPSMC_StopFanControl ((uint8_t)0x5C) #define PPSMC_MSG_NoDisplay ((uint8_t)0x5D) +#define PPSMC_NoDisplay ((uint8_t)0x5D) #define PPSMC_MSG_HasDisplay ((uint8_t)0x5E) +#define PPSMC_HasDisplay ((uint8_t)0x5E) #define PPSMC_MSG_UVDPowerOFF ((uint8_t)0x60) #define PPSMC_MSG_UVDPowerON ((uint8_t)0x61) #define PPSMC_MSG_EnableULV ((uint8_t)0x62) @@ -108,6 +110,7 @@ typedef uint8_t PPSMC_Result; #define PPSMC_MSG_DisableDTE ((uint8_t)0x88) #define PPSMC_MSG_ThrottleOVRDSCLKDS ((uint8_t)0x96) #define PPSMC_MSG_CancelThrottleOVRDSCLKDS ((uint8_t)0x97) +#define PPSMC_MSG_EnableACDCGPIOInterrupt ((uint16_t) 0x149) /* CI/KV/KB */ #define PPSMC_MSG_UVDDPM_SetEnabledMask ((uint16_t) 0x12D) @@ -161,6 +164,7 @@ typedef uint8_t PPSMC_Result; #define PPSMC_MSG_MASTER_DeepSleep_OFF ((uint16_t) 0x190) #define PPSMC_MSG_Remove_DC_Clamp ((uint16_t) 0x191) #define PPSMC_MSG_SetFanPwmMax ((uint16_t) 0x19A) +#define PPSMC_MSG_SetFanRpmMax ((uint16_t) 0x205) #define PPSMC_MSG_ENABLE_THERMAL_DPM ((uint16_t) 0x19C) #define PPSMC_MSG_DISABLE_THERMAL_DPM ((uint16_t) 0x19D) diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 36d97195f..f6e37bdc1 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -254,19 +254,6 @@ static void sdma_v2_4_ring_emit_ib(struct amdgpu_ring *ring, unsigned vm_id, bool ctx_switch) { u32 vmid = vm_id & 0xf; - u32 next_rptr = ring->wptr + 5; - - while ((next_rptr & 7) != 2) - next_rptr++; - - next_rptr += 6; - - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR)); - amdgpu_ring_write(ring, lower_32_bits(ring->next_rptr_gpu_addr) & 0xfffffffc); - amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); - amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1)); - amdgpu_ring_write(ring, next_rptr); /* IB packet must end on a 8 DW boundary */ sdma_v2_4_ring_insert_nop(ring, (10 - (ring->wptr & 7)) % 8); @@ -405,7 +392,7 @@ static void sdma_v2_4_enable(struct amdgpu_device *adev, bool enable) u32 f32_cntl; int i; - if (enable == false) { + if (!enable) { sdma_v2_4_gfx_stop(adev); sdma_v2_4_rlc_stop(adev); } @@ -579,19 +566,21 @@ static int sdma_v2_4_start(struct amdgpu_device *adev) { int r; - if (!adev->firmware.smu_load) { - r = sdma_v2_4_load_microcode(adev); - if (r) - return r; - } else { - r = adev->smu.smumgr_funcs->check_fw_load_finish(adev, - AMDGPU_UCODE_ID_SDMA0); - if (r) - return -EINVAL; - r = adev->smu.smumgr_funcs->check_fw_load_finish(adev, - AMDGPU_UCODE_ID_SDMA1); - if (r) - return -EINVAL; + if (!adev->pp_enabled) { + if (!adev->firmware.smu_load) { + r = sdma_v2_4_load_microcode(adev); + if (r) + return r; + } else { + r = adev->smu.smumgr_funcs->check_fw_load_finish(adev, + AMDGPU_UCODE_ID_SDMA0); + if (r) + return -EINVAL; + r = adev->smu.smumgr_funcs->check_fw_load_finish(adev, + AMDGPU_UCODE_ID_SDMA1); + if (r) + return -EINVAL; + } } /* halt the engine before programing */ @@ -678,20 +667,19 @@ static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring) * Test a simple IB in the DMA ring (VI). * Returns 0 on success, error on failure. */ -static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) +static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; struct fence *f = NULL; - unsigned i; unsigned index; - int r; u32 tmp = 0; u64 gpu_addr; + long r; r = amdgpu_wb_get(adev, &index); if (r) { - dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r); + dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r); return r; } @@ -701,7 +689,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) memset(&ib, 0, sizeof(ib)); r = amdgpu_ib_get(adev, NULL, 256, &ib); if (r) { - DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); + DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r); goto err0; } @@ -720,28 +708,25 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) if (r) goto err1; - r = fence_wait(f, false); - if (r) { - DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); + r = fence_wait_timeout(f, false, timeout); + if (r == 0) { + DRM_ERROR("amdgpu: IB test timed out\n"); + r = -ETIMEDOUT; goto err1; - } - for (i = 0; i < adev->usec_timeout; i++) { - tmp = le32_to_cpu(adev->wb.wb[index]); - if (tmp == 0xDEADBEEF) - break; - DRM_UDELAY(1); - } - if (i < adev->usec_timeout) { - DRM_INFO("ib test on ring %d succeeded in %u usecs\n", - ring->idx, i); + } else if (r < 0) { + DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); goto err1; + } + tmp = le32_to_cpu(adev->wb.wb[index]); + if (tmp == 0xDEADBEEF) { + DRM_INFO("ib test on ring %d succeeded\n", ring->idx); + r = 0; } else { DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); r = -EINVAL; } err1: - fence_put(f); amdgpu_ib_free(adev, &ib, NULL); fence_put(f); err0: diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 95c44942e..82bab2fb4 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -405,18 +405,6 @@ static void sdma_v3_0_ring_emit_ib(struct amdgpu_ring *ring, unsigned vm_id, bool ctx_switch) { u32 vmid = vm_id & 0xf; - u32 next_rptr = ring->wptr + 5; - - while ((next_rptr & 7) != 2) - next_rptr++; - next_rptr += 6; - - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR)); - amdgpu_ring_write(ring, lower_32_bits(ring->next_rptr_gpu_addr) & 0xfffffffc); - amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr)); - amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1)); - amdgpu_ring_write(ring, next_rptr); /* IB packet must end on a 8 DW boundary */ sdma_v3_0_ring_insert_nop(ring, (10 - (ring->wptr & 7)) % 8); @@ -606,7 +594,7 @@ static void sdma_v3_0_enable(struct amdgpu_device *adev, bool enable) u32 f32_cntl; int i; - if (enable == false) { + if (!enable) { sdma_v3_0_gfx_stop(adev); sdma_v3_0_rlc_stop(adev); } @@ -898,20 +886,19 @@ static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring) * Test a simple IB in the DMA ring (VI). * Returns 0 on success, error on failure. */ -static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) +static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; struct fence *f = NULL; - unsigned i; unsigned index; - int r; u32 tmp = 0; u64 gpu_addr; + long r; r = amdgpu_wb_get(adev, &index); if (r) { - dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r); + dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r); return r; } @@ -921,7 +908,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) memset(&ib, 0, sizeof(ib)); r = amdgpu_ib_get(adev, NULL, 256, &ib); if (r) { - DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); + DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r); goto err0; } @@ -940,27 +927,24 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) if (r) goto err1; - r = fence_wait(f, false); - if (r) { - DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); + r = fence_wait_timeout(f, false, timeout); + if (r == 0) { + DRM_ERROR("amdgpu: IB test timed out\n"); + r = -ETIMEDOUT; goto err1; - } - for (i = 0; i < adev->usec_timeout; i++) { - tmp = le32_to_cpu(adev->wb.wb[index]); - if (tmp == 0xDEADBEEF) - break; - DRM_UDELAY(1); - } - if (i < adev->usec_timeout) { - DRM_INFO("ib test on ring %d succeeded in %u usecs\n", - ring->idx, i); + } else if (r < 0) { + DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r); goto err1; + } + tmp = le32_to_cpu(adev->wb.wb[index]); + if (tmp == 0xDEADBEEF) { + DRM_INFO("ib test on ring %d succeeded\n", ring->idx); + r = 0; } else { DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); r = -EINVAL; } err1: - fence_put(f); amdgpu_ib_free(adev, &ib, NULL); fence_put(f); err0: diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c index 083893dd6..940de1836 100644 --- a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c +++ b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c @@ -173,7 +173,7 @@ static int tonga_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg) { if (!tonga_is_smc_ram_running(adev)) { - return -EINVAL;; + return -EINVAL; } if (wait_smu_response(adev)) { diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c index f07551476..132e613ed 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c @@ -34,6 +34,8 @@ #include "oss/oss_2_0_d.h" #include "oss/oss_2_0_sh_mask.h" +#include "bif/bif_4_1_d.h" + static void uvd_v4_2_mc_resume(struct amdgpu_device *adev); static void uvd_v4_2_init_cg(struct amdgpu_device *adev); static void uvd_v4_2_set_ring_funcs(struct amdgpu_device *adev); @@ -438,6 +440,32 @@ static void uvd_v4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq amdgpu_ring_write(ring, 2); } +/** + * uvd_v4_2_ring_emit_hdp_flush - emit an hdp flush + * + * @ring: amdgpu_ring pointer + * + * Emits an hdp flush. + */ +static void uvd_v4_2_ring_emit_hdp_flush(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET0(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0)); + amdgpu_ring_write(ring, 0); +} + +/** + * uvd_v4_2_ring_hdp_invalidate - emit an hdp invalidate + * + * @ring: amdgpu_ring pointer + * + * Emits an hdp invalidate. + */ +static void uvd_v4_2_ring_emit_hdp_invalidate(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET0(mmHDP_DEBUG0, 0)); + amdgpu_ring_write(ring, 1); +} + /** * uvd_v4_2_ring_test_ring - register write test * @@ -498,49 +526,6 @@ static void uvd_v4_2_ring_emit_ib(struct amdgpu_ring *ring, amdgpu_ring_write(ring, ib->length_dw); } -/** - * uvd_v4_2_ring_test_ib - test ib execution - * - * @ring: amdgpu_ring pointer - * - * Test if we can successfully execute an IB - */ -static int uvd_v4_2_ring_test_ib(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - struct fence *fence = NULL; - int r; - - r = amdgpu_asic_set_uvd_clocks(adev, 53300, 40000); - if (r) { - DRM_ERROR("amdgpu: failed to raise UVD clocks (%d).\n", r); - return r; - } - - r = amdgpu_uvd_get_create_msg(ring, 1, NULL); - if (r) { - DRM_ERROR("amdgpu: failed to get create msg (%d).\n", r); - goto error; - } - - r = amdgpu_uvd_get_destroy_msg(ring, 1, true, &fence); - if (r) { - DRM_ERROR("amdgpu: failed to get destroy ib (%d).\n", r); - goto error; - } - - r = fence_wait(fence, false); - if (r) { - DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); - goto error; - } - DRM_INFO("ib test on ring %d succeeded\n", ring->idx); -error: - fence_put(fence); - amdgpu_asic_set_uvd_clocks(adev, 0, 0); - return r; -} - /** * uvd_v4_2_mc_resume - memory controller programming * @@ -763,10 +748,14 @@ static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = { .parse_cs = amdgpu_uvd_ring_parse_cs, .emit_ib = uvd_v4_2_ring_emit_ib, .emit_fence = uvd_v4_2_ring_emit_fence, + .emit_hdp_flush = uvd_v4_2_ring_emit_hdp_flush, + .emit_hdp_invalidate = uvd_v4_2_ring_emit_hdp_invalidate, .test_ring = uvd_v4_2_ring_test_ring, - .test_ib = uvd_v4_2_ring_test_ib, + .test_ib = amdgpu_uvd_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, .pad_ib = amdgpu_ring_generic_pad_ib, + .begin_use = amdgpu_uvd_ring_begin_use, + .end_use = amdgpu_uvd_ring_end_use, }; static void uvd_v4_2_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c index e0a76a883..101de136b 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c @@ -31,6 +31,7 @@ #include "uvd/uvd_5_0_sh_mask.h" #include "oss/oss_2_0_d.h" #include "oss/oss_2_0_sh_mask.h" +#include "bif/bif_5_0_d.h" #include "vi.h" static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev); @@ -488,6 +489,32 @@ static void uvd_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq amdgpu_ring_write(ring, 2); } +/** + * uvd_v5_0_ring_emit_hdp_flush - emit an hdp flush + * + * @ring: amdgpu_ring pointer + * + * Emits an hdp flush. + */ +static void uvd_v5_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET0(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0)); + amdgpu_ring_write(ring, 0); +} + +/** + * uvd_v5_0_ring_hdp_invalidate - emit an hdp invalidate + * + * @ring: amdgpu_ring pointer + * + * Emits an hdp invalidate. + */ +static void uvd_v5_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET0(mmHDP_DEBUG0, 0)); + amdgpu_ring_write(ring, 1); +} + /** * uvd_v5_0_ring_test_ring - register write test * @@ -550,49 +577,6 @@ static void uvd_v5_0_ring_emit_ib(struct amdgpu_ring *ring, amdgpu_ring_write(ring, ib->length_dw); } -/** - * uvd_v5_0_ring_test_ib - test ib execution - * - * @ring: amdgpu_ring pointer - * - * Test if we can successfully execute an IB - */ -static int uvd_v5_0_ring_test_ib(struct amdgpu_ring *ring) -{ - struct amdgpu_device *adev = ring->adev; - struct fence *fence = NULL; - int r; - - r = amdgpu_asic_set_uvd_clocks(adev, 53300, 40000); - if (r) { - DRM_ERROR("amdgpu: failed to raise UVD clocks (%d).\n", r); - return r; - } - - r = amdgpu_uvd_get_create_msg(ring, 1, NULL); - if (r) { - DRM_ERROR("amdgpu: failed to get create msg (%d).\n", r); - goto error; - } - - r = amdgpu_uvd_get_destroy_msg(ring, 1, true, &fence); - if (r) { - DRM_ERROR("amdgpu: failed to get destroy ib (%d).\n", r); - goto error; - } - - r = fence_wait(fence, false); - if (r) { - DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); - goto error; - } - DRM_INFO("ib test on ring %d succeeded\n", ring->idx); -error: - fence_put(fence); - amdgpu_asic_set_uvd_clocks(adev, 0, 0); - return r; -} - static bool uvd_v5_0_is_idle(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -815,10 +799,14 @@ static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = { .parse_cs = amdgpu_uvd_ring_parse_cs, .emit_ib = uvd_v5_0_ring_emit_ib, .emit_fence = uvd_v5_0_ring_emit_fence, + .emit_hdp_flush = uvd_v5_0_ring_emit_hdp_flush, + .emit_hdp_invalidate = uvd_v5_0_ring_emit_hdp_invalidate, .test_ring = uvd_v5_0_ring_test_ring, - .test_ib = uvd_v5_0_ring_test_ib, + .test_ib = amdgpu_uvd_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, .pad_ib = amdgpu_ring_generic_pad_ib, + .begin_use = amdgpu_uvd_ring_begin_use, + .end_use = amdgpu_uvd_ring_end_use, }; static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c index c9929d665..7f21102bf 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c @@ -33,6 +33,8 @@ #include "oss/oss_2_0_sh_mask.h" #include "smu/smu_7_1_3_d.h" #include "smu/smu_7_1_3_sh_mask.h" +#include "bif/bif_5_1_d.h" +#include "gmc/gmc_8_1_d.h" #include "vi.h" static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev); @@ -385,8 +387,8 @@ static int uvd_v6_0_start(struct amdgpu_device *adev) uint32_t mp_swap_cntl; int i, j, r; - /*disable DPG */ - WREG32_P(mmUVD_POWER_STATUS, 0, ~(1 << 2)); + /* disable DPG */ + WREG32_P(mmUVD_POWER_STATUS, 0, ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); /* disable byte swapping */ lmi_swap_cntl = 0; @@ -405,17 +407,21 @@ static int uvd_v6_0_start(struct amdgpu_device *adev) } /* disable interupt */ - WREG32_P(mmUVD_MASTINT_EN, 0, ~(1 << 1)); + WREG32_P(mmUVD_MASTINT_EN, 0, ~UVD_MASTINT_EN__VCPU_EN_MASK); /* stall UMC and register bus before resetting VCPU */ - WREG32_P(mmUVD_LMI_CTRL2, 1 << 8, ~(1 << 8)); + WREG32_P(mmUVD_LMI_CTRL2, UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); mdelay(1); /* put LMI, VCPU, RBC etc... into reset */ - WREG32(mmUVD_SOFT_RESET, UVD_SOFT_RESET__LMI_SOFT_RESET_MASK | - UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK | UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK | - UVD_SOFT_RESET__RBC_SOFT_RESET_MASK | UVD_SOFT_RESET__CSM_SOFT_RESET_MASK | - UVD_SOFT_RESET__CXW_SOFT_RESET_MASK | UVD_SOFT_RESET__TAP_SOFT_RESET_MASK | + WREG32(mmUVD_SOFT_RESET, + UVD_SOFT_RESET__LMI_SOFT_RESET_MASK | + UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK | + UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK | + UVD_SOFT_RESET__RBC_SOFT_RESET_MASK | + UVD_SOFT_RESET__CSM_SOFT_RESET_MASK | + UVD_SOFT_RESET__CXW_SOFT_RESET_MASK | + UVD_SOFT_RESET__TAP_SOFT_RESET_MASK | UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK); mdelay(5); @@ -424,8 +430,13 @@ static int uvd_v6_0_start(struct amdgpu_device *adev) mdelay(5); /* initialize UVD memory controller */ - WREG32(mmUVD_LMI_CTRL, 0x40 | (1 << 8) | (1 << 13) | - (1 << 21) | (1 << 9) | (1 << 20)); + WREG32(mmUVD_LMI_CTRL, + (0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | + UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | + UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | + UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | + UVD_LMI_CTRL__REQ_MODE_MASK | + UVD_LMI_CTRL__DISABLE_ON_FWV_FAIL_MASK); #ifdef __BIG_ENDIAN /* swap (8 in 32) RB and IB */ @@ -447,10 +458,10 @@ static int uvd_v6_0_start(struct amdgpu_device *adev) mdelay(5); /* enable VCPU clock */ - WREG32(mmUVD_VCPU_CNTL, 1 << 9); + WREG32(mmUVD_VCPU_CNTL, UVD_VCPU_CNTL__CLK_EN_MASK); /* enable UMC */ - WREG32_P(mmUVD_LMI_CTRL2, 0, ~(1 << 8)); + WREG32_P(mmUVD_LMI_CTRL2, 0, ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); /* boot up the VCPU */ WREG32(mmUVD_SOFT_RESET, 0); @@ -484,10 +495,12 @@ static int uvd_v6_0_start(struct amdgpu_device *adev) return r; } /* enable master interrupt */ - WREG32_P(mmUVD_MASTINT_EN, 3 << 1, ~(3 << 1)); + WREG32_P(mmUVD_MASTINT_EN, + (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK), + ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK)); /* clear the bit 4 of UVD_STATUS */ - WREG32_P(mmUVD_STATUS, 0, ~(2 << 1)); + WREG32_P(mmUVD_STATUS, 0, ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); rb_bufsz = order_base_2(ring->ring_size); tmp = 0; @@ -580,6 +593,32 @@ static void uvd_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq amdgpu_ring_write(ring, 2); } +/** + * uvd_v6_0_ring_emit_hdp_flush - emit an hdp flush + * + * @ring: amdgpu_ring pointer + * + * Emits an hdp flush. + */ +static void uvd_v6_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET0(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0)); + amdgpu_ring_write(ring, 0); +} + +/** + * uvd_v6_0_ring_hdp_invalidate - emit an hdp invalidate + * + * @ring: amdgpu_ring pointer + * + * Emits an hdp invalidate. + */ +static void uvd_v6_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET0(mmHDP_DEBUG0, 0)); + amdgpu_ring_write(ring, 1); +} + /** * uvd_v6_0_ring_test_ring - register write test * @@ -634,6 +673,9 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib, unsigned vm_id, bool ctx_switch) { + amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_VMID, 0)); + amdgpu_ring_write(ring, vm_id); + amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_LOW, 0)); amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr)); amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH, 0)); @@ -642,39 +684,55 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring, amdgpu_ring_write(ring, ib->length_dw); } -/** - * uvd_v6_0_ring_test_ib - test ib execution - * - * @ring: amdgpu_ring pointer - * - * Test if we can successfully execute an IB - */ -static int uvd_v6_0_ring_test_ib(struct amdgpu_ring *ring) +static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring, + unsigned vm_id, uint64_t pd_addr) { - struct fence *fence = NULL; - int r; + uint32_t reg; - r = amdgpu_uvd_get_create_msg(ring, 1, NULL); - if (r) { - DRM_ERROR("amdgpu: failed to get create msg (%d).\n", r); - goto error; - } + if (vm_id < 8) + reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id; + else + reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8; - r = amdgpu_uvd_get_destroy_msg(ring, 1, true, &fence); - if (r) { - DRM_ERROR("amdgpu: failed to get destroy ib (%d).\n", r); - goto error; - } + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0)); + amdgpu_ring_write(ring, reg << 2); + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0)); + amdgpu_ring_write(ring, pd_addr >> 12); + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0)); + amdgpu_ring_write(ring, 0x8); - r = fence_wait(fence, false); - if (r) { - DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); - goto error; - } - DRM_INFO("ib test on ring %d succeeded\n", ring->idx); -error: - fence_put(fence); - return r; + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0)); + amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2); + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0)); + amdgpu_ring_write(ring, 1 << vm_id); + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0)); + amdgpu_ring_write(ring, 0x8); + + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0)); + amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2); + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0)); + amdgpu_ring_write(ring, 0); + amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH8, 0)); + amdgpu_ring_write(ring, 1 << vm_id); /* mask */ + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0)); + amdgpu_ring_write(ring, 0xC); +} + +static void uvd_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) +{ + uint32_t seq = ring->fence_drv.sync_seq; + uint64_t addr = ring->fence_drv.gpu_addr; + + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0)); + amdgpu_ring_write(ring, lower_32_bits(addr)); + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0)); + amdgpu_ring_write(ring, upper_32_bits(addr)); + amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH8, 0)); + amdgpu_ring_write(ring, 0xffffffff); /* mask */ + amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH9, 0)); + amdgpu_ring_write(ring, seq); + amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0)); + amdgpu_ring_write(ring, 0xE); } static bool uvd_v6_0_is_idle(void *handle) @@ -847,7 +905,8 @@ static int uvd_v6_0_set_clockgating_state(void *handle, bool enable = (state == AMD_CG_STATE_GATE) ? true : false; static int curstate = -1; - if (adev->asic_type == CHIP_FIJI) + if (adev->asic_type == CHIP_FIJI || + adev->asic_type == CHIP_POLARIS10) uvd_v6_set_bypass_mode(adev, enable); if (!(adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG)) @@ -912,22 +971,51 @@ const struct amd_ip_funcs uvd_v6_0_ip_funcs = { .set_powergating_state = uvd_v6_0_set_powergating_state, }; -static const struct amdgpu_ring_funcs uvd_v6_0_ring_funcs = { +static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = { .get_rptr = uvd_v6_0_ring_get_rptr, .get_wptr = uvd_v6_0_ring_get_wptr, .set_wptr = uvd_v6_0_ring_set_wptr, .parse_cs = amdgpu_uvd_ring_parse_cs, .emit_ib = uvd_v6_0_ring_emit_ib, .emit_fence = uvd_v6_0_ring_emit_fence, + .emit_hdp_flush = uvd_v6_0_ring_emit_hdp_flush, + .emit_hdp_invalidate = uvd_v6_0_ring_emit_hdp_invalidate, + .test_ring = uvd_v6_0_ring_test_ring, + .test_ib = amdgpu_uvd_ring_test_ib, + .insert_nop = amdgpu_ring_insert_nop, + .pad_ib = amdgpu_ring_generic_pad_ib, + .begin_use = amdgpu_uvd_ring_begin_use, + .end_use = amdgpu_uvd_ring_end_use, +}; + +static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = { + .get_rptr = uvd_v6_0_ring_get_rptr, + .get_wptr = uvd_v6_0_ring_get_wptr, + .set_wptr = uvd_v6_0_ring_set_wptr, + .parse_cs = NULL, + .emit_ib = uvd_v6_0_ring_emit_ib, + .emit_fence = uvd_v6_0_ring_emit_fence, + .emit_vm_flush = uvd_v6_0_ring_emit_vm_flush, + .emit_pipeline_sync = uvd_v6_0_ring_emit_pipeline_sync, + .emit_hdp_flush = uvd_v6_0_ring_emit_hdp_flush, + .emit_hdp_invalidate = uvd_v6_0_ring_emit_hdp_invalidate, .test_ring = uvd_v6_0_ring_test_ring, - .test_ib = uvd_v6_0_ring_test_ib, + .test_ib = amdgpu_uvd_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, .pad_ib = amdgpu_ring_generic_pad_ib, + .begin_use = amdgpu_uvd_ring_begin_use, + .end_use = amdgpu_uvd_ring_end_use, }; static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev) { - adev->uvd.ring.funcs = &uvd_v6_0_ring_funcs; + if (adev->asic_type >= CHIP_POLARIS10) { + adev->uvd.ring.funcs = &uvd_v6_0_ring_vm_funcs; + DRM_INFO("UVD is enabled in VM mode\n"); + } else { + adev->uvd.ring.funcs = &uvd_v6_0_ring_phys_funcs; + DRM_INFO("UVD is enabled in physical mode\n"); + } } static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = { diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c index 45d92aceb..80a37a602 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c @@ -594,6 +594,8 @@ static const struct amdgpu_ring_funcs vce_v2_0_ring_funcs = { .test_ib = amdgpu_vce_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, .pad_ib = amdgpu_ring_generic_pad_ib, + .begin_use = amdgpu_vce_ring_begin_use, + .end_use = amdgpu_vce_ring_end_use, }; static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index 30e8099e9..c271abffd 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -43,6 +43,7 @@ #define mmVCE_LMI_VCPU_CACHE_40BIT_BAR0 0x8616 #define mmVCE_LMI_VCPU_CACHE_40BIT_BAR1 0x8617 #define mmVCE_LMI_VCPU_CACHE_40BIT_BAR2 0x8618 +#define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK 0x02 #define VCE_V3_0_FW_SIZE (384 * 1024) #define VCE_V3_0_STACK_SIZE (64 * 1024) @@ -51,6 +52,7 @@ static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx); static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev); static void vce_v3_0_set_irq_funcs(struct amdgpu_device *adev); +static int vce_v3_0_wait_for_idle(void *handle); /** * vce_v3_0_ring_get_rptr - get read pointer @@ -205,6 +207,32 @@ static void vce_v3_0_set_vce_sw_clock_gating(struct amdgpu_device *adev, vce_v3_0_override_vce_clock_gating(adev, false); } +static int vce_v3_0_firmware_loaded(struct amdgpu_device *adev) +{ + int i, j; + + for (i = 0; i < 10; ++i) { + for (j = 0; j < 100; ++j) { + uint32_t status = RREG32(mmVCE_STATUS); + + if (status & VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK) + return 0; + mdelay(10); + } + + DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n"); + WREG32_P(mmVCE_SOFT_RESET, + VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, + ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK); + mdelay(10); + WREG32_P(mmVCE_SOFT_RESET, 0, + ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK); + mdelay(10); + } + + return -ETIMEDOUT; +} + /** * vce_v3_0_start - start VCE block * @@ -215,11 +243,24 @@ static void vce_v3_0_set_vce_sw_clock_gating(struct amdgpu_device *adev, static int vce_v3_0_start(struct amdgpu_device *adev) { struct amdgpu_ring *ring; - int idx, i, j, r; + int idx, r; + + ring = &adev->vce.ring[0]; + WREG32(mmVCE_RB_RPTR, ring->wptr); + WREG32(mmVCE_RB_WPTR, ring->wptr); + WREG32(mmVCE_RB_BASE_LO, ring->gpu_addr); + WREG32(mmVCE_RB_BASE_HI, upper_32_bits(ring->gpu_addr)); + WREG32(mmVCE_RB_SIZE, ring->ring_size / 4); + + ring = &adev->vce.ring[1]; + WREG32(mmVCE_RB_RPTR2, ring->wptr); + WREG32(mmVCE_RB_WPTR2, ring->wptr); + WREG32(mmVCE_RB_BASE_LO2, ring->gpu_addr); + WREG32(mmVCE_RB_BASE_HI2, upper_32_bits(ring->gpu_addr)); + WREG32(mmVCE_RB_SIZE2, ring->ring_size / 4); mutex_lock(&adev->grbm_idx_mutex); for (idx = 0; idx < 2; ++idx) { - if (adev->vce.harvest_config & (1 << idx)) continue; @@ -233,48 +274,24 @@ static int vce_v3_0_start(struct amdgpu_device *adev) vce_v3_0_mc_resume(adev, idx); - /* set BUSY flag */ - WREG32_P(mmVCE_STATUS, 1, ~1); + WREG32_P(mmVCE_STATUS, VCE_STATUS__JOB_BUSY_MASK, + ~VCE_STATUS__JOB_BUSY_MASK); + if (adev->asic_type >= CHIP_STONEY) WREG32_P(mmVCE_VCPU_CNTL, 1, ~0x200001); else WREG32_P(mmVCE_VCPU_CNTL, VCE_VCPU_CNTL__CLK_EN_MASK, ~VCE_VCPU_CNTL__CLK_EN_MASK); - WREG32_P(mmVCE_SOFT_RESET, - VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, - ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK); - - mdelay(100); - WREG32_P(mmVCE_SOFT_RESET, 0, ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK); - for (i = 0; i < 10; ++i) { - uint32_t status; - for (j = 0; j < 100; ++j) { - status = RREG32(mmVCE_STATUS); - if (status & 2) - break; - mdelay(10); - } - r = 0; - if (status & 2) - break; - - DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n"); - WREG32_P(mmVCE_SOFT_RESET, - VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, - ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK); - mdelay(10); - WREG32_P(mmVCE_SOFT_RESET, 0, - ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK); - mdelay(10); - r = -1; - } + mdelay(100); + + r = vce_v3_0_firmware_loaded(adev); /* clear BUSY flag */ - WREG32_P(mmVCE_STATUS, 0, ~1); + WREG32_P(mmVCE_STATUS, 0, ~VCE_STATUS__JOB_BUSY_MASK); /* Set Clock-Gating off */ if (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG) @@ -290,19 +307,46 @@ static int vce_v3_0_start(struct amdgpu_device *adev) WREG32_P(mmGRBM_GFX_INDEX, 0, ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK); mutex_unlock(&adev->grbm_idx_mutex); - ring = &adev->vce.ring[0]; - WREG32(mmVCE_RB_RPTR, ring->wptr); - WREG32(mmVCE_RB_WPTR, ring->wptr); - WREG32(mmVCE_RB_BASE_LO, ring->gpu_addr); - WREG32(mmVCE_RB_BASE_HI, upper_32_bits(ring->gpu_addr)); - WREG32(mmVCE_RB_SIZE, ring->ring_size / 4); + return 0; +} - ring = &adev->vce.ring[1]; - WREG32(mmVCE_RB_RPTR2, ring->wptr); - WREG32(mmVCE_RB_WPTR2, ring->wptr); - WREG32(mmVCE_RB_BASE_LO2, ring->gpu_addr); - WREG32(mmVCE_RB_BASE_HI2, upper_32_bits(ring->gpu_addr)); - WREG32(mmVCE_RB_SIZE2, ring->ring_size / 4); +static int vce_v3_0_stop(struct amdgpu_device *adev) +{ + int idx; + + mutex_lock(&adev->grbm_idx_mutex); + for (idx = 0; idx < 2; ++idx) { + if (adev->vce.harvest_config & (1 << idx)) + continue; + + if (idx == 0) + WREG32_P(mmGRBM_GFX_INDEX, 0, + ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK); + else + WREG32_P(mmGRBM_GFX_INDEX, + GRBM_GFX_INDEX__VCE_INSTANCE_MASK, + ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK); + + if (adev->asic_type >= CHIP_STONEY) + WREG32_P(mmVCE_VCPU_CNTL, 0, ~0x200001); + else + WREG32_P(mmVCE_VCPU_CNTL, 0, + ~VCE_VCPU_CNTL__CLK_EN_MASK); + /* hold on ECPU */ + WREG32_P(mmVCE_SOFT_RESET, + VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, + ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK); + + /* clear BUSY flag */ + WREG32_P(mmVCE_STATUS, 0, ~VCE_STATUS__JOB_BUSY_MASK); + + /* Set Clock-Gating off */ + if (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG) + vce_v3_0_set_vce_sw_clock_gating(adev, false); + } + + WREG32_P(mmGRBM_GFX_INDEX, 0, ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK); + mutex_unlock(&adev->grbm_idx_mutex); return 0; } @@ -441,7 +485,14 @@ static int vce_v3_0_hw_init(void *handle) static int vce_v3_0_hw_fini(void *handle) { - return 0; + int r; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + r = vce_v3_0_wait_for_idle(handle); + if (r) + return r; + + return vce_v3_0_stop(adev); } static int vce_v3_0_suspend(void *handle) @@ -604,6 +655,18 @@ static int vce_v3_0_process_interrupt(struct amdgpu_device *adev, return 0; } +static void vce_v3_set_bypass_mode(struct amdgpu_device *adev, bool enable) +{ + u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL); + + if (enable) + tmp |= GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK; + else + tmp &= ~GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK; + + WREG32_SMC(ixGCK_DFS_BYPASS_CNTL, tmp); +} + static int vce_v3_0_set_clockgating_state(void *handle, enum amd_clockgating_state state) { @@ -611,6 +674,9 @@ static int vce_v3_0_set_clockgating_state(void *handle, bool enable = (state == AMD_CG_STATE_GATE) ? true : false; int i; + if (adev->asic_type == CHIP_POLARIS10) + vce_v3_set_bypass_mode(adev, enable); + if (!(adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)) return 0; @@ -701,6 +767,8 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_funcs = { .test_ib = amdgpu_vce_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, .pad_ib = amdgpu_ring_generic_pad_ib, + .begin_use = amdgpu_vce_ring_begin_use, + .end_use = amdgpu_vce_ring_end_use, }; static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index d8fca2e11..d2c7e4962 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -200,6 +200,29 @@ static void vi_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v) spin_unlock_irqrestore(&adev->didt_idx_lock, flags); } +static u32 vi_gc_cac_rreg(struct amdgpu_device *adev, u32 reg) +{ + unsigned long flags; + u32 r; + + spin_lock_irqsave(&adev->gc_cac_idx_lock, flags); + WREG32(mmGC_CAC_IND_INDEX, (reg)); + r = RREG32(mmGC_CAC_IND_DATA); + spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags); + return r; +} + +static void vi_gc_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v) +{ + unsigned long flags; + + spin_lock_irqsave(&adev->gc_cac_idx_lock, flags); + WREG32(mmGC_CAC_IND_INDEX, (reg)); + WREG32(mmGC_CAC_IND_DATA, (v)); + spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags); +} + + static const u32 tonga_mgcg_cgcg_init[] = { mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100, @@ -530,12 +553,12 @@ static uint32_t vi_read_indexed_register(struct amdgpu_device *adev, u32 se_num, mutex_lock(&adev->grbm_idx_mutex); if (se_num != 0xffffffff || sh_num != 0xffffffff) - gfx_v8_0_select_se_sh(adev, se_num, sh_num); + amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff); val = RREG32(reg_offset); if (se_num != 0xffffffff || sh_num != 0xffffffff) - gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); return val; } @@ -594,7 +617,7 @@ static int vi_read_register(struct amdgpu_device *adev, u32 se_num, return -EINVAL; } -static void vi_gpu_pci_config_reset(struct amdgpu_device *adev) +static int vi_gpu_pci_config_reset(struct amdgpu_device *adev) { u32 i; @@ -609,11 +632,14 @@ static void vi_gpu_pci_config_reset(struct amdgpu_device *adev) /* wait for asic to come out of reset */ for (i = 0; i < adev->usec_timeout; i++) { - if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) - break; + if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) { + /* enable BM */ + pci_set_master(adev->pdev); + return 0; + } udelay(1); } - + return -EINVAL; } static void vi_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung) @@ -639,13 +665,15 @@ static void vi_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hun */ static int vi_asic_reset(struct amdgpu_device *adev) { + int r; + vi_set_bios_scratch_engine_hung(adev, true); - vi_gpu_pci_config_reset(adev); + r = vi_gpu_pci_config_reset(adev); vi_set_bios_scratch_engine_hung(adev, false); - return 0; + return r; } static int vi_set_uvd_clock(struct amdgpu_device *adev, u32 clock, @@ -1130,9 +1158,6 @@ static const struct amdgpu_asic_funcs vi_asic_funcs = .set_uvd_clocks = &vi_set_uvd_clocks, .set_vce_clocks = &vi_set_vce_clocks, .get_virtual_caps = &vi_get_virtual_caps, - /* these should be moved to their own ip modules */ - .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter, - .wait_for_mc_idle = &gmc_v8_0_mc_wait_for_idle, }; static int vi_common_early_init(void *handle) @@ -1153,6 +1178,8 @@ static int vi_common_early_init(void *handle) adev->uvd_ctx_wreg = &vi_uvd_ctx_wreg; adev->didt_rreg = &vi_didt_rreg; adev->didt_wreg = &vi_didt_wreg; + adev->gc_cac_rreg = &vi_gc_cac_rreg; + adev->gc_cac_wreg = &vi_gc_cac_wreg; adev->asic_funcs = &vi_asic_funcs; @@ -1226,12 +1253,18 @@ static int vi_common_early_init(void *handle) adev->cg_flags = AMD_CG_SUPPORT_UVD_MGCG | AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS | + AMD_CG_SUPPORT_GFX_RLC_LS | + AMD_CG_SUPPORT_GFX_CP_LS | + AMD_CG_SUPPORT_GFX_CGTS | + AMD_CG_SUPPORT_GFX_MGLS | + AMD_CG_SUPPORT_GFX_CGTS_LS | + AMD_CG_SUPPORT_GFX_CGCG | + AMD_CG_SUPPORT_GFX_CGLS | AMD_CG_SUPPORT_BIF_LS | AMD_CG_SUPPORT_HDP_MGCG | AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_SDMA_MGCG | AMD_CG_SUPPORT_SDMA_LS; - adev->pg_flags = 0; adev->external_rev_id = adev->rev_id + 0x1; break; default: diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h index ec4036a09..a625b9137 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -187,12 +187,12 @@ int init_pipelines(struct device_queue_manager *dqm, unsigned int get_first_pipe(struct device_queue_manager *dqm); unsigned int get_pipes_num(struct device_queue_manager *dqm); -extern inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) +static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) { return (pdd->lds_base >> 16) & 0xFF; } -extern inline unsigned int +static inline unsigned int get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd) { return (pdd->lds_base >> 60) & 0x0E; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c index e621eba63..a7d3cb3fe 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c @@ -184,7 +184,7 @@ u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, sizeof(u32)) + inx; pr_debug("kfd: get kernel queue doorbell\n" - " doorbell offset == 0x%08d\n" + " doorbell offset == 0x%08X\n" " kernel address == 0x%08lX\n", *doorbell_off, (uintptr_t)(kfd->doorbell_kernel_ptr + inx)); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index d0d5f4baf..80113c335 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -617,10 +617,7 @@ int kgd2kfd_resume(struct kfd_dev *kfd); int kfd_init_apertures(struct kfd_process *process); /* Queue Context Management */ -inline uint32_t lower_32(uint64_t x); -inline uint32_t upper_32(uint64_t x); struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd); -inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m); int init_queue(struct queue **q, struct queue_properties properties); void uninit_queue(struct queue *q); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 7708d90b9..4f3849ac8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -63,13 +63,12 @@ static struct kfd_process *create_process(const struct task_struct *thread); void kfd_process_create_wq(void) { if (!kfd_process_wq) - kfd_process_wq = create_workqueue("kfd_process_wq"); + kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0); } void kfd_process_destroy_wq(void) { if (kfd_process_wq) { - flush_workqueue(kfd_process_wq); destroy_workqueue(kfd_process_wq); kfd_process_wq = NULL; } @@ -330,6 +329,7 @@ err_process_pqm_init: synchronize_rcu(); mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm); err_mmu_notifier: + mutex_destroy(&process->mutex); kfd_pasid_free(process->pasid); err_alloc_pasid: kfree(process->queues); diff --git a/drivers/gpu/drm/amd/include/amd_pcie.h b/drivers/gpu/drm/amd/include/amd_pcie.h index 7c2a916c1..5eb895fd9 100644 --- a/drivers/gpu/drm/amd/include/amd_pcie.h +++ b/drivers/gpu/drm/amd/include/amd_pcie.h @@ -37,6 +37,13 @@ #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_MASK 0x0000FFFF #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_SHIFT 0 +/* gen: chipset 1/2, asic 1/2/3 */ +#define AMDGPU_DEFAULT_PCIE_GEN_MASK (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 \ + | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 \ + | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 \ + | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 \ + | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3) + /* Following flags shows PCIe lane width switch supported in driver which are decided by chipset and ASIC */ #define CAIL_PCIE_LINK_WIDTH_SUPPORT_X1 0x00010000 #define CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 0x00020000 @@ -47,4 +54,11 @@ #define CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 0x00400000 #define CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT 16 +/* 1/2/4/8/16 lanes */ +#define AMDGPU_DEFAULT_PCIE_MLW_MASK (CAIL_PCIE_LINK_WIDTH_SUPPORT_X1 \ + | CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 \ + | CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 \ + | CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 \ + | CAIL_PCIE_LINK_WIDTH_SUPPORT_X16) + #endif diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h index afce1edbe..a74a0d2ff 100644 --- a/drivers/gpu/drm/amd/include/amd_shared.h +++ b/drivers/gpu/drm/amd/include/amd_shared.h @@ -25,15 +25,6 @@ #define AMD_MAX_USEC_TIMEOUT 100000 /* 100 ms */ -/* -* Supported GPU families (aligned with amdgpu_drm.h) -*/ -#define AMD_FAMILY_UNKNOWN 0 -#define AMD_FAMILY_CI 120 /* Bonaire, Hawaii */ -#define AMD_FAMILY_KV 125 /* Kaveri, Kabini, Mullins */ -#define AMD_FAMILY_VI 130 /* Iceland, Tonga */ -#define AMD_FAMILY_CZ 135 /* Carrizo */ - /* * Supported ASIC types */ @@ -120,6 +111,8 @@ enum amd_powergating_state { #define AMD_PG_SUPPORT_SDMA (1 << 8) #define AMD_PG_SUPPORT_ACP (1 << 9) #define AMD_PG_SUPPORT_SAMU (1 << 10) +#define AMD_PG_SUPPORT_GFX_QUICK_MG (1 << 11) +#define AMD_PG_SUPPORT_GFX_PIPELINE (1 << 12) enum amd_pm_state_type { /* not used for dpm */ diff --git a/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h index 293329719..809759f7b 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h +++ b/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h @@ -27,6 +27,7 @@ #define mmMM_INDEX 0x0 #define mmMM_INDEX_HI 0x6 #define mmMM_DATA 0x1 +#define mmCC_BIF_BX_STRAP2 0x152A #define mmBIF_MM_INDACCESS_CNTL 0x1500 #define mmBIF_DOORBELL_APER_EN 0x1501 #define mmBUS_CNTL 0x1508 diff --git a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h index ebaf67bb1..90ff7c8a6 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h +++ b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h @@ -2823,4 +2823,7 @@ #define mmDC_EDC_CSINVOC_CNT 0x3192 #define mmDC_EDC_RESTORE_CNT 0x3193 +#define mmGC_CAC_IND_INDEX 0x129a +#define mmGC_CAC_IND_DATA 0x129b + #endif /* GFX_8_0_D_H */ diff --git a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_sh_mask.h index 7d722458d..4070ca3a6 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_sh_mask.h @@ -8730,8 +8730,6 @@ #define RLC_GPM_STAT__DYN_CU_POWERING_DOWN__SHIFT 0x10 #define RLC_GPM_STAT__ABORTED_PD_SEQUENCE_MASK 0x20000 #define RLC_GPM_STAT__ABORTED_PD_SEQUENCE__SHIFT 0x11 -#define RLC_GPM_STAT__RESERVED_MASK 0xfc0000 -#define RLC_GPM_STAT__RESERVED__SHIFT 0x12 #define RLC_GPM_STAT__PG_ERROR_STATUS_MASK 0xff000000 #define RLC_GPM_STAT__PG_ERROR_STATUS__SHIFT 0x18 #define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL_MASK 0x3f @@ -8764,8 +8762,10 @@ #define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE__SHIFT 0x12 #define RLC_PG_CNTL__SMU_HANDSHAKE_ENABLE_MASK 0x80000 #define RLC_PG_CNTL__SMU_HANDSHAKE_ENABLE__SHIFT 0x13 -#define RLC_PG_CNTL__RESERVED1_MASK 0xf00000 -#define RLC_PG_CNTL__RESERVED1__SHIFT 0x14 +#define RLC_PG_CNTL__QUICK_PG_ENABLE_MASK 0x100000 +#define RLC_PG_CNTL__QUICK_PG_ENABLE__SHIFT 0x14 +#define RLC_PG_CNTL__RESERVED1_MASK 0xe00000 +#define RLC_PG_CNTL__RESERVED1__SHIFT 0x15 #define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY_MASK 0xff #define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY__SHIFT 0x0 #define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY_MASK 0xff00 @@ -9102,8 +9102,6 @@ #define RLC_GPM_LOG_CONT__CONT__SHIFT 0x0 #define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK 0xff #define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT 0x0 -#define RLC_PG_DELAY_3__RESERVED_MASK 0xffffff00 -#define RLC_PG_DELAY_3__RESERVED__SHIFT 0x8 #define RLC_GPM_INT_DISABLE_TH0__DISABLE_MASK 0xffffffff #define RLC_GPM_INT_DISABLE_TH0__DISABLE__SHIFT 0x0 #define RLC_GPM_INT_DISABLE_TH1__DISABLE_MASK 0xffffffff @@ -9124,14 +9122,8 @@ #define RLC_SRM_DEBUG_SELECT__RESERVED__SHIFT 0x8 #define RLC_SRM_DEBUG__DATA_MASK 0xffffffff #define RLC_SRM_DEBUG__DATA__SHIFT 0x0 -#define RLC_SRM_ARAM_ADDR__ADDR_MASK 0x3ff -#define RLC_SRM_ARAM_ADDR__ADDR__SHIFT 0x0 -#define RLC_SRM_ARAM_ADDR__RESERVED_MASK 0xfffffc00 -#define RLC_SRM_ARAM_ADDR__RESERVED__SHIFT 0xa #define RLC_SRM_ARAM_DATA__DATA_MASK 0xffffffff #define RLC_SRM_ARAM_DATA__DATA__SHIFT 0x0 -#define RLC_SRM_DRAM_ADDR__ADDR_MASK 0x3ff -#define RLC_SRM_DRAM_ADDR__ADDR__SHIFT 0x0 #define RLC_SRM_DRAM_ADDR__RESERVED_MASK 0xfffffc00 #define RLC_SRM_DRAM_ADDR__RESERVED__SHIFT 0xa #define RLC_SRM_DRAM_DATA__DATA_MASK 0xffffffff @@ -17946,8 +17938,6 @@ #define VGT_TESS_DISTRIBUTION__ACCUM_TRI__SHIFT 0x8 #define VGT_TESS_DISTRIBUTION__ACCUM_QUAD_MASK 0xff0000 #define VGT_TESS_DISTRIBUTION__ACCUM_QUAD__SHIFT 0x10 -#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT_MASK 0xff000000 -#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT__SHIFT 0x18 #define VGT_TF_RING_SIZE__SIZE_MASK 0xffff #define VGT_TF_RING_SIZE__SIZE__SHIFT 0x0 #define VGT_SYS_CONFIG__DUAL_CORE_EN_MASK 0x1 @@ -20502,8 +20492,6 @@ #define DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT 0x4 #define DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20 #define DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5 -#define DIDT_SQ_CTRL0__UNUSED_0_MASK 0xffffffc0 -#define DIDT_SQ_CTRL0__UNUSED_0__SHIFT 0x6 #define DIDT_SQ_CTRL1__MIN_POWER_MASK 0xffff #define DIDT_SQ_CTRL1__MIN_POWER__SHIFT 0x0 #define DIDT_SQ_CTRL1__MAX_POWER_MASK 0xffff0000 @@ -20558,8 +20546,6 @@ #define DIDT_DB_CTRL0__DIDT_CTRL_RST__SHIFT 0x4 #define DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20 #define DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5 -#define DIDT_DB_CTRL0__UNUSED_0_MASK 0xffffffc0 -#define DIDT_DB_CTRL0__UNUSED_0__SHIFT 0x6 #define DIDT_DB_CTRL1__MIN_POWER_MASK 0xffff #define DIDT_DB_CTRL1__MIN_POWER__SHIFT 0x0 #define DIDT_DB_CTRL1__MAX_POWER_MASK 0xffff0000 @@ -20614,8 +20600,6 @@ #define DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT 0x4 #define DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20 #define DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5 -#define DIDT_TD_CTRL0__UNUSED_0_MASK 0xffffffc0 -#define DIDT_TD_CTRL0__UNUSED_0__SHIFT 0x6 #define DIDT_TD_CTRL1__MIN_POWER_MASK 0xffff #define DIDT_TD_CTRL1__MIN_POWER__SHIFT 0x0 #define DIDT_TD_CTRL1__MAX_POWER_MASK 0xffff0000 @@ -20670,8 +20654,6 @@ #define DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT 0x4 #define DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20 #define DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5 -#define DIDT_TCP_CTRL0__UNUSED_0_MASK 0xffffffc0 -#define DIDT_TCP_CTRL0__UNUSED_0__SHIFT 0x6 #define DIDT_TCP_CTRL1__MIN_POWER_MASK 0xffff #define DIDT_TCP_CTRL1__MIN_POWER__SHIFT 0x0 #define DIDT_TCP_CTRL1__MAX_POWER_MASK 0xffff0000 @@ -20726,8 +20708,6 @@ #define DIDT_DBR_CTRL0__DIDT_CTRL_RST__SHIFT 0x4 #define DIDT_DBR_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20 #define DIDT_DBR_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5 -#define DIDT_DBR_CTRL0__UNUSED_0_MASK 0xffffffc0 -#define DIDT_DBR_CTRL0__UNUSED_0__SHIFT 0x6 #define DIDT_DBR_CTRL1__MIN_POWER_MASK 0xffff #define DIDT_DBR_CTRL1__MIN_POWER__SHIFT 0x0 #define DIDT_DBR_CTRL1__MAX_POWER_MASK 0xffff0000 @@ -20773,4 +20753,84 @@ #define DIDT_DBR_WEIGHT8_11__WEIGHT11_MASK 0xff000000 #define DIDT_DBR_WEIGHT8_11__WEIGHT11__SHIFT 0x18 +#define DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK 0x00000001 +#define DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT 0x00000000 + +#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK 0x0000007e +#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK 0x00001f80L +#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT 0x00000001 +#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT 0x00000007 + +#define DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK 0x1fffe000L +#define DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT 0x0000000d + +#define DIDT_SQ_STALL_CTRL__UNUSED_0_MASK 0xe0000000L +#define DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT 0x0000001d + +#define DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK 0x00000001L +#define DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT 0x00000000 + +#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK 0x00007ffeL +#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT 0x00000001 +#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK 0x1fff8000L +#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT 0x0000000f + +#define DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK 0x00000001L +#define DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT 0x00000000 + +#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK 0x0000007eL +#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK 0x00001f80L +#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT 0x00000001 +#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT 0x00000007 + +#define DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK 0x1fffe000L +#define DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT 0x0000000d + +#define DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK 0x00000fc0L +#define DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK 0x0003f000L +#define DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT 0x00000006 +#define DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT 0x0000000c + +#define DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK 0x00000001L +#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK 0x00007ffeL +#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK 0x1fff8000L + +#define DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT 0x00000000 +#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT 0x00000001 +#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT 0x0000000f + +#define DIDT_TD_STALL_CTRL__UNUSED_0_MASK 0xe0000000L +#define DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT 0x0000001d + +#define DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK 0x00000fc0L +#define DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK 0x0003f000L +#define DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT 0x00000006 +#define DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT 0x0000000c + +#define DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK 0x00000001L +#define DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT 0x00000000 + +#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK 0x0000007eL +#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK 0x00001f80L +#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT 0x00000001 +#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT 0x00000007 + +#define DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK 0x1fffe000L +#define DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT 0x0000000d + +#define DIDT_TCP_STALL_CTRL__UNUSED_0_MASK 0xe0000000L +#define DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT 0x0000001d + +#define DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK 0x00000001L +#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK 0x00007ffeL +#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK 0x1fff8000L +#define DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT 0x00000000 +#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT 0x00000001 +#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT 0x0000000f + +#define DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK 0x00000fc0L +#define DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK 0x0003f000L +#define DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT 0x00000006 +#define DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT 0x0000000c + #endif /* GFX_8_0_SH_MASK_H */ diff --git a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h index 6f6fb3474..ec69869c5 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h +++ b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h @@ -111,6 +111,8 @@ #define mmUVD_MIF_RECON1_ADDR_CONFIG 0x39c5 #define ixUVD_MIF_SCLR_ADDR_CONFIG 0x4 #define mmUVD_JPEG_ADDR_CONFIG 0x3a1f +#define mmUVD_GP_SCRATCH8 0x3c0a +#define mmUVD_GP_SCRATCH9 0x3c0b #define mmUVD_GP_SCRATCH4 0x3d38 #endif /* UVD_6_0_D_H */ diff --git a/drivers/gpu/drm/amd/include/cgs_common.h b/drivers/gpu/drm/amd/include/cgs_common.h index 7464daf89..b86aba9d0 100644 --- a/drivers/gpu/drm/amd/include/cgs_common.h +++ b/drivers/gpu/drm/amd/include/cgs_common.h @@ -49,6 +49,7 @@ enum cgs_ind_reg { CGS_IND_REG__SMC, CGS_IND_REG__UVD_CTX, CGS_IND_REG__DIDT, + CGS_IND_REG_GC_CAC, CGS_IND_REG__AUDIO_ENDPT }; @@ -112,20 +113,23 @@ enum cgs_system_info_id { CGS_SYSTEM_INFO_ADAPTER_BDF_ID = 1, CGS_SYSTEM_INFO_PCIE_GEN_INFO, CGS_SYSTEM_INFO_PCIE_MLW, + CGS_SYSTEM_INFO_PCIE_DEV, + CGS_SYSTEM_INFO_PCIE_REV, CGS_SYSTEM_INFO_CG_FLAGS, CGS_SYSTEM_INFO_PG_FLAGS, CGS_SYSTEM_INFO_GFX_CU_INFO, + CGS_SYSTEM_INFO_GFX_SE_INFO, CGS_SYSTEM_INFO_ID_MAXIMUM, }; struct cgs_system_info { - uint64_t size; - uint64_t info_id; + uint64_t size; + enum cgs_system_info_id info_id; union { - void *ptr; - uint64_t value; + void *ptr; + uint64_t value; }; - uint64_t padding[13]; + uint64_t padding[13]; }; /* @@ -158,6 +162,10 @@ struct cgs_firmware_info { uint16_t feature_version; uint32_t image_size; uint64_t mc_addr; + + /* only for smc firmware */ + uint32_t ucode_start_address; + void *kptr; }; @@ -189,7 +197,6 @@ typedef unsigned long cgs_handle_t; struct cgs_acpi_method_argument { uint32_t type; - uint32_t method_length; uint32_t data_length; union{ uint32_t value; diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c index e629f8a9f..abbb658bd 100644 --- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c +++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c @@ -176,7 +176,7 @@ static int pp_hw_fini(void *handle) static bool pp_is_idle(void *handle) { - return 0; + return false; } static int pp_wait_for_idle(void *handle) @@ -536,6 +536,10 @@ int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_event event_id, void *input, case AMD_PP_EVENT_COMPLETE_INIT: ret = pem_handle_event(pp_handle->eventmgr, event_id, &data); break; + case AMD_PP_EVENT_READJUST_POWER_STATE: + pp_handle->hwmgr->current_ps = pp_handle->hwmgr->boot_ps; + ret = pem_handle_event(pp_handle->eventmgr, event_id, &data); + break; default: break; } @@ -740,12 +744,12 @@ static int pp_dpm_get_pp_table(void *handle, char **table) PP_CHECK_HW(hwmgr); - if (hwmgr->hwmgr_func->get_pp_table == NULL) { - printk(KERN_INFO "%s was not implemented.\n", __func__); - return 0; - } + if (!hwmgr->soft_pp_table) + return -EINVAL; - return hwmgr->hwmgr_func->get_pp_table(hwmgr, table); + *table = (char *)hwmgr->soft_pp_table; + + return hwmgr->soft_pp_table_size; } static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size) @@ -759,12 +763,23 @@ static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size) PP_CHECK_HW(hwmgr); - if (hwmgr->hwmgr_func->set_pp_table == NULL) { - printk(KERN_INFO "%s was not implemented.\n", __func__); - return 0; + if (!hwmgr->hardcode_pp_table) { + hwmgr->hardcode_pp_table = + kzalloc(hwmgr->soft_pp_table_size, GFP_KERNEL); + + if (!hwmgr->hardcode_pp_table) + return -ENOMEM; + + /* to avoid powerplay crash when hardcode pptable is empty */ + memcpy(hwmgr->hardcode_pp_table, hwmgr->soft_pp_table, + hwmgr->soft_pp_table_size); } - return hwmgr->hwmgr_func->set_pp_table(hwmgr, buf, size); + memcpy(hwmgr->hardcode_pp_table, buf, size); + + hwmgr->soft_pp_table = hwmgr->hardcode_pp_table; + + return amd_powerplay_reset(handle); } static int pp_dpm_force_clock_level(void *handle, @@ -806,6 +821,82 @@ static int pp_dpm_print_clock_levels(void *handle, return hwmgr->hwmgr_func->print_clock_levels(hwmgr, type, buf); } +static int pp_dpm_get_sclk_od(void *handle) +{ + struct pp_hwmgr *hwmgr; + + if (!handle) + return -EINVAL; + + hwmgr = ((struct pp_instance *)handle)->hwmgr; + + PP_CHECK_HW(hwmgr); + + if (hwmgr->hwmgr_func->get_sclk_od == NULL) { + printk(KERN_INFO "%s was not implemented.\n", __func__); + return 0; + } + + return hwmgr->hwmgr_func->get_sclk_od(hwmgr); +} + +static int pp_dpm_set_sclk_od(void *handle, uint32_t value) +{ + struct pp_hwmgr *hwmgr; + + if (!handle) + return -EINVAL; + + hwmgr = ((struct pp_instance *)handle)->hwmgr; + + PP_CHECK_HW(hwmgr); + + if (hwmgr->hwmgr_func->set_sclk_od == NULL) { + printk(KERN_INFO "%s was not implemented.\n", __func__); + return 0; + } + + return hwmgr->hwmgr_func->set_sclk_od(hwmgr, value); +} + +static int pp_dpm_get_mclk_od(void *handle) +{ + struct pp_hwmgr *hwmgr; + + if (!handle) + return -EINVAL; + + hwmgr = ((struct pp_instance *)handle)->hwmgr; + + PP_CHECK_HW(hwmgr); + + if (hwmgr->hwmgr_func->get_mclk_od == NULL) { + printk(KERN_INFO "%s was not implemented.\n", __func__); + return 0; + } + + return hwmgr->hwmgr_func->get_mclk_od(hwmgr); +} + +static int pp_dpm_set_mclk_od(void *handle, uint32_t value) +{ + struct pp_hwmgr *hwmgr; + + if (!handle) + return -EINVAL; + + hwmgr = ((struct pp_instance *)handle)->hwmgr; + + PP_CHECK_HW(hwmgr); + + if (hwmgr->hwmgr_func->set_mclk_od == NULL) { + printk(KERN_INFO "%s was not implemented.\n", __func__); + return 0; + } + + return hwmgr->hwmgr_func->set_mclk_od(hwmgr, value); +} + const struct amd_powerplay_funcs pp_dpm_funcs = { .get_temperature = pp_dpm_get_temperature, .load_firmware = pp_dpm_load_fw, @@ -828,6 +919,10 @@ const struct amd_powerplay_funcs pp_dpm_funcs = { .set_pp_table = pp_dpm_set_pp_table, .force_clock_level = pp_dpm_force_clock_level, .print_clock_levels = pp_dpm_print_clock_levels, + .get_sclk_od = pp_dpm_get_sclk_od, + .set_sclk_od = pp_dpm_set_sclk_od, + .get_mclk_od = pp_dpm_get_mclk_od, + .set_mclk_od = pp_dpm_set_mclk_od, }; static int amd_pp_instance_init(struct amd_pp_init *pp_init, @@ -909,6 +1004,44 @@ int amd_powerplay_fini(void *handle) return 0; } +int amd_powerplay_reset(void *handle) +{ + struct pp_instance *instance = (struct pp_instance *)handle; + struct pp_eventmgr *eventmgr; + struct pem_event_data event_data = { {0} }; + int ret; + + if (instance == NULL) + return -EINVAL; + + eventmgr = instance->eventmgr; + if (!eventmgr || !eventmgr->pp_eventmgr_fini) + return -EINVAL; + + eventmgr->pp_eventmgr_fini(eventmgr); + + ret = pp_sw_fini(handle); + if (ret) + return ret; + + kfree(instance->hwmgr->ps); + + ret = pp_sw_init(handle); + if (ret) + return ret; + + hw_init_power_state_table(instance->hwmgr); + + if (eventmgr == NULL || eventmgr->pp_eventmgr_init == NULL) + return -EINVAL; + + ret = eventmgr->pp_eventmgr_init(eventmgr); + if (ret) + return ret; + + return pem_handle_event(eventmgr, AMD_PP_EVENT_COMPLETE_INIT, &event_data); +} + /* export this function to DAL */ int amd_powerplay_display_configuration_change(void *handle, diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c index d6635cc4b..635fc4b48 100644 --- a/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c +++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c @@ -30,7 +30,6 @@ static const pem_event_action * const initialize_event[] = { system_config_tasks, setup_asic_tasks, enable_dynamic_state_management_tasks, - enable_clock_power_gatings_tasks, get_2d_performance_state_tasks, set_performance_state_tasks, initialize_thermal_controller_tasks, @@ -140,7 +139,6 @@ static const pem_event_action * const resume_event[] = { setup_asic_tasks, enable_stutter_mode_tasks, /*must do this in boot state and before SMC is started */ enable_dynamic_state_management_tasks, - enable_clock_power_gatings_tasks, enable_disable_bapm_tasks, initialize_thermal_controller_tasks, get_2d_performance_state_tasks, diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c index 5cd123472..b6f45fd01 100644 --- a/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c +++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c @@ -132,8 +132,7 @@ int pem_task_enable_dynamic_state_management(struct pp_eventmgr *eventmgr, struc int pem_task_disable_dynamic_state_management(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data) { - /* TODO */ - return 0; + return phm_disable_dynamic_state_management(eventmgr->hwmgr); } int pem_task_enable_clock_power_gatings_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data) diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c index 436fc16da..2028980f1 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c @@ -177,12 +177,12 @@ int cz_dpm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate) cz_dpm_powerdown_uvd(hwmgr); } else { cz_dpm_powerup_uvd(hwmgr); - cgs_set_clockgating_state(hwmgr->device, - AMD_IP_BLOCK_TYPE_UVD, - AMD_PG_STATE_GATE); cgs_set_powergating_state(hwmgr->device, AMD_IP_BLOCK_TYPE_UVD, AMD_CG_STATE_UNGATE); + cgs_set_clockgating_state(hwmgr->device, + AMD_IP_BLOCK_TYPE_UVD, + AMD_PG_STATE_GATE); cz_dpm_update_uvd_dpm(hwmgr, false); } @@ -206,25 +206,26 @@ int cz_dpm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate) AMD_IP_BLOCK_TYPE_VCE, AMD_PG_STATE_GATE); cz_enable_disable_vce_dpm(hwmgr, false); - /* TODO: to figure out why vce can't be poweroff*/ + cz_dpm_powerdown_vce(hwmgr); cz_hwmgr->vce_power_gated = true; } else { cz_dpm_powerup_vce(hwmgr); cz_hwmgr->vce_power_gated = false; - cgs_set_clockgating_state( - hwmgr->device, - AMD_IP_BLOCK_TYPE_VCE, - AMD_PG_STATE_GATE); cgs_set_powergating_state( hwmgr->device, AMD_IP_BLOCK_TYPE_VCE, AMD_CG_STATE_UNGATE); + cgs_set_clockgating_state( + hwmgr->device, + AMD_IP_BLOCK_TYPE_VCE, + AMD_PG_STATE_GATE); cz_dpm_update_vce_dpm(hwmgr); cz_enable_disable_vce_dpm(hwmgr, true); return 0; } } } else { + cz_hwmgr->vce_power_gated = bgate; cz_dpm_update_vce_dpm(hwmgr); cz_enable_disable_vce_dpm(hwmgr, !bgate); return 0; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c index 1f14c477d..8cc0df9b5 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c @@ -1167,9 +1167,9 @@ static int cz_apply_state_adjust_rules(struct pp_hwmgr *hwmgr, cz_ps->action = cz_current_ps->action; - if ((force_high == false) && (cz_ps->action == FORCE_HIGH)) + if (!force_high && (cz_ps->action == FORCE_HIGH)) cz_ps->action = CANCEL_FORCE_HIGH; - else if ((force_high == true) && (cz_ps->action != FORCE_HIGH)) + else if (force_high && (cz_ps->action != FORCE_HIGH)) cz_ps->action = FORCE_HIGH; else cz_ps->action = DO_NOTHING; @@ -1180,6 +1180,13 @@ static int cz_apply_state_adjust_rules(struct pp_hwmgr *hwmgr, static int cz_hwmgr_backend_init(struct pp_hwmgr *hwmgr) { int result = 0; + struct cz_hwmgr *data; + + data = kzalloc(sizeof(struct cz_hwmgr), GFP_KERNEL); + if (data == NULL) + return -ENOMEM; + + hwmgr->backend = data; result = cz_initialize_dpm_defaults(hwmgr); if (result != 0) { @@ -1649,7 +1656,7 @@ static void cz_hw_print_display_cfg( struct cz_hwmgr *hw_data = (struct cz_hwmgr *)(hwmgr->backend); uint32_t data = 0; - if (hw_data->cc6_settings.cc6_setting_changed == true) { + if (hw_data->cc6_settings.cc6_setting_changed) { hw_data->cc6_settings.cc6_setting_changed = false; @@ -1909,15 +1916,7 @@ static const struct pp_hwmgr_func cz_hwmgr_funcs = { int cz_hwmgr_init(struct pp_hwmgr *hwmgr) { - struct cz_hwmgr *cz_hwmgr; - int ret = 0; - - cz_hwmgr = kzalloc(sizeof(struct cz_hwmgr), GFP_KERNEL); - if (cz_hwmgr == NULL) - return -ENOMEM; - - hwmgr->backend = cz_hwmgr; hwmgr->hwmgr_func = &cz_hwmgr_funcs; hwmgr->pptable_func = &pptable_funcs; - return ret; + return 0; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c index e1b649bd5..5afe82068 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c @@ -56,7 +56,7 @@ int fiji_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate) fiji_update_uvd_dpm(hwmgr, false); cgs_set_clockgating_state(hwmgr->device, AMD_IP_BLOCK_TYPE_UVD, - AMD_PG_STATE_UNGATE); + AMD_CG_STATE_UNGATE); } return 0; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c index 92912ab20..120a9e2c3 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c @@ -581,25 +581,24 @@ static int fiji_patch_boot_state(struct pp_hwmgr *hwmgr, static int fiji_hwmgr_backend_fini(struct pp_hwmgr *hwmgr) { - struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); - - if (data->soft_pp_table) { - kfree(data->soft_pp_table); - data->soft_pp_table = NULL; - } - return phm_hwmgr_backend_fini(hwmgr); } static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr) { - struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + struct fiji_hwmgr *data; uint32_t i; struct phm_ppt_v1_information *table_info = (struct phm_ppt_v1_information *)(hwmgr->pptable); bool stay_in_boot; int result; + data = kzalloc(sizeof(struct fiji_hwmgr), GFP_KERNEL); + if (data == NULL) + return -ENOMEM; + + hwmgr->backend = data; + data->dll_default_on = false; data->sram_end = SMC_RAM_END; @@ -699,7 +698,7 @@ static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr) if (0 == result) { struct cgs_system_info sys_info = {0}; - data->is_tlu_enabled = 0; + data->is_tlu_enabled = false; hwmgr->platform_descriptor.hardwareActivityPerformanceLevels = FIJI_MAX_HARDWARE_POWERLEVELS; hwmgr->platform_descriptor.hardwarePerformanceLevels = 2; @@ -734,7 +733,7 @@ static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr) sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO; result = cgs_query_system_info(hwmgr->device, &sys_info); if (result) - data->pcie_gen_cap = 0x30007; + data->pcie_gen_cap = AMDGPU_DEFAULT_PCIE_GEN_MASK; else data->pcie_gen_cap = (uint32_t)sys_info.value; if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) @@ -743,7 +742,7 @@ static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr) sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW; result = cgs_query_system_info(hwmgr->device, &sys_info); if (result) - data->pcie_lane_cap = 0x2f0000; + data->pcie_lane_cap = AMDGPU_DEFAULT_PCIE_MLW_MASK; else data->pcie_lane_cap = (uint32_t)sys_info.value; } else { @@ -1236,6 +1235,34 @@ static int fiji_program_voting_clients(struct pp_hwmgr *hwmgr) return 0; } +static int fiji_clear_voting_clients(struct pp_hwmgr *hwmgr) +{ + /* Reset voting clients before disabling DPM */ + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, + SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 1); + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, + SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 1); + + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_0, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_1, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_2, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_3, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_4, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_5, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_6, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_7, 0); + + return 0; +} + /** * Get the location of various tables inside the FW image. * @@ -1362,6 +1389,17 @@ static int fiji_copy_and_switch_arb_sets(struct pp_hwmgr *hwmgr, return 0; } +/** +* Call SMC to reset S0/S1 to S1 and Reset SMIO to initial value +* +* @param hwmgr the address of the powerplay hardware manager. +* @return if success then 0; +*/ +static int fiji_reset_to_default(struct pp_hwmgr *hwmgr) +{ + return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_ResetToDefaults); +} + /** * Initial switch from ARB F0->F1 * @@ -1375,6 +1413,21 @@ static int fiji_initial_switch_from_arbf0_to_f1(struct pp_hwmgr *hwmgr) MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1); } +static int fiji_force_switch_to_arbf0(struct pp_hwmgr *hwmgr) +{ + uint32_t tmp; + + tmp = (cgs_read_ind_register(hwmgr->device, + CGS_IND_REG__SMC, ixSMC_SCRATCH9) & + 0x0000ff00) >> 8; + + if (tmp == MC_CG_ARB_FREQ_F0) + return 0; + + return fiji_copy_and_switch_arb_sets(hwmgr, + tmp, MC_CG_ARB_FREQ_F0); +} + static int fiji_reset_single_dpm_table(struct pp_hwmgr *hwmgr, struct fiji_single_dpm_table *dpm_table, uint32_t count) { @@ -1397,7 +1450,7 @@ static void fiji_setup_pcie_table_entry( { dpm_table->dpm_levels[index].value = pcie_gen; dpm_table->dpm_levels[index].param1 = pcie_lanes; - dpm_table->dpm_levels[index].enabled = 1; + dpm_table->dpm_levels[index].enabled = true; } static int fiji_setup_default_pcie_table(struct pp_hwmgr *hwmgr) @@ -1609,7 +1662,6 @@ static int fiji_populate_cac_table(struct pp_hwmgr *hwmgr, { uint32_t count; uint8_t index; - int result = 0; struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); struct phm_ppt_v1_information *table_info = (struct phm_ppt_v1_information *)(hwmgr->pptable); @@ -1631,7 +1683,7 @@ static int fiji_populate_cac_table(struct pp_hwmgr *hwmgr, VOLTAGE_SCALE)) / 25); } - return result; + return 0; } /** @@ -3177,6 +3229,17 @@ static int fiji_enable_ulv(struct pp_hwmgr *hwmgr) return 0; } +static int fiji_disable_ulv(struct pp_hwmgr *hwmgr) +{ + struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + struct fiji_ulv_parm *ulv = &(data->ulv); + + if (ulv->ulv_supported) + return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DisableULV); + + return 0; +} + static int fiji_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr) { if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, @@ -3197,6 +3260,21 @@ static int fiji_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr) return 0; } +static int fiji_disable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr) +{ + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_SclkDeepSleep)) { + if (smum_send_msg_to_smc(hwmgr->smumgr, + PPSMC_MSG_MASTER_DeepSleep_OFF)) { + PP_ASSERT_WITH_CODE(false, + "Attempt to disable Master Deep Sleep switch failed!", + return -1); + } + } + + return 0; +} + static int fiji_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) { struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); @@ -3357,6 +3435,70 @@ static int fiji_start_dpm(struct pp_hwmgr *hwmgr) return 0; } +static int fiji_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) +{ + struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + + /* disable SCLK dpm */ + if (!data->sclk_dpm_key_disabled) + PP_ASSERT_WITH_CODE( + (smum_send_msg_to_smc(hwmgr->smumgr, + PPSMC_MSG_DPM_Disable) == 0), + "Failed to disable SCLK DPM!", + return -1); + + /* disable MCLK dpm */ + if (!data->mclk_dpm_key_disabled) { + PP_ASSERT_WITH_CODE( + (smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, + PPSMC_MSG_MCLKDPM_SetEnabledMask, 1) == 0), + "Failed to force MCLK DPM0!", + return -1); + + PP_ASSERT_WITH_CODE( + (smum_send_msg_to_smc(hwmgr->smumgr, + PPSMC_MSG_MCLKDPM_Disable) == 0), + "Failed to disable MCLK DPM!", + return -1); + } + + return 0; +} + +static int fiji_stop_dpm(struct pp_hwmgr *hwmgr) +{ + struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + + /* disable general power management */ + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT, + GLOBAL_PWRMGT_EN, 0); + /* disable sclk deep sleep */ + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL, + DYNAMIC_PM_EN, 0); + + /* disable PCIE dpm */ + if (!data->pcie_dpm_key_disabled) { + PP_ASSERT_WITH_CODE( + (smum_send_msg_to_smc(hwmgr->smumgr, + PPSMC_MSG_PCIeDPM_Disable) == 0), + "Failed to disable pcie DPM during DPM Stop Function!", + return -1); + } + + if (fiji_disable_sclk_mclk_dpm(hwmgr)) { + printk(KERN_ERR "Failed to disable Sclk DPM and Mclk DPM!"); + return -1; + } + + PP_ASSERT_WITH_CODE( + (smum_send_msg_to_smc(hwmgr->smumgr, + PPSMC_MSG_Voltage_Cntl_Disable) == 0), + "Failed to disable voltage DPM during DPM Stop Function!", + return -1); + + return 0; +} + static void fiji_set_dpm_event_sources(struct pp_hwmgr *hwmgr, uint32_t sources) { @@ -3415,6 +3557,23 @@ static int fiji_enable_thermal_auto_throttle(struct pp_hwmgr *hwmgr) return fiji_enable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal); } +static int fiji_disable_auto_throttle_source(struct pp_hwmgr *hwmgr, + PHM_AutoThrottleSource source) +{ + struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + + if (data->active_auto_throttle_sources & (1 << source)) { + data->active_auto_throttle_sources &= ~(1 << source); + fiji_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources); + } + return 0; +} + +static int fiji_disable_thermal_auto_throttle(struct pp_hwmgr *hwmgr) +{ + return fiji_disable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal); +} + static int fiji_enable_dpm_tasks(struct pp_hwmgr *hwmgr) { int tmp_result, result = 0; @@ -3529,6 +3688,64 @@ static int fiji_enable_dpm_tasks(struct pp_hwmgr *hwmgr) return result; } +static int fiji_disable_dpm_tasks(struct pp_hwmgr *hwmgr) +{ + int tmp_result, result = 0; + + tmp_result = (fiji_is_dpm_running(hwmgr)) ? 0 : -1; + PP_ASSERT_WITH_CODE(tmp_result == 0, + "DPM is not running right now, no need to disable DPM!", + return 0); + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_ThermalController)) + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, + GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 1); + + tmp_result = fiji_disable_power_containment(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to disable power containment!", result = tmp_result); + + tmp_result = fiji_disable_smc_cac(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to disable SMC CAC!", result = tmp_result); + + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, + CG_SPLL_SPREAD_SPECTRUM, SSEN, 0); + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, + GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 0); + + tmp_result = fiji_disable_thermal_auto_throttle(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to disable thermal auto throttle!", result = tmp_result); + + tmp_result = fiji_stop_dpm(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to stop DPM!", result = tmp_result); + + tmp_result = fiji_disable_deep_sleep_master_switch(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to disable deep sleep master switch!", result = tmp_result); + + tmp_result = fiji_disable_ulv(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to disable ULV!", result = tmp_result); + + tmp_result = fiji_clear_voting_clients(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to clear voting clients!", result = tmp_result); + + tmp_result = fiji_reset_to_default(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to reset to default!", result = tmp_result); + + tmp_result = fiji_force_switch_to_arbf0(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to force to switch arbf0!", result = tmp_result); + + return result; +} + static int fiji_force_dpm_highest(struct pp_hwmgr *hwmgr) { struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); @@ -4171,8 +4388,9 @@ static int fiji_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if ((0 == data->sclk_dpm_key_disabled) && (data->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) { - PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr), - "Trying to freeze SCLK DPM when DPM is disabled",); + PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr), + "Trying to freeze SCLK DPM when DPM is disabled", + ); PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_SCLKDPM_FreezeLevel), "Failed to freeze SCLK DPM during FreezeSclkMclkDPM Function!", @@ -4182,8 +4400,9 @@ static int fiji_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if ((0 == data->mclk_dpm_key_disabled) && (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) { - PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr), - "Trying to freeze MCLK DPM when DPM is disabled",); + PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr), + "Trying to freeze MCLK DPM when DPM is disabled", + ); PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_MCLKDPM_FreezeLevel), "Failed to freeze MCLK DPM during FreezeSclkMclkDPM Function!", @@ -4353,7 +4572,6 @@ static int fiji_trim_single_dpm_states(struct pp_hwmgr *hwmgr, static int fiji_trim_dpm_states(struct pp_hwmgr *hwmgr, const struct fiji_power_state *fiji_ps) { - int result = 0; struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); uint32_t high_limit_count; @@ -4373,7 +4591,7 @@ static int fiji_trim_dpm_states(struct pp_hwmgr *hwmgr, fiji_ps->performance_levels[0].memory_clock, fiji_ps->performance_levels[high_limit_count].memory_clock); - return result; + return 0; } static int fiji_generate_dpm_level_enable_mask( @@ -4632,8 +4850,9 @@ static int fiji_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) (data->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) { - PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr), - "Trying to Unfreeze SCLK DPM when DPM is disabled",); + PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr), + "Trying to Unfreeze SCLK DPM when DPM is disabled", + ); PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_SCLKDPM_UnfreezeLevel), "Failed to unfreeze SCLK DPM during UnFreezeSclkMclkDPM Function!", @@ -4643,8 +4862,9 @@ static int fiji_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if ((0 == data->mclk_dpm_key_disabled) && (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) { - PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr), - "Trying to Unfreeze MCLK DPM when DPM is disabled",); + PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr), + "Trying to Unfreeze MCLK DPM when DPM is disabled", + ); PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_SCLKDPM_UnfreezeLevel), "Failed to unfreeze MCLK DPM during UnFreezeSclkMclkDPM Function!", @@ -5071,42 +5291,6 @@ static int fiji_get_fan_control_mode(struct pp_hwmgr *hwmgr) CG_FDO_CTRL2, FDO_PWM_MODE); } -static int fiji_get_pp_table(struct pp_hwmgr *hwmgr, char **table) -{ - struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); - - if (!data->soft_pp_table) { - data->soft_pp_table = kmemdup(hwmgr->soft_pp_table, - hwmgr->soft_pp_table_size, - GFP_KERNEL); - if (!data->soft_pp_table) - return -ENOMEM; - } - - *table = (char *)&data->soft_pp_table; - - return hwmgr->soft_pp_table_size; -} - -static int fiji_set_pp_table(struct pp_hwmgr *hwmgr, const char *buf, size_t size) -{ - struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); - - if (!data->soft_pp_table) { - data->soft_pp_table = kzalloc(hwmgr->soft_pp_table_size, GFP_KERNEL); - if (!data->soft_pp_table) - return -ENOMEM; - } - - memcpy(data->soft_pp_table, buf, size); - - hwmgr->soft_pp_table = data->soft_pp_table; - - /* TODO: re-init powerplay to implement modified pptable */ - - return 0; -} - static int fiji_force_clock_level(struct pp_hwmgr *hwmgr, enum pp_clock_type type, uint32_t mask) { @@ -5276,12 +5460,96 @@ bool fiji_check_smc_update_required_for_display_configuration(struct pp_hwmgr *h return is_update_required; } +static int fiji_get_sclk_od(struct pp_hwmgr *hwmgr) +{ + struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + struct fiji_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table); + struct fiji_single_dpm_table *golden_sclk_table = + &(data->golden_dpm_table.sclk_table); + int value; + + value = (sclk_table->dpm_levels[sclk_table->count - 1].value - + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) * + 100 / + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value; + + return value; +} + +static int fiji_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value) +{ + struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + struct fiji_single_dpm_table *golden_sclk_table = + &(data->golden_dpm_table.sclk_table); + struct pp_power_state *ps; + struct fiji_power_state *fiji_ps; + + if (value > 20) + value = 20; + + ps = hwmgr->request_ps; + + if (ps == NULL) + return -EINVAL; + + fiji_ps = cast_phw_fiji_power_state(&ps->hardware); + + fiji_ps->performance_levels[fiji_ps->performance_level_count - 1].engine_clock = + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value * + value / 100 + + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value; + + return 0; +} + +static int fiji_get_mclk_od(struct pp_hwmgr *hwmgr) +{ + struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + struct fiji_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table); + struct fiji_single_dpm_table *golden_mclk_table = + &(data->golden_dpm_table.mclk_table); + int value; + + value = (mclk_table->dpm_levels[mclk_table->count - 1].value - + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) * + 100 / + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value; + + return value; +} + +static int fiji_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value) +{ + struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + struct fiji_single_dpm_table *golden_mclk_table = + &(data->golden_dpm_table.mclk_table); + struct pp_power_state *ps; + struct fiji_power_state *fiji_ps; + + if (value > 20) + value = 20; + + ps = hwmgr->request_ps; + + if (ps == NULL) + return -EINVAL; + + fiji_ps = cast_phw_fiji_power_state(&ps->hardware); + + fiji_ps->performance_levels[fiji_ps->performance_level_count - 1].memory_clock = + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value * + value / 100 + + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value; + + return 0; +} static const struct pp_hwmgr_func fiji_hwmgr_funcs = { .backend_init = &fiji_hwmgr_backend_init, .backend_fini = &fiji_hwmgr_backend_fini, .asic_setup = &fiji_setup_asic_task, .dynamic_state_management_enable = &fiji_enable_dpm_tasks, + .dynamic_state_management_disable = &fiji_disable_dpm_tasks, .force_dpm_level = &fiji_dpm_force_dpm_level, .get_num_of_pp_table_entries = &tonga_get_number_of_powerplay_table_entries, .get_power_state_size = &fiji_get_power_state_size, @@ -5314,24 +5582,18 @@ static const struct pp_hwmgr_func fiji_hwmgr_funcs = { .get_fan_control_mode = fiji_get_fan_control_mode, .check_states_equal = fiji_check_states_equal, .check_smc_update_required_for_display_configuration = fiji_check_smc_update_required_for_display_configuration, - .get_pp_table = fiji_get_pp_table, - .set_pp_table = fiji_set_pp_table, .force_clock_level = fiji_force_clock_level, .print_clock_levels = fiji_print_clock_levels, + .get_sclk_od = fiji_get_sclk_od, + .set_sclk_od = fiji_set_sclk_od, + .get_mclk_od = fiji_get_mclk_od, + .set_mclk_od = fiji_set_mclk_od, }; int fiji_hwmgr_init(struct pp_hwmgr *hwmgr) { - struct fiji_hwmgr *data; - int ret = 0; - - data = kzalloc(sizeof(struct fiji_hwmgr), GFP_KERNEL); - if (data == NULL) - return -ENOMEM; - - hwmgr->backend = data; hwmgr->hwmgr_func = &fiji_hwmgr_funcs; hwmgr->pptable_func = &tonga_pptable_funcs; pp_fiji_thermal_initialize(hwmgr); - return ret; + return 0; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h index 170edf5a7..bf67c2a92 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h @@ -302,9 +302,6 @@ struct fiji_hwmgr { bool pg_acp_init; bool frtc_enabled; bool frtc_status_changed; - - /* soft pptable for re-uploading into smu */ - void *soft_pp_table; }; /* To convert to Q8.8 format for firmware */ diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c index db23a4068..44658451a 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c @@ -72,18 +72,19 @@ void fiji_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr) fiji_hwmgr->dte_tj_offset = tmp; if (!tmp) { - phm_cap_set(hwmgr->platform_descriptor.platformCaps, - PHM_PlatformCaps_PowerContainment); - phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_CAC); fiji_hwmgr->fast_watermark_threshold = 100; - tmp = 1; - fiji_hwmgr->enable_dte_feature = tmp ? false : true; - fiji_hwmgr->enable_tdc_limit_feature = tmp ? true : false; - fiji_hwmgr->enable_pkg_pwr_tracking_feature = tmp ? true : false; + if (hwmgr->powercontainment_enabled) { + phm_cap_set(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_PowerContainment); + tmp = 1; + fiji_hwmgr->enable_dte_feature = tmp ? false : true; + fiji_hwmgr->enable_tdc_limit_feature = tmp ? true : false; + fiji_hwmgr->enable_pkg_pwr_tracking_feature = tmp ? true : false; + } } } @@ -459,6 +460,23 @@ int fiji_enable_smc_cac(struct pp_hwmgr *hwmgr) return result; } +int fiji_disable_smc_cac(struct pp_hwmgr *hwmgr) +{ + struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + int result = 0; + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_CAC) && data->cac_enabled) { + int smc_result = smum_send_msg_to_smc(hwmgr->smumgr, + (uint16_t)(PPSMC_MSG_DisableCac)); + PP_ASSERT_WITH_CODE((smc_result == 0), + "Failed to disable CAC in SMC.", result = -1); + + data->cac_enabled = false; + } + return result; +} + int fiji_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n) { struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); @@ -528,6 +546,48 @@ int fiji_enable_power_containment(struct pp_hwmgr *hwmgr) return result; } +int fiji_disable_power_containment(struct pp_hwmgr *hwmgr) +{ + struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend); + int result = 0; + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_PowerContainment) && + data->power_containment_features) { + int smc_result; + + if (data->power_containment_features & + POWERCONTAINMENT_FEATURE_TDCLimit) { + smc_result = smum_send_msg_to_smc(hwmgr->smumgr, + (uint16_t)(PPSMC_MSG_TDCLimitDisable)); + PP_ASSERT_WITH_CODE((smc_result == 0), + "Failed to disable TDCLimit in SMC.", + result = smc_result); + } + + if (data->power_containment_features & + POWERCONTAINMENT_FEATURE_DTE) { + smc_result = smum_send_msg_to_smc(hwmgr->smumgr, + (uint16_t)(PPSMC_MSG_DisableDTE)); + PP_ASSERT_WITH_CODE((smc_result == 0), + "Failed to disable DTE in SMC.", + result = smc_result); + } + + if (data->power_containment_features & + POWERCONTAINMENT_FEATURE_PkgPwrLimit) { + smc_result = smum_send_msg_to_smc(hwmgr->smumgr, + (uint16_t)(PPSMC_MSG_PkgPwrLimitDisable)); + PP_ASSERT_WITH_CODE((smc_result == 0), + "Failed to disable PkgPwrTracking in SMC.", + result = smc_result); + } + data->power_containment_features = 0; + } + + return result; +} + int fiji_power_control_set_level(struct pp_hwmgr *hwmgr) { struct phm_ppt_v1_information *table_info = diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h index 55e58200f..fec772421 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h @@ -36,6 +36,19 @@ enum fiji_pt_config_reg_type { #define POWERCONTAINMENT_FEATURE_TDCLimit 0x00000002 #define POWERCONTAINMENT_FEATURE_PkgPwrLimit 0x00000004 +#define DIDT_SQ_CTRL0__UNUSED_0_MASK 0xffffffc0 +#define DIDT_SQ_CTRL0__UNUSED_0__SHIFT 0x6 +#define DIDT_TD_CTRL0__UNUSED_0_MASK 0xffffffc0 +#define DIDT_TD_CTRL0__UNUSED_0__SHIFT 0x6 +#define DIDT_TCP_CTRL0__UNUSED_0_MASK 0xffffffc0 +#define DIDT_TCP_CTRL0__UNUSED_0__SHIFT 0x6 +#define DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK 0xe0000000 +#define DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT 0x0000001d +#define DIDT_TD_TUNING_CTRL__UNUSED_0_MASK 0xe0000000 +#define DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT 0x0000001d +#define DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK 0xe0000000 +#define DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT 0x0000001d + struct fiji_pt_config_reg { uint32_t offset; uint32_t mask; @@ -58,7 +71,9 @@ void fiji_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr); int fiji_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr); int fiji_populate_pm_fuses(struct pp_hwmgr *hwmgr); int fiji_enable_smc_cac(struct pp_hwmgr *hwmgr); +int fiji_disable_smc_cac(struct pp_hwmgr *hwmgr); int fiji_enable_power_containment(struct pp_hwmgr *hwmgr); +int fiji_disable_power_containment(struct pp_hwmgr *hwmgr); int fiji_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n); int fiji_power_control_set_level(struct pp_hwmgr *hwmgr); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c index 7a705cee0..a6abe81bc 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c @@ -59,8 +59,8 @@ int phm_dispatch_table(struct pp_hwmgr *hwmgr, struct phm_runtime_table_header *rt_table, void *input, void *output) { - int result = 0; - void *temp_storage = NULL; + int result; + void *temp_storage; if (hwmgr == NULL || rt_table == NULL) { printk(KERN_ERR "[ powerplay ] Invalid Parameter!\n"); @@ -73,12 +73,13 @@ int phm_dispatch_table(struct pp_hwmgr *hwmgr, printk(KERN_ERR "[ powerplay ] Could not allocate table temporary storage\n"); return -ENOMEM; } + } else { + temp_storage = NULL; } result = phm_run_table(hwmgr, rt_table, input, output, temp_storage); - if (NULL != temp_storage) - kfree(temp_storage); + kfree(temp_storage); return result; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c index efb77eda7..789f98ad2 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c @@ -154,6 +154,30 @@ int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr) return ret; } +int phm_disable_dynamic_state_management(struct pp_hwmgr *hwmgr) +{ + int ret = -1; + bool enabled; + + PHM_FUNC_CHECK(hwmgr); + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_TablelessHardwareInterface)) { + if (hwmgr->hwmgr_func->dynamic_state_management_disable) + ret = hwmgr->hwmgr_func->dynamic_state_management_disable(hwmgr); + } else { + ret = phm_dispatch_table(hwmgr, + &(hwmgr->disable_dynamic_state_management), + NULL, NULL); + } + + enabled = ret == 0 ? false : true; + + cgs_notify_dpm_enabled(hwmgr->device, enabled); + + return ret; +} + int phm_force_dpm_levels(struct pp_hwmgr *hwmgr, enum amd_dpm_forced_level level) { PHM_FUNC_CHECK(hwmgr); @@ -314,7 +338,7 @@ int phm_store_dal_configuration_data(struct pp_hwmgr *hwmgr, if (hwmgr->hwmgr_func->store_cc6_data == NULL) return -EINVAL; - /* to do pass other display configuration in furture */ + /* TODO: pass other display configuration in the future */ if (hwmgr->hwmgr_func->store_cc6_data) hwmgr->hwmgr_func->store_cc6_data(hwmgr, diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c index 20f20e075..27e07624a 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "cgs_common.h" #include "power_state.h" #include "hwmgr.h" @@ -58,12 +59,13 @@ int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle) hwmgr->hw_revision = pp_init->rev_id; hwmgr->usec_timeout = AMD_MAX_USEC_TIMEOUT; hwmgr->power_source = PP_PowerSource_AC; + hwmgr->powercontainment_enabled = pp_init->powercontainment_enabled; switch (hwmgr->chip_family) { - case AMD_FAMILY_CZ: + case AMDGPU_FAMILY_CZ: cz_hwmgr_init(hwmgr); break; - case AMD_FAMILY_VI: + case AMDGPU_FAMILY_VI: switch (hwmgr->chip_id) { case CHIP_TONGA: tonga_hwmgr_init(hwmgr); @@ -94,6 +96,8 @@ int hwmgr_fini(struct pp_hwmgr *hwmgr) return -EINVAL; /* do hwmgr finish*/ + kfree(hwmgr->hardcode_pp_table); + kfree(hwmgr->backend); kfree(hwmgr->start_thermal_controller.function_list); @@ -530,7 +534,7 @@ int phm_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr /* initialize vddc_dep_on_dal_pwrl table */ table_size = sizeof(uint32_t) + 4 * sizeof(struct phm_clock_voltage_dependency_record); - table_clk_vlt = (struct phm_clock_voltage_dependency_table *)kzalloc(table_size, GFP_KERNEL); + table_clk_vlt = kzalloc(table_size, GFP_KERNEL); if (NULL == table_clk_vlt) { printk(KERN_ERR "[ powerplay ] Can not allocate space for vddc_dep_on_dal_pwrl! \n"); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.c index 8f142a74a..b5edb5105 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.c @@ -106,11 +106,17 @@ int polaris10_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate) data->uvd_power_gated = bgate; if (bgate) { + cgs_set_clockgating_state(hwmgr->device, + AMD_IP_BLOCK_TYPE_UVD, + AMD_CG_STATE_GATE); polaris10_update_uvd_dpm(hwmgr, true); polaris10_phm_powerdown_uvd(hwmgr); } else { polaris10_phm_powerup_uvd(hwmgr); polaris10_update_uvd_dpm(hwmgr, false); + cgs_set_clockgating_state(hwmgr->device, + AMD_IP_BLOCK_TYPE_UVD, + AMD_CG_STATE_UNGATE); } return 0; @@ -125,11 +131,19 @@ int polaris10_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate) data->vce_power_gated = bgate; - if (bgate) + if (bgate) { + cgs_set_clockgating_state(hwmgr->device, + AMD_IP_BLOCK_TYPE_VCE, + AMD_CG_STATE_GATE); + polaris10_update_vce_dpm(hwmgr, true); polaris10_phm_powerdown_vce(hwmgr); - else + } else { polaris10_phm_powerup_vce(hwmgr); - + polaris10_update_vce_dpm(hwmgr, false); + cgs_set_clockgating_state(hwmgr->device, + AMD_IP_BLOCK_TYPE_VCE, + AMD_CG_STATE_UNGATE); + } return 0; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c index 91e25f942..769636a0c 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c @@ -389,6 +389,34 @@ static int polaris10_program_voting_clients(struct pp_hwmgr *hwmgr) return 0; } +static int polaris10_clear_voting_clients(struct pp_hwmgr *hwmgr) +{ + /* Reset voting clients before disabling DPM */ + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, + SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 1); + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, + SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 1); + + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_0, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_1, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_2, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_3, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_4, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_5, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_6, 0); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, + ixCG_FREQ_TRAN_VOTING_7, 0); + + return 0; +} + /** * Get the location of various tables inside the FW image. * @@ -515,6 +543,11 @@ static int polaris10_copy_and_switch_arb_sets(struct pp_hwmgr *hwmgr, return 0; } +static int polaris10_reset_to_default(struct pp_hwmgr *hwmgr) +{ + return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_ResetToDefaults); +} + /** * Initial switch from ARB F0->F1 * @@ -528,6 +561,21 @@ static int polaris10_initial_switch_from_arbf0_to_f1(struct pp_hwmgr *hwmgr) MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1); } +static int polaris10_force_switch_to_arbf0(struct pp_hwmgr *hwmgr) +{ + uint32_t tmp; + + tmp = (cgs_read_ind_register(hwmgr->device, + CGS_IND_REG__SMC, ixSMC_SCRATCH9) & + 0x0000ff00) >> 8; + + if (tmp == MC_CG_ARB_FREQ_F0) + return 0; + + return polaris10_copy_and_switch_arb_sets(hwmgr, + tmp, MC_CG_ARB_FREQ_F0); +} + static int polaris10_setup_default_pcie_table(struct pp_hwmgr *hwmgr) { struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); @@ -1356,9 +1404,9 @@ static int polaris10_populate_all_memory_levels(struct pp_hwmgr *hwmgr) return result; } - /* in order to prevent MC activity from stutter mode to push DPM up. + /* In order to prevent MC activity from stutter mode to push DPM up, * the UVD change complements this by putting the MCLK in - * a higher state by default such that we are not effected by + * a higher state by default such that we are not affected by * up threshold or and MCLK DPM latency. */ levels[0].ActivityLevel = 0x1f; @@ -1425,7 +1473,7 @@ static int polaris10_populate_smc_acpi_level(struct pp_hwmgr *hwmgr, /* Get MinVoltage and Frequency from DPM0, * already converted to SMC_UL */ - sclk_frequency = data->dpm_table.sclk_table.dpm_levels[0].value; + sclk_frequency = data->vbios_boot_state.sclk_bootup_value; result = polaris10_get_dependency_volt_by_clk(hwmgr, table_info->vdd_dep_on_sclk, sclk_frequency, @@ -1461,8 +1509,7 @@ static int polaris10_populate_smc_acpi_level(struct pp_hwmgr *hwmgr, /* Get MinVoltage and Frequency from DPM0, already converted to SMC_UL */ - table->MemoryACPILevel.MclkFrequency = - data->dpm_table.mclk_table.dpm_levels[0].value; + table->MemoryACPILevel.MclkFrequency = data->vbios_boot_state.mclk_bootup_value; result = polaris10_get_dependency_volt_by_clk(hwmgr, table_info->vdd_dep_on_mclk, table->MemoryACPILevel.MclkFrequency, @@ -1780,7 +1827,7 @@ static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr) { uint32_t ro, efuse, volt_without_cks, volt_with_cks, value, max, min; struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); - uint8_t i, stretch_amount, stretch_amount2, volt_offset = 0; + uint8_t i, stretch_amount, volt_offset = 0; struct phm_ppt_v1_information *table_info = (struct phm_ppt_v1_information *)(hwmgr->pptable); struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table = @@ -1831,11 +1878,8 @@ static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr) data->smc_state_table.LdoRefSel = (table_info->cac_dtp_table->ucCKS_LDO_REFSEL != 0) ? table_info->cac_dtp_table->ucCKS_LDO_REFSEL : 6; /* Populate CKS Lookup Table */ - if (stretch_amount == 1 || stretch_amount == 2 || stretch_amount == 5) - stretch_amount2 = 0; - else if (stretch_amount == 3 || stretch_amount == 4) - stretch_amount2 = 1; - else { + if (stretch_amount != 1 && stretch_amount != 2 && stretch_amount != 3 && + stretch_amount != 4 && stretch_amount != 5) { phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_ClockStretcher); PP_ASSERT_WITH_CODE(false, @@ -1890,9 +1934,8 @@ static int polaris10_populate_vr_config(struct pp_hwmgr *hwmgr, if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) { config = VR_SVI2_PLANE_2; table->VRConfig |= (config << VRCONF_MVDD_SHIFT); - } else if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) { - config = VR_SMIO_PATTERN_2; - table->VRConfig |= (config << VRCONF_MVDD_SHIFT); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start + + offsetof(SMU74_SoftRegisters, AllowMvddSwitch), 0x1); } else { config = VR_STATIC_VOLTAGE; table->VRConfig |= (config << VRCONF_MVDD_SHIFT); @@ -2262,6 +2305,17 @@ static int polaris10_enable_ulv(struct pp_hwmgr *hwmgr) return 0; } +static int polaris10_disable_ulv(struct pp_hwmgr *hwmgr) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + struct polaris10_ulv_parm *ulv = &(data->ulv); + + if (ulv->ulv_supported) + return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DisableULV); + + return 0; +} + static int polaris10_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr) { if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, @@ -2282,6 +2336,21 @@ static int polaris10_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr) return 0; } +static int polaris10_disable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr) +{ + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_SclkDeepSleep)) { + if (smum_send_msg_to_smc(hwmgr->smumgr, + PPSMC_MSG_MASTER_DeepSleep_OFF)) { + PP_ASSERT_WITH_CODE(false, + "Attempt to disable Master Deep Sleep switch failed!", + return -1); + } + } + + return 0; +} + static int polaris10_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) { struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); @@ -2379,6 +2448,58 @@ static int polaris10_start_dpm(struct pp_hwmgr *hwmgr) return 0; } +static int polaris10_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + + /* disable SCLK dpm */ + if (!data->sclk_dpm_key_disabled) + PP_ASSERT_WITH_CODE( + (smum_send_msg_to_smc(hwmgr->smumgr, + PPSMC_MSG_DPM_Disable) == 0), + "Failed to disable SCLK DPM!", + return -1); + + /* disable MCLK dpm */ + if (!data->mclk_dpm_key_disabled) { + PP_ASSERT_WITH_CODE( + (smum_send_msg_to_smc(hwmgr->smumgr, + PPSMC_MSG_MCLKDPM_Disable) == 0), + "Failed to disable MCLK DPM!", + return -1); + } + + return 0; +} + +static int polaris10_stop_dpm(struct pp_hwmgr *hwmgr) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + + /* disable general power management */ + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT, + GLOBAL_PWRMGT_EN, 0); + /* disable sclk deep sleep */ + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL, + DYNAMIC_PM_EN, 0); + + /* disable PCIE dpm */ + if (!data->pcie_dpm_key_disabled) { + PP_ASSERT_WITH_CODE( + (smum_send_msg_to_smc(hwmgr->smumgr, + PPSMC_MSG_PCIeDPM_Disable) == 0), + "Failed to disable pcie DPM during DPM Stop Function!", + return -1); + } + + if (polaris10_disable_sclk_mclk_dpm(hwmgr)) { + printk(KERN_ERR "Failed to disable Sclk DPM and Mclk DPM!"); + return -1; + } + + return 0; +} + static void polaris10_set_dpm_event_sources(struct pp_hwmgr *hwmgr, uint32_t sources) { bool protection; @@ -2436,6 +2557,23 @@ static int polaris10_enable_thermal_auto_throttle(struct pp_hwmgr *hwmgr) return polaris10_enable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal); } +static int polaris10_disable_auto_throttle_source(struct pp_hwmgr *hwmgr, + PHM_AutoThrottleSource source) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + + if (data->active_auto_throttle_sources & (1 << source)) { + data->active_auto_throttle_sources &= ~(1 << source); + polaris10_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources); + } + return 0; +} + +static int polaris10_disable_thermal_auto_throttle(struct pp_hwmgr *hwmgr) +{ + return polaris10_disable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal); +} + int polaris10_pcie_performance_request(struct pp_hwmgr *hwmgr) { struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); @@ -2530,6 +2668,10 @@ int polaris10_enable_dpm_tasks(struct pp_hwmgr *hwmgr) PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to enable deep sleep master switch!", result = tmp_result); + tmp_result = polaris10_enable_didt_config(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to enable deep sleep master switch!", result = tmp_result); + tmp_result = polaris10_start_dpm(hwmgr); PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to start DPM!", result = tmp_result); @@ -2559,8 +2701,60 @@ int polaris10_enable_dpm_tasks(struct pp_hwmgr *hwmgr) int polaris10_disable_dpm_tasks(struct pp_hwmgr *hwmgr) { + int tmp_result, result = 0; - return 0; + tmp_result = (polaris10_is_dpm_running(hwmgr)) ? 0 : -1; + PP_ASSERT_WITH_CODE(tmp_result == 0, + "DPM is not running right now, no need to disable DPM!", + return 0); + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_ThermalController)) + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, + GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 1); + + tmp_result = polaris10_disable_power_containment(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to disable power containment!", result = tmp_result); + + tmp_result = polaris10_disable_smc_cac(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to disable SMC CAC!", result = tmp_result); + + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, + CG_SPLL_SPREAD_SPECTRUM, SSEN, 0); + PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, + GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 0); + + tmp_result = polaris10_disable_thermal_auto_throttle(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to disable thermal auto throttle!", result = tmp_result); + + tmp_result = polaris10_stop_dpm(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to stop DPM!", result = tmp_result); + + tmp_result = polaris10_disable_deep_sleep_master_switch(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to disable deep sleep master switch!", result = tmp_result); + + tmp_result = polaris10_disable_ulv(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to disable ULV!", result = tmp_result); + + tmp_result = polaris10_clear_voting_clients(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to clear voting clients!", result = tmp_result); + + tmp_result = polaris10_reset_to_default(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to reset to default!", result = tmp_result); + + tmp_result = polaris10_force_switch_to_arbf0(hwmgr); + PP_ASSERT_WITH_CODE((tmp_result == 0), + "Failed to force to switch arbf0!", result = tmp_result); + + return result; } int polaris10_reset_asic_tasks(struct pp_hwmgr *hwmgr) @@ -2571,13 +2765,6 @@ int polaris10_reset_asic_tasks(struct pp_hwmgr *hwmgr) int polaris10_hwmgr_backend_fini(struct pp_hwmgr *hwmgr) { - struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); - - if (data->soft_pp_table) { - kfree(data->soft_pp_table); - data->soft_pp_table = NULL; - } - return phm_hwmgr_backend_fini(hwmgr); } @@ -2624,17 +2811,22 @@ int polaris10_set_features_platform_caps(struct pp_hwmgr *hwmgr) PHM_PlatformCaps_DynamicUVDState); /* power tune caps Assume disabled */ - phm_cap_unset(hwmgr->platform_descriptor.platformCaps, + phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping); - phm_cap_unset(hwmgr->platform_descriptor.platformCaps, + phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping); - phm_cap_unset(hwmgr->platform_descriptor.platformCaps, + phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping); - phm_cap_unset(hwmgr->platform_descriptor.platformCaps, + phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping); - phm_cap_set(hwmgr->platform_descriptor.platformCaps, - PHM_PlatformCaps_PowerContainment); + if (hwmgr->powercontainment_enabled) + phm_cap_set(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_PowerContainment); + else + phm_cap_unset(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_PowerContainment); + phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_CAC); @@ -2706,12 +2898,12 @@ static int polaris10_get_evv_voltages(struct pp_hwmgr *hwmgr) } } - - PP_ASSERT_WITH_CODE(0 == atomctrl_get_voltage_evv_on_sclk_ai(hwmgr, - VOLTAGE_TYPE_VDDC, sclk, vv_id, &vddc), - "Error retrieving EVV voltage value!", - continue); - + if (atomctrl_get_voltage_evv_on_sclk_ai(hwmgr, + VOLTAGE_TYPE_VDDC, + sclk, vv_id, &vddc) != 0) { + printk(KERN_WARNING "failed to retrieving EVV voltage!\n"); + continue; + } /* need to make sure vddc is less than 2v or else, it could burn the ASIC. * real voltage level in unit of 0.01mv */ @@ -2968,13 +3160,19 @@ int polaris10_patch_voltage_workaround(struct pp_hwmgr *hwmgr) int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr) { - struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + struct polaris10_hwmgr *data; struct pp_atomctrl_gpio_pin_assignment gpio_pin_assignment; uint32_t temp_reg; int result; struct phm_ppt_v1_information *table_info = (struct phm_ppt_v1_information *)(hwmgr->pptable); + data = kzalloc(sizeof(struct polaris10_hwmgr), GFP_KERNEL); + if (data == NULL) + return -ENOMEM; + + hwmgr->backend = data; + data->dll_default_on = false; data->sram_end = SMC_RAM_END; data->mclk_dpm0_activity_target = 0xa; @@ -3063,7 +3261,7 @@ int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr) if (0 == result) { struct cgs_system_info sys_info = {0}; - data->is_tlu_enabled = 0; + data->is_tlu_enabled = false; hwmgr->platform_descriptor.hardwareActivityPerformanceLevels = POLARIS10_MAX_HARDWARE_POWERLEVELS; @@ -3148,7 +3346,7 @@ int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr) sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO; result = cgs_query_system_info(hwmgr->device, &sys_info); if (result) - data->pcie_gen_cap = 0x30007; + data->pcie_gen_cap = AMDGPU_DEFAULT_PCIE_GEN_MASK; else data->pcie_gen_cap = (uint32_t)sys_info.value; if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) @@ -3157,7 +3355,7 @@ int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr) sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW; result = cgs_query_system_info(hwmgr->device, &sys_info); if (result) - data->pcie_lane_cap = 0x2f0000; + data->pcie_lane_cap = AMDGPU_DEFAULT_PCIE_MLW_MASK; else data->pcie_lane_cap = (uint32_t)sys_info.value; @@ -3446,6 +3644,7 @@ static int polaris10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr, hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DisableMclkSwitchingForFrameLock); + disable_mclk_switching = (1 < info.display_count) || disable_mclk_switching_for_frame_lock; @@ -3950,8 +4149,8 @@ static int polaris10_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if ((0 == data->sclk_dpm_key_disabled) && (data->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) { - PP_ASSERT_WITH_CODE(true == polaris10_is_dpm_running(hwmgr), - "Trying to freeze SCLK DPM when DPM is disabled", + PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr), + "Trying to freeze SCLK DPM when DPM is disabled", ); PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_SCLKDPM_FreezeLevel), @@ -3962,8 +4161,8 @@ static int polaris10_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if ((0 == data->mclk_dpm_key_disabled) && (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) { - PP_ASSERT_WITH_CODE(true == polaris10_is_dpm_running(hwmgr), - "Trying to freeze MCLK DPM when DPM is disabled", + PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr), + "Trying to freeze MCLK DPM when DPM is disabled", ); PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_MCLKDPM_FreezeLevel), @@ -4123,7 +4322,6 @@ static int polaris10_trim_single_dpm_states(struct pp_hwmgr *hwmgr, static int polaris10_trim_dpm_states(struct pp_hwmgr *hwmgr, const struct polaris10_power_state *polaris10_ps) { - int result = 0; struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); uint32_t high_limit_count; @@ -4143,7 +4341,7 @@ static int polaris10_trim_dpm_states(struct pp_hwmgr *hwmgr, polaris10_ps->performance_levels[0].memory_clock, polaris10_ps->performance_levels[high_limit_count].memory_clock); - return result; + return 0; } static int polaris10_generate_dpm_level_enable_mask( @@ -4226,25 +4424,20 @@ int polaris10_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate) return polaris10_enable_disable_uvd_dpm(hwmgr, !bgate); } -static int polaris10_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input) +int polaris10_update_vce_dpm(struct pp_hwmgr *hwmgr, bool bgate) { - const struct phm_set_power_state_input *states = - (const struct phm_set_power_state_input *)input; struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); - const struct polaris10_power_state *polaris10_nps = - cast_const_phw_polaris10_power_state(states->pnew_state); - const struct polaris10_power_state *polaris10_cps = - cast_const_phw_polaris10_power_state(states->pcurrent_state); - uint32_t mm_boot_level_offset, mm_boot_level_value; struct phm_ppt_v1_information *table_info = (struct phm_ppt_v1_information *)(hwmgr->pptable); - if (polaris10_nps->vce_clks.evclk > 0 && - (polaris10_cps == NULL || polaris10_cps->vce_clks.evclk == 0)) { - - data->smc_state_table.VceBootLevel = + if (!bgate) { + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_StablePState)) + data->smc_state_table.VceBootLevel = (uint8_t) (table_info->mm_dep_table->count - 1); + else + data->smc_state_table.VceBootLevel = 0; mm_boot_level_offset = data->dpm_table_start + offsetof(SMU74_Discrete_DpmTable, VceBootLevel); @@ -4257,18 +4450,14 @@ static int polaris10_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input) cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value); - if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState)) { + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState)) smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_VCEDPM_SetEnabledMask, (uint32_t)1 << data->smc_state_table.VceBootLevel); - - polaris10_enable_disable_vce_dpm(hwmgr, true); - } else if (polaris10_nps->vce_clks.evclk == 0 && - polaris10_cps != NULL && - polaris10_cps->vce_clks.evclk > 0) - polaris10_enable_disable_vce_dpm(hwmgr, false); } + polaris10_enable_disable_vce_dpm(hwmgr, !bgate); + return 0; } @@ -4353,8 +4542,8 @@ static int polaris10_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) (data->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) { - PP_ASSERT_WITH_CODE(true == polaris10_is_dpm_running(hwmgr), - "Trying to Unfreeze SCLK DPM when DPM is disabled", + PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr), + "Trying to Unfreeze SCLK DPM when DPM is disabled", ); PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_SCLKDPM_UnfreezeLevel), @@ -4365,8 +4554,8 @@ static int polaris10_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if ((0 == data->mclk_dpm_key_disabled) && (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) { - PP_ASSERT_WITH_CODE(true == polaris10_is_dpm_running(hwmgr), - "Trying to Unfreeze MCLK DPM when DPM is disabled", + PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr), + "Trying to Unfreeze MCLK DPM when DPM is disabled", ); PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_SCLKDPM_UnfreezeLevel), @@ -4422,6 +4611,8 @@ static int polaris10_notify_smc_display(struct pp_hwmgr *hwmgr) return (smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)PPSMC_HasDisplay) == 0) ? 0 : -EINVAL; } + + static int polaris10_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *input) { int tmp_result, result = 0; @@ -4455,11 +4646,6 @@ static int polaris10_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *i "Failed to generate DPM level enabled mask!", result = tmp_result); - tmp_result = polaris10_update_vce_dpm(hwmgr, input); - PP_ASSERT_WITH_CODE((0 == tmp_result), - "Failed to update VCE DPM!", - result = tmp_result); - tmp_result = polaris10_update_sclk_threshold(hwmgr); PP_ASSERT_WITH_CODE((0 == tmp_result), "Failed to update SCLK threshold!", @@ -4530,6 +4716,7 @@ int polaris10_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwm if (num_active_displays > 1) /* to do && (pHwMgr->pPECI->displayConfiguration.bMultiMonitorInSync != TRUE)) */ polaris10_notify_smc_display_change(hwmgr, false); + return 0; } @@ -4579,6 +4766,7 @@ int polaris10_program_display_gap(struct pp_hwmgr *hwmgr) cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start + offsetof(SMU74_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us)); + return 0; } @@ -4820,42 +5008,6 @@ int polaris10_setup_asic_task(struct pp_hwmgr *hwmgr) return result; } -static int polaris10_get_pp_table(struct pp_hwmgr *hwmgr, char **table) -{ - struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); - - if (!data->soft_pp_table) { - data->soft_pp_table = kmemdup(hwmgr->soft_pp_table, - hwmgr->soft_pp_table_size, - GFP_KERNEL); - if (!data->soft_pp_table) - return -ENOMEM; - } - - *table = (char *)&data->soft_pp_table; - - return hwmgr->soft_pp_table_size; -} - -static int polaris10_set_pp_table(struct pp_hwmgr *hwmgr, const char *buf, size_t size) -{ - struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); - - if (!data->soft_pp_table) { - data->soft_pp_table = kzalloc(hwmgr->soft_pp_table_size, GFP_KERNEL); - if (!data->soft_pp_table) - return -ENOMEM; - } - - memcpy(data->soft_pp_table, buf, size); - - hwmgr->soft_pp_table = data->soft_pp_table; - - /* TODO: re-init powerplay to implement modified pptable */ - - return 0; -} - static int polaris10_force_clock_level(struct pp_hwmgr *hwmgr, enum pp_clock_type type, uint32_t mask) { @@ -4998,6 +5150,89 @@ static int polaris10_get_fan_control_mode(struct pp_hwmgr *hwmgr) CG_FDO_CTRL2, FDO_PWM_MODE); } +static int polaris10_get_sclk_od(struct pp_hwmgr *hwmgr) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + struct polaris10_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table); + struct polaris10_single_dpm_table *golden_sclk_table = + &(data->golden_dpm_table.sclk_table); + int value; + + value = (sclk_table->dpm_levels[sclk_table->count - 1].value - + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) * + 100 / + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value; + + return value; +} + +static int polaris10_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + struct polaris10_single_dpm_table *golden_sclk_table = + &(data->golden_dpm_table.sclk_table); + struct pp_power_state *ps; + struct polaris10_power_state *polaris10_ps; + + if (value > 20) + value = 20; + + ps = hwmgr->request_ps; + + if (ps == NULL) + return -EINVAL; + + polaris10_ps = cast_phw_polaris10_power_state(&ps->hardware); + + polaris10_ps->performance_levels[polaris10_ps->performance_level_count - 1].engine_clock = + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value * + value / 100 + + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value; + + return 0; +} + +static int polaris10_get_mclk_od(struct pp_hwmgr *hwmgr) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + struct polaris10_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table); + struct polaris10_single_dpm_table *golden_mclk_table = + &(data->golden_dpm_table.mclk_table); + int value; + + value = (mclk_table->dpm_levels[mclk_table->count - 1].value - + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) * + 100 / + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value; + + return value; +} + +static int polaris10_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + struct polaris10_single_dpm_table *golden_mclk_table = + &(data->golden_dpm_table.mclk_table); + struct pp_power_state *ps; + struct polaris10_power_state *polaris10_ps; + + if (value > 20) + value = 20; + + ps = hwmgr->request_ps; + + if (ps == NULL) + return -EINVAL; + + polaris10_ps = cast_phw_polaris10_power_state(&ps->hardware); + + polaris10_ps->performance_levels[polaris10_ps->performance_level_count - 1].memory_clock = + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value * + value / 100 + + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value; + + return 0; +} static const struct pp_hwmgr_func polaris10_hwmgr_funcs = { .backend_init = &polaris10_hwmgr_backend_init, .backend_fini = &polaris10_hwmgr_backend_fini, @@ -5036,22 +5271,17 @@ static const struct pp_hwmgr_func polaris10_hwmgr_funcs = { .check_states_equal = polaris10_check_states_equal, .set_fan_control_mode = polaris10_set_fan_control_mode, .get_fan_control_mode = polaris10_get_fan_control_mode, - .get_pp_table = polaris10_get_pp_table, - .set_pp_table = polaris10_set_pp_table, .force_clock_level = polaris10_force_clock_level, .print_clock_levels = polaris10_print_clock_levels, .enable_per_cu_power_gating = polaris10_phm_enable_per_cu_power_gating, + .get_sclk_od = polaris10_get_sclk_od, + .set_sclk_od = polaris10_set_sclk_od, + .get_mclk_od = polaris10_get_mclk_od, + .set_mclk_od = polaris10_set_mclk_od, }; int polaris10_hwmgr_init(struct pp_hwmgr *hwmgr) { - struct polaris10_hwmgr *data; - - data = kzalloc (sizeof(struct polaris10_hwmgr), GFP_KERNEL); - if (data == NULL) - return -ENOMEM; - - hwmgr->backend = data; hwmgr->hwmgr_func = &polaris10_hwmgr_funcs; hwmgr->pptable_func = &tonga_pptable_funcs; pp_polaris10_thermal_initialize(hwmgr); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h index afc343482..33c33947e 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h @@ -309,10 +309,6 @@ struct polaris10_hwmgr { uint32_t up_hyst; uint32_t disable_dpm_mask; bool apply_optimized_settings; - - /* soft pptable for re-uploading into smu */ - void *soft_pp_table; - uint32_t avfs_vdroop_override_setting; bool apply_avfs_cks_off_voltage; uint32_t frame_time_x2; @@ -356,6 +352,6 @@ int polaris10_hwmgr_init(struct pp_hwmgr *hwmgr); int polaris10_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate); int polaris10_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate); int polaris10_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable); - +int polaris10_update_vce_dpm(struct pp_hwmgr *hwmgr, bool bgate); #endif diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.c index ae96f14b8..b9cb240a1 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.c @@ -28,10 +28,360 @@ #include "polaris10_smumgr.h" #include "smu74_discrete.h" #include "pp_debug.h" +#include "gca/gfx_8_0_d.h" +#include "gca/gfx_8_0_sh_mask.h" +#include "oss/oss_3_0_sh_mask.h" #define VOLTAGE_SCALE 4 #define POWERTUNE_DEFAULT_SET_MAX 1 +uint32_t DIDTBlock_Info = SQ_IR_MASK | TCP_IR_MASK | TD_PCC_MASK; + +struct polaris10_pt_config_reg GCCACConfig_Polaris10[] = { +/* --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + * Offset Mask Shift Value Type + * --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + */ + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x00060013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x00860013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x01060013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x01860013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x02060013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x02860013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x03060013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x03860013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x04060013, POLARIS10_CONFIGREG_GC_CAC_IND }, + + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x000E0013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x008E0013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x010E0013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x018E0013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x020E0013, POLARIS10_CONFIGREG_GC_CAC_IND }, + + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x00100013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x00900013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x01100013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x01900013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x02100013, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x02900013, POLARIS10_CONFIGREG_GC_CAC_IND }, + + { 0xFFFFFFFF } +}; + +struct polaris10_pt_config_reg GCCACConfig_Polaris11[] = { +/* --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + * Offset Mask Shift Value Type + * --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + */ + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x00060011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x00860011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x01060011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x01860011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x02060011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x02860011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x03060011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x03860011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x04060011, POLARIS10_CONFIGREG_GC_CAC_IND }, + + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x000E0011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x008E0011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x010E0011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x018E0011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x020E0011, POLARIS10_CONFIGREG_GC_CAC_IND }, + + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x00100011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x00900011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x01100011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x01900011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x02100011, POLARIS10_CONFIGREG_GC_CAC_IND }, + { ixGC_CAC_CNTL, 0xFFFFFFFF, 0, 0x02900011, POLARIS10_CONFIGREG_GC_CAC_IND }, + + { 0xFFFFFFFF } +}; + +struct polaris10_pt_config_reg DIDTConfig_Polaris10[] = { +/* --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + * Offset Mask Shift Value Type + * --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + */ + { ixDIDT_SQ_WEIGHT0_3, DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK, DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT, 0x0073, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT0_3, DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK, DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT, 0x00ab, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT0_3, DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK, DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT, 0x0084, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT0_3, DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK, DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT, 0x005a, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_WEIGHT4_7, DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK, DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT, 0x0067, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT4_7, DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK, DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT, 0x0084, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT4_7, DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK, DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT, 0x0027, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT4_7, DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK, DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT, 0x0046, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_WEIGHT8_11, DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK, DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT, 0x00aa, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT8_11, DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK, DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT8_11, DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK, DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT8_11, DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK, DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_CTRL1, DIDT_SQ_CTRL1__MIN_POWER_MASK, DIDT_SQ_CTRL1__MIN_POWER__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL1, DIDT_SQ_CTRL1__MAX_POWER_MASK, DIDT_SQ_CTRL1__MAX_POWER__SHIFT, 0xffff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_CTRL_OCP, DIDT_SQ_CTRL_OCP__UNUSED_0_MASK, DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL_OCP, DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK, DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT, 0xffff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK, DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT, 0x3853, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__UNUSED_0_MASK, DIDT_SQ_CTRL2__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK, DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT, 0x005a, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__UNUSED_1_MASK, DIDT_SQ_CTRL2__UNUSED_1__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK, DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__UNUSED_2_MASK, DIDT_SQ_CTRL2__UNUSED_2__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_STALL_CTRL, DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK, DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_STALL_CTRL, DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK, DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_STALL_CTRL, DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK, DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_STALL_CTRL, DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK, DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_STALL_CTRL, DIDT_SQ_STALL_CTRL__UNUSED_0_MASK, DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_TUNING_CTRL, DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK, DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_TUNING_CTRL, DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK, DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT, 0x3853, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_TUNING_CTRL, DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK, DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT, 0x3153, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_TUNING_CTRL, DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK, DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK, DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK, DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__PHASE_OFFSET_MASK, DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK, DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK, DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK, DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT, 0x0010, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK, DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT, 0x0010, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__UNUSED_0_MASK, DIDT_SQ_CTRL0__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_WEIGHT0_3, DIDT_TD_WEIGHT0_3__WEIGHT0_MASK, DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT, 0x000a, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT0_3, DIDT_TD_WEIGHT0_3__WEIGHT1_MASK, DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT, 0x0010, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT0_3, DIDT_TD_WEIGHT0_3__WEIGHT2_MASK, DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT, 0x0017, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT0_3, DIDT_TD_WEIGHT0_3__WEIGHT3_MASK, DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT, 0x002f, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_WEIGHT4_7, DIDT_TD_WEIGHT4_7__WEIGHT4_MASK, DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT, 0x0046, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT4_7, DIDT_TD_WEIGHT4_7__WEIGHT5_MASK, DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT, 0x005d, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT4_7, DIDT_TD_WEIGHT4_7__WEIGHT6_MASK, DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT4_7, DIDT_TD_WEIGHT4_7__WEIGHT7_MASK, DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_CTRL1, DIDT_TD_CTRL1__MIN_POWER_MASK, DIDT_TD_CTRL1__MIN_POWER__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL1, DIDT_TD_CTRL1__MAX_POWER_MASK, DIDT_TD_CTRL1__MAX_POWER__SHIFT, 0xffff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_CTRL_OCP, DIDT_TD_CTRL_OCP__UNUSED_0_MASK, DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL_OCP, DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK, DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT, 0x00ff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK, DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT, 0x3fff, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__UNUSED_0_MASK, DIDT_TD_CTRL2__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK, DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT, 0x000f, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__UNUSED_1_MASK, DIDT_TD_CTRL2__UNUSED_1__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK, DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__UNUSED_2_MASK, DIDT_TD_CTRL2__UNUSED_2__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_STALL_CTRL, DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK, DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_STALL_CTRL, DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK, DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_STALL_CTRL, DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK, DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_STALL_CTRL, DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK, DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_STALL_CTRL, DIDT_TD_STALL_CTRL__UNUSED_0_MASK, DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_TUNING_CTRL, DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK, DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_TUNING_CTRL, DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK, DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT, 0x0dde, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_TUNING_CTRL, DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK, DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT, 0x0dde, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_TUNING_CTRL, DIDT_TD_TUNING_CTRL__UNUSED_0_MASK, DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK, DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__USE_REF_CLOCK_MASK, DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__PHASE_OFFSET_MASK, DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK, DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK, DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK, DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT, 0x0009, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK, DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT, 0x0009, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__UNUSED_0_MASK, DIDT_TD_CTRL0__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_WEIGHT0_3, DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK, DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT, 0x0004, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT0_3, DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK, DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT, 0x0037, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT0_3, DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK, DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT0_3, DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK, DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT, 0x00ff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_WEIGHT4_7, DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK, DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT, 0x0054, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT4_7, DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK, DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT4_7, DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK, DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT4_7, DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK, DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_CTRL1, DIDT_TCP_CTRL1__MIN_POWER_MASK, DIDT_TCP_CTRL1__MIN_POWER__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL1, DIDT_TCP_CTRL1__MAX_POWER_MASK, DIDT_TCP_CTRL1__MAX_POWER__SHIFT, 0xffff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_CTRL_OCP, DIDT_TCP_CTRL_OCP__UNUSED_0_MASK, DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL_OCP, DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK, DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT, 0xffff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK, DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT, 0x3dde, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__UNUSED_0_MASK, DIDT_TCP_CTRL2__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK, DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT, 0x0032, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__UNUSED_1_MASK, DIDT_TCP_CTRL2__UNUSED_1__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK, DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__UNUSED_2_MASK, DIDT_TCP_CTRL2__UNUSED_2__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_STALL_CTRL, DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK, DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_STALL_CTRL, DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK, DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_STALL_CTRL, DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK, DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_STALL_CTRL, DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK, DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_STALL_CTRL, DIDT_TCP_STALL_CTRL__UNUSED_0_MASK, DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_TUNING_CTRL, DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK, DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_TUNING_CTRL, DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK, DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT, 0x3dde, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_TUNING_CTRL, DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK, DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT, 0x3dde, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_TUNING_CTRL, DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK, DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK, DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK, DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__PHASE_OFFSET_MASK, DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK, DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK, DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK, DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT, 0x0010, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK, DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT, 0x0010, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__UNUSED_0_MASK, DIDT_TCP_CTRL0__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { 0xFFFFFFFF } +}; + +struct polaris10_pt_config_reg DIDTConfig_Polaris11[] = { +/* --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + * Offset Mask Shift Value Type + * --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + */ + { ixDIDT_SQ_WEIGHT0_3, DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK, DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT, 0x0073, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT0_3, DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK, DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT, 0x00ab, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT0_3, DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK, DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT, 0x0084, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT0_3, DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK, DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT, 0x005a, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_WEIGHT4_7, DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK, DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT, 0x0067, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT4_7, DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK, DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT, 0x0084, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT4_7, DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK, DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT, 0x0027, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT4_7, DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK, DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT, 0x0046, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_WEIGHT8_11, DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK, DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT, 0x00aa, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT8_11, DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK, DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT8_11, DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK, DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_WEIGHT8_11, DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK, DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_CTRL1, DIDT_SQ_CTRL1__MIN_POWER_MASK, DIDT_SQ_CTRL1__MIN_POWER__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL1, DIDT_SQ_CTRL1__MAX_POWER_MASK, DIDT_SQ_CTRL1__MAX_POWER__SHIFT, 0xffff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_CTRL_OCP, DIDT_SQ_CTRL_OCP__UNUSED_0_MASK, DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL_OCP, DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK, DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT, 0xffff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK, DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT, 0x3853, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__UNUSED_0_MASK, DIDT_SQ_CTRL2__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK, DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT, 0x005a, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__UNUSED_1_MASK, DIDT_SQ_CTRL2__UNUSED_1__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK, DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL2, DIDT_SQ_CTRL2__UNUSED_2_MASK, DIDT_SQ_CTRL2__UNUSED_2__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_STALL_CTRL, DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK, DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_STALL_CTRL, DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK, DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_STALL_CTRL, DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK, DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_STALL_CTRL, DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK, DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_STALL_CTRL, DIDT_SQ_STALL_CTRL__UNUSED_0_MASK, DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_TUNING_CTRL, DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK, DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_TUNING_CTRL, DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK, DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT, 0x3853, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_TUNING_CTRL, DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK, DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT, 0x3153, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_TUNING_CTRL, DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK, DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK, DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK, DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__PHASE_OFFSET_MASK, DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK, DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK, DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK, DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT, 0x0010, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK, DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT, 0x0010, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_SQ_CTRL0, DIDT_SQ_CTRL0__UNUSED_0_MASK, DIDT_SQ_CTRL0__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_WEIGHT0_3, DIDT_TD_WEIGHT0_3__WEIGHT0_MASK, DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT, 0x000a, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT0_3, DIDT_TD_WEIGHT0_3__WEIGHT1_MASK, DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT, 0x0010, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT0_3, DIDT_TD_WEIGHT0_3__WEIGHT2_MASK, DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT, 0x0017, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT0_3, DIDT_TD_WEIGHT0_3__WEIGHT3_MASK, DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT, 0x002f, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_WEIGHT4_7, DIDT_TD_WEIGHT4_7__WEIGHT4_MASK, DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT, 0x0046, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT4_7, DIDT_TD_WEIGHT4_7__WEIGHT5_MASK, DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT, 0x005d, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT4_7, DIDT_TD_WEIGHT4_7__WEIGHT6_MASK, DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_WEIGHT4_7, DIDT_TD_WEIGHT4_7__WEIGHT7_MASK, DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_CTRL1, DIDT_TD_CTRL1__MIN_POWER_MASK, DIDT_TD_CTRL1__MIN_POWER__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL1, DIDT_TD_CTRL1__MAX_POWER_MASK, DIDT_TD_CTRL1__MAX_POWER__SHIFT, 0xffff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_CTRL_OCP, DIDT_TD_CTRL_OCP__UNUSED_0_MASK, DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL_OCP, DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK, DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT, 0x00ff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK, DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT, 0x3fff, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__UNUSED_0_MASK, DIDT_TD_CTRL2__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK, DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT, 0x000f, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__UNUSED_1_MASK, DIDT_TD_CTRL2__UNUSED_1__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK, DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL2, DIDT_TD_CTRL2__UNUSED_2_MASK, DIDT_TD_CTRL2__UNUSED_2__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_STALL_CTRL, DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK, DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_STALL_CTRL, DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK, DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_STALL_CTRL, DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK, DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_STALL_CTRL, DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK, DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_STALL_CTRL, DIDT_TD_STALL_CTRL__UNUSED_0_MASK, DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_TUNING_CTRL, DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK, DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_TUNING_CTRL, DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK, DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT, 0x0dde, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_TUNING_CTRL, DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK, DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT, 0x0dde, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_TUNING_CTRL, DIDT_TD_TUNING_CTRL__UNUSED_0_MASK, DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK, DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__USE_REF_CLOCK_MASK, DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__PHASE_OFFSET_MASK, DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK, DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK, DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK, DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT, 0x0008, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK, DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT, 0x0008, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TD_CTRL0, DIDT_TD_CTRL0__UNUSED_0_MASK, DIDT_TD_CTRL0__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_WEIGHT0_3, DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK, DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT, 0x0004, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT0_3, DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK, DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT, 0x0037, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT0_3, DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK, DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT0_3, DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK, DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT, 0x00ff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_WEIGHT4_7, DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK, DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT, 0x0054, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT4_7, DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK, DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT4_7, DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK, DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_WEIGHT4_7, DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK, DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_CTRL1, DIDT_TCP_CTRL1__MIN_POWER_MASK, DIDT_TCP_CTRL1__MIN_POWER__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL1, DIDT_TCP_CTRL1__MAX_POWER_MASK, DIDT_TCP_CTRL1__MAX_POWER__SHIFT, 0xffff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_CTRL_OCP, DIDT_TCP_CTRL_OCP__UNUSED_0_MASK, DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL_OCP, DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK, DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT, 0xffff, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK, DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT, 0x3dde, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__UNUSED_0_MASK, DIDT_TCP_CTRL2__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK, DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT, 0x0032, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__UNUSED_1_MASK, DIDT_TCP_CTRL2__UNUSED_1__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK, DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL2, DIDT_TCP_CTRL2__UNUSED_2_MASK, DIDT_TCP_CTRL2__UNUSED_2__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_STALL_CTRL, DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK, DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_STALL_CTRL, DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK, DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_STALL_CTRL, DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK, DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_STALL_CTRL, DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK, DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_STALL_CTRL, DIDT_TCP_STALL_CTRL__UNUSED_0_MASK, DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_TUNING_CTRL, DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK, DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_TUNING_CTRL, DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK, DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT, 0x3dde, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_TUNING_CTRL, DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK, DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT, 0x3dde, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_TUNING_CTRL, DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK, DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK, DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT, 0x0001, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK, DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__PHASE_OFFSET_MASK, DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK, DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK, DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK, DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT, 0x0010, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK, DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT, 0x0010, POLARIS10_CONFIGREG_DIDT_IND }, + { ixDIDT_TCP_CTRL0, DIDT_TCP_CTRL0__UNUSED_0_MASK, DIDT_TCP_CTRL0__UNUSED_0__SHIFT, 0x0000, POLARIS10_CONFIGREG_DIDT_IND }, + { 0xFFFFFFFF } +}; + static const struct polaris10_pt_defaults polaris10_power_tune_data_set_array[POWERTUNE_DEFAULT_SET_MAX] = { /* sviLoadLIneEn, SviLoadLineVddC, TDC_VDDC_ThrottleReleaseLimitPerc, TDC_MAWt, * TdcWaterfallCtl, DTEAmbientTempBase, DisplayCac, BAPM_TEMP_GRADIENT */ @@ -209,6 +559,187 @@ static int polaris10_min_max_vgnb_lpml_id_from_bapm_vddc(struct pp_hwmgr *hwmgr) return 0; } +static int polaris10_enable_didt(struct pp_hwmgr *hwmgr, const bool enable) +{ + + uint32_t en = enable ? 1 : 0; + int32_t result = 0; + uint32_t data; + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping)) { + data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_CTRL0); + data &= ~DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK; + data |= ((en << DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_CTRL0, data); + DIDTBlock_Info &= ~SQ_Enable_MASK; + DIDTBlock_Info |= en << SQ_Enable_SHIFT; + } + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping)) { + data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_CTRL0); + data &= ~DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK; + data |= ((en << DIDT_DB_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_CTRL0, data); + DIDTBlock_Info &= ~DB_Enable_MASK; + DIDTBlock_Info |= en << DB_Enable_SHIFT; + } + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping)) { + data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_CTRL0); + data &= ~DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK; + data |= ((en << DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_CTRL0, data); + DIDTBlock_Info &= ~TD_Enable_MASK; + DIDTBlock_Info |= en << TD_Enable_SHIFT; + } + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) { + data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_CTRL0); + data &= ~DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK; + data |= ((en << DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK); + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_CTRL0, data); + DIDTBlock_Info &= ~TCP_Enable_MASK; + DIDTBlock_Info |= en << TCP_Enable_SHIFT; + } + + if (enable) + result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_Didt_Block_Function, DIDTBlock_Info); + + return result; +} + +static int polaris10_program_pt_config_registers(struct pp_hwmgr *hwmgr, + struct polaris10_pt_config_reg *cac_config_regs) +{ + struct polaris10_pt_config_reg *config_regs = cac_config_regs; + uint32_t cache = 0; + uint32_t data = 0; + + PP_ASSERT_WITH_CODE((config_regs != NULL), "Invalid config register table.", return -EINVAL); + + while (config_regs->offset != 0xFFFFFFFF) { + if (config_regs->type == POLARIS10_CONFIGREG_CACHE) + cache |= ((config_regs->value << config_regs->shift) & config_regs->mask); + else { + switch (config_regs->type) { + case POLARIS10_CONFIGREG_SMC_IND: + data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, config_regs->offset); + break; + + case POLARIS10_CONFIGREG_DIDT_IND: + data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, config_regs->offset); + break; + + case POLARIS10_CONFIGREG_GC_CAC_IND: + data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG_GC_CAC, config_regs->offset); + break; + + default: + data = cgs_read_register(hwmgr->device, config_regs->offset); + break; + } + + data &= ~config_regs->mask; + data |= ((config_regs->value << config_regs->shift) & config_regs->mask); + data |= cache; + + switch (config_regs->type) { + case POLARIS10_CONFIGREG_SMC_IND: + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, config_regs->offset, data); + break; + + case POLARIS10_CONFIGREG_DIDT_IND: + cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, config_regs->offset, data); + break; + + case POLARIS10_CONFIGREG_GC_CAC_IND: + cgs_write_ind_register(hwmgr->device, CGS_IND_REG_GC_CAC, config_regs->offset, data); + break; + + default: + cgs_write_register(hwmgr->device, config_regs->offset, data); + break; + } + cache = 0; + } + + config_regs++; + } + + return 0; +} + +int polaris10_enable_didt_config(struct pp_hwmgr *hwmgr) +{ + int result; + uint32_t num_se = 0; + uint32_t count, value, value2; + struct cgs_system_info sys_info = {0}; + + sys_info.size = sizeof(struct cgs_system_info); + sys_info.info_id = CGS_SYSTEM_INFO_GFX_SE_INFO; + result = cgs_query_system_info(hwmgr->device, &sys_info); + + + if (result == 0) + num_se = sys_info.value; + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping) || + phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping) || + phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping) || + phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) { + + /* TO DO Pre DIDT disable clock gating */ + value = 0; + value2 = cgs_read_register(hwmgr->device, mmGRBM_GFX_INDEX); + for (count = 0; count < num_se; count++) { + value = SYS_GRBM_GFX_INDEX_DATA__INSTANCE_BROADCAST_WRITES_MASK + | SYS_GRBM_GFX_INDEX_DATA__SH_BROADCAST_WRITES_MASK + | (count << SYS_GRBM_GFX_INDEX_DATA__SE_INDEX__SHIFT); + cgs_write_register(hwmgr->device, mmGRBM_GFX_INDEX, value); + + if (hwmgr->chip_id == CHIP_POLARIS10) { + result = polaris10_program_pt_config_registers(hwmgr, GCCACConfig_Polaris10); + PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result); + result = polaris10_program_pt_config_registers(hwmgr, DIDTConfig_Polaris10); + PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result); + } else if (hwmgr->chip_id == CHIP_POLARIS11) { + result = polaris10_program_pt_config_registers(hwmgr, GCCACConfig_Polaris11); + PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result); + result = polaris10_program_pt_config_registers(hwmgr, DIDTConfig_Polaris11); + PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result); + } + } + cgs_write_register(hwmgr->device, mmGRBM_GFX_INDEX, value2); + + result = polaris10_enable_didt(hwmgr, true); + PP_ASSERT_WITH_CODE((result == 0), "EnableDiDt failed.", return result); + + /* TO DO Post DIDT enable clock gating */ + } + + return 0; +} + +int polaris10_disable_didt_config(struct pp_hwmgr *hwmgr) +{ + int result; + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping) || + phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping) || + phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping) || + phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) { + /* TO DO Pre DIDT disable clock gating */ + + result = polaris10_enable_didt(hwmgr, false); + PP_ASSERT_WITH_CODE((result == 0), "Post DIDT enable clock gating failed.", return result); + /* TO DO Post DIDT enable clock gating */ + } + + return 0; +} + + static int polaris10_populate_bapm_vddc_base_leakage_sidd(struct pp_hwmgr *hwmgr) { struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); @@ -312,6 +843,23 @@ int polaris10_enable_smc_cac(struct pp_hwmgr *hwmgr) return result; } +int polaris10_disable_smc_cac(struct pp_hwmgr *hwmgr) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + int result = 0; + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_CAC) && data->cac_enabled) { + int smc_result = smum_send_msg_to_smc(hwmgr->smumgr, + (uint16_t)(PPSMC_MSG_DisableCac)); + PP_ASSERT_WITH_CODE((smc_result == 0), + "Failed to disable CAC in SMC.", result = -1); + + data->cac_enabled = false; + } + return result; +} + int polaris10_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n) { struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); @@ -373,6 +921,48 @@ int polaris10_enable_power_containment(struct pp_hwmgr *hwmgr) return result; } +int polaris10_disable_power_containment(struct pp_hwmgr *hwmgr) +{ + struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend); + int result = 0; + + if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, + PHM_PlatformCaps_PowerContainment) && + data->power_containment_features) { + int smc_result; + + if (data->power_containment_features & + POWERCONTAINMENT_FEATURE_TDCLimit) { + smc_result = smum_send_msg_to_smc(hwmgr->smumgr, + (uint16_t)(PPSMC_MSG_TDCLimitDisable)); + PP_ASSERT_WITH_CODE((smc_result == 0), + "Failed to disable TDCLimit in SMC.", + result = smc_result); + } + + if (data->power_containment_features & + POWERCONTAINMENT_FEATURE_DTE) { + smc_result = smum_send_msg_to_smc(hwmgr->smumgr, + (uint16_t)(PPSMC_MSG_DisableDTE)); + PP_ASSERT_WITH_CODE((smc_result == 0), + "Failed to disable DTE in SMC.", + result = smc_result); + } + + if (data->power_containment_features & + POWERCONTAINMENT_FEATURE_PkgPwrLimit) { + smc_result = smum_send_msg_to_smc(hwmgr->smumgr, + (uint16_t)(PPSMC_MSG_PkgPwrLimitDisable)); + PP_ASSERT_WITH_CODE((smc_result == 0), + "Failed to disable PkgPwrTracking in SMC.", + result = smc_result); + } + data->power_containment_features = 0; + } + + return result; +} + int polaris10_power_control_set_level(struct pp_hwmgr *hwmgr) { struct phm_ppt_v1_information *table_info = diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.h b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.h index 68bc1cb6d..bc78e28f0 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.h @@ -27,15 +27,37 @@ enum polaris10_pt_config_reg_type { POLARIS10_CONFIGREG_MMR = 0, POLARIS10_CONFIGREG_SMC_IND, POLARIS10_CONFIGREG_DIDT_IND, + POLARIS10_CONFIGREG_GC_CAC_IND, POLARIS10_CONFIGREG_CACHE, POLARIS10_CONFIGREG_MAX }; +#define DIDT_SQ_CTRL0__UNUSED_0_MASK 0xfffc0000 +#define DIDT_SQ_CTRL0__UNUSED_0__SHIFT 0x12 +#define DIDT_TD_CTRL0__UNUSED_0_MASK 0xfffc0000 +#define DIDT_TD_CTRL0__UNUSED_0__SHIFT 0x12 +#define DIDT_TCP_CTRL0__UNUSED_0_MASK 0xfffc0000 +#define DIDT_TCP_CTRL0__UNUSED_0__SHIFT 0x12 +#define DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK 0xc0000000 +#define DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT 0x0000001e +#define DIDT_TD_TUNING_CTRL__UNUSED_0_MASK 0xc0000000 +#define DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT 0x0000001e +#define DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK 0xc0000000 +#define DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT 0x0000001e + /* PowerContainment Features */ #define POWERCONTAINMENT_FEATURE_DTE 0x00000001 #define POWERCONTAINMENT_FEATURE_TDCLimit 0x00000002 #define POWERCONTAINMENT_FEATURE_PkgPwrLimit 0x00000004 +#define ixGC_CAC_CNTL 0x0000 +#define ixDIDT_SQ_STALL_CTRL 0x0004 +#define ixDIDT_SQ_TUNING_CTRL 0x0005 +#define ixDIDT_TD_STALL_CTRL 0x0044 +#define ixDIDT_TD_TUNING_CTRL 0x0045 +#define ixDIDT_TCP_STALL_CTRL 0x0064 +#define ixDIDT_TCP_TUNING_CTRL 0x0065 + struct polaris10_pt_config_reg { uint32_t offset; uint32_t mask; @@ -62,9 +84,11 @@ void polaris10_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr); int polaris10_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr); int polaris10_populate_pm_fuses(struct pp_hwmgr *hwmgr); int polaris10_enable_smc_cac(struct pp_hwmgr *hwmgr); +int polaris10_disable_smc_cac(struct pp_hwmgr *hwmgr); int polaris10_enable_power_containment(struct pp_hwmgr *hwmgr); +int polaris10_disable_power_containment(struct pp_hwmgr *hwmgr); int polaris10_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n); int polaris10_power_control_set_level(struct pp_hwmgr *hwmgr); - +int polaris10_enable_didt_config(struct pp_hwmgr *hwmgr); #endif /* POLARIS10_POWERTUNE_H */ diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c index a3c38bbd1..1944d289f 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c @@ -66,7 +66,7 @@ int acpi_pcie_perf_request(void *device, uint8_t perf_req, bool advertise) int result; struct cgs_system_info info = {0}; - if( 0 != acpi_atcs_notify_pcie_device_ready(device)) + if (acpi_atcs_notify_pcie_device_ready(device)) return -EINVAL; info.size = sizeof(struct cgs_system_info); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c index ffc7c0dd3..26f3e30d0 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c @@ -179,13 +179,12 @@ int atomctrl_set_engine_dram_timings_rv770( /* They are both in 10KHz Units. */ engine_clock_parameters.ulTargetEngineClock = - (uint32_t) engine_clock & SET_CLOCK_FREQ_MASK; - engine_clock_parameters.ulTargetEngineClock |= - (COMPUTE_ENGINE_PLL_PARAM << 24); + cpu_to_le32((engine_clock & SET_CLOCK_FREQ_MASK) | + ((COMPUTE_ENGINE_PLL_PARAM << 24))); /* in 10 khz units.*/ engine_clock_parameters.sReserved.ulClock = - (uint32_t) memory_clock & SET_CLOCK_FREQ_MASK; + cpu_to_le32(memory_clock & SET_CLOCK_FREQ_MASK); return cgs_atom_exec_cmd_table(hwmgr->device, GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings), &engine_clock_parameters); @@ -252,7 +251,7 @@ int atomctrl_get_memory_pll_dividers_si( COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_1 mpll_parameters; int result; - mpll_parameters.ulClock = (uint32_t) clock_value; + mpll_parameters.ulClock = cpu_to_le32(clock_value); mpll_parameters.ucInputFlag = (uint8_t)((strobe_mode) ? 1 : 0); result = cgs_atom_exec_cmd_table @@ -262,9 +261,9 @@ int atomctrl_get_memory_pll_dividers_si( if (0 == result) { mpll_param->mpll_fb_divider.clk_frac = - mpll_parameters.ulFbDiv.usFbDivFrac; + le16_to_cpu(mpll_parameters.ulFbDiv.usFbDivFrac); mpll_param->mpll_fb_divider.cl_kf = - mpll_parameters.ulFbDiv.usFbDiv; + le16_to_cpu(mpll_parameters.ulFbDiv.usFbDiv); mpll_param->mpll_post_divider = (uint32_t)mpll_parameters.ucPostDiv; mpll_param->vco_mode = @@ -300,7 +299,7 @@ int atomctrl_get_memory_pll_dividers_vi(struct pp_hwmgr *hwmgr, COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_2 mpll_parameters; int result; - mpll_parameters.ulClock.ulClock = (uint32_t)clock_value; + mpll_parameters.ulClock.ulClock = cpu_to_le32(clock_value); result = cgs_atom_exec_cmd_table(hwmgr->device, GetIndexIntoMasterTable(COMMAND, ComputeMemoryClockParam), @@ -320,7 +319,7 @@ int atomctrl_get_engine_pll_dividers_kong(struct pp_hwmgr *hwmgr, COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 pll_parameters; int result; - pll_parameters.ulClock = clock_value; + pll_parameters.ulClock = cpu_to_le32(clock_value); result = cgs_atom_exec_cmd_table (hwmgr->device, @@ -329,7 +328,7 @@ int atomctrl_get_engine_pll_dividers_kong(struct pp_hwmgr *hwmgr, if (0 == result) { dividers->pll_post_divider = pll_parameters.ucPostDiv; - dividers->real_clock = pll_parameters.ulClock; + dividers->real_clock = le32_to_cpu(pll_parameters.ulClock); } return result; @@ -343,7 +342,7 @@ int atomctrl_get_engine_pll_dividers_vi( COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 pll_patameters; int result; - pll_patameters.ulClock.ulClock = clock_value; + pll_patameters.ulClock.ulClock = cpu_to_le32(clock_value); pll_patameters.ulClock.ucPostDiv = COMPUTE_GPUCLK_INPUT_FLAG_SCLK; result = cgs_atom_exec_cmd_table @@ -355,12 +354,12 @@ int atomctrl_get_engine_pll_dividers_vi( dividers->pll_post_divider = pll_patameters.ulClock.ucPostDiv; dividers->real_clock = - pll_patameters.ulClock.ulClock; + le32_to_cpu(pll_patameters.ulClock.ulClock); dividers->ul_fb_div.ul_fb_div_frac = - pll_patameters.ulFbDiv.usFbDivFrac; + le16_to_cpu(pll_patameters.ulFbDiv.usFbDivFrac); dividers->ul_fb_div.ul_fb_div = - pll_patameters.ulFbDiv.usFbDiv; + le16_to_cpu(pll_patameters.ulFbDiv.usFbDiv); dividers->uc_pll_ref_div = pll_patameters.ucPllRefDiv; @@ -380,7 +379,7 @@ int atomctrl_get_engine_pll_dividers_ai(struct pp_hwmgr *hwmgr, COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_7 pll_patameters; int result; - pll_patameters.ulClock.ulClock = clock_value; + pll_patameters.ulClock.ulClock = cpu_to_le32(clock_value); pll_patameters.ulClock.ucPostDiv = COMPUTE_GPUCLK_INPUT_FLAG_SCLK; result = cgs_atom_exec_cmd_table @@ -412,7 +411,7 @@ int atomctrl_get_dfs_pll_dividers_vi( COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 pll_patameters; int result; - pll_patameters.ulClock.ulClock = clock_value; + pll_patameters.ulClock.ulClock = cpu_to_le32(clock_value); pll_patameters.ulClock.ucPostDiv = COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK; @@ -425,12 +424,12 @@ int atomctrl_get_dfs_pll_dividers_vi( dividers->pll_post_divider = pll_patameters.ulClock.ucPostDiv; dividers->real_clock = - pll_patameters.ulClock.ulClock; + le32_to_cpu(pll_patameters.ulClock.ulClock); dividers->ul_fb_div.ul_fb_div_frac = - pll_patameters.ulFbDiv.usFbDivFrac; + le16_to_cpu(pll_patameters.ulFbDiv.usFbDivFrac); dividers->ul_fb_div.ul_fb_div = - pll_patameters.ulFbDiv.usFbDiv; + le16_to_cpu(pll_patameters.ulFbDiv.usFbDiv); dividers->uc_pll_ref_div = pll_patameters.ucPllRefDiv; @@ -519,13 +518,13 @@ int atomctrl_get_voltage_table_v3( for (i = 0; i < voltage_object->asGpioVoltageObj.ucGpioEntryNum; i++) { voltage_table->entries[i].value = - voltage_object->asGpioVoltageObj.asVolGpioLut[i].usVoltageValue; + le16_to_cpu(voltage_object->asGpioVoltageObj.asVolGpioLut[i].usVoltageValue); voltage_table->entries[i].smio_low = - voltage_object->asGpioVoltageObj.asVolGpioLut[i].ulVoltageId; + le32_to_cpu(voltage_object->asGpioVoltageObj.asVolGpioLut[i].ulVoltageId); } voltage_table->mask_low = - voltage_object->asGpioVoltageObj.ulGpioMaskVal; + le32_to_cpu(voltage_object->asGpioVoltageObj.ulGpioMaskVal); voltage_table->count = voltage_object->asGpioVoltageObj.ucGpioEntryNum; voltage_table->phase_delay = @@ -552,13 +551,13 @@ static bool atomctrl_lookup_gpio_pin( pin_assignment->ucGpioPinBitShift; gpio_pin_assignment->us_gpio_pin_aindex = le16_to_cpu(pin_assignment->usGpioPin_AIndex); - return false; + return true; } offset += offsetof(ATOM_GPIO_PIN_ASSIGNMENT, ucGPIO_ID) + 1; } - return true; + return false; } /** @@ -650,8 +649,8 @@ int atomctrl_calculate_voltage_evv_on_sclk( return -1; if (getASICProfilingInfo->asHeader.ucTableFormatRevision < 3 || - (getASICProfilingInfo->asHeader.ucTableFormatRevision == 3 && - getASICProfilingInfo->asHeader.ucTableContentRevision < 4)) + (getASICProfilingInfo->asHeader.ucTableFormatRevision == 3 && + getASICProfilingInfo->asHeader.ucTableContentRevision < 4)) return -1; /*----------------------------------------------------------- @@ -662,37 +661,37 @@ int atomctrl_calculate_voltage_evv_on_sclk( switch (dpm_level) { case 1: - fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm1); - fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM1, 1000); + fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm1)); + fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM1), 1000); break; case 2: - fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm2); - fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM2, 1000); + fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm2)); + fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM2), 1000); break; case 3: - fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm3); - fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM3, 1000); + fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm3)); + fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM3), 1000); break; case 4: - fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm4); - fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM4, 1000); + fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm4)); + fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM4), 1000); break; case 5: - fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm5); - fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM5, 1000); + fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm5)); + fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM5), 1000); break; case 6: - fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm6); - fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM6, 1000); + fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm6)); + fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM6), 1000); break; case 7: - fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm7); - fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM7, 1000); + fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm7)); + fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM7), 1000); break; default: printk(KERN_ERR "DPM Level not supported\n"); fPowerDPMx = Convert_ULONG_ToFraction(1); - fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM0, 1000); + fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM0), 1000); } /*------------------------- @@ -716,9 +715,9 @@ int atomctrl_calculate_voltage_evv_on_sclk( return result; /* Finally, the actual fuse value */ - ul_RO_fused = sOutput_FuseValues.ulEfuseValue; - fMin = GetScaledFraction(sRO_fuse.ulEfuseMin, 1); - fRange = GetScaledFraction(sRO_fuse.ulEfuseEncodeRange, 1); + ul_RO_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue); + fMin = GetScaledFraction(le32_to_cpu(sRO_fuse.ulEfuseMin), 1); + fRange = GetScaledFraction(le32_to_cpu(sRO_fuse.ulEfuseEncodeRange), 1); fRO_fused = fDecodeLinearFuse(ul_RO_fused, fMin, fRange, sRO_fuse.ucEfuseLength); sCACm_fuse = getASICProfilingInfo->sCACm; @@ -736,9 +735,9 @@ int atomctrl_calculate_voltage_evv_on_sclk( if (result) return result; - ul_CACm_fused = sOutput_FuseValues.ulEfuseValue; - fMin = GetScaledFraction(sCACm_fuse.ulEfuseMin, 1000); - fRange = GetScaledFraction(sCACm_fuse.ulEfuseEncodeRange, 1000); + ul_CACm_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue); + fMin = GetScaledFraction(le32_to_cpu(sCACm_fuse.ulEfuseMin), 1000); + fRange = GetScaledFraction(le32_to_cpu(sCACm_fuse.ulEfuseEncodeRange), 1000); fCACm_fused = fDecodeLinearFuse(ul_CACm_fused, fMin, fRange, sCACm_fuse.ucEfuseLength); @@ -756,9 +755,9 @@ int atomctrl_calculate_voltage_evv_on_sclk( if (result) return result; - ul_CACb_fused = sOutput_FuseValues.ulEfuseValue; - fMin = GetScaledFraction(sCACb_fuse.ulEfuseMin, 1000); - fRange = GetScaledFraction(sCACb_fuse.ulEfuseEncodeRange, 1000); + ul_CACb_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue); + fMin = GetScaledFraction(le32_to_cpu(sCACb_fuse.ulEfuseMin), 1000); + fRange = GetScaledFraction(le32_to_cpu(sCACb_fuse.ulEfuseEncodeRange), 1000); fCACb_fused = fDecodeLinearFuse(ul_CACb_fused, fMin, fRange, sCACb_fuse.ucEfuseLength); @@ -777,9 +776,9 @@ int atomctrl_calculate_voltage_evv_on_sclk( if (result) return result; - ul_Kt_Beta_fused = sOutput_FuseValues.ulEfuseValue; - fAverage = GetScaledFraction(sKt_Beta_fuse.ulEfuseEncodeAverage, 1000); - fRange = GetScaledFraction(sKt_Beta_fuse.ulEfuseEncodeRange, 1000); + ul_Kt_Beta_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue); + fAverage = GetScaledFraction(le32_to_cpu(sKt_Beta_fuse.ulEfuseEncodeAverage), 1000); + fRange = GetScaledFraction(le32_to_cpu(sKt_Beta_fuse.ulEfuseEncodeRange), 1000); fKt_Beta_fused = fDecodeLogisticFuse(ul_Kt_Beta_fused, fAverage, fRange, sKt_Beta_fuse.ucEfuseLength); @@ -798,9 +797,9 @@ int atomctrl_calculate_voltage_evv_on_sclk( if (result) return result; - ul_Kv_m_fused = sOutput_FuseValues.ulEfuseValue; - fAverage = GetScaledFraction(sKv_m_fuse.ulEfuseEncodeAverage, 1000); - fRange = GetScaledFraction((sKv_m_fuse.ulEfuseEncodeRange & 0x7fffffff), 1000); + ul_Kv_m_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue); + fAverage = GetScaledFraction(le32_to_cpu(sKv_m_fuse.ulEfuseEncodeAverage), 1000); + fRange = GetScaledFraction((le32_to_cpu(sKv_m_fuse.ulEfuseEncodeRange) & 0x7fffffff), 1000); fRange = fMultiply(fRange, ConvertToFraction(-1)); fKv_m_fused = fDecodeLogisticFuse(ul_Kv_m_fused, @@ -820,9 +819,9 @@ int atomctrl_calculate_voltage_evv_on_sclk( if (result) return result; - ul_Kv_b_fused = sOutput_FuseValues.ulEfuseValue; - fAverage = GetScaledFraction(sKv_b_fuse.ulEfuseEncodeAverage, 1000); - fRange = GetScaledFraction(sKv_b_fuse.ulEfuseEncodeRange, 1000); + ul_Kv_b_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue); + fAverage = GetScaledFraction(le32_to_cpu(sKv_b_fuse.ulEfuseEncodeAverage), 1000); + fRange = GetScaledFraction(le32_to_cpu(sKv_b_fuse.ulEfuseEncodeRange), 1000); fKv_b_fused = fDecodeLogisticFuse(ul_Kv_b_fused, fAverage, fRange, sKv_b_fuse.ucEfuseLength); @@ -851,9 +850,9 @@ int atomctrl_calculate_voltage_evv_on_sclk( if (result) return result; - ul_FT_Lkg_V0NORM = sOutput_FuseValues.ulEfuseValue; - fLn_MaxDivMin = GetScaledFraction(getASICProfilingInfo->ulLkgEncodeLn_MaxDivMin, 10000); - fMin = GetScaledFraction(getASICProfilingInfo->ulLkgEncodeMin, 10000); + ul_FT_Lkg_V0NORM = le32_to_cpu(sOutput_FuseValues.ulEfuseValue); + fLn_MaxDivMin = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulLkgEncodeLn_MaxDivMin), 10000); + fMin = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulLkgEncodeMin), 10000); fFT_Lkg_V0NORM = fDecodeLeakageID(ul_FT_Lkg_V0NORM, fLn_MaxDivMin, fMin, getASICProfilingInfo->ucLkgEfuseLength); @@ -863,40 +862,40 @@ int atomctrl_calculate_voltage_evv_on_sclk( * PART 2 - Grabbing all required values *------------------------------------------- */ - fSM_A0 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A0, 1000000), + fSM_A0 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A0), 1000000), ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A0_sign))); - fSM_A1 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A1, 1000000), + fSM_A1 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A1), 1000000), ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A1_sign))); - fSM_A2 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A2, 100000), + fSM_A2 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A2), 100000), ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A2_sign))); - fSM_A3 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A3, 1000000), + fSM_A3 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A3), 1000000), ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A3_sign))); - fSM_A4 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A4, 1000000), + fSM_A4 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A4), 1000000), ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A4_sign))); - fSM_A5 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A5, 1000), + fSM_A5 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A5), 1000), ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A5_sign))); - fSM_A6 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A6, 1000), + fSM_A6 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A6), 1000), ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A6_sign))); - fSM_A7 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A7, 1000), + fSM_A7 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A7), 1000), ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A7_sign))); - fMargin_RO_a = ConvertToFraction(getASICProfilingInfo->ulMargin_RO_a); - fMargin_RO_b = ConvertToFraction(getASICProfilingInfo->ulMargin_RO_b); - fMargin_RO_c = ConvertToFraction(getASICProfilingInfo->ulMargin_RO_c); + fMargin_RO_a = ConvertToFraction(le32_to_cpu(getASICProfilingInfo->ulMargin_RO_a)); + fMargin_RO_b = ConvertToFraction(le32_to_cpu(getASICProfilingInfo->ulMargin_RO_b)); + fMargin_RO_c = ConvertToFraction(le32_to_cpu(getASICProfilingInfo->ulMargin_RO_c)); - fMargin_fixed = ConvertToFraction(getASICProfilingInfo->ulMargin_fixed); + fMargin_fixed = ConvertToFraction(le32_to_cpu(getASICProfilingInfo->ulMargin_fixed)); fMargin_FMAX_mean = GetScaledFraction( - getASICProfilingInfo->ulMargin_Fmax_mean, 10000); + le32_to_cpu(getASICProfilingInfo->ulMargin_Fmax_mean), 10000); fMargin_Plat_mean = GetScaledFraction( - getASICProfilingInfo->ulMargin_plat_mean, 10000); + le32_to_cpu(getASICProfilingInfo->ulMargin_plat_mean), 10000); fMargin_FMAX_sigma = GetScaledFraction( - getASICProfilingInfo->ulMargin_Fmax_sigma, 10000); + le32_to_cpu(getASICProfilingInfo->ulMargin_Fmax_sigma), 10000); fMargin_Plat_sigma = GetScaledFraction( - getASICProfilingInfo->ulMargin_plat_sigma, 10000); + le32_to_cpu(getASICProfilingInfo->ulMargin_plat_sigma), 10000); fMargin_DC_sigma = GetScaledFraction( - getASICProfilingInfo->ulMargin_DC_sigma, 100); + le32_to_cpu(getASICProfilingInfo->ulMargin_DC_sigma), 100); fMargin_DC_sigma = fDivide(fMargin_DC_sigma, ConvertToFraction(1000)); fCACm_fused = fDivide(fCACm_fused, ConvertToFraction(100)); @@ -908,14 +907,14 @@ int atomctrl_calculate_voltage_evv_on_sclk( fSclk = GetScaledFraction(sclk, 100); fV_max = fDivide(GetScaledFraction( - getASICProfilingInfo->ulMaxVddc, 1000), ConvertToFraction(4)); - fT_prod = GetScaledFraction(getASICProfilingInfo->ulBoardCoreTemp, 10); - fLKG_Factor = GetScaledFraction(getASICProfilingInfo->ulEvvLkgFactor, 100); - fT_FT = GetScaledFraction(getASICProfilingInfo->ulLeakageTemp, 10); + le32_to_cpu(getASICProfilingInfo->ulMaxVddc), 1000), ConvertToFraction(4)); + fT_prod = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulBoardCoreTemp), 10); + fLKG_Factor = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulEvvLkgFactor), 100); + fT_FT = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulLeakageTemp), 10); fV_FT = fDivide(GetScaledFraction( - getASICProfilingInfo->ulLeakageVoltage, 1000), ConvertToFraction(4)); + le32_to_cpu(getASICProfilingInfo->ulLeakageVoltage), 1000), ConvertToFraction(4)); fV_min = fDivide(GetScaledFraction( - getASICProfilingInfo->ulMinVddc, 1000), ConvertToFraction(4)); + le32_to_cpu(getASICProfilingInfo->ulMinVddc), 1000), ConvertToFraction(4)); /*----------------------- * PART 3 @@ -925,7 +924,7 @@ int atomctrl_calculate_voltage_evv_on_sclk( fA_Term = fAdd(fMargin_RO_a, fAdd(fMultiply(fSM_A4, fSclk), fSM_A5)); fB_Term = fAdd(fAdd(fMultiply(fSM_A2, fSclk), fSM_A6), fMargin_RO_b); fC_Term = fAdd(fMargin_RO_c, - fAdd(fMultiply(fSM_A0,fLkg_FT), + fAdd(fMultiply(fSM_A0, fLkg_FT), fAdd(fMultiply(fSM_A1, fMultiply(fLkg_FT, fSclk)), fAdd(fMultiply(fSM_A3, fSclk), fSubtract(fSM_A7, fRO_fused))))); @@ -1063,9 +1062,55 @@ int atomctrl_get_voltage_evv_on_sclk( get_voltage_info_param_space.ucVoltageMode = ATOM_GET_VOLTAGE_EVV_VOLTAGE; get_voltage_info_param_space.usVoltageLevel = - virtual_voltage_Id; + cpu_to_le16(virtual_voltage_Id); get_voltage_info_param_space.ulSCLKFreq = - sclk; + cpu_to_le32(sclk); + + result = cgs_atom_exec_cmd_table(hwmgr->device, + GetIndexIntoMasterTable(COMMAND, GetVoltageInfo), + &get_voltage_info_param_space); + + if (0 != result) + return result; + + *voltage = le16_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 *) + (&get_voltage_info_param_space))->usVoltageLevel); + + return result; +} + +/** + * atomctrl_get_voltage_evv gets voltage via call to ATOM COMMAND table. + * @param hwmgr input: pointer to hwManager + * @param virtual_voltage_id input: voltage id which match per voltage DPM state: 0xff01, 0xff02.. 0xff08 + * @param voltage output: real voltage level in unit of mv + */ +int atomctrl_get_voltage_evv(struct pp_hwmgr *hwmgr, + uint16_t virtual_voltage_id, + uint16_t *voltage) +{ + int result; + int entry_id; + GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2 get_voltage_info_param_space; + + /* search for leakage voltage ID 0xff01 ~ 0xff08 and sckl */ + for (entry_id = 0; entry_id < hwmgr->dyn_state.vddc_dependency_on_sclk->count; entry_id++) { + if (hwmgr->dyn_state.vddc_dependency_on_sclk->entries[entry_id].v == virtual_voltage_id) { + /* found */ + break; + } + } + + PP_ASSERT_WITH_CODE(entry_id < hwmgr->dyn_state.vddc_dependency_on_sclk->count, + "Can't find requested voltage id in vddc_dependency_on_sclk table!", + return -EINVAL; + ); + + get_voltage_info_param_space.ucVoltageType = VOLTAGE_TYPE_VDDC; + get_voltage_info_param_space.ucVoltageMode = ATOM_GET_VOLTAGE_EVV_VOLTAGE; + get_voltage_info_param_space.usVoltageLevel = virtual_voltage_id; + get_voltage_info_param_space.ulSCLKFreq = + cpu_to_le32(hwmgr->dyn_state.vddc_dependency_on_sclk->entries[entry_id].clk); result = cgs_atom_exec_cmd_table(hwmgr->device, GetIndexIntoMasterTable(COMMAND, GetVoltageInfo), @@ -1074,8 +1119,8 @@ int atomctrl_get_voltage_evv_on_sclk( if (0 != result) return result; - *voltage = ((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 *) - (&get_voltage_info_param_space))->usVoltageLevel; + *voltage = le16_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 *) + (&get_voltage_info_param_space))->usVoltageLevel); return result; } @@ -1165,8 +1210,8 @@ static int asic_internal_ss_get_ss_asignment(struct pp_hwmgr *hwmgr, if (entry_found) { ssEntry->speed_spectrum_percentage = - ssInfo->usSpreadSpectrumPercentage; - ssEntry->speed_spectrum_rate = ssInfo->usSpreadRateInKhz; + le16_to_cpu(ssInfo->usSpreadSpectrumPercentage); + ssEntry->speed_spectrum_rate = le16_to_cpu(ssInfo->usSpreadRateInKhz); if (((GET_DATA_TABLE_MAJOR_REVISION(table) == 2) && (GET_DATA_TABLE_MINOR_REVISION(table) >= 2)) || @@ -1222,7 +1267,7 @@ int atomctrl_read_efuse(void *device, uint16_t start_index, int result; READ_EFUSE_VALUE_PARAMETER efuse_param; - efuse_param.sEfuse.usEfuseIndex = (start_index / 32) * 4; + efuse_param.sEfuse.usEfuseIndex = cpu_to_le16((start_index / 32) * 4); efuse_param.sEfuse.ucBitShift = (uint8_t) (start_index - ((start_index / 32) * 32)); efuse_param.sEfuse.ucBitLength = (uint8_t) @@ -1232,19 +1277,21 @@ int atomctrl_read_efuse(void *device, uint16_t start_index, GetIndexIntoMasterTable(COMMAND, ReadEfuseValue), &efuse_param); if (!result) - *efuse = efuse_param.ulEfuseValue & mask; + *efuse = le32_to_cpu(efuse_param.ulEfuseValue) & mask; return result; } int atomctrl_set_ac_timing_ai(struct pp_hwmgr *hwmgr, uint32_t memory_clock, - uint8_t level) + uint8_t level) { DYNAMICE_MEMORY_SETTINGS_PARAMETER_V2_1 memory_clock_parameters; int result; - memory_clock_parameters.asDPMMCReg.ulClock.ulClockFreq = memory_clock & SET_CLOCK_FREQ_MASK; - memory_clock_parameters.asDPMMCReg.ulClock.ulComputeClockFlag = ADJUST_MC_SETTING_PARAM; + memory_clock_parameters.asDPMMCReg.ulClock.ulClockFreq = + memory_clock & SET_CLOCK_FREQ_MASK; + memory_clock_parameters.asDPMMCReg.ulClock.ulComputeClockFlag = + ADJUST_MC_SETTING_PARAM; memory_clock_parameters.asDPMMCReg.ucMclkDPMState = level; result = cgs_atom_exec_cmd_table @@ -1264,8 +1311,8 @@ int atomctrl_get_voltage_evv_on_sclk_ai(struct pp_hwmgr *hwmgr, uint8_t voltage_ get_voltage_info_param_space.ucVoltageType = voltage_type; get_voltage_info_param_space.ucVoltageMode = ATOM_GET_VOLTAGE_EVV_VOLTAGE; - get_voltage_info_param_space.usVoltageLevel = virtual_voltage_Id; - get_voltage_info_param_space.ulSCLKFreq = sclk; + get_voltage_info_param_space.usVoltageLevel = cpu_to_le16(virtual_voltage_Id); + get_voltage_info_param_space.ulSCLKFreq = cpu_to_le32(sclk); result = cgs_atom_exec_cmd_table(hwmgr->device, GetIndexIntoMasterTable(COMMAND, GetVoltageInfo), @@ -1274,7 +1321,7 @@ int atomctrl_get_voltage_evv_on_sclk_ai(struct pp_hwmgr *hwmgr, uint8_t voltage_ if (0 != result) return result; - *voltage = ((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_3 *)(&get_voltage_info_param_space))->ulVoltageLevel; + *voltage = le32_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_3 *)(&get_voltage_info_param_space))->ulVoltageLevel); return result; } @@ -1295,15 +1342,19 @@ int atomctrl_get_smc_sclk_range_table(struct pp_hwmgr *hwmgr, struct pp_atom_ctr for (i = 0; i < psmu_info->ucSclkEntryNum; i++) { table->entry[i].ucVco_setting = psmu_info->asSclkFcwRangeEntry[i].ucVco_setting; table->entry[i].ucPostdiv = psmu_info->asSclkFcwRangeEntry[i].ucPostdiv; - table->entry[i].usFcw_pcc = psmu_info->asSclkFcwRangeEntry[i].ucFcw_pcc; - table->entry[i].usFcw_trans_upper = psmu_info->asSclkFcwRangeEntry[i].ucFcw_trans_upper; - table->entry[i].usRcw_trans_lower = psmu_info->asSclkFcwRangeEntry[i].ucRcw_trans_lower; + table->entry[i].usFcw_pcc = + le16_to_cpu(psmu_info->asSclkFcwRangeEntry[i].ucFcw_pcc); + table->entry[i].usFcw_trans_upper = + le16_to_cpu(psmu_info->asSclkFcwRangeEntry[i].ucFcw_trans_upper); + table->entry[i].usRcw_trans_lower = + le16_to_cpu(psmu_info->asSclkFcwRangeEntry[i].ucRcw_trans_lower); } return 0; } -int atomctrl_get_avfs_information(struct pp_hwmgr *hwmgr, struct pp_atom_ctrl__avfs_parameters *param) +int atomctrl_get_avfs_information(struct pp_hwmgr *hwmgr, + struct pp_atom_ctrl__avfs_parameters *param) { ATOM_ASIC_PROFILING_INFO_V3_6 *profile = NULL; @@ -1317,30 +1368,30 @@ int atomctrl_get_avfs_information(struct pp_hwmgr *hwmgr, struct pp_atom_ctrl__a if (!profile) return -1; - param->ulAVFS_meanNsigma_Acontant0 = profile->ulAVFS_meanNsigma_Acontant0; - param->ulAVFS_meanNsigma_Acontant1 = profile->ulAVFS_meanNsigma_Acontant1; - param->ulAVFS_meanNsigma_Acontant2 = profile->ulAVFS_meanNsigma_Acontant2; - param->usAVFS_meanNsigma_DC_tol_sigma = profile->usAVFS_meanNsigma_DC_tol_sigma; - param->usAVFS_meanNsigma_Platform_mean = profile->usAVFS_meanNsigma_Platform_mean; - param->usAVFS_meanNsigma_Platform_sigma = profile->usAVFS_meanNsigma_Platform_sigma; - param->ulGB_VDROOP_TABLE_CKSOFF_a0 = profile->ulGB_VDROOP_TABLE_CKSOFF_a0; - param->ulGB_VDROOP_TABLE_CKSOFF_a1 = profile->ulGB_VDROOP_TABLE_CKSOFF_a1; - param->ulGB_VDROOP_TABLE_CKSOFF_a2 = profile->ulGB_VDROOP_TABLE_CKSOFF_a2; - param->ulGB_VDROOP_TABLE_CKSON_a0 = profile->ulGB_VDROOP_TABLE_CKSON_a0; - param->ulGB_VDROOP_TABLE_CKSON_a1 = profile->ulGB_VDROOP_TABLE_CKSON_a1; - param->ulGB_VDROOP_TABLE_CKSON_a2 = profile->ulGB_VDROOP_TABLE_CKSON_a2; - param->ulAVFSGB_FUSE_TABLE_CKSOFF_m1 = profile->ulAVFSGB_FUSE_TABLE_CKSOFF_m1; - param->usAVFSGB_FUSE_TABLE_CKSOFF_m2 = profile->usAVFSGB_FUSE_TABLE_CKSOFF_m2; - param->ulAVFSGB_FUSE_TABLE_CKSOFF_b = profile->ulAVFSGB_FUSE_TABLE_CKSOFF_b; - param->ulAVFSGB_FUSE_TABLE_CKSON_m1 = profile->ulAVFSGB_FUSE_TABLE_CKSON_m1; - param->usAVFSGB_FUSE_TABLE_CKSON_m2 = profile->usAVFSGB_FUSE_TABLE_CKSON_m2; - param->ulAVFSGB_FUSE_TABLE_CKSON_b = profile->ulAVFSGB_FUSE_TABLE_CKSON_b; - param->usMaxVoltage_0_25mv = profile->usMaxVoltage_0_25mv; + param->ulAVFS_meanNsigma_Acontant0 = le32_to_cpu(profile->ulAVFS_meanNsigma_Acontant0); + param->ulAVFS_meanNsigma_Acontant1 = le32_to_cpu(profile->ulAVFS_meanNsigma_Acontant1); + param->ulAVFS_meanNsigma_Acontant2 = le32_to_cpu(profile->ulAVFS_meanNsigma_Acontant2); + param->usAVFS_meanNsigma_DC_tol_sigma = le16_to_cpu(profile->usAVFS_meanNsigma_DC_tol_sigma); + param->usAVFS_meanNsigma_Platform_mean = le16_to_cpu(profile->usAVFS_meanNsigma_Platform_mean); + param->usAVFS_meanNsigma_Platform_sigma = le16_to_cpu(profile->usAVFS_meanNsigma_Platform_sigma); + param->ulGB_VDROOP_TABLE_CKSOFF_a0 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSOFF_a0); + param->ulGB_VDROOP_TABLE_CKSOFF_a1 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSOFF_a1); + param->ulGB_VDROOP_TABLE_CKSOFF_a2 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSOFF_a2); + param->ulGB_VDROOP_TABLE_CKSON_a0 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSON_a0); + param->ulGB_VDROOP_TABLE_CKSON_a1 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSON_a1); + param->ulGB_VDROOP_TABLE_CKSON_a2 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSON_a2); + param->ulAVFSGB_FUSE_TABLE_CKSOFF_m1 = le32_to_cpu(profile->ulAVFSGB_FUSE_TABLE_CKSOFF_m1); + param->usAVFSGB_FUSE_TABLE_CKSOFF_m2 = le16_to_cpu(profile->usAVFSGB_FUSE_TABLE_CKSOFF_m2); + param->ulAVFSGB_FUSE_TABLE_CKSOFF_b = le32_to_cpu(profile->ulAVFSGB_FUSE_TABLE_CKSOFF_b); + param->ulAVFSGB_FUSE_TABLE_CKSON_m1 = le32_to_cpu(profile->ulAVFSGB_FUSE_TABLE_CKSON_m1); + param->usAVFSGB_FUSE_TABLE_CKSON_m2 = le16_to_cpu(profile->usAVFSGB_FUSE_TABLE_CKSON_m2); + param->ulAVFSGB_FUSE_TABLE_CKSON_b = le32_to_cpu(profile->ulAVFSGB_FUSE_TABLE_CKSON_b); + param->usMaxVoltage_0_25mv = le16_to_cpu(profile->usMaxVoltage_0_25mv); param->ucEnableGB_VDROOP_TABLE_CKSOFF = profile->ucEnableGB_VDROOP_TABLE_CKSOFF; param->ucEnableGB_VDROOP_TABLE_CKSON = profile->ucEnableGB_VDROOP_TABLE_CKSON; param->ucEnableGB_FUSE_TABLE_CKSOFF = profile->ucEnableGB_FUSE_TABLE_CKSOFF; param->ucEnableGB_FUSE_TABLE_CKSON = profile->ucEnableGB_FUSE_TABLE_CKSON; - param->usPSM_Age_ComFactor = profile->usPSM_Age_ComFactor; + param->usPSM_Age_ComFactor = le16_to_cpu(profile->usPSM_Age_ComFactor); param->ucEnableApplyAVFS_CKS_OFF_Voltage = profile->ucEnableApplyAVFS_CKS_OFF_Voltage; return 0; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h index 1e35a9625..fc898afce 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h @@ -281,6 +281,7 @@ struct pp_atom_ctrl__avfs_parameters { extern bool atomctrl_get_pp_assign_pin(struct pp_hwmgr *hwmgr, const uint32_t pinId, pp_atomctrl_gpio_pin_assignment *gpio_pin_assignment); extern int atomctrl_get_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type, uint32_t sclk, uint16_t virtual_voltage_Id, uint16_t *voltage); +extern int atomctrl_get_voltage_evv(struct pp_hwmgr *hwmgr, uint16_t virtual_voltage_id, uint16_t *voltage); extern uint32_t atomctrl_get_mpll_reference_clock(struct pp_hwmgr *hwmgr); extern int atomctrl_get_memory_clock_spread_spectrum(struct pp_hwmgr *hwmgr, const uint32_t memory_clock, pp_atomctrl_internal_ss_info *ssInfo); extern int atomctrl_get_engine_clock_spread_spectrum(struct pp_hwmgr *hwmgr, const uint32_t engine_clock, pp_atomctrl_internal_ss_info *ssInfo); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h index 009bd5963..8f50a0383 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h @@ -50,55 +50,45 @@ typedef union _fInt { * Function Declarations * ------------------------------------------------------------------------------- */ -fInt ConvertToFraction(int); /* Use this to convert an INT to a FINT */ -fInt Convert_ULONG_ToFraction(uint32_t); /* Use this to convert an uint32_t to a FINT */ -fInt GetScaledFraction(int, int); /* Use this to convert an INT to a FINT after scaling it by a factor */ -int ConvertBackToInteger(fInt); /* Convert a FINT back to an INT that is scaled by 1000 (i.e. last 3 digits are the decimal digits) */ - -fInt fNegate(fInt); /* Returns -1 * input fInt value */ -fInt fAdd (fInt, fInt); /* Returns the sum of two fInt numbers */ -fInt fSubtract (fInt A, fInt B); /* Returns A-B - Sometimes easier than Adding negative numbers */ -fInt fMultiply (fInt, fInt); /* Returns the product of two fInt numbers */ -fInt fDivide (fInt A, fInt B); /* Returns A/B */ -fInt fGetSquare(fInt); /* Returns the square of a fInt number */ -fInt fSqrt(fInt); /* Returns the Square Root of a fInt number */ - -int uAbs(int); /* Returns the Absolute value of the Int */ -fInt fAbs(fInt); /* Returns the Absolute value of the fInt */ -int uPow(int base, int exponent); /* Returns base^exponent an INT */ - -void SolveQuadracticEqn(fInt, fInt, fInt, fInt[]); /* Returns the 2 roots via the array */ -bool Equal(fInt, fInt); /* Returns true if two fInts are equal to each other */ -bool GreaterThan(fInt A, fInt B); /* Returns true if A > B */ - -fInt fExponential(fInt exponent); /* Can be used to calculate e^exponent */ -fInt fNaturalLog(fInt value); /* Can be used to calculate ln(value) */ +static fInt ConvertToFraction(int); /* Use this to convert an INT to a FINT */ +static fInt Convert_ULONG_ToFraction(uint32_t); /* Use this to convert an uint32_t to a FINT */ +static fInt GetScaledFraction(int, int); /* Use this to convert an INT to a FINT after scaling it by a factor */ +static int ConvertBackToInteger(fInt); /* Convert a FINT back to an INT that is scaled by 1000 (i.e. last 3 digits are the decimal digits) */ + +static fInt fNegate(fInt); /* Returns -1 * input fInt value */ +static fInt fAdd (fInt, fInt); /* Returns the sum of two fInt numbers */ +static fInt fSubtract (fInt A, fInt B); /* Returns A-B - Sometimes easier than Adding negative numbers */ +static fInt fMultiply (fInt, fInt); /* Returns the product of two fInt numbers */ +static fInt fDivide (fInt A, fInt B); /* Returns A/B */ +static fInt fGetSquare(fInt); /* Returns the square of a fInt number */ +static fInt fSqrt(fInt); /* Returns the Square Root of a fInt number */ + +static int uAbs(int); /* Returns the Absolute value of the Int */ +static int uPow(int base, int exponent); /* Returns base^exponent an INT */ + +static void SolveQuadracticEqn(fInt, fInt, fInt, fInt[]); /* Returns the 2 roots via the array */ +static bool Equal(fInt, fInt); /* Returns true if two fInts are equal to each other */ +static bool GreaterThan(fInt A, fInt B); /* Returns true if A > B */ + +static fInt fExponential(fInt exponent); /* Can be used to calculate e^exponent */ +static fInt fNaturalLog(fInt value); /* Can be used to calculate ln(value) */ /* Fuse decoding functions * ------------------------------------------------------------------------------------- */ -fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t bitlength); -fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint32_t bitlength); -fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, uint32_t bitlength); +static fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t bitlength); +static fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint32_t bitlength); +static fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, uint32_t bitlength); /* Internal Support Functions - Use these ONLY for testing or adding to internal functions * ------------------------------------------------------------------------------------- * Some of the following functions take two INTs as their input - This is unsafe for a variety of reasons. */ -fInt Add (int, int); /* Add two INTs and return Sum as FINT */ -fInt Multiply (int, int); /* Multiply two INTs and return Product as FINT */ -fInt Divide (int, int); /* You get the idea... */ -fInt fNegate(fInt); +static fInt Divide (int, int); /* Divide two INTs and return result as FINT */ +static fInt fNegate(fInt); -int uGetScaledDecimal (fInt); /* Internal function */ -int GetReal (fInt A); /* Internal function */ - -/* Future Additions and Incomplete Functions - * ------------------------------------------------------------------------------------- - */ -int GetRoundedValue(fInt); /* Incomplete function - Useful only when Precision is lacking */ - /* Let us say we have 2.126 but can only handle 2 decimal points. We could */ - /* either chop of 6 and keep 2.12 or use this function to get 2.13, which is more accurate */ +static int uGetScaledDecimal (fInt); /* Internal function */ +static int GetReal (fInt A); /* Internal function */ /* ------------------------------------------------------------------------------------- * TROUBLESHOOTING INFORMATION @@ -115,7 +105,7 @@ int GetRoundedValue(fInt); /* Incomplete function - Usef * START OF CODE * ------------------------------------------------------------------------------------- */ -fInt fExponential(fInt exponent) /*Can be used to calculate e^exponent*/ +static fInt fExponential(fInt exponent) /*Can be used to calculate e^exponent*/ { uint32_t i; bool bNegated = false; @@ -154,7 +144,7 @@ fInt fExponential(fInt exponent) /*Can be used to calculate e^exponent*/ return solution; } -fInt fNaturalLog(fInt value) +static fInt fNaturalLog(fInt value) { uint32_t i; fInt upper_bound = Divide(8, 1000); @@ -179,7 +169,7 @@ fInt fNaturalLog(fInt value) return (fAdd(solution, error_term)); } -fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t bitlength) +static fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t bitlength) { fInt f_fuse_value = Convert_ULONG_ToFraction(fuse_value); fInt f_bit_max_value = Convert_ULONG_ToFraction((uPow(2, bitlength)) - 1); @@ -194,7 +184,7 @@ fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t b } -fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint32_t bitlength) +static fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint32_t bitlength) { fInt f_fuse_value = Convert_ULONG_ToFraction(fuse_value); fInt f_bit_max_value = Convert_ULONG_ToFraction((uPow(2, bitlength)) - 1); @@ -212,7 +202,7 @@ fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint return f_decoded_value; } -fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, uint32_t bitlength) +static fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, uint32_t bitlength) { fInt fLeakage; fInt f_bit_max_value = Convert_ULONG_ToFraction((uPow(2, bitlength)) - 1); @@ -225,7 +215,7 @@ fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, return fLeakage; } -fInt ConvertToFraction(int X) /*Add all range checking here. Is it possible to make fInt a private declaration? */ +static fInt ConvertToFraction(int X) /*Add all range checking here. Is it possible to make fInt a private declaration? */ { fInt temp; @@ -237,13 +227,13 @@ fInt ConvertToFraction(int X) /*Add all range checking here. Is it possible to m return temp; } -fInt fNegate(fInt X) +static fInt fNegate(fInt X) { fInt CONSTANT_NEGONE = ConvertToFraction(-1); return (fMultiply(X, CONSTANT_NEGONE)); } -fInt Convert_ULONG_ToFraction(uint32_t X) +static fInt Convert_ULONG_ToFraction(uint32_t X) { fInt temp; @@ -255,7 +245,7 @@ fInt Convert_ULONG_ToFraction(uint32_t X) return temp; } -fInt GetScaledFraction(int X, int factor) +static fInt GetScaledFraction(int X, int factor) { int times_shifted, factor_shifted; bool bNEGATED; @@ -304,7 +294,7 @@ fInt GetScaledFraction(int X, int factor) } /* Addition using two fInts */ -fInt fAdd (fInt X, fInt Y) +static fInt fAdd (fInt X, fInt Y) { fInt Sum; @@ -314,7 +304,7 @@ fInt fAdd (fInt X, fInt Y) } /* Addition using two fInts */ -fInt fSubtract (fInt X, fInt Y) +static fInt fSubtract (fInt X, fInt Y) { fInt Difference; @@ -323,7 +313,7 @@ fInt fSubtract (fInt X, fInt Y) return Difference; } -bool Equal(fInt A, fInt B) +static bool Equal(fInt A, fInt B) { if (A.full == B.full) return true; @@ -331,7 +321,7 @@ bool Equal(fInt A, fInt B) return false; } -bool GreaterThan(fInt A, fInt B) +static bool GreaterThan(fInt A, fInt B) { if (A.full > B.full) return true; @@ -339,7 +329,7 @@ bool GreaterThan(fInt A, fInt B) return false; } -fInt fMultiply (fInt X, fInt Y) /* Uses 64-bit integers (int64_t) */ +static fInt fMultiply (fInt X, fInt Y) /* Uses 64-bit integers (int64_t) */ { fInt Product; int64_t tempProduct; @@ -363,7 +353,7 @@ fInt fMultiply (fInt X, fInt Y) /* Uses 64-bit integers (int64_t) */ return Product; } -fInt fDivide (fInt X, fInt Y) +static fInt fDivide (fInt X, fInt Y) { fInt fZERO, fQuotient; int64_t longlongX, longlongY; @@ -384,7 +374,7 @@ fInt fDivide (fInt X, fInt Y) return fQuotient; } -int ConvertBackToInteger (fInt A) /*THIS is the function that will be used to check with the Golden settings table*/ +static int ConvertBackToInteger (fInt A) /*THIS is the function that will be used to check with the Golden settings table*/ { fInt fullNumber, scaledDecimal, scaledReal; @@ -397,13 +387,13 @@ int ConvertBackToInteger (fInt A) /*THIS is the function that will be used to ch return fullNumber.full; } -fInt fGetSquare(fInt A) +static fInt fGetSquare(fInt A) { return fMultiply(A,A); } /* x_new = x_old - (x_old^2 - C) / (2 * x_old) */ -fInt fSqrt(fInt num) +static fInt fSqrt(fInt num) { fInt F_divide_Fprime, Fprime; fInt test; @@ -460,7 +450,7 @@ fInt fSqrt(fInt num) return (x_new); } -void SolveQuadracticEqn(fInt A, fInt B, fInt C, fInt Roots[]) +static void SolveQuadracticEqn(fInt A, fInt B, fInt C, fInt Roots[]) { fInt *pRoots = &Roots[0]; fInt temp, root_first, root_second; @@ -498,52 +488,13 @@ void SolveQuadracticEqn(fInt A, fInt B, fInt C, fInt Roots[]) * ----------------------------------------------------------------------------- */ -/* Addition using two normal ints - Temporary - Use only for testing purposes?. */ -fInt Add (int X, int Y) -{ - fInt A, B, Sum; - - A.full = (X << SHIFT_AMOUNT); - B.full = (Y << SHIFT_AMOUNT); - - Sum.full = A.full + B.full; - - return Sum; -} - /* Conversion Functions */ -int GetReal (fInt A) +static int GetReal (fInt A) { return (A.full >> SHIFT_AMOUNT); } -/* Temporarily Disabled */ -int GetRoundedValue(fInt A) /*For now, round the 3rd decimal place */ -{ - /* ROUNDING TEMPORARLY DISABLED - int temp = A.full; - int decimal_cutoff, decimal_mask = 0x000001FF; - decimal_cutoff = temp & decimal_mask; - if (decimal_cutoff > 0x147) { - temp += 673; - }*/ - - return ConvertBackToInteger(A)/10000; /*Temporary - in case this was used somewhere else */ -} - -fInt Multiply (int X, int Y) -{ - fInt A, B, Product; - - A.full = X << SHIFT_AMOUNT; - B.full = Y << SHIFT_AMOUNT; - - Product = fMultiply(A, B); - - return Product; -} - -fInt Divide (int X, int Y) +static fInt Divide (int X, int Y) { fInt A, B, Quotient; @@ -555,7 +506,7 @@ fInt Divide (int X, int Y) return Quotient; } -int uGetScaledDecimal (fInt A) /*Converts the fractional portion to whole integers - Costly function */ +static int uGetScaledDecimal (fInt A) /*Converts the fractional portion to whole integers - Costly function */ { int dec[PRECISION]; int i, scaledDecimal = 0, tmp = A.partial.decimal; @@ -570,7 +521,7 @@ int uGetScaledDecimal (fInt A) /*Converts the fractional portion to whole intege return scaledDecimal; } -int uPow(int base, int power) +static int uPow(int base, int power) { if (power == 0) return 1; @@ -578,15 +529,7 @@ int uPow(int base, int power) return (base)*uPow(base, power - 1); } -fInt fAbs(fInt A) -{ - if (A.partial.real < 0) - return (fMultiply(A, ConvertToFraction(-1))); - else - return A; -} - -int uAbs(int X) +static int uAbs(int X) { if (X < 0) return (X * -1); @@ -594,7 +537,7 @@ int uAbs(int X) return X; } -fInt fRoundUpByStepSize(fInt A, fInt fStepSize, bool error_term) +static fInt fRoundUpByStepSize(fInt A, fInt fStepSize, bool error_term) { fInt solution; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c index 2f1a14fe0..6c321b0d8 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c @@ -794,19 +794,35 @@ static const ATOM_PPLIB_STATE_V2 *get_state_entry_v2( static const ATOM_PPLIB_POWERPLAYTABLE *get_powerplay_table( struct pp_hwmgr *hwmgr) { - const void *table_addr = NULL; + const void *table_addr = hwmgr->soft_pp_table; uint8_t frev, crev; uint16_t size; - table_addr = cgs_atom_get_data_table(hwmgr->device, - GetIndexIntoMasterTable(DATA, PowerPlayInfo), - &size, &frev, &crev); + if (!table_addr) { + table_addr = cgs_atom_get_data_table(hwmgr->device, + GetIndexIntoMasterTable(DATA, PowerPlayInfo), + &size, &frev, &crev); - hwmgr->soft_pp_table = table_addr; + hwmgr->soft_pp_table = table_addr; + hwmgr->soft_pp_table_size = size; + } return (const ATOM_PPLIB_POWERPLAYTABLE *)table_addr; } +int pp_tables_get_response_times(struct pp_hwmgr *hwmgr, + uint32_t *vol_rep_time, uint32_t *bb_rep_time) +{ + const ATOM_PPLIB_POWERPLAYTABLE *powerplay_tab = get_powerplay_table(hwmgr); + + PP_ASSERT_WITH_CODE(NULL != powerplay_tab, + "Missing PowerPlay Table!", return -EINVAL); + + *vol_rep_time = (uint32_t)le16_to_cpu(powerplay_tab->usVoltageTime); + *bb_rep_time = (uint32_t)le16_to_cpu(powerplay_tab->usBackbiasTime); + + return 0; +} int pp_tables_get_num_of_entries(struct pp_hwmgr *hwmgr, unsigned long *num_of_entries) @@ -1499,7 +1515,7 @@ int get_number_of_vce_state_table_entries( const ATOM_PPLIB_VCE_State_Table *vce_table = get_vce_state_table(hwmgr, table); - if (vce_table > 0) + if (vce_table) return vce_table->numEntries; return 0; @@ -1589,11 +1605,6 @@ static int pp_tables_initialize(struct pp_hwmgr *hwmgr) static int pp_tables_uninitialize(struct pp_hwmgr *hwmgr) { - if (NULL != hwmgr->soft_pp_table) { - kfree(hwmgr->soft_pp_table); - hwmgr->soft_pp_table = NULL; - } - if (NULL != hwmgr->dyn_state.vddc_dependency_on_sclk) { kfree(hwmgr->dyn_state.vddc_dependency_on_sclk); hwmgr->dyn_state.vddc_dependency_on_sclk = NULL; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h index 304348024..baddaa756 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h @@ -32,16 +32,19 @@ struct pp_hw_power_state; extern const struct pp_table_func pptable_funcs; typedef int (*pp_tables_hw_clock_info_callback)(struct pp_hwmgr *hwmgr, - struct pp_hw_power_state *hw_ps, - unsigned int index, - const void *clock_info); + struct pp_hw_power_state *hw_ps, + unsigned int index, + const void *clock_info); int pp_tables_get_num_of_entries(struct pp_hwmgr *hwmgr, - unsigned long *num_of_entries); + unsigned long *num_of_entries); int pp_tables_get_entry(struct pp_hwmgr *hwmgr, - unsigned long entry_index, - struct pp_power_state *ps, - pp_tables_hw_clock_info_callback func); + unsigned long entry_index, + struct pp_power_state *ps, + pp_tables_hw_clock_info_callback func); + +int pp_tables_get_response_times(struct pp_hwmgr *hwmgr, + uint32_t *vol_rep_time, uint32_t *bb_rep_time); #endif diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c index 5d0f655bf..c7dc11122 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c @@ -571,7 +571,7 @@ int tonga_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if (0 == data->sclk_dpm_key_disabled) { /* Checking if DPM is running. If we discover hang because of this, we should skip this message.*/ PP_ASSERT_WITH_CODE( - (0 == tonga_is_dpm_running(hwmgr)), + !tonga_is_dpm_running(hwmgr), "Trying to Disable SCLK DPM when DPM is disabled", return -1 ); @@ -587,7 +587,7 @@ int tonga_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if (0 == data->mclk_dpm_key_disabled) { /* Checking if DPM is running. If we discover hang because of this, we should skip this message. */ PP_ASSERT_WITH_CODE( - (0 == tonga_is_dpm_running(hwmgr)), + !tonga_is_dpm_running(hwmgr), "Trying to Disable MCLK DPM when DPM is disabled", return -1 ); @@ -614,7 +614,7 @@ int tonga_stop_dpm(struct pp_hwmgr *hwmgr) if (0 == data->pcie_dpm_key_disabled) { /* Checking if DPM is running. If we discover hang because of this, we should skip this message.*/ PP_ASSERT_WITH_CODE( - (0 == tonga_is_dpm_running(hwmgr)), + !tonga_is_dpm_running(hwmgr), "Trying to Disable PCIE DPM when DPM is disabled", return -1 ); @@ -630,7 +630,7 @@ int tonga_stop_dpm(struct pp_hwmgr *hwmgr) /* Checking if DPM is running. If we discover hang because of this, we should skip this message.*/ PP_ASSERT_WITH_CODE( - (0 == tonga_is_dpm_running(hwmgr)), + !tonga_is_dpm_running(hwmgr), "Trying to Disable Voltage CNTL when DPM is disabled", return -1 ); @@ -688,8 +688,9 @@ int tonga_dpm_force_state(struct pp_hwmgr *hwmgr, uint32_t n) uint32_t level_mask = 1 << n; /* Checking if DPM is running. If we discover hang because of this, we should skip this message. */ - PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr), - "Trying to force SCLK when DPM is disabled", return -1;); + PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr), + "Trying to force SCLK when DPM is disabled", + return -1;); if (0 == data->sclk_dpm_key_disabled) return (0 == smum_send_msg_to_smc_with_parameter( hwmgr->smumgr, @@ -712,8 +713,9 @@ int tonga_dpm_force_state_mclk(struct pp_hwmgr *hwmgr, uint32_t n) uint32_t level_mask = 1 << n; /* Checking if DPM is running. If we discover hang because of this, we should skip this message. */ - PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr), - "Trying to Force MCLK when DPM is disabled", return -1;); + PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr), + "Trying to Force MCLK when DPM is disabled", + return -1;); if (0 == data->mclk_dpm_key_disabled) return (0 == smum_send_msg_to_smc_with_parameter( hwmgr->smumgr, @@ -735,8 +737,9 @@ int tonga_dpm_force_state_pcie(struct pp_hwmgr *hwmgr, uint32_t n) tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend); /* Checking if DPM is running. If we discover hang because of this, we should skip this message.*/ - PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr), - "Trying to Force PCIE level when DPM is disabled", return -1;); + PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr), + "Trying to Force PCIE level when DPM is disabled", + return -1;); if (0 == data->pcie_dpm_key_disabled) return (0 == smum_send_msg_to_smc_with_parameter( hwmgr->smumgr, @@ -774,7 +777,7 @@ int tonga_process_firmware_header(struct pp_hwmgr *hwmgr) uint32_t tmp; int result; - bool error = 0; + bool error = false; result = tonga_read_smc_sram_dword(hwmgr->smumgr, SMU72_FIRMWARE_HEADER_LOCATION + @@ -933,11 +936,11 @@ int tonga_init_power_gate_state(struct pp_hwmgr *hwmgr) { tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend); - data->uvd_power_gated = 0; - data->vce_power_gated = 0; - data->samu_power_gated = 0; - data->acp_power_gated = 0; - data->pg_acp_init = 1; + data->uvd_power_gated = false; + data->vce_power_gated = false; + data->samu_power_gated = false; + data->acp_power_gated = false; + data->pg_acp_init = true; return 0; } @@ -955,7 +958,7 @@ int tonga_check_for_dpm_running(struct pp_hwmgr *hwmgr) * because we may have test scenarios that need us intentionly disable SCLK/MCLK DPM, * whereas voltage control is a fundemental change that will not be disabled */ - return (0 == tonga_is_dpm_running(hwmgr) ? 0 : 1); + return (!tonga_is_dpm_running(hwmgr) ? 0 : 1); } /** @@ -968,7 +971,7 @@ int tonga_check_for_dpm_stopped(struct pp_hwmgr *hwmgr) { tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend); - if (0 != tonga_is_dpm_running(hwmgr)) { + if (tonga_is_dpm_running(hwmgr)) { /* If HW Virtualization is enabled, dpm_table_start will not have a valid value */ if (!data->dpm_table_start) { return 1; @@ -991,7 +994,7 @@ static int tonga_trim_voltage_table(struct pp_hwmgr *hwmgr, { uint32_t table_size, i, j; uint16_t vvalue; - bool bVoltageFound = 0; + bool bVoltageFound = false; pp_atomctrl_voltage_table *table; PP_ASSERT_WITH_CODE((NULL != voltage_table), "Voltage Table empty.", return -1;); @@ -1007,11 +1010,11 @@ static int tonga_trim_voltage_table(struct pp_hwmgr *hwmgr, for (i = 0; i < voltage_table->count; i++) { vvalue = voltage_table->entries[i].value; - bVoltageFound = 0; + bVoltageFound = false; for (j = 0; j < table->count; j++) { if (vvalue == table->entries[j].value) { - bVoltageFound = 1; + bVoltageFound = true; break; } } @@ -1331,7 +1334,6 @@ static int tonga_populate_cac_tables(struct pp_hwmgr *hwmgr, { uint32_t count; uint8_t index; - int result = 0; tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend); struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable); struct phm_ppt_v1_voltage_lookup_table *vddgfx_lookup_table = pptable_info->vddgfx_lookup_table; @@ -1378,7 +1380,7 @@ static int tonga_populate_cac_tables(struct pp_hwmgr *hwmgr, } } - return result; + return 0; } @@ -2042,7 +2044,7 @@ static int tonga_populate_single_memory_level( if ((data->mclk_stutter_mode_threshold != 0) && (memory_clock <= data->mclk_stutter_mode_threshold) && - (data->is_uvd_enabled == 0) + (!data->is_uvd_enabled) && (PHM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL, STUTTER_ENABLE) & 0x1) && (data->display_timing.num_existing_displays <= 2) && (data->display_timing.num_existing_displays != 0)) @@ -2705,7 +2707,7 @@ static int tonga_reset_single_dpm_table( dpm_table->count = count; for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++) { - dpm_table->dpm_levels[i].enabled = 0; + dpm_table->dpm_levels[i].enabled = false; } return 0; @@ -2718,7 +2720,7 @@ static void tonga_setup_pcie_table_entry( { dpm_table->dpm_levels[index].value = pcie_gen; dpm_table->dpm_levels[index].param1 = pcie_lanes; - dpm_table->dpm_levels[index].enabled = 1; + dpm_table->dpm_levels[index].enabled = true; } static int tonga_setup_default_pcie_tables(struct pp_hwmgr *hwmgr) @@ -2828,7 +2830,7 @@ static int tonga_setup_default_dpm_tables(struct pp_hwmgr *hwmgr) allowed_vdd_sclk_table->entries[i].clk) { data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].value = allowed_vdd_sclk_table->entries[i].clk; - data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].enabled = 1; /*(i==0) ? 1 : 0; to do */ + data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].enabled = true; /*(i==0) ? 1 : 0; to do */ data->dpm_table.sclk_table.count++; } } @@ -2842,7 +2844,7 @@ static int tonga_setup_default_dpm_tables(struct pp_hwmgr *hwmgr) allowed_vdd_mclk_table->entries[i].clk) { data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].value = allowed_vdd_mclk_table->entries[i].clk; - data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].enabled = 1; /*(i==0) ? 1 : 0; */ + data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].enabled = true; /*(i==0) ? 1 : 0; */ data->dpm_table.mclk_table.count++; } } @@ -3026,8 +3028,8 @@ int tonga_init_smc_table(struct pp_hwmgr *hwmgr) reg_value = 0; if ((0 == reg_value) && - (0 == atomctrl_get_pp_assign_pin(hwmgr, - VDDC_VRHOT_GPIO_PINID, &gpio_pin_assignment))) { + (atomctrl_get_pp_assign_pin(hwmgr, VDDC_VRHOT_GPIO_PINID, + &gpio_pin_assignment))) { table->VRHotGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift; phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_RegulatorHot); @@ -3040,8 +3042,8 @@ int tonga_init_smc_table(struct pp_hwmgr *hwmgr) /* ACDC Switch GPIO */ reg_value = 0; if ((0 == reg_value) && - (0 == atomctrl_get_pp_assign_pin(hwmgr, - PP_AC_DC_SWITCH_GPIO_PINID, &gpio_pin_assignment))) { + (atomctrl_get_pp_assign_pin(hwmgr, PP_AC_DC_SWITCH_GPIO_PINID, + &gpio_pin_assignment))) { table->AcDcGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift; phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_AutomaticDCTransition); @@ -3063,8 +3065,7 @@ int tonga_init_smc_table(struct pp_hwmgr *hwmgr) } reg_value = 0; - if ((0 == reg_value) && - (0 == atomctrl_get_pp_assign_pin(hwmgr, + if ((0 == reg_value) && (atomctrl_get_pp_assign_pin(hwmgr, THERMAL_INT_OUTPUT_GPIO_PINID, &gpio_pin_assignment))) { phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_ThermalOutGPIO); @@ -3135,7 +3136,7 @@ int tonga_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr) if (0 == data->sclk_dpm_key_disabled) { /* Checking if DPM is running. If we discover hang because of this, we should skip this message.*/ - if (0 != tonga_is_dpm_running(hwmgr)) + if (tonga_is_dpm_running(hwmgr)) printk(KERN_ERR "[ powerplay ] Trying to set Enable Mask when DPM is disabled \n"); if (0 != data->dpm_level_enable_mask.sclk_dpm_enable_mask) { @@ -3150,7 +3151,7 @@ int tonga_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr) if (0 == data->mclk_dpm_key_disabled) { /* Checking if DPM is running. If we discover hang because of this, we should skip this message.*/ - if (0 != tonga_is_dpm_running(hwmgr)) + if (tonga_is_dpm_running(hwmgr)) printk(KERN_ERR "[ powerplay ] Trying to set Enable Mask when DPM is disabled \n"); if (0 != data->dpm_level_enable_mask.mclk_dpm_enable_mask) { @@ -3261,7 +3262,7 @@ int tonga_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwm /* initialize vddc_dep_on_dal_pwrl table */ table_size = sizeof(uint32_t) + 4 * sizeof(struct phm_clock_voltage_dependency_record); - table_clk_vlt = (struct phm_clock_voltage_dependency_table *)kzalloc(table_size, GFP_KERNEL); + table_clk_vlt = kzalloc(table_size, GFP_KERNEL); if (NULL == table_clk_vlt) { printk(KERN_ERR "[ powerplay ] Can not allocate space for vddc_dep_on_dal_pwrl! \n"); @@ -3336,9 +3337,9 @@ int tonga_unforce_dpm_levels(struct pp_hwmgr *hwmgr) tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend); int result = 1; - PP_ASSERT_WITH_CODE (0 == tonga_is_dpm_running(hwmgr), - "Trying to Unforce DPM when DPM is disabled. Returning without sending SMC message.", - return result); + PP_ASSERT_WITH_CODE (!tonga_is_dpm_running(hwmgr), + "Trying to Unforce DPM when DPM is disabled. Returning without sending SMC message.", + return result); if (0 == data->pcie_dpm_key_disabled) { PP_ASSERT_WITH_CODE((0 == smum_send_msg_to_smc( @@ -3742,7 +3743,7 @@ uint8_t tonga_get_memory_modile_index(struct pp_hwmgr *hwmgr) bool tonga_check_s0_mc_reg_index(uint16_t inReg, uint16_t *outReg) { - bool result = 1; + bool result = true; switch (inReg) { case mmMC_SEQ_RAS_TIMING: @@ -3826,7 +3827,7 @@ bool tonga_check_s0_mc_reg_index(uint16_t inReg, uint16_t *outReg) break; default: - result = 0; + result = false; break; } @@ -4422,13 +4423,6 @@ int tonga_reset_asic_tasks(struct pp_hwmgr *hwmgr) int tonga_hwmgr_backend_fini(struct pp_hwmgr *hwmgr) { - struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend); - - if (data->soft_pp_table) { - kfree(data->soft_pp_table); - data->soft_pp_table = NULL; - } - return phm_hwmgr_backend_fini(hwmgr); } @@ -4442,7 +4436,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr) { int result = 0; SMU72_Discrete_DpmTable *table = NULL; - tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend); + tonga_hwmgr *data; pp_atomctrl_gpio_pin_assignment gpio_pin_assignment; struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable); phw_tonga_ulv_parm *ulv; @@ -4451,7 +4445,13 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr) PP_ASSERT_WITH_CODE((NULL != hwmgr), "Invalid Parameter!", return -1;); - data->dll_defaule_on = 0; + data = kzalloc(sizeof(struct tonga_hwmgr), GFP_KERNEL); + if (data == NULL) + return -ENOMEM; + + hwmgr->backend = data; + + data->dll_defaule_on = false; data->sram_end = SMC_RAM_END; data->activity_target[0] = PPTONGA_TARGETACTIVITY_DFLT; @@ -4557,13 +4557,13 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr) /* ULV Support*/ ulv = &(data->ulv); - ulv->ulv_supported = 0; + ulv->ulv_supported = false; /* Initalize Dynamic State Adjustment Rule Settings*/ result = tonga_initializa_dynamic_state_adjustment_rule_settings(hwmgr); if (result) printk(KERN_ERR "[ powerplay ] tonga_initializa_dynamic_state_adjustment_rule_settings failed!\n"); - data->uvd_enabled = 0; + data->uvd_enabled = false; table = &(data->smc_state_table); @@ -4571,7 +4571,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr) * if ucGPIO_ID=VDDC_PCC_GPIO_PINID in GPIO_LUTable, * Peak Current Control feature is enabled and we should program PCC HW register */ - if (0 == atomctrl_get_pp_assign_pin(hwmgr, VDDC_PCC_GPIO_PINID, &gpio_pin_assignment)) { + if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_PCC_GPIO_PINID, &gpio_pin_assignment)) { uint32_t temp_reg = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCNB_PWRMGT_CNTL); @@ -4610,7 +4610,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr) phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SMU7); - data->vddc_phase_shed_control = 0; + data->vddc_phase_shed_control = false; phm_cap_unset(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_UVDPowerGating); @@ -4629,7 +4629,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr) } if (0 == result) { - data->is_tlu_enabled = 0; + data->is_tlu_enabled = false; hwmgr->platform_descriptor.hardwareActivityPerformanceLevels = TONGA_MAX_HARDWARE_POWERLEVELS; hwmgr->platform_descriptor.hardwarePerformanceLevels = 2; @@ -4639,7 +4639,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr) sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO; result = cgs_query_system_info(hwmgr->device, &sys_info); if (result) - data->pcie_gen_cap = 0x30007; + data->pcie_gen_cap = AMDGPU_DEFAULT_PCIE_GEN_MASK; else data->pcie_gen_cap = (uint32_t)sys_info.value; if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) @@ -4648,7 +4648,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr) sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW; result = cgs_query_system_info(hwmgr->device, &sys_info); if (result) - data->pcie_lane_cap = 0x2f0000; + data->pcie_lane_cap = AMDGPU_DEFAULT_PCIE_MLW_MASK; else data->pcie_lane_cap = (uint32_t)sys_info.value; } else { @@ -5310,9 +5310,8 @@ static int tonga_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if ((0 == data->sclk_dpm_key_disabled) && (data->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) { - PP_ASSERT_WITH_CODE( - 0 == tonga_is_dpm_running(hwmgr), - "Trying to freeze SCLK DPM when DPM is disabled", + PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr), + "Trying to freeze SCLK DPM when DPM is disabled", ); PP_ASSERT_WITH_CODE( 0 == smum_send_msg_to_smc(hwmgr->smumgr, @@ -5324,8 +5323,8 @@ static int tonga_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if ((0 == data->mclk_dpm_key_disabled) && (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) { - PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr), - "Trying to freeze MCLK DPM when DPM is disabled", + PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr), + "Trying to freeze MCLK DPM when DPM is disabled", ); PP_ASSERT_WITH_CODE( 0 == smum_send_msg_to_smc(hwmgr->smumgr, @@ -5460,7 +5459,6 @@ static int tonga_trim_single_dpm_states(struct pp_hwmgr *hwmgr, static int tonga_trim_dpm_states(struct pp_hwmgr *hwmgr, const struct tonga_power_state *hw_state) { - int result = 0; struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend); uint32_t high_limit_count; @@ -5480,7 +5478,7 @@ static int tonga_trim_dpm_states(struct pp_hwmgr *hwmgr, const struct tonga_powe hw_state->performance_levels[0].memory_clock, hw_state->performance_levels[high_limit_count].memory_clock); - return result; + return 0; } static int tonga_generate_dpm_level_enable_mask(struct pp_hwmgr *hwmgr, const void *input) @@ -5627,8 +5625,8 @@ static int tonga_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) (data->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) { - PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr), - "Trying to Unfreeze SCLK DPM when DPM is disabled", + PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr), + "Trying to Unfreeze SCLK DPM when DPM is disabled", ); PP_ASSERT_WITH_CODE( 0 == smum_send_msg_to_smc(hwmgr->smumgr, @@ -5640,9 +5638,8 @@ static int tonga_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr) if ((0 == data->mclk_dpm_key_disabled) && (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) { - PP_ASSERT_WITH_CODE( - 0 == tonga_is_dpm_running(hwmgr), - "Trying to Unfreeze MCLK DPM when DPM is disabled", + PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr), + "Trying to Unfreeze MCLK DPM when DPM is disabled", ); PP_ASSERT_WITH_CODE( 0 == smum_send_msg_to_smc(hwmgr->smumgr, @@ -6031,42 +6028,6 @@ static int tonga_get_fan_control_mode(struct pp_hwmgr *hwmgr) CG_FDO_CTRL2, FDO_PWM_MODE); } -static int tonga_get_pp_table(struct pp_hwmgr *hwmgr, char **table) -{ - struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend); - - if (!data->soft_pp_table) { - data->soft_pp_table = kmemdup(hwmgr->soft_pp_table, - hwmgr->soft_pp_table_size, - GFP_KERNEL); - if (!data->soft_pp_table) - return -ENOMEM; - } - - *table = (char *)&data->soft_pp_table; - - return hwmgr->soft_pp_table_size; -} - -static int tonga_set_pp_table(struct pp_hwmgr *hwmgr, const char *buf, size_t size) -{ - struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend); - - if (!data->soft_pp_table) { - data->soft_pp_table = kzalloc(hwmgr->soft_pp_table_size, GFP_KERNEL); - if (!data->soft_pp_table) - return -ENOMEM; - } - - memcpy(data->soft_pp_table, buf, size); - - hwmgr->soft_pp_table = data->soft_pp_table; - - /* TODO: re-init powerplay to implement modified pptable */ - - return 0; -} - static int tonga_force_clock_level(struct pp_hwmgr *hwmgr, enum pp_clock_type type, uint32_t mask) { @@ -6174,11 +6135,96 @@ static int tonga_print_clock_levels(struct pp_hwmgr *hwmgr, return size; } +static int tonga_get_sclk_od(struct pp_hwmgr *hwmgr) +{ + struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend); + struct tonga_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table); + struct tonga_single_dpm_table *golden_sclk_table = + &(data->golden_dpm_table.sclk_table); + int value; + + value = (sclk_table->dpm_levels[sclk_table->count - 1].value - + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) * + 100 / + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value; + + return value; +} + +static int tonga_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value) +{ + struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend); + struct tonga_single_dpm_table *golden_sclk_table = + &(data->golden_dpm_table.sclk_table); + struct pp_power_state *ps; + struct tonga_power_state *tonga_ps; + + if (value > 20) + value = 20; + + ps = hwmgr->request_ps; + + if (ps == NULL) + return -EINVAL; + + tonga_ps = cast_phw_tonga_power_state(&ps->hardware); + + tonga_ps->performance_levels[tonga_ps->performance_level_count - 1].engine_clock = + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value * + value / 100 + + golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value; + + return 0; +} + +static int tonga_get_mclk_od(struct pp_hwmgr *hwmgr) +{ + struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend); + struct tonga_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table); + struct tonga_single_dpm_table *golden_mclk_table = + &(data->golden_dpm_table.mclk_table); + int value; + + value = (mclk_table->dpm_levels[mclk_table->count - 1].value - + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) * + 100 / + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value; + + return value; +} + +static int tonga_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value) +{ + struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend); + struct tonga_single_dpm_table *golden_mclk_table = + &(data->golden_dpm_table.mclk_table); + struct pp_power_state *ps; + struct tonga_power_state *tonga_ps; + + if (value > 20) + value = 20; + + ps = hwmgr->request_ps; + + if (ps == NULL) + return -EINVAL; + + tonga_ps = cast_phw_tonga_power_state(&ps->hardware); + + tonga_ps->performance_levels[tonga_ps->performance_level_count - 1].memory_clock = + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value * + value / 100 + + golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value; + + return 0; +} + static const struct pp_hwmgr_func tonga_hwmgr_funcs = { .backend_init = &tonga_hwmgr_backend_init, .backend_fini = &tonga_hwmgr_backend_fini, .asic_setup = &tonga_setup_asic_task, .dynamic_state_management_enable = &tonga_enable_dpm_tasks, + .dynamic_state_management_disable = &tonga_disable_dpm_tasks, .apply_state_adjust_rules = tonga_apply_state_adjust_rules, .force_dpm_level = &tonga_force_dpm_level, .power_state_set = tonga_set_power_state_tasks, @@ -6212,22 +6258,16 @@ static const struct pp_hwmgr_func tonga_hwmgr_funcs = { .check_states_equal = tonga_check_states_equal, .set_fan_control_mode = tonga_set_fan_control_mode, .get_fan_control_mode = tonga_get_fan_control_mode, - .get_pp_table = tonga_get_pp_table, - .set_pp_table = tonga_set_pp_table, .force_clock_level = tonga_force_clock_level, .print_clock_levels = tonga_print_clock_levels, + .get_sclk_od = tonga_get_sclk_od, + .set_sclk_od = tonga_set_sclk_od, + .get_mclk_od = tonga_get_mclk_od, + .set_mclk_od = tonga_set_mclk_od, }; int tonga_hwmgr_init(struct pp_hwmgr *hwmgr) { - tonga_hwmgr *data; - - data = kzalloc (sizeof(tonga_hwmgr), GFP_KERNEL); - if (data == NULL) - return -ENOMEM; - memset(data, 0x00, sizeof(tonga_hwmgr)); - - hwmgr->backend = data; hwmgr->hwmgr_func = &tonga_hwmgr_funcs; hwmgr->pptable_func = &tonga_pptable_funcs; pp_tonga_thermal_initialize(hwmgr); diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h index 573cd39fe..3961884bf 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h @@ -352,9 +352,6 @@ struct tonga_hwmgr { bool samu_power_gated; /* 1: gated, 0:not gated */ bool acp_power_gated; /* 1: gated, 0:not gated */ bool pg_acp_init; - - /* soft pptable for re-uploading into smu */ - void *soft_pp_table; }; typedef struct tonga_hwmgr tonga_hwmgr; diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c index dccc859f6..cfb647f76 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c @@ -167,8 +167,7 @@ static int get_vddc_lookup_table( table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_voltage_lookup_record) * max_levels; - table = (phm_ppt_v1_voltage_lookup_table *) - kzalloc(table_size, GFP_KERNEL); + table = kzalloc(table_size, GFP_KERNEL); if (NULL == table) return -ENOMEM; @@ -327,7 +326,7 @@ static int get_valid_clk( table_size = sizeof(uint32_t) + sizeof(uint32_t) * clk_volt_pp_table->count; - table = (struct phm_clock_array *)kzalloc(table_size, GFP_KERNEL); + table = kzalloc(table_size, GFP_KERNEL); if (NULL == table) return -ENOMEM; @@ -377,8 +376,7 @@ static int get_mclk_voltage_dependency_table( table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_clock_voltage_dependency_record) * mclk_dep_table->ucNumEntries; - mclk_table = (phm_ppt_v1_clock_voltage_dependency_table *) - kzalloc(table_size, GFP_KERNEL); + mclk_table = kzalloc(table_size, GFP_KERNEL); if (NULL == mclk_table) return -ENOMEM; @@ -424,8 +422,7 @@ static int get_sclk_voltage_dependency_table( table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_clock_voltage_dependency_record) * tonga_table->ucNumEntries; - sclk_table = (phm_ppt_v1_clock_voltage_dependency_table *) - kzalloc(table_size, GFP_KERNEL); + sclk_table = kzalloc(table_size, GFP_KERNEL); if (NULL == sclk_table) return -ENOMEM; @@ -456,8 +453,7 @@ static int get_sclk_voltage_dependency_table( table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_clock_voltage_dependency_record) * polaris_table->ucNumEntries; - sclk_table = (phm_ppt_v1_clock_voltage_dependency_table *) - kzalloc(table_size, GFP_KERNEL); + sclk_table = kzalloc(table_size, GFP_KERNEL); if (NULL == sclk_table) return -ENOMEM; @@ -504,7 +500,7 @@ static int get_pcie_table( table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_pcie_record) * atom_pcie_table->ucNumEntries; - pcie_table = (phm_ppt_v1_pcie_table *)kzalloc(table_size, GFP_KERNEL); + pcie_table = kzalloc(table_size, GFP_KERNEL); if (pcie_table == NULL) return -ENOMEM; @@ -541,7 +537,7 @@ static int get_pcie_table( table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_pcie_record) * atom_pcie_table->ucNumEntries; - pcie_table = (phm_ppt_v1_pcie_table *)kzalloc(table_size, GFP_KERNEL); + pcie_table = kzalloc(table_size, GFP_KERNEL); if (pcie_table == NULL) return -ENOMEM; @@ -695,8 +691,7 @@ static int get_mm_clock_voltage_table( table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_mm_clock_voltage_dependency_record) * mm_dependency_table->ucNumEntries; - mm_table = (phm_ppt_v1_mm_clock_voltage_dependency_table *) - kzalloc(table_size, GFP_KERNEL); + mm_table = kzalloc(table_size, GFP_KERNEL); if (NULL == mm_table) return -ENOMEM; @@ -1073,13 +1068,9 @@ int tonga_pp_tables_initialize(struct pp_hwmgr *hwmgr) int tonga_pp_tables_uninitialize(struct pp_hwmgr *hwmgr) { - int result = 0; struct phm_ppt_v1_information *pp_table_information = (struct phm_ppt_v1_information *)(hwmgr->pptable); - if (NULL != hwmgr->soft_pp_table) - hwmgr->soft_pp_table = NULL; - kfree(pp_table_information->vdd_dep_on_sclk); pp_table_information->vdd_dep_on_sclk = NULL; @@ -1116,7 +1107,7 @@ int tonga_pp_tables_uninitialize(struct pp_hwmgr *hwmgr) kfree(hwmgr->pptable); hwmgr->pptable = NULL; - return result; + return 0; } const struct pp_table_func tonga_pptable_funcs = { diff --git a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h index 50b367d44..b764c8c05 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h +++ b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h @@ -132,6 +132,7 @@ struct amd_pp_init { uint32_t chip_family; uint32_t chip_id; uint32_t rev_id; + bool powercontainment_enabled; }; enum amd_pp_display_config_type{ AMD_PP_DisplayConfigType_None = 0, @@ -342,6 +343,10 @@ struct amd_powerplay_funcs { int (*set_pp_table)(void *handle, const char *buf, size_t size); int (*force_clock_level)(void *handle, enum pp_clock_type type, uint32_t mask); int (*print_clock_levels)(void *handle, enum pp_clock_type type, char *buf); + int (*get_sclk_od)(void *handle); + int (*set_sclk_od)(void *handle, uint32_t value); + int (*get_mclk_od)(void *handle); + int (*set_mclk_od)(void *handle, uint32_t value); }; struct amd_powerplay { @@ -355,6 +360,8 @@ int amd_powerplay_init(struct amd_pp_init *pp_init, int amd_powerplay_fini(void *handle); +int amd_powerplay_reset(void *handle); + int amd_powerplay_display_configuration_change(void *handle, const struct amd_pp_display_configuration *input); diff --git a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h index 56f712c7d..962cb5385 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h +++ b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h @@ -340,6 +340,7 @@ extern int phm_powergate_vce(struct pp_hwmgr *hwmgr, bool gate); extern int phm_powerdown_uvd(struct pp_hwmgr *hwmgr); extern int phm_setup_asic(struct pp_hwmgr *hwmgr); extern int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr); +extern int phm_disable_dynamic_state_management(struct pp_hwmgr *hwmgr); extern void phm_init_dynamic_caps(struct pp_hwmgr *hwmgr); extern bool phm_is_hw_access_blocked(struct pp_hwmgr *hwmgr); extern int phm_block_hw_access(struct pp_hwmgr *hwmgr, bool block); diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h index 77e8e33d5..bf0d2accf 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h +++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h @@ -278,6 +278,8 @@ struct pp_hwmgr_func { int (*dynamic_state_management_enable)( struct pp_hwmgr *hw_mgr); + int (*dynamic_state_management_disable)( + struct pp_hwmgr *hw_mgr); int (*patch_boot_state)(struct pp_hwmgr *hwmgr, struct pp_hw_power_state *hw_ps); @@ -333,11 +335,13 @@ struct pp_hwmgr_func { int (*get_clock_by_type)(struct pp_hwmgr *hwmgr, enum amd_pp_clock_type type, struct amd_pp_clocks *clocks); int (*get_max_high_clocks)(struct pp_hwmgr *hwmgr, struct amd_pp_simple_clock_info *clocks); int (*power_off_asic)(struct pp_hwmgr *hwmgr); - int (*get_pp_table)(struct pp_hwmgr *hwmgr, char **table); - int (*set_pp_table)(struct pp_hwmgr *hwmgr, const char *buf, size_t size); int (*force_clock_level)(struct pp_hwmgr *hwmgr, enum pp_clock_type type, uint32_t mask); int (*print_clock_levels)(struct pp_hwmgr *hwmgr, enum pp_clock_type type, char *buf); int (*enable_per_cu_power_gating)(struct pp_hwmgr *hwmgr, bool enable); + int (*get_sclk_od)(struct pp_hwmgr *hwmgr); + int (*set_sclk_od)(struct pp_hwmgr *hwmgr, uint32_t value); + int (*get_mclk_od)(struct pp_hwmgr *hwmgr); + int (*set_mclk_od)(struct pp_hwmgr *hwmgr, uint32_t value); }; struct pp_table_func { @@ -580,6 +584,7 @@ struct pp_hwmgr { struct pp_smumgr *smumgr; const void *soft_pp_table; uint32_t soft_pp_table_size; + void *hardcode_pp_table; bool need_pp_table_upload; enum amd_dpm_forced_level dpm_level; bool block_hw_access; @@ -609,6 +614,7 @@ struct pp_hwmgr { uint32_t num_ps; struct pp_thermal_controller_info thermal_controller; bool fan_ctrl_is_in_default_mode; + bool powercontainment_enabled; uint32_t fan_ctrl_default_mode; uint32_t tmin; struct phm_microcode_version_info microcode_version_info; diff --git a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h index fc9e3d1dd..3c235f017 100644 --- a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h +++ b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h @@ -131,6 +131,12 @@ extern int smu_free_memory(void *device, void *handle); smum_wait_on_indirect_register(smumgr, \ mm##port##_INDEX, index, value, mask) +#define SMUM_WAIT_INDIRECT_REGISTER(smumgr, port, reg, value, mask) \ + SMUM_WAIT_INDIRECT_REGISTER_GIVEN_INDEX(smumgr, port, ix##reg, value, mask) + +#define SMUM_WAIT_INDIRECT_FIELD(smumgr, port, reg, field, fieldval) \ + SMUM_WAIT_INDIRECT_REGISTER(smumgr, port, reg, (fieldval) << SMUM_FIELD_SHIFT(reg, field), \ + SMUM_FIELD_MASK(reg, field) ) #define SMUM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(smumgr, \ index, value, mask) \ @@ -158,6 +164,10 @@ extern int smu_free_memory(void *device, void *handle); (SMUM_FIELD_MASK(reg, field) & ((field_val) << \ SMUM_FIELD_SHIFT(reg, field)))) +#define SMUM_READ_INDIRECT_FIELD(device, port, reg, field) \ + SMUM_GET_FIELD(cgs_read_ind_register(device, port, ix##reg), \ + reg, field) + #define SMUM_WAIT_VFPF_INDIRECT_REGISTER_GIVEN_INDEX(smumgr, \ port, index, value, mask) \ smum_wait_on_indirect_register(smumgr, \ @@ -191,6 +201,13 @@ extern int smu_free_memory(void *device, void *handle); SMUM_SET_FIELD(cgs_read_ind_register(device, port, ix##reg), \ reg, field, fieldval)) + +#define SMUM_WRITE_INDIRECT_FIELD(device, port, reg, field, fieldval) \ + cgs_write_ind_register(device, port, ix##reg, \ + SMUM_SET_FIELD(cgs_read_ind_register(device, port, ix##reg), \ + reg, field, fieldval)) + + #define SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, port, reg, field, fieldval) \ SMUM_WAIT_VFPF_INDIRECT_REGISTER(smumgr, port, reg, \ (fieldval) << SMUM_FIELD_SHIFT(reg, field), \ @@ -200,4 +217,16 @@ extern int smu_free_memory(void *device, void *handle); SMUM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL(smumgr, port, reg, \ (fieldval) << SMUM_FIELD_SHIFT(reg, field), \ SMUM_FIELD_MASK(reg, field)) + +#define SMUM_WAIT_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(smumgr, port, index, value, mask) \ + smum_wait_for_indirect_register_unequal(smumgr, \ + mm##port##_INDEX, index, value, mask) + +#define SMUM_WAIT_INDIRECT_REGISTER_UNEQUAL(smumgr, port, reg, value, mask) \ + SMUM_WAIT_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(smumgr, port, ix##reg, value, mask) + +#define SMUM_WAIT_INDIRECT_FIELD_UNEQUAL(smumgr, port, reg, field, fieldval) \ + SMUM_WAIT_INDIRECT_REGISTER_UNEQUAL(smumgr, port, reg, (fieldval) << SMUM_FIELD_SHIFT(reg, field), \ + SMUM_FIELD_MASK(reg, field) ) + #endif diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c index 0728c1e3d..7723473e5 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "pp_instance.h" #include "smumgr.h" #include "cgs_common.h" @@ -52,10 +53,10 @@ int smum_init(struct amd_pp_init *pp_init, struct pp_instance *handle) handle->smu_mgr = smumgr; switch (smumgr->chip_family) { - case AMD_FAMILY_CZ: + case AMDGPU_FAMILY_CZ: cz_smum_init(smumgr); break; - case AMD_FAMILY_VI: + case AMDGPU_FAMILY_VI: switch (smumgr->chip_id) { case CHIP_TONGA: tonga_smum_init(smumgr); diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c index b22722eab..f42c536b3 100644 --- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c +++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c @@ -479,7 +479,6 @@ static int tonga_request_smu_reload_fw(struct pp_smumgr *smumgr) struct tonga_smumgr *tonga_smu = (struct tonga_smumgr *)(smumgr->backend); uint16_t fw_to_load; - int result = 0; struct SMU_DRAMData_TOC *toc; /** * First time this gets called during SmuMgr init, @@ -563,7 +562,7 @@ static int tonga_request_smu_reload_fw(struct pp_smumgr *smumgr) smumgr, PPSMC_MSG_LoadUcodes, fw_to_load), "Fail to Request SMU Load uCode", return 0); - return result; + return 0; } static int tonga_request_smu_load_specific_fw(struct pp_smumgr *smumgr, diff --git a/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h b/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h index c89dc7777..b961a1c6c 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h @@ -26,7 +26,7 @@ TRACE_EVENT(amd_sched_job, TP_fast_assign( __entry->entity = sched_job->s_entity; __entry->sched_job = sched_job; - __entry->fence = &sched_job->s_fence->base; + __entry->fence = &sched_job->s_fence->finished; __entry->name = sched_job->sched->name; __entry->job_count = kfifo_len( &sched_job->s_entity->job_queue) / sizeof(sched_job); @@ -46,7 +46,7 @@ TRACE_EVENT(amd_sched_process_job, ), TP_fast_assign( - __entry->fence = &fence->base; + __entry->fence = &fence->finished; ), TP_printk("fence=%p signaled", __entry->fence) ); diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index c16248cee..963a24d46 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -32,6 +32,7 @@ static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity); static void amd_sched_wakeup(struct amd_gpu_scheduler *sched); +static void amd_sched_process_job(struct fence *f, struct fence_cb *cb); struct kmem_cache *sched_fence_slab; atomic_t sched_fence_slab_ref = ATOMIC_INIT(0); @@ -140,7 +141,7 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched, return r; atomic_set(&entity->fence_seq, 0); - entity->fence_context = fence_context_alloc(1); + entity->fence_context = fence_context_alloc(2); return 0; } @@ -251,17 +252,21 @@ static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity) s_fence = to_amd_sched_fence(fence); if (s_fence && s_fence->sched == sched) { - /* Fence is from the same scheduler */ - if (test_bit(AMD_SCHED_FENCE_SCHEDULED_BIT, &fence->flags)) { - /* Ignore it when it is already scheduled */ - fence_put(entity->dependency); - return false; - } - /* Wait for fence to be scheduled */ - entity->cb.func = amd_sched_entity_clear_dep; - list_add_tail(&entity->cb.node, &s_fence->scheduled_cb); - return true; + /* + * Fence is from the same scheduler, only need to wait for + * it to be scheduled + */ + fence = fence_get(&s_fence->scheduled); + fence_put(entity->dependency); + entity->dependency = fence; + if (!fence_add_callback(fence, &entity->cb, + amd_sched_entity_clear_dep)) + return true; + + /* Ignore it when it is already scheduled */ + fence_put(fence); + return false; } if (!fence_add_callback(entity->dependency, &entity->cb, @@ -319,46 +324,114 @@ static bool amd_sched_entity_in(struct amd_sched_job *sched_job) return added; } -static void amd_sched_free_job(struct fence *f, struct fence_cb *cb) { - struct amd_sched_job *job = container_of(cb, struct amd_sched_job, cb_free_job); - schedule_work(&job->work_free_job); -} - /* job_finish is called after hw fence signaled, and * the job had already been deleted from ring_mirror_list */ -void amd_sched_job_finish(struct amd_sched_job *s_job) +static void amd_sched_job_finish(struct work_struct *work) { - struct amd_sched_job *next; + struct amd_sched_job *s_job = container_of(work, struct amd_sched_job, + finish_work); struct amd_gpu_scheduler *sched = s_job->sched; + /* remove job from ring_mirror_list */ + spin_lock(&sched->job_list_lock); + list_del_init(&s_job->node); if (sched->timeout != MAX_SCHEDULE_TIMEOUT) { - if (cancel_delayed_work(&s_job->work_tdr)) - amd_sched_job_put(s_job); + struct amd_sched_job *next; + + spin_unlock(&sched->job_list_lock); + cancel_delayed_work_sync(&s_job->work_tdr); + spin_lock(&sched->job_list_lock); /* queue TDR for next job */ next = list_first_entry_or_null(&sched->ring_mirror_list, struct amd_sched_job, node); - if (next) { - INIT_DELAYED_WORK(&next->work_tdr, s_job->timeout_callback); - amd_sched_job_get(next); + if (next) schedule_delayed_work(&next->work_tdr, sched->timeout); - } } + spin_unlock(&sched->job_list_lock); + sched->ops->free_job(s_job); } -void amd_sched_job_begin(struct amd_sched_job *s_job) +static void amd_sched_job_finish_cb(struct fence *f, struct fence_cb *cb) +{ + struct amd_sched_job *job = container_of(cb, struct amd_sched_job, + finish_cb); + schedule_work(&job->finish_work); +} + +static void amd_sched_job_begin(struct amd_sched_job *s_job) { struct amd_gpu_scheduler *sched = s_job->sched; + spin_lock(&sched->job_list_lock); + list_add_tail(&s_job->node, &sched->ring_mirror_list); if (sched->timeout != MAX_SCHEDULE_TIMEOUT && - list_first_entry_or_null(&sched->ring_mirror_list, struct amd_sched_job, node) == s_job) - { - INIT_DELAYED_WORK(&s_job->work_tdr, s_job->timeout_callback); - amd_sched_job_get(s_job); + list_first_entry_or_null(&sched->ring_mirror_list, + struct amd_sched_job, node) == s_job) + schedule_delayed_work(&s_job->work_tdr, sched->timeout); + spin_unlock(&sched->job_list_lock); +} + +static void amd_sched_job_timedout(struct work_struct *work) +{ + struct amd_sched_job *job = container_of(work, struct amd_sched_job, + work_tdr.work); + + job->sched->ops->timedout_job(job); +} + +void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched) +{ + struct amd_sched_job *s_job; + + spin_lock(&sched->job_list_lock); + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { + if (fence_remove_callback(s_job->s_fence->parent, &s_job->s_fence->cb)) { + fence_put(s_job->s_fence->parent); + s_job->s_fence->parent = NULL; + } + } + atomic_set(&sched->hw_rq_count, 0); + spin_unlock(&sched->job_list_lock); +} + +void amd_sched_job_recovery(struct amd_gpu_scheduler *sched) +{ + struct amd_sched_job *s_job, *tmp; + int r; + + spin_lock(&sched->job_list_lock); + s_job = list_first_entry_or_null(&sched->ring_mirror_list, + struct amd_sched_job, node); + if (s_job && sched->timeout != MAX_SCHEDULE_TIMEOUT) schedule_delayed_work(&s_job->work_tdr, sched->timeout); + + list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { + struct amd_sched_fence *s_fence = s_job->s_fence; + struct fence *fence; + + spin_unlock(&sched->job_list_lock); + fence = sched->ops->run_job(s_job); + atomic_inc(&sched->hw_rq_count); + if (fence) { + s_fence->parent = fence_get(fence); + r = fence_add_callback(fence, &s_fence->cb, + amd_sched_process_job); + if (r == -ENOENT) + amd_sched_process_job(fence, &s_fence->cb); + else if (r) + DRM_ERROR("fence add callback failed (%d)\n", + r); + fence_put(fence); + } else { + DRM_ERROR("Failed to run job!\n"); + amd_sched_process_job(NULL, &s_fence->cb); + } + spin_lock(&sched->job_list_lock); } + spin_unlock(&sched->job_list_lock); } /** @@ -372,36 +445,29 @@ void amd_sched_entity_push_job(struct amd_sched_job *sched_job) { struct amd_sched_entity *entity = sched_job->s_entity; - sched_job->use_sched = 1; - fence_add_callback(&sched_job->s_fence->base, - &sched_job->cb_free_job, amd_sched_free_job); trace_amd_sched_job(sched_job); + fence_add_callback(&sched_job->s_fence->finished, &sched_job->finish_cb, + amd_sched_job_finish_cb); wait_event(entity->sched->job_scheduled, amd_sched_entity_in(sched_job)); } /* init a sched_job with basic field */ int amd_sched_job_init(struct amd_sched_job *job, - struct amd_gpu_scheduler *sched, - struct amd_sched_entity *entity, - void (*timeout_cb)(struct work_struct *work), - void (*free_cb)(struct kref *refcount), - void *owner, struct fence **fence) + struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + void *owner) { - INIT_LIST_HEAD(&job->node); - kref_init(&job->refcount); job->sched = sched; job->s_entity = entity; job->s_fence = amd_sched_fence_create(entity, owner); if (!job->s_fence) return -ENOMEM; - job->s_fence->s_job = job; - job->timeout_callback = timeout_cb; - job->free_callback = free_cb; + INIT_WORK(&job->finish_work, amd_sched_job_finish); + INIT_LIST_HEAD(&job->node); + INIT_DELAYED_WORK(&job->work_tdr, amd_sched_job_timedout); - if (fence) - *fence = &job->s_fence->base; return 0; } @@ -450,23 +516,25 @@ static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) struct amd_sched_fence *s_fence = container_of(cb, struct amd_sched_fence, cb); struct amd_gpu_scheduler *sched = s_fence->sched; - unsigned long flags; atomic_dec(&sched->hw_rq_count); - - /* remove job from ring_mirror_list */ - spin_lock_irqsave(&sched->job_list_lock, flags); - list_del_init(&s_fence->s_job->node); - sched->ops->finish_job(s_fence->s_job); - spin_unlock_irqrestore(&sched->job_list_lock, flags); - - amd_sched_fence_signal(s_fence); + amd_sched_fence_finished(s_fence); trace_amd_sched_process_job(s_fence); - fence_put(&s_fence->base); + fence_put(&s_fence->finished); wake_up_interruptible(&sched->wake_up_worker); } +static bool amd_sched_blocked(struct amd_gpu_scheduler *sched) +{ + if (kthread_should_park()) { + kthread_parkme(); + return true; + } + + return false; +} + static int amd_sched_main(void *param) { struct sched_param sparam = {.sched_priority = 1}; @@ -476,14 +544,15 @@ static int amd_sched_main(void *param) sched_setscheduler(current, SCHED_FIFO, &sparam); while (!kthread_should_stop()) { - struct amd_sched_entity *entity; + struct amd_sched_entity *entity = NULL; struct amd_sched_fence *s_fence; struct amd_sched_job *sched_job; struct fence *fence; wait_event_interruptible(sched->wake_up_worker, - (entity = amd_sched_select_entity(sched)) || - kthread_should_stop()); + (!amd_sched_blocked(sched) && + (entity = amd_sched_select_entity(sched))) || + kthread_should_stop()); if (!entity) continue; @@ -495,16 +564,19 @@ static int amd_sched_main(void *param) s_fence = sched_job->s_fence; atomic_inc(&sched->hw_rq_count); - amd_sched_job_pre_schedule(sched, sched_job); + amd_sched_job_begin(sched_job); + fence = sched->ops->run_job(sched_job); amd_sched_fence_scheduled(s_fence); if (fence) { + s_fence->parent = fence_get(fence); r = fence_add_callback(fence, &s_fence->cb, amd_sched_process_job); if (r == -ENOENT) amd_sched_process_job(fence, &s_fence->cb); else if (r) - DRM_ERROR("fence add callback failed (%d)\n", r); + DRM_ERROR("fence add callback failed (%d)\n", + r); fence_put(fence); } else { DRM_ERROR("Failed to run job!\n"); diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 070095a94..7cbbbfb50 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -27,8 +27,6 @@ #include #include -#define AMD_SCHED_FENCE_SCHEDULED_BIT FENCE_FLAG_USER_BITS - struct amd_gpu_scheduler; struct amd_sched_rq; @@ -68,36 +66,34 @@ struct amd_sched_rq { }; struct amd_sched_fence { - struct fence base; + struct fence scheduled; + struct fence finished; struct fence_cb cb; - struct list_head scheduled_cb; + struct fence *parent; struct amd_gpu_scheduler *sched; spinlock_t lock; void *owner; - struct amd_sched_job *s_job; }; struct amd_sched_job { - struct kref refcount; struct amd_gpu_scheduler *sched; struct amd_sched_entity *s_entity; struct amd_sched_fence *s_fence; - bool use_sched; /* true if the job goes to scheduler */ - struct fence_cb cb_free_job; - struct work_struct work_free_job; - struct list_head node; - struct delayed_work work_tdr; - void (*timeout_callback) (struct work_struct *work); - void (*free_callback)(struct kref *refcount); + struct fence_cb finish_cb; + struct work_struct finish_work; + struct list_head node; + struct delayed_work work_tdr; }; -extern const struct fence_ops amd_sched_fence_ops; +extern const struct fence_ops amd_sched_fence_ops_scheduled; +extern const struct fence_ops amd_sched_fence_ops_finished; static inline struct amd_sched_fence *to_amd_sched_fence(struct fence *f) { - struct amd_sched_fence *__f = container_of(f, struct amd_sched_fence, base); + if (f->ops == &amd_sched_fence_ops_scheduled) + return container_of(f, struct amd_sched_fence, scheduled); - if (__f->base.ops == &amd_sched_fence_ops) - return __f; + if (f->ops == &amd_sched_fence_ops_finished) + return container_of(f, struct amd_sched_fence, finished); return NULL; } @@ -109,8 +105,8 @@ static inline struct amd_sched_fence *to_amd_sched_fence(struct fence *f) struct amd_sched_backend_ops { struct fence *(*dependency)(struct amd_sched_job *sched_job); struct fence *(*run_job)(struct amd_sched_job *sched_job); - void (*begin_job)(struct amd_sched_job *sched_job); - void (*finish_job)(struct amd_sched_job *sched_job); + void (*timedout_job)(struct amd_sched_job *sched_job); + void (*free_job)(struct amd_sched_job *sched_job); }; enum amd_sched_priority { @@ -152,25 +148,11 @@ void amd_sched_entity_push_job(struct amd_sched_job *sched_job); struct amd_sched_fence *amd_sched_fence_create( struct amd_sched_entity *s_entity, void *owner); void amd_sched_fence_scheduled(struct amd_sched_fence *fence); -void amd_sched_fence_signal(struct amd_sched_fence *fence); +void amd_sched_fence_finished(struct amd_sched_fence *fence); int amd_sched_job_init(struct amd_sched_job *job, - struct amd_gpu_scheduler *sched, - struct amd_sched_entity *entity, - void (*timeout_cb)(struct work_struct *work), - void (*free_cb)(struct kref* refcount), - void *owner, struct fence **fence); -void amd_sched_job_pre_schedule(struct amd_gpu_scheduler *sched , - struct amd_sched_job *s_job); -void amd_sched_job_finish(struct amd_sched_job *s_job); -void amd_sched_job_begin(struct amd_sched_job *s_job); -static inline void amd_sched_job_get(struct amd_sched_job *job) { - if (job) - kref_get(&job->refcount); -} - -static inline void amd_sched_job_put(struct amd_sched_job *job) { - if (job) - kref_put(&job->refcount, job->free_callback); -} - + struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + void *owner); +void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched); +void amd_sched_job_recovery(struct amd_gpu_scheduler *sched); #endif diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c index 2a732c490..6b63beaf7 100644 --- a/drivers/gpu/drm/amd/scheduler/sched_fence.c +++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c @@ -27,7 +27,8 @@ #include #include "gpu_scheduler.h" -struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *s_entity, void *owner) +struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *entity, + void *owner) { struct amd_sched_fence *fence = NULL; unsigned seq; @@ -36,46 +37,37 @@ struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *s_entity if (fence == NULL) return NULL; - INIT_LIST_HEAD(&fence->scheduled_cb); fence->owner = owner; - fence->sched = s_entity->sched; + fence->sched = entity->sched; spin_lock_init(&fence->lock); - seq = atomic_inc_return(&s_entity->fence_seq); - fence_init(&fence->base, &amd_sched_fence_ops, &fence->lock, - s_entity->fence_context, seq); + seq = atomic_inc_return(&entity->fence_seq); + fence_init(&fence->scheduled, &amd_sched_fence_ops_scheduled, + &fence->lock, entity->fence_context, seq); + fence_init(&fence->finished, &amd_sched_fence_ops_finished, + &fence->lock, entity->fence_context + 1, seq); return fence; } -void amd_sched_fence_signal(struct amd_sched_fence *fence) +void amd_sched_fence_scheduled(struct amd_sched_fence *fence) { - int ret = fence_signal(&fence->base); + int ret = fence_signal(&fence->scheduled); + if (!ret) - FENCE_TRACE(&fence->base, "signaled from irq context\n"); + FENCE_TRACE(&fence->scheduled, "signaled from irq context\n"); else - FENCE_TRACE(&fence->base, "was already signaled\n"); -} - -void amd_sched_job_pre_schedule(struct amd_gpu_scheduler *sched , - struct amd_sched_job *s_job) -{ - unsigned long flags; - spin_lock_irqsave(&sched->job_list_lock, flags); - list_add_tail(&s_job->node, &sched->ring_mirror_list); - sched->ops->begin_job(s_job); - spin_unlock_irqrestore(&sched->job_list_lock, flags); + FENCE_TRACE(&fence->scheduled, "was already signaled\n"); } -void amd_sched_fence_scheduled(struct amd_sched_fence *s_fence) +void amd_sched_fence_finished(struct amd_sched_fence *fence) { - struct fence_cb *cur, *tmp; + int ret = fence_signal(&fence->finished); - set_bit(AMD_SCHED_FENCE_SCHEDULED_BIT, &s_fence->base.flags); - list_for_each_entry_safe(cur, tmp, &s_fence->scheduled_cb, node) { - list_del_init(&cur->node); - cur->func(&s_fence->base, cur); - } + if (!ret) + FENCE_TRACE(&fence->finished, "signaled from irq context\n"); + else + FENCE_TRACE(&fence->finished, "was already signaled\n"); } static const char *amd_sched_fence_get_driver_name(struct fence *fence) @@ -105,6 +97,8 @@ static void amd_sched_fence_free(struct rcu_head *rcu) { struct fence *f = container_of(rcu, struct fence, rcu); struct amd_sched_fence *fence = to_amd_sched_fence(f); + + fence_put(fence->parent); kmem_cache_free(sched_fence_slab, fence); } @@ -116,16 +110,41 @@ static void amd_sched_fence_free(struct rcu_head *rcu) * This function is called when the reference count becomes zero. * It just RCU schedules freeing up the fence. */ -static void amd_sched_fence_release(struct fence *f) +static void amd_sched_fence_release_scheduled(struct fence *f) { - call_rcu(&f->rcu, amd_sched_fence_free); + struct amd_sched_fence *fence = to_amd_sched_fence(f); + + call_rcu(&fence->finished.rcu, amd_sched_fence_free); } -const struct fence_ops amd_sched_fence_ops = { +/** + * amd_sched_fence_release_scheduled - drop extra reference + * + * @f: fence + * + * Drop the extra reference from the scheduled fence to the base fence. + */ +static void amd_sched_fence_release_finished(struct fence *f) +{ + struct amd_sched_fence *fence = to_amd_sched_fence(f); + + fence_put(&fence->scheduled); +} + +const struct fence_ops amd_sched_fence_ops_scheduled = { + .get_driver_name = amd_sched_fence_get_driver_name, + .get_timeline_name = amd_sched_fence_get_timeline_name, + .enable_signaling = amd_sched_fence_enable_signaling, + .signaled = NULL, + .wait = fence_default_wait, + .release = amd_sched_fence_release_scheduled, +}; + +const struct fence_ops amd_sched_fence_ops_finished = { .get_driver_name = amd_sched_fence_get_driver_name, .get_timeline_name = amd_sched_fence_get_timeline_name, .enable_signaling = amd_sched_fence_enable_signaling, .signaled = NULL, .wait = fence_default_wait, - .release = amd_sched_fence_release, + .release = amd_sched_fence_release_finished, }; diff --git a/drivers/gpu/drm/arc/Kconfig b/drivers/gpu/drm/arc/Kconfig index f9a13b658..f47d88ba4 100644 --- a/drivers/gpu/drm/arc/Kconfig +++ b/drivers/gpu/drm/arc/Kconfig @@ -2,7 +2,6 @@ config DRM_ARCPGU tristate "ARC PGU" depends on DRM && OF select DRM_KMS_CMA_HELPER - select DRM_KMS_FB_HELPER select DRM_KMS_HELPER help Choose this option if you have an ARC PGU controller. diff --git a/drivers/gpu/drm/arc/Makefile b/drivers/gpu/drm/arc/Makefile index d48fda70f..73de56a01 100644 --- a/drivers/gpu/drm/arc/Makefile +++ b/drivers/gpu/drm/arc/Makefile @@ -1,2 +1,2 @@ -arcpgu-y := arcpgu_crtc.o arcpgu_hdmi.o arcpgu_drv.o +arcpgu-y := arcpgu_crtc.o arcpgu_hdmi.o arcpgu_sim.o arcpgu_drv.o obj-$(CONFIG_DRM_ARCPGU) += arcpgu.o diff --git a/drivers/gpu/drm/arc/arcpgu.h b/drivers/gpu/drm/arc/arcpgu.h index 86574b698..e8fcf3ab1 100644 --- a/drivers/gpu/drm/arc/arcpgu.h +++ b/drivers/gpu/drm/arc/arcpgu.h @@ -22,7 +22,6 @@ struct arcpgu_drm_private { struct clk *clk; struct drm_fbdev_cma *fbdev; struct drm_framebuffer *fb; - struct list_head event_list; struct drm_crtc crtc; struct drm_plane *plane; }; @@ -43,6 +42,7 @@ static inline u32 arc_pgu_read(struct arcpgu_drm_private *arcpgu, int arc_pgu_setup_crtc(struct drm_device *dev); int arcpgu_drm_hdmi_init(struct drm_device *drm, struct device_node *np); +int arcpgu_drm_sim_init(struct drm_device *drm, struct device_node *np); struct drm_fbdev_cma *arcpgu_fbdev_cma_init(struct drm_device *dev, unsigned int preferred_bpp, unsigned int num_crtc, unsigned int max_conn_count); diff --git a/drivers/gpu/drm/arc/arcpgu_crtc.c b/drivers/gpu/drm/arc/arcpgu_crtc.c index 92f8beff8..ee0a61c28 100644 --- a/drivers/gpu/drm/arc/arcpgu_crtc.c +++ b/drivers/gpu/drm/arc/arcpgu_crtc.c @@ -145,20 +145,14 @@ static int arc_pgu_crtc_atomic_check(struct drm_crtc *crtc, static void arc_pgu_crtc_atomic_begin(struct drm_crtc *crtc, struct drm_crtc_state *state) { - struct arcpgu_drm_private *arcpgu = crtc_to_arcpgu_priv(crtc); - unsigned long flags; - - if (crtc->state->event) { - struct drm_pending_vblank_event *event = crtc->state->event; + struct drm_pending_vblank_event *event = crtc->state->event; + if (event) { crtc->state->event = NULL; - event->pipe = drm_crtc_index(crtc); - - WARN_ON(drm_crtc_vblank_get(crtc) != 0); - spin_lock_irqsave(&crtc->dev->event_lock, flags); - list_add_tail(&event->base.link, &arcpgu->event_list); - spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + spin_lock_irq(&crtc->dev->event_lock); + drm_crtc_send_vblank_event(crtc, event); + spin_unlock_irq(&crtc->dev->event_lock); } } diff --git a/drivers/gpu/drm/arc/arcpgu_drv.c b/drivers/gpu/drm/arc/arcpgu_drv.c index 76e187a5b..6d4ff3473 100644 --- a/drivers/gpu/drm/arc/arcpgu_drv.c +++ b/drivers/gpu/drm/arc/arcpgu_drv.c @@ -28,21 +28,14 @@ static void arcpgu_fb_output_poll_changed(struct drm_device *dev) { struct arcpgu_drm_private *arcpgu = dev->dev_private; - if (arcpgu->fbdev) - drm_fbdev_cma_hotplug_event(arcpgu->fbdev); -} - -static int arcpgu_atomic_commit(struct drm_device *dev, - struct drm_atomic_state *state, bool async) -{ - return drm_atomic_helper_commit(dev, state, false); + drm_fbdev_cma_hotplug_event(arcpgu->fbdev); } static struct drm_mode_config_funcs arcpgu_drm_modecfg_funcs = { .fb_create = drm_fb_cma_create, .output_poll_changed = arcpgu_fb_output_poll_changed, .atomic_check = drm_atomic_helper_check, - .atomic_commit = arcpgu_atomic_commit, + .atomic_commit = drm_atomic_helper_commit, }; static void arcpgu_setup_mode_config(struct drm_device *drm) @@ -55,7 +48,7 @@ static void arcpgu_setup_mode_config(struct drm_device *drm) drm->mode_config.funcs = &arcpgu_drm_modecfg_funcs; } -int arcpgu_gem_mmap(struct file *filp, struct vm_area_struct *vma) +static int arcpgu_gem_mmap(struct file *filp, struct vm_area_struct *vma) { int ret; @@ -81,22 +74,6 @@ static const struct file_operations arcpgu_drm_ops = { .mmap = arcpgu_gem_mmap, }; -static void arcpgu_preclose(struct drm_device *drm, struct drm_file *file) -{ - struct arcpgu_drm_private *arcpgu = drm->dev_private; - struct drm_pending_vblank_event *e, *t; - unsigned long flags; - - spin_lock_irqsave(&drm->event_lock, flags); - list_for_each_entry_safe(e, t, &arcpgu->event_list, base.link) { - if (e->base.file_priv != file) - continue; - list_del(&e->base.link); - e->base.destroy(&e->base); - } - spin_unlock_irqrestore(&drm->event_lock, flags); -} - static void arcpgu_lastclose(struct drm_device *drm) { struct arcpgu_drm_private *arcpgu = drm->dev_private; @@ -122,16 +99,12 @@ static int arcpgu_load(struct drm_device *drm) if (IS_ERR(arcpgu->clk)) return PTR_ERR(arcpgu->clk); - INIT_LIST_HEAD(&arcpgu->event_list); - arcpgu_setup_mode_config(drm); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); arcpgu->regs = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(arcpgu->regs)) { - dev_err(drm->dev, "Could not remap IO mem\n"); + if (IS_ERR(arcpgu->regs)) return PTR_ERR(arcpgu->regs); - } dev_info(drm->dev, "arc_pgu ID: 0x%x\n", arc_pgu_read(arcpgu, ARCPGU_REG_ID)); @@ -149,15 +122,17 @@ static int arcpgu_load(struct drm_device *drm) /* find the encoder node and initialize it */ encoder_node = of_parse_phandle(drm->dev->of_node, "encoder-slave", 0); - if (!encoder_node) { - dev_err(drm->dev, "failed to get an encoder slave node\n"); - return -ENODEV; + if (encoder_node) { + ret = arcpgu_drm_hdmi_init(drm, encoder_node); + of_node_put(encoder_node); + if (ret < 0) + return ret; + } else { + ret = arcpgu_drm_sim_init(drm, NULL); + if (ret < 0) + return ret; } - ret = arcpgu_drm_hdmi_init(drm, encoder_node); - if (ret < 0) - return ret; - drm_mode_config_reset(drm); drm_kms_helper_poll_init(drm); @@ -174,7 +149,7 @@ static int arcpgu_load(struct drm_device *drm) return 0; } -int arcpgu_unload(struct drm_device *drm) +static int arcpgu_unload(struct drm_device *drm) { struct arcpgu_drm_private *arcpgu = drm->dev_private; @@ -192,7 +167,6 @@ int arcpgu_unload(struct drm_device *drm) static struct drm_driver arcpgu_drm_driver = { .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME | DRIVER_ATOMIC, - .preclose = arcpgu_preclose, .lastclose = arcpgu_lastclose, .name = "drm-arcpgu", .desc = "ARC PGU Controller", @@ -207,7 +181,7 @@ static struct drm_driver arcpgu_drm_driver = { .get_vblank_counter = drm_vblank_no_hw_counter, .prime_handle_to_fd = drm_gem_prime_handle_to_fd, .prime_fd_to_handle = drm_gem_prime_fd_to_handle, - .gem_free_object = drm_gem_cma_free_object, + .gem_free_object_unlocked = drm_gem_cma_free_object, .gem_vm_ops = &drm_gem_cma_vm_ops, .gem_prime_export = drm_gem_prime_export, .gem_prime_import = drm_gem_prime_import, @@ -235,15 +209,8 @@ static int arcpgu_probe(struct platform_device *pdev) if (ret) goto err_unload; - ret = drm_connector_register_all(drm); - if (ret) - goto err_unregister; - return 0; -err_unregister: - drm_dev_unregister(drm); - err_unload: arcpgu_unload(drm); @@ -257,7 +224,6 @@ static int arcpgu_remove(struct platform_device *pdev) { struct drm_device *drm = platform_get_drvdata(pdev); - drm_connector_unregister_all(drm); drm_dev_unregister(drm); arcpgu_unload(drm); drm_dev_unref(drm); diff --git a/drivers/gpu/drm/arc/arcpgu_hdmi.c b/drivers/gpu/drm/arc/arcpgu_hdmi.c index 08b6baeb3..b7a8b2ac4 100644 --- a/drivers/gpu/drm/arc/arcpgu_hdmi.c +++ b/drivers/gpu/drm/arc/arcpgu_hdmi.c @@ -46,23 +46,6 @@ static int arcpgu_drm_connector_get_modes(struct drm_connector *connector) return sfuncs->get_modes(&slave->base, connector); } -struct drm_encoder * -arcpgu_drm_connector_best_encoder(struct drm_connector *connector) -{ - struct drm_encoder_slave *slave; - struct arcpgu_drm_connector *con = - container_of(connector, struct arcpgu_drm_connector, connector); - - slave = con->encoder_slave; - if (slave == NULL) { - dev_err(connector->dev->dev, - "connector_best_encoder: cannot find slave encoder for connector\n"); - return NULL; - } - - return &slave->base; -} - static enum drm_connector_status arcpgu_drm_connector_detect(struct drm_connector *connector, bool force) { @@ -97,7 +80,6 @@ static void arcpgu_drm_connector_destroy(struct drm_connector *connector) static const struct drm_connector_helper_funcs arcpgu_drm_connector_helper_funcs = { .get_modes = arcpgu_drm_connector_get_modes, - .best_encoder = arcpgu_drm_connector_best_encoder, }; static const struct drm_connector_funcs arcpgu_drm_connector_funcs = { diff --git a/drivers/gpu/drm/arc/arcpgu_sim.c b/drivers/gpu/drm/arc/arcpgu_sim.c new file mode 100644 index 000000000..2bf06d715 --- /dev/null +++ b/drivers/gpu/drm/arc/arcpgu_sim.c @@ -0,0 +1,128 @@ +/* + * ARC PGU DRM driver. + * + * Copyright (C) 2016 Synopsys, Inc. (www.synopsys.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include + +#include "arcpgu.h" + +#define XRES_DEF 640 +#define YRES_DEF 480 + +#define XRES_MAX 8192 +#define YRES_MAX 8192 + + +struct arcpgu_drm_connector { + struct drm_connector connector; + struct drm_encoder_slave *encoder_slave; +}; + +static int arcpgu_drm_connector_get_modes(struct drm_connector *connector) +{ + int count; + + count = drm_add_modes_noedid(connector, XRES_MAX, YRES_MAX); + drm_set_preferred_mode(connector, XRES_DEF, YRES_DEF); + return count; +} + +static enum drm_connector_status +arcpgu_drm_connector_detect(struct drm_connector *connector, bool force) +{ + return connector_status_connected; +} + +static void arcpgu_drm_connector_destroy(struct drm_connector *connector) +{ + drm_connector_unregister(connector); + drm_connector_cleanup(connector); +} + +static const struct drm_connector_helper_funcs +arcpgu_drm_connector_helper_funcs = { + .get_modes = arcpgu_drm_connector_get_modes, +}; + +static const struct drm_connector_funcs arcpgu_drm_connector_funcs = { + .dpms = drm_helper_connector_dpms, + .reset = drm_atomic_helper_connector_reset, + .detect = arcpgu_drm_connector_detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .destroy = arcpgu_drm_connector_destroy, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static struct drm_encoder_funcs arcpgu_drm_encoder_funcs = { + .destroy = drm_encoder_cleanup, +}; + +int arcpgu_drm_sim_init(struct drm_device *drm, struct device_node *np) +{ + struct arcpgu_drm_connector *arcpgu_connector; + struct drm_encoder_slave *encoder; + struct drm_connector *connector; + int ret; + + encoder = devm_kzalloc(drm->dev, sizeof(*encoder), GFP_KERNEL); + if (encoder == NULL) + return -ENOMEM; + + encoder->base.possible_crtcs = 1; + encoder->base.possible_clones = 0; + + ret = drm_encoder_init(drm, &encoder->base, &arcpgu_drm_encoder_funcs, + DRM_MODE_ENCODER_VIRTUAL, NULL); + if (ret) + return ret; + + arcpgu_connector = devm_kzalloc(drm->dev, sizeof(*arcpgu_connector), + GFP_KERNEL); + if (!arcpgu_connector) { + ret = -ENOMEM; + goto error_encoder_cleanup; + } + + connector = &arcpgu_connector->connector; + drm_connector_helper_add(connector, &arcpgu_drm_connector_helper_funcs); + + ret = drm_connector_init(drm, connector, &arcpgu_drm_connector_funcs, + DRM_MODE_CONNECTOR_VIRTUAL); + if (ret < 0) { + dev_err(drm->dev, "failed to initialize drm connector\n"); + goto error_encoder_cleanup; + } + + ret = drm_mode_connector_attach_encoder(connector, &encoder->base); + if (ret < 0) { + dev_err(drm->dev, "could not attach connector to encoder\n"); + drm_connector_unregister(connector); + goto error_connector_cleanup; + } + + arcpgu_connector->encoder_slave = encoder; + + return 0; + +error_connector_cleanup: + drm_connector_cleanup(connector); + +error_encoder_cleanup: + drm_encoder_cleanup(&encoder->base); + return ret; +} diff --git a/drivers/gpu/drm/arm/Kconfig b/drivers/gpu/drm/arm/Kconfig index eaed454e0..9a18e1bd5 100644 --- a/drivers/gpu/drm/arm/Kconfig +++ b/drivers/gpu/drm/arm/Kconfig @@ -9,7 +9,6 @@ config DRM_HDLCD depends on COMMON_CLK select DRM_ARM select DRM_KMS_HELPER - select DRM_KMS_FB_HELPER select DRM_KMS_CMA_HELPER help Choose this option if you have an ARM High Definition Colour LCD @@ -25,3 +24,19 @@ config DRM_HDLCD_SHOW_UNDERRUN Enable this option to show in red colour the pixels that the HDLCD device did not fetch from framebuffer due to underrun conditions. + +config DRM_MALI_DISPLAY + tristate "ARM Mali Display Processor" + depends on DRM && OF && (ARM || ARM64) + depends on COMMON_CLK + select DRM_ARM + select DRM_KMS_HELPER + select DRM_KMS_CMA_HELPER + select DRM_GEM_CMA_HELPER + select VIDEOMODE_HELPERS + help + Choose this option if you want to compile the ARM Mali Display + Processor driver. It supports the DP500, DP550 and DP650 variants + of the hardware. + + If compiled as a module it will be called mali-dp. diff --git a/drivers/gpu/drm/arm/Makefile b/drivers/gpu/drm/arm/Makefile index 89dcb7bab..bb8b158ff 100644 --- a/drivers/gpu/drm/arm/Makefile +++ b/drivers/gpu/drm/arm/Makefile @@ -1,2 +1,4 @@ hdlcd-y := hdlcd_drv.o hdlcd_crtc.o obj-$(CONFIG_DRM_HDLCD) += hdlcd.o +mali-dp-y := malidp_drv.o malidp_hw.o malidp_planes.o malidp_crtc.o +obj-$(CONFIG_DRM_MALI_DISPLAY) += mali-dp.o diff --git a/drivers/gpu/drm/arm/hdlcd_crtc.c b/drivers/gpu/drm/arm/hdlcd_crtc.c index 0813c2f06..48019ae22 100644 --- a/drivers/gpu/drm/arm/hdlcd_crtc.c +++ b/drivers/gpu/drm/arm/hdlcd_crtc.c @@ -196,30 +196,11 @@ static void hdlcd_crtc_atomic_begin(struct drm_crtc *crtc, } } -static void hdlcd_crtc_atomic_flush(struct drm_crtc *crtc, - struct drm_crtc_state *state) -{ -} - -static bool hdlcd_crtc_mode_fixup(struct drm_crtc *crtc, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) -{ - return true; -} - static const struct drm_crtc_helper_funcs hdlcd_crtc_helper_funcs = { - .mode_fixup = hdlcd_crtc_mode_fixup, - .mode_set = drm_helper_crtc_mode_set, - .mode_set_base = drm_helper_crtc_mode_set_base, - .mode_set_nofb = hdlcd_crtc_mode_set_nofb, .enable = hdlcd_crtc_enable, .disable = hdlcd_crtc_disable, - .prepare = hdlcd_crtc_disable, - .commit = hdlcd_crtc_enable, .atomic_check = hdlcd_crtc_atomic_check, .atomic_begin = hdlcd_crtc_atomic_begin, - .atomic_flush = hdlcd_crtc_atomic_flush, }; static int hdlcd_plane_atomic_check(struct drm_plane *plane, diff --git a/drivers/gpu/drm/arm/hdlcd_drv.c b/drivers/gpu/drm/arm/hdlcd_drv.c index a6ca36f00..d83b46a30 100644 --- a/drivers/gpu/drm/arm/hdlcd_drv.c +++ b/drivers/gpu/drm/arm/hdlcd_drv.c @@ -102,21 +102,14 @@ static void hdlcd_fb_output_poll_changed(struct drm_device *drm) { struct hdlcd_drm_private *hdlcd = drm->dev_private; - if (hdlcd->fbdev) - drm_fbdev_cma_hotplug_event(hdlcd->fbdev); -} - -static int hdlcd_atomic_commit(struct drm_device *dev, - struct drm_atomic_state *state, bool nonblock) -{ - return drm_atomic_helper_commit(dev, state, false); + drm_fbdev_cma_hotplug_event(hdlcd->fbdev); } static const struct drm_mode_config_funcs hdlcd_mode_config_funcs = { .fb_create = drm_fb_cma_create, .output_poll_changed = hdlcd_fb_output_poll_changed, .atomic_check = drm_atomic_helper_check, - .atomic_commit = hdlcd_atomic_commit, + .atomic_commit = drm_atomic_helper_commit, }; static void hdlcd_setup_mode_config(struct drm_device *drm) @@ -296,7 +289,7 @@ static struct drm_driver hdlcd_driver = { .get_vblank_counter = drm_vblank_no_hw_counter, .enable_vblank = hdlcd_enable_vblank, .disable_vblank = hdlcd_disable_vblank, - .gem_free_object = drm_gem_cma_free_object, + .gem_free_object_unlocked = drm_gem_cma_free_object, .gem_vm_ops = &drm_gem_cma_vm_ops, .dumb_create = drm_gem_cma_dumb_create, .dumb_map_offset = drm_gem_cma_dumb_map_offset, diff --git a/drivers/gpu/drm/arm/malidp_crtc.c b/drivers/gpu/drm/arm/malidp_crtc.c new file mode 100644 index 000000000..08e6a71f5 --- /dev/null +++ b/drivers/gpu/drm/arm/malidp_crtc.c @@ -0,0 +1,216 @@ +/* + * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. + * Author: Liviu Dudau + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * ARM Mali DP500/DP550/DP650 driver (crtc operations) + */ + +#include +#include +#include +#include +#include +#include +#include