1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
|
/*
* Coherency fabric (Aurora) support for Armada 370, 375, 38x and XP
* platforms.
*
* Copyright (C) 2012 Marvell
*
* Yehuda Yitschak <yehuday@marvell.com>
* Gregory Clement <gregory.clement@free-electrons.com>
* Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
*
* This file is licensed under the terms of the GNU General Public
* License version 2. This program is licensed "as is" without any
* warranty of any kind, whether express or implied.
*
* The Armada 370, 375, 38x and XP SOCs have a coherency fabric which is
* responsible for ensuring hardware coherency between all CPUs and between
* CPUs and I/O masters. This file initializes the coherency fabric and
* supplies basic routines for configuring and controlling hardware coherency
*/
#define pr_fmt(fmt) "mvebu-coherency: " fmt
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/of_address.h>
#include <linux/io.h>
#include <linux/smp.h>
#include <linux/dma-mapping.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/mbus.h>
#include <linux/pci.h>
#include <asm/smp_plat.h>
#include <asm/cacheflush.h>
#include <asm/mach/map.h>
#include <asm/dma-mapping.h>
#include "coherency.h"
#include "mvebu-soc-id.h"
unsigned long coherency_phys_base;
void __iomem *coherency_base;
static void __iomem *coherency_cpu_base;
/* Coherency fabric registers */
#define IO_SYNC_BARRIER_CTL_OFFSET 0x0
enum {
COHERENCY_FABRIC_TYPE_NONE,
COHERENCY_FABRIC_TYPE_ARMADA_370_XP,
COHERENCY_FABRIC_TYPE_ARMADA_375,
COHERENCY_FABRIC_TYPE_ARMADA_380,
};
static const struct of_device_id of_coherency_table[] = {
{.compatible = "marvell,coherency-fabric",
.data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_370_XP },
{.compatible = "marvell,armada-375-coherency-fabric",
.data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_375 },
{.compatible = "marvell,armada-380-coherency-fabric",
.data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_380 },
{ /* end of list */ },
};
/* Functions defined in coherency_ll.S */
int ll_enable_coherency(void);
void ll_add_cpu_to_smp_group(void);
static int mvebu_hwcc_notifier(struct notifier_block *nb,
unsigned long event, void *__dev)
{
struct device *dev = __dev;
if (event != BUS_NOTIFY_ADD_DEVICE)
return NOTIFY_DONE;
set_dma_ops(dev, &arm_coherent_dma_ops);
return NOTIFY_OK;
}
static struct notifier_block mvebu_hwcc_nb = {
.notifier_call = mvebu_hwcc_notifier,
};
static struct notifier_block mvebu_hwcc_pci_nb = {
.notifier_call = mvebu_hwcc_notifier,
};
static void __init armada_370_coherency_init(struct device_node *np)
{
struct resource res;
of_address_to_resource(np, 0, &res);
coherency_phys_base = res.start;
/*
* Ensure secondary CPUs will see the updated value,
* which they read before they join the coherency
* fabric, and therefore before they are coherent with
* the boot CPU cache.
*/
sync_cache_w(&coherency_phys_base);
coherency_base = of_iomap(np, 0);
coherency_cpu_base = of_iomap(np, 1);
set_cpu_coherent();
}
/*
* This ioremap hook is used on Armada 375/38x to ensure that PCIe
* memory areas are mapped as MT_UNCACHED instead of MT_DEVICE. This
* is needed as a workaround for a deadlock issue between the PCIe
* interface and the cache controller.
*/
static void __iomem *
armada_pcie_wa_ioremap_caller(phys_addr_t phys_addr, size_t size,
unsigned int mtype, void *caller)
{
struct resource pcie_mem;
mvebu_mbus_get_pcie_mem_aperture(&pcie_mem);
if (pcie_mem.start <= phys_addr && (phys_addr + size) <= pcie_mem.end)
mtype = MT_UNCACHED;
return __arm_ioremap_caller(phys_addr, size, mtype, caller);
}
static void __init armada_375_380_coherency_init(struct device_node *np)
{
struct device_node *cache_dn;
coherency_cpu_base = of_iomap(np, 0);
arch_ioremap_caller = armada_pcie_wa_ioremap_caller;
/*
* We should switch the PL310 to I/O coherency mode only if
* I/O coherency is actually enabled.
*/
if (!coherency_available())
return;
/*
* Add the PL310 property "arm,io-coherent". This makes sure the
* outer sync operation is not used, which allows to
* workaround the system erratum that causes deadlocks when
* doing PCIe in an SMP situation on Armada 375 and Armada
* 38x.
*/
for_each_compatible_node(cache_dn, NULL, "arm,pl310-cache") {
struct property *p;
p = kzalloc(sizeof(*p), GFP_KERNEL);
p->name = kstrdup("arm,io-coherent", GFP_KERNEL);
of_add_property(cache_dn, p);
}
}
static int coherency_type(void)
{
struct device_node *np;
const struct of_device_id *match;
int type;
/*
* The coherency fabric is needed:
* - For coherency between processors on Armada XP, so only
* when SMP is enabled.
* - For coherency between the processor and I/O devices, but
* this coherency requires many pre-requisites (write
* allocate cache policy, shareable pages, SMP bit set) that
* are only meant in SMP situations.
*
* Note that this means that on Armada 370, there is currently
* no way to use hardware I/O coherency, because even when
* CONFIG_SMP is enabled, is_smp() returns false due to the
* Armada 370 being a single-core processor. To lift this
* limitation, we would have to find a way to make the cache
* policy set to write-allocate (on all Armada SoCs), and to
* set the shareable attribute in page tables (on all Armada
* SoCs except the Armada 370). Unfortunately, such decisions
* are taken very early in the kernel boot process, at a point
* where we don't know yet on which SoC we are running.
*/
if (!is_smp())
return COHERENCY_FABRIC_TYPE_NONE;
np = of_find_matching_node_and_match(NULL, of_coherency_table, &match);
if (!np)
return COHERENCY_FABRIC_TYPE_NONE;
type = (int) match->data;
of_node_put(np);
return type;
}
int set_cpu_coherent(void)
{
int type = coherency_type();
if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP) {
if (!coherency_base) {
pr_warn("Can't make current CPU cache coherent.\n");
pr_warn("Coherency fabric is not initialized\n");
return 1;
}
ll_add_cpu_to_smp_group();
return ll_enable_coherency();
}
return 0;
}
int coherency_available(void)
{
return coherency_type() != COHERENCY_FABRIC_TYPE_NONE;
}
int __init coherency_init(void)
{
int type = coherency_type();
struct device_node *np;
np = of_find_matching_node(NULL, of_coherency_table);
if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP)
armada_370_coherency_init(np);
else if (type == COHERENCY_FABRIC_TYPE_ARMADA_375 ||
type == COHERENCY_FABRIC_TYPE_ARMADA_380)
armada_375_380_coherency_init(np);
of_node_put(np);
return 0;
}
static int __init coherency_late_init(void)
{
if (coherency_available())
bus_register_notifier(&platform_bus_type,
&mvebu_hwcc_nb);
return 0;
}
postcore_initcall(coherency_late_init);
#if IS_ENABLED(CONFIG_PCI)
static int __init coherency_pci_init(void)
{
if (coherency_available())
bus_register_notifier(&pci_bus_type,
&mvebu_hwcc_pci_nb);
return 0;
}
arch_initcall(coherency_pci_init);
#endif
|
