1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
|
/*
* Userspace implementations of gettimeofday() and friends.
*
* Copyright (C) 2012 ARM Limited
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* Author: Will Deacon <will.deacon@arm.com>
*/
#include <linux/linkage.h>
#include <asm/asm-offsets.h>
#include <asm/unistd.h>
#define NSEC_PER_SEC_LO16 0xca00
#define NSEC_PER_SEC_HI16 0x3b9a
vdso_data .req x6
use_syscall .req w7
seqcnt .req w8
.macro seqcnt_acquire
9999: ldr seqcnt, [vdso_data, #VDSO_TB_SEQ_COUNT]
tbnz seqcnt, #0, 9999b
dmb ishld
ldr use_syscall, [vdso_data, #VDSO_USE_SYSCALL]
.endm
.macro seqcnt_read, cnt
dmb ishld
ldr \cnt, [vdso_data, #VDSO_TB_SEQ_COUNT]
.endm
.macro seqcnt_check, cnt, fail
cmp \cnt, seqcnt
b.ne \fail
.endm
.text
/* int __kernel_gettimeofday(struct timeval *tv, struct timezone *tz); */
ENTRY(__kernel_gettimeofday)
.cfi_startproc
mov x2, x30
.cfi_register x30, x2
/* Acquire the sequence counter and get the timespec. */
adr vdso_data, _vdso_data
1: seqcnt_acquire
cbnz use_syscall, 4f
/* If tv is NULL, skip to the timezone code. */
cbz x0, 2f
bl __do_get_tspec
seqcnt_check w9, 1b
/* Convert ns to us. */
mov x13, #1000
lsl x13, x13, x12
udiv x11, x11, x13
stp x10, x11, [x0, #TVAL_TV_SEC]
2:
/* If tz is NULL, return 0. */
cbz x1, 3f
ldp w4, w5, [vdso_data, #VDSO_TZ_MINWEST]
stp w4, w5, [x1, #TZ_MINWEST]
3:
mov x0, xzr
ret x2
4:
/* Syscall fallback. */
mov x8, #__NR_gettimeofday
svc #0
ret x2
.cfi_endproc
ENDPROC(__kernel_gettimeofday)
/* int __kernel_clock_gettime(clockid_t clock_id, struct timespec *tp); */
ENTRY(__kernel_clock_gettime)
.cfi_startproc
cmp w0, #CLOCK_REALTIME
ccmp w0, #CLOCK_MONOTONIC, #0x4, ne
b.ne 2f
mov x2, x30
.cfi_register x30, x2
/* Get kernel timespec. */
adr vdso_data, _vdso_data
1: seqcnt_acquire
cbnz use_syscall, 7f
bl __do_get_tspec
seqcnt_check w9, 1b
mov x30, x2
cmp w0, #CLOCK_MONOTONIC
b.ne 6f
/* Get wtm timespec. */
ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC]
/* Check the sequence counter. */
seqcnt_read w9
seqcnt_check w9, 1b
b 4f
2:
cmp w0, #CLOCK_REALTIME_COARSE
ccmp w0, #CLOCK_MONOTONIC_COARSE, #0x4, ne
b.ne 8f
/* xtime_coarse_nsec is already right-shifted */
mov x12, #0
/* Get coarse timespec. */
adr vdso_data, _vdso_data
3: seqcnt_acquire
ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC]
/* Get wtm timespec. */
ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC]
/* Check the sequence counter. */
seqcnt_read w9
seqcnt_check w9, 3b
cmp w0, #CLOCK_MONOTONIC_COARSE
b.ne 6f
4:
/* Add on wtm timespec. */
add x10, x10, x13
lsl x14, x14, x12
add x11, x11, x14
/* Normalise the new timespec. */
mov x15, #NSEC_PER_SEC_LO16
movk x15, #NSEC_PER_SEC_HI16, lsl #16
lsl x15, x15, x12
cmp x11, x15
b.lt 5f
sub x11, x11, x15
add x10, x10, #1
5:
cmp x11, #0
b.ge 6f
add x11, x11, x15
sub x10, x10, #1
6: /* Store to the user timespec. */
lsr x11, x11, x12
stp x10, x11, [x1, #TSPEC_TV_SEC]
mov x0, xzr
ret
7:
mov x30, x2
8: /* Syscall fallback. */
mov x8, #__NR_clock_gettime
svc #0
ret
.cfi_endproc
ENDPROC(__kernel_clock_gettime)
/* int __kernel_clock_getres(clockid_t clock_id, struct timespec *res); */
ENTRY(__kernel_clock_getres)
.cfi_startproc
cmp w0, #CLOCK_REALTIME
ccmp w0, #CLOCK_MONOTONIC, #0x4, ne
b.ne 1f
ldr x2, 5f
b 2f
1:
cmp w0, #CLOCK_REALTIME_COARSE
ccmp w0, #CLOCK_MONOTONIC_COARSE, #0x4, ne
b.ne 4f
ldr x2, 6f
2:
cbz w1, 3f
stp xzr, x2, [x1]
3: /* res == NULL. */
mov w0, wzr
ret
4: /* Syscall fallback. */
mov x8, #__NR_clock_getres
svc #0
ret
5:
.quad CLOCK_REALTIME_RES
6:
.quad CLOCK_COARSE_RES
.cfi_endproc
ENDPROC(__kernel_clock_getres)
/*
* Read the current time from the architected counter.
* Expects vdso_data to be initialised.
* Clobbers the temporary registers (x9 - x15).
* Returns:
* - w9 = vDSO sequence counter
* - (x10, x11) = (ts->tv_sec, shifted ts->tv_nsec)
* - w12 = cs_shift
*/
ENTRY(__do_get_tspec)
.cfi_startproc
/* Read from the vDSO data page. */
ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST]
ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC]
ldp w11, w12, [vdso_data, #VDSO_CS_MULT]
seqcnt_read w9
/* Read the virtual counter. */
isb
mrs x15, cntvct_el0
/* Calculate cycle delta and convert to ns. */
sub x10, x15, x10
/* We can only guarantee 56 bits of precision. */
movn x15, #0xff00, lsl #48
and x10, x15, x10
mul x10, x10, x11
/* Use the kernel time to calculate the new timespec. */
mov x11, #NSEC_PER_SEC_LO16
movk x11, #NSEC_PER_SEC_HI16, lsl #16
lsl x11, x11, x12
add x15, x10, x14
udiv x14, x15, x11
add x10, x13, x14
mul x13, x14, x11
sub x11, x15, x13
ret
.cfi_endproc
ENDPROC(__do_get_tspec)
|
