From 57f0f512b273f60d52568b8c6b77e17f5636edc0 Mon Sep 17 00:00:00 2001 From: André Fabian Silva Delgado Date: Wed, 5 Aug 2015 17:04:01 -0300 Subject: Initial import --- arch/metag/lib/usercopy.c | 1354 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1354 insertions(+) create mode 100644 arch/metag/lib/usercopy.c (limited to 'arch/metag/lib/usercopy.c') diff --git a/arch/metag/lib/usercopy.c b/arch/metag/lib/usercopy.c new file mode 100644 index 000000000..b3ebfe9c8 --- /dev/null +++ b/arch/metag/lib/usercopy.c @@ -0,0 +1,1354 @@ +/* + * User address space access functions. + * The non-inlined parts of asm-metag/uaccess.h are here. + * + * Copyright (C) 2006, Imagination Technologies. + * Copyright (C) 2000, Axis Communications AB. + * + * Written by Hans-Peter Nilsson. + * Pieces used from memcpy, originally by Kenny Ranerup long time ago. + * Modified for Meta by Will Newton. + */ + +#include +#include +#include /* def of L1_CACHE_BYTES */ + +#define USE_RAPF +#define RAPF_MIN_BUF_SIZE (3*L1_CACHE_BYTES) + + +/* The "double write" in this code is because the Meta will not fault + * immediately unless the memory pipe is forced to by e.g. a data stall or + * another memory op. The second write should be discarded by the write + * combiner so should have virtually no cost. + */ + +#define __asm_copy_user_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + asm volatile ( \ + COPY \ + "1:\n" \ + " .section .fixup,\"ax\"\n" \ + " MOV D1Ar1,#0\n" \ + FIXUP \ + " MOVT D1Ar1,#HI(1b)\n" \ + " JUMP D1Ar1,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + TENTRY \ + " .previous\n" \ + : "=r" (to), "=r" (from), "=r" (ret) \ + : "0" (to), "1" (from), "2" (ret) \ + : "D1Ar1", "memory") + + +#define __asm_copy_to_user_1(to, from, ret) \ + __asm_copy_user_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "2: SETB [%0++],D1Ar1\n", \ + "3: ADD %2,%2,#1\n", \ + " .long 2b,3b\n") + +#define __asm_copy_to_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_user_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + " SETW [%0],D1Ar1\n" \ + "2: SETW [%0++],D1Ar1\n" COPY, \ + "3: ADD %2,%2,#2\n" FIXUP, \ + " .long 2b,3b\n" TENTRY) + +#define __asm_copy_to_user_2(to, from, ret) \ + __asm_copy_to_user_2x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_3(to, from, ret) \ + __asm_copy_to_user_2x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "4: SETB [%0++],D1Ar1\n", \ + "5: ADD %2,%2,#1\n", \ + " .long 4b,5b\n") + +#define __asm_copy_to_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_user_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + " SETD [%0],D1Ar1\n" \ + "2: SETD [%0++],D1Ar1\n" COPY, \ + "3: ADD %2,%2,#4\n" FIXUP, \ + " .long 2b,3b\n" TENTRY) + +#define __asm_copy_to_user_4(to, from, ret) \ + __asm_copy_to_user_4x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_5(to, from, ret) \ + __asm_copy_to_user_4x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "4: SETB [%0++],D1Ar1\n", \ + "5: ADD %2,%2,#1\n", \ + " .long 4b,5b\n") + +#define __asm_copy_to_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_4x_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + " SETW [%0],D1Ar1\n" \ + "4: SETW [%0++],D1Ar1\n" COPY, \ + "5: ADD %2,%2,#2\n" FIXUP, \ + " .long 4b,5b\n" TENTRY) + +#define __asm_copy_to_user_6(to, from, ret) \ + __asm_copy_to_user_6x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_7(to, from, ret) \ + __asm_copy_to_user_6x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "6: SETB [%0++],D1Ar1\n", \ + "7: ADD %2,%2,#1\n", \ + " .long 6b,7b\n") + +#define __asm_copy_to_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_4x_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + " SETD [%0],D1Ar1\n" \ + "4: SETD [%0++],D1Ar1\n" COPY, \ + "5: ADD %2,%2,#4\n" FIXUP, \ + " .long 4b,5b\n" TENTRY) + +#define __asm_copy_to_user_8(to, from, ret) \ + __asm_copy_to_user_8x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_9(to, from, ret) \ + __asm_copy_to_user_8x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "6: SETB [%0++],D1Ar1\n", \ + "7: ADD %2,%2,#1\n", \ + " .long 6b,7b\n") + +#define __asm_copy_to_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_8x_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + " SETW [%0],D1Ar1\n" \ + "6: SETW [%0++],D1Ar1\n" COPY, \ + "7: ADD %2,%2,#2\n" FIXUP, \ + " .long 6b,7b\n" TENTRY) + +#define __asm_copy_to_user_10(to, from, ret) \ + __asm_copy_to_user_10x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_11(to, from, ret) \ + __asm_copy_to_user_10x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "8: SETB [%0++],D1Ar1\n", \ + "9: ADD %2,%2,#1\n", \ + " .long 8b,9b\n") + +#define __asm_copy_to_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_8x_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + " SETD [%0],D1Ar1\n" \ + "6: SETD [%0++],D1Ar1\n" COPY, \ + "7: ADD %2,%2,#4\n" FIXUP, \ + " .long 6b,7b\n" TENTRY) +#define __asm_copy_to_user_12(to, from, ret) \ + __asm_copy_to_user_12x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_13(to, from, ret) \ + __asm_copy_to_user_12x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "8: SETB [%0++],D1Ar1\n", \ + "9: ADD %2,%2,#1\n", \ + " .long 8b,9b\n") + +#define __asm_copy_to_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_12x_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + " SETW [%0],D1Ar1\n" \ + "8: SETW [%0++],D1Ar1\n" COPY, \ + "9: ADD %2,%2,#2\n" FIXUP, \ + " .long 8b,9b\n" TENTRY) + +#define __asm_copy_to_user_14(to, from, ret) \ + __asm_copy_to_user_14x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_15(to, from, ret) \ + __asm_copy_to_user_14x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "10: SETB [%0++],D1Ar1\n", \ + "11: ADD %2,%2,#1\n", \ + " .long 10b,11b\n") + +#define __asm_copy_to_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_12x_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + " SETD [%0],D1Ar1\n" \ + "8: SETD [%0++],D1Ar1\n" COPY, \ + "9: ADD %2,%2,#4\n" FIXUP, \ + " .long 8b,9b\n" TENTRY) + +#define __asm_copy_to_user_16(to, from, ret) \ + __asm_copy_to_user_16x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_8x64(to, from, ret) \ + asm volatile ( \ + " GETL D0Ar2,D1Ar1,[%1++]\n" \ + " SETL [%0],D0Ar2,D1Ar1\n" \ + "2: SETL [%0++],D0Ar2,D1Ar1\n" \ + "1:\n" \ + " .section .fixup,\"ax\"\n" \ + "3: ADD %2,%2,#8\n" \ + " MOVT D0Ar2,#HI(1b)\n" \ + " JUMP D0Ar2,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .long 2b,3b\n" \ + " .previous\n" \ + : "=r" (to), "=r" (from), "=r" (ret) \ + : "0" (to), "1" (from), "2" (ret) \ + : "D1Ar1", "D0Ar2", "memory") + +/* + * optimized copying loop using RAPF when 64 bit aligned + * + * n will be automatically decremented inside the loop + * ret will be left intact. if error occurs we will rewind + * so that the original non optimized code will fill up + * this value correctly. + * + * on fault: + * > n will hold total number of uncopied bytes + * + * > {'to','from'} will be rewind back so that + * the non-optimized code will do the proper fix up + * + * DCACHE drops the cacheline which helps in reducing cache + * pollution. + * + * We introduce an extra SETL at the end of the loop to + * ensure we don't fall off the loop before we catch all + * erros. + * + * NOTICE: + * LSM_STEP in TXSTATUS must be cleared in fix up code. + * since we're using M{S,G}ETL, a fault might happen at + * any address in the middle of M{S,G}ETL causing + * the value of LSM_STEP to be incorrect which can + * cause subsequent use of M{S,G}ET{L,D} to go wrong. + * ie: if LSM_STEP was 1 when a fault occurs, the + * next call to M{S,G}ET{L,D} will skip the first + * copy/getting as it think that the first 1 has already + * been done. + * + */ +#define __asm_copy_user_64bit_rapf_loop( \ + to, from, ret, n, id, FIXUP) \ + asm volatile ( \ + ".balign 8\n" \ + "MOV RAPF, %1\n" \ + "MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \ + "MOV D0Ar6, #0\n" \ + "LSR D1Ar5, %3, #6\n" \ + "SUB TXRPT, D1Ar5, #2\n" \ + "MOV RAPF, %1\n" \ + "$Lloop"id":\n" \ + "ADD RAPF, %1, #64\n" \ + "21:\n" \ + "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "22:\n" \ + "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #32\n" \ + "23:\n" \ + "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "24:\n" \ + "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #32\n" \ + "DCACHE [%1+#-64], D0Ar6\n" \ + "BR $Lloop"id"\n" \ + \ + "MOV RAPF, %1\n" \ + "25:\n" \ + "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "26:\n" \ + "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #32\n" \ + "27:\n" \ + "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "28:\n" \ + "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %0, %0, #8\n" \ + "29:\n" \ + "SETL [%0++], D0.7, D1.7\n" \ + "SUB %3, %3, #32\n" \ + "1:" \ + "DCACHE [%1+#-64], D0Ar6\n" \ + "GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \ + "GETL D0FrT, D1RtP, [A0StP+#-32]\n" \ + "GETL D0.5, D1.5, [A0StP+#-24]\n" \ + "GETL D0.6, D1.6, [A0StP+#-16]\n" \ + "GETL D0.7, D1.7, [A0StP+#-8]\n" \ + "SUB A0StP, A0StP, #40\n" \ + " .section .fixup,\"ax\"\n" \ + "4:\n" \ + " ADD %0, %0, #8\n" \ + "3:\n" \ + " MOV D0Ar2, TXSTATUS\n" \ + " MOV D1Ar1, TXSTATUS\n" \ + " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \ + " MOV TXSTATUS, D1Ar1\n" \ + FIXUP \ + " MOVT D0Ar2,#HI(1b)\n" \ + " JUMP D0Ar2,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .long 21b,3b\n" \ + " .long 22b,3b\n" \ + " .long 23b,3b\n" \ + " .long 24b,3b\n" \ + " .long 25b,3b\n" \ + " .long 26b,3b\n" \ + " .long 27b,3b\n" \ + " .long 28b,3b\n" \ + " .long 29b,4b\n" \ + " .previous\n" \ + : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \ + : "0" (to), "1" (from), "2" (ret), "3" (n) \ + : "D1Ar1", "D0Ar2", "memory") + +/* rewind 'to' and 'from' pointers when a fault occurs + * + * Rationale: + * A fault always occurs on writing to user buffer. A fault + * is at a single address, so we need to rewind by only 4 + * bytes. + * Since we do a complete read from kernel buffer before + * writing, we need to rewind it also. The amount to be + * rewind equals the number of faulty writes in MSETD + * which is: [4 - (LSM_STEP-1)]*8 + * LSM_STEP is bits 10:8 in TXSTATUS which is already read + * and stored in D0Ar2 + * + * NOTE: If a fault occurs at the last operation in M{G,S}ETL + * LSM_STEP will be 0. ie: we do 4 writes in our case, if + * a fault happens at the 4th write, LSM_STEP will be 0 + * instead of 4. The code copes with that. + * + * n is updated by the number of successful writes, which is: + * n = n - (LSM_STEP-1)*8 + */ +#define __asm_copy_to_user_64bit_rapf_loop(to, from, ret, n, id)\ + __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \ + "LSR D0Ar2, D0Ar2, #8\n" \ + "AND D0Ar2, D0Ar2, #0x7\n" \ + "ADDZ D0Ar2, D0Ar2, #4\n" \ + "SUB D0Ar2, D0Ar2, #1\n" \ + "MOV D1Ar1, #4\n" \ + "SUB D0Ar2, D1Ar1, D0Ar2\n" \ + "LSL D0Ar2, D0Ar2, #3\n" \ + "LSL D1Ar1, D1Ar1, #3\n" \ + "SUB D1Ar1, D1Ar1, D0Ar2\n" \ + "SUB %0, %0, #8\n" \ + "SUB %1, %1,D0Ar2\n" \ + "SUB %3, %3, D1Ar1\n") + +/* + * optimized copying loop using RAPF when 32 bit aligned + * + * n will be automatically decremented inside the loop + * ret will be left intact. if error occurs we will rewind + * so that the original non optimized code will fill up + * this value correctly. + * + * on fault: + * > n will hold total number of uncopied bytes + * + * > {'to','from'} will be rewind back so that + * the non-optimized code will do the proper fix up + * + * DCACHE drops the cacheline which helps in reducing cache + * pollution. + * + * We introduce an extra SETD at the end of the loop to + * ensure we don't fall off the loop before we catch all + * erros. + * + * NOTICE: + * LSM_STEP in TXSTATUS must be cleared in fix up code. + * since we're using M{S,G}ETL, a fault might happen at + * any address in the middle of M{S,G}ETL causing + * the value of LSM_STEP to be incorrect which can + * cause subsequent use of M{S,G}ET{L,D} to go wrong. + * ie: if LSM_STEP was 1 when a fault occurs, the + * next call to M{S,G}ET{L,D} will skip the first + * copy/getting as it think that the first 1 has already + * been done. + * + */ +#define __asm_copy_user_32bit_rapf_loop( \ + to, from, ret, n, id, FIXUP) \ + asm volatile ( \ + ".balign 8\n" \ + "MOV RAPF, %1\n" \ + "MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \ + "MOV D0Ar6, #0\n" \ + "LSR D1Ar5, %3, #6\n" \ + "SUB TXRPT, D1Ar5, #2\n" \ + "MOV RAPF, %1\n" \ + "$Lloop"id":\n" \ + "ADD RAPF, %1, #64\n" \ + "21:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "22:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "23:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "24:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "25:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "26:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "27:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "28:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "DCACHE [%1+#-64], D0Ar6\n" \ + "BR $Lloop"id"\n" \ + \ + "MOV RAPF, %1\n" \ + "29:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "30:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "31:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "32:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "33:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "34:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "35:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "36:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %0, %0, #4\n" \ + "37:\n" \ + "SETD [%0++], D0.7\n" \ + "SUB %3, %3, #16\n" \ + "1:" \ + "DCACHE [%1+#-64], D0Ar6\n" \ + "GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \ + "GETL D0FrT, D1RtP, [A0StP+#-32]\n" \ + "GETL D0.5, D1.5, [A0StP+#-24]\n" \ + "GETL D0.6, D1.6, [A0StP+#-16]\n" \ + "GETL D0.7, D1.7, [A0StP+#-8]\n" \ + "SUB A0StP, A0StP, #40\n" \ + " .section .fixup,\"ax\"\n" \ + "4:\n" \ + " ADD %0, %0, #4\n" \ + "3:\n" \ + " MOV D0Ar2, TXSTATUS\n" \ + " MOV D1Ar1, TXSTATUS\n" \ + " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \ + " MOV TXSTATUS, D1Ar1\n" \ + FIXUP \ + " MOVT D0Ar2,#HI(1b)\n" \ + " JUMP D0Ar2,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .long 21b,3b\n" \ + " .long 22b,3b\n" \ + " .long 23b,3b\n" \ + " .long 24b,3b\n" \ + " .long 25b,3b\n" \ + " .long 26b,3b\n" \ + " .long 27b,3b\n" \ + " .long 28b,3b\n" \ + " .long 29b,3b\n" \ + " .long 30b,3b\n" \ + " .long 31b,3b\n" \ + " .long 32b,3b\n" \ + " .long 33b,3b\n" \ + " .long 34b,3b\n" \ + " .long 35b,3b\n" \ + " .long 36b,3b\n" \ + " .long 37b,4b\n" \ + " .previous\n" \ + : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \ + : "0" (to), "1" (from), "2" (ret), "3" (n) \ + : "D1Ar1", "D0Ar2", "memory") + +/* rewind 'to' and 'from' pointers when a fault occurs + * + * Rationale: + * A fault always occurs on writing to user buffer. A fault + * is at a single address, so we need to rewind by only 4 + * bytes. + * Since we do a complete read from kernel buffer before + * writing, we need to rewind it also. The amount to be + * rewind equals the number of faulty writes in MSETD + * which is: [4 - (LSM_STEP-1)]*4 + * LSM_STEP is bits 10:8 in TXSTATUS which is already read + * and stored in D0Ar2 + * + * NOTE: If a fault occurs at the last operation in M{G,S}ETL + * LSM_STEP will be 0. ie: we do 4 writes in our case, if + * a fault happens at the 4th write, LSM_STEP will be 0 + * instead of 4. The code copes with that. + * + * n is updated by the number of successful writes, which is: + * n = n - (LSM_STEP-1)*4 + */ +#define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\ + __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \ + "LSR D0Ar2, D0Ar2, #8\n" \ + "AND D0Ar2, D0Ar2, #0x7\n" \ + "ADDZ D0Ar2, D0Ar2, #4\n" \ + "SUB D0Ar2, D0Ar2, #1\n" \ + "MOV D1Ar1, #4\n" \ + "SUB D0Ar2, D1Ar1, D0Ar2\n" \ + "LSL D0Ar2, D0Ar2, #2\n" \ + "LSL D1Ar1, D1Ar1, #2\n" \ + "SUB D1Ar1, D1Ar1, D0Ar2\n" \ + "SUB %0, %0, #4\n" \ + "SUB %1, %1, D0Ar2\n" \ + "SUB %3, %3, D1Ar1\n") + +unsigned long __copy_user(void __user *pdst, const void *psrc, + unsigned long n) +{ + register char __user *dst asm ("A0.2") = pdst; + register const char *src asm ("A1.2") = psrc; + unsigned long retn = 0; + + if (n == 0) + return 0; + + if ((unsigned long) src & 1) { + __asm_copy_to_user_1(dst, src, retn); + n--; + } + if ((unsigned long) dst & 1) { + /* Worst case - byte copy */ + while (n > 0) { + __asm_copy_to_user_1(dst, src, retn); + n--; + } + } + if (((unsigned long) src & 2) && n >= 2) { + __asm_copy_to_user_2(dst, src, retn); + n -= 2; + } + if ((unsigned long) dst & 2) { + /* Second worst case - word copy */ + while (n >= 2) { + __asm_copy_to_user_2(dst, src, retn); + n -= 2; + } + } + +#ifdef USE_RAPF + /* 64 bit copy loop */ + if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) { + if (n >= RAPF_MIN_BUF_SIZE) { + /* copy user using 64 bit rapf copy */ + __asm_copy_to_user_64bit_rapf_loop(dst, src, retn, + n, "64cu"); + } + while (n >= 8) { + __asm_copy_to_user_8x64(dst, src, retn); + n -= 8; + } + } + if (n >= RAPF_MIN_BUF_SIZE) { + /* copy user using 32 bit rapf copy */ + __asm_copy_to_user_32bit_rapf_loop(dst, src, retn, n, "32cu"); + } +#else + /* 64 bit copy loop */ + if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) { + while (n >= 8) { + __asm_copy_to_user_8x64(dst, src, retn); + n -= 8; + } + } +#endif + + while (n >= 16) { + __asm_copy_to_user_16(dst, src, retn); + n -= 16; + } + + while (n >= 4) { + __asm_copy_to_user_4(dst, src, retn); + n -= 4; + } + + switch (n) { + case 0: + break; + case 1: + __asm_copy_to_user_1(dst, src, retn); + break; + case 2: + __asm_copy_to_user_2(dst, src, retn); + break; + case 3: + __asm_copy_to_user_3(dst, src, retn); + break; + } + + return retn; +} +EXPORT_SYMBOL(__copy_user); + +#define __asm_copy_from_user_1(to, from, ret) \ + __asm_copy_user_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "2: SETB [%0++],D1Ar1\n", \ + "3: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 2b,3b\n") + +#define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_user_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + "2: SETW [%0++],D1Ar1\n" COPY, \ + "3: ADD %2,%2,#2\n" \ + " SETW [%0++],D1Ar1\n" FIXUP, \ + " .long 2b,3b\n" TENTRY) + +#define __asm_copy_from_user_2(to, from, ret) \ + __asm_copy_from_user_2x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_3(to, from, ret) \ + __asm_copy_from_user_2x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "4: SETB [%0++],D1Ar1\n", \ + "5: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 4b,5b\n") + +#define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_user_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + "2: SETD [%0++],D1Ar1\n" COPY, \ + "3: ADD %2,%2,#4\n" \ + " SETD [%0++],D1Ar1\n" FIXUP, \ + " .long 2b,3b\n" TENTRY) + +#define __asm_copy_from_user_4(to, from, ret) \ + __asm_copy_from_user_4x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_5(to, from, ret) \ + __asm_copy_from_user_4x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "4: SETB [%0++],D1Ar1\n", \ + "5: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 4b,5b\n") + +#define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_from_user_4x_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + "4: SETW [%0++],D1Ar1\n" COPY, \ + "5: ADD %2,%2,#2\n" \ + " SETW [%0++],D1Ar1\n" FIXUP, \ + " .long 4b,5b\n" TENTRY) + +#define __asm_copy_from_user_6(to, from, ret) \ + __asm_copy_from_user_6x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_7(to, from, ret) \ + __asm_copy_from_user_6x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "6: SETB [%0++],D1Ar1\n", \ + "7: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 6b,7b\n") + +#define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_from_user_4x_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + "4: SETD [%0++],D1Ar1\n" COPY, \ + "5: ADD %2,%2,#4\n" \ + " SETD [%0++],D1Ar1\n" FIXUP, \ + " .long 4b,5b\n" TENTRY) + +#define __asm_copy_from_user_8(to, from, ret) \ + __asm_copy_from_user_8x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_9(to, from, ret) \ + __asm_copy_from_user_8x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "6: SETB [%0++],D1Ar1\n", \ + "7: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 6b,7b\n") + +#define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_from_user_8x_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + "6: SETW [%0++],D1Ar1\n" COPY, \ + "7: ADD %2,%2,#2\n" \ + " SETW [%0++],D1Ar1\n" FIXUP, \ + " .long 6b,7b\n" TENTRY) + +#define __asm_copy_from_user_10(to, from, ret) \ + __asm_copy_from_user_10x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_11(to, from, ret) \ + __asm_copy_from_user_10x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "8: SETB [%0++],D1Ar1\n", \ + "9: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 8b,9b\n") + +#define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_from_user_8x_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + "6: SETD [%0++],D1Ar1\n" COPY, \ + "7: ADD %2,%2,#4\n" \ + " SETD [%0++],D1Ar1\n" FIXUP, \ + " .long 6b,7b\n" TENTRY) + +#define __asm_copy_from_user_12(to, from, ret) \ + __asm_copy_from_user_12x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_13(to, from, ret) \ + __asm_copy_from_user_12x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "8: SETB [%0++],D1Ar1\n", \ + "9: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 8b,9b\n") + +#define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_from_user_12x_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + "8: SETW [%0++],D1Ar1\n" COPY, \ + "9: ADD %2,%2,#2\n" \ + " SETW [%0++],D1Ar1\n" FIXUP, \ + " .long 8b,9b\n" TENTRY) + +#define __asm_copy_from_user_14(to, from, ret) \ + __asm_copy_from_user_14x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_15(to, from, ret) \ + __asm_copy_from_user_14x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "10: SETB [%0++],D1Ar1\n", \ + "11: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 10b,11b\n") + +#define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_from_user_12x_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + "8: SETD [%0++],D1Ar1\n" COPY, \ + "9: ADD %2,%2,#4\n" \ + " SETD [%0++],D1Ar1\n" FIXUP, \ + " .long 8b,9b\n" TENTRY) + +#define __asm_copy_from_user_16(to, from, ret) \ + __asm_copy_from_user_16x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_8x64(to, from, ret) \ + asm volatile ( \ + " GETL D0Ar2,D1Ar1,[%1++]\n" \ + "2: SETL [%0++],D0Ar2,D1Ar1\n" \ + "1:\n" \ + " .section .fixup,\"ax\"\n" \ + " MOV D1Ar1,#0\n" \ + " MOV D0Ar2,#0\n" \ + "3: ADD %2,%2,#8\n" \ + " SETL [%0++],D0Ar2,D1Ar1\n" \ + " MOVT D0Ar2,#HI(1b)\n" \ + " JUMP D0Ar2,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .long 2b,3b\n" \ + " .previous\n" \ + : "=a" (to), "=r" (from), "=r" (ret) \ + : "0" (to), "1" (from), "2" (ret) \ + : "D1Ar1", "D0Ar2", "memory") + +/* rewind 'from' pointer when a fault occurs + * + * Rationale: + * A fault occurs while reading from user buffer, which is the + * source. Since the fault is at a single address, we only + * need to rewind by 8 bytes. + * Since we don't write to kernel buffer until we read first, + * the kernel buffer is at the right state and needn't be + * corrected. + */ +#define __asm_copy_from_user_64bit_rapf_loop(to, from, ret, n, id) \ + __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \ + "SUB %1, %1, #8\n") + +/* rewind 'from' pointer when a fault occurs + * + * Rationale: + * A fault occurs while reading from user buffer, which is the + * source. Since the fault is at a single address, we only + * need to rewind by 4 bytes. + * Since we don't write to kernel buffer until we read first, + * the kernel buffer is at the right state and needn't be + * corrected. + */ +#define __asm_copy_from_user_32bit_rapf_loop(to, from, ret, n, id) \ + __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \ + "SUB %1, %1, #4\n") + + +/* Copy from user to kernel, zeroing the bytes that were inaccessible in + userland. The return-value is the number of bytes that were + inaccessible. */ +unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, + unsigned long n) +{ + register char *dst asm ("A0.2") = pdst; + register const char __user *src asm ("A1.2") = psrc; + unsigned long retn = 0; + + if (n == 0) + return 0; + + if ((unsigned long) src & 1) { + __asm_copy_from_user_1(dst, src, retn); + n--; + } + if ((unsigned long) dst & 1) { + /* Worst case - byte copy */ + while (n > 0) { + __asm_copy_from_user_1(dst, src, retn); + n--; + if (retn) + goto copy_exception_bytes; + } + } + if (((unsigned long) src & 2) && n >= 2) { + __asm_copy_from_user_2(dst, src, retn); + n -= 2; + } + if ((unsigned long) dst & 2) { + /* Second worst case - word copy */ + while (n >= 2) { + __asm_copy_from_user_2(dst, src, retn); + n -= 2; + if (retn) + goto copy_exception_bytes; + } + } + + /* We only need one check after the unalignment-adjustments, + because if both adjustments were done, either both or + neither reference had an exception. */ + if (retn != 0) + goto copy_exception_bytes; + +#ifdef USE_RAPF + /* 64 bit copy loop */ + if (!(((unsigned long) src | (unsigned long) dst) & 7)) { + if (n >= RAPF_MIN_BUF_SIZE) { + /* Copy using fast 64bit rapf */ + __asm_copy_from_user_64bit_rapf_loop(dst, src, retn, + n, "64cuz"); + } + while (n >= 8) { + __asm_copy_from_user_8x64(dst, src, retn); + n -= 8; + if (retn) + goto copy_exception_bytes; + } + } + + if (n >= RAPF_MIN_BUF_SIZE) { + /* Copy using fast 32bit rapf */ + __asm_copy_from_user_32bit_rapf_loop(dst, src, retn, + n, "32cuz"); + } +#else + /* 64 bit copy loop */ + if (!(((unsigned long) src | (unsigned long) dst) & 7)) { + while (n >= 8) { + __asm_copy_from_user_8x64(dst, src, retn); + n -= 8; + if (retn) + goto copy_exception_bytes; + } + } +#endif + + while (n >= 4) { + __asm_copy_from_user_4(dst, src, retn); + n -= 4; + + if (retn) + goto copy_exception_bytes; + } + + /* If we get here, there were no memory read faults. */ + switch (n) { + /* These copies are at least "naturally aligned" (so we don't + have to check each byte), due to the src alignment code. + The *_3 case *will* get the correct count for retn. */ + case 0: + /* This case deliberately left in (if you have doubts check the + generated assembly code). */ + break; + case 1: + __asm_copy_from_user_1(dst, src, retn); + break; + case 2: + __asm_copy_from_user_2(dst, src, retn); + break; + case 3: + __asm_copy_from_user_3(dst, src, retn); + break; + } + + /* If we get here, retn correctly reflects the number of failing + bytes. */ + return retn; + + copy_exception_bytes: + /* We already have "retn" bytes cleared, and need to clear the + remaining "n" bytes. A non-optimized simple byte-for-byte in-line + memset is preferred here, since this isn't speed-critical code and + we'd rather have this a leaf-function than calling memset. */ + { + char *endp; + for (endp = dst + n; dst < endp; dst++) + *dst = 0; + } + + return retn + n; +} +EXPORT_SYMBOL(__copy_user_zeroing); + +#define __asm_clear_8x64(to, ret) \ + asm volatile ( \ + " MOV D0Ar2,#0\n" \ + " MOV D1Ar1,#0\n" \ + " SETL [%0],D0Ar2,D1Ar1\n" \ + "2: SETL [%0++],D0Ar2,D1Ar1\n" \ + "1:\n" \ + " .section .fixup,\"ax\"\n" \ + "3: ADD %1,%1,#8\n" \ + " MOVT D0Ar2,#HI(1b)\n" \ + " JUMP D0Ar2,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .long 2b,3b\n" \ + " .previous\n" \ + : "=r" (to), "=r" (ret) \ + : "0" (to), "1" (ret) \ + : "D1Ar1", "D0Ar2", "memory") + +/* Zero userspace. */ + +#define __asm_clear(to, ret, CLEAR, FIXUP, TENTRY) \ + asm volatile ( \ + " MOV D1Ar1,#0\n" \ + CLEAR \ + "1:\n" \ + " .section .fixup,\"ax\"\n" \ + FIXUP \ + " MOVT D1Ar1,#HI(1b)\n" \ + " JUMP D1Ar1,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + TENTRY \ + " .previous" \ + : "=r" (to), "=r" (ret) \ + : "0" (to), "1" (ret) \ + : "D1Ar1", "memory") + +#define __asm_clear_1(to, ret) \ + __asm_clear(to, ret, \ + " SETB [%0],D1Ar1\n" \ + "2: SETB [%0++],D1Ar1\n", \ + "3: ADD %1,%1,#1\n", \ + " .long 2b,3b\n") + +#define __asm_clear_2(to, ret) \ + __asm_clear(to, ret, \ + " SETW [%0],D1Ar1\n" \ + "2: SETW [%0++],D1Ar1\n", \ + "3: ADD %1,%1,#2\n", \ + " .long 2b,3b\n") + +#define __asm_clear_3(to, ret) \ + __asm_clear(to, ret, \ + "2: SETW [%0++],D1Ar1\n" \ + " SETB [%0],D1Ar1\n" \ + "3: SETB [%0++],D1Ar1\n", \ + "4: ADD %1,%1,#2\n" \ + "5: ADD %1,%1,#1\n", \ + " .long 2b,4b\n" \ + " .long 3b,5b\n") + +#define __asm_clear_4x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ + __asm_clear(to, ret, \ + " SETD [%0],D1Ar1\n" \ + "2: SETD [%0++],D1Ar1\n" CLEAR, \ + "3: ADD %1,%1,#4\n" FIXUP, \ + " .long 2b,3b\n" TENTRY) + +#define __asm_clear_4(to, ret) \ + __asm_clear_4x_cont(to, ret, "", "", "") + +#define __asm_clear_8x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ + __asm_clear_4x_cont(to, ret, \ + " SETD [%0],D1Ar1\n" \ + "4: SETD [%0++],D1Ar1\n" CLEAR, \ + "5: ADD %1,%1,#4\n" FIXUP, \ + " .long 4b,5b\n" TENTRY) + +#define __asm_clear_8(to, ret) \ + __asm_clear_8x_cont(to, ret, "", "", "") + +#define __asm_clear_12x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ + __asm_clear_8x_cont(to, ret, \ + " SETD [%0],D1Ar1\n" \ + "6: SETD [%0++],D1Ar1\n" CLEAR, \ + "7: ADD %1,%1,#4\n" FIXUP, \ + " .long 6b,7b\n" TENTRY) + +#define __asm_clear_12(to, ret) \ + __asm_clear_12x_cont(to, ret, "", "", "") + +#define __asm_clear_16x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ + __asm_clear_12x_cont(to, ret, \ + " SETD [%0],D1Ar1\n" \ + "8: SETD [%0++],D1Ar1\n" CLEAR, \ + "9: ADD %1,%1,#4\n" FIXUP, \ + " .long 8b,9b\n" TENTRY) + +#define __asm_clear_16(to, ret) \ + __asm_clear_16x_cont(to, ret, "", "", "") + +unsigned long __do_clear_user(void __user *pto, unsigned long pn) +{ + register char __user *dst asm ("D0Re0") = pto; + register unsigned long n asm ("D1Re0") = pn; + register unsigned long retn asm ("D0Ar6") = 0; + + if ((unsigned long) dst & 1) { + __asm_clear_1(dst, retn); + n--; + } + + if ((unsigned long) dst & 2) { + __asm_clear_2(dst, retn); + n -= 2; + } + + /* 64 bit copy loop */ + if (!((__force unsigned long) dst & 7)) { + while (n >= 8) { + __asm_clear_8x64(dst, retn); + n -= 8; + } + } + + while (n >= 16) { + __asm_clear_16(dst, retn); + n -= 16; + } + + while (n >= 4) { + __asm_clear_4(dst, retn); + n -= 4; + } + + switch (n) { + case 0: + break; + case 1: + __asm_clear_1(dst, retn); + break; + case 2: + __asm_clear_2(dst, retn); + break; + case 3: + __asm_clear_3(dst, retn); + break; + } + + return retn; +} +EXPORT_SYMBOL(__do_clear_user); + +unsigned char __get_user_asm_b(const void __user *addr, long *err) +{ + register unsigned char x asm ("D0Re0") = 0; + asm volatile ( + " GETB %0,[%2]\n" + "1:\n" + " GETB %0,[%2]\n" + "2:\n" + " .section .fixup,\"ax\"\n" + "3: MOV D0FrT,%3\n" + " SETD [%1],D0FrT\n" + " MOVT D0FrT,#HI(2b)\n" + " JUMP D0FrT,#LO(2b)\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + " .long 1b,3b\n" + " .previous\n" + : "=r" (x) + : "r" (err), "r" (addr), "P" (-EFAULT) + : "D0FrT"); + return x; +} +EXPORT_SYMBOL(__get_user_asm_b); + +unsigned short __get_user_asm_w(const void __user *addr, long *err) +{ + register unsigned short x asm ("D0Re0") = 0; + asm volatile ( + " GETW %0,[%2]\n" + "1:\n" + " GETW %0,[%2]\n" + "2:\n" + " .section .fixup,\"ax\"\n" + "3: MOV D0FrT,%3\n" + " SETD [%1],D0FrT\n" + " MOVT D0FrT,#HI(2b)\n" + " JUMP D0FrT,#LO(2b)\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + " .long 1b,3b\n" + " .previous\n" + : "=r" (x) + : "r" (err), "r" (addr), "P" (-EFAULT) + : "D0FrT"); + return x; +} +EXPORT_SYMBOL(__get_user_asm_w); + +unsigned int __get_user_asm_d(const void __user *addr, long *err) +{ + register unsigned int x asm ("D0Re0") = 0; + asm volatile ( + " GETD %0,[%2]\n" + "1:\n" + " GETD %0,[%2]\n" + "2:\n" + " .section .fixup,\"ax\"\n" + "3: MOV D0FrT,%3\n" + " SETD [%1],D0FrT\n" + " MOVT D0FrT,#HI(2b)\n" + " JUMP D0FrT,#LO(2b)\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + " .long 1b,3b\n" + " .previous\n" + : "=r" (x) + : "r" (err), "r" (addr), "P" (-EFAULT) + : "D0FrT"); + return x; +} +EXPORT_SYMBOL(__get_user_asm_d); + +long __put_user_asm_b(unsigned int x, void __user *addr) +{ + register unsigned int err asm ("D0Re0") = 0; + asm volatile ( + " MOV %0,#0\n" + " SETB [%2],%1\n" + "1:\n" + " SETB [%2],%1\n" + "2:\n" + ".section .fixup,\"ax\"\n" + "3: MOV %0,%3\n" + " MOVT D0FrT,#HI(2b)\n" + " JUMP D0FrT,#LO(2b)\n" + ".previous\n" + ".section __ex_table,\"a\"\n" + " .long 1b,3b\n" + ".previous" + : "=r"(err) + : "d" (x), "a" (addr), "P"(-EFAULT) + : "D0FrT"); + return err; +} +EXPORT_SYMBOL(__put_user_asm_b); + +long __put_user_asm_w(unsigned int x, void __user *addr) +{ + register unsigned int err asm ("D0Re0") = 0; + asm volatile ( + " MOV %0,#0\n" + " SETW [%2],%1\n" + "1:\n" + " SETW [%2],%1\n" + "2:\n" + ".section .fixup,\"ax\"\n" + "3: MOV %0,%3\n" + " MOVT D0FrT,#HI(2b)\n" + " JUMP D0FrT,#LO(2b)\n" + ".previous\n" + ".section __ex_table,\"a\"\n" + " .long 1b,3b\n" + ".previous" + : "=r"(err) + : "d" (x), "a" (addr), "P"(-EFAULT) + : "D0FrT"); + return err; +} +EXPORT_SYMBOL(__put_user_asm_w); + +long __put_user_asm_d(unsigned int x, void __user *addr) +{ + register unsigned int err asm ("D0Re0") = 0; + asm volatile ( + " MOV %0,#0\n" + " SETD [%2],%1\n" + "1:\n" + " SETD [%2],%1\n" + "2:\n" + ".section .fixup,\"ax\"\n" + "3: MOV %0,%3\n" + " MOVT D0FrT,#HI(2b)\n" + " JUMP D0FrT,#LO(2b)\n" + ".previous\n" + ".section __ex_table,\"a\"\n" + " .long 1b,3b\n" + ".previous" + : "=r"(err) + : "d" (x), "a" (addr), "P"(-EFAULT) + : "D0FrT"); + return err; +} +EXPORT_SYMBOL(__put_user_asm_d); + +long __put_user_asm_l(unsigned long long x, void __user *addr) +{ + register unsigned int err asm ("D0Re0") = 0; + asm volatile ( + " MOV %0,#0\n" + " SETL [%2],%1,%t1\n" + "1:\n" + " SETL [%2],%1,%t1\n" + "2:\n" + ".section .fixup,\"ax\"\n" + "3: MOV %0,%3\n" + " MOVT D0FrT,#HI(2b)\n" + " JUMP D0FrT,#LO(2b)\n" + ".previous\n" + ".section __ex_table,\"a\"\n" + " .long 1b,3b\n" + ".previous" + : "=r"(err) + : "d" (x), "a" (addr), "P"(-EFAULT) + : "D0FrT"); + return err; +} +EXPORT_SYMBOL(__put_user_asm_l); + +long strnlen_user(const char __user *src, long count) +{ + long res; + + if (!access_ok(VERIFY_READ, src, 0)) + return 0; + + asm volatile (" MOV D0Ar4, %1\n" + " MOV D0Ar6, %2\n" + "0:\n" + " SUBS D0FrT, D0Ar6, #0\n" + " SUB D0Ar6, D0Ar6, #1\n" + " BLE 2f\n" + " GETB D0FrT, [D0Ar4+#1++]\n" + "1:\n" + " TST D0FrT, #255\n" + " BNE 0b\n" + "2:\n" + " SUB %0, %2, D0Ar6\n" + "3:\n" + " .section .fixup,\"ax\"\n" + "4:\n" + " MOV %0, #0\n" + " MOVT D0FrT,#HI(3b)\n" + " JUMP D0FrT,#LO(3b)\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + " .long 1b,4b\n" + " .previous\n" + : "=r" (res) + : "r" (src), "r" (count) + : "D0FrT", "D0Ar4", "D0Ar6", "cc"); + + return res; +} +EXPORT_SYMBOL(strnlen_user); + +long __strncpy_from_user(char *dst, const char __user *src, long count) +{ + long res; + + if (count == 0) + return 0; + + /* + * Currently, in 2.4.0-test9, most ports use a simple byte-copy loop. + * So do we. + * + * This code is deduced from: + * + * char tmp2; + * long tmp1, tmp3; + * tmp1 = count; + * while ((*dst++ = (tmp2 = *src++)) != 0 + * && --tmp1) + * ; + * + * res = count - tmp1; + * + * with tweaks. + */ + + asm volatile (" MOV %0,%3\n" + "1:\n" + " GETB D0FrT,[%2++]\n" + "2:\n" + " CMP D0FrT,#0\n" + " SETB [%1++],D0FrT\n" + " BEQ 3f\n" + " SUBS %0,%0,#1\n" + " BNZ 1b\n" + "3:\n" + " SUB %0,%3,%0\n" + "4:\n" + " .section .fixup,\"ax\"\n" + "5:\n" + " MOV %0,%7\n" + " MOVT D0FrT,#HI(4b)\n" + " JUMP D0FrT,#LO(4b)\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + " .long 2b,5b\n" + " .previous" + : "=r" (res), "=r" (dst), "=r" (src), "=r" (count) + : "3" (count), "1" (dst), "2" (src), "P" (-EFAULT) + : "D0FrT", "memory", "cc"); + + return res; +} +EXPORT_SYMBOL(__strncpy_from_user); -- cgit v1.2.3-54-g00ecf