arm64: Only use the inline asm for get_user/put_user

Previously the code was using a branch to the corresponding functions
defined in .S files. From a performance perspective, inlined
get_user/put_user are better but with a few KB increase in the resulting
Image size.

The patch also reorganises the uaccess.h file so to look consistent.

Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Reported-by: Arnd Bergmann <arnd@arndb.de>

5 files changed, 91 insertions, 322 deletions

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 09d7b5361f3..328ac944957 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -52,13 +52,6 @@ struct exception_table_entry
 
 extern int fixup_exception(struct pt_regs *regs);
 
-/*
- * These two are intentionally not defined anywhere - if the kernel
- * code generates any references to them, that's a bug.
- */
-extern long __get_user_bad(void);
-extern long __put_user_bad(void);
-
 #define KERNEL_DS	(-1UL)
 #define get_ds()	(KERNEL_DS)
 
@@ -78,7 +71,7 @@ static inline void set_fs(mm_segment_t fs)
 #define __addr_ok(addr)							\
 ({									\
 	unsigned long flag;						\
-	asm("cmp %1, %0; cset %0, lo"				\
+	asm("cmp %1, %0; cset %0, lo"					\
 		: "=&r" (flag)						\
 		: "r" (addr), "0" (current_thread_info()->addr_limit)	\
 		: "cc");						\
@@ -94,11 +87,11 @@ static inline void set_fs(mm_segment_t fs)
  *
  * This needs 65-bit arithmetic.
  */
-#define __range_ok(addr,size)						\
+#define __range_ok(addr, size)						\
 ({									\
 	unsigned long flag, roksum;					\
 	__chk_user_ptr(addr);						\
-	asm("adds %1, %1, %3; ccmp %1, %4, #2, cc; cset %0, cc"	\
+	asm("adds %1, %1, %3; ccmp %1, %4, #2, cc; cset %0, cc"		\
 		: "=&r" (flag), "=&r" (roksum)				\
 		: "1" (addr), "Ir" (size),				\
 		  "r" (current_thread_info()->addr_limit)		\
@@ -106,174 +99,99 @@ static inline void set_fs(mm_segment_t fs)
 	flag;								\
 })
 
-/*
- * Single-value transfer routines.  They automatically use the right
- * size if we just have the right pointer type.  Note that the functions
- * which read from user space (*get_*) need to take care not to leak
- * kernel data even if the calling code is buggy and fails to check
- * the return value.  This means zeroing out the destination variable
- * or buffer on error.  Normally this is done out of line by the
- * fixup code, but there are a few places where it intrudes on the
- * main code path.  When we only write to user space, there is no
- * problem.
- */
-extern long __get_user_1(void *);
-extern long __get_user_2(void *);
-extern long __get_user_4(void *);
-extern long __get_user_8(void *);
-
-#define __get_user_x(__r2,__p,__e,__s,__i...)				\
-	   asm volatile(						\
-		__asmeq("%0", "x0") __asmeq("%1", "x2")			\
-		"bl	__get_user_" #__s				\
-		: "=&r" (__e), "=r" (__r2)				\
-		: "0" (__p)						\
-		: __i, "cc")
-
-#define get_user(x,p)							\
-	({								\
-		register const typeof(*(p)) __user *__p asm("x0") = (p);\
-		register unsigned long __r2 asm("x2");			\
-		register long __e asm("x0");				\
-		switch (sizeof(*(__p))) {				\
-		case 1:							\
-			__get_user_x(__r2, __p, __e, 1, "x30");		\
-			break;						\
-		case 2:							\
-			__get_user_x(__r2, __p, __e, 2, "x3", "x30");	\
-			break;						\
-		case 4:							\
-			__get_user_x(__r2, __p, __e, 4, "x30");		\
-			break;						\
-		case 8:							\
-			__get_user_x(__r2, __p, __e, 8, "x30");		\
-			break;						\
-		default: __e = __get_user_bad(); break;			\
-		}							\
-		x = (typeof(*(p))) __r2;				\
-		__e;							\
-	})
-
-#define __get_user_unaligned __get_user
-
-extern long __put_user_1(void *, unsigned long);
-extern long __put_user_2(void *, unsigned long);
-extern long __put_user_4(void *, unsigned long);
-extern long __put_user_8(void *, unsigned long);
-
-#define __put_user_x(__r2,__p,__e,__s)					\
-	   asm volatile(						\
-		__asmeq("%0", "x0") __asmeq("%2", "x2")			\
-		"bl	__put_user_" #__s				\
-		: "=&r" (__e)						\
-		: "0" (__p), "r" (__r2)					\
-		: "x8", "x30", "cc")
-
-#define put_user(x,p)							\
-	({								\
-		register const typeof(*(p)) __r2 asm("x2") = (x);	\
-		register const typeof(*(p)) __user *__p asm("x0") = (p);\
-		register long __e asm("x0");				\
-		switch (sizeof(*(__p))) {				\
-		case 1:							\
-			__put_user_x(__r2, __p, __e, 1);		\
-			break;						\
-		case 2:							\
-			__put_user_x(__r2, __p, __e, 2);		\
-			break;						\
-		case 4:							\
-			__put_user_x(__r2, __p, __e, 4);		\
-			break;						\
-		case 8:							\
-			__put_user_x(__r2, __p, __e, 8);		\
-			break;						\
-		default: __e = __put_user_bad(); break;			\
-		}							\
-		__e;							\
-	})
-
-#define __put_user_unaligned __put_user
-
-#define access_ok(type,addr,size)	__range_ok(addr,size)
+#define access_ok(type, addr, size)	__range_ok(addr, size)
 
 /*
- * The "__xxx" versions of the user access functions do not verify the
- * address space - it must have been done previously with a separate
- * "access_ok()" call.
+ * The "__xxx" versions of the user access functions do not verify the address
+ * space - it must have been done previously with a separate "access_ok()"
+ * call.
  *
- * The "xxx_error" versions set the third argument to EFAULT if an
- * error occurs, and leave it unchanged on success.  Note that these
- * versions are void (ie, don't return a value as such).
+ * The "__xxx_error" versions set the third argument to -EFAULT if an error
+ * occurs, and leave it unchanged on success.
  */
-#define __get_user(x,ptr)						\
-({									\
-	long __gu_err = 0;						\
-	__get_user_err((x),(ptr),__gu_err);				\
-	__gu_err;							\
-})
-
-#define __get_user_error(x,ptr,err)					\
-({									\
-	__get_user_err((x),(ptr),err);					\
-	(void) 0;							\
-})
+#define __get_user_asm(instr, reg, x, addr, err)			\
+	asm volatile(							\
+	"1:	" instr "	" reg "1, [%2]\n"			\
+	"2:\n"								\
+	"	.section .fixup, \"ax\"\n"				\
+	"	.align	2\n"						\
+	"3:	mov	%0, %3\n"					\
+	"	mov	%1, #0\n"					\
+	"	b	2b\n"						\
+	"	.previous\n"						\
+	"	.section __ex_table,\"a\"\n"				\
+	"	.align	3\n"						\
+	"	.quad	1b, 3b\n"					\
+	"	.previous"						\
+	: "+r" (err), "=&r" (x)						\
+	: "r" (addr), "i" (-EFAULT))
 
-#define __get_user_err(x,ptr,err)					\
+#define __get_user_err(x, ptr, err)					\
 do {									\
-	unsigned long __gu_addr = (unsigned long)(ptr);			\
-	unsigned long __gu_val;						\
 	__chk_user_ptr(ptr);						\
 	switch (sizeof(*(ptr))) {					\
 	case 1:								\
-		__get_user_asm("ldrb", "%w", __gu_val, __gu_addr, err);	\
+		__get_user_asm("ldrb", "%w", (x), (ptr), err);		\
 		break;							\
 	case 2:								\
-		__get_user_asm("ldrh", "%w", __gu_val, __gu_addr, err);	\
+		__get_user_asm("ldrh", "%w", (x), (ptr), err);		\
 		break;							\
 	case 4:								\
-		__get_user_asm("ldr", "%w", __gu_val, __gu_addr, err);	\
+		__get_user_asm("ldr", "%w", (x), (ptr), err);		\
 		break;							\
 	case 8:								\
-		__get_user_asm("ldr", "%",  __gu_val, __gu_addr, err);	\
+		__get_user_asm("ldr", "%",  (x), (ptr), err);		\
 		break;							\
 	default:							\
-		(__gu_val) = __get_user_bad();				\
+		BUILD_BUG();						\
 	}								\
-	(x) = (__typeof__(*(ptr)))__gu_val;				\
 } while (0)
 
-#define __get_user_asm(instr, reg, x, addr, err)			\
+#define __get_user(x, ptr)						\
+({									\
+	long __gu_err = 0;						\
+	__get_user_err((x), (ptr), __gu_err);				\
+	__gu_err;							\
+})
+
+#define __get_user_error(x, ptr, err)					\
+({									\
+	__get_user_err((x), (ptr), (err));				\
+	(void)0;							\
+})
+
+#define __get_user_unaligned __get_user
+
+#define get_user(x, ptr)						\
+({									\
+	long __gu_err = 0;						\
+	unsigned long __gu_val = 0;					\
+	might_sleep();							\
+	if (access_ok(VERIFY_READ, (ptr), sizeof(*ptr)))		\
+		__get_user_err(__gu_val, (ptr), __gu_err);		\
+	else								\
+		__gu_err = -EFAULT;					\
+	(x) = (__typeof__(*(ptr)))__gu_val;				\
+	__gu_err;							\
+})
+
+#define __put_user_asm(instr, reg, x, __pu_addr, err)			\
 	asm volatile(							\
 	"1:	" instr "	" reg "1, [%2]\n"			\
 	"2:\n"								\
-	"	.section .fixup, \"ax\"\n"				\
+	"	.section .fixup,\"ax\"\n"				\
 	"	.align	2\n"						\
 	"3:	mov	%0, %3\n"					\
-	"	mov	%1, #0\n"					\
 	"	b	2b\n"						\
 	"	.previous\n"						\
 	"	.section __ex_table,\"a\"\n"				\
 	"	.align	3\n"						\
 	"	.quad	1b, 3b\n"					\
 	"	.previous"						\
-	: "+r" (err), "=&r" (x)						\
-	: "r" (addr), "i" (-EFAULT)					\
-	: "cc")
-
-#define __put_user(x,ptr)						\
-({									\
-	long __pu_err = 0;						\
-	__put_user_err((x),(ptr),__pu_err);				\
-	__pu_err;							\
-})
-
-#define __put_user_error(x,ptr,err)					\
-({									\
-	__put_user_err((x),(ptr),err);					\
-	(void) 0;							\
-})
+	: "+r" (err)							\
+	: "r" (x), "r" (__pu_addr), "i" (-EFAULT))
 
-#define __put_user_err(x,ptr,err)					\
+#define __put_user_err(x, ptr, err)					\
 do {									\
 	unsigned long __pu_addr = (unsigned long)(ptr);			\
 	__typeof__(*(ptr)) __pu_val = (x);				\
@@ -292,26 +210,35 @@ do {									\
 		__put_user_asm("str",  "%",  __pu_val, __pu_addr, err);	\
 		break;							\
 	default:							\
-		__put_user_bad();					\
+		BUILD_BUG();						\
 	}								\
 } while (0)
 
-#define __put_user_asm(instr, reg, x, __pu_addr, err)			\
-	asm volatile(							\
-	"1:	" instr "	" reg "1, [%2]\n"			\
-	"2:\n"								\
-	"	.section .fixup,\"ax\"\n"				\
-	"	.align	2\n"						\
-	"3:	mov	%0, %3\n"					\
-	"	b	2b\n"						\
-	"	.previous\n"						\
-	"	.section __ex_table,\"a\"\n"				\
-	"	.align	3\n"						\
-	"	.quad	1b, 3b\n"					\
-	"	.previous"						\
-	: "+r" (err)							\
-	: "r" (x), "r" (__pu_addr), "i" (-EFAULT)			\
-	: "cc")
+#define __put_user(x, ptr)						\
+({									\
+	long __pu_err = 0;						\
+	__put_user_err((x), (ptr), __pu_err);				\
+	__pu_err;							\
+})
+
+#define __put_user_error(x, ptr, err)					\
+({									\
+	__put_user_err((x), (ptr), (err));				\
+	(void)0;							\
+})
+
+#define __put_user_unaligned __put_user
+
+#define put_user(x, ptr)						\
+({									\
+	long __gu_err = 0;						\
+	might_sleep();							\
+	if (access_ok(VERIFY_WRITE, (ptr), sizeof(*ptr)))		\
+		__put_user_err((x), (ptr), __gu_err);			\
+	else								\
+		__gu_err = -EFAULT;					\
+	__gu_err;							\
+})
 
 extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n);
 extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n);
diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c
index 46315732125..cef3925eaf6 100644
--- a/arch/arm64/kernel/arm64ksyms.c
+++ b/arch/arm64/kernel/arm64ksyms.c
@@ -39,15 +39,6 @@ EXPORT_SYMBOL(__copy_from_user);
 EXPORT_SYMBOL(__copy_to_user);
 EXPORT_SYMBOL(__clear_user);
 
-EXPORT_SYMBOL(__get_user_1);
-EXPORT_SYMBOL(__get_user_2);
-EXPORT_SYMBOL(__get_user_4);
-
-EXPORT_SYMBOL(__put_user_1);
-EXPORT_SYMBOL(__put_user_2);
-EXPORT_SYMBOL(__put_user_4);
-EXPORT_SYMBOL(__put_user_8);
-
 	/* bitops */
 EXPORT_SYMBOL(__atomic_hash);
 
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index ae71baede39..2fb7f6092aa 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -1,5 +1,4 @@
 lib-y		:= bitops.o delay.o					\
-		   strncpy_from_user.o strnlen_user.o			\
-		   clear_user.o getuser.o putuser.o			\
+		   strncpy_from_user.o strnlen_user.o clear_user.o	\
 		   copy_from_user.o copy_to_user.o copy_in_user.o	\
 		   copy_page.o clear_page.o
diff --git a/arch/arm64/lib/getuser.S b/arch/arm64/lib/getuser.S
deleted file mode 100644
index 1b4da229603..00000000000
--- a/arch/arm64/lib/getuser.S
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Based on arch/arm/lib/getuser.S
- *
- * Copyright (C) 2012 ARM Ltd.
- * Idea from x86 version, (C) Copyright 1998 Linus Torvalds
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- *
- * These functions have a non-standard call interface to make them more
- * efficient, especially as they return an error value in addition to
- * the "real" return value.
- *
- * __get_user_X
- *
- * Inputs:	x0 contains the address
- * Outputs:	x0 is the error code
- *		x2, x3 contains the zero-extended value
- *		lr corrupted
- *
- * No other registers must be altered.  (see <asm/uaccess.h>
- * for specific ASM register usage).
- *
- * Note also that it is intended that __get_user_bad is not global.
- */
-
-#include <linux/linkage.h>
-#include <asm/errno.h>
-
-ENTRY(__get_user_1)
-1:	ldrb	w2, [x0]
-	mov	x0, #0
-	ret
-ENDPROC(__get_user_1)
-
-ENTRY(__get_user_2)
-2:	ldrh	w2, [x0]
-	mov	x0, #0
-	ret
-ENDPROC(__get_user_2)
-
-ENTRY(__get_user_4)
-3:	ldr	w2, [x0]
-	mov	x0, #0
-	ret
-ENDPROC(__get_user_4)
-
-ENTRY(__get_user_8)
-4:	ldr	x2, [x0]
-	mov	x0, #0
-	ret
-ENDPROC(__get_user_4)
-
-__get_user_bad:
-	mov	x2, #0
-	mov	x0, #-EFAULT
-	ret
-ENDPROC(__get_user_bad)
-
-.section __ex_table, "a"
-	.quad	1b, __get_user_bad
-	.quad	2b, __get_user_bad
-	.quad	3b, __get_user_bad
-	.quad	4b, __get_user_bad
-.previous
diff --git a/arch/arm64/lib/putuser.S b/arch/arm64/lib/putuser.S
deleted file mode 100644
index 62d4a42adb9..00000000000
--- a/arch/arm64/lib/putuser.S
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Based on arch/arm/lib/putuser.S
- *
- * Copyright (C) 2012 ARM Ltd.
- * Idea from x86 version, (C) Copyright 1998 Linus Torvalds
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * These functions have a non-standard call interface to make
- * them more efficient, especially as they return an error
- * value in addition to the "real" return value.
- *
- * __put_user_X
- *
- * Inputs:	x0 contains the address
- *		x2, x3 contains the value
- * Outputs:	x0 is the error code
- *		lr corrupted
- *
- * No other registers must be altered.  (see <asm/uaccess.h>
- * for specific ASM register usage).
- *
- * Note that it is intended that __put_user_bad is not global.
- */
-
-#include <linux/linkage.h>
-#include <asm/errno.h>
-
-ENTRY(__put_user_1)
-1:	strb	w2, [x0]
-	mov	x0, #0
-	ret
-ENDPROC(__put_user_1)
-
-ENTRY(__put_user_2)
-2:	strh	w2, [x0]
-	mov	x0, #0
-	ret
-ENDPROC(__put_user_2)
-
-ENTRY(__put_user_4)
-3:	str	w2, [x0]
-	mov	x0, #0
-	ret
-ENDPROC(__put_user_4)
-
-ENTRY(__put_user_8)
-4:	str	x2, [x0]
-	mov	x0, #0
-	ret
-ENDPROC(__put_user_8)
-
-__put_user_bad:
-	mov	x0, #-EFAULT
-	ret
-ENDPROC(__put_user_bad)
-
-.section __ex_table, "a"
-	.quad	1b, __put_user_bad
-	.quad	2b, __put_user_bad
-	.quad	3b, __put_user_bad
-	.quad	4b, __put_user_bad
-.previous