arm64: percpu: Implement this_cpu operations

The generic this_cpu operations disable interrupts to ensure that the
requested operation is protected from pre-emption. For arm64, this is
overkill and can hurt throughput and latency.

This patch provides arm64 specific implementations for the this_cpu
operations. Rather than disable interrupts, we use the exclusive
monitor or atomic operations as appropriate.

The following operations are implemented: add, add_return, and, or,
read, write, xchg. We also wire up a cmpxchg implementation from
cmpxchg.h.

Testing was performed using the percpu_test module and hackbench on a
Juno board running 3.18-rc3.

Signed-off-by: Steve Capper <steve.capper@linaro.org>
---

This patch applies on top of one I sent out earlier:
"arm64: xchg: Implement cmpxchg_double"
http://lists.infradead.org/pipermail/linux-arm-kernel/2014-October/294705.html

My two least favourite things in the kernel are excessive pre-processor
use and assembler, so this patch has been fun to write ;-).

I've tried to make the inline assembler clearer with named parameters,
it really upset checkpatch, if it upsets people then please shout and I
will change it.

Cheers,

Hi Steve,

On 6 November 2014 12:12, Steve Capper <steve.capper@linaro.org> wrote:
> The generic this_cpu operations disable interrupts to ensure that the
> requested operation is protected from pre-emption. For arm64, this is
> overkill and can hurt throughput and latency.
>
> This patch provides arm64 specific implementations for the this_cpu
> operations. Rather than disable interrupts, we use the exclusive
> monitor or atomic operations as appropriate.
>
> The following operations are implemented: add, add_return, and, or,
> read, write, xchg. We also wire up a cmpxchg implementation from
> cmpxchg.h.
>
> Testing was performed using the percpu_test module and hackbench on a
> Juno board running 3.18-rc3.
>

Got any numbers?

> Signed-off-by: Steve Capper <steve.capper@linaro.org>
> ---
>
> This patch applies on top of one I sent out earlier:
> "arm64: xchg: Implement cmpxchg_double"
> http://lists.infradead.org/pipermail/linux-arm-kernel/2014-October/294705.html
>
> My two least favourite things in the kernel are excessive pre-processor
> use and assembler, so this patch has been fun to write ;-).
>
> I've tried to make the inline assembler clearer with named parameters,
> it really upset checkpatch, if it upsets people then please shout and I
> will change it.
>
> Cheers,
> --
> Steve
>
> ---
>  arch/arm64/include/asm/cmpxchg.h |   6 +-
>  arch/arm64/include/asm/percpu.h  | 231 +++++++++++++++++++++++++++++++++++++++
>  2 files changed, 235 insertions(+), 2 deletions(-)
>
> diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
> index 3e02245..3e51f49 100644
> --- a/arch/arm64/include/asm/cmpxchg.h
> +++ b/arch/arm64/include/asm/cmpxchg.h
> @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
>         __ret; \
>  })
>
> -#define this_cpu_cmpxchg_8(ptr, o, n) \
> -       cmpxchg(raw_cpu_ptr(&(ptr)), o, n);
> +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
>
>  #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
>         cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
> diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> index 5279e57..e751681 100644
> --- a/arch/arm64/include/asm/percpu.h
> +++ b/arch/arm64/include/asm/percpu.h
> @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
>
>  #endif /* CONFIG_SMP */
>
> +#define PERCPU_OP(op, asm_op)                                          \
> +static inline unsigned long __percpu_##op(void *ptr,                   \
> +                       unsigned long val, int size)                    \
> +{                                                                      \
> +       unsigned long loop, ret;                                        \
> +                                                                       \
> +       switch (size) {                                                 \
> +       case 1:                                                         \
> +               do {                                                    \
> +                       asm ("//__per_cpu_" #op "_1\n"                  \
> +                       "ldxrb    %w[ret], %[ptr]\n"                    \
> +                       #asm_op " %w[ret], %w[ret], %w[val]\n"          \
> +                       "stxrb    %w[loop], %w[ret], %[ptr]\n"          \
> +                       : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
> +                         [ptr] "+Q"(*(u8 *)ptr)                        \
> +                       : [val] "Ir" (val));                            \
> +               } while (loop);                                         \
> +               break;                                                  \
> +       case 2:                                                         \
> +               do {                                                    \
> +                       asm ("//__per_cpu_" #op "_2\n"                  \
> +                       "ldxrh    %w[ret], %[ptr]\n"                    \
> +                       #asm_op " %w[ret], %w[ret], %w[val]\n"          \
> +                       "stxrh    %w[loop], %w[ret], %[ptr]\n"          \
> +                       : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
> +                         [ptr]  "+Q"(*(u16 *)ptr)                      \
> +                       : [val] "Ir" (val));                            \
> +               } while (loop);                                         \
> +               break;                                                  \
> +       case 4:                                                         \
> +               do {                                                    \
> +                       asm ("//__per_cpu_" #op "_4\n"                  \
> +                       "ldxr     %w[ret], %[ptr]\n"                    \
> +                       #asm_op " %w[ret], %w[ret], %w[val]\n"          \
> +                       "stxr     %w[loop], %w[ret], %[ptr]\n"          \
> +                       : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
> +                         [ptr] "+Q"(*(u32 *)ptr)                       \
> +                       : [val] "Ir" (val));                            \
> +               } while (loop);                                         \
> +               break;                                                  \
> +       case 8:                                                         \
> +               do {                                                    \
> +                       asm ("//__per_cpu_" #op "_8\n"                  \
> +                       "ldxr     %[ret], %[ptr]\n"                     \
> +                       #asm_op " %[ret], %[ret], %[val]\n"             \
> +                       "stxr     %w[loop], %[ret], %[ptr]\n"           \
> +                       : [loop] "=&r" (loop), [ret] "=&r" (ret),       \
> +                         [ptr] "+Q"(*(u64 *)ptr)                       \
> +                       : [val] "Ir" (val));                            \
> +               } while (loop);                                         \
> +               break;                                                  \
> +       default:                                                        \
> +               BUILD_BUG();                                            \
> +       }                                                               \
> +                                                                       \
> +       return ret;                                                     \
> +}
> +
> +PERCPU_OP(add, add)
> +PERCPU_OP(and, and)
> +PERCPU_OP(or, orr)
> +#undef PERCPU_OP
> +
> +static inline unsigned long __percpu_read(void *ptr, int size)
> +{
> +       unsigned long ret;
> +
> +       switch (size) {
> +       case 1:
> +               asm ("//__per_cpu_read_1\n"
> +                       "ldrb %w[ret], %[ptr]\n" :
> +                       [ret] "=&r"(ret), [ptr] "+Q"(*(u8 *)ptr));
> +               break;
> +       case 2:
> +               asm ("//__per_cpu_read_2\n"
> +                       "ldrh %w[ret], %[ptr]\n" :
> +                       [ret] "=&r"(ret), [ptr] "+Q"(*(u16 *)ptr));
> +               break;
> +       case 4:
> +               asm ("//__per_cpu_read_4\n"
> +                       "ldr %w[ret], %[ptr]\n" :
> +                       [ret] "=&r"(ret), [ptr] "+Q"(*(u32 *)ptr));
> +               break;
> +       case 8:
> +               asm ("//__per_cpu_read_8\n"
> +                       "ldr %[ret], %[ptr]\n" :
> +                       [ret] "=&r"(ret), [ptr] "+Q"(*(u64 *)ptr));
> +               break;
> +       default:
> +               BUILD_BUG();
> +       }
> +
> +       return ret;
> +}
> +

Why are the 'ptr' references '+Q' outputs here rather than 'Q' inputs?

> +static inline void __percpu_write(void *ptr, unsigned long val, int size)
> +{
> +       switch (size) {
> +       case 1:
> +               asm ("//__per_cpu_write_1\n"
> +                       "strb %w[val], %[ptr]\n" :
> +                       [ptr] "+Q"(*(u8 *)ptr) : [val] "r"(val));
> +               break;
> +       case 2:
> +               asm ("//__per_cpu_write_2\n"
> +                       "strh %w[val], %[ptr]\n" :
> +                       [ptr] "+Q"(*(u16 *)ptr) : [val] "r"(val));
> +               break;
> +       case 4:
> +               asm ("//__per_cpu_write_4\n"
> +                       "str %w[val], %[ptr]\n" :
> +                       [ptr] "+Q"(*(u32 *)ptr) : [val] "r"(val));
> +               break;
> +       case 8:
> +               asm ("//__per_cpu_write_8\n"
> +                       "str %[val], %[ptr]\n" :
> +                       [ptr] "+Q"(*(u64 *)ptr) : [val] "r"(val));
> +               break;
> +       default:
> +               BUILD_BUG();
> +       }
> +}
> +

... and similarly, why are these '+Q' and not just '=Q' ?

> +static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
> +                                               int size)
> +{
> +       unsigned long ret, loop;
> +
> +       switch (size) {
> +       case 1:
> +               do {
> +                       asm ("//__percpu_xchg_1\n"
> +                       "ldxrb %w[ret], %[ptr]\n"
> +                       "stxrb %w[loop], %w[val], %[ptr]\n"
> +                       : [loop] "=&r"(loop), [ret] "=&r"(ret),
> +                         [ptr] "+Q"(*(u8 *)ptr)
> +                       : [val] "r" (val));
> +               } while (loop);
> +               break;
> +       case 2:
> +               do {
> +                       asm ("//__percpu_xchg_2\n"
> +                       "ldxrh %w[ret], %[ptr]\n"
> +                       "stxrh %w[loop], %w[val], %[ptr]\n"
> +                       : [loop] "=&r"(loop), [ret] "=&r"(ret),
> +                         [ptr] "+Q"(*(u16 *)ptr)
> +                       : [val] "r" (val));
> +               } while (loop);
> +               break;
> +       case 4:
> +               do {
> +                       asm ("//__percpu_xchg_4\n"
> +                       "ldxr %w[ret], %[ptr]\n"
> +                       "stxr %w[loop], %w[val], %[ptr]\n"
> +                       : [loop] "=&r"(loop), [ret] "=&r"(ret),
> +                         [ptr] "+Q"(*(u32 *)ptr)
> +                       : [val] "r" (val));
> +               } while (loop);
> +               break;
> +       case 8:
> +               do {
> +                       asm ("//__percpu_xchg_8\n"
> +                       "ldxr %[ret], %[ptr]\n"
> +                       "stxr %w[loop], %[val], %[ptr]\n"
> +                       : [loop] "=&r"(loop), [ret] "=&r"(ret),
> +                         [ptr] "+Q"(*(u64 *)ptr)
> +                       : [val] "r" (val));
> +               } while (loop);
> +               break;
> +       default:
> +               BUILD_BUG();
> +       }
> +
> +       return ret;
> +}
> +
> +#define _percpu_add(pcp, val) \
> +       __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
> +
> +#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val))
> +
> +#define _percpu_and(pcp, val) \
> +       __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
> +
> +#define _percpu_or(pcp, val) \
> +       __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp))
> +
> +#define _percpu_read(pcp) (typeof(pcp))        \
> +       (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp)))
> +
> +#define _percpu_write(pcp, val) \
> +       __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))
> +
> +#define _percpu_xchg(pcp, val) (typeof(pcp)) \
> +       (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)))
> +
> +#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
> +#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
> +#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
> +#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
> +
> +#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
> +#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
> +#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
> +#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
> +
> +#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
> +#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
> +#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
> +#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
> +
> +#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
> +#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
> +#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
> +#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
> +
> +#define this_cpu_read_1(pcp) _percpu_read(pcp)
> +#define this_cpu_read_2(pcp) _percpu_read(pcp)
> +#define this_cpu_read_4(pcp) _percpu_read(pcp)
> +#define this_cpu_read_8(pcp) _percpu_read(pcp)
> +
> +#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
> +#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
> +#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
> +#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
> +
> +#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
> +#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
> +#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
> +#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
> +
>  #include <asm-generic/percpu.h>
>
>  #endif /* __ASM_PERCPU_H */
> --
> 1.9.3
>
>
> _______________________________________________
> linux-arm-kernel mailing list
> linux-arm-kernel@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

On Thu, Nov 06, 2014 at 12:26:46PM +0100, Ard Biesheuvel wrote:
> Hi Steve,

Hey Ard,

> 
> On 6 November 2014 12:12, Steve Capper <steve.capper@linaro.org> wrote:
> > The generic this_cpu operations disable interrupts to ensure that the
> > requested operation is protected from pre-emption. For arm64, this is
> > overkill and can hurt throughput and latency.
> >
> > This patch provides arm64 specific implementations for the this_cpu
> > operations. Rather than disable interrupts, we use the exclusive
> > monitor or atomic operations as appropriate.
> >
> > The following operations are implemented: add, add_return, and, or,
> > read, write, xchg. We also wire up a cmpxchg implementation from
> > cmpxchg.h.
> >
> > Testing was performed using the percpu_test module and hackbench on a
> > Juno board running 3.18-rc3.
> >
> 
> Got any numbers?

For `./hackbench 100 process 1000' on a Juno system running all cores I get:

      w/o patch | with patch
 ---------------+------------
 mean     47.05 | 44.93
 stddev    0.38 |  0.68
 ---------------+------------
(times in seconds, rounded to 2d.p., six runs performed)

So a just under 5% speed boost.

[...]

> > diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> > index 5279e57..e751681 100644
> > --- a/arch/arm64/include/asm/percpu.h
> > +++ b/arch/arm64/include/asm/percpu.h
> > @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)

[...]

> > +static inline unsigned long __percpu_read(void *ptr, int size)
> > +{
> > +       unsigned long ret;
> > +
> > +       switch (size) {
> > +       case 1:
> > +               asm ("//__per_cpu_read_1\n"
> > +                       "ldrb %w[ret], %[ptr]\n" :
> > +                       [ret] "=&r"(ret), [ptr] "+Q"(*(u8 *)ptr));
> > +               break;
> > +       case 2:
> > +               asm ("//__per_cpu_read_2\n"
> > +                       "ldrh %w[ret], %[ptr]\n" :
> > +                       [ret] "=&r"(ret), [ptr] "+Q"(*(u16 *)ptr));
> > +               break;
> > +       case 4:
> > +               asm ("//__per_cpu_read_4\n"
> > +                       "ldr %w[ret], %[ptr]\n" :
> > +                       [ret] "=&r"(ret), [ptr] "+Q"(*(u32 *)ptr));
> > +               break;
> > +       case 8:
> > +               asm ("//__per_cpu_read_8\n"
> > +                       "ldr %[ret], %[ptr]\n" :
> > +                       [ret] "=&r"(ret), [ptr] "+Q"(*(u64 *)ptr));
> > +               break;
> > +       default:
> > +               BUILD_BUG();
> > +       }
> > +
> > +       return ret;
> > +}
> > +
> 
> Why are the 'ptr' references '+Q' outputs here rather than 'Q' inputs?
> 

Because I fudged the constraint :-).
Thanks, you're quite right it should be 'Q' constraint as we're only reading
from that address.

> > +static inline void __percpu_write(void *ptr, unsigned long val, int size)
> > +{
> > +       switch (size) {
> > +       case 1:
> > +               asm ("//__per_cpu_write_1\n"
> > +                       "strb %w[val], %[ptr]\n" :
> > +                       [ptr] "+Q"(*(u8 *)ptr) : [val] "r"(val));
> > +               break;
> > +       case 2:
> > +               asm ("//__per_cpu_write_2\n"
> > +                       "strh %w[val], %[ptr]\n" :
> > +                       [ptr] "+Q"(*(u16 *)ptr) : [val] "r"(val));
> > +               break;
> > +       case 4:
> > +               asm ("//__per_cpu_write_4\n"
> > +                       "str %w[val], %[ptr]\n" :
> > +                       [ptr] "+Q"(*(u32 *)ptr) : [val] "r"(val));
> > +               break;
> > +       case 8:
> > +               asm ("//__per_cpu_write_8\n"
> > +                       "str %[val], %[ptr]\n" :
> > +                       [ptr] "+Q"(*(u64 *)ptr) : [val] "r"(val));
> > +               break;
> > +       default:
> > +               BUILD_BUG();
> > +       }
> > +}
> > +
> 
> ... and similarly, why are these '+Q' and not just '=Q' ?

Another mistake on my part, it should be "=Q" as we're only writing to that
address.

Thanks for going through this carefully Ard, I'll send out a corrected V2 once
I've sanity checked it.

Cheers,

Hi Steve,

Thanks for looking at this!

On Thu, Nov 06, 2014 at 11:12:57AM +0000, Steve Capper wrote:
> The generic this_cpu operations disable interrupts to ensure that the
> requested operation is protected from pre-emption. For arm64, this is
> overkill and can hurt throughput and latency.
> 
> This patch provides arm64 specific implementations for the this_cpu
> operations. Rather than disable interrupts, we use the exclusive
> monitor or atomic operations as appropriate.
> 
> The following operations are implemented: add, add_return, and, or,
> read, write, xchg. We also wire up a cmpxchg implementation from
> cmpxchg.h.
> 
> Testing was performed using the percpu_test module and hackbench on a
> Juno board running 3.18-rc3.

[...]

> diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
> index 3e02245..3e51f49 100644
> --- a/arch/arm64/include/asm/cmpxchg.h
> +++ b/arch/arm64/include/asm/cmpxchg.h
> @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
>  	__ret; \
>  })
>  
> -#define this_cpu_cmpxchg_8(ptr, o, n) \
> -	cmpxchg(raw_cpu_ptr(&(ptr)), o, n);
> +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)

You can use cmpxchg_local here, as we don't require barrier semantics.

>  #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
>  	cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
> diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> index 5279e57..e751681 100644
> --- a/arch/arm64/include/asm/percpu.h
> +++ b/arch/arm64/include/asm/percpu.h
> @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
>  
>  #endif /* CONFIG_SMP */
>  
> +#define PERCPU_OP(op, asm_op)						\
> +static inline unsigned long __percpu_##op(void *ptr,			\
> +			unsigned long val, int size)			\
> +{									\
> +	unsigned long loop, ret;					\
> +									\
> +	switch (size) {							\
> +	case 1:								\
> +		do {							\
> +			asm ("//__per_cpu_" #op "_1\n"			\
> +			"ldxrb	  %w[ret], %[ptr]\n"			\
> +			#asm_op " %w[ret], %w[ret], %w[val]\n"		\
> +			"stxrb	  %w[loop], %w[ret], %[ptr]\n"		\
> +			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
> +			  [ptr] "+Q"(*(u8 *)ptr)			\
> +			: [val] "Ir" (val));				\
> +		} while (loop);						\
> +		break;							\

Curious, but do you see any difference in code generation over an explicit
cbnz, like we use in the ATOMIC_OP macro?

> +	case 2:								\
> +		do {							\
> +			asm ("//__per_cpu_" #op "_2\n"			\
> +			"ldxrh	  %w[ret], %[ptr]\n"			\
> +			#asm_op " %w[ret], %w[ret], %w[val]\n"		\
> +			"stxrh	  %w[loop], %w[ret], %[ptr]\n"		\
> +			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
> +			  [ptr]  "+Q"(*(u16 *)ptr)			\
> +			: [val] "Ir" (val));				\
> +		} while (loop);						\
> +		break;							\
> +	case 4:								\
> +		do {							\
> +			asm ("//__per_cpu_" #op "_4\n"			\
> +			"ldxr	  %w[ret], %[ptr]\n"			\
> +			#asm_op " %w[ret], %w[ret], %w[val]\n"		\
> +			"stxr	  %w[loop], %w[ret], %[ptr]\n"		\
> +			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
> +			  [ptr] "+Q"(*(u32 *)ptr)			\
> +			: [val] "Ir" (val));				\
> +		} while (loop);						\
> +		break;							\
> +	case 8:								\
> +		do {							\
> +			asm ("//__per_cpu_" #op "_8\n"			\
> +			"ldxr	  %[ret], %[ptr]\n"			\
> +			#asm_op " %[ret], %[ret], %[val]\n"		\
> +			"stxr	  %w[loop], %[ret], %[ptr]\n"		\
> +			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
> +			  [ptr] "+Q"(*(u64 *)ptr)			\
> +			: [val] "Ir" (val));				\
> +		} while (loop);						\
> +		break;							\
> +	default:							\
> +		BUILD_BUG();						\
> +	}								\
> +									\
> +	return ret;							\
> +}
> +
> +PERCPU_OP(add, add)
> +PERCPU_OP(and, and)
> +PERCPU_OP(or, orr)

Can you use these to generate local_t versions too?

Will

On Thu, Nov 06, 2014 at 12:27:53PM +0000, Will Deacon wrote:
> Hi Steve,
> 
> Thanks for looking at this!

Hey Will,
No problem, it's quite beneficial for performance.

> 
> On Thu, Nov 06, 2014 at 11:12:57AM +0000, Steve Capper wrote:
> > The generic this_cpu operations disable interrupts to ensure that the
> > requested operation is protected from pre-emption. For arm64, this is
> > overkill and can hurt throughput and latency.
> > 
> > This patch provides arm64 specific implementations for the this_cpu
> > operations. Rather than disable interrupts, we use the exclusive
> > monitor or atomic operations as appropriate.
> > 
> > The following operations are implemented: add, add_return, and, or,
> > read, write, xchg. We also wire up a cmpxchg implementation from
> > cmpxchg.h.
> > 
> > Testing was performed using the percpu_test module and hackbench on a
> > Juno board running 3.18-rc3.
> 
> [...]
> 
> > diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
> > index 3e02245..3e51f49 100644
> > --- a/arch/arm64/include/asm/cmpxchg.h
> > +++ b/arch/arm64/include/asm/cmpxchg.h
> > @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
> >  	__ret; \
> >  })
> >  
> > -#define this_cpu_cmpxchg_8(ptr, o, n) \
> > -	cmpxchg(raw_cpu_ptr(&(ptr)), o, n);
> > +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> > +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> > +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> > +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n)
> 
> You can use cmpxchg_local here, as we don't require barrier semantics.

Agreed, thanks, I will update that.

> 
> >  #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
> >  	cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \
> > diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
> > index 5279e57..e751681 100644
> > --- a/arch/arm64/include/asm/percpu.h
> > +++ b/arch/arm64/include/asm/percpu.h
> > @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void)
> >  
> >  #endif /* CONFIG_SMP */
> >  
> > +#define PERCPU_OP(op, asm_op)						\
> > +static inline unsigned long __percpu_##op(void *ptr,			\
> > +			unsigned long val, int size)			\
> > +{									\
> > +	unsigned long loop, ret;					\
> > +									\
> > +	switch (size) {							\
> > +	case 1:								\
> > +		do {							\
> > +			asm ("//__per_cpu_" #op "_1\n"			\
> > +			"ldxrb	  %w[ret], %[ptr]\n"			\
> > +			#asm_op " %w[ret], %w[ret], %w[val]\n"		\
> > +			"stxrb	  %w[loop], %w[ret], %[ptr]\n"		\
> > +			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
> > +			  [ptr] "+Q"(*(u8 *)ptr)			\
> > +			: [val] "Ir" (val));				\
> > +		} while (loop);						\
> > +		break;							\
> 
> Curious, but do you see any difference in code generation over an explicit
> cbnz, like we use in the ATOMIC_OP macro?

I've not noticed any substancial difference in the code paths I've
inspected. Theoretically, a compiler that is extremely averse to
branches could do some unrolling with this.

I decided to give the compiler as much control as possible, so elected
to put the minimum amount of assembler in.

> 
> > +	case 2:								\
> > +		do {							\
> > +			asm ("//__per_cpu_" #op "_2\n"			\
> > +			"ldxrh	  %w[ret], %[ptr]\n"			\
> > +			#asm_op " %w[ret], %w[ret], %w[val]\n"		\
> > +			"stxrh	  %w[loop], %w[ret], %[ptr]\n"		\
> > +			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
> > +			  [ptr]  "+Q"(*(u16 *)ptr)			\
> > +			: [val] "Ir" (val));				\
> > +		} while (loop);						\
> > +		break;							\
> > +	case 4:								\
> > +		do {							\
> > +			asm ("//__per_cpu_" #op "_4\n"			\
> > +			"ldxr	  %w[ret], %[ptr]\n"			\
> > +			#asm_op " %w[ret], %w[ret], %w[val]\n"		\
> > +			"stxr	  %w[loop], %w[ret], %[ptr]\n"		\
> > +			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
> > +			  [ptr] "+Q"(*(u32 *)ptr)			\
> > +			: [val] "Ir" (val));				\
> > +		} while (loop);						\
> > +		break;							\
> > +	case 8:								\
> > +		do {							\
> > +			asm ("//__per_cpu_" #op "_8\n"			\
> > +			"ldxr	  %[ret], %[ptr]\n"			\
> > +			#asm_op " %[ret], %[ret], %[val]\n"		\
> > +			"stxr	  %w[loop], %[ret], %[ptr]\n"		\
> > +			: [loop] "=&r" (loop), [ret] "=&r" (ret),	\
> > +			  [ptr] "+Q"(*(u64 *)ptr)			\
> > +			: [val] "Ir" (val));				\
> > +		} while (loop);						\
> > +		break;							\
> > +	default:							\
> > +		BUILD_BUG();						\
> > +	}								\
> > +									\
> > +	return ret;							\
> > +}
> > +
> > +PERCPU_OP(add, add)
> > +PERCPU_OP(and, and)
> > +PERCPU_OP(or, orr)
> 
> Can you use these to generate local_t versions too?

Sure, I forgot about them, I will add them to V3.

Cheers,