path: root/renderscript-toolkit/src/main/cpp/Resize_neon.S

/*
 * Copyright (C) 2015 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#define ENTRY(f) .text; .align 4; .globl f; .type f,#function; f: .fnstart
#define END(f) .fnend; .size f, .-f;

.eabi_attribute 25,1 @Tag_ABI_align8_preserved
.arm

/* Fixed-point precision after vertical pass -- 16 bit data minus 1 sign and 1
 * integer (bicubic has a little overshoot).  It would also be possible to add
 * a temporary DC bias to eliminate the sign bit for more precision, but that's
 * extra arithmetic.
 */
.set VERTBITS, 14

/* The size of the scratch buffer in which we store our vertically convolved
 * intermediates.
 */
.set CHUNKSHIFT, 7
.set CHUNKSIZE, (1 << CHUNKSHIFT)

/* The number of components processed in a single iteration of the innermost
 * loop.
 */
.set VECSHIFT, 3
.set VECSIZE, (1<<VECSHIFT)

/* Read four different lines (except at edges where addresses may be clamped,
 * which is why we don't simply take base and stride registers), and multiply
 * and accumulate them by the coefficients in d6[0..3], leaving the results in
 * q12.  This gives eight 16-bit results representing a horizontal line of 2-8
 * input pixels (depending on number of components per pixel) to be fed into
 * the horizontal scaling pass.
 *
 * Input coefficients are 16-bit unsigned fixed-point (although [0] and [3] are
 * known to represent negative values and VMLS is used to implement this).
 * Output is VERTBITS signed fixed-point, which must leave room for a little
 * bit of overshoot beyond [0,1.0).
 */
.macro vert8, dstlo=d24, dsthi=d25
        vld1.u8     d16, [r4]!
        vld1.u8     d18, [r5]!
        vld1.u8     d20, [r6]!
        vld1.u8     d22, [r7]!
        vmovl.u8    q8, d16
        vmovl.u8    q9, d18
        vmovl.u8    q10, d20
        vmovl.u8    q11, d22
        vmull.u16   q12, d18, d6[1]
        vmull.u16   q13, d19, d6[1]
        vmlsl.u16   q12, d16, d6[0]
        vmlsl.u16   q13, d17, d6[0]
        vmlal.u16   q12, d20, d6[2]
        vmlal.u16   q13, d21, d6[2]
        vmlsl.u16   q12, d22, d6[3]
        vmlsl.u16   q13, d23, d6[3]

        /* Shift by 8 (bits per pixel), plus 16 (the fixed-point multiplies),
         * minus VERTBITS (the number of fraction bits we want to keep from
         * here on).
         */
        vqshrn.s32  \dstlo, q12, #8 + 16 - VERTBITS
        vqshrn.s32  \dsthi, q13, #8 + 16 - VERTBITS
.endm

/* As above, but only four 16-bit results into d25.
 */
.macro vert4
        vld1.u32    d16[0], [r4]!
        vld1.u32    d18[0], [r5]!
        vld1.u32    d20[0], [r6]!
        vld1.u32    d22[0], [r7]!
        vmovl.u8    q8, d16
        vmovl.u8    q9, d18
        vmovl.u8    q10, d20
        vmovl.u8    q11, d22
        vmull.u16   q12, d18, d6[1]
        vmlsl.u16   q12, d16, d6[0]
        vmlal.u16   q12, d20, d6[2]
        vmlsl.u16   q12, d22, d6[3]
        vqshrn.s32  d25, q12, #8 + 16 - VERTBITS
.endm


/* During horizontal resize having CHUNKSIZE input available means being able
 * to produce a varying amount of output, depending on the phase of the data.
 * This function calculates the minimum number of VECSIZE chunks extracted from
 * a CHUNKSIZE window (r1), and the threshold value for when the count will be
 * one higher than that (r0).
 * These work out, conveniently, to be the quotient and remainder from:
 *      (CHUNKSIZE + xinc * VECSIZE - 1) / (xinc * VECSIZE)
 *
 * The two values can be packed together in a uint64_t for convenience; and
 * they are, in fact, used this way as an arithmetic short-cut later on.
 */

/* uint64_t rsdIntrinsicResize_oscctl_K(uint32_t xinc); */
ENTRY(rsdIntrinsicResize_oscctl_K)
        lsl         r2, r0, #VECSHIFT
        movw        r0, #:lower16:(CHUNKSIZE << 16) - 1
        movt        r0, #:upper16:(CHUNKSIZE << 16) - 1
        add         r0, r0, r2
#if defined(ARCH_ARM_USE_UDIV)
        udiv        r1, r0, r2
        mls         r0, r1, r2, r0
#else
        clz         r3, r2
        clz         r1, r0
        subs        r3, r3, r1
        movlt       r3, #0
        mov         r1, #1
        lsl         r2, r2, r3
        lsl         r3, r1, r3
        mov         r1, #0
1:      cmp         r2, r0
        addls       r1, r3
        subls       r0, r2
        lsrs        r3, r3, #1
        lsr         r2, r2, #1
        bne         1b
#endif
        bx          lr
END(rsdIntrinsicResize_oscctl_K)

/* Iterate to generate the uchar1, uchar2, and uchar4 versions of the code.
 * For the most part the vertical pass (the outer loop) is the same for all
 * versions.  Exceptions are handled in-line with conditional assembly.
 */
.irp comp, 1, 2, 4
.if \comp == 1
.set COMPONENT_SHIFT, 0
.elseif \comp == 2
.set COMPONENT_SHIFT, 1
.elseif \comp == 4
.set COMPONENT_SHIFT, 2
.else
.error "Unknown component count"
.endif
.set COMPONENT_COUNT, (1 << COMPONENT_SHIFT)
.set LOOP_OUTPUT_SIZE, (VECSIZE * COMPONENT_COUNT)

.set BUFFER_SIZE, (CHUNKSIZE * 2 + 4) * COMPONENT_COUNT * 2
.set OSC_STORE, (BUFFER_SIZE + 0)
.set OSCSTEP_STORE, (BUFFER_SIZE + 4)
.set OSCCTL_STORE, (BUFFER_SIZE + 8)
.set AVAIL_STORE, (BUFFER_SIZE + 16)
.set SP_STORE, (BUFFER_SIZE + 24)   /* should be +20, but rounded up to make a legal constant somewhere */

/* void rsdIntrinsicResizeB\comp\()_K(
 *             uint8_t * restrict dst,          // r0
 *             size_t count,                    // r1
 *             uint32_t xf,                     // r2
 *             uint32_t xinc,                   // r3
 *             uint8_t const * restrict srcn,   // [sp]     -> [sp,#104] -> r4
 *             uint8_t const * restrict src0,   // [sp,#4]  -> [sp,#108] -> r5
 *             uint8_t const * restrict src1,   // [sp,#8]  -> [sp,#112] -> r6
 *             uint8_t const * restrict src2,   // [sp,#12] -> [sp,#116] -> r7
 *             size_t xclip,                    // [sp,#16] -> [sp,#120]
 *             size_t avail,                    // [sp,#20] -> [sp,#124] -> lr
 *             uint64_t osc_ctl,                // [sp,#24] -> [sp,#128]
 *             int32_t const *yr);              // [sp,#32] -> [sp,#136] -> d8 (copied to d6 for scalar access)
 */
ENTRY(rsdIntrinsicResizeB\comp\()_K)
            push        {r4,r5,r6,r7,r8,r9,r10,r11,r12,lr}
            vpush       {d8-d15}

            /* align the working buffer on the stack to make it easy to use bit
             * twiddling for address calculations and bounds tests.
             */
            sub         r12, sp, #BUFFER_SIZE + 32
            mov         lr, sp
            bfc         r12, #0, #CHUNKSHIFT + 1 + COMPONENT_SHIFT + 1
            mov         sp, r12
            str         lr, [sp,#SP_STORE]

            ldr         r8, [lr,#136]           // yr
            adr         r9, 8f
            vld1.s32    {q4}, [r8]
            vld1.s16    {q5}, [r9]
            vqmovun.s32 d8, q4                  // yr
            vdup.s16    q6, r2
            vdup.s16    q7, r3
            vmla.s16    q6, q5, q7              // vxf
            vshl.s16    q7, q7, #VECSHIFT       // vxinc

            ldrd        r4,r5, [lr,#104]        // srcn, src0
            ldrd        r6,r7, [lr,#112]        // src1, src2

            /* Compute starting condition for oscillator used to compute ahead
             * of time how many iterations are possible before needing to
             * refill the working buffer.  This is based on the fixed-point
             * index of the last element in the vector of pixels processed in
             * each iteration, counting up until it would overflow.
             */
            sub         r8, r2, r3
            mov         r9, r3, LSL #VECSHIFT
            add         r8, r8, r9

            ldrd        r10,r11, [lr,#128]      // osc_ctl

            str         r8, [sp,#OSC_STORE]
            str         r9, [sp,#OSCSTEP_STORE]
            str         r10, [sp,#OSCCTL_STORE]
            str         r11, [sp,#OSCCTL_STORE+4]
            ldrd        r10,r11, [lr,#120]      // xclip,avail


            /* r4-r7 contain pointers to the four lines of input to be
             * convolved.  These pointers have been clamped vertically and
             * horizontally (which is why it's not a simple row/stride pair),
             * and the xclip argument (now in r10) indicates how many pixels
             * from true the x position of the pointer is.  This value should
             * be 0, 1, or 2 only.
             *
             * Start by placing four pixels worth of input at the far end of
             * the buffer.  As many as two of these may be clipped, so four
             * pixels are fetched, and then the first pixel is duplicated and
             * the data shifted according to xclip.  The source pointers are
             * then also adjusted according to xclip so that subsequent fetches
             * match.
             */
            vmov        d6, d8  /* make y coeffs available for vert4 and vert8 macros */

            sub         r8, r12, r10, LSL #COMPONENT_SHIFT + 1
            add         r9, r12, #(2 * CHUNKSIZE - 4) * COMPONENT_COUNT * 2
            add         r8, r8, #4 * COMPONENT_COUNT * 2
.if \comp == 1
            vert4
            vdup.s16    d24, d25[0]
            vst1.s16    {q12}, [r12]
            vld1.s16    {d24}, [r8]
            vst1.s16    {d24}, [r9]
.elseif \comp == 2
            vert8
            vdup.u32    q11, d24[0]
            vst1.s16    {q11,q12}, [r12]
            vld1.s16    {q12}, [r8]
            vst1.s16    {q12}, [r9]
.elseif \comp == 4
            vert8       d28, d29
            vert8       d30, d31
            vmov.u64    d24, d28
            vmov.u64    d25, d28
            vmov.u64    d26, d28
            vmov.u64    d27, d28
            vst1.s16    {q12,q13}, [r12]!
            vst1.s16    {q14,q15}, [r12]
            sub         r12, r12, #32
            vld1.s16    {q11,q12}, [r8]
            vst1.s16    {q11,q12}, [r9]
.endif
            /* Count off four pixels into the working buffer, and move count to
             * its new home.
             */
            sub         lr, r11, #4
            /* Incoming pointers were to the first _legal_ pixel.  Four pixels
             * were read unconditionally, but some may have been discarded by
             * xclip, so we rewind the pointers to compensate.
             */
            sub         r4, r4, r10, LSL #COMPONENT_SHIFT
            sub         r5, r5, r10, LSL #COMPONENT_SHIFT
            sub         r6, r6, r10, LSL #COMPONENT_SHIFT
            sub         r7, r7, r10, LSL #COMPONENT_SHIFT

            /* First tap starts where we just pre-filled, at the end of the
             * buffer.
             */
            add         r2, r2, #(CHUNKSIZE * 2 - 4) << 16

            /* Use overflowing arithmetic to implement wraparound array
             * indexing.
             */
            mov         r2, r2, LSL #(15 - CHUNKSHIFT)
            mov         r3, r3, LSL #(15 - CHUNKSHIFT)

            str         lr, [sp,#AVAIL_STORE]

            /* Start of outermost loop.
             * Fetch CHUNKSIZE pixels into scratch buffer, then calculate the
             * number of iterations of the inner loop that can be performed and
             * get into that.
             *
             * The fill is complicated by the possibility of running out of
             * input before the scratch buffer is filled.  If this isn't a risk
             * then it's handled by the simple loop at 2:, otherwise the
             * horrible loop at 3:.
             */
1:          ldr         lr, [sp,#AVAIL_STORE]   /* get number of pixels available */
            vmov        d6, d8              /* put y scaling coefficients somewhere handy */
            subs        lr, #CHUNKSIZE
            bge         2f                  /* if at least CHUNKSIZE are available... */
            add         lr, #CHUNKSIZE      /* if they're not... */
            b           4f
            /* ..just sneaking a literal in here after this unconditional branch.. */
8:          .hword      0, 1, 2, 3, 4, 5, 6, 7
            /* basic fill loop, processing 8 bytes at a time until there are
             * fewer than eight bytes available.
             */
3:          vert8
            sub         lr, lr, #8 / COMPONENT_COUNT
            vst1.s16    {q12}, [r12]!
4:          cmp         lr, #8 / COMPONENT_COUNT - 1
            bgt         3b
.if \comp == 4
            blt         3f
            /* The last pixel (four bytes) if necessary */
            vert4
.else
            cmp         lr, #1
            blt         3f
            /* The last pixels if necessary */
            sub         r4, r4, #8
            sub         r5, r5, #8
            sub         r6, r6, #8
            sub         r7, r7, #8
            add         r4, r4, lr, LSL #COMPONENT_SHIFT
            add         r5, r5, lr, LSL #COMPONENT_SHIFT
            add         r6, r6, lr, LSL #COMPONENT_SHIFT
            add         r7, r7, lr, LSL #COMPONENT_SHIFT
            vert8
            sub         lr, sp, lr, LSL #COMPONENT_SHIFT + 1
            sub         sp, sp, #32
            sub         lr, lr, #16
.if \comp == 1
            vdup.s16    q13, d25[3]
.elseif \comp == 2
            vdup.u32    q13, d25[1]
.endif
            vst1.s16    {q12,q13}, [sp]
            vld1.s16    {q12}, [lr]
            add         sp, sp, #32
            b           4f
.endif
            /* Keep filling until we get to the end of this chunk of the buffer */
3:
.if \comp == 1
            vdup.s16    q12, d25[3]
.elseif \comp == 2
            vdup.u32    q12, d25[1]
.elseif \comp == 4
            vmov.u64    d24, d25
.endif
4:          vst1.s16    {q12}, [r12]!
            tst         r12, #(CHUNKSIZE - 1) * COMPONENT_COUNT * 2
            bne         3b
            b           4f

.align 4
2:          /* Quickly pull a chunk of data into the working buffer.
             */
            vert8
            vst1.s16    {q12}, [r12]!
            vert8
            vst1.s16    {q12}, [r12]!
            tst         r12, #(CHUNKSIZE - 1) * COMPONENT_COUNT * 2
            bne         2b
            cmp         lr, #0
            bne         3f
4:          /* if we end with 0 pixels left we'll have nothing handy to spread
             * across to the right, so we rewind a bit.
             */
            mov         lr, #1
            sub         r4, r4, #COMPONENT_COUNT
            sub         r5, r5, #COMPONENT_COUNT
            sub         r6, r6, #COMPONENT_COUNT
            sub         r7, r7, #COMPONENT_COUNT
3:          str         lr, [sp,#AVAIL_STORE]       /* done with available pixel count */
            add         lr, sp, #OSC_STORE
            ldrd        r8,r9, [lr,#0]              /* need osc, osc_step soon */
            ldrd        r10,r11, [lr,#OSCCTL_STORE-OSC_STORE] /* need osc_ctl too */

            /* copy four taps (width of cubic window) to far end for overflow
             * address handling
             */
            sub         lr, r12, #CHUNKSIZE * COMPONENT_COUNT * 2
            eor         r12, lr, #CHUNKSIZE * COMPONENT_COUNT * 2
.if \comp == 1
            vld1.s16    {d28}, [lr]
.elseif \comp == 2
            vld1.s16    {q14}, [lr]
.elseif \comp == 4
            vld1.s16    {q14,q15}, [lr]
.endif
            add         lr, r12, #CHUNKSIZE * COMPONENT_COUNT * 2
.if \comp == 1
            vst1.s16    {d28}, [lr]
.elseif \comp == 2
            vst1.s16    {q14}, [lr]
.elseif \comp == 4
            vst1.s16    {q14,q15}, [lr]
.endif
            /* r11 contains the maximum possible iteration count, but if r8 is
             * greater than r10 then this indicates that the count must be
             * reduced by one for this iteration to avoid reading past the end
             * of the available data.
             */
            cmp             r10, r8
            sbc         lr, r11, #0

            mla         r8, lr, r9, r8
            sub         r8, r8, #(CHUNKSIZE << 16)

            str         r8, [sp,#OSC_STORE]         /* done with osc */

            /* prefer to count pixels, rather than vectors, to clarify the tail
             * store case on exit.
             */
            mov         lr, lr, LSL #VECSHIFT
            cmp         lr, r1
            movgt       lr, r1

            sub         r1, r1, lr

            mov         lr, lr, LSL #COMPONENT_SHIFT

            vmov.i16    d10, #3
            vmov.i16    d11, #0x8000

            cmp         lr, #0
            bgt         3f
            cmp         r1, #0
            bgt         1b     /* an extreme case where we shouldn't use code in this structure */
            b           9f

            .align 4
2:          /* Inner loop continues here, but starts at 3:, see end of loop
             * below for explanation. */
.if LOOP_OUTPUT_SIZE == 4
            vst1.u32    {d16[0]}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 8
            vst1.u8     {d16}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 16
            vst1.u8     {q8}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 32
            vst1.u8     {q8,q9}, [r0]!
.endif
            /* Inner loop:  here the four x coefficients for each tap are
             * calculated in vector code, and the addresses are calculated in
             * scalar code, and these calculations are interleaved.
             */
3:          vshr.u16    q8, q6, #1
            mov         r8, r2, LSR #(31 - CHUNKSHIFT)
            vqrdmulh.s16 q9, q8, q8
            add         r2, r2, r3
            vqrdmulh.s16 q10, q9, q8
            mov         r9, r2, LSR #(31 - CHUNKSHIFT)
            vshll.s16   q11, d18, #2
            vshll.s16   q12, d19, #2
            add         r2, r2, r3
            vmlsl.s16   q11, d20, d10
            vmlsl.s16   q12, d21, d10
            mov         r10, r2, LSR #(31 - CHUNKSHIFT)

            vhadd.s16   q0, q10, q8
            add         r2, r2, r3
            vsub.s16    q0, q9, q0
            mov         r11, r2, LSR #(31 - CHUNKSHIFT)

            vaddw.s16   q1, q11, d18
            vaddw.s16   q13, q12, d19
            add         r2, r2, r3
            vshrn.s32   d2, q1, #1
            vshrn.s32   d3, q13, #1
            add         r8, sp, r8, LSL #(COMPONENT_SHIFT + 1)
            vsub.s16    d2, d2, d11
            vsub.s16    d3, d3, d11 // TODO: find a wider d11 and use q-reg operation
            add         r9, sp, r9, LSL #(COMPONENT_SHIFT + 1)

            vaddw.s16   q2, q11, d16
            vaddw.s16   q13, q12, d17
            add         r10, sp, r10, LSL #(COMPONENT_SHIFT + 1)
            vshrn.s32   d4, q2, #1
            vshrn.s32   d5, q13, #1
            add         r11, sp, r11, LSL #(COMPONENT_SHIFT + 1)
            vneg.s16    q2, q2

            vhsub.s16   q3, q10, q9

            /* increment the x fractional parts (oveflow is ignored, as the
             * scalar arithmetic shadows this addition with full precision).
             */
            vadd.s16    q6, q6, q7

            /* At this point we have four pointers in r8-r11, pointing to the
             * four taps in the scratch buffer that must be convolved together
             * to produce an output pixel (one output pixel per pointer).
             * These pointers usually overlap, but their spacing is irregular
             * so resolving the redundancy through L1 is a pragmatic solution.
             *
             * The scratch buffer is made of signed 16-bit data, holding over
             * some extra precision, and overshoot, from the vertical pass.
             *
             * We also have the 16-bit unsigned fixed-point weights for each
             * of the four taps in q0 - q3.  That's eight pixels worth of
             * coefficients when we have only four pointers, so calculations
             * for four more pixels are interleaved with the fetch and permute
             * code for each variant in the following code.
             *
             * The data arrangement is less than ideal for any pixel format,
             * but permuting loads help to mitigate most of the problems.
             *
             * Note also that the two outside taps of a bicubic are negative,
             * but these coefficients are unsigned.  The sign is hard-coded by
             * use of multiply-and-subtract operations.
             */
.if \comp == 1
            /* The uchar 1 case.
             * Issue one lanewise vld4.s16 to load four consecutive pixels from
             * one pointer (one pixel) into four different registers; then load
             * four consecutive s16 values from the next pointer (pixel) into
             * the next lane of those four registers, etc., so that we finish
             * with q12 - q15 representing the four taps, and each lane
             * representing a separate pixel.
             *
             * The first vld4 uses a splat to avoid any false dependency on
             * the previous state of the register.
             */
            vld4.s16    {d24[],d26[],d28[],d30[]}, [r8]
            mov         r8, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3
            vld4.s16    {d24[1],d26[1],d28[1],d30[1]}, [r9]
            add         r8, sp, r8, LSL #(COMPONENT_SHIFT + 1)
            mov         r9, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3
            vld4.s16    {d24[2],d26[2],d28[2],d30[2]}, [r10]
            add         r9, sp, r9, LSL #(COMPONENT_SHIFT + 1)
            mov         r10, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3
            vld4.s16    {d24[3],d26[3],d28[3],d30[3]}, [r11]
            add         r10, sp, r10, LSL #(COMPONENT_SHIFT + 1)
            mov         r11, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3
            vld4.s16    {d25[],d27[],d29[],d31[]}, [r8]
            add         r11, sp, r11, LSL #(COMPONENT_SHIFT + 1)
            vld4.s16    {d25[1],d27[1],d29[1],d31[1]}, [r9]
            vld4.s16    {d25[2],d27[2],d29[2],d31[2]}, [r10]
            vld4.s16    {d25[3],d27[3],d29[3],d31[3]}, [r11]

            vmull.s16   q8, d24, d0
            vmull.s16   q9, d25, d1
            vmlsl.s16   q8, d26, d2
            vmlsl.s16   q9, d27, d3
            vmlsl.s16   q8, d28, d4
            vmlsl.s16   q9, d29, d5
            vmlal.s16   q8, d30, d6
            vmlal.s16   q9, d31, d7

            subs        lr, lr, #LOOP_OUTPUT_SIZE

            vqrshrn.s32 d16, q8, #15
            vqrshrn.s32 d17, q9, #15

            vqrshrun.s16 d16, q8, #VERTBITS - 8
.elseif \comp == 2
            /* The uchar2 case:
             * This time load pairs of values into adjacent lanes in q12 - q15
             * by aliasing them as u32 data; leaving room for only four pixels,
             * so the process has to be done twice.  This also means that the
             * coefficient registers fail to align with the coefficient data
             * (eight separate pixels), so that has to be doubled-up to match.
             */
            vld4.u32    {d24[],d26[],d28[],d30[]}, [r8]
            mov         r8, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3
            vld4.u32    {d24[1],d26[1],d28[1],d30[1]}, [r9]
            add         r8, sp, r8, LSL #(COMPONENT_SHIFT + 1)
            mov         r9, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3
            vld4.u32    {d25[],d27[],d29[],d31[]}, [r10]
            add         r9, sp, r9, LSL #(COMPONENT_SHIFT + 1)
            mov         r10, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3
            vld4.u32    {d25[1],d27[1],d29[1],d31[1]}, [r11]
            add         r10, sp, r10, LSL #(COMPONENT_SHIFT + 1)
            mov         r11, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3

            /* double-up coefficients to align with component pairs */
            vmov        d20, d0
            add         r11, sp, r11, LSL #(COMPONENT_SHIFT + 1)
            vmov        d21, d2
            vmov        d22, d4
            vmov        d23, d6
            vzip.s16    d0, d20
            vzip.s16    d2, d21
            vzip.s16    d4, d22
            vzip.s16    d6, d23

            vmull.s16   q8, d24, d0
            vmull.s16   q9, d25, d20
            vmlsl.s16   q8, d26, d2
            vmlsl.s16   q9, d27, d21
            vmlsl.s16   q8, d28, d4
            vmlsl.s16   q9, d29, d22
            vmlal.s16   q8, d30, d6
            vmlal.s16   q9, d31, d23

            vqrshrn.s32 d16, q8, #15
            vqrshrn.s32 d17, q9, #15

            vld4.u32    {d24[],d26[],d28[],d30[]}, [r8]
            vld4.u32    {d24[1],d26[1],d28[1],d30[1]}, [r9]
            vld4.u32    {d25[],d27[],d29[],d31[]}, [r10]
            vld4.u32    {d25[1],d27[1],d29[1],d31[1]}, [r11]

            /* double-up coefficients to align with component pairs */
            vmov        d0, d1
            vmov        d2, d3
            vmov        d4, d5
            vmov        d6, d7
            vzip.s16    d0, d1
            vzip.s16    d2, d3
            vzip.s16    d4, d5
            vzip.s16    d6, d7

            vmull.s16   q10, d24, d0
            vmull.s16   q11, d25, d1
            vmlsl.s16   q10, d26, d2
            vmlsl.s16   q11, d27, d3
            vmlsl.s16   q10, d28, d4
            vmlsl.s16   q11, d29, d5
            vmlal.s16   q10, d30, d6
            vmlal.s16   q11, d31, d7

            subs        lr, lr, #LOOP_OUTPUT_SIZE

            vqrshrn.s32 d18, q10, #15
            vqrshrn.s32 d19, q11, #15

            vqrshrun.s16 d16, q8, #VERTBITS - 8
            vqrshrun.s16 d17, q9, #VERTBITS - 8
.elseif \comp == 4
            /* The uchar4 case.
             * This case is comparatively painless because four s16s are the
             * smallest addressable unit for a vmul-by-scalar.  Rather than
             * permute the data, simply arrange the multiplies to suit the way
             * the data comes in.  That's a lot of data, though, so things
             * progress in pairs of pixels at a time.
             */
            vld1.s16    {q12,q13}, [r8]
            mov         r8, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3
            vld1.s16    {q14,q15}, [r9]
            add         r8, sp, r8, LSL #(COMPONENT_SHIFT + 1)
            mov         r9, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3

            vmull.s16   q8, d24, d0[0]
            vmull.s16   q9, d28, d0[1]
            vmlsl.s16   q8, d25, d2[0]
            vmlsl.s16   q9, d29, d2[1]
            vmlsl.s16   q8, d26, d4[0]
            vmlsl.s16   q9, d30, d4[1]
            vmlal.s16   q8, d27, d6[0]
            vmlal.s16   q9, d31, d6[1]

            /* And two more...  */
            vld1.s16    {q12,q13}, [r10]
            add         r9, sp, r9, LSL #(COMPONENT_SHIFT + 1)
            mov         r10, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3
            vld1.s16    {q14,q15}, [r11]
            add         r10, sp, r10, LSL #(COMPONENT_SHIFT + 1)
            mov         r11, r2, LSR #(31 - CHUNKSHIFT)
            add         r2, r2, r3

            vqrshrn.s32 d16, q8, #15
            add         r11, sp, r11, LSL #(COMPONENT_SHIFT + 1)
            vqrshrn.s32 d17, q9, #15

            vmull.s16   q10, d24, d0[2]
            vmull.s16   q11, d28, d0[3]
            vmlsl.s16   q10, d25, d2[2]
            vmlsl.s16   q11, d29, d2[3]
            vmlsl.s16   q10, d26, d4[2]
            vmlsl.s16   q11, d30, d4[3]
            vmlal.s16   q10, d27, d6[2]
            vmlal.s16   q11, d31, d6[3]

            vqrshrn.s32 d18, q10, #15
            vqrshrn.s32 d19, q11, #15

            vqrshrun.s16 d16, q8, #VERTBITS - 8
            vqrshrun.s16 d17, q9, #VERTBITS - 8

            /* And two more...  */
            vld1.s16    {q12,q13}, [r8]
            vld1.s16    {q14,q15}, [r9]

            vmull.s16   q10, d24, d1[0]
            vmull.s16   q11, d28, d1[1]
            vmlsl.s16   q10, d25, d3[0]
            vmlsl.s16   q11, d29, d3[1]
            vmlsl.s16   q10, d26, d5[0]
            vmlsl.s16   q11, d30, d5[1]
            vmlal.s16   q10, d27, d7[0]
            vmlal.s16   q11, d31, d7[1]

            /* And two more...  */
            vld1.s16    {q12,q13}, [r10]
            vld1.s16    {q14,q15}, [r11]

            subs        lr, lr, #LOOP_OUTPUT_SIZE

            vqrshrn.s32 d18, q10, #15
            vqrshrn.s32 d19, q11, #15

            vmull.s16   q10, d24, d1[2]
            vmull.s16   q11, d28, d1[3]
            vmlsl.s16   q10, d25, d3[2]
            vmlsl.s16   q11, d29, d3[3]
            vmlsl.s16   q10, d26, d5[2]
            vmlsl.s16   q11, d30, d5[3]
            vmlal.s16   q10, d27, d7[2]
            vmlal.s16   q11, d31, d7[3]

            vqrshrn.s32 d20, q10, #15
            vqrshrn.s32 d21, q11, #15

            vqrshrun.s16 d18, q9, #VERTBITS - 8
            vqrshrun.s16 d19, q10, #VERTBITS - 8
.endif
            bgt         2b      /* continue inner loop */
            /* The inner loop has already been limited to ensure that none of
             * the earlier iterations could overfill the output, so the store
             * appears within the loop but after the conditional branch (at the
             * top).  At the end, provided it won't overfill, perform the final
             * store here.  If it would, then break out to the tricky tail case
             * instead.
             */
            blt         1f
            /* Store the amount of data appropriate to the configuration of the
             * instance being assembled.
             */
.if LOOP_OUTPUT_SIZE == 4
            vst1.u32    {d16[0]}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 8
            vst1.u8     {d16}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 16
            vst1.u8     {q8}, [r0]!
.elseif LOOP_OUTPUT_SIZE == 32
            vst1.u8     {q8,q9}, [r0]!
.endif
            b           1b              /* resume outer loop */
            /* Partial tail store case:
             * Different versions of the code need different subsets of the
             * following partial stores.  Here the number of components and the
             * size of the chunk of data produced by each inner loop iteration
             * is tested to figure out whether or not each phrase is relevant.
             */
.if 16 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 16
1:          tst         lr, #16
            beq         1f
            vst1.u8     {q8}, [r0]!
            vmov        q8, q9
.endif
.if 8 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 8
1:          tst         lr, #8
            beq         1f
            vst1.u8     {d16}, [r0]!
            vmov.u8     d16, d17
.endif
.if 4 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 4
1:          tst         lr, #4
            beq         1f
            vst1.u32    {d16[0]}, [r0]!
            vext.u32    d16, d16, d16, #1
.endif
.if 2 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 2
1:          tst         lr, #2
            beq         1f
            vst1.u16    {d16[0]}, [r0]!
            vext.u16    d16, d16, d16, #1
.endif
.if 1 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 1
1:          tst         lr, #1
            beq         1f
            vst1.u8     {d16[0]}, [r0]!
.endif
1:
9:          ldr         sp, [sp,#SP_STORE]
            vpop        {d8-d15}
            pop         {r4,r5,r6,r7,r8,r9,r10,r11,r12,pc}
END(rsdIntrinsicResizeB\comp\()_K)
.endr