/* * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define ENTRY(f) .text; .align 4; .globl f; .type f,#function; f: #define PRIVATE(f) .text; .align 4; .type f,#function; f: #define END(f) .size f, .-f; .set FRACTION_BITS, 7 .set MAX_R, 25 /* A quick way of making a line of code conditional on some other condition. * Use `.set cc, 1` or `.set cc, 0` to enable or disable lines prefixed with * `ifcc`: */ .macro ifcc zzz:vararg .if cc \zzz .endif .endm /* Fetch 16 columns of bytes (regardless of image format), convolve these * vertically, and leave them in the register file. If working near the top or * bottom of an image then clamp the addressing while loading the data in. * * The convolution is fully unrolled for windows up to max_r, with the * outermost edges calculated first. This way it's possible to branch directly * into the relevant part of the code for an arbitrary convolution radius. Two * variants of the loop are produced; one eliminates the clamping code for a * slight speed advantage. * * Where the macro is called with reg=x, the specified register is taken to * contain a pre-calculated pointer into one of the two loops. * * Input: * x1 -- src * x2 -- pitch * x5 -- r * x6 -- rup * x7 -- rdn * x12 -- switch index * v0-v3 -- coefficient table * x13 = -pitch * x15 = top-row in * x19 = bottom-row in * Output: * x1 += 16 * v10,v11 -- 16 convolved columns * Modifies: * x10 = upper row pointer * x11 = lower row pointer * v12-v15 = temporary sums */ .macro fetch, max_r=MAX_R, labelc=1, labelnc=2, reg=x12 /*{{{*/ .ifc \reg,x12 ; .set cc, 1 ; .else ; .set cc, 0 ; .endif ld1 {v15.16b}, [x1], #16 mov x10, x15 uxtl v14.8h, v15.8b // prfm PLDL1KEEP,[x1, #16] // TODO: confirm uxtl2 v15.8h, v15.16b .if \max_r < 16 // approximate ifcc adr \reg, 1f .else ifcc adrp \reg, 1f ifcc add \reg, \reg, #:lo12:1f .endif umull v12.4s, v14.4h, v0.h[0] ifcc sub \reg, \reg, x5, LSL #6 umull2 v13.4s, v14.8h, v0.h[0] mov x11, x19 umull v14.4s, v15.4h, v0.h[0] ifcc add \reg, \reg, x5, LSL #3 umull2 v15.4s, v15.8h, v0.h[0] br \reg .irp rowclamp, 1, 0 .set cc, \rowclamp .align 4 .irp dreg, 4, 3, 2, 1, 0 ; .irp lane, 7, 6, 5, 4, 3, 2, 1, 0 ; .irp doth, .h .set i, \dreg * 8 + \lane .if 0 < i && i <= \max_r ld1 {v10.16b}, [x10], x2 ifcc cmp x6, #i ld1 {v11.16b}, [x11], x13 ifcc csel x10, x15, x10, lo uaddl v16.8h, v10.8b, v11.8b ifcc cmp x7, #i uaddl2 v11.8h, v10.16b, v11.16b ifcc csel x11, x19, x11, lo umlal v12.4s, v16.4h, v\dreg\doth[\lane] umlal2 v13.4s, v16.8h, v\dreg\doth[\lane] // prfm PLDL1KEEP,[x10, #32] // TODO: confirm nop umlal v14.4s, v11.4h, v\dreg\doth[\lane] // prfm PLDL1KEEP,[x11, #32] // TODO: confirm nop umlal2 v15.4s, v11.8h, v\dreg\doth[\lane] .endif .endr ; .endr ; .endr .if \rowclamp == 1 1: \labelc : b 2f .else 2: \labelnc : .endif .endr uqrshrn v10.4h, v12.4s, #16 - FRACTION_BITS add x15, x15, #16 uqrshrn2 v10.8h, v13.4s, #16 - FRACTION_BITS add x19, x19, #16 uqrshrn v11.4h, v14.4s, #16 - FRACTION_BITS uqrshrn2 v11.8h, v15.4s, #16 - FRACTION_BITS .endm /*}}}*/ /* Some portion of the convolution window (as much as will fit, and all of it * for the uchar1 cases) is kept in the register file to avoid unnecessary * memory accesses. This forces the horizontal loops to be unrolled because * there's no indexed addressing into the register file. * * As in the fetch macro, the operations are ordered from outside to inside, so * that jumping into the middle of the block bypasses the unwanted window taps. * * There are several variants of the macro because of the fixed offets of the * taps -- the wider the maximum radius the further the centre tap is from the * most recently fetched data. This means that pre-filling the window requires * more data that won't be used and it means that rotating the window involves * more mov operations. * * When the buffer gets too big the buffer at [x9] is used. * * Input: * v16-v31,v4-v11 -- convoltion window * x9 -- pointer to additional convolution window data * Output: * x9 -- updated buffer pointer (if used) * d31 -- result to be stored * Modifies: * x12 -- temp buffer pointer * v12-v13 -- temporaries for load and vext operations. * v14-v15 -- intermediate sums */ #define TUNED_LIST1 8, 16 .macro hconv1_8/*{{{*/ umull v14.4s, v9.4h, v0.h[0] umull2 v15.4s, v9.8h, v0.h[0] adr x16, 100f ldrsh x12, [x16, x5, LSL #1] add x12, x12, x16 br x12 100: .hword -4 .hword 101f-100b .hword 102f-100b .hword 103f-100b .hword 104f-100b .hword 105f-100b .hword 106f-100b .hword 107f-100b .hword 108f-100b .align 4 108: umlal v14.4s, v8.4h, v1.h[0] umlal2 v15.4s, v8.8h, v1.h[0] umlal v14.4s, v10.4h, v1.h[0] umlal2 v15.4s, v10.8h, v1.h[0] 107: ext v12.16b, v8.16b, v9.16b, #1*2 ext v13.16b, v9.16b, v10.16b, #7*2 umlal v14.4s, v12.4h, v0.h[7] umlal2 v15.4s, v12.8h, v0.h[7] umlal v14.4s, v13.4h, v0.h[7] umlal2 v15.4s, v13.8h, v0.h[7] 106: ext v12.16b, v8.16b, v9.16b, #2*2 ext v13.16b, v9.16b, v10.16b, #6*2 umlal v14.4s, v12.4h, v0.h[6] umlal2 v15.4s, v12.8h, v0.h[6] umlal v14.4s, v13.4h, v0.h[6] umlal2 v15.4s, v13.8h, v0.h[6] 105: ext v12.16b, v8.16b, v9.16b, #3*2 ext v13.16b, v9.16b, v10.16b, #5*2 umlal v14.4s, v12.4h, v0.h[5] umlal2 v15.4s, v12.8h, v0.h[5] umlal v14.4s, v13.4h, v0.h[5] umlal2 v15.4s, v13.8h, v0.h[5] 104: //ext v12.16b, v8.16b, v9.16b, #4*2 //ext v13.16b, v9.16b, v10.16b, #4*2 umlal2 v14.4s, v8.8h, v0.h[4] umlal v15.4s, v9.4h, v0.h[4] umlal2 v14.4s, v9.8h, v0.h[4] umlal v15.4s, v10.4h, v0.h[4] 103: ext v12.16b, v8.16b, v9.16b, #5*2 ext v13.16b, v9.16b, v10.16b, #3*2 umlal v14.4s, v12.4h, v0.h[3] umlal2 v15.4s, v12.8h, v0.h[3] umlal v14.4s, v13.4h, v0.h[3] umlal2 v15.4s, v13.8h, v0.h[3] 102: ext v12.16b, v8.16b, v9.16b, #6*2 ext v13.16b, v9.16b, v10.16b, #2*2 umlal v14.4s, v12.4h, v0.h[2] umlal2 v15.4s, v12.8h, v0.h[2] umlal v14.4s, v13.4h, v0.h[2] umlal2 v15.4s, v13.8h, v0.h[2] 101: ext v12.16b, v8.16b, v9.16b, #7*2 ext v13.16b, v9.16b, v10.16b, #1*2 umlal v14.4s, v12.4h, v0.h[1] umlal2 v15.4s, v12.8h, v0.h[1] umlal v14.4s, v13.4h, v0.h[1] umlal2 v15.4s, v13.8h, v0.h[1] uqrshrn v14.4h, v14.4s, #16 uqrshrn2 v14.8h, v15.4s, #16 uqrshrn v15.8b, v14.8h, #FRACTION_BITS mov v8.16b, v9.16b mov v9.16b, v10.16b mov v10.16b, v11.16b .endm/*}}}*/ .macro hconv1_16/*{{{*/ umull v14.4s, v8.4h, v0.h[0] umull2 v15.4s, v8.8h, v0.h[0] adr x16, 100f ldrsh x12, [x16, x5, LSL #1] add x12, x12, x16 br x12 100: .hword -4 .hword 101f-100b .hword 102f-100b .hword 103f-100b .hword 104f-100b .hword 105f-100b .hword 106f-100b .hword 107f-100b .hword 108f-100b .hword 109f-100b .hword 110f-100b .hword 111f-100b .hword 112f-100b .hword 113f-100b .hword 114f-100b .hword 115f-100b .hword 116f-100b .align 4 116: //ext v12.16b, v6.16b, v7.16b, #0*2 //ext v13.16b, v10.16b, v11.16b, #0*2 umlal v14.4s, v6.4h, v2.h[0] umlal2 v15.4s, v6.8h, v2.h[0] umlal v14.4s, v10.4h, v2.h[0] umlal2 v15.4s, v10.8h, v2.h[0] 115: ext v12.16b, v6.16b, v7.16b, #1*2 ext v13.16b, v9.16b, v10.16b, #7*2 umlal v14.4s, v12.4h, v1.h[7] umlal2 v15.4s, v12.8h, v1.h[7] umlal v14.4s, v13.4h, v1.h[7] umlal2 v15.4s, v13.8h, v1.h[7] 114: ext v12.16b, v6.16b, v7.16b, #2*2 ext v13.16b, v9.16b, v10.16b, #6*2 umlal v14.4s, v12.4h, v1.h[6] umlal2 v15.4s, v12.8h, v1.h[6] umlal v14.4s, v13.4h, v1.h[6] umlal2 v15.4s, v13.8h, v1.h[6] 113: ext v12.16b, v6.16b, v7.16b, #3*2 ext v13.16b, v9.16b, v10.16b, #5*2 umlal v14.4s, v12.4h, v1.h[5] umlal2 v15.4s, v12.8h, v1.h[5] umlal v14.4s, v13.4h, v1.h[5] umlal2 v15.4s, v13.8h, v1.h[5] 112: //ext v12.16b, v6.16b, v7.16b, #4*2 //ext v13.16b, v9.16b, v10.16b, #4*2 umlal2 v14.4s, v6.8h, v1.h[4] umlal v15.4s, v7.4h, v1.h[4] umlal2 v14.4s, v9.8h, v1.h[4] umlal v15.4s, v10.4h, v1.h[4] 111: ext v12.16b, v6.16b, v7.16b, #5*2 ext v13.16b, v9.16b, v10.16b, #3*2 umlal v14.4s, v12.4h, v1.h[3] umlal2 v15.4s, v12.8h, v1.h[3] umlal v14.4s, v13.4h, v1.h[3] umlal2 v15.4s, v13.8h, v1.h[3] 110: ext v12.16b, v6.16b, v7.16b, #6*2 ext v13.16b, v9.16b, v10.16b, #2*2 umlal v14.4s, v12.4h, v1.h[2] umlal2 v15.4s, v12.8h, v1.h[2] umlal v14.4s, v13.4h, v1.h[2] umlal2 v15.4s, v13.8h, v1.h[2] 109: ext v12.16b, v6.16b, v7.16b, #7*2 ext v13.16b, v9.16b, v10.16b, #1*2 umlal v14.4s, v12.4h, v1.h[1] umlal2 v15.4s, v12.8h, v1.h[1] umlal v14.4s, v13.4h, v1.h[1] umlal2 v15.4s, v13.8h, v1.h[1] 108: //ext v12.16b, v7.16b, v8.16b, #0*2 //ext v13.16b, v9.16b, v10.16b, #0*2 umlal v14.4s, v7.4h, v1.h[0] umlal2 v15.4s, v7.8h, v1.h[0] umlal v14.4s, v9.4h, v1.h[0] umlal2 v15.4s, v9.8h, v1.h[0] 107: ext v12.16b, v7.16b, v8.16b, #1*2 ext v13.16b, v8.16b, v9.16b, #7*2 umlal v14.4s, v12.4h, v0.h[7] umlal2 v15.4s, v12.8h, v0.h[7] umlal v14.4s, v13.4h, v0.h[7] umlal2 v15.4s, v13.8h, v0.h[7] 106: ext v12.16b, v7.16b, v8.16b, #2*2 ext v13.16b, v8.16b, v9.16b, #6*2 umlal v14.4s, v12.4h, v0.h[6] umlal2 v15.4s, v12.8h, v0.h[6] umlal v14.4s, v13.4h, v0.h[6] umlal2 v15.4s, v13.8h, v0.h[6] 105: ext v12.16b, v7.16b, v8.16b, #3*2 ext v13.16b, v8.16b, v9.16b, #5*2 umlal v14.4s, v12.4h, v0.h[5] umlal2 v15.4s, v12.8h, v0.h[5] umlal v14.4s, v13.4h, v0.h[5] umlal2 v15.4s, v13.8h, v0.h[5] 104: //ext v12.16b, v7.16b, v8.16b, #4*2 //ext v13.16b, v8.16b, v9.16b, #4*2 umlal2 v14.4s, v7.8h, v0.h[4] umlal v15.4s, v8.4h, v0.h[4] umlal2 v14.4s, v8.8h, v0.h[4] umlal v15.4s, v9.4h, v0.h[4] 103: ext v12.16b, v7.16b, v8.16b, #5*2 ext v13.16b, v8.16b, v9.16b, #3*2 umlal v14.4s, v12.4h, v0.h[3] umlal2 v15.4s, v12.8h, v0.h[3] umlal v14.4s, v13.4h, v0.h[3] umlal2 v15.4s, v13.8h, v0.h[3] 102: ext v12.16b, v7.16b, v8.16b, #6*2 ext v13.16b, v8.16b, v9.16b, #2*2 umlal v14.4s, v12.4h, v0.h[2] umlal2 v15.4s, v12.8h, v0.h[2] umlal v14.4s, v13.4h, v0.h[2] umlal2 v15.4s, v13.8h, v0.h[2] 101: ext v12.16b, v7.16b, v8.16b, #7*2 ext v13.16b, v8.16b, v9.16b, #1*2 umlal v14.4s, v12.4h, v0.h[1] umlal2 v15.4s, v12.8h, v0.h[1] umlal v14.4s, v13.4h, v0.h[1] umlal2 v15.4s, v13.8h, v0.h[1] uqrshrn v14.4h, v14.4s, #16 uqrshrn2 v14.8h, v15.4s, #16 uqrshrn v15.8b, v14.8h, #FRACTION_BITS mov v6.16b, v7.16b mov v7.16b, v8.16b mov v8.16b, v9.16b mov v9.16b, v10.16b mov v10.16b, v11.16b .endm/*}}}*/ .macro hconv1_25/*{{{*/ ext v12.16b, v6.16b, v7.16b, #7*2 umull v14.4s, v12.4h, v0.h[0] umull2 v15.4s, v12.8h, v0.h[0] adr x16, 100f ldrsh x12, [x16, x5, LSL #1] add x12, x12, x16 br x12 100: .hword -4 .hword 101f-100b .hword 102f-100b .hword 103f-100b .hword 104f-100b .hword 105f-100b .hword 106f-100b .hword 107f-100b .hword 108f-100b .hword 109f-100b .hword 110f-100b .hword 111f-100b .hword 112f-100b .hword 113f-100b .hword 114f-100b .hword 115f-100b .hword 116f-100b .hword 117f-100b .hword 118f-100b .hword 119f-100b .hword 120f-100b .hword 121f-100b .hword 122f-100b .hword 123f-100b .hword 124f-100b .hword 125f-100b .align 4 125: ext v12.16b, v31.16b, v4.16b, #6*2 ext v13.16b, v10.16b, v11.16b, #0*2 umlal v14.4s, v12.4h, v3.h[1] umlal2 v15.4s, v12.8h, v3.h[1] umlal v14.4s, v13.4h, v3.h[1] umlal2 v15.4s, v13.8h, v3.h[1] 124: ext v12.16b, v31.16b, v4.16b, #7*2 ext v13.16b, v9.16b, v10.16b, #7*2 umlal v14.4s, v12.4h, v3.h[0] umlal2 v15.4s, v12.8h, v3.h[0] umlal v14.4s, v13.4h, v3.h[0] umlal2 v15.4s, v13.8h, v3.h[0] 123: ext v12.16b, v4.16b, v5.16b, #0*2 ext v13.16b, v9.16b, v10.16b, #6*2 umlal v14.4s, v12.4h, v2.h[7] umlal2 v15.4s, v12.8h, v2.h[7] umlal v14.4s, v13.4h, v2.h[7] umlal2 v15.4s, v13.8h, v2.h[7] 122: ext v12.16b, v4.16b, v5.16b, #1*2 ext v13.16b, v9.16b, v10.16b, #5*2 umlal v14.4s, v12.4h, v2.h[6] umlal2 v15.4s, v12.8h, v2.h[6] umlal v14.4s, v13.4h, v2.h[6] umlal2 v15.4s, v13.8h, v2.h[6] 121: ext v12.16b, v4.16b, v5.16b, #2*2 ext v13.16b, v9.16b, v10.16b, #4*2 umlal v14.4s, v12.4h, v2.h[5] umlal2 v15.4s, v12.8h, v2.h[5] umlal v14.4s, v13.4h, v2.h[5] umlal2 v15.4s, v13.8h, v2.h[5] 120: ext v12.16b, v4.16b, v5.16b, #3*2 ext v13.16b, v9.16b, v10.16b, #3*2 umlal v14.4s, v12.4h, v2.h[4] umlal2 v15.4s, v12.8h, v2.h[4] umlal v14.4s, v13.4h, v2.h[4] umlal2 v15.4s, v13.8h, v2.h[4] 119: ext v12.16b, v4.16b, v5.16b, #4*2 ext v13.16b, v9.16b, v10.16b, #2*2 umlal v14.4s, v12.4h, v2.h[3] umlal2 v15.4s, v12.8h, v2.h[3] umlal v14.4s, v13.4h, v2.h[3] umlal2 v15.4s, v13.8h, v2.h[3] 118: ext v12.16b, v4.16b, v5.16b, #5*2 ext v13.16b, v9.16b, v10.16b, #1*2 umlal v14.4s, v12.4h, v2.h[2] umlal2 v15.4s, v12.8h, v2.h[2] umlal v14.4s, v13.4h, v2.h[2] umlal2 v15.4s, v13.8h, v2.h[2] 117: ext v12.16b, v4.16b, v5.16b, #6*2 ext v13.16b, v9.16b, v10.16b, #0*2 umlal v14.4s, v12.4h, v2.h[1] umlal2 v15.4s, v12.8h, v2.h[1] umlal v14.4s, v13.4h, v2.h[1] umlal2 v15.4s, v13.8h, v2.h[1] 116: ext v12.16b, v4.16b, v5.16b, #7*2 ext v13.16b, v8.16b, v9.16b, #7*2 umlal v14.4s, v12.4h, v2.h[0] umlal2 v15.4s, v12.8h, v2.h[0] umlal v14.4s, v13.4h, v2.h[0] umlal2 v15.4s, v13.8h, v2.h[0] 115: ext v12.16b, v5.16b, v6.16b, #0*2 ext v13.16b, v8.16b, v9.16b, #6*2 umlal v14.4s, v12.4h, v1.h[7] umlal2 v15.4s, v12.8h, v1.h[7] umlal v14.4s, v13.4h, v1.h[7] umlal2 v15.4s, v13.8h, v1.h[7] 114: ext v12.16b, v5.16b, v6.16b, #1*2 ext v13.16b, v8.16b, v9.16b, #5*2 umlal v14.4s, v12.4h, v1.h[6] umlal2 v15.4s, v12.8h, v1.h[6] umlal v14.4s, v13.4h, v1.h[6] umlal2 v15.4s, v13.8h, v1.h[6] 113: ext v12.16b, v5.16b, v6.16b, #2*2 ext v13.16b, v8.16b, v9.16b, #4*2 umlal v14.4s, v12.4h, v1.h[5] umlal2 v15.4s, v12.8h, v1.h[5] umlal v14.4s, v13.4h, v1.h[5] umlal2 v15.4s, v13.8h, v1.h[5] 112: ext v12.16b, v5.16b, v6.16b, #3*2 ext v13.16b, v8.16b, v9.16b, #3*2 umlal v14.4s, v12.4h, v1.h[4] umlal2 v15.4s, v12.8h, v1.h[4] umlal v14.4s, v13.4h, v1.h[4] umlal2 v15.4s, v13.8h, v1.h[4] 111: ext v12.16b, v5.16b, v6.16b, #4*2 ext v13.16b, v8.16b, v9.16b, #2*2 umlal v14.4s, v12.4h, v1.h[3] umlal2 v15.4s, v12.8h, v1.h[3] umlal v14.4s, v13.4h, v1.h[3] umlal2 v15.4s, v13.8h, v1.h[3] 110: ext v12.16b, v5.16b, v6.16b, #5*2 ext v13.16b, v8.16b, v9.16b, #1*2 umlal v14.4s, v12.4h, v1.h[2] umlal2 v15.4s, v12.8h, v1.h[2] umlal v14.4s, v13.4h, v1.h[2] umlal2 v15.4s, v13.8h, v1.h[2] 109: ext v12.16b, v5.16b, v6.16b, #6*2 ext v13.16b, v8.16b, v9.16b, #0*2 umlal v14.4s, v12.4h, v1.h[1] umlal2 v15.4s, v12.8h, v1.h[1] umlal v14.4s, v13.4h, v1.h[1] umlal2 v15.4s, v13.8h, v1.h[1] 108: ext v12.16b, v5.16b, v6.16b, #7*2 ext v13.16b, v7.16b, v8.16b, #7*2 umlal v14.4s, v12.4h, v1.h[0] umlal2 v15.4s, v12.8h, v1.h[0] umlal v14.4s, v13.4h, v1.h[0] umlal2 v15.4s, v13.8h, v1.h[0] 107: ext v12.16b, v6.16b, v7.16b, #0*2 ext v13.16b, v7.16b, v8.16b, #6*2 umlal v14.4s, v12.4h, v0.h[7] umlal2 v15.4s, v12.8h, v0.h[7] umlal v14.4s, v13.4h, v0.h[7] umlal2 v15.4s, v13.8h, v0.h[7] 106: ext v12.16b, v6.16b, v7.16b, #1*2 ext v13.16b, v7.16b, v8.16b, #5*2 umlal v14.4s, v12.4h, v0.h[6] umlal2 v15.4s, v12.8h, v0.h[6] umlal v14.4s, v13.4h, v0.h[6] umlal2 v15.4s, v13.8h, v0.h[6] 105: ext v12.16b, v6.16b, v7.16b, #2*2 ext v13.16b, v7.16b, v8.16b, #4*2 umlal v14.4s, v12.4h, v0.h[5] umlal2 v15.4s, v12.8h, v0.h[5] umlal v14.4s, v13.4h, v0.h[5] umlal2 v15.4s, v13.8h, v0.h[5] 104: ext v12.16b, v6.16b, v7.16b, #3*2 ext v13.16b, v7.16b, v8.16b, #3*2 umlal v14.4s, v12.4h, v0.h[4] umlal2 v15.4s, v12.8h, v0.h[4] umlal v14.4s, v13.4h, v0.h[4] umlal2 v15.4s, v13.8h, v0.h[4] 103: ext v12.16b, v6.16b, v7.16b, #4*2 ext v13.16b, v7.16b, v8.16b, #2*2 umlal v14.4s, v12.4h, v0.h[3] umlal2 v15.4s, v12.8h, v0.h[3] umlal v14.4s, v13.4h, v0.h[3] umlal2 v15.4s, v13.8h, v0.h[3] 102: ext v12.16b, v6.16b, v7.16b, #5*2 ext v13.16b, v7.16b, v8.16b, #1*2 umlal v14.4s, v12.4h, v0.h[2] umlal2 v15.4s, v12.8h, v0.h[2] umlal v14.4s, v13.4h, v0.h[2] umlal2 v15.4s, v13.8h, v0.h[2] 101: ext v12.16b, v6.16b, v7.16b, #6*2 ext v13.16b, v7.16b, v8.16b, #0*2 umlal v14.4s, v12.4h, v0.h[1] umlal2 v15.4s, v12.8h, v0.h[1] umlal v14.4s, v13.4h, v0.h[1] umlal2 v15.4s, v13.8h, v0.h[1] uqrshrn v14.4h, v14.4s, #16 uqrshrn2 v14.8h, v15.4s, #16 uqrshrn v15.8b, v14.8h, #FRACTION_BITS mov v31.16b, v4.16b mov v4.16b, v5.16b mov v5.16b, v6.16b mov v6.16b, v7.16b mov v7.16b, v8.16b mov v8.16b, v9.16b mov v9.16b, v10.16b mov v10.16b, v11.16b .endm/*}}}*/ #define TUNED_LIST4 6, 12, 20 .macro hconv4_6/*{{{*/ umull v14.4s, v7.4h, v0.h[0] umull2 v15.4s, v7.8h, v0.h[0] adr x16, 100f ldrsh x12, [x16, x5, LSL #1] add x12, x12, x16 br x12 100: .hword -4 .hword 101f-100b .hword 102f-100b .hword 103f-100b .hword 104f-100b .hword 105f-100b .hword 106f-100b .align 4 106: umlal v14.4s, v4.4h, v0.h[6] umlal2 v15.4s, v4.8h, v0.h[6] umlal v14.4s, v10.4h, v0.h[6] umlal2 v15.4s, v10.8h, v0.h[6] 105: umlal2 v14.4s, v4.8h, v0.h[5] umlal v15.4s, v5.4h, v0.h[5] umlal2 v14.4s, v9.8h, v0.h[5] umlal v15.4s, v10.4h, v0.h[5] 104: umlal v14.4s, v5.4h, v0.h[4] umlal2 v15.4s, v5.8h, v0.h[4] umlal v14.4s, v9.4h, v0.h[4] umlal2 v15.4s, v9.8h, v0.h[4] 103: umlal2 v14.4s, v5.8h, v0.h[3] umlal v15.4s, v6.4h, v0.h[3] umlal2 v14.4s, v8.8h, v0.h[3] umlal v15.4s, v9.4h, v0.h[3] 102: umlal v14.4s, v6.4h, v0.h[2] umlal2 v15.4s, v6.8h, v0.h[2] umlal v14.4s, v8.4h, v0.h[2] umlal2 v15.4s, v8.8h, v0.h[2] 101: umlal2 v14.4s, v6.8h, v0.h[1] umlal v15.4s, v7.4h, v0.h[1] umlal2 v14.4s, v7.8h, v0.h[1] umlal v15.4s, v8.4h, v0.h[1] uqrshrn v14.4h, v14.4s, #16 uqrshrn2 v14.8h, v15.4s, #16 uqrshrn v15.8b, v14.8h, #FRACTION_BITS mov v4.16b, v5.16b mov v5.16b, v6.16b mov v6.16b, v7.16b mov v7.16b, v8.16b mov v8.16b, v9.16b mov v9.16b, v10.16b mov v10.16b, v11.16b .endm/*}}}*/ .macro hconv4_12/*{{{*/ umull v14.4s, v4.4h, v0.h[0] umull2 v15.4s, v4.8h, v0.h[0] adr x16, 100f ldrsh x12, [x16, x5, LSL #1] add x12, x12, x16 br x12 100: .hword -4 .hword 101f-100b .hword 102f-100b .hword 103f-100b .hword 104f-100b .hword 105f-100b .hword 106f-100b .hword 107f-100b .hword 108f-100b .hword 109f-100b .hword 110f-100b .hword 111f-100b .hword 112f-100b .align 4 112: umlal v14.4s, v26.4h, v1.h[4] umlal2 v15.4s, v26.8h, v1.h[4] umlal v14.4s, v10.4h, v1.h[4] umlal2 v15.4s, v10.8h, v1.h[4] 111: umlal2 v14.4s, v26.8h, v1.h[3] umlal v15.4s, v27.4h, v1.h[3] umlal2 v14.4s, v9.8h, v1.h[3] umlal v15.4s, v10.4h, v1.h[3] 110: umlal v14.4s, v27.4h, v1.h[2] umlal2 v15.4s, v27.8h, v1.h[2] umlal v14.4s, v9.4h, v1.h[2] umlal2 v15.4s, v9.8h, v1.h[2] 109: umlal2 v14.4s, v27.8h, v1.h[1] umlal v15.4s, v28.4h, v1.h[1] umlal2 v14.4s, v8.8h, v1.h[1] umlal v15.4s, v9.4h, v1.h[1] 108: umlal v14.4s, v28.4h, v1.h[0] umlal2 v15.4s, v28.8h, v1.h[0] umlal v14.4s, v8.4h, v1.h[0] umlal2 v15.4s, v8.8h, v1.h[0] 107: umlal2 v14.4s, v28.8h, v0.h[7] umlal v15.4s, v29.4h, v0.h[7] umlal2 v14.4s, v7.8h, v0.h[7] umlal v15.4s, v8.4h, v0.h[7] 106: umlal v14.4s, v29.4h, v0.h[6] umlal2 v15.4s, v29.8h, v0.h[6] umlal v14.4s, v7.4h, v0.h[6] umlal2 v15.4s, v7.8h, v0.h[6] 105: umlal2 v14.4s, v29.8h, v0.h[5] umlal v15.4s, v30.4h, v0.h[5] umlal2 v14.4s, v6.8h, v0.h[5] umlal v15.4s, v7.4h, v0.h[5] 104: umlal v14.4s, v30.4h, v0.h[4] umlal2 v15.4s, v30.8h, v0.h[4] umlal v14.4s, v6.4h, v0.h[4] umlal2 v15.4s, v6.8h, v0.h[4] 103: umlal2 v14.4s, v30.8h, v0.h[3] umlal v15.4s, v31.4h, v0.h[3] umlal2 v14.4s, v5.8h, v0.h[3] umlal v15.4s, v6.4h, v0.h[3] 102: umlal v14.4s, v31.4h, v0.h[2] umlal2 v15.4s, v31.8h, v0.h[2] umlal v14.4s, v5.4h, v0.h[2] umlal2 v15.4s, v5.8h, v0.h[2] 101: umlal2 v14.4s, v31.8h, v0.h[1] umlal v15.4s, v4.4h, v0.h[1] umlal2 v14.4s, v4.8h, v0.h[1] umlal v15.4s, v5.4h, v0.h[1] uqrshrn v14.4h, v14.4s, #16 uqrshrn2 v14.8h, v15.4s, #16 uqrshrn v15.8b, v14.8h, #FRACTION_BITS mov v26.16b, v27.16b mov v27.16b, v28.16b mov v28.16b, v29.16b mov v29.16b, v30.16b mov v30.16b, v31.16b mov v31.16b, v4.16b mov v4.16b, v5.16b mov v5.16b, v6.16b mov v6.16b, v7.16b mov v7.16b, v8.16b mov v8.16b, v9.16b mov v9.16b, v10.16b mov v10.16b, v11.16b .endm/*}}}*/ .macro hconv4_20/*{{{*/ umull v14.4s, v28.4h, v0.h[0] umull2 v15.4s, v28.8h, v0.h[0] adr x16, 100f ldrsh x12, [x16, x5, LSL #1] add x12, x12, x16 br x12 100: .hword -4 .hword 101f-100b .hword 102f-100b .hword 103f-100b .hword 104f-100b .hword 105f-100b .hword 106f-100b .hword 107f-100b .hword 108f-100b .hword 109f-100b .hword 110f-100b .hword 111f-100b .hword 112f-100b .hword 113f-100b .hword 114f-100b .hword 115f-100b .hword 116f-100b .hword 117f-100b .hword 118f-100b .hword 119f-100b .hword 120f-100b .align 4 120: umlal v14.4s, v18.4h, v2.h[4] umlal2 v15.4s, v18.8h, v2.h[4] umlal v14.4s, v10.4h, v2.h[4] umlal2 v15.4s, v10.8h, v2.h[4] 119: umlal2 v14.4s, v18.8h, v2.h[3] umlal v15.4s, v19.4h, v2.h[3] umlal2 v14.4s, v9.8h, v2.h[3] umlal v15.4s, v10.4h, v2.h[3] 118: umlal v14.4s, v19.4h, v2.h[2] umlal2 v15.4s, v19.8h, v2.h[2] umlal v14.4s, v9.4h, v2.h[2] umlal2 v15.4s, v9.8h, v2.h[2] 117: umlal2 v14.4s, v19.8h, v2.h[1] umlal v15.4s, v20.4h, v2.h[1] umlal2 v14.4s, v8.8h, v2.h[1] umlal v15.4s, v9.4h, v2.h[1] 116: umlal v14.4s, v20.4h, v2.h[0] umlal2 v15.4s, v20.8h, v2.h[0] umlal v14.4s, v8.4h, v2.h[0] umlal2 v15.4s, v8.8h, v2.h[0] 115: umlal2 v14.4s, v20.8h, v1.h[7] umlal v15.4s, v21.4h, v1.h[7] umlal2 v14.4s, v7.8h, v1.h[7] umlal v15.4s, v8.4h, v1.h[7] 114: umlal v14.4s, v21.4h, v1.h[6] umlal2 v15.4s, v21.8h, v1.h[6] umlal v14.4s, v7.4h, v1.h[6] umlal2 v15.4s, v7.8h, v1.h[6] 113: umlal2 v14.4s, v21.8h, v1.h[5] umlal v15.4s, v22.4h, v1.h[5] umlal2 v14.4s, v6.8h, v1.h[5] umlal v15.4s, v7.4h, v1.h[5] 112: umlal v14.4s, v22.4h, v1.h[4] umlal2 v15.4s, v22.8h, v1.h[4] umlal v14.4s, v6.4h, v1.h[4] umlal2 v15.4s, v6.8h, v1.h[4] 111: umlal2 v14.4s, v22.8h, v1.h[3] umlal v15.4s, v23.4h, v1.h[3] umlal2 v14.4s, v5.8h, v1.h[3] umlal v15.4s, v6.4h, v1.h[3] 110: umlal v14.4s, v23.4h, v1.h[2] umlal2 v15.4s, v23.8h, v1.h[2] umlal v14.4s, v5.4h, v1.h[2] umlal2 v15.4s, v5.8h, v1.h[2] 109: umlal2 v14.4s, v23.8h, v1.h[1] umlal v15.4s, v24.4h, v1.h[1] umlal2 v14.4s, v4.8h, v1.h[1] umlal v15.4s, v5.4h, v1.h[1] 108: umlal v14.4s, v24.4h, v1.h[0] umlal2 v15.4s, v24.8h, v1.h[0] umlal v14.4s, v4.4h, v1.h[0] umlal2 v15.4s, v4.8h, v1.h[0] 107: umlal2 v14.4s, v24.8h, v0.h[7] umlal v15.4s, v25.4h, v0.h[7] umlal2 v14.4s, v31.8h, v0.h[7] umlal v15.4s, v4.4h, v0.h[7] 106: umlal v14.4s, v25.4h, v0.h[6] umlal2 v15.4s, v25.8h, v0.h[6] umlal v14.4s, v31.4h, v0.h[6] umlal2 v15.4s, v31.8h, v0.h[6] 105: umlal2 v14.4s, v25.8h, v0.h[5] umlal v15.4s, v26.4h, v0.h[5] umlal2 v14.4s, v30.8h, v0.h[5] umlal v15.4s, v31.4h, v0.h[5] 104: umlal v14.4s, v26.4h, v0.h[4] umlal2 v15.4s, v26.8h, v0.h[4] umlal v14.4s, v30.4h, v0.h[4] umlal2 v15.4s, v30.8h, v0.h[4] 103: umlal2 v14.4s, v26.8h, v0.h[3] umlal v15.4s, v27.4h, v0.h[3] umlal2 v14.4s, v29.8h, v0.h[3] umlal v15.4s, v30.4h, v0.h[3] 102: umlal v14.4s, v27.4h, v0.h[2] umlal2 v15.4s, v27.8h, v0.h[2] umlal v14.4s, v29.4h, v0.h[2] umlal2 v15.4s, v29.8h, v0.h[2] 101: umlal2 v14.4s, v27.8h, v0.h[1] umlal v15.4s, v28.4h, v0.h[1] umlal2 v14.4s, v28.8h, v0.h[1] umlal v15.4s, v29.4h, v0.h[1] uqrshrn v14.4h, v14.4s, #16 uqrshrn2 v14.8h, v15.4s, #16 uqrshrn v15.8b, v14.8h, #FRACTION_BITS mov v18.16b, v19.16b mov v19.16b, v20.16b mov v20.16b, v21.16b mov v21.16b, v22.16b mov v22.16b, v23.16b mov v23.16b, v24.16b mov v24.16b, v25.16b mov v25.16b, v26.16b mov v26.16b, v27.16b mov v27.16b, v28.16b mov v28.16b, v29.16b mov v29.16b, v30.16b mov v30.16b, v31.16b mov v31.16b, v4.16b mov v4.16b, v5.16b mov v5.16b, v6.16b mov v6.16b, v7.16b mov v7.16b, v8.16b mov v8.16b, v9.16b mov v9.16b, v10.16b mov v10.16b, v11.16b .endm/*}}}*/ .macro hconv4_25/*{{{*/ umull2 v14.4s, v25.8h, v0.h[0] umull v15.4s, v26.4h, v0.h[0] adr x16, 100f ldrsh x12, [x16, x5, LSL #1] add x12, x12, x16 br x12 100: .hword -4 .hword 101f-100b .hword 102f-100b .hword 103f-100b .hword 104f-100b .hword 105f-100b .hword 106f-100b .hword 107f-100b .hword 108f-100b .hword 109f-100b .hword 110f-100b .hword 111f-100b .hword 112f-100b .hword 113f-100b .hword 114f-100b .hword 115f-100b .hword 116f-100b .hword 117f-100b .hword 118f-100b .hword 119f-100b .hword 120f-100b .hword 121f-100b .hword 122f-100b .hword 123f-100b .hword 124f-100b .hword 125f-100b .align 4 125: ld1 {v12.8h}, [x9] umlal v14.4s, v12.4h, v3.h[1] umlal2 v15.4s, v12.8h, v3.h[1] umlal v14.4s, v10.4h, v3.h[1] umlal2 v15.4s, v10.8h, v3.h[1] 124: add x12, x9, #0x08 bic x12, x12, #0x40 ld1 {v12.4h}, [x12], #8 bic x12, x12, #0x40 ld1 {v13.4h}, [x12] umlal v14.4s, v12.4h, v3.h[0] umlal v15.4s, v13.4h, v3.h[0] umlal2 v14.4s, v9.8h, v3.h[0] umlal v15.4s, v10.4h, v3.h[0] 123: add x12, x9, #0x10 bic x12, x12, #0x40 ld1 {v12.8h}, [x12] umlal v14.4s, v12.4h, v2.h[7] umlal2 v15.4s, v12.8h, v2.h[7] umlal v14.4s, v9.4h, v2.h[7] umlal2 v15.4s, v9.8h, v2.h[7] 122: add x12, x9, #0x18 bic x12, x12, #0x40 ld1 {v12.4h}, [x12], #8 bic x12, x12, #0x40 ld1 {v13.4h}, [x12] umlal v14.4s, v12.4h, v2.h[6] umlal v15.4s, v13.4h, v2.h[6] umlal2 v14.4s, v8.8h, v2.h[6] umlal v15.4s, v9.4h, v2.h[6] 121: add x12, x9, #0x20 bic x12, x12, #0x40 ld1 {v12.8h}, [x12] umlal v14.4s, v12.4h, v2.h[5] umlal2 v15.4s, v12.8h, v2.h[5] umlal v14.4s, v8.4h, v2.h[5] umlal2 v15.4s, v8.8h, v2.h[5] 120: add x12, x9, #0x28 bic x12, x12, #0x40 ld1 {v12.4h}, [x12], #8 bic x12, x12, #0x40 ld1 {v13.4h}, [x12] umlal v14.4s, v12.4h, v2.h[4] umlal v15.4s, v13.4h, v2.h[4] umlal2 v14.4s, v7.8h, v2.h[4] umlal v15.4s, v8.4h, v2.h[4] 119: add x12, x9, #0x30 bic x12, x12, #0x40 ld1 {v12.8h}, [x12] umlal v14.4s, v12.4h, v2.h[3] umlal2 v15.4s, v12.8h, v2.h[3] umlal v14.4s, v7.4h, v2.h[3] umlal2 v15.4s, v7.8h, v2.h[3] 118: add x12, x9, #0x38 bic x12, x12, #0x40 ld1 {v12.4h}, [x12] umlal v14.4s, v12.4h, v2.h[2] umlal v15.4s, v17.4h, v2.h[2] umlal2 v14.4s, v6.8h, v2.h[2] umlal v15.4s, v7.4h, v2.h[2] 117: umlal v14.4s, v17.4h, v2.h[1] umlal2 v15.4s, v17.8h, v2.h[1] umlal v14.4s, v6.4h, v2.h[1] umlal2 v15.4s, v6.8h, v2.h[1] 116: umlal2 v14.4s, v17.8h, v2.h[0] umlal v15.4s, v18.4h, v2.h[0] umlal2 v14.4s, v5.8h, v2.h[0] umlal v15.4s, v6.4h, v2.h[0] 115: umlal v14.4s, v18.4h, v1.h[7] umlal2 v15.4s, v18.8h, v1.h[7] umlal v14.4s, v5.4h, v1.h[7] umlal2 v15.4s, v5.8h, v1.h[7] 114: umlal2 v14.4s, v18.8h, v1.h[6] umlal v15.4s, v19.4h, v1.h[6] umlal2 v14.4s, v4.8h, v1.h[6] umlal v15.4s, v5.4h, v1.h[6] 113: umlal v14.4s, v19.4h, v1.h[5] umlal2 v15.4s, v19.8h, v1.h[5] umlal v14.4s, v4.4h, v1.h[5] umlal2 v15.4s, v4.8h, v1.h[5] 112: umlal2 v14.4s, v19.8h, v1.h[4] umlal v15.4s, v20.4h, v1.h[4] umlal2 v14.4s, v31.8h, v1.h[4] umlal v15.4s, v4.4h, v1.h[4] 111: umlal v14.4s, v20.4h, v1.h[3] umlal2 v15.4s, v20.8h, v1.h[3] umlal v14.4s, v31.4h, v1.h[3] umlal2 v15.4s, v31.8h, v1.h[3] 110: umlal2 v14.4s, v20.8h, v1.h[2] umlal v15.4s, v21.4h, v1.h[2] umlal2 v14.4s, v30.8h, v1.h[2] umlal v15.4s, v31.4h, v1.h[2] 109: umlal v14.4s, v21.4h, v1.h[1] umlal2 v15.4s, v21.8h, v1.h[1] umlal v14.4s, v30.4h, v1.h[1] umlal2 v15.4s, v30.8h, v1.h[1] 108: umlal2 v14.4s, v21.8h, v1.h[0] umlal v15.4s, v22.4h, v1.h[0] umlal2 v14.4s, v29.8h, v1.h[0] umlal v15.4s, v30.4h, v1.h[0] 107: umlal v14.4s, v22.4h, v0.h[7] umlal2 v15.4s, v22.8h, v0.h[7] umlal v14.4s, v29.4h, v0.h[7] umlal2 v15.4s, v29.8h, v0.h[7] 106: umlal2 v14.4s, v22.8h, v0.h[6] umlal v15.4s, v23.4h, v0.h[6] umlal2 v14.4s, v28.8h, v0.h[6] umlal v15.4s, v29.4h, v0.h[6] 105: umlal v14.4s, v23.4h, v0.h[5] umlal2 v15.4s, v23.8h, v0.h[5] umlal v14.4s, v28.4h, v0.h[5] umlal2 v15.4s, v28.8h, v0.h[5] 104: umlal2 v14.4s, v23.8h, v0.h[4] umlal v15.4s, v24.4h, v0.h[4] umlal2 v14.4s, v27.8h, v0.h[4] umlal v15.4s, v28.4h, v0.h[4] 103: umlal v14.4s, v24.4h, v0.h[3] umlal2 v15.4s, v24.8h, v0.h[3] umlal v14.4s, v27.4h, v0.h[3] umlal2 v15.4s, v27.8h, v0.h[3] 102: umlal2 v14.4s, v24.8h, v0.h[2] umlal v15.4s, v25.4h, v0.h[2] umlal2 v14.4s, v26.8h, v0.h[2] umlal v15.4s, v27.4h, v0.h[2] 101: umlal v14.4s, v25.4h, v0.h[1] umlal2 v15.4s, v25.8h, v0.h[1] umlal v14.4s, v26.4h, v0.h[1] umlal2 v15.4s, v26.8h, v0.h[1] uqrshrn v14.4h, v14.4s, #16 uqrshrn2 v14.8h, v15.4s, #16 uqrshrn v15.8b, v14.8h, #FRACTION_BITS st1 {v17.16b}, [x9], #16 bic x9, x9, #0x40 mov v17.16b, v18.16b mov v18.16b, v19.16b mov v19.16b, v20.16b mov v20.16b, v21.16b mov v21.16b, v22.16b mov v22.16b, v23.16b mov v23.16b, v24.16b mov v24.16b, v25.16b mov v25.16b, v26.16b mov v26.16b, v27.16b mov v27.16b, v28.16b mov v28.16b, v29.16b mov v29.16b, v30.16b mov v30.16b, v31.16b mov v31.16b, v4.16b mov v4.16b, v5.16b mov v5.16b, v6.16b mov v6.16b, v7.16b mov v7.16b, v8.16b mov v8.16b, v9.16b mov v9.16b, v10.16b mov v10.16b, v11.16b .endm/*}}}*/ /* Dedicated function wrapper for the fetch macro, for the cases where * performance isn't that important, to keep code size down. */ PRIVATE(fetch_generic_asm) stp x10, x11, [sp, #-16]! fetch ldp x10, x11, [sp], #16 ret END(fetch_generic_asm) /* Given values in v10 and v11, and an index in x11, sweep the (x11&15)th value * across to fill the rest of the register pair. Used for filling the right * hand edge of the window when starting too close to the right hand edge of * the image. * Also returns a dup-ed copy of the last element in v12 for the tail-fill * case (this happens incidentally in common path, but must be done * deliberately in the fast-out path). */ PRIVATE(prefetch_clampright1) ands x12, x11, #15 beq 1f sub x12, x12, #1 sub sp, sp, #64 st1 {v10.8h,v11.8h}, [sp] add x12, sp, x12, LSL #1 ld1r {v12.8h}, [x12] st1 {v12.8h}, [x12], #16 st1 {v12.8h}, [x12] ld1 {v10.8h,v11.8h}, [sp] add sp, sp, #64 ret 1: dup v12.8h, v11.h[7] ret END(prefetch_clampright1) PRIVATE(prefetch_clampright4) ands x12, x11, #15 beq 1f sub x12, x12, #4 sub sp, sp, #64 st1 {v10.8h,v11.8h}, [sp] add x12, sp, x12, LSL #1 ld1r {v12.2d}, [x12] st1 {v12.8h}, [x12], #16 st1 {v12.8h}, [x12] ld1 {v10.8h,v11.8h}, [sp] add sp, sp, #64 ret 1: dup v12.2d, v11.d[1] ret END(prefetch_clampright4) /* Helpers for prefetch, below. */ .macro prefetch_out qa, qb, store, qsa, qsb, qsb_hi .if \store == 2 .ifc \qsa,\qsb st1 {\qsa}, [x9], #16 st1 {\qsb}, [x9], #16 .else st1 {\qsa,\qsb}, [x9], #32 .endif .elseif \store == 1 bic x9, x9, #0x40 st1 {\qsa}, [x9], #16 mov \qb, \qsb .elseif \store == 0 mov \qa, \qsa mov \qb, \qsb .endif .endm .macro prefetch_one qa, qb, rem, c, store=0, step=1 .set i, (need - 16) - \rem .if i >= 0 1: cmp x10, #i+16 blo 2f prefetch_out \qa, \qb, \store, v9.16b, v9.16b, v9.d[1] b 1f 2: cmp x11, #i+16 bls 3f prefetch_out \qa, \qb, \store, v10.16b, v11.16b, v11.d[1] bl fetch_generic_asm b 2f 3: bl prefetch_clampright\step prefetch_out \qa, \qb, \store, v10.16b, v11.16b, v11.d[1] 4: b 4f+4 //v12 contains pad word from prefetch_clampright call prefetch_out \qa, \qb, \store, v12.16b, v12.16b, v12.d[1] .if \rem > 0 b 4f+4 .else 1: 2: 3: 4: nop .endif .endif .endm /* Fill the convolution window with context data. The aim here is to load * exactly rlf + rrt columns, and in the main loop to read as many columns as * will be written. This is complicated by the need to handle cases when the * input starts very close to the left or right (or both) edges of the image, * and where these do not fall on 16-byte boundaries. * * Input: * x1 -- src * x2 -- pitch * x3 -- count * x4 -- inlen * x5 -- r * x6 -- rup * x7 -- rdn * x8 -- rlf * x9 -- buffer (if needed) * x13 = -pitch * x15 = top-row in * x19 = bottom-row in * Output: * x1 += rlf + min(count, rrt) * Modifies: * x10 -- fill start index in the window * x11 -- fill stop index in the window * x12 -- scratch */ .macro prefetch step=1, max_r=25 .set need, ((\max_r + \max_r) * \step + 15) & ~15 .if \step == 1 mov x10, #need - (\max_r * \step) sub x10, x10, x8 .else mov x10, #need - (\max_r * \step) sub x10, x10, x8, LSL #2 .endif add x11, x10, x4 subs x11, x11, #need csel x11, xzr, x11, hi add x11, x11, #need bl fetch_generic_asm .if \step == 1 dup v9.8h, v10.h[0] .else dup v9.2d, v10.d[0] .endif ands x12, x10, #15 beq 2f sub sp, sp, #32 st1 {v10.8h,v11.8h}, [sp] sub x12, sp, x12, LSL #1 sub sp, sp, #16 st1 {v9.8h}, [sp] sub sp, sp, #16 st1 {v9.8h}, [sp] ld1 {v10.8h,v11.8h}, [x12] add sp, sp, #64 sub x1, x1, x10 sub x15, x15, x10 sub x19, x19, x10 bic x10, x10, #15 add x1, x1, x10 add x15, x15, x10 add x19, x19, x10 2: .if \step > 1 /* it's only in the uchar2 and uchar4 cases where the register file * is insufficient (given MAX_R <= 25). */ prefetch_one xx, xx, 192, c=\max_r, step=\step, store=2 prefetch_one xx, xx, 176, c=\max_r, step=\step, store=2 prefetch_one xx, v17.16b, 160, c=\max_r, step=\step, store=1 prefetch_one v18.16b, v19.16b, 144, c=\max_r, step=\step, store=0 prefetch_one v20.16b, v21.16b, 128, c=\max_r, step=\step, store=0 prefetch_one v22.16b, v23.16b, 112, c=\max_r, step=\step, store=0 prefetch_one v24.16b, v25.16b, 96, c=\max_r, step=\step, store=0 prefetch_one v26.16b, v27.16b, 80, c=\max_r, step=\step, store=0 prefetch_one v28.16b, v29.16b, 64, c=\max_r, step=\step, store=0 .endif prefetch_one v30.16b, v31.16b, 48, c=\max_r, step=\step, store=0 prefetch_one v4.16b, v5.16b, 32, c=\max_r, step=\step, store=0 prefetch_one v6.16b, v7.16b, 16, c=\max_r, step=\step, store=0 prefetch_one v8.16b, v9.16b, 0, c=\max_r, step=\step, store=0 .if \step == 1 add x10, x8, #\max_r * \step .else lsl x10, x8, #2 add x10, x10, #\max_r * \step .endif subs x4, x4, x10 csel x4, xzr, x4, lo .endm /* The main loop. * * Input: * x0 = dst * x1 = src * x2 = pitch * x3 = count * x4 = inlen * x5 = r * x6 = rup * x7 = rdn * x9 = buffer * x13 = -pitch * x15 = top-row in * x19 = bottom-row in * Modifies * x8 = fetch code pointer */ .macro mainloop core, step=1, max_r=25, labelc="", labelnc="" adrp x8, \labelnc add x8, x8, #:lo12:\labelnc sub x8, x8, x5, LSL #5 sub x8, x8, x5, LSL #3 cmp x5, x6 ccmp x5, x7, #0, eq beq 5f /* if (r != rup || r != rdn) then the address-clamping table should * be used rather than the short-cut version. */ adrp x8, \labelc add x8, x8, #:lo12:\labelc sub x8, x8, x5, LSL #6 add x8, x8, x5, LSL #3 b 5f .align 4 3: fetch max_r=\max_r, labelc=\labelc, labelnc=\labelnc, reg=x8 /* For each call to fetch two are made to \core. It would be * preferable to have twice the work done in \core. */ \core st1 {v15.8b}, [x0], #8 \core st1 {v15.8b}, [x0], #8 sub x3, x3, #16 5: subs x4, x4, #16 bhs 3b adds x4, x4, #16 bne 1f .if \step==1 dup v10.8h, v9.h[7] dup v11.8h, v9.h[7] .else dup v10.2d, v9.d[1] dup v11.2d, v9.d[1] .endif b 4f 1: sub x1, x1, #16 sub x15, x15, #16 sub x19, x19, #16 add x1, x1, x4 add x15, x15, x4 add x19, x19, x4 bl fetch_generic_asm .if \step==1 dup v12.8h, v11.h[7] .else dup v12.2d, v11.d[1] .endif sub x4, xzr, x4 tbz x4, #3, 1f mov v10.16b, v11.16b mov v11.16b, v12.16b 1: tbz x4, #2, 1f ext v10.16b, v10.16b, v11.16b, #4*2 ext v11.16b, v11.16b, v12.16b, #4*2 1: tbz x4, #1, 1f ext v10.16b, v10.16b, v11.16b, #2*2 ext v11.16b, v11.16b, v12.16b, #2*2 1: tbz x4, #0, 4f ext v10.16b, v10.16b, v11.16b, #1*2 ext v11.16b, v11.16b, v12.16b, #1*2 4: cbz x3, 5f 3: \core .if \step==1 dup v11.8h, v11.h[7] .else dup v11.2d, v11.d[1] .endif subs x3, x3, #8 blo 4f st1 {v15.8b}, [x0], #8 beq 5f b 3b 4: tbz x3, #2, 1f st1 {v15.s}[0], [x0], #4 ext v15.8b, v15.8b, v15.8b, #4 1: tbz x3, #1, 1f st1 {v15.h}[0], [x0], #2 ext v15.8b, v15.8b, v15.8b, #2 1: tbz x3, #0, 5f st1 {v15.b}[0], [x0], #1 ext v15.8b, v15.8b, v15.8b, #1 5: nop .endm .irep r, TUNED_LIST1, 25 PRIVATE(convolve1_\r) stp x29,x30, [sp, #-16]! prefetch step=1, max_r=\r mainloop core=hconv1_\r, step=1, max_r=\r, labelc=.Lcnv1_\r, labelnc=.Lcnvnc1_\r ldp x29,x30, [sp], #16 ret END(convolve1_\r) .endr .irep r, TUNED_LIST4, 25 PRIVATE(convolve4_\r) sub x12, sp, #0x040 bic x9, x12, #0x07f mov sp, x9 stp x12,x30, [sp, #-16]! /* x9 now points to a buffer on the stack whose address has the low * 7 bits clear. This allows easy address calculation in the * wrap-around cases. */ prefetch step=4, max_r=\r mainloop core=hconv4_\r, step=4, max_r=\r, labelc=.Lcnv4_\r, labelnc=.Lcnvnc4_\r ldp x12,x30, [sp] add sp, x12, #0x40 ret END(convolve4_\r) .endr /* void rsdIntrinsicBlurU1_K( * void *out, // x0 * void *in, // x1 * size_t w, // x2 * size_t h, // x3 * size_t p, // x4 * size_t x, // x5 * size_t y, // x6 * size_t count, // x7 * size_t r, // [sp] * uint16_t *tab); // [sp,#8] */ ENTRY(rsdIntrinsicBlurU1_K) stp x19,x30, [sp, #-16]! sub x8, sp, #32 sub sp, sp, #64 st1 {v8.1d - v11.1d}, [sp] st1 {v12.1d - v15.1d}, [x8] mov x8, x5 // x ldr w5, [sp,#80] // r sub x9, x2, x8 sub x10, x3, x6 mov x2, x4 // pitch mov x3, x7 // count sub x7, x10, #1 sub x9, x9, x3 ldr x12, [sp, #88] // tab add x1, x1, x8 cmp x6, x5 csel x6, x5, x6, hs cmp x7, x5 csel x7, x5, x7, hs cmp x8, x5 csel x8, x5, x8, hs cmp x9, x5 csel x9, x5, x9, hs add x4, x8, x9 add x4, x4, x3 sub x1, x1, x8 sub x13, xzr, x2 msub x15, x2, x6, x1 madd x19, x2, x7, x1 ld1 {v0.8h,v1.8h}, [x12], #32 ld1 {v2.8h,v3.8h}, [x12], #32 adr x30, 1f .irep r, TUNED_LIST1 cmp x5, #\r bls convolve1_\r .endr b convolve1_25 1: ld1 {v8.1d - v11.1d}, [sp], #32 ld1 {v12.1d - v15.1d}, [sp], #32 ldp x19,x30, [sp], #16 ret END(rsdIntrinsicBlurU1_K) /* void rsdIntrinsicBlurU4_K( * void *out, // x0 * void *in, // x1 * size_t w, // x2 * size_t h, // x3 * size_t p, // x4 * size_t x, // x5 * size_t y, // x6 * size_t count, // x7 * size_t r, // [sp] * uint16_t *tab); // [sp,#8] */ ENTRY(rsdIntrinsicBlurU4_K) stp x19,x30, [sp, #-16]! sub x8, sp, #32 sub sp, sp, #64 st1 {v8.1d - v11.1d}, [sp] st1 {v12.1d - v15.1d}, [x8] mov x8, x5 // x ldr w5, [sp,#80] // r sub x9, x2, x8 sub x10, x3, x6 mov x2, x4 // pitch mov x3, x7 // count sub x7, x10, #1 sub x9, x9, x3 ldr x12, [sp, #88] add x1, x1, x8, LSL #2 cmp x6, x5 csel x6, x5, x6, hs cmp x7, x5 csel x7, x5, x7, hs cmp x8, x5 csel x8, x5, x8, hs cmp x9, x5 csel x9, x5, x9, hs lsl x3, x3, #2 add x4, x8, x9 add x4, x3, x4, LSL #2 sub x1, x1, x8, LSL #2 sub x13, xzr, x2 msub x15, x2, x6, x1 madd x19, x2, x7, x1 ld1 {v0.8h,v1.8h}, [x12], #32 ld1 {v2.8h,v3.8h}, [x12], #32 adr x30, 1f .irep r, TUNED_LIST4 cmp x5, #\r bls convolve4_\r .endr b convolve4_25 1: ld1 {v8.1d - v11.1d}, [sp], #32 ld1 {v12.1d - v15.1d}, [sp], #32 ldp x19,x30, [sp], #16 ret END(rsdIntrinsicBlurU4_K)