1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
|
/*
* Copyright (C) 2015 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define ENTRY(f) .text; .align 4; .globl f; .type f,#function; f:
#define END(f) .size f, .-f;
/* Fixed-point precision after vertical pass -- 16 bit data minus 1 sign and 1
* integer (bicubic has a little overshoot). It would also be possible to add
* a temporary DC bias to eliminate the sign bit for more precision, but that's
* extra arithmetic.
*/
.set VERTBITS, 14
/* The size of the scratch buffer in which we store our vertically convolved
* intermediates.
*/
.set CHUNKSHIFT, 7 /* 5 tests better for uchar4, but 7 is necessary for ridiculous (10:1) scale factors */
.set CHUNKSIZE, (1 << CHUNKSHIFT)
/* The number of components processed in a single iteration of the innermost
* loop.
*/
.set VECSHIFT, 3
.set VECSIZE, (1<<VECSHIFT)
/* Read four different lines (except at edges where addresses may be clamped,
* which is why we don't simply take base and stride registers), and multiply
* and accumulate them by the coefficients in v3[0..3], leaving the results in
* v12. This gives eight 16-bit results representing a horizontal line of 2-8
* input pixels (depending on number of components per pixel) to be fed into
* the horizontal scaling pass.
*
* Input coefficients are 16-bit unsigned fixed-point (although [0] and [3] are
* known to represent negative values and VMLS is used to implement this).
* Output is VERTBITS signed fixed-point, which must leave room for a little
* v12. This gives eight 16-bit results.
*/
.macro vert8, dstlo=v12.4h, dsthi=v12.8h
ld1 {v8.8b}, [x4], #8
ld1 {v9.8b}, [x5], #8
ld1 {v10.8b}, [x6], #8
ld1 {v11.8b}, [x7], #8
uxtl v8.8h, v8.8b
uxtl v9.8h, v9.8b
uxtl v10.8h, v10.8b
uxtl v11.8h, v11.8b
umull v12.4s, v9.4h, v3.h[1]
umull2 v13.4s, v9.8h, v3.h[1]
umlsl v12.4s, v8.4h, v3.h[0]
umlsl2 v13.4s, v8.8h, v3.h[0]
umlal v12.4s, v10.4h, v3.h[2]
umlal2 v13.4s, v10.8h, v3.h[2]
umlsl v12.4s, v11.4h, v3.h[3]
umlsl2 v13.4s, v11.8h, v3.h[3]
/* Shift by 8 (bits per pixel), plus 16 (the fixed-point multiplies),
* minus VERTBITS (the number of fraction bits we want to keep from
* here on).
*/
sqshrn \dstlo, v12.4s, #8 + (16 - VERTBITS)
sqshrn2 \dsthi, v13.4s, #8 + (16 - VERTBITS)
.endm
/* As above, but only four 16-bit results into v12hi.
*/
.macro vert4, dst=v12.8h
ld1 {v8.s}[0], [x4], #4
ld1 {v9.s}[0], [x5], #4
ld1 {v10.s}[0], [x6], #4
ld1 {v11.s}[0], [x7], #4
uxtl v8.8h, v8.8b
uxtl v9.8h, v9.8b
uxtl v10.8h, v10.8b
uxtl v11.8h, v11.8b
umull v12.4s, v9.4h, v3.h[1]
umlsl v12.4s, v8.4h, v3.h[0]
umlal v12.4s, v10.4h, v3.h[2]
umlsl v12.4s, v11.4h, v3.h[3]
.ifc \dst,v12.8h
sqshrn2 \dst, v12.4s, #8 + (16 - VERTBITS)
.else
sqshrn \dst, v12.4s, #8 + (16 - VERTBITS)
.endif
.endm
/* During horizontal resize having CHUNKSIZE input available means being able
* to produce a varying amount of output, depending on the phase of the data.
* This function calculates the minimum number of VECSIZE chunks extracted from
* a CHUNKSIZE window (x1), and the threshold value for when the count will be
* one higher than that (x0).
* These work out, conveniently, to be the quotient and remainder from:
* (CHUNKSIZE + xinc * VECSIZE - 1) / (xinc * VECSIZE)
*
* The two values are packed together in a uint64_t for convenience; and
* they are, in fact, used this way as an arithmetic short-cut later on.
*/
/* uint64_t rsdIntrinsicResize_oscctl_K(uint32_t xinc) */
ENTRY(rsdIntrinsicResize_oscctl_K)
lsl x2, x0, #VECSHIFT
mov x0, #(CHUNKSIZE << 16) - 1
add x0, x0, x2
udiv x1, x0, x2
msub x0, x1, x2, x0
add x0, x0, x1, LSL #32
ret
END(rsdIntrinsicResize_oscctl_K)
/* Iterate to generate the uchar1, uchar2, and uchar4 versions of the code.
* For the most part the vertical pass (the outer loop) is the same for all
* versions. Exceptions are handled in-line with conditional assembly.
*/
.irp comp, 1, 2, 4
.if \comp == 1
.set COMPONENT_SHIFT, 0
.elseif \comp == 2
.set COMPONENT_SHIFT, 1
.elseif \comp == 4
.set COMPONENT_SHIFT, 2
.else
.error "Unknown component count"
.endif
.set COMPONENT_COUNT, (1 << COMPONENT_SHIFT)
.set LOOP_OUTPUT_SIZE, (VECSIZE * COMPONENT_COUNT)
.set BUFFER_SIZE, (CHUNKSIZE * 2 + 4) * COMPONENT_COUNT * 2
/* void rsdIntrinsicResizeB1_K(
* uint8_t * restrict dst, // x0
* size_t count, // x1
* uint32_t xf, // x2
* uint32_t xinc, // x3
* uint8_t const * restrict srcn, // x4
* uint8_t const * restrict src0, // x5
* uint8_t const * restrict src1, // x6
* uint8_t const * restrict src2, // x7
* size_t xclip, // [sp,#0] -> [sp,#64] -> x12
* size_t avail, // [sp,#8] -> [sp,#72] -> x11
* uint64_t osc_ctl, // [sp,#16] -> [sp,#80] -> x10
* int32 const *yr, // [sp,#24] -> [sp,#88] -> v4 (copied to v3 for scalar access)
*/
ENTRY(rsdIntrinsicResizeB\comp\()_K)
sub x8, sp, #32
sub sp, sp, #64
st1 {v8.1d - v11.1d}, [sp]
st1 {v12.1d - v15.1d}, [x8]
/* align the working buffer on the stack to make it easy to use bit
* twiddling for address calculations.
*/
sub x12, sp, #BUFFER_SIZE
bic x12, x12, #(1 << (CHUNKSHIFT + 1 + COMPONENT_SHIFT + 1)) - 1
ldr x8, [sp,#88] // yr
adr x9, 8f
ld1 {v4.4s}, [x8]
ld1 {v5.8h}, [x9]
sqxtun v4.4h, v4.4s // yr
dup v6.8h, w2
dup v7.8h, w3
mla v6.8h, v5.8h, v7.8h // vxf
shl v7.8h, v7.8h, #VECSHIFT // vxinc
/* Compute starting condition for oscillator used to compute ahead
* of time how many iterations are possible before needing to
* refill the working buffer. This is based on the fixed-point
* index of the last element in the vector of pixels processed in
* each iteration, counting up until it would overflow.
*/
sub x8, x2, x3
lsl x9, x3, #VECSHIFT
add x8, x8, x9
ldr x10, [sp,#80] // osc_ctl
ldp x13,x11, [sp,#64] // xclip, avail
mov x18, sp
mov sp, x12
/* x4-x7 contain pointers to the four lines of input to be
* convolved. These pointers have been clamped vertically and
* horizontally (which is why it's not a simple row/stride pair),
* and the xclip argument (now in x13) indicates how many pixels
* from true the x position of the pointer is. This value should
* be 0, 1, or 2 only.
*
* Start by placing four pixels worth of input at the far end of
* the buffer. As many as two of these may be clipped, so four
* pixels are fetched, and then the first pixel is duplicated and
* the data shifted according to xclip. The source pointers are
* then also adjusted according to xclip so that subsequent fetches
* match.
*/
mov v3.8b, v4.8b /* make y coeffs available for vert4 and vert8 macros */
sub x14, x12, x13, LSL #(COMPONENT_SHIFT + 1)
add x15, x12, #(2 * CHUNKSIZE - 4) * COMPONENT_COUNT * 2
add x14, x14, #4 * COMPONENT_COUNT * 2
.if \comp == 1
vert4 v12.4h
dup v11.4h, v12.h[0]
st1 {v11.4h,v12.4h}, [x12]
ld1 {v12.4h}, [x14]
st1 {v12.4h}, [x15]
.elseif \comp == 2
vert8
dup v11.4s, v12.s[0]
st1 {v11.8h,v12.8h}, [x12]
ld1 {v12.8h}, [x14]
st1 {v12.8h}, [x15]
.elseif \comp == 4
vert8 v14.4h, v14.8h
vert8 v15.4h, v15.8h
dup v12.2d, v14.d[0]
dup v13.2d, v14.d[0]
st1 {v12.8h,v13.8h}, [x12], #32
st1 {v14.8h,v15.8h}, [x12]
sub x12, x12, #32
ld1 {v11.8h,v12.8h}, [x14]
st1 {v11.8h,v12.8h}, [x15]
.endif
/* Count off four pixels into the working buffer.
*/
sub x11, x11, #4
/* Incoming pointers were to the first _legal_ pixel. Four pixels
* were read unconditionally, but some may have been discarded by
* xclip, so we rewind the pointers to compensate.
*/
sub x4, x4, x13, LSL #(COMPONENT_SHIFT)
sub x5, x5, x13, LSL #(COMPONENT_SHIFT)
sub x6, x6, x13, LSL #(COMPONENT_SHIFT)
sub x7, x7, x13, LSL #(COMPONENT_SHIFT)
/* First tap starts where we just pre-filled, at the end of the
* buffer.
*/
add x2, x2, #(CHUNKSIZE * 2 - 4) << 16
/* Use overflowing arithmetic to implement wraparound array
* indexing.
*/
lsl x2, x2, #(47 - CHUNKSHIFT)
lsl x3, x3, #(47 - CHUNKSHIFT)
/* Start of outermost loop.
* Fetch CHUNKSIZE pixels into scratch buffer, then calculate the
* number of iterations of the inner loop that can be performed and
* get into that.
*
* The fill is complicated by the possibility of running out of
* input before the scratch buffer is filled. If this isn't a risk
* then it's handled by the simple loop at 2:, otherwise the
* horrible loop at 3:.
*/
1: mov v3.8b, v4.8b /* put y scaling coefficients somewhere handy */
subs x11, x11, #CHUNKSIZE
bge 2f /* if at least CHUNKSIZE are available... */
add x11, x11, #CHUNKSIZE /* if they're not... */
b 4f
/* ..just sneaking a literal in here after this unconditional branch.. */
8: .hword 0, 1, 2, 3, 4, 5, 6, 7
/* basic fill loop, processing 8 bytes at a time until there are
* fewer than eight bytes available.
*/
3: vert8
sub x11, x11, #8 / COMPONENT_COUNT
st1 {v12.8h}, [x12], #16
4: cmp x11, #8 / COMPONENT_COUNT - 1
bgt 3b
.if \comp == 4
blt 3f
/* The last pixel (four bytes) if necessary */
vert4
.else
cmp x11, #1
blt 3f
/* The last pixels if necessary */
sub x4, x4, #8
sub x5, x5, #8
sub x6, x6, #8
sub x7, x7, #8
add x4, x4, x11, LSL #(COMPONENT_SHIFT)
add x5, x5, x11, LSL #(COMPONENT_SHIFT)
add x6, x6, x11, LSL #(COMPONENT_SHIFT)
add x7, x7, x11, LSL #(COMPONENT_SHIFT)
vert8
sub x11, sp, x11, LSL #(COMPONENT_SHIFT + 1)
sub sp, sp, #32
sub x11, x11, #16
.if \comp == 1
dup v13.8h, v12.h[7]
.elseif \comp == 2
dup v13.4s, v12.s[3]
.endif
st1 {v12.8h,v13.8h}, [sp]
ld1 {v12.8h}, [x11]
add sp, sp, #32
b 4f
.endif
/* Keep filling until we get to the end of this chunk of the buffer */
3:
.if \comp == 1
dup v12.8h, v12.h[7]
.elseif \comp == 2
dup v12.4s, v12.s[3]
.elseif \comp == 4
dup v12.2d, v12.d[1]
.endif
4: st1 {v12.8h}, [x12], #16
tst x12, #(CHUNKSIZE - 1) * COMPONENT_COUNT * 2
bne 3b
b 4f
.align 4
2: /* Quickly pull a chunk of data into the working buffer.
*/
vert8
st1 {v12.8h}, [x12], #16
vert8
st1 {v12.8h}, [x12], #16
tst x12, #(CHUNKSIZE - 1) * COMPONENT_COUNT * 2
bne 2b
cmp x11, #0
bne 3f
4: /* if we end with 0 pixels left we'll have nothing handy to spread
* across to the right, so we rewind a bit.
*/
mov x11, #1
sub x4, x4, #COMPONENT_COUNT
sub x5, x5, #COMPONENT_COUNT
sub x6, x6, #COMPONENT_COUNT
sub x7, x7, #COMPONENT_COUNT
3: /* copy four taps (width of cubic window) to far end for overflow
* address handling
*/
sub x13, x12, #CHUNKSIZE * COMPONENT_COUNT * 2
eor x12, x13, #CHUNKSIZE * COMPONENT_COUNT * 2
.if \comp == 1
ld1 {v14.4h}, [x13]
.elseif \comp == 2
ld1 {v14.8h}, [x13]
.elseif \comp == 4
ld1 {v14.8h,v15.8h}, [x13]
.endif
add x13, x12, #CHUNKSIZE * COMPONENT_COUNT * 2
.if \comp == 1
st1 {v14.4h}, [x13]
.elseif \comp == 2
st1 {v14.8h}, [x13]
.elseif \comp == 4
st1 {v14.8h,v15.8h}, [x13]
.endif
/* The high 32-bits of x10 contains the maximum possible iteration
* count, but if x8 is greater than the low 32-bits of x10 then
* this indicates that the count must be reduced by one for this
* iteration to avoid reading past the end of the available data.
*/
sub x13, x10, x8
lsr x13, x13, #32
madd x8, x13, x9, x8
sub x8, x8, #(CHUNKSIZE << 16)
/* prefer to count pixels, rather than vectors, to clarify the tail
* store case on exit.
*/
lsl x13, x13, #VECSHIFT
cmp x13, x1
csel x13, x1, x13, gt
sub x1, x1, x13
lsl x13, x13, #COMPONENT_SHIFT
mov w14, #0x8000
movi v30.8h, #3
dup v31.8h, w14
cmp x13, #0
bgt 3f
cmp x1, #0
bgt 1b /* an extreme case where we shouldn't use code in this structure */
b 9f
.align 4
2: /* Inner loop continues here, but starts at 3:, see end of loop
* below for explanation. */
.if LOOP_OUTPUT_SIZE == 4
st1 {v8.s}[0], [x0], #4
.elseif LOOP_OUTPUT_SIZE == 8
st1 {v8.8b}, [x0], #8
.elseif LOOP_OUTPUT_SIZE == 16
st1 {v8.16b}, [x0], #16
.elseif LOOP_OUTPUT_SIZE == 32
st1 {v8.16b,v9.16b}, [x0], #32
.endif
/* Inner loop: here the four x coefficients for each tap are
* calculated in vector code, and the addresses are calculated in
* scalar code, and these calculations are interleaved.
*/
3: ushr v8.8h, v6.8h, #1 // sxf
lsr x14, x2, #(63 - CHUNKSHIFT)
sqrdmulh v9.8h, v8.8h, v8.8h // sxf**2
add x2, x2, x3
sqrdmulh v10.8h, v9.8h, v8.8h // sxf**3
lsr x15, x2, #(63 - CHUNKSHIFT)
sshll v11.4s, v9.4h, #2
sshll2 v12.4s, v9.8h, #2
add x2, x2, x3
smlsl v11.4s, v10.4h, v30.4h
smlsl2 v12.4s, v10.8h, v30.8h
lsr x16, x2, #(63 - CHUNKSHIFT)
shadd v0.8h, v10.8h, v8.8h
add x2, x2, x3
sub v0.8h, v9.8h, v0.8h
lsr x17, x2, #(63 - CHUNKSHIFT)
saddw v1.4s, v11.4s, v9.4h
saddw2 v13.4s, v12.4s, v9.8h
add x2, x2, x3
shrn v1.4h, v1.4s, #1
shrn2 v1.8h, v13.4s, #1
add x14, sp, x14, LSL #(COMPONENT_SHIFT + 1)
sub v1.8h, v1.8h, v31.8h
add x15, sp, x15, LSL #(COMPONENT_SHIFT + 1)
saddw v2.4s, v11.4s, v8.4h
saddw2 v13.4s, v12.4s, v8.8h
add x16, sp, x16, LSL #(COMPONENT_SHIFT + 1)
shrn v2.4h, v2.4s, #1
shrn2 v2.8h, v13.4s, #1
add x17, sp, x17, LSL #(COMPONENT_SHIFT + 1)
neg v2.8h, v2.8h
shsub v3.8h, v10.8h, v9.8h
/* increment the x fractional parts (oveflow is ignored, as the
* scalar arithmetic shadows this addition with full precision).
*/
add v6.8h, v6.8h, v7.8h
/* At this point we have four pointers in x8-x11, pointing to the
* four taps in the scratch buffer that must be convolved together
* to produce an output pixel (one output pixel per pointer).
* These pointers usually overlap, but their spacing is irregular
* so resolving the redundancy through L1 is a pragmatic solution.
*
* The scratch buffer is made of signed 16-bit data, holding over
* some extra precision, and overshoot, from the vertical pass.
*
* We also have the 16-bit unsigned fixed-point weights for each
* of the four taps in v0 - v3. That's eight pixels worth of
* coefficients when we have only four pointers, so calculations
* for four more pixels are interleaved with the fetch and permute
* code for each variant in the following code.
*
* The data arrangement is less than ideal for any pixel format,
* but permuting loads help to mitigate most of the problems.
*
* Note also that the two outside taps of a bicubic are negative,
* but these coefficients are unsigned. The sign is hard-coded by
* use of multiply-and-subtract operations.
*/
.if \comp == 1
/* The uchar 1 case.
* Issue one lanewise ld4.h to load four consecutive pixels from
* one pointer (one pixel) into four different registers; then load
* four consecutive s16 values from the next pointer (pixel) into
* the next lane of those four registers, etc., so that we finish
* with v12 - v15 representing the four taps, and each lane
* representing a separate pixel.
*
* The first ld4 uses a splat to avoid any false dependency on
* the previous state of the register.
*/
ld4r {v12.8h,v13.8h,v14.8h,v15.8h}, [x14]
lsr x14, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
ld4 {v12.h,v13.h,v14.h,v15.h}[1], [x15]
add x14, sp, x14, LSL #(COMPONENT_SHIFT + 1)
lsr x15, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
ld4 {v12.h,v13.h,v14.h,v15.h}[2], [x16]
add x15, sp, x15, LSL #(COMPONENT_SHIFT + 1)
lsr x16, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
ld4 {v12.h,v13.h,v14.h,v15.h}[3], [x17]
add x16, sp, x16, LSL #(COMPONENT_SHIFT + 1)
lsr x17, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
ld4 {v12.h,v13.h,v14.h,v15.h}[4], [x14]
add x17, sp, x17, LSL #(COMPONENT_SHIFT + 1)
ld4 {v12.h,v13.h,v14.h,v15.h}[5], [x15]
ld4 {v12.h,v13.h,v14.h,v15.h}[6], [x16]
ld4 {v12.h,v13.h,v14.h,v15.h}[7], [x17]
smull v8.4s, v12.4h, v0.4h
smull2 v9.4s, v12.8h, v0.8h
smlsl v8.4s, v13.4h, v1.4h
smlsl2 v9.4s, v13.8h, v1.8h
smlsl v8.4s, v14.4h, v2.4h
smlsl2 v9.4s, v14.8h, v2.8h
smlal v8.4s, v15.4h, v3.4h
smlal2 v9.4s, v15.8h, v3.8h
subs x13, x13, #LOOP_OUTPUT_SIZE
sqrshrn v8.4h, v8.4s, #15
sqrshrn2 v8.8h, v9.4s, #15
sqrshrun v8.8b, v8.8h, #VERTBITS - 8
.elseif \comp == 2
/* The uchar2 case:
* This time load pairs of values into adjacent lanes in v12 - v15
* by aliasing them as u32 data; leaving room for only four pixels,
* so the process has to be done twice. This also means that the
* coefficient registers fail to align with the coefficient data
* (eight separate pixels), so that has to be doubled-up to match.
*/
ld4r {v12.4s,v13.4s,v14.4s,v15.4s}, [x14]
lsr x14, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
ld4 {v12.s,v13.s,v14.s,v15.s}[1], [x15]
add x14, sp, x14, LSL #(COMPONENT_SHIFT + 1)
lsr x15, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
ld4 {v12.s,v13.s,v14.s,v15.s}[2], [x16]
add x15, sp, x15, LSL #(COMPONENT_SHIFT + 1)
lsr x16, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
ld4 {v12.s,v13.s,v14.s,v15.s}[3], [x17]
add x16, sp, x16, LSL #(COMPONENT_SHIFT + 1)
lsr x17, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
/* double-up coefficients to align with component pairs */
zip1 v16.8h, v0.8h, v0.8h
add x17, sp, x17, LSL #(COMPONENT_SHIFT + 1)
zip1 v17.8h, v1.8h, v1.8h
zip1 v18.8h, v2.8h, v2.8h
zip1 v19.8h, v3.8h, v3.8h
smull v8.4s, v12.4h, v16.4h
smull2 v9.4s, v12.8h, v16.8h
smlsl v8.4s, v13.4h, v17.4h
smlsl2 v9.4s, v13.8h, v17.8h
smlsl v8.4s, v14.4h, v18.4h
smlsl2 v9.4s, v14.8h, v18.8h
smlal v8.4s, v15.4h, v19.4h
smlal2 v9.4s, v15.8h, v19.8h
sqrshrn v8.4h, v8.4s, #15
sqrshrn2 v8.8h, v9.4s, #15
ld4r {v12.4s,v13.4s,v14.4s,v15.4s}, [x14]
ld4 {v12.s,v13.s,v14.s,v15.s}[1], [x15]
ld4 {v12.s,v13.s,v14.s,v15.s}[2], [x16]
ld4 {v12.s,v13.s,v14.s,v15.s}[3], [x17]
/* double-up coefficients to align with component pairs */
zip2 v16.8h, v0.8h, v0.8h
zip2 v17.8h, v1.8h, v1.8h
zip2 v18.8h, v2.8h, v2.8h
zip2 v19.8h, v3.8h, v3.8h
smull v10.4s, v12.4h, v16.4h
smull2 v11.4s, v12.8h, v16.8h
smlsl v10.4s, v13.4h, v17.4h
smlsl2 v11.4s, v13.8h, v17.8h
smlsl v10.4s, v14.4h, v18.4h
smlsl2 v11.4s, v14.8h, v18.8h
smlal v10.4s, v15.4h, v19.4h
smlal2 v11.4s, v15.8h, v19.8h
subs x13, x13, #LOOP_OUTPUT_SIZE
sqrshrn v9.4h, v10.4s, #15
sqrshrn2 v9.8h, v11.4s, #15
sqrshrun v8.8b, v8.8h, #VERTBITS - 8
sqrshrun2 v8.16b, v9.8h, #VERTBITS - 8
.elseif \comp == 4
/* The uchar4 case.
* This case is comparatively painless because four s16s are the
* smallest addressable unit for a vmul-by-scalar. Rather than
* permute the data, simply arrange the multiplies to suit the way
* the data comes in. That's a lot of data, though, so things
* progress in pairs of pixels at a time.
*/
ld1 {v12.8h,v13.8h}, [x14]
lsr x14, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
ld1 {v14.8h,v15.8h}, [x15]
add x14, sp, x14, LSL #(COMPONENT_SHIFT + 1)
lsr x15, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
smull v8.4s, v12.4h, v0.h[0]
smull v9.4s, v14.4h, v0.h[1]
smlsl2 v8.4s, v12.8h, v1.h[0]
smlsl2 v9.4s, v14.8h, v1.h[1]
smlsl v8.4s, v13.4h, v2.h[0]
smlsl v9.4s, v15.4h, v2.h[1]
smlal2 v8.4s, v13.8h, v3.h[0]
smlal2 v9.4s, v15.8h, v3.h[1]
/* And two more... */
ld1 {v12.8h,v13.8h}, [x16]
add x15, sp, x15, LSL #(COMPONENT_SHIFT + 1)
lsr x16, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
ld1 {v14.8h,v15.8h}, [x17]
add x16, sp, x16, LSL #(COMPONENT_SHIFT + 1)
lsr x17, x2, #(63 - CHUNKSHIFT)
add x2, x2, x3
sqrshrn v8.4h, v8.4s, #15
add x17, sp, x17, LSL #(COMPONENT_SHIFT + 1)
sqrshrn2 v8.8h, v9.4s, #15
smull v10.4s, v12.4h, v0.h[2]
smull v11.4s, v14.4h, v0.h[3]
smlsl2 v10.4s, v12.8h, v1.h[2]
smlsl2 v11.4s, v14.8h, v1.h[3]
smlsl v10.4s, v13.4h, v2.h[2]
smlsl v11.4s, v15.4h, v2.h[3]
smlal2 v10.4s, v13.8h, v3.h[2]
smlal2 v11.4s, v15.8h, v3.h[3]
sqrshrn v9.4h, v10.4s, #15
sqrshrn2 v9.8h, v11.4s, #15
sqrshrun v8.8b, v8.8h, #VERTBITS - 8
sqrshrun2 v8.16b, v9.8h, #VERTBITS - 8
/* And two more... */
ld1 {v12.8h,v13.8h}, [x14]
ld1 {v14.8h,v15.8h}, [x15]
smull v10.4s, v12.4h, v0.h[4]
smull v11.4s, v14.4h, v0.h[5]
smlsl2 v10.4s, v12.8h, v1.h[4]
smlsl2 v11.4s, v14.8h, v1.h[5]
smlsl v10.4s, v13.4h, v2.h[4]
smlsl v11.4s, v15.4h, v2.h[5]
smlal2 v10.4s, v13.8h, v3.h[4]
smlal2 v11.4s, v15.8h, v3.h[5]
/* And two more... */
ld1 {v12.8h,v13.8h}, [x16]
ld1 {v14.8h,v15.8h}, [x17]
subs x13, x13, #LOOP_OUTPUT_SIZE
sqrshrn v9.4h, v10.4s, #15
sqrshrn2 v9.8h, v11.4s, #15
smull v10.4s, v12.4h, v0.h[6]
smull v11.4s, v14.4h, v0.h[7]
smlsl2 v10.4s, v12.8h, v1.h[6]
smlsl2 v11.4s, v14.8h, v1.h[7]
smlsl v10.4s, v13.4h, v2.h[6]
smlsl v11.4s, v15.4h, v2.h[7]
smlal2 v10.4s, v13.8h, v3.h[6]
smlal2 v11.4s, v15.8h, v3.h[7]
sqrshrn v10.4h, v10.4s, #15
sqrshrn2 v10.8h, v11.4s, #15
sqrshrun v9.8b, v9.8h, #VERTBITS - 8
sqrshrun2 v9.16b, v10.8h, #VERTBITS - 8
.endif
bgt 2b /* continue inner loop */
/* The inner loop has already been limited to ensure that none of
* the earlier iterations could overfill the output, so the store
* appears within the loop but after the conditional branch (at the
* top). At the end, provided it won't overfill, perform the final
* store here. If it would, then break out to the tricky tail case
* instead.
*/
blt 1f
/* Store the amount of data appropriate to the configuration of the
* instance being assembled.
*/
.if LOOP_OUTPUT_SIZE == 4
st1 {v8.s}[0], [x0], #4
.elseif LOOP_OUTPUT_SIZE == 8
st1 {v8.8b}, [x0], #8
.elseif LOOP_OUTPUT_SIZE == 16
st1 {v8.16b}, [x0], #16
.elseif LOOP_OUTPUT_SIZE == 32
st1 {v8.16b,v9.16b}, [x0], #32
.endif
b 1b /* resume outer loop */
/* Partial tail store case:
* Different versions of the code need different subsets of the
* following partial stores. Here the number of components and the
* size of the chunk of data produced by each inner loop iteration
* is tested to figure out whether or not each phrase is relevant.
*/
.if 16 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 16
1: tst x13, #16
beq 1f
st1 {v8.16b}, [x0], #16
mov v8.16b, v9.16b
.endif
.if 8 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 8
1: tst x13, #8
beq 1f
st1 {v8.8b}, [x0], #8
ext v8.16b, v8.16b, v8.16b, #8
.endif
.if 4 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 4
1: tst x13, #4
beq 1f
st1 {v8.s}[0], [x0], #4
ext v8.8b, v8.8b, v8.8b, #4
.endif
.if 2 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 2
1: tst x13, #2
beq 1f
st1 {v8.h}[0], [x0], #2
ext v8.8b, v8.8b, v8.8b, #2
.endif
.if 1 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 1
1: tst x13, #1
beq 1f
st1 {v8.b}[0], [x0], #1
.endif
1:
9: mov sp, x18
ld1 {v8.1d - v11.1d}, [sp], #32
ld1 {v12.1d - v15.1d}, [sp], #32
ret
END(rsdIntrinsicResizeB\comp\()_K)
.endr
|