aboutsummaryrefslogtreecommitdiff
path: root/src/cpu
diff options
context:
space:
mode:
Diffstat (limited to 'src/cpu')
-rw-r--r--src/cpu/ppc/vm/assembler_ppc.hpp8
-rw-r--r--src/cpu/ppc/vm/assembler_ppc.inline.hpp27
-rw-r--r--src/cpu/ppc/vm/macroAssembler_ppc.hpp34
-rw-r--r--src/cpu/ppc/vm/macroAssembler_ppc_sha.cpp1136
-rw-r--r--src/cpu/ppc/vm/stubGenerator_ppc.cpp390
-rw-r--r--src/cpu/ppc/vm/stubRoutines_ppc_64.hpp2
-rw-r--r--src/cpu/ppc/vm/vm_version_ppc.cpp54
-rw-r--r--src/cpu/ppc/vm/vm_version_ppc.hpp3
-rw-r--r--src/cpu/x86/vm/stubRoutines_x86_64.hpp2
-rw-r--r--src/cpu/x86/vm/x86_64.ad61
10 files changed, 1518 insertions, 199 deletions
diff --git a/src/cpu/ppc/vm/assembler_ppc.hpp b/src/cpu/ppc/vm/assembler_ppc.hpp
index b14f2b6d0..88ac813a1 100644
--- a/src/cpu/ppc/vm/assembler_ppc.hpp
+++ b/src/cpu/ppc/vm/assembler_ppc.hpp
@@ -2000,7 +2000,8 @@ class Assembler : public AbstractAssembler {
inline void vsbox( VectorRegister d, VectorRegister a);
// SHA (introduced with Power 8)
- // Not yet implemented.
+ inline void vshasigmad(VectorRegister d, VectorRegister a, bool st, int six);
+ inline void vshasigmaw(VectorRegister d, VectorRegister a, bool st, int six);
// Vector Binary Polynomial Multiplication (introduced with Power 8)
inline void vpmsumb( VectorRegister d, VectorRegister a, VectorRegister b);
@@ -2096,6 +2097,11 @@ class Assembler : public AbstractAssembler {
inline void lvsl( VectorRegister d, Register s2);
inline void lvsr( VectorRegister d, Register s2);
+ // Endianess specific concatenation of 2 loaded vectors.
+ inline void load_perm(VectorRegister perm, Register addr);
+ inline void vec_perm(VectorRegister first_dest, VectorRegister second, VectorRegister perm);
+ inline void vec_perm(VectorRegister dest, VectorRegister first, VectorRegister second, VectorRegister perm);
+
// RegisterOrConstant versions.
// These emitters choose between the versions using two registers and
// those with register and immediate, depending on the content of roc.
diff --git a/src/cpu/ppc/vm/assembler_ppc.inline.hpp b/src/cpu/ppc/vm/assembler_ppc.inline.hpp
index 1d2c05e2a..5d8066372 100644
--- a/src/cpu/ppc/vm/assembler_ppc.inline.hpp
+++ b/src/cpu/ppc/vm/assembler_ppc.inline.hpp
@@ -789,7 +789,8 @@ inline void Assembler::vncipherlast(VectorRegister d, VectorRegister a, VectorRe
inline void Assembler::vsbox( VectorRegister d, VectorRegister a) { emit_int32( VSBOX_OPCODE | vrt(d) | vra(a) ); }
// SHA (introduced with Power 8)
-// Not yet implemented.
+inline void Assembler::vshasigmad(VectorRegister d, VectorRegister a, bool st, int six) { emit_int32( VSHASIGMAD_OPCODE | vrt(d) | vra(a) | vst(st) | vsix(six)); }
+inline void Assembler::vshasigmaw(VectorRegister d, VectorRegister a, bool st, int six) { emit_int32( VSHASIGMAW_OPCODE | vrt(d) | vra(a) | vst(st) | vsix(six)); }
// Vector Binary Polynomial Multiplication (introduced with Power 8)
inline void Assembler::vpmsumb( VectorRegister d, VectorRegister a, VectorRegister b) { emit_int32( VPMSUMB_OPCODE | vrt(d) | vra(a) | vrb(b)); }
@@ -887,6 +888,30 @@ inline void Assembler::stvxl( VectorRegister d, Register s2) { emit_int32( STVXL
inline void Assembler::lvsl( VectorRegister d, Register s2) { emit_int32( LVSL_OPCODE | vrt(d) | rb(s2)); }
inline void Assembler::lvsr( VectorRegister d, Register s2) { emit_int32( LVSR_OPCODE | vrt(d) | rb(s2)); }
+inline void Assembler::load_perm(VectorRegister perm, Register addr) {
+#if defined(VM_LITTLE_ENDIAN)
+ lvsr(perm, addr);
+#else
+ lvsl(perm, addr);
+#endif
+}
+
+inline void Assembler::vec_perm(VectorRegister first_dest, VectorRegister second, VectorRegister perm) {
+#if defined(VM_LITTLE_ENDIAN)
+ vperm(first_dest, second, first_dest, perm);
+#else
+ vperm(first_dest, first_dest, second, perm);
+#endif
+}
+
+inline void Assembler::vec_perm(VectorRegister dest, VectorRegister first, VectorRegister second, VectorRegister perm) {
+#if defined(VM_LITTLE_ENDIAN)
+ vperm(dest, second, first, perm);
+#else
+ vperm(dest, first, second, perm);
+#endif
+}
+
inline void Assembler::load_const(Register d, void* x, Register tmp) {
load_const(d, (long)x, tmp);
}
diff --git a/src/cpu/ppc/vm/macroAssembler_ppc.hpp b/src/cpu/ppc/vm/macroAssembler_ppc.hpp
index 3c6cea51b..afcec9a47 100644
--- a/src/cpu/ppc/vm/macroAssembler_ppc.hpp
+++ b/src/cpu/ppc/vm/macroAssembler_ppc.hpp
@@ -667,6 +667,40 @@ class MacroAssembler: public Assembler {
void kernel_crc32_singleByte(Register crc, Register buf, Register len, Register table, Register tmp);
+ // SHA-2 auxiliary functions and public interfaces
+ private:
+ void sha256_deque(const VectorRegister src,
+ const VectorRegister dst1, const VectorRegister dst2, const VectorRegister dst3);
+ void sha256_load_h_vec(const VectorRegister a, const VectorRegister e, const Register hptr);
+ void sha256_round(const VectorRegister* hs, const int total_hs, int& h_cnt, const VectorRegister kpw);
+ void sha256_load_w_plus_k_vec(const Register buf_in, const VectorRegister* ws,
+ const int total_ws, const Register k, const VectorRegister* kpws,
+ const int total_kpws);
+ void sha256_calc_4w(const VectorRegister w0, const VectorRegister w1,
+ const VectorRegister w2, const VectorRegister w3, const VectorRegister kpw0,
+ const VectorRegister kpw1, const VectorRegister kpw2, const VectorRegister kpw3,
+ const Register j, const Register k);
+ void sha256_update_sha_state(const VectorRegister a, const VectorRegister b,
+ const VectorRegister c, const VectorRegister d, const VectorRegister e,
+ const VectorRegister f, const VectorRegister g, const VectorRegister h,
+ const Register hptr);
+
+ void sha512_load_w_vec(const Register buf_in, const VectorRegister* ws, const int total_ws);
+ void sha512_update_sha_state(const Register state, const VectorRegister* hs, const int total_hs);
+ void sha512_round(const VectorRegister* hs, const int total_hs, int& h_cnt, const VectorRegister kpw);
+ void sha512_load_h_vec(const Register state, const VectorRegister* hs, const int total_hs);
+ void sha512_calc_2w(const VectorRegister w0, const VectorRegister w1,
+ const VectorRegister w2, const VectorRegister w3,
+ const VectorRegister w4, const VectorRegister w5,
+ const VectorRegister w6, const VectorRegister w7,
+ const VectorRegister kpw0, const VectorRegister kpw1, const Register j,
+ const VectorRegister vRb, const Register k);
+
+ public:
+ void sha256(bool multi_block);
+ void sha512(bool multi_block);
+
+
//
// Debugging
//
diff --git a/src/cpu/ppc/vm/macroAssembler_ppc_sha.cpp b/src/cpu/ppc/vm/macroAssembler_ppc_sha.cpp
new file mode 100644
index 000000000..7a82ed3f9
--- /dev/null
+++ b/src/cpu/ppc/vm/macroAssembler_ppc_sha.cpp
@@ -0,0 +1,1136 @@
+// Copyright (c) 2017 Instituto de Pesquisas Eldorado. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+
+// Implemented according to "Descriptions of SHA-256, SHA-384, and SHA-512"
+// (http://www.iwar.org.uk/comsec/resources/cipher/sha256-384-512.pdf).
+
+#include "asm/macroAssembler.inline.hpp"
+#include "runtime/stubRoutines.hpp"
+
+/**********************************************************************
+ * SHA 256
+ *********************************************************************/
+
+void MacroAssembler::sha256_deque(const VectorRegister src,
+ const VectorRegister dst1,
+ const VectorRegister dst2,
+ const VectorRegister dst3) {
+ vsldoi (dst1, src, src, 12);
+ vsldoi (dst2, src, src, 8);
+ vsldoi (dst3, src, src, 4);
+}
+
+void MacroAssembler::sha256_round(const VectorRegister* hs,
+ const int total_hs,
+ int& h_cnt,
+ const VectorRegister kpw) {
+ // convenience registers: cycle from 0-7 downwards
+ const VectorRegister a = hs[(total_hs + 0 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister b = hs[(total_hs + 1 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister c = hs[(total_hs + 2 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister d = hs[(total_hs + 3 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister e = hs[(total_hs + 4 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister f = hs[(total_hs + 5 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister g = hs[(total_hs + 6 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister h = hs[(total_hs + 7 - (h_cnt % total_hs)) % total_hs];
+ // temporaries
+ VectorRegister ch = VR0;
+ VectorRegister maj = VR1;
+ VectorRegister bsa = VR2;
+ VectorRegister bse = VR3;
+ VectorRegister vt0 = VR4;
+ VectorRegister vt1 = VR5;
+ VectorRegister vt2 = VR6;
+ VectorRegister vt3 = VR7;
+
+ vsel (ch, g, f, e);
+ vxor (maj, a, b);
+ vshasigmaw (bse, e, 1, 0xf);
+ vadduwm (vt2, ch, kpw);
+ vadduwm (vt1, h, bse);
+ vsel (maj, b, c, maj);
+ vadduwm (vt3, vt1, vt2);
+ vshasigmaw (bsa, a, 1, 0);
+ vadduwm (vt0, bsa, maj);
+
+ vadduwm (d, d, vt3);
+ vadduwm (h, vt3, vt0);
+
+ // advance vector pointer to the next iteration
+ h_cnt++;
+}
+
+void MacroAssembler::sha256_load_h_vec(const VectorRegister a,
+ const VectorRegister e,
+ const Register hptr) {
+ // temporaries
+ Register tmp = R8;
+ VectorRegister vt0 = VR0;
+ VectorRegister vRb = VR6;
+ // labels
+ Label sha256_aligned;
+
+ andi_ (tmp, hptr, 0xf);
+ lvx (a, hptr);
+ addi (tmp, hptr, 16);
+ lvx (e, tmp);
+ beq (CCR0, sha256_aligned);
+
+ // handle unaligned accesses
+ load_perm(vRb, hptr);
+ addi (tmp, hptr, 32);
+ vec_perm(a, e, vRb);
+
+ lvx (vt0, tmp);
+ vec_perm(e, vt0, vRb);
+
+ // aligned accesses
+ bind(sha256_aligned);
+}
+
+void MacroAssembler::sha256_load_w_plus_k_vec(const Register buf_in,
+ const VectorRegister* ws,
+ const int total_ws,
+ const Register k,
+ const VectorRegister* kpws,
+ const int total_kpws) {
+ Label w_aligned, after_w_load;
+
+ Register tmp = R8;
+ VectorRegister vt0 = VR0;
+ VectorRegister vt1 = VR1;
+ VectorRegister vRb = VR6;
+
+ andi_ (tmp, buf_in, 0xF);
+ beq (CCR0, w_aligned); // address ends with 0x0, not 0x8
+
+ // deal with unaligned addresses
+ lvx (ws[0], buf_in);
+ load_perm(vRb, buf_in);
+
+ for (int n = 1; n < total_ws; n++) {
+ VectorRegister w_cur = ws[n];
+ VectorRegister w_prev = ws[n-1];
+
+ addi (tmp, buf_in, n * 16);
+ lvx (w_cur, tmp);
+ vec_perm(w_prev, w_cur, vRb);
+ }
+ addi (tmp, buf_in, total_ws * 16);
+ lvx (vt0, tmp);
+ vec_perm(ws[total_ws-1], vt0, vRb);
+ b (after_w_load);
+
+ bind(w_aligned);
+
+ // deal with aligned addresses
+ lvx(ws[0], buf_in);
+ for (int n = 1; n < total_ws; n++) {
+ VectorRegister w = ws[n];
+ addi (tmp, buf_in, n * 16);
+ lvx (w, tmp);
+ }
+
+ bind(after_w_load);
+
+#if defined(VM_LITTLE_ENDIAN)
+ // Byte swapping within int values
+ li (tmp, 8);
+ lvsl (vt0, tmp);
+ vspltisb (vt1, 0xb);
+ vxor (vt1, vt0, vt1);
+ for (int n = 0; n < total_ws; n++) {
+ VectorRegister w = ws[n];
+ vec_perm(w, w, vt1);
+ }
+#endif
+
+ // Loading k, which is always aligned to 16-bytes
+ lvx (kpws[0], k);
+ for (int n = 1; n < total_kpws; n++) {
+ VectorRegister kpw = kpws[n];
+ addi (tmp, k, 16 * n);
+ lvx (kpw, tmp);
+ }
+
+ // Add w to K
+ assert(total_ws == total_kpws, "Redesign the loop below");
+ for (int n = 0; n < total_kpws; n++) {
+ VectorRegister kpw = kpws[n];
+ VectorRegister w = ws[n];
+
+ vadduwm (kpw, kpw, w);
+ }
+}
+
+void MacroAssembler::sha256_calc_4w(const VectorRegister w0,
+ const VectorRegister w1,
+ const VectorRegister w2,
+ const VectorRegister w3,
+ const VectorRegister kpw0,
+ const VectorRegister kpw1,
+ const VectorRegister kpw2,
+ const VectorRegister kpw3,
+ const Register j,
+ const Register k) {
+ // Temporaries
+ const VectorRegister vt0 = VR0;
+ const VectorRegister vt1 = VR1;
+ const VectorSRegister vsrt1 = vt1->to_vsr();
+ const VectorRegister vt2 = VR2;
+ const VectorRegister vt3 = VR3;
+ const VectorSRegister vst3 = vt3->to_vsr();
+ const VectorRegister vt4 = VR4;
+
+ // load to k[j]
+ lvx (vt0, j, k);
+
+ // advance j
+ addi (j, j, 16); // 16 bytes were read
+
+#if defined(VM_LITTLE_ENDIAN)
+ // b = w[j-15], w[j-14], w[j-13], w[j-12]
+ vsldoi (vt1, w1, w0, 12);
+
+ // c = w[j-7], w[j-6], w[j-5], w[j-4]
+ vsldoi (vt2, w3, w2, 12);
+
+#else
+ // b = w[j-15], w[j-14], w[j-13], w[j-12]
+ vsldoi (vt1, w0, w1, 4);
+
+ // c = w[j-7], w[j-6], w[j-5], w[j-4]
+ vsldoi (vt2, w2, w3, 4);
+#endif
+
+ // d = w[j-2], w[j-1], w[j-4], w[j-3]
+ vsldoi (vt3, w3, w3, 8);
+
+ // b = s0(w[j-15]) , s0(w[j-14]) , s0(w[j-13]) , s0(w[j-12])
+ vshasigmaw (vt1, vt1, 0, 0);
+
+ // d = s1(w[j-2]) , s1(w[j-1]) , s1(w[j-4]) , s1(w[j-3])
+ vshasigmaw (vt3, vt3, 0, 0xf);
+
+ // c = s0(w[j-15]) + w[j-7],
+ // s0(w[j-14]) + w[j-6],
+ // s0(w[j-13]) + w[j-5],
+ // s0(w[j-12]) + w[j-4]
+ vadduwm (vt2, vt1, vt2);
+
+ // c = s0(w[j-15]) + w[j-7] + w[j-16],
+ // s0(w[j-14]) + w[j-6] + w[j-15],
+ // s0(w[j-13]) + w[j-5] + w[j-14],
+ // s0(w[j-12]) + w[j-4] + w[j-13]
+ vadduwm (vt2, vt2, w0);
+
+ // e = s0(w[j-15]) + w[j-7] + w[j-16] + s1(w[j-2]), // w[j]
+ // s0(w[j-14]) + w[j-6] + w[j-15] + s1(w[j-1]), // w[j+1]
+ // s0(w[j-13]) + w[j-5] + w[j-14] + s1(w[j-4]), // UNDEFINED
+ // s0(w[j-12]) + w[j-4] + w[j-13] + s1(w[j-3]) // UNDEFINED
+ vadduwm (vt4, vt2, vt3);
+
+ // At this point, e[0] and e[1] are the correct values to be stored at w[j]
+ // and w[j+1].
+ // e[2] and e[3] are not considered.
+ // b = s1(w[j]) , s1(s(w[j+1]) , UNDEFINED , UNDEFINED
+ vshasigmaw (vt1, vt4, 0, 0xf);
+
+ // v5 = s1(w[j-2]) , s1(w[j-1]) , s1(w[j]) , s1(w[j+1])
+#if defined(VM_LITTLE_ENDIAN)
+ xxmrgld (vst3, vsrt1, vst3);
+#else
+ xxmrghd (vst3, vst3, vsrt1);
+#endif
+
+ // c = s0(w[j-15]) + w[j-7] + w[j-16] + s1(w[j-2]), // w[j]
+ // s0(w[j-14]) + w[j-6] + w[j-15] + s1(w[j-1]), // w[j+1]
+ // s0(w[j-13]) + w[j-5] + w[j-14] + s1(w[j]), // w[j+2]
+ // s0(w[j-12]) + w[j-4] + w[j-13] + s1(w[j+1]) // w[j+4]
+ vadduwm (vt2, vt2, vt3);
+
+ // Updating w0 to w3 to hold the new previous 16 values from w.
+ vmr (w0, w1);
+ vmr (w1, w2);
+ vmr (w2, w3);
+ vmr (w3, vt2);
+
+ // store k + w to v9 (4 values at once)
+#if defined(VM_LITTLE_ENDIAN)
+ vadduwm (kpw0, vt2, vt0);
+
+ vsldoi (kpw1, kpw0, kpw0, 12);
+ vsldoi (kpw2, kpw0, kpw0, 8);
+ vsldoi (kpw3, kpw0, kpw0, 4);
+#else
+ vadduwm (kpw3, vt2, vt0);
+
+ vsldoi (kpw2, kpw3, kpw3, 12);
+ vsldoi (kpw1, kpw3, kpw3, 8);
+ vsldoi (kpw0, kpw3, kpw3, 4);
+#endif
+}
+
+void MacroAssembler::sha256_update_sha_state(const VectorRegister a,
+ const VectorRegister b_,
+ const VectorRegister c,
+ const VectorRegister d,
+ const VectorRegister e,
+ const VectorRegister f,
+ const VectorRegister g,
+ const VectorRegister h,
+ const Register hptr) {
+ // temporaries
+ VectorRegister vt0 = VR0;
+ VectorRegister vt1 = VR1;
+ VectorRegister vt2 = VR2;
+ VectorRegister vt3 = VR3;
+ VectorRegister vt4 = VR4;
+ VectorRegister vt5 = VR5;
+ VectorRegister vaux = VR6;
+ VectorRegister vRb = VR6;
+ Register tmp = R8;
+ Register of16 = R8;
+ Register of32 = R9;
+ Label state_load_aligned;
+
+ // Load hptr
+ andi_ (tmp, hptr, 0xf);
+ li (of16, 16);
+ lvx (vt0, hptr);
+ lvx (vt5, of16, hptr);
+ beq (CCR0, state_load_aligned);
+
+ // handle unaligned accesses
+ li (of32, 32);
+ load_perm(vRb, hptr);
+
+ vec_perm(vt0, vt5, vRb); // vt0 = hptr[0]..hptr[3]
+
+ lvx (vt1, hptr, of32);
+ vec_perm(vt5, vt1, vRb); // vt5 = hptr[4]..hptr[7]
+
+ // aligned accesses
+ bind(state_load_aligned);
+
+#if defined(VM_LITTLE_ENDIAN)
+ vmrglw (vt1, b_, a); // vt1 = {a, b, ?, ?}
+ vmrglw (vt2, d, c); // vt2 = {c, d, ?, ?}
+ vmrglw (vt3, f, e); // vt3 = {e, f, ?, ?}
+ vmrglw (vt4, h, g); // vt4 = {g, h, ?, ?}
+ xxmrgld (vt1->to_vsr(), vt2->to_vsr(), vt1->to_vsr()); // vt1 = {a, b, c, d}
+ xxmrgld (vt3->to_vsr(), vt4->to_vsr(), vt3->to_vsr()); // vt3 = {e, f, g, h}
+ vadduwm (a, vt0, vt1); // a = {a+hptr[0], b+hptr[1], c+hptr[2], d+hptr[3]}
+ vadduwm (e, vt5, vt3); // e = {e+hptr[4], f+hptr[5], g+hptr[6], h+hptr[7]}
+
+ // Save hptr back, works for any alignment
+ xxswapd (vt0->to_vsr(), a->to_vsr());
+ stxvd2x (vt0->to_vsr(), hptr);
+ xxswapd (vt5->to_vsr(), e->to_vsr());
+ stxvd2x (vt5->to_vsr(), of16, hptr);
+#else
+ vmrglw (vt1, a, b_); // vt1 = {a, b, ?, ?}
+ vmrglw (vt2, c, d); // vt2 = {c, d, ?, ?}
+ vmrglw (vt3, e, f); // vt3 = {e, f, ?, ?}
+ vmrglw (vt4, g, h); // vt4 = {g, h, ?, ?}
+ xxmrgld (vt1->to_vsr(), vt1->to_vsr(), vt2->to_vsr()); // vt1 = {a, b, c, d}
+ xxmrgld (vt3->to_vsr(), vt3->to_vsr(), vt4->to_vsr()); // vt3 = {e, f, g, h}
+ vadduwm (d, vt0, vt1); // d = {a+hptr[0], b+hptr[1], c+hptr[2], d+hptr[3]}
+ vadduwm (h, vt5, vt3); // h = {e+hptr[4], f+hptr[5], g+hptr[6], h+hptr[7]}
+
+ // Save hptr back, works for any alignment
+ stxvd2x (d->to_vsr(), hptr);
+ stxvd2x (h->to_vsr(), of16, hptr);
+#endif
+}
+
+static const uint32_t sha256_round_table[64] __attribute((aligned(16))) = {
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
+ 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+ 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
+ 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+ 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
+ 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+ 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+ 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+ 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+ 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
+ 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
+ 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+ 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
+ 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
+};
+static const uint32_t *sha256_round_consts = sha256_round_table;
+
+// R3_ARG1 - byte[] Input string with padding but in Big Endian
+// R4_ARG2 - int[] SHA.state (at first, the root of primes)
+// R5_ARG3 - int offset
+// R6_ARG4 - int limit
+//
+// Internal Register usage:
+// R7 - k
+// R8 - tmp | j | of16
+// R9 - of32
+// VR0-VR8 - ch, maj, bsa, bse, vt0-vt3 | vt0-vt5, vaux/vRb
+// VR9-VR16 - a-h
+// VR17-VR20 - w0-w3
+// VR21-VR23 - vRb | vaux0-vaux2
+// VR24-VR27 - kpw0-kpw3
+void MacroAssembler::sha256(bool multi_block) {
+ static const ssize_t buf_size = 64;
+ static const uint8_t w_size = sizeof(sha256_round_table)/sizeof(uint32_t);
+#ifdef AIX
+ // malloc provides 16 byte alignment
+ if (((uintptr_t)sha256_round_consts & 0xF) != 0) {
+ uint32_t *new_round_consts = (uint32_t*)malloc(sizeof(sha256_round_table));
+ guarantee(new_round_consts, "oom");
+ memcpy(new_round_consts, sha256_round_consts, sizeof(sha256_round_table));
+ sha256_round_consts = (const uint32_t*)new_round_consts;
+ }
+#endif
+
+ Register buf_in = R3_ARG1;
+ Register state = R4_ARG2;
+ Register ofs = R5_ARG3;
+ Register limit = R6_ARG4;
+
+ Label sha_loop, core_loop;
+
+ // Save non-volatile vector registers in the red zone
+ static const VectorRegister nv[] = {
+ VR20, VR21, VR22, VR23, VR24, VR25, VR26, VR27/*, VR28, VR29, VR30, VR31*/
+ };
+ static const uint8_t nv_size = sizeof(nv) / sizeof (VectorRegister);
+
+ for (int c = 0; c < nv_size; c++) {
+ Register tmp = R8;
+ li (tmp, (c - (nv_size)) * 16);
+ stvx(nv[c], tmp, R1);
+ }
+
+ // Load hash state to registers
+ VectorRegister a = VR9;
+ VectorRegister b = VR10;
+ VectorRegister c = VR11;
+ VectorRegister d = VR12;
+ VectorRegister e = VR13;
+ VectorRegister f = VR14;
+ VectorRegister g = VR15;
+ VectorRegister h = VR16;
+ static const VectorRegister hs[] = {a, b, c, d, e, f, g, h};
+ static const int total_hs = sizeof(hs)/sizeof(VectorRegister);
+ // counter for cycling through hs vector to avoid register moves between iterations
+ int h_cnt = 0;
+
+ // Load a-h registers from the memory pointed by state
+#if defined(VM_LITTLE_ENDIAN)
+ sha256_load_h_vec(a, e, state);
+#else
+ sha256_load_h_vec(d, h, state);
+#endif
+
+ // keep k loaded also during MultiBlock loops
+ Register k = R7;
+ assert(((uintptr_t)sha256_round_consts & 0xF) == 0, "k alignment");
+ load_const_optimized(k, (address)sha256_round_consts, R0);
+
+ // Avoiding redundant loads
+ if (multi_block) {
+ align(OptoLoopAlignment);
+ }
+ bind(sha_loop);
+#if defined(VM_LITTLE_ENDIAN)
+ sha256_deque(a, b, c, d);
+ sha256_deque(e, f, g, h);
+#else
+ sha256_deque(d, c, b, a);
+ sha256_deque(h, g, f, e);
+#endif
+
+ // Load 16 elements from w out of the loop.
+ // Order of the int values is Endianess specific.
+ VectorRegister w0 = VR17;
+ VectorRegister w1 = VR18;
+ VectorRegister w2 = VR19;
+ VectorRegister w3 = VR20;
+ static const VectorRegister ws[] = {w0, w1, w2, w3};
+ static const int total_ws = sizeof(ws)/sizeof(VectorRegister);
+
+ VectorRegister kpw0 = VR24;
+ VectorRegister kpw1 = VR25;
+ VectorRegister kpw2 = VR26;
+ VectorRegister kpw3 = VR27;
+ static const VectorRegister kpws[] = {kpw0, kpw1, kpw2, kpw3};
+ static const int total_kpws = sizeof(kpws)/sizeof(VectorRegister);
+
+ sha256_load_w_plus_k_vec(buf_in, ws, total_ws, k, kpws, total_kpws);
+
+ // Cycle through the first 16 elements
+ assert(total_ws == total_kpws, "Redesign the loop below");
+ for (int n = 0; n < total_ws; n++) {
+ VectorRegister vaux0 = VR21;
+ VectorRegister vaux1 = VR22;
+ VectorRegister vaux2 = VR23;
+
+ sha256_deque(kpws[n], vaux0, vaux1, vaux2);
+
+#if defined(VM_LITTLE_ENDIAN)
+ sha256_round(hs, total_hs, h_cnt, kpws[n]);
+ sha256_round(hs, total_hs, h_cnt, vaux0);
+ sha256_round(hs, total_hs, h_cnt, vaux1);
+ sha256_round(hs, total_hs, h_cnt, vaux2);
+#else
+ sha256_round(hs, total_hs, h_cnt, vaux2);
+ sha256_round(hs, total_hs, h_cnt, vaux1);
+ sha256_round(hs, total_hs, h_cnt, vaux0);
+ sha256_round(hs, total_hs, h_cnt, kpws[n]);
+#endif
+ }
+
+ Register tmp = R8;
+ // loop the 16th to the 64th iteration by 8 steps
+ li (tmp, (w_size - 16) / total_hs);
+ mtctr(tmp);
+
+ // j will be aligned to 4 for loading words.
+ // Whenever read, advance the pointer (e.g: when j is used in a function)
+ Register j = R8;
+ li (j, 16*4);
+
+ align(OptoLoopAlignment);
+ bind(core_loop);
+
+ // due to VectorRegister rotate, always iterate in multiples of total_hs
+ for (int n = 0; n < total_hs/4; n++) {
+ sha256_calc_4w(w0, w1, w2, w3, kpw0, kpw1, kpw2, kpw3, j, k);
+ sha256_round(hs, total_hs, h_cnt, kpw0);
+ sha256_round(hs, total_hs, h_cnt, kpw1);
+ sha256_round(hs, total_hs, h_cnt, kpw2);
+ sha256_round(hs, total_hs, h_cnt, kpw3);
+ }
+
+ bdnz (core_loop);
+
+ // Update hash state
+ sha256_update_sha_state(a, b, c, d, e, f, g, h, state);
+
+ if (multi_block) {
+ addi(buf_in, buf_in, buf_size);
+ addi(ofs, ofs, buf_size);
+ cmplw(CCR0, ofs, limit);
+ ble(CCR0, sha_loop);
+
+ // return ofs
+ mr(R3_RET, ofs);
+ }
+
+ // Restore non-volatile registers
+ for (int c = 0; c < nv_size; c++) {
+ Register tmp = R8;
+ li (tmp, (c - (nv_size)) * 16);
+ lvx(nv[c], tmp, R1);
+ }
+}
+
+
+/**********************************************************************
+ * SHA 512
+ *********************************************************************/
+
+void MacroAssembler::sha512_load_w_vec(const Register buf_in,
+ const VectorRegister* ws,
+ const int total_ws) {
+ Register tmp = R8;
+ VectorRegister vRb = VR8;
+ VectorRegister aux = VR9;
+ Label is_aligned, after_alignment;
+
+ andi_ (tmp, buf_in, 0xF);
+ beq (CCR0, is_aligned); // address ends with 0x0, not 0x8
+
+ // deal with unaligned addresses
+ lvx (ws[0], buf_in);
+ load_perm(vRb, buf_in);
+
+ for (int n = 1; n < total_ws; n++) {
+ VectorRegister w_cur = ws[n];
+ VectorRegister w_prev = ws[n-1];
+ addi (tmp, buf_in, n * 16);
+ lvx (w_cur, tmp);
+ vec_perm(w_prev, w_cur, vRb);
+ }
+ addi (tmp, buf_in, total_ws * 16);
+ lvx (aux, tmp);
+ vec_perm(ws[total_ws-1], aux, vRb);
+ b (after_alignment);
+
+ bind(is_aligned);
+ lvx (ws[0], buf_in);
+ for (int n = 1; n < total_ws; n++) {
+ VectorRegister w = ws[n];
+ addi (tmp, buf_in, n * 16);
+ lvx (w, tmp);
+ }
+
+ bind(after_alignment);
+}
+
+// Update hash state
+void MacroAssembler::sha512_update_sha_state(const Register state,
+ const VectorRegister* hs,
+ const int total_hs) {
+
+#if defined(VM_LITTLE_ENDIAN)
+ int start_idx = 0;
+#else
+ int start_idx = 1;
+#endif
+
+ // load initial hash from the memory pointed by state
+ VectorRegister ini_a = VR10;
+ VectorRegister ini_c = VR12;
+ VectorRegister ini_e = VR14;
+ VectorRegister ini_g = VR16;
+ static const VectorRegister inis[] = {ini_a, ini_c, ini_e, ini_g};
+ static const int total_inis = sizeof(inis)/sizeof(VectorRegister);
+
+ Label state_save_aligned, after_state_save_aligned;
+
+ Register addr = R7;
+ Register tmp = R8;
+ VectorRegister vRb = VR8;
+ VectorRegister aux = VR9;
+
+ andi_(tmp, state, 0xf);
+ beq(CCR0, state_save_aligned);
+ // deal with unaligned addresses
+
+ {
+ VectorRegister a = hs[0];
+ VectorRegister b_ = hs[1];
+ VectorRegister c = hs[2];
+ VectorRegister d = hs[3];
+ VectorRegister e = hs[4];
+ VectorRegister f = hs[5];
+ VectorRegister g = hs[6];
+ VectorRegister h = hs[7];
+ load_perm(vRb, state);
+ lvx (ini_a, state);
+ addi (addr, state, 16);
+
+ lvx (ini_c, addr);
+ addi (addr, state, 32);
+ vec_perm(ini_a, ini_c, vRb);
+
+ lvx (ini_e, addr);
+ addi (addr, state, 48);
+ vec_perm(ini_c, ini_e, vRb);
+
+ lvx (ini_g, addr);
+ addi (addr, state, 64);
+ vec_perm(ini_e, ini_g, vRb);
+
+ lvx (aux, addr);
+ vec_perm(ini_g, aux, vRb);
+
+#if defined(VM_LITTLE_ENDIAN)
+ xxmrgld(a->to_vsr(), b_->to_vsr(), a->to_vsr());
+ xxmrgld(c->to_vsr(), d->to_vsr(), c->to_vsr());
+ xxmrgld(e->to_vsr(), f->to_vsr(), e->to_vsr());
+ xxmrgld(g->to_vsr(), h->to_vsr(), g->to_vsr());
+#else
+ xxmrgld(b_->to_vsr(), a->to_vsr(), b_->to_vsr());
+ xxmrgld(d->to_vsr(), c->to_vsr(), d->to_vsr());
+ xxmrgld(f->to_vsr(), e->to_vsr(), f->to_vsr());
+ xxmrgld(h->to_vsr(), g->to_vsr(), h->to_vsr());
+#endif
+
+ for (int n = start_idx; n < total_hs; n += 2) {
+ VectorRegister h_cur = hs[n];
+ VectorRegister ini_cur = inis[n/2];
+
+ vaddudm(h_cur, ini_cur, h_cur);
+ }
+
+ for (int n = start_idx; n < total_hs; n += 2) {
+ VectorRegister h_cur = hs[n];
+
+ mfvrd (tmp, h_cur);
+#if defined(VM_LITTLE_ENDIAN)
+ std (tmp, 8*n + 8, state);
+#else
+ std (tmp, 8*n - 8, state);
+#endif
+ vsldoi (aux, h_cur, h_cur, 8);
+ mfvrd (tmp, aux);
+ std (tmp, 8*n + 0, state);
+ }
+
+ b (after_state_save_aligned);
+ }
+
+ bind(state_save_aligned);
+ {
+ for (int n = 0; n < total_hs; n += 2) {
+#if defined(VM_LITTLE_ENDIAN)
+ VectorRegister h_cur = hs[n];
+ VectorRegister h_next = hs[n+1];
+#else
+ VectorRegister h_cur = hs[n+1];
+ VectorRegister h_next = hs[n];
+#endif
+ VectorRegister ini_cur = inis[n/2];
+
+ if (n/2 == 0) {
+ lvx(ini_cur, state);
+ } else {
+ addi(addr, state, (n/2) * 16);
+ lvx(ini_cur, addr);
+ }
+ xxmrgld(h_cur->to_vsr(), h_next->to_vsr(), h_cur->to_vsr());
+ }
+
+ for (int n = start_idx; n < total_hs; n += 2) {
+ VectorRegister h_cur = hs[n];
+ VectorRegister ini_cur = inis[n/2];
+
+ vaddudm(h_cur, ini_cur, h_cur);
+ }
+
+ for (int n = start_idx; n < total_hs; n += 2) {
+ VectorRegister h_cur = hs[n];
+
+ if (n/2 == 0) {
+ stvx(h_cur, state);
+ } else {
+ addi(addr, state, (n/2) * 16);
+ stvx(h_cur, addr);
+ }
+ }
+ }
+
+ bind(after_state_save_aligned);
+}
+
+// Use h_cnt to cycle through hs elements but also increment it at the end
+void MacroAssembler::sha512_round(const VectorRegister* hs,
+ const int total_hs, int& h_cnt,
+ const VectorRegister kpw) {
+
+ // convenience registers: cycle from 0-7 downwards
+ const VectorRegister a = hs[(total_hs + 0 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister b = hs[(total_hs + 1 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister c = hs[(total_hs + 2 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister d = hs[(total_hs + 3 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister e = hs[(total_hs + 4 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister f = hs[(total_hs + 5 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister g = hs[(total_hs + 6 - (h_cnt % total_hs)) % total_hs];
+ const VectorRegister h = hs[(total_hs + 7 - (h_cnt % total_hs)) % total_hs];
+ // temporaries
+ const VectorRegister Ch = VR20;
+ const VectorRegister Maj = VR21;
+ const VectorRegister bsa = VR22;
+ const VectorRegister bse = VR23;
+ const VectorRegister tmp1 = VR24;
+ const VectorRegister tmp2 = VR25;
+
+ vsel (Ch, g, f, e);
+ vxor (Maj, a, b);
+ vshasigmad(bse, e, 1, 0xf);
+ vaddudm (tmp2, Ch, kpw);
+ vaddudm (tmp1, h, bse);
+ vsel (Maj, b, c, Maj);
+ vaddudm (tmp1, tmp1, tmp2);
+ vshasigmad(bsa, a, 1, 0);
+ vaddudm (tmp2, bsa, Maj);
+ vaddudm (d, d, tmp1);
+ vaddudm (h, tmp1, tmp2);
+
+ // advance vector pointer to the next iteration
+ h_cnt++;
+}
+
+void MacroAssembler::sha512_calc_2w(const VectorRegister w0,
+ const VectorRegister w1,
+ const VectorRegister w2,
+ const VectorRegister w3,
+ const VectorRegister w4,
+ const VectorRegister w5,
+ const VectorRegister w6,
+ const VectorRegister w7,
+ const VectorRegister kpw0,
+ const VectorRegister kpw1,
+ const Register j,
+ const VectorRegister vRb,
+ const Register k) {
+ // Temporaries
+ const VectorRegister VR_a = VR20;
+ const VectorRegister VR_b = VR21;
+ const VectorRegister VR_c = VR22;
+ const VectorRegister VR_d = VR23;
+
+ // load to k[j]
+ lvx (VR_a, j, k);
+ // advance j
+ addi (j, j, 16); // 16 bytes were read
+
+#if defined(VM_LITTLE_ENDIAN)
+ // v6 = w[j-15], w[j-14]
+ vperm (VR_b, w1, w0, vRb);
+ // v12 = w[j-7], w[j-6]
+ vperm (VR_c, w5, w4, vRb);
+#else
+ // v6 = w[j-15], w[j-14]
+ vperm (VR_b, w0, w1, vRb);
+ // v12 = w[j-7], w[j-6]
+ vperm (VR_c, w4, w5, vRb);
+#endif
+
+ // v6 = s0(w[j-15]) , s0(w[j-14])
+ vshasigmad (VR_b, VR_b, 0, 0);
+ // v5 = s1(w[j-2]) , s1(w[j-1])
+ vshasigmad (VR_d, w7, 0, 0xf);
+ // v6 = s0(w[j-15]) + w[j-7] , s0(w[j-14]) + w[j-6]
+ vaddudm (VR_b, VR_b, VR_c);
+ // v8 = s1(w[j-2]) + w[j-16] , s1(w[j-1]) + w[j-15]
+ vaddudm (VR_d, VR_d, w0);
+ // v9 = s0(w[j-15]) + w[j-7] + w[j-16] + s1(w[j-2]), // w[j]
+ // s0(w[j-14]) + w[j-6] + w[j-15] + s1(w[j-1]), // w[j+1]
+ vaddudm (VR_c, VR_d, VR_b);
+ // Updating w0 to w7 to hold the new previous 16 values from w.
+ vmr (w0, w1);
+ vmr (w1, w2);
+ vmr (w2, w3);
+ vmr (w3, w4);
+ vmr (w4, w5);
+ vmr (w5, w6);
+ vmr (w6, w7);
+ vmr (w7, VR_c);
+
+#if defined(VM_LITTLE_ENDIAN)
+ // store k + w to kpw0 (2 values at once)
+ vaddudm (kpw0, VR_c, VR_a);
+ // kpw1 holds (k + w)[1]
+ vsldoi (kpw1, kpw0, kpw0, 8);
+#else
+ // store k + w to kpw0 (2 values at once)
+ vaddudm (kpw1, VR_c, VR_a);
+ // kpw1 holds (k + w)[1]
+ vsldoi (kpw0, kpw1, kpw1, 8);
+#endif
+}
+
+void MacroAssembler::sha512_load_h_vec(const Register state,
+ const VectorRegister* hs,
+ const int total_hs) {
+#if defined(VM_LITTLE_ENDIAN)
+ VectorRegister a = hs[0];
+ VectorRegister g = hs[6];
+ int start_idx = 0;
+#else
+ VectorRegister a = hs[1];
+ VectorRegister g = hs[7];
+ int start_idx = 1;
+#endif
+
+ Register addr = R7;
+ VectorRegister vRb = VR8;
+ Register tmp = R8;
+ Label state_aligned, after_state_aligned;
+
+ andi_(tmp, state, 0xf);
+ beq(CCR0, state_aligned);
+
+ // deal with unaligned addresses
+ VectorRegister aux = VR9;
+
+ lvx(hs[start_idx], state);
+ load_perm(vRb, state);
+
+ for (int n = start_idx + 2; n < total_hs; n += 2) {
+ VectorRegister h_cur = hs[n];
+ VectorRegister h_prev2 = hs[n - 2];
+ addi(addr, state, (n/2) * 16);
+ lvx(h_cur, addr);
+ vec_perm(h_prev2, h_cur, vRb);
+ }
+ addi(addr, state, (total_hs/2) * 16);
+ lvx (aux, addr);
+ vec_perm(hs[total_hs - 2 + start_idx], aux, vRb);
+ b (after_state_aligned);
+
+ bind(state_aligned);
+
+ // deal with aligned addresses
+ lvx(hs[start_idx], state);
+
+ for (int n = start_idx + 2; n < total_hs; n += 2) {
+ VectorRegister h_cur = hs[n];
+ addi(addr, state, (n/2) * 16);
+ lvx(h_cur, addr);
+ }
+
+ bind(after_state_aligned);
+}
+
+static const uint64_t sha512_round_table[80] __attribute((aligned(16))) = {
+ 0x428a2f98d728ae22, 0x7137449123ef65cd,
+ 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc,
+ 0x3956c25bf348b538, 0x59f111f1b605d019,
+ 0x923f82a4af194f9b, 0xab1c5ed5da6d8118,
+ 0xd807aa98a3030242, 0x12835b0145706fbe,
+ 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2,
+ 0x72be5d74f27b896f, 0x80deb1fe3b1696b1,
+ 0x9bdc06a725c71235, 0xc19bf174cf692694,
+ 0xe49b69c19ef14ad2, 0xefbe4786384f25e3,
+ 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65,
+ 0x2de92c6f592b0275, 0x4a7484aa6ea6e483,
+ 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5,
+ 0x983e5152ee66dfab, 0xa831c66d2db43210,
+ 0xb00327c898fb213f, 0xbf597fc7beef0ee4,
+ 0xc6e00bf33da88fc2, 0xd5a79147930aa725,
+ 0x06ca6351e003826f, 0x142929670a0e6e70,
+ 0x27b70a8546d22ffc, 0x2e1b21385c26c926,
+ 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df,
+ 0x650a73548baf63de, 0x766a0abb3c77b2a8,
+ 0x81c2c92e47edaee6, 0x92722c851482353b,
+ 0xa2bfe8a14cf10364, 0xa81a664bbc423001,
+ 0xc24b8b70d0f89791, 0xc76c51a30654be30,
+ 0xd192e819d6ef5218, 0xd69906245565a910,
+ 0xf40e35855771202a, 0x106aa07032bbd1b8,
+ 0x19a4c116b8d2d0c8, 0x1e376c085141ab53,
+ 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8,
+ 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb,
+ 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3,
+ 0x748f82ee5defb2fc, 0x78a5636f43172f60,
+ 0x84c87814a1f0ab72, 0x8cc702081a6439ec,
+ 0x90befffa23631e28, 0xa4506cebde82bde9,
+ 0xbef9a3f7b2c67915, 0xc67178f2e372532b,
+ 0xca273eceea26619c, 0xd186b8c721c0c207,
+ 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178,
+ 0x06f067aa72176fba, 0x0a637dc5a2c898a6,
+ 0x113f9804bef90dae, 0x1b710b35131c471b,
+ 0x28db77f523047d84, 0x32caab7b40c72493,
+ 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c,
+ 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a,
+ 0x5fcb6fab3ad6faec, 0x6c44198c4a475817,
+};
+static const uint64_t *sha512_round_consts = sha512_round_table;
+
+// R3_ARG1 - byte[] Input string with padding but in Big Endian
+// R4_ARG2 - int[] SHA.state (at first, the root of primes)
+// R5_ARG3 - int offset
+// R6_ARG4 - int limit
+//
+// Internal Register usage:
+// R7 R8 R9 - volatile temporaries
+// VR0-VR7 - a-h
+// VR8 - vRb
+// VR9 - aux (highly volatile, use with care)
+// VR10-VR17 - w0-w7 | ini_a-ini_h
+// VR18 - vsp16 | kplusw0
+// VR19 - vsp32 | kplusw1
+// VR20-VR25 - sha512_calc_2w and sha512_round temporaries
+void MacroAssembler::sha512(bool multi_block) {
+ static const ssize_t buf_size = 128;
+ static const uint8_t w_size = sizeof(sha512_round_table)/sizeof(uint64_t);
+#ifdef AIX
+ // malloc provides 16 byte alignment
+ if (((uintptr_t)sha512_round_consts & 0xF) != 0) {
+ uint64_t *new_round_consts = (uint64_t*)malloc(sizeof(sha512_round_table));
+ guarantee(new_round_consts, "oom");
+ memcpy(new_round_consts, sha512_round_consts, sizeof(sha512_round_table));
+ sha512_round_consts = (const uint64_t*)new_round_consts;
+ }
+#endif
+
+ Register buf_in = R3_ARG1;
+ Register state = R4_ARG2;
+ Register ofs = R5_ARG3;
+ Register limit = R6_ARG4;
+
+ Label sha_loop, core_loop;
+
+ // Save non-volatile vector registers in the red zone
+ static const VectorRegister nv[] = {
+ VR20, VR21, VR22, VR23, VR24, VR25/*, VR26, VR27, VR28, VR29, VR30, VR31*/
+ };
+ static const uint8_t nv_size = sizeof(nv) / sizeof (VectorRegister);
+
+ for (int c = 0; c < nv_size; c++) {
+ Register idx = R7;
+ li (idx, (c - (nv_size)) * 16);
+ stvx(nv[c], idx, R1);
+ }
+
+ // Load hash state to registers
+ VectorRegister a = VR0;
+ VectorRegister b = VR1;
+ VectorRegister c = VR2;
+ VectorRegister d = VR3;
+ VectorRegister e = VR4;
+ VectorRegister f = VR5;
+ VectorRegister g = VR6;
+ VectorRegister h = VR7;
+ static const VectorRegister hs[] = {a, b, c, d, e, f, g, h};
+ static const int total_hs = sizeof(hs)/sizeof(VectorRegister);
+ // counter for cycling through hs vector to avoid register moves between iterations
+ int h_cnt = 0;
+
+ // Load a-h registers from the memory pointed by state
+ sha512_load_h_vec(state, hs, total_hs);
+
+ Register k = R9;
+ assert(((uintptr_t)sha512_round_consts & 0xF) == 0, "k alignment");
+ load_const_optimized(k, (address)sha512_round_consts, R0);
+
+ if (multi_block) {
+ align(OptoLoopAlignment);
+ }
+ bind(sha_loop);
+
+ for (int n = 0; n < total_hs; n += 2) {
+#if defined(VM_LITTLE_ENDIAN)
+ VectorRegister h_cur = hs[n];
+ VectorRegister h_next = hs[n + 1];
+#else
+ VectorRegister h_cur = hs[n + 1];
+ VectorRegister h_next = hs[n];
+#endif
+ vsldoi (h_next, h_cur, h_cur, 8);
+ }
+
+ // Load 16 elements from w out of the loop.
+ // Order of the long values is Endianess specific.
+ VectorRegister w0 = VR10;
+ VectorRegister w1 = VR11;
+ VectorRegister w2 = VR12;
+ VectorRegister w3 = VR13;
+ VectorRegister w4 = VR14;
+ VectorRegister w5 = VR15;
+ VectorRegister w6 = VR16;
+ VectorRegister w7 = VR17;
+ static const VectorRegister ws[] = {w0, w1, w2, w3, w4, w5, w6, w7};
+ static const int total_ws = sizeof(ws)/sizeof(VectorRegister);
+
+ // Load 16 w into vectors and setup vsl for vperm
+ sha512_load_w_vec(buf_in, ws, total_ws);
+
+#if defined(VM_LITTLE_ENDIAN)
+ VectorRegister vsp16 = VR18;
+ VectorRegister vsp32 = VR19;
+ VectorRegister shiftarg = VR9;
+
+ vspltisw(vsp16, 8);
+ vspltisw(shiftarg, 1);
+ vsl (vsp16, vsp16, shiftarg);
+ vsl (vsp32, vsp16, shiftarg);
+
+ VectorRegister vsp8 = VR9;
+ vspltish(vsp8, 8);
+
+ // Convert input from Big Endian to Little Endian
+ for (int c = 0; c < total_ws; c++) {
+ VectorRegister w = ws[c];
+ vrlh (w, w, vsp8);
+ }
+ for (int c = 0; c < total_ws; c++) {
+ VectorRegister w = ws[c];
+ vrlw (w, w, vsp16);
+ }
+ for (int c = 0; c < total_ws; c++) {
+ VectorRegister w = ws[c];
+ vrld (w, w, vsp32);
+ }
+#endif
+
+ Register Rb = R10;
+ VectorRegister vRb = VR8;
+ li (Rb, 8);
+ load_perm(vRb, Rb);
+
+ VectorRegister kplusw0 = VR18;
+ VectorRegister kplusw1 = VR19;
+
+ Register addr = R7;
+
+ for (int n = 0; n < total_ws; n++) {
+ VectorRegister w = ws[n];
+
+ if (n == 0) {
+ lvx (kplusw0, k);
+ } else {
+ addi (addr, k, n * 16);
+ lvx (kplusw0, addr);
+ }
+#if defined(VM_LITTLE_ENDIAN)
+ vaddudm(kplusw0, kplusw0, w);
+ vsldoi (kplusw1, kplusw0, kplusw0, 8);
+#else
+ vaddudm(kplusw1, kplusw0, w);
+ vsldoi (kplusw0, kplusw1, kplusw1, 8);
+#endif
+
+ sha512_round(hs, total_hs, h_cnt, kplusw0);
+ sha512_round(hs, total_hs, h_cnt, kplusw1);
+ }
+
+ Register tmp = R8;
+ li (tmp, (w_size-16)/total_hs);
+ mtctr (tmp);
+ // j will be aligned to 4 for loading words.
+ // Whenever read, advance the pointer (e.g: when j is used in a function)
+ Register j = tmp;
+ li (j, 8*16);
+
+ align(OptoLoopAlignment);
+ bind(core_loop);
+
+ // due to VectorRegister rotate, always iterate in multiples of total_hs
+ for (int n = 0; n < total_hs/2; n++) {
+ sha512_calc_2w(w0, w1, w2, w3, w4, w5, w6, w7, kplusw0, kplusw1, j, vRb, k);
+ sha512_round(hs, total_hs, h_cnt, kplusw0);
+ sha512_round(hs, total_hs, h_cnt, kplusw1);
+ }
+
+ bdnz (core_loop);
+
+ sha512_update_sha_state(state, hs, total_hs);
+
+ if (multi_block) {
+ addi(buf_in, buf_in, buf_size);
+ addi(ofs, ofs, buf_size);
+ cmplw(CCR0, ofs, limit);
+ ble(CCR0, sha_loop);
+
+ // return ofs
+ mr(R3_RET, ofs);
+ }
+
+ // Restore non-volatile registers
+ for (int c = 0; c < nv_size; c++) {
+ Register idx = R7;
+ li (idx, (c - (nv_size)) * 16);
+ lvx(nv[c], idx, R1);
+ }
+}
diff --git a/src/cpu/ppc/vm/stubGenerator_ppc.cpp b/src/cpu/ppc/vm/stubGenerator_ppc.cpp
index 12dd846ce..7f2c4427f 100644
--- a/src/cpu/ppc/vm/stubGenerator_ppc.cpp
+++ b/src/cpu/ppc/vm/stubGenerator_ppc.cpp
@@ -2224,7 +2224,7 @@ class StubGenerator: public StubCodeGenerator {
return start;
}
- // Arguments for generated stub (little endian only):
+ // Arguments for generated stub:
// R3_ARG1 - source byte array address
// R4_ARG2 - destination byte array address
// R5_ARG3 - round key array
@@ -2243,7 +2243,6 @@ class StubGenerator: public StubCodeGenerator {
Register keylen = R8;
Register temp = R9;
Register keypos = R10;
- Register hex = R11;
Register fifteen = R12;
VectorRegister vRet = VR0;
@@ -2263,164 +2262,170 @@ class StubGenerator: public StubCodeGenerator {
VectorRegister vTmp3 = VR11;
VectorRegister vTmp4 = VR12;
- VectorRegister vLow = VR13;
- VectorRegister vHigh = VR14;
-
- __ li (hex, 16);
__ li (fifteen, 15);
- __ vspltisb (fSplt, 0x0f);
// load unaligned from[0-15] to vsRet
__ lvx (vRet, from);
__ lvx (vTmp1, fifteen, from);
__ lvsl (fromPerm, from);
+#ifdef VM_LITTLE_ENDIAN
+ __ vspltisb (fSplt, 0x0f);
__ vxor (fromPerm, fromPerm, fSplt);
+#endif
__ vperm (vRet, vRet, vTmp1, fromPerm);
// load keylen (44 or 52 or 60)
__ lwz (keylen, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT), key);
// to load keys
- __ lvsr (keyPerm, key);
- __ vxor (vTmp2, vTmp2, vTmp2);
+ __ load_perm (keyPerm, key);
+#ifdef VM_LITTLE_ENDIAN
__ vspltisb (vTmp2, -16);
__ vrld (keyPerm, keyPerm, vTmp2);
__ vrld (keyPerm, keyPerm, vTmp2);
__ vsldoi (keyPerm, keyPerm, keyPerm, 8);
+#endif
- // load the 1st round key to vKey1
- __ li (keypos, 0);
+ // load the 1st round key to vTmp1
+ __ lvx (vTmp1, key);
+ __ li (keypos, 16);
__ lvx (vKey1, keypos, key);
- __ addi (keypos, keypos, 16);
- __ lvx (vTmp1, keypos, key);
- __ vperm (vKey1, vTmp1, vKey1, keyPerm);
+ __ vec_perm (vTmp1, vKey1, keyPerm);
// 1st round
- __ vxor (vRet, vRet, vKey1);
+ __ vxor (vRet, vRet, vTmp1);
// load the 2nd round key to vKey1
- __ addi (keypos, keypos, 16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey1, vTmp2, vTmp1, keyPerm);
+ __ li (keypos, 32);
+ __ lvx (vKey2, keypos, key);
+ __ vec_perm (vKey1, vKey2, keyPerm);
// load the 3rd round key to vKey2
- __ addi (keypos, keypos, 16);
- __ lvx (vTmp1, keypos, key);
- __ vperm (vKey2, vTmp1, vTmp2, keyPerm);
+ __ li (keypos, 48);
+ __ lvx (vKey3, keypos, key);
+ __ vec_perm (vKey2, vKey3, keyPerm);
// load the 4th round key to vKey3
- __ addi (keypos, keypos, 16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey3, vTmp2, vTmp1, keyPerm);
+ __ li (keypos, 64);
+ __ lvx (vKey4, keypos, key);
+ __ vec_perm (vKey3, vKey4, keyPerm);
// load the 5th round key to vKey4
- __ addi (keypos, keypos, 16);
+ __ li (keypos, 80);
__ lvx (vTmp1, keypos, key);
- __ vperm (vKey4, vTmp1, vTmp2, keyPerm);
+ __ vec_perm (vKey4, vTmp1, keyPerm);
// 2nd - 5th rounds
- __ vcipher (vRet, vRet, vKey1);
- __ vcipher (vRet, vRet, vKey2);
- __ vcipher (vRet, vRet, vKey3);
- __ vcipher (vRet, vRet, vKey4);
+ __ vcipher (vRet, vRet, vKey1);
+ __ vcipher (vRet, vRet, vKey2);
+ __ vcipher (vRet, vRet, vKey3);
+ __ vcipher (vRet, vRet, vKey4);
// load the 6th round key to vKey1
- __ addi (keypos, keypos, 16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey1, vTmp2, vTmp1, keyPerm);
+ __ li (keypos, 96);
+ __ lvx (vKey2, keypos, key);
+ __ vec_perm (vKey1, vTmp1, vKey2, keyPerm);
// load the 7th round key to vKey2
- __ addi (keypos, keypos, 16);
- __ lvx (vTmp1, keypos, key);
- __ vperm (vKey2, vTmp1, vTmp2, keyPerm);
+ __ li (keypos, 112);
+ __ lvx (vKey3, keypos, key);
+ __ vec_perm (vKey2, vKey3, keyPerm);
// load the 8th round key to vKey3
- __ addi (keypos, keypos, 16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey3, vTmp2, vTmp1, keyPerm);
+ __ li (keypos, 128);
+ __ lvx (vKey4, keypos, key);
+ __ vec_perm (vKey3, vKey4, keyPerm);
// load the 9th round key to vKey4
- __ addi (keypos, keypos, 16);
+ __ li (keypos, 144);
__ lvx (vTmp1, keypos, key);
- __ vperm (vKey4, vTmp1, vTmp2, keyPerm);
+ __ vec_perm (vKey4, vTmp1, keyPerm);
// 6th - 9th rounds
- __ vcipher (vRet, vRet, vKey1);
- __ vcipher (vRet, vRet, vKey2);
- __ vcipher (vRet, vRet, vKey3);
- __ vcipher (vRet, vRet, vKey4);
+ __ vcipher (vRet, vRet, vKey1);
+ __ vcipher (vRet, vRet, vKey2);
+ __ vcipher (vRet, vRet, vKey3);
+ __ vcipher (vRet, vRet, vKey4);
// load the 10th round key to vKey1
- __ addi (keypos, keypos, 16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey1, vTmp2, vTmp1, keyPerm);
+ __ li (keypos, 160);
+ __ lvx (vKey2, keypos, key);
+ __ vec_perm (vKey1, vTmp1, vKey2, keyPerm);
// load the 11th round key to vKey2
- __ addi (keypos, keypos, 16);
+ __ li (keypos, 176);
__ lvx (vTmp1, keypos, key);
- __ vperm (vKey2, vTmp1, vTmp2, keyPerm);
+ __ vec_perm (vKey2, vTmp1, keyPerm);
// if all round keys are loaded, skip next 4 rounds
__ cmpwi (CCR0, keylen, 44);
__ beq (CCR0, L_doLast);
// 10th - 11th rounds
- __ vcipher (vRet, vRet, vKey1);
- __ vcipher (vRet, vRet, vKey2);
+ __ vcipher (vRet, vRet, vKey1);
+ __ vcipher (vRet, vRet, vKey2);
// load the 12th round key to vKey1
- __ addi (keypos, keypos, 16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey1, vTmp2, vTmp1, keyPerm);
+ __ li (keypos, 192);
+ __ lvx (vKey2, keypos, key);
+ __ vec_perm (vKey1, vTmp1, vKey2, keyPerm);
// load the 13th round key to vKey2
- __ addi (keypos, keypos, 16);
+ __ li (keypos, 208);
__ lvx (vTmp1, keypos, key);
- __ vperm (vKey2, vTmp1, vTmp2, keyPerm);
+ __ vec_perm (vKey2, vTmp1, keyPerm);
// if all round keys are loaded, skip next 2 rounds
__ cmpwi (CCR0, keylen, 52);
__ beq (CCR0, L_doLast);
// 12th - 13th rounds
- __ vcipher (vRet, vRet, vKey1);
- __ vcipher (vRet, vRet, vKey2);
+ __ vcipher (vRet, vRet, vKey1);
+ __ vcipher (vRet, vRet, vKey2);
// load the 14th round key to vKey1
- __ addi (keypos, keypos, 16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey1, vTmp2, vTmp1, keyPerm);
+ __ li (keypos, 224);
+ __ lvx (vKey2, keypos, key);
+ __ vec_perm (vKey1, vTmp1, vKey2, keyPerm);
// load the 15th round key to vKey2
- __ addi (keypos, keypos, 16);
+ __ li (keypos, 240);
__ lvx (vTmp1, keypos, key);
- __ vperm (vKey2, vTmp1, vTmp2, keyPerm);
+ __ vec_perm (vKey2, vTmp1, keyPerm);
__ bind(L_doLast);
// last two rounds
- __ vcipher (vRet, vRet, vKey1);
- __ vcipherlast (vRet, vRet, vKey2);
-
- __ neg (temp, to);
- __ lvsr (toPerm, temp);
- __ vspltisb (vTmp2, -1);
- __ vxor (vTmp1, vTmp1, vTmp1);
- __ vperm (vTmp2, vTmp2, vTmp1, toPerm);
- __ vxor (toPerm, toPerm, fSplt);
+ __ vcipher (vRet, vRet, vKey1);
+ __ vcipherlast (vRet, vRet, vKey2);
+
+ // store result (unaligned)
+#ifdef VM_LITTLE_ENDIAN
+ __ lvsl (toPerm, to);
+#else
+ __ lvsr (toPerm, to);
+#endif
+ __ vspltisb (vTmp3, -1);
+ __ vspltisb (vTmp4, 0);
__ lvx (vTmp1, to);
- __ vperm (vRet, vRet, vRet, toPerm);
- __ vsel (vTmp1, vTmp1, vRet, vTmp2);
- __ lvx (vTmp4, fifteen, to);
+ __ lvx (vTmp2, fifteen, to);
+#ifdef VM_LITTLE_ENDIAN
+ __ vperm (vTmp3, vTmp3, vTmp4, toPerm); // generate select mask
+ __ vxor (toPerm, toPerm, fSplt); // swap bytes
+#else
+ __ vperm (vTmp3, vTmp4, vTmp3, toPerm); // generate select mask
+#endif
+ __ vperm (vTmp4, vRet, vRet, toPerm); // rotate data
+ __ vsel (vTmp2, vTmp4, vTmp2, vTmp3);
+ __ vsel (vTmp1, vTmp1, vTmp4, vTmp3);
+ __ stvx (vTmp2, fifteen, to); // store this one first (may alias)
__ stvx (vTmp1, to);
- __ vsel (vRet, vRet, vTmp4, vTmp2);
- __ stvx (vRet, fifteen, to);
__ blr();
return start;
}
- // Arguments for generated stub (little endian only):
+ // Arguments for generated stub:
// R3_ARG1 - source byte array address
// R4_ARG2 - destination byte array address
// R5_ARG3 - K (key) in little endian int array
@@ -2442,7 +2447,6 @@ class StubGenerator: public StubCodeGenerator {
Register keylen = R8;
Register temp = R9;
Register keypos = R10;
- Register hex = R11;
Register fifteen = R12;
VectorRegister vRet = VR0;
@@ -2463,30 +2467,30 @@ class StubGenerator: public StubCodeGenerator {
VectorRegister vTmp3 = VR12;
VectorRegister vTmp4 = VR13;
- VectorRegister vLow = VR14;
- VectorRegister vHigh = VR15;
-
- __ li (hex, 16);
__ li (fifteen, 15);
- __ vspltisb (fSplt, 0x0f);
// load unaligned from[0-15] to vsRet
__ lvx (vRet, from);
__ lvx (vTmp1, fifteen, from);
__ lvsl (fromPerm, from);
+#ifdef VM_LITTLE_ENDIAN
+ __ vspltisb (fSplt, 0x0f);
__ vxor (fromPerm, fromPerm, fSplt);
+#endif
__ vperm (vRet, vRet, vTmp1, fromPerm); // align [and byte swap in LE]
// load keylen (44 or 52 or 60)
__ lwz (keylen, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT), key);
// to load keys
- __ lvsr (keyPerm, key);
+ __ load_perm (keyPerm, key);
+#ifdef VM_LITTLE_ENDIAN
__ vxor (vTmp2, vTmp2, vTmp2);
__ vspltisb (vTmp2, -16);
__ vrld (keyPerm, keyPerm, vTmp2);
__ vrld (keyPerm, keyPerm, vTmp2);
__ vsldoi (keyPerm, keyPerm, keyPerm, 8);
+#endif
__ cmpwi (CCR0, keylen, 44);
__ beq (CCR0, L_do44);
@@ -2494,32 +2498,32 @@ class StubGenerator: public StubCodeGenerator {
__ cmpwi (CCR0, keylen, 52);
__ beq (CCR0, L_do52);
- // load the 15th round key to vKey11
+ // load the 15th round key to vKey1
__ li (keypos, 240);
- __ lvx (vTmp1, keypos, key);
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey1, vTmp1, vTmp2, keyPerm);
+ __ lvx (vKey1, keypos, key);
+ __ li (keypos, 224);
+ __ lvx (vKey2, keypos, key);
+ __ vec_perm (vKey1, vKey2, vKey1, keyPerm);
- // load the 14th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp1, keypos, key);
- __ vperm (vKey2, vTmp2, vTmp1, keyPerm);
+ // load the 14th round key to vKey2
+ __ li (keypos, 208);
+ __ lvx (vKey3, keypos, key);
+ __ vec_perm (vKey2, vKey3, vKey2, keyPerm);
- // load the 13th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey3, vTmp1, vTmp2, keyPerm);
+ // load the 13th round key to vKey3
+ __ li (keypos, 192);
+ __ lvx (vKey4, keypos, key);
+ __ vec_perm (vKey3, vKey4, vKey3, keyPerm);
- // load the 12th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp1, keypos, key);
- __ vperm (vKey4, vTmp2, vTmp1, keyPerm);
+ // load the 12th round key to vKey4
+ __ li (keypos, 176);
+ __ lvx (vKey5, keypos, key);
+ __ vec_perm (vKey4, vKey5, vKey4, keyPerm);
- // load the 11th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey5, vTmp1, vTmp2, keyPerm);
+ // load the 11th round key to vKey5
+ __ li (keypos, 160);
+ __ lvx (vTmp1, keypos, key);
+ __ vec_perm (vKey5, vTmp1, vKey5, keyPerm);
// 1st - 5th rounds
__ vxor (vRet, vRet, vKey1);
@@ -2532,22 +2536,22 @@ class StubGenerator: public StubCodeGenerator {
__ bind (L_do52);
- // load the 13th round key to vKey11
+ // load the 13th round key to vKey1
__ li (keypos, 208);
- __ lvx (vTmp1, keypos, key);
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey1, vTmp1, vTmp2, keyPerm);
+ __ lvx (vKey1, keypos, key);
+ __ li (keypos, 192);
+ __ lvx (vKey2, keypos, key);
+ __ vec_perm (vKey1, vKey2, vKey1, keyPerm);
- // load the 12th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp1, keypos, key);
- __ vperm (vKey2, vTmp2, vTmp1, keyPerm);
+ // load the 12th round key to vKey2
+ __ li (keypos, 176);
+ __ lvx (vKey3, keypos, key);
+ __ vec_perm (vKey2, vKey3, vKey2, keyPerm);
- // load the 11th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey3, vTmp1, vTmp2, keyPerm);
+ // load the 11th round key to vKey3
+ __ li (keypos, 160);
+ __ lvx (vTmp1, keypos, key);
+ __ vec_perm (vKey3, vTmp1, vKey3, keyPerm);
// 1st - 3rd rounds
__ vxor (vRet, vRet, vKey1);
@@ -2558,42 +2562,42 @@ class StubGenerator: public StubCodeGenerator {
__ bind (L_do44);
- // load the 11th round key to vKey11
+ // load the 11th round key to vKey1
__ li (keypos, 176);
+ __ lvx (vKey1, keypos, key);
+ __ li (keypos, 160);
__ lvx (vTmp1, keypos, key);
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey1, vTmp1, vTmp2, keyPerm);
+ __ vec_perm (vKey1, vTmp1, vKey1, keyPerm);
// 1st round
__ vxor (vRet, vRet, vKey1);
__ bind (L_doLast);
- // load the 10th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp1, keypos, key);
- __ vperm (vKey1, vTmp2, vTmp1, keyPerm);
+ // load the 10th round key to vKey1
+ __ li (keypos, 144);
+ __ lvx (vKey2, keypos, key);
+ __ vec_perm (vKey1, vKey2, vTmp1, keyPerm);
- // load the 9th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey2, vTmp1, vTmp2, keyPerm);
+ // load the 9th round key to vKey2
+ __ li (keypos, 128);
+ __ lvx (vKey3, keypos, key);
+ __ vec_perm (vKey2, vKey3, vKey2, keyPerm);
- // load the 8th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp1, keypos, key);
- __ vperm (vKey3, vTmp2, vTmp1, keyPerm);
+ // load the 8th round key to vKey3
+ __ li (keypos, 112);
+ __ lvx (vKey4, keypos, key);
+ __ vec_perm (vKey3, vKey4, vKey3, keyPerm);
- // load the 7th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey4, vTmp1, vTmp2, keyPerm);
+ // load the 7th round key to vKey4
+ __ li (keypos, 96);
+ __ lvx (vKey5, keypos, key);
+ __ vec_perm (vKey4, vKey5, vKey4, keyPerm);
- // load the 6th round key to vKey10
- __ addi (keypos, keypos, -16);
+ // load the 6th round key to vKey5
+ __ li (keypos, 80);
__ lvx (vTmp1, keypos, key);
- __ vperm (vKey5, vTmp2, vTmp1, keyPerm);
+ __ vec_perm (vKey5, vTmp1, vKey5, keyPerm);
// last 10th - 6th rounds
__ vncipher (vRet, vRet, vKey1);
@@ -2602,30 +2606,29 @@ class StubGenerator: public StubCodeGenerator {
__ vncipher (vRet, vRet, vKey4);
__ vncipher (vRet, vRet, vKey5);
- // load the 5th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey1, vTmp1, vTmp2, keyPerm);
+ // load the 5th round key to vKey1
+ __ li (keypos, 64);
+ __ lvx (vKey2, keypos, key);
+ __ vec_perm (vKey1, vKey2, vTmp1, keyPerm);
- // load the 4th round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp1, keypos, key);
- __ vperm (vKey2, vTmp2, vTmp1, keyPerm);
+ // load the 4th round key to vKey2
+ __ li (keypos, 48);
+ __ lvx (vKey3, keypos, key);
+ __ vec_perm (vKey2, vKey3, vKey2, keyPerm);
- // load the 3rd round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey3, vTmp1, vTmp2, keyPerm);
+ // load the 3rd round key to vKey3
+ __ li (keypos, 32);
+ __ lvx (vKey4, keypos, key);
+ __ vec_perm (vKey3, vKey4, vKey3, keyPerm);
- // load the 2nd round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp1, keypos, key);
- __ vperm (vKey4, vTmp2, vTmp1, keyPerm);
+ // load the 2nd round key to vKey4
+ __ li (keypos, 16);
+ __ lvx (vKey5, keypos, key);
+ __ vec_perm (vKey4, vKey5, vKey4, keyPerm);
- // load the 1st round key to vKey10
- __ addi (keypos, keypos, -16);
- __ lvx (vTmp2, keypos, key);
- __ vperm (vKey5, vTmp1, vTmp2, keyPerm);
+ // load the 1st round key to vKey5
+ __ lvx (vTmp1, key);
+ __ vec_perm (vKey5, vTmp1, vKey5, keyPerm);
// last 5th - 1th rounds
__ vncipher (vRet, vRet, vKey1);
@@ -2634,24 +2637,54 @@ class StubGenerator: public StubCodeGenerator {
__ vncipher (vRet, vRet, vKey4);
__ vncipherlast (vRet, vRet, vKey5);
- __ neg (temp, to);
- __ lvsr (toPerm, temp);
- __ vspltisb (vTmp2, -1);
- __ vxor (vTmp1, vTmp1, vTmp1);
- __ vperm (vTmp2, vTmp2, vTmp1, toPerm);
- __ vxor (toPerm, toPerm, fSplt);
+ // store result (unaligned)
+#ifdef VM_LITTLE_ENDIAN
+ __ lvsl (toPerm, to);
+#else
+ __ lvsr (toPerm, to);
+#endif
+ __ vspltisb (vTmp3, -1);
+ __ vspltisb (vTmp4, 0);
__ lvx (vTmp1, to);
- __ vperm (vRet, vRet, vRet, toPerm);
- __ vsel (vTmp1, vTmp1, vRet, vTmp2);
- __ lvx (vTmp4, fifteen, to);
+ __ lvx (vTmp2, fifteen, to);
+#ifdef VM_LITTLE_ENDIAN
+ __ vperm (vTmp3, vTmp3, vTmp4, toPerm); // generate select mask
+ __ vxor (toPerm, toPerm, fSplt); // swap bytes
+#else
+ __ vperm (vTmp3, vTmp4, vTmp3, toPerm); // generate select mask
+#endif
+ __ vperm (vTmp4, vRet, vRet, toPerm); // rotate data
+ __ vsel (vTmp2, vTmp4, vTmp2, vTmp3);
+ __ vsel (vTmp1, vTmp1, vTmp4, vTmp3);
+ __ stvx (vTmp2, fifteen, to); // store this one first (may alias)
__ stvx (vTmp1, to);
- __ vsel (vRet, vRet, vTmp4, vTmp2);
- __ stvx (vRet, fifteen, to);
__ blr();
return start;
}
+ address generate_sha256_implCompress(bool multi_block, const char *name) {
+ assert(UseSHA, "need SHA instructions");
+ StubCodeMark mark(this, "StubRoutines", name);
+ address start = __ function_entry();
+
+ __ sha256 (multi_block);
+
+ __ blr();
+ return start;
+ }
+
+ address generate_sha512_implCompress(bool multi_block, const char *name) {
+ assert(UseSHA, "need SHA instructions");
+ StubCodeMark mark(this, "StubRoutines", name);
+ address start = __ function_entry();
+
+ __ sha512 (multi_block);
+
+ __ blr();
+ return start;
+ }
+
void generate_arraycopy_stubs() {
// Note: the disjoint stubs must be generated first, some of
// the conjoint stubs use them.
@@ -2881,6 +2914,15 @@ class StubGenerator: public StubCodeGenerator {
StubRoutines::_montgomerySquare
= CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square);
}
+
+ if (UseSHA256Intrinsics) {
+ StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
+ StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
+ }
+ if (UseSHA512Intrinsics) {
+ StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress");
+ StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB");
+ }
}
public:
diff --git a/src/cpu/ppc/vm/stubRoutines_ppc_64.hpp b/src/cpu/ppc/vm/stubRoutines_ppc_64.hpp
index 3655c1f78..e6e90ae76 100644
--- a/src/cpu/ppc/vm/stubRoutines_ppc_64.hpp
+++ b/src/cpu/ppc/vm/stubRoutines_ppc_64.hpp
@@ -34,7 +34,7 @@ static bool returns_to_call_stub(address return_pc) { return return_pc == _call_
enum platform_dependent_constants {
code_size1 = 20000, // simply increase if too small (assembler will crash if too small)
- code_size2 = 20000 // simply increase if too small (assembler will crash if too small)
+ code_size2 = 24000 // simply increase if too small (assembler will crash if too small)
};
// CRC32 Intrinsics.
diff --git a/src/cpu/ppc/vm/vm_version_ppc.cpp b/src/cpu/ppc/vm/vm_version_ppc.cpp
index 6761c02a2..bbc52691b 100644
--- a/src/cpu/ppc/vm/vm_version_ppc.cpp
+++ b/src/cpu/ppc/vm/vm_version_ppc.cpp
@@ -110,7 +110,7 @@ void VM_Version::initialize() {
// Create and print feature-string.
char buf[(num_features+1) * 16]; // Max 16 chars per feature.
jio_snprintf(buf, sizeof(buf),
- "ppc64%s%s%s%s%s%s%s%s%s%s%s%s%s",
+ "ppc64%s%s%s%s%s%s%s%s%s%s%s%s%s%s",
(has_fsqrt() ? " fsqrt" : ""),
(has_isel() ? " isel" : ""),
(has_lxarxeh() ? " lxarxeh" : ""),
@@ -124,7 +124,8 @@ void VM_Version::initialize() {
(has_vcipher() ? " aes" : ""),
(has_vpmsumb() ? " vpmsumb" : ""),
(has_mfdscr() ? " mfdscr" : ""),
- (has_vsx() ? " vsx" : "")
+ (has_vsx() ? " vsx" : ""),
+ (has_vshasig() ? " sha" : "")
// Make sure number of %s matches num_features!
);
_features_str = strdup(buf);
@@ -173,7 +174,6 @@ void VM_Version::initialize() {
}
// The AES intrinsic stubs require AES instruction support.
-#if defined(VM_LITTLE_ENDIAN)
if (has_vcipher()) {
if (FLAG_IS_DEFAULT(UseAES)) {
UseAES = true;
@@ -194,29 +194,43 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseAESIntrinsics, false);
}
-#else
- if (UseAES) {
- warning("AES instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseAES, false);
- }
- if (UseAESIntrinsics) {
- if (!FLAG_IS_DEFAULT(UseAESIntrinsics))
- warning("AES intrinsics are not available on this CPU");
- FLAG_SET_DEFAULT(UseAESIntrinsics, false);
- }
-#endif
-
- if (UseSHA) {
- warning("SHA instructions are not available on this CPU");
+ if (has_vshasig()) {
+ if (FLAG_IS_DEFAULT(UseSHA)) {
+ UseSHA = true;
+ }
+ } else if (UseSHA) {
+ if (!FLAG_IS_DEFAULT(UseSHA))
+ warning("SHA instructions are not available on this CPU");
FLAG_SET_DEFAULT(UseSHA, false);
}
- if (UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics) {
- warning("SHA intrinsics are not available on this CPU");
+
+ if (UseSHA1Intrinsics) {
+ warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
+ }
+
+ if (UseSHA && has_vshasig()) {
+ if (FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
+ }
+ } else if (UseSHA256Intrinsics) {
+ warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU.");
FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
+ }
+
+ if (UseSHA && has_vshasig()) {
+ if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
+ }
+ } else if (UseSHA512Intrinsics) {
+ warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU.");
FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
}
+ if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA, false);
+ }
+
if (FLAG_IS_DEFAULT(UseMontgomeryMultiplyIntrinsic)) {
UseMontgomeryMultiplyIntrinsic = true;
}
@@ -503,6 +517,7 @@ void VM_Version::determine_features() {
a->vpmsumb(VR0, VR1, VR2); // code[12] -> vpmsumb
a->mfdscr(R0); // code[13] -> mfdscr
a->lxvd2x(VSR0, R3_ARG1); // code[14] -> vsx
+ a->vshasigmaw(VR0, VR1, 1, 0xF); // code[15] -> vshasig
a->blr();
// Emit function to set one cache line to zero. Emit function descriptor and get pointer to it.
@@ -551,6 +566,7 @@ void VM_Version::determine_features() {
if (code[feature_cntr++]) features |= vpmsumb_m;
if (code[feature_cntr++]) features |= mfdscr_m;
if (code[feature_cntr++]) features |= vsx_m;
+ if (code[feature_cntr++]) features |= vshasig_m;
// Print the detection code.
if (PrintAssembly) {
diff --git a/src/cpu/ppc/vm/vm_version_ppc.hpp b/src/cpu/ppc/vm/vm_version_ppc.hpp
index 6245e1c64..86243a1a8 100644
--- a/src/cpu/ppc/vm/vm_version_ppc.hpp
+++ b/src/cpu/ppc/vm/vm_version_ppc.hpp
@@ -47,6 +47,7 @@ protected:
vpmsumb,
mfdscr,
vsx,
+ vshasig,
num_features // last entry to count features
};
enum Feature_Flag_Set {
@@ -63,6 +64,7 @@ protected:
dcba_m = (1 << dcba ),
lqarx_m = (1 << lqarx ),
vcipher_m = (1 << vcipher),
+ vshasig_m = (1 << vshasig),
vpmsumb_m = (1 << vpmsumb),
mfdscr_m = (1 << mfdscr ),
vsx_m = (1 << vsx ),
@@ -99,6 +101,7 @@ public:
static bool has_vpmsumb() { return (_features & vpmsumb_m) != 0; }
static bool has_mfdscr() { return (_features & mfdscr_m) != 0; }
static bool has_vsx() { return (_features & vsx_m) != 0; }
+ static bool has_vshasig() { return (_features & vshasig_m) != 0; }
static const char* cpu_features() { return _features_str; }
diff --git a/src/cpu/x86/vm/stubRoutines_x86_64.hpp b/src/cpu/x86/vm/stubRoutines_x86_64.hpp
index 15922b8ae..b048fd74e 100644
--- a/src/cpu/x86/vm/stubRoutines_x86_64.hpp
+++ b/src/cpu/x86/vm/stubRoutines_x86_64.hpp
@@ -33,7 +33,7 @@ static bool returns_to_call_stub(address return_pc) { return return_pc == _
enum platform_dependent_constants {
code_size1 = 19000, // simply increase if too small (assembler will crash if too small)
- code_size2 = 23000 // simply increase if too small (assembler will crash if too small)
+ code_size2 = 24000 // simply increase if too small (assembler will crash if too small)
};
class x86 {
diff --git a/src/cpu/x86/vm/x86_64.ad b/src/cpu/x86/vm/x86_64.ad
index e20b02d2b..e36604faf 100644
--- a/src/cpu/x86/vm/x86_64.ad
+++ b/src/cpu/x86/vm/x86_64.ad
@@ -3740,6 +3740,23 @@ operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
%}
%}
+// Indirect Memory Plus Positive Index Register Plus Offset Operand
+operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
+%{
+ constraint(ALLOC_IN_RC(ptr_reg));
+ predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
+ match(AddP (AddP reg (ConvI2L idx)) off);
+
+ op_cost(10);
+ format %{"[$reg + $off + $idx]" %}
+ interface(MEMORY_INTER) %{
+ base($reg);
+ index($idx);
+ scale(0x0);
+ disp($off);
+ %}
+%}
+
// Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
%{
@@ -3891,6 +3908,23 @@ operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale
%}
%}
+// Indirect Memory Times Plus Positive Index Register Plus Offset Operand
+operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
+%{
+ constraint(ALLOC_IN_RC(ptr_reg));
+ predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
+ match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
+
+ op_cost(10);
+ format %{"[$reg + $off + $idx]" %}
+ interface(MEMORY_INTER) %{
+ base($reg);
+ index($idx);
+ scale(0x0);
+ disp($off);
+ %}
+%}
+
// Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
%{
@@ -4082,11 +4116,11 @@ operand cmpOpUCF2() %{
// case of this is memory operands.
opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
- indIndexScale, indIndexScaleOffset, indPosIndexScaleOffset,
+ indIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
indCompressedOopOffset,
indirectNarrow, indOffset8Narrow, indOffset32Narrow,
indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
- indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow);
+ indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
//----------PIPELINE-----------------------------------------------------------
// Rules which define the behavior of the target architectures pipeline.
@@ -5120,6 +5154,17 @@ instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
ins_pipe(ialu_reg_reg_fat);
%}
+instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
+%{
+ match(Set dst mem);
+
+ ins_cost(110);
+ format %{ "leaq $dst, $mem\t# ptr posidxoff" %}
+ opcode(0x8D);
+ ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
+ ins_pipe(ialu_reg_reg_fat);
+%}
+
instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
%{
match(Set dst mem);
@@ -5204,6 +5249,18 @@ instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
ins_pipe(ialu_reg_reg_fat);
%}
+instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
+%{
+ predicate(Universe::narrow_oop_shift() == 0);
+ match(Set dst mem);
+
+ ins_cost(110);
+ format %{ "leaq $dst, $mem\t# ptr posidxoffnarrow" %}
+ opcode(0x8D);
+ ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
+ ins_pipe(ialu_reg_reg_fat);
+%}
+
instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
%{
predicate(Universe::narrow_oop_shift() == 0);
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-20 15:05:49 +0000