// Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build 386, !gccgo, !appengine #include "textflag.h" DATA iv0<>+0x00(SB)/4, $0x6a09e667 DATA iv0<>+0x04(SB)/4, $0xbb67ae85 DATA iv0<>+0x08(SB)/4, $0x3c6ef372 DATA iv0<>+0x0c(SB)/4, $0xa54ff53a GLOBL iv0<>(SB), (NOPTR+RODATA), $16 DATA iv1<>+0x00(SB)/4, $0x510e527f DATA iv1<>+0x04(SB)/4, $0x9b05688c DATA iv1<>+0x08(SB)/4, $0x1f83d9ab DATA iv1<>+0x0c(SB)/4, $0x5be0cd19 GLOBL iv1<>(SB), (NOPTR+RODATA), $16 DATA rol16<>+0x00(SB)/8, $0x0504070601000302 DATA rol16<>+0x08(SB)/8, $0x0D0C0F0E09080B0A GLOBL rol16<>(SB), (NOPTR+RODATA), $16 DATA rol8<>+0x00(SB)/8, $0x0407060500030201 DATA rol8<>+0x08(SB)/8, $0x0C0F0E0D080B0A09 GLOBL rol8<>(SB), (NOPTR+RODATA), $16 DATA counter<>+0x00(SB)/8, $0x40 DATA counter<>+0x08(SB)/8, $0x0 GLOBL counter<>(SB), (NOPTR+RODATA), $16 #define ROTL_SSE2(n, t, v) \ MOVO v, t; \ PSLLL $n, t; \ PSRLL $(32-n), v; \ PXOR t, v #define ROTL_SSSE3(c, v) \ PSHUFB c, v #define ROUND_SSE2(v0, v1, v2, v3, m0, m1, m2, m3, t) \ PADDL m0, v0; \ PADDL v1, v0; \ PXOR v0, v3; \ ROTL_SSE2(16, t, v3); \ PADDL v3, v2; \ PXOR v2, v1; \ ROTL_SSE2(20, t, v1); \ PADDL m1, v0; \ PADDL v1, v0; \ PXOR v0, v3; \ ROTL_SSE2(24, t, v3); \ PADDL v3, v2; \ PXOR v2, v1; \ ROTL_SSE2(25, t, v1); \ PSHUFL $0x39, v1, v1; \ PSHUFL $0x4E, v2, v2; \ PSHUFL $0x93, v3, v3; \ PADDL m2, v0; \ PADDL v1, v0; \ PXOR v0, v3; \ ROTL_SSE2(16, t, v3); \ PADDL v3, v2; \ PXOR v2, v1; \ ROTL_SSE2(20, t, v1); \ PADDL m3, v0; \ PADDL v1, v0; \ PXOR v0, v3; \ ROTL_SSE2(24, t, v3); \ PADDL v3, v2; \ PXOR v2, v1; \ ROTL_SSE2(25, t, v1); \ PSHUFL $0x39, v3, v3; \ PSHUFL $0x4E, v2, v2; \ PSHUFL $0x93, v1, v1 #define ROUND_SSSE3(v0, v1, v2, v3, m0, m1, m2, m3, t, c16, c8) \ PADDL m0, v0; \ PADDL v1, v0; \ PXOR v0, v3; \ ROTL_SSSE3(c16, v3); \ PADDL v3, v2; \ PXOR v2, v1; \ ROTL_SSE2(20, t, v1); \ PADDL m1, v0; \ PADDL v1, v0; \ PXOR v0, v3; \ ROTL_SSSE3(c8, v3); \ PADDL v3, v2; \ PXOR v2, v1; \ ROTL_SSE2(25, t, v1); \ PSHUFL $0x39, v1, v1; \ PSHUFL $0x4E, v2, v2; \ PSHUFL $0x93, v3, v3; \ PADDL m2, v0; \ PADDL v1, v0; \ PXOR v0, v3; \ ROTL_SSSE3(c16, v3); \ PADDL v3, v2; \ PXOR v2, v1; \ ROTL_SSE2(20, t, v1); \ PADDL m3, v0; \ PADDL v1, v0; \ PXOR v0, v3; \ ROTL_SSSE3(c8, v3); \ PADDL v3, v2; \ PXOR v2, v1; \ ROTL_SSE2(25, t, v1); \ PSHUFL $0x39, v3, v3; \ PSHUFL $0x4E, v2, v2; \ PSHUFL $0x93, v1, v1 #define PRECOMPUTE(dst, off, src, t) \ MOVL 0*4(src), t; \ MOVL t, 0*4+off+0(dst); \ MOVL t, 9*4+off+64(dst); \ MOVL t, 5*4+off+128(dst); \ MOVL t, 14*4+off+192(dst); \ MOVL t, 4*4+off+256(dst); \ MOVL t, 2*4+off+320(dst); \ MOVL t, 8*4+off+384(dst); \ MOVL t, 12*4+off+448(dst); \ MOVL t, 3*4+off+512(dst); \ MOVL t, 15*4+off+576(dst); \ MOVL 1*4(src), t; \ MOVL t, 4*4+off+0(dst); \ MOVL t, 8*4+off+64(dst); \ MOVL t, 14*4+off+128(dst); \ MOVL t, 5*4+off+192(dst); \ MOVL t, 12*4+off+256(dst); \ MOVL t, 11*4+off+320(dst); \ MOVL t, 1*4+off+384(dst); \ MOVL t, 6*4+off+448(dst); \ MOVL t, 10*4+off+512(dst); \ MOVL t, 3*4+off+576(dst); \ MOVL 2*4(src), t; \ MOVL t, 1*4+off+0(dst); \ MOVL t, 13*4+off+64(dst); \ MOVL t, 6*4+off+128(dst); \ MOVL t, 8*4+off+192(dst); \ MOVL t, 2*4+off+256(dst); \ MOVL t, 0*4+off+320(dst); \ MOVL t, 14*4+off+384(dst); \ MOVL t, 11*4+off+448(dst); \ MOVL t, 12*4+off+512(dst); \ MOVL t, 4*4+off+576(dst); \ MOVL 3*4(src), t; \ MOVL t, 5*4+off+0(dst); \ MOVL t, 15*4+off+64(dst); \ MOVL t, 9*4+off+128(dst); \ MOVL t, 1*4+off+192(dst); \ MOVL t, 11*4+off+256(dst); \ MOVL t, 7*4+off+320(dst); \ MOVL t, 13*4+off+384(dst); \ MOVL t, 3*4+off+448(dst); \ MOVL t, 6*4+off+512(dst); \ MOVL t, 10*4+off+576(dst); \ MOVL 4*4(src), t; \ MOVL t, 2*4+off+0(dst); \ MOVL t, 1*4+off+64(dst); \ MOVL t, 15*4+off+128(dst); \ MOVL t, 10*4+off+192(dst); \ MOVL t, 6*4+off+256(dst); \ MOVL t, 8*4+off+320(dst); \ MOVL t, 3*4+off+384(dst); \ MOVL t, 13*4+off+448(dst); \ MOVL t, 14*4+off+512(dst); \ MOVL t, 5*4+off+576(dst); \ MOVL 5*4(src), t; \ MOVL t, 6*4+off+0(dst); \ MOVL t, 11*4+off+64(dst); \ MOVL t, 2*4+off+128(dst); \ MOVL t, 9*4+off+192(dst); \ MOVL t, 1*4+off+256(dst); \ MOVL t, 13*4+off+320(dst); \ MOVL t, 4*4+off+384(dst); \ MOVL t, 8*4+off+448(dst); \ MOVL t, 15*4+off+512(dst); \ MOVL t, 7*4+off+576(dst); \ MOVL 6*4(src), t; \ MOVL t, 3*4+off+0(dst); \ MOVL t, 7*4+off+64(dst); \ MOVL t, 13*4+off+128(dst); \ MOVL t, 12*4+off+192(dst); \ MOVL t, 10*4+off+256(dst); \ MOVL t, 1*4+off+320(dst); \ MOVL t, 9*4+off+384(dst); \ MOVL t, 14*4+off+448(dst); \ MOVL t, 0*4+off+512(dst); \ MOVL t, 6*4+off+576(dst); \ MOVL 7*4(src), t; \ MOVL t, 7*4+off+0(dst); \ MOVL t, 14*4+off+64(dst); \ MOVL t, 10*4+off+128(dst); \ MOVL t, 0*4+off+192(dst); \ MOVL t, 5*4+off+256(dst); \ MOVL t, 9*4+off+320(dst); \ MOVL t, 12*4+off+384(dst); \ MOVL t, 1*4+off+448(dst); \ MOVL t, 13*4+off+512(dst); \ MOVL t, 2*4+off+576(dst); \ MOVL 8*4(src), t; \ MOVL t, 8*4+off+0(dst); \ MOVL t, 5*4+off+64(dst); \ MOVL t, 4*4+off+128(dst); \ MOVL t, 15*4+off+192(dst); \ MOVL t, 14*4+off+256(dst); \ MOVL t, 3*4+off+320(dst); \ MOVL t, 11*4+off+384(dst); \ MOVL t, 10*4+off+448(dst); \ MOVL t, 7*4+off+512(dst); \ MOVL t, 1*4+off+576(dst); \ MOVL 9*4(src), t; \ MOVL t, 12*4+off+0(dst); \ MOVL t, 2*4+off+64(dst); \ MOVL t, 11*4+off+128(dst); \ MOVL t, 4*4+off+192(dst); \ MOVL t, 0*4+off+256(dst); \ MOVL t, 15*4+off+320(dst); \ MOVL t, 10*4+off+384(dst); \ MOVL t, 7*4+off+448(dst); \ MOVL t, 5*4+off+512(dst); \ MOVL t, 9*4+off+576(dst); \ MOVL 10*4(src), t; \ MOVL t, 9*4+off+0(dst); \ MOVL t, 4*4+off+64(dst); \ MOVL t, 8*4+off+128(dst); \ MOVL t, 13*4+off+192(dst); \ MOVL t, 3*4+off+256(dst); \ MOVL t, 5*4+off+320(dst); \ MOVL t, 7*4+off+384(dst); \ MOVL t, 15*4+off+448(dst); \ MOVL t, 11*4+off+512(dst); \ MOVL t, 0*4+off+576(dst); \ MOVL 11*4(src), t; \ MOVL t, 13*4+off+0(dst); \ MOVL t, 10*4+off+64(dst); \ MOVL t, 0*4+off+128(dst); \ MOVL t, 3*4+off+192(dst); \ MOVL t, 9*4+off+256(dst); \ MOVL t, 6*4+off+320(dst); \ MOVL t, 15*4+off+384(dst); \ MOVL t, 4*4+off+448(dst); \ MOVL t, 2*4+off+512(dst); \ MOVL t, 12*4+off+576(dst); \ MOVL 12*4(src), t; \ MOVL t, 10*4+off+0(dst); \ MOVL t, 12*4+off+64(dst); \ MOVL t, 1*4+off+128(dst); \ MOVL t, 6*4+off+192(dst); \ MOVL t, 13*4+off+256(dst); \ MOVL t, 4*4+off+320(dst); \ MOVL t, 0*4+off+384(dst); \ MOVL t, 2*4+off+448(dst); \ MOVL t, 8*4+off+512(dst); \ MOVL t, 14*4+off+576(dst); \ MOVL 13*4(src), t; \ MOVL t, 14*4+off+0(dst); \ MOVL t, 3*4+off+64(dst); \ MOVL t, 7*4+off+128(dst); \ MOVL t, 2*4+off+192(dst); \ MOVL t, 15*4+off+256(dst); \ MOVL t, 12*4+off+320(dst); \ MOVL t, 6*4+off+384(dst); \ MOVL t, 0*4+off+448(dst); \ MOVL t, 9*4+off+512(dst); \ MOVL t, 11*4+off+576(dst); \ MOVL 14*4(src), t; \ MOVL t, 11*4+off+0(dst); \ MOVL t, 0*4+off+64(dst); \ MOVL t, 12*4+off+128(dst); \ MOVL t, 7*4+off+192(dst); \ MOVL t, 8*4+off+256(dst); \ MOVL t, 14*4+off+320(dst); \ MOVL t, 2*4+off+384(dst); \ MOVL t, 5*4+off+448(dst); \ MOVL t, 1*4+off+512(dst); \ MOVL t, 13*4+off+576(dst); \ MOVL 15*4(src), t; \ MOVL t, 15*4+off+0(dst); \ MOVL t, 6*4+off+64(dst); \ MOVL t, 3*4+off+128(dst); \ MOVL t, 11*4+off+192(dst); \ MOVL t, 7*4+off+256(dst); \ MOVL t, 10*4+off+320(dst); \ MOVL t, 5*4+off+384(dst); \ MOVL t, 9*4+off+448(dst); \ MOVL t, 4*4+off+512(dst); \ MOVL t, 8*4+off+576(dst) // func hashBlocksSSE2(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte) TEXT ·hashBlocksSSE2(SB), 4, $0-24 MOVL h+0(FP), AX MOVL c+4(FP), BX MOVL flag+8(FP), CX MOVL blocks_base+12(FP), SI MOVL blocks_len+16(FP), DX MOVL SP, BP ANDL $0xFFFFFFF0, SP SUBL $(16+16+640), SP MOVL CX, 8(SP) MOVL 0(BX), CX MOVL CX, 0(SP) MOVL 4(BX), CX MOVL CX, 4(SP) XORL CX, CX MOVL CX, 12(SP) MOVOU 0(AX), X0 MOVOU 16(AX), X1 MOVOU counter<>(SB), X2 loop: MOVO X0, X4 MOVO X1, X5 MOVOU iv0<>(SB), X6 MOVOU iv1<>(SB), X7 MOVO 0(SP), X3 PADDQ X2, X3 PXOR X3, X7 MOVO X3, 0(SP) PRECOMPUTE(SP, 16, SI, CX) ROUND_SSE2(X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X3) ROUND_SSE2(X4, X5, X6, X7, 16+64(SP), 32+64(SP), 48+64(SP), 64+64(SP), X3) ROUND_SSE2(X4, X5, X6, X7, 16+128(SP), 32+128(SP), 48+128(SP), 64+128(SP), X3) ROUND_SSE2(X4, X5, X6, X7, 16+192(SP), 32+192(SP), 48+192(SP), 64+192(SP), X3) ROUND_SSE2(X4, X5, X6, X7, 16+256(SP), 32+256(SP), 48+256(SP), 64+256(SP), X3) ROUND_SSE2(X4, X5, X6, X7, 16+320(SP), 32+320(SP), 48+320(SP), 64+320(SP), X3) ROUND_SSE2(X4, X5, X6, X7, 16+384(SP), 32+384(SP), 48+384(SP), 64+384(SP), X3) ROUND_SSE2(X4, X5, X6, X7, 16+448(SP), 32+448(SP), 48+448(SP), 64+448(SP), X3) ROUND_SSE2(X4, X5, X6, X7, 16+512(SP), 32+512(SP), 48+512(SP), 64+512(SP), X3) ROUND_SSE2(X4, X5, X6, X7, 16+576(SP), 32+576(SP), 48+576(SP), 64+576(SP), X3) PXOR X4, X0 PXOR X5, X1 PXOR X6, X0 PXOR X7, X1 LEAL 64(SI), SI SUBL $64, DX JNE loop MOVL 0(SP), CX MOVL CX, 0(BX) MOVL 4(SP), CX MOVL CX, 4(BX) MOVOU X0, 0(AX) MOVOU X1, 16(AX) MOVL BP, SP RET // func hashBlocksSSSE3(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte) TEXT ·hashBlocksSSSE3(SB), 4, $0-24 MOVL h+0(FP), AX MOVL c+4(FP), BX MOVL flag+8(FP), CX MOVL blocks_base+12(FP), SI MOVL blocks_len+16(FP), DX MOVL SP, BP ANDL $0xFFFFFFF0, SP SUBL $(16+16+640+32), SP MOVL CX, 8(SP) MOVL 0(BX), CX MOVL CX, 0(SP) MOVL 4(BX), CX MOVL CX, 4(SP) XORL CX, CX MOVL CX, 12(SP) MOVOU 0(AX), X0 MOVOU 16(AX), X1 MOVOU counter<>(SB), X2 loop: MOVO X0, 656(SP) MOVO X1, 672(SP) MOVO X0, X4 MOVO X1, X5 MOVOU iv0<>(SB), X6 MOVOU iv1<>(SB), X7 MOVO 0(SP), X3 PADDQ X2, X3 PXOR X3, X7 MOVO X3, 0(SP) MOVOU rol16<>(SB), X0 MOVOU rol8<>(SB), X1 PRECOMPUTE(SP, 16, SI, CX) ROUND_SSSE3(X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X3, X0, X1) ROUND_SSSE3(X4, X5, X6, X7, 16+64(SP), 32+64(SP), 48+64(SP), 64+64(SP), X3, X0, X1) ROUND_SSSE3(X4, X5, X6, X7, 16+128(SP), 32+128(SP), 48+128(SP), 64+128(SP), X3, X0, X1) ROUND_SSSE3(X4, X5, X6, X7, 16+192(SP), 32+192(SP), 48+192(SP), 64+192(SP), X3, X0, X1) ROUND_SSSE3(X4, X5, X6, X7, 16+256(SP), 32+256(SP), 48+256(SP), 64+256(SP), X3, X0, X1) ROUND_SSSE3(X4, X5, X6, X7, 16+320(SP), 32+320(SP), 48+320(SP), 64+320(SP), X3, X0, X1) ROUND_SSSE3(X4, X5, X6, X7, 16+384(SP), 32+384(SP), 48+384(SP), 64+384(SP), X3, X0, X1) ROUND_SSSE3(X4, X5, X6, X7, 16+448(SP), 32+448(SP), 48+448(SP), 64+448(SP), X3, X0, X1) ROUND_SSSE3(X4, X5, X6, X7, 16+512(SP), 32+512(SP), 48+512(SP), 64+512(SP), X3, X0, X1) ROUND_SSSE3(X4, X5, X6, X7, 16+576(SP), 32+576(SP), 48+576(SP), 64+576(SP), X3, X0, X1) MOVO 656(SP), X0 MOVO 672(SP), X1 PXOR X4, X0 PXOR X5, X1 PXOR X6, X0 PXOR X7, X1 LEAL 64(SI), SI SUBL $64, DX JNE loop MOVL 0(SP), CX MOVL CX, 0(BX) MOVL 4(SP), CX MOVL CX, 4(BX) MOVOU X0, 0(AX) MOVOU X1, 16(AX) MOVL BP, SP RET // func supportSSSE3() bool TEXT ·supportSSSE3(SB), 4, $0-1 MOVL $1, AX CPUID MOVL CX, BX ANDL $0x1, BX // supports SSE3 JZ FALSE ANDL $0x200, CX // supports SSSE3 JZ FALSE MOVB $1, ret+0(FP) RET FALSE: MOVB $0, ret+0(FP) RET // func supportSSE2() bool TEXT ·supportSSE2(SB), 4, $0-1 MOVL $1, AX CPUID SHRL $26, DX ANDL $1, DX // DX != 0 if support SSE2 MOVB DX, ret+0(FP) RET