scrypt: 2x faster.
Work on uint32 slices instead of bytes.
Replace usage of Salsa20/8 from salsa package with
the specialized version.
benchmark old ns/op new ns/op delta
BenchmarkKey 266430525 126657130 -52.46%
R=agl
CC=golang-dev
https://golang.org/cl/7139050
diff --git a/scrypt/scrypt.go b/scrypt/scrypt.go
index 1c01fa5..f7a9fe1 100644
--- a/scrypt/scrypt.go
+++ b/scrypt/scrypt.go
@@ -9,69 +9,203 @@
import (
"crypto/sha256"
- "encoding/binary"
"errors"
"code.google.com/p/go.crypto/pbkdf2"
- "code.google.com/p/go.crypto/salsa20/salsa"
)
const maxInt = int(^uint(0) >> 1)
-// blockCopy copies n bytes from src into dst.
-func blockCopy(dst, src []byte, n int) {
+// blockCopy copies n numbers from src into dst.
+func blockCopy(dst, src []uint32, n int) {
copy(dst, src[:n])
}
-// blockXOR XORs bytes from dst with n bytes from src.
-func blockXOR(dst, src []byte, n int) {
+// blockXOR XORs numbers from dst with n numbers from src.
+func blockXOR(dst, src []uint32, n int) {
for i, v := range src[:n] {
dst[i] ^= v
}
}
-func blockMix(b, y []byte, r int) {
- var x [64]byte
+// salsaXOR applies Salsa20/8 to the XOR of 16 numbers from tmp and in,
+// and puts the result into both both tmp and out.
+func salsaXOR(tmp *[16]uint32, in, out []uint32) {
+ w0 := tmp[0] ^ in[0]
+ w1 := tmp[1] ^ in[1]
+ w2 := tmp[2] ^ in[2]
+ w3 := tmp[3] ^ in[3]
+ w4 := tmp[4] ^ in[4]
+ w5 := tmp[5] ^ in[5]
+ w6 := tmp[6] ^ in[6]
+ w7 := tmp[7] ^ in[7]
+ w8 := tmp[8] ^ in[8]
+ w9 := tmp[9] ^ in[9]
+ w10 := tmp[10] ^ in[10]
+ w11 := tmp[11] ^ in[11]
+ w12 := tmp[12] ^ in[12]
+ w13 := tmp[13] ^ in[13]
+ w14 := tmp[14] ^ in[14]
+ w15 := tmp[15] ^ in[15]
- blockCopy(x[:], b[(2*r-1)*64:], 64)
+ x0, x1, x2, x3, x4, x5, x6, x7, x8 := w0, w1, w2, w3, w4, w5, w6, w7, w8
+ x9, x10, x11, x12, x13, x14, x15 := w9, w10, w11, w12, w13, w14, w15
- for i := 0; i < 2*r*64; i += 64 {
- blockXOR(x[:], b[i:], 64)
- salsa.Core208(&x, &x)
- blockCopy(y[i:], x[:], 64)
+ for i := 0; i < 8; i += 2 {
+ u := x0 + x12
+ x4 ^= u<<7 | u>>(32-7)
+ u = x4 + x0
+ x8 ^= u<<9 | u>>(32-9)
+ u = x8 + x4
+ x12 ^= u<<13 | u>>(32-13)
+ u = x12 + x8
+ x0 ^= u<<18 | u>>(32-18)
+
+ u = x5 + x1
+ x9 ^= u<<7 | u>>(32-7)
+ u = x9 + x5
+ x13 ^= u<<9 | u>>(32-9)
+ u = x13 + x9
+ x1 ^= u<<13 | u>>(32-13)
+ u = x1 + x13
+ x5 ^= u<<18 | u>>(32-18)
+
+ u = x10 + x6
+ x14 ^= u<<7 | u>>(32-7)
+ u = x14 + x10
+ x2 ^= u<<9 | u>>(32-9)
+ u = x2 + x14
+ x6 ^= u<<13 | u>>(32-13)
+ u = x6 + x2
+ x10 ^= u<<18 | u>>(32-18)
+
+ u = x15 + x11
+ x3 ^= u<<7 | u>>(32-7)
+ u = x3 + x15
+ x7 ^= u<<9 | u>>(32-9)
+ u = x7 + x3
+ x11 ^= u<<13 | u>>(32-13)
+ u = x11 + x7
+ x15 ^= u<<18 | u>>(32-18)
+
+ u = x0 + x3
+ x1 ^= u<<7 | u>>(32-7)
+ u = x1 + x0
+ x2 ^= u<<9 | u>>(32-9)
+ u = x2 + x1
+ x3 ^= u<<13 | u>>(32-13)
+ u = x3 + x2
+ x0 ^= u<<18 | u>>(32-18)
+
+ u = x5 + x4
+ x6 ^= u<<7 | u>>(32-7)
+ u = x6 + x5
+ x7 ^= u<<9 | u>>(32-9)
+ u = x7 + x6
+ x4 ^= u<<13 | u>>(32-13)
+ u = x4 + x7
+ x5 ^= u<<18 | u>>(32-18)
+
+ u = x10 + x9
+ x11 ^= u<<7 | u>>(32-7)
+ u = x11 + x10
+ x8 ^= u<<9 | u>>(32-9)
+ u = x8 + x11
+ x9 ^= u<<13 | u>>(32-13)
+ u = x9 + x8
+ x10 ^= u<<18 | u>>(32-18)
+
+ u = x15 + x14
+ x12 ^= u<<7 | u>>(32-7)
+ u = x12 + x15
+ x13 ^= u<<9 | u>>(32-9)
+ u = x13 + x12
+ x14 ^= u<<13 | u>>(32-13)
+ u = x14 + x13
+ x15 ^= u<<18 | u>>(32-18)
}
+ x0 += w0
+ x1 += w1
+ x2 += w2
+ x3 += w3
+ x4 += w4
+ x5 += w5
+ x6 += w6
+ x7 += w7
+ x8 += w8
+ x9 += w9
+ x10 += w10
+ x11 += w11
+ x12 += w12
+ x13 += w13
+ x14 += w14
+ x15 += w15
- for i := 0; i < r; i++ {
- blockCopy(b[i*64:], y[i*2*64:], 64)
- }
+ out[0], tmp[0] = x0, x0
+ out[1], tmp[1] = x1, x1
+ out[2], tmp[2] = x2, x2
+ out[3], tmp[3] = x3, x3
+ out[4], tmp[4] = x4, x4
+ out[5], tmp[5] = x5, x5
+ out[6], tmp[6] = x6, x6
+ out[7], tmp[7] = x7, x7
+ out[8], tmp[8] = x8, x8
+ out[9], tmp[9] = x9, x9
+ out[10], tmp[10] = x10, x10
+ out[11], tmp[11] = x11, x11
+ out[12], tmp[12] = x12, x12
+ out[13], tmp[13] = x13, x13
+ out[14], tmp[14] = x14, x14
+ out[15], tmp[15] = x15, x15
+}
- for i := 0; i < r; i++ {
- blockCopy(b[(i+r)*64:], y[(i*2+1)*64:], 64)
+func blockMix(tmp *[16]uint32, in, out []uint32, r int) {
+ blockCopy(tmp[:], in[(2*r-1)*16:], 16)
+ for i := 0; i < 2*r; i += 2 {
+ salsaXOR(tmp, in[i*16:], out[i*8:])
+ salsaXOR(tmp, in[i*16+16:], out[i*8+r*16:])
}
}
-func integer(b []byte, r int) uint64 {
- return binary.LittleEndian.Uint64(b[(2*r-1)*64:])
+func integer(b []uint32, r int) uint64 {
+ j := (2*r - 1) * 16
+ return uint64(b[j]) | uint64(b[j+1])<<32
}
-func smix(b []byte, r, N int, v, xy []byte) {
+func smix(b []byte, r, N int, v, xy []uint32) {
+ var tmp [16]uint32
x := xy
- y := xy[128*r:]
+ y := xy[32*r:]
- blockCopy(x, b, 128*r)
-
- for i := 0; i < N; i++ {
- blockCopy(v[i*128*r:], x, 128*r)
- blockMix(x, y, r)
+ j := 0
+ for i := 0; i < 32*r; i++ {
+ x[i] = uint32(b[j]) | uint32(b[j+1])<<8 | uint32(b[j+2])<<16 | uint32(b[j+3])<<24
+ j += 4
}
+ for i := 0; i < N; i += 2 {
+ blockCopy(v[i*(32*r):], x, 32*r)
+ blockMix(&tmp, x, y, r)
- for i := 0; i < N; i++ {
+ blockCopy(v[(i+1)*(32*r):], y, 32*r)
+ blockMix(&tmp, y, x, r)
+ }
+ for i := 0; i < N; i += 2 {
j := int(integer(x, r) & uint64(N-1))
- blockXOR(x, v[j*128*r:], 128*r)
- blockMix(x, y, r)
- }
+ blockXOR(x, v[j*(32*r):], 32*r)
+ blockMix(&tmp, x, y, r)
- blockCopy(b, x, 128*r)
+ j = int(integer(y, r) & uint64(N-1))
+ blockXOR(y, v[j*(32*r):], 32*r)
+ blockMix(&tmp, y, x, r)
+ }
+ j = 0
+ for _, v := range x[:32*r] {
+ b[j+0] = byte(v >> 0)
+ b[j+1] = byte(v >> 8)
+ b[j+2] = byte(v >> 16)
+ b[j+3] = byte(v >> 24)
+ j += 4
+ }
}
// Key derives a key from the password, salt, and cost parameters, returning
@@ -97,8 +231,8 @@
return nil, errors.New("scrypt: parameters are too large")
}
- xy := make([]byte, 256*r)
- v := make([]byte, 128*r*N)
+ xy := make([]uint32, 64*r)
+ v := make([]uint32, 32*N*r)
b := pbkdf2.Key(password, salt, 1, p*128*r, sha256.New)
for i := 0; i < p; i++ {
