internal/sm2ec: improve purego implementation's performance #274

internal/sm2ec/sm2p256.go

@@ -2,37 +2,38 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Code generated by generate.go. DO NOT EDIT.
-
 //go:build purego || !(amd64 || arm64 || s390x || ppc64le)
 
 package sm2ec
 
 import (
 	"crypto/subtle"
+	_ "embed"
 	"errors"
-	"github.com/emmansun/gmsm/internal/sm2ec/fiat"
+	"math/bits"
+	"runtime"
 	"sync"
-)
+	"unsafe"
 
-// sm2p256ElementLength is the length of an element of the base or scalar field,
+	"github.com/emmansun/gmsm/internal/byteorder"
-// which have the same bytes length for all NIST P curves.
+	"github.com/emmansun/gmsm/internal/sm2ec/fiat"
-const sm2p256ElementLength = 32
+)
 
 // SM2P256Point is a SM2P256 point. The zero value is NOT valid.
 type SM2P256Point struct {
 	// The point is represented in projective coordinates (X:Y:Z),
-	// where x = X/Z and y = Y/Z.
+	// where x = X/Z and y = Y/Z. Infinity is (0:1:0).
-	x, y, z *fiat.SM2P256Element
+	//
+	// fiat.SM2P256Element is a base field element in [0, P-1] in the Montgomery
+	// domain as four limbs in little-endian order value.
+	x, y, z fiat.SM2P256Element
 }
 
 // NewSM2P256Point returns a new SM2P256Point representing the point at infinity point.
 func NewSM2P256Point() *SM2P256Point {
-	return &SM2P256Point{
+	p := &SM2P256Point{}
-		x: new(fiat.SM2P256Element),
+	p.y.One()
-		y: new(fiat.SM2P256Element).One(),
+	return p
-		z: new(fiat.SM2P256Element),
-	}
 }
 
 // SetGenerator sets p to the canonical generator and returns p.
@@ -45,12 +46,16 @@ func (p *SM2P256Point) SetGenerator() *SM2P256Point {
 
 // Set sets p = q and returns p.
 func (p *SM2P256Point) Set(q *SM2P256Point) *SM2P256Point {
-	p.x.Set(q.x)
+	p.x.Set(&q.x)
-	p.y.Set(q.y)
+	p.y.Set(&q.y)
-	p.z.Set(q.z)
+	p.z.Set(&q.z)
 	return p
 }
 
+const p256ElementLength = 32
+const p256UncompressedLength = 1 + 2*p256ElementLength
+const p256CompressedLength = 1 + p256ElementLength
+
 // SetBytes sets p to the compressed, uncompressed, or infinity value encoded in
 // b, as specified in SEC 1, Version 2.0, Section 2.3.4. If the point is not on
 // the curve, it returns nil and an error, and the receiver is unchanged.
@@ -61,12 +66,12 @@ func (p *SM2P256Point) SetBytes(b []byte) (*SM2P256Point, error) {
 	case len(b) == 1 && b[0] == 0:
 		return p.Set(NewSM2P256Point()), nil
 	// Uncompressed form.
-	case len(b) == 1+2*sm2p256ElementLength && b[0] == 4:
+	case len(b) == p256UncompressedLength && b[0] == 4:
-		x, err := new(fiat.SM2P256Element).SetBytes(b[1 : 1+sm2p256ElementLength])
+		x, err := new(fiat.SM2P256Element).SetBytes(b[1 : 1+p256ElementLength])
 		if err != nil {
 			return nil, err
 		}
-		y, err := new(fiat.SM2P256Element).SetBytes(b[1+sm2p256ElementLength:])
+		y, err := new(fiat.SM2P256Element).SetBytes(b[1+p256ElementLength:])
 		if err != nil {
 			return nil, err
 		}
@@ -78,7 +83,7 @@ func (p *SM2P256Point) SetBytes(b []byte) (*SM2P256Point, error) {
 		p.z.One()
 		return p, nil
 	// Compressed form.
-	case len(b) == 1+sm2p256ElementLength && (b[0] == 2 || b[0] == 3):
+	case len(b) == p256CompressedLength && (b[0] == 2 || b[0] == 3):
 		x, err := new(fiat.SM2P256Element).SetBytes(b[1:])
 		if err != nil {
 			return nil, err
@@ -92,7 +97,7 @@ func (p *SM2P256Point) SetBytes(b []byte) (*SM2P256Point, error) {
 		// significant bit, based on the encoding type byte.
 		otherRoot := new(fiat.SM2P256Element)
 		otherRoot.Sub(otherRoot, y)
-		cond := y.Bytes()[sm2p256ElementLength-1]&1 ^ b[0]&1
+		cond := y.Bytes()[p256ElementLength-1]&1 ^ b[0]&1
 		y.Select(otherRoot, y, int(cond))
 		p.x.Set(x)
 		p.y.Set(y)
@@ -142,17 +147,17 @@ func sm2p256CheckOnCurve(x, y *fiat.SM2P256Element) error {
 func (p *SM2P256Point) Bytes() []byte {
 	// This function is outlined to make the allocations inline in the caller
 	// rather than happen on the heap.
-	var out [1 + 2*sm2p256ElementLength]byte
+	var out [p256UncompressedLength]byte
 	return p.bytes(&out)
 }
 
-func (p *SM2P256Point) bytes(out *[1 + 2*sm2p256ElementLength]byte) []byte {
+func (p *SM2P256Point) bytes(out *[p256UncompressedLength]byte) []byte {
 	if p.z.IsZero() == 1 {
 		return append(out[:0], 0)
 	}
-	zinv := new(fiat.SM2P256Element).Invert(p.z)
+	zinv := new(fiat.SM2P256Element).Invert(&p.z)
-	x := new(fiat.SM2P256Element).Mul(p.x, zinv)
+	x := new(fiat.SM2P256Element).Mul(&p.x, zinv)
-	y := new(fiat.SM2P256Element).Mul(p.y, zinv)
+	y := new(fiat.SM2P256Element).Mul(&p.y, zinv)
 	buf := append(out[:0], 4)
 	buf = append(buf, x.Bytes()...)
 	buf = append(buf, y.Bytes()...)
@@ -164,16 +169,16 @@ func (p *SM2P256Point) bytes(out *[1 + 2*sm2p256ElementLength]byte) []byte {
 func (p *SM2P256Point) BytesX() ([]byte, error) {
 	// This function is outlined to make the allocations inline in the caller
 	// rather than happen on the heap.
-	var out [sm2p256ElementLength]byte
+	var out [p256ElementLength]byte
 	return p.bytesX(&out)
 }
 
-func (p *SM2P256Point) bytesX(out *[sm2p256ElementLength]byte) ([]byte, error) {
+func (p *SM2P256Point) bytesX(out *[p256ElementLength]byte) ([]byte, error) {
 	if p.z.IsZero() == 1 {
 		return nil, errors.New("SM2P256 point is the point at infinity")
 	}
-	zinv := new(fiat.SM2P256Element).Invert(p.z)
+	zinv := new(fiat.SM2P256Element).Invert(&p.z)
-	x := new(fiat.SM2P256Element).Mul(p.x, zinv)
+	x := new(fiat.SM2P256Element).Mul(&p.x, zinv)
 	return append(out[:0], x.Bytes()...), nil
 }
 
@@ -183,21 +188,21 @@ func (p *SM2P256Point) bytesX(out *[sm2p256ElementLength]byte) ([]byte, error) {
 func (p *SM2P256Point) BytesCompressed() []byte {
 	// This function is outlined to make the allocations inline in the caller
 	// rather than happen on the heap.
-	var out [1 + sm2p256ElementLength]byte
+	var out [p256CompressedLength]byte
 	return p.bytesCompressed(&out)
 }
 
-func (p *SM2P256Point) bytesCompressed(out *[1 + sm2p256ElementLength]byte) []byte {
+func (p *SM2P256Point) bytesCompressed(out *[p256CompressedLength]byte) []byte {
 	if p.z.IsZero() == 1 {
 		return append(out[:0], 0)
 	}
-	zinv := new(fiat.SM2P256Element).Invert(p.z)
+	zinv := new(fiat.SM2P256Element).Invert(&p.z)
-	x := new(fiat.SM2P256Element).Mul(p.x, zinv)
+	x := new(fiat.SM2P256Element).Mul(&p.x, zinv)
-	y := new(fiat.SM2P256Element).Mul(p.y, zinv)
+	y := new(fiat.SM2P256Element).Mul(&p.y, zinv)
 	// Encode the sign of the y coordinate (indicated by the least significant
 	// bit) as the encoding type (2 or 3).
 	buf := append(out[:0], 2)
-	buf[0] |= y.Bytes()[sm2p256ElementLength-1] & 1
+	buf[0] |= y.Bytes()[p256ElementLength-1] & 1
 	buf = append(buf, x.Bytes()...)
 	return buf
 }
@@ -206,21 +211,21 @@ func (p *SM2P256Point) bytesCompressed(out *[1 + sm2p256ElementLength]byte) []by
 func (q *SM2P256Point) Add(p1, p2 *SM2P256Point) *SM2P256Point {
 	// Complete addition formula for a = -3 from "Complete addition formulas for
 	// prime order elliptic curves" (https://eprint.iacr.org/2015/1060), §A.2.
-	t0 := new(fiat.SM2P256Element).Mul(p1.x, p2.x)     // t0 := X1 * X2
+	t0 := new(fiat.SM2P256Element).Mul(&p1.x, &p2.x)   // t0 := X1 * X2
-	t1 := new(fiat.SM2P256Element).Mul(p1.y, p2.y)     // t1 := Y1 * Y2
+	t1 := new(fiat.SM2P256Element).Mul(&p1.y, &p2.y)   // t1 := Y1 * Y2
-	t2 := new(fiat.SM2P256Element).Mul(p1.z, p2.z)     // t2 := Z1 * Z2
+	t2 := new(fiat.SM2P256Element).Mul(&p1.z, &p2.z)   // t2 := Z1 * Z2
-	t3 := new(fiat.SM2P256Element).Add(p1.x, p1.y)     // t3 := X1 + Y1
+	t3 := new(fiat.SM2P256Element).Add(&p1.x, &p1.y)   // t3 := X1 + Y1
-	t4 := new(fiat.SM2P256Element).Add(p2.x, p2.y)     // t4 := X2 + Y2
+	t4 := new(fiat.SM2P256Element).Add(&p2.x, &p2.y)   // t4 := X2 + Y2
 	t3.Mul(t3, t4)                                     // t3 := t3 * t4
 	t4.Add(t0, t1)                                     // t4 := t0 + t1
 	t3.Sub(t3, t4)                                     // t3 := t3 - t4
-	t4.Add(p1.y, p1.z)                                 // t4 := Y1 + Z1
+	t4.Add(&p1.y, &p1.z)                               // t4 := Y1 + Z1
-	x3 := new(fiat.SM2P256Element).Add(p2.y, p2.z)     // X3 := Y2 + Z2
+	x3 := new(fiat.SM2P256Element).Add(&p2.y, &p2.z)   // X3 := Y2 + Z2
 	t4.Mul(t4, x3)                                     // t4 := t4 * X3
 	x3.Add(t1, t2)                                     // X3 := t1 + t2
 	t4.Sub(t4, x3)                                     // t4 := t4 - X3
-	x3.Add(p1.x, p1.z)                                 // X3 := X1 + Z1
+	x3.Add(&p1.x, &p1.z)                               // X3 := X1 + Z1
-	y3 := new(fiat.SM2P256Element).Add(p2.x, p2.z)     // Y3 := X2 + Z2
+	y3 := new(fiat.SM2P256Element).Add(&p2.x, &p2.z)   // Y3 := X2 + Z2
 	x3.Mul(x3, y3)                                     // X3 := X3 * Y3
 	y3.Add(t0, t2)                                     // Y3 := t0 + t2
 	y3.Sub(x3, y3)                                     // Y3 := X3 - Y3
@@ -260,12 +265,12 @@ func (q *SM2P256Point) Add(p1, p2 *SM2P256Point) *SM2P256Point {
 func (q *SM2P256Point) Double(p *SM2P256Point) *SM2P256Point {
 	// Complete addition formula for a = -3 from "Complete addition formulas for
 	// prime order elliptic curves" (https://eprint.iacr.org/2015/1060), §A.2.
-	t0 := new(fiat.SM2P256Element).Square(p.x)         // t0 := X ^ 2
+	t0 := new(fiat.SM2P256Element).Square(&p.x)        // t0 := X ^ 2
-	t1 := new(fiat.SM2P256Element).Square(p.y)         // t1 := Y ^ 2
+	t1 := new(fiat.SM2P256Element).Square(&p.y)        // t1 := Y ^ 2
-	t2 := new(fiat.SM2P256Element).Square(p.z)         // t2 := Z ^ 2
+	t2 := new(fiat.SM2P256Element).Square(&p.z)        // t2 := Z ^ 2
-	t3 := new(fiat.SM2P256Element).Mul(p.x, p.y)       // t3 := X * Y
+	t3 := new(fiat.SM2P256Element).Mul(&p.x, &p.y)     // t3 := X * Y
 	t3.Add(t3, t3)                                     // t3 := t3 + t3
-	z3 := new(fiat.SM2P256Element).Mul(p.x, p.z)       // Z3 := X * Z
+	z3 := new(fiat.SM2P256Element).Mul(&p.x, &p.z)     // Z3 := X * Z
 	z3.Add(z3, z3)                                     // Z3 := Z3 + Z3
 	y3 := new(fiat.SM2P256Element).Mul(sm2p256B(), t2) // Y3 := b * t2
 	y3.Sub(y3, z3)                                     // Y3 := Y3 - Z3
@@ -287,7 +292,7 @@ func (q *SM2P256Point) Double(p *SM2P256Point) *SM2P256Point {
 	t0.Sub(t0, t2)                                     // t0 := t0 - t2
 	t0.Mul(t0, z3)                                     // t0 := t0 * Z3
 	y3.Add(y3, t0)                                     // Y3 := Y3 + t0
-	t0.Mul(p.y, p.z)                                   // t0 := Y * Z
+	t0.Mul(&p.y, &p.z)                                 // t0 := Y * Z
 	t0.Add(t0, t0)                                     // t0 := t0 + t0
 	z3.Mul(t0, z3)                                     // Z3 := t0 * Z3
 	x3.Sub(x3, z3)                                     // X3 := X3 - Z3
@@ -301,133 +306,341 @@ func (q *SM2P256Point) Double(p *SM2P256Point) *SM2P256Point {
 	return q
 }
 
-// Select sets q to p1 if cond == 1, and to p2 if cond == 0.
+// sm2P256AffinePoint is a point in affine coordinates (x, y). x and y are still
-func (q *SM2P256Point) Select(p1, p2 *SM2P256Point, cond int) *SM2P256Point {
+// Montgomery domain elements. The point can't be the point at infinity.
-	q.x.Select(p1.x, p2.x, cond)
+type sm2P256AffinePoint struct {
-	q.y.Select(p1.y, p2.y, cond)
+	x, y fiat.SM2P256Element
-	q.z.Select(p1.z, p2.z, cond)
+}
+
+func (p *sm2P256AffinePoint) Projective() *SM2P256Point {
+	pp := &SM2P256Point{x: p.x, y: p.y}
+	pp.z.One()
+	return pp
+}
+
+// AddAffine sets q = p1 + p2, if infinity == 0, and to p1 if infinity == 1.
+// p2 can't be the point at infinity as it can't be represented in affine
+// coordinates, instead callers can set p2 to an arbitrary point and set
+// infinity to 1.
+func (q *SM2P256Point) AddAffine(p1 *SM2P256Point, p2 *sm2P256AffinePoint, infinity int) *SM2P256Point {
+	// Complete mixed addition formula for a = -3 from "Complete addition
+	// formulas for prime order elliptic curves"
+	// (https://eprint.iacr.org/2015/1060), Algorithm 5.
+
+	t0 := new(fiat.SM2P256Element).Mul(&p1.x, &p2.x)      // t0 ← X1 · X2
+	t1 := new(fiat.SM2P256Element).Mul(&p1.y, &p2.y)      // t1 ← Y1 · Y2
+	t3 := new(fiat.SM2P256Element).Add(&p2.x, &p2.y)      // t3 ← X2 + Y2
+	t4 := new(fiat.SM2P256Element).Add(&p1.x, &p1.y)      // t4 ← X1 + Y1
+	t3.Mul(t3, t4)                                        // t3 ← t3 · t4
+	t4.Add(t0, t1)                                        // t4 ← t0 + t1
+	t3.Sub(t3, t4)                                        // t3 ← t3 − t4
+	t4.Mul(&p2.y, &p1.z)                                  // t4 ← Y2 · Z1
+	t4.Add(t4, &p1.y)                                     // t4 ← t4 + Y1
+	y3 := new(fiat.SM2P256Element).Mul(&p2.x, &p1.z)      // Y3 ← X2 · Z1
+	y3.Add(y3, &p1.x)                                     // Y3 ← Y3 + X1
+	z3 := new(fiat.SM2P256Element).Mul(sm2p256B(), &p1.z) // Z3 ← b  · Z1
+	x3 := new(fiat.SM2P256Element).Sub(y3, z3)            // X3 ← Y3 − Z3
+	z3.Add(x3, x3)                                        // Z3 ← X3 + X3
+	x3.Add(x3, z3)                                        // X3 ← X3 + Z3
+	z3.Sub(t1, x3)                                        // Z3 ← t1 − X3
+	x3.Add(t1, x3)                                        // X3 ← t1 + X3
+	y3.Mul(sm2p256B(), y3)                                // Y3 ← b  · Y3
+	t1.Add(&p1.z, &p1.z)                                  // t1 ← Z1 + Z1
+	t2 := new(fiat.SM2P256Element).Add(t1, &p1.z)         // t2 ← t1 + Z1
+	y3.Sub(y3, t2)                                        // Y3 ← Y3 − t2
+	y3.Sub(y3, t0)                                        // Y3 ← Y3 − t0
+	t1.Add(y3, y3)                                        // t1 ← Y3 + Y3
+	y3.Add(t1, y3)                                        // Y3 ← t1 + Y3
+	t1.Add(t0, t0)                                        // t1 ← t0 + t0
+	t0.Add(t1, t0)                                        // t0 ← t1 + t0
+	t0.Sub(t0, t2)                                        // t0 ← t0 − t2
+	t1.Mul(t4, y3)                                        // t1 ← t4 · Y3
+	t2.Mul(t0, y3)                                        // t2 ← t0 · Y3
+	y3.Mul(x3, z3)                                        // Y3 ← X3 · Z3
+	y3.Add(y3, t2)                                        // Y3 ← Y3 + t2
+	x3.Mul(t3, x3)                                        // X3 ← t3 · X3
+	x3.Sub(x3, t1)                                        // X3 ← X3 − t1
+	z3.Mul(t4, z3)                                        // Z3 ← t4 · Z3
+	t1.Mul(t3, t0)                                        // t1 ← t3 · t0
+	z3.Add(z3, t1)                                        // Z3 ← Z3 + t1
+
+	q.x.Select(&p1.x, x3, infinity)
+	q.y.Select(&p1.y, y3, infinity)
+	q.z.Select(&p1.z, z3, infinity)
 	return q
 }
 
-// A sm2p256Table holds the first 15 multiples of a point at offset -1, so [1]P
+// Select sets q to p1 if cond == 1, and to p2 if cond == 0.
-// is at table[0], [15]P is at table[14], and [0]P is implicitly the identity
+func (q *SM2P256Point) Select(p1, p2 *SM2P256Point, cond int) *SM2P256Point {
-// point.
+	q.x.Select(&p1.x, &p2.x, cond)
-type sm2p256Table [15]*SM2P256Point
+	q.y.Select(&p1.y, &p2.y, cond)
+	q.z.Select(&p1.z, &p2.z, cond)
+	return q
+}
+
+// p256OrdElement is a SM2 P256 scalar field element in [0, ord(G)-1] in the
+// Montgomery domain (with R 2²⁵⁶) as four uint64 limbs in little-endian order.
+type p256OrdElement [4]uint64
+
+// SetBytes sets s to the big-endian value of x, reducing it as necessary.
+func (s *p256OrdElement) SetBytes(x []byte) (*p256OrdElement, error) {
+	if len(x) != 32 {
+		return nil, errors.New("invalid scalar length")
+	}
+
+	s[0] = byteorder.BEUint64(x[24:])
+	s[1] = byteorder.BEUint64(x[16:])
+	s[2] = byteorder.BEUint64(x[8:])
+	s[3] = byteorder.BEUint64(x[:])
+
+	// Ensure s is in the range [0, ord(G)-1]. Since 2 * ord(G) > 2²⁵⁶, we can
+	// just conditionally subtract ord(G), keeping the result if it doesn't
+	// underflow.
+	t0, b := bits.Sub64(s[0], 0x53bbf40939d54123, 0)
+	t1, b := bits.Sub64(s[1], 0x7203df6b21c6052b, b)
+	t2, b := bits.Sub64(s[2], 0xffffffffffffffff, b)
+	t3, b := bits.Sub64(s[3], 0xfffffffeffffffff, b)
+	tMask := b - 1 // zero if subtraction underflowed
+	s[0] ^= (t0 ^ s[0]) & tMask
+	s[1] ^= (t1 ^ s[1]) & tMask
+	s[2] ^= (t2 ^ s[2]) & tMask
+	s[3] ^= (t3 ^ s[3]) & tMask
+
+	return s, nil
+}
+
+func (s *p256OrdElement) Bytes() []byte {
+	var out [32]byte
+	byteorder.BEPutUint64(out[24:], s[0])
+	byteorder.BEPutUint64(out[16:], s[1])
+	byteorder.BEPutUint64(out[8:], s[2])
+	byteorder.BEPutUint64(out[:], s[3])
+	return out[:]
+}
+
+// Rsh returns the 64 least significant bits of x >> n. n must be lower
+// than 256. The value of n leaks through timing side-channels.
+func (s *p256OrdElement) Rsh(n int) uint64 {
+	i := n / 64
+	n = n % 64
+	res := s[i] >> n
+	// Shift in the more significant limb, if present.
+	if i := i + 1; i < len(s) {
+		res |= s[i] << (64 - n)
+	}
+	return res
+}
+
+// sm2p256Table is a table of the first 32 multiples of a point. Points are stored
+// at an index offset of -1 so [8]P is at index 7, P is at 0, and [16]P is at 15.
+// [0]P is the point at infinity and it's not stored.
+type sm2p256Table [32]SM2P256Point
 
 // Select selects the n-th multiple of the table base point into p. It works in
-// constant time by iterating over every entry of the table. n must be in [0, 15].
+// constant time. n must be in [0, 16]. If n is 0, p is set to the identity point.
 func (table *sm2p256Table) Select(p *SM2P256Point, n uint8) {
-	if n >= 16 {
+	if n > 32 {
 		panic("sm2ec: internal error: sm2p256Table called with out-of-bounds value")
 	}
 	p.Set(NewSM2P256Point())
-	for i, f := range table {
+	for i := uint8(1); i <= 32; i++ {
-		cond := subtle.ConstantTimeByteEq(uint8(i+1), n)
+		cond := subtle.ConstantTimeByteEq(i, n)
-		p.Select(f, p, cond)
+		p.Select(&table[i-1], p, cond)
 	}
 }
 
+// Compute populates the table to the first 32 multiples of q.
+func (table *sm2p256Table) Compute(q *SM2P256Point) *sm2p256Table {
+	table[0].Set(q)
+	for i := 1; i < 32; i += 2 {
+		table[i].Double(&table[i/2])
+		if i+1 < 32 {
+			table[i+1].Add(&table[i], q)
+		}
+	}
+	return table
+}
+
+func boothW6(in uint64) (uint8, int) {
+	s := ^((in >> 6) - 1)
+	d := (1 << 7) - in - 1
+	d = (d & s) | (in & (^s))
+	d = (d >> 1) + (d & 1)
+	return uint8(d), int(s & 1)
+}
+
 // ScalarMult sets p = scalar * q, and returns p.
 func (p *SM2P256Point) ScalarMult(q *SM2P256Point, scalar []byte) (*SM2P256Point, error) {
-	// Compute a sm2p256Table for the base point q. The explicit NewSM2P256Point
+	s, err := new(p256OrdElement).SetBytes(scalar)
-	// calls get inlined, letting the allocations live on the stack.
+	if err != nil {
-	var table = sm2p256Table{NewSM2P256Point(), NewSM2P256Point(), NewSM2P256Point(),
+		return nil, err
-		NewSM2P256Point(), NewSM2P256Point(), NewSM2P256Point(), NewSM2P256Point(),
-		NewSM2P256Point(), NewSM2P256Point(), NewSM2P256Point(), NewSM2P256Point(),
-		NewSM2P256Point(), NewSM2P256Point(), NewSM2P256Point(), NewSM2P256Point()}
-	table[0].Set(q)
-	for i := 1; i < 15; i += 2 {
-		table[i].Double(table[i/2])
-		table[i+1].Add(table[i], q)
 	}
 
-	// Instead of doing the classic double-and-add chain, we do it with a
+	// Start scanning the window from the most significant bits. We move by
-	// four-bit window: we double four times, and then add [0-15]P.
+	// 6 bits at a time and need to finish at -1, so -1 + 6 * 42 = 251.
+	index := 251
+
+	sel, sign := boothW6(s.Rsh(index))
+	// sign is always zero because the boothW6 input here is at
+	// most two bits long, so the top bit is never set.
+	_ = sign
+
+	// Neither Select nor Add have exceptions for the point at infinity /
+	// selector zero, so we don't need to check for it here or in the loop.
+	table := new(sm2p256Table).Compute(q)
+	table.Select(p, sel)
+
 	t := NewSM2P256Point()
-	p.Set(NewSM2P256Point())
+	for index >= 5 {
-	for i, byte := range scalar {
+		index -= 6
-		// No need to double on the first iteration, as p is the identity at
+
-		// this point, and [N]∞ = ∞.
-		if i != 0 {
 		p.Double(p)
 		p.Double(p)
 		p.Double(p)
 		p.Double(p)
+		p.Double(p)
+		p.Double(p)
+
+		if index >= 0 {
+			sel, sign = boothW6(s.Rsh(index) & 0x7f)
+		} else {
+			// Booth encoding considers a virtual zero bit at index -1,
+			// so we shift left the least significant limb.
+			wvalue := (s[0] << 1) & 0x7f
+			sel, sign = boothW6(wvalue)
 		}
 
-		windowValue := byte >> 4
+		table.Select(t, sel)
-		table.Select(t, windowValue)
+		t.Negate(sign)
-		p.Add(p, t)
-
-		p.Double(p)
-		p.Double(p)
-		p.Double(p)
-		p.Double(p)
-
-		windowValue = byte & 0b1111
-		table.Select(t, windowValue)
 		p.Add(p, t)
 	}
 
 	return p, nil
 }
 
-var sm2p256GeneratorTable *[sm2p256ElementLength * 2]sm2p256Table
+// Negate sets p to -p, if cond == 1, and to p if cond == 0.
-var sm2p256GeneratorTableOnce sync.Once
+func (p *SM2P256Point) Negate(cond int) *SM2P256Point {
-
+	negY := new(fiat.SM2P256Element)
-// generatorTable returns a sequence of sm2p256Tables. The first table contains
+	negY.Sub(negY, &p.y)
-// multiples of G. Each successive table is the previous table doubled four
+	p.y.Select(negY, &p.y, cond)
-// times.
+	return p
-func (p *SM2P256Point) generatorTable() *[sm2p256ElementLength * 2]sm2p256Table {
-	sm2p256GeneratorTableOnce.Do(func() {
-		sm2p256GeneratorTable = new([sm2p256ElementLength * 2]sm2p256Table)
-		base := NewSM2P256Point().SetGenerator()
-		for i := 0; i < sm2p256ElementLength*2; i++ {
-			sm2p256GeneratorTable[i][0] = NewSM2P256Point().Set(base)
-			for j := 1; j < 15; j++ {
-				sm2p256GeneratorTable[i][j] = NewSM2P256Point().Add(sm2p256GeneratorTable[i][j-1], base)
-			}
-			base.Double(base)
-			base.Double(base)
-			base.Double(base)
-			base.Double(base)
-		}
-	})
-	return sm2p256GeneratorTable
 }
 
-// ScalarBaseMult sets p = scalar * B, where B is the canonical generator, and
+// sm2P256AffineTable is a table of the first 32 multiples of a point. Points are
-// returns p.
+// stored at an index offset of -1 like in p256Table, and [0]P is not stored.
+type sm2P256AffineTable [32]sm2P256AffinePoint
+
+// Select selects the n-th multiple of the table base point into p. It works in
+// constant time. n can be in [0, 32], but (unlike p256Table.Select) if n is 0,
+// p is set to an undefined value.
+func (table *sm2P256AffineTable) Select(p *sm2P256AffinePoint, n uint8) {
+	if n > 32 {
+		panic("nistec: internal error: sm2P256AffineTable.Select called with out-of-bounds value")
+	}
+	for i := uint8(1); i <= 32; i++ {
+		cond := subtle.ConstantTimeByteEq(i, n)
+		p.x.Select(&table[i-1].x, &p.x, cond)
+		p.y.Select(&table[i-1].y, &p.y, cond)
+	}
+}
+
+// sm2p256GeneratorTable is a series of precomputed multiples of G, the canonical
+// generator. The first sm2P256AffineTable contains multiples of G. The second one
+// multiples of [2⁶]G, the third one of [2¹²]G, and so on, where each successive
+// table is the previous table doubled six times. Six is the width of the
+// sliding window used in ScalarBaseMult, and having each table already
+// pre-doubled lets us avoid the doublings between windows entirely. This table
+// aliases into p256PrecomputedEmbed.
+var sm2p256GeneratorTable *[43]sm2P256AffineTable
+
+//go:embed p256_asm_table.bin
+var p256PrecomputedEmbed string
+
+func init() {
+	p256PrecomputedPtr := (*unsafe.Pointer)(unsafe.Pointer(&p256PrecomputedEmbed))
+	// BigEndian architectures need to reverse the byte order of the table.
+	if runtime.GOARCH == "armbe" ||
+		runtime.GOARCH == "arm64be" ||
+		runtime.GOARCH == "ppc" ||
+		runtime.GOARCH == "ppc64" ||
+		runtime.GOARCH == "mips" ||
+		runtime.GOARCH == "mips64" ||
+		runtime.GOARCH == "sparc" ||
+		runtime.GOARCH == "sparc64" ||
+		runtime.GOARCH == "s390" {
+		var newTable [43 * 32 * 2 * 4]uint64
+		for i, x := range (*[43 * 32 * 2 * 4][8]byte)(*p256PrecomputedPtr) {
+			newTable[i] = byteorder.LEUint64(x[:])
+		}
+		newTablePtr := unsafe.Pointer(&newTable)
+		p256PrecomputedPtr = &newTablePtr
+	}
+	sm2p256GeneratorTable = (*[43]sm2P256AffineTable)(*p256PrecomputedPtr)
+}
+
+// ScalarBaseMult sets p = scalar * generator, where scalar is a 32-byte big
+// endian value, and returns r. If scalar is not 32 bytes long, ScalarBaseMult
+// returns an error and the receiver is unchanged.
 func (p *SM2P256Point) ScalarBaseMult(scalar []byte) (*SM2P256Point, error) {
-	if len(scalar) != sm2p256ElementLength {
+	// This function works like ScalarMult above, but the table is fixed and
-		return nil, errors.New("invalid scalar length")
+	// "pre-doubled" for each iteration, so instead of doubling we move to the
+	// next table at each iteration.
+
+	s, err := new(p256OrdElement).SetBytes(scalar)
+	if err != nil {
+		return nil, err
 	}
-	tables := p.generatorTable()
 
-	// This is also a scalar multiplication with a four-bit window like in
+	// Start scanning the window from the most significant bits. We move by
-	// ScalarMult, but in this case the doublings are precomputed. The value
+	// 6 bits at a time and need to finish at -1, so -1 + 6 * 42 = 251.
-	// [windowValue]G added at iteration k would normally get doubled
+	index := 251
-	// (totIterations-k)×4 times, but with a larger precomputation we can
-	// instead add [2^((totIterations-k)×4)][windowValue]G and avoid the
-	// doublings between iterations.
-	t := NewSM2P256Point()
-	p.Set(NewSM2P256Point())
-	tableIndex := len(tables) - 1
-	for _, byte := range scalar {
-		windowValue := byte >> 4
-		tables[tableIndex].Select(t, windowValue)
-		p.Add(p, t)
-		tableIndex--
 
-		windowValue = byte & 0b1111
+	sel, sign := boothW6(s.Rsh(index))
-		tables[tableIndex].Select(t, windowValue)
+	// sign is always zero because the boothW6 input here is at
-		p.Add(p, t)
+	// most five bits long, so the top bit is never set.
-		tableIndex--
+	_ = sign
+
+	t := &sm2P256AffinePoint{}
+	table := &sm2p256GeneratorTable[(index+1)/6]
+	table.Select(t, sel)
+
+	// Select's output is undefined if the selector is zero, when it should be
+	// the point at infinity (because infinity can't be represented in affine
+	// coordinates). Here we conditionally set p to the infinity if sel is zero.
+	// In the loop, that's handled by AddAffine.
+	selIsZero := subtle.ConstantTimeByteEq(sel, 0)
+	p.Select(NewSM2P256Point(), t.Projective(), selIsZero)
+
+	for index >= 5 {
+		index -= 6
+
+		if index >= 0 {
+			sel, sign = boothW6(s.Rsh(index) & 0b1111111)
+		} else {
+			// Booth encoding considers a virtual zero bit at index -1,
+			// so we shift left the least significant limb.
+			wvalue := (s[0] << 1) & 0b1111111
+			sel, sign = boothW6(wvalue)
+		}
+
+		table := &sm2p256GeneratorTable[(index+1)/6]
+		table.Select(t, sel)
+		t.Negate(sign)
+		selIsZero := subtle.ConstantTimeByteEq(sel, 0)
+		p.AddAffine(p, t, selIsZero)
 	}
 
 	return p, nil
 }
 
+// Negate sets p to -p, if cond == 1, and to p if cond == 0.
+func (p *sm2P256AffinePoint) Negate(cond int) *sm2P256AffinePoint {
+	negY := new(fiat.SM2P256Element)
+	negY.Sub(negY, &p.y)
+	p.y.Select(negY, &p.y, cond)
+	return p
+}
+
 // sm2p256Sqrt sets e to a square root of x. If x is not a square, sm2p256Sqrt returns
 // false and e is unchanged. e and x can overlap.
 func sm2p256Sqrt(e, x *fiat.SM2P256Element) (isSquare bool) {
@@ -480,47 +693,32 @@ func sm2p256SqrtCandidate(z, x *fiat.SM2P256Element) {
 	t2.Square(z)
 	t3.Square(t2)
 	t1.Square(t3)
-	t4.Square(t1)
+	p256Square(t4, t1, 3)
-	for s := 1; s < 3; s++ {
-		t4.Square(t4)
-	}
 	t3.Mul(t3, t4)
-	for s := 0; s < 5; s++ {
+	p256Square(t3, t3, 5)
-		t3.Square(t3)
-	}
 	t1.Mul(t1, t3)
-	t3.Square(t1)
+	p256Square(t3, t1, 2)
-	for s := 1; s < 2; s++ {
-		t3.Square(t3)
-	}
 	t2.Mul(t2, t3)
-	for s := 0; s < 14; s++ {
+	p256Square(t2, t2, 14)
-		t2.Square(t2)
-	}
 	t1.Mul(t1, t2)
 	t0.Mul(t0, t1)
-	for s := 0; s < 4; s++ {
+	p256Square(t0, t0, 4)
-		t0.Square(t0)
+	p256Square(t1, t0, 31)
-	}
-	t1.Square(t0)
-	for s := 1; s < 31; s++ {
-		t1.Square(t1)
-	}
 	t0.Mul(t0, t1)
-	for s := 0; s < 32; s++ {
+	p256Square(t1, t1, 32)
-		t1.Square(t1)
-	}
 	t1.Mul(t0, t1)
-	for s := 0; s < 62; s++ {
+	p256Square(t1, t1, 62)
-		t1.Square(t1)
-	}
 	t0.Mul(t0, t1)
 	z.Mul(z, t0)
-	for s := 0; s < 32; s++ {
+	p256Square(z, z, 32)
-		z.Square(z)
-	}
 	z.Mul(x, z)
-	for s := 0; s < 62; s++ {
+	p256Square(z, z, 62)
-		z.Square(z)
+}
+
+// p256Square sets e to the square of x, repeated n times > 1.
+func p256Square(e, x *fiat.SM2P256Element, n int) {
+	e.Square(x)
+	for i := 1; i < n; i++ {
+		e.Square(e)
 	}
 }