zstd: x86 assembler implementation of sequenceDecs.decode

Differences with the Go implementation:
- check ml and mo in the main loop,
- s.seqSize and litRemain are checked in the end.

Author: WojciechMula

.gitignore

@@ -23,3 +23,10 @@ _testmain.go
 *.test
 *.prof
 /s2/cmd/_s2sx/sfx-exe
+
+# Linux perf files
+perf.data
+perf.data.old
+
+# gdb history
+.gdb_history

zstd/autogen.go

@@ -0,0 +1,4 @@
+package zstd
+
+//go:generate go run generate.go
+//go:generate asmfmt -w seqdec_amd64.s

zstd/generate.go

@@ -0,0 +1,74 @@
+//go:build ignore
+// +build ignore
+
+package main
+
+import (
+	"log"
+	"os"
+	"path"
+	"text/template"
+)
+
+func main() {
+	mapping := []struct {
+		template string
+		output   string
+	}{{
+		template: "seqdec_amd64.s.in",
+		output:   "seqdec_amd64.s",
+	},
+	}
+
+	for i := range mapping {
+
+		state := make(map[string]string)
+
+		funcMap := template.FuncMap{
+			"var": func(name string) string { return state[name] },
+			"set": func(name, value string) string {
+				state[name] = value
+				return ""
+			},
+		}
+
+		input := mapping[i].template
+		output := mapping[i].output
+		if !shouldRegenerate(input, output) {
+			log.Printf("%q is up to date", output)
+			continue
+		}
+
+		tmpl, err := template.New(path.Base(input)).Funcs(funcMap).ParseFiles(input)
+		die(err)
+
+		out, err := os.Create(output)
+		die(err)
+		defer out.Close()
+
+		log.Printf("Generating %q from %q", output, input)
+		err = tmpl.Execute(out, nil)
+		die(err)
+	}
+}
+
+func die(err error) {
+	if err != nil {
+		log.Fatal(err)
+		os.Exit(1)
+	}
+}
+
+func shouldRegenerate(srcpath, dstpath string) bool {
+	src, err1 := os.Stat(srcpath)
+	if err1 != nil {
+		return true // I/O errors will be rediscovered later
+	}
+
+	dst, err2 := os.Stat(dstpath)
+	if err2 != nil {
+		return true
+	}
+
+	return src.ModTime().After(dst.ModTime())
+}

zstd/seqdec.go

@@ -98,150 +98,6 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, out []byte) erro
 	return nil
 }
 
-// decode sequences from the stream with the provided history.
-func (s *sequenceDecs) decode(seqs []seqVals) error {
-	br := s.br
-
-	// Grab full sizes tables, to avoid bounds checks.
-	llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize]
-	llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state
-	s.seqSize = 0
-	litRemain := len(s.literals)
-	maxBlockSize := maxCompressedBlockSize
-	if s.windowSize < maxBlockSize {
-		maxBlockSize = s.windowSize
-	}
-	for i := range seqs {
-		var ll, mo, ml int
-		if br.off > 4+((maxOffsetBits+16+16)>>3) {
-			// inlined function:
-			// ll, mo, ml = s.nextFast(br, llState, mlState, ofState)
-
-			// Final will not read from stream.
-			var llB, mlB, moB uint8
-			ll, llB = llState.final()
-			ml, mlB = mlState.final()
-			mo, moB = ofState.final()
-
-			// extra bits are stored in reverse order.
-			br.fillFast()
-			mo += br.getBits(moB)
-			if s.maxBits > 32 {
-				br.fillFast()
-			}
-			ml += br.getBits(mlB)
-			ll += br.getBits(llB)
-
-			if moB > 1 {
-				s.prevOffset[2] = s.prevOffset[1]
-				s.prevOffset[1] = s.prevOffset[0]
-				s.prevOffset[0] = mo
-			} else {
-				// mo = s.adjustOffset(mo, ll, moB)
-				// Inlined for rather big speedup
-				if ll == 0 {
-					// There is an exception though, when current sequence's literals_length = 0.
-					// In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2,
-					// an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte.
-					mo++
-				}
-
-				if mo == 0 {
-					mo = s.prevOffset[0]
-				} else {
-					var temp int
-					if mo == 3 {
-						temp = s.prevOffset[0] - 1
-					} else {
-						temp = s.prevOffset[mo]
-					}
-
-					if temp == 0 {
-						// 0 is not valid; input is corrupted; force offset to 1
-						println("WARNING: temp was 0")
-						temp = 1
-					}
-
-					if mo != 1 {
-						s.prevOffset[2] = s.prevOffset[1]
-					}
-					s.prevOffset[1] = s.prevOffset[0]
-					s.prevOffset[0] = temp
-					mo = temp
-				}
-			}
-			br.fillFast()
-		} else {
-			if br.overread() {
-				if debugDecoder {
-					printf("reading sequence %d, exceeded available data\n", i)
-				}
-				return io.ErrUnexpectedEOF
-			}
-			ll, mo, ml = s.next(br, llState, mlState, ofState)
-			br.fill()
-		}
-
-		if debugSequences {
-			println("Seq", i, "Litlen:", ll, "mo:", mo, "(abs) ml:", ml)
-		}
-		// Evaluate.
-		// We might be doing this async, so do it early.
-		if mo == 0 && ml > 0 {
-			return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml)
-		}
-		if ml > maxMatchLen {
-			return fmt.Errorf("match len (%d) bigger than max allowed length", ml)
-		}
-		s.seqSize += ll + ml
-		if s.seqSize > maxBlockSize {
-			return fmt.Errorf("output (%d) bigger than max block size (%d)", s.seqSize, maxBlockSize)
-		}
-		litRemain -= ll
-		if litRemain < 0 {
-			return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, litRemain+ll)
-		}
-		seqs[i] = seqVals{
-			ll: ll,
-			ml: ml,
-			mo: mo,
-		}
-		if i == len(seqs)-1 {
-			// This is the last sequence, so we shouldn't update state.
-			break
-		}
-
-		// Manually inlined, ~ 5-20% faster
-		// Update all 3 states at once. Approx 20% faster.
-		nBits := llState.nbBits() + mlState.nbBits() + ofState.nbBits()
-		if nBits == 0 {
-			llState = llTable[llState.newState()&maxTableMask]
-			mlState = mlTable[mlState.newState()&maxTableMask]
-			ofState = ofTable[ofState.newState()&maxTableMask]
-		} else {
-			bits := br.get32BitsFast(nBits)
-			lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31))
-			llState = llTable[(llState.newState()+lowBits)&maxTableMask]
-
-			lowBits = uint16(bits >> (ofState.nbBits() & 31))
-			lowBits &= bitMask[mlState.nbBits()&15]
-			mlState = mlTable[(mlState.newState()+lowBits)&maxTableMask]
-
-			lowBits = uint16(bits) & bitMask[ofState.nbBits()&15]
-			ofState = ofTable[(ofState.newState()+lowBits)&maxTableMask]
-		}
-	}
-	s.seqSize += litRemain
-	if s.seqSize > maxBlockSize {
-		return fmt.Errorf("output (%d) bigger than max block size (%d)", s.seqSize, maxBlockSize)
-	}
-	err := br.close()
-	if err != nil {
-		printf("Closing sequences: %v, %+v\n", err, *br)
-	}
-	return err
-}
-
 // execute will execute the decoded sequence with the provided history.
 // The sequence must be evaluated before being sent.
 func (s *sequenceDecs) execute(seqs []seqVals, hist []byte) error {

zstd/seqdec_amd64.go

@@ -0,0 +1,86 @@
+//go:build amd64 && !appengine && !noasm && gc
+// +build amd64,!appengine,!noasm,gc
+
+package zstd
+
+import (
+	"fmt"
+)
+
+type decodeAsmContext struct {
+	llTable   []decSymbol
+	mlTable   []decSymbol
+	ofTable   []decSymbol
+	llState   decSymbol
+	mlState   decSymbol
+	ofState   decSymbol
+	iteration int
+	seqs      []seqVals
+	litRemain int
+}
+
+// error reported when mo == 0 && ml > 0
+const errorMatchLenOfsMismatch = 1
+
+// error reported when ml > maxMatchLen
+const errorMatchLenTooBig = 2
+
+// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm.
+//
+// Please refer to seqdec_generic.go for the reference implementation.
+func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
+
+// decode sequences from the stream without the provided history.
+func (s *sequenceDecs) decode(seqs []seqVals) error {
+	br := s.br
+
+	maxBlockSize := maxCompressedBlockSize
+	if s.windowSize < maxBlockSize {
+		maxBlockSize = s.windowSize
+	}
+
+	ctx := decodeAsmContext{
+		llTable:   s.litLengths.fse.dt[:maxTablesize],
+		mlTable:   s.matchLengths.fse.dt[:maxTablesize],
+		ofTable:   s.offsets.fse.dt[:maxTablesize],
+		llState:   s.litLengths.state.state,
+		mlState:   s.matchLengths.state.state,
+		ofState:   s.offsets.state.state,
+		seqs:      seqs,
+		iteration: len(seqs) - 1,
+		litRemain: len(s.literals),
+	}
+
+	s.seqSize = 0
+
+	errCode := sequenceDecs_decode_amd64(s, br, &ctx)
+	if errCode != 0 {
+		i := len(s.literals) - ctx.iteration
+		switch errCode {
+		case errorMatchLenOfsMismatch:
+			ml := ctx.seqs[i].ml
+			return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml)
+
+		case errorMatchLenTooBig:
+			ml := ctx.seqs[i].ml
+			return fmt.Errorf("match len (%d) bigger than max allowed length", ml)
+		}
+
+		return fmt.Errorf("sequenceDecs_decode_amd64 returned erronous code %d", errCode)
+	}
+
+	if ctx.litRemain < 0 {
+		return fmt.Errorf("literal count is too big: total available %d, total requested %d",
+			len(s.literals), len(s.literals)-ctx.litRemain)
+	}
+
+	s.seqSize += ctx.litRemain
+	if s.seqSize > maxBlockSize {
+		return fmt.Errorf("output (%d) bigger than max block size (%d)", s.seqSize, maxBlockSize)
+	}
+	err := br.close()
+	if err != nil {
+		printf("Closing sequences: %v, %+v\n", err, *br)
+	}
+	return err
+}