Matcher interface; benchmarks

Also a bugfix re default rebuild logic.

Also pruner.NewMatcher() copies the default rebuild trigger.

Author: jsmorph

lib/core_matcher.go

@@ -108,15 +108,15 @@ func (m *CoreMatcher) MatchesForJSONEvent(event []byte) ([]X, error) {
 	if err != nil {
 		return nil, err
 	}
-	matches := m.MatchesForFields(fields)
-	return matches, nil
+	return m.MatchesForFields(fields)
+
 }
 
 // MatchesForFields takes a list of Field structures and sorts them by pathname; the fields in a pattern to
 //  matched are similarly sorted; thus running an automaton over them works
-func (m *CoreMatcher) MatchesForFields(fields []Field) []X {
+func (m *CoreMatcher) MatchesForFields(fields []Field) ([]X, error) {
 	sort.Slice(fields, func(i, j int) bool { return string(fields[i].Path) < string(fields[j].Path) })
-	return m.matchesForSortedFields(fields).matches()
+	return m.matchesForSortedFields(fields).matches(), nil
 }
 
 // proposedTransition represents a suggestion that the name/value pair at fields[fieldIndex] might allow a transition

lib/fj.go

@@ -72,7 +72,7 @@ func (fj *FJ) FlattenAndMatch(event []byte) ([]X, error) {
 	if err != nil {
 		return nil, err
 	}
-	return fj.matcher.MatchesForFields(fields), nil
+	return fj.matcher.MatchesForFields(fields)
 }
 
 // Flatten implements the Flattener interface. It assumes that the event is immutable - if you modify the event

lib/matcher.go

@@ -3,6 +3,6 @@ package quamina
 type Matcher interface {
 	AddPattern(x X, pat string) error
 	MatchesForJSONEvent(event []byte) ([]X, error)
-	MatchesForFields(fields []Field) []X
+	MatchesForFields(fields []Field) ([]X, error)
 	DeletePattern(x X) error
 }

pruner/pruner.go

@@ -62,7 +62,7 @@ type Matcher struct {
 	// Matcher is the underlying matcher that does the hard work.
 	//
 	// Maybe Matcher should maybe not be embedded or public.
-	*quamina.Matcher
+	Matcher *quamina.CoreMatcher
 
 	// live is live set of patterns.
 	live LivePatternsState
@@ -108,6 +108,7 @@ func newTooMuchFiltering(ratio float64, min int64) *tooMuchFiltering {
 }
 
 func (t *tooMuchFiltering) Rebuild(added bool, s *Stats) bool {
+
 	if added {
 		// No need to think when we're adding a pattern since
 		// that operation cannot result in an increase of
@@ -123,11 +124,11 @@ func (t *tooMuchFiltering) Rebuild(added bool, s *Stats) bool {
 
 	// We won't rebuild if nothing's been emitted yet.
 	//
-	// In isolating, that heuristic is arguable, but for this
+	// In isolation, this heuristic is arguable, but for this
 	// policy we need it.  Otherwise we'll divide by zero, and
 	// nobody wants that.
 	if s.Emitted == 0 {
-		return true
+		return false
 	}
 
 	var (
@@ -170,10 +171,11 @@ func NewMatcher(s LivePatternsState) *Matcher {
 	if s == nil {
 		s = NewMemState()
 	}
+	trigger := *defaultRebuildTrigger // Copy
 	return &Matcher{
-		Matcher:        quamina.NewMatcher(),
+		Matcher:        quamina.NewCoreMatcher(),
 		live:           s,
-		rebuildTrigger: defaultRebuildTrigger,
+		rebuildTrigger: &trigger,
 	}
 }
 
@@ -236,7 +238,10 @@ func (m *Matcher) MatchesForJSONEvent(event []byte) ([]quamina.X, error) {
 
 func (m *Matcher) MatchesForFields(fields []quamina.Field) ([]quamina.X, error) {
 
-	xs := m.Matcher.MatchesForFields(fields)
+	xs, err := m.Matcher.MatchesForFields(fields)
+	if err != nil {
+		return nil, err
+	}
 
 	// Remove any X that isn't in the live set.
 
@@ -273,7 +278,7 @@ func (m *Matcher) MatchesForFields(fields []quamina.Field) ([]quamina.X, error)
 //
 // The return boolean when true indicates that at least one pattern
 // for x was removed.
-func (m *Matcher) DeletePattern(x quamina.X) (bool, error) {
+func (m *Matcher) DeletePattern(x quamina.X) error {
 	// Maybe better to return (int,error) as in
 	// LivePatternStats.Delete(), or maybe just return an error
 	// and nothing else.
@@ -289,7 +294,7 @@ func (m *Matcher) DeletePattern(x quamina.X) (bool, error) {
 		}
 	}
 
-	return 0 < n, err
+	return err
 }
 
 // Rebuild rebuilds the matcher state based on only live patterns.
@@ -313,7 +318,7 @@ func (m *Matcher) rebuild(fearlessly bool) error {
 
 	var (
 		then = time.Now()
-		m1   = quamina.NewMatcher()
+		m1   = quamina.NewCoreMatcher()
 	)
 
 	if fearlessly {

pruner/pruner_test.go

@@ -1,11 +1,14 @@
 package pruner
 
 import (
+	"bufio"
 	"fmt"
 	"log"
-	quamina "quamina/lib"
+	"os"
 	"testing"
 	"time"
+
+	quamina "quamina/lib"
 )
 
 var verbose = false
@@ -50,15 +53,11 @@ func TestBasic(t *testing.T) {
 
 	m.printStats()
 
-	if have, err := m.DeletePattern(id); err != nil {
+	if err := m.DeletePattern(id); err != nil {
 		t.Fatal(err)
-	} else if !have {
-		t.Fatal(have)
 	}
-	if have, err := m.DeletePattern(id); err != nil {
+	if err := m.DeletePattern(id); err != nil {
 		t.Fatal(err)
-	} else if have {
-		t.Fatal(have)
 	}
 
 	m.printStats()
@@ -92,8 +91,8 @@ func TestBasic(t *testing.T) {
 
 func TestRebuildSome(t *testing.T) {
 	var (
+		n = int(2 * defaultRebuildTrigger.MinAction)
 		m = NewMatcher(nil)
-		n = int(defaultRebuildTrigger.MinAction + 100)
 	)
 
 	populate := func() {
@@ -107,10 +106,8 @@ func TestRebuildSome(t *testing.T) {
 
 	depopulate := func() {
 		for i := 0; i < n; i += 2 {
-			if had, err := m.DeletePattern(i); err != nil {
+			if err := m.DeletePattern(i); err != nil {
 				t.Fatal(err)
-			} else if !had {
-				t.Fatal(i)
 			}
 		}
 		// Maybe check a lot more often.
@@ -153,6 +150,7 @@ func TestRebuildSome(t *testing.T) {
 		m.printStats()
 		depopulate()
 		query(false)
+		m.printStats()
 		if s := m.Stats(); 0 == s.RebuildDuration {
 			t.Fatal(s)
 		}
@@ -195,7 +193,7 @@ func TestTriggerTooManyFilteredDenom(t *testing.T) {
 	if err := m.AddPattern(1, `{"likes":["tacos"]}`); err != nil {
 		t.Fatal(err)
 	}
-	if _, err := m.DeletePattern(1); err != nil {
+	if err := m.DeletePattern(1); err != nil {
 		t.Fatal(err)
 	}
 	_, err := m.MatchesForJSONEvent([]byte(`{"likes":"tacos"}`))
@@ -217,12 +215,12 @@ func TestTriggerRebuild(t *testing.T) {
 		doomed  = func(id int) bool {
 			return id%2 == 0
 		}
-		printState = func() {
-			logf("state:")
-			for id, p := range m.live.(*MemState).m {
-				logf("  %v -> %s", id, p)
-			}
-		}
+		// printState = func() {
+		// 	logf("state:")
+		// 	for id, p := range m.live.(*MemState).m {
+		// 		logf("  %v -> %s", id, p)
+		// 	}
+		// }
 	)
 
 	trigger.MinAction = 5
@@ -235,13 +233,13 @@ func TestTriggerRebuild(t *testing.T) {
 		}
 
 		if doomed(i) {
-			if _, err := m.DeletePattern(i); err != nil {
+			if err := m.DeletePattern(i); err != nil {
 				t.Fatal(err)
 			}
 		}
 	}
 
-	printState()
+	// printState()
 	m.printStats()
 
 	for i := 0; i < n; i++ {
@@ -261,7 +259,7 @@ func TestTriggerRebuild(t *testing.T) {
 		}
 	}
 
-	printState()
+	// printState()
 	m.printStats()
 
 	s := m.Stats()
@@ -314,7 +312,7 @@ func TestBadState(t *testing.T) {
 	if err := m.AddPattern(1, `{"likes":["queso"]}`); err == nil {
 		t.Fatal("expected error")
 	}
-	if _, err := m.DeletePattern(1); err == nil {
+	if err := m.DeletePattern(1); err == nil {
 		t.Fatal("expected error")
 	}
 	if err := m.Rebuild(false); err == nil {
@@ -433,9 +431,7 @@ func TestMultiplePatternsWithSameId(t *testing.T) {
 		t.Fatal(s.Live)
 	}
 
-	if had, err := m.DeletePattern(id); err != nil {
-		t.Fatal(err)
-	} else if !had {
+	if err := m.DeletePattern(id); err != nil {
 		t.Fatal(err)
 	}
 
@@ -450,3 +446,91 @@ func TestMultiplePatternsWithSameId(t *testing.T) {
 	}
 
 }
+
+func BenchmarkCityLotsCore(b *testing.B) {
+	benchmarkCityLots(b, quamina.NewCoreMatcher())
+}
+
+func BenchmarkCityLotsPruner(b *testing.B) {
+	benchmarkCityLots(b, NewMatcher(nil))
+}
+
+// benchmarkCityLots is distilled from TestCityLots.
+func benchmarkCityLots(b *testing.B, m quamina.Matcher) {
+
+	oneMeg := 1024 * 1024
+	file, err := os.Open("../test_data/citylots.jlines")
+	if err != nil {
+		b.Errorf("Can't open file %s", err)
+	}
+	defer file.Close()
+
+	patterns := []string{
+		`{ "properties": { "STREET": [ "CRANLEIGH" ] } }`,
+		`{ "properties": { "STREET": [ "17TH" ], "ODD_EVEN": [ "E"] } }`,
+		`{ "geometry": { "coordinates": [ 37.807807921694092 ] } }`,
+		`{ "properties": { "MAPBLKLOT": ["0011008"], "BLKLOT": ["0011008"]},  "geometry": { "coordinates": [ 37.807807921694092 ] } } `,
+	}
+	names := []string{
+		"CRANLEIGH",
+		"17TH Even",
+		"Geometry",
+		"0011008",
+	}
+
+	scanner := bufio.NewScanner(file)
+	buf := make([]byte, oneMeg)
+	scanner.Buffer(buf, oneMeg)
+
+	var fj quamina.Flattener
+	switch vv := m.(type) {
+	case *Matcher:
+		fj = quamina.NewFJ(vv.Matcher)
+		vv.DisableRebuild()
+	case *quamina.CoreMatcher:
+		fj = quamina.NewFJ(vv)
+	default:
+		b.Fatalf("%T", vv)
+	}
+
+	for i := range names {
+		err = m.AddPattern(names[i], patterns[i])
+		if err != nil {
+			b.Errorf("AddPattern error %s", err)
+		}
+	}
+	results := make(map[quamina.X]int)
+
+	lineCount := 0
+	var lines [][]byte
+	for scanner.Scan() {
+		lineCount++
+		lines = append(lines, []byte(scanner.Text()))
+	}
+	lineCount = 0
+
+	b.ResetTimer()
+
+	for _, line := range lines {
+		matches, err := fj.FlattenAndMatch(line)
+		if err != nil {
+			b.Errorf("Matches4JSON error %s on %s", err, line)
+		}
+		lineCount++
+		if lineCount == b.N {
+			break
+		}
+		for _, match := range matches {
+			count, ok := results[match]
+			if !ok {
+				count = 0
+			}
+			results[match] = count + 1
+		}
+	}
+
+	err = scanner.Err()
+	if err != nil {
+		b.Errorf("Scanner error %s", err)
+	}
+}