Switched from map[string] to smallTable based value matching.

Author: timbray

lib/benchmarks_test.go

@@ -61,7 +61,9 @@ func TestCityLots(t *testing.T) {
 	if err != nil {
 		t.Error("Can't open file: " + err.Error())
 	}
-	defer file.Close()
+	defer func(file *os.File) {
+		_ = file.Close()
+	}(file)
 
 	patterns := []string{
 		`{ "properties": { "STREET": [ "CRANLEIGH" ] } }`,

lib/field_matcher.go

@@ -1,46 +1,45 @@
 package quamina
 
 // fieldMatchState has a map which is keyed by the field pathSegments values that can start transitions from this
-//  state; for eacrh such field, there is a valueMatcher which, given the field's value, determines whether
+//  state; for each such field, there is a valueMatcher which, given the field's value, determines whether
 //  the automaton progresses to another fieldMatchState
 // matches contains the X values that arrival at this state implies have matched
 // existsFalseFailures reports the condition that traversal has occurred by matching a field which is named in an
 //  exists:false pattern, and the named X's should be subtracted from the matches list being built up by a match project
 // The matches field contains a list of the patterns that have been matched if traversal arrives at this state
 type fieldMatchState struct {
-	transitions         map[string]*valueMatchState
+	transitions         map[string]*valueMatcher
 	matches             []X
 	existsFalseFailures *matchSet
 }
 
 func newFieldMatchState() *fieldMatchState {
-	return &fieldMatchState{transitions: make(map[string]*valueMatchState), existsFalseFailures: newMatchSet()}
+	return &fieldMatchState{transitions: make(map[string]*valueMatcher), existsFalseFailures: newMatchSet()}
 }
 
 func (m *fieldMatchState) addTransition(field *patternField) []*fieldMatchState {
 
 	// transition from a fieldMatchstate might already be present; create a new empty one if not
-	valueMatcher, ok := m.transitions[field.path]
+	vm, ok := m.transitions[field.path]
 	if !ok {
-		valueMatcher = newValueMatchState()
-		m.transitions[field.path] = valueMatcher
+		vm = newValueMatcher()
+		m.transitions[field.path] = vm
 	}
 
 	// suppose I'm adding the first pattern to a matcher and it has "x": [1, 2]. In principle the branches on
 	//  "x": 1 and "x": 2 could go to tne same next state. But we have to make a unique next state for each of them
 	//  because some future other pattern might have "x": [2, 3] and thus we need a separate branch to potentially
 	//  match two patterns on "x": 2 but not "x": 1. If you were optimizing the automaton for size you might detect
 	//  cases where this doesn't happen and reduce the number of fieldMatchstates
-
 	var nextFieldMatchers []*fieldMatchState
 	for _, val := range field.vals {
-		nextFieldMatchers = append(nextFieldMatchers, valueMatcher.addTransition(val))
+		nextFieldMatchers = append(nextFieldMatchers, vm.addTransition(val))
 	}
 	return nextFieldMatchers
 }
 
 // transitionOn returns one or more fieldMatchStates you can transition to on a field's name/value combination,
-//  or nil if no trnasitions are possible.
+//  or nil if no transitions are possible.
 func (m *fieldMatchState) transitionOn(field *Field) []*fieldMatchState {
 
 	// are there transitions on this field name?
@@ -51,15 +50,3 @@ func (m *fieldMatchState) transitionOn(field *Field) []*fieldMatchState {
 
 	return valMatcher.transitionOn(field.Val)
 }
-
-/* for debugging
-func (m *fieldMatchState) String() string {
-	var keys []string
-	for k := range m.transitions {
-		p := strings.ReplaceAll(k, "\n", "**")
-		keys = append(keys, p)
-	}
-	keys = append(keys, fmt.Sprintf(" Matches: %d", len(m.matches)))
-	return strings.Join(keys, " / ")
-}
-*/

lib/fj.go

@@ -661,7 +661,7 @@ func (fj *FJ) stepOneArrayElement() {
 	fj.arrayTrail[len(fj.arrayTrail)-1].Pos++
 }
 
-// ch fetches the next byte from the event, while tracking line/column numbers. It doesn't check array bounds,
+// ch fetches the next byte from the event. It doesn't check array bounds,
 //  so it's the caller's responsibility to ensure we haven't run off the end of the event.
 func (fj *FJ) ch() byte {
 	return fj.event[fj.eventIndex]

lib/flattener.go

@@ -10,7 +10,9 @@ type Field struct {
 	ArrayTrail []ArrayPos
 }
 
-// Flattener by example:
+// Flattener is provided as an interface in the hope that flatterners for other non-JSON message formats might
+//  be implemented.
+// How it needs to work, by JSON example:
 // { "a": 1, "b": "two", "c": true", "d": nil, "e": { "e1": 2, "e2":, 3.02e-5} "f": [33, "x"]} }
 // should produce
 // "a", "1"
@@ -30,7 +32,6 @@ type Field struct {
 // "a", "1"
 // "f", "33"
 // "f", "\"x\""
-
 type Flattener interface {
 	Flatten(event []byte, nameTracker NameTracker) ([]Field, error)
 	Reset()

lib/match_set.go

@@ -1,5 +1,7 @@
 package quamina
 
+// matchSet is what it says on the tin; implements a set semantic on matches, which are of type X. These could all
+//  be implemented as match[X]bool but this makes the calling code more readable.
 type matchSet struct {
 	set map[X]bool
 }
@@ -11,26 +13,12 @@ func newMatchSet() *matchSet {
 func (m *matchSet) addX(x X) {
 	m.set[x] = true
 }
-func (m *matchSet) removeX(x X) {
-	delete(m.set, x)
-}
+
 func (m *matchSet) contains(x X) bool {
 	_, ok := m.set[x]
 	return ok
 }
 
-func (m *matchSet) subtractXs(xs []X) {
-	for _, x := range xs {
-		delete(m.set, x)
-	}
-}
-
-func (m *matchSet) addSet(addend *matchSet) {
-	for x := range addend.set {
-		m.set[x] = true
-	}
-}
-
 func (m *matchSet) matches() []X {
 	matches := make([]X, 0, len(m.set))
 	for x := range m.set {

lib/matcher.go

@@ -13,7 +13,7 @@ import (
 )
 
 // Matcher uses a finite automaton to implement the MatchesForJSONEvent and MatchesForFields functions.
-// startState is the start of the atomaton
+// startState is the start of the automaton
 // namesUsed is a map of field names that are used in any of the patterns that this automaton encodes. Typically,
 //  patterns only consider a subset of the fields in an incoming data object, and there is no reason to consider
 //  fields that do not appear in patterns when using the automaton for matching
@@ -66,7 +66,7 @@ func (m *Matcher) AddPattern(x X, patternJSON string) error {
 
 	// "states" now holds the set of terminal states arrived at by matching each field in the pattern,
 	//   so update the matches value to indicate this (skipping those that are only there to serve
-	//   exists:false processing
+	//   exists:false processing)
 	for _, endState := range states {
 		if !endState.existsFalseFailures.contains(x) {
 			endState.matches = append(endState.matches, x)
@@ -108,9 +108,9 @@ func (m *Matcher) matchesForSortedFields(fields []Field) *matchSet {
 
 	failedExistsFalseMatches := newMatchSet()
 
-	// The idea is that we addX potential field transitions to the proposals list; any time such a transition
+	// The idea is that we add potential field transitions to the proposals list; any time such a transition
 	//  succeeds, i.e. matches a particular field and moves to a new state, we propose transitions from that
-	//  state on all the following fields in the list
+	//  state on all the following fields in the event
 	// Start by giving each field a chance to match against the start state. Doing it by pre-allocating the
 	//  proposals and filling in their values is observably faster than the more idiomatic append()
 	proposals := make([]proposedTransition, len(fields))
@@ -134,7 +134,7 @@ func (m *Matcher) matchesForSortedFields(fields []Field) *matchSet {
 		// for each state in the set of transitions from the proposed state
 		for _, nextState := range nextStates {
 
-			// if arriving at this state means we've matched one or more patterns, record that fact (& lose dupes)
+			// if arriving at this state means we've matched one or more patterns, record that fact
 			for _, nextMatch := range nextState.matches {
 				matches.addX(nextMatch)
 			}
@@ -187,21 +187,3 @@ func (m *Matcher) IsNameUsed(label []byte) bool {
 	_, ok := m.namesUsed[string(label)]
 	return ok
 }
-
-// for debugging
-/*
-func prettyField(f Field) string {
-	p := string(f.Path)
-	p = strings.ReplaceAll(p, "\n", "/")
-	v := string(f.Val)
-	pv := p + "*" + v + " "
-	for _, ap := range f.ArrayTrail {
-		pv = pv + fmt.Sprintf("@%d.%d ", ap.Array, ap.Pos)
-	}
-	return pv
-}
-
-	log := fmt.Sprintf("S: %v, F: %s, matches %d", prop.state, prop.fields[prop.fieldIndex], len(nextStates))
-	log = strings.ReplaceAll(log, "\n", "**")
-	fmt.Println(log)
-*/

lib/matcher_test.go

@@ -15,21 +15,14 @@ func TestBasicMatching(t *testing.T) {
 		t.Error(err.Error())
 	}
 	shouldMatch := []string{
-		`{"a": 2, "b": "3", "x": 33}`,
 		`{"b": "3", "a": 1}`,
+		`{"a": 2, "b": "3", "x": 33}`,
 	}
 	shouldNotMatch := []string{
 		`{"b": "3", "a": 6}`,
 		`{"a": 2}`,
 		`{"b": "3"}`,
 	}
-	for _, shouldNot := range shouldNotMatch {
-		var matches []X
-		matches, err = m.MatchesForJSONEvent([]byte(shouldNot))
-		if len(matches) != 0 {
-			t.Error("Matched: " + shouldNot)
-		}
-	}
 	for _, should := range shouldMatch {
 		var matches []X
 		matches, err = m.MatchesForJSONEvent([]byte(should))
@@ -40,6 +33,13 @@ func TestBasicMatching(t *testing.T) {
 			t.Errorf("event %s, LM %d", should, len(matches))
 		}
 	}
+	for _, shouldNot := range shouldNotMatch {
+		var matches []X
+		matches, err = m.MatchesForJSONEvent([]byte(shouldNot))
+		if len(matches) != 0 {
+			t.Error("Matched: " + shouldNot)
+		}
+	}
 }
 
 func TestExerciseMatching(t *testing.T) {
@@ -49,7 +49,7 @@ func TestExerciseMatching(t *testing.T) {
             "Height": 600,
             "Title":  "View from 15th Floor",
             "Thumbnail": {
-                "Url":    "http://www.example.com/image/481989943",
+                "Url":    "https://www.example.com/image/481989943",
                 "Height": 125,
                 "Width":  100
             },
@@ -128,64 +128,9 @@ func TestSimpleAddPattern(t *testing.T) {
 		t.Errorf("s0 trans len %d", len(s0.transitions))
 	}
 
-	v0, ok := s0.transitions["a"]
+	_, ok := s0.transitions["a"]
 	if !ok {
 		t.Error("No trans from start on 'a'")
 	}
-	if len(v0.valueTransitions) != 2 {
-		t.Errorf("v1 trans %d wanted 2", len(v0.valueTransitions))
-	}
-	s1, ok := v0.valueTransitions["1"]
-	if !ok {
-		t.Error("no trans on 1 fro s1")
-	}
-	s2, ok := v0.valueTransitions["2"]
-	if !ok {
-		t.Error("no trans on 2 from s2")
-	}
-	if len(s1.transitions) != 1 {
-		t.Errorf("s1 trans len %d", len(s1.transitions))
-	}
-	if len(s2.transitions) != 1 {
-		t.Errorf("s2 trans len %d", len(s2.transitions))
-	}
-	v1, ok := s1.transitions["b"]
-	if !ok {
-		t.Error("no trans on b from s1")
-	}
-	v2, ok := s2.transitions["b"]
-	if !ok {
-		t.Error("no trans on b from s2")
-	}
-	for _, v := range []*valueMatchState{v1, v2} {
-		if len(v.valueTransitions) != 2 {
-			t.Errorf("trans len on %v = %d", v, len(v.valueTransitions))
-		}
-		s3, ok := v.valueTransitions["1"]
-		if !ok {
-			t.Error("no trans on 1 at s3")
-		}
-		if len(s3.transitions) != 0 {
-			t.Errorf("len trans s3 = %d", len(s3.transitions))
-		}
-		if len(s3.matches) != 1 {
-			t.Errorf("s3 matches %d", len(s3.matches))
-		}
-		if s3.matches[0] != x {
-			t.Error("s3 match mismatch")
-		}
-		s4, ok := v.valueTransitions[`"3"`]
-		if !ok {
-			t.Error(`no trans on "3" at s4`)
-		}
-		if len(s4.transitions) != 0 {
-			t.Errorf("len trans s4 = %d", len(s4.transitions))
-		}
-		if len(s4.matches) != 1 {
-			t.Errorf("s4 matches %d", len(s4.matches))
-		}
-		if s4.matches[0] != x {
-			t.Error("s4 match mismatch")
-		}
-	}
+	// TODO: Consider hand-checking the smallValueMacher structure
 }

lib/pattern.go

@@ -28,6 +28,24 @@ type patternField struct {
 	vals []typedVal
 }
 
+// for debugging
+func (f *patternField) String() string {
+	s := fmt.Sprintf("p=%s: ", f.path)
+	var v string
+	for _, val := range f.vals {
+		switch val.vType {
+		case existsTrueType:
+			v = " exists:true"
+		case existsFalseType:
+			v = " exists:false"
+		default:
+			v = " <" + val.val + ">"
+		}
+		s += v
+	}
+	return s
+}
+
 type patternBuild struct {
 	jd         *json.Decoder
 	path       []string

lib/pattern_test.go

@@ -36,34 +36,34 @@ func TestPatternFromJSON(t *testing.T) {
 		`{"x": [ {"exists": true} ] }`,
 		`{"x": { "y": [ {"exists": false} ] } }`,
 	}
-	w1 := []*patternField{&patternField{path: "x", vals: []typedVal{typedVal{numberType, "2"}}}}
-	w2 := []*patternField{&patternField{path: "x", vals: []typedVal{
+	w1 := []*patternField{{path: "x", vals: []typedVal{{numberType, "2"}}}}
+	w2 := []*patternField{{path: "x", vals: []typedVal{
 		{literalType, "null"},
 		{literalType, "true"},
 		{literalType, "false"},
 		{stringType, `"hopp"`},
 		{numberType, "3.072e-11"},
-	}}}
+		}}}
 	w3 := []*patternField{
-		&patternField{path: "x\na", vals: []typedVal{
+		{path: "x\na", vals: []typedVal{
 			{numberType, "27"},
 			{numberType, "28"},
 		}},
-		&patternField{path: "x\nb\nm", vals: []typedVal{
+		{path: "x\nb\nm", vals: []typedVal{
 			{stringType, `"a"`},
 			{stringType, `"b"`},
 		}},
 	}
 	w4 := []*patternField{
-		&patternField{path: "x", vals: []typedVal{
+		{path: "x", vals: []typedVal{
 			{vType: existsTrueType, val: ""},
 		},
-		}}
+				}}
 	w5 := []*patternField{
-		&patternField{path: "x\ny", vals: []typedVal{
+		{path: "x\ny", vals: []typedVal{
 			{vType: existsFalseType, val: ""},
 		},
-		}}
+				}}
 	wanted := [][]*patternField{w1, w2, w3, w4, w5}
 
 	for i, good := range goods {