Move the libs/pubsub package to internal scope (#7451)

No API changes, merely changes the import path.

internal/pubsub/query/bench_test.go

@@ -0,0 +1,58 @@
+package query_test
+
+import (
+	"testing"
+
+	"github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/internal/pubsub/query"
+)
+
+const testQuery = `tm.events.type='NewBlock' AND abci.account.name='Igor'`
+
+var testEvents = []types.Event{
+	{
+		Type: "tm.events",
+		Attributes: []types.EventAttribute{{
+			Key:   "index",
+			Value: "25",
+		}, {
+			Key:   "type",
+			Value: "NewBlock",
+		}},
+	},
+	{
+		Type: "abci.account",
+		Attributes: []types.EventAttribute{{
+			Key:   "name",
+			Value: "Anya",
+		}, {
+			Key:   "name",
+			Value: "Igor",
+		}},
+	},
+}
+
+func BenchmarkParseCustom(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_, err := query.New(testQuery)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkMatchCustom(b *testing.B) {
+	q, err := query.New(testQuery)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ok, err := q.Matches(testEvents)
+		if err != nil {
+			b.Fatal(err)
+		} else if !ok {
+			b.Error("no match")
+		}
+	}
+}

internal/pubsub/query/query.go

@@ -0,0 +1,327 @@
+// Package query implements the custom query format used to filter event
+// subscriptions in Tendermint.
+//
+// Query expressions describe properties of events and their attributes, using
+// strings like:
+//
+//    abci.invoice.number = 22 AND abci.invoice.owner = 'Ivan'
+//
+// Query expressions can handle attribute values encoding numbers, strings,
+// dates, and timestamps.  The complete query grammar is described in the
+// query/syntax package.
+//
+package query
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/internal/pubsub/query/syntax"
+)
+
+// All is a query that matches all events.
+var All *Query
+
+// A Query is the compiled form of a query.
+type Query struct {
+	ast   syntax.Query
+	conds []condition
+}
+
+// New parses and compiles the query expression into an executable query.
+func New(query string) (*Query, error) {
+	ast, err := syntax.Parse(query)
+	if err != nil {
+		return nil, err
+	}
+	return Compile(ast)
+}
+
+// MustCompile compiles the query expression into an executable query.
+// In case of error, MustCompile will panic.
+//
+// This is intended for use in program initialization; use query.New if you
+// need to check errors.
+func MustCompile(query string) *Query {
+	q, err := New(query)
+	if err != nil {
+		panic(err)
+	}
+	return q
+}
+
+// Compile compiles the given query AST so it can be used to match events.
+func Compile(ast syntax.Query) (*Query, error) {
+	conds := make([]condition, len(ast))
+	for i, q := range ast {
+		cond, err := compileCondition(q)
+		if err != nil {
+			return nil, fmt.Errorf("compile %s: %w", q, err)
+		}
+		conds[i] = cond
+	}
+	return &Query{ast: ast, conds: conds}, nil
+}
+
+// Matches satisfies part of the pubsub.Query interface.  This implementation
+// never reports an error. A nil *Query matches all events.
+func (q *Query) Matches(events []types.Event) (bool, error) {
+	if q == nil {
+		return true, nil
+	}
+	return q.matchesEvents(events), nil
+}
+
+// String matches part of the pubsub.Query interface.
+func (q *Query) String() string {
+	if q == nil {
+		return "<empty>"
+	}
+	return q.ast.String()
+}
+
+// Syntax returns the syntax tree representation of q.
+func (q *Query) Syntax() syntax.Query {
+	if q == nil {
+		return nil
+	}
+	return q.ast
+}
+
+// matchesEvents reports whether all the conditions match the given events.
+func (q *Query) matchesEvents(events []types.Event) bool {
+	for _, cond := range q.conds {
+		if !cond.matchesAny(events) {
+			return false
+		}
+	}
+	return len(events) != 0
+}
+
+// A condition is a compiled match condition.  A condition matches an event if
+// the event has the designated type, contains an attribute with the given
+// name, and the match function returns true for the attribute value.
+type condition struct {
+	tag   string // e.g., "tx.hash"
+	match func(s string) bool
+}
+
+// findAttr returns a slice of attribute values from event matching the
+// condition tag, and reports whether the event type strictly equals the
+// condition tag.
+func (c condition) findAttr(event types.Event) ([]string, bool) {
+	if !strings.HasPrefix(c.tag, event.Type) {
+		return nil, false // type does not match tag
+	} else if len(c.tag) == len(event.Type) {
+		return nil, true // type == tag
+	}
+	var vals []string
+	for _, attr := range event.Attributes {
+		fullName := event.Type + "." + attr.Key
+		if fullName == c.tag {
+			vals = append(vals, attr.Value)
+		}
+	}
+	return vals, false
+}
+
+// matchesAny reports whether c matches at least one of the given events.
+func (c condition) matchesAny(events []types.Event) bool {
+	for _, event := range events {
+		if c.matchesEvent(event) {
+			return true
+		}
+	}
+	return false
+}
+
+// matchesEvent reports whether c matches the given event.
+func (c condition) matchesEvent(event types.Event) bool {
+	vs, tagEqualsType := c.findAttr(event)
+	if len(vs) == 0 {
+		// As a special case, a condition tag that exactly matches the event type
+		// is matched against an empty string. This allows existence checks to
+		// work for type-only queries.
+		if tagEqualsType {
+			return c.match("")
+		}
+		return false
+	}
+
+	// At this point, we have candidate values.
+	for _, v := range vs {
+		if c.match(v) {
+			return true
+		}
+	}
+	return false
+}
+
+func compileCondition(cond syntax.Condition) (condition, error) {
+	out := condition{tag: cond.Tag}
+
+	// Handle existence checks separately to simplify the logic below for
+	// comparisons that take arguments.
+	if cond.Op == syntax.TExists {
+		out.match = func(string) bool { return true }
+		return out, nil
+	}
+
+	// All the other operators require an argument.
+	if cond.Arg == nil {
+		return condition{}, fmt.Errorf("missing argument for %v", cond.Op)
+	}
+
+	// Precompile the argument value matcher.
+	argType := cond.Arg.Type
+	var argValue interface{}
+
+	switch argType {
+	case syntax.TString:
+		argValue = cond.Arg.Value()
+	case syntax.TNumber:
+		argValue = cond.Arg.Number()
+	case syntax.TTime, syntax.TDate:
+		argValue = cond.Arg.Time()
+	default:
+		return condition{}, fmt.Errorf("unknown argument type %v", argType)
+	}
+
+	mcons := opTypeMap[cond.Op][argType]
+	if mcons == nil {
+		return condition{}, fmt.Errorf("invalid op/arg combination (%v, %v)", cond.Op, argType)
+	}
+	out.match = mcons(argValue)
+	return out, nil
+}
+
+// TODO(creachadair): The existing implementation allows anything number shaped
+// to be treated as a number. This preserves the parts of that behavior we had
+// tests for, but we should probably get rid of that.
+var extractNum = regexp.MustCompile(`^\d+(\.\d+)?`)
+
+func parseNumber(s string) (float64, error) {
+	return strconv.ParseFloat(extractNum.FindString(s), 64)
+}
+
+// A map of operator ⇒ argtype ⇒ match-constructor.
+// An entry does not exist if the combination is not valid.
+//
+// Disable the dupl lint for this map. The result isn't even correct.
+//nolint:dupl
+var opTypeMap = map[syntax.Token]map[syntax.Token]func(interface{}) func(string) bool{
+	syntax.TContains: {
+		syntax.TString: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				return strings.Contains(s, v.(string))
+			}
+		},
+	},
+	syntax.TEq: {
+		syntax.TString: func(v interface{}) func(string) bool {
+			return func(s string) bool { return s == v.(string) }
+		},
+		syntax.TNumber: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				w, err := parseNumber(s)
+				return err == nil && w == v.(float64)
+			}
+		},
+		syntax.TDate: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				ts, err := syntax.ParseDate(s)
+				return err == nil && ts.Equal(v.(time.Time))
+			}
+		},
+		syntax.TTime: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				ts, err := syntax.ParseTime(s)
+				return err == nil && ts.Equal(v.(time.Time))
+			}
+		},
+	},
+	syntax.TLt: {
+		syntax.TNumber: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				w, err := parseNumber(s)
+				return err == nil && w < v.(float64)
+			}
+		},
+		syntax.TDate: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				ts, err := syntax.ParseDate(s)
+				return err == nil && ts.Before(v.(time.Time))
+			}
+		},
+		syntax.TTime: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				ts, err := syntax.ParseTime(s)
+				return err == nil && ts.Before(v.(time.Time))
+			}
+		},
+	},
+	syntax.TLeq: {
+		syntax.TNumber: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				w, err := parseNumber(s)
+				return err == nil && w <= v.(float64)
+			}
+		},
+		syntax.TDate: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				ts, err := syntax.ParseDate(s)
+				return err == nil && !ts.After(v.(time.Time))
+			}
+		},
+		syntax.TTime: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				ts, err := syntax.ParseTime(s)
+				return err == nil && !ts.After(v.(time.Time))
+			}
+		},
+	},
+	syntax.TGt: {
+		syntax.TNumber: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				w, err := parseNumber(s)
+				return err == nil && w > v.(float64)
+			}
+		},
+		syntax.TDate: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				ts, err := syntax.ParseDate(s)
+				return err == nil && ts.After(v.(time.Time))
+			}
+		},
+		syntax.TTime: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				ts, err := syntax.ParseTime(s)
+				return err == nil && ts.After(v.(time.Time))
+			}
+		},
+	},
+	syntax.TGeq: {
+		syntax.TNumber: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				w, err := parseNumber(s)
+				return err == nil && w >= v.(float64)
+			}
+		},
+		syntax.TDate: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				ts, err := syntax.ParseDate(s)
+				return err == nil && !ts.Before(v.(time.Time))
+			}
+		},
+		syntax.TTime: func(v interface{}) func(string) bool {
+			return func(s string) bool {
+				ts, err := syntax.ParseTime(s)
+				return err == nil && !ts.Before(v.(time.Time))
+			}
+		},
+	},
+}

internal/pubsub/query/query_test.go

@@ -0,0 +1,275 @@
+package query_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/internal/pubsub"
+	"github.com/tendermint/tendermint/internal/pubsub/query"
+	"github.com/tendermint/tendermint/internal/pubsub/query/syntax"
+)
+
+var _ pubsub.Query = (*query.Query)(nil)
+
+// Example events from the OpenAPI documentation:
+//  https://github.com/tendermint/tendermint/blob/master/rpc/openapi/openapi.yaml
+//
+// Redactions:
+//
+//   - Add an explicit "tm" event for the built-in attributes.
+//   - Remove Index fields (not relevant to tests).
+//   - Add explicit balance values (to use in tests).
+//
+var apiEvents = []types.Event{
+	{
+		Type: "tm",
+		Attributes: []types.EventAttribute{
+			{Key: "event", Value: "Tx"},
+			{Key: "hash", Value: "XYZ"},
+			{Key: "height", Value: "5"},
+		},
+	},
+	{
+		Type: "rewards.withdraw",
+		Attributes: []types.EventAttribute{
+			{Key: "address", Value: "AddrA"},
+			{Key: "source", Value: "SrcX"},
+			{Key: "amount", Value: "100"},
+			{Key: "balance", Value: "1500"},
+		},
+	},
+	{
+		Type: "rewards.withdraw",
+		Attributes: []types.EventAttribute{
+			{Key: "address", Value: "AddrB"},
+			{Key: "source", Value: "SrcY"},
+			{Key: "amount", Value: "45"},
+			{Key: "balance", Value: "999"},
+		},
+	},
+	{
+		Type: "transfer",
+		Attributes: []types.EventAttribute{
+			{Key: "sender", Value: "AddrC"},
+			{Key: "recipient", Value: "AddrD"},
+			{Key: "amount", Value: "160"},
+		},
+	},
+}
+
+func TestCompiledMatches(t *testing.T) {
+	var (
+		txDate = "2017-01-01"
+		txTime = "2018-05-03T14:45:00Z"
+	)
+
+	testCases := []struct {
+		s       string
+		events  []types.Event
+		matches bool
+	}{
+		{`tm.events.type='NewBlock'`,
+			newTestEvents(`tm|events.type=NewBlock`),
+			true},
+		{`tx.gas > 7`,
+			newTestEvents(`tx|gas=8`),
+			true},
+		{`transfer.amount > 7`,
+			newTestEvents(`transfer|amount=8stake`),
+			true},
+		{`transfer.amount > 7`,
+			newTestEvents(`transfer|amount=8.045`),
+			true},
+		{`transfer.amount > 7.043`,
+			newTestEvents(`transfer|amount=8.045stake`),
+			true},
+		{`transfer.amount > 8.045`,
+			newTestEvents(`transfer|amount=8.045stake`),
+			false},
+		{`tx.gas > 7 AND tx.gas < 9`,
+			newTestEvents(`tx|gas=8`),
+			true},
+		{`body.weight >= 3.5`,
+			newTestEvents(`body|weight=3.5`),
+			true},
+		{`account.balance < 1000.0`,
+			newTestEvents(`account|balance=900`),
+			true},
+		{`apples.kg <= 4`,
+			newTestEvents(`apples|kg=4.0`),
+			true},
+		{`body.weight >= 4.5`,
+			newTestEvents(`body|weight=4.5`),
+			true},
+		{`oranges.kg < 4 AND watermellons.kg > 10`,
+			newTestEvents(`oranges|kg=3`, `watermellons|kg=12`),
+			true},
+		{`peaches.kg < 4`,
+			newTestEvents(`peaches|kg=5`),
+			false},
+		{`tx.date > DATE 2017-01-01`,
+			newTestEvents(`tx|date=` + time.Now().Format(syntax.DateFormat)),
+			true},
+		{`tx.date = DATE 2017-01-01`,
+			newTestEvents(`tx|date=` + txDate),
+			true},
+		{`tx.date = DATE 2018-01-01`,
+			newTestEvents(`tx|date=` + txDate),
+			false},
+		{`tx.time >= TIME 2013-05-03T14:45:00Z`,
+			newTestEvents(`tx|time=` + time.Now().Format(syntax.TimeFormat)),
+			true},
+		{`tx.time = TIME 2013-05-03T14:45:00Z`,
+			newTestEvents(`tx|time=` + txTime),
+			false},
+		{`abci.owner.name CONTAINS 'Igor'`,
+			newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
+			true},
+		{`abci.owner.name CONTAINS 'Igor'`,
+			newTestEvents(`abci|owner.name=Pavel|owner.name=Ivan`),
+			false},
+		{`abci.owner.name = 'Igor'`,
+			newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
+			true},
+		{`abci.owner.name = 'Ivan'`,
+			newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
+			true},
+		{`abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'`,
+			newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
+			true},
+		{`abci.owner.name = 'Ivan' AND abci.owner.name = 'John'`,
+			newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`),
+			false},
+		{`tm.events.type='NewBlock'`,
+			newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`),
+			true},
+		{`app.name = 'fuzzed'`,
+			newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`),
+			true},
+		{`tm.events.type='NewBlock' AND app.name = 'fuzzed'`,
+			newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`),
+			true},
+		{`tm.events.type='NewHeader' AND app.name = 'fuzzed'`,
+			newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`),
+			false},
+		{`slash EXISTS`,
+			newTestEvents(`slash|reason=missing_signature|power=6000`),
+			true},
+		{`slash EXISTS`,
+			newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`),
+			false},
+		{`slash.reason EXISTS AND slash.power > 1000`,
+			newTestEvents(`slash|reason=missing_signature|power=6000`),
+			true},
+		{`slash.reason EXISTS AND slash.power > 1000`,
+			newTestEvents(`slash|reason=missing_signature|power=500`),
+			false},
+		{`slash.reason EXISTS`,
+			newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`),
+			false},
+
+		// Test cases based on the OpenAPI examples.
+		{`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA'`,
+			apiEvents, true},
+		{`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA' AND rewards.withdraw.source = 'SrcY'`,
+			apiEvents, true},
+		{`tm.event = 'Tx' AND transfer.sender = 'AddrA'`,
+			apiEvents, false},
+		{`tm.event = 'Tx' AND transfer.sender = 'AddrC'`,
+			apiEvents, true},
+		{`tm.event = 'Tx' AND transfer.sender = 'AddrZ'`,
+			apiEvents, false},
+		{`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrZ'`,
+			apiEvents, false},
+		{`tm.event = 'Tx' AND rewards.withdraw.source = 'W'`,
+			apiEvents, false},
+	}
+
+	// NOTE: The original implementation allowed arbitrary prefix matches on
+	// attribute tags, e.g., "sl" would match "slash".
+	//
+	// That is weird and probably wrong: "foo.ba" should not match "foo.bar",
+	// or there is no way to distinguish the case where there were two values
+	// for "foo.bar" or one value each for "foo.ba" and "foo.bar".
+	//
+	// Apart from a single test case, I could not find any attested usage of
+	// this implementation detail. It isn't documented in the OpenAPI docs and
+	// is not shown in any of the example inputs.
+	//
+	// On that basis, I removed that test case. This implementation still does
+	// correctly handle variable type/attribute splits ("x", "y.z" / "x.y", "z")
+	// since that was required by the original "flattened" event representation.
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("%02d", i+1), func(t *testing.T) {
+			c, err := query.New(tc.s)
+			if err != nil {
+				t.Fatalf("NewCompiled %#q: unexpected error: %v", tc.s, err)
+			}
+
+			got, err := c.Matches(tc.events)
+			if err != nil {
+				t.Errorf("Query: %#q\nInput: %+v\nMatches: got error %v",
+					tc.s, tc.events, err)
+			}
+			if got != tc.matches {
+				t.Errorf("Query: %#q\nInput: %+v\nMatches: got %v, want %v",
+					tc.s, tc.events, got, tc.matches)
+			}
+		})
+	}
+}
+
+func TestAllMatchesAll(t *testing.T) {
+	events := newTestEvents(
+		``,
+		`Asher|Roth=`,
+		`Route|66=`,
+		`Rilly|Blue=`,
+	)
+	for i := 0; i < len(events); i++ {
+		match, err := query.All.Matches(events[:i])
+		if err != nil {
+			t.Errorf("Matches failed: %v", err)
+		} else if !match {
+			t.Errorf("Did not match on %+v ", events[:i])
+		}
+	}
+}
+
+// newTestEvent constructs an Event message from a template string.
+// The format is "type|attr1=val1|attr2=val2|...".
+func newTestEvent(s string) types.Event {
+	var event types.Event
+	parts := strings.Split(s, "|")
+	event.Type = parts[0]
+	if len(parts) == 1 {
+		return event // type only, no attributes
+	}
+	for _, kv := range parts[1:] {
+		key, val := splitKV(kv)
+		event.Attributes = append(event.Attributes, types.EventAttribute{
+			Key:   key,
+			Value: val,
+		})
+	}
+	return event
+}
+
+// newTestEvents constructs a slice of Event messages by applying newTestEvent
+// to each element of ss.
+func newTestEvents(ss ...string) []types.Event {
+	events := make([]types.Event, len(ss))
+	for i, s := range ss {
+		events[i] = newTestEvent(s)
+	}
+	return events
+}
+
+func splitKV(s string) (key, value string) {
+	kv := strings.SplitN(s, "=", 2)
+	return kv[0], kv[1]
+}

internal/pubsub/query/syntax/doc.go

@@ -0,0 +1,34 @@
+// Package syntax defines a scanner and parser for the Tendermint event filter
+// query language. A query selects events by their types and attribute values.
+//
+// Grammar
+//
+// The grammar of the query language is defined by the following EBNF:
+//
+//   query      = conditions EOF
+//   conditions = condition {"AND" condition}
+//   condition  = tag comparison
+//   comparison = equal / order / contains / "EXISTS"
+//   equal      = "=" (date / number / time / value)
+//   order      = cmp (date / number / time)
+//   contains   = "CONTAINS" value
+//   cmp        = "<" / "<=" / ">" / ">="
+//
+// The lexical terms are defined here using RE2 regular expression notation:
+//
+//   // The name of an event attribute (type.value)
+//   tag    = #'\w+(\.\w+)*'
+//
+//   // A datestamp (YYYY-MM-DD)
+//   date   = #'DATE \d{4}-\d{2}-\d{2}'
+//
+//   // A number with optional fractional parts (0, 10, 3.25)
+//   number = #'\d+(\.\d+)?'
+//
+//   // An RFC3339 timestamp (2021-11-23T22:04:19-09:00)
+//   time   = #'TIME \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([-+]\d{2}:\d{2}|Z)'
+//
+//   // A quoted literal string value ('a b c')
+//   value  = #'\'[^\']*\''
+//
+package syntax

internal/pubsub/query/syntax/parser.go

@@ -0,0 +1,213 @@
+package syntax
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Parse parses the specified query string. It is shorthand for constructing a
+// parser for s and calling its Parse method.
+func Parse(s string) (Query, error) {
+	return NewParser(strings.NewReader(s)).Parse()
+}
+
+// Query is the root of the parse tree for a query.  A query is the conjunction
+// of one or more conditions.
+type Query []Condition
+
+func (q Query) String() string {
+	ss := make([]string, len(q))
+	for i, cond := range q {
+		ss[i] = cond.String()
+	}
+	return strings.Join(ss, " AND ")
+}
+
+// A Condition is a single conditional expression, consisting of a tag, a
+// comparison operator, and an optional argument. The type of the argument
+// depends on the operator.
+type Condition struct {
+	Tag string
+	Op  Token
+	Arg *Arg
+
+	opText string
+}
+
+func (c Condition) String() string {
+	s := c.Tag + " " + c.opText
+	if c.Arg != nil {
+		return s + " " + c.Arg.String()
+	}
+	return s
+}
+
+// An Arg is the argument of a comparison operator.
+type Arg struct {
+	Type Token
+	text string
+}
+
+func (a *Arg) String() string {
+	if a == nil {
+		return ""
+	}
+	switch a.Type {
+	case TString:
+		return "'" + a.text + "'"
+	case TTime:
+		return "TIME " + a.text
+	case TDate:
+		return "DATE " + a.text
+	default:
+		return a.text
+	}
+}
+
+// Number returns the value of the argument text as a number, or a NaN if the
+// text does not encode a valid number value.
+func (a *Arg) Number() float64 {
+	if a == nil {
+		return -1
+	}
+	v, err := strconv.ParseFloat(a.text, 64)
+	if err == nil && v >= 0 {
+		return v
+	}
+	return math.NaN()
+}
+
+// Time returns the value of the argument text as a time, or the zero value if
+// the text does not encode a timestamp or datestamp.
+func (a *Arg) Time() time.Time {
+	var ts time.Time
+	if a == nil {
+		return ts
+	}
+	var err error
+	switch a.Type {
+	case TDate:
+		ts, err = ParseDate(a.text)
+	case TTime:
+		ts, err = ParseTime(a.text)
+	}
+	if err == nil {
+		return ts
+	}
+	return time.Time{}
+}
+
+// Value returns the value of the argument text as a string, or "".
+func (a *Arg) Value() string {
+	if a == nil {
+		return ""
+	}
+	return a.text
+}
+
+// Parser is a query expression parser. The grammar for query expressions is
+// defined in the syntax package documentation.
+type Parser struct {
+	scanner *Scanner
+}
+
+// NewParser constructs a new parser that reads the input from r.
+func NewParser(r io.Reader) *Parser {
+	return &Parser{scanner: NewScanner(r)}
+}
+
+// Parse parses the complete input and returns the resulting query.
+func (p *Parser) Parse() (Query, error) {
+	cond, err := p.parseCond()
+	if err != nil {
+		return nil, err
+	}
+	conds := []Condition{cond}
+	for p.scanner.Next() != io.EOF {
+		if tok := p.scanner.Token(); tok != TAnd {
+			return nil, fmt.Errorf("offset %d: got %v, want %v", p.scanner.Pos(), tok, TAnd)
+		}
+		cond, err := p.parseCond()
+		if err != nil {
+			return nil, err
+		}
+		conds = append(conds, cond)
+	}
+	return conds, nil
+}
+
+// parseCond parses a conditional expression: tag OP value.
+func (p *Parser) parseCond() (Condition, error) {
+	var cond Condition
+	if err := p.require(TTag); err != nil {
+		return cond, err
+	}
+	cond.Tag = p.scanner.Text()
+	if err := p.require(TLeq, TGeq, TLt, TGt, TEq, TContains, TExists); err != nil {
+		return cond, err
+	}
+	cond.Op = p.scanner.Token()
+	cond.opText = p.scanner.Text()
+
+	var err error
+	switch cond.Op {
+	case TLeq, TGeq, TLt, TGt:
+		err = p.require(TNumber, TTime, TDate)
+	case TEq:
+		err = p.require(TNumber, TTime, TDate, TString)
+	case TContains:
+		err = p.require(TString)
+	case TExists:
+		// no argument
+		return cond, nil
+	default:
+		return cond, fmt.Errorf("offset %d: unexpected operator %v", p.scanner.Pos(), cond.Op)
+	}
+	if err != nil {
+		return cond, err
+	}
+	cond.Arg = &Arg{Type: p.scanner.Token(), text: p.scanner.Text()}
+	return cond, nil
+}
+
+// require advances the scanner and requires that the resulting token is one of
+// the specified token types.
+func (p *Parser) require(tokens ...Token) error {
+	if err := p.scanner.Next(); err != nil {
+		return fmt.Errorf("offset %d: %w", p.scanner.Pos(), err)
+	}
+	got := p.scanner.Token()
+	for _, tok := range tokens {
+		if tok == got {
+			return nil
+		}
+	}
+	return fmt.Errorf("offset %d: got %v, wanted %s", p.scanner.Pos(), got, tokLabel(tokens))
+}
+
+// tokLabel makes a human-readable summary string for the given token types.
+func tokLabel(tokens []Token) string {
+	if len(tokens) == 1 {
+		return tokens[0].String()
+	}
+	last := len(tokens) - 1
+	ss := make([]string, len(tokens)-1)
+	for i, tok := range tokens[:last] {
+		ss[i] = tok.String()
+	}
+	return strings.Join(ss, ", ") + " or " + tokens[last].String()
+}
+
+// ParseDate parses s as a date string in the format used by DATE values.
+func ParseDate(s string) (time.Time, error) {
+	return time.Parse("2006-01-02", s)
+}
+
+// ParseTime parses s as a timestamp in the format used by TIME values.
+func ParseTime(s string) (time.Time, error) {
+	return time.Parse(time.RFC3339, s)
+}

internal/pubsub/query/syntax/scanner.go

@@ -0,0 +1,312 @@
+package syntax
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+	"unicode"
+)
+
+// Token is the type of a lexical token in the query grammar.
+type Token byte
+
+const (
+	TInvalid  = iota // invalid or unknown token
+	TTag             // field tag: x.y
+	TString          // string value: 'foo bar'
+	TNumber          // number: 0, 15.5, 100
+	TTime            // timestamp: TIME yyyy-mm-ddThh:mm:ss([-+]hh:mm|Z)
+	TDate            // datestamp: DATE yyyy-mm-dd
+	TAnd             // operator: AND
+	TContains        // operator: CONTAINS
+	TExists          // operator: EXISTS
+	TEq              // operator: =
+	TLt              // operator: <
+	TLeq             // operator: <=
+	TGt              // operator: >
+	TGeq             // operator: >=
+
+	// Do not reorder these values without updating the scanner code.
+)
+
+var tString = [...]string{
+	TInvalid:  "invalid token",
+	TTag:      "tag",
+	TString:   "string",
+	TNumber:   "number",
+	TTime:     "timestamp",
+	TDate:     "datestamp",
+	TAnd:      "AND operator",
+	TContains: "CONTAINS operator",
+	TExists:   "EXISTS operator",
+	TEq:       "= operator",
+	TLt:       "< operator",
+	TLeq:      "<= operator",
+	TGt:       "> operator",
+	TGeq:      ">= operator",
+}
+
+func (t Token) String() string {
+	v := int(t)
+	if v > len(tString) {
+		return "unknown token type"
+	}
+	return tString[v]
+}
+
+const (
+	// TimeFormat is the format string used for timestamp values.
+	TimeFormat = time.RFC3339
+
+	// DateFormat is the format string used for datestamp values.
+	DateFormat = "2006-01-02"
+)
+
+// Scanner reads lexical tokens of the query language from an input stream.
+// Each call to Next advances the scanner to the next token, or reports an
+// error.
+type Scanner struct {
+	r   *bufio.Reader
+	buf bytes.Buffer
+	tok Token
+	err error
+
+	pos, last, end int
+}
+
+// NewScanner constructs a new scanner that reads from r.
+func NewScanner(r io.Reader) *Scanner { return &Scanner{r: bufio.NewReader(r)} }
+
+// Next advances s to the next token in the input, or reports an error.  At the
+// end of input, Next returns io.EOF.
+func (s *Scanner) Next() error {
+	s.buf.Reset()
+	s.pos = s.end
+	s.tok = TInvalid
+	s.err = nil
+
+	for {
+		ch, err := s.rune()
+		if err != nil {
+			return s.fail(err)
+		}
+		if unicode.IsSpace(ch) {
+			s.pos = s.end
+			continue // skip whitespace
+		}
+		if '0' <= ch && ch <= '9' {
+			return s.scanNumber(ch)
+		} else if isTagRune(ch) {
+			return s.scanTagLike(ch)
+		}
+		switch ch {
+		case '\'':
+			return s.scanString(ch)
+		case '<', '>', '=':
+			return s.scanCompare(ch)
+		default:
+			return s.invalid(ch)
+		}
+	}
+}
+
+// Token returns the type of the current input token.
+func (s *Scanner) Token() Token { return s.tok }
+
+// Text returns the text of the current input token.
+func (s *Scanner) Text() string { return s.buf.String() }
+
+// Pos returns the start offset of the current token in the input.
+func (s *Scanner) Pos() int { return s.pos }
+
+// Err returns the last error reported by Next, if any.
+func (s *Scanner) Err() error { return s.err }
+
+// scanNumber scans for numbers with optional fractional parts.
+// Examples: 0, 1, 3.14
+func (s *Scanner) scanNumber(first rune) error {
+	s.buf.WriteRune(first)
+	if err := s.scanWhile(isDigit); err != nil {
+		return err
+	}
+
+	ch, err := s.rune()
+	if err != nil && err != io.EOF {
+		return err
+	}
+	if ch == '.' {
+		s.buf.WriteRune(ch)
+		if err := s.scanWhile(isDigit); err != nil {
+			return err
+		}
+	} else {
+		s.unrune()
+	}
+	s.tok = TNumber
+	return nil
+}
+
+func (s *Scanner) scanString(first rune) error {
+	// discard opening quote
+	for {
+		ch, err := s.rune()
+		if err != nil {
+			return s.fail(err)
+		} else if ch == first {
+			// discard closing quote
+			s.tok = TString
+			return nil
+		}
+		s.buf.WriteRune(ch)
+	}
+}
+
+func (s *Scanner) scanCompare(first rune) error {
+	s.buf.WriteRune(first)
+	switch first {
+	case '=':
+		s.tok = TEq
+		return nil
+	case '<':
+		s.tok = TLt
+	case '>':
+		s.tok = TGt
+	default:
+		return s.invalid(first)
+	}
+
+	ch, err := s.rune()
+	if err == io.EOF {
+		return nil // the assigned token is correct
+	} else if err != nil {
+		return s.fail(err)
+	}
+	if ch == '=' {
+		s.buf.WriteRune(ch)
+		s.tok++ // depends on token order
+		return nil
+	}
+	s.unrune()
+	return nil
+}
+
+func (s *Scanner) scanTagLike(first rune) error {
+	s.buf.WriteRune(first)
+	var hasSpace bool
+	for {
+		ch, err := s.rune()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return s.fail(err)
+		}
+		if !isTagRune(ch) {
+			hasSpace = ch == ' ' // to check for TIME, DATE
+			break
+		}
+		s.buf.WriteRune(ch)
+	}
+
+	text := s.buf.String()
+	switch text {
+	case "TIME":
+		if hasSpace {
+			return s.scanTimestamp()
+		}
+		s.tok = TTag
+	case "DATE":
+		if hasSpace {
+			return s.scanDatestamp()
+		}
+		s.tok = TTag
+	case "AND":
+		s.tok = TAnd
+	case "EXISTS":
+		s.tok = TExists
+	case "CONTAINS":
+		s.tok = TContains
+	default:
+		s.tok = TTag
+	}
+	s.unrune()
+	return nil
+}
+
+func (s *Scanner) scanTimestamp() error {
+	s.buf.Reset() // discard "TIME" label
+	if err := s.scanWhile(isTimeRune); err != nil {
+		return err
+	}
+	if ts, err := time.Parse(TimeFormat, s.buf.String()); err != nil {
+		return s.fail(fmt.Errorf("invalid TIME value: %w", err))
+	} else if y := ts.Year(); y < 1900 || y > 2999 {
+		return s.fail(fmt.Errorf("timestamp year %d out of range", ts.Year()))
+	}
+	s.tok = TTime
+	return nil
+}
+
+func (s *Scanner) scanDatestamp() error {
+	s.buf.Reset() // discard "DATE" label
+	if err := s.scanWhile(isDateRune); err != nil {
+		return err
+	}
+	if ts, err := time.Parse(DateFormat, s.buf.String()); err != nil {
+		return s.fail(fmt.Errorf("invalid DATE value: %w", err))
+	} else if y := ts.Year(); y < 1900 || y > 2999 {
+		return s.fail(fmt.Errorf("datestamp year %d out of range", ts.Year()))
+	}
+	s.tok = TDate
+	return nil
+}
+
+func (s *Scanner) scanWhile(ok func(rune) bool) error {
+	for {
+		ch, err := s.rune()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return s.fail(err)
+		} else if !ok(ch) {
+			s.unrune()
+			return nil
+		}
+		s.buf.WriteRune(ch)
+	}
+}
+
+func (s *Scanner) rune() (rune, error) {
+	ch, nb, err := s.r.ReadRune()
+	s.last = nb
+	s.end += nb
+	return ch, err
+}
+
+func (s *Scanner) unrune() {
+	_ = s.r.UnreadRune()
+	s.end -= s.last
+}
+
+func (s *Scanner) fail(err error) error {
+	s.err = err
+	return err
+}
+
+func (s *Scanner) invalid(ch rune) error {
+	return s.fail(fmt.Errorf("invalid input %c at offset %d", ch, s.end))
+}
+
+func isDigit(r rune) bool { return '0' <= r && r <= '9' }
+
+func isTagRune(r rune) bool {
+	return r == '.' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
+}
+
+func isTimeRune(r rune) bool {
+	return strings.ContainsRune("-T:+Z", r) || isDigit(r)
+}
+
+func isDateRune(r rune) bool { return isDigit(r) || r == '-' }

internal/pubsub/query/syntax/syntax_test.go

@@ -0,0 +1,190 @@
+package syntax_test
+
+import (
+	"io"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/tendermint/tendermint/internal/pubsub/query/syntax"
+)
+
+func TestScanner(t *testing.T) {
+	tests := []struct {
+		input string
+		want  []syntax.Token
+	}{
+		// Empty inputs
+		{"", nil},
+		{"  ", nil},
+		{"\t\n ", nil},
+
+		// Numbers
+		{`0 123`, []syntax.Token{syntax.TNumber, syntax.TNumber}},
+		{`0.32 3.14`, []syntax.Token{syntax.TNumber, syntax.TNumber}},
+
+		// Tags
+		{`foo foo.bar`, []syntax.Token{syntax.TTag, syntax.TTag}},
+
+		// Strings (values)
+		{` '' x 'x' 'x y'`, []syntax.Token{syntax.TString, syntax.TTag, syntax.TString, syntax.TString}},
+		{` 'you are not your job' `, []syntax.Token{syntax.TString}},
+
+		// Comparison operators
+		{`< <= = > >=`, []syntax.Token{
+			syntax.TLt, syntax.TLeq, syntax.TEq, syntax.TGt, syntax.TGeq,
+		}},
+
+		// Mixed values of various kinds.
+		{`x AND y`, []syntax.Token{syntax.TTag, syntax.TAnd, syntax.TTag}},
+		{`x.y CONTAINS 'z'`, []syntax.Token{syntax.TTag, syntax.TContains, syntax.TString}},
+		{`foo EXISTS`, []syntax.Token{syntax.TTag, syntax.TExists}},
+		{`and AND`, []syntax.Token{syntax.TTag, syntax.TAnd}},
+
+		// Timestamp
+		{`TIME 2021-11-23T15:16:17Z`, []syntax.Token{syntax.TTime}},
+
+		// Datestamp
+		{`DATE 2021-11-23`, []syntax.Token{syntax.TDate}},
+	}
+
+	for _, test := range tests {
+		s := syntax.NewScanner(strings.NewReader(test.input))
+		var got []syntax.Token
+		for s.Next() == nil {
+			got = append(got, s.Token())
+		}
+		if err := s.Err(); err != io.EOF {
+			t.Errorf("Next: unexpected error: %v", err)
+		}
+
+		if !reflect.DeepEqual(got, test.want) {
+			t.Logf("Scanner input: %q", test.input)
+			t.Errorf("Wrong tokens:\ngot:  %+v\nwant: %+v", got, test.want)
+		}
+	}
+}
+
+func TestScannerErrors(t *testing.T) {
+	tests := []struct {
+		input string
+	}{
+		{`'incomplete string`},
+		{`-23`},
+		{`&`},
+		{`DATE xyz-pdq`},
+		{`DATE xyzp-dq-zv`},
+		{`DATE 0000-00-00`},
+		{`DATE 0000-00-000`},
+		{`DATE 2021-01-99`},
+		{`TIME 2021-01-01T34:56:78Z`},
+		{`TIME 2021-01-99T14:56:08Z`},
+		{`TIME 2021-01-99T34:56:08`},
+		{`TIME 2021-01-99T34:56:11+3`},
+	}
+	for _, test := range tests {
+		s := syntax.NewScanner(strings.NewReader(test.input))
+		if err := s.Next(); err == nil {
+			t.Errorf("Next: got %v (%#q), want error", s.Token(), s.Text())
+		}
+	}
+}
+
+// These parser tests were copied from the original implementation of the query
+// parser, and are preserved here as a compatibility check.
+func TestParseValid(t *testing.T) {
+	tests := []struct {
+		input string
+		valid bool
+	}{
+		{"tm.events.type='NewBlock'", true},
+		{"tm.events.type = 'NewBlock'", true},
+		{"tm.events.name = ''", true},
+		{"tm.events.type='TIME'", true},
+		{"tm.events.type='DATE'", true},
+		{"tm.events.type='='", true},
+		{"tm.events.type='TIME", false},
+		{"tm.events.type=TIME'", false},
+		{"tm.events.type==", false},
+		{"tm.events.type=NewBlock", false},
+		{">==", false},
+		{"tm.events.type 'NewBlock' =", false},
+		{"tm.events.type>'NewBlock'", false},
+		{"", false},
+		{"=", false},
+		{"='NewBlock'", false},
+		{"tm.events.type=", false},
+
+		{"tm.events.typeNewBlock", false},
+		{"tm.events.type'NewBlock'", false},
+		{"'NewBlock'", false},
+		{"NewBlock", false},
+		{"", false},
+
+		{"tm.events.type='NewBlock' AND abci.account.name='Igor'", true},
+		{"tm.events.type='NewBlock' AND", false},
+		{"tm.events.type='NewBlock' AN", false},
+		{"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false},
+		{"AND tm.events.type='NewBlock' ", false},
+
+		{"abci.account.name CONTAINS 'Igor'", true},
+
+		{"tx.date > DATE 2013-05-03", true},
+		{"tx.date < DATE 2013-05-03", true},
+		{"tx.date <= DATE 2013-05-03", true},
+		{"tx.date >= DATE 2013-05-03", true},
+		{"tx.date >= DAT 2013-05-03", false},
+		{"tx.date <= DATE2013-05-03", false},
+		{"tx.date <= DATE -05-03", false},
+		{"tx.date >= DATE 20130503", false},
+		{"tx.date >= DATE 2013+01-03", false},
+		// incorrect year, month, day
+		{"tx.date >= DATE 0013-01-03", false},
+		{"tx.date >= DATE 2013-31-03", false},
+		{"tx.date >= DATE 2013-01-83", false},
+
+		{"tx.date > TIME 2013-05-03T14:45:00+07:00", true},
+		{"tx.date < TIME 2013-05-03T14:45:00-02:00", true},
+		{"tx.date <= TIME 2013-05-03T14:45:00Z", true},
+		{"tx.date >= TIME 2013-05-03T14:45:00Z", true},
+		{"tx.date >= TIME2013-05-03T14:45:00Z", false},
+		{"tx.date = IME 2013-05-03T14:45:00Z", false},
+		{"tx.date = TIME 2013-05-:45:00Z", false},
+		{"tx.date >= TIME 2013-05-03T14:45:00", false},
+		{"tx.date >= TIME 0013-00-00T14:45:00Z", false},
+		{"tx.date >= TIME 2013+05=03T14:45:00Z", false},
+
+		{"account.balance=100", true},
+		{"account.balance >= 200", true},
+		{"account.balance >= -300", false},
+		{"account.balance >>= 400", false},
+		{"account.balance=33.22.1", false},
+
+		{"slashing.amount EXISTS", true},
+		{"slashing.amount EXISTS AND account.balance=100", true},
+		{"account.balance=100 AND slashing.amount EXISTS", true},
+		{"slashing EXISTS", true},
+
+		{"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true},
+		{"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false},
+	}
+
+	for _, test := range tests {
+		q, err := syntax.Parse(test.input)
+		if test.valid != (err == nil) {
+			t.Errorf("Parse %#q: valid %v got err=%v", test.input, test.valid, err)
+		}
+
+		// For valid queries, check that the query round-trips.
+		if test.valid {
+			qstr := q.String()
+			r, err := syntax.Parse(qstr)
+			if err != nil {
+				t.Errorf("Reparse %#q failed: %v", qstr, err)
+			}
+			if rstr := r.String(); rstr != qstr {
+				t.Errorf("Reparse diff\nold: %#q\nnew: %#q", qstr, rstr)
+			}
+		}
+	}
+}