Move the libs/pubsub package to internal scope (#7451)

No API changes, merely changes the import path.
This commit is contained in:
M. J. Fromberger
2021-12-15 07:09:32 -08:00
committed by GitHub
parent f3278e8b68
commit 82738eb016
50 changed files with 55 additions and 54 deletions
+34
View File
@@ -0,0 +1,34 @@
// Package syntax defines a scanner and parser for the Tendermint event filter
// query language. A query selects events by their types and attribute values.
//
// Grammar
//
// The grammar of the query language is defined by the following EBNF:
//
// query = conditions EOF
// conditions = condition {"AND" condition}
// condition = tag comparison
// comparison = equal / order / contains / "EXISTS"
// equal = "=" (date / number / time / value)
// order = cmp (date / number / time)
// contains = "CONTAINS" value
// cmp = "<" / "<=" / ">" / ">="
//
// The lexical terms are defined here using RE2 regular expression notation:
//
// // The name of an event attribute (type.value)
// tag = #'\w+(\.\w+)*'
//
// // A datestamp (YYYY-MM-DD)
// date = #'DATE \d{4}-\d{2}-\d{2}'
//
// // A number with optional fractional parts (0, 10, 3.25)
// number = #'\d+(\.\d+)?'
//
// // An RFC3339 timestamp (2021-11-23T22:04:19-09:00)
// time = #'TIME \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([-+]\d{2}:\d{2}|Z)'
//
// // A quoted literal string value ('a b c')
// value = #'\'[^\']*\''
//
package syntax
+213
View File
@@ -0,0 +1,213 @@
package syntax
import (
"fmt"
"io"
"math"
"strconv"
"strings"
"time"
)
// Parse parses the specified query string. It is shorthand for constructing a
// parser for s and calling its Parse method.
func Parse(s string) (Query, error) {
return NewParser(strings.NewReader(s)).Parse()
}
// Query is the root of the parse tree for a query. A query is the conjunction
// of one or more conditions.
type Query []Condition
func (q Query) String() string {
ss := make([]string, len(q))
for i, cond := range q {
ss[i] = cond.String()
}
return strings.Join(ss, " AND ")
}
// A Condition is a single conditional expression, consisting of a tag, a
// comparison operator, and an optional argument. The type of the argument
// depends on the operator.
type Condition struct {
Tag string
Op Token
Arg *Arg
opText string
}
func (c Condition) String() string {
s := c.Tag + " " + c.opText
if c.Arg != nil {
return s + " " + c.Arg.String()
}
return s
}
// An Arg is the argument of a comparison operator.
type Arg struct {
Type Token
text string
}
func (a *Arg) String() string {
if a == nil {
return ""
}
switch a.Type {
case TString:
return "'" + a.text + "'"
case TTime:
return "TIME " + a.text
case TDate:
return "DATE " + a.text
default:
return a.text
}
}
// Number returns the value of the argument text as a number, or a NaN if the
// text does not encode a valid number value.
func (a *Arg) Number() float64 {
if a == nil {
return -1
}
v, err := strconv.ParseFloat(a.text, 64)
if err == nil && v >= 0 {
return v
}
return math.NaN()
}
// Time returns the value of the argument text as a time, or the zero value if
// the text does not encode a timestamp or datestamp.
func (a *Arg) Time() time.Time {
var ts time.Time
if a == nil {
return ts
}
var err error
switch a.Type {
case TDate:
ts, err = ParseDate(a.text)
case TTime:
ts, err = ParseTime(a.text)
}
if err == nil {
return ts
}
return time.Time{}
}
// Value returns the value of the argument text as a string, or "".
func (a *Arg) Value() string {
if a == nil {
return ""
}
return a.text
}
// Parser is a query expression parser. The grammar for query expressions is
// defined in the syntax package documentation.
type Parser struct {
scanner *Scanner
}
// NewParser constructs a new parser that reads the input from r.
func NewParser(r io.Reader) *Parser {
return &Parser{scanner: NewScanner(r)}
}
// Parse parses the complete input and returns the resulting query.
func (p *Parser) Parse() (Query, error) {
cond, err := p.parseCond()
if err != nil {
return nil, err
}
conds := []Condition{cond}
for p.scanner.Next() != io.EOF {
if tok := p.scanner.Token(); tok != TAnd {
return nil, fmt.Errorf("offset %d: got %v, want %v", p.scanner.Pos(), tok, TAnd)
}
cond, err := p.parseCond()
if err != nil {
return nil, err
}
conds = append(conds, cond)
}
return conds, nil
}
// parseCond parses a conditional expression: tag OP value.
func (p *Parser) parseCond() (Condition, error) {
var cond Condition
if err := p.require(TTag); err != nil {
return cond, err
}
cond.Tag = p.scanner.Text()
if err := p.require(TLeq, TGeq, TLt, TGt, TEq, TContains, TExists); err != nil {
return cond, err
}
cond.Op = p.scanner.Token()
cond.opText = p.scanner.Text()
var err error
switch cond.Op {
case TLeq, TGeq, TLt, TGt:
err = p.require(TNumber, TTime, TDate)
case TEq:
err = p.require(TNumber, TTime, TDate, TString)
case TContains:
err = p.require(TString)
case TExists:
// no argument
return cond, nil
default:
return cond, fmt.Errorf("offset %d: unexpected operator %v", p.scanner.Pos(), cond.Op)
}
if err != nil {
return cond, err
}
cond.Arg = &Arg{Type: p.scanner.Token(), text: p.scanner.Text()}
return cond, nil
}
// require advances the scanner and requires that the resulting token is one of
// the specified token types.
func (p *Parser) require(tokens ...Token) error {
if err := p.scanner.Next(); err != nil {
return fmt.Errorf("offset %d: %w", p.scanner.Pos(), err)
}
got := p.scanner.Token()
for _, tok := range tokens {
if tok == got {
return nil
}
}
return fmt.Errorf("offset %d: got %v, wanted %s", p.scanner.Pos(), got, tokLabel(tokens))
}
// tokLabel makes a human-readable summary string for the given token types.
func tokLabel(tokens []Token) string {
if len(tokens) == 1 {
return tokens[0].String()
}
last := len(tokens) - 1
ss := make([]string, len(tokens)-1)
for i, tok := range tokens[:last] {
ss[i] = tok.String()
}
return strings.Join(ss, ", ") + " or " + tokens[last].String()
}
// ParseDate parses s as a date string in the format used by DATE values.
func ParseDate(s string) (time.Time, error) {
return time.Parse("2006-01-02", s)
}
// ParseTime parses s as a timestamp in the format used by TIME values.
func ParseTime(s string) (time.Time, error) {
return time.Parse(time.RFC3339, s)
}
+312
View File
@@ -0,0 +1,312 @@
package syntax
import (
"bufio"
"bytes"
"fmt"
"io"
"strings"
"time"
"unicode"
)
// Token is the type of a lexical token in the query grammar.
type Token byte
const (
TInvalid = iota // invalid or unknown token
TTag // field tag: x.y
TString // string value: 'foo bar'
TNumber // number: 0, 15.5, 100
TTime // timestamp: TIME yyyy-mm-ddThh:mm:ss([-+]hh:mm|Z)
TDate // datestamp: DATE yyyy-mm-dd
TAnd // operator: AND
TContains // operator: CONTAINS
TExists // operator: EXISTS
TEq // operator: =
TLt // operator: <
TLeq // operator: <=
TGt // operator: >
TGeq // operator: >=
// Do not reorder these values without updating the scanner code.
)
var tString = [...]string{
TInvalid: "invalid token",
TTag: "tag",
TString: "string",
TNumber: "number",
TTime: "timestamp",
TDate: "datestamp",
TAnd: "AND operator",
TContains: "CONTAINS operator",
TExists: "EXISTS operator",
TEq: "= operator",
TLt: "< operator",
TLeq: "<= operator",
TGt: "> operator",
TGeq: ">= operator",
}
func (t Token) String() string {
v := int(t)
if v > len(tString) {
return "unknown token type"
}
return tString[v]
}
const (
// TimeFormat is the format string used for timestamp values.
TimeFormat = time.RFC3339
// DateFormat is the format string used for datestamp values.
DateFormat = "2006-01-02"
)
// Scanner reads lexical tokens of the query language from an input stream.
// Each call to Next advances the scanner to the next token, or reports an
// error.
type Scanner struct {
r *bufio.Reader
buf bytes.Buffer
tok Token
err error
pos, last, end int
}
// NewScanner constructs a new scanner that reads from r.
func NewScanner(r io.Reader) *Scanner { return &Scanner{r: bufio.NewReader(r)} }
// Next advances s to the next token in the input, or reports an error. At the
// end of input, Next returns io.EOF.
func (s *Scanner) Next() error {
s.buf.Reset()
s.pos = s.end
s.tok = TInvalid
s.err = nil
for {
ch, err := s.rune()
if err != nil {
return s.fail(err)
}
if unicode.IsSpace(ch) {
s.pos = s.end
continue // skip whitespace
}
if '0' <= ch && ch <= '9' {
return s.scanNumber(ch)
} else if isTagRune(ch) {
return s.scanTagLike(ch)
}
switch ch {
case '\'':
return s.scanString(ch)
case '<', '>', '=':
return s.scanCompare(ch)
default:
return s.invalid(ch)
}
}
}
// Token returns the type of the current input token.
func (s *Scanner) Token() Token { return s.tok }
// Text returns the text of the current input token.
func (s *Scanner) Text() string { return s.buf.String() }
// Pos returns the start offset of the current token in the input.
func (s *Scanner) Pos() int { return s.pos }
// Err returns the last error reported by Next, if any.
func (s *Scanner) Err() error { return s.err }
// scanNumber scans for numbers with optional fractional parts.
// Examples: 0, 1, 3.14
func (s *Scanner) scanNumber(first rune) error {
s.buf.WriteRune(first)
if err := s.scanWhile(isDigit); err != nil {
return err
}
ch, err := s.rune()
if err != nil && err != io.EOF {
return err
}
if ch == '.' {
s.buf.WriteRune(ch)
if err := s.scanWhile(isDigit); err != nil {
return err
}
} else {
s.unrune()
}
s.tok = TNumber
return nil
}
func (s *Scanner) scanString(first rune) error {
// discard opening quote
for {
ch, err := s.rune()
if err != nil {
return s.fail(err)
} else if ch == first {
// discard closing quote
s.tok = TString
return nil
}
s.buf.WriteRune(ch)
}
}
func (s *Scanner) scanCompare(first rune) error {
s.buf.WriteRune(first)
switch first {
case '=':
s.tok = TEq
return nil
case '<':
s.tok = TLt
case '>':
s.tok = TGt
default:
return s.invalid(first)
}
ch, err := s.rune()
if err == io.EOF {
return nil // the assigned token is correct
} else if err != nil {
return s.fail(err)
}
if ch == '=' {
s.buf.WriteRune(ch)
s.tok++ // depends on token order
return nil
}
s.unrune()
return nil
}
func (s *Scanner) scanTagLike(first rune) error {
s.buf.WriteRune(first)
var hasSpace bool
for {
ch, err := s.rune()
if err == io.EOF {
break
} else if err != nil {
return s.fail(err)
}
if !isTagRune(ch) {
hasSpace = ch == ' ' // to check for TIME, DATE
break
}
s.buf.WriteRune(ch)
}
text := s.buf.String()
switch text {
case "TIME":
if hasSpace {
return s.scanTimestamp()
}
s.tok = TTag
case "DATE":
if hasSpace {
return s.scanDatestamp()
}
s.tok = TTag
case "AND":
s.tok = TAnd
case "EXISTS":
s.tok = TExists
case "CONTAINS":
s.tok = TContains
default:
s.tok = TTag
}
s.unrune()
return nil
}
func (s *Scanner) scanTimestamp() error {
s.buf.Reset() // discard "TIME" label
if err := s.scanWhile(isTimeRune); err != nil {
return err
}
if ts, err := time.Parse(TimeFormat, s.buf.String()); err != nil {
return s.fail(fmt.Errorf("invalid TIME value: %w", err))
} else if y := ts.Year(); y < 1900 || y > 2999 {
return s.fail(fmt.Errorf("timestamp year %d out of range", ts.Year()))
}
s.tok = TTime
return nil
}
func (s *Scanner) scanDatestamp() error {
s.buf.Reset() // discard "DATE" label
if err := s.scanWhile(isDateRune); err != nil {
return err
}
if ts, err := time.Parse(DateFormat, s.buf.String()); err != nil {
return s.fail(fmt.Errorf("invalid DATE value: %w", err))
} else if y := ts.Year(); y < 1900 || y > 2999 {
return s.fail(fmt.Errorf("datestamp year %d out of range", ts.Year()))
}
s.tok = TDate
return nil
}
func (s *Scanner) scanWhile(ok func(rune) bool) error {
for {
ch, err := s.rune()
if err == io.EOF {
return nil
} else if err != nil {
return s.fail(err)
} else if !ok(ch) {
s.unrune()
return nil
}
s.buf.WriteRune(ch)
}
}
func (s *Scanner) rune() (rune, error) {
ch, nb, err := s.r.ReadRune()
s.last = nb
s.end += nb
return ch, err
}
func (s *Scanner) unrune() {
_ = s.r.UnreadRune()
s.end -= s.last
}
func (s *Scanner) fail(err error) error {
s.err = err
return err
}
func (s *Scanner) invalid(ch rune) error {
return s.fail(fmt.Errorf("invalid input %c at offset %d", ch, s.end))
}
func isDigit(r rune) bool { return '0' <= r && r <= '9' }
func isTagRune(r rune) bool {
return r == '.' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}
func isTimeRune(r rune) bool {
return strings.ContainsRune("-T:+Z", r) || isDigit(r)
}
func isDateRune(r rune) bool { return isDigit(r) || r == '-' }
+190
View File
@@ -0,0 +1,190 @@
package syntax_test
import (
"io"
"reflect"
"strings"
"testing"
"github.com/tendermint/tendermint/internal/pubsub/query/syntax"
)
func TestScanner(t *testing.T) {
tests := []struct {
input string
want []syntax.Token
}{
// Empty inputs
{"", nil},
{" ", nil},
{"\t\n ", nil},
// Numbers
{`0 123`, []syntax.Token{syntax.TNumber, syntax.TNumber}},
{`0.32 3.14`, []syntax.Token{syntax.TNumber, syntax.TNumber}},
// Tags
{`foo foo.bar`, []syntax.Token{syntax.TTag, syntax.TTag}},
// Strings (values)
{` '' x 'x' 'x y'`, []syntax.Token{syntax.TString, syntax.TTag, syntax.TString, syntax.TString}},
{` 'you are not your job' `, []syntax.Token{syntax.TString}},
// Comparison operators
{`< <= = > >=`, []syntax.Token{
syntax.TLt, syntax.TLeq, syntax.TEq, syntax.TGt, syntax.TGeq,
}},
// Mixed values of various kinds.
{`x AND y`, []syntax.Token{syntax.TTag, syntax.TAnd, syntax.TTag}},
{`x.y CONTAINS 'z'`, []syntax.Token{syntax.TTag, syntax.TContains, syntax.TString}},
{`foo EXISTS`, []syntax.Token{syntax.TTag, syntax.TExists}},
{`and AND`, []syntax.Token{syntax.TTag, syntax.TAnd}},
// Timestamp
{`TIME 2021-11-23T15:16:17Z`, []syntax.Token{syntax.TTime}},
// Datestamp
{`DATE 2021-11-23`, []syntax.Token{syntax.TDate}},
}
for _, test := range tests {
s := syntax.NewScanner(strings.NewReader(test.input))
var got []syntax.Token
for s.Next() == nil {
got = append(got, s.Token())
}
if err := s.Err(); err != io.EOF {
t.Errorf("Next: unexpected error: %v", err)
}
if !reflect.DeepEqual(got, test.want) {
t.Logf("Scanner input: %q", test.input)
t.Errorf("Wrong tokens:\ngot: %+v\nwant: %+v", got, test.want)
}
}
}
func TestScannerErrors(t *testing.T) {
tests := []struct {
input string
}{
{`'incomplete string`},
{`-23`},
{`&`},
{`DATE xyz-pdq`},
{`DATE xyzp-dq-zv`},
{`DATE 0000-00-00`},
{`DATE 0000-00-000`},
{`DATE 2021-01-99`},
{`TIME 2021-01-01T34:56:78Z`},
{`TIME 2021-01-99T14:56:08Z`},
{`TIME 2021-01-99T34:56:08`},
{`TIME 2021-01-99T34:56:11+3`},
}
for _, test := range tests {
s := syntax.NewScanner(strings.NewReader(test.input))
if err := s.Next(); err == nil {
t.Errorf("Next: got %v (%#q), want error", s.Token(), s.Text())
}
}
}
// These parser tests were copied from the original implementation of the query
// parser, and are preserved here as a compatibility check.
func TestParseValid(t *testing.T) {
tests := []struct {
input string
valid bool
}{
{"tm.events.type='NewBlock'", true},
{"tm.events.type = 'NewBlock'", true},
{"tm.events.name = ''", true},
{"tm.events.type='TIME'", true},
{"tm.events.type='DATE'", true},
{"tm.events.type='='", true},
{"tm.events.type='TIME", false},
{"tm.events.type=TIME'", false},
{"tm.events.type==", false},
{"tm.events.type=NewBlock", false},
{">==", false},
{"tm.events.type 'NewBlock' =", false},
{"tm.events.type>'NewBlock'", false},
{"", false},
{"=", false},
{"='NewBlock'", false},
{"tm.events.type=", false},
{"tm.events.typeNewBlock", false},
{"tm.events.type'NewBlock'", false},
{"'NewBlock'", false},
{"NewBlock", false},
{"", false},
{"tm.events.type='NewBlock' AND abci.account.name='Igor'", true},
{"tm.events.type='NewBlock' AND", false},
{"tm.events.type='NewBlock' AN", false},
{"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false},
{"AND tm.events.type='NewBlock' ", false},
{"abci.account.name CONTAINS 'Igor'", true},
{"tx.date > DATE 2013-05-03", true},
{"tx.date < DATE 2013-05-03", true},
{"tx.date <= DATE 2013-05-03", true},
{"tx.date >= DATE 2013-05-03", true},
{"tx.date >= DAT 2013-05-03", false},
{"tx.date <= DATE2013-05-03", false},
{"tx.date <= DATE -05-03", false},
{"tx.date >= DATE 20130503", false},
{"tx.date >= DATE 2013+01-03", false},
// incorrect year, month, day
{"tx.date >= DATE 0013-01-03", false},
{"tx.date >= DATE 2013-31-03", false},
{"tx.date >= DATE 2013-01-83", false},
{"tx.date > TIME 2013-05-03T14:45:00+07:00", true},
{"tx.date < TIME 2013-05-03T14:45:00-02:00", true},
{"tx.date <= TIME 2013-05-03T14:45:00Z", true},
{"tx.date >= TIME 2013-05-03T14:45:00Z", true},
{"tx.date >= TIME2013-05-03T14:45:00Z", false},
{"tx.date = IME 2013-05-03T14:45:00Z", false},
{"tx.date = TIME 2013-05-:45:00Z", false},
{"tx.date >= TIME 2013-05-03T14:45:00", false},
{"tx.date >= TIME 0013-00-00T14:45:00Z", false},
{"tx.date >= TIME 2013+05=03T14:45:00Z", false},
{"account.balance=100", true},
{"account.balance >= 200", true},
{"account.balance >= -300", false},
{"account.balance >>= 400", false},
{"account.balance=33.22.1", false},
{"slashing.amount EXISTS", true},
{"slashing.amount EXISTS AND account.balance=100", true},
{"account.balance=100 AND slashing.amount EXISTS", true},
{"slashing EXISTS", true},
{"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true},
{"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false},
}
for _, test := range tests {
q, err := syntax.Parse(test.input)
if test.valid != (err == nil) {
t.Errorf("Parse %#q: valid %v got err=%v", test.input, test.valid, err)
}
// For valid queries, check that the query round-trips.
if test.valid {
qstr := q.String()
r, err := syntax.Parse(qstr)
if err != nil {
t.Errorf("Reparse %#q failed: %v", qstr, err)
}
if rstr := r.String(); rstr != qstr {
t.Errorf("Reparse diff\nold: %#q\nnew: %#q", qstr, rstr)
}
}
}
}