mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-10-01 04:05:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

abci: Refactor tagging events using list of lists (#3643)

Browse Source 
## PR

This PR introduces a fundamental breaking change to the structure of ABCI response and tx tags and the way they're processed. Namely, the SDK can support more complex and aggregated events for distribution and slashing. In addition, block responses can include duplicate keys in events.

    Implement new Event type. An event has a type and a list of KV pairs (ie. list-of-lists). Typical events may look like:

"rewards": [{"amount": "5000uatom", "validator": "...", "recipient": "..."}]
"sender": [{"address": "...", "balance": "100uatom"}]

The events are indexed by {even.type}.{even.attribute[i].key}/.... In this case a client would subscribe or query for rewards.recipient='...'

    ABCI response types and related types now include Events []Event instead of Tags []cmn.KVPair.
    PubSub logic now publishes/matches against map[string][]string instead of map[string]string to support duplicate keys in response events (from #1385). A match is successful if the value is found in the slice of strings.

closes: #1859
closes: #2905

## Commits:

* Implement Event ABCI type and updates responses to use events

* Update messages_test.go

* Update kvstore.go

* Update event_bus.go

* Update subscription.go

* Update pubsub.go

* Update kvstore.go

* Update query logic to handle slice of strings in events

* Update Empty#Matches and unit tests

* Update pubsub logic

* Update EventBus#Publish

* Update kv tx indexer

* Update godocs

* Update ResultEvent to use slice of strings; update RPC

* Update more tests

* Update abci.md

* Check for key in validateAndStringifyEvents

* Fix KV indexer to skip empty keys

* Fix linting errors

* Update CHANGELOG_PENDING.md

* Update docs/spec/abci/abci.md

Co-Authored-By: Federico Kunze <31522760+fedekunze@users.noreply.github.com>

* Update abci/types/types.proto

Co-Authored-By: Ethan Buchman <ethan@coinculture.info>

* Update docs/spec/abci/abci.md

Co-Authored-By: Ethan Buchman <ethan@coinculture.info>

* Update libs/pubsub/query/query.go

Co-Authored-By: Ethan Buchman <ethan@coinculture.info>

* Update match function to match if ANY value matches

* Implement TestSubscribeDuplicateKeys

* Update TestMatches to include multi-key test cases

* Update events.go

* Update Query interface godoc

* Update match godoc

* Add godoc for matchValue

* DRY-up tx indexing

* Return error from PublishWithEvents in EventBus#Publish

* Update PublishEventNewBlockHeader to return an error

* Fix build

* Update events doc in ABCI

* Update ABCI events godoc

* Implement TestEventBusPublishEventTxDuplicateKeys

* Update TestSubscribeDuplicateKeys to be table-driven

* Remove mod file

* Remove markdown from events godoc

* Implement TestTxSearchDeprecatedIndexing test
This commit is contained in:
Alexander Bezobchuk
2019-06-12 14:03:45 +02:00
committed by Anton Kaliaev
co-authored by Federico Kunze Ethan Buchman
parent 8b7ca8fd99
commit ab0835463f
25 changed files with 1405 additions and 551 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/pubsub/query/empty.go
+1 -1
View File
@@ -5,7 +5,7 @@ type Empty struct {
}
// Matches always returns true.
func (Empty) Matches(tags map[string]string) bool {
func (Empty) Matches(tags map[string][]string) bool {
return true
}
libs/pubsub/query/empty_test.go
+4 -4
View File
@@ -10,8 +10,8 @@ import (
func TestEmptyQueryMatchesAnything(t *testing.T) {
q := query.Empty{}
assert.True(t, q.Matches(map[string]string{}))
assert.True(t, q.Matches(map[string]string{"Asher": "Roth"}))
assert.True(t, q.Matches(map[string]string{"Route": "66"}))
assert.True(t, q.Matches(map[string]string{"Route": "66", "Billy": "Blue"}))
assert.True(t, q.Matches(map[string][]string{}))
assert.True(t, q.Matches(map[string][]string{"Asher": {"Roth"}}))
assert.True(t, q.Matches(map[string][]string{"Route": {"66"}}))
assert.True(t, q.Matches(map[string][]string{"Route": {"66"}, "Billy": {"Blue"}}))
}
libs/pubsub/query/query.go
+50 -22
View File
@@ -148,12 +148,14 @@ func (q *Query) Conditions() []Condition {
return conditions
}
// Matches returns true if the query matches the given set of tags, false otherwise.
// Matches returns true if the query matches against any event in the given set
// of events, false otherwise. For each event, a match exists if the query is
// matched against *any* value in a slice of values.
//
// For example, query "name=John" matches tags = {"name": "John"}. More
// examples could be found in parser_test.go and query_test.go.
func (q *Query) Matches(tags map[string]string) bool {
if len(tags) == 0 {
// For example, query "name=John" matches events = {"name": ["John", "Eric"]}.
// More examples could be found in parser_test.go and query_test.go.
func (q *Query) Matches(events map[string][]string) bool {
if len(events) == 0 {
return false
}
@@ -162,7 +164,8 @@ func (q *Query) Matches(tags map[string]string) bool {
var tag string
var op Operator
// tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7")
// tokens must be in the following order:
// tag ("tx.gas") -> operator ("=") -> operand ("7")
for _, token := range q.parser.Tokens() {
switch token.pegRule {
@@ -188,7 +191,7 @@ func (q *Query) Matches(tags map[string]string) bool {
// see if the triplet (tag, operator, operand) matches any tag
// "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" }
if !match(tag, op, reflect.ValueOf(valueWithoutSingleQuotes), tags) {
if !match(tag, op, reflect.ValueOf(valueWithoutSingleQuotes), events) {
return false
}
case rulenumber:
@@ -198,7 +201,7 @@ func (q *Query) Matches(tags map[string]string) bool {
if err != nil {
panic(fmt.Sprintf("got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", err, number))
}
if !match(tag, op, reflect.ValueOf(value), tags) {
if !match(tag, op, reflect.ValueOf(value), events) {
return false
}
} else {
@@ -206,7 +209,7 @@ func (q *Query) Matches(tags map[string]string) bool {
if err != nil {
panic(fmt.Sprintf("got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", err, number))
}
if !match(tag, op, reflect.ValueOf(value), tags) {
if !match(tag, op, reflect.ValueOf(value), events) {
return false
}
}
@@ -215,7 +218,7 @@ func (q *Query) Matches(tags map[string]string) bool {
if err != nil {
panic(fmt.Sprintf("got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", err, buffer[begin:end]))
}
if !match(tag, op, reflect.ValueOf(value), tags) {
if !match(tag, op, reflect.ValueOf(value), events) {
return false
}
case ruledate:
@@ -223,7 +226,7 @@ func (q *Query) Matches(tags map[string]string) bool {
if err != nil {
panic(fmt.Sprintf("got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", err, buffer[begin:end]))
}
if !match(tag, op, reflect.ValueOf(value), tags) {
if !match(tag, op, reflect.ValueOf(value), events) {
return false
}
}
@@ -232,34 +235,53 @@ func (q *Query) Matches(tags map[string]string) bool {
return true
}
// match returns true if the given triplet (tag, operator, operand) matches any tag.
// match returns true if the given triplet (tag, operator, operand) matches any
// value in an event for that key.
//
// First, it looks up the tag in tags and if it finds one, tries to compare the
// value from it to the operand using the operator.
// First, it looks up the key in the events and if it finds one, tries to compare
// all the values from it to the operand using the operator.
//
// "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" }
func match(tag string, op Operator, operand reflect.Value, tags map[string]string) bool {
// "tx.gas", "=", "7", {"tx": [{"gas": 7, "ID": "4AE393495334"}]}
func match(tag string, op Operator, operand reflect.Value, events map[string][]string) bool {
// look up the tag from the query in tags
value, ok := tags[tag]
values, ok := events[tag]
if !ok {
return false
}
for _, value := range values {
// return true if any value in the set of the event's values matches
if matchValue(value, op, operand) {
return true
}
}
return false
}
// matchValue will attempt to match a string value against an operation an
// operand. A boolean is returned representing the match result. It will panic
// if an error occurs or if the operand is invalid.
func matchValue(value string, op Operator, operand reflect.Value) bool {
switch operand.Kind() {
case reflect.Struct: // time
operandAsTime := operand.Interface().(time.Time)
// try our best to convert value from tags to time.Time
var (
v time.Time
err error
)
if strings.ContainsAny(value, "T") {
v, err = time.Parse(TimeLayout, value)
} else {
v, err = time.Parse(DateLayout, value)
}
if err != nil {
panic(fmt.Sprintf("Failed to convert value %v from tag to time.Time: %v", value, err))
panic(fmt.Sprintf("failed to convert value %v from tag to time.Time: %v", value, err))
}
switch op {
case OpLessEqual:
return v.Before(operandAsTime) || v.Equal(operandAsTime)
@@ -272,14 +294,17 @@ func match(tag string, op Operator, operand reflect.Value, tags map[string]strin
case OpEqual:
return v.Equal(operandAsTime)
}
case reflect.Float64:
operandFloat64 := operand.Interface().(float64)
var v float64
// try our best to convert value from tags to float64
v, err := strconv.ParseFloat(value, 64)
if err != nil {
panic(fmt.Sprintf("Failed to convert value %v from tag to float64: %v", value, err))
panic(fmt.Sprintf("failed to convert value %v from tag to float64: %v", value, err))
}
switch op {
case OpLessEqual:
return v <= operandFloat64
@@ -292,6 +317,7 @@ func match(tag string, op Operator, operand reflect.Value, tags map[string]strin
case OpEqual:
return v == operandFloat64
}
case reflect.Int64:
operandInt := operand.Interface().(int64)
var v int64
@@ -299,7 +325,7 @@ func match(tag string, op Operator, operand reflect.Value, tags map[string]strin
if strings.ContainsAny(value, ".") {
v1, err := strconv.ParseFloat(value, 64)
if err != nil {
panic(fmt.Sprintf("Failed to convert value %v from tag to float64: %v", value, err))
panic(fmt.Sprintf("failed to convert value %v from tag to float64: %v", value, err))
}
v = int64(v1)
} else {
@@ -307,7 +333,7 @@ func match(tag string, op Operator, operand reflect.Value, tags map[string]strin
// try our best to convert value from tags to int64
v, err = strconv.ParseInt(value, 10, 64)
if err != nil {
panic(fmt.Sprintf("Failed to convert value %v from tag to int64: %v", value, err))
panic(fmt.Sprintf("failed to convert value %v from tag to int64: %v", value, err))
}
}
switch op {
@@ -322,6 +348,7 @@ func match(tag string, op Operator, operand reflect.Value, tags map[string]strin
case OpEqual:
return v == operandInt
}
case reflect.String:
switch op {
case OpEqual:
@@ -329,8 +356,9 @@ func match(tag string, op Operator, operand reflect.Value, tags map[string]strin
case OpContains:
return strings.Contains(value, operand.String())
}
default:
panic(fmt.Sprintf("Unknown kind of operand %v", operand.Kind()))
panic(fmt.Sprintf("unknown kind of operand %v", operand.Kind()))
}
return false
libs/pubsub/query/query_test.go
+31 -19
View File
@@ -19,30 +19,40 @@ func TestMatches(t *testing.T) {
testCases := []struct {
s string
tags map[string]string
events map[string][]string
err bool
matches bool
}{
{"tm.events.type='NewBlock'", map[string]string{"tm.events.type": "NewBlock"}, false, true},
{"tm.events.type='NewBlock'", map[string][]string{"tm.events.type": {"NewBlock"}}, false, true},
{"tx.gas > 7", map[string]string{"tx.gas": "8"}, false, true},
{"tx.gas > 7 AND tx.gas < 9", map[string]string{"tx.gas": "8"}, false, true},
{"body.weight >= 3.5", map[string]string{"body.weight": "3.5"}, false, true},
{"account.balance < 1000.0", map[string]string{"account.balance": "900"}, false, true},
{"apples.kg <= 4", map[string]string{"apples.kg": "4.0"}, false, true},
{"body.weight >= 4.5", map[string]string{"body.weight": fmt.Sprintf("%v", float32(4.5))}, false, true},
{"oranges.kg < 4 AND watermellons.kg > 10", map[string]string{"oranges.kg": "3", "watermellons.kg": "12"}, false, true},
{"peaches.kg < 4", map[string]string{"peaches.kg": "5"}, false, false},
{"tx.gas > 7", map[string][]string{"tx.gas": {"8"}}, false, true},
{"tx.gas > 7 AND tx.gas < 9", map[string][]string{"tx.gas": {"8"}}, false, true},
{"body.weight >= 3.5", map[string][]string{"body.weight": {"3.5"}}, false, true},
{"account.balance < 1000.0", map[string][]string{"account.balance": {"900"}}, false, true},
{"apples.kg <= 4", map[string][]string{"apples.kg": {"4.0"}}, false, true},
{"body.weight >= 4.5", map[string][]string{"body.weight": {fmt.Sprintf("%v", float32(4.5))}}, false, true},
{"oranges.kg < 4 AND watermellons.kg > 10", map[string][]string{"oranges.kg": {"3"}, "watermellons.kg": {"12"}}, false, true},
{"peaches.kg < 4", map[string][]string{"peaches.kg": {"5"}}, false, false},
{"tx.date > DATE 2017-01-01", map[string]string{"tx.date": time.Now().Format(query.DateLayout)}, false, true},
{"tx.date = DATE 2017-01-01", map[string]string{"tx.date": txDate}, false, true},
{"tx.date = DATE 2018-01-01", map[string]string{"tx.date": txDate}, false, false},
{"tx.date > DATE 2017-01-01", map[string][]string{"tx.date": {time.Now().Format(query.DateLayout)}}, false, true},
{"tx.date = DATE 2017-01-01", map[string][]string{"tx.date": {txDate}}, false, true},
{"tx.date = DATE 2018-01-01", map[string][]string{"tx.date": {txDate}}, false, false},
{"tx.time >= TIME 2013-05-03T14:45:00Z", map[string]string{"tx.time": time.Now().Format(query.TimeLayout)}, false, true},
{"tx.time = TIME 2013-05-03T14:45:00Z", map[string]string{"tx.time": txTime}, false, false},
{"tx.time >= TIME 2013-05-03T14:45:00Z", map[string][]string{"tx.time": {time.Now().Format(query.TimeLayout)}}, false, true},
{"tx.time = TIME 2013-05-03T14:45:00Z", map[string][]string{"tx.time": {txTime}}, false, false},
{"abci.owner.name CONTAINS 'Igor'", map[string]string{"abci.owner.name": "Igor,Ivan"}, false, true},
{"abci.owner.name CONTAINS 'Igor'", map[string]string{"abci.owner.name": "Pavel,Ivan"}, false, false},
{"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Igor,Ivan"}}, false, true},
{"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Pavel,Ivan"}}, false, false},
{"abci.owner.name = 'Igor'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, false, true},
{"abci.owner.name = 'Ivan'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, false, true},
{"abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, false, true},
{"abci.owner.name = 'Ivan' AND abci.owner.name = 'John'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, false, false},
{"tm.events.type='NewBlock'", map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, false, true},
{"app.name = 'fuzzed'", map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, false, true},
{"tm.events.type='NewBlock' AND app.name = 'fuzzed'", map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, false, true},
{"tm.events.type='NewHeader' AND app.name = 'fuzzed'", map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, false, false},
}
for _, tc := range testCases {
@@ -51,10 +61,12 @@ func TestMatches(t *testing.T) {
require.Nil(t, err)
}
require.NotNil(t, q, "Query '%s' should not be nil", tc.s)
if tc.matches {
assert.True(t, q.Matches(tc.tags), "Query '%s' should match %v", tc.s, tc.tags)
assert.True(t, q.Matches(tc.events), "Query '%s' should match %v", tc.s, tc.events)
} else {
assert.False(t, q.Matches(tc.tags), "Query '%s' should not match %v", tc.s, tc.tags)
assert.False(t, q.Matches(tc.events), "Query '%s' should not match %v", tc.s, tc.events)
}
}
}