mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-05-01 04:45:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

indexer: allow indexing an event at runtime (#4466)

Browse Source 
The PR added a new field `index` to event attribute, that will cause indexer service to index the event if set to true.
This commit is contained in:
Diep Pham
2020-04-22 17:07:03 +07:00
committed by GitHub
parent 0a159c2613
commit 843d63f935
14 changed files with 757 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG_PENDING.md
View File

@@ -24,4 +24,6 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
### IMPROVEMENTS:
- [txindex] [\#4466](https://github.com/tendermint/tendermint/pull/4466) Allow to index an event at runtime (@favadi)
### BUG FIXES:

5
abci/example/kvstore/kvstore.go
View File

@@ -10,7 +10,6 @@ import (
"github.com/tendermint/tendermint/abci/example/code"
"github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/kv"
"github.com/tendermint/tendermint/version"
)
@@ -99,9 +98,11 @@ func (app *Application) DeliverTx(req types.RequestDeliverTx) types.ResponseDeli
events := []types.Event{
{
Type: "app",
Attributes: []kv.Pair{
Attributes: []types.EventAttribute{
{Key: []byte("creator"), Value: []byte("Cosmoshi Netowoko")},
{Key: []byte("key"), Value: key},
{Key: []byte("index_key"), Value: []byte("index is working"), Index: true},
{Key: []byte("noindex_key"), Value: []byte("index is working"), Index: false},
},
},
}

6
abci/types/messages_test.go
View File

@@ -8,8 +8,6 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/tendermint/tendermint/libs/kv"
)
func TestMarshalJSON(t *testing.T) {
@@ -24,7 +22,7 @@ func TestMarshalJSON(t *testing.T) {
Events: []Event{
{
Type: "testEvent",
Attributes: []kv.Pair{
Attributes: []EventAttribute{
{Key: []byte("pho"), Value: []byte("bo")},
},
},
@@ -91,7 +89,7 @@ func TestWriteReadMessage2(t *testing.T) {
Events: []Event{
{
Type: "testEvent",
Attributes: []kv.Pair{
Attributes: []EventAttribute{
{Key: []byte("abc"), Value: []byte("def")},
},
},

803
abci/types/types.pb.go
View File

File diff suppressed because it is too large Load Diff

13
abci/types/types.proto
View File

@@ -6,7 +6,6 @@ option go_package = "github.com/tendermint/tendermint/abci/types";
// https://github.com/gogo/protobuf/blob/master/extensions.md
import "third_party/proto/gogoproto/gogo.proto";
import "crypto/merkle/merkle.proto";
import "libs/kv/types.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/duration.proto";
@@ -247,10 +246,18 @@ message LastCommitInfo {
repeated VoteInfo votes = 2 [(gogoproto.nullable) = false];
}
// EventAttribute represents an event to the indexing service.
message EventAttribute {
bytes key = 1;
bytes value = 2;
bool index = 3;
}
message Event {
string type = 1;
repeated tendermint.libs.kv.Pair attributes = 2
[(gogoproto.nullable) = false, (gogoproto.jsontag) = "attributes,omitempty"];
repeated EventAttribute attributes = 2
[(gogoproto.nullable) = false, (gogoproto.jsontag) = "attributes,omitempty"];
}
//----------------------------------------

125
abci/types/typespb_test.go
View File

@@ -13,7 +13,6 @@ import (
_ "github.com/golang/protobuf/ptypes/duration"
_ "github.com/golang/protobuf/ptypes/timestamp"
_ "github.com/tendermint/tendermint/crypto/merkle"
_ "github.com/tendermint/tendermint/libs/kv"
math "math"
math_rand "math/rand"
testing "testing"
@@ -1706,6 +1705,62 @@ func TestLastCommitInfoMarshalTo(t *testing.T) {
}
}
func TestEventAttributeProto(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedEventAttribute(popr, false)
dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &EventAttribute{}
if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
littlefuzz := make([]byte, len(dAtA))
copy(littlefuzz, dAtA)
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
if len(littlefuzz) > 0 {
fuzzamount := 100
for i := 0; i < fuzzamount; i++ {
littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
}
// shouldn't panic
_ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
}
}
func TestEventAttributeMarshalTo(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedEventAttribute(popr, false)
size := p.Size()
dAtA := make([]byte, size)
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
_, err := p.MarshalTo(dAtA)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &EventAttribute{}
if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func TestEventProto(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
@@ -2806,6 +2861,24 @@ func TestLastCommitInfoJSON(t *testing.T) {
t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
}
}
func TestEventAttributeJSON(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedEventAttribute(popr, true)
marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{}
jsondata, err := marshaler.MarshalToString(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &EventAttribute{}
err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
}
}
func TestEventJSON(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
@@ -3826,6 +3899,34 @@ func TestLastCommitInfoProtoCompactText(t *testing.T) {
}
}
func TestEventAttributeProtoText(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedEventAttribute(popr, true)
dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p)
msg := &EventAttribute{}
if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func TestEventAttributeProtoCompactText(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedEventAttribute(popr, true)
dAtA := github_com_gogo_protobuf_proto.CompactTextString(p)
msg := &EventAttribute{}
if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func TestEventProtoText(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
@@ -4766,6 +4867,28 @@ func TestLastCommitInfoSize(t *testing.T) {
}
}
func TestEventAttributeSize(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedEventAttribute(popr, true)
size2 := github_com_gogo_protobuf_proto.Size(p)
dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
size := p.Size()
if len(dAtA) != size {
t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
}
if size2 != size {
t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
}
size3 := github_com_gogo_protobuf_proto.Size(p)
if size3 != size {
t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
}
}
func TestEventSize(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))

12
docs/app-dev/indexing-transactions.md
View File

@@ -76,10 +76,11 @@ func (app *KVStoreApplication) DeliverTx(req types.RequestDeliverTx) types.Resul
events := []abci.Event{
{
Type: "transfer",
Attributes: kv.Pairs{
kv.Pair{Key: []byte("sender"), Value: []byte("Bob")},
kv.Pair{Key: []byte("recipient"), Value: []byte("Alice")},
kv.Pair{Key: []byte("balance"), Value: []byte("100")},
Attributes: []abci.EventAttribute{
{Key: []byte("sender"), Value: []byte("Bob")},
{Key: []byte("recipient"), Value: []byte("Alice")},
{Key: []byte("balance"), Value: []byte("100")},
{Key: []byte("note"), Value: []byte("nothing"), Index: true},
},
},
}
@@ -98,6 +99,9 @@ Note, there are a few predefined event types:
Tendermint will throw a warning if you try to use any of the above keys.
The index will be added if the `Index` field of attribute is set to true. In above example, querying
using `transfer.note` will work.
## Querying Transactions
You can query the transaction results by calling `/tx_search` RPC endpoint:

2
libs/kv/types.proto
View File

@@ -17,6 +17,6 @@ option (gogoproto.testgen_all) = true;
// Abstract types
message Pair {
bytes key = 1;
bytes key = 1;
bytes value = 2;
}

14
rpc/client/rpc_test.go
View File

@@ -493,6 +493,20 @@ func TestTxSearch(t *testing.T) {
t.Fatal("expected a lot of transactions")
}
// query using an index key
result, err = c.TxSearch("app.index_key='index is working'", false, 1, 30, "asc")
require.Nil(t, err)
if len(result.Txs) == 0 {
t.Fatal("expected a lot of transactions")
}
// query using an noindex key
result, err = c.TxSearch("app.noindex_key='index is working'", false, 1, 30, "asc")
require.Nil(t, err)
if len(result.Txs) != 0 {
t.Fatal("expected no transaction")
}
// query using a compositeKey (see kvstore application) and height
result, err = c.TxSearch("app.creator='Cosmoshi Netowoko' AND tx.height<10000", true, 1, 30, "asc")
require.Nil(t, err)

5
state/state_test.go
View File

@@ -17,7 +17,6 @@ import (
abci "github.com/tendermint/tendermint/abci/types"
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/crypto/ed25519"
"github.com/tendermint/tendermint/libs/kv"
"github.com/tendermint/tendermint/libs/rand"
tmrand "github.com/tendermint/tendermint/libs/rand"
sm "github.com/tendermint/tendermint/state"
@@ -136,8 +135,8 @@ func TestABCIResponsesSaveLoad2(t *testing.T) {
{
Data: []byte("Gotcha!"),
Events: []abci.Event{
{Type: "type1", Attributes: []kv.Pair{{Key: []byte("a"), Value: []byte("1")}}},
{Type: "type2", Attributes: []kv.Pair{{Key: []byte("build"), Value: []byte("stuff")}}},
{Type: "type1", Attributes: []abci.EventAttribute{{Key: []byte("a"), Value: []byte("1")}}},
{Type: "type2", Attributes: []abci.EventAttribute{{Key: []byte("build"), Value: []byte("stuff")}}},
},
},
},

2
state/txindex/kv/kv.go
View File

@@ -153,7 +153,7 @@ func (txi *TxIndex) indexEvents(result *types.TxResult, hash []byte, store dbm.S
}
compositeTag := fmt.Sprintf("%s.%s", event.Type, string(attr.Key))
if txi.indexAllEvents || tmstring.StringInSlice(compositeTag, txi.compositeKeysToIndex) {
if txi.indexAllEvents || tmstring.StringInSlice(compositeTag, txi.compositeKeysToIndex) || attr.GetIndex() {
store.Set(keyForEvent(compositeTag, attr.Value, result), hash)
}
}

3
state/txindex/kv/kv_bench_test.go
View File

@@ -10,7 +10,6 @@ import (
dbm "github.com/tendermint/tm-db"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/kv"
"github.com/tendermint/tendermint/libs/pubsub/query"
"github.com/tendermint/tendermint/types"
)
@@ -33,7 +32,7 @@ func BenchmarkTxSearch(b *testing.B) {
events := []abci.Event{
{
Type: "transfer",
Attributes: []kv.Pair{
Attributes: []abci.EventAttribute{
{Key: []byte("address"), Value: []byte(fmt.Sprintf("address_%d", i%100))},
{Key: []byte("amount"), Value: []byte("50")},
},

27
state/txindex/kv/kv_test.go
View File

@@ -13,7 +13,6 @@ import (
db "github.com/tendermint/tm-db"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/kv"
"github.com/tendermint/tendermint/libs/pubsub/query"
tmrand "github.com/tendermint/tendermint/libs/rand"
"github.com/tendermint/tendermint/state/txindex"
@@ -71,9 +70,9 @@ func TestTxSearch(t *testing.T) {
indexer := NewTxIndex(db.NewMemDB(), IndexEvents(allowedKeys))
txResult := txResultWithEvents([]abci.Event{
{Type: "account", Attributes: []kv.Pair{{Key: []byte("number"), Value: []byte("1")}}},
{Type: "account", Attributes: []kv.Pair{{Key: []byte("owner"), Value: []byte("Ivan")}}},
{Type: "", Attributes: []kv.Pair{{Key: []byte("not_allowed"), Value: []byte("Vlad")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("1")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("owner"), Value: []byte("Ivan")}}},
{Type: "", Attributes: []abci.EventAttribute{{Key: []byte("not_allowed"), Value: []byte("Vlad")}}},
})
hash := txResult.Tx.Hash()
@@ -140,9 +139,9 @@ func TestTxSearchWithCancelation(t *testing.T) {
indexer := NewTxIndex(db.NewMemDB(), IndexEvents(allowedKeys))
txResult := txResultWithEvents([]abci.Event{
{Type: "account", Attributes: []kv.Pair{{Key: []byte("number"), Value: []byte("1")}}},
{Type: "account", Attributes: []kv.Pair{{Key: []byte("owner"), Value: []byte("Ivan")}}},
{Type: "", Attributes: []kv.Pair{{Key: []byte("not_allowed"), Value: []byte("Vlad")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("1")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("owner"), Value: []byte("Ivan")}}},
{Type: "", Attributes: []abci.EventAttribute{{Key: []byte("not_allowed"), Value: []byte("Vlad")}}},
})
err := indexer.Index(txResult)
require.NoError(t, err)
@@ -160,7 +159,7 @@ func TestTxSearchDeprecatedIndexing(t *testing.T) {
// index tx using events indexing (composite key)
txResult1 := txResultWithEvents([]abci.Event{
{Type: "account", Attributes: []kv.Pair{{Key: []byte("number"), Value: []byte("1")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("1")}}},
})
hash1 := txResult1.Tx.Hash()
@@ -231,8 +230,8 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) {
indexer := NewTxIndex(db.NewMemDB(), IndexEvents(allowedKeys))
txResult := txResultWithEvents([]abci.Event{
{Type: "account", Attributes: []kv.Pair{{Key: []byte("number"), Value: []byte("1")}}},
{Type: "account", Attributes: []kv.Pair{{Key: []byte("number"), Value: []byte("2")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("1")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("2")}}},
})
err := indexer.Index(txResult)
@@ -253,7 +252,7 @@ func TestTxSearchMultipleTxs(t *testing.T) {
// indexed first, but bigger height (to test the order of transactions)
txResult := txResultWithEvents([]abci.Event{
{Type: "account", Attributes: []kv.Pair{{Key: []byte("number"), Value: []byte("1")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("1")}}},
})
txResult.Tx = types.Tx("Bob's account")
@@ -264,7 +263,7 @@ func TestTxSearchMultipleTxs(t *testing.T) {
// indexed second, but smaller height (to test the order of transactions)
txResult2 := txResultWithEvents([]abci.Event{
{Type: "account", Attributes: []kv.Pair{{Key: []byte("number"), Value: []byte("2")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("2")}}},
})
txResult2.Tx = types.Tx("Alice's account")
txResult2.Height = 1
@@ -275,7 +274,7 @@ func TestTxSearchMultipleTxs(t *testing.T) {
// indexed third (to test the order of transactions)
txResult3 := txResultWithEvents([]abci.Event{
{Type: "account", Attributes: []kv.Pair{{Key: []byte("number"), Value: []byte("3")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number"), Value: []byte("3")}}},
})
txResult3.Tx = types.Tx("Jack's account")
txResult3.Height = 1
@@ -286,7 +285,7 @@ func TestTxSearchMultipleTxs(t *testing.T) {
// indexed fourth (to test we don't include txs with similar events)
// https://github.com/tendermint/tendermint/issues/2908
txResult4 := txResultWithEvents([]abci.Event{
{Type: "account", Attributes: []kv.Pair{{Key: []byte("number.id"), Value: []byte("1")}}},
{Type: "account", Attributes: []abci.EventAttribute{{Key: []byte("number.id"), Value: []byte("1")}}},
})
txResult4.Tx = types.Tx("Mike's account")
txResult4.Height = 2

17
types/event_bus_test.go
View File

@@ -11,7 +11,6 @@ import (
"github.com/stretchr/testify/require"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/kv"
tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
tmquery "github.com/tendermint/tendermint/libs/pubsub/query"
tmrand "github.com/tendermint/tendermint/libs/rand"
@@ -27,7 +26,7 @@ func TestEventBusPublishEventTx(t *testing.T) {
result := abci.ResponseDeliverTx{
Data: []byte("bar"),
Events: []abci.Event{
{Type: "testType", Attributes: []kv.Pair{{Key: []byte("baz"), Value: []byte("1")}}},
{Type: "testType", Attributes: []abci.EventAttribute{{Key: []byte("baz"), Value: []byte("1")}}},
},
}
@@ -71,12 +70,12 @@ func TestEventBusPublishEventNewBlock(t *testing.T) {
block := MakeBlock(0, []Tx{}, nil, []Evidence{})
resultBeginBlock := abci.ResponseBeginBlock{
Events: []abci.Event{
{Type: "testType", Attributes: []kv.Pair{{Key: []byte("baz"), Value: []byte("1")}}},
{Type: "testType", Attributes: []abci.EventAttribute{{Key: []byte("baz"), Value: []byte("1")}}},
},
}
resultEndBlock := abci.ResponseEndBlock{
Events: []abci.Event{
{Type: "testType", Attributes: []kv.Pair{{Key: []byte("foz"), Value: []byte("2")}}},
{Type: "testType", Attributes: []abci.EventAttribute{{Key: []byte("foz"), Value: []byte("2")}}},
},
}
@@ -121,7 +120,7 @@ func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) {
Events: []abci.Event{
{
Type: "transfer",
Attributes: []kv.Pair{
Attributes: []abci.EventAttribute{
{Key: []byte("sender"), Value: []byte("foo")},
{Key: []byte("recipient"), Value: []byte("bar")},
{Key: []byte("amount"), Value: []byte("5")},
@@ -129,7 +128,7 @@ func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) {
},
{
Type: "transfer",
Attributes: []kv.Pair{
Attributes: []abci.EventAttribute{
{Key: []byte("sender"), Value: []byte("baz")},
{Key: []byte("recipient"), Value: []byte("cat")},
{Key: []byte("amount"), Value: []byte("13")},
@@ -137,7 +136,7 @@ func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) {
},
{
Type: "withdraw.rewards",
Attributes: []kv.Pair{
Attributes: []abci.EventAttribute{
{Key: []byte("address"), Value: []byte("bar")},
{Key: []byte("source"), Value: []byte("iceman")},
{Key: []byte("amount"), Value: []byte("33")},
@@ -218,12 +217,12 @@ func TestEventBusPublishEventNewBlockHeader(t *testing.T) {
block := MakeBlock(0, []Tx{}, nil, []Evidence{})
resultBeginBlock := abci.ResponseBeginBlock{
Events: []abci.Event{
{Type: "testType", Attributes: []kv.Pair{{Key: []byte("baz"), Value: []byte("1")}}},
{Type: "testType", Attributes: []abci.EventAttribute{{Key: []byte("baz"), Value: []byte("1")}}},
},
}
resultEndBlock := abci.ResponseEndBlock{
Events: []abci.Event{
{Type: "testType", Attributes: []kv.Pair{{Key: []byte("foz"), Value: []byte("2")}}},
{Type: "testType", Attributes: []abci.EventAttribute{{Key: []byte("foz"), Value: []byte("2")}}},
},
}