From 82907c84fa53501696b21931c9e97c7ee99d2f75 Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 13 Jun 2022 19:20:54 +0200 Subject: [PATCH 1/3] sink/psql: json marshal instead of proto (#8637) Storing transaction records as JSON makes it simpler for clients of the index. --- CHANGELOG_PENDING.md | 3 ++- internal/state/indexer/sink/psql/psql.go | 9 ++++++--- internal/state/indexer/sink/psql/psql_test.go | 12 ++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index f33198403..608abc06d 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -32,7 +32,8 @@ Special thanks to external contributors on this release: - [abci] \#8605 Remove info, log, events, gasUsed and mempoolError fields from ResponseCheckTx as they are not used by Tendermint. (@jmalicevic) - [abci] \#8664 Move `app_hash` parameter from `Commit` to `FinalizeBlock`. (@sergio-mena) - [abci] \#8656 Added cli command for `PrepareProposal`. (@jmalicevic) - + - [sink/psql] \#8637 tx_results emitted from psql sink are now json encoded, previously they were protobuf encoded + - P2P Protocol - [p2p] \#7035 Remove legacy P2P routing implementation and associated configuration options. (@tychoish) diff --git a/internal/state/indexer/sink/psql/psql.go b/internal/state/indexer/sink/psql/psql.go index c06383264..57f5e5c3d 100644 --- a/internal/state/indexer/sink/psql/psql.go +++ b/internal/state/indexer/sink/psql/psql.go @@ -9,8 +9,7 @@ import ( "strings" "time" - "github.com/gogo/protobuf/proto" - + "github.com/gogo/protobuf/jsonpb" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/internal/pubsub/query" "github.com/tendermint/tendermint/internal/state/indexer" @@ -177,12 +176,16 @@ INSERT INTO `+tableBlocks+` (height, chain_id, created_at) }) } +var ( + jsonpbMarshaller = jsonpb.Marshaler{} +) + func (es *EventSink) IndexTxEvents(txrs []*abci.TxResult) error { ts := time.Now().UTC() for _, txr := range txrs { // Encode the result message in protobuf wire format for indexing. - resultData, err := proto.Marshal(txr) + resultData, err := jsonpbMarshaller.MarshalToString(txr) if err != nil { return fmt.Errorf("marshaling tx_result: %w", err) } diff --git a/internal/state/indexer/sink/psql/psql_test.go b/internal/state/indexer/sink/psql/psql_test.go index 72d14b5d8..2625d7245 100644 --- a/internal/state/indexer/sink/psql/psql_test.go +++ b/internal/state/indexer/sink/psql/psql_test.go @@ -1,6 +1,7 @@ package psql import ( + "bytes" "context" "database/sql" "flag" @@ -12,7 +13,7 @@ import ( "time" "github.com/adlio/schema" - "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/jsonpb" "github.com/ory/dockertest" "github.com/ory/dockertest/docker" "github.com/stretchr/testify/assert" @@ -151,6 +152,8 @@ func TestType(t *testing.T) { assert.Equal(t, indexer.PSQL, psqlSink.Type()) } +var jsonpbUnmarshaller = jsonpb.Unmarshaler{} + func TestIndexing(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -278,13 +281,14 @@ func loadTxResult(hash []byte) (*abci.TxResult, error) { hashString := fmt.Sprintf("%X", hash) var resultData []byte if err := testDB().QueryRow(` -SELECT tx_result FROM `+tableTxResults+` WHERE tx_hash = $1; -`, hashString).Scan(&resultData); err != nil { + SELECT tx_result FROM `+tableTxResults+` WHERE tx_hash = $1; + `, hashString).Scan(&resultData); err != nil { return nil, fmt.Errorf("lookup transaction for hash %q failed: %v", hashString, err) } + reader := bytes.NewBuffer(resultData) txr := new(abci.TxResult) - if err := proto.Unmarshal(resultData, txr); err != nil { + if err := jsonpbUnmarshaller.Unmarshal(reader, txr); err != nil { return nil, fmt.Errorf("unmarshaling txr: %w", err) } From 21bbbe3e2a1ad133241abdefdaf6874f72fbbc4d Mon Sep 17 00:00:00 2001 From: Jeeyong Um Date: Tue, 14 Jun 2022 16:50:55 +0800 Subject: [PATCH 2/3] mempool: fix typos in test (#8746) --- internal/mempool/priority_queue_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/mempool/priority_queue_test.go b/internal/mempool/priority_queue_test.go index ddc84806d..90f611162 100644 --- a/internal/mempool/priority_queue_test.go +++ b/internal/mempool/priority_queue_test.go @@ -90,7 +90,7 @@ func TestTxPriorityQueue_GetEvictableTxs(t *testing.T) { expectedLen int }{ { - name: "larest priority; single tx", + name: "largest priority; single tx", priority: int64(max + 1), txSize: 5, totalSize: totalSize, @@ -98,7 +98,7 @@ func TestTxPriorityQueue_GetEvictableTxs(t *testing.T) { expectedLen: 1, }, { - name: "larest priority; multi tx", + name: "largest priority; multi tx", priority: int64(max + 1), txSize: 17, totalSize: totalSize, @@ -106,7 +106,7 @@ func TestTxPriorityQueue_GetEvictableTxs(t *testing.T) { expectedLen: 4, }, { - name: "larest priority; out of capacity", + name: "largest priority; out of capacity", priority: int64(max + 1), txSize: totalSize + 1, totalSize: totalSize, From a4cf8939b88b70072f40a87e87516496ee3cc347 Mon Sep 17 00:00:00 2001 From: Jeeyong Um Date: Tue, 14 Jun 2022 18:54:35 +0800 Subject: [PATCH 3/3] mempool: fix error message check in test (#8750) --- internal/mempool/mempool_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mempool/mempool_test.go b/internal/mempool/mempool_test.go index 538cb3e1f..42fb13bdc 100644 --- a/internal/mempool/mempool_test.go +++ b/internal/mempool/mempool_test.go @@ -631,7 +631,7 @@ func TestTxMempool_CheckTxPostCheckError(t *testing.T) { require.NoError(t, txmp.CheckTx(ctx, tx, callback, TxInfo{SenderID: 0})) } else { err = txmp.CheckTx(ctx, tx, callback, TxInfo{SenderID: 0}) - fmt.Print(err.Error()) + require.EqualError(t, err, "test error") } }) }