wip: start refactor of merkle package

crypto/merkle/simple_map.go

@@ -1,7 +1,6 @@
 package merkle
 
 import (
-	"github.com/tendermint/tendermint/crypto/tmhash"
 	cmn "github.com/tendermint/tendermint/libs/common"
 )
 
@@ -35,7 +34,7 @@ func (sm *simpleMap) Set(key string, value Hasher) {
 	})
 }
 
-// Hash Merkle root hash of items sorted by key
+// Hash computes the Merkle root hash of items sorted by key
 // (UNSTABLE: and by value too if duplicate key).
 func (sm *simpleMap) Hash() []byte {
 	sm.Sort()
@@ -66,23 +65,17 @@ func (sm *simpleMap) KVPairs() cmn.KVPairs {
 // then hashed.
 type KVPair cmn.KVPair
 
-func (kv KVPair) Hash() []byte {
-	hasher := tmhash.New()
-	err := encodeByteSlice(hasher, kv.Key)
-	if err != nil {
-		panic(err)
-	}
-	err = encodeByteSlice(hasher, kv.Value)
-	if err != nil {
-		panic(err)
-	}
-	return hasher.Sum(nil)
+func (kv KVPair) Bytes() []byte {
+	val := make([]byte, len(kv.Key)+len(kv.Value))
+	copy(val, kv.Key)
+	copy(val[len(kv.Key):], kv.Value)
+	return val
 }
 
 func hashKVPairs(kvs cmn.KVPairs) []byte {
-	kvsH := make([]Hasher, len(kvs))
+	kvsH := make([][]byte, len(kvs))
 	for i, kvp := range kvs {
-		kvsH[i] = KVPair(kvp)
+		kvsH[i] = KVPair(kvp).Bytes()
 	}
-	return SimpleHashFromHashers(kvsH)
+	return SimpleHashFromByteSlices(kvsH)
 }

crypto/merkle/simple_proof.go

@@ -3,6 +3,8 @@ package merkle
 import (
 	"bytes"
 	"fmt"
+
+	"github.com/tendermint/tendermint/crypto/tmhash"
 )
 
 // SimpleProof represents a simple merkle proof.
@@ -12,8 +14,8 @@ type SimpleProof struct {
 
 // SimpleProofsFromHashers computes inclusion proof for given items.
 // proofs[0] is the proof for items[0].
-func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) {
-	trails, rootSPN := trailsFromHashers(items)
+func SimpleProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*SimpleProof) {
+	trails, rootSPN := trailsFromByteSlices(items)
 	rootHash = rootSPN.Hash
 	proofs = make([]*SimpleProof, len(items))
 	for i, trail := range trails {
@@ -34,12 +36,12 @@ func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[strin
 	}
 	sm.Sort()
 	kvs := sm.kvs
-	kvsH := make([]Hasher, 0, len(kvs))
+	kvsB := make([][]byte, 0, len(kvs))
 	for _, kvp := range kvs {
-		kvsH = append(kvsH, KVPair(kvp))
+		kvsB = append(kvsB, KVPair(kvp).Bytes())
 	}
 
-	rootHash, proofList := SimpleProofsFromHashers(kvsH)
+	rootHash, proofList := SimpleProofsFromByteSlices(kvsB)
 	proofs = make(map[string]*SimpleProof)
 	keys = make([]string, len(proofList))
 	for i, kvp := range kvs {
@@ -49,9 +51,16 @@ func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs map[strin
 	return
 }
 
-// Verify that leafHash is a leaf hash of the simple-merkle-tree
+// Verify that leaf is a leaf hash of the simple-merkle-tree
 // which hashes to rootHash.
-func (sp *SimpleProof) Verify(index int, total int, leafHash []byte, rootHash []byte) bool {
+func (sp *SimpleProof) Verify(index int, total int, leaf []byte, rootHash []byte) bool {
+	leafHash := tmhash.Sum(append([]byte{0}, leaf...))
+	return sp.VerifyHash(index, total, leafHash, rootHash)
+}
+
+// VerifyHash that leafHash is a leaf hash of the simple-merkle-tree
+// which hashes to rootHash.
+func (sp *SimpleProof) VerifyHash(index int, total int, leafHash []byte, rootHash []byte) bool {
 	computedHash := computeHashFromAunts(index, total, leafHash, sp.Aunts)
 	return computedHash != nil && bytes.Equal(computedHash, rootHash)
 }
@@ -138,17 +147,19 @@ func (spn *SimpleProofNode) FlattenAunts() [][]byte {
 
 // trails[0].Hash is the leaf hash for items[0].
 // trails[i].Parent.Parent....Parent == root for all i.
-func trailsFromHashers(items []Hasher) (trails []*SimpleProofNode, root *SimpleProofNode) {
+func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *SimpleProofNode) {
 	// Recursive impl.
 	switch len(items) {
 	case 0:
 		return nil, nil
 	case 1:
-		trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil}
+		hash0 := tmhash.Sum(append(LeafHashPrefix, items[0]...))
+		trail := &SimpleProofNode{hash0, nil, nil, nil}
 		return []*SimpleProofNode{trail}, trail
 	default:
-		lefts, leftRoot := trailsFromHashers(items[:(len(items)+1)/2])
-		rights, rightRoot := trailsFromHashers(items[(len(items)+1)/2:])
+		k := getSplitPoint(len(items))
+		lefts, leftRoot := trailsFromByteSlices(items[:k])
+		rights, rightRoot := trailsFromByteSlices(items[k:])
 		rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash)
 		root := &SimpleProofNode{rootHash, nil, nil, nil}
 		leftRoot.Parent = root

crypto/merkle/simple_tree.go

@@ -1,28 +1,35 @@
 package merkle
 
 import (
+	"fmt"
+	"math/bits"
+
 	"github.com/tendermint/tendermint/crypto/tmhash"
 )
 
 // SimpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right).
 func SimpleHashFromTwoHashes(left, right []byte) []byte {
 	var hasher = tmhash.New()
-	err := encodeByteSlice(hasher, left)
+	_, err := hasher.Write([]byte{1})
 	if err != nil {
 		panic(err)
 	}
-	err = encodeByteSlice(hasher, right)
+	_, err = hasher.Write(left)
+	if err != nil {
+		panic(err)
+	}
+	_, err = hasher.Write(right)
 	if err != nil {
 		panic(err)
 	}
 	return hasher.Sum(nil)
 }
 
-// SimpleHashFromHashers computes a Merkle tree from items that can be hashed.
-func SimpleHashFromHashers(items []Hasher) []byte {
+// SimpleHashFromByteSlices computes a Merkle tree from items that are byte slices.
+func SimpleHashFromByteSlices(items [][]byte) []byte {
 	hashes := make([][]byte, len(items))
 	for i, item := range items {
-		hash := item.Hash()
+		hash := tmhash.Sum(append([]byte{0}, item...))
 		hashes[i] = hash
 	}
 	return simpleHashFromHashes(hashes)
@@ -51,8 +58,23 @@ func simpleHashFromHashes(hashes [][]byte) []byte {
 	case 1:
 		return hashes[0]
 	default:
-		left := simpleHashFromHashes(hashes[:(len(hashes)+1)/2])
-		right := simpleHashFromHashes(hashes[(len(hashes)+1)/2:])
+		k := getSplitPoint(len(hashes))
+		left := simpleHashFromHashes(hashes[:k])
+		right := simpleHashFromHashes(hashes[k:])
 		return SimpleHashFromTwoHashes(left, right)
 	}
 }
+
+func getSplitPoint(length int) int {
+	if length < 1 {
+		panic("Trying to split a tree with size < 1")
+	}
+	uLength := uint(length)
+	bitlen := bits.Len(uLength)
+	k := 1 << uint(bitlen-1)
+	if k == length {
+		k >>= 1
+	}
+	fmt.Println(length, k)
+	return k
+}

crypto/merkle/simple_tree_test.go

@@ -2,33 +2,28 @@ package merkle
 
 import (
 	"bytes"
+	"crypto/rand"
 
 	cmn "github.com/tendermint/tendermint/libs/common"
 	. "github.com/tendermint/tendermint/libs/test"
 
 	"testing"
-
-	"github.com/tendermint/tendermint/crypto/tmhash"
 )
 
-type testItem []byte
-
-func (tI testItem) Hash() []byte {
-	return []byte(tI)
-}
-
 func TestSimpleProof(t *testing.T) {
 
 	total := 100
+	sliceSize := 100
 
-	items := make([]Hasher, total)
+	items := make([][]byte, total)
 	for i := 0; i < total; i++ {
-		items[i] = testItem(cmn.RandBytes(tmhash.Size))
+		items[i] = make([]byte, sliceSize)
+		rand.Read(items[i])
 	}
 
-	rootHash := SimpleHashFromHashers(items)
+	rootHash := SimpleHashFromByteSlices(items)
 
-	rootHash2, proofs := SimpleProofsFromHashers(items)
+	rootHash2, proofs := SimpleProofsFromByteSlices(items)
 
 	if !bytes.Equal(rootHash, rootHash2) {
 		t.Errorf("Unmatched root hashes: %X vs %X", rootHash, rootHash2)
@@ -36,18 +31,17 @@ func TestSimpleProof(t *testing.T) {
 
 	// For each item, check the trail.
 	for i, item := range items {
-		itemHash := item.Hash()
 		proof := proofs[i]
 
 		// Verify success
-		ok := proof.Verify(i, total, itemHash, rootHash)
+		ok := proof.Verify(i, total, item, rootHash)
 		if !ok {
 			t.Errorf("Verification failed for index %v.", i)
 		}
 
 		// Wrong item index should make it fail
 		{
-			ok = proof.Verify((i+1)%total, total, itemHash, rootHash)
+			ok = proof.Verify((i+1)%total, total, item, rootHash)
 			if ok {
 				t.Errorf("Expected verification to fail for wrong index %v.", i)
 			}
@@ -57,7 +51,7 @@ func TestSimpleProof(t *testing.T) {
 		origAunts := proof.Aunts
 		proof.Aunts = append(proof.Aunts, cmn.RandBytes(32))
 		{
-			ok = proof.Verify(i, total, itemHash, rootHash)
+			ok = proof.Verify(i, total, item, rootHash)
 			if ok {
 				t.Errorf("Expected verification to fail for wrong trail length.")
 			}
@@ -67,7 +61,7 @@ func TestSimpleProof(t *testing.T) {
 		// Trail too short should make it fail
 		proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1]
 		{
-			ok = proof.Verify(i, total, itemHash, rootHash)
+			ok = proof.Verify(i, total, item, rootHash)
 			if ok {
 				t.Errorf("Expected verification to fail for wrong trail length.")
 			}
@@ -75,13 +69,13 @@ func TestSimpleProof(t *testing.T) {
 		proof.Aunts = origAunts
 
 		// Mutating the itemHash should make it fail.
-		ok = proof.Verify(i, total, MutateByteSlice(itemHash), rootHash)
+		ok = proof.Verify(i, total, MutateByteSlice(item), rootHash)
 		if ok {
 			t.Errorf("Expected verification to fail for mutated leaf hash")
 		}
 
 		// Mutating the rootHash should make it fail.
-		ok = proof.Verify(i, total, itemHash, MutateByteSlice(rootHash))
+		ok = proof.Verify(i, total, item, MutateByteSlice(rootHash))
 		if ok {
 			t.Errorf("Expected verification to fail for mutated root hash")
 		}

crypto/merkle/types.go

@@ -6,6 +6,11 @@ import (
 	amino "github.com/tendermint/go-amino"
 )
 
+var (
+	LeafHashPrefix  = []byte{0}
+	InnerHashPrefix = []byte{1}
+)
+
 // Tree is a Merkle tree interface.
 type Tree interface {
 	Size() (size int)