Unified deterministic merkle tree computation. Now we have 2 types of

trees... IAVL (nondeterministic) and Simple (deterministic)

merkle/simple_tree.go

@@ -0,0 +1,173 @@
+/*
+Computes a deterministic minimal height merkle tree hash.
+If the number of items is not a power of two, some leaves
+will be at different levels, leaning left.
+
+Use this for shorter deterministic trees, such as the validator list.
+For larger datasets, use IAVLTree.
+
+				*
+			   / \
+			 /     \
+		   /         \
+		 /             \
+		*               *
+	   / \             / \
+	  /   \           /   \
+	 /     \         /     \
+	*       *       *       h6
+   / \     / \     / \
+  h0  h1  h2  h3  h4  h5
+
+*/
+
+package merkle
+
+import (
+	"bytes"
+	"crypto/sha256"
+
+	. "github.com/tendermint/tendermint/binary"
+)
+
+func HashFromTwoHashes(left []byte, right []byte) []byte {
+	var n int64
+	var err error
+	var hasher = sha256.New()
+	WriteByteSlice(hasher, left, &n, &err)
+	WriteByteSlice(hasher, right, &n, &err)
+	if err != nil {
+		panic(err)
+	}
+	return hasher.Sum(nil)
+}
+
+func HashFromHashes(hashes [][]byte) []byte {
+	// Recursive impl.
+	switch len(hashes) {
+	case 0:
+		panic("Cannot call HashFromHashes() with 0 length arg")
+	case 1:
+		return hashes[0]
+	default:
+		left := HashFromHashes(hashes[:(len(hashes)+1)/2])
+		right := HashFromHashes(hashes[(len(hashes)+1)/2:])
+		return HashFromTwoHashes(left, right)
+	}
+}
+
+// Convenience for HashFromHashes.
+func HashFromBinaries(items []Binary) []byte {
+	hashes := [][]byte{}
+	for _, item := range items {
+		hasher := sha256.New()
+		_, err := item.WriteTo(hasher)
+		if err != nil {
+			panic(err)
+		}
+		hash := hasher.Sum(nil)
+		hashes = append(hashes, hash)
+	}
+	return HashFromHashes(hashes)
+}
+
+// Convenience for HashFromHashes.
+func HashFromHashables(items []Hashable) []byte {
+	hashes := [][]byte{}
+	for _, item := range items {
+		hash := item.Hash()
+		hashes = append(hashes, hash)
+	}
+	return HashFromHashes(hashes)
+}
+
+type HashTrail struct {
+	Hash   []byte
+	Parent *HashTrail
+	Left   *HashTrail
+	Right  *HashTrail
+}
+
+func (ht *HashTrail) Flatten() [][]byte {
+	// Nonrecursive impl.
+	trail := [][]byte{}
+	for ht != nil {
+		if ht.Left != nil {
+			trail = append(trail, ht.Left.Hash)
+		} else if ht.Right != nil {
+			trail = append(trail, ht.Right.Hash)
+		} else {
+			break
+		}
+		ht = ht.Parent
+	}
+	return trail
+}
+
+// returned trails[0].Hash is the leaf hash.
+// trails[0].Parent.Hash is the hash above that, etc.
+func HashTrailsFromHashables(items []Hashable) (trails []*HashTrail, root *HashTrail) {
+	// Recursive impl.
+	switch len(items) {
+	case 0:
+		panic("Cannot call HashTrailsFromHashables() with 0 length arg")
+	case 1:
+		trail := &HashTrail{items[0].Hash(), nil, nil, nil}
+		return []*HashTrail{trail}, trail
+	default:
+		lefts, leftRoot := HashTrailsFromHashables(items[:(len(items)+1)/2])
+		rights, rightRoot := HashTrailsFromHashables(items[(len(items)+1)/2:])
+		rootHash := HashFromTwoHashes(leftRoot.Hash, rightRoot.Hash)
+		root := &HashTrail{rootHash, nil, nil, nil}
+		leftRoot.Parent = root
+		leftRoot.Right = rightRoot
+		rightRoot.Parent = root
+		rightRoot.Left = leftRoot
+		return append(lefts, rights...), root
+	}
+}
+
+// Ensures that leafHash is part of rootHash.
+func VerifyHashTrail(index uint, total uint, leafHash []byte, trail [][]byte, rootHash []byte) bool {
+	computedRoot := ComputeRootFromTrail(index, total, leafHash, trail)
+	if computedRoot == nil {
+		return false
+	}
+	return bytes.Equal(computedRoot, rootHash)
+}
+
+// Use the leafHash and trail to get the root merkle hash.
+// If the length of the trail slice isn't exactly correct, the result is nil.
+func ComputeRootFromTrail(index uint, total uint, leafHash []byte, trail [][]byte) []byte {
+	// Recursive impl.
+	if index >= total {
+		return nil
+	}
+	switch total {
+	case 0:
+		panic("Cannot call ComputeRootFromTrail() with 0 total")
+	case 1:
+		if len(trail) != 0 {
+			return nil
+		}
+		return leafHash
+	default:
+		if len(trail) == 0 {
+			return nil
+		}
+		numLeft := (total + 1) / 2
+		if index < numLeft {
+			leftRoot := ComputeRootFromTrail(index, numLeft, leafHash, trail[:len(trail)-1])
+			if leftRoot == nil {
+				return nil
+			}
+			return HashFromTwoHashes(leftRoot, trail[len(trail)-1])
+		} else {
+			rightRoot := ComputeRootFromTrail(index-numLeft, total-numLeft, leafHash, trail[:len(trail)-1])
+			if rightRoot == nil {
+				return nil
+			}
+			return HashFromTwoHashes(trail[len(trail)-1], rightRoot)
+		}
+	}
+}

merkle/simple_tree_test.go

@@ -0,0 +1,74 @@
+package merkle
+
+import (
+	. "github.com/tendermint/tendermint/common"
+
+	"bytes"
+	"testing"
+)
+
+type testItem []byte
+
+func (tI testItem) Hash() []byte {
+	return []byte(tI)
+}
+
+func TestMerkleTrails(t *testing.T) {
+
+	numItems := uint(100)
+
+	items := make([]Hashable, numItems)
+	for i := uint(0); i < numItems; i++ {
+		items[i] = testItem(RandBytes(32))
+	}
+
+	root := HashFromHashables(items)
+
+	trails, rootTrail := HashTrailsFromHashables(items)
+
+	// Assert that HashFromHashables and HashTrailsFromHashables are compatible.
+	if !bytes.Equal(root, rootTrail.Hash) {
+		t.Errorf("Root mismatch:\n%X vs\n%X", root, rootTrail.Hash)
+	}
+
+	// For each item, check the trail.
+	for i, item := range items {
+		itemHash := item.Hash()
+		flatTrail := trails[i].Flatten()
+
+		// Verify success
+		ok := VerifyHashTrail(uint(i), numItems, itemHash, flatTrail, root)
+		if !ok {
+			t.Errorf("Verification failed for index %v.", i)
+		}
+
+		// Wrong item index should make it fail
+		ok = VerifyHashTrail(uint(i)+1, numItems, itemHash, flatTrail, root)
+		if ok {
+			t.Errorf("Expected verification to fail for wrong index %v.", i)
+		}
+
+		// Trail too long should make it fail
+		trail2 := append(flatTrail, RandBytes(32))
+		ok = VerifyHashTrail(uint(i), numItems, itemHash, trail2, root)
+		if ok {
+			t.Errorf("Expected verification to fail for wrong trail length.")
+		}
+
+		// Trail too short should make it fail
+		trail2 = flatTrail[:len(flatTrail)-1]
+		ok = VerifyHashTrail(uint(i), numItems, itemHash, trail2, root)
+		if ok {
+			t.Errorf("Expected verification to fail for wrong trail length.")
+		}
+
+		// Mutating the itemHash should make it fail.
+		itemHash2 := make([]byte, len(itemHash))
+		copy(itemHash2, itemHash)
+		itemHash2[0] += byte(0x01)
+		ok = VerifyHashTrail(uint(i), numItems, itemHash2, flatTrail, root)
+		if ok {
+			t.Errorf("Expected verification to fail for mutated leaf hash")
+		}
+	}
+}

merkle/util.go

@@ -1,230 +1,10 @@
 package merkle
 
 import (
-	"bytes"
-	"crypto/sha256"
 	"fmt"
-
-	. "github.com/tendermint/tendermint/binary"
 )
 
-func HashFromTwoHashes(left []byte, right []byte) []byte {
+// Prints the in-memory children recursively.
-	var n int64
-	var err error
-	var hasher = sha256.New()
-	WriteByteSlice(hasher, left, &n, &err)
-	WriteByteSlice(hasher, right, &n, &err)
-	if err != nil {
-		panic(err)
-	}
-	return hasher.Sum(nil)
-}
-
-/*
-Computes a deterministic minimal height merkle tree hash.
-If the number of items is not a power of two, some leaves
-will be at different levels.
-
-				*
-			   / \
-			 /     \
-		   /         \
-		 /             \
-		*               *
-	   / \             / \
-	  /   \           /   \
-	 /     \         /     \
-	*       h2      *       *
-   / \             / \     / \
-  h0  h1          h3  h4  h5  h6
-
-*/
-func HashFromHashes(hashes [][]byte) []byte {
-	switch len(hashes) {
-	case 0:
-		return nil
-	case 1:
-		return hashes[0]
-	default:
-		left := HashFromHashes(hashes[:len(hashes)/2])
-		right := HashFromHashes(hashes[len(hashes)/2:])
-		return HashFromTwoHashes(left, right)
-	}
-}
-
-// Convenience for HashFromHashes.
-func HashFromBinaries(items []Binary) []byte {
-	hashes := [][]byte{}
-	for _, item := range items {
-		hasher := sha256.New()
-		_, err := item.WriteTo(hasher)
-		if err != nil {
-			panic(err)
-		}
-		hash := hasher.Sum(nil)
-		hashes = append(hashes, hash)
-	}
-	return HashFromHashes(hashes)
-}
-
-// Convenience for HashFromHashes.
-func HashFromHashables(items []Hashable) []byte {
-	hashes := [][]byte{}
-	for _, item := range items {
-		hash := item.Hash()
-		hashes = append(hashes, hash)
-	}
-	return HashFromHashes(hashes)
-}
-
-/*
-Calculates an array of hashes, useful for deriving hash trails.
-
-				7
-			   / \
-			 /     \
-		   /         \
-		 /             \
-		3                11
-	   / \              / \
-	  /   \            /   \
-	 /     \          /     \
-	1       5        9       13
-   / \     / \      / \      / \
-  0   2   4   6    8   10  12   14
-  h0  h1  h2  h3   h4  h5  h6   h7
-
-(diagram and idea borrowed from libswift)
-
-The hashes provided get assigned to even indices.
-The derived merkle hashes get assigned to odd indices.
-If "hashes" is not of length power of 2, it is padded
-with blank (zeroed) hashes.
-*/
-func HashTreeFromHashes(hashes [][]byte) [][]byte {
-
-	// Make length of "hashes" a power of 2
-	hashesLen := uint32(len(hashes))
-	fullLen := uint32(1)
-	for {
-		if fullLen >= hashesLen {
-			break
-		} else {
-			fullLen <<= 1
-		}
-	}
-	blank := make([]byte, len(hashes[0]))
-	for i := hashesLen; i < fullLen; i++ {
-		hashes = append(hashes, blank)
-	}
-
-	// The result is twice the length minus one.
-	res := make([][]byte, len(hashes)*2-1)
-	for i, hash := range hashes {
-		res[i*2] = hash
-	}
-
-	// Fill all the hashes recursively.
-	fillTreeRoot(res, 0, len(res)-1)
-	return res
-}
-
-// Fill in the blanks.
-func fillTreeRoot(res [][]byte, start, end int) []byte {
-	if start == end {
-		return res[start]
-	} else {
-		mid := (start + end) / 2
-		left := fillTreeRoot(res, start, mid-1)
-		right := fillTreeRoot(res, mid+1, end)
-		root := HashFromTwoHashes(left, right)
-		res[mid] = root
-		return root
-	}
-}
-
-// Convenience for HashTreeFromHashes.
-func HashTreeFromHashables(items []Hashable) [][]byte {
-	hashes := [][]byte{}
-	for _, item := range items {
-		hash := item.Hash()
-		hashes = append(hashes, hash)
-	}
-	return HashTreeFromHashes(hashes)
-}
-
-/*
-Given the original index of an item,
-(e.g. for h5 in the diagram above, the index is 5, not 10)
-returns a trail of hashes, which along with the index can be
-used to calculate the merkle root.
-
-See VerifyHashTrailForIndex()
-*/
-func HashTrailForIndex(hashTree [][]byte, index int) [][]byte {
-	trail := [][]byte{}
-	index *= 2
-
-	// We start from the leaf layer and work our way up.
-	// Notice the indices in the diagram:
-	// 0  2  4 ... offset 0, stride 2
-	// 1  5  9 ... offset 1, stride 4
-	// 3 11 19 ... offset 3, stride 8
-	// 7 23 39 ... offset 7, stride 16 etc.
-
-	offset := 0
-	stride := 2
-
-	for {
-		// Calculate sibling of index.
-		var next int
-		if ((index-offset)/stride)%2 == 0 {
-			next = index + stride
-		} else {
-			next = index - stride
-		}
-		if next >= len(hashTree) {
-			break
-		}
-		// Insert sibling hash to trail.
-		trail = append(trail, hashTree[next])
-
-		index = (index + next) / 2
-		offset += stride / 2
-		stride *= 2
-	}
-
-	return trail
-}
-
-// Ensures that leafHash is part of rootHash.
-func VerifyHashTrailForIndex(index int, leafHash []byte, trail [][]byte, rootHash []byte) bool {
-	index *= 2
-	offset := 0
-	stride := 2
-
-	tempHash := make([]byte, len(leafHash))
-	copy(tempHash, leafHash)
-
-	for i := 0; i < len(trail); i++ {
-		var next int
-		if ((index-offset)/stride)%2 == 0 {
-			next = index + stride
-			tempHash = HashFromTwoHashes(tempHash, trail[i])
-		} else {
-			next = index - stride
-			tempHash = HashFromTwoHashes(trail[i], tempHash)
-		}
-		index = (index + next) / 2
-		offset += stride / 2
-		stride *= 2
-	}
-
-	return bytes.Equal(rootHash, tempHash)
-}
-
-//-----------------------------------------------------------------------------
-
 func PrintIAVLNode(node *IAVLNode) {
 	fmt.Println("==== NODE")
 	if node != nil {

merkle/util_test.go

@@ -1,35 +0,0 @@
-package merkle
-
-import (
-	. "github.com/tendermint/tendermint/common"
-
-	"testing"
-)
-
-// TODO: Actually test. All this does is help debug.
-// consensus/part_set tests some of this functionality.
-func TestHashTreeMerkleTrail(t *testing.T) {
-
-	numHashes := 5
-
-	// Make some fake "hashes".
-	hashes := make([][]byte, numHashes)
-	for i := 0; i < numHashes; i++ {
-		hashes[i] = RandBytes(32)
-		t.Logf("hash %v\t%X\n", i, hashes[i])
-	}
-
-	hashTree := HashTreeFromHashes(hashes)
-	for i := 0; i < len(hashTree); i++ {
-		t.Logf("tree %v\t%X\n", i, hashTree[i])
-	}
-
-	for i := 0; i < numHashes; i++ {
-		t.Logf("trail %v\n", i)
-		trail := HashTrailForIndex(hashTree, i)
-		for j := 0; j < len(trail); j++ {
-			t.Logf("index: %v, hash: %X\n", j, trail[j])
-		}
-	}
-
-}