From ad85c28653cc9e7e7c46ec7390fea47b0fac3947 Mon Sep 17 00:00:00 2001 From: tau3 Date: Sun, 16 Feb 2020 02:22:54 +0400 Subject: [PATCH] use highway hash for addrbook internal hashing --- go.mod | 3 +- go.sum | 14 +++---- p2p/pex/addrbook.go | 90 +++++++++++++++++++++++++++++---------------- 3 files changed, 66 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index b52f7ca4b..0da5c8831 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/libp2p/go-buffer-pool v0.0.2 github.com/magiconair/properties v1.8.1 + github.com/minio/highwayhash v1.0.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v0.9.3 github.com/rcrowley/go-metrics v0.0.0-20180503174638-e2704e165165 @@ -31,6 +32,6 @@ require ( github.com/tendermint/tm-db v0.4.0 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 golang.org/x/net v0.0.0-20190628185345-da137c7871d7 - golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect + golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4 // indirect google.golang.org/grpc v1.27.1 ) diff --git a/go.sum b/go.sum index 19cfd4cb8..e6f6d994c 100644 --- a/go.sum +++ b/go.sum @@ -137,6 +137,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643 h1:hLDRPB66XQT/8+wG9WsDpiCvZf1yKO7sz7scAjSlBa0= github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= +github.com/minio/highwayhash v1.0.0 h1:iMSDhgUILCr0TNm8LWlSjF8N0ZIj2qbO8WHp6Q/J2BA= +github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -151,8 +153,6 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.0 h1:J8lpUdobwIeCI7OiSxHqEwJUKvJwicL5+3v1oe2Yb4k= -github.com/pkg/errors v0.9.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -199,8 +199,6 @@ github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= -github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -257,17 +255,19 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6Zh golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4 h1:sfkvUWPNGwSV+8/fNqctR5lS2AqCSqYwXdrjCxp/dXo= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -288,8 +288,6 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index 4efb4d1e0..8f82331a1 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -5,20 +5,19 @@ package pex import ( - "crypto/sha256" "encoding/binary" "fmt" - "math" - "math/rand" - "net" - "sync" - "time" - + "github.com/minio/highwayhash" "github.com/tendermint/tendermint/crypto" tmmath "github.com/tendermint/tendermint/libs/math" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/libs/service" "github.com/tendermint/tendermint/p2p" + "math" + "math/rand" + "net" + "sync" + "time" ) const ( @@ -628,7 +627,10 @@ func (a *addrBook) addAddress(addr, src *p2p.NetAddress) error { ka = newKnownAddress(addr, src) } - bucket := a.calcNewBucket(addr, src) + bucket, err := a.calcNewBucket(addr, src) + if err != nil { + return err + } a.addToNewBucket(ka, bucket) return nil } @@ -691,15 +693,15 @@ func (a *addrBook) expireNew(bucketIdx int) { // Promotes an address from new to old. If the destination bucket is full, // demote the oldest one to a "new" bucket. // TODO: Demote more probabilistically? -func (a *addrBook) moveToOld(ka *knownAddress) { +func (a *addrBook) moveToOld(ka *knownAddress) error { // Sanity check if ka.isOld() { a.Logger.Error(fmt.Sprintf("Cannot promote address that is already old %v", ka)) - return + return nil } if len(ka.Buckets) == 0 { a.Logger.Error(fmt.Sprintf("Cannot promote address that isn't in any new buckets %v", ka)) - return + return nil } // Remove from all (new) buckets. @@ -708,13 +710,19 @@ func (a *addrBook) moveToOld(ka *knownAddress) { ka.BucketType = bucketTypeOld // Try to add it to its oldBucket destination. - oldBucketIdx := a.calcOldBucket(ka.Addr) + oldBucketIdx, err := a.calcOldBucket(ka.Addr) + if err != nil { + return err + } added := a.addToOldBucket(ka, oldBucketIdx) if !added { // No room; move the oldest to a new bucket oldest := a.pickOldest(bucketTypeOld, oldBucketIdx) a.removeFromBucket(oldest, bucketTypeOld, oldBucketIdx) - newBucketIdx := a.calcNewBucket(oldest.Addr, oldest.Src) + newBucketIdx, err := a.calcNewBucket(oldest.Addr, oldest.Src) + if err != nil { + return err + } a.addToNewBucket(oldest, newBucketIdx) // Finally, add our ka to old bucket again. @@ -723,19 +731,23 @@ func (a *addrBook) moveToOld(ka *knownAddress) { a.Logger.Error(fmt.Sprintf("Could not re-add ka %v to oldBucketIdx %v", ka, oldBucketIdx)) } } + return nil } //--------------------------------------------------------------------- // calculate bucket placements -// doublesha256( key + sourcegroup + +// hash( key + sourcegroup + // int64(doublesha256(key + group + sourcegroup))%bucket_per_group ) % num_new_buckets -func (a *addrBook) calcNewBucket(addr, src *p2p.NetAddress) int { +func (a *addrBook) calcNewBucket(addr, src *p2p.NetAddress) (int, error) { data1 := []byte{} data1 = append(data1, []byte(a.key)...) data1 = append(data1, []byte(a.groupKey(addr))...) data1 = append(data1, []byte(a.groupKey(src))...) - hash1 := doubleSha256(data1) + hash1, err := hash(data1) + if err != nil { + return 0, err + } hash64 := binary.BigEndian.Uint64(hash1) hash64 %= newBucketsPerGroup var hashbuf [8]byte @@ -745,17 +757,24 @@ func (a *addrBook) calcNewBucket(addr, src *p2p.NetAddress) int { data2 = append(data2, a.groupKey(src)...) data2 = append(data2, hashbuf[:]...) - hash2 := doubleSha256(data2) - return int(binary.BigEndian.Uint64(hash2) % newBucketCount) + hash2, err := hash(data2) + if err != nil { + return 0, err + } + result := int(binary.BigEndian.Uint64(hash2) % newBucketCount) + return result, nil } -// doublesha256( key + group + -// int64(doublesha256(key + addr))%buckets_per_group ) % num_old_buckets -func (a *addrBook) calcOldBucket(addr *p2p.NetAddress) int { +// hash( key + group + +// hash(doublesha256(key + addr))%buckets_per_group ) % num_old_buckets +func (a *addrBook) calcOldBucket(addr *p2p.NetAddress) (int, error) { data1 := []byte{} data1 = append(data1, []byte(a.key)...) data1 = append(data1, []byte(addr.String())...) - hash1 := doubleSha256(data1) + hash1, err := hash(data1) + if err != nil { + return 0, err + } hash64 := binary.BigEndian.Uint64(hash1) hash64 %= oldBucketsPerGroup var hashbuf [8]byte @@ -765,8 +784,12 @@ func (a *addrBook) calcOldBucket(addr *p2p.NetAddress) int { data2 = append(data2, a.groupKey(addr)...) data2 = append(data2, hashbuf[:]...) - hash2 := doubleSha256(data2) - return int(binary.BigEndian.Uint64(hash2) % oldBucketCount) + hash2, err := hash(data2) + if err != nil { + return 0, err + } + result := int(binary.BigEndian.Uint64(hash2) % oldBucketCount) + return result, nil } // Return a string representing the network group of this address. @@ -818,12 +841,15 @@ func (a *addrBook) groupKey(na *p2p.NetAddress) string { return (&net.IPNet{IP: na.IP, Mask: net.CIDRMask(bits, 128)}).String() } -// doubleSha256 calculates sha256(sha256(b)) and returns the resulting bytes. -func doubleSha256(b []byte) []byte { - hasher := sha256.New() - hasher.Write(b) // nolint:errcheck - sum := hasher.Sum(nil) - hasher.Reset() - hasher.Write(sum) // nolint:errcheck - return hasher.Sum(nil) +// hash function calculates hash and returns the resulting bytes. +func hash(b []byte) ([]byte, error) { + highwayKeyLength := 32 + key := make([]byte, highwayKeyLength) + rand.Read(key) + hasher, err := highwayhash.New64(key) + if err != nil { + return nil, err + } + hasher.Write(b) + return hasher.Sum(nil), nil }