commit 16372365c47cc9387d40325ea51ecef63e949c75 Author: Jae Kwon Date: Wed Oct 21 12:15:19 2015 -0700 First commit diff --git a/array.go b/array.go new file mode 100644 index 000000000..adedc42be --- /dev/null +++ b/array.go @@ -0,0 +1,5 @@ +package common + +func Arr(items ...interface{}) []interface{} { + return items +} diff --git a/async.go b/async.go new file mode 100644 index 000000000..1d302c344 --- /dev/null +++ b/async.go @@ -0,0 +1,15 @@ +package common + +import "sync" + +func Parallel(tasks ...func()) { + var wg sync.WaitGroup + wg.Add(len(tasks)) + for _, task := range tasks { + go func(task func()) { + task() + wg.Done() + }(task) + } + wg.Wait() +} diff --git a/bit_array.go b/bit_array.go new file mode 100644 index 000000000..dc006f0eb --- /dev/null +++ b/bit_array.go @@ -0,0 +1,275 @@ +package common + +import ( + "fmt" + "math/rand" + "strings" + "sync" +) + +type BitArray struct { + mtx sync.Mutex + Bits int `json:"bits"` // NOTE: persisted via reflect, must be exported + Elems []uint64 `json:"elems"` // NOTE: persisted via reflect, must be exported +} + +// There is no BitArray whose Size is 0. Use nil instead. +func NewBitArray(bits int) *BitArray { + if bits == 0 { + return nil + } + return &BitArray{ + Bits: bits, + Elems: make([]uint64, (bits+63)/64), + } +} + +func (bA *BitArray) Size() int { + if bA == nil { + return 0 + } + return bA.Bits +} + +// NOTE: behavior is undefined if i >= bA.Bits +func (bA *BitArray) GetIndex(i int) bool { + if bA == nil { + return false + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.getIndex(i) +} + +func (bA *BitArray) getIndex(i int) bool { + if i >= bA.Bits { + return false + } + return bA.Elems[i/64]&(uint64(1)< 0 +} + +// NOTE: behavior is undefined if i >= bA.Bits +func (bA *BitArray) SetIndex(i int, v bool) bool { + if bA == nil { + return false + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.setIndex(i, v) +} + +func (bA *BitArray) setIndex(i int, v bool) bool { + if i >= bA.Bits { + return false + } + if v { + bA.Elems[i/64] |= (uint64(1) << uint(i%64)) + } else { + bA.Elems[i/64] &= ^(uint64(1) << uint(i%64)) + } + return true +} + +func (bA *BitArray) Copy() *BitArray { + if bA == nil { + return nil + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.copy() +} + +func (bA *BitArray) copy() *BitArray { + c := make([]uint64, len(bA.Elems)) + copy(c, bA.Elems) + return &BitArray{ + Bits: bA.Bits, + Elems: c, + } +} + +func (bA *BitArray) copyBits(bits int) *BitArray { + c := make([]uint64, (bits+63)/64) + copy(c, bA.Elems) + return &BitArray{ + Bits: bits, + Elems: c, + } +} + +// Returns a BitArray of larger bits size. +func (bA *BitArray) Or(o *BitArray) *BitArray { + if bA == nil { + o.Copy() + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + c := bA.copyBits(MaxInt(bA.Bits, o.Bits)) + for i := 0; i < len(c.Elems); i++ { + c.Elems[i] |= o.Elems[i] + } + return c +} + +// Returns a BitArray of smaller bit size. +func (bA *BitArray) And(o *BitArray) *BitArray { + if bA == nil { + return nil + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.and(o) +} + +func (bA *BitArray) and(o *BitArray) *BitArray { + c := bA.copyBits(MinInt(bA.Bits, o.Bits)) + for i := 0; i < len(c.Elems); i++ { + c.Elems[i] &= o.Elems[i] + } + return c +} + +func (bA *BitArray) Not() *BitArray { + if bA == nil { + return nil // Degenerate + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + c := bA.copy() + for i := 0; i < len(c.Elems); i++ { + c.Elems[i] = ^c.Elems[i] + } + return c +} + +func (bA *BitArray) Sub(o *BitArray) *BitArray { + if bA == nil { + return nil + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + if bA.Bits > o.Bits { + c := bA.copy() + for i := 0; i < len(o.Elems)-1; i++ { + c.Elems[i] &= ^c.Elems[i] + } + i := len(o.Elems) - 1 + if i >= 0 { + for idx := i * 64; idx < o.Bits; idx++ { + // NOTE: each individual GetIndex() call to o is safe. + c.setIndex(idx, c.getIndex(idx) && !o.GetIndex(idx)) + } + } + return c + } else { + return bA.and(o.Not()) // Note degenerate case where o == nil + } +} + +func (bA *BitArray) IsFull() bool { + if bA == nil { + return true + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + + // Check all elements except the last + for _, elem := range bA.Elems[:len(bA.Elems)-1] { + if (^elem) != 0 { + return false + } + } + + // Check that the last element has (lastElemBits) 1's + lastElemBits := (bA.Bits+63)%64 + 1 + lastElem := bA.Elems[len(bA.Elems)-1] + return (lastElem+1)&((uint64(1)< 0 { + randBitStart := rand.Intn(64) + for j := 0; j < 64; j++ { + bitIdx := ((j + randBitStart) % 64) + if (bA.Elems[elemIdx] & (uint64(1) << uint(bitIdx))) > 0 { + return 64*elemIdx + bitIdx, true + } + } + PanicSanity("should not happen") + } + } else { + // Special case for last elem, to ignore straggler bits + elemBits := bA.Bits % 64 + if elemBits == 0 { + elemBits = 64 + } + randBitStart := rand.Intn(elemBits) + for j := 0; j < elemBits; j++ { + bitIdx := ((j + randBitStart) % elemBits) + if (bA.Elems[elemIdx] & (uint64(1) << uint(bitIdx))) > 0 { + return 64*elemIdx + bitIdx, true + } + } + } + } + return 0, false +} + +func (bA *BitArray) String() string { + if bA == nil { + return "nil-BitArray" + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.stringIndented("") +} + +func (bA *BitArray) StringIndented(indent string) string { + if bA == nil { + return "nil-BitArray" + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.stringIndented(indent) +} + +func (bA *BitArray) stringIndented(indent string) string { + + lines := []string{} + bits := "" + for i := 0; i < bA.Bits; i++ { + if bA.getIndex(i) { + bits += "X" + } else { + bits += "_" + } + if i%100 == 99 { + lines = append(lines, bits) + bits = "" + } + if i%10 == 9 { + bits += " " + } + if i%50 == 49 { + bits += " " + } + } + if len(bits) > 0 { + lines = append(lines, bits) + } + return fmt.Sprintf("BA{%v:%v}", bA.Bits, strings.Join(lines, indent)) +} diff --git a/bit_array_test.go b/bit_array_test.go new file mode 100644 index 000000000..93274aab0 --- /dev/null +++ b/bit_array_test.go @@ -0,0 +1,120 @@ +package common + +import ( + "testing" +) + +func randBitArray(bits int) (*BitArray, []byte) { + src := RandBytes((bits + 7) / 8) + bA := NewBitArray(bits) + for i := 0; i < len(src); i++ { + for j := 0; j < 8; j++ { + if i*8+j >= bits { + return bA, src + } + setBit := src[i]&(1< 0 + bA.SetIndex(i*8+j, setBit) + } + } + return bA, src +} + +func TestAnd(t *testing.T) { + + bA1, _ := randBitArray(51) + bA2, _ := randBitArray(31) + bA3 := bA1.And(bA2) + + if bA3.Bits != 31 { + t.Error("Expected min bits", bA3.Bits) + } + if len(bA3.Elems) != len(bA2.Elems) { + t.Error("Expected min elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) && bA2.GetIndex(i) + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i)) + } + } +} + +func TestOr(t *testing.T) { + + bA1, _ := randBitArray(51) + bA2, _ := randBitArray(31) + bA3 := bA1.Or(bA2) + + if bA3.Bits != 51 { + t.Error("Expected max bits") + } + if len(bA3.Elems) != len(bA1.Elems) { + t.Error("Expected max elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) || bA2.GetIndex(i) + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i)) + } + } +} + +func TestSub1(t *testing.T) { + + bA1, _ := randBitArray(31) + bA2, _ := randBitArray(51) + bA3 := bA1.Sub(bA2) + + if bA3.Bits != bA1.Bits { + t.Error("Expected bA1 bits") + } + if len(bA3.Elems) != len(bA1.Elems) { + t.Error("Expected bA1 elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) + if bA2.GetIndex(i) { + expected = false + } + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i)) + } + } +} + +func TestSub2(t *testing.T) { + + bA1, _ := randBitArray(51) + bA2, _ := randBitArray(31) + bA3 := bA1.Sub(bA2) + + if bA3.Bits != bA1.Bits { + t.Error("Expected bA1 bits") + } + if len(bA3.Elems) != len(bA1.Elems) { + t.Error("Expected bA1 elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) + if i < bA2.Bits && bA2.GetIndex(i) { + expected = false + } + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3") + } + } +} + +func TestPickRandom(t *testing.T) { + for idx := 0; idx < 123; idx++ { + bA1 := NewBitArray(123) + bA1.SetIndex(idx, true) + index, ok := bA1.PickRandom() + if !ok { + t.Fatal("Expected to pick element but got none") + } + if index != idx { + t.Fatalf("Expected to pick element at %v but got wrong index", idx) + } + } +} diff --git a/byteslice.go b/byteslice.go new file mode 100644 index 000000000..be828f065 --- /dev/null +++ b/byteslice.go @@ -0,0 +1,44 @@ +package common + +import ( + "bytes" +) + +func Fingerprint(slice []byte) []byte { + fingerprint := make([]byte, 6) + copy(fingerprint, slice) + return fingerprint +} + +func IsZeros(slice []byte) bool { + for _, byt := range slice { + if byt != byte(0) { + return false + } + } + return true +} + +func RightPadBytes(slice []byte, l int) []byte { + if l < len(slice) { + return slice + } + padded := make([]byte, l) + copy(padded[0:len(slice)], slice) + return padded +} + +func LeftPadBytes(slice []byte, l int) []byte { + if l < len(slice) { + return slice + } + padded := make([]byte, l) + copy(padded[l-len(slice):], slice) + return padded +} + +func TrimmedString(b []byte) string { + trimSet := string([]byte{0}) + return string(bytes.TrimLeft(b, trimSet)) + +} diff --git a/cmap.go b/cmap.go new file mode 100644 index 000000000..5de6fa2fa --- /dev/null +++ b/cmap.go @@ -0,0 +1,62 @@ +package common + +import "sync" + +// CMap is a goroutine-safe map +type CMap struct { + m map[string]interface{} + l sync.Mutex +} + +func NewCMap() *CMap { + return &CMap{ + m: make(map[string]interface{}, 0), + } +} + +func (cm *CMap) Set(key string, value interface{}) { + cm.l.Lock() + defer cm.l.Unlock() + cm.m[key] = value +} + +func (cm *CMap) Get(key string) interface{} { + cm.l.Lock() + defer cm.l.Unlock() + return cm.m[key] +} + +func (cm *CMap) Has(key string) bool { + cm.l.Lock() + defer cm.l.Unlock() + _, ok := cm.m[key] + return ok +} + +func (cm *CMap) Delete(key string) { + cm.l.Lock() + defer cm.l.Unlock() + delete(cm.m, key) +} + +func (cm *CMap) Size() int { + cm.l.Lock() + defer cm.l.Unlock() + return len(cm.m) +} + +func (cm *CMap) Clear() { + cm.l.Lock() + defer cm.l.Unlock() + cm.m = make(map[string]interface{}, 0) +} + +func (cm *CMap) Values() []interface{} { + cm.l.Lock() + defer cm.l.Unlock() + items := []interface{}{} + for _, v := range cm.m { + items = append(items, v) + } + return items +} diff --git a/colors.go b/colors.go new file mode 100644 index 000000000..776b22e2e --- /dev/null +++ b/colors.go @@ -0,0 +1,84 @@ +package common + +import ( + "fmt" + "strings" +) + +const ( + ANSIReset = "\x1b[0m" + ANSIBright = "\x1b[1m" + ANSIDim = "\x1b[2m" + ANSIUnderscore = "\x1b[4m" + ANSIBlink = "\x1b[5m" + ANSIReverse = "\x1b[7m" + ANSIHidden = "\x1b[8m" + + ANSIFgBlack = "\x1b[30m" + ANSIFgRed = "\x1b[31m" + ANSIFgGreen = "\x1b[32m" + ANSIFgYellow = "\x1b[33m" + ANSIFgBlue = "\x1b[34m" + ANSIFgMagenta = "\x1b[35m" + ANSIFgCyan = "\x1b[36m" + ANSIFgWhite = "\x1b[37m" + + ANSIBgBlack = "\x1b[40m" + ANSIBgRed = "\x1b[41m" + ANSIBgGreen = "\x1b[42m" + ANSIBgYellow = "\x1b[43m" + ANSIBgBlue = "\x1b[44m" + ANSIBgMagenta = "\x1b[45m" + ANSIBgCyan = "\x1b[46m" + ANSIBgWhite = "\x1b[47m" +) + +// color the string s with color 'color' +// unless s is already colored +func treat(s string, color string) string { + if len(s) > 2 && s[:2] == "\x1b[" { + return s + } else { + return color + s + ANSIReset + } +} + +func treatAll(color string, args ...interface{}) string { + var parts []string + for _, arg := range args { + parts = append(parts, treat(fmt.Sprintf("%v", arg), color)) + } + return strings.Join(parts, "") +} + +func Black(args ...interface{}) string { + return treatAll(ANSIFgBlack, args...) +} + +func Red(args ...interface{}) string { + return treatAll(ANSIFgRed, args...) +} + +func Green(args ...interface{}) string { + return treatAll(ANSIFgGreen, args...) +} + +func Yellow(args ...interface{}) string { + return treatAll(ANSIFgYellow, args...) +} + +func Blue(args ...interface{}) string { + return treatAll(ANSIFgBlue, args...) +} + +func Magenta(args ...interface{}) string { + return treatAll(ANSIFgMagenta, args...) +} + +func Cyan(args ...interface{}) string { + return treatAll(ANSIFgCyan, args...) +} + +func White(args ...interface{}) string { + return treatAll(ANSIFgWhite, args...) +} diff --git a/errors.go b/errors.go new file mode 100644 index 000000000..e168a75b7 --- /dev/null +++ b/errors.go @@ -0,0 +1,45 @@ +package common + +import ( + "fmt" +) + +type StackError struct { + Err interface{} + Stack []byte +} + +func (se StackError) String() string { + return fmt.Sprintf("Error: %v\nStack: %s", se.Err, se.Stack) +} + +func (se StackError) Error() string { + return se.String() +} + +//-------------------------------------------------------------------------------------------------- +// panic wrappers + +// A panic resulting from a sanity check means there is a programmer error +// and some gaurantee is not satisfied. +func PanicSanity(v interface{}) { + panic(Fmt("Paniced on a Sanity Check: %v", v)) +} + +// A panic here means something has gone horribly wrong, in the form of data corruption or +// failure of the operating system. In a correct/healthy system, these should never fire. +// If they do, it's indicative of a much more serious problem. +func PanicCrisis(v interface{}) { + panic(Fmt("Paniced on a Crisis: %v", v)) +} + +// Indicates a failure of consensus. Someone was malicious or something has +// gone horribly wrong. These should really boot us into an "emergency-recover" mode +func PanicConsensus(v interface{}) { + panic(Fmt("Paniced on a Consensus Failure: %v", v)) +} + +// For those times when we're not sure if we should panic +func PanicQ(v interface{}) { + panic(Fmt("Paniced questionably: %v", v)) +} diff --git a/heap.go b/heap.go new file mode 100644 index 000000000..4a96d7aaa --- /dev/null +++ b/heap.go @@ -0,0 +1,103 @@ +package common + +import ( + "container/heap" +) + +type Comparable interface { + Less(o interface{}) bool +} + +//----------------------------------------------------------------------------- + +/* +Example usage: + h := NewHeap() + + h.Push(String("msg1"), 1) + h.Push(String("msg3"), 3) + h.Push(String("msg2"), 2) + + fmt.Println(h.Pop()) + fmt.Println(h.Pop()) + fmt.Println(h.Pop()) +*/ + +type Heap struct { + pq priorityQueue +} + +func NewHeap() *Heap { + return &Heap{pq: make([]*pqItem, 0)} +} + +func (h *Heap) Len() int64 { + return int64(len(h.pq)) +} + +func (h *Heap) Push(value interface{}, priority Comparable) { + heap.Push(&h.pq, &pqItem{value: value, priority: priority}) +} + +func (h *Heap) Peek() interface{} { + if len(h.pq) == 0 { + return nil + } + return h.pq[0].value +} + +func (h *Heap) Update(value interface{}, priority Comparable) { + h.pq.Update(h.pq[0], value, priority) +} + +func (h *Heap) Pop() interface{} { + item := heap.Pop(&h.pq).(*pqItem) + return item.value +} + +//----------------------------------------------------------------------------- + +/////////////////////// +// From: http://golang.org/pkg/container/heap/#example__priorityQueue + +type pqItem struct { + value interface{} + priority Comparable + index int +} + +type priorityQueue []*pqItem + +func (pq priorityQueue) Len() int { return len(pq) } + +func (pq priorityQueue) Less(i, j int) bool { + return pq[i].priority.Less(pq[j].priority) +} + +func (pq priorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] + pq[i].index = i + pq[j].index = j +} + +func (pq *priorityQueue) Push(x interface{}) { + n := len(*pq) + item := x.(*pqItem) + item.index = n + *pq = append(*pq, item) +} + +func (pq *priorityQueue) Pop() interface{} { + old := *pq + n := len(old) + item := old[n-1] + item.index = -1 // for safety + *pq = old[0 : n-1] + return item +} + +func (pq *priorityQueue) Update(item *pqItem, value interface{}, priority Comparable) { + item.value = value + item.priority = priority + heap.Fix(pq, item.index) +} diff --git a/int.go b/int.go new file mode 100644 index 000000000..50e86a072 --- /dev/null +++ b/int.go @@ -0,0 +1,55 @@ +package common + +import ( + "encoding/binary" + "sort" +) + +// Sort for []uint64 + +type Uint64Slice []uint64 + +func (p Uint64Slice) Len() int { return len(p) } +func (p Uint64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Uint64Slice) Sort() { sort.Sort(p) } + +func SearchUint64s(a []uint64, x uint64) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +func (p Uint64Slice) Search(x uint64) int { return SearchUint64s(p, x) } + +//----------------------------------------------------------------------------- + +func PutUint64LE(dest []byte, i uint64) { + binary.LittleEndian.PutUint64(dest, i) +} + +func GetUint64LE(src []byte) uint64 { + return binary.LittleEndian.Uint64(src) +} + +func PutUint64BE(dest []byte, i uint64) { + binary.BigEndian.PutUint64(dest, i) +} + +func GetUint64BE(src []byte) uint64 { + return binary.BigEndian.Uint64(src) +} + +func PutInt64LE(dest []byte, i int64) { + binary.LittleEndian.PutUint64(dest, uint64(i)) +} + +func GetInt64LE(src []byte) int64 { + return int64(binary.LittleEndian.Uint64(src)) +} + +func PutInt64BE(dest []byte, i int64) { + binary.BigEndian.PutUint64(dest, uint64(i)) +} + +func GetInt64BE(src []byte) int64 { + return int64(binary.BigEndian.Uint64(src)) +} diff --git a/io.go b/io.go new file mode 100644 index 000000000..378c19fc6 --- /dev/null +++ b/io.go @@ -0,0 +1,75 @@ +package common + +import ( + "bytes" + "errors" + "io" +) + +type PrefixedReader struct { + Prefix []byte + reader io.Reader +} + +func NewPrefixedReader(prefix []byte, reader io.Reader) *PrefixedReader { + return &PrefixedReader{prefix, reader} +} + +func (pr *PrefixedReader) Read(p []byte) (n int, err error) { + if len(pr.Prefix) > 0 { + read := copy(p, pr.Prefix) + pr.Prefix = pr.Prefix[read:] + return read, nil + } else { + return pr.reader.Read(p) + } +} + +// NOTE: Not goroutine safe +type BufferCloser struct { + bytes.Buffer + Closed bool +} + +func NewBufferCloser(buf []byte) *BufferCloser { + return &BufferCloser{ + *bytes.NewBuffer(buf), + false, + } +} + +func (bc *BufferCloser) Close() error { + if bc.Closed { + return errors.New("BufferCloser already closed") + } + bc.Closed = true + return nil +} + +func (bc *BufferCloser) Write(p []byte) (n int, err error) { + if bc.Closed { + return 0, errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.Write(p) +} + +func (bc *BufferCloser) WriteByte(c byte) error { + if bc.Closed { + return errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.WriteByte(c) +} + +func (bc *BufferCloser) WriteRune(r rune) (n int, err error) { + if bc.Closed { + return 0, errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.WriteRune(r) +} + +func (bc *BufferCloser) WriteString(s string) (n int, err error) { + if bc.Closed { + return 0, errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.WriteString(s) +} diff --git a/math.go b/math.go new file mode 100644 index 000000000..b037d1a71 --- /dev/null +++ b/math.go @@ -0,0 +1,157 @@ +package common + +func MaxInt8(a, b int8) int8 { + if a > b { + return a + } + return b +} + +func MaxUint8(a, b uint8) uint8 { + if a > b { + return a + } + return b +} + +func MaxInt16(a, b int16) int16 { + if a > b { + return a + } + return b +} + +func MaxUint16(a, b uint16) uint16 { + if a > b { + return a + } + return b +} + +func MaxInt32(a, b int32) int32 { + if a > b { + return a + } + return b +} + +func MaxUint32(a, b uint32) uint32 { + if a > b { + return a + } + return b +} + +func MaxInt64(a, b int64) int64 { + if a > b { + return a + } + return b +} + +func MaxUint64(a, b uint64) uint64 { + if a > b { + return a + } + return b +} + +func MaxInt(a, b int) int { + if a > b { + return a + } + return b +} + +func MaxUint(a, b uint) uint { + if a > b { + return a + } + return b +} + +//----------------------------------------------------------------------------- + +func MinInt8(a, b int8) int8 { + if a < b { + return a + } + return b +} + +func MinUint8(a, b uint8) uint8 { + if a < b { + return a + } + return b +} + +func MinInt16(a, b int16) int16 { + if a < b { + return a + } + return b +} + +func MinUint16(a, b uint16) uint16 { + if a < b { + return a + } + return b +} + +func MinInt32(a, b int32) int32 { + if a < b { + return a + } + return b +} + +func MinUint32(a, b uint32) uint32 { + if a < b { + return a + } + return b +} + +func MinInt64(a, b int64) int64 { + if a < b { + return a + } + return b +} + +func MinUint64(a, b uint64) uint64 { + if a < b { + return a + } + return b +} + +func MinInt(a, b int) int { + if a < b { + return a + } + return b +} + +func MinUint(a, b uint) uint { + if a < b { + return a + } + return b +} + +//----------------------------------------------------------------------------- + +func ExpUint64(a, b uint64) uint64 { + accum := uint64(1) + for b > 0 { + if b&1 == 1 { + accum *= a + } + a *= a + b >>= 1 + } + return accum +} diff --git a/os.go b/os.go new file mode 100644 index 000000000..170c6f82a --- /dev/null +++ b/os.go @@ -0,0 +1,225 @@ +package common + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "os/signal" + "strings" + "sync" + "time" +) + +var ( + GoPath = os.Getenv("GOPATH") +) + +func TrapSignal(cb func()) { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + signal.Notify(c, os.Kill) + go func() { + for sig := range c { + fmt.Printf("captured %v, exiting...\n", sig) + if cb != nil { + cb() + } + os.Exit(1) + } + }() + select {} +} + +func Exit(s string) { + fmt.Printf(s + "\n") + os.Exit(1) +} + +func EnsureDir(dir string) error { + if _, err := os.Stat(dir); os.IsNotExist(err) { + err := os.MkdirAll(dir, 0700) + if err != nil { + return fmt.Errorf("Could not create directory %v. %v", dir, err) + } + } + return nil +} + +func FileExists(filePath string) bool { + _, err := os.Stat(filePath) + return !os.IsNotExist(err) +} + +func ReadFile(filePath string) ([]byte, error) { + return ioutil.ReadFile(filePath) +} + +func MustReadFile(filePath string) []byte { + fileBytes, err := ioutil.ReadFile(filePath) + if err != nil { + Exit(Fmt("MustReadFile failed: %v", err)) + return nil + } + return fileBytes +} + +func WriteFile(filePath string, contents []byte) error { + err := ioutil.WriteFile(filePath, contents, 0600) + if err != nil { + return err + } + // fmt.Printf("File written to %v.\n", filePath) + return nil +} + +func MustWriteFile(filePath string, contents []byte) { + err := WriteFile(filePath, contents) + if err != nil { + Exit(Fmt("MustWriteFile failed: %v", err)) + } +} + +// Writes to newBytes to filePath. +// Guaranteed not to lose *both* oldBytes and newBytes, +// (assuming that the OS is perfect) +func WriteFileAtomic(filePath string, newBytes []byte) error { + // If a file already exists there, copy to filePath+".bak" (overwrite anything) + if _, err := os.Stat(filePath); !os.IsNotExist(err) { + fileBytes, err := ioutil.ReadFile(filePath) + if err != nil { + return fmt.Errorf("Could not read file %v. %v", filePath, err) + } + err = ioutil.WriteFile(filePath+".bak", fileBytes, 0600) + if err != nil { + return fmt.Errorf("Could not write file %v. %v", filePath+".bak", err) + } + } + // Write newBytes to filePath.new + err := ioutil.WriteFile(filePath+".new", newBytes, 0600) + if err != nil { + return fmt.Errorf("Could not write file %v. %v", filePath+".new", err) + } + // Move filePath.new to filePath + err = os.Rename(filePath+".new", filePath) + return err +} + +//-------------------------------------------------------------------------------- + +/* AutoFile usage + +// Create/Append to ./autofile_test +af, err := OpenAutoFile("autofile_test") +if err != nil { + panic(err) +} + +// Stream of writes. +// During this time, the file may be moved e.g. by logRotate. +for i := 0; i < 60; i++ { + af.Write([]byte(Fmt("LOOP(%v)", i))) + time.Sleep(time.Second) +} + +// Close the AutoFile +err = af.Close() +if err != nil { + panic(err) +} +*/ + +const autoFileOpenDuration = 1000 * time.Millisecond + +// Automatically closes and re-opens file for writing. +// This is useful for using a log file with the logrotate tool. +type AutoFile struct { + Path string + ticker *time.Ticker + mtx sync.Mutex + file *os.File +} + +func OpenAutoFile(path string) (af *AutoFile, err error) { + af = &AutoFile{ + Path: path, + ticker: time.NewTicker(autoFileOpenDuration), + } + if err = af.openFile(); err != nil { + return + } + go af.processTicks() + return +} + +func (af *AutoFile) Close() error { + af.ticker.Stop() + af.mtx.Lock() + err := af.closeFile() + af.mtx.Unlock() + return err +} + +func (af *AutoFile) processTicks() { + for { + _, ok := <-af.ticker.C + if !ok { + return // Done. + } + af.mtx.Lock() + af.closeFile() + af.mtx.Unlock() + } +} + +func (af *AutoFile) closeFile() (err error) { + file := af.file + if file == nil { + return nil + } + af.file = nil + return file.Close() +} + +func (af *AutoFile) Write(b []byte) (n int, err error) { + af.mtx.Lock() + defer af.mtx.Unlock() + if af.file == nil { + if err = af.openFile(); err != nil { + return + } + } + return af.file.Write(b) +} + +func (af *AutoFile) openFile() error { + file, err := os.OpenFile(af.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return err + } + af.file = file + return nil +} + +func Tempfile(prefix string) (*os.File, string) { + file, err := ioutil.TempFile("", prefix) + if err != nil { + PanicCrisis(err) + } + return file, file.Name() +} + +func Prompt(prompt string, defaultValue string) (string, error) { + fmt.Print(prompt) + reader := bufio.NewReader(os.Stdin) + line, err := reader.ReadString('\n') + if err != nil { + return defaultValue, err + } else { + line = strings.TrimSpace(line) + if line == "" { + return defaultValue, nil + } + return line, nil + } +} diff --git a/random.go b/random.go new file mode 100644 index 000000000..645601154 --- /dev/null +++ b/random.go @@ -0,0 +1,145 @@ +package common + +import ( + crand "crypto/rand" + "encoding/hex" + "math/rand" + "time" +) + +const ( + strChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" // 62 characters +) + +func init() { + // Seed math/rand with "secure" int64 + b := CRandBytes(8) + var seed uint64 + for i := 0; i < 8; i++ { + seed |= uint64(b[i]) + seed <<= 8 + } + rand.Seed(int64(seed)) +} + +// Constructs an alphanumeric string of given length. +func RandStr(length int) string { + chars := []byte{} +MAIN_LOOP: + for { + val := rand.Int63() + for i := 0; i < 10; i++ { + v := int(val & 0x3f) // rightmost 6 bits + if v >= 62 { // only 62 characters in strChars + val >>= 6 + continue + } else { + chars = append(chars, strChars[v]) + if len(chars) == length { + break MAIN_LOOP + } + val >>= 6 + } + } + } + + return string(chars) +} + +func RandUint16() uint16 { + return uint16(rand.Uint32() & (1<<16 - 1)) +} + +func RandUint32() uint32 { + return rand.Uint32() +} + +func RandUint64() uint64 { + return uint64(rand.Uint32())<<32 + uint64(rand.Uint32()) +} + +func RandUint() uint { + return uint(rand.Int()) +} + +func RandInt16() int16 { + return int16(rand.Uint32() & (1<<16 - 1)) +} + +func RandInt32() int32 { + return int32(rand.Uint32()) +} + +func RandInt64() int64 { + return int64(rand.Uint32())<<32 + int64(rand.Uint32()) +} + +func RandInt() int { + return rand.Int() +} + +// Distributed pseudo-exponentially to test for various cases +func RandUint16Exp() uint16 { + bits := rand.Uint32() % 16 + if bits == 0 { + return 0 + } + n := uint16(1 << (bits - 1)) + n += uint16(rand.Int31()) & ((1 << (bits - 1)) - 1) + return n +} + +// Distributed pseudo-exponentially to test for various cases +func RandUint32Exp() uint32 { + bits := rand.Uint32() % 32 + if bits == 0 { + return 0 + } + n := uint32(1 << (bits - 1)) + n += uint32(rand.Int31()) & ((1 << (bits - 1)) - 1) + return n +} + +// Distributed pseudo-exponentially to test for various cases +func RandUint64Exp() uint64 { + bits := rand.Uint32() % 64 + if bits == 0 { + return 0 + } + n := uint64(1 << (bits - 1)) + n += uint64(rand.Int63()) & ((1 << (bits - 1)) - 1) + return n +} + +func RandFloat32() float32 { + return rand.Float32() +} + +func RandTime() time.Time { + return time.Unix(int64(RandUint64Exp()), 0) +} + +func RandBytes(n int) []byte { + bs := make([]byte, n) + for i := 0; i < n; i++ { + bs[i] = byte(rand.Intn(256)) + } + return bs +} + +//----------------------------------------------------------------------------- +// CRand* methods are crypto safe. + +func CRandBytes(numBytes int) []byte { + b := make([]byte, numBytes) + _, err := crand.Read(b) + if err != nil { + PanicCrisis(err) + } + return b +} + +// RandHex(24) gives 96 bits of randomness, strong enough for most purposes. +func CRandHex(numDigits int) string { + return hex.EncodeToString(CRandBytes(numDigits / 2)) +} diff --git a/repeat_timer.go b/repeat_timer.go new file mode 100644 index 000000000..e2aa18ea8 --- /dev/null +++ b/repeat_timer.go @@ -0,0 +1,72 @@ +package common + +import "time" +import "sync" + +/* +RepeatTimer repeatedly sends a struct{}{} to .Ch after each "dur" period. +It's good for keeping connections alive. +A RepeatTimer must be Stop()'d or it will keep a goroutine alive. +*/ +type RepeatTimer struct { + Ch chan time.Time + + mtx sync.Mutex + name string + ticker *time.Ticker + quit chan struct{} + dur time.Duration +} + +func NewRepeatTimer(name string, dur time.Duration) *RepeatTimer { + var t = &RepeatTimer{ + Ch: make(chan time.Time), + ticker: time.NewTicker(dur), + quit: make(chan struct{}), + name: name, + dur: dur, + } + go t.fireRoutine(t.ticker) + return t +} + +func (t *RepeatTimer) fireRoutine(ticker *time.Ticker) { + for { + select { + case t_ := <-ticker.C: + t.Ch <- t_ + case <-t.quit: + return + } + } +} + +// Wait the duration again before firing. +func (t *RepeatTimer) Reset() { + t.Stop() + + t.mtx.Lock() // Lock + defer t.mtx.Unlock() + + t.ticker = time.NewTicker(t.dur) + t.quit = make(chan struct{}) + go t.fireRoutine(t.ticker) +} + +// For ease of .Stop()'ing services before .Start()'ing them, +// we ignore .Stop()'s on nil RepeatTimers. +func (t *RepeatTimer) Stop() bool { + if t == nil { + return false + } + t.mtx.Lock() // Lock + defer t.mtx.Unlock() + + exists := t.ticker != nil + if exists { + t.ticker.Stop() + t.ticker = nil + close(t.quit) + } + return exists +} diff --git a/service.go b/service.go new file mode 100644 index 000000000..05b2adedd --- /dev/null +++ b/service.go @@ -0,0 +1,154 @@ +/* + +Classical-inheritance-style service declarations. +Services can be started, then stopped. +Users can override the OnStart/OnStop methods. +These methods are guaranteed to be called at most once. +Caller must ensure that Start() and Stop() are not called concurrently. +It is ok to call Stop() without calling Start() first. +Services cannot be re-started unless otherwise documented. + +Typical usage: + +type FooService struct { + BaseService + // private fields +} + +func NewFooService() *FooService { + fs := &FooService{ + // init + } + fs.BaseService = *NewBaseService(log, "FooService", fs) + return fs +} + +func (fs *FooService) OnStart() error { + fs.BaseService.OnStart() // Always call the overridden method. + // initialize private fields + // start subroutines, etc. +} + +func (fs *FooService) OnStop() error { + fs.BaseService.OnStop() // Always call the overridden method. + // close/destroy private fields + // stop subroutines, etc. +} + +*/ +package common + +import "sync/atomic" +import "github.com/tendermint/tendermint/Godeps/_workspace/src/github.com/tendermint/log15" + +type Service interface { + Start() (bool, error) + OnStart() error + + Stop() bool + OnStop() + + IsRunning() bool + + String() string +} + +type BaseService struct { + log log15.Logger + name string + started uint32 // atomic + stopped uint32 // atomic + + // The "subclass" of BaseService + impl Service +} + +func NewBaseService(log log15.Logger, name string, impl Service) *BaseService { + return &BaseService{ + log: log, + name: name, + impl: impl, + } +} + +// Implements Servce +func (bs *BaseService) Start() (bool, error) { + if atomic.CompareAndSwapUint32(&bs.started, 0, 1) { + if atomic.LoadUint32(&bs.stopped) == 1 { + if bs.log != nil { + bs.log.Warn(Fmt("Not starting %v -- already stopped", bs.name), "impl", bs.impl) + } + return false, nil + } else { + if bs.log != nil { + bs.log.Notice(Fmt("Starting %v", bs.name), "impl", bs.impl) + } + } + err := bs.impl.OnStart() + return true, err + } else { + if bs.log != nil { + bs.log.Info(Fmt("Not starting %v -- already started", bs.name), "impl", bs.impl) + } + return false, nil + } +} + +// Implements Service +func (bs *BaseService) OnStart() error { return nil } + +// Implements Service +func (bs *BaseService) Stop() bool { + if atomic.CompareAndSwapUint32(&bs.stopped, 0, 1) { + if bs.log != nil { + bs.log.Notice(Fmt("Stopping %v", bs.name), "impl", bs.impl) + } + bs.impl.OnStop() + return true + } else { + if bs.log != nil { + bs.log.Notice(Fmt("Not stopping %v", bs.name), "impl", bs.impl) + } + return false + } +} + +// Implements Service +func (bs *BaseService) OnStop() {} + +// Implements Service +func (bs *BaseService) IsRunning() bool { + return atomic.LoadUint32(&bs.started) == 1 && atomic.LoadUint32(&bs.stopped) == 0 +} + +// Implements Servce +func (bs *BaseService) String() string { + return bs.name +} + +//---------------------------------------- + +type QuitService struct { + BaseService + Quit chan struct{} +} + +func NewQuitService(log log15.Logger, name string, impl Service) *QuitService { + return &QuitService{ + BaseService: *NewBaseService(log, name, impl), + Quit: nil, + } +} + +// NOTE: when overriding OnStart, must call .QuitService.OnStart(). +func (qs *QuitService) OnStart() error { + qs.Quit = make(chan struct{}) + return nil +} + +// NOTE: when overriding OnStop, must call .QuitService.OnStop(). +func (qs *QuitService) OnStop() { + if qs.Quit != nil { + close(qs.Quit) + } +} diff --git a/string.go b/string.go new file mode 100644 index 000000000..a4d221b74 --- /dev/null +++ b/string.go @@ -0,0 +1,24 @@ +package common + +import ( + "fmt" + "strings" +) + +var Fmt = fmt.Sprintf + +func RightPadString(s string, totalLength int) string { + remaining := totalLength - len(s) + if remaining > 0 { + s = s + strings.Repeat(" ", remaining) + } + return s +} + +func LeftPadString(s string, totalLength int) string { + remaining := totalLength - len(s) + if remaining > 0 { + s = strings.Repeat(" ", remaining) + s + } + return s +} diff --git a/test/assert.go b/test/assert.go new file mode 100644 index 000000000..a6ffed0ce --- /dev/null +++ b/test/assert.go @@ -0,0 +1,14 @@ +package test + +import ( + "testing" +) + +func AssertPanics(t *testing.T, msg string, f func()) { + defer func() { + if err := recover(); err == nil { + t.Errorf("Should have panic'd, but didn't: %v", msg) + } + }() + f() +} diff --git a/test/mutate.go b/test/mutate.go new file mode 100644 index 000000000..39bf90557 --- /dev/null +++ b/test/mutate.go @@ -0,0 +1,28 @@ +package test + +import ( + . "github.com/tendermint/tendermint/common" +) + +// Contract: !bytes.Equal(input, output) && len(input) >= len(output) +func MutateByteSlice(bytez []byte) []byte { + // If bytez is empty, panic + if len(bytez) == 0 { + panic("Cannot mutate an empty bytez") + } + + // Copy bytez + mBytez := make([]byte, len(bytez)) + copy(mBytez, bytez) + bytez = mBytez + + // Try a random mutation + switch RandInt() % 2 { + case 0: // Mutate a single byte + bytez[RandInt()%len(bytez)] += byte(RandInt()%255 + 1) + case 1: // Remove an arbitrary byte + pos := RandInt() % len(bytez) + bytez = append(bytez[:pos], bytez[pos+1:]...) + } + return bytez +} diff --git a/throttle_timer.go b/throttle_timer.go new file mode 100644 index 000000000..0b40a60c2 --- /dev/null +++ b/throttle_timer.go @@ -0,0 +1,57 @@ +package common + +import ( + "sync/atomic" + "time" +) + +/* +ThrottleTimer fires an event at most "dur" after each .Set() call. +If a short burst of .Set() calls happens, ThrottleTimer fires once. +If a long continuous burst of .Set() calls happens, ThrottleTimer fires +at most once every "dur". +*/ +type ThrottleTimer struct { + Name string + Ch chan struct{} + quit chan struct{} + dur time.Duration + timer *time.Timer + isSet uint32 +} + +func NewThrottleTimer(name string, dur time.Duration) *ThrottleTimer { + var ch = make(chan struct{}) + var quit = make(chan struct{}) + var t = &ThrottleTimer{Name: name, Ch: ch, dur: dur, quit: quit} + t.timer = time.AfterFunc(dur, t.fireRoutine) + t.timer.Stop() + return t +} + +func (t *ThrottleTimer) fireRoutine() { + select { + case t.Ch <- struct{}{}: + atomic.StoreUint32(&t.isSet, 0) + case <-t.quit: + // do nothing + default: + t.timer.Reset(t.dur) + } +} + +func (t *ThrottleTimer) Set() { + if atomic.CompareAndSwapUint32(&t.isSet, 0, 1) { + t.timer.Reset(t.dur) + } +} + +// For ease of .Stop()'ing services before .Start()'ing them, +// we ignore .Stop()'s on nil ThrottleTimers +func (t *ThrottleTimer) Stop() bool { + if t == nil { + return false + } + close(t.quit) + return t.timer.Stop() +} diff --git a/word.go b/word.go new file mode 100644 index 000000000..4072482b8 --- /dev/null +++ b/word.go @@ -0,0 +1,91 @@ +package common + +import ( + "bytes" + "sort" +) + +var ( + Zero256 = Word256{0} + One256 = Word256{1} +) + +type Word256 [32]byte + +func (w Word256) String() string { return string(w[:]) } +func (w Word256) TrimmedString() string { return TrimmedString(w.Bytes()) } +func (w Word256) Copy() Word256 { return w } +func (w Word256) Bytes() []byte { return w[:] } // copied. +func (w Word256) Prefix(n int) []byte { return w[:n] } +func (w Word256) Postfix(n int) []byte { return w[32-n:] } +func (w Word256) IsZero() bool { + accum := byte(0) + for _, byt := range w { + accum |= byt + } + return accum == 0 +} +func (w Word256) Compare(other Word256) int { + return bytes.Compare(w[:], other[:]) +} + +func Uint64ToWord256(i uint64) Word256 { + buf := [8]byte{} + PutUint64BE(buf[:], i) + return LeftPadWord256(buf[:]) +} + +func Int64ToWord256(i int64) Word256 { + buf := [8]byte{} + PutInt64BE(buf[:], i) + return LeftPadWord256(buf[:]) +} + +func RightPadWord256(bz []byte) (word Word256) { + copy(word[:], bz) + return +} + +func LeftPadWord256(bz []byte) (word Word256) { + copy(word[32-len(bz):], bz) + return +} + +func Uint64FromWord256(word Word256) uint64 { + buf := word.Postfix(8) + return GetUint64BE(buf) +} + +func Int64FromWord256(word Word256) int64 { + buf := word.Postfix(8) + return GetInt64BE(buf) +} + +//------------------------------------- + +type Tuple256 struct { + First Word256 + Second Word256 +} + +func (tuple Tuple256) Compare(other Tuple256) int { + firstCompare := tuple.First.Compare(other.First) + if firstCompare == 0 { + return tuple.Second.Compare(other.Second) + } else { + return firstCompare + } +} + +func Tuple256Split(t Tuple256) (Word256, Word256) { + return t.First, t.Second +} + +type Tuple256Slice []Tuple256 + +func (p Tuple256Slice) Len() int { return len(p) } +func (p Tuple256Slice) Less(i, j int) bool { + return p[i].Compare(p[j]) < 0 +} +func (p Tuple256Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Tuple256Slice) Sort() { sort.Sort(p) }