mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-09-29 19:25:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "detele everything"

Browse Source 
This reverts commit d02c5d1e30.
This commit is contained in:
Zach Ramsay
2018-07-10 11:22:25 -04:00
parent 2f4ab0c068
commit 44dad6d70b
721 changed files with 99035 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/common/LICENSE
+193
View File
@@ -0,0 +1,193 @@
Tendermint Go-Common
Copyright (C) 2015 Tendermint
Apache License
Version 2.0, January 2004
https://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
libs/common/async.go
+175
View File
@@ -0,0 +1,175 @@
package common
import (
"sync/atomic"
)
//----------------------------------------
// Task
// val: the value returned after task execution.
// err: the error returned during task completion.
// abort: tells Parallel to return, whether or not all tasks have completed.
type Task func(i int) (val interface{}, err error, abort bool)
type TaskResult struct {
Value interface{}
Error error
}
type TaskResultCh <-chan TaskResult
type taskResultOK struct {
TaskResult
OK bool
}
type TaskResultSet struct {
chz []TaskResultCh
results []taskResultOK
}
func newTaskResultSet(chz []TaskResultCh) *TaskResultSet {
return &TaskResultSet{
chz: chz,
results: make([]taskResultOK, len(chz)),
}
}
func (trs *TaskResultSet) Channels() []TaskResultCh {
return trs.chz
}
func (trs *TaskResultSet) LatestResult(index int) (TaskResult, bool) {
if len(trs.results) <= index {
return TaskResult{}, false
}
resultOK := trs.results[index]
return resultOK.TaskResult, resultOK.OK
}
// NOTE: Not concurrency safe.
// Writes results to trs.results without waiting for all tasks to complete.
func (trs *TaskResultSet) Reap() *TaskResultSet {
for i := 0; i < len(trs.results); i++ {
var trch = trs.chz[i]
select {
case result, ok := <-trch:
if ok {
// Write result.
trs.results[i] = taskResultOK{
TaskResult: result,
OK: true,
}
} else {
// We already wrote it.
}
default:
// Do nothing.
}
}
return trs
}
// NOTE: Not concurrency safe.
// Like Reap() but waits until all tasks have returned or panic'd.
func (trs *TaskResultSet) Wait() *TaskResultSet {
for i := 0; i < len(trs.results); i++ {
var trch = trs.chz[i]
result, ok := <-trch
if ok {
// Write result.
trs.results[i] = taskResultOK{
TaskResult: result,
OK: true,
}
} else {
// We already wrote it.
}
}
return trs
}
// Returns the firstmost (by task index) error as
// discovered by all previous Reap() calls.
func (trs *TaskResultSet) FirstValue() interface{} {
for _, result := range trs.results {
if result.Value != nil {
return result.Value
}
}
return nil
}
// Returns the firstmost (by task index) error as
// discovered by all previous Reap() calls.
func (trs *TaskResultSet) FirstError() error {
for _, result := range trs.results {
if result.Error != nil {
return result.Error
}
}
return nil
}
//----------------------------------------
// Parallel
// Run tasks in parallel, with ability to abort early.
// Returns ok=false iff any of the tasks returned abort=true.
// NOTE: Do not implement quit features here. Instead, provide convenient
// concurrent quit-like primitives, passed implicitly via Task closures. (e.g.
// it's not Parallel's concern how you quit/abort your tasks).
func Parallel(tasks ...Task) (trs *TaskResultSet, ok bool) {
var taskResultChz = make([]TaskResultCh, len(tasks)) // To return.
var taskDoneCh = make(chan bool, len(tasks)) // A "wait group" channel, early abort if any true received.
var numPanics = new(int32) // Keep track of panics to set ok=false later.
ok = true // We will set it to false iff any tasks panic'd or returned abort.
// Start all tasks in parallel in separate goroutines.
// When the task is complete, it will appear in the
// respective taskResultCh (associated by task index).
for i, task := range tasks {
var taskResultCh = make(chan TaskResult, 1) // Capacity for 1 result.
taskResultChz[i] = taskResultCh
go func(i int, task Task, taskResultCh chan TaskResult) {
// Recovery
defer func() {
if pnk := recover(); pnk != nil {
atomic.AddInt32(numPanics, 1)
// Send panic to taskResultCh.
taskResultCh <- TaskResult{nil, ErrorWrap(pnk, "Panic in task")}
// Closing taskResultCh lets trs.Wait() work.
close(taskResultCh)
// Decrement waitgroup.
taskDoneCh <- false
}
}()
// Run the task.
var val, err, abort = task(i)
// Send val/err to taskResultCh.
// NOTE: Below this line, nothing must panic/
taskResultCh <- TaskResult{val, err}
// Closing taskResultCh lets trs.Wait() work.
close(taskResultCh)
// Decrement waitgroup.
taskDoneCh <- abort
}(i, task, taskResultCh)
}
// Wait until all tasks are done, or until abort.
// DONE_LOOP:
for i := 0; i < len(tasks); i++ {
abort := <-taskDoneCh
if abort {
ok = false
break
}
}
// Ok is also false if there were any panics.
// We must do this check here (after DONE_LOOP).
ok = ok && (atomic.LoadInt32(numPanics) == 0)
return newTaskResultSet(taskResultChz).Reap(), ok
}
libs/common/async_test.go
+156
View File
@@ -0,0 +1,156 @@
package common
import (
"errors"
"fmt"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestParallel(t *testing.T) {
// Create tasks.
var counter = new(int32)
var tasks = make([]Task, 100*1000)
for i := 0; i < len(tasks); i++ {
tasks[i] = func(i int) (res interface{}, err error, abort bool) {
atomic.AddInt32(counter, 1)
return -1 * i, nil, false
}
}
// Run in parallel.
var trs, ok = Parallel(tasks...)
assert.True(t, ok)
// Verify.
assert.Equal(t, int(*counter), len(tasks), "Each task should have incremented the counter already")
var failedTasks int
for i := 0; i < len(tasks); i++ {
taskResult, ok := trs.LatestResult(i)
if !ok {
assert.Fail(t, "Task #%v did not complete.", i)
failedTasks++
} else if taskResult.Error != nil {
assert.Fail(t, "Task should not have errored but got %v", taskResult.Error)
failedTasks++
} else if !assert.Equal(t, -1*i, taskResult.Value.(int)) {
assert.Fail(t, "Task should have returned %v but got %v", -1*i, taskResult.Value.(int))
failedTasks++
} else {
// Good!
}
}
assert.Equal(t, failedTasks, 0, "No task should have failed")
assert.Nil(t, trs.FirstError(), "There should be no errors")
assert.Equal(t, 0, trs.FirstValue(), "First value should be 0")
}
func TestParallelAbort(t *testing.T) {
var flow1 = make(chan struct{}, 1)
var flow2 = make(chan struct{}, 1)
var flow3 = make(chan struct{}, 1) // Cap must be > 0 to prevent blocking.
var flow4 = make(chan struct{}, 1)
// Create tasks.
var tasks = []Task{
func(i int) (res interface{}, err error, abort bool) {
assert.Equal(t, i, 0)
flow1 <- struct{}{}
return 0, nil, false
},
func(i int) (res interface{}, err error, abort bool) {
assert.Equal(t, i, 1)
flow2 <- <-flow1
return 1, errors.New("some error"), false
},
func(i int) (res interface{}, err error, abort bool) {
assert.Equal(t, i, 2)
flow3 <- <-flow2
return 2, nil, true
},
func(i int) (res interface{}, err error, abort bool) {
assert.Equal(t, i, 3)
<-flow4
return 3, nil, false
},
}
// Run in parallel.
var taskResultSet, ok = Parallel(tasks...)
assert.False(t, ok, "ok should be false since we aborted task #2.")
// Verify task #3.
// Initially taskResultSet.chz[3] sends nothing since flow4 didn't send.
waitTimeout(t, taskResultSet.chz[3], "Task #3")
// Now let the last task (#3) complete after abort.
flow4 <- <-flow3
// Wait until all tasks have returned or panic'd.
taskResultSet.Wait()
// Verify task #0, #1, #2.
checkResult(t, taskResultSet, 0, 0, nil, nil)
checkResult(t, taskResultSet, 1, 1, errors.New("some error"), nil)
checkResult(t, taskResultSet, 2, 2, nil, nil)
checkResult(t, taskResultSet, 3, 3, nil, nil)
}
func TestParallelRecover(t *testing.T) {
// Create tasks.
var tasks = []Task{
func(i int) (res interface{}, err error, abort bool) {
return 0, nil, false
},
func(i int) (res interface{}, err error, abort bool) {
return 1, errors.New("some error"), false
},
func(i int) (res interface{}, err error, abort bool) {
panic(2)
},
}
// Run in parallel.
var taskResultSet, ok = Parallel(tasks...)
assert.False(t, ok, "ok should be false since we panic'd in task #2.")
// Verify task #0, #1, #2.
checkResult(t, taskResultSet, 0, 0, nil, nil)
checkResult(t, taskResultSet, 1, 1, errors.New("some error"), nil)
checkResult(t, taskResultSet, 2, nil, nil, 2)
}
// Wait for result
func checkResult(t *testing.T, taskResultSet *TaskResultSet, index int, val interface{}, err error, pnk interface{}) {
taskResult, ok := taskResultSet.LatestResult(index)
taskName := fmt.Sprintf("Task #%v", index)
assert.True(t, ok, "TaskResultCh unexpectedly closed for %v", taskName)
assert.Equal(t, val, taskResult.Value, taskName)
if err != nil {
assert.Equal(t, err, taskResult.Error, taskName)
} else if pnk != nil {
assert.Equal(t, pnk, taskResult.Error.(Error).Data(), taskName)
} else {
assert.Nil(t, taskResult.Error, taskName)
}
}
// Wait for timeout (no result)
func waitTimeout(t *testing.T, taskResultCh TaskResultCh, taskName string) {
select {
case _, ok := <-taskResultCh:
if !ok {
assert.Fail(t, "TaskResultCh unexpectedly closed (%v)", taskName)
} else {
assert.Fail(t, "TaskResultCh unexpectedly returned for %v", taskName)
}
case <-time.After(1 * time.Second): // TODO use deterministic time?
// Good!
}
}
libs/common/bit_array.go
+378
View File
@@ -0,0 +1,378 @@
package common
import (
"encoding/binary"
"fmt"
"regexp"
"strings"
"sync"
)
type BitArray struct {
mtx sync.Mutex
Bits int `json:"bits"` // NOTE: persisted via reflect, must be exported
Elems []uint64 `json:"elems"` // NOTE: persisted via reflect, must be exported
}
// There is no BitArray whose Size is 0. Use nil instead.
func NewBitArray(bits int) *BitArray {
if bits <= 0 {
return nil
}
return &BitArray{
Bits: bits,
Elems: make([]uint64, (bits+63)/64),
}
}
func (bA *BitArray) Size() int {
if bA == nil {
return 0
}
return bA.Bits
}
// NOTE: behavior is undefined if i >= bA.Bits
func (bA *BitArray) GetIndex(i int) bool {
if bA == nil {
return false
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
return bA.getIndex(i)
}
func (bA *BitArray) getIndex(i int) bool {
if i >= bA.Bits {
return false
}
return bA.Elems[i/64]&(uint64(1)<<uint(i%64)) > 0
}
// NOTE: behavior is undefined if i >= bA.Bits
func (bA *BitArray) SetIndex(i int, v bool) bool {
if bA == nil {
return false
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
return bA.setIndex(i, v)
}
func (bA *BitArray) setIndex(i int, v bool) bool {
if i >= bA.Bits {
return false
}
if v {
bA.Elems[i/64] |= (uint64(1) << uint(i%64))
} else {
bA.Elems[i/64] &= ^(uint64(1) << uint(i%64))
}
return true
}
func (bA *BitArray) Copy() *BitArray {
if bA == nil {
return nil
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
return bA.copy()
}
func (bA *BitArray) copy() *BitArray {
c := make([]uint64, len(bA.Elems))
copy(c, bA.Elems)
return &BitArray{
Bits: bA.Bits,
Elems: c,
}
}
func (bA *BitArray) copyBits(bits int) *BitArray {
c := make([]uint64, (bits+63)/64)
copy(c, bA.Elems)
return &BitArray{
Bits: bits,
Elems: c,
}
}
// Returns a BitArray of larger bits size.
func (bA *BitArray) Or(o *BitArray) *BitArray {
if bA == nil && o == nil {
return nil
}
if bA == nil && o != nil {
return o.Copy()
}
if o == nil {
return bA.Copy()
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
c := bA.copyBits(MaxInt(bA.Bits, o.Bits))
for i := 0; i < len(c.Elems); i++ {
c.Elems[i] |= o.Elems[i]
}
return c
}
// Returns a BitArray of smaller bit size.
func (bA *BitArray) And(o *BitArray) *BitArray {
if bA == nil || o == nil {
return nil
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
return bA.and(o)
}
func (bA *BitArray) and(o *BitArray) *BitArray {
c := bA.copyBits(MinInt(bA.Bits, o.Bits))
for i := 0; i < len(c.Elems); i++ {
c.Elems[i] &= o.Elems[i]
}
return c
}
func (bA *BitArray) Not() *BitArray {
if bA == nil {
return nil // Degenerate
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
c := bA.copy()
for i := 0; i < len(c.Elems); i++ {
c.Elems[i] = ^c.Elems[i]
}
return c
}
func (bA *BitArray) Sub(o *BitArray) *BitArray {
if bA == nil || o == nil {
// TODO: Decide if we should do 1's complement here?
return nil
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
if bA.Bits > o.Bits {
c := bA.copy()
for i := 0; i < len(o.Elems)-1; i++ {
c.Elems[i] &= ^c.Elems[i]
}
i := len(o.Elems) - 1
if i >= 0 {
for idx := i * 64; idx < o.Bits; idx++ {
// NOTE: each individual GetIndex() call to o is safe.
c.setIndex(idx, c.getIndex(idx) && !o.GetIndex(idx))
}
}
return c
}
return bA.and(o.Not()) // Note degenerate case where o == nil
}
func (bA *BitArray) IsEmpty() bool {
if bA == nil {
return true // should this be opposite?
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
for _, e := range bA.Elems {
if e > 0 {
return false
}
}
return true
}
func (bA *BitArray) IsFull() bool {
if bA == nil {
return true
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
// Check all elements except the last
for _, elem := range bA.Elems[:len(bA.Elems)-1] {
if (^elem) != 0 {
return false
}
}
// Check that the last element has (lastElemBits) 1's
lastElemBits := (bA.Bits+63)%64 + 1
lastElem := bA.Elems[len(bA.Elems)-1]
return (lastElem+1)&((uint64(1)<<uint(lastElemBits))-1) == 0
}
func (bA *BitArray) PickRandom() (int, bool) {
if bA == nil {
return 0, false
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
length := len(bA.Elems)
if length == 0 {
return 0, false
}
randElemStart := RandIntn(length)
for i := 0; i < length; i++ {
elemIdx := ((i + randElemStart) % length)
if elemIdx < length-1 {
if bA.Elems[elemIdx] > 0 {
randBitStart := RandIntn(64)
for j := 0; j < 64; j++ {
bitIdx := ((j + randBitStart) % 64)
if (bA.Elems[elemIdx] & (uint64(1) << uint(bitIdx))) > 0 {
return 64*elemIdx + bitIdx, true
}
}
PanicSanity("should not happen")
}
} else {
// Special case for last elem, to ignore straggler bits
elemBits := bA.Bits % 64
if elemBits == 0 {
elemBits = 64
}
randBitStart := RandIntn(elemBits)
for j := 0; j < elemBits; j++ {
bitIdx := ((j + randBitStart) % elemBits)
if (bA.Elems[elemIdx] & (uint64(1) << uint(bitIdx))) > 0 {
return 64*elemIdx + bitIdx, true
}
}
}
}
return 0, false
}
// String returns a string representation of BitArray: BA{<bit-string>},
// where <bit-string> is a sequence of 'x' (1) and '_' (0).
// The <bit-string> includes spaces and newlines to help people.
// For a simple sequence of 'x' and '_' characters with no spaces or newlines,
// see the MarshalJSON() method.
// Example: "BA{_x_}" or "nil-BitArray" for nil.
func (bA *BitArray) String() string {
return bA.StringIndented("")
}
func (bA *BitArray) StringIndented(indent string) string {
if bA == nil {
return "nil-BitArray"
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
return bA.stringIndented(indent)
}
func (bA *BitArray) stringIndented(indent string) string {
lines := []string{}
bits := ""
for i := 0; i < bA.Bits; i++ {
if bA.getIndex(i) {
bits += "x"
} else {
bits += "_"
}
if i%100 == 99 {
lines = append(lines, bits)
bits = ""
}
if i%10 == 9 {
bits += indent
}
if i%50 == 49 {
bits += indent
}
}
if len(bits) > 0 {
lines = append(lines, bits)
}
return fmt.Sprintf("BA{%v:%v}", bA.Bits, strings.Join(lines, indent))
}
func (bA *BitArray) Bytes() []byte {
bA.mtx.Lock()
defer bA.mtx.Unlock()
numBytes := (bA.Bits + 7) / 8
bytes := make([]byte, numBytes)
for i := 0; i < len(bA.Elems); i++ {
elemBytes := [8]byte{}
binary.LittleEndian.PutUint64(elemBytes[:], bA.Elems[i])
copy(bytes[i*8:], elemBytes[:])
}
return bytes
}
// NOTE: other bitarray o is not locked when reading,
// so if necessary, caller must copy or lock o prior to calling Update.
// If bA is nil, does nothing.
func (bA *BitArray) Update(o *BitArray) {
if bA == nil || o == nil {
return
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
copy(bA.Elems, o.Elems)
}
// MarshalJSON implements json.Marshaler interface by marshaling bit array
// using a custom format: a string of '-' or 'x' where 'x' denotes the 1 bit.
func (bA *BitArray) MarshalJSON() ([]byte, error) {
if bA == nil {
return []byte("null"), nil
}
bA.mtx.Lock()
defer bA.mtx.Unlock()
bits := `"`
for i := 0; i < bA.Bits; i++ {
if bA.getIndex(i) {
bits += `x`
} else {
bits += `_`
}
}
bits += `"`
return []byte(bits), nil
}
var bitArrayJSONRegexp = regexp.MustCompile(`\A"([_x]*)"\z`)
// UnmarshalJSON implements json.Unmarshaler interface by unmarshaling a custom
// JSON description.
func (bA *BitArray) UnmarshalJSON(bz []byte) error {
b := string(bz)
if b == "null" {
// This is required e.g. for encoding/json when decoding
// into a pointer with pre-allocated BitArray.
bA.Bits = 0
bA.Elems = nil
return nil
}
// Validate 'b'.
match := bitArrayJSONRegexp.FindStringSubmatch(b)
if match == nil {
return fmt.Errorf("BitArray in JSON should be a string of format %q but got %s", bitArrayJSONRegexp.String(), b)
}
bits := match[1]
// Construct new BitArray and copy over.
numBits := len(bits)
bA2 := NewBitArray(numBits)
for i := 0; i < numBits; i++ {
if bits[i] == 'x' {
bA2.SetIndex(i, true)
}
}
*bA = *bA2
return nil
}
libs/common/bit_array_test.go
+267
View File
@@ -0,0 +1,267 @@
package common
import (
"bytes"
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func randBitArray(bits int) (*BitArray, []byte) {
src := RandBytes((bits + 7) / 8)
bA := NewBitArray(bits)
for i := 0; i < len(src); i++ {
for j := 0; j < 8; j++ {
if i*8+j >= bits {
return bA, src
}
setBit := src[i]&(1<<uint(j)) > 0
bA.SetIndex(i*8+j, setBit)
}
}
return bA, src
}
func TestAnd(t *testing.T) {
bA1, _ := randBitArray(51)
bA2, _ := randBitArray(31)
bA3 := bA1.And(bA2)
var bNil *BitArray
require.Equal(t, bNil.And(bA1), (*BitArray)(nil))
require.Equal(t, bA1.And(nil), (*BitArray)(nil))
require.Equal(t, bNil.And(nil), (*BitArray)(nil))
if bA3.Bits != 31 {
t.Error("Expected min bits", bA3.Bits)
}
if len(bA3.Elems) != len(bA2.Elems) {
t.Error("Expected min elems length")
}
for i := 0; i < bA3.Bits; i++ {
expected := bA1.GetIndex(i) && bA2.GetIndex(i)
if bA3.GetIndex(i) != expected {
t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i))
}
}
}
func TestOr(t *testing.T) {
bA1, _ := randBitArray(51)
bA2, _ := randBitArray(31)
bA3 := bA1.Or(bA2)
bNil := (*BitArray)(nil)
require.Equal(t, bNil.Or(bA1), bA1)
require.Equal(t, bA1.Or(nil), bA1)
require.Equal(t, bNil.Or(nil), (*BitArray)(nil))
if bA3.Bits != 51 {
t.Error("Expected max bits")
}
if len(bA3.Elems) != len(bA1.Elems) {
t.Error("Expected max elems length")
}
for i := 0; i < bA3.Bits; i++ {
expected := bA1.GetIndex(i) || bA2.GetIndex(i)
if bA3.GetIndex(i) != expected {
t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i))
}
}
}
func TestSub1(t *testing.T) {
bA1, _ := randBitArray(31)
bA2, _ := randBitArray(51)
bA3 := bA1.Sub(bA2)
bNil := (*BitArray)(nil)
require.Equal(t, bNil.Sub(bA1), (*BitArray)(nil))
require.Equal(t, bA1.Sub(nil), (*BitArray)(nil))
require.Equal(t, bNil.Sub(nil), (*BitArray)(nil))
if bA3.Bits != bA1.Bits {
t.Error("Expected bA1 bits")
}
if len(bA3.Elems) != len(bA1.Elems) {
t.Error("Expected bA1 elems length")
}
for i := 0; i < bA3.Bits; i++ {
expected := bA1.GetIndex(i)
if bA2.GetIndex(i) {
expected = false
}
if bA3.GetIndex(i) != expected {
t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i))
}
}
}
func TestSub2(t *testing.T) {
bA1, _ := randBitArray(51)
bA2, _ := randBitArray(31)
bA3 := bA1.Sub(bA2)
bNil := (*BitArray)(nil)
require.Equal(t, bNil.Sub(bA1), (*BitArray)(nil))
require.Equal(t, bA1.Sub(nil), (*BitArray)(nil))
require.Equal(t, bNil.Sub(nil), (*BitArray)(nil))
if bA3.Bits != bA1.Bits {
t.Error("Expected bA1 bits")
}
if len(bA3.Elems) != len(bA1.Elems) {
t.Error("Expected bA1 elems length")
}
for i := 0; i < bA3.Bits; i++ {
expected := bA1.GetIndex(i)
if i < bA2.Bits && bA2.GetIndex(i) {
expected = false
}
if bA3.GetIndex(i) != expected {
t.Error("Wrong bit from bA3")
}
}
}
func TestPickRandom(t *testing.T) {
for idx := 0; idx < 123; idx++ {
bA1 := NewBitArray(123)
bA1.SetIndex(idx, true)
index, ok := bA1.PickRandom()
if !ok {
t.Fatal("Expected to pick element but got none")
}
if index != idx {
t.Fatalf("Expected to pick element at %v but got wrong index", idx)
}
}
}
func TestBytes(t *testing.T) {
bA := NewBitArray(4)
bA.SetIndex(0, true)
check := func(bA *BitArray, bz []byte) {
if !bytes.Equal(bA.Bytes(), bz) {
panic(Fmt("Expected %X but got %X", bz, bA.Bytes()))
}
}
check(bA, []byte{0x01})
bA.SetIndex(3, true)
check(bA, []byte{0x09})
bA = NewBitArray(9)
check(bA, []byte{0x00, 0x00})
bA.SetIndex(7, true)
check(bA, []byte{0x80, 0x00})
bA.SetIndex(8, true)
check(bA, []byte{0x80, 0x01})
bA = NewBitArray(16)
check(bA, []byte{0x00, 0x00})
bA.SetIndex(7, true)
check(bA, []byte{0x80, 0x00})
bA.SetIndex(8, true)
check(bA, []byte{0x80, 0x01})
bA.SetIndex(9, true)
check(bA, []byte{0x80, 0x03})
}
func TestEmptyFull(t *testing.T) {
ns := []int{47, 123}
for _, n := range ns {
bA := NewBitArray(n)
if !bA.IsEmpty() {
t.Fatal("Expected bit array to be empty")
}
for i := 0; i < n; i++ {
bA.SetIndex(i, true)
}
if !bA.IsFull() {
t.Fatal("Expected bit array to be full")
}
}
}
func TestUpdateNeverPanics(t *testing.T) {
newRandBitArray := func(n int) *BitArray {
ba, _ := randBitArray(n)
return ba
}
pairs := []struct {
a, b *BitArray
}{
{nil, nil},
{newRandBitArray(10), newRandBitArray(12)},
{newRandBitArray(23), newRandBitArray(23)},
{newRandBitArray(37), nil},
{nil, NewBitArray(10)},
}
for _, pair := range pairs {
a, b := pair.a, pair.b
a.Update(b)
b.Update(a)
}
}
func TestNewBitArrayNeverCrashesOnNegatives(t *testing.T) {
bitList := []int{-127, -128, -1 << 31}
for _, bits := range bitList {
_ = NewBitArray(bits)
}
}
func TestJSONMarshalUnmarshal(t *testing.T) {
bA1 := NewBitArray(0)
bA2 := NewBitArray(1)
bA3 := NewBitArray(1)
bA3.SetIndex(0, true)
bA4 := NewBitArray(5)
bA4.SetIndex(0, true)
bA4.SetIndex(1, true)
testCases := []struct {
bA *BitArray
marshalledBA string
}{
{nil, `null`},
{bA1, `null`},
{bA2, `"_"`},
{bA3, `"x"`},
{bA4, `"xx___"`},
}
for _, tc := range testCases {
t.Run(tc.bA.String(), func(t *testing.T) {
bz, err := json.Marshal(tc.bA)
require.NoError(t, err)
assert.Equal(t, tc.marshalledBA, string(bz))
var unmarshalledBA *BitArray
err = json.Unmarshal(bz, &unmarshalledBA)
require.NoError(t, err)
if tc.bA == nil {
require.Nil(t, unmarshalledBA)
} else {
require.NotNil(t, unmarshalledBA)
assert.EqualValues(t, tc.bA.Bits, unmarshalledBA.Bits)
if assert.EqualValues(t, tc.bA.String(), unmarshalledBA.String()) {
assert.EqualValues(t, tc.bA.Elems, unmarshalledBA.Elems)
}
}
})
}
}
libs/common/bytes.go
+62
View File
@@ -0,0 +1,62 @@
package common
import (
"encoding/hex"
"fmt"
"strings"
)
// The main purpose of HexBytes is to enable HEX-encoding for json/encoding.
type HexBytes []byte
// Marshal needed for protobuf compatibility
func (bz HexBytes) Marshal() ([]byte, error) {
return bz, nil
}
// Unmarshal needed for protobuf compatibility
func (bz *HexBytes) Unmarshal(data []byte) error {
*bz = data
return nil
}
// This is the point of Bytes.
func (bz HexBytes) MarshalJSON() ([]byte, error) {
s := strings.ToUpper(hex.EncodeToString(bz))
jbz := make([]byte, len(s)+2)
jbz[0] = '"'
copy(jbz[1:], []byte(s))
jbz[len(jbz)-1] = '"'
return jbz, nil
}
// This is the point of Bytes.
func (bz *HexBytes) UnmarshalJSON(data []byte) error {
if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' {
return fmt.Errorf("Invalid hex string: %s", data)
}
bz2, err := hex.DecodeString(string(data[1 : len(data)-1]))
if err != nil {
return err
}
*bz = bz2
return nil
}
// Allow it to fulfill various interfaces in light-client, etc...
func (bz HexBytes) Bytes() []byte {
return bz
}
func (bz HexBytes) String() string {
return strings.ToUpper(hex.EncodeToString(bz))
}
func (bz HexBytes) Format(s fmt.State, verb rune) {
switch verb {
case 'p':
s.Write([]byte(fmt.Sprintf("%p", bz)))
default:
s.Write([]byte(fmt.Sprintf("%X", []byte(bz))))
}
}
libs/common/bytes_test.go
+65
View File
@@ -0,0 +1,65 @@
package common
import (
"encoding/json"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
// This is a trivial test for protobuf compatibility.
func TestMarshal(t *testing.T) {
bz := []byte("hello world")
dataB := HexBytes(bz)
bz2, err := dataB.Marshal()
assert.Nil(t, err)
assert.Equal(t, bz, bz2)
var dataB2 HexBytes
err = (&dataB2).Unmarshal(bz)
assert.Nil(t, err)
assert.Equal(t, dataB, dataB2)
}
// Test that the hex encoding works.
func TestJSONMarshal(t *testing.T) {
type TestStruct struct {
B1 []byte
B2 HexBytes
}
cases := []struct {
input []byte
expected string
}{
{[]byte(``), `{"B1":"","B2":""}`},
{[]byte(`a`), `{"B1":"YQ==","B2":"61"}`},
{[]byte(`abc`), `{"B1":"YWJj","B2":"616263"}`},
}
for i, tc := range cases {
t.Run(fmt.Sprintf("Case %d", i), func(t *testing.T) {
ts := TestStruct{B1: tc.input, B2: tc.input}
// Test that it marshals correctly to JSON.
jsonBytes, err := json.Marshal(ts)
if err != nil {
t.Fatal(err)
}
assert.Equal(t, string(jsonBytes), tc.expected)
// TODO do fuzz testing to ensure that unmarshal fails
// Test that unmarshaling works correctly.
ts2 := TestStruct{}
err = json.Unmarshal(jsonBytes, &ts2)
if err != nil {
t.Fatal(err)
}
assert.Equal(t, ts2.B1, tc.input)
assert.Equal(t, ts2.B2, HexBytes(tc.input))
})
}
}
libs/common/byteslice.go
+73
View File
@@ -0,0 +1,73 @@
package common
import (
"bytes"
)
// Fingerprint returns the first 6 bytes of a byte slice.
// If the slice is less than 6 bytes, the fingerprint
// contains trailing zeroes.
func Fingerprint(slice []byte) []byte {
fingerprint := make([]byte, 6)
copy(fingerprint, slice)
return fingerprint
}
func IsZeros(slice []byte) bool {
for _, byt := range slice {
if byt != byte(0) {
return false
}
}
return true
}
func RightPadBytes(slice []byte, l int) []byte {
if l < len(slice) {
return slice
}
padded := make([]byte, l)
copy(padded[0:len(slice)], slice)
return padded
}
func LeftPadBytes(slice []byte, l int) []byte {
if l < len(slice) {
return slice
}
padded := make([]byte, l)
copy(padded[l-len(slice):], slice)
return padded
}
func TrimmedString(b []byte) string {
trimSet := string([]byte{0})
return string(bytes.TrimLeft(b, trimSet))
}
// PrefixEndBytes returns the end byteslice for a noninclusive range
// that would include all byte slices for which the input is the prefix
func PrefixEndBytes(prefix []byte) []byte {
if prefix == nil {
return nil
}
end := make([]byte, len(prefix))
copy(end, prefix)
finished := false
for !finished {
if end[len(end)-1] != byte(255) {
end[len(end)-1]++
finished = true
} else {
end = end[:len(end)-1]
if len(end) == 0 {
end = nil
finished = true
}
}
}
return end
}
libs/common/byteslice_test.go
+28
View File
@@ -0,0 +1,28 @@
package common
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestPrefixEndBytes(t *testing.T) {
assert := assert.New(t)
var testCases = []struct {
prefix []byte
expected []byte
}{
{[]byte{byte(55), byte(255), byte(255), byte(0)}, []byte{byte(55), byte(255), byte(255), byte(1)}},
{[]byte{byte(55), byte(255), byte(255), byte(15)}, []byte{byte(55), byte(255), byte(255), byte(16)}},
{[]byte{byte(55), byte(200), byte(255)}, []byte{byte(55), byte(201)}},
{[]byte{byte(55), byte(255), byte(255)}, []byte{byte(56)}},
{[]byte{byte(255), byte(255), byte(255)}, nil},
{nil, nil},
}
for _, test := range testCases {
end := PrefixEndBytes(test.prefix)
assert.Equal(test.expected, end)
}
}
libs/common/cmap.go
+73
View File
@@ -0,0 +1,73 @@
package common
import "sync"
// CMap is a goroutine-safe map
type CMap struct {
m map[string]interface{}
l sync.Mutex
}
func NewCMap() *CMap {
return &CMap{
m: make(map[string]interface{}),
}
}
func (cm *CMap) Set(key string, value interface{}) {
cm.l.Lock()
defer cm.l.Unlock()
cm.m[key] = value
}
func (cm *CMap) Get(key string) interface{} {
cm.l.Lock()
defer cm.l.Unlock()
return cm.m[key]
}
func (cm *CMap) Has(key string) bool {
cm.l.Lock()
defer cm.l.Unlock()
_, ok := cm.m[key]
return ok
}
func (cm *CMap) Delete(key string) {
cm.l.Lock()
defer cm.l.Unlock()
delete(cm.m, key)
}
func (cm *CMap) Size() int {
cm.l.Lock()
defer cm.l.Unlock()
return len(cm.m)
}
func (cm *CMap) Clear() {
cm.l.Lock()
defer cm.l.Unlock()
cm.m = make(map[string]interface{})
}
func (cm *CMap) Keys() []string {
cm.l.Lock()
defer cm.l.Unlock()
keys := []string{}
for k := range cm.m {
keys = append(keys, k)
}
return keys
}
func (cm *CMap) Values() []interface{} {
cm.l.Lock()
defer cm.l.Unlock()
items := []interface{}{}
for _, v := range cm.m {
items = append(items, v)
}
return items
}
libs/common/cmap_test.go
+53
View File
@@ -0,0 +1,53 @@
package common
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestIterateKeysWithValues(t *testing.T) {
cmap := NewCMap()
for i := 1; i <= 10; i++ {
cmap.Set(fmt.Sprintf("key%d", i), fmt.Sprintf("value%d", i))
}
// Testing size
assert.Equal(t, 10, cmap.Size())
assert.Equal(t, 10, len(cmap.Keys()))
assert.Equal(t, 10, len(cmap.Values()))
// Iterating Keys, checking for matching Value
for _, key := range cmap.Keys() {
val := strings.Replace(key, "key", "value", -1)
assert.Equal(t, val, cmap.Get(key))
}
// Test if all keys are within []Keys()
keys := cmap.Keys()
for i := 1; i <= 10; i++ {
assert.Contains(t, keys, fmt.Sprintf("key%d", i), "cmap.Keys() should contain key")
}
// Delete 1 Key
cmap.Delete("key1")
assert.NotEqual(t, len(keys), len(cmap.Keys()), "[]keys and []Keys() should not be equal, they are copies, one item was removed")
}
func TestContains(t *testing.T) {
cmap := NewCMap()
cmap.Set("key1", "value1")
// Test for known values
assert.True(t, cmap.Has("key1"))
assert.Equal(t, "value1", cmap.Get("key1"))
// Test for unknown values
assert.False(t, cmap.Has("key2"))
assert.Nil(t, cmap.Get("key2"))
}
libs/common/colors.go
+95
View File
@@ -0,0 +1,95 @@
package common
import (
"fmt"
"strings"
)
const (
ANSIReset = "\x1b[0m"
ANSIBright = "\x1b[1m"
ANSIDim = "\x1b[2m"
ANSIUnderscore = "\x1b[4m"
ANSIBlink = "\x1b[5m"
ANSIReverse = "\x1b[7m"
ANSIHidden = "\x1b[8m"
ANSIFgBlack = "\x1b[30m"
ANSIFgRed = "\x1b[31m"
ANSIFgGreen = "\x1b[32m"
ANSIFgYellow = "\x1b[33m"
ANSIFgBlue = "\x1b[34m"
ANSIFgMagenta = "\x1b[35m"
ANSIFgCyan = "\x1b[36m"
ANSIFgWhite = "\x1b[37m"
ANSIBgBlack = "\x1b[40m"
ANSIBgRed = "\x1b[41m"
ANSIBgGreen = "\x1b[42m"
ANSIBgYellow = "\x1b[43m"
ANSIBgBlue = "\x1b[44m"
ANSIBgMagenta = "\x1b[45m"
ANSIBgCyan = "\x1b[46m"
ANSIBgWhite = "\x1b[47m"
)
// color the string s with color 'color'
// unless s is already colored
func treat(s string, color string) string {
if len(s) > 2 && s[:2] == "\x1b[" {
return s
}
return color + s + ANSIReset
}
func treatAll(color string, args ...interface{}) string {
var parts []string
for _, arg := range args {
parts = append(parts, treat(fmt.Sprintf("%v", arg), color))
}
return strings.Join(parts, "")
}
func Black(args ...interface{}) string {
return treatAll(ANSIFgBlack, args...)
}
func Red(args ...interface{}) string {
return treatAll(ANSIFgRed, args...)
}
func Green(args ...interface{}) string {
return treatAll(ANSIFgGreen, args...)
}
func Yellow(args ...interface{}) string {
return treatAll(ANSIFgYellow, args...)
}
func Blue(args ...interface{}) string {
return treatAll(ANSIFgBlue, args...)
}
func Magenta(args ...interface{}) string {
return treatAll(ANSIFgMagenta, args...)
}
func Cyan(args ...interface{}) string {
return treatAll(ANSIFgCyan, args...)
}
func White(args ...interface{}) string {
return treatAll(ANSIFgWhite, args...)
}
func ColoredBytes(data []byte, textColor, bytesColor func(...interface{}) string) string {
s := ""
for _, b := range data {
if 0x21 <= b && b < 0x7F {
s += textColor(string(b))
} else {
s += bytesColor(Fmt("%02X", b))
}
}
return s
}
libs/common/date.go
+43
View File
@@ -0,0 +1,43 @@
package common
import (
"strings"
"time"
"github.com/pkg/errors"
)
// TimeLayout helps to parse a date string of the format YYYY-MM-DD
// Intended to be used with the following function:
// time.Parse(TimeLayout, date)
var TimeLayout = "2006-01-02" //this represents YYYY-MM-DD
// ParseDateRange parses a date range string of the format start:end
// where the start and end date are of the format YYYY-MM-DD.
// The parsed dates are time.Time and will return the zero time for
// unbounded dates, ex:
// unbounded start: :2000-12-31
// unbounded end: 2000-12-31:
func ParseDateRange(dateRange string) (startDate, endDate time.Time, err error) {
dates := strings.Split(dateRange, ":")
if len(dates) != 2 {
err = errors.New("bad date range, must be in format date:date")
return
}
parseDate := func(date string) (out time.Time, err error) {
if len(date) == 0 {
return
}
out, err = time.Parse(TimeLayout, date)
return
}
startDate, err = parseDate(dates[0])
if err != nil {
return
}
endDate, err = parseDate(dates[1])
if err != nil {
return
}
return
}
libs/common/date_test.go
+46
View File
@@ -0,0 +1,46 @@
package common
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
var (
date = time.Date(2015, time.Month(12), 31, 0, 0, 0, 0, time.UTC)
date2 = time.Date(2016, time.Month(12), 31, 0, 0, 0, 0, time.UTC)
zero time.Time
)
func TestParseDateRange(t *testing.T) {
assert := assert.New(t)
var testDates = []struct {
dateStr string
start time.Time
end time.Time
errNil bool
}{
{"2015-12-31:2016-12-31", date, date2, true},
{"2015-12-31:", date, zero, true},
{":2016-12-31", zero, date2, true},
{"2016-12-31", zero, zero, false},
{"2016-31-12:", zero, zero, false},
{":2016-31-12", zero, zero, false},
}
for _, test := range testDates {
start, end, err := ParseDateRange(test.dateStr)
if test.errNil {
assert.Nil(err)
testPtr := func(want, have time.Time) {
assert.True(have.Equal(want))
}
testPtr(test.start, start)
testPtr(test.end, end)
} else {
assert.NotNil(err)
}
}
}
libs/common/errors.go
+246
View File
@@ -0,0 +1,246 @@
package common
import (
"fmt"
"runtime"
)
//----------------------------------------
// Convenience method.
func ErrorWrap(cause interface{}, format string, args ...interface{}) Error {
if causeCmnError, ok := cause.(*cmnError); ok {
msg := Fmt(format, args...)
return causeCmnError.Stacktrace().Trace(1, msg)
} else if cause == nil {
return newCmnError(FmtError{format, args}).Stacktrace()
} else {
// NOTE: causeCmnError is a typed nil here.
msg := Fmt(format, args...)
return newCmnError(cause).Stacktrace().Trace(1, msg)
}
}
//----------------------------------------
// Error & cmnError
/*
Usage with arbitrary error data:
```go
// Error construction
type MyError struct{}
var err1 error = NewErrorWithData(MyError{}, "my message")
...
// Wrapping
var err2 error = ErrorWrap(err1, "another message")
if (err1 != err2) { panic("should be the same")
...
// Error handling
switch err2.Data().(type){
case MyError: ...
default: ...
}
```
*/
type Error interface {
Error() string
Stacktrace() Error
Trace(offset int, format string, args ...interface{}) Error
Data() interface{}
}
// New Error with formatted message.
// The Error's Data will be a FmtError type.
func NewError(format string, args ...interface{}) Error {
err := FmtError{format, args}
return newCmnError(err)
}
// New Error with specified data.
func NewErrorWithData(data interface{}) Error {
return newCmnError(data)
}
type cmnError struct {
data interface{} // associated data
msgtraces []msgtraceItem // all messages traced
stacktrace []uintptr // first stack trace
}
var _ Error = &cmnError{}
// NOTE: do not expose.
func newCmnError(data interface{}) *cmnError {
return &cmnError{
data: data,
msgtraces: nil,
stacktrace: nil,
}
}
// Implements error.
func (err *cmnError) Error() string {
return fmt.Sprintf("%v", err)
}
// Captures a stacktrace if one was not already captured.
func (err *cmnError) Stacktrace() Error {
if err.stacktrace == nil {
var offset = 3
var depth = 32
err.stacktrace = captureStacktrace(offset, depth)
}
return err
}
// Add tracing information with msg.
// Set n=0 unless wrapped with some function, then n > 0.
func (err *cmnError) Trace(offset int, format string, args ...interface{}) Error {
msg := Fmt(format, args...)
return err.doTrace(msg, offset)
}
// Return the "data" of this error.
// Data could be used for error handling/switching,
// or for holding general error/debug information.
func (err *cmnError) Data() interface{} {
return err.data
}
func (err *cmnError) doTrace(msg string, n int) Error {
pc, _, _, _ := runtime.Caller(n + 2) // +1 for doTrace(). +1 for the caller.
// Include file & line number & msg.
// Do not include the whole stack trace.
err.msgtraces = append(err.msgtraces, msgtraceItem{
pc: pc,
msg: msg,
})
return err
}
func (err *cmnError) Format(s fmt.State, verb rune) {
switch verb {
case 'p':
s.Write([]byte(fmt.Sprintf("%p", &err)))
default:
if s.Flag('#') {
s.Write([]byte("--= Error =--\n"))
// Write data.
s.Write([]byte(fmt.Sprintf("Data: %#v\n", err.data)))
// Write msg trace items.
s.Write([]byte(fmt.Sprintf("Msg Traces:\n")))
for i, msgtrace := range err.msgtraces {
s.Write([]byte(fmt.Sprintf(" %4d %s\n", i, msgtrace.String())))
}
// Write stack trace.
if err.stacktrace != nil {
s.Write([]byte(fmt.Sprintf("Stack Trace:\n")))
for i, pc := range err.stacktrace {
fnc := runtime.FuncForPC(pc)
file, line := fnc.FileLine(pc)
s.Write([]byte(fmt.Sprintf(" %4d %s:%d\n", i, file, line)))
}
}
s.Write([]byte("--= /Error =--\n"))
} else {
// Write msg.
s.Write([]byte(fmt.Sprintf("Error{%v}", err.data))) // TODO tick-esc?
}
}
}
//----------------------------------------
// stacktrace & msgtraceItem
func captureStacktrace(offset int, depth int) []uintptr {
var pcs = make([]uintptr, depth)
n := runtime.Callers(offset, pcs)
return pcs[0:n]
}
type msgtraceItem struct {
pc uintptr
msg string
}
func (mti msgtraceItem) String() string {
fnc := runtime.FuncForPC(mti.pc)
file, line := fnc.FileLine(mti.pc)
return fmt.Sprintf("%s:%d - %s",
file, line,
mti.msg,
)
}
//----------------------------------------
// fmt error
/*
FmtError is the data type for NewError() (e.g. NewError().Data().(FmtError))
Theoretically it could be used to switch on the format string.
```go
// Error construction
var err1 error = NewError("invalid username %v", "BOB")
var err2 error = NewError("another kind of error")
...
// Error handling
switch err1.Data().(cmn.FmtError).Format() {
case "invalid username %v": ...
case "another kind of error": ...
default: ...
}
```
*/
type FmtError struct {
format string
args []interface{}
}
func (fe FmtError) Error() string {
return fmt.Sprintf(fe.format, fe.args...)
}
func (fe FmtError) String() string {
return fmt.Sprintf("FmtError{format:%v,args:%v}",
fe.format, fe.args)
}
func (fe FmtError) Format() string {
return fe.format
}
//----------------------------------------
// Panic wrappers
// XXX DEPRECATED
// A panic resulting from a sanity check means there is a programmer error
// and some guarantee is not satisfied.
// XXX DEPRECATED
func PanicSanity(v interface{}) {
panic(Fmt("Panicked on a Sanity Check: %v", v))
}
// A panic here means something has gone horribly wrong, in the form of data corruption or
// failure of the operating system. In a correct/healthy system, these should never fire.
// If they do, it's indicative of a much more serious problem.
// XXX DEPRECATED
func PanicCrisis(v interface{}) {
panic(Fmt("Panicked on a Crisis: %v", v))
}
// Indicates a failure of consensus. Someone was malicious or something has
// gone horribly wrong. These should really boot us into an "emergency-recover" mode
// XXX DEPRECATED
func PanicConsensus(v interface{}) {
panic(Fmt("Panicked on a Consensus Failure: %v", v))
}
// For those times when we're not sure if we should panic
// XXX DEPRECATED
func PanicQ(v interface{}) {
panic(Fmt("Panicked questionably: %v", v))
}
libs/common/errors_test.go
+101
View File
@@ -0,0 +1,101 @@
package common
import (
fmt "fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestErrorPanic(t *testing.T) {
type pnk struct {
msg string
}
capturePanic := func() (err Error) {
defer func() {
if r := recover(); r != nil {
err = ErrorWrap(r, "This is the message in ErrorWrap(r, message).")
}
}()
panic(pnk{"something"})
}
var err = capturePanic()
assert.Equal(t, pnk{"something"}, err.Data())
assert.Equal(t, "Error{{something}}", fmt.Sprintf("%v", err))
assert.Contains(t, fmt.Sprintf("%#v", err), "This is the message in ErrorWrap(r, message).")
assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0")
}
func TestErrorWrapSomething(t *testing.T) {
var err = ErrorWrap("something", "formatter%v%v", 0, 1)
assert.Equal(t, "something", err.Data())
assert.Equal(t, "Error{something}", fmt.Sprintf("%v", err))
assert.Regexp(t, `formatter01\n`, fmt.Sprintf("%#v", err))
assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0")
}
func TestErrorWrapNothing(t *testing.T) {
var err = ErrorWrap(nil, "formatter%v%v", 0, 1)
assert.Equal(t,
FmtError{"formatter%v%v", []interface{}{0, 1}},
err.Data())
assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err))
assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`)
assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0")
}
func TestErrorNewError(t *testing.T) {
var err = NewError("formatter%v%v", 0, 1)
assert.Equal(t,
FmtError{"formatter%v%v", []interface{}{0, 1}},
err.Data())
assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err))
assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`)
assert.NotContains(t, fmt.Sprintf("%#v", err), "Stack Trace")
}
func TestErrorNewErrorWithStacktrace(t *testing.T) {
var err = NewError("formatter%v%v", 0, 1).Stacktrace()
assert.Equal(t,
FmtError{"formatter%v%v", []interface{}{0, 1}},
err.Data())
assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err))
assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`)
assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0")
}
func TestErrorNewErrorWithTrace(t *testing.T) {
var err = NewError("formatter%v%v", 0, 1)
err.Trace(0, "trace %v", 1)
err.Trace(0, "trace %v", 2)
err.Trace(0, "trace %v", 3)
assert.Equal(t,
FmtError{"formatter%v%v", []interface{}{0, 1}},
err.Data())
assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err))
assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`)
dump := fmt.Sprintf("%#v", err)
assert.NotContains(t, dump, "Stack Trace")
assert.Regexp(t, `common/errors_test\.go:[0-9]+ - trace 1`, dump)
assert.Regexp(t, `common/errors_test\.go:[0-9]+ - trace 2`, dump)
assert.Regexp(t, `common/errors_test\.go:[0-9]+ - trace 3`, dump)
}
func TestErrorWrapError(t *testing.T) {
var err1 error = NewError("my message")
var err2 error = ErrorWrap(err1, "another message")
assert.Equal(t, err1, err2)
}
libs/common/heap.go
+125
View File
@@ -0,0 +1,125 @@
package common
import (
"bytes"
"container/heap"
)
/*
Example usage:
```
h := NewHeap()
h.Push("msg1", 1)
h.Push("msg3", 3)
h.Push("msg2", 2)
fmt.Println(h.Pop()) // msg1
fmt.Println(h.Pop()) // msg2
fmt.Println(h.Pop()) // msg3
```
*/
type Heap struct {
pq priorityQueue
}
func NewHeap() *Heap {
return &Heap{pq: make([]*pqItem, 0)}
}
func (h *Heap) Len() int64 {
return int64(len(h.pq))
}
func (h *Heap) Push(value interface{}, priority int) {
heap.Push(&h.pq, &pqItem{value: value, priority: cmpInt(priority)})
}
func (h *Heap) PushBytes(value interface{}, priority []byte) {
heap.Push(&h.pq, &pqItem{value: value, priority: cmpBytes(priority)})
}
func (h *Heap) PushComparable(value interface{}, priority Comparable) {
heap.Push(&h.pq, &pqItem{value: value, priority: priority})
}
func (h *Heap) Peek() interface{} {
if len(h.pq) == 0 {
return nil
}
return h.pq[0].value
}
func (h *Heap) Update(value interface{}, priority Comparable) {
h.pq.Update(h.pq[0], value, priority)
}
func (h *Heap) Pop() interface{} {
item := heap.Pop(&h.pq).(*pqItem)
return item.value
}
//-----------------------------------------------------------------------------
// From: http://golang.org/pkg/container/heap/#example__priorityQueue
type pqItem struct {
value interface{}
priority Comparable
index int
}
type priorityQueue []*pqItem
func (pq priorityQueue) Len() int { return len(pq) }
func (pq priorityQueue) Less(i, j int) bool {
return pq[i].priority.Less(pq[j].priority)
}
func (pq priorityQueue) Swap(i, j int) {
pq[i], pq[j] = pq[j], pq[i]
pq[i].index = i
pq[j].index = j
}
func (pq *priorityQueue) Push(x interface{}) {
n := len(*pq)
item := x.(*pqItem)
item.index = n
*pq = append(*pq, item)
}
func (pq *priorityQueue) Pop() interface{} {
old := *pq
n := len(old)
item := old[n-1]
item.index = -1 // for safety
*pq = old[0 : n-1]
return item
}
func (pq *priorityQueue) Update(item *pqItem, value interface{}, priority Comparable) {
item.value = value
item.priority = priority
heap.Fix(pq, item.index)
}
//--------------------------------------------------------------------------------
// Comparable
type Comparable interface {
Less(o interface{}) bool
}
type cmpInt int
func (i cmpInt) Less(o interface{}) bool {
return int(i) < int(o.(cmpInt))
}
type cmpBytes []byte
func (bz cmpBytes) Less(o interface{}) bool {
return bytes.Compare([]byte(bz), []byte(o.(cmpBytes))) < 0
}
libs/common/int.go
+65
View File
@@ -0,0 +1,65 @@
package common
import (
"encoding/binary"
"sort"
)
// Sort for []uint64
type Uint64Slice []uint64
func (p Uint64Slice) Len() int { return len(p) }
func (p Uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
func (p Uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p Uint64Slice) Sort() { sort.Sort(p) }
func SearchUint64s(a []uint64, x uint64) int {
return sort.Search(len(a), func(i int) bool { return a[i] >= x })
}
func (p Uint64Slice) Search(x uint64) int { return SearchUint64s(p, x) }
//--------------------------------------------------------------------------------
func PutUint64LE(dest []byte, i uint64) {
binary.LittleEndian.PutUint64(dest, i)
}
func GetUint64LE(src []byte) uint64 {
return binary.LittleEndian.Uint64(src)
}
func PutUint64BE(dest []byte, i uint64) {
binary.BigEndian.PutUint64(dest, i)
}
func GetUint64BE(src []byte) uint64 {
return binary.BigEndian.Uint64(src)
}
func PutInt64LE(dest []byte, i int64) {
binary.LittleEndian.PutUint64(dest, uint64(i))
}
func GetInt64LE(src []byte) int64 {
return int64(binary.LittleEndian.Uint64(src))
}
func PutInt64BE(dest []byte, i int64) {
binary.BigEndian.PutUint64(dest, uint64(i))
}
func GetInt64BE(src []byte) int64 {
return int64(binary.BigEndian.Uint64(src))
}
// IntInSlice returns true if a is found in the list.
func IntInSlice(a int, list []int) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
libs/common/int_test.go
+14
View File
@@ -0,0 +1,14 @@
package common
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestIntInSlice(t *testing.T) {
assert.True(t, IntInSlice(1, []int{1, 2, 3}))
assert.False(t, IntInSlice(4, []int{1, 2, 3}))
assert.True(t, IntInSlice(0, []int{0}))
assert.False(t, IntInSlice(0, []int{}))
}
libs/common/io.go
+74
View File
@@ -0,0 +1,74 @@
package common
import (
"bytes"
"errors"
"io"
)
type PrefixedReader struct {
Prefix []byte
reader io.Reader
}
func NewPrefixedReader(prefix []byte, reader io.Reader) *PrefixedReader {
return &PrefixedReader{prefix, reader}
}
func (pr *PrefixedReader) Read(p []byte) (n int, err error) {
if len(pr.Prefix) > 0 {
read := copy(p, pr.Prefix)
pr.Prefix = pr.Prefix[read:]
return read, nil
}
return pr.reader.Read(p)
}
// NOTE: Not goroutine safe
type BufferCloser struct {
bytes.Buffer
Closed bool
}
func NewBufferCloser(buf []byte) *BufferCloser {
return &BufferCloser{
*bytes.NewBuffer(buf),
false,
}
}
func (bc *BufferCloser) Close() error {
if bc.Closed {
return errors.New("BufferCloser already closed")
}
bc.Closed = true
return nil
}
func (bc *BufferCloser) Write(p []byte) (n int, err error) {
if bc.Closed {
return 0, errors.New("Cannot write to closed BufferCloser")
}
return bc.Buffer.Write(p)
}
func (bc *BufferCloser) WriteByte(c byte) error {
if bc.Closed {
return errors.New("Cannot write to closed BufferCloser")
}
return bc.Buffer.WriteByte(c)
}
func (bc *BufferCloser) WriteRune(r rune) (n int, err error) {
if bc.Closed {
return 0, errors.New("Cannot write to closed BufferCloser")
}
return bc.Buffer.WriteRune(r)
}
func (bc *BufferCloser) WriteString(s string) (n int, err error) {
if bc.Closed {
return 0, errors.New("Cannot write to closed BufferCloser")
}
return bc.Buffer.WriteString(s)
}
libs/common/kvpair.go
+67
View File
@@ -0,0 +1,67 @@
package common
import (
"bytes"
"sort"
)
//----------------------------------------
// KVPair
/*
Defined in types.proto
type KVPair struct {
Key []byte
Value []byte
}
*/
type KVPairs []KVPair
// Sorting
func (kvs KVPairs) Len() int { return len(kvs) }
func (kvs KVPairs) Less(i, j int) bool {
switch bytes.Compare(kvs[i].Key, kvs[j].Key) {
case -1:
return true
case 0:
return bytes.Compare(kvs[i].Value, kvs[j].Value) < 0
case 1:
return false
default:
panic("invalid comparison result")
}
}
func (kvs KVPairs) Swap(i, j int) { kvs[i], kvs[j] = kvs[j], kvs[i] }
func (kvs KVPairs) Sort() { sort.Sort(kvs) }
//----------------------------------------
// KI64Pair
/*
Defined in types.proto
type KI64Pair struct {
Key []byte
Value int64
}
*/
type KI64Pairs []KI64Pair
// Sorting
func (kvs KI64Pairs) Len() int { return len(kvs) }
func (kvs KI64Pairs) Less(i, j int) bool {
switch bytes.Compare(kvs[i].Key, kvs[j].Key) {
case -1:
return true
case 0:
return kvs[i].Value < kvs[j].Value
case 1:
return false
default:
panic("invalid comparison result")
}
}
func (kvs KI64Pairs) Swap(i, j int) { kvs[i], kvs[j] = kvs[j], kvs[i] }
func (kvs KI64Pairs) Sort() { sort.Sort(kvs) }
libs/common/math.go
+157
View File
@@ -0,0 +1,157 @@
package common
func MaxInt8(a, b int8) int8 {
if a > b {
return a
}
return b
}
func MaxUint8(a, b uint8) uint8 {
if a > b {
return a
}
return b
}
func MaxInt16(a, b int16) int16 {
if a > b {
return a
}
return b
}
func MaxUint16(a, b uint16) uint16 {
if a > b {
return a
}
return b
}
func MaxInt32(a, b int32) int32 {
if a > b {
return a
}
return b
}
func MaxUint32(a, b uint32) uint32 {
if a > b {
return a
}
return b
}
func MaxInt64(a, b int64) int64 {
if a > b {
return a
}
return b
}
func MaxUint64(a, b uint64) uint64 {
if a > b {
return a
}
return b
}
func MaxInt(a, b int) int {
if a > b {
return a
}
return b
}
func MaxUint(a, b uint) uint {
if a > b {
return a
}
return b
}
//-----------------------------------------------------------------------------
func MinInt8(a, b int8) int8 {
if a < b {
return a
}
return b
}
func MinUint8(a, b uint8) uint8 {
if a < b {
return a
}
return b
}
func MinInt16(a, b int16) int16 {
if a < b {
return a
}
return b
}
func MinUint16(a, b uint16) uint16 {
if a < b {
return a
}
return b
}
func MinInt32(a, b int32) int32 {
if a < b {
return a
}
return b
}
func MinUint32(a, b uint32) uint32 {
if a < b {
return a
}
return b
}
func MinInt64(a, b int64) int64 {
if a < b {
return a
}
return b
}
func MinUint64(a, b uint64) uint64 {
if a < b {
return a
}
return b
}
func MinInt(a, b int) int {
if a < b {
return a
}
return b
}
func MinUint(a, b uint) uint {
if a < b {
return a
}
return b
}
//-----------------------------------------------------------------------------
func ExpUint64(a, b uint64) uint64 {
accum := uint64(1)
for b > 0 {
if b&1 == 1 {
accum *= a
}
a *= a
b >>= 1
}
return accum
}
libs/common/net.go
+26
View File
@@ -0,0 +1,26 @@
package common
import (
"net"
"strings"
)
// Connect dials the given address and returns a net.Conn. The protoAddr argument should be prefixed with the protocol,
// eg. "tcp://127.0.0.1:8080" or "unix:///tmp/test.sock"
func Connect(protoAddr string) (net.Conn, error) {
proto, address := ProtocolAndAddress(protoAddr)
conn, err := net.Dial(proto, address)
return conn, err
}
// ProtocolAndAddress splits an address into the protocol and address components.
// For instance, "tcp://127.0.0.1:8080" will be split into "tcp" and "127.0.0.1:8080".
// If the address has no protocol prefix, the default is "tcp".
func ProtocolAndAddress(listenAddr string) (string, string) {
protocol, address := "tcp", listenAddr
parts := strings.SplitN(address, "://", 2)
if len(parts) == 2 {
protocol, address = parts[0], parts[1]
}
return protocol, address
}
libs/common/net_test.go
+38
View File
@@ -0,0 +1,38 @@
package common
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestProtocolAndAddress(t *testing.T) {
cases := []struct {
fullAddr string
proto string
addr string
}{
{
"tcp://mydomain:80",
"tcp",
"mydomain:80",
},
{
"mydomain:80",
"tcp",
"mydomain:80",
},
{
"unix://mydomain:80",
"unix",
"mydomain:80",
},
}
for _, c := range cases {
proto, addr := ProtocolAndAddress(c.fullAddr)
assert.Equal(t, proto, c.proto)
assert.Equal(t, addr, c.addr)
}
}
libs/common/nil.go
+29
View File
@@ -0,0 +1,29 @@
package common
import "reflect"
// Go lacks a simple and safe way to see if something is a typed nil.
// See:
// - https://dave.cheney.net/2017/08/09/typed-nils-in-go-2
// - https://groups.google.com/forum/#!topic/golang-nuts/wnH302gBa4I/discussion
// - https://github.com/golang/go/issues/21538
func IsTypedNil(o interface{}) bool {
rv := reflect.ValueOf(o)
switch rv.Kind() {
case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Slice:
return rv.IsNil()
default:
return false
}
}
// Returns true if it has zero length.
func IsEmpty(o interface{}) bool {
rv := reflect.ValueOf(o)
switch rv.Kind() {
case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String:
return rv.Len() == 0
default:
return false
}
}
libs/common/os.go
+195
View File
@@ -0,0 +1,195 @@
package common
import (
"bufio"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"os/signal"
"path/filepath"
"strings"
"syscall"
)
var gopath string
// GoPath returns GOPATH env variable value. If it is not set, this function
// will try to call `go env GOPATH` subcommand.
func GoPath() string {
if gopath != "" {
return gopath
}
path := os.Getenv("GOPATH")
if len(path) == 0 {
goCmd := exec.Command("go", "env", "GOPATH")
out, err := goCmd.Output()
if err != nil {
panic(fmt.Sprintf("failed to determine gopath: %v", err))
}
path = string(out)
}
gopath = path
return path
}
// TrapSignal catches the SIGTERM and executes cb function. After that it exits
// with code 1.
func TrapSignal(cb func()) {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
go func() {
for sig := range c {
fmt.Printf("captured %v, exiting...\n", sig)
if cb != nil {
cb()
}
os.Exit(1)
}
}()
select {}
}
// Kill the running process by sending itself SIGTERM.
func Kill() error {
p, err := os.FindProcess(os.Getpid())
if err != nil {
return err
}
return p.Signal(syscall.SIGTERM)
}
func Exit(s string) {
fmt.Printf(s + "\n")
os.Exit(1)
}
func EnsureDir(dir string, mode os.FileMode) error {
if _, err := os.Stat(dir); os.IsNotExist(err) {
err := os.MkdirAll(dir, mode)
if err != nil {
return fmt.Errorf("Could not create directory %v. %v", dir, err)
}
}
return nil
}
func IsDirEmpty(name string) (bool, error) {
f, err := os.Open(name)
if err != nil {
if os.IsNotExist(err) {
return true, err
}
// Otherwise perhaps a permission
// error or some other error.
return false, err
}
defer f.Close()
_, err = f.Readdirnames(1) // Or f.Readdir(1)
if err == io.EOF {
return true, nil
}
return false, err // Either not empty or error, suits both cases
}
func FileExists(filePath string) bool {
_, err := os.Stat(filePath)
return !os.IsNotExist(err)
}
func ReadFile(filePath string) ([]byte, error) {
return ioutil.ReadFile(filePath)
}
func MustReadFile(filePath string) []byte {
fileBytes, err := ioutil.ReadFile(filePath)
if err != nil {
Exit(Fmt("MustReadFile failed: %v", err))
return nil
}
return fileBytes
}
func WriteFile(filePath string, contents []byte, mode os.FileMode) error {
return ioutil.WriteFile(filePath, contents, mode)
}
func MustWriteFile(filePath string, contents []byte, mode os.FileMode) {
err := WriteFile(filePath, contents, mode)
if err != nil {
Exit(Fmt("MustWriteFile failed: %v", err))
}
}
// WriteFileAtomic creates a temporary file with data and the perm given and
// swaps it atomically with filename if successful.
func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error {
var (
dir = filepath.Dir(filename)
tempFile = filepath.Join(dir, "write-file-atomic-"+RandStr(32))
// Override in case it does exist, create in case it doesn't and force kernel
// flush, which still leaves the potential of lingering disk cache.
flag = os.O_WRONLY | os.O_CREATE | os.O_SYNC | os.O_TRUNC
)
f, err := os.OpenFile(tempFile, flag, perm)
if err != nil {
return err
}
// Clean up in any case. Defer stacking order is last-in-first-out.
defer os.Remove(f.Name())
defer f.Close()
if n, err := f.Write(data); err != nil {
return err
} else if n < len(data) {
return io.ErrShortWrite
}
// Close the file before renaming it, otherwise it will cause "The process
// cannot access the file because it is being used by another process." on windows.
f.Close()
return os.Rename(f.Name(), filename)
}
//--------------------------------------------------------------------------------
func Tempfile(prefix string) (*os.File, string) {
file, err := ioutil.TempFile("", prefix)
if err != nil {
PanicCrisis(err)
}
return file, file.Name()
}
func Tempdir(prefix string) (*os.File, string) {
tempDir := os.TempDir() + "/" + prefix + RandStr(12)
err := EnsureDir(tempDir, 0700)
if err != nil {
panic(Fmt("Error creating temp dir: %v", err))
}
dir, err := os.Open(tempDir)
if err != nil {
panic(Fmt("Error opening temp dir: %v", err))
}
return dir, tempDir
}
//--------------------------------------------------------------------------------
func Prompt(prompt string, defaultValue string) (string, error) {
fmt.Print(prompt)
reader := bufio.NewReader(os.Stdin)
line, err := reader.ReadString('\n')
if err != nil {
return defaultValue, err
}
line = strings.TrimSpace(line)
if line == "" {
return defaultValue, nil
}
return line, nil
}
libs/common/os_test.go
+91
View File
@@ -0,0 +1,91 @@
package common
import (
"bytes"
"io/ioutil"
"math/rand"
"os"
"testing"
"time"
)
func TestWriteFileAtomic(t *testing.T) {
var (
seed = rand.New(rand.NewSource(time.Now().UnixNano()))
data = []byte(RandStr(seed.Intn(2048)))
old = RandBytes(seed.Intn(2048))
perm os.FileMode = 0600
)
f, err := ioutil.TempFile("/tmp", "write-atomic-test-")
if err != nil {
t.Fatal(err)
}
defer os.Remove(f.Name())
if err = ioutil.WriteFile(f.Name(), old, 0664); err != nil {
t.Fatal(err)
}
if err = WriteFileAtomic(f.Name(), data, perm); err != nil {
t.Fatal(err)
}
rData, err := ioutil.ReadFile(f.Name())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, rData) {
t.Fatalf("data mismatch: %v != %v", data, rData)
}
stat, err := os.Stat(f.Name())
if err != nil {
t.Fatal(err)
}
if have, want := stat.Mode().Perm(), perm; have != want {
t.Errorf("have %v, want %v", have, want)
}
}
func TestGoPath(t *testing.T) {
// restore original gopath upon exit
path := os.Getenv("GOPATH")
defer func() {
_ = os.Setenv("GOPATH", path)
}()
err := os.Setenv("GOPATH", "~/testgopath")
if err != nil {
t.Fatal(err)
}
path = GoPath()
if path != "~/testgopath" {
t.Fatalf("should get GOPATH env var value, got %v", path)
}
os.Unsetenv("GOPATH")
path = GoPath()
if path != "~/testgopath" {
t.Fatalf("subsequent calls should return the same value, got %v", path)
}
}
func TestGoPathWithoutEnvVar(t *testing.T) {
// restore original gopath upon exit
path := os.Getenv("GOPATH")
defer func() {
_ = os.Setenv("GOPATH", path)
}()
os.Unsetenv("GOPATH")
// reset cache
gopath = ""
path = GoPath()
if path == "" || path == "~/testgopath" {
t.Fatalf("should get nonempty result of calling go env GOPATH, got %v", path)
}
}
libs/common/random.go
+357
View File
@@ -0,0 +1,357 @@
package common
import (
crand "crypto/rand"
mrand "math/rand"
"sync"
"time"
)
const (
strChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" // 62 characters
)
// pseudo random number generator.
// seeded with OS randomness (crand)
type Rand struct {
sync.Mutex
rand *mrand.Rand
}
var grand *Rand
func init() {
grand = NewRand()
grand.init()
}
func NewRand() *Rand {
rand := &Rand{}
rand.init()
return rand
}
func (r *Rand) init() {
bz := cRandBytes(8)
var seed uint64
for i := 0; i < 8; i++ {
seed |= uint64(bz[i])
seed <<= 8
}
r.reset(int64(seed))
}
func (r *Rand) reset(seed int64) {
r.rand = mrand.New(mrand.NewSource(seed))
}
//----------------------------------------
// Global functions
func Seed(seed int64) {
grand.Seed(seed)
}
func RandStr(length int) string {
return grand.Str(length)
}
func RandUint16() uint16 {
return grand.Uint16()
}
func RandUint32() uint32 {
return grand.Uint32()
}
func RandUint64() uint64 {
return grand.Uint64()
}
func RandUint() uint {
return grand.Uint()
}
func RandInt16() int16 {
return grand.Int16()
}
func RandInt32() int32 {
return grand.Int32()
}
func RandInt64() int64 {
return grand.Int64()
}
func RandInt() int {
return grand.Int()
}
func RandInt31() int32 {
return grand.Int31()
}
func RandInt31n(n int32) int32 {
return grand.Int31n(n)
}
func RandInt63() int64 {
return grand.Int63()
}
func RandInt63n(n int64) int64 {
return grand.Int63n(n)
}
func RandUint16Exp() uint16 {
return grand.Uint16Exp()
}
func RandUint32Exp() uint32 {
return grand.Uint32Exp()
}
func RandUint64Exp() uint64 {
return grand.Uint64Exp()
}
func RandFloat32() float32 {
return grand.Float32()
}
func RandFloat64() float64 {
return grand.Float64()
}
func RandTime() time.Time {
return grand.Time()
}
func RandBytes(n int) []byte {
return grand.Bytes(n)
}
func RandIntn(n int) int {
return grand.Intn(n)
}
func RandPerm(n int) []int {
return grand.Perm(n)
}
//----------------------------------------
// Rand methods
func (r *Rand) Seed(seed int64) {
r.Lock()
r.reset(seed)
r.Unlock()
}
// Constructs an alphanumeric string of given length.
// It is not safe for cryptographic usage.
func (r *Rand) Str(length int) string {
chars := []byte{}
MAIN_LOOP:
for {
val := r.Int63()
for i := 0; i < 10; i++ {
v := int(val & 0x3f) // rightmost 6 bits
if v >= 62 { // only 62 characters in strChars
val >>= 6
continue
} else {
chars = append(chars, strChars[v])
if len(chars) == length {
break MAIN_LOOP
}
val >>= 6
}
}
}
return string(chars)
}
// It is not safe for cryptographic usage.
func (r *Rand) Uint16() uint16 {
return uint16(r.Uint32() & (1<<16 - 1))
}
// It is not safe for cryptographic usage.
func (r *Rand) Uint32() uint32 {
r.Lock()
u32 := r.rand.Uint32()
r.Unlock()
return u32
}
// It is not safe for cryptographic usage.
func (r *Rand) Uint64() uint64 {
return uint64(r.Uint32())<<32 + uint64(r.Uint32())
}
// It is not safe for cryptographic usage.
func (r *Rand) Uint() uint {
r.Lock()
i := r.rand.Int()
r.Unlock()
return uint(i)
}
// It is not safe for cryptographic usage.
func (r *Rand) Int16() int16 {
return int16(r.Uint32() & (1<<16 - 1))
}
// It is not safe for cryptographic usage.
func (r *Rand) Int32() int32 {
return int32(r.Uint32())
}
// It is not safe for cryptographic usage.
func (r *Rand) Int64() int64 {
return int64(r.Uint64())
}
// It is not safe for cryptographic usage.
func (r *Rand) Int() int {
r.Lock()
i := r.rand.Int()
r.Unlock()
return i
}
// It is not safe for cryptographic usage.
func (r *Rand) Int31() int32 {
r.Lock()
i31 := r.rand.Int31()
r.Unlock()
return i31
}
// It is not safe for cryptographic usage.
func (r *Rand) Int31n(n int32) int32 {
r.Lock()
i31n := r.rand.Int31n(n)
r.Unlock()
return i31n
}
// It is not safe for cryptographic usage.
func (r *Rand) Int63() int64 {
r.Lock()
i63 := r.rand.Int63()
r.Unlock()
return i63
}
// It is not safe for cryptographic usage.
func (r *Rand) Int63n(n int64) int64 {
r.Lock()
i63n := r.rand.Int63n(n)
r.Unlock()
return i63n
}
// Distributed pseudo-exponentially to test for various cases
// It is not safe for cryptographic usage.
func (r *Rand) Uint16Exp() uint16 {
bits := r.Uint32() % 16
if bits == 0 {
return 0
}
n := uint16(1 << (bits - 1))
n += uint16(r.Int31()) & ((1 << (bits - 1)) - 1)
return n
}
// Distributed pseudo-exponentially to test for various cases
// It is not safe for cryptographic usage.
func (r *Rand) Uint32Exp() uint32 {
bits := r.Uint32() % 32
if bits == 0 {
return 0
}
n := uint32(1 << (bits - 1))
n += uint32(r.Int31()) & ((1 << (bits - 1)) - 1)
return n
}
// Distributed pseudo-exponentially to test for various cases
// It is not safe for cryptographic usage.
func (r *Rand) Uint64Exp() uint64 {
bits := r.Uint32() % 64
if bits == 0 {
return 0
}
n := uint64(1 << (bits - 1))
n += uint64(r.Int63()) & ((1 << (bits - 1)) - 1)
return n
}
// It is not safe for cryptographic usage.
func (r *Rand) Float32() float32 {
r.Lock()
f32 := r.rand.Float32()
r.Unlock()
return f32
}
// It is not safe for cryptographic usage.
func (r *Rand) Float64() float64 {
r.Lock()
f64 := r.rand.Float64()
r.Unlock()
return f64
}
// It is not safe for cryptographic usage.
func (r *Rand) Time() time.Time {
return time.Unix(int64(r.Uint64Exp()), 0)
}
// RandBytes returns n random bytes from the OS's source of entropy ie. via crypto/rand.
// It is not safe for cryptographic usage.
func (r *Rand) Bytes(n int) []byte {
// cRandBytes isn't guaranteed to be fast so instead
// use random bytes generated from the internal PRNG
bs := make([]byte, n)
for i := 0; i < len(bs); i++ {
bs[i] = byte(r.Int() & 0xFF)
}
return bs
}
// RandIntn returns, as an int, a non-negative pseudo-random number in [0, n).
// It panics if n <= 0.
// It is not safe for cryptographic usage.
func (r *Rand) Intn(n int) int {
r.Lock()
i := r.rand.Intn(n)
r.Unlock()
return i
}
// RandPerm returns a pseudo-random permutation of n integers in [0, n).
// It is not safe for cryptographic usage.
func (r *Rand) Perm(n int) []int {
r.Lock()
perm := r.rand.Perm(n)
r.Unlock()
return perm
}
// NOTE: This relies on the os's random number generator.
// For real security, we should salt that with some seed.
// See github.com/tendermint/go-crypto for a more secure reader.
func cRandBytes(numBytes int) []byte {
b := make([]byte, numBytes)
_, err := crand.Read(b)
if err != nil {
PanicCrisis(err)
}
return b
}
libs/common/random_test.go
+121
View File
@@ -0,0 +1,121 @@
package common
import (
"bytes"
"encoding/json"
"fmt"
mrand "math/rand"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestRandStr(t *testing.T) {
l := 243
s := RandStr(l)
assert.Equal(t, l, len(s))
}
func TestRandBytes(t *testing.T) {
l := 243
b := RandBytes(l)
assert.Equal(t, l, len(b))
}
func TestRandIntn(t *testing.T) {
n := 243
for i := 0; i < 100; i++ {
x := RandIntn(n)
assert.True(t, x < n)
}
}
// Test to make sure that we never call math.rand().
// We do this by ensuring that outputs are deterministic.
func TestDeterminism(t *testing.T) {
var firstOutput string
// Set math/rand's seed for the sake of debugging this test.
// (It isn't strictly necessary).
mrand.Seed(1)
for i := 0; i < 100; i++ {
output := testThemAll()
if i == 0 {
firstOutput = output
} else {
if firstOutput != output {
t.Errorf("Run #%d's output was different from first run.\nfirst: %v\nlast: %v",
i, firstOutput, output)
}
}
}
}
func testThemAll() string {
// Such determinism.
grand.reset(1)
// Use it.
out := new(bytes.Buffer)
perm := RandPerm(10)
blob, _ := json.Marshal(perm)
fmt.Fprintf(out, "perm: %s\n", blob)
fmt.Fprintf(out, "randInt: %d\n", RandInt())
fmt.Fprintf(out, "randUint: %d\n", RandUint())
fmt.Fprintf(out, "randIntn: %d\n", RandIntn(97))
fmt.Fprintf(out, "randInt31: %d\n", RandInt31())
fmt.Fprintf(out, "randInt32: %d\n", RandInt32())
fmt.Fprintf(out, "randInt63: %d\n", RandInt63())
fmt.Fprintf(out, "randInt64: %d\n", RandInt64())
fmt.Fprintf(out, "randUint32: %d\n", RandUint32())
fmt.Fprintf(out, "randUint64: %d\n", RandUint64())
fmt.Fprintf(out, "randUint16Exp: %d\n", RandUint16Exp())
fmt.Fprintf(out, "randUint32Exp: %d\n", RandUint32Exp())
fmt.Fprintf(out, "randUint64Exp: %d\n", RandUint64Exp())
return out.String()
}
func TestRngConcurrencySafety(t *testing.T) {
var wg sync.WaitGroup
for i := 0; i < 100; i++ {
wg.Add(1)
go func() {
defer wg.Done()
_ = RandUint64()
<-time.After(time.Millisecond * time.Duration(RandIntn(100)))
_ = RandPerm(3)
}()
}
wg.Wait()
}
func BenchmarkRandBytes10B(b *testing.B) {
benchmarkRandBytes(b, 10)
}
func BenchmarkRandBytes100B(b *testing.B) {
benchmarkRandBytes(b, 100)
}
func BenchmarkRandBytes1KiB(b *testing.B) {
benchmarkRandBytes(b, 1024)
}
func BenchmarkRandBytes10KiB(b *testing.B) {
benchmarkRandBytes(b, 10*1024)
}
func BenchmarkRandBytes100KiB(b *testing.B) {
benchmarkRandBytes(b, 100*1024)
}
func BenchmarkRandBytes1MiB(b *testing.B) {
benchmarkRandBytes(b, 1024*1024)
}
func benchmarkRandBytes(b *testing.B, n int) {
for i := 0; i < b.N; i++ {
_ = RandBytes(n)
}
b.ReportAllocs()
}
libs/common/repeat_timer.go
+232
View File
@@ -0,0 +1,232 @@
package common
import (
"sync"
"time"
)
// Used by RepeatTimer the first time,
// and every time it's Reset() after Stop().
type TickerMaker func(dur time.Duration) Ticker
// Ticker is a basic ticker interface.
type Ticker interface {
// Never changes, never closes.
Chan() <-chan time.Time
// Stopping a stopped Ticker will panic.
Stop()
}
//----------------------------------------
// defaultTicker
var _ Ticker = (*defaultTicker)(nil)
type defaultTicker time.Ticker
func defaultTickerMaker(dur time.Duration) Ticker {
ticker := time.NewTicker(dur)
return (*defaultTicker)(ticker)
}
// Implements Ticker
func (t *defaultTicker) Chan() <-chan time.Time {
return t.C
}
// Implements Ticker
func (t *defaultTicker) Stop() {
((*time.Ticker)(t)).Stop()
}
//----------------------------------------
// LogicalTickerMaker
// Construct a TickerMaker that always uses `source`.
// It's useful for simulating a deterministic clock.
func NewLogicalTickerMaker(source chan time.Time) TickerMaker {
return func(dur time.Duration) Ticker {
return newLogicalTicker(source, dur)
}
}
type logicalTicker struct {
source <-chan time.Time
ch chan time.Time
quit chan struct{}
}
func newLogicalTicker(source <-chan time.Time, interval time.Duration) Ticker {
lt := &logicalTicker{
source: source,
ch: make(chan time.Time),
quit: make(chan struct{}),
}
go lt.fireRoutine(interval)
return lt
}
// We need a goroutine to read times from t.source
// and fire on t.Chan() when `interval` has passed.
func (t *logicalTicker) fireRoutine(interval time.Duration) {
source := t.source
// Init `lasttime`
lasttime := time.Time{}
select {
case lasttime = <-source:
case <-t.quit:
return
}
// Init `lasttime` end
for {
select {
case newtime := <-source:
elapsed := newtime.Sub(lasttime)
if interval <= elapsed {
// Block for determinism until the ticker is stopped.
select {
case t.ch <- newtime:
case <-t.quit:
return
}
// Reset timeleft.
// Don't try to "catch up" by sending more.
// "Ticker adjusts the intervals or drops ticks to make up for
// slow receivers" - https://golang.org/pkg/time/#Ticker
lasttime = newtime
}
case <-t.quit:
return // done
}
}
}
// Implements Ticker
func (t *logicalTicker) Chan() <-chan time.Time {
return t.ch // immutable
}
// Implements Ticker
func (t *logicalTicker) Stop() {
close(t.quit) // it *should* panic when stopped twice.
}
//---------------------------------------------------------------------
/*
RepeatTimer repeatedly sends a struct{}{} to `.Chan()` after each `dur`
period. (It's good for keeping connections alive.)
A RepeatTimer must be stopped, or it will keep a goroutine alive.
*/
type RepeatTimer struct {
name string
ch chan time.Time
tm TickerMaker
mtx sync.Mutex
dur time.Duration
ticker Ticker
quit chan struct{}
}
// NewRepeatTimer returns a RepeatTimer with a defaultTicker.
func NewRepeatTimer(name string, dur time.Duration) *RepeatTimer {
return NewRepeatTimerWithTickerMaker(name, dur, defaultTickerMaker)
}
// NewRepeatTimerWithTicker returns a RepeatTimer with the given ticker
// maker.
func NewRepeatTimerWithTickerMaker(name string, dur time.Duration, tm TickerMaker) *RepeatTimer {
var t = &RepeatTimer{
name: name,
ch: make(chan time.Time),
tm: tm,
dur: dur,
ticker: nil,
quit: nil,
}
t.reset()
return t
}
// receive ticks on ch, send out on t.ch
func (t *RepeatTimer) fireRoutine(ch <-chan time.Time, quit <-chan struct{}) {
for {
select {
case tick := <-ch:
select {
case t.ch <- tick:
case <-quit:
return
}
case <-quit: // NOTE: `t.quit` races.
return
}
}
}
func (t *RepeatTimer) Chan() <-chan time.Time {
return t.ch
}
func (t *RepeatTimer) Stop() {
t.mtx.Lock()
defer t.mtx.Unlock()
t.stop()
}
// Wait the duration again before firing.
func (t *RepeatTimer) Reset() {
t.mtx.Lock()
defer t.mtx.Unlock()
t.reset()
}
//----------------------------------------
// Misc.
// CONTRACT: (non-constructor) caller should hold t.mtx.
func (t *RepeatTimer) reset() {
if t.ticker != nil {
t.stop()
}
t.ticker = t.tm(t.dur)
t.quit = make(chan struct{})
go t.fireRoutine(t.ticker.Chan(), t.quit)
}
// CONTRACT: caller should hold t.mtx.
func (t *RepeatTimer) stop() {
if t.ticker == nil {
/*
Similar to the case of closing channels twice:
https://groups.google.com/forum/#!topic/golang-nuts/rhxMiNmRAPk
Stopping a RepeatTimer twice implies that you do
not know whether you are done or not.
If you're calling stop on a stopped RepeatTimer,
you probably have race conditions.
*/
panic("Tried to stop a stopped RepeatTimer")
}
t.ticker.Stop()
t.ticker = nil
/*
From https://golang.org/pkg/time/#Ticker:
"Stop the ticker to release associated resources"
"After Stop, no more ticks will be sent"
So we shouldn't have to do the below.
select {
case <-t.ch:
// read off channel if there's anything there
default:
}
*/
close(t.quit)
}
libs/common/repeat_timer_test.go
+137
View File
@@ -0,0 +1,137 @@
package common
import (
"math/rand"
"sync"
"testing"
"time"
"github.com/fortytw2/leaktest"
"github.com/stretchr/testify/assert"
)
func TestDefaultTicker(t *testing.T) {
ticker := defaultTickerMaker(time.Millisecond * 10)
<-ticker.Chan()
ticker.Stop()
}
func TestRepeatTimer(t *testing.T) {
ch := make(chan time.Time, 100)
mtx := new(sync.Mutex)
// tick() fires from start to end
// (exclusive) in milliseconds with incr.
// It locks on mtx, so subsequent calls
// run in series.
tick := func(startMs, endMs, incrMs time.Duration) {
mtx.Lock()
go func() {
for tMs := startMs; tMs < endMs; tMs += incrMs {
lt := time.Time{}
lt = lt.Add(tMs * time.Millisecond)
ch <- lt
}
mtx.Unlock()
}()
}
// tock consumes Ticker.Chan() events and checks them against the ms in "timesMs".
tock := func(t *testing.T, rt *RepeatTimer, timesMs []int64) {
// Check against timesMs.
for _, timeMs := range timesMs {
tyme := <-rt.Chan()
sinceMs := tyme.Sub(time.Time{}) / time.Millisecond
assert.Equal(t, timeMs, int64(sinceMs))
}
// TODO detect number of running
// goroutines to ensure that
// no other times will fire.
// See https://github.com/tendermint/tendermint/libs/issues/120.
time.Sleep(time.Millisecond * 100)
done := true
select {
case <-rt.Chan():
done = false
default:
}
assert.True(t, done)
}
tm := NewLogicalTickerMaker(ch)
rt := NewRepeatTimerWithTickerMaker("bar", time.Second, tm)
/* NOTE: Useful for debugging deadlocks...
go func() {
time.Sleep(time.Second * 3)
trace := make([]byte, 102400)
count := runtime.Stack(trace, true)
fmt.Printf("Stack of %d bytes: %s\n", count, trace)
}()
*/
tick(0, 1000, 10)
tock(t, rt, []int64{})
tick(1000, 2000, 10)
tock(t, rt, []int64{1000})
tick(2005, 5000, 10)
tock(t, rt, []int64{2005, 3005, 4005})
tick(5001, 5999, 1)
// Read 5005 instead of 5001 because
// it's 1 second greater than 4005.
tock(t, rt, []int64{5005})
tick(6000, 7005, 1)
tock(t, rt, []int64{6005})
tick(7033, 8032, 1)
tock(t, rt, []int64{7033})
// After a reset, nothing happens
// until two ticks are received.
rt.Reset()
tock(t, rt, []int64{})
tick(8040, 8041, 1)
tock(t, rt, []int64{})
tick(9555, 9556, 1)
tock(t, rt, []int64{9555})
// After a stop, nothing more is sent.
rt.Stop()
tock(t, rt, []int64{})
// Another stop panics.
assert.Panics(t, func() { rt.Stop() })
}
func TestRepeatTimerReset(t *testing.T) {
// check that we are not leaking any go-routines
defer leaktest.Check(t)()
timer := NewRepeatTimer("test", 20*time.Millisecond)
defer timer.Stop()
// test we don't receive tick before duration ms.
select {
case <-timer.Chan():
t.Fatal("did not expect to receive tick")
default:
}
timer.Reset()
// test we receive tick after Reset is called
select {
case <-timer.Chan():
// all good
case <-time.After(40 * time.Millisecond):
t.Fatal("expected to receive tick after reset")
}
// just random calls
for i := 0; i < 100; i++ {
time.Sleep(time.Duration(rand.Intn(40)) * time.Millisecond)
timer.Reset()
}
}
libs/common/service.go
+205
View File
@@ -0,0 +1,205 @@
package common
import (
"errors"
"fmt"
"sync/atomic"
"github.com/tendermint/tendermint/libs/log"
)
var (
ErrAlreadyStarted = errors.New("already started")
ErrAlreadyStopped = errors.New("already stopped")
)
// Service defines a service that can be started, stopped, and reset.
type Service interface {
// Start the service.
// If it's already started or stopped, will return an error.
// If OnStart() returns an error, it's returned by Start()
Start() error
OnStart() error
// Stop the service.
// If it's already stopped, will return an error.
// OnStop must never error.
Stop() error
OnStop()
// Reset the service.
// Panics by default - must be overwritten to enable reset.
Reset() error
OnReset() error
// Return true if the service is running
IsRunning() bool
// Quit returns a channel, which is closed once service is stopped.
Quit() <-chan struct{}
// String representation of the service
String() string
// SetLogger sets a logger.
SetLogger(log.Logger)
}
/*
Classical-inheritance-style service declarations. Services can be started, then
stopped, then optionally restarted.
Users can override the OnStart/OnStop methods. In the absence of errors, these
methods are guaranteed to be called at most once. If OnStart returns an error,
service won't be marked as started, so the user can call Start again.
A call to Reset will panic, unless OnReset is overwritten, allowing
OnStart/OnStop to be called again.
The caller must ensure that Start and Stop are not called concurrently.
It is ok to call Stop without calling Start first.
Typical usage:
type FooService struct {
BaseService
// private fields
}
func NewFooService() *FooService {
fs := &FooService{
// init
}
fs.BaseService = *NewBaseService(log, "FooService", fs)
return fs
}
func (fs *FooService) OnStart() error {
fs.BaseService.OnStart() // Always call the overridden method.
// initialize private fields
// start subroutines, etc.
}
func (fs *FooService) OnStop() error {
fs.BaseService.OnStop() // Always call the overridden method.
// close/destroy private fields
// stop subroutines, etc.
}
*/
type BaseService struct {
Logger log.Logger
name string
started uint32 // atomic
stopped uint32 // atomic
quit chan struct{}
// The "subclass" of BaseService
impl Service
}
// NewBaseService creates a new BaseService.
func NewBaseService(logger log.Logger, name string, impl Service) *BaseService {
if logger == nil {
logger = log.NewNopLogger()
}
return &BaseService{
Logger: logger,
name: name,
quit: make(chan struct{}),
impl: impl,
}
}
// SetLogger implements Service by setting a logger.
func (bs *BaseService) SetLogger(l log.Logger) {
bs.Logger = l
}
// Start implements Service by calling OnStart (if defined). An error will be
// returned if the service is already running or stopped. Not to start the
// stopped service, you need to call Reset.
func (bs *BaseService) Start() error {
if atomic.CompareAndSwapUint32(&bs.started, 0, 1) {
if atomic.LoadUint32(&bs.stopped) == 1 {
bs.Logger.Error(Fmt("Not starting %v -- already stopped", bs.name), "impl", bs.impl)
return ErrAlreadyStopped
}
bs.Logger.Info(Fmt("Starting %v", bs.name), "impl", bs.impl)
err := bs.impl.OnStart()
if err != nil {
// revert flag
atomic.StoreUint32(&bs.started, 0)
return err
}
return nil
}
bs.Logger.Debug(Fmt("Not starting %v -- already started", bs.name), "impl", bs.impl)
return ErrAlreadyStarted
}
// OnStart implements Service by doing nothing.
// NOTE: Do not put anything in here,
// that way users don't need to call BaseService.OnStart()
func (bs *BaseService) OnStart() error { return nil }
// Stop implements Service by calling OnStop (if defined) and closing quit
// channel. An error will be returned if the service is already stopped.
func (bs *BaseService) Stop() error {
if atomic.CompareAndSwapUint32(&bs.stopped, 0, 1) {
bs.Logger.Info(Fmt("Stopping %v", bs.name), "impl", bs.impl)
bs.impl.OnStop()
close(bs.quit)
return nil
}
bs.Logger.Debug(Fmt("Stopping %v (ignoring: already stopped)", bs.name), "impl", bs.impl)
return ErrAlreadyStopped
}
// OnStop implements Service by doing nothing.
// NOTE: Do not put anything in here,
// that way users don't need to call BaseService.OnStop()
func (bs *BaseService) OnStop() {}
// Reset implements Service by calling OnReset callback (if defined). An error
// will be returned if the service is running.
func (bs *BaseService) Reset() error {
if !atomic.CompareAndSwapUint32(&bs.stopped, 1, 0) {
bs.Logger.Debug(Fmt("Can't reset %v. Not stopped", bs.name), "impl", bs.impl)
return fmt.Errorf("can't reset running %s", bs.name)
}
// whether or not we've started, we can reset
atomic.CompareAndSwapUint32(&bs.started, 1, 0)
bs.quit = make(chan struct{})
return bs.impl.OnReset()
}
// OnReset implements Service by panicking.
func (bs *BaseService) OnReset() error {
PanicSanity("The service cannot be reset")
return nil
}
// IsRunning implements Service by returning true or false depending on the
// service's state.
func (bs *BaseService) IsRunning() bool {
return atomic.LoadUint32(&bs.started) == 1 && atomic.LoadUint32(&bs.stopped) == 0
}
// Wait blocks until the service is stopped.
func (bs *BaseService) Wait() {
<-bs.quit
}
// String implements Servce by returning a string representation of the service.
func (bs *BaseService) String() string {
return bs.name
}
// Quit Implements Service by returning a quit channel.
func (bs *BaseService) Quit() <-chan struct{} {
return bs.quit
}
libs/common/service_test.go
+54
View File
@@ -0,0 +1,54 @@
package common
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
type testService struct {
BaseService
}
func (testService) OnReset() error {
return nil
}
func TestBaseServiceWait(t *testing.T) {
ts := &testService{}
ts.BaseService = *NewBaseService(nil, "TestService", ts)
ts.Start()
waitFinished := make(chan struct{})
go func() {
ts.Wait()
waitFinished <- struct{}{}
}()
go ts.Stop()
select {
case <-waitFinished:
// all good
case <-time.After(100 * time.Millisecond):
t.Fatal("expected Wait() to finish within 100 ms.")
}
}
func TestBaseServiceReset(t *testing.T) {
ts := &testService{}
ts.BaseService = *NewBaseService(nil, "TestService", ts)
ts.Start()
err := ts.Reset()
require.Error(t, err, "expected cant reset service error")
ts.Stop()
err = ts.Reset()
require.NoError(t, err)
err = ts.Start()
require.NoError(t, err)
}
libs/common/string.go
+89
View File
@@ -0,0 +1,89 @@
package common
import (
"encoding/hex"
"fmt"
"strings"
)
// Like fmt.Sprintf, but skips formatting if args are empty.
var Fmt = func(format string, a ...interface{}) string {
if len(a) == 0 {
return format
}
return fmt.Sprintf(format, a...)
}
// IsHex returns true for non-empty hex-string prefixed with "0x"
func IsHex(s string) bool {
if len(s) > 2 && strings.EqualFold(s[:2], "0x") {
_, err := hex.DecodeString(s[2:])
return err == nil
}
return false
}
// StripHex returns hex string without leading "0x"
func StripHex(s string) string {
if IsHex(s) {
return s[2:]
}
return s
}
// StringInSlice returns true if a is found the list.
func StringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
// SplitAndTrim slices s into all subslices separated by sep and returns a
// slice of the string s with all leading and trailing Unicode code points
// contained in cutset removed. If sep is empty, SplitAndTrim splits after each
// UTF-8 sequence. First part is equivalent to strings.SplitN with a count of
// -1.
func SplitAndTrim(s, sep, cutset string) []string {
if s == "" {
return []string{}
}
spl := strings.Split(s, sep)
for i := 0; i < len(spl); i++ {
spl[i] = strings.Trim(spl[i], cutset)
}
return spl
}
// Returns true if s is a non-empty printable non-tab ascii character.
func IsASCIIText(s string) bool {
if len(s) == 0 {
return false
}
for _, b := range []byte(s) {
if 32 <= b && b <= 126 {
// good
} else {
return false
}
}
return true
}
// NOTE: Assumes that s is ASCII as per IsASCIIText(), otherwise panics.
func ASCIITrim(s string) string {
r := make([]byte, 0, len(s))
for _, b := range []byte(s) {
if b == 32 {
continue // skip space
} else if 32 < b && b <= 126 {
r = append(r, b)
} else {
panic(fmt.Sprintf("non-ASCII (non-tab) char 0x%X", b))
}
}
return string(r)
}
libs/common/string_test.go
+74
View File
@@ -0,0 +1,74 @@
package common
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestStringInSlice(t *testing.T) {
assert.True(t, StringInSlice("a", []string{"a", "b", "c"}))
assert.False(t, StringInSlice("d", []string{"a", "b", "c"}))
assert.True(t, StringInSlice("", []string{""}))
assert.False(t, StringInSlice("", []string{}))
}
func TestIsHex(t *testing.T) {
notHex := []string{
"", " ", "a", "x", "0", "0x", "0X", "0x ", "0X ", "0X a",
"0xf ", "0x f", "0xp", "0x-",
"0xf", "0XBED", "0xF", "0xbed", // Odd lengths
}
for _, v := range notHex {
assert.False(t, IsHex(v), "%q is not hex", v)
}
hex := []string{
"0x00", "0x0a", "0x0F", "0xFFFFFF", "0Xdeadbeef", "0x0BED",
"0X12", "0X0A",
}
for _, v := range hex {
assert.True(t, IsHex(v), "%q is hex", v)
}
}
func TestSplitAndTrim(t *testing.T) {
testCases := []struct {
s string
sep string
cutset string
expected []string
}{
{"a,b,c", ",", " ", []string{"a", "b", "c"}},
{" a , b , c ", ",", " ", []string{"a", "b", "c"}},
{" a, b, c ", ",", " ", []string{"a", "b", "c"}},
{" , ", ",", " ", []string{"", ""}},
{" ", ",", " ", []string{""}},
}
for _, tc := range testCases {
assert.Equal(t, tc.expected, SplitAndTrim(tc.s, tc.sep, tc.cutset), "%s", tc.s)
}
}
func TestIsASCIIText(t *testing.T) {
notASCIIText := []string{
"", "\xC2", "\xC2\xA2", "\xFF", "\x80", "\xF0", "\n", "\t",
}
for _, v := range notASCIIText {
assert.False(t, IsHex(v), "%q is not ascii-text", v)
}
asciiText := []string{
" ", ".", "x", "$", "_", "abcdefg;", "-", "0x00", "0", "123",
}
for _, v := range asciiText {
assert.True(t, IsASCIIText(v), "%q is ascii-text", v)
}
}
func TestASCIITrim(t *testing.T) {
assert.Equal(t, ASCIITrim(" "), "")
assert.Equal(t, ASCIITrim(" a"), "a")
assert.Equal(t, ASCIITrim("a "), "a")
assert.Equal(t, ASCIITrim(" a "), "a")
assert.Panics(t, func() { ASCIITrim("\xC2\xA2") })
}
libs/common/throttle_timer.go
+75
View File
@@ -0,0 +1,75 @@
package common
import (
"sync"
"time"
)
/*
ThrottleTimer fires an event at most "dur" after each .Set() call.
If a short burst of .Set() calls happens, ThrottleTimer fires once.
If a long continuous burst of .Set() calls happens, ThrottleTimer fires
at most once every "dur".
*/
type ThrottleTimer struct {
Name string
Ch chan struct{}
quit chan struct{}
dur time.Duration
mtx sync.Mutex
timer *time.Timer
isSet bool
}
func NewThrottleTimer(name string, dur time.Duration) *ThrottleTimer {
var ch = make(chan struct{})
var quit = make(chan struct{})
var t = &ThrottleTimer{Name: name, Ch: ch, dur: dur, quit: quit}
t.mtx.Lock()
t.timer = time.AfterFunc(dur, t.fireRoutine)
t.mtx.Unlock()
t.timer.Stop()
return t
}
func (t *ThrottleTimer) fireRoutine() {
t.mtx.Lock()
defer t.mtx.Unlock()
select {
case t.Ch <- struct{}{}:
t.isSet = false
case <-t.quit:
// do nothing
default:
t.timer.Reset(t.dur)
}
}
func (t *ThrottleTimer) Set() {
t.mtx.Lock()
defer t.mtx.Unlock()
if !t.isSet {
t.isSet = true
t.timer.Reset(t.dur)
}
}
func (t *ThrottleTimer) Unset() {
t.mtx.Lock()
defer t.mtx.Unlock()
t.isSet = false
t.timer.Stop()
}
// For ease of .Stop()'ing services before .Start()'ing them,
// we ignore .Stop()'s on nil ThrottleTimers
func (t *ThrottleTimer) Stop() bool {
if t == nil {
return false
}
close(t.quit)
t.mtx.Lock()
defer t.mtx.Unlock()
return t.timer.Stop()
}
libs/common/throttle_timer_test.go
+78
View File
@@ -0,0 +1,78 @@
package common
import (
"sync"
"testing"
"time"
// make govet noshadow happy...
asrt "github.com/stretchr/testify/assert"
)
type thCounter struct {
input chan struct{}
mtx sync.Mutex
count int
}
func (c *thCounter) Increment() {
c.mtx.Lock()
c.count++
c.mtx.Unlock()
}
func (c *thCounter) Count() int {
c.mtx.Lock()
val := c.count
c.mtx.Unlock()
return val
}
// Read should run in a go-routine and
// updates count by one every time a packet comes in
func (c *thCounter) Read() {
for range c.input {
c.Increment()
}
}
func TestThrottle(test *testing.T) {
assert := asrt.New(test)
ms := 50
delay := time.Duration(ms) * time.Millisecond
longwait := time.Duration(2) * delay
t := NewThrottleTimer("foo", delay)
// start at 0
c := &thCounter{input: t.Ch}
assert.Equal(0, c.Count())
go c.Read()
// waiting does nothing
time.Sleep(longwait)
assert.Equal(0, c.Count())
// send one event adds one
t.Set()
time.Sleep(longwait)
assert.Equal(1, c.Count())
// send a burst adds one
for i := 0; i < 5; i++ {
t.Set()
}
time.Sleep(longwait)
assert.Equal(2, c.Count())
// send 12, over 2 delay sections, adds 3
short := time.Duration(ms/5) * time.Millisecond
for i := 0; i < 13; i++ {
t.Set()
time.Sleep(short)
}
time.Sleep(longwait)
assert.Equal(5, c.Count())
close(t.Ch)
}
libs/common/types.pb.go
+98
View File
@@ -0,0 +1,98 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: common/types.proto
/*
Package common is a generated protocol buffer package.
It is generated from these files:
common/types.proto
It has these top-level messages:
KVPair
KI64Pair
*/
//nolint: gas
package common
import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
// Define these here for compatibility but use tmlibs/common.KVPair.
type KVPair struct {
Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
}
func (m *KVPair) Reset() { *m = KVPair{} }
func (m *KVPair) String() string { return proto.CompactTextString(m) }
func (*KVPair) ProtoMessage() {}
func (*KVPair) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
func (m *KVPair) GetKey() []byte {
if m != nil {
return m.Key
}
return nil
}
func (m *KVPair) GetValue() []byte {
if m != nil {
return m.Value
}
return nil
}
// Define these here for compatibility but use tmlibs/common.KI64Pair.
type KI64Pair struct {
Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Value int64 `protobuf:"varint,2,opt,name=value" json:"value,omitempty"`
}
func (m *KI64Pair) Reset() { *m = KI64Pair{} }
func (m *KI64Pair) String() string { return proto.CompactTextString(m) }
func (*KI64Pair) ProtoMessage() {}
func (*KI64Pair) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *KI64Pair) GetKey() []byte {
if m != nil {
return m.Key
}
return nil
}
func (m *KI64Pair) GetValue() int64 {
if m != nil {
return m.Value
}
return 0
}
func init() {
proto.RegisterType((*KVPair)(nil), "common.KVPair")
proto.RegisterType((*KI64Pair)(nil), "common.KI64Pair")
}
func init() { proto.RegisterFile("common/types.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 107 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4a, 0xce, 0xcf, 0xcd,
0xcd, 0xcf, 0xd3, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62,
0x83, 0x88, 0x29, 0x19, 0x70, 0xb1, 0x79, 0x87, 0x05, 0x24, 0x66, 0x16, 0x09, 0x09, 0x70, 0x31,
0x67, 0xa7, 0x56, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0xf0, 0x04, 0x81, 0x98, 0x42, 0x22, 0x5c, 0xac,
0x65, 0x89, 0x39, 0xa5, 0xa9, 0x12, 0x4c, 0x60, 0x31, 0x08, 0x47, 0xc9, 0x88, 0x8b, 0xc3, 0xdb,
0xd3, 0xcc, 0x84, 0x18, 0x3d, 0xcc, 0x50, 0x3d, 0x49, 0x6c, 0x60, 0x4b, 0x8d, 0x01, 0x01, 0x00,
0x00, 0xff, 0xff, 0xd8, 0xf1, 0xc3, 0x8c, 0x8a, 0x00, 0x00, 0x00,
}
libs/common/types.proto
+17
View File
@@ -0,0 +1,17 @@
syntax = "proto3";
package common;
//----------------------------------------
// Abstract types
// Define these here for compatibility but use tmlibs/common.KVPair.
message KVPair {
bytes key = 1;
bytes value = 2;
}
// Define these here for compatibility but use tmlibs/common.KI64Pair.
message KI64Pair {
bytes key = 1;
int64 value = 2;
}
libs/common/word.go
+90
View File
@@ -0,0 +1,90 @@
package common
import (
"bytes"
"sort"
)
var (
Zero256 = Word256{0}
One256 = Word256{1}
)
type Word256 [32]byte
func (w Word256) String() string { return string(w[:]) }
func (w Word256) TrimmedString() string { return TrimmedString(w.Bytes()) }
func (w Word256) Copy() Word256 { return w }
func (w Word256) Bytes() []byte { return w[:] } // copied.
func (w Word256) Prefix(n int) []byte { return w[:n] }
func (w Word256) Postfix(n int) []byte { return w[32-n:] }
func (w Word256) IsZero() bool {
accum := byte(0)
for _, byt := range w {
accum |= byt
}
return accum == 0
}
func (w Word256) Compare(other Word256) int {
return bytes.Compare(w[:], other[:])
}
func Uint64ToWord256(i uint64) Word256 {
buf := [8]byte{}
PutUint64BE(buf[:], i)
return LeftPadWord256(buf[:])
}
func Int64ToWord256(i int64) Word256 {
buf := [8]byte{}
PutInt64BE(buf[:], i)
return LeftPadWord256(buf[:])
}
func RightPadWord256(bz []byte) (word Word256) {
copy(word[:], bz)
return
}
func LeftPadWord256(bz []byte) (word Word256) {
copy(word[32-len(bz):], bz)
return
}
func Uint64FromWord256(word Word256) uint64 {
buf := word.Postfix(8)
return GetUint64BE(buf)
}
func Int64FromWord256(word Word256) int64 {
buf := word.Postfix(8)
return GetInt64BE(buf)
}
//-------------------------------------
type Tuple256 struct {
First Word256
Second Word256
}
func (tuple Tuple256) Compare(other Tuple256) int {
firstCompare := tuple.First.Compare(other.First)
if firstCompare == 0 {
return tuple.Second.Compare(other.Second)
}
return firstCompare
}
func Tuple256Split(t Tuple256) (Word256, Word256) {
return t.First, t.Second
}
type Tuple256Slice []Tuple256
func (p Tuple256Slice) Len() int { return len(p) }
func (p Tuple256Slice) Less(i, j int) bool {
return p[i].Compare(p[j]) < 0
}
func (p Tuple256Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p Tuple256Slice) Sort() { sort.Sort(p) }