mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-04-17 14:11:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1842 from tendermint/bucky/merge-tmlibs

Browse Source 
Bucky/merge tmlibs
This commit is contained in:
Ethan Buchman
2018-07-02 12:31:39 -04:00
committed by GitHub
parent af703620d4 eccdce11c5
commit 60f233a4bc
310 changed files with 16322 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
Gopkg.lock generated
View File

@@ -16,7 +16,10 @@
[[projects]]
branch = "master"
name = "github.com/btcsuite/btcutil"
packages = ["base58"]
packages = [
"base58",
"bech32"
]
revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4"
[[projects]]
@@ -113,7 +116,6 @@
".",
"hcl/ast",
"hcl/parser",
"hcl/printer",
"hcl/scanner",
"hcl/strconv",
"hcl/token",
@@ -211,7 +213,7 @@
"nfs",
"xfs"
]
revision = "94663424ae5ae9856b40a9f170762b4197024661"
revision = "40f013a808ec4fa79def444a1a56de4d1727efcb"
[[projects]]
branch = "master"
@@ -237,8 +239,8 @@
[[projects]]
name = "github.com/spf13/cobra"
packages = ["."]
revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
version = "v0.0.3"
revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b"
version = "v0.0.1"
[[projects]]
branch = "master"
@@ -255,8 +257,8 @@
[[projects]]
name = "github.com/spf13/viper"
packages = ["."]
revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736"
version = "v1.0.2"
revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
version = "v1.0.0"
[[projects]]
name = "github.com/stretchr/testify"
@@ -284,20 +286,7 @@
"leveldb/table",
"leveldb/util"
]
revision = "e2150783cd35f5b607daca48afd8c57ec54cc995"
[[projects]]
name = "github.com/tendermint/abci"
packages = [
"client",
"example/code",
"example/counter",
"example/kvstore",
"server",
"types"
]
revision = "198dccf0ddfd1bb176f87657e3286a05a6ed9540"
version = "v0.12.0"
revision = "0d5a0ceb10cf9ab89fdd744cc8c50a83134f6697"
[[projects]]
branch = "master"
@@ -315,23 +304,6 @@
revision = "2106ca61d91029c931fd54968c2bb02dc96b1412"
version = "0.10.1"
[[projects]]
name = "github.com/tendermint/tmlibs"
packages = [
"autofile",
"cli",
"cli/flags",
"clist",
"common",
"db",
"flowrate",
"log",
"merkle",
"test"
]
revision = "692f1d86a6e2c0efa698fd1e4541b68c74ffaf38"
version = "v0.8.4"
[[projects]]
branch = "master"
name = "golang.org/x/crypto"
@@ -342,6 +314,7 @@
"curve25519",
"hkdf",
"internal/chacha20",
"internal/subtle",
"nacl/box",
"nacl/secretbox",
"openpgp/armor",
@@ -350,7 +323,7 @@
"ripemd160",
"salsa20/salsa"
]
revision = "8ac0e0d97ce45cd83d1d7243c060cb8461dda5e9"
revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602"
[[projects]]
branch = "master"
@@ -365,7 +338,7 @@
"netutil",
"trace"
]
revision = "db08ff08e8622530d9ed3a0e8ac279f6d4c02196"
revision = "4cb1c02c05b0e749b0365f61ae859a8e0cfceed9"
[[projects]]
branch = "master"
@@ -374,7 +347,7 @@
"cpu",
"unix"
]
revision = "a9e25c09b96b8870693763211309e213c6ef299d"
revision = "7138fd3d9dc8335c567ca206f4333fb75eb05d56"
[[projects]]
name = "golang.org/x/text"
@@ -407,9 +380,13 @@
packages = [
".",
"balancer",
"balancer/base",
"balancer/roundrobin",
"codes",
"connectivity",
"credentials",
"encoding",
"encoding/proto",
"grpclb/grpc_lb_v1/messages",
"grpclog",
"internal",
@@ -418,13 +395,15 @@
"naming",
"peer",
"resolver",
"resolver/dns",
"resolver/passthrough",
"stats",
"status",
"tap",
"transport"
]
revision = "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e"
version = "v1.7.5"
revision = "d11072e7ca9811b1100b80ca0269ac831f06d024"
version = "v1.11.3"
[[projects]]
name = "gopkg.in/yaml.v2"
@@ -435,6 +414,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "d17038089dd6383ff5028229d4026bb92f5c7adc7e9c1cd52584237e2e5fd431"
inputs-digest = "71753a9d4ece4252d23941f116f5ff66c0d5da730a099e5a9867491d223ed93b"
solver-name = "gps-cdcl"
solver-version = 1

18
Gopkg.toml
View File

@@ -35,15 +35,15 @@
[[constraint]]
name = "github.com/go-kit/kit"
version = "~0.6.0"
version = "=0.6.0"
[[constraint]]
name = "github.com/gogo/protobuf"
version = "~1.0.0"
version = "=1.0.0"
[[constraint]]
name = "github.com/golang/protobuf"
version = "~1.0.0"
version = "=1.0.0"
[[constraint]]
name = "github.com/gorilla/websocket"
@@ -51,7 +51,7 @@
[[constraint]]
name = "github.com/pkg/errors"
version = "~0.8.0"
version = "=0.8.0"
[[constraint]]
name = "github.com/rcrowley/go-metrics"
@@ -59,11 +59,11 @@
[[constraint]]
name = "github.com/spf13/cobra"
version = "~0.0.1"
version = "=0.0.1"
[[constraint]]
name = "github.com/spf13/viper"
version = "~1.0.0"
version = "=1.0.0"
[[constraint]]
name = "github.com/stretchr/testify"
@@ -73,13 +73,9 @@
name = "github.com/tendermint/go-amino"
version = "~0.10.1"
[[override]]
name = "github.com/tendermint/tmlibs"
version = "~0.8.4"
[[constraint]]
name = "google.golang.org/grpc"
version = "~1.7.3"
version = "~1.11.3"
# this got updated and broke, so locked to an old working commit ...
[[override]]

2
abci/client/client.go
View File

@@ -5,7 +5,7 @@ import (
"sync"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
const (

2
abci/client/grpc_client.go
View File

@@ -10,7 +10,7 @@ import (
grpc "google.golang.org/grpc"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
var _ Client = (*grpcClient)(nil)

2
abci/client/local_client.go
View File

@@ -4,7 +4,7 @@ import (
"sync"
types "github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
var _ Client = (*localClient)(nil)

2
abci/client/socket_client.go
View File

@@ -11,7 +11,7 @@ import (
"time"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
const reqQueueSize = 256 // TODO make configurable

4
abci/cmd/abci-cli/abci-cli.go
View File

@@ -11,8 +11,8 @@ import (
"github.com/spf13/cobra"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
abcicli "github.com/tendermint/tendermint/abci/client"
"github.com/tendermint/tendermint/abci/example/code"

2
abci/example/counter/counter.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/tendermint/tendermint/abci/example/code"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
type CounterApplication struct {

4
abci/example/example_test.go
View File

@@ -11,8 +11,8 @@ import (
"golang.org/x/net/context"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
abcicli "github.com/tendermint/tendermint/abci/client"
"github.com/tendermint/tendermint/abci/example/code"

2
abci/example/kvstore/helpers.go
View File

@@ -2,7 +2,7 @@ package kvstore
import (
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
// RandVal creates one random validator, with a key derived

4
abci/example/kvstore/kvstore.go
View File

@@ -8,8 +8,8 @@ import (
"github.com/tendermint/tendermint/abci/example/code"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
dbm "github.com/tendermint/tmlibs/db"
cmn "github.com/tendermint/tendermint/libs/common"
dbm "github.com/tendermint/tendermint/libs/db"
)
var (

4
abci/example/kvstore/kvstore_test.go
View File

@@ -8,8 +8,8 @@ import (
"github.com/stretchr/testify/require"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
abcicli "github.com/tendermint/tendermint/abci/client"
"github.com/tendermint/tendermint/abci/example/code"

6
abci/example/kvstore/persistent_kvstore.go
View File

@@ -9,9 +9,9 @@ import (
"github.com/tendermint/tendermint/abci/example/code"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
dbm "github.com/tendermint/tmlibs/db"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
dbm "github.com/tendermint/tendermint/libs/db"
"github.com/tendermint/tendermint/libs/log"
)
const (

2
abci/server/grpc_server.go
View File

@@ -6,7 +6,7 @@ import (
"google.golang.org/grpc"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
type GRPCServer struct {

2
abci/server/server.go
View File

@@ -13,7 +13,7 @@ import (
"fmt"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
func NewServer(protoAddr, transport string, app types.Application) (cmn.Service, error) {

2
abci/server/socket_server.go
View File

@@ -8,7 +8,7 @@ import (
"sync"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
// var maxNumberConnections = 2

2
abci/tests/benchmarks/parallel/parallel.go
View File

@@ -6,7 +6,7 @@ import (
"log"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
func main() {

2
abci/tests/benchmarks/simple/simple.go
View File

@@ -8,7 +8,7 @@ import (
"reflect"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
func main() {

2
abci/tests/server/client.go
View File

@@ -7,7 +7,7 @@ import (
abcicli "github.com/tendermint/tendermint/abci/client"
"github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
func InitChain(client abcicli.Client) error {

2
abci/tests/test_app/app.go
View File

@@ -7,7 +7,7 @@ import (
abcicli "github.com/tendermint/tendermint/abci/client"
"github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tmlibs/log"
"github.com/tendermint/tendermint/libs/log"
)
func startClient(abciType string) abcicli.Client {

2
abci/types/messages_test.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/stretchr/testify/assert"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
func TestMarshalJSON(t *testing.T) {

2
abci/types/types.pb.go
View File

@@ -50,7 +50,7 @@ import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
import _ "github.com/gogo/protobuf/gogoproto"
import common "github.com/tendermint/tmlibs/common"
import common "github.com/tendermint/tendermint/libs/common"
import context "golang.org/x/net/context"
import grpc "google.golang.org/grpc"

2
abci/types/util.go
View File

@@ -5,7 +5,7 @@ import (
"encoding/json"
"sort"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
//------------------------------------------------------------------------------

2
benchmarks/map_test.go
View File

@@ -3,7 +3,7 @@ package benchmarks
import (
"testing"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
func BenchmarkSomething(b *testing.B) {

2
benchmarks/os_test.go
View File

@@ -4,7 +4,7 @@ import (
"os"
"testing"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
func BenchmarkFileWrite(b *testing.B) {

2
benchmarks/simu/counter.go
View File

@@ -7,7 +7,7 @@ import (
"time"
rpcclient "github.com/tendermint/tendermint/rpc/lib/client"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
func main() {

6
blockchain/pool.go
View File

@@ -8,9 +8,9 @@ import (
"sync/atomic"
"time"
cmn "github.com/tendermint/tmlibs/common"
flow "github.com/tendermint/tmlibs/flowrate"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
flow "github.com/tendermint/tendermint/libs/flowrate"
"github.com/tendermint/tendermint/libs/log"
"github.com/tendermint/tendermint/p2p"
"github.com/tendermint/tendermint/types"

4
blockchain/pool_test.go
View File

@@ -5,8 +5,8 @@ import (
"testing"
"time"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
"github.com/tendermint/tendermint/p2p"
"github.com/tendermint/tendermint/types"

4
blockchain/reactor.go
View File

@@ -9,8 +9,8 @@ import (
"github.com/tendermint/tendermint/p2p"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
)
const (

6
blockchain/reactor_test.go
View File

@@ -4,9 +4,9 @@ import (
"net"
"testing"
cmn "github.com/tendermint/tmlibs/common"
dbm "github.com/tendermint/tmlibs/db"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
dbm "github.com/tendermint/tendermint/libs/db"
"github.com/tendermint/tendermint/libs/log"
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/p2p"

4
blockchain/store.go
View File

@@ -4,8 +4,8 @@ import (
"fmt"
"sync"
cmn "github.com/tendermint/tmlibs/common"
dbm "github.com/tendermint/tmlibs/db"
cmn "github.com/tendermint/tendermint/libs/common"
dbm "github.com/tendermint/tendermint/libs/db"
"github.com/tendermint/tendermint/types"
)

14
blockchain/store_test.go
View File

@@ -10,8 +10,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/tendermint/tmlibs/db"
"github.com/tendermint/tmlibs/log"
"github.com/tendermint/tendermint/libs/db"
"github.com/tendermint/tendermint/libs/log"
"github.com/tendermint/tendermint/types"
)
@@ -153,14 +153,14 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
parts: validPartSet,
seenCommit: seenCommit1,
corruptCommitInDB: true, // Corrupt the DB's commit entry
wantPanic: "Error reading block commit",
wantPanic: "unmarshal to types.Commit failed",
},
{
block: newBlock(&header1, commitAtH10),
parts: validPartSet,
seenCommit: seenCommit1,
wantPanic: "Error reading block",
wantPanic: "unmarshal to types.BlockMeta failed",
corruptBlockInDB: true, // Corrupt the DB's block entry
},
@@ -179,7 +179,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
seenCommit: seenCommit1,
corruptSeenCommitInDB: true,
wantPanic: "Error reading block seen commit",
wantPanic: "unmarshal to types.Commit failed",
},
{
@@ -287,7 +287,7 @@ func TestLoadBlockPart(t *testing.T) {
db.Set(calcBlockPartKey(height, index), []byte("Tendermint"))
res, _, panicErr = doFn(loadPart)
require.NotNil(t, panicErr, "expecting a non-nil panic")
require.Contains(t, panicErr.Error(), "Error reading block part")
require.Contains(t, panicErr.Error(), "unmarshal to types.Part failed")
// 3. A good block serialized and saved to the DB should be retrievable
db.Set(calcBlockPartKey(height, index), cdc.MustMarshalBinaryBare(part1))
@@ -316,7 +316,7 @@ func TestLoadBlockMeta(t *testing.T) {
db.Set(calcBlockMetaKey(height), []byte("Tendermint-Meta"))
res, _, panicErr = doFn(loadMeta)
require.NotNil(t, panicErr, "expecting a non-nil panic")
require.Contains(t, panicErr.Error(), "Error reading block meta")
require.Contains(t, panicErr.Error(), "unmarshal to types.BlockMeta")
// 3. A good blockMeta serialized and saved to the DB should be retrievable
meta := &types.BlockMeta{}

4
cmd/priv_val_server/main.go
View File

@@ -5,8 +5,8 @@ import (
"os"
crypto "github.com/tendermint/tendermint/crypto"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
"github.com/tendermint/tendermint/privval"
)

2
cmd/tendermint/commands/gen_node_key.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/spf13/cobra"
"github.com/tendermint/tendermint/p2p"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
// GenNodeKeyCmd allows the generation of a node key. It prints node's ID to

2
cmd/tendermint/commands/init.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/tendermint/tendermint/p2p"
"github.com/tendermint/tendermint/privval"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
// InitFilesCmd initialises a fresh Tendermint Core instance.

2
cmd/tendermint/commands/lite.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/spf13/cobra"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/lite/proxy"
rpcclient "github.com/tendermint/tendermint/rpc/client"

2
cmd/tendermint/commands/reset_priv_validator.go
View File

@@ -6,7 +6,7 @@ import (
"github.com/spf13/cobra"
"github.com/tendermint/tendermint/privval"
"github.com/tendermint/tmlibs/log"
"github.com/tendermint/tendermint/libs/log"
)
// ResetAllCmd removes the database of this Tendermint core

6
cmd/tendermint/commands/root.go
View File

@@ -7,9 +7,9 @@ import (
"github.com/spf13/viper"
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tmlibs/cli"
tmflags "github.com/tendermint/tmlibs/cli/flags"
"github.com/tendermint/tmlibs/log"
"github.com/tendermint/tendermint/libs/cli"
tmflags "github.com/tendermint/tendermint/libs/cli/flags"
"github.com/tendermint/tendermint/libs/log"
)
var (

4
cmd/tendermint/commands/root_test.go
View File

@@ -14,8 +14,8 @@ import (
"github.com/stretchr/testify/require"
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tmlibs/cli"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tendermint/libs/cli"
cmn "github.com/tendermint/tendermint/libs/common"
)
var (

2
cmd/tendermint/commands/testnet.go
View File

@@ -14,7 +14,7 @@ import (
"github.com/tendermint/tendermint/p2p"
"github.com/tendermint/tendermint/privval"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
var (

2
cmd/tendermint/main.go
View File

@@ -4,7 +4,7 @@ import (
"os"
"path/filepath"
"github.com/tendermint/tmlibs/cli"
"github.com/tendermint/tendermint/libs/cli"
cmd "github.com/tendermint/tendermint/cmd/tendermint/commands"
cfg "github.com/tendermint/tendermint/config"

2
config/toml.go
View File

@@ -6,7 +6,7 @@ import (
"path/filepath"
"text/template"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
var configTemplate *template.Template

2
consensus/byzantine_test.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/tendermint/tendermint/p2p"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
func init() {

6
consensus/common_test.go
View File

@@ -22,9 +22,9 @@ import (
"github.com/tendermint/tendermint/privval"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
dbm "github.com/tendermint/tmlibs/db"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
dbm "github.com/tendermint/tendermint/libs/db"
"github.com/tendermint/tendermint/libs/log"
"github.com/tendermint/tendermint/abci/example/counter"
"github.com/tendermint/tendermint/abci/example/kvstore"

2
consensus/mempool_test.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/tendermint/tendermint/abci/example/code"
abci "github.com/tendermint/tendermint/abci/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/types"
)

4
consensus/reactor.go
View File

@@ -9,8 +9,8 @@ import (
"github.com/pkg/errors"
amino "github.com/tendermint/go-amino"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
cstypes "github.com/tendermint/tendermint/consensus/types"
tmevents "github.com/tendermint/tendermint/libs/events"

4
consensus/reactor_test.go
View File

@@ -11,8 +11,8 @@ import (
"time"
"github.com/tendermint/tendermint/abci/example/kvstore"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/p2p"

8
consensus/replay.go
View File

@@ -11,10 +11,10 @@ import (
"time"
abci "github.com/tendermint/tendermint/abci/types"
//auto "github.com/tendermint/tmlibs/autofile"
cmn "github.com/tendermint/tmlibs/common"
dbm "github.com/tendermint/tmlibs/db"
"github.com/tendermint/tmlibs/log"
//auto "github.com/tendermint/tendermint/libs/autofile"
cmn "github.com/tendermint/tendermint/libs/common"
dbm "github.com/tendermint/tendermint/libs/db"
"github.com/tendermint/tendermint/libs/log"
"github.com/tendermint/tendermint/proxy"
sm "github.com/tendermint/tendermint/state"

6
consensus/replay_file.go
View File

@@ -16,9 +16,9 @@ import (
"github.com/tendermint/tendermint/proxy"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
dbm "github.com/tendermint/tmlibs/db"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
dbm "github.com/tendermint/tendermint/libs/db"
"github.com/tendermint/tendermint/libs/log"
)
const (

8
consensus/replay_test.go
View File

@@ -19,16 +19,16 @@ import (
"github.com/tendermint/tendermint/abci/example/kvstore"
abci "github.com/tendermint/tendermint/abci/types"
crypto "github.com/tendermint/tendermint/crypto"
auto "github.com/tendermint/tmlibs/autofile"
cmn "github.com/tendermint/tmlibs/common"
dbm "github.com/tendermint/tmlibs/db"
auto "github.com/tendermint/tendermint/libs/autofile"
cmn "github.com/tendermint/tendermint/libs/common"
dbm "github.com/tendermint/tendermint/libs/db"
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/privval"
"github.com/tendermint/tendermint/proxy"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
"github.com/tendermint/tmlibs/log"
"github.com/tendermint/tendermint/libs/log"
)
var consensusReplayConfig *cfg.Config

4
consensus/state.go
View File

@@ -10,8 +10,8 @@ import (
"time"
fail "github.com/ebuchman/fail-test"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
cfg "github.com/tendermint/tendermint/config"
cstypes "github.com/tendermint/tendermint/consensus/types"

4
consensus/state_test.go
View File

@@ -10,8 +10,8 @@ import (
cstypes "github.com/tendermint/tendermint/consensus/types"
tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
)
func init() {

4
consensus/ticker.go
View File

@@ -3,8 +3,8 @@ package consensus
import (
"time"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/log"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/log"
)
var (

2
consensus/types/height_vote_set.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/tendermint/tendermint/p2p"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
type RoundVoteSet struct {

2
consensus/types/height_vote_set_test.go
View File

@@ -6,7 +6,7 @@ import (
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
var config *cfg.Config // NOTE: must be reset for each _test.go file

2
consensus/types/peer_round_state.go
View File

@@ -5,7 +5,7 @@ import (
"time"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
//-----------------------------------------------------------------------------

2
consensus/types/round_state.go
View File

@@ -6,7 +6,7 @@ import (
"time"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
//-----------------------------------------------------------------------------

2
consensus/types/round_state_test.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/tendermint/go-amino"
"github.com/tendermint/tendermint/crypto"
"github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
func BenchmarkRoundStateDeepCopy(b *testing.B) {

2
consensus/version.go
View File

@@ -1,7 +1,7 @@
package consensus
import (
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
// kind of arbitrary

4
consensus/wal.go
View File

@@ -12,8 +12,8 @@ import (
amino "github.com/tendermint/go-amino"
"github.com/tendermint/tendermint/types"
auto "github.com/tendermint/tmlibs/autofile"
cmn "github.com/tendermint/tmlibs/common"
auto "github.com/tendermint/tendermint/libs/autofile"
cmn "github.com/tendermint/tendermint/libs/common"
)
const (

8
consensus/wal_generator.go
View File

@@ -17,10 +17,10 @@ import (
"github.com/tendermint/tendermint/proxy"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
auto "github.com/tendermint/tmlibs/autofile"
cmn "github.com/tendermint/tmlibs/common"
"github.com/tendermint/tmlibs/db"
"github.com/tendermint/tmlibs/log"
auto "github.com/tendermint/tendermint/libs/autofile"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/libs/db"
"github.com/tendermint/tendermint/libs/log"
)
// WALWithNBlocks generates a consensus WAL. It does this by spining up a

2
consensus/wal_test.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/tendermint/tendermint/consensus/types"
tmtypes "github.com/tendermint/tendermint/types"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

2
crypto/merkle/simple_map.go
View File

@@ -2,7 +2,7 @@ package merkle
import (
"github.com/tendermint/tendermint/crypto/tmhash"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
)
// Merkle tree from a map.

4
crypto/merkle/simple_tree_test.go
View File

@@ -3,8 +3,8 @@ package merkle
import (
"bytes"
cmn "github.com/tendermint/tmlibs/common"
. "github.com/tendermint/tmlibs/test"
cmn "github.com/tendermint/tendermint/libs/common"
. "github.com/tendermint/tendermint/libs/test"
"github.com/tendermint/tendermint/crypto/tmhash"
"testing"

2
crypto/pub_key.go
View File

@@ -12,7 +12,7 @@ import (
"github.com/tendermint/ed25519"
"github.com/tendermint/ed25519/extra25519"
cmn "github.com/tendermint/tmlibs/common"
cmn "github.com/tendermint/tendermint/libs/common"
"github.com/tendermint/tendermint/crypto/tmhash"
)

2
crypto/random.go
View File

@@ -9,7 +9,7 @@ import (
"io"
"sync"
. "github.com/tendermint/tmlibs/common"
. "github.com/tendermint/tendermint/libs/common"
)
var gRandInfo *randInfo

2
crypto/signature.go
View File

@@ -5,7 +5,7 @@ import (
"crypto/subtle"
. "github.com/tendermint/tmlibs/common"
. "github.com/tendermint/tendermint/libs/common"
)
func SignatureFromBytes(pubKeyBytes []byte) (pubKey Signature, err error) {

2
crypto/symmetric.go
View File

@@ -3,7 +3,7 @@ package crypto
import (
"errors"
. "github.com/tendermint/tmlibs/common"
. "github.com/tendermint/tendermint/libs/common"
"golang.org/x/crypto/nacl/secretbox"
)

6
evidence/pool.go
View File

@@ -4,9 +4,9 @@ import (
"fmt"
"sync"
clist "github.com/tendermint/tmlibs/clist"
dbm "github.com/tendermint/tmlibs/db"
"github.com/tendermint/tmlibs/log"
clist "github.com/tendermint/tendermint/libs/clist"
dbm "github.com/tendermint/tendermint/libs/db"
"github.com/tendermint/tendermint/libs/log"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"

2
evidence/pool_test.go
View File

@@ -9,7 +9,7 @@ import (
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
dbm "github.com/tendermint/tmlibs/db"
dbm "github.com/tendermint/tendermint/libs/db"
)
var mockState = sm.State{}

4
evidence/reactor.go
View File

@@ -6,8 +6,8 @@ import (
"time"
"github.com/tendermint/go-amino"
clist "github.com/tendermint/tmlibs/clist"
"github.com/tendermint/tmlibs/log"
clist "github.com/tendermint/tendermint/libs/clist"
"github.com/tendermint/tendermint/libs/log"
"github.com/tendermint/tendermint/p2p"
"github.com/tendermint/tendermint/types"

4
evidence/reactor_test.go
View File

@@ -10,8 +10,8 @@ import (
"github.com/go-kit/kit/log/term"
dbm "github.com/tendermint/tmlibs/db"
"github.com/tendermint/tmlibs/log"
dbm "github.com/tendermint/tendermint/libs/db"
"github.com/tendermint/tendermint/libs/log"
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/p2p"

2
evidence/store.go
View File

@@ -4,7 +4,7 @@ import (
"fmt"
"github.com/tendermint/tendermint/types"
dbm "github.com/tendermint/tmlibs/db"
dbm "github.com/tendermint/tendermint/libs/db"
)
/*

2
evidence/store_test.go
View File

@@ -5,7 +5,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/tendermint/tendermint/types"
dbm "github.com/tendermint/tmlibs/db"
dbm "github.com/tendermint/tendermint/libs/db"
)
//-------------------------------------------

19
libs/.editorconfig Normal file
View File

@@ -0,0 +1,19 @@
# top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
[Makefile]
indent_style = tab
[*.sh]
indent_style = tab
[*.proto]
indent_style = space
indent_size = 2

5
libs/.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
*.sw[opqr]
vendor
.glide
pubsub/query/fuzz_test/output

438
libs/CHANGELOG.md Normal file
View File

@@ -0,0 +1,438 @@
# Changelog
## 0.9.0
*June 24th, 2018*
BREAKING:
- [events, pubsub] Removed - moved to github.com/tendermint/tendermint
- [merkle] Use 20-bytes of SHA256 instead of RIPEMD160. NOTE: this package is
moving to github.com/tendermint/go-crypto !
- [common] Remove gogoproto from KVPair types
- [common] Error simplification, #220
FEATURES:
- [db/remotedb] New DB type using an external CLevelDB process via
GRPC
- [autofile] logjack command for piping stdin to a rotating file
- [bech32] New package. NOTE: should move out of here - it's just two small
functions
- [common] ColoredBytes([]byte) string for printing mixed ascii and bytes
- [db] DebugDB uses ColoredBytes()
## 0.8.4
*June 5, 2018*
IMPROVEMENTS:
- [autofile] Flush on Stop; Close() method to Flush and close file
## 0.8.3
*May 21, 2018*
FEATURES:
- [common] ASCIITrim()
## 0.8.2 (April 23rd, 2018)
FEATURES:
- [pubsub] TagMap, NewTagMap
- [merkle] SimpleProofsFromMap()
- [common] IsASCIIText()
- [common] PrefixEndBytes // e.g. increment or nil
- [common] BitArray.MarshalJSON/.UnmarshalJSON
- [common] BitArray uses 'x' not 'X' for String() and above.
- [db] DebugDB shows better colorized output
BUG FIXES:
- [common] Fix TestParallelAbort nondeterministic failure #201/#202
- [db] PrefixDB Iterator/ReverseIterator fixes
- [db] DebugDB fixes
## 0.8.1 (April 5th, 2018)
FEATURES:
- [common] Error.Error() includes cause
- [common] IsEmpty() for 0 length
## 0.8.0 (April 4th, 2018)
BREAKING:
- [merkle] `PutVarint->PutUvarint` in encodeByteSlice
- [db] batch.WriteSync()
- [common] Refactored and fixed `Parallel` function
- [common] Refactored `Rand` functionality
- [common] Remove unused `Right/LeftPadString` functions
- [common] Remove StackError, introduce Error interface (to replace use of pkg/errors)
FEATURES:
- [db] NewPrefixDB for a DB with all keys prefixed
- [db] NewDebugDB prints everything during operation
- [common] SplitAndTrim func
- [common] rand.Float64(), rand.Int63n(n), rand.Int31n(n) and global equivalents
- [common] HexBytes Format()
BUG FIXES:
- [pubsub] Fix unsubscribing
- [cli] Return config errors
- [common] Fix WriteFileAtomic Windows bug
## 0.7.1 (March 22, 2018)
IMPROVEMENTS:
- glide -> dep
BUG FIXES:
- [common] Fix panic in NewBitArray for negative bits
- [common] Fix and simplify WriteFileAtomic so it cleans up properly
## 0.7.0 (February 20, 2018)
BREAKING:
- [db] Major API upgrade. See `db/types.go`.
- [common] added `Quit() <-chan struct{}` to Service interface.
The returned channel is closed when service is stopped.
- [common] Remove HTTP functions
- [common] Heap.Push takes an `int`, new Heap.PushComparable takes the comparable.
- [logger] Removed. Use `log`
- [merkle] Major API updade - uses cmn.KVPairs.
- [cli] WriteDemoConfig -> WriteConfigValues
- [all] Remove go-wire dependency!
FEATURES:
- [db] New FSDB that uses the filesystem directly
- [common] HexBytes
- [common] KVPair and KI64Pair (protobuf based key-value pair objects)
IMPROVEMENTS:
- [clist] add WaitChan() to CList, NextWaitChan() and PrevWaitChan()
to CElement. These can be used instead of blocking `*Wait()` methods
if you need to be able to send quit signal and not block forever
- [common] IsHex handles 0x-prefix
BUG FIXES:
- [common] BitArray check for nil arguments
- [common] Fix memory leak in RepeatTimer
## 0.6.0 (December 29, 2017)
BREAKING:
- [cli] remove --root
- [pubsub] add String() method to Query interface
IMPROVEMENTS:
- [common] use a thread-safe and well seeded non-crypto rng
BUG FIXES
- [clist] fix misuse of wait group
- [common] introduce Ticker interface and logicalTicker for better testing of timers
## 0.5.0 (December 5, 2017)
BREAKING:
- [common] replace Service#Start, Service#Stop first return value (bool) with an
error (ErrAlreadyStarted, ErrAlreadyStopped)
- [common] replace Service#Reset first return value (bool) with an error
- [process] removed
FEATURES:
- [common] IntInSlice and StringInSlice functions
- [pubsub/query] introduce `Condition` struct, expose `Operator`, and add `query.Conditions()`
## 0.4.1 (November 27, 2017)
FEATURES:
- [common] `Keys()` method on `CMap`
IMPROVEMENTS:
- [log] complex types now encoded as "%+v" by default if `String()` method is undefined (previously resulted in error)
- [log] logger logs its own errors
BUG FIXES:
- [common] fixed `Kill()` to build on Windows (Windows does not have `syscall.Kill`)
## 0.4.0 (October 26, 2017)
BREAKING:
- [common] GoPath is now a function
- [db] `DB` and `Iterator` interfaces have new methods to better support iteration
FEATURES:
- [autofile] `Read([]byte)` and `Write([]byte)` methods on `Group` to support binary WAL
- [common] `Kill()` sends SIGTERM to the current process
IMPROVEMENTS:
- comments and linting
BUG FIXES:
- [events] fix allocation error prefixing cache with 1000 empty events
## 0.3.2 (October 2, 2017)
BUG FIXES:
- [autofile] fix AutoFile.Sync() to open file if it's been closed
- [db] fix MemDb.Close() to not empty the database (ie. its just a noop)
## 0.3.1 (September 22, 2017)
BUG FIXES:
- [common] fix WriteFileAtomic to not use /tmp, which can be on another device
## 0.3.0 (September 22, 2017)
BREAKING CHANGES:
- [log] logger functions no longer returns an error
- [common] NewBaseService takes the new logger
- [cli] RunCaptureWithArgs now captures stderr and stdout
- +func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (stdout, stderr string, err error)
- -func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (output string, err error)
FEATURES:
- [common] various common HTTP functionality
- [common] Date range parsing from string (ex. "2015-12-31:2017-12-31")
- [common] ProtocolAndAddress function
- [pubsub] New package for publish-subscribe with more advanced filtering
BUG FIXES:
- [common] fix atomicity of WriteFileAtomic by calling fsync
- [db] fix memDb iteration index out of range
- [autofile] fix Flush by calling fsync
## 0.2.2 (June 16, 2017)
FEATURES:
- [common] IsHex and StripHex for handling `0x` prefixed hex strings
- [log] NewTracingLogger returns a logger that output error traces, ala `github.com/pkg/errors`
IMPROVEMENTS:
- [cli] Error handling for tests
- [cli] Support dashes in ENV variables
BUG FIXES:
- [flowrate] Fix non-deterministic test failures
## 0.2.1 (June 2, 2017)
FEATURES:
- [cli] Log level parsing moved here from tendermint repo
## 0.2.0 (May 18, 2017)
BREAKING CHANGES:
- [common] NewBaseService takes the new logger
FEATURES:
- [cli] New library to standardize building command line tools
- [log] New logging library
BUG FIXES:
- [autofile] Close file before rotating
## 0.1.0 (May 1, 2017)
Initial release, combines what were previously independent repos:
- go-autofile
- go-clist
- go-common
- go-db
- go-events
- go-flowrate
- go-logger
- go-merkle
- go-process

3
libs/CODEOWNERS Normal file
View File

@@ -0,0 +1,3 @@
* @melekes @ebuchman
*.md @zramsay
*.rst @zramsay

281
libs/Gopkg.lock generated Normal file
View File

@@ -0,0 +1,281 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
branch = "master"
name = "github.com/btcsuite/btcutil"
packages = ["bech32"]
revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/fortytw2/leaktest"
packages = ["."]
revision = "3b724c3d7b8729a35bf4e577f71653aec6e53513"
[[projects]]
name = "github.com/fsnotify/fsnotify"
packages = ["."]
revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
version = "v1.4.7"
[[projects]]
name = "github.com/go-kit/kit"
packages = [
"log",
"log/level",
"log/term"
]
revision = "4dc7be5d2d12881735283bcab7352178e190fc71"
version = "v0.6.0"
[[projects]]
name = "github.com/go-logfmt/logfmt"
packages = ["."]
revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5"
version = "v0.3.0"
[[projects]]
name = "github.com/go-stack/stack"
packages = ["."]
revision = "817915b46b97fd7bb80e8ab6b69f01a53ac3eebf"
version = "v1.6.0"
[[projects]]
name = "github.com/golang/protobuf"
packages = [
"proto",
"ptypes",
"ptypes/any",
"ptypes/duration",
"ptypes/timestamp"
]
revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/golang/snappy"
packages = ["."]
revision = "553a641470496b2327abcac10b36396bd98e45c9"
[[projects]]
branch = "master"
name = "github.com/hashicorp/hcl"
packages = [
".",
"hcl/ast",
"hcl/parser",
"hcl/scanner",
"hcl/strconv",
"hcl/token",
"json/parser",
"json/scanner",
"json/token"
]
revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8"
[[projects]]
name = "github.com/inconshreveable/mousetrap"
packages = ["."]
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
version = "v1.0"
[[projects]]
branch = "master"
name = "github.com/jmhodges/levigo"
packages = ["."]
revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9"
[[projects]]
branch = "master"
name = "github.com/kr/logfmt"
packages = ["."]
revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
[[projects]]
name = "github.com/magiconair/properties"
packages = ["."]
revision = "49d762b9817ba1c2e9d0c69183c2b4a8b8f1d934"
[[projects]]
name = "github.com/mitchellh/mapstructure"
packages = ["."]
revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff"
[[projects]]
name = "github.com/pelletier/go-toml"
packages = ["."]
revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8"
version = "v1.1.0"
[[projects]]
name = "github.com/pkg/errors"
packages = ["."]
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
[[projects]]
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
name = "github.com/spf13/afero"
packages = [
".",
"mem"
]
revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c"
version = "v1.0.2"
[[projects]]
name = "github.com/spf13/cast"
packages = ["."]
revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4"
version = "v1.1.0"
[[projects]]
name = "github.com/spf13/cobra"
packages = ["."]
revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b"
version = "v0.0.1"
[[projects]]
branch = "master"
name = "github.com/spf13/jwalterweatherman"
packages = ["."]
revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394"
[[projects]]
name = "github.com/spf13/pflag"
packages = ["."]
revision = "97afa5e7ca8a08a383cb259e06636b5e2cc7897f"
[[projects]]
name = "github.com/spf13/viper"
packages = ["."]
revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
version = "v1.0.0"
[[projects]]
name = "github.com/stretchr/testify"
packages = [
"assert",
"require"
]
revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
version = "v1.2.1"
[[projects]]
name = "github.com/syndtr/goleveldb"
packages = [
"leveldb",
"leveldb/cache",
"leveldb/comparer",
"leveldb/errors",
"leveldb/filter",
"leveldb/iterator",
"leveldb/journal",
"leveldb/memdb",
"leveldb/opt",
"leveldb/storage",
"leveldb/table",
"leveldb/util"
]
revision = "b89cc31ef7977104127d34c1bd31ebd1a9db2199"
[[projects]]
branch = "master"
name = "golang.org/x/net"
packages = [
"context",
"http/httpguts",
"http2",
"http2/hpack",
"idna",
"internal/timeseries",
"trace"
]
revision = "d11bb6cd8e3c4e60239c9cb20ef68586d74500d0"
[[projects]]
name = "golang.org/x/sys"
packages = ["unix"]
revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
[[projects]]
name = "golang.org/x/text"
packages = [
"collate",
"collate/build",
"internal/colltab",
"internal/gen",
"internal/tag",
"internal/triegen",
"internal/ucd",
"language",
"secure/bidirule",
"transform",
"unicode/bidi",
"unicode/cldr",
"unicode/norm",
"unicode/rangetable"
]
revision = "c01e4764d870b77f8abe5096ee19ad20d80e8075"
[[projects]]
branch = "master"
name = "google.golang.org/genproto"
packages = ["googleapis/rpc/status"]
revision = "86e600f69ee4704c6efbf6a2a40a5c10700e76c2"
[[projects]]
name = "google.golang.org/grpc"
packages = [
".",
"balancer",
"balancer/base",
"balancer/roundrobin",
"codes",
"connectivity",
"credentials",
"encoding",
"encoding/proto",
"grpclb/grpc_lb_v1/messages",
"grpclog",
"internal",
"keepalive",
"metadata",
"naming",
"peer",
"resolver",
"resolver/dns",
"resolver/passthrough",
"stats",
"status",
"tap",
"transport"
]
revision = "d11072e7ca9811b1100b80ca0269ac831f06d024"
version = "v1.11.3"
[[projects]]
name = "gopkg.in/yaml.v2"
packages = ["."]
revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4"
version = "v2.0.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "e0c0af880b57928787ea78a820abefd2759e6aee4cba18e67ab36b80e62ad581"
solver-name = "gps-cdcl"
solver-version = 1

69
libs/Gopkg.toml Normal file
View File

@@ -0,0 +1,69 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
#
# [prune]
# non-go = false
# go-tests = true
# unused-packages = true
[[constraint]]
branch = "master"
name = "github.com/fortytw2/leaktest"
[[constraint]]
name = "github.com/go-kit/kit"
version = "0.6.0"
[[constraint]]
name = "github.com/go-logfmt/logfmt"
version = "0.3.0"
[[constraint]]
name = "github.com/gogo/protobuf"
version = "1.0.0"
[[constraint]]
branch = "master"
name = "github.com/jmhodges/levigo"
[[constraint]]
name = "github.com/pkg/errors"
version = "0.8.0"
[[constraint]]
name = "github.com/spf13/cobra"
version = "0.0.1"
[[constraint]]
name = "github.com/spf13/viper"
version = "1.0.0"
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.2.1"
[[constraint]]
name = "github.com/btcsuite/btcutil"
branch ="master"
[prune]
go-tests = true
unused-packages = true

193
libs/LICENSE Normal file
View File

@@ -0,0 +1,193 @@
Tendermint Libraries
Copyright (C) 2017 Tendermint
Apache License
Version 2.0, January 2004
https://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

137
libs/Makefile Normal file
View File

@@ -0,0 +1,137 @@
GOTOOLS = \
github.com/golang/dep/cmd/dep \
github.com/golang/protobuf/protoc-gen-go \
github.com/square/certstrap
# github.com/alecthomas/gometalinter.v2 \
GOTOOLS_CHECK = dep gometalinter.v2 protoc protoc-gen-go
INCLUDE = -I=. -I=${GOPATH}/src
all: check get_vendor_deps protoc grpc_dbserver build test install metalinter
check: check_tools
########################################
### Build
protoc:
## If you get the following error,
## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory"
## See https://stackoverflow.com/a/25518702
protoc $(INCLUDE) --go_out=plugins=grpc:. common/*.proto
@echo "--> adding nolint declarations to protobuf generated files"
@awk '/package common/ { print "//nolint: gas"; print; next }1' common/types.pb.go > common/types.pb.go.new
@mv common/types.pb.go.new common/types.pb.go
build:
# Nothing to build!
install:
# Nothing to install!
########################################
### Tools & dependencies
check_tools:
@# https://stackoverflow.com/a/25668869
@echo "Found tools: $(foreach tool,$(GOTOOLS_CHECK),\
$(if $(shell which $(tool)),$(tool),$(error "No $(tool) in PATH")))"
get_tools:
@echo "--> Installing tools"
go get -u -v $(GOTOOLS)
# @gometalinter.v2 --install
get_protoc:
@# https://github.com/google/protobuf/releases
curl -L https://github.com/google/protobuf/releases/download/v3.4.1/protobuf-cpp-3.4.1.tar.gz | tar xvz && \
cd protobuf-3.4.1 && \
DIST_LANG=cpp ./configure && \
make && \
make install && \
cd .. && \
rm -rf protobuf-3.4.1
update_tools:
@echo "--> Updating tools"
@go get -u $(GOTOOLS)
get_vendor_deps:
@rm -rf vendor/
@echo "--> Running dep ensure"
@dep ensure
########################################
### Testing
gen_certs: clean_certs
## Generating certificates for TLS testing...
certstrap init --common-name "tendermint.com" --passphrase ""
certstrap request-cert -ip "::" --passphrase ""
certstrap sign "::" --CA "tendermint.com" --passphrase ""
mv out/::.crt out/::.key db/remotedb
clean_certs:
## Cleaning TLS testing certificates...
rm -rf out
rm -f db/remotedb/::.crt db/remotedb/::.key
test: gen_certs
GOCACHE=off go test -tags gcc $(shell go list ./... | grep -v vendor)
make clean_certs
test100:
@for i in {1..100}; do make test; done
########################################
### Formatting, linting, and vetting
fmt:
@go fmt ./...
metalinter:
@echo "==> Running linter"
gometalinter.v2 --vendor --deadline=600s --disable-all \
--enable=deadcode \
--enable=goconst \
--enable=goimports \
--enable=gosimple \
--enable=ineffassign \
--enable=megacheck \
--enable=misspell \
--enable=staticcheck \
--enable=safesql \
--enable=structcheck \
--enable=unconvert \
--enable=unused \
--enable=varcheck \
--enable=vetshadow \
./...
#--enable=maligned \
#--enable=gas \
#--enable=aligncheck \
#--enable=dupl \
#--enable=errcheck \
#--enable=gocyclo \
#--enable=golint \ <== comments on anything exported
#--enable=gotype \
#--enable=interfacer \
#--enable=unparam \
#--enable=vet \
metalinter_all:
protoc $(INCLUDE) --lint_out=. types/*.proto
gometalinter.v2 --vendor --deadline=600s --enable-all --disable=lll ./...
# To avoid unintended conflicts with file names, always add to .PHONY
# unless there is a reason not to.
# https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
.PHONY: check protoc build check_tools get_tools get_protoc update_tools get_vendor_deps test fmt metalinter metalinter_all gen_certs clean_certs
grpc_dbserver:
protoc -I db/remotedb/proto/ db/remotedb/proto/defs.proto --go_out=plugins=grpc:db/remotedb/proto

49
libs/README.md Normal file
View File

@@ -0,0 +1,49 @@
# TMLIBS
This repo is a home for various small packages.
## autofile
Autofile is file access with automatic log rotation. A group of files is maintained and rotation happens
when the leading file gets too big. Provides a reader for reading from the file group.
## cli
CLI wraps the `cobra` and `viper` packages and handles some common elements of building a CLI like flags and env vars for the home directory and the logger.
## clist
Clist provides a linekd list that is safe for concurrent access by many readers.
## common
Common provides a hodgepodge of useful functions.
## db
DB provides a database interface and a number of implementions, including ones using an in-memory map, the filesystem directory structure,
an implemention of LevelDB in Go, and the official LevelDB in C.
## events
Events is a synchronous PubSub package.
## flowrate
Flowrate is a fork of https://github.com/mxk/go-flowrate that added a `SetREMA` method.
## log
Log is a log package structured around key-value pairs that allows logging level to be set differently for different keys.
## merkle
Merkle provides a simple static merkle tree and corresponding proofs.
## process
Process is a simple utility for spawning OS processes.
## pubsub
PubSub is an asynchronous PubSub package.

1
libs/autofile/README.md Normal file
View File

@@ -0,0 +1 @@
# go-autofile

142
libs/autofile/autofile.go Normal file
View File

@@ -0,0 +1,142 @@
package autofile
import (
"os"
"sync"
"time"
cmn "github.com/tendermint/tendermint/libs/common"
)
/* AutoFile usage
// Create/Append to ./autofile_test
af, err := OpenAutoFile("autofile_test")
if err != nil {
panic(err)
}
// Stream of writes.
// During this time, the file may be moved e.g. by logRotate.
for i := 0; i < 60; i++ {
af.Write([]byte(Fmt("LOOP(%v)", i)))
time.Sleep(time.Second)
}
// Close the AutoFile
err = af.Close()
if err != nil {
panic(err)
}
*/
const autoFileOpenDuration = 1000 * time.Millisecond
// Automatically closes and re-opens file for writing.
// This is useful for using a log file with the logrotate tool.
type AutoFile struct {
ID string
Path string
ticker *time.Ticker
mtx sync.Mutex
file *os.File
}
func OpenAutoFile(path string) (af *AutoFile, err error) {
af = &AutoFile{
ID: cmn.RandStr(12) + ":" + path,
Path: path,
ticker: time.NewTicker(autoFileOpenDuration),
}
if err = af.openFile(); err != nil {
return
}
go af.processTicks()
sighupWatchers.addAutoFile(af)
return
}
func (af *AutoFile) Close() error {
af.ticker.Stop()
err := af.closeFile()
sighupWatchers.removeAutoFile(af)
return err
}
func (af *AutoFile) processTicks() {
for {
_, ok := <-af.ticker.C
if !ok {
return // Done.
}
af.closeFile()
}
}
func (af *AutoFile) closeFile() (err error) {
af.mtx.Lock()
defer af.mtx.Unlock()
file := af.file
if file == nil {
return nil
}
af.file = nil
return file.Close()
}
func (af *AutoFile) Write(b []byte) (n int, err error) {
af.mtx.Lock()
defer af.mtx.Unlock()
if af.file == nil {
if err = af.openFile(); err != nil {
return
}
}
n, err = af.file.Write(b)
return
}
func (af *AutoFile) Sync() error {
af.mtx.Lock()
defer af.mtx.Unlock()
if af.file == nil {
if err := af.openFile(); err != nil {
return err
}
}
return af.file.Sync()
}
func (af *AutoFile) openFile() error {
file, err := os.OpenFile(af.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return err
}
af.file = file
return nil
}
func (af *AutoFile) Size() (int64, error) {
af.mtx.Lock()
defer af.mtx.Unlock()
if af.file == nil {
err := af.openFile()
if err != nil {
if err == os.ErrNotExist {
return 0, nil
}
return -1, err
}
}
stat, err := af.file.Stat()
if err != nil {
return -1, err
}
return stat.Size(), nil
}

71
libs/autofile/autofile_test.go Normal file
View File

@@ -0,0 +1,71 @@
package autofile
import (
"os"
"sync/atomic"
"syscall"
"testing"
"time"
cmn "github.com/tendermint/tendermint/libs/common"
)
func TestSIGHUP(t *testing.T) {
// First, create an AutoFile writing to a tempfile dir
file, name := cmn.Tempfile("sighup_test")
if err := file.Close(); err != nil {
t.Fatalf("Error creating tempfile: %v", err)
}
// Here is the actual AutoFile
af, err := OpenAutoFile(name)
if err != nil {
t.Fatalf("Error creating autofile: %v", err)
}
// Write to the file.
_, err = af.Write([]byte("Line 1\n"))
if err != nil {
t.Fatalf("Error writing to autofile: %v", err)
}
_, err = af.Write([]byte("Line 2\n"))
if err != nil {
t.Fatalf("Error writing to autofile: %v", err)
}
// Move the file over
err = os.Rename(name, name+"_old")
if err != nil {
t.Fatalf("Error moving autofile: %v", err)
}
// Send SIGHUP to self.
oldSighupCounter := atomic.LoadInt32(&sighupCounter)
syscall.Kill(syscall.Getpid(), syscall.SIGHUP)
// Wait a bit... signals are not handled synchronously.
for atomic.LoadInt32(&sighupCounter) == oldSighupCounter {
time.Sleep(time.Millisecond * 10)
}
// Write more to the file.
_, err = af.Write([]byte("Line 3\n"))
if err != nil {
t.Fatalf("Error writing to autofile: %v", err)
}
_, err = af.Write([]byte("Line 4\n"))
if err != nil {
t.Fatalf("Error writing to autofile: %v", err)
}
if err := af.Close(); err != nil {
t.Fatalf("Error closing autofile")
}
// Both files should exist
if body := cmn.MustReadFile(name + "_old"); string(body) != "Line 1\nLine 2\n" {
t.Errorf("Unexpected body %s", body)
}
if body := cmn.MustReadFile(name); string(body) != "Line 3\nLine 4\n" {
t.Errorf("Unexpected body %s", body)
}
}

107
libs/autofile/cmd/logjack.go Normal file
View File

@@ -0,0 +1,107 @@
package main
import (
"flag"
"fmt"
"io"
"os"
"strconv"
"strings"
auto "github.com/tendermint/tendermint/libs/autofile"
cmn "github.com/tendermint/tendermint/libs/common"
)
const Version = "0.0.1"
const readBufferSize = 1024 // 1KB at a time
// Parse command-line options
func parseFlags() (headPath string, chopSize int64, limitSize int64, version bool) {
var flagSet = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
var chopSizeStr, limitSizeStr string
flagSet.StringVar(&headPath, "head", "logjack.out", "Destination (head) file.")
flagSet.StringVar(&chopSizeStr, "chop", "100M", "Move file if greater than this")
flagSet.StringVar(&limitSizeStr, "limit", "10G", "Only keep this much (for each specified file). Remove old files.")
flagSet.BoolVar(&version, "version", false, "Version")
flagSet.Parse(os.Args[1:])
chopSize = parseBytesize(chopSizeStr)
limitSize = parseBytesize(limitSizeStr)
return
}
func main() {
// Read options
headPath, chopSize, limitSize, version := parseFlags()
if version {
fmt.Printf("logjack version %v\n", Version)
return
}
// Open Group
group, err := auto.OpenGroup(headPath)
if err != nil {
fmt.Printf("logjack couldn't create output file %v\n", headPath)
os.Exit(1)
}
group.SetHeadSizeLimit(chopSize)
group.SetTotalSizeLimit(limitSize)
err = group.Start()
if err != nil {
fmt.Printf("logjack couldn't start with file %v\n", headPath)
os.Exit(1)
}
go func() {
// Forever, read from stdin and write to AutoFile.
buf := make([]byte, readBufferSize)
for {
n, err := os.Stdin.Read(buf)
group.Write(buf[:n])
group.Flush()
if err != nil {
group.Stop()
if err == io.EOF {
os.Exit(0)
} else {
fmt.Println("logjack errored")
os.Exit(1)
}
}
}
}()
// Trap signal
cmn.TrapSignal(func() {
fmt.Println("logjack shutting down")
})
}
func parseBytesize(chopSize string) int64 {
// Handle suffix multiplier
var multiplier int64 = 1
if strings.HasSuffix(chopSize, "T") {
multiplier = 1042 * 1024 * 1024 * 1024
chopSize = chopSize[:len(chopSize)-1]
}
if strings.HasSuffix(chopSize, "G") {
multiplier = 1042 * 1024 * 1024
chopSize = chopSize[:len(chopSize)-1]
}
if strings.HasSuffix(chopSize, "M") {
multiplier = 1042 * 1024
chopSize = chopSize[:len(chopSize)-1]
}
if strings.HasSuffix(chopSize, "K") {
multiplier = 1042
chopSize = chopSize[:len(chopSize)-1]
}
// Parse the numeric part
chopSizeInt, err := strconv.Atoi(chopSize)
if err != nil {
panic(err)
}
return int64(chopSizeInt) * multiplier
}

747
libs/autofile/group.go Normal file
View File

@@ -0,0 +1,747 @@
package autofile
import (
"bufio"
"errors"
"fmt"
"io"
"log"
"os"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
cmn "github.com/tendermint/tendermint/libs/common"
)
const (
groupCheckDuration = 5000 * time.Millisecond
defaultHeadSizeLimit = 10 * 1024 * 1024 // 10MB
defaultTotalSizeLimit = 1 * 1024 * 1024 * 1024 // 1GB
maxFilesToRemove = 4 // needs to be greater than 1
)
/*
You can open a Group to keep restrictions on an AutoFile, like
the maximum size of each chunk, and/or the total amount of bytes
stored in the group.
The first file to be written in the Group.Dir is the head file.
Dir/
- <HeadPath>
Once the Head file reaches the size limit, it will be rotated.
Dir/
- <HeadPath>.000 // First rolled file
- <HeadPath> // New head path, starts empty.
// The implicit index is 001.
As more files are written, the index numbers grow...
Dir/
- <HeadPath>.000 // First rolled file
- <HeadPath>.001 // Second rolled file
- ...
- <HeadPath> // New head path
The Group can also be used to binary-search for some line,
assuming that marker lines are written occasionally.
*/
type Group struct {
cmn.BaseService
ID string
Head *AutoFile // The head AutoFile to write to
headBuf *bufio.Writer
Dir string // Directory that contains .Head
ticker *time.Ticker
mtx sync.Mutex
headSizeLimit int64
totalSizeLimit int64
minIndex int // Includes head
maxIndex int // Includes head, where Head will move to
// TODO: When we start deleting files, we need to start tracking GroupReaders
// and their dependencies.
}
// OpenGroup creates a new Group with head at headPath. It returns an error if
// it fails to open head file.
func OpenGroup(headPath string) (g *Group, err error) {
dir := path.Dir(headPath)
head, err := OpenAutoFile(headPath)
if err != nil {
return nil, err
}
g = &Group{
ID: "group:" + head.ID,
Head: head,
headBuf: bufio.NewWriterSize(head, 4096*10),
Dir: dir,
ticker: time.NewTicker(groupCheckDuration),
headSizeLimit: defaultHeadSizeLimit,
totalSizeLimit: defaultTotalSizeLimit,
minIndex: 0,
maxIndex: 0,
}
g.BaseService = *cmn.NewBaseService(nil, "Group", g)
gInfo := g.readGroupInfo()
g.minIndex = gInfo.MinIndex
g.maxIndex = gInfo.MaxIndex
return
}
// OnStart implements Service by starting the goroutine that checks file and
// group limits.
func (g *Group) OnStart() error {
go g.processTicks()
return nil
}
// OnStop implements Service by stopping the goroutine described above.
// NOTE: g.Head must be closed separately using Close.
func (g *Group) OnStop() {
g.ticker.Stop()
g.Flush() // flush any uncommitted data
}
// Close closes the head file. The group must be stopped by this moment.
func (g *Group) Close() {
g.Flush() // flush any uncommitted data
g.mtx.Lock()
_ = g.Head.closeFile()
g.mtx.Unlock()
}
// SetHeadSizeLimit allows you to overwrite default head size limit - 10MB.
func (g *Group) SetHeadSizeLimit(limit int64) {
g.mtx.Lock()
g.headSizeLimit = limit
g.mtx.Unlock()
}
// HeadSizeLimit returns the current head size limit.
func (g *Group) HeadSizeLimit() int64 {
g.mtx.Lock()
defer g.mtx.Unlock()
return g.headSizeLimit
}
// SetTotalSizeLimit allows you to overwrite default total size limit of the
// group - 1GB.
func (g *Group) SetTotalSizeLimit(limit int64) {
g.mtx.Lock()
g.totalSizeLimit = limit
g.mtx.Unlock()
}
// TotalSizeLimit returns total size limit of the group.
func (g *Group) TotalSizeLimit() int64 {
g.mtx.Lock()
defer g.mtx.Unlock()
return g.totalSizeLimit
}
// MaxIndex returns index of the last file in the group.
func (g *Group) MaxIndex() int {
g.mtx.Lock()
defer g.mtx.Unlock()
return g.maxIndex
}
// MinIndex returns index of the first file in the group.
func (g *Group) MinIndex() int {
g.mtx.Lock()
defer g.mtx.Unlock()
return g.minIndex
}
// Write writes the contents of p into the current head of the group. It
// returns the number of bytes written. If nn < len(p), it also returns an
// error explaining why the write is short.
// NOTE: Writes are buffered so they don't write synchronously
// TODO: Make it halt if space is unavailable
func (g *Group) Write(p []byte) (nn int, err error) {
g.mtx.Lock()
defer g.mtx.Unlock()
return g.headBuf.Write(p)
}
// WriteLine writes line into the current head of the group. It also appends "\n".
// NOTE: Writes are buffered so they don't write synchronously
// TODO: Make it halt if space is unavailable
func (g *Group) WriteLine(line string) error {
g.mtx.Lock()
defer g.mtx.Unlock()
_, err := g.headBuf.Write([]byte(line + "\n"))
return err
}
// Flush writes any buffered data to the underlying file and commits the
// current content of the file to stable storage.
func (g *Group) Flush() error {
g.mtx.Lock()
defer g.mtx.Unlock()
err := g.headBuf.Flush()
if err == nil {
err = g.Head.Sync()
}
return err
}
func (g *Group) processTicks() {
for {
_, ok := <-g.ticker.C
if !ok {
return // Done.
}
g.checkHeadSizeLimit()
g.checkTotalSizeLimit()
}
}
// NOTE: for testing
func (g *Group) stopTicker() {
g.ticker.Stop()
}
// NOTE: this function is called manually in tests.
func (g *Group) checkHeadSizeLimit() {
limit := g.HeadSizeLimit()
if limit == 0 {
return
}
size, err := g.Head.Size()
if err != nil {
panic(err)
}
if size >= limit {
g.RotateFile()
}
}
func (g *Group) checkTotalSizeLimit() {
limit := g.TotalSizeLimit()
if limit == 0 {
return
}
gInfo := g.readGroupInfo()
totalSize := gInfo.TotalSize
for i := 0; i < maxFilesToRemove; i++ {
index := gInfo.MinIndex + i
if totalSize < limit {
return
}
if index == gInfo.MaxIndex {
// Special degenerate case, just do nothing.
log.Println("WARNING: Group's head " + g.Head.Path + "may grow without bound")
return
}
pathToRemove := filePathForIndex(g.Head.Path, index, gInfo.MaxIndex)
fileInfo, err := os.Stat(pathToRemove)
if err != nil {
log.Println("WARNING: Failed to fetch info for file @" + pathToRemove)
continue
}
err = os.Remove(pathToRemove)
if err != nil {
log.Println(err)
return
}
totalSize -= fileInfo.Size()
}
}
// RotateFile causes group to close the current head and assign it some index.
// Note it does not create a new head.
func (g *Group) RotateFile() {
g.mtx.Lock()
defer g.mtx.Unlock()
headPath := g.Head.Path
if err := g.Head.closeFile(); err != nil {
panic(err)
}
indexPath := filePathForIndex(headPath, g.maxIndex, g.maxIndex+1)
if err := os.Rename(headPath, indexPath); err != nil {
panic(err)
}
g.maxIndex++
}
// NewReader returns a new group reader.
// CONTRACT: Caller must close the returned GroupReader.
func (g *Group) NewReader(index int) (*GroupReader, error) {
r := newGroupReader(g)
err := r.SetIndex(index)
if err != nil {
return nil, err
}
return r, nil
}
// Returns -1 if line comes after, 0 if found, 1 if line comes before.
type SearchFunc func(line string) (int, error)
// Searches for the right file in Group, then returns a GroupReader to start
// streaming lines.
// Returns true if an exact match was found, otherwise returns the next greater
// line that starts with prefix.
// CONTRACT: Caller must close the returned GroupReader
func (g *Group) Search(prefix string, cmp SearchFunc) (*GroupReader, bool, error) {
g.mtx.Lock()
minIndex, maxIndex := g.minIndex, g.maxIndex
g.mtx.Unlock()
// Now minIndex/maxIndex may change meanwhile,
// but it shouldn't be a big deal
// (maybe we'll want to limit scanUntil though)
for {
curIndex := (minIndex + maxIndex + 1) / 2
// Base case, when there's only 1 choice left.
if minIndex == maxIndex {
r, err := g.NewReader(maxIndex)
if err != nil {
return nil, false, err
}
match, err := scanUntil(r, prefix, cmp)
if err != nil {
r.Close()
return nil, false, err
}
return r, match, err
}
// Read starting roughly at the middle file,
// until we find line that has prefix.
r, err := g.NewReader(curIndex)
if err != nil {
return nil, false, err
}
foundIndex, line, err := scanNext(r, prefix)
r.Close()
if err != nil {
return nil, false, err
}
// Compare this line to our search query.
val, err := cmp(line)
if err != nil {
return nil, false, err
}
if val < 0 {
// Line will come later
minIndex = foundIndex
} else if val == 0 {
// Stroke of luck, found the line
r, err := g.NewReader(foundIndex)
if err != nil {
return nil, false, err
}
match, err := scanUntil(r, prefix, cmp)
if !match {
panic("Expected match to be true")
}
if err != nil {
r.Close()
return nil, false, err
}
return r, true, err
} else {
// We passed it
maxIndex = curIndex - 1
}
}
}
// Scans and returns the first line that starts with 'prefix'
// Consumes line and returns it.
func scanNext(r *GroupReader, prefix string) (int, string, error) {
for {
line, err := r.ReadLine()
if err != nil {
return 0, "", err
}
if !strings.HasPrefix(line, prefix) {
continue
}
index := r.CurIndex()
return index, line, nil
}
}
// Returns true iff an exact match was found.
// Pushes line, does not consume it.
func scanUntil(r *GroupReader, prefix string, cmp SearchFunc) (bool, error) {
for {
line, err := r.ReadLine()
if err != nil {
return false, err
}
if !strings.HasPrefix(line, prefix) {
continue
}
val, err := cmp(line)
if err != nil {
return false, err
}
if val < 0 {
continue
} else if val == 0 {
r.PushLine(line)
return true, nil
} else {
r.PushLine(line)
return false, nil
}
}
}
// Searches backwards for the last line in Group with prefix.
// Scans each file forward until the end to find the last match.
func (g *Group) FindLast(prefix string) (match string, found bool, err error) {
g.mtx.Lock()
minIndex, maxIndex := g.minIndex, g.maxIndex
g.mtx.Unlock()
r, err := g.NewReader(maxIndex)
if err != nil {
return "", false, err
}
defer r.Close()
// Open files from the back and read
GROUP_LOOP:
for i := maxIndex; i >= minIndex; i-- {
err := r.SetIndex(i)
if err != nil {
return "", false, err
}
// Scan each line and test whether line matches
for {
line, err := r.ReadLine()
if err == io.EOF {
if found {
return match, found, nil
}
continue GROUP_LOOP
} else if err != nil {
return "", false, err
}
if strings.HasPrefix(line, prefix) {
match = line
found = true
}
if r.CurIndex() > i {
if found {
return match, found, nil
}
continue GROUP_LOOP
}
}
}
return
}
// GroupInfo holds information about the group.
type GroupInfo struct {
MinIndex int // index of the first file in the group, including head
MaxIndex int // index of the last file in the group, including head
TotalSize int64 // total size of the group
HeadSize int64 // size of the head
}
// Returns info after scanning all files in g.Head's dir.
func (g *Group) ReadGroupInfo() GroupInfo {
g.mtx.Lock()
defer g.mtx.Unlock()
return g.readGroupInfo()
}
// Index includes the head.
// CONTRACT: caller should have called g.mtx.Lock
func (g *Group) readGroupInfo() GroupInfo {
groupDir := filepath.Dir(g.Head.Path)
headBase := filepath.Base(g.Head.Path)
var minIndex, maxIndex int = -1, -1
var totalSize, headSize int64 = 0, 0
dir, err := os.Open(groupDir)
if err != nil {
panic(err)
}
defer dir.Close()
fiz, err := dir.Readdir(0)
if err != nil {
panic(err)
}
// For each file in the directory, filter by pattern
for _, fileInfo := range fiz {
if fileInfo.Name() == headBase {
fileSize := fileInfo.Size()
totalSize += fileSize
headSize = fileSize
continue
} else if strings.HasPrefix(fileInfo.Name(), headBase) {
fileSize := fileInfo.Size()
totalSize += fileSize
indexedFilePattern := regexp.MustCompile(`^.+\.([0-9]{3,})$`)
submatch := indexedFilePattern.FindSubmatch([]byte(fileInfo.Name()))
if len(submatch) != 0 {
// Matches
fileIndex, err := strconv.Atoi(string(submatch[1]))
if err != nil {
panic(err)
}
if maxIndex < fileIndex {
maxIndex = fileIndex
}
if minIndex == -1 || fileIndex < minIndex {
minIndex = fileIndex
}
}
}
}
// Now account for the head.
if minIndex == -1 {
// If there were no numbered files,
// then the head is index 0.
minIndex, maxIndex = 0, 0
} else {
// Otherwise, the head file is 1 greater
maxIndex++
}
return GroupInfo{minIndex, maxIndex, totalSize, headSize}
}
func filePathForIndex(headPath string, index int, maxIndex int) string {
if index == maxIndex {
return headPath
}
return fmt.Sprintf("%v.%03d", headPath, index)
}
//--------------------------------------------------------------------------------
// GroupReader provides an interface for reading from a Group.
type GroupReader struct {
*Group
mtx sync.Mutex
curIndex int
curFile *os.File
curReader *bufio.Reader
curLine []byte
}
func newGroupReader(g *Group) *GroupReader {
return &GroupReader{
Group: g,
curIndex: 0,
curFile: nil,
curReader: nil,
curLine: nil,
}
}
// Close closes the GroupReader by closing the cursor file.
func (gr *GroupReader) Close() error {
gr.mtx.Lock()
defer gr.mtx.Unlock()
if gr.curReader != nil {
err := gr.curFile.Close()
gr.curIndex = 0
gr.curReader = nil
gr.curFile = nil
gr.curLine = nil
return err
}
return nil
}
// Read implements io.Reader, reading bytes from the current Reader
// incrementing index until enough bytes are read.
func (gr *GroupReader) Read(p []byte) (n int, err error) {
lenP := len(p)
if lenP == 0 {
return 0, errors.New("given empty slice")
}
gr.mtx.Lock()
defer gr.mtx.Unlock()
// Open file if not open yet
if gr.curReader == nil {
if err = gr.openFile(gr.curIndex); err != nil {
return 0, err
}
}
// Iterate over files until enough bytes are read
var nn int
for {
nn, err = gr.curReader.Read(p[n:])
n += nn
if err == io.EOF {
if n >= lenP {
return n, nil
}
// Open the next file
if err1 := gr.openFile(gr.curIndex + 1); err1 != nil {
return n, err1
}
} else if err != nil {
return n, err
} else if nn == 0 { // empty file
return n, err
}
}
}
// ReadLine reads a line (without delimiter).
// just return io.EOF if no new lines found.
func (gr *GroupReader) ReadLine() (string, error) {
gr.mtx.Lock()
defer gr.mtx.Unlock()
// From PushLine
if gr.curLine != nil {
line := string(gr.curLine)
gr.curLine = nil
return line, nil
}
// Open file if not open yet
if gr.curReader == nil {
err := gr.openFile(gr.curIndex)
if err != nil {
return "", err
}
}
// Iterate over files until line is found
var linePrefix string
for {
bytesRead, err := gr.curReader.ReadBytes('\n')
if err == io.EOF {
// Open the next file
if err1 := gr.openFile(gr.curIndex + 1); err1 != nil {
return "", err1
}
if len(bytesRead) > 0 && bytesRead[len(bytesRead)-1] == byte('\n') {
return linePrefix + string(bytesRead[:len(bytesRead)-1]), nil
}
linePrefix += string(bytesRead)
continue
} else if err != nil {
return "", err
}
return linePrefix + string(bytesRead[:len(bytesRead)-1]), nil
}
}
// IF index > gr.Group.maxIndex, returns io.EOF
// CONTRACT: caller should hold gr.mtx
func (gr *GroupReader) openFile(index int) error {
// Lock on Group to ensure that head doesn't move in the meanwhile.
gr.Group.mtx.Lock()
defer gr.Group.mtx.Unlock()
if index > gr.Group.maxIndex {
return io.EOF
}
curFilePath := filePathForIndex(gr.Head.Path, index, gr.Group.maxIndex)
curFile, err := os.Open(curFilePath)
if err != nil {
return err
}
curReader := bufio.NewReader(curFile)
// Update gr.cur*
if gr.curFile != nil {
gr.curFile.Close() // TODO return error?
}
gr.curIndex = index
gr.curFile = curFile
gr.curReader = curReader
gr.curLine = nil
return nil
}
// PushLine makes the given line the current one, so the next time somebody
// calls ReadLine, this line will be returned.
// panics if called twice without calling ReadLine.
func (gr *GroupReader) PushLine(line string) {
gr.mtx.Lock()
defer gr.mtx.Unlock()
if gr.curLine == nil {
gr.curLine = []byte(line)
} else {
panic("PushLine failed, already have line")
}
}
// CurIndex returns cursor's file index.
func (gr *GroupReader) CurIndex() int {
gr.mtx.Lock()
defer gr.mtx.Unlock()
return gr.curIndex
}
// SetIndex sets the cursor's file index to index by opening a file at this
// position.
func (gr *GroupReader) SetIndex(index int) error {
gr.mtx.Lock()
defer gr.mtx.Unlock()
return gr.openFile(index)
}
//--------------------------------------------------------------------------------
// A simple SearchFunc that assumes that the marker is of form
// <prefix><number>.
// For example, if prefix is '#HEIGHT:', the markers of expected to be of the form:
//
// #HEIGHT:1
// ...
// #HEIGHT:2
// ...
func MakeSimpleSearchFunc(prefix string, target int) SearchFunc {
return func(line string) (int, error) {
if !strings.HasPrefix(line, prefix) {
return -1, errors.New(cmn.Fmt("Marker line did not have prefix: %v", prefix))
}
i, err := strconv.Atoi(line[len(prefix):])
if err != nil {
return -1, errors.New(cmn.Fmt("Failed to parse marker line: %v", err.Error()))
}
if target < i {
return 1, nil
} else if target == i {
return 0, nil
} else {
return -1, nil
}
}
}

438
libs/autofile/group_test.go Normal file
View File

@@ -0,0 +1,438 @@
package autofile
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
cmn "github.com/tendermint/tendermint/libs/common"
)
// NOTE: Returned group has ticker stopped
func createTestGroup(t *testing.T, headSizeLimit int64) *Group {
testID := cmn.RandStr(12)
testDir := "_test_" + testID
err := cmn.EnsureDir(testDir, 0700)
require.NoError(t, err, "Error creating dir")
headPath := testDir + "/myfile"
g, err := OpenGroup(headPath)
require.NoError(t, err, "Error opening Group")
g.SetHeadSizeLimit(headSizeLimit)
g.stopTicker()
require.NotEqual(t, nil, g, "Failed to create Group")
return g
}
func destroyTestGroup(t *testing.T, g *Group) {
g.Close()
err := os.RemoveAll(g.Dir)
require.NoError(t, err, "Error removing test Group directory")
}
func assertGroupInfo(t *testing.T, gInfo GroupInfo, minIndex, maxIndex int, totalSize, headSize int64) {
assert.Equal(t, minIndex, gInfo.MinIndex)
assert.Equal(t, maxIndex, gInfo.MaxIndex)
assert.Equal(t, totalSize, gInfo.TotalSize)
assert.Equal(t, headSize, gInfo.HeadSize)
}
func TestCheckHeadSizeLimit(t *testing.T) {
g := createTestGroup(t, 1000*1000)
// At first, there are no files.
assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 0, 0)
// Write 1000 bytes 999 times.
for i := 0; i < 999; i++ {
err := g.WriteLine(cmn.RandStr(999))
require.NoError(t, err, "Error appending to head")
}
g.Flush()
assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 999000, 999000)
// Even calling checkHeadSizeLimit manually won't rotate it.
g.checkHeadSizeLimit()
assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 999000, 999000)
// Write 1000 more bytes.
err := g.WriteLine(cmn.RandStr(999))
require.NoError(t, err, "Error appending to head")
g.Flush()
// Calling checkHeadSizeLimit this time rolls it.
g.checkHeadSizeLimit()
assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 1000000, 0)
// Write 1000 more bytes.
err = g.WriteLine(cmn.RandStr(999))
require.NoError(t, err, "Error appending to head")
g.Flush()
// Calling checkHeadSizeLimit does nothing.
g.checkHeadSizeLimit()
assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 1001000, 1000)
// Write 1000 bytes 999 times.
for i := 0; i < 999; i++ {
err = g.WriteLine(cmn.RandStr(999))
require.NoError(t, err, "Error appending to head")
}
g.Flush()
assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 2000000, 1000000)
// Calling checkHeadSizeLimit rolls it again.
g.checkHeadSizeLimit()
assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2000000, 0)
// Write 1000 more bytes.
_, err = g.Head.Write([]byte(cmn.RandStr(999) + "\n"))
require.NoError(t, err, "Error appending to head")
g.Flush()
assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2001000, 1000)
// Calling checkHeadSizeLimit does nothing.
g.checkHeadSizeLimit()
assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2001000, 1000)
// Cleanup
destroyTestGroup(t, g)
}
func TestSearch(t *testing.T) {
g := createTestGroup(t, 10*1000)
// Create some files in the group that have several INFO lines in them.
// Try to put the INFO lines in various spots.
for i := 0; i < 100; i++ {
// The random junk at the end ensures that this INFO linen
// is equally likely to show up at the end.
_, err := g.Head.Write([]byte(fmt.Sprintf("INFO %v %v\n", i, cmn.RandStr(123))))
require.NoError(t, err, "Failed to write to head")
g.checkHeadSizeLimit()
for j := 0; j < 10; j++ {
_, err1 := g.Head.Write([]byte(cmn.RandStr(123) + "\n"))
require.NoError(t, err1, "Failed to write to head")
g.checkHeadSizeLimit()
}
}
// Create a search func that searches for line
makeSearchFunc := func(target int) SearchFunc {
return func(line string) (int, error) {
parts := strings.Split(line, " ")
if len(parts) != 3 {
return -1, errors.New("Line did not have 3 parts")
}
i, err := strconv.Atoi(parts[1])
if err != nil {
return -1, errors.New("Failed to parse INFO: " + err.Error())
}
if target < i {
return 1, nil
} else if target == i {
return 0, nil
} else {
return -1, nil
}
}
}
// Now search for each number
for i := 0; i < 100; i++ {
t.Log("Testing for i", i)
gr, match, err := g.Search("INFO", makeSearchFunc(i))
require.NoError(t, err, "Failed to search for line")
assert.True(t, match, "Expected Search to return exact match")
line, err := gr.ReadLine()
require.NoError(t, err, "Failed to read line after search")
if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", i)) {
t.Fatal("Failed to get correct line")
}
// Make sure we can continue to read from there.
cur := i + 1
for {
line, err := gr.ReadLine()
if err == io.EOF {
if cur == 99+1 {
// OK!
break
} else {
t.Fatal("Got EOF after the wrong INFO #")
}
} else if err != nil {
t.Fatal("Error reading line", err)
}
if !strings.HasPrefix(line, "INFO ") {
continue
}
if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", cur)) {
t.Fatalf("Unexpected INFO #. Expected %v got:\n%v", cur, line)
}
cur++
}
gr.Close()
}
// Now search for something that is too small.
// We should get the first available line.
{
gr, match, err := g.Search("INFO", makeSearchFunc(-999))
require.NoError(t, err, "Failed to search for line")
assert.False(t, match, "Expected Search to not return exact match")
line, err := gr.ReadLine()
require.NoError(t, err, "Failed to read line after search")
if !strings.HasPrefix(line, "INFO 0 ") {
t.Error("Failed to fetch correct line, which is the earliest INFO")
}
err = gr.Close()
require.NoError(t, err, "Failed to close GroupReader")
}
// Now search for something that is too large.
// We should get an EOF error.
{
gr, _, err := g.Search("INFO", makeSearchFunc(999))
assert.Equal(t, io.EOF, err)
assert.Nil(t, gr)
}
// Cleanup
destroyTestGroup(t, g)
}
func TestRotateFile(t *testing.T) {
g := createTestGroup(t, 0)
g.WriteLine("Line 1")
g.WriteLine("Line 2")
g.WriteLine("Line 3")
g.Flush()
g.RotateFile()
g.WriteLine("Line 4")
g.WriteLine("Line 5")
g.WriteLine("Line 6")
g.Flush()
// Read g.Head.Path+"000"
body1, err := ioutil.ReadFile(g.Head.Path + ".000")
assert.NoError(t, err, "Failed to read first rolled file")
if string(body1) != "Line 1\nLine 2\nLine 3\n" {
t.Errorf("Got unexpected contents: [%v]", string(body1))
}
// Read g.Head.Path
body2, err := ioutil.ReadFile(g.Head.Path)
assert.NoError(t, err, "Failed to read first rolled file")
if string(body2) != "Line 4\nLine 5\nLine 6\n" {
t.Errorf("Got unexpected contents: [%v]", string(body2))
}
// Cleanup
destroyTestGroup(t, g)
}
func TestFindLast1(t *testing.T) {
g := createTestGroup(t, 0)
g.WriteLine("Line 1")
g.WriteLine("Line 2")
g.WriteLine("# a")
g.WriteLine("Line 3")
g.Flush()
g.RotateFile()
g.WriteLine("Line 4")
g.WriteLine("Line 5")
g.WriteLine("Line 6")
g.WriteLine("# b")
g.Flush()
match, found, err := g.FindLast("#")
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, "# b", match)
// Cleanup
destroyTestGroup(t, g)
}
func TestFindLast2(t *testing.T) {
g := createTestGroup(t, 0)
g.WriteLine("Line 1")
g.WriteLine("Line 2")
g.WriteLine("Line 3")
g.Flush()
g.RotateFile()
g.WriteLine("# a")
g.WriteLine("Line 4")
g.WriteLine("Line 5")
g.WriteLine("# b")
g.WriteLine("Line 6")
g.Flush()
match, found, err := g.FindLast("#")
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, "# b", match)
// Cleanup
destroyTestGroup(t, g)
}
func TestFindLast3(t *testing.T) {
g := createTestGroup(t, 0)
g.WriteLine("Line 1")
g.WriteLine("# a")
g.WriteLine("Line 2")
g.WriteLine("# b")
g.WriteLine("Line 3")
g.Flush()
g.RotateFile()
g.WriteLine("Line 4")
g.WriteLine("Line 5")
g.WriteLine("Line 6")
g.Flush()
match, found, err := g.FindLast("#")
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, "# b", match)
// Cleanup
destroyTestGroup(t, g)
}
func TestFindLast4(t *testing.T) {
g := createTestGroup(t, 0)
g.WriteLine("Line 1")
g.WriteLine("Line 2")
g.WriteLine("Line 3")
g.Flush()
g.RotateFile()
g.WriteLine("Line 4")
g.WriteLine("Line 5")
g.WriteLine("Line 6")
g.Flush()
match, found, err := g.FindLast("#")
assert.NoError(t, err)
assert.False(t, found)
assert.Empty(t, match)
// Cleanup
destroyTestGroup(t, g)
}
func TestWrite(t *testing.T) {
g := createTestGroup(t, 0)
written := []byte("Medusa")
g.Write(written)
g.Flush()
read := make([]byte, len(written))
gr, err := g.NewReader(0)
require.NoError(t, err, "failed to create reader")
_, err = gr.Read(read)
assert.NoError(t, err, "failed to read data")
assert.Equal(t, written, read)
// Cleanup
destroyTestGroup(t, g)
}
// test that Read reads the required amount of bytes from all the files in the
// group and returns no error if n == size of the given slice.
func TestGroupReaderRead(t *testing.T) {
g := createTestGroup(t, 0)
professor := []byte("Professor Monster")
g.Write(professor)
g.Flush()
g.RotateFile()
frankenstein := []byte("Frankenstein's Monster")
g.Write(frankenstein)
g.Flush()
totalWrittenLength := len(professor) + len(frankenstein)
read := make([]byte, totalWrittenLength)
gr, err := g.NewReader(0)
require.NoError(t, err, "failed to create reader")
n, err := gr.Read(read)
assert.NoError(t, err, "failed to read data")
assert.Equal(t, totalWrittenLength, n, "not enough bytes read")
professorPlusFrankenstein := professor
professorPlusFrankenstein = append(professorPlusFrankenstein, frankenstein...)
assert.Equal(t, professorPlusFrankenstein, read)
// Cleanup
destroyTestGroup(t, g)
}
// test that Read returns an error if number of bytes read < size of
// the given slice. Subsequent call should return 0, io.EOF.
func TestGroupReaderRead2(t *testing.T) {
g := createTestGroup(t, 0)
professor := []byte("Professor Monster")
g.Write(professor)
g.Flush()
g.RotateFile()
frankenstein := []byte("Frankenstein's Monster")
frankensteinPart := []byte("Frankenstein")
g.Write(frankensteinPart) // note writing only a part
g.Flush()
totalLength := len(professor) + len(frankenstein)
read := make([]byte, totalLength)
gr, err := g.NewReader(0)
require.NoError(t, err, "failed to create reader")
// 1) n < (size of the given slice), io.EOF
n, err := gr.Read(read)
assert.Equal(t, io.EOF, err)
assert.Equal(t, len(professor)+len(frankensteinPart), n, "Read more/less bytes than it is in the group")
// 2) 0, io.EOF
n, err = gr.Read([]byte("0"))
assert.Equal(t, io.EOF, err)
assert.Equal(t, 0, n)
// Cleanup
destroyTestGroup(t, g)
}
func TestMinIndex(t *testing.T) {
g := createTestGroup(t, 0)
assert.Zero(t, g.MinIndex(), "MinIndex should be zero at the beginning")
// Cleanup
destroyTestGroup(t, g)
}
func TestMaxIndex(t *testing.T) {
g := createTestGroup(t, 0)
assert.Zero(t, g.MaxIndex(), "MaxIndex should be zero at the beginning")
g.WriteLine("Line 1")
g.Flush()
g.RotateFile()
assert.Equal(t, 1, g.MaxIndex(), "MaxIndex should point to the last file")
// Cleanup
destroyTestGroup(t, g)
}

63
libs/autofile/sighup_watcher.go Normal file
View File

@@ -0,0 +1,63 @@
package autofile
import (
"os"
"os/signal"
"sync"
"sync/atomic"
"syscall"
)
func init() {
initSighupWatcher()
}
var sighupWatchers *SighupWatcher
var sighupCounter int32 // For testing
func initSighupWatcher() {
sighupWatchers = newSighupWatcher()
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP)
go func() {
for range c {
sighupWatchers.closeAll()
atomic.AddInt32(&sighupCounter, 1)
}
}()
}
// Watchces for SIGHUP events and notifies registered AutoFiles
type SighupWatcher struct {
mtx sync.Mutex
autoFiles map[string]*AutoFile
}
func newSighupWatcher() *SighupWatcher {
return &SighupWatcher{
autoFiles: make(map[string]*AutoFile, 10),
}
}
func (w *SighupWatcher) addAutoFile(af *AutoFile) {
w.mtx.Lock()
w.autoFiles[af.ID] = af
w.mtx.Unlock()
}
// If AutoFile isn't registered or was already removed, does nothing.
func (w *SighupWatcher) removeAutoFile(af *AutoFile) {
w.mtx.Lock()
delete(w.autoFiles, af.ID)
w.mtx.Unlock()
}
func (w *SighupWatcher) closeAll() {
w.mtx.Lock()
for _, af := range w.autoFiles {
af.closeFile()
}
w.mtx.Unlock()
}

29
libs/bech32/bech32.go Normal file
View File

@@ -0,0 +1,29 @@
package bech32
import (
"github.com/btcsuite/btcutil/bech32"
"github.com/pkg/errors"
)
//ConvertAndEncode converts from a base64 encoded byte string to base32 encoded byte string and then to bech32
func ConvertAndEncode(hrp string, data []byte) (string, error) {
converted, err := bech32.ConvertBits(data, 8, 5, true)
if err != nil {
return "", errors.Wrap(err, "encoding bech32 failed")
}
return bech32.Encode(hrp, converted)
}
//DecodeAndConvert decodes a bech32 encoded string and converts to base64 encoded bytes
func DecodeAndConvert(bech string) (string, []byte, error) {
hrp, data, err := bech32.Decode(bech)
if err != nil {
return "", nil, errors.Wrap(err, "decoding bech32 failed")
}
converted, err := bech32.ConvertBits(data, 5, 8, false)
if err != nil {
return "", nil, errors.Wrap(err, "decoding bech32 failed")
}
return hrp, converted, nil
}

31
libs/bech32/bech32_test.go Normal file
View File

@@ -0,0 +1,31 @@
package bech32_test
import (
"bytes"
"crypto/sha256"
"testing"
"github.com/tendermint/tendermint/libs/bech32"
)
func TestEncodeAndDecode(t *testing.T) {
sum := sha256.Sum256([]byte("hello world\n"))
bech, err := bech32.ConvertAndEncode("shasum", sum[:])
if err != nil {
t.Error(err)
}
hrp, data, err := bech32.DecodeAndConvert(bech)
if err != nil {
t.Error(err)
}
if hrp != "shasum" {
t.Error("Invalid hrp")
}
if !bytes.Equal(data, sum[:]) {
t.Error("Invalid decode")
}
}

21
libs/circle.yml Normal file
View File

@@ -0,0 +1,21 @@
machine:
environment:
GOPATH: "${HOME}/.go_workspace"
PROJECT_PARENT_PATH: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME"
PROJECT_PATH: $GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
hosts:
localhost: 127.0.0.1
dependencies:
override:
- mkdir -p "$PROJECT_PARENT_PATH"
- ln -sf "$HOME/$CIRCLE_PROJECT_REPONAME/" "$PROJECT_PATH"
post:
- go version
test:
override:
- cd $PROJECT_PATH && make get_tools && make get_vendor_deps && bash ./test.sh
post:
- cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt
- cd "$PROJECT_PATH" && mv coverage.txt "${CIRCLE_ARTIFACTS}"

86
libs/cli/flags/log_level.go Normal file
View File

@@ -0,0 +1,86 @@
package flags
import (
"fmt"
"strings"
"github.com/pkg/errors"
"github.com/tendermint/tendermint/libs/log"
)
const (
defaultLogLevelKey = "*"
)
// ParseLogLevel parses complex log level - comma-separated
// list of module:level pairs with an optional *:level pair (* means
// all other modules).
//
// Example:
// ParseLogLevel("consensus:debug,mempool:debug,*:error", log.NewTMLogger(os.Stdout), "info")
func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (log.Logger, error) {
if lvl == "" {
return nil, errors.New("Empty log level")
}
l := lvl
// prefix simple one word levels (e.g. "info") with "*"
if !strings.Contains(l, ":") {
l = defaultLogLevelKey + ":" + l
}
options := make([]log.Option, 0)
isDefaultLogLevelSet := false
var option log.Option
var err error
list := strings.Split(l, ",")
for _, item := range list {
moduleAndLevel := strings.Split(item, ":")
if len(moduleAndLevel) != 2 {
return nil, fmt.Errorf("Expected list in a form of \"module:level\" pairs, given pair %s, list %s", item, list)
}
module := moduleAndLevel[0]
level := moduleAndLevel[1]
if module == defaultLogLevelKey {
option, err = log.AllowLevel(level)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("Failed to parse default log level (pair %s, list %s)", item, l))
}
options = append(options, option)
isDefaultLogLevelSet = true
} else {
switch level {
case "debug":
option = log.AllowDebugWith("module", module)
case "info":
option = log.AllowInfoWith("module", module)
case "error":
option = log.AllowErrorWith("module", module)
case "none":
option = log.AllowNoneWith("module", module)
default:
return nil, fmt.Errorf("Expected either \"info\", \"debug\", \"error\" or \"none\" log level, given %s (pair %s, list %s)", level, item, list)
}
options = append(options, option)
}
}
// if "*" is not provided, set default global level
if !isDefaultLogLevelSet {
option, err = log.AllowLevel(defaultLogLevelValue)
if err != nil {
return nil, err
}
options = append(options, option)
}
return log.NewFilter(logger, options...), nil
}

94
libs/cli/flags/log_level_test.go Normal file
View File

@@ -0,0 +1,94 @@
package flags_test
import (
"bytes"
"strings"
"testing"
tmflags "github.com/tendermint/tendermint/libs/cli/flags"
"github.com/tendermint/tendermint/libs/log"
)
const (
defaultLogLevelValue = "info"
)
func TestParseLogLevel(t *testing.T) {
var buf bytes.Buffer
jsonLogger := log.NewTMJSONLogger(&buf)
correctLogLevels := []struct {
lvl string
expectedLogLines []string
}{
{"mempool:error", []string{
``, // if no default is given, assume info
``,
`{"_msg":"Mesmero","level":"error","module":"mempool"}`,
`{"_msg":"Mind","level":"info","module":"state"}`, // if no default is given, assume info
``}},
{"mempool:error,*:debug", []string{
`{"_msg":"Kingpin","level":"debug","module":"wire"}`,
``,
`{"_msg":"Mesmero","level":"error","module":"mempool"}`,
`{"_msg":"Mind","level":"info","module":"state"}`,
`{"_msg":"Gideon","level":"debug"}`}},
{"*:debug,wire:none", []string{
``,
`{"_msg":"Kitty Pryde","level":"info","module":"mempool"}`,
`{"_msg":"Mesmero","level":"error","module":"mempool"}`,
`{"_msg":"Mind","level":"info","module":"state"}`,
`{"_msg":"Gideon","level":"debug"}`}},
}
for _, c := range correctLogLevels {
logger, err := tmflags.ParseLogLevel(c.lvl, jsonLogger, defaultLogLevelValue)
if err != nil {
t.Fatal(err)
}
buf.Reset()
logger.With("module", "wire").Debug("Kingpin")
if have := strings.TrimSpace(buf.String()); c.expectedLogLines[0] != have {
t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[0], have, c.lvl)
}
buf.Reset()
logger.With("module", "mempool").Info("Kitty Pryde")
if have := strings.TrimSpace(buf.String()); c.expectedLogLines[1] != have {
t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[1], have, c.lvl)
}
buf.Reset()
logger.With("module", "mempool").Error("Mesmero")
if have := strings.TrimSpace(buf.String()); c.expectedLogLines[2] != have {
t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[2], have, c.lvl)
}
buf.Reset()
logger.With("module", "state").Info("Mind")
if have := strings.TrimSpace(buf.String()); c.expectedLogLines[3] != have {
t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[3], have, c.lvl)
}
buf.Reset()
logger.Debug("Gideon")
if have := strings.TrimSpace(buf.String()); c.expectedLogLines[4] != have {
t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[4], have, c.lvl)
}
}
incorrectLogLevel := []string{"some", "mempool:some", "*:some,mempool:error"}
for _, lvl := range incorrectLogLevel {
if _, err := tmflags.ParseLogLevel(lvl, jsonLogger, defaultLogLevelValue); err == nil {
t.Fatalf("Expected %s to produce error", lvl)
}
}
}

87
libs/cli/helper.go Normal file
View File

@@ -0,0 +1,87 @@
package cli
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
)
// WriteConfigVals writes a toml file with the given values.
// It returns an error if writing was impossible.
func WriteConfigVals(dir string, vals map[string]string) error {
data := ""
for k, v := range vals {
data = data + fmt.Sprintf("%s = \"%s\"\n", k, v)
}
cfile := filepath.Join(dir, "config.toml")
return ioutil.WriteFile(cfile, []byte(data), 0666)
}
// RunWithArgs executes the given command with the specified command line args
// and environmental variables set. It returns any error returned from cmd.Execute()
func RunWithArgs(cmd Executable, args []string, env map[string]string) error {
oargs := os.Args
oenv := map[string]string{}
// defer returns the environment back to normal
defer func() {
os.Args = oargs
for k, v := range oenv {
os.Setenv(k, v)
}
}()
// set the args and env how we want them
os.Args = args
for k, v := range env {
// backup old value if there, to restore at end
oenv[k] = os.Getenv(k)
err := os.Setenv(k, v)
if err != nil {
return err
}
}
// and finally run the command
return cmd.Execute()
}
// RunCaptureWithArgs executes the given command with the specified command
// line args and environmental variables set. It returns string fields
// representing output written to stdout and stderr, additionally any error
// from cmd.Execute() is also returned
func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (stdout, stderr string, err error) {
oldout, olderr := os.Stdout, os.Stderr // keep backup of the real stdout
rOut, wOut, _ := os.Pipe()
rErr, wErr, _ := os.Pipe()
os.Stdout, os.Stderr = wOut, wErr
defer func() {
os.Stdout, os.Stderr = oldout, olderr // restoring the real stdout
}()
// copy the output in a separate goroutine so printing can't block indefinitely
copyStd := func(reader *os.File) *(chan string) {
stdC := make(chan string)
go func() {
var buf bytes.Buffer
// io.Copy will end when we call reader.Close() below
io.Copy(&buf, reader)
stdC <- buf.String()
}()
return &stdC
}
outC := copyStd(rOut)
errC := copyStd(rErr)
// now run the command
err = RunWithArgs(cmd, args, env)
// and grab the stdout to return
wOut.Close()
wErr.Close()
stdout = <-*outC
stderr = <-*errC
return stdout, stderr, err
}

157
libs/cli/setup.go Normal file
View File

@@ -0,0 +1,157 @@
package cli
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
const (
HomeFlag = "home"
TraceFlag = "trace"
OutputFlag = "output"
EncodingFlag = "encoding"
)
// Executable is the minimal interface to *corba.Command, so we can
// wrap if desired before the test
type Executable interface {
Execute() error
}
// PrepareBaseCmd is meant for tendermint and other servers
func PrepareBaseCmd(cmd *cobra.Command, envPrefix, defaultHome string) Executor {
cobra.OnInitialize(func() { initEnv(envPrefix) })
cmd.PersistentFlags().StringP(HomeFlag, "", defaultHome, "directory for config and data")
cmd.PersistentFlags().Bool(TraceFlag, false, "print out full stack trace on errors")
cmd.PersistentPreRunE = concatCobraCmdFuncs(bindFlagsLoadViper, cmd.PersistentPreRunE)
return Executor{cmd, os.Exit}
}
// PrepareMainCmd is meant for client side libs that want some more flags
//
// This adds --encoding (hex, btc, base64) and --output (text, json) to
// the command. These only really make sense in interactive commands.
func PrepareMainCmd(cmd *cobra.Command, envPrefix, defaultHome string) Executor {
cmd.PersistentFlags().StringP(EncodingFlag, "e", "hex", "Binary encoding (hex|b64|btc)")
cmd.PersistentFlags().StringP(OutputFlag, "o", "text", "Output format (text|json)")
cmd.PersistentPreRunE = concatCobraCmdFuncs(validateOutput, cmd.PersistentPreRunE)
return PrepareBaseCmd(cmd, envPrefix, defaultHome)
}
// initEnv sets to use ENV variables if set.
func initEnv(prefix string) {
copyEnvVars(prefix)
// env variables with TM prefix (eg. TM_ROOT)
viper.SetEnvPrefix(prefix)
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
viper.AutomaticEnv()
}
// This copies all variables like TMROOT to TM_ROOT,
// so we can support both formats for the user
func copyEnvVars(prefix string) {
prefix = strings.ToUpper(prefix)
ps := prefix + "_"
for _, e := range os.Environ() {
kv := strings.SplitN(e, "=", 2)
if len(kv) == 2 {
k, v := kv[0], kv[1]
if strings.HasPrefix(k, prefix) && !strings.HasPrefix(k, ps) {
k2 := strings.Replace(k, prefix, ps, 1)
os.Setenv(k2, v)
}
}
}
}
// Executor wraps the cobra Command with a nicer Execute method
type Executor struct {
*cobra.Command
Exit func(int) // this is os.Exit by default, override in tests
}
type ExitCoder interface {
ExitCode() int
}
// execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func (e Executor) Execute() error {
e.SilenceUsage = true
e.SilenceErrors = true
err := e.Command.Execute()
if err != nil {
if viper.GetBool(TraceFlag) {
fmt.Fprintf(os.Stderr, "ERROR: %+v\n", err)
} else {
fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
}
// return error code 1 by default, can override it with a special error type
exitCode := 1
if ec, ok := err.(ExitCoder); ok {
exitCode = ec.ExitCode()
}
e.Exit(exitCode)
}
return err
}
type cobraCmdFunc func(cmd *cobra.Command, args []string) error
// Returns a single function that calls each argument function in sequence
// RunE, PreRunE, PersistentPreRunE, etc. all have this same signature
func concatCobraCmdFuncs(fs ...cobraCmdFunc) cobraCmdFunc {
return func(cmd *cobra.Command, args []string) error {
for _, f := range fs {
if f != nil {
if err := f(cmd, args); err != nil {
return err
}
}
}
return nil
}
}
// Bind all flags and read the config into viper
func bindFlagsLoadViper(cmd *cobra.Command, args []string) error {
// cmd.Flags() includes flags from this command and all persistent flags from the parent
if err := viper.BindPFlags(cmd.Flags()); err != nil {
return err
}
homeDir := viper.GetString(HomeFlag)
viper.Set(HomeFlag, homeDir)
viper.SetConfigName("config") // name of config file (without extension)
viper.AddConfigPath(homeDir) // search root directory
viper.AddConfigPath(filepath.Join(homeDir, "config")) // search root directory /config
// If a config file is found, read it in.
if err := viper.ReadInConfig(); err == nil {
// stderr, so if we redirect output to json file, this doesn't appear
// fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
} else if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
// ignore not found error, return other errors
return err
}
return nil
}
func validateOutput(cmd *cobra.Command, args []string) error {
// validate output format
output := viper.GetString(OutputFlag)
switch output {
case "text", "json":
default:
return errors.Errorf("Unsupported output format: %s", output)
}
return nil
}

237
libs/cli/setup_test.go Normal file
View File

@@ -0,0 +1,237 @@
package cli
import (
"fmt"
"io/ioutil"
"strconv"
"strings"
"testing"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSetupEnv(t *testing.T) {
cases := []struct {
args []string
env map[string]string
expected string
}{
{nil, nil, ""},
{[]string{"--foobar", "bang!"}, nil, "bang!"},
// make sure reset is good
{nil, nil, ""},
// test both variants of the prefix
{nil, map[string]string{"DEMO_FOOBAR": "good"}, "good"},
{nil, map[string]string{"DEMOFOOBAR": "silly"}, "silly"},
// and that cli overrides env...
{[]string{"--foobar", "important"},
map[string]string{"DEMO_FOOBAR": "ignored"}, "important"},
}
for idx, tc := range cases {
i := strconv.Itoa(idx)
// test command that store value of foobar in local variable
var foo string
demo := &cobra.Command{
Use: "demo",
RunE: func(cmd *cobra.Command, args []string) error {
foo = viper.GetString("foobar")
return nil
},
}
demo.Flags().String("foobar", "", "Some test value from config")
cmd := PrepareBaseCmd(demo, "DEMO", "/qwerty/asdfgh") // some missing dir..
cmd.Exit = func(int) {}
viper.Reset()
args := append([]string{cmd.Use}, tc.args...)
err := RunWithArgs(cmd, args, tc.env)
require.Nil(t, err, i)
assert.Equal(t, tc.expected, foo, i)
}
}
func tempDir() string {
cdir, err := ioutil.TempDir("", "test-cli")
if err != nil {
panic(err)
}
return cdir
}
func TestSetupConfig(t *testing.T) {
// we pre-create two config files we can refer to in the rest of
// the test cases.
cval1 := "fubble"
conf1 := tempDir()
err := WriteConfigVals(conf1, map[string]string{"boo": cval1})
require.Nil(t, err)
cases := []struct {
args []string
env map[string]string
expected string
expectedTwo string
}{
{nil, nil, "", ""},
// setting on the command line
{[]string{"--boo", "haha"}, nil, "haha", ""},
{[]string{"--two-words", "rocks"}, nil, "", "rocks"},
{[]string{"--home", conf1}, nil, cval1, ""},
// test both variants of the prefix
{nil, map[string]string{"RD_BOO": "bang"}, "bang", ""},
{nil, map[string]string{"RD_TWO_WORDS": "fly"}, "", "fly"},
{nil, map[string]string{"RDTWO_WORDS": "fly"}, "", "fly"},
{nil, map[string]string{"RD_HOME": conf1}, cval1, ""},
{nil, map[string]string{"RDHOME": conf1}, cval1, ""},
}
for idx, tc := range cases {
i := strconv.Itoa(idx)
// test command that store value of foobar in local variable
var foo, two string
boo := &cobra.Command{
Use: "reader",
RunE: func(cmd *cobra.Command, args []string) error {
foo = viper.GetString("boo")
two = viper.GetString("two-words")
return nil
},
}
boo.Flags().String("boo", "", "Some test value from config")
boo.Flags().String("two-words", "", "Check out env handling -")
cmd := PrepareBaseCmd(boo, "RD", "/qwerty/asdfgh") // some missing dir...
cmd.Exit = func(int) {}
viper.Reset()
args := append([]string{cmd.Use}, tc.args...)
err := RunWithArgs(cmd, args, tc.env)
require.Nil(t, err, i)
assert.Equal(t, tc.expected, foo, i)
assert.Equal(t, tc.expectedTwo, two, i)
}
}
type DemoConfig struct {
Name string `mapstructure:"name"`
Age int `mapstructure:"age"`
Unused int `mapstructure:"unused"`
}
func TestSetupUnmarshal(t *testing.T) {
// we pre-create two config files we can refer to in the rest of
// the test cases.
cval1, cval2 := "someone", "else"
conf1 := tempDir()
err := WriteConfigVals(conf1, map[string]string{"name": cval1})
require.Nil(t, err)
// even with some ignored fields, should be no problem
conf2 := tempDir()
err = WriteConfigVals(conf2, map[string]string{"name": cval2, "foo": "bar"})
require.Nil(t, err)
// unused is not declared on a flag and remains from base
base := DemoConfig{
Name: "default",
Age: 42,
Unused: -7,
}
c := func(name string, age int) DemoConfig {
r := base
// anything set on the flags as a default is used over
// the default config object
r.Name = "from-flag"
if name != "" {
r.Name = name
}
if age != 0 {
r.Age = age
}
return r
}
cases := []struct {
args []string
env map[string]string
expected DemoConfig
}{
{nil, nil, c("", 0)},
// setting on the command line
{[]string{"--name", "haha"}, nil, c("haha", 0)},
{[]string{"--home", conf1}, nil, c(cval1, 0)},
// test both variants of the prefix
{nil, map[string]string{"MR_AGE": "56"}, c("", 56)},
{nil, map[string]string{"MR_HOME": conf1}, c(cval1, 0)},
{[]string{"--age", "17"}, map[string]string{"MRHOME": conf2}, c(cval2, 17)},
}
for idx, tc := range cases {
i := strconv.Itoa(idx)
// test command that store value of foobar in local variable
cfg := base
marsh := &cobra.Command{
Use: "marsh",
RunE: func(cmd *cobra.Command, args []string) error {
return viper.Unmarshal(&cfg)
},
}
marsh.Flags().String("name", "from-flag", "Some test value from config")
// if we want a flag to use the proper default, then copy it
// from the default config here
marsh.Flags().Int("age", base.Age, "Some test value from config")
cmd := PrepareBaseCmd(marsh, "MR", "/qwerty/asdfgh") // some missing dir...
cmd.Exit = func(int) {}
viper.Reset()
args := append([]string{cmd.Use}, tc.args...)
err := RunWithArgs(cmd, args, tc.env)
require.Nil(t, err, i)
assert.Equal(t, tc.expected, cfg, i)
}
}
func TestSetupTrace(t *testing.T) {
cases := []struct {
args []string
env map[string]string
long bool
expected string
}{
{nil, nil, false, "Trace flag = false"},
{[]string{"--trace"}, nil, true, "Trace flag = true"},
{[]string{"--no-such-flag"}, nil, false, "unknown flag: --no-such-flag"},
{nil, map[string]string{"DBG_TRACE": "true"}, true, "Trace flag = true"},
}
for idx, tc := range cases {
i := strconv.Itoa(idx)
// test command that store value of foobar in local variable
trace := &cobra.Command{
Use: "trace",
RunE: func(cmd *cobra.Command, args []string) error {
return errors.Errorf("Trace flag = %t", viper.GetBool(TraceFlag))
},
}
cmd := PrepareBaseCmd(trace, "DBG", "/qwerty/asdfgh") // some missing dir..
cmd.Exit = func(int) {}
viper.Reset()
args := append([]string{cmd.Use}, tc.args...)
stdout, stderr, err := RunCaptureWithArgs(cmd, args, tc.env)
require.NotNil(t, err, i)
require.Equal(t, "", stdout, i)
require.NotEqual(t, "", stderr, i)
msg := strings.Split(stderr, "\n")
desired := fmt.Sprintf("ERROR: %s", tc.expected)
assert.Equal(t, desired, msg[0], i)
if tc.long && assert.True(t, len(msg) > 2, i) {
// the next line starts the stack trace...
assert.Contains(t, msg[1], "TestSetupTrace", i)
assert.Contains(t, msg[2], "setup_test.go", i)
}
}
}

384
libs/clist/clist.go Normal file
View File

@@ -0,0 +1,384 @@
package clist
/*
The purpose of CList is to provide a goroutine-safe linked-list.
This list can be traversed concurrently by any number of goroutines.
However, removed CElements cannot be added back.
NOTE: Not all methods of container/list are (yet) implemented.
NOTE: Removed elements need to DetachPrev or DetachNext consistently
to ensure garbage collection of removed elements.
*/
import (
"sync"
)
/*
CElement is an element of a linked-list
Traversal from a CElement is goroutine-safe.
We can't avoid using WaitGroups or for-loops given the documentation
spec without re-implementing the primitives that already exist in
golang/sync. Notice that WaitGroup allows many go-routines to be
simultaneously released, which is what we want. Mutex doesn't do
this. RWMutex does this, but it's clumsy to use in the way that a
WaitGroup would be used -- and we'd end up having two RWMutex's for
prev/next each, which is doubly confusing.
sync.Cond would be sort-of useful, but we don't need a write-lock in
the for-loop. Use sync.Cond when you need serial access to the
"condition". In our case our condition is if `next != nil || removed`,
and there's no reason to serialize that condition for goroutines
waiting on NextWait() (since it's just a read operation).
*/
type CElement struct {
mtx sync.RWMutex
prev *CElement
prevWg *sync.WaitGroup
prevWaitCh chan struct{}
next *CElement
nextWg *sync.WaitGroup
nextWaitCh chan struct{}
removed bool
Value interface{} // immutable
}
// Blocking implementation of Next().
// May return nil iff CElement was tail and got removed.
func (e *CElement) NextWait() *CElement {
for {
e.mtx.RLock()
next := e.next
nextWg := e.nextWg
removed := e.removed
e.mtx.RUnlock()
if next != nil || removed {
return next
}
nextWg.Wait()
// e.next doesn't necessarily exist here.
// That's why we need to continue a for-loop.
}
}
// Blocking implementation of Prev().
// May return nil iff CElement was head and got removed.
func (e *CElement) PrevWait() *CElement {
for {
e.mtx.RLock()
prev := e.prev
prevWg := e.prevWg
removed := e.removed
e.mtx.RUnlock()
if prev != nil || removed {
return prev
}
prevWg.Wait()
}
}
// PrevWaitChan can be used to wait until Prev becomes not nil. Once it does,
// channel will be closed.
func (e *CElement) PrevWaitChan() <-chan struct{} {
e.mtx.RLock()
defer e.mtx.RUnlock()
return e.prevWaitCh
}
// NextWaitChan can be used to wait until Next becomes not nil. Once it does,
// channel will be closed.
func (e *CElement) NextWaitChan() <-chan struct{} {
e.mtx.RLock()
defer e.mtx.RUnlock()
return e.nextWaitCh
}
// Nonblocking, may return nil if at the end.
func (e *CElement) Next() *CElement {
e.mtx.RLock()
defer e.mtx.RUnlock()
return e.next
}
// Nonblocking, may return nil if at the end.
func (e *CElement) Prev() *CElement {
e.mtx.RLock()
defer e.mtx.RUnlock()
return e.prev
}
func (e *CElement) Removed() bool {
e.mtx.RLock()
defer e.mtx.RUnlock()
return e.removed
}
func (e *CElement) DetachNext() {
if !e.Removed() {
panic("DetachNext() must be called after Remove(e)")
}
e.mtx.Lock()
defer e.mtx.Unlock()
e.next = nil
}
func (e *CElement) DetachPrev() {
if !e.Removed() {
panic("DetachPrev() must be called after Remove(e)")
}
e.mtx.Lock()
defer e.mtx.Unlock()
e.prev = nil
}
// NOTE: This function needs to be safe for
// concurrent goroutines waiting on nextWg.
func (e *CElement) SetNext(newNext *CElement) {
e.mtx.Lock()
defer e.mtx.Unlock()
oldNext := e.next
e.next = newNext
if oldNext != nil && newNext == nil {
// See https://golang.org/pkg/sync/:
//
// If a WaitGroup is reused to wait for several independent sets of
// events, new Add calls must happen after all previous Wait calls have
// returned.
e.nextWg = waitGroup1() // WaitGroups are difficult to re-use.
e.nextWaitCh = make(chan struct{})
}
if oldNext == nil && newNext != nil {
e.nextWg.Done()
close(e.nextWaitCh)
}
}
// NOTE: This function needs to be safe for
// concurrent goroutines waiting on prevWg
func (e *CElement) SetPrev(newPrev *CElement) {
e.mtx.Lock()
defer e.mtx.Unlock()
oldPrev := e.prev
e.prev = newPrev
if oldPrev != nil && newPrev == nil {
e.prevWg = waitGroup1() // WaitGroups are difficult to re-use.
e.prevWaitCh = make(chan struct{})
}
if oldPrev == nil && newPrev != nil {
e.prevWg.Done()
close(e.prevWaitCh)
}
}
func (e *CElement) SetRemoved() {
e.mtx.Lock()
defer e.mtx.Unlock()
e.removed = true
// This wakes up anyone waiting in either direction.
if e.prev == nil {
e.prevWg.Done()
close(e.prevWaitCh)
}
if e.next == nil {
e.nextWg.Done()
close(e.nextWaitCh)
}
}
//--------------------------------------------------------------------------------
// CList represents a linked list.
// The zero value for CList is an empty list ready to use.
// Operations are goroutine-safe.
type CList struct {
mtx sync.RWMutex
wg *sync.WaitGroup
waitCh chan struct{}
head *CElement // first element
tail *CElement // last element
len int // list length
}
func (l *CList) Init() *CList {
l.mtx.Lock()
defer l.mtx.Unlock()
l.wg = waitGroup1()
l.waitCh = make(chan struct{})
l.head = nil
l.tail = nil
l.len = 0
return l
}
func New() *CList { return new(CList).Init() }
func (l *CList) Len() int {
l.mtx.RLock()
defer l.mtx.RUnlock()
return l.len
}
func (l *CList) Front() *CElement {
l.mtx.RLock()
defer l.mtx.RUnlock()
return l.head
}
func (l *CList) FrontWait() *CElement {
// Loop until the head is non-nil else wait and try again
for {
l.mtx.RLock()
head := l.head
wg := l.wg
l.mtx.RUnlock()
if head != nil {
return head
}
wg.Wait()
// NOTE: If you think l.head exists here, think harder.
}
}
func (l *CList) Back() *CElement {
l.mtx.RLock()
defer l.mtx.RUnlock()
return l.tail
}
func (l *CList) BackWait() *CElement {
for {
l.mtx.RLock()
tail := l.tail
wg := l.wg
l.mtx.RUnlock()
if tail != nil {
return tail
}
wg.Wait()
// l.tail doesn't necessarily exist here.
// That's why we need to continue a for-loop.
}
}
// WaitChan can be used to wait until Front or Back becomes not nil. Once it
// does, channel will be closed.
func (l *CList) WaitChan() <-chan struct{} {
l.mtx.Lock()
defer l.mtx.Unlock()
return l.waitCh
}
func (l *CList) PushBack(v interface{}) *CElement {
l.mtx.Lock()
defer l.mtx.Unlock()
// Construct a new element
e := &CElement{
prev: nil,
prevWg: waitGroup1(),
prevWaitCh: make(chan struct{}),
next: nil,
nextWg: waitGroup1(),
nextWaitCh: make(chan struct{}),
removed: false,
Value: v,
}
// Release waiters on FrontWait/BackWait maybe
if l.len == 0 {
l.wg.Done()
close(l.waitCh)
}
l.len++
// Modify the tail
if l.tail == nil {
l.head = e
l.tail = e
} else {
e.SetPrev(l.tail) // We must init e first.
l.tail.SetNext(e) // This will make e accessible.
l.tail = e // Update the list.
}
return e
}
// CONTRACT: Caller must call e.DetachPrev() and/or e.DetachNext() to avoid memory leaks.
// NOTE: As per the contract of CList, removed elements cannot be added back.
func (l *CList) Remove(e *CElement) interface{} {
l.mtx.Lock()
defer l.mtx.Unlock()
prev := e.Prev()
next := e.Next()
if l.head == nil || l.tail == nil {
panic("Remove(e) on empty CList")
}
if prev == nil && l.head != e {
panic("Remove(e) with false head")
}
if next == nil && l.tail != e {
panic("Remove(e) with false tail")
}
// If we're removing the only item, make CList FrontWait/BackWait wait.
if l.len == 1 {
l.wg = waitGroup1() // WaitGroups are difficult to re-use.
l.waitCh = make(chan struct{})
}
// Update l.len
l.len--
// Connect next/prev and set head/tail
if prev == nil {
l.head = next
} else {
prev.SetNext(next)
}
if next == nil {
l.tail = prev
} else {
next.SetPrev(prev)
}
// Set .Done() on e, otherwise waiters will wait forever.
e.SetRemoved()
return e.Value
}
func waitGroup1() (wg *sync.WaitGroup) {
wg = &sync.WaitGroup{}
wg.Add(1)
return
}

293
libs/clist/clist_test.go Normal file
View File

@@ -0,0 +1,293 @@
package clist
import (
"fmt"
"math/rand"
"runtime"
"sync/atomic"
"testing"
"time"
)
func TestSmall(t *testing.T) {
l := New()
el1 := l.PushBack(1)
el2 := l.PushBack(2)
el3 := l.PushBack(3)
if l.Len() != 3 {
t.Error("Expected len 3, got ", l.Len())
}
//fmt.Printf("%p %v\n", el1, el1)
//fmt.Printf("%p %v\n", el2, el2)
//fmt.Printf("%p %v\n", el3, el3)
r1 := l.Remove(el1)
//fmt.Printf("%p %v\n", el1, el1)
//fmt.Printf("%p %v\n", el2, el2)
//fmt.Printf("%p %v\n", el3, el3)
r2 := l.Remove(el2)
//fmt.Printf("%p %v\n", el1, el1)
//fmt.Printf("%p %v\n", el2, el2)
//fmt.Printf("%p %v\n", el3, el3)
r3 := l.Remove(el3)
if r1 != 1 {
t.Error("Expected 1, got ", r1)
}
if r2 != 2 {
t.Error("Expected 2, got ", r2)
}
if r3 != 3 {
t.Error("Expected 3, got ", r3)
}
if l.Len() != 0 {
t.Error("Expected len 0, got ", l.Len())
}
}
/*
This test is quite hacky because it relies on SetFinalizer
which isn't guaranteed to run at all.
*/
// nolint: megacheck
func _TestGCFifo(t *testing.T) {
const numElements = 1000000
l := New()
gcCount := new(uint64)
// SetFinalizer doesn't work well with circular structures,
// so we construct a trivial non-circular structure to
// track.
type value struct {
Int int
}
done := make(chan struct{})
for i := 0; i < numElements; i++ {
v := new(value)
v.Int = i
l.PushBack(v)
runtime.SetFinalizer(v, func(v *value) {
atomic.AddUint64(gcCount, 1)
})
}
for el := l.Front(); el != nil; {
l.Remove(el)
//oldEl := el
el = el.Next()
//oldEl.DetachPrev()
//oldEl.DetachNext()
}
runtime.GC()
time.Sleep(time.Second * 3)
runtime.GC()
time.Sleep(time.Second * 3)
_ = done
if *gcCount != numElements {
t.Errorf("Expected gcCount to be %v, got %v", numElements,
*gcCount)
}
}
/*
This test is quite hacky because it relies on SetFinalizer
which isn't guaranteed to run at all.
*/
// nolint: megacheck
func _TestGCRandom(t *testing.T) {
const numElements = 1000000
l := New()
gcCount := 0
// SetFinalizer doesn't work well with circular structures,
// so we construct a trivial non-circular structure to
// track.
type value struct {
Int int
}
for i := 0; i < numElements; i++ {
v := new(value)
v.Int = i
l.PushBack(v)
runtime.SetFinalizer(v, func(v *value) {
gcCount++
})
}
els := make([]*CElement, 0, numElements)
for el := l.Front(); el != nil; el = el.Next() {
els = append(els, el)
}
for _, i := range rand.Perm(numElements) {
el := els[i]
l.Remove(el)
_ = el.Next()
}
runtime.GC()
time.Sleep(time.Second * 3)
if gcCount != numElements {
t.Errorf("Expected gcCount to be %v, got %v", numElements,
gcCount)
}
}
func TestScanRightDeleteRandom(t *testing.T) {
const numElements = 10000
const numTimes = 1000
const numScanners = 10
l := New()
stop := make(chan struct{})
els := make([]*CElement, numElements)
for i := 0; i < numElements; i++ {
el := l.PushBack(i)
els[i] = el
}
// Launch scanner routines that will rapidly iterate over elements.
for i := 0; i < numScanners; i++ {
go func(scannerID int) {
var el *CElement
restartCounter := 0
counter := 0
FOR_LOOP:
for {
select {
case <-stop:
fmt.Println("stopped")
break FOR_LOOP
default:
}
if el == nil {
el = l.FrontWait()
restartCounter++
}
el = el.Next()
counter++
}
fmt.Printf("Scanner %v restartCounter: %v counter: %v\n", scannerID, restartCounter, counter)
}(i)
}
// Remove an element, push back an element.
for i := 0; i < numTimes; i++ {
// Pick an element to remove
rmElIdx := rand.Intn(len(els))
rmEl := els[rmElIdx]
// Remove it
l.Remove(rmEl)
//fmt.Print(".")
// Insert a new element
newEl := l.PushBack(-1*i - 1)
els[rmElIdx] = newEl
if i%100000 == 0 {
fmt.Printf("Pushed %vK elements so far...\n", i/1000)
}
}
// Stop scanners
close(stop)
time.Sleep(time.Second * 1)
// And remove all the elements.
for el := l.Front(); el != nil; el = el.Next() {
l.Remove(el)
}
if l.Len() != 0 {
t.Fatal("Failed to remove all elements from CList")
}
}
func TestWaitChan(t *testing.T) {
l := New()
ch := l.WaitChan()
// 1) add one element to an empty list
go l.PushBack(1)
<-ch
// 2) and remove it
el := l.Front()
v := l.Remove(el)
if v != 1 {
t.Fatal("where is 1 coming from?")
}
// 3) test iterating forward and waiting for Next (NextWaitChan and Next)
el = l.PushBack(0)
done := make(chan struct{})
pushed := 0
go func() {
for i := 1; i < 100; i++ {
l.PushBack(i)
pushed++
time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
}
close(done)
}()
next := el
seen := 0
FOR_LOOP:
for {
select {
case <-next.NextWaitChan():
next = next.Next()
seen++
if next == nil {
continue
}
case <-done:
break FOR_LOOP
case <-time.After(10 * time.Second):
t.Fatal("max execution time")
}
}
if pushed != seen {
t.Fatalf("number of pushed items (%d) not equal to number of seen items (%d)", pushed, seen)
}
// 4) test iterating backwards (PrevWaitChan and Prev)
prev := next
seen = 0
FOR_LOOP2:
for {
select {
case <-prev.PrevWaitChan():
prev = prev.Prev()
seen++
if prev == nil {
t.Fatal("expected PrevWaitChan to block forever on nil when reached first elem")
}
case <-time.After(5 * time.Second):
break FOR_LOOP2
}
}
if pushed != seen {
t.Fatalf("number of pushed items (%d) not equal to number of seen items (%d)", pushed, seen)
}
}

Some files were not shown because too many files have changed in this diff Show More