mv tmlibs files to libs dir

libs/cli/flags/log_level.go

@@ -0,0 +1,86 @@
+package flags
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/tendermint/tmlibs/log"
+)
+
+const (
+	defaultLogLevelKey = "*"
+)
+
+// ParseLogLevel parses complex log level - comma-separated
+// list of module:level pairs with an optional *:level pair (* means
+// all other modules).
+//
+// Example:
+//		ParseLogLevel("consensus:debug,mempool:debug,*:error", log.NewTMLogger(os.Stdout), "info")
+func ParseLogLevel(lvl string, logger log.Logger, defaultLogLevelValue string) (log.Logger, error) {
+	if lvl == "" {
+		return nil, errors.New("Empty log level")
+	}
+
+	l := lvl
+
+	// prefix simple one word levels (e.g. "info") with "*"
+	if !strings.Contains(l, ":") {
+		l = defaultLogLevelKey + ":" + l
+	}
+
+	options := make([]log.Option, 0)
+
+	isDefaultLogLevelSet := false
+	var option log.Option
+	var err error
+
+	list := strings.Split(l, ",")
+	for _, item := range list {
+		moduleAndLevel := strings.Split(item, ":")
+
+		if len(moduleAndLevel) != 2 {
+			return nil, fmt.Errorf("Expected list in a form of \"module:level\" pairs, given pair %s, list %s", item, list)
+		}
+
+		module := moduleAndLevel[0]
+		level := moduleAndLevel[1]
+
+		if module == defaultLogLevelKey {
+			option, err = log.AllowLevel(level)
+			if err != nil {
+				return nil, errors.Wrap(err, fmt.Sprintf("Failed to parse default log level (pair %s, list %s)", item, l))
+			}
+			options = append(options, option)
+			isDefaultLogLevelSet = true
+		} else {
+			switch level {
+			case "debug":
+				option = log.AllowDebugWith("module", module)
+			case "info":
+				option = log.AllowInfoWith("module", module)
+			case "error":
+				option = log.AllowErrorWith("module", module)
+			case "none":
+				option = log.AllowNoneWith("module", module)
+			default:
+				return nil, fmt.Errorf("Expected either \"info\", \"debug\", \"error\" or \"none\" log level, given %s (pair %s, list %s)", level, item, list)
+			}
+			options = append(options, option)
+
+		}
+	}
+
+	// if "*" is not provided, set default global level
+	if !isDefaultLogLevelSet {
+		option, err = log.AllowLevel(defaultLogLevelValue)
+		if err != nil {
+			return nil, err
+		}
+		options = append(options, option)
+	}
+
+	return log.NewFilter(logger, options...), nil
+}

libs/cli/flags/log_level_test.go

@@ -0,0 +1,94 @@
+package flags_test
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	tmflags "github.com/tendermint/tmlibs/cli/flags"
+	"github.com/tendermint/tmlibs/log"
+)
+
+const (
+	defaultLogLevelValue = "info"
+)
+
+func TestParseLogLevel(t *testing.T) {
+	var buf bytes.Buffer
+	jsonLogger := log.NewTMJSONLogger(&buf)
+
+	correctLogLevels := []struct {
+		lvl              string
+		expectedLogLines []string
+	}{
+		{"mempool:error", []string{
+			``, // if no default is given, assume info
+			``,
+			`{"_msg":"Mesmero","level":"error","module":"mempool"}`,
+			`{"_msg":"Mind","level":"info","module":"state"}`, // if no default is given, assume info
+			``}},
+
+		{"mempool:error,*:debug", []string{
+			`{"_msg":"Kingpin","level":"debug","module":"wire"}`,
+			``,
+			`{"_msg":"Mesmero","level":"error","module":"mempool"}`,
+			`{"_msg":"Mind","level":"info","module":"state"}`,
+			`{"_msg":"Gideon","level":"debug"}`}},
+
+		{"*:debug,wire:none", []string{
+			``,
+			`{"_msg":"Kitty Pryde","level":"info","module":"mempool"}`,
+			`{"_msg":"Mesmero","level":"error","module":"mempool"}`,
+			`{"_msg":"Mind","level":"info","module":"state"}`,
+			`{"_msg":"Gideon","level":"debug"}`}},
+	}
+
+	for _, c := range correctLogLevels {
+		logger, err := tmflags.ParseLogLevel(c.lvl, jsonLogger, defaultLogLevelValue)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		buf.Reset()
+
+		logger.With("module", "wire").Debug("Kingpin")
+		if have := strings.TrimSpace(buf.String()); c.expectedLogLines[0] != have {
+			t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[0], have, c.lvl)
+		}
+
+		buf.Reset()
+
+		logger.With("module", "mempool").Info("Kitty Pryde")
+		if have := strings.TrimSpace(buf.String()); c.expectedLogLines[1] != have {
+			t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[1], have, c.lvl)
+		}
+
+		buf.Reset()
+
+		logger.With("module", "mempool").Error("Mesmero")
+		if have := strings.TrimSpace(buf.String()); c.expectedLogLines[2] != have {
+			t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[2], have, c.lvl)
+		}
+
+		buf.Reset()
+
+		logger.With("module", "state").Info("Mind")
+		if have := strings.TrimSpace(buf.String()); c.expectedLogLines[3] != have {
+			t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[3], have, c.lvl)
+		}
+
+		buf.Reset()
+
+		logger.Debug("Gideon")
+		if have := strings.TrimSpace(buf.String()); c.expectedLogLines[4] != have {
+			t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[4], have, c.lvl)
+		}
+	}
+
+	incorrectLogLevel := []string{"some", "mempool:some", "*:some,mempool:error"}
+	for _, lvl := range incorrectLogLevel {
+		if _, err := tmflags.ParseLogLevel(lvl, jsonLogger, defaultLogLevelValue); err == nil {
+			t.Fatalf("Expected %s to produce error", lvl)
+		}
+	}
+}

libs/cli/helper.go

@@ -0,0 +1,87 @@
+package cli
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// WriteConfigVals writes a toml file with the given values.
+// It returns an error if writing was impossible.
+func WriteConfigVals(dir string, vals map[string]string) error {
+	data := ""
+	for k, v := range vals {
+		data = data + fmt.Sprintf("%s = \"%s\"\n", k, v)
+	}
+	cfile := filepath.Join(dir, "config.toml")
+	return ioutil.WriteFile(cfile, []byte(data), 0666)
+}
+
+// RunWithArgs executes the given command with the specified command line args
+// and environmental variables set. It returns any error returned from cmd.Execute()
+func RunWithArgs(cmd Executable, args []string, env map[string]string) error {
+	oargs := os.Args
+	oenv := map[string]string{}
+	// defer returns the environment back to normal
+	defer func() {
+		os.Args = oargs
+		for k, v := range oenv {
+			os.Setenv(k, v)
+		}
+	}()
+
+	// set the args and env how we want them
+	os.Args = args
+	for k, v := range env {
+		// backup old value if there, to restore at end
+		oenv[k] = os.Getenv(k)
+		err := os.Setenv(k, v)
+		if err != nil {
+			return err
+		}
+	}
+
+	// and finally run the command
+	return cmd.Execute()
+}
+
+// RunCaptureWithArgs executes the given command with the specified command
+// line args and environmental variables set. It returns string fields
+// representing output written to stdout and stderr, additionally any error
+// from cmd.Execute() is also returned
+func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (stdout, stderr string, err error) {
+	oldout, olderr := os.Stdout, os.Stderr // keep backup of the real stdout
+	rOut, wOut, _ := os.Pipe()
+	rErr, wErr, _ := os.Pipe()
+	os.Stdout, os.Stderr = wOut, wErr
+	defer func() {
+		os.Stdout, os.Stderr = oldout, olderr // restoring the real stdout
+	}()
+
+	// copy the output in a separate goroutine so printing can't block indefinitely
+	copyStd := func(reader *os.File) *(chan string) {
+		stdC := make(chan string)
+		go func() {
+			var buf bytes.Buffer
+			// io.Copy will end when we call reader.Close() below
+			io.Copy(&buf, reader)
+			stdC <- buf.String()
+		}()
+		return &stdC
+	}
+	outC := copyStd(rOut)
+	errC := copyStd(rErr)
+
+	// now run the command
+	err = RunWithArgs(cmd, args, env)
+
+	// and grab the stdout to return
+	wOut.Close()
+	wErr.Close()
+	stdout = <-*outC
+	stderr = <-*errC
+	return stdout, stderr, err
+}

libs/cli/setup.go

@@ -0,0 +1,157 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+const (
+	HomeFlag     = "home"
+	TraceFlag    = "trace"
+	OutputFlag   = "output"
+	EncodingFlag = "encoding"
+)
+
+// Executable is the minimal interface to *corba.Command, so we can
+// wrap if desired before the test
+type Executable interface {
+	Execute() error
+}
+
+// PrepareBaseCmd is meant for tendermint and other servers
+func PrepareBaseCmd(cmd *cobra.Command, envPrefix, defaultHome string) Executor {
+	cobra.OnInitialize(func() { initEnv(envPrefix) })
+	cmd.PersistentFlags().StringP(HomeFlag, "", defaultHome, "directory for config and data")
+	cmd.PersistentFlags().Bool(TraceFlag, false, "print out full stack trace on errors")
+	cmd.PersistentPreRunE = concatCobraCmdFuncs(bindFlagsLoadViper, cmd.PersistentPreRunE)
+	return Executor{cmd, os.Exit}
+}
+
+// PrepareMainCmd is meant for client side libs that want some more flags
+//
+// This adds --encoding (hex, btc, base64) and --output (text, json) to
+// the command.  These only really make sense in interactive commands.
+func PrepareMainCmd(cmd *cobra.Command, envPrefix, defaultHome string) Executor {
+	cmd.PersistentFlags().StringP(EncodingFlag, "e", "hex", "Binary encoding (hex|b64|btc)")
+	cmd.PersistentFlags().StringP(OutputFlag, "o", "text", "Output format (text|json)")
+	cmd.PersistentPreRunE = concatCobraCmdFuncs(validateOutput, cmd.PersistentPreRunE)
+	return PrepareBaseCmd(cmd, envPrefix, defaultHome)
+}
+
+// initEnv sets to use ENV variables if set.
+func initEnv(prefix string) {
+	copyEnvVars(prefix)
+
+	// env variables with TM prefix (eg. TM_ROOT)
+	viper.SetEnvPrefix(prefix)
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
+	viper.AutomaticEnv()
+}
+
+// This copies all variables like TMROOT to TM_ROOT,
+// so we can support both formats for the user
+func copyEnvVars(prefix string) {
+	prefix = strings.ToUpper(prefix)
+	ps := prefix + "_"
+	for _, e := range os.Environ() {
+		kv := strings.SplitN(e, "=", 2)
+		if len(kv) == 2 {
+			k, v := kv[0], kv[1]
+			if strings.HasPrefix(k, prefix) && !strings.HasPrefix(k, ps) {
+				k2 := strings.Replace(k, prefix, ps, 1)
+				os.Setenv(k2, v)
+			}
+		}
+	}
+}
+
+// Executor wraps the cobra Command with a nicer Execute method
+type Executor struct {
+	*cobra.Command
+	Exit func(int) // this is os.Exit by default, override in tests
+}
+
+type ExitCoder interface {
+	ExitCode() int
+}
+
+// execute adds all child commands to the root command sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func (e Executor) Execute() error {
+	e.SilenceUsage = true
+	e.SilenceErrors = true
+	err := e.Command.Execute()
+	if err != nil {
+		if viper.GetBool(TraceFlag) {
+			fmt.Fprintf(os.Stderr, "ERROR: %+v\n", err)
+		} else {
+			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
+		}
+
+		// return error code 1 by default, can override it with a special error type
+		exitCode := 1
+		if ec, ok := err.(ExitCoder); ok {
+			exitCode = ec.ExitCode()
+		}
+		e.Exit(exitCode)
+	}
+	return err
+}
+
+type cobraCmdFunc func(cmd *cobra.Command, args []string) error
+
+// Returns a single function that calls each argument function in sequence
+// RunE, PreRunE, PersistentPreRunE, etc. all have this same signature
+func concatCobraCmdFuncs(fs ...cobraCmdFunc) cobraCmdFunc {
+	return func(cmd *cobra.Command, args []string) error {
+		for _, f := range fs {
+			if f != nil {
+				if err := f(cmd, args); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+}
+
+// Bind all flags and read the config into viper
+func bindFlagsLoadViper(cmd *cobra.Command, args []string) error {
+	// cmd.Flags() includes flags from this command and all persistent flags from the parent
+	if err := viper.BindPFlags(cmd.Flags()); err != nil {
+		return err
+	}
+
+	homeDir := viper.GetString(HomeFlag)
+	viper.Set(HomeFlag, homeDir)
+	viper.SetConfigName("config")                         // name of config file (without extension)
+	viper.AddConfigPath(homeDir)                          // search root directory
+	viper.AddConfigPath(filepath.Join(homeDir, "config")) // search root directory /config
+
+	// If a config file is found, read it in.
+	if err := viper.ReadInConfig(); err == nil {
+		// stderr, so if we redirect output to json file, this doesn't appear
+		// fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
+	} else if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
+		// ignore not found error, return other errors
+		return err
+	}
+	return nil
+}
+
+func validateOutput(cmd *cobra.Command, args []string) error {
+	// validate output format
+	output := viper.GetString(OutputFlag)
+	switch output {
+	case "text", "json":
+	default:
+		return errors.Errorf("Unsupported output format: %s", output)
+	}
+	return nil
+}

libs/cli/setup_test.go

@@ -0,0 +1,237 @@
+package cli
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSetupEnv(t *testing.T) {
+	cases := []struct {
+		args     []string
+		env      map[string]string
+		expected string
+	}{
+		{nil, nil, ""},
+		{[]string{"--foobar", "bang!"}, nil, "bang!"},
+		// make sure reset is good
+		{nil, nil, ""},
+		// test both variants of the prefix
+		{nil, map[string]string{"DEMO_FOOBAR": "good"}, "good"},
+		{nil, map[string]string{"DEMOFOOBAR": "silly"}, "silly"},
+		// and that cli overrides env...
+		{[]string{"--foobar", "important"},
+			map[string]string{"DEMO_FOOBAR": "ignored"}, "important"},
+	}
+
+	for idx, tc := range cases {
+		i := strconv.Itoa(idx)
+		// test command that store value of foobar in local variable
+		var foo string
+		demo := &cobra.Command{
+			Use: "demo",
+			RunE: func(cmd *cobra.Command, args []string) error {
+				foo = viper.GetString("foobar")
+				return nil
+			},
+		}
+		demo.Flags().String("foobar", "", "Some test value from config")
+		cmd := PrepareBaseCmd(demo, "DEMO", "/qwerty/asdfgh") // some missing dir..
+		cmd.Exit = func(int) {}
+
+		viper.Reset()
+		args := append([]string{cmd.Use}, tc.args...)
+		err := RunWithArgs(cmd, args, tc.env)
+		require.Nil(t, err, i)
+		assert.Equal(t, tc.expected, foo, i)
+	}
+}
+
+func tempDir() string {
+	cdir, err := ioutil.TempDir("", "test-cli")
+	if err != nil {
+		panic(err)
+	}
+	return cdir
+}
+
+func TestSetupConfig(t *testing.T) {
+	// we pre-create two config files we can refer to in the rest of
+	// the test cases.
+	cval1 := "fubble"
+	conf1 := tempDir()
+	err := WriteConfigVals(conf1, map[string]string{"boo": cval1})
+	require.Nil(t, err)
+
+	cases := []struct {
+		args        []string
+		env         map[string]string
+		expected    string
+		expectedTwo string
+	}{
+		{nil, nil, "", ""},
+		// setting on the command line
+		{[]string{"--boo", "haha"}, nil, "haha", ""},
+		{[]string{"--two-words", "rocks"}, nil, "", "rocks"},
+		{[]string{"--home", conf1}, nil, cval1, ""},
+		// test both variants of the prefix
+		{nil, map[string]string{"RD_BOO": "bang"}, "bang", ""},
+		{nil, map[string]string{"RD_TWO_WORDS": "fly"}, "", "fly"},
+		{nil, map[string]string{"RDTWO_WORDS": "fly"}, "", "fly"},
+		{nil, map[string]string{"RD_HOME": conf1}, cval1, ""},
+		{nil, map[string]string{"RDHOME": conf1}, cval1, ""},
+	}
+
+	for idx, tc := range cases {
+		i := strconv.Itoa(idx)
+		// test command that store value of foobar in local variable
+		var foo, two string
+		boo := &cobra.Command{
+			Use: "reader",
+			RunE: func(cmd *cobra.Command, args []string) error {
+				foo = viper.GetString("boo")
+				two = viper.GetString("two-words")
+				return nil
+			},
+		}
+		boo.Flags().String("boo", "", "Some test value from config")
+		boo.Flags().String("two-words", "", "Check out env handling -")
+		cmd := PrepareBaseCmd(boo, "RD", "/qwerty/asdfgh") // some missing dir...
+		cmd.Exit = func(int) {}
+
+		viper.Reset()
+		args := append([]string{cmd.Use}, tc.args...)
+		err := RunWithArgs(cmd, args, tc.env)
+		require.Nil(t, err, i)
+		assert.Equal(t, tc.expected, foo, i)
+		assert.Equal(t, tc.expectedTwo, two, i)
+	}
+}
+
+type DemoConfig struct {
+	Name   string `mapstructure:"name"`
+	Age    int    `mapstructure:"age"`
+	Unused int    `mapstructure:"unused"`
+}
+
+func TestSetupUnmarshal(t *testing.T) {
+	// we pre-create two config files we can refer to in the rest of
+	// the test cases.
+	cval1, cval2 := "someone", "else"
+	conf1 := tempDir()
+	err := WriteConfigVals(conf1, map[string]string{"name": cval1})
+	require.Nil(t, err)
+	// even with some ignored fields, should be no problem
+	conf2 := tempDir()
+	err = WriteConfigVals(conf2, map[string]string{"name": cval2, "foo": "bar"})
+	require.Nil(t, err)
+
+	// unused is not declared on a flag and remains from base
+	base := DemoConfig{
+		Name:   "default",
+		Age:    42,
+		Unused: -7,
+	}
+	c := func(name string, age int) DemoConfig {
+		r := base
+		// anything set on the flags as a default is used over
+		// the default config object
+		r.Name = "from-flag"
+		if name != "" {
+			r.Name = name
+		}
+		if age != 0 {
+			r.Age = age
+		}
+		return r
+	}
+
+	cases := []struct {
+		args     []string
+		env      map[string]string
+		expected DemoConfig
+	}{
+		{nil, nil, c("", 0)},
+		// setting on the command line
+		{[]string{"--name", "haha"}, nil, c("haha", 0)},
+		{[]string{"--home", conf1}, nil, c(cval1, 0)},
+		// test both variants of the prefix
+		{nil, map[string]string{"MR_AGE": "56"}, c("", 56)},
+		{nil, map[string]string{"MR_HOME": conf1}, c(cval1, 0)},
+		{[]string{"--age", "17"}, map[string]string{"MRHOME": conf2}, c(cval2, 17)},
+	}
+
+	for idx, tc := range cases {
+		i := strconv.Itoa(idx)
+		// test command that store value of foobar in local variable
+		cfg := base
+		marsh := &cobra.Command{
+			Use: "marsh",
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return viper.Unmarshal(&cfg)
+			},
+		}
+		marsh.Flags().String("name", "from-flag", "Some test value from config")
+		// if we want a flag to use the proper default, then copy it
+		// from the default config here
+		marsh.Flags().Int("age", base.Age, "Some test value from config")
+		cmd := PrepareBaseCmd(marsh, "MR", "/qwerty/asdfgh") // some missing dir...
+		cmd.Exit = func(int) {}
+
+		viper.Reset()
+		args := append([]string{cmd.Use}, tc.args...)
+		err := RunWithArgs(cmd, args, tc.env)
+		require.Nil(t, err, i)
+		assert.Equal(t, tc.expected, cfg, i)
+	}
+}
+
+func TestSetupTrace(t *testing.T) {
+	cases := []struct {
+		args     []string
+		env      map[string]string
+		long     bool
+		expected string
+	}{
+		{nil, nil, false, "Trace flag = false"},
+		{[]string{"--trace"}, nil, true, "Trace flag = true"},
+		{[]string{"--no-such-flag"}, nil, false, "unknown flag: --no-such-flag"},
+		{nil, map[string]string{"DBG_TRACE": "true"}, true, "Trace flag = true"},
+	}
+
+	for idx, tc := range cases {
+		i := strconv.Itoa(idx)
+		// test command that store value of foobar in local variable
+		trace := &cobra.Command{
+			Use: "trace",
+			RunE: func(cmd *cobra.Command, args []string) error {
+				return errors.Errorf("Trace flag = %t", viper.GetBool(TraceFlag))
+			},
+		}
+		cmd := PrepareBaseCmd(trace, "DBG", "/qwerty/asdfgh") // some missing dir..
+		cmd.Exit = func(int) {}
+
+		viper.Reset()
+		args := append([]string{cmd.Use}, tc.args...)
+		stdout, stderr, err := RunCaptureWithArgs(cmd, args, tc.env)
+		require.NotNil(t, err, i)
+		require.Equal(t, "", stdout, i)
+		require.NotEqual(t, "", stderr, i)
+		msg := strings.Split(stderr, "\n")
+		desired := fmt.Sprintf("ERROR: %s", tc.expected)
+		assert.Equal(t, desired, msg[0], i)
+		if tc.long && assert.True(t, len(msg) > 2, i) {
+			// the next line starts the stack trace...
+			assert.Contains(t, msg[1], "TestSetupTrace", i)
+			assert.Contains(t, msg[2], "setup_test.go", i)
+		}
+	}
+}