Add abci repo

abci/example/code/code.go

@@ -0,0 +1,9 @@
+package code
+
+// Return codes for the examples
+const (
+	CodeTypeOK            uint32 = 0
+	CodeTypeEncodingError uint32 = 1
+	CodeTypeBadNonce      uint32 = 2
+	CodeTypeUnauthorized  uint32 = 3
+)

abci/example/counter/counter.go

@@ -0,0 +1,104 @@
+package counter
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/tendermint/abci/example/code"
+	"github.com/tendermint/abci/types"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+type CounterApplication struct {
+	types.BaseApplication
+
+	hashCount int
+	txCount   int
+	serial    bool
+}
+
+func NewCounterApplication(serial bool) *CounterApplication {
+	return &CounterApplication{serial: serial}
+}
+
+func (app *CounterApplication) Info(req types.RequestInfo) types.ResponseInfo {
+	return types.ResponseInfo{Data: cmn.Fmt("{\"hashes\":%v,\"txs\":%v}", app.hashCount, app.txCount)}
+}
+
+func (app *CounterApplication) SetOption(req types.RequestSetOption) types.ResponseSetOption {
+	key, value := req.Key, req.Value
+	if key == "serial" && value == "on" {
+		app.serial = true
+	} else {
+		/*
+			TODO Panic and have the ABCI server pass an exception.
+			The client can call SetOptionSync() and get an `error`.
+			return types.ResponseSetOption{
+				Error: cmn.Fmt("Unknown key (%s) or value (%s)", key, value),
+			}
+		*/
+		return types.ResponseSetOption{}
+	}
+
+	return types.ResponseSetOption{}
+}
+
+func (app *CounterApplication) DeliverTx(tx []byte) types.ResponseDeliverTx {
+	if app.serial {
+		if len(tx) > 8 {
+			return types.ResponseDeliverTx{
+				Code: code.CodeTypeEncodingError,
+				Log:  fmt.Sprintf("Max tx size is 8 bytes, got %d", len(tx))}
+		}
+		tx8 := make([]byte, 8)
+		copy(tx8[len(tx8)-len(tx):], tx)
+		txValue := binary.BigEndian.Uint64(tx8)
+		if txValue != uint64(app.txCount) {
+			return types.ResponseDeliverTx{
+				Code: code.CodeTypeBadNonce,
+				Log:  fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.txCount, txValue)}
+		}
+	}
+	app.txCount++
+	return types.ResponseDeliverTx{Code: code.CodeTypeOK}
+}
+
+func (app *CounterApplication) CheckTx(tx []byte) types.ResponseCheckTx {
+	if app.serial {
+		if len(tx) > 8 {
+			return types.ResponseCheckTx{
+				Code: code.CodeTypeEncodingError,
+				Log:  fmt.Sprintf("Max tx size is 8 bytes, got %d", len(tx))}
+		}
+		tx8 := make([]byte, 8)
+		copy(tx8[len(tx8)-len(tx):], tx)
+		txValue := binary.BigEndian.Uint64(tx8)
+		if txValue < uint64(app.txCount) {
+			return types.ResponseCheckTx{
+				Code: code.CodeTypeBadNonce,
+				Log:  fmt.Sprintf("Invalid nonce. Expected >= %v, got %v", app.txCount, txValue)}
+		}
+	}
+	return types.ResponseCheckTx{Code: code.CodeTypeOK}
+}
+
+func (app *CounterApplication) Commit() (resp types.ResponseCommit) {
+	app.hashCount++
+	if app.txCount == 0 {
+		return types.ResponseCommit{}
+	}
+	hash := make([]byte, 8)
+	binary.BigEndian.PutUint64(hash, uint64(app.txCount))
+	return types.ResponseCommit{Data: hash}
+}
+
+func (app *CounterApplication) Query(reqQuery types.RequestQuery) types.ResponseQuery {
+	switch reqQuery.Path {
+	case "hash":
+		return types.ResponseQuery{Value: []byte(cmn.Fmt("%v", app.hashCount))}
+	case "tx":
+		return types.ResponseQuery{Value: []byte(cmn.Fmt("%v", app.txCount))}
+	default:
+		return types.ResponseQuery{Log: cmn.Fmt("Invalid query path. Expected hash or tx, got %v", reqQuery.Path)}
+	}
+}

abci/example/example.go

@@ -0,0 +1,3 @@
+package example
+
+// so the go tool doesn't return errors about no buildable go files ...

abci/example/example_test.go

@@ -0,0 +1,154 @@
+package example
+
+import (
+	"fmt"
+	"net"
+	"reflect"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+
+	"golang.org/x/net/context"
+
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
+
+	abcicli "github.com/tendermint/abci/client"
+	"github.com/tendermint/abci/example/code"
+	"github.com/tendermint/abci/example/kvstore"
+	abciserver "github.com/tendermint/abci/server"
+	"github.com/tendermint/abci/types"
+)
+
+func TestKVStore(t *testing.T) {
+	fmt.Println("### Testing KVStore")
+	testStream(t, kvstore.NewKVStoreApplication())
+}
+
+func TestBaseApp(t *testing.T) {
+	fmt.Println("### Testing BaseApp")
+	testStream(t, types.NewBaseApplication())
+}
+
+func TestGRPC(t *testing.T) {
+	fmt.Println("### Testing GRPC")
+	testGRPCSync(t, types.NewGRPCApplication(types.NewBaseApplication()))
+}
+
+func testStream(t *testing.T, app types.Application) {
+	numDeliverTxs := 200000
+
+	// Start the listener
+	server := abciserver.NewSocketServer("unix://test.sock", app)
+	server.SetLogger(log.TestingLogger().With("module", "abci-server"))
+	if err := server.Start(); err != nil {
+		t.Fatalf("Error starting socket server: %v", err.Error())
+	}
+	defer server.Stop()
+
+	// Connect to the socket
+	client := abcicli.NewSocketClient("unix://test.sock", false)
+	client.SetLogger(log.TestingLogger().With("module", "abci-client"))
+	if err := client.Start(); err != nil {
+		t.Fatalf("Error starting socket client: %v", err.Error())
+	}
+	defer client.Stop()
+
+	done := make(chan struct{})
+	counter := 0
+	client.SetResponseCallback(func(req *types.Request, res *types.Response) {
+		// Process response
+		switch r := res.Value.(type) {
+		case *types.Response_DeliverTx:
+			counter++
+			if r.DeliverTx.Code != code.CodeTypeOK {
+				t.Error("DeliverTx failed with ret_code", r.DeliverTx.Code)
+			}
+			if counter > numDeliverTxs {
+				t.Fatalf("Too many DeliverTx responses. Got %d, expected %d", counter, numDeliverTxs)
+			}
+			if counter == numDeliverTxs {
+				go func() {
+					time.Sleep(time.Second * 2) // Wait for a bit to allow counter overflow
+					close(done)
+				}()
+				return
+			}
+		case *types.Response_Flush:
+			// ignore
+		default:
+			t.Error("Unexpected response type", reflect.TypeOf(res.Value))
+		}
+	})
+
+	// Write requests
+	for counter := 0; counter < numDeliverTxs; counter++ {
+		// Send request
+		reqRes := client.DeliverTxAsync([]byte("test"))
+		_ = reqRes
+		// check err ?
+
+		// Sometimes send flush messages
+		if counter%123 == 0 {
+			client.FlushAsync()
+			// check err ?
+		}
+	}
+
+	// Send final flush message
+	client.FlushAsync()
+
+	<-done
+}
+
+//-------------------------
+// test grpc
+
+func dialerFunc(addr string, timeout time.Duration) (net.Conn, error) {
+	return cmn.Connect(addr)
+}
+
+func testGRPCSync(t *testing.T, app *types.GRPCApplication) {
+	numDeliverTxs := 2000
+
+	// Start the listener
+	server := abciserver.NewGRPCServer("unix://test.sock", app)
+	server.SetLogger(log.TestingLogger().With("module", "abci-server"))
+	if err := server.Start(); err != nil {
+		t.Fatalf("Error starting GRPC server: %v", err.Error())
+	}
+	defer server.Stop()
+
+	// Connect to the socket
+	conn, err := grpc.Dial("unix://test.sock", grpc.WithInsecure(), grpc.WithDialer(dialerFunc))
+	if err != nil {
+		t.Fatalf("Error dialing GRPC server: %v", err.Error())
+	}
+	defer conn.Close()
+
+	client := types.NewABCIApplicationClient(conn)
+
+	// Write requests
+	for counter := 0; counter < numDeliverTxs; counter++ {
+		// Send request
+		response, err := client.DeliverTx(context.Background(), &types.RequestDeliverTx{[]byte("test")})
+		if err != nil {
+			t.Fatalf("Error in GRPC DeliverTx: %v", err.Error())
+		}
+		counter++
+		if response.Code != code.CodeTypeOK {
+			t.Error("DeliverTx failed with ret_code", response.Code)
+		}
+		if counter > numDeliverTxs {
+			t.Fatal("Too many DeliverTx responses")
+		}
+		t.Log("response", counter)
+		if counter == numDeliverTxs {
+			go func() {
+				time.Sleep(time.Second * 2) // Wait for a bit to allow counter overflow
+			}()
+		}
+
+	}
+}

abci/example/js/.gitignore

@@ -0,0 +1 @@
+node_modules

abci/example/js/README.md

@@ -0,0 +1 @@
+This example has been moved here: https://github.com/tendermint/js-abci/tree/master/example

abci/example/kvstore/README.md

@@ -0,0 +1,31 @@
+# KVStore
+
+There are two app's here: the KVStoreApplication and the PersistentKVStoreApplication.
+
+## KVStoreApplication
+
+The KVStoreApplication is a simple merkle key-value store. 
+Transactions of the form `key=value` are stored as key-value pairs in the tree.
+Transactions without an `=` sign set the value to the key.
+The app has no replay protection (other than what the mempool provides).
+
+## PersistentKVStoreApplication
+
+The PersistentKVStoreApplication wraps the KVStoreApplication
+and provides two additional features:
+
+1) persistence of state across app restarts (using Tendermint's ABCI-Handshake mechanism)
+2) validator set changes
+
+The state is persisted in leveldb along with the last block committed,
+and the Handshake allows any necessary blocks to be replayed.
+Validator set changes are effected using the following transaction format:
+
+```
+val:pubkey1/power1,addr2/power2,addr3/power3"
+```
+
+where `power1` is the new voting power for the validator with `pubkey1` (possibly a new one).
+There is no sybil protection against new validators joining. 
+Validators can be removed by setting their power to `0`.
+

abci/example/kvstore/helpers.go

@@ -0,0 +1,38 @@
+package kvstore
+
+import (
+	"github.com/tendermint/abci/types"
+	cmn "github.com/tendermint/tmlibs/common"
+)
+
+// RandVal creates one random validator, with a key derived
+// from the input value
+func RandVal(i int) types.Validator {
+	addr := cmn.RandBytes(20)
+	pubkey := cmn.RandBytes(32)
+	power := cmn.RandUint16() + 1
+	v := types.Ed25519Validator(pubkey, int64(power))
+	v.Address = addr
+	return v
+}
+
+// RandVals returns a list of cnt validators for initializing
+// the application. Note that the keys are deterministically
+// derived from the index in the array, while the power is
+// random (Change this if not desired)
+func RandVals(cnt int) []types.Validator {
+	res := make([]types.Validator, cnt)
+	for i := 0; i < cnt; i++ {
+		res[i] = RandVal(i)
+	}
+	return res
+}
+
+// InitKVStore initializes the kvstore app with some data,
+// which allows tests to pass and is fine as long as you
+// don't make any tx that modify the validator state
+func InitKVStore(app *PersistentKVStoreApplication) {
+	app.InitChain(types.RequestInitChain{
+		Validators: RandVals(1),
+	})
+}

abci/example/kvstore/kvstore.go

@@ -0,0 +1,126 @@
+package kvstore
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+
+	"github.com/tendermint/abci/example/code"
+	"github.com/tendermint/abci/types"
+	cmn "github.com/tendermint/tmlibs/common"
+	dbm "github.com/tendermint/tmlibs/db"
+)
+
+var (
+	stateKey        = []byte("stateKey")
+	kvPairPrefixKey = []byte("kvPairKey:")
+)
+
+type State struct {
+	db      dbm.DB
+	Size    int64  `json:"size"`
+	Height  int64  `json:"height"`
+	AppHash []byte `json:"app_hash"`
+}
+
+func loadState(db dbm.DB) State {
+	stateBytes := db.Get(stateKey)
+	var state State
+	if len(stateBytes) != 0 {
+		err := json.Unmarshal(stateBytes, &state)
+		if err != nil {
+			panic(err)
+		}
+	}
+	state.db = db
+	return state
+}
+
+func saveState(state State) {
+	stateBytes, err := json.Marshal(state)
+	if err != nil {
+		panic(err)
+	}
+	state.db.Set(stateKey, stateBytes)
+}
+
+func prefixKey(key []byte) []byte {
+	return append(kvPairPrefixKey, key...)
+}
+
+//---------------------------------------------------
+
+var _ types.Application = (*KVStoreApplication)(nil)
+
+type KVStoreApplication struct {
+	types.BaseApplication
+
+	state State
+}
+
+func NewKVStoreApplication() *KVStoreApplication {
+	state := loadState(dbm.NewMemDB())
+	return &KVStoreApplication{state: state}
+}
+
+func (app *KVStoreApplication) Info(req types.RequestInfo) (resInfo types.ResponseInfo) {
+	return types.ResponseInfo{Data: fmt.Sprintf("{\"size\":%v}", app.state.Size)}
+}
+
+// tx is either "key=value" or just arbitrary bytes
+func (app *KVStoreApplication) DeliverTx(tx []byte) types.ResponseDeliverTx {
+	var key, value []byte
+	parts := bytes.Split(tx, []byte("="))
+	if len(parts) == 2 {
+		key, value = parts[0], parts[1]
+	} else {
+		key, value = tx, tx
+	}
+	app.state.db.Set(prefixKey(key), value)
+	app.state.Size += 1
+
+	tags := []cmn.KVPair{
+		{[]byte("app.creator"), []byte("jae")},
+		{[]byte("app.key"), key},
+	}
+	return types.ResponseDeliverTx{Code: code.CodeTypeOK, Tags: tags}
+}
+
+func (app *KVStoreApplication) CheckTx(tx []byte) types.ResponseCheckTx {
+	return types.ResponseCheckTx{Code: code.CodeTypeOK}
+}
+
+func (app *KVStoreApplication) Commit() types.ResponseCommit {
+	// Using a memdb - just return the big endian size of the db
+	appHash := make([]byte, 8)
+	binary.PutVarint(appHash, app.state.Size)
+	app.state.AppHash = appHash
+	app.state.Height += 1
+	saveState(app.state)
+	return types.ResponseCommit{Data: appHash}
+}
+
+func (app *KVStoreApplication) Query(reqQuery types.RequestQuery) (resQuery types.ResponseQuery) {
+	if reqQuery.Prove {
+		value := app.state.db.Get(prefixKey(reqQuery.Data))
+		resQuery.Index = -1 // TODO make Proof return index
+		resQuery.Key = reqQuery.Data
+		resQuery.Value = value
+		if value != nil {
+			resQuery.Log = "exists"
+		} else {
+			resQuery.Log = "does not exist"
+		}
+		return
+	} else {
+		value := app.state.db.Get(prefixKey(reqQuery.Data))
+		resQuery.Value = value
+		if value != nil {
+			resQuery.Log = "exists"
+		} else {
+			resQuery.Log = "does not exist"
+		}
+		return
+	}
+}

abci/example/kvstore/kvstore_test.go

@@ -0,0 +1,310 @@
+package kvstore
+
+import (
+	"bytes"
+	"io/ioutil"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
+
+	abcicli "github.com/tendermint/abci/client"
+	"github.com/tendermint/abci/example/code"
+	abciserver "github.com/tendermint/abci/server"
+	"github.com/tendermint/abci/types"
+)
+
+func testKVStore(t *testing.T, app types.Application, tx []byte, key, value string) {
+	ar := app.DeliverTx(tx)
+	require.False(t, ar.IsErr(), ar)
+	// repeating tx doesn't raise error
+	ar = app.DeliverTx(tx)
+	require.False(t, ar.IsErr(), ar)
+
+	// make sure query is fine
+	resQuery := app.Query(types.RequestQuery{
+		Path: "/store",
+		Data: []byte(key),
+	})
+	require.Equal(t, code.CodeTypeOK, resQuery.Code)
+	require.Equal(t, value, string(resQuery.Value))
+
+	// make sure proof is fine
+	resQuery = app.Query(types.RequestQuery{
+		Path:  "/store",
+		Data:  []byte(key),
+		Prove: true,
+	})
+	require.EqualValues(t, code.CodeTypeOK, resQuery.Code)
+	require.Equal(t, value, string(resQuery.Value))
+}
+
+func TestKVStoreKV(t *testing.T) {
+	kvstore := NewKVStoreApplication()
+	key := "abc"
+	value := key
+	tx := []byte(key)
+	testKVStore(t, kvstore, tx, key, value)
+
+	value = "def"
+	tx = []byte(key + "=" + value)
+	testKVStore(t, kvstore, tx, key, value)
+}
+
+func TestPersistentKVStoreKV(t *testing.T) {
+	dir, err := ioutil.TempDir("/tmp", "abci-kvstore-test") // TODO
+	if err != nil {
+		t.Fatal(err)
+	}
+	kvstore := NewPersistentKVStoreApplication(dir)
+	key := "abc"
+	value := key
+	tx := []byte(key)
+	testKVStore(t, kvstore, tx, key, value)
+
+	value = "def"
+	tx = []byte(key + "=" + value)
+	testKVStore(t, kvstore, tx, key, value)
+}
+
+func TestPersistentKVStoreInfo(t *testing.T) {
+	dir, err := ioutil.TempDir("/tmp", "abci-kvstore-test") // TODO
+	if err != nil {
+		t.Fatal(err)
+	}
+	kvstore := NewPersistentKVStoreApplication(dir)
+	InitKVStore(kvstore)
+	height := int64(0)
+
+	resInfo := kvstore.Info(types.RequestInfo{})
+	if resInfo.LastBlockHeight != height {
+		t.Fatalf("expected height of %d, got %d", height, resInfo.LastBlockHeight)
+	}
+
+	// make and apply block
+	height = int64(1)
+	hash := []byte("foo")
+	header := types.Header{
+		Height: int64(height),
+	}
+	kvstore.BeginBlock(types.RequestBeginBlock{hash, header, nil, nil})
+	kvstore.EndBlock(types.RequestEndBlock{header.Height})
+	kvstore.Commit()
+
+	resInfo = kvstore.Info(types.RequestInfo{})
+	if resInfo.LastBlockHeight != height {
+		t.Fatalf("expected height of %d, got %d", height, resInfo.LastBlockHeight)
+	}
+
+}
+
+// add a validator, remove a validator, update a validator
+func TestValUpdates(t *testing.T) {
+	dir, err := ioutil.TempDir("/tmp", "abci-kvstore-test") // TODO
+	if err != nil {
+		t.Fatal(err)
+	}
+	kvstore := NewPersistentKVStoreApplication(dir)
+
+	// init with some validators
+	total := 10
+	nInit := 5
+	vals := RandVals(total)
+	// iniitalize with the first nInit
+	kvstore.InitChain(types.RequestInitChain{
+		Validators: vals[:nInit],
+	})
+
+	vals1, vals2 := vals[:nInit], kvstore.Validators()
+	valsEqual(t, vals1, vals2)
+
+	var v1, v2, v3 types.Validator
+
+	// add some validators
+	v1, v2 = vals[nInit], vals[nInit+1]
+	diff := []types.Validator{v1, v2}
+	tx1 := MakeValSetChangeTx(v1.PubKey, v1.Power)
+	tx2 := MakeValSetChangeTx(v2.PubKey, v2.Power)
+
+	makeApplyBlock(t, kvstore, 1, diff, tx1, tx2)
+
+	vals1, vals2 = vals[:nInit+2], kvstore.Validators()
+	valsEqual(t, vals1, vals2)
+
+	// remove some validators
+	v1, v2, v3 = vals[nInit-2], vals[nInit-1], vals[nInit]
+	v1.Power = 0
+	v2.Power = 0
+	v3.Power = 0
+	diff = []types.Validator{v1, v2, v3}
+	tx1 = MakeValSetChangeTx(v1.PubKey, v1.Power)
+	tx2 = MakeValSetChangeTx(v2.PubKey, v2.Power)
+	tx3 := MakeValSetChangeTx(v3.PubKey, v3.Power)
+
+	makeApplyBlock(t, kvstore, 2, diff, tx1, tx2, tx3)
+
+	vals1 = append(vals[:nInit-2], vals[nInit+1])
+	vals2 = kvstore.Validators()
+	valsEqual(t, vals1, vals2)
+
+	// update some validators
+	v1 = vals[0]
+	if v1.Power == 5 {
+		v1.Power = 6
+	} else {
+		v1.Power = 5
+	}
+	diff = []types.Validator{v1}
+	tx1 = MakeValSetChangeTx(v1.PubKey, v1.Power)
+
+	makeApplyBlock(t, kvstore, 3, diff, tx1)
+
+	vals1 = append([]types.Validator{v1}, vals1[1:]...)
+	vals2 = kvstore.Validators()
+	valsEqual(t, vals1, vals2)
+
+}
+
+func makeApplyBlock(t *testing.T, kvstore types.Application, heightInt int, diff []types.Validator, txs ...[]byte) {
+	// make and apply block
+	height := int64(heightInt)
+	hash := []byte("foo")
+	header := types.Header{
+		Height: height,
+	}
+
+	kvstore.BeginBlock(types.RequestBeginBlock{hash, header, nil, nil})
+	for _, tx := range txs {
+		if r := kvstore.DeliverTx(tx); r.IsErr() {
+			t.Fatal(r)
+		}
+	}
+	resEndBlock := kvstore.EndBlock(types.RequestEndBlock{header.Height})
+	kvstore.Commit()
+
+	valsEqual(t, diff, resEndBlock.ValidatorUpdates)
+
+}
+
+// order doesn't matter
+func valsEqual(t *testing.T, vals1, vals2 []types.Validator) {
+	if len(vals1) != len(vals2) {
+		t.Fatalf("vals dont match in len. got %d, expected %d", len(vals2), len(vals1))
+	}
+	sort.Sort(types.Validators(vals1))
+	sort.Sort(types.Validators(vals2))
+	for i, v1 := range vals1 {
+		v2 := vals2[i]
+		if !bytes.Equal(v1.PubKey.Data, v2.PubKey.Data) ||
+			v1.Power != v2.Power {
+			t.Fatalf("vals dont match at index %d. got %X/%d , expected %X/%d", i, v2.PubKey, v2.Power, v1.PubKey, v1.Power)
+		}
+	}
+}
+
+func makeSocketClientServer(app types.Application, name string) (abcicli.Client, cmn.Service, error) {
+	// Start the listener
+	socket := cmn.Fmt("unix://%s.sock", name)
+	logger := log.TestingLogger()
+
+	server := abciserver.NewSocketServer(socket, app)
+	server.SetLogger(logger.With("module", "abci-server"))
+	if err := server.Start(); err != nil {
+		return nil, nil, err
+	}
+
+	// Connect to the socket
+	client := abcicli.NewSocketClient(socket, false)
+	client.SetLogger(logger.With("module", "abci-client"))
+	if err := client.Start(); err != nil {
+		server.Stop()
+		return nil, nil, err
+	}
+
+	return client, server, nil
+}
+
+func makeGRPCClientServer(app types.Application, name string) (abcicli.Client, cmn.Service, error) {
+	// Start the listener
+	socket := cmn.Fmt("unix://%s.sock", name)
+	logger := log.TestingLogger()
+
+	gapp := types.NewGRPCApplication(app)
+	server := abciserver.NewGRPCServer(socket, gapp)
+	server.SetLogger(logger.With("module", "abci-server"))
+	if err := server.Start(); err != nil {
+		return nil, nil, err
+	}
+
+	client := abcicli.NewGRPCClient(socket, true)
+	client.SetLogger(logger.With("module", "abci-client"))
+	if err := client.Start(); err != nil {
+		server.Stop()
+		return nil, nil, err
+	}
+	return client, server, nil
+}
+
+func TestClientServer(t *testing.T) {
+	// set up socket app
+	kvstore := NewKVStoreApplication()
+	client, server, err := makeSocketClientServer(kvstore, "kvstore-socket")
+	require.Nil(t, err)
+	defer server.Stop()
+	defer client.Stop()
+
+	runClientTests(t, client)
+
+	// set up grpc app
+	kvstore = NewKVStoreApplication()
+	gclient, gserver, err := makeGRPCClientServer(kvstore, "kvstore-grpc")
+	require.Nil(t, err)
+	defer gserver.Stop()
+	defer gclient.Stop()
+
+	runClientTests(t, gclient)
+}
+
+func runClientTests(t *testing.T, client abcicli.Client) {
+	// run some tests....
+	key := "abc"
+	value := key
+	tx := []byte(key)
+	testClient(t, client, tx, key, value)
+
+	value = "def"
+	tx = []byte(key + "=" + value)
+	testClient(t, client, tx, key, value)
+}
+
+func testClient(t *testing.T, app abcicli.Client, tx []byte, key, value string) {
+	ar, err := app.DeliverTxSync(tx)
+	require.NoError(t, err)
+	require.False(t, ar.IsErr(), ar)
+	// repeating tx doesn't raise error
+	ar, err = app.DeliverTxSync(tx)
+	require.NoError(t, err)
+	require.False(t, ar.IsErr(), ar)
+
+	// make sure query is fine
+	resQuery, err := app.QuerySync(types.RequestQuery{
+		Path: "/store",
+		Data: []byte(key),
+	})
+	require.Nil(t, err)
+	require.Equal(t, code.CodeTypeOK, resQuery.Code)
+	require.Equal(t, value, string(resQuery.Value))
+
+	// make sure proof is fine
+	resQuery, err = app.QuerySync(types.RequestQuery{
+		Path:  "/store",
+		Data:  []byte(key),
+		Prove: true,
+	})
+	require.Nil(t, err)
+	require.Equal(t, code.CodeTypeOK, resQuery.Code)
+	require.Equal(t, value, string(resQuery.Value))
+}

abci/example/kvstore/persistent_kvstore.go

@@ -0,0 +1,200 @@
+package kvstore
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/tendermint/abci/example/code"
+	"github.com/tendermint/abci/types"
+	cmn "github.com/tendermint/tmlibs/common"
+	dbm "github.com/tendermint/tmlibs/db"
+	"github.com/tendermint/tmlibs/log"
+)
+
+const (
+	ValidatorSetChangePrefix string = "val:"
+)
+
+//-----------------------------------------
+
+var _ types.Application = (*PersistentKVStoreApplication)(nil)
+
+type PersistentKVStoreApplication struct {
+	app *KVStoreApplication
+
+	// validator set
+	ValUpdates []types.Validator
+
+	logger log.Logger
+}
+
+func NewPersistentKVStoreApplication(dbDir string) *PersistentKVStoreApplication {
+	name := "kvstore"
+	db, err := dbm.NewGoLevelDB(name, dbDir)
+	if err != nil {
+		panic(err)
+	}
+
+	state := loadState(db)
+
+	return &PersistentKVStoreApplication{
+		app:    &KVStoreApplication{state: state},
+		logger: log.NewNopLogger(),
+	}
+}
+
+func (app *PersistentKVStoreApplication) SetLogger(l log.Logger) {
+	app.logger = l
+}
+
+func (app *PersistentKVStoreApplication) Info(req types.RequestInfo) types.ResponseInfo {
+	res := app.app.Info(req)
+	res.LastBlockHeight = app.app.state.Height
+	res.LastBlockAppHash = app.app.state.AppHash
+	return res
+}
+
+func (app *PersistentKVStoreApplication) SetOption(req types.RequestSetOption) types.ResponseSetOption {
+	return app.app.SetOption(req)
+}
+
+// tx is either "val:pubkey/power" or "key=value" or just arbitrary bytes
+func (app *PersistentKVStoreApplication) DeliverTx(tx []byte) types.ResponseDeliverTx {
+	// if it starts with "val:", update the validator set
+	// format is "val:pubkey/power"
+	if isValidatorTx(tx) {
+		// update validators in the merkle tree
+		// and in app.ValUpdates
+		return app.execValidatorTx(tx)
+	}
+
+	// otherwise, update the key-value store
+	return app.app.DeliverTx(tx)
+}
+
+func (app *PersistentKVStoreApplication) CheckTx(tx []byte) types.ResponseCheckTx {
+	return app.app.CheckTx(tx)
+}
+
+// Commit will panic if InitChain was not called
+func (app *PersistentKVStoreApplication) Commit() types.ResponseCommit {
+	return app.app.Commit()
+}
+
+func (app *PersistentKVStoreApplication) Query(reqQuery types.RequestQuery) types.ResponseQuery {
+	return app.app.Query(reqQuery)
+}
+
+// Save the validators in the merkle tree
+func (app *PersistentKVStoreApplication) InitChain(req types.RequestInitChain) types.ResponseInitChain {
+	for _, v := range req.Validators {
+		r := app.updateValidator(v)
+		if r.IsErr() {
+			app.logger.Error("Error updating validators", "r", r)
+		}
+	}
+	return types.ResponseInitChain{}
+}
+
+// Track the block hash and header information
+func (app *PersistentKVStoreApplication) BeginBlock(req types.RequestBeginBlock) types.ResponseBeginBlock {
+	// reset valset changes
+	app.ValUpdates = make([]types.Validator, 0)
+	return types.ResponseBeginBlock{}
+}
+
+// Update the validator set
+func (app *PersistentKVStoreApplication) EndBlock(req types.RequestEndBlock) types.ResponseEndBlock {
+	return types.ResponseEndBlock{ValidatorUpdates: app.ValUpdates}
+}
+
+//---------------------------------------------
+// update validators
+
+func (app *PersistentKVStoreApplication) Validators() (validators []types.Validator) {
+	itr := app.app.state.db.Iterator(nil, nil)
+	for ; itr.Valid(); itr.Next() {
+		if isValidatorTx(itr.Key()) {
+			validator := new(types.Validator)
+			err := types.ReadMessage(bytes.NewBuffer(itr.Value()), validator)
+			if err != nil {
+				panic(err)
+			}
+			validators = append(validators, *validator)
+		}
+	}
+	return
+}
+
+func MakeValSetChangeTx(pubkey types.PubKey, power int64) []byte {
+	return []byte(cmn.Fmt("val:%X/%d", pubkey.Data, power))
+}
+
+func isValidatorTx(tx []byte) bool {
+	return strings.HasPrefix(string(tx), ValidatorSetChangePrefix)
+}
+
+// format is "val:pubkey/power"
+// pubkey is raw 32-byte ed25519 key
+func (app *PersistentKVStoreApplication) execValidatorTx(tx []byte) types.ResponseDeliverTx {
+	tx = tx[len(ValidatorSetChangePrefix):]
+
+	//get the pubkey and power
+	pubKeyAndPower := strings.Split(string(tx), "/")
+	if len(pubKeyAndPower) != 2 {
+		return types.ResponseDeliverTx{
+			Code: code.CodeTypeEncodingError,
+			Log:  fmt.Sprintf("Expected 'pubkey/power'. Got %v", pubKeyAndPower)}
+	}
+	pubkeyS, powerS := pubKeyAndPower[0], pubKeyAndPower[1]
+
+	// decode the pubkey
+	pubkey, err := hex.DecodeString(pubkeyS)
+	if err != nil {
+		return types.ResponseDeliverTx{
+			Code: code.CodeTypeEncodingError,
+			Log:  fmt.Sprintf("Pubkey (%s) is invalid hex", pubkeyS)}
+	}
+
+	// decode the power
+	power, err := strconv.ParseInt(powerS, 10, 64)
+	if err != nil {
+		return types.ResponseDeliverTx{
+			Code: code.CodeTypeEncodingError,
+			Log:  fmt.Sprintf("Power (%s) is not an int", powerS)}
+	}
+
+	// update
+	return app.updateValidator(types.Ed25519Validator(pubkey, int64(power)))
+}
+
+// add, update, or remove a validator
+func (app *PersistentKVStoreApplication) updateValidator(v types.Validator) types.ResponseDeliverTx {
+	key := []byte("val:" + string(v.PubKey.Data))
+	if v.Power == 0 {
+		// remove validator
+		if !app.app.state.db.Has(key) {
+			return types.ResponseDeliverTx{
+				Code: code.CodeTypeUnauthorized,
+				Log:  fmt.Sprintf("Cannot remove non-existent validator %X", key)}
+		}
+		app.app.state.db.Delete(key)
+	} else {
+		// add or update validator
+		value := bytes.NewBuffer(make([]byte, 0))
+		if err := types.WriteMessage(&v, value); err != nil {
+			return types.ResponseDeliverTx{
+				Code: code.CodeTypeEncodingError,
+				Log:  fmt.Sprintf("Error encoding validator: %v", err)}
+		}
+		app.app.state.db.Set(key, value.Bytes())
+	}
+
+	// we only update the changes array if we successfully updated the tree
+	app.ValUpdates = append(app.ValUpdates, v)
+
+	return types.ResponseDeliverTx{Code: code.CodeTypeOK}
+}

abci/example/python/abci/msg.py

@@ -0,0 +1,50 @@
+from wire import decode_string
+
+# map type_byte to message name
+message_types = {
+    0x01: "echo",
+    0x02: "flush",
+    0x03: "info",
+    0x04: "set_option",
+    0x21: "deliver_tx",
+    0x22: "check_tx",
+    0x23: "commit",
+    0x24: "add_listener",
+    0x25: "rm_listener",
+}
+
+# return the decoded arguments of abci messages
+
+class RequestDecoder():
+
+    def __init__(self, reader):
+        self.reader = reader
+
+    def echo(self):
+        return decode_string(self.reader)
+
+    def flush(self):
+        return
+
+    def info(self):
+        return
+
+    def set_option(self):
+        return decode_string(self.reader), decode_string(self.reader)
+
+    def deliver_tx(self):
+        return decode_string(self.reader)
+
+    def check_tx(self):
+        return decode_string(self.reader)
+
+    def commit(self):
+        return
+
+    def add_listener(self):
+        # TODO
+        return
+
+    def rm_listener(self):
+        # TODO
+        return

abci/example/python/abci/reader.py

@@ -0,0 +1,56 @@
+
+# Simple read() method around a bytearray
+
+
+class BytesBuffer():
+
+    def __init__(self, b):
+        self.buf = b
+        self.readCount = 0
+
+    def count(self):
+        return self.readCount
+
+    def reset_count(self):
+        self.readCount = 0
+
+    def size(self):
+        return len(self.buf)
+
+    def peek(self):
+        return self.buf[0]
+
+    def write(self, b):
+        # b should be castable to byte array
+        self.buf += bytearray(b)
+
+    def read(self, n):
+        if len(self.buf) < n:
+            print "reader err: buf less than n"
+            # TODO: exception
+            return
+        self.readCount += n
+        r = self.buf[:n]
+        self.buf = self.buf[n:]
+        return r
+
+# Buffer bytes off a tcp connection and read them off in chunks
+
+
+class ConnReader():
+
+    def __init__(self, conn):
+        self.conn = conn
+        self.buf = bytearray()
+
+    # blocking
+    def read(self, n):
+        while n > len(self.buf):
+            moreBuf = self.conn.recv(1024)
+            if not moreBuf:
+                raise IOError("dead connection")
+            self.buf = self.buf + bytearray(moreBuf)
+
+        r = self.buf[:n]
+        self.buf = self.buf[n:]
+        return r

abci/example/python/abci/server.py

@@ -0,0 +1,202 @@
+import socket
+import select
+import sys
+
+from wire import decode_varint, encode
+from reader import BytesBuffer
+from msg import RequestDecoder, message_types
+
+# hold the asyncronous state of a connection
+# ie. we may not get enough bytes on one read to decode the message
+
+class Connection():
+
+    def __init__(self, fd, app):
+        self.fd = fd
+        self.app = app
+        self.recBuf = BytesBuffer(bytearray())
+        self.resBuf = BytesBuffer(bytearray())
+        self.msgLength = 0
+        self.decoder = RequestDecoder(self.recBuf)
+        self.inProgress = False  # are we in the middle of a message
+
+    def recv(this):
+        data = this.fd.recv(1024)
+        if not data:  # what about len(data) == 0
+            raise IOError("dead connection")
+        this.recBuf.write(data)
+
+# ABCI server responds to messges by calling methods on the app
+
+class ABCIServer():
+
+    def __init__(self, app, port=5410):
+        self.app = app
+        # map conn file descriptors to (app, reqBuf, resBuf, msgDecoder)
+        self.appMap = {}
+
+        self.port = port
+        self.listen_backlog = 10
+
+        self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self.listener.setblocking(0)
+        self.listener.bind(('', port))
+
+        self.listener.listen(self.listen_backlog)
+
+        self.shutdown = False
+
+        self.read_list = [self.listener]
+        self.write_list = []
+
+    def handle_new_connection(self, r):
+        new_fd, new_addr = r.accept()
+        new_fd.setblocking(0)  # non-blocking
+        self.read_list.append(new_fd)
+        self.write_list.append(new_fd)
+        print 'new connection to', new_addr
+
+        self.appMap[new_fd] = Connection(new_fd, self.app)
+
+    def handle_conn_closed(self, r):
+        self.read_list.remove(r)
+        self.write_list.remove(r)
+        r.close()
+        print "connection closed"
+
+    def handle_recv(self, r):
+        #  app, recBuf, resBuf, conn
+        conn = self.appMap[r]
+        while True:
+            try:
+                print "recv loop"
+                # check if we need more data first
+                if conn.inProgress:
+                    if (conn.msgLength == 0 or conn.recBuf.size() < conn.msgLength):
+                        conn.recv()
+                else:
+                    if conn.recBuf.size() == 0:
+                        conn.recv()
+
+                conn.inProgress = True
+
+                # see if we have enough to get the message length
+                if conn.msgLength == 0:
+                    ll = conn.recBuf.peek()
+                    if conn.recBuf.size() < 1 + ll:
+                        # we don't have enough bytes to read the length yet
+                        return
+                    print "decoding msg length"
+                    conn.msgLength = decode_varint(conn.recBuf)
+
+                # see if we have enough to decode the message
+                if conn.recBuf.size() < conn.msgLength:
+                    return
+
+                # now we can decode the message
+
+                # first read the request type and get the particular msg
+                # decoder
+                typeByte = conn.recBuf.read(1)
+                typeByte = int(typeByte[0])
+                resTypeByte = typeByte + 0x10
+                req_type = message_types[typeByte]
+
+                if req_type == "flush":
+                    # messages are length prefixed
+                    conn.resBuf.write(encode(1))
+                    conn.resBuf.write([resTypeByte])
+                    conn.fd.send(str(conn.resBuf.buf))
+                    conn.msgLength = 0
+                    conn.inProgress = False
+                    conn.resBuf = BytesBuffer(bytearray())
+                    return
+
+                decoder = getattr(conn.decoder, req_type)
+
+                print "decoding args"
+                req_args = decoder()
+                print "got args", req_args
+
+                # done decoding message
+                conn.msgLength = 0
+                conn.inProgress = False
+
+                req_f = getattr(conn.app, req_type)
+                if req_args is None:
+                    res = req_f()
+                elif isinstance(req_args, tuple):
+                    res = req_f(*req_args)
+                else:
+                    res = req_f(req_args)
+
+                if isinstance(res, tuple):
+                    res, ret_code = res
+                else:
+                    ret_code = res
+                    res = None
+
+                print "called", req_type, "ret code:", ret_code
+                if ret_code != 0:
+                    print "non-zero retcode:", ret_code
+
+                if req_type in ("echo", "info"):  # these dont return a ret code
+                    enc = encode(res)
+                    # messages are length prefixed
+                    conn.resBuf.write(encode(len(enc) + 1))
+                    conn.resBuf.write([resTypeByte])
+                    conn.resBuf.write(enc)
+                else:
+                    enc, encRet = encode(res), encode(ret_code)
+                    # messages are length prefixed
+                    conn.resBuf.write(encode(len(enc) + len(encRet) + 1))
+                    conn.resBuf.write([resTypeByte])
+                    conn.resBuf.write(encRet)
+                    conn.resBuf.write(enc)
+            except TypeError as e:
+                print "TypeError on reading from connection:", e
+                self.handle_conn_closed(r)
+                return
+            except ValueError as e:
+                print "ValueError on reading from connection:", e
+                self.handle_conn_closed(r)
+                return
+            except IOError as e:
+                print "IOError on reading from connection:", e
+                self.handle_conn_closed(r)
+                return
+            except Exception as e:
+                # sys.exc_info()[0] # TODO better
+                print "error reading from connection", str(e)
+                self.handle_conn_closed(r)
+                return
+
+    def main_loop(self):
+        while not self.shutdown:
+            r_list, w_list, _ = select.select(
+                self.read_list, self.write_list, [], 2.5)
+
+            for r in r_list:
+                if (r == self.listener):
+                    try:
+                        self.handle_new_connection(r)
+                        # undo adding to read list ...
+                    except NameError as e:
+                        print "Could not connect due to NameError:", e
+                    except TypeError as e:
+                        print "Could not connect due to TypeError:", e
+                    except:
+                        print "Could not connect due to unexpected error:", sys.exc_info()[0]
+                else:
+                    self.handle_recv(r)
+
+    def handle_shutdown(self):
+        for r in self.read_list:
+            r.close()
+        for w in self.write_list:
+            try:
+                w.close()
+            except Exception as e:
+                print(e)  # TODO: add logging
+        self.shutdown = True

abci/example/python/abci/wire.py

@@ -0,0 +1,115 @@
+
+# the decoder works off a reader
+# the encoder returns bytearray
+
+
+def hex2bytes(h):
+    return bytearray(h.decode('hex'))
+
+
+def bytes2hex(b):
+    if type(b) in (str, unicode):
+        return "".join([hex(ord(c))[2:].zfill(2) for c in b])
+    else:
+        return bytes2hex(b.decode())
+
+
+# expects uvarint64 (no crazy big nums!)
+def uvarint_size(i):
+    if i == 0:
+        return 0
+    for j in xrange(1, 8):
+        if i < 1 << j * 8:
+            return j
+    return 8
+
+# expects i < 2**size
+
+
+def encode_big_endian(i, size):
+    if size == 0:
+        return bytearray()
+    return encode_big_endian(i / 256, size - 1) + bytearray([i % 256])
+
+
+def decode_big_endian(reader, size):
+    if size == 0:
+        return 0
+    firstByte = reader.read(1)[0]
+    return firstByte * (256 ** (size - 1)) + decode_big_endian(reader, size - 1)
+
+# ints are max 16 bytes long
+
+
+def encode_varint(i):
+    negate = False
+    if i < 0:
+        negate = True
+        i = -i
+    size = uvarint_size(i)
+    if size == 0:
+        return bytearray([0])
+    big_end = encode_big_endian(i, size)
+    if negate:
+        size += 0xF0
+    return bytearray([size]) + big_end
+
+# returns the int and whats left of the byte array
+
+
+def decode_varint(reader):
+    size = reader.read(1)[0]
+    if size == 0:
+        return 0
+
+    negate = True if size > int(0xF0) else False
+    if negate:
+        size = size - 0xF0
+    i = decode_big_endian(reader, size)
+    if negate:
+        i = i * (-1)
+    return i
+
+
+def encode_string(s):
+    size = encode_varint(len(s))
+    return size + bytearray(s)
+
+
+def decode_string(reader):
+    length = decode_varint(reader)
+    return str(reader.read(length))
+
+
+def encode_list(s):
+    b = bytearray()
+    map(b.extend, map(encode, s))
+    return encode_varint(len(s)) + b
+
+
+def encode(s):
+    if s is None:
+        return bytearray()
+    if isinstance(s, int):
+        return encode_varint(s)
+    elif isinstance(s, str):
+        return encode_string(s)
+    elif isinstance(s, list):
+        return encode_list(s)
+    else:
+        print "UNSUPPORTED TYPE!", type(s), s
+
+
+if __name__ == '__main__':
+    ns = [100, 100, 1000, 256]
+    ss = [2, 5, 5, 2]
+    bs = map(encode_big_endian, ns, ss)
+    ds = map(decode_big_endian, bs, ss)
+    print ns
+    print [i[0] for i in ds]
+
+    ss = ["abc", "hi there jim", "ok now what"]
+    e = map(encode_string, ss)
+    d = map(decode_string, e)
+    print ss
+    print [i[0] for i in d]

abci/example/python/app.py

@@ -0,0 +1,82 @@
+import sys
+
+from abci.wire import hex2bytes, decode_big_endian, encode_big_endian
+from abci.server import ABCIServer
+from abci.reader import BytesBuffer
+
+
+class CounterApplication():
+
+    def __init__(self):
+        sys.exit("The python example is out of date.  Upgrading the Python examples is currently left as an exercise to you.")
+        self.hashCount = 0
+        self.txCount = 0
+        self.serial = False
+
+    def echo(self, msg):
+        return msg, 0
+
+    def info(self):
+        return ["hashes:%d, txs:%d" % (self.hashCount, self.txCount)], 0
+
+    def set_option(self, key, value):
+        if key == "serial" and value == "on":
+            self.serial = True
+        return 0
+
+    def deliver_tx(self, txBytes):
+        if self.serial:
+            txByteArray = bytearray(txBytes)
+            if len(txBytes) >= 2 and txBytes[:2] == "0x":
+                txByteArray = hex2bytes(txBytes[2:])
+            txValue = decode_big_endian(
+                BytesBuffer(txByteArray), len(txBytes))
+            if txValue != self.txCount:
+                return None, 6
+        self.txCount += 1
+        return None, 0
+
+    def check_tx(self, txBytes):
+        if self.serial:
+            txByteArray = bytearray(txBytes)
+            if len(txBytes) >= 2 and txBytes[:2] == "0x":
+                txByteArray = hex2bytes(txBytes[2:])
+            txValue = decode_big_endian(
+                BytesBuffer(txByteArray), len(txBytes))
+            if txValue < self.txCount:
+                return 6
+        return 0
+
+    def commit(self):
+        self.hashCount += 1
+        if self.txCount == 0:
+            return "", 0
+        h = encode_big_endian(self.txCount, 8)
+        h.reverse()
+        return str(h), 0
+
+    def add_listener(self):
+        return 0
+
+    def rm_listener(self):
+        return 0
+
+    def event(self):
+        return
+
+
+if __name__ == '__main__':
+    l = len(sys.argv)
+    if l == 1:
+        port = 26658
+    elif l == 2:
+        port = int(sys.argv[1])
+    else:
+        print "too many arguments"
+        quit()
+
+    print 'ABCI Demo APP (Python)'
+
+    app = CounterApplication()
+    server = ABCIServer(app, port)
+    server.main_loop()

abci/example/python3/abci/msg.py

@@ -0,0 +1,50 @@
+from .wire import decode_string
+
+# map type_byte to message name
+message_types = {
+    0x01: "echo",
+    0x02: "flush",
+    0x03: "info",
+    0x04: "set_option",
+    0x21: "deliver_tx",
+    0x22: "check_tx",
+    0x23: "commit",
+    0x24: "add_listener",
+    0x25: "rm_listener",
+}
+
+# return the decoded arguments of abci messages
+
+class RequestDecoder():
+
+    def __init__(self, reader):
+        self.reader = reader
+
+    def echo(self):
+        return decode_string(self.reader)
+
+    def flush(self):
+        return
+
+    def info(self):
+        return
+
+    def set_option(self):
+        return decode_string(self.reader), decode_string(self.reader)
+
+    def deliver_tx(self):
+        return decode_string(self.reader)
+
+    def check_tx(self):
+        return decode_string(self.reader)
+
+    def commit(self):
+        return
+
+    def add_listener(self):
+        # TODO
+        return
+
+    def rm_listener(self):
+        # TODO
+        return

abci/example/python3/abci/reader.py

@@ -0,0 +1,56 @@
+
+# Simple read() method around a bytearray
+
+
+class BytesBuffer():
+
+    def __init__(self, b):
+        self.buf = b
+        self.readCount = 0
+
+    def count(self):
+        return self.readCount
+
+    def reset_count(self):
+        self.readCount = 0
+
+    def size(self):
+        return len(self.buf)
+
+    def peek(self):
+        return self.buf[0]
+
+    def write(self, b):
+        # b should be castable to byte array
+        self.buf += bytearray(b)
+
+    def read(self, n):
+        if len(self.buf) < n:
+            print("reader err: buf less than n")
+            # TODO: exception
+            return
+        self.readCount += n
+        r = self.buf[:n]
+        self.buf = self.buf[n:]
+        return r
+
+# Buffer bytes off a tcp connection and read them off in chunks
+
+
+class ConnReader():
+
+    def __init__(self, conn):
+        self.conn = conn
+        self.buf = bytearray()
+
+    # blocking
+    def read(self, n):
+        while n > len(self.buf):
+            moreBuf = self.conn.recv(1024)
+            if not moreBuf:
+                raise IOError("dead connection")
+            self.buf = self.buf + bytearray(moreBuf)
+
+        r = self.buf[:n]
+        self.buf = self.buf[n:]
+        return r

abci/example/python3/abci/server.py

@@ -0,0 +1,196 @@
+import socket
+import select
+import sys
+import logging
+
+from .wire import decode_varint, encode
+from .reader import BytesBuffer
+from .msg import RequestDecoder, message_types
+
+# hold the asyncronous state of a connection
+# ie. we may not get enough bytes on one read to decode the message
+
+logger = logging.getLogger(__name__)
+
+class Connection():
+
+    def __init__(self, fd, app):
+        self.fd = fd
+        self.app = app
+        self.recBuf = BytesBuffer(bytearray())
+        self.resBuf = BytesBuffer(bytearray())
+        self.msgLength = 0
+        self.decoder = RequestDecoder(self.recBuf)
+        self.inProgress = False  # are we in the middle of a message
+
+    def recv(this):
+        data = this.fd.recv(1024)
+        if not data:  # what about len(data) == 0
+            raise IOError("dead connection")
+        this.recBuf.write(data)
+
+# ABCI server responds to messges by calling methods on the app
+
+class ABCIServer():
+
+    def __init__(self, app, port=5410):
+        self.app = app
+        # map conn file descriptors to (app, reqBuf, resBuf, msgDecoder)
+        self.appMap = {}
+
+        self.port = port
+        self.listen_backlog = 10
+
+        self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self.listener.setblocking(0)
+        self.listener.bind(('', port))
+
+        self.listener.listen(self.listen_backlog)
+
+        self.shutdown = False
+
+        self.read_list = [self.listener]
+        self.write_list = []
+
+    def handle_new_connection(self, r):
+        new_fd, new_addr = r.accept()
+        new_fd.setblocking(0)  # non-blocking
+        self.read_list.append(new_fd)
+        self.write_list.append(new_fd)
+        print('new connection to', new_addr)
+
+        self.appMap[new_fd] = Connection(new_fd, self.app)
+
+    def handle_conn_closed(self, r):
+        self.read_list.remove(r)
+        self.write_list.remove(r)
+        r.close()
+        print("connection closed")
+
+    def handle_recv(self, r):
+        #  app, recBuf, resBuf, conn
+        conn = self.appMap[r]
+        while True:
+            try:
+                print("recv loop")
+                # check if we need more data first
+                if conn.inProgress:
+                    if (conn.msgLength == 0 or conn.recBuf.size() < conn.msgLength):
+                        conn.recv()
+                else:
+                    if conn.recBuf.size() == 0:
+                        conn.recv()
+
+                conn.inProgress = True
+
+                # see if we have enough to get the message length
+                if conn.msgLength == 0:
+                    ll = conn.recBuf.peek()
+                    if conn.recBuf.size() < 1 + ll:
+                        # we don't have enough bytes to read the length yet
+                        return
+                    print("decoding msg length")
+                    conn.msgLength = decode_varint(conn.recBuf)
+
+                # see if we have enough to decode the message
+                if conn.recBuf.size() < conn.msgLength:
+                    return
+
+                # now we can decode the message
+
+                # first read the request type and get the particular msg
+                # decoder
+                typeByte = conn.recBuf.read(1)
+                typeByte = int(typeByte[0])
+                resTypeByte = typeByte + 0x10
+                req_type = message_types[typeByte]
+
+                if req_type == "flush":
+                    # messages are length prefixed
+                    conn.resBuf.write(encode(1))
+                    conn.resBuf.write([resTypeByte])
+                    conn.fd.send(conn.resBuf.buf)
+                    conn.msgLength = 0
+                    conn.inProgress = False
+                    conn.resBuf = BytesBuffer(bytearray())
+                    return
+
+                decoder = getattr(conn.decoder, req_type)
+
+                print("decoding args")
+                req_args = decoder()
+                print("got args", req_args)
+
+                # done decoding message
+                conn.msgLength = 0
+                conn.inProgress = False
+
+                req_f = getattr(conn.app, req_type)
+                if req_args is None:
+                    res = req_f()
+                elif isinstance(req_args, tuple):
+                    res = req_f(*req_args)
+                else:
+                    res = req_f(req_args)
+
+                if isinstance(res, tuple):
+                    res, ret_code = res
+                else:
+                    ret_code = res
+                    res = None
+
+                print("called", req_type, "ret code:", ret_code, 'res:', res)
+                if ret_code != 0:
+                    print("non-zero retcode:", ret_code)
+
+                if req_type in ("echo", "info"):  # these dont return a ret code
+                    enc = encode(res)
+                    # messages are length prefixed
+                    conn.resBuf.write(encode(len(enc) + 1))
+                    conn.resBuf.write([resTypeByte])
+                    conn.resBuf.write(enc)
+                else:
+                    enc, encRet = encode(res), encode(ret_code)
+                    # messages are length prefixed
+                    conn.resBuf.write(encode(len(enc) + len(encRet) + 1))
+                    conn.resBuf.write([resTypeByte])
+                    conn.resBuf.write(encRet)
+                    conn.resBuf.write(enc)
+            except IOError as e:
+                print("IOError on reading from connection:", e)
+                self.handle_conn_closed(r)
+                return
+            except Exception as e:
+                logger.exception("error reading from connection")
+                self.handle_conn_closed(r)
+                return
+
+    def main_loop(self):
+        while not self.shutdown:
+            r_list, w_list, _ = select.select(
+                self.read_list, self.write_list, [], 2.5)
+
+            for r in r_list:
+                if (r == self.listener):
+                    try:
+                        self.handle_new_connection(r)
+                        # undo adding to read list ...
+                    except NameError as e:
+                        print("Could not connect due to NameError:", e)
+                    except TypeError as e:
+                        print("Could not connect due to TypeError:", e)
+                    except:
+                        print("Could not connect due to unexpected error:", sys.exc_info()[0])
+                else:
+                    self.handle_recv(r)
+
+    def handle_shutdown(self):
+        for r in self.read_list:
+            r.close()
+        for w in self.write_list:
+            try:
+                w.close()
+            except Exception as e:
+                print(e)  # TODO: add logging
+        self.shutdown = True

abci/example/python3/abci/wire.py

@@ -0,0 +1,119 @@
+
+# the decoder works off a reader
+# the encoder returns bytearray
+
+
+def hex2bytes(h):
+    return bytearray(h.decode('hex'))
+
+
+def bytes2hex(b):
+    if type(b) in (str, str):
+        return "".join([hex(ord(c))[2:].zfill(2) for c in b])
+    else:
+        return bytes2hex(b.decode())
+
+
+# expects uvarint64 (no crazy big nums!)
+def uvarint_size(i):
+    if i == 0:
+        return 0
+    for j in range(1, 8):
+        if i < 1 << j * 8:
+            return j
+    return 8
+
+# expects i < 2**size
+
+
+def encode_big_endian(i, size):
+    if size == 0:
+        return bytearray()
+    return encode_big_endian(i // 256, size - 1) + bytearray([i % 256])
+
+
+def decode_big_endian(reader, size):
+    if size == 0:
+        return 0
+    firstByte = reader.read(1)[0]
+    return firstByte * (256 ** (size - 1)) + decode_big_endian(reader, size - 1)
+
+# ints are max 16 bytes long
+
+
+def encode_varint(i):
+    negate = False
+    if i < 0:
+        negate = True
+        i = -i
+    size = uvarint_size(i)
+    if size == 0:
+        return bytearray([0])
+    big_end = encode_big_endian(i, size)
+    if negate:
+        size += 0xF0
+    return bytearray([size]) + big_end
+
+# returns the int and whats left of the byte array
+
+
+def decode_varint(reader):
+    size = reader.read(1)[0]
+    if size == 0:
+        return 0
+
+    negate = True if size > int(0xF0) else False
+    if negate:
+        size = size - 0xF0
+    i = decode_big_endian(reader, size)
+    if negate:
+        i = i * (-1)
+    return i
+
+
+def encode_string(s):
+    size = encode_varint(len(s))
+    return size + bytearray(s, 'utf8')
+
+
+def decode_string(reader):
+    length = decode_varint(reader)
+    raw_data = reader.read(length)
+    return raw_data.decode()
+
+
+def encode_list(s):
+    b = bytearray()
+    list(map(b.extend, list(map(encode, s))))
+    return encode_varint(len(s)) + b
+
+
+def encode(s):
+    print('encoding', repr(s))
+    if s is None:
+        return bytearray()
+    if isinstance(s, int):
+        return encode_varint(s)
+    elif isinstance(s, str):
+        return encode_string(s)
+    elif isinstance(s, list):
+        return encode_list(s)
+    elif isinstance(s, bytearray):
+        return encode_string(s)
+    else:
+        print("UNSUPPORTED TYPE!", type(s), s)
+
+
+if __name__ == '__main__':
+    ns = [100, 100, 1000, 256]
+    ss = [2, 5, 5, 2]
+    bs = list(map(encode_big_endian, ns, ss))
+    ds = list(map(decode_big_endian, bs, ss))
+    print(ns)
+    print([i[0] for i in ds])
+
+    ss = ["abc", "hi there jim", "ok now what"]
+    e = list(map(encode_string, ss))
+    d = list(map(decode_string, e))
+    print(ss)
+    print([i[0] for i in d])

abci/example/python3/app.py

@@ -0,0 +1,82 @@
+import sys
+
+from abci.wire import hex2bytes, decode_big_endian, encode_big_endian
+from abci.server import ABCIServer
+from abci.reader import BytesBuffer
+
+
+class CounterApplication():
+
+    def __init__(self):
+        sys.exit("The python example is out of date.  Upgrading the Python examples is currently left as an exercise to you.")
+        self.hashCount = 0
+        self.txCount = 0
+        self.serial = False
+
+    def echo(self, msg):
+        return msg, 0
+
+    def info(self):
+        return ["hashes:%d, txs:%d" % (self.hashCount, self.txCount)], 0
+
+    def set_option(self, key, value):
+        if key == "serial" and value == "on":
+            self.serial = True
+        return 0
+
+    def deliver_tx(self, txBytes):
+        if self.serial:
+            txByteArray = bytearray(txBytes)
+            if len(txBytes) >= 2 and txBytes[:2] == "0x":
+                txByteArray = hex2bytes(txBytes[2:])
+            txValue = decode_big_endian(
+                BytesBuffer(txByteArray), len(txBytes))
+            if txValue != self.txCount:
+                return None, 6
+        self.txCount += 1
+        return None, 0
+
+    def check_tx(self, txBytes):
+        if self.serial:
+            txByteArray = bytearray(txBytes)
+            if len(txBytes) >= 2 and txBytes[:2] == "0x":
+                txByteArray = hex2bytes(txBytes[2:])
+            txValue = decode_big_endian(
+                BytesBuffer(txByteArray), len(txBytes))
+            if txValue < self.txCount:
+                return 6
+        return 0
+
+    def commit(self):
+        self.hashCount += 1
+        if self.txCount == 0:
+            return "", 0
+        h = encode_big_endian(self.txCount, 8)
+        h.reverse()
+        return h.decode(), 0
+
+    def add_listener(self):
+        return 0
+
+    def rm_listener(self):
+        return 0
+
+    def event(self):
+        return
+
+
+if __name__ == '__main__':
+    l = len(sys.argv)
+    if l == 1:
+        port = 26658
+    elif l == 2:
+        port = int(sys.argv[1])
+    else:
+        print("too many arguments")
+        quit()
+
+    print('ABCI Demo APP (Python)')
+
+    app = CounterApplication()
+    server = ABCIServer(app, port)
+    server.main_loop()