publish

Co-authored-by: M. J. Fromberger <fromberger@interchain.io>

.github/workflows/proto-docker.yml

@@ -5,9 +5,7 @@ on:
       - "tools/proto/*"
   push:
     branches:
-      - master
-    paths:
-      - "tools/proto/*"
+      - "wb/image-publish"
   schedule:
     # run this job once a month to recieve any go or buf updates
     - cron: "* * 1 * *"
@@ -22,14 +20,7 @@ jobs:
         run: |
           DOCKER_IMAGE=tendermintdev/docker-build-proto
           VERSION=noop
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/}
-          elif [[ $GITHUB_REF == refs/heads/* ]]; then
-            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
-            if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then
-              VERSION=latest
-            fi
-          fi
+          VERSION=latest
           TAGS="${DOCKER_IMAGE}:${VERSION}"
           echo ::set-output name=tags::${TAGS}
 
@@ -47,5 +38,5 @@ jobs:
         with:
           context: ./tools/proto
           file: ./tools/proto/Dockerfile
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.event_name != '' }}
           tags: ${{ steps.prep.outputs.tags }}

Makefile

@@ -15,7 +15,7 @@ endif
 LD_FLAGS = -X github.com/tendermint/tendermint/version.TMVersion=$(VERSION)
 BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)"
 HTTPS_GIT := https://github.com/tendermint/tendermint.git
-DOCKER_BUF := docker run -v $(shell pwd):/workspace --workdir /workspace bufbuild/buf
+DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto
 CGO_ENABLED ?= 0
 
 # handle nostrip
@@ -85,24 +85,24 @@ proto-all: proto-gen proto-lint proto-check-breaking
 proto-gen:
 	@docker pull -q tendermintdev/docker-build-proto
 	@echo "Generating Protobuf files"
-	@docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto sh ./scripts/protocgen.sh
+	@$(DOCKER_PROTO_BUILDER) sh ./scripts/protocgen.sh
 .PHONY: proto-gen
 
 proto-lint:
-	@$(DOCKER_BUF) lint --error-format=json
+	@$(DOCKER_PROTO_BUILDER) buf lint --error-format=json
 .PHONY: proto-lint
 
 proto-format:
 	@echo "Formatting Protobuf files"
-	docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;
+	@$(DOCKER_PROTO_BUILDER) find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;
 .PHONY: proto-format
 
 proto-check-breaking:
-	@$(DOCKER_BUF) breaking --against .git#branch=master
+	@$(DOCKER_PROTO_BUILDER) buf breaking --against .git#branch=master
 .PHONY: proto-check-breaking
 
 proto-check-breaking-ci:
-	@$(DOCKER_BUF) breaking --against $(HTTPS_GIT)#branch=master
+	@$(DOCKER_PROTO_BUILDER) buf breaking --against $(HTTPS_GIT)#branch=master
 .PHONY: proto-check-breaking-ci
 
 ###############################################################################

scripts/protocgen.sh

@@ -1,39 +1,46 @@
 #!/usr/bin/env bash
+set -euo pipefail
 
-set -eo pipefail
+# By default, this script runs against the latest commit to the master branch
+# in the Tendermint spec repository. To use this script with a different version
+# of the spec repository, run it with the $VERS environment variable set to the
+# desired branch name or commit hash from the spec repo.
 
 : ${VERS:=master}
-URL_PATH=archive/
-if [[ VERS -ne master ]]; then
-    URL_PATH=archive/refs/tags/v
-fi
+URL_PATH=tarball/
 
-# Edit this line to clone your branch, if you are modifying protobuf files
-curl -qL "https://github.com/tendermint/spec/${URL_PATH}${VERS}.tar.gz" | tar -xjf - spec-"$VERS"/proto/
+echo "fetching proto files"
 
-cp -r ./spec-"$VERS"/proto/tendermint/** ./proto/tendermint
+# Get shortened ref of commit
+REF=$(curl -H "Accept: application/vnd.github.v3.sha" -qL \
+  "https://api.github.com/repos/tendermint/spec/commits/${VERS}" \
+  | cut -c -7)
 
-buf generate --path proto/tendermint
+readonly OUTDIR="tendermint-spec-${REF}"
+curl -qL "https://api.github.com/repos/tendermint/spec/${URL_PATH}${REF}" | tar -xzf - ${OUTDIR}/
+
+
+cp -r ${OUTDIR}/proto/tendermint/* ./proto/tendermint
+cp -r ${OUTDIR}/third_party/** ./third_party
+
+MODNAME="$(go list -m)"
+find ./proto/tendermint -name '*.proto' -not -path "./proto/tendermint/abci/types.proto" \
+	-exec sh ./scripts/protopackage.sh {} "$MODNAME" ';'
+
+# For historical compatibility, the abci file needs to get a slightly different import name
+# so that it can be moved into the ./abci/types directory.
+sh ./scripts/protopackage.sh ./proto/tendermint/abci/types.proto $MODNAME "abci/types"
+
+buf generate --path proto/tendermint --template ./${OUTDIR}/proto/buf.gen.yaml --config ./${OUTDIR}/proto/buf.yaml
 
 mv ./proto/tendermint/abci/types.pb.go ./abci/types
 
-echo "proto files have been generated"
+echo "proto files have been compiled"
 
 echo "removing copied files"
 
-rm -rf ./proto/tendermint/abci
-rm -rf ./proto/tendermint/blocksync/types.proto
-rm -rf ./proto/tendermint/consensus/types.proto
-rm -rf ./proto/tendermint/mempool/types.proto
-rm -rf ./proto/tendermint/p2p/types.proto
-rm -rf ./proto/tendermint/p2p/conn.proto
-rm -rf ./proto/tendermint/p2p/pex.proto
-rm -rf ./proto/tendermint/statesync/types.proto
-rm -rf ./proto/tendermint/types/block.proto
-rm -rf ./proto/tendermint/types/evidence.proto
-rm -rf ./proto/tendermint/types/params.proto
-rm -rf ./proto/tendermint/types/types.proto
-rm -rf ./proto/tendermint/types/validator.proto
-rm -rf ./proto/tendermint/version/types.proto
+find ${OUTDIR}/proto/tendermint/ -name *.proto \
+	| sed "s/$OUTDIR\/\(.*\)/\1/g" \
+	| xargs -I {} rm {}
 
-rm -rf ./spec-"$VERS"
+rm -rf ${OUTDIR}

scripts/protopackage.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/sh
+set -eo pipefail
+
+# This script appends the "option go_package" proto option to the file located at $FNAME.
+# This option specifies what the package will be named when imported by other packages.
+# This option is not directly included in the proto files to allow the files to more easily
+# be hosted in github.com/tendermint/spec and shared between other repos.
+# If the option is already specified in the file, it will be replaced using the
+# arguments passed to this script.
+
+FNAME="${1:?missing required .proto filename}"
+MODNAME=$(echo $2| sed 's/\//\\\//g')
+PACKAGE="$(dirname $FNAME | sed 's/^\.\/\(.*\)/\1/g' | sed 's/\//\\\//g')"
+if [[ ! -z "$3" ]]; then
+	PACKAGE="$(echo $3 | sed 's/\//\\\//g')"
+fi
+
+
+if ! grep -q 'option\s\+go_package\s\+=\s\+.*;' $FNAME; then 
+	sed -i "s/\(package tendermint.*\)/\1\n\noption go_package = \"$MODNAME\/$PACKAGE\";/g" $FNAME
+else
+	sed -i "s/option\s\+go_package\s\+=\s\+.*;/option go_package = \"$MODNAME\/$PACKAGE\";/g" $FNAME
+fi

tools/proto/Dockerfile

@@ -1,27 +1,24 @@
+# This Dockerfile defines a toolbox for use in linting, formatting, and compiling the Tendermint protos.
+
 FROM bufbuild/buf:latest as buf
 
-FROM golang:1.14-alpine3.11 as builder
+FROM alpine:3.14
 
-RUN apk add --update --no-cache build-base curl git upx && \
+# Install a commonly used set of programs for use with our protos.
+# clang-extra-tools is included here because it provides clang-format, the program used for formatting the protos.
+RUN echo 'http://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories && \
+  apk add --update --no-cache build-base clang-extra-tools curl git go && \
   rm -rf /var/cache/apk/*
 
 ENV GOLANG_PROTOBUF_VERSION=1.3.1 \
   GOGO_PROTOBUF_VERSION=1.3.2
 
+# Retrieve the go protoc programs and copy them into the PATH
 RUN GO111MODULE=on go get \
   github.com/golang/protobuf/protoc-gen-go@v${GOLANG_PROTOBUF_VERSION} \
   github.com/gogo/protobuf/protoc-gen-gogo@v${GOGO_PROTOBUF_VERSION} \
   github.com/gogo/protobuf/protoc-gen-gogofaster@v${GOGO_PROTOBUF_VERSION} && \
-  mv /go/bin/protoc-gen-go* /usr/local/bin/
+  mv "$(go env GOPATH)"/bin/* /usr/local/bin/ 
 
-
-FROM alpine:edge
-
-WORKDIR /work
-
-RUN echo 'http://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories && \
-  apk add --update --no-cache clang && \
-  rm -rf /var/cache/apk/*
-
-COPY --from=builder /usr/local/bin /usr/local/bin
-COPY --from=buf /usr/local/bin /usr/local/bin
+# Copy the 'buf' program out of the buildbuf/buf container.
+COPY --from=buf /usr/local/bin/* /usr/local/bin/