diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 89797a581..e773526fd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -40,7 +40,7 @@ jobs: platforms: all - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1.5.0 + uses: docker/setup-buildx-action@v1.6.0 - name: Login to DockerHub if: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/proto-docker.yml b/.github/workflows/proto-docker.yml index ed31025b9..ee26bd111 100644 --- a/.github/workflows/proto-docker.yml +++ b/.github/workflows/proto-docker.yml @@ -34,7 +34,7 @@ jobs: echo ::set-output name=tags::${TAGS} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1.5.0 + uses: docker/setup-buildx-action@v1.6.0 - name: Login to DockerHub uses: docker/login-action@v1.10.0 diff --git a/go.mod b/go.mod index 6168555e8..f2bada024 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/go-kit/kit v0.11.0 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.2 - github.com/golangci/golangci-lint v1.42.0 + github.com/golangci/golangci-lint v1.42.1 github.com/google/orderedcode v0.0.1 github.com/google/uuid v1.3.0 github.com/gorilla/websocket v1.4.2 diff --git a/go.sum b/go.sum index d1022f6d3..824e128fb 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Antonboom/errname v0.1.3 h1:qKV8gSzPzBqrG/q0dgraZXJCymWt6KuD9+Y7K7xtzN8= -github.com/Antonboom/errname v0.1.3/go.mod h1:jRXo3m0E0EuCnK3wbsSVH3X55Z4iTDLl6ZfCxwFj4TM= +github.com/Antonboom/errname v0.1.4 h1:lGSlI42Gm4bI1e+IITtXJXvxFM8N7naWimVFKcb0McY= +github.com/Antonboom/errname v0.1.4/go.mod h1:jRXo3m0E0EuCnK3wbsSVH3X55Z4iTDLl6ZfCxwFj4TM= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -341,8 +341,8 @@ github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZB github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks= github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.42.0 h1:hqf1zo6zY3GKGjjBk3ttdH22tGwF6ZRpk6j6xyJmE8I= -github.com/golangci/golangci-lint v1.42.0/go.mod h1:wgkGQnU9lOUFvTFo5QBSOvaSSddEV21Z1zYkJSbppZA= +github.com/golangci/golangci-lint v1.42.1 h1:nC4WyrbdnNdohDVUoNKjy/4N4FTM1gCFaVeXecy6vzM= +github.com/golangci/golangci-lint v1.42.1/go.mod h1:MuInrVlgg2jq4do6XI1jbkErbVHVbwdrLLtGv6p2wPI= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= @@ -596,8 +596,8 @@ github.com/mbilski/exhaustivestruct v1.2.0 h1:wCBmUnSYufAHO6J4AVWY6ff+oxWxsVFrwg github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc= github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 h1:QASJXOGm2RZ5Ardbc86qNFvby9AqkLDibfChMtAg5QM= github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= -github.com/mgechev/revive v1.1.0 h1:TvabpsolbtlzZTyJcgMRN38MHrgi8C0DhmGE5dhscGY= -github.com/mgechev/revive v1.1.0/go.mod h1:PKqk4L74K6wVNwY2b6fr+9Qqr/3hIsHVfZCJdbvozrY= +github.com/mgechev/revive v1.1.1 h1:mkXNHP14Y6tfq+ocnQaiKEtgJDM41yaoyQq4qn6TD/4= +github.com/mgechev/revive v1.1.1/go.mod h1:PKqk4L74K6wVNwY2b6fr+9Qqr/3hIsHVfZCJdbvozrY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -714,8 +714,8 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/polyfloyd/go-errorlint v0.0.0-20210510181950-ab96adb96fea h1:Sk6Xawg57ZkjXmFYD1xCHSKN6FtYM+km51MM7Lveyyc= -github.com/polyfloyd/go-errorlint v0.0.0-20210510181950-ab96adb96fea/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw= +github.com/polyfloyd/go-errorlint v0.0.0-20210722154253-910bb7978349 h1:Kq/3kL0k033ds3tyez5lFPrfQ74fNJ+OqCclRipubwA= +github.com/polyfloyd/go-errorlint v0.0.0-20210722154253-910bb7978349/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -863,8 +863,8 @@ github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzH github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/tendermint/tm-db v0.6.4 h1:3N2jlnYQkXNQclQwd/eKV/NzlqPlfK21cpRRIx80XXQ= github.com/tendermint/tm-db v0.6.4/go.mod h1:dptYhIpJ2M5kUuenLr+Yyf3zQOv1SgBZcl8/BmWlMBw= -github.com/tetafro/godot v1.4.8 h1:rhuUH+tBrx24yVAr6Ox3/UxcsiUPPJcGhinfLdbdew0= -github.com/tetafro/godot v1.4.8/go.mod h1:LR3CJpxDVGlYOWn3ZZg1PgNZdTUvzsZWu8xaEohUpn8= +github.com/tetafro/godot v1.4.9 h1:wsNd0RuUxISqqudFqchsSsMqsM188DoZVPBeKl87tP0= +github.com/tetafro/godot v1.4.9/go.mod h1:LR3CJpxDVGlYOWn3ZZg1PgNZdTUvzsZWu8xaEohUpn8= github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 h1:ig99OeTyDwQWhPe2iw9lwfQVF1KB3Q4fpP3X7/2VBG8= github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= diff --git a/internal/evidence/pool.go b/internal/evidence/pool.go index fa722f4f1..2a48fe032 100644 --- a/internal/evidence/pool.go +++ b/internal/evidence/pool.go @@ -516,10 +516,13 @@ func (evpool *Pool) processConsensusBuffer(state sm.State) { // Check the height of the conflicting votes and fetch the corresponding time and validator set // to produce the valid evidence - var dve *types.DuplicateVoteEvidence + var ( + dve *types.DuplicateVoteEvidence + err error + ) switch { case voteSet.VoteA.Height == state.LastBlockHeight: - dve = types.NewDuplicateVoteEvidence( + dve, err = types.NewDuplicateVoteEvidence( voteSet.VoteA, voteSet.VoteB, state.LastBlockTime, @@ -527,8 +530,8 @@ func (evpool *Pool) processConsensusBuffer(state sm.State) { ) case voteSet.VoteA.Height < state.LastBlockHeight: - valSet, err := evpool.stateDB.LoadValidators(voteSet.VoteA.Height) - if err != nil { + valSet, dbErr := evpool.stateDB.LoadValidators(voteSet.VoteA.Height) + if dbErr != nil { evpool.logger.Error("failed to load validator set for conflicting votes", "height", voteSet.VoteA.Height, "err", err) continue @@ -538,7 +541,7 @@ func (evpool *Pool) processConsensusBuffer(state sm.State) { evpool.logger.Error("failed to load block time for conflicting votes", "height", voteSet.VoteA.Height) continue } - dve = types.NewDuplicateVoteEvidence( + dve, err = types.NewDuplicateVoteEvidence( voteSet.VoteA, voteSet.VoteB, blockMeta.Header.Time, @@ -554,6 +557,10 @@ func (evpool *Pool) processConsensusBuffer(state sm.State) { "state.LastBlockHeight", state.LastBlockHeight) continue } + if err != nil { + evpool.logger.Error("error in generating evidence from votes", "err", err) + continue + } // check if we already have this evidence if evpool.isPending(dve) { diff --git a/internal/evidence/reactor_test.go b/internal/evidence/reactor_test.go index b098eb373..1cf995731 100644 --- a/internal/evidence/reactor_test.go +++ b/internal/evidence/reactor_test.go @@ -534,12 +534,13 @@ func TestEvidenceListSerialization(t *testing.T) { valSet := types.NewValidatorSet([]*types.Validator{val}) - dupl := types.NewDuplicateVoteEvidence( + dupl, err := types.NewDuplicateVoteEvidence( exampleVote(1), exampleVote(2), defaultEvidenceTime, valSet, ) + require.NoError(t, err) testCases := map[string]struct { evidenceList []types.Evidence diff --git a/networks/remote/ansible/inventory/digital_ocean.py b/networks/remote/ansible/inventory/digital_ocean.py index 24ba64370..383b329a1 100755 --- a/networks/remote/ansible/inventory/digital_ocean.py +++ b/networks/remote/ansible/inventory/digital_ocean.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -''' +""" DigitalOcean external inventory script ====================================== @@ -22,7 +22,7 @@ found. You can force this script to use the cache with --force-cache. ---- Configuration is read from `digital_ocean.ini`, then from environment variables, -then and command-line arguments. +and then from command-line arguments. Most notably, the DigitalOcean API Token must be specified. It can be specified in the INI file or with the following environment variables: @@ -40,6 +40,7 @@ is to use the output of the --env option with export: The following groups are generated from --list: - ID (droplet ID) - NAME (droplet NAME) + - digital_ocean - image_ID - image_NAME - distro_NAME (distribution NAME from image) @@ -73,14 +74,12 @@ For each host, the following variables are registered: ----- ``` -usage: digital_ocean.py [-h] [--list] [--host HOST] [--all] - [--droplets] [--regions] [--images] [--sizes] - [--ssh-keys] [--domains] [--pretty] - [--cache-path CACHE_PATH] - [--cache-max_age CACHE_MAX_AGE] - [--force-cache] - [--refresh-cache] - [--api-token API_TOKEN] +usage: digital_ocean.py [-h] [--list] [--host HOST] [--all] [--droplets] + [--regions] [--images] [--sizes] [--ssh-keys] + [--domains] [--tags] [--pretty] + [--cache-path CACHE_PATH] + [--cache-max_age CACHE_MAX_AGE] [--force-cache] + [--refresh-cache] [--env] [--api-token API_TOKEN] Produce an Ansible Inventory file based on DigitalOcean credentials @@ -91,65 +90,129 @@ optional arguments: --host HOST Get all Ansible inventory variables about a specific Droplet --all List all DigitalOcean information as JSON - --droplets List Droplets as JSON + --droplets, -d List Droplets as JSON --regions List Regions as JSON --images List Images as JSON --sizes List Sizes as JSON --ssh-keys List SSH keys as JSON --domains List Domains as JSON + --tags List Tags as JSON --pretty, -p Pretty-print results --cache-path CACHE_PATH Path to the cache files (default: .) --cache-max_age CACHE_MAX_AGE Maximum age of the cached items (default: 0) --force-cache Only use data from the cache - --refresh-cache Force refresh of cache by making API requests to + --refresh-cache, -r Force refresh of cache by making API requests to DigitalOcean (default: False - use cache files) + --env, -e Display DO_API_TOKEN --api-token API_TOKEN, -a API_TOKEN DigitalOcean API Token ``` -''' +""" # (c) 2013, Evan Wies +# (c) 2017, Ansible Project +# (c) 2017, Abhijeet Kasurde # # Inspired by the EC2 inventory plugin: # https://github.com/ansible/ansible/blob/devel/contrib/inventory/ec2.py # -# This file is part of Ansible, -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type ###################################################################### -import os -import sys -import re import argparse -from time import time -import ConfigParser import ast +import os +import re +import requests +import sys +from time import time try: - import json + import ConfigParser except ImportError: - import simplejson as json + import configparser as ConfigParser -try: - from dopy.manager import DoManager -except ImportError as e: - sys.exit("failed=True msg='`dopy` library required for this script'") +import json + + +class DoManager: + def __init__(self, api_token): + self.api_token = api_token + self.api_endpoint = 'https://api.digitalocean.com/v2' + self.headers = {'Authorization': 'Bearer {0}'.format(self.api_token), + 'Content-type': 'application/json'} + self.timeout = 60 + + def _url_builder(self, path): + if path[0] == '/': + path = path[1:] + return '%s/%s' % (self.api_endpoint, path) + + def send(self, url, method='GET', data=None): + url = self._url_builder(url) + data = json.dumps(data) + try: + if method == 'GET': + resp_data = {} + incomplete = True + while incomplete: + resp = requests.get(url, data=data, headers=self.headers, timeout=self.timeout) + json_resp = resp.json() + + for key, value in json_resp.items(): + if isinstance(value, list) and key in resp_data: + resp_data[key] += value + else: + resp_data[key] = value + + try: + url = json_resp['links']['pages']['next'] + except KeyError: + incomplete = False + + except ValueError as e: + sys.exit("Unable to parse result from %s: %s" % (url, e)) + return resp_data + + def all_active_droplets(self): + resp = self.send('droplets/') + return resp['droplets'] + + def all_regions(self): + resp = self.send('regions/') + return resp['regions'] + + def all_images(self, filter_name='global'): + params = {'filter': filter_name} + resp = self.send('images/', data=params) + return resp['images'] + + def sizes(self): + resp = self.send('sizes/') + return resp['sizes'] + + def all_ssh_keys(self): + resp = self.send('account/keys') + return resp['ssh_keys'] + + def all_domains(self): + resp = self.send('domains/') + return resp['domains'] + + def show_droplet(self, droplet_id): + resp = self.send('droplets/%s' % droplet_id) + return resp['droplet'] + + def all_tags(self): + resp = self.send('tags') + return resp['tags'] class DigitalOceanInventory(object): @@ -159,7 +222,7 @@ class DigitalOceanInventory(object): ########################################################################### def __init__(self): - ''' Main execution path ''' + """Main execution path """ # DigitalOceanInventory data self.data = {} # All DigitalOcean data @@ -178,9 +241,9 @@ class DigitalOceanInventory(object): # Verify credentials were set if not hasattr(self, 'api_token'): - sys.stderr.write('''Could not find values for DigitalOcean api_token. -They must be specified via either ini file, command line argument (--api-token), -or environment variables (DO_API_TOKEN)\n''') + msg = 'Could not find values for DigitalOcean api_token. They must be specified via either ini file, ' \ + 'command line argument (--api-token), or environment variables (DO_API_TOKEN)\n' + sys.stderr.write(msg) sys.exit(-1) # env command, show DigitalOcean credentials @@ -196,10 +259,10 @@ or environment variables (DO_API_TOKEN)\n''') self.load_from_cache() if len(self.data) == 0: if self.args.force_cache: - sys.stderr.write('''Cache is empty and --force-cache was specified\n''') + sys.stderr.write('Cache is empty and --force-cache was specified\n') sys.exit(-1) - self.manager = DoManager(None, self.api_token, api_version=2) + self.manager = DoManager(self.api_token) # Pick the json_data to print based on the CLI command if self.args.droplets: @@ -220,6 +283,9 @@ or environment variables (DO_API_TOKEN)\n''') elif self.args.domains: self.load_from_digital_ocean('domains') json_data = {'domains': self.data['domains']} + elif self.args.tags: + self.load_from_digital_ocean('tags') + json_data = {'tags': self.data['tags']} elif self.args.all: self.load_from_digital_ocean() json_data = self.data @@ -234,19 +300,19 @@ or environment variables (DO_API_TOKEN)\n''') self.write_to_cache() if self.args.pretty: - print(json.dumps(json_data, sort_keys=True, indent=2)) + print(json.dumps(json_data, indent=2)) else: print(json.dumps(json_data)) - # That's all she wrote... ########################################################################### # Script configuration ########################################################################### def read_settings(self): - ''' Reads the settings from the digital_ocean.ini file ''' - config = ConfigParser.SafeConfigParser() - config.read(os.path.dirname(os.path.realpath(__file__)) + '/digital_ocean.ini') + """ Reads the settings from the digital_ocean.ini file """ + config = ConfigParser.ConfigParser() + config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'digital_ocean.ini') + config.read(config_path) # Credentials if config.has_option('digital_ocean', 'api_token'): @@ -267,7 +333,7 @@ or environment variables (DO_API_TOKEN)\n''') self.group_variables = ast.literal_eval(config.get('digital_ocean', 'group_variables')) def read_environment(self): - ''' Reads the settings from environment variables ''' + """ Reads the settings from environment variables """ # Setup credentials if os.getenv("DO_API_TOKEN"): self.api_token = os.getenv("DO_API_TOKEN") @@ -275,7 +341,7 @@ or environment variables (DO_API_TOKEN)\n''') self.api_token = os.getenv("DO_API_KEY") def read_cli_args(self): - ''' Command line argument processing ''' + """ Command line argument processing """ parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on DigitalOcean credentials') parser.add_argument('--list', action='store_true', help='List all active Droplets as Ansible inventory (default: True)') @@ -288,6 +354,7 @@ or environment variables (DO_API_TOKEN)\n''') parser.add_argument('--sizes', action='store_true', help='List Sizes as JSON') parser.add_argument('--ssh-keys', action='store_true', help='List SSH keys as JSON') parser.add_argument('--domains', action='store_true', help='List Domains as JSON') + parser.add_argument('--tags', action='store_true', help='List Tags as JSON') parser.add_argument('--pretty', '-p', action='store_true', help='Pretty-print results') @@ -309,6 +376,7 @@ or environment variables (DO_API_TOKEN)\n''') if (not self.args.droplets and not self.args.regions and not self.args.images and not self.args.sizes and not self.args.ssh_keys and not self.args.domains and + not self.args.tags and not self.args.all and not self.args.host): self.args.list = True @@ -317,7 +385,7 @@ or environment variables (DO_API_TOKEN)\n''') ########################################################################### def load_from_digital_ocean(self, resource=None): - '''Get JSON from DigitalOcean API''' + """Get JSON from DigitalOcean API """ if self.args.force_cache and os.path.isfile(self.cache_filename): return # We always get fresh droplets @@ -333,7 +401,7 @@ or environment variables (DO_API_TOKEN)\n''') self.data['regions'] = self.manager.all_regions() self.cache_refreshed = True if resource == 'images' or resource is None: - self.data['images'] = self.manager.all_images(filter=None) + self.data['images'] = self.manager.all_images() self.cache_refreshed = True if resource == 'sizes' or resource is None: self.data['sizes'] = self.manager.sizes() @@ -344,9 +412,27 @@ or environment variables (DO_API_TOKEN)\n''') if resource == 'domains' or resource is None: self.data['domains'] = self.manager.all_domains() self.cache_refreshed = True + if resource == 'tags' or resource is None: + self.data['tags'] = self.manager.all_tags() + self.cache_refreshed = True + + def add_inventory_group(self, key): + """ Method to create group dict """ + host_dict = {'hosts': [], 'vars': {}} + self.inventory[key] = host_dict + return + + def add_host(self, group, host): + """ Helper method to reduce host duplication """ + if group not in self.inventory: + self.add_inventory_group(group) + + if host not in self.inventory[group]['hosts']: + self.inventory[group]['hosts'].append(host) + return def build_inventory(self): - '''Build Ansible inventory of droplets''' + """ Build Ansible inventory of droplets """ self.inventory = { 'all': { 'hosts': [], @@ -357,52 +443,44 @@ or environment variables (DO_API_TOKEN)\n''') # add all droplets by id and name for droplet in self.data['droplets']: - # when using private_networking, the API reports the private one in "ip_address". - if 'private_networking' in droplet['features'] and not self.use_private_network: - for net in droplet['networks']['v4']: - if net['type'] == 'public': - dest = net['ip_address'] - else: - continue - else: - dest = droplet['ip_address'] + for net in droplet['networks']['v4']: + if net['type'] == 'public': + dest = net['ip_address'] + else: + continue self.inventory['all']['hosts'].append(dest) - self.inventory[droplet['id']] = [dest] - self.inventory[droplet['name']] = [dest] + self.add_host(droplet['id'], dest) + + self.add_host(droplet['name'], dest) # groups that are always present - for group in ('region_' + droplet['region']['slug'], + for group in ('digital_ocean', + 'region_' + droplet['region']['slug'], 'image_' + str(droplet['image']['id']), 'size_' + droplet['size']['slug'], - 'distro_' + self.to_safe(droplet['image']['distribution']), + 'distro_' + DigitalOceanInventory.to_safe(droplet['image']['distribution']), 'status_' + droplet['status']): - if group not in self.inventory: - self.inventory[group] = {'hosts': [], 'vars': {}} - self.inventory[group]['hosts'].append(dest) + self.add_host(group, dest) # groups that are not always present for group in (droplet['image']['slug'], droplet['image']['name']): if group: - image = 'image_' + self.to_safe(group) - if image not in self.inventory: - self.inventory[image] = {'hosts': [], 'vars': {}} - self.inventory[image]['hosts'].append(dest) + image = 'image_' + DigitalOceanInventory.to_safe(group) + self.add_host(image, dest) if droplet['tags']: for tag in droplet['tags']: - if tag not in self.inventory: - self.inventory[tag] = {'hosts': [], 'vars': {}} - self.inventory[tag]['hosts'].append(dest) + self.add_host(tag, dest) # hostvars info = self.do_namespace(droplet) self.inventory['_meta']['hostvars'][dest] = info def load_droplet_variables_for_host(self): - '''Generate a JSON response to a --host call''' + """ Generate a JSON response to a --host call """ host = int(self.args.host) droplet = self.manager.show_droplet(host) info = self.do_namespace(droplet) @@ -413,7 +491,7 @@ or environment variables (DO_API_TOKEN)\n''') ########################################################################### def is_cache_valid(self): - ''' Determines if the cache files have expired, or if it is still valid ''' + """ Determines if the cache files have expired, or if it is still valid """ if os.path.isfile(self.cache_filename): mod_time = os.path.getmtime(self.cache_filename) current_time = time() @@ -422,11 +500,10 @@ or environment variables (DO_API_TOKEN)\n''') return False def load_from_cache(self): - ''' Reads the data from the cache file and assigns it to member variables as Python Objects''' + """ Reads the data from the cache file and assigns it to member variables as Python Objects """ try: - cache = open(self.cache_filename, 'r') - json_data = cache.read() - cache.close() + with open(self.cache_filename, 'r') as cache: + json_data = cache.read() data = json.loads(json_data) except IOError: data = {'data': {}, 'inventory': {}} @@ -435,31 +512,24 @@ or environment variables (DO_API_TOKEN)\n''') self.inventory = data['inventory'] def write_to_cache(self): - ''' Writes data in JSON format to a file ''' + """ Writes data in JSON format to a file """ data = {'data': self.data, 'inventory': self.inventory} - json_data = json.dumps(data, sort_keys=True, indent=2) + json_data = json.dumps(data, indent=2) - cache = open(self.cache_filename, 'w') - cache.write(json_data) - cache.close() + with open(self.cache_filename, 'w') as cache: + cache.write(json_data) ########################################################################### # Utilities ########################################################################### + @staticmethod + def to_safe(word): + """ Converts 'bad' characters in a string to underscores so they can be used as Ansible groups """ + return re.sub(r"[^A-Za-z0-9\-.]", "_", word) - def push(self, my_dict, key, element): - ''' Pushed an element onto an array that may not have been defined in the dict ''' - if key in my_dict: - my_dict[key].append(element) - else: - my_dict[key] = [element] - - def to_safe(self, word): - ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups ''' - return re.sub("[^A-Za-z0-9\-\.]", "_", word) - - def do_namespace(self, data): - ''' Returns a copy of the dictionary with all the keys put in a 'do_' namespace ''' + @staticmethod + def do_namespace(data): + """ Returns a copy of the dictionary with all the keys put in a 'do_' namespace """ info = {} for k, v in data.items(): info['do_' + k] = v diff --git a/networks/remote/terraform/cluster/main.tf b/networks/remote/terraform/cluster/main.tf index 98ab37cee..15a913b30 100644 --- a/networks/remote/terraform/cluster/main.tf +++ b/networks/remote/terraform/cluster/main.tf @@ -1,3 +1,12 @@ +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + } +} + resource "digitalocean_tag" "cluster" { name = "${var.name}" } diff --git a/networks/remote/terraform/cluster/variables.tf b/networks/remote/terraform/cluster/variables.tf index 1b6a70072..0dc66fafe 100644 --- a/networks/remote/terraform/cluster/variables.tf +++ b/networks/remote/terraform/cluster/variables.tf @@ -4,13 +4,13 @@ variable "name" { variable "regions" { description = "Regions to launch in" - type = "list" + type = list default = ["AMS3", "FRA1", "LON1", "NYC3", "SFO2", "SGP1", "TOR1"] } variable "ssh_key" { description = "SSH key filename to copy to the nodes" - type = "string" + type = string } variable "instance_size" { diff --git a/networks/remote/terraform/main.tf b/networks/remote/terraform/main.tf index a768ee13a..470734694 100644 --- a/networks/remote/terraform/main.tf +++ b/networks/remote/terraform/main.tf @@ -1,5 +1,14 @@ #Terraform Configuration +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + } +} + variable "DO_API_TOKEN" { description = "DigitalOcean Access Token" } @@ -11,7 +20,7 @@ variable "TESTNET_NAME" { variable "SSH_KEY_FILE" { description = "SSH public key file to be used on the nodes" - type = "string" + type = string } variable "SERVERS" { diff --git a/rpc/client/evidence_test.go b/rpc/client/evidence_test.go index 0ff158e56..5626b7f48 100644 --- a/rpc/client/evidence_test.go +++ b/rpc/client/evidence_test.go @@ -29,7 +29,7 @@ var defaultTestTime = time.Date(2018, 10, 10, 8, 20, 13, 695936996, time.UTC) func newEvidence(t *testing.T, val *privval.FilePV, vote *types.Vote, vote2 *types.Vote, chainID string) *types.DuplicateVoteEvidence { - + t.Helper() var err error v := vote.ToProto() @@ -44,7 +44,9 @@ func newEvidence(t *testing.T, val *privval.FilePV, validator := types.NewValidator(val.Key.PubKey, 10) valSet := types.NewValidatorSet([]*types.Validator{validator}) - return types.NewDuplicateVoteEvidence(vote, vote2, defaultTestTime, valSet) + ev, err := types.NewDuplicateVoteEvidence(vote, vote2, defaultTestTime, valSet) + require.NoError(t, err) + return ev } func makeEvidences( diff --git a/test/e2e/runner/evidence.go b/test/e2e/runner/evidence.go index 35646ccdb..9249d7958 100644 --- a/test/e2e/runner/evidence.go +++ b/test/e2e/runner/evidence.go @@ -197,10 +197,10 @@ func generateDuplicateVoteEvidence( chainID string, time time.Time, ) (*types.DuplicateVoteEvidence, error) { - // nolint:gosec // G404: Use of weak random number generator - privVal := privVals[rand.Intn(len(privVals))] - - valIdx, _ := vals.GetByAddress(privVal.PrivKey.PubKey().Address()) + privVal, valIdx, err := getRandomValidatorIndex(privVals, vals) + if err != nil { + return nil, err + } voteA, err := factory.MakeVote(privVal, chainID, valIdx, height, 0, 2, makeRandomBlockID(), time) if err != nil { return nil, err @@ -209,14 +209,27 @@ func generateDuplicateVoteEvidence( if err != nil { return nil, err } - ev := types.NewDuplicateVoteEvidence(voteA, voteB, time, vals) - if ev == nil { - return nil, fmt.Errorf("could not generate evidence a=%v b=%v vals=%v", voteA, voteB, vals) + ev, err := types.NewDuplicateVoteEvidence(voteA, voteB, time, vals) + if err != nil { + return nil, fmt.Errorf("could not generate evidence: %w", err) } return ev, nil } +// getRandomValidatorIndex picks a random validator from a slice of mock PrivVals that's +// also part of the validator set, returning the PrivVal and its index in the validator set +func getRandomValidatorIndex(privVals []types.MockPV, vals *types.ValidatorSet) (types.MockPV, int32, error) { + for _, idx := range rand.Perm(len(privVals)) { + pv := privVals[idx] + valIdx, _ := vals.GetByAddress(pv.PrivKey.PubKey().Address()) + if valIdx >= 0 { + return pv, valIdx, nil + } + } + return types.MockPV{}, -1, errors.New("no private validator found in validator set") +} + func readPrivKey(keyFilePath string) (crypto.PrivKey, error) { keyJSONBytes, err := ioutil.ReadFile(keyFilePath) if err != nil { diff --git a/types/evidence.go b/types/evidence.go index 40ff85e5e..330850ea3 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -46,15 +46,20 @@ type DuplicateVoteEvidence struct { var _ Evidence = &DuplicateVoteEvidence{} // NewDuplicateVoteEvidence creates DuplicateVoteEvidence with right ordering given -// two conflicting votes. If one of the votes is nil, evidence returned is nil as well -func NewDuplicateVoteEvidence(vote1, vote2 *Vote, blockTime time.Time, valSet *ValidatorSet) *DuplicateVoteEvidence { +// two conflicting votes. If either of the votes is nil, the val set is nil or the voter is +// not in the val set, an error is returned +func NewDuplicateVoteEvidence(vote1, vote2 *Vote, blockTime time.Time, valSet *ValidatorSet, +) (*DuplicateVoteEvidence, error) { var voteA, voteB *Vote - if vote1 == nil || vote2 == nil || valSet == nil { - return nil + if vote1 == nil || vote2 == nil { + return nil, errors.New("missing vote") + } + if valSet == nil { + return nil, errors.New("missing validator set") } idx, val := valSet.GetByAddress(vote1.ValidatorAddress) if idx == -1 { - return nil + return nil, errors.New("validator not in validator set") } if strings.Compare(vote1.BlockID.Key(), vote2.BlockID.Key()) == -1 { @@ -70,7 +75,7 @@ func NewDuplicateVoteEvidence(vote1, vote2 *Vote, blockTime time.Time, valSet *V TotalVotingPower: valSet.TotalVotingPower(), ValidatorPower: val.VotingPower, Timestamp: blockTime, - } + }, nil } // ABCI returns the application relevant representation of the evidence @@ -92,7 +97,7 @@ func (dve *DuplicateVoteEvidence) Bytes() []byte { pbe := dve.ToProto() bz, err := pbe.Marshal() if err != nil { - panic(err) + panic("marshaling duplicate vote evidence to bytes: " + err.Error()) } return bz @@ -260,11 +265,11 @@ func (l *LightClientAttackEvidence) ABCI() []abci.Evidence { func (l *LightClientAttackEvidence) Bytes() []byte { pbe, err := l.ToProto() if err != nil { - panic(err) + panic("converting light client attack evidence to proto: " + err.Error()) } bz, err := pbe.Marshal() if err != nil { - panic(err) + panic("marshaling light client attack evidence to bytes: " + err.Error()) } return bz } @@ -684,7 +689,11 @@ func NewMockDuplicateVoteEvidenceWithValidator(height int64, time time.Time, vB := voteB.ToProto() _ = pv.SignVote(context.Background(), chainID, vB) voteB.Signature = vB.Signature - return NewDuplicateVoteEvidence(voteA, voteB, time, NewValidatorSet([]*Validator{val})) + ev, err := NewDuplicateVoteEvidence(voteA, voteB, time, NewValidatorSet([]*Validator{val})) + if err != nil { + panic("constructing mock duplicate vote evidence: " + err.Error()) + } + return ev } func makeMockVote(height int64, round, index int32, addr Address, diff --git a/types/evidence_test.go b/types/evidence_test.go index 9d54797e4..5110bcb1d 100644 --- a/types/evidence_test.go +++ b/types/evidence_test.go @@ -85,7 +85,8 @@ func TestDuplicateVoteEvidenceValidation(t *testing.T) { vote1 := makeVote(t, val, chainID, math.MaxInt32, math.MaxInt64, math.MaxInt32, 0x02, blockID, defaultVoteTime) vote2 := makeVote(t, val, chainID, math.MaxInt32, math.MaxInt64, math.MaxInt32, 0x02, blockID2, defaultVoteTime) valSet := NewValidatorSet([]*Validator{val.ExtractIntoValidator(10)}) - ev := NewDuplicateVoteEvidence(vote1, vote2, defaultVoteTime, valSet) + ev, err := NewDuplicateVoteEvidence(vote1, vote2, defaultVoteTime, valSet) + require.NoError(t, err) tc.malleateEvidence(ev) assert.Equal(t, tc.expectErr, ev.ValidateBasic() != nil, "Validate Basic had an unexpected result") })