From ee7af3da63295bd185933b6539a2ab212b047b3a Mon Sep 17 00:00:00 2001 From: Yoshiyuki Mineo Date: Mon, 1 Jun 2015 16:45:19 +0900 Subject: [PATCH] Initial commit --- .gitignore | 1 + .travis.yml | 8 ++ LICENSE | 21 ++++ README.md | 80 +++++++++++++++ awsutil/awsutil.go | 61 ++++++++++++ example/Dockerfile | 6 ++ example/Dockerrun.aws.json | 8 ++ example/README.md | 21 ++++ example/linux64_build.sh | 2 + example/sonyflake_server.go | 42 ++++++++ sonyflake.go | 183 ++++++++++++++++++++++++++++++++++ sonyflake_test.go | 192 ++++++++++++++++++++++++++++++++++++ 12 files changed, 625 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 awsutil/awsutil.go create mode 100644 example/Dockerfile create mode 100644 example/Dockerrun.aws.json create mode 100644 example/README.md create mode 100755 example/linux64_build.sh create mode 100644 example/sonyflake_server.go create mode 100644 sonyflake.go create mode 100644 sonyflake_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..86d68c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +example/sonyflake_server diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..1bc8b93 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: go +go: + - 1.4 + - tip +sudo: false +script: + - go test -v . + - cd example && ./linux64_build.sh diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..81795bf --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright 2015 Sony Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..fb94d74 --- /dev/null +++ b/README.md @@ -0,0 +1,80 @@ +Sonyflake +========= + +Sonyflake is a distributed unique ID generator inspired by [Twitter's Snowflake](https://blog.twitter.com/2010/announcing-snowflake). +A Sonyflake ID is composed of + + 39 bits for time in units of 10 msec + 8 bits for a sequence number + 16 bits for a machine id + +Installation +------------ + +``` +go get github.com/sony/sonyflake +``` + +Usage +----- + +The function NewSonyflake creates a new Sonyflake instance. + +```go +func NewSonyflake(st Settings) *Sonyflake +``` + +You can configure Sonyflake by the struct Settings: + +```go +type Settings struct { + StartTime time.Time + MachineID func() (uint16, error) + CheckMachineID func(uint16) bool +} +``` + +- StartTime is the time since which the Sonyflake time is defined as the elapsed time. + If StartTime is 0, the start time of the Sonyflake is set to "2014-09-01 00:00:00 +0000 UTC". + If StartTime is ahead of the current time, Sonyflake is not created. + +- MachineID returns the unique ID of the Sonyflake instance. + If MachineID returns an error, Sonyflake is not created. + If MachineID is nil, default MachineID is used. + Default MachineID returns the lower 16 bits of the private IP address. + +- CheckMachineID validates the uniqueness of the machine ID. + If CheckMachineID returns false, Sonyflake is not created. + If CheckMachineID is nil, no validation is done. + +In order to get a new unique ID, you just have to call the method NextID. + +```go +func (sf *Sonyflake) NextID() (uint64, error) +``` + +NextID can continue to generate IDs for about 174 years from StartTime. +But after the Sonyflake time is over the limit, NextID returns an error. + +AWS VPC and Docker +------------------ + +The [awsutil](https://github.com/sony/sonyflake/blob/master/awsutil) package provides +the function AmazonEC2MachineID that returns the lower 16-bit private IP address of the Amazon EC2 instance. +It also works correctly on Docker +by retrieving [instance metadata](http://docs.aws.amazon.com/en_us/AWSEC2/latest/UserGuide/ec2-instance-metadata.html). + +[AWS VPC](http://docs.aws.amazon.com/en_us/AmazonVPC/latest/UserGuide/VPC_Subnets.html) +is assigned a single CIDR with a netmask between /28 and /16. +So if each EC2 instance has a unique private IP address in AWS VPC, +the lower 16 bits of the address is also unique. +In this common case, you can use AmazonEC2MachineID as Settings.MachineID. + +See [example](https://github.com/sony/sonyflake/blob/master/example) that runs Sonyflake on AWS Elastic Beanstalk. + +License +------- + +The MIT License (MIT) + +See [LICENSE](https://github.com/sony/sonyflake/blob/master/LICENSE) for details. diff --git a/awsutil/awsutil.go b/awsutil/awsutil.go new file mode 100644 index 0000000..0223245 --- /dev/null +++ b/awsutil/awsutil.go @@ -0,0 +1,61 @@ +// Package awsutil provides utility functions for using Sonyflake on AWS. +package awsutil + +import ( + "errors" + "io/ioutil" + "net" + "net/http" + "os/exec" + "regexp" + "strconv" + "time" +) + +func amazonEC2PrivateIPv4() (net.IP, error) { + res, err := http.Get("http://169.254.169.254/latest/meta-data/local-ipv4") + if err != nil { + return nil, err + } + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + ip := net.ParseIP(string(body)) + if ip == nil { + return nil, errors.New("invalid ip address") + } + return ip.To4(), nil +} + +// AmazonEC2MachineID retrieves the private IP address of the Amazon EC2 instance +// and returns its lower 16 bits. +// It works correctly on Docker as well. +func AmazonEC2MachineID() (uint16, error) { + ip, err := amazonEC2PrivateIPv4() + if err != nil { + return 0, err + } + + return uint16(ip[2])<<8 + uint16(ip[3]), nil +} + +// TimeDifference returns the time difference between the localhost and the given NTP server. +func TimeDifference(server string) (time.Duration, error) { + output, err := exec.Command("/usr/sbin/ntpdate", "-q", server).CombinedOutput() + if err != nil { + return time.Duration(0), err + } + + re, _ := regexp.Compile("offset (.*) sec") + submatched := re.FindSubmatch(output) + if len(submatched) != 2 { + return time.Duration(0), errors.New("invalid ntpdate output") + } + + f, err := strconv.ParseFloat(string(submatched[1]), 64) + return time.Duration(f*1000) * time.Millisecond, nil +} diff --git a/example/Dockerfile b/example/Dockerfile new file mode 100644 index 0000000..6344389 --- /dev/null +++ b/example/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:14.04 + +ADD ./sonyflake_server / +ENTRYPOINT ["/sonyflake_server"] + +EXPOSE 8080 diff --git a/example/Dockerrun.aws.json b/example/Dockerrun.aws.json new file mode 100644 index 0000000..eb8d0a2 --- /dev/null +++ b/example/Dockerrun.aws.json @@ -0,0 +1,8 @@ +{ + "AWSEBDockerrunVersion": "1", + "Ports": [ + { + "ContainerPort": "8080" + } + ] +} diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..17fbcab --- /dev/null +++ b/example/README.md @@ -0,0 +1,21 @@ +Example +======= + +This example runs Sonyflake on AWS Elastic Beanstalk. + +Setup +----- + +1. Build the cross compiler for linux/amd64 if using other platforms. + + ``` + cd $GOROOT/src && GOOS=linux GOARCH=amd64 ./make.bash + ``` + +2. Build sonyflake_server in the example directory. + + ``` + ./linux64_build.sh + ``` + +3. Upload the example directory to AWS Elastic Beanstalk. diff --git a/example/linux64_build.sh b/example/linux64_build.sh new file mode 100755 index 0000000..a164783 --- /dev/null +++ b/example/linux64_build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +GOOS=linux GOARCH=amd64 go build sonyflake_server.go diff --git a/example/sonyflake_server.go b/example/sonyflake_server.go new file mode 100644 index 0000000..5b96e5f --- /dev/null +++ b/example/sonyflake_server.go @@ -0,0 +1,42 @@ +package main + +import ( + "encoding/json" + "net/http" + + "github.com/sony/sonyflake" + "github.com/sony/sonyflake/awsutil" +) + +var sf *sonyflake.Sonyflake + +func init() { + var st sonyflake.Settings + st.MachineID = awsutil.AmazonEC2MachineID + sf = sonyflake.NewSonyflake(st) + if sf == nil { + panic("sonyflake not created") + } +} + +func handler(w http.ResponseWriter, r *http.Request) { + id, err := sf.NextID() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + body, err := json.Marshal(sonyflake.Decompose(id)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header()["Content-Type"] = []string{"application/json; charset=utf-8"} + w.Write(body) +} + +func main() { + http.HandleFunc("/", handler) + http.ListenAndServe(":8080", nil) +} diff --git a/sonyflake.go b/sonyflake.go new file mode 100644 index 0000000..80376b6 --- /dev/null +++ b/sonyflake.go @@ -0,0 +1,183 @@ +// Package sonyflake implements Sonyflake, a distributed unique ID generator inspired by Twitter's Snowflake. +// +// A Sonyflake ID is composed of +// 39 bits for time in units of 10 msec +// 8 bits for a sequence number +// 16 bits for a machine id +package sonyflake + +import ( + "errors" + "net" + "sync" + "time" +) + +// These constants are the bit lengths of Sonyflake ID parts. +const ( + BitLenTime = 39 // bit length of time + BitLenSequence = 8 // bit length of sequence number + BitLenMachineID = 63 - BitLenTime - BitLenSequence // bit length of machine id +) + +// Settings configures Sonyflake: +// +// StartTime is the time since which the Sonyflake time is defined as the elapsed time. +// If StartTime is 0, the start time of the Sonyflake is set to "2014-09-01 00:00:00 +0000 UTC". +// If StartTime is ahead of the current time, Sonyflake is not created. +// +// MachineID returns the unique ID of the Sonyflake instance. +// If MachineID returns an error, Sonyflake is not created. +// If MachineID is nil, default MachineID is used. +// Default MachineID returns the lower 16 bits of the private IP address. +// +// CheckMachineID validates the uniqueness of the machine ID. +// If CheckMachineID returns false, Sonyflake is not created. +// If CheckMachineID is nil, no validation is done. +type Settings struct { + StartTime time.Time + MachineID func() (uint16, error) + CheckMachineID func(uint16) bool +} + +// Sonyflake is a distributed unique ID generator. +type Sonyflake struct { + mutex *sync.Mutex + startTime int64 + elapsedTime int64 + sequence uint16 + machineID uint16 +} + +// NewSonyflake returns a new Sonyflake configured with the given Settings. +// NewSonyflake returns nil in the following cases: +// - Settings.StartTime is ahead of the current time. +// - Settings.MachineID returns an error. +// - Settings.CheckMachineID returns false. +func NewSonyflake(st Settings) *Sonyflake { + sf := new(Sonyflake) + sf.mutex = new(sync.Mutex) + sf.sequence = uint16(1<= current + sf.sequence = (sf.sequence + 1) & maskSequence + if sf.sequence == 0 { + sf.elapsedTime++ + overtime := sf.elapsedTime - current + time.Sleep(sleepTime((overtime))) + } + } + + return sf.toID() +} + +const sonyflakeTimeUnit = 1e7 // nsec, i.e. 10 msec + +func toSonyflakeTime(t time.Time) int64 { + return t.UTC().UnixNano() / sonyflakeTimeUnit +} + +func currentElapsedTime(startTime int64) int64 { + return toSonyflakeTime(time.Now()) - startTime +} + +func sleepTime(overtime int64) time.Duration { + return time.Duration(overtime)*10*time.Millisecond - + time.Duration(time.Now().UTC().UnixNano()%sonyflakeTimeUnit)*time.Nanosecond +} + +func (sf *Sonyflake) toID() (uint64, error) { + if sf.elapsedTime >= 1<= 16 && ip[1] < 32) || ip[0] == 192 && ip[1] == 168) +} + +func lower16BitPrivateIP() (uint16, error) { + ip, err := privateIPv4() + if err != nil { + return 0, err + } + + return uint16(ip[2])<<8 + uint16(ip[3]), nil +} + +// Decompose returns a set of Sonyflake ID parts. +func Decompose(id uint64) map[string]uint64 { + const maskSequence = uint64((1<> 63 + time := id >> (BitLenSequence + BitLenMachineID) + sequence := id & maskSequence >> BitLenMachineID + machineID := id & maskMachineID + return map[string]uint64{ + "id": id, + "msb": msb, + "time": time, + "sequence": sequence, + "machine-id": machineID, + } +} diff --git a/sonyflake_test.go b/sonyflake_test.go new file mode 100644 index 0000000..4edcb0f --- /dev/null +++ b/sonyflake_test.go @@ -0,0 +1,192 @@ +package sonyflake + +import ( + "fmt" + "runtime" + "testing" + "time" + + "github.com/deckarep/golang-set" +) + +var sf *Sonyflake + +var startTime int64 +var machineID uint64 + +func init() { + var st Settings + st.StartTime = time.Now() + + sf = NewSonyflake(st) + if sf == nil { + panic("sonyflake not created") + } + + startTime = toSonyflakeTime(st.StartTime) + + ip, _ := lower16BitPrivateIP() + machineID = uint64(ip) +} + +func nextID(t *testing.T) uint64 { + id, err := sf.NextID() + if err != nil { + t.Fatal("id not generated") + } + return id +} + +func TestSonyflakeOnce(t *testing.T) { + sleepTime := uint64(50) + time.Sleep(time.Duration(sleepTime) * 10 * time.Millisecond) + + id := nextID(t) + parts := Decompose(id) + + actualMSB := parts["msb"] + if actualMSB != 0 { + t.Errorf("unexpected msb: %d", actualMSB) + } + + actualTime := parts["time"] + if actualTime < sleepTime || actualTime > sleepTime+1 { + t.Errorf("unexpected time: %d", actualTime) + } + + actualSequence := parts["sequence"] + if actualSequence != 0 { + t.Errorf("unexpected sequence: %d", actualSequence) + } + + actualMachineID := parts["machine-id"] + if actualMachineID != machineID { + t.Errorf("unexpected machine id: %d", actualMachineID) + } + + fmt.Println("sonyflake id:", id) + fmt.Println("decompose:", parts) +} + +func currentTime() int64 { + return toSonyflakeTime(time.Now()) +} + +func TestSonyflakeFor10Sec(t *testing.T) { + var numID uint32 + var lastID uint64 + var maxSequence uint64 + + initial := currentTime() + current := initial + for current-initial < 1000 { + id := nextID(t) + parts := Decompose(id) + numID++ + + if id <= lastID { + t.Fatal("duplicated id") + } + lastID = id + + current = currentTime() + + actualMSB := parts["msb"] + if actualMSB != 0 { + t.Errorf("unexpected msb: %d", actualMSB) + } + + actualTime := int64(parts["time"]) + overtime := startTime + actualTime - current + if overtime > 0 { + t.Errorf("unexpected overtime: %d", overtime) + } + + actualSequence := parts["sequence"] + if maxSequence < actualSequence { + maxSequence = actualSequence + } + + actualMachineID := parts["machine-id"] + if actualMachineID != machineID { + t.Errorf("unexpected machine id: %d", actualMachineID) + } + } + + if maxSequence != 1<