Initial commit

.gitignore

@@ -0,0 +1 @@
+example/sonyflake_server

.travis.yml

@@ -0,0 +1,8 @@
+language: go
+go:
+  - 1.4
+  - tip
+sudo: false
+script:
+  - go test -v .
+  - cd example && ./linux64_build.sh

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright 2015 Sony Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,80 @@
+Sonyflake
+=========
+
+Sonyflake is a distributed unique ID generator inspired by [Twitter's Snowflake](https://blog.twitter.com/2010/announcing-snowflake).  
+A Sonyflake ID is composed of
+
+    39 bits for time in units of 10 msec
+     8 bits for a sequence number
+    16 bits for a machine id
+
+Installation
+------------
+
+```
+go get github.com/sony/sonyflake
+```
+
+Usage
+-----
+
+The function NewSonyflake creates a new Sonyflake instance.
+
+```go
+func NewSonyflake(st Settings) *Sonyflake
+```
+
+You can configure Sonyflake by the struct Settings:
+
+```go
+type Settings struct {
+	StartTime      time.Time
+	MachineID      func() (uint16, error)
+	CheckMachineID func(uint16) bool
+}
+```
+
+- StartTime is the time since which the Sonyflake time is defined as the elapsed time.
+  If StartTime is 0, the start time of the Sonyflake is set to "2014-09-01 00:00:00 +0000 UTC".
+  If StartTime is ahead of the current time, Sonyflake is not created.
+
+- MachineID returns the unique ID of the Sonyflake instance.
+  If MachineID returns an error, Sonyflake is not created.
+  If MachineID is nil, default MachineID is used.
+  Default MachineID returns the lower 16 bits of the private IP address.
+
+- CheckMachineID validates the uniqueness of the machine ID.
+  If CheckMachineID returns false, Sonyflake is not created.
+  If CheckMachineID is nil, no validation is done.
+
+In order to get a new unique ID, you just have to call the method NextID.
+
+```go
+func (sf *Sonyflake) NextID() (uint64, error)
+```
+
+NextID can continue to generate IDs for about 174 years from StartTime.
+But after the Sonyflake time is over the limit, NextID returns an error.
+
+AWS VPC and Docker
+------------------
+
+The [awsutil](https://github.com/sony/sonyflake/blob/master/awsutil) package provides
+the function AmazonEC2MachineID that returns the lower 16-bit private IP address of the Amazon EC2 instance.
+It also works correctly on Docker
+by retrieving [instance metadata](http://docs.aws.amazon.com/en_us/AWSEC2/latest/UserGuide/ec2-instance-metadata.html).
+
+[AWS VPC](http://docs.aws.amazon.com/en_us/AmazonVPC/latest/UserGuide/VPC_Subnets.html)
+is assigned a single CIDR with a netmask between /28 and /16.
+So if each EC2 instance has a unique private IP address in AWS VPC,
+the lower 16 bits of the address is also unique.
+In this common case, you can use AmazonEC2MachineID as Settings.MachineID.
+
+See [example](https://github.com/sony/sonyflake/blob/master/example) that runs Sonyflake on AWS Elastic Beanstalk.
+
+License
+-------
+
+The MIT License (MIT)
+
+See [LICENSE](https://github.com/sony/sonyflake/blob/master/LICENSE) for details.

awsutil/awsutil.go

@@ -0,0 +1,61 @@
+// Package awsutil provides utility functions for using Sonyflake on AWS.
+package awsutil
+
+import (
+	"errors"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"time"
+)
+
+func amazonEC2PrivateIPv4() (net.IP, error) {
+	res, err := http.Get("http://169.254.169.254/latest/meta-data/local-ipv4")
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	ip := net.ParseIP(string(body))
+	if ip == nil {
+		return nil, errors.New("invalid ip address")
+	}
+	return ip.To4(), nil
+}
+
+// AmazonEC2MachineID retrieves the private IP address of the Amazon EC2 instance
+// and returns its lower 16 bits.
+// It works correctly on Docker as well.
+func AmazonEC2MachineID() (uint16, error) {
+	ip, err := amazonEC2PrivateIPv4()
+	if err != nil {
+		return 0, err
+	}
+
+	return uint16(ip[2])<<8 + uint16(ip[3]), nil
+}
+
+// TimeDifference returns the time difference between the localhost and the given NTP server.
+func TimeDifference(server string) (time.Duration, error) {
+	output, err := exec.Command("/usr/sbin/ntpdate", "-q", server).CombinedOutput()
+	if err != nil {
+		return time.Duration(0), err
+	}
+
+	re, _ := regexp.Compile("offset (.*) sec")
+	submatched := re.FindSubmatch(output)
+	if len(submatched) != 2 {
+		return time.Duration(0), errors.New("invalid ntpdate output")
+	}
+
+	f, err := strconv.ParseFloat(string(submatched[1]), 64)
+	return time.Duration(f*1000) * time.Millisecond, nil
+}

example/Dockerfile

@@ -0,0 +1,6 @@
+FROM ubuntu:14.04
+
+ADD ./sonyflake_server /
+ENTRYPOINT ["/sonyflake_server"]
+
+EXPOSE 8080

example/Dockerrun.aws.json

@@ -0,0 +1,8 @@
+{
+  "AWSEBDockerrunVersion": "1",
+  "Ports": [
+    {
+      "ContainerPort": "8080"
+    }
+  ]
+}

example/README.md

@@ -0,0 +1,21 @@
+Example
+=======
+
+This example runs Sonyflake on AWS Elastic Beanstalk.
+
+Setup
+-----
+
+1. Build the cross compiler for linux/amd64 if using other platforms.
+
+  ```
+  cd $GOROOT/src && GOOS=linux GOARCH=amd64 ./make.bash
+  ```
+
+2. Build sonyflake_server in the example directory.
+
+  ```
+  ./linux64_build.sh
+  ```
+
+3. Upload the example directory to AWS Elastic Beanstalk.

example/linux64_build.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+GOOS=linux GOARCH=amd64 go build sonyflake_server.go

example/sonyflake_server.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/sony/sonyflake"
+	"github.com/sony/sonyflake/awsutil"
+)
+
+var sf *sonyflake.Sonyflake
+
+func init() {
+	var st sonyflake.Settings
+	st.MachineID = awsutil.AmazonEC2MachineID
+	sf = sonyflake.NewSonyflake(st)
+	if sf == nil {
+		panic("sonyflake not created")
+	}
+}
+
+func handler(w http.ResponseWriter, r *http.Request) {
+	id, err := sf.NextID()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	body, err := json.Marshal(sonyflake.Decompose(id))
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header()["Content-Type"] = []string{"application/json; charset=utf-8"}
+	w.Write(body)
+}
+
+func main() {
+	http.HandleFunc("/", handler)
+	http.ListenAndServe(":8080", nil)
+}

sonyflake.go

@@ -0,0 +1,183 @@
+// Package sonyflake implements Sonyflake, a distributed unique ID generator inspired by Twitter's Snowflake.
+//
+// A Sonyflake ID is composed of
+//     39 bits for time in units of 10 msec
+//      8 bits for a sequence number
+//     16 bits for a machine id
+package sonyflake
+
+import (
+	"errors"
+	"net"
+	"sync"
+	"time"
+)
+
+// These constants are the bit lengths of Sonyflake ID parts.
+const (
+	BitLenTime      = 39                               // bit length of time
+	BitLenSequence  = 8                                // bit length of sequence number
+	BitLenMachineID = 63 - BitLenTime - BitLenSequence // bit length of machine id
+)
+
+// Settings configures Sonyflake:
+//
+// StartTime is the time since which the Sonyflake time is defined as the elapsed time.
+// If StartTime is 0, the start time of the Sonyflake is set to "2014-09-01 00:00:00 +0000 UTC".
+// If StartTime is ahead of the current time, Sonyflake is not created.
+//
+// MachineID returns the unique ID of the Sonyflake instance.
+// If MachineID returns an error, Sonyflake is not created.
+// If MachineID is nil, default MachineID is used.
+// Default MachineID returns the lower 16 bits of the private IP address.
+//
+// CheckMachineID validates the uniqueness of the machine ID.
+// If CheckMachineID returns false, Sonyflake is not created.
+// If CheckMachineID is nil, no validation is done.
+type Settings struct {
+	StartTime      time.Time
+	MachineID      func() (uint16, error)
+	CheckMachineID func(uint16) bool
+}
+
+// Sonyflake is a distributed unique ID generator.
+type Sonyflake struct {
+	mutex       *sync.Mutex
+	startTime   int64
+	elapsedTime int64
+	sequence    uint16
+	machineID   uint16
+}
+
+// NewSonyflake returns a new Sonyflake configured with the given Settings.
+// NewSonyflake returns nil in the following cases:
+// - Settings.StartTime is ahead of the current time.
+// - Settings.MachineID returns an error.
+// - Settings.CheckMachineID returns false.
+func NewSonyflake(st Settings) *Sonyflake {
+	sf := new(Sonyflake)
+	sf.mutex = new(sync.Mutex)
+	sf.sequence = uint16(1<<BitLenSequence - 1)
+
+	if st.StartTime.After(time.Now()) {
+		return nil
+	}
+	if st.StartTime.IsZero() {
+		sf.startTime = toSonyflakeTime(time.Date(2014, 9, 1, 0, 0, 0, 0, time.UTC))
+	} else {
+		sf.startTime = toSonyflakeTime(st.StartTime)
+	}
+
+	var err error
+	if st.MachineID == nil {
+		sf.machineID, err = lower16BitPrivateIP()
+	} else {
+		sf.machineID, err = st.MachineID()
+	}
+	if err != nil || (st.CheckMachineID != nil && !st.CheckMachineID(sf.machineID)) {
+		return nil
+	}
+
+	return sf
+}
+
+// NextID generates a next unique ID.
+// After the Sonyflake time overflows, NextID returns an error.
+func (sf *Sonyflake) NextID() (uint64, error) {
+	const maskSequence = uint16(1<<BitLenSequence - 1)
+
+	sf.mutex.Lock()
+	defer sf.mutex.Unlock()
+
+	current := currentElapsedTime(sf.startTime)
+	if sf.elapsedTime < current {
+		sf.elapsedTime = current
+		sf.sequence = 0
+	} else { // sf.elapsedTime >= current
+		sf.sequence = (sf.sequence + 1) & maskSequence
+		if sf.sequence == 0 {
+			sf.elapsedTime++
+			overtime := sf.elapsedTime - current
+			time.Sleep(sleepTime((overtime)))
+		}
+	}
+
+	return sf.toID()
+}
+
+const sonyflakeTimeUnit = 1e7 // nsec, i.e. 10 msec
+
+func toSonyflakeTime(t time.Time) int64 {
+	return t.UTC().UnixNano() / sonyflakeTimeUnit
+}
+
+func currentElapsedTime(startTime int64) int64 {
+	return toSonyflakeTime(time.Now()) - startTime
+}
+
+func sleepTime(overtime int64) time.Duration {
+	return time.Duration(overtime)*10*time.Millisecond -
+		time.Duration(time.Now().UTC().UnixNano()%sonyflakeTimeUnit)*time.Nanosecond
+}
+
+func (sf *Sonyflake) toID() (uint64, error) {
+	if sf.elapsedTime >= 1<<BitLenTime {
+		return 0, errors.New("over the time limit")
+	}
+
+	return uint64(sf.elapsedTime)<<(BitLenSequence+BitLenMachineID) |
+		uint64(sf.sequence)<<BitLenMachineID |
+		uint64(sf.machineID), nil
+}
+
+func privateIPv4() (net.IP, error) {
+	as, err := net.InterfaceAddrs()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, a := range as {
+		ipnet, ok := a.(*net.IPNet)
+		if !ok || ipnet.IP.IsLoopback() {
+			continue
+		}
+
+		ip := ipnet.IP.To4()
+		if isPrivateIPv4(ip) {
+			return ip, nil
+		}
+	}
+	return nil, errors.New("no private ip address")
+}
+
+func isPrivateIPv4(ip net.IP) bool {
+	return ip != nil &&
+		(ip[0] == 10 || ip[0] == 172 && (ip[1] >= 16 && ip[1] < 32) || ip[0] == 192 && ip[1] == 168)
+}
+
+func lower16BitPrivateIP() (uint16, error) {
+	ip, err := privateIPv4()
+	if err != nil {
+		return 0, err
+	}
+
+	return uint16(ip[2])<<8 + uint16(ip[3]), nil
+}
+
+// Decompose returns a set of Sonyflake ID parts.
+func Decompose(id uint64) map[string]uint64 {
+	const maskSequence = uint64((1<<BitLenSequence - 1) << BitLenMachineID)
+	const maskMachineID = uint64(1<<BitLenMachineID - 1)
+
+	msb := id >> 63
+	time := id >> (BitLenSequence + BitLenMachineID)
+	sequence := id & maskSequence >> BitLenMachineID
+	machineID := id & maskMachineID
+	return map[string]uint64{
+		"id":         id,
+		"msb":        msb,
+		"time":       time,
+		"sequence":   sequence,
+		"machine-id": machineID,
+	}
+}

sonyflake_test.go

@@ -0,0 +1,192 @@
+package sonyflake
+
+import (
+	"fmt"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/deckarep/golang-set"
+)
+
+var sf *Sonyflake
+
+var startTime int64
+var machineID uint64
+
+func init() {
+	var st Settings
+	st.StartTime = time.Now()
+
+	sf = NewSonyflake(st)
+	if sf == nil {
+		panic("sonyflake not created")
+	}
+
+	startTime = toSonyflakeTime(st.StartTime)
+
+	ip, _ := lower16BitPrivateIP()
+	machineID = uint64(ip)
+}
+
+func nextID(t *testing.T) uint64 {
+	id, err := sf.NextID()
+	if err != nil {
+		t.Fatal("id not generated")
+	}
+	return id
+}
+
+func TestSonyflakeOnce(t *testing.T) {
+	sleepTime := uint64(50)
+	time.Sleep(time.Duration(sleepTime) * 10 * time.Millisecond)
+
+	id := nextID(t)
+	parts := Decompose(id)
+
+	actualMSB := parts["msb"]
+	if actualMSB != 0 {
+		t.Errorf("unexpected msb: %d", actualMSB)
+	}
+
+	actualTime := parts["time"]
+	if actualTime < sleepTime || actualTime > sleepTime+1 {
+		t.Errorf("unexpected time: %d", actualTime)
+	}
+
+	actualSequence := parts["sequence"]
+	if actualSequence != 0 {
+		t.Errorf("unexpected sequence: %d", actualSequence)
+	}
+
+	actualMachineID := parts["machine-id"]
+	if actualMachineID != machineID {
+		t.Errorf("unexpected machine id: %d", actualMachineID)
+	}
+
+	fmt.Println("sonyflake id:", id)
+	fmt.Println("decompose:", parts)
+}
+
+func currentTime() int64 {
+	return toSonyflakeTime(time.Now())
+}
+
+func TestSonyflakeFor10Sec(t *testing.T) {
+	var numID uint32
+	var lastID uint64
+	var maxSequence uint64
+
+	initial := currentTime()
+	current := initial
+	for current-initial < 1000 {
+		id := nextID(t)
+		parts := Decompose(id)
+		numID++
+
+		if id <= lastID {
+			t.Fatal("duplicated id")
+		}
+		lastID = id
+
+		current = currentTime()
+
+		actualMSB := parts["msb"]
+		if actualMSB != 0 {
+			t.Errorf("unexpected msb: %d", actualMSB)
+		}
+
+		actualTime := int64(parts["time"])
+		overtime := startTime + actualTime - current
+		if overtime > 0 {
+			t.Errorf("unexpected overtime: %d", overtime)
+		}
+
+		actualSequence := parts["sequence"]
+		if maxSequence < actualSequence {
+			maxSequence = actualSequence
+		}
+
+		actualMachineID := parts["machine-id"]
+		if actualMachineID != machineID {
+			t.Errorf("unexpected machine id: %d", actualMachineID)
+		}
+	}
+
+	if maxSequence != 1<<BitLenSequence-1 {
+		t.Errorf("unexpected max sequence: %d", maxSequence)
+	}
+	fmt.Println("max sequence:", maxSequence)
+	fmt.Println("number of id:", numID)
+}
+
+func TestSonyflakeInParallel(t *testing.T) {
+	numCPU := runtime.NumCPU()
+	runtime.GOMAXPROCS(numCPU)
+	fmt.Println("number of cpu:", numCPU)
+
+	consumer := make(chan uint64)
+
+	const numID = 10000
+	generate := func() {
+		for i := 0; i < numID; i++ {
+			consumer <- nextID(t)
+		}
+	}
+
+	const numGenerator = 10
+	for i := 0; i < numGenerator; i++ {
+		go generate()
+	}
+
+	set := mapset.NewSet()
+	for i := 0; i < numID*numGenerator; i++ {
+		id := <-consumer
+		if set.Contains(id) {
+			t.Fatal("duplicated id")
+		} else {
+			set.Add(id)
+		}
+	}
+	fmt.Println("number of id:", set.Cardinality())
+}
+
+func TestNilSonyflake(t *testing.T) {
+	var startInFuture Settings
+	startInFuture.StartTime = time.Now().Add(time.Duration(1) * time.Minute)
+	if NewSonyflake(startInFuture) != nil {
+		t.Errorf("sonyflake starting in the future")
+	}
+
+	var noMachineID Settings
+	noMachineID.MachineID = func() (uint16, error) {
+		return 0, fmt.Errorf("no machine id")
+	}
+	if NewSonyflake(noMachineID) != nil {
+		t.Errorf("sonyflake with no machine id")
+	}
+
+	var invalidMachineID Settings
+	invalidMachineID.CheckMachineID = func(uint16) bool {
+		return false
+	}
+	if NewSonyflake(invalidMachineID) != nil {
+		t.Errorf("sonyflake with invalid machine id")
+	}
+}
+
+func pseudoSleep(period time.Duration) {
+	sf.startTime -= int64(period) / sonyflakeTimeUnit
+}
+
+func TestNextIDError(t *testing.T) {
+	year := time.Duration(365*24) * time.Hour
+	pseudoSleep(time.Duration(174) * year)
+	nextID(t)
+
+	pseudoSleep(time.Duration(1) * year)
+	_, err := sf.NextID()
+	if err == nil {
+		t.Errorf("time is not over")
+	}
+}