This commit is contained in:
Yoshiyuki Mineo
2025-05-02 01:01:34 +09:00
parent 8d195df6f7
commit 6ff253cccd
14 changed files with 821 additions and 24 deletions

43
.github/workflows/test-v1.yml vendored Normal file
View File

@@ -0,0 +1,43 @@
name: CI for v1 Module
on:
push:
paths-ignore:
- 'v2/**'
pull_request:
paths-ignore:
- 'v2/**'
jobs:
test-v1:
strategy:
matrix:
go-version: [1.23.x, 1.24.x]
os: [ubuntu-latest]
runs-on: ${{matrix.os}}
defaults:
run:
working-directory: ./
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: ${{matrix.go-version}}
- name: gofmt
run: test -z "`gofmt -l .`"
- name: golint
run: test -z "`golint ./...`"
- name: go test
run: go test -v ./...
- name: Build example
run: cd example && ./linux64_build.sh

46
.github/workflows/test-v2.yml vendored Normal file
View File

@@ -0,0 +1,46 @@
name: CI for v2 Module
on:
push:
paths:
- 'v2/**'
pull_request:
paths:
- 'v2/**'
jobs:
test-v2:
strategy:
matrix:
go-version: [1.23.x, 1.24.x]
os: [ubuntu-latest]
runs-on: ${{matrix.os}}
defaults:
run:
working-directory: ./v2
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: ./v2/go.mod
- name: Tidy modules
run: go mod tidy
- name: Run tests
run: go test ./... -v
- name: gofmt
run: test -z "`gofmt -l .`"
- name: golint
run: test -z "`golint ./...`"
- name: Build example
run: cd example && ./linux64_build.sh

View File

@@ -1,24 +0,0 @@
on: [push, pull_request]
name: Test
jobs:
test:
strategy:
matrix:
go-version: [1.23.x, 1.24.x]
os: [ubuntu-latest]
runs-on: ${{matrix.os}}
steps:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: ${{matrix.go-version}}
- name: Checkout
uses: actions/checkout@v2
- name: gofmt
run: test -z "`gofmt -l .`"
- name: golint
run: test -z "`golint ./...`"
- name: go test
run: go test -v ./...
- name: Build example
run: cd example && ./linux64_build.sh

64
v2/awsutil/awsutil.go Normal file
View File

@@ -0,0 +1,64 @@
// Package awsutil provides utility functions for using Sonyflake on AWS.
package awsutil
import (
"errors"
"io/ioutil"
"net"
"net/http"
"os/exec"
"regexp"
"strconv"
"time"
)
func amazonEC2PrivateIPv4() (net.IP, error) {
res, err := http.Get("http://169.254.169.254/latest/meta-data/local-ipv4")
if err != nil {
return nil, err
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
ip := net.ParseIP(string(body))
if ip == nil {
return nil, errors.New("invalid ip address")
}
return ip.To4(), nil
}
// AmazonEC2MachineID retrieves the private IP address of the Amazon EC2 instance
// and returns its lower 16 bits.
// It works correctly on Docker as well.
func AmazonEC2MachineID() (uint16, error) {
ip, err := amazonEC2PrivateIPv4()
if err != nil {
return 0, err
}
return uint16(ip[2])<<8 + uint16(ip[3]), nil
}
// TimeDifference returns the time difference between the localhost and the given NTP server.
func TimeDifference(server string) (time.Duration, error) {
output, err := exec.Command("/usr/sbin/ntpdate", "-q", server).CombinedOutput()
if err != nil {
return time.Duration(0), err
}
re, _ := regexp.Compile("offset (.*) sec")
submatched := re.FindSubmatch(output)
if len(submatched) != 2 {
return time.Duration(0), errors.New("invalid ntpdate output")
}
f, err := strconv.ParseFloat(string(submatched[1]), 64)
if err != nil {
return time.Duration(0), err
}
return time.Duration(f*1000) * time.Millisecond, nil
}

6
v2/example/Dockerfile Normal file
View File

@@ -0,0 +1,6 @@
FROM ubuntu:14.04
ADD ./sonyflake_server /
ENTRYPOINT ["/sonyflake_server"]
EXPOSE 8080

View File

@@ -0,0 +1,8 @@
{
"AWSEBDockerrunVersion": "1",
"Ports": [
{
"ContainerPort": "8080"
}
]
}

21
v2/example/README.md Normal file
View File

@@ -0,0 +1,21 @@
Example
=======
This example runs Sonyflake on AWS Elastic Beanstalk.
Setup
-----
1. Build the cross compiler for linux/amd64 if using other platforms.
```
cd $GOROOT/src && GOOS=linux GOARCH=amd64 ./make.bash
```
2. Build sonyflake_server in the example directory.
```
./linux64_build.sh
```
3. Upload the example directory to AWS Elastic Beanstalk.

2
v2/example/linux64_build.sh Executable file
View File

@@ -0,0 +1,2 @@
#!/bin/sh
GOOS=linux GOARCH=amd64 go build sonyflake_server.go

View File

@@ -0,0 +1,42 @@
package main
import (
"encoding/json"
"net/http"
"github.com/sony/sonyflake/v2"
"github.com/sony/sonyflake/v2/awsutil"
)
var sf *sonyflake.Sonyflake
func init() {
var st sonyflake.Settings
st.MachineID = awsutil.AmazonEC2MachineID
sf = sonyflake.NewSonyflake(st)
if sf == nil {
panic("sonyflake not created")
}
}
func handler(w http.ResponseWriter, r *http.Request) {
id, err := sf.NextID()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
body, err := json.Marshal(sonyflake.Decompose(id))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header()["Content-Type"] = []string{"application/json; charset=utf-8"}
w.Write(body)
}
func main() {
http.HandleFunc("/", handler)
http.ListenAndServe(":8080", nil)
}

3
v2/go.mod Normal file
View File

@@ -0,0 +1,3 @@
module github.com/sony/sonyflake/v2
go 1.22

35
v2/mock/sonyflake_mock.go Normal file
View File

@@ -0,0 +1,35 @@
// Package mock offers implementations of interfaces defined in types.go
// This allows complete control over input / output for any given method that consumes
// a given type
package mock
import (
"fmt"
"net"
"github.com/sony/sonyflake/v2/types"
)
// NewSuccessfulInterfaceAddrs returns a single private IP address
func NewSuccessfulInterfaceAddrs() types.InterfaceAddrs {
ifat := make([]net.Addr, 0, 1)
ifat = append(ifat, &net.IPNet{IP: []byte{192, 168, 0, 1}, Mask: []byte{255, 0, 0, 0}})
return func() ([]net.Addr, error) {
return ifat, nil
}
}
// NewFailingInterfaceAddrs returns an error
func NewFailingInterfaceAddrs() types.InterfaceAddrs {
return func() ([]net.Addr, error) {
return nil, fmt.Errorf("test error")
}
}
// NewFailingInterfaceAddrs returns an empty slice of addresses
func NewNilInterfaceAddrs() types.InterfaceAddrs {
return func() ([]net.Addr, error) {
return []net.Addr{}, nil
}
}

229
v2/sonyflake.go Normal file
View File

@@ -0,0 +1,229 @@
// Package sonyflake implements Sonyflake, a distributed unique ID generator inspired by Twitter's Snowflake.
//
// A Sonyflake ID is composed of
//
// 39 bits for time in units of 10 msec
// 8 bits for a sequence number
// 16 bits for a machine id
package sonyflake
import (
"errors"
"net"
"sync"
"time"
"github.com/sony/sonyflake/v2/types"
)
// These constants are the bit lengths of Sonyflake ID parts.
const (
BitLenTime = 39 // bit length of time
BitLenSequence = 8 // bit length of sequence number
BitLenMachineID = 63 - BitLenTime - BitLenSequence // bit length of machine id
)
// Settings configures Sonyflake:
//
// StartTime is the time since which the Sonyflake time is defined as the elapsed time.
// If StartTime is 0, the start time of the Sonyflake is set to "2014-09-01 00:00:00 +0000 UTC".
// If StartTime is ahead of the current time, Sonyflake is not created.
//
// MachineID returns the unique ID of the Sonyflake instance.
// If MachineID returns an error, Sonyflake is not created.
// If MachineID is nil, default MachineID is used.
// Default MachineID returns the lower 16 bits of the private IP address.
//
// CheckMachineID validates the uniqueness of the machine ID.
// If CheckMachineID returns false, Sonyflake is not created.
// If CheckMachineID is nil, no validation is done.
type Settings struct {
StartTime time.Time
MachineID func() (uint16, error)
CheckMachineID func(uint16) bool
}
// Sonyflake is a distributed unique ID generator.
type Sonyflake struct {
mutex *sync.Mutex
startTime int64
elapsedTime int64
sequence uint16
machineID uint16
}
var (
ErrStartTimeAhead = errors.New("start time is ahead of now")
ErrNoPrivateAddress = errors.New("no private ip address")
ErrOverTimeLimit = errors.New("over the time limit")
ErrInvalidMachineID = errors.New("invalid machine id")
)
var defaultInterfaceAddrs = net.InterfaceAddrs
// New returns a new Sonyflake configured with the given Settings.
// New returns an error in the following cases:
// - Settings.StartTime is ahead of the current time.
// - Settings.MachineID returns an error.
// - Settings.CheckMachineID returns false.
func New(st Settings) (*Sonyflake, error) {
if st.StartTime.After(time.Now()) {
return nil, ErrStartTimeAhead
}
sf := new(Sonyflake)
sf.mutex = new(sync.Mutex)
sf.sequence = uint16(1<<BitLenSequence - 1)
if st.StartTime.IsZero() {
sf.startTime = toSonyflakeTime(time.Date(2014, 9, 1, 0, 0, 0, 0, time.UTC))
} else {
sf.startTime = toSonyflakeTime(st.StartTime)
}
var err error
if st.MachineID == nil {
sf.machineID, err = lower16BitPrivateIP(defaultInterfaceAddrs)
} else {
sf.machineID, err = st.MachineID()
}
if err != nil {
return nil, err
}
if st.CheckMachineID != nil && !st.CheckMachineID(sf.machineID) {
return nil, ErrInvalidMachineID
}
return sf, nil
}
// NewSonyflake returns a new Sonyflake configured with the given Settings.
// NewSonyflake returns nil in the following cases:
// - Settings.StartTime is ahead of the current time.
// - Settings.MachineID returns an error.
// - Settings.CheckMachineID returns false.
func NewSonyflake(st Settings) *Sonyflake {
sf, _ := New(st)
return sf
}
// NextID generates a next unique ID.
// After the Sonyflake time overflows, NextID returns an error.
func (sf *Sonyflake) NextID() (uint64, error) {
const maskSequence = uint16(1<<BitLenSequence - 1)
sf.mutex.Lock()
defer sf.mutex.Unlock()
current := currentElapsedTime(sf.startTime)
if sf.elapsedTime < current {
sf.elapsedTime = current
sf.sequence = 0
} else { // sf.elapsedTime >= current
sf.sequence = (sf.sequence + 1) & maskSequence
if sf.sequence == 0 {
sf.elapsedTime++
overtime := sf.elapsedTime - current
time.Sleep(sleepTime((overtime)))
}
}
return sf.toID()
}
const sonyflakeTimeUnit = 1e7 // nsec, i.e. 10 msec
func toSonyflakeTime(t time.Time) int64 {
return t.UTC().UnixNano() / sonyflakeTimeUnit
}
func currentElapsedTime(startTime int64) int64 {
return toSonyflakeTime(time.Now()) - startTime
}
func sleepTime(overtime int64) time.Duration {
return time.Duration(overtime*sonyflakeTimeUnit) -
time.Duration(time.Now().UTC().UnixNano()%sonyflakeTimeUnit)
}
func (sf *Sonyflake) toID() (uint64, error) {
if sf.elapsedTime >= 1<<BitLenTime {
return 0, ErrOverTimeLimit
}
return uint64(sf.elapsedTime)<<(BitLenSequence+BitLenMachineID) |
uint64(sf.sequence)<<BitLenMachineID |
uint64(sf.machineID), nil
}
func privateIPv4(interfaceAddrs types.InterfaceAddrs) (net.IP, error) {
as, err := interfaceAddrs()
if err != nil {
return nil, err
}
for _, a := range as {
ipnet, ok := a.(*net.IPNet)
if !ok || ipnet.IP.IsLoopback() {
continue
}
ip := ipnet.IP.To4()
if isPrivateIPv4(ip) {
return ip, nil
}
}
return nil, ErrNoPrivateAddress
}
func isPrivateIPv4(ip net.IP) bool {
// Allow private IP addresses (RFC1918) and link-local addresses (RFC3927)
return ip != nil &&
(ip[0] == 10 || ip[0] == 172 && (ip[1] >= 16 && ip[1] < 32) || ip[0] == 192 && ip[1] == 168 || ip[0] == 169 && ip[1] == 254)
}
func lower16BitPrivateIP(interfaceAddrs types.InterfaceAddrs) (uint16, error) {
ip, err := privateIPv4(interfaceAddrs)
if err != nil {
return 0, err
}
return uint16(ip[2])<<8 + uint16(ip[3]), nil
}
// ElapsedTime returns the elapsed time when the given Sonyflake ID was generated.
func ElapsedTime(id uint64) time.Duration {
return time.Duration(elapsedTime(id) * sonyflakeTimeUnit)
}
func elapsedTime(id uint64) uint64 {
return id >> (BitLenSequence + BitLenMachineID)
}
// SequenceNumber returns the sequence number of a Sonyflake ID.
func SequenceNumber(id uint64) uint64 {
const maskSequence = uint64((1<<BitLenSequence - 1) << BitLenMachineID)
return id & maskSequence >> BitLenMachineID
}
// MachineID returns the machine ID of a Sonyflake ID.
func MachineID(id uint64) uint64 {
const maskMachineID = uint64(1<<BitLenMachineID - 1)
return id & maskMachineID
}
// Decompose returns a set of Sonyflake ID parts.
func Decompose(id uint64) map[string]uint64 {
msb := id >> 63
time := elapsedTime(id)
sequence := SequenceNumber(id)
machineID := MachineID(id)
return map[string]uint64{
"id": id,
"msb": msb,
"time": time,
"sequence": sequence,
"machine-id": machineID,
}
}

314
v2/sonyflake_test.go Normal file
View File

@@ -0,0 +1,314 @@
package sonyflake
import (
"errors"
"fmt"
"net"
"runtime"
"testing"
"time"
"github.com/sony/sonyflake/v2/mock"
"github.com/sony/sonyflake/v2/types"
)
var sf *Sonyflake
var startTime int64
var machineID uint64
func init() {
var st Settings
st.StartTime = time.Now()
sf = NewSonyflake(st)
if sf == nil {
panic("sonyflake not created")
}
startTime = toSonyflakeTime(st.StartTime)
ip, _ := lower16BitPrivateIP(defaultInterfaceAddrs)
machineID = uint64(ip)
}
func nextID(t *testing.T) uint64 {
id, err := sf.NextID()
if err != nil {
t.Fatal("id not generated")
}
return id
}
func TestNew(t *testing.T) {
genError := fmt.Errorf("an error occurred while generating ID")
tests := []struct {
name string
settings Settings
err error
}{
{
name: "failure: time ahead",
settings: Settings{
StartTime: time.Now().Add(time.Minute),
},
err: ErrStartTimeAhead,
},
{
name: "failure: machine ID",
settings: Settings{
MachineID: func() (uint16, error) {
return 0, genError
},
},
err: genError,
},
{
name: "failure: invalid machine ID",
settings: Settings{
CheckMachineID: func(uint16) bool {
return false
},
},
err: ErrInvalidMachineID,
},
{
name: "success",
settings: Settings{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
sonyflake, err := New(test.settings)
if !errors.Is(err, test.err) {
t.Fatalf("unexpected value, want %#v, got %#v", test.err, err)
}
if sonyflake == nil && err == nil {
t.Fatal("unexpected value, sonyflake should not be nil")
}
})
}
}
func TestSonyflakeOnce(t *testing.T) {
sleepTime := time.Duration(50 * sonyflakeTimeUnit)
time.Sleep(sleepTime)
id := nextID(t)
actualTime := ElapsedTime(id)
if actualTime < sleepTime || actualTime > sleepTime+sonyflakeTimeUnit {
t.Errorf("unexpected time: %d", actualTime)
}
actualSequence := SequenceNumber(id)
if actualSequence != 0 {
t.Errorf("unexpected sequence: %d", actualSequence)
}
actualMachineID := MachineID(id)
if actualMachineID != machineID {
t.Errorf("unexpected machine id: %d", actualMachineID)
}
fmt.Println("sonyflake id:", id)
fmt.Println("decompose:", Decompose(id))
}
func currentTime() int64 {
return toSonyflakeTime(time.Now())
}
func TestSonyflakeFor10Sec(t *testing.T) {
var numID uint32
var lastID uint64
var maxSequence uint64
initial := currentTime()
current := initial
for current-initial < 1000 {
id := nextID(t)
parts := Decompose(id)
numID++
if id == lastID {
t.Fatal("duplicated id")
}
if id < lastID {
t.Fatal("must increase with time")
}
lastID = id
current = currentTime()
actualMSB := parts["msb"]
if actualMSB != 0 {
t.Errorf("unexpected msb: %d", actualMSB)
}
actualTime := int64(parts["time"])
overtime := startTime + actualTime - current
if overtime > 0 {
t.Errorf("unexpected overtime: %d", overtime)
}
actualSequence := parts["sequence"]
if maxSequence < actualSequence {
maxSequence = actualSequence
}
actualMachineID := parts["machine-id"]
if actualMachineID != machineID {
t.Errorf("unexpected machine id: %d", actualMachineID)
}
}
if maxSequence != 1<<BitLenSequence-1 {
t.Errorf("unexpected max sequence: %d", maxSequence)
}
fmt.Println("max sequence:", maxSequence)
fmt.Println("number of id:", numID)
}
func TestSonyflakeInParallel(t *testing.T) {
numCPU := runtime.NumCPU()
runtime.GOMAXPROCS(numCPU)
fmt.Println("number of cpu:", numCPU)
consumer := make(chan uint64)
const numID = 10000
generate := func() {
for i := 0; i < numID; i++ {
consumer <- nextID(t)
}
}
const numGenerator = 10
for i := 0; i < numGenerator; i++ {
go generate()
}
set := make(map[uint64]struct{})
for i := 0; i < numID*numGenerator; i++ {
id := <-consumer
if _, ok := set[id]; ok {
t.Fatal("duplicated id")
}
set[id] = struct{}{}
}
fmt.Println("number of id:", len(set))
}
func pseudoSleep(period time.Duration) {
sf.startTime -= int64(period) / sonyflakeTimeUnit
}
func TestNextIDError(t *testing.T) {
year := time.Duration(365*24) * time.Hour
pseudoSleep(time.Duration(174) * year)
nextID(t)
pseudoSleep(time.Duration(1) * year)
_, err := sf.NextID()
if err == nil {
t.Errorf("time is not over")
}
}
func TestPrivateIPv4(t *testing.T) {
testCases := []struct {
description string
expected net.IP
interfaceAddrs types.InterfaceAddrs
error string
}{
{
description: "InterfaceAddrs returns an error",
expected: nil,
interfaceAddrs: mock.NewFailingInterfaceAddrs(),
error: "test error",
},
{
description: "InterfaceAddrs returns an empty or nil list",
expected: nil,
interfaceAddrs: mock.NewNilInterfaceAddrs(),
error: "no private ip address",
},
{
description: "InterfaceAddrs returns one or more IPs",
expected: net.IP{192, 168, 0, 1},
interfaceAddrs: mock.NewSuccessfulInterfaceAddrs(),
error: "",
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
actual, err := privateIPv4(tc.interfaceAddrs)
if (err != nil) && (tc.error == "") {
t.Errorf("expected no error, but got: %s", err)
return
} else if (err != nil) && (tc.error != "") {
return
}
if net.IP.Equal(actual, tc.expected) {
return
} else {
t.Errorf("error: expected: %s, but got: %s", tc.expected, actual)
}
})
}
}
func TestLower16BitPrivateIP(t *testing.T) {
testCases := []struct {
description string
expected uint16
interfaceAddrs types.InterfaceAddrs
error string
}{
{
description: "InterfaceAddrs returns an empty or nil list",
expected: 0,
interfaceAddrs: mock.NewNilInterfaceAddrs(),
error: "no private ip address",
},
{
description: "InterfaceAddrs returns one or more IPs",
expected: 1,
interfaceAddrs: mock.NewSuccessfulInterfaceAddrs(),
error: "",
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
actual, err := lower16BitPrivateIP(tc.interfaceAddrs)
if (err != nil) && (tc.error == "") {
t.Errorf("expected no error, but got: %s", err)
return
} else if (err != nil) && (tc.error != "") {
return
}
if actual == tc.expected {
return
} else {
t.Errorf("error: expected: %v, but got: %v", tc.expected, actual)
}
})
}
}
func TestSonyflakeTimeUnit(t *testing.T) {
if time.Duration(sonyflakeTimeUnit) != 10*time.Millisecond {
t.Errorf("unexpected time unit")
}
}

8
v2/types/types.go Normal file
View File

@@ -0,0 +1,8 @@
// Package Types defines type signatures used throughout SonyFlake. This allows for
// fine-tuned control over imports, and the ability to mock out imports as well
package types
import "net"
// InterfaceAddrs defines the interface used for retrieving network addresses
type InterfaceAddrs func() ([]net.Addr, error)