From 6ff253cccdc20e8f0b375e5666ef9bc7f8fcd9f1 Mon Sep 17 00:00:00 2001 From: Yoshiyuki Mineo Date: Fri, 2 May 2025 01:01:34 +0900 Subject: [PATCH] Add v2 --- .github/workflows/test-v1.yml | 43 +++++ .github/workflows/test-v2.yml | 46 +++++ .github/workflows/test.yml | 24 --- v2/awsutil/awsutil.go | 64 +++++++ v2/example/Dockerfile | 6 + v2/example/Dockerrun.aws.json | 8 + v2/example/README.md | 21 +++ v2/example/linux64_build.sh | 2 + v2/example/sonyflake_server.go | 42 +++++ v2/go.mod | 3 + v2/mock/sonyflake_mock.go | 35 ++++ v2/sonyflake.go | 229 ++++++++++++++++++++++++ v2/sonyflake_test.go | 314 +++++++++++++++++++++++++++++++++ v2/types/types.go | 8 + 14 files changed, 821 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/test-v1.yml create mode 100644 .github/workflows/test-v2.yml delete mode 100644 .github/workflows/test.yml create mode 100644 v2/awsutil/awsutil.go create mode 100644 v2/example/Dockerfile create mode 100644 v2/example/Dockerrun.aws.json create mode 100644 v2/example/README.md create mode 100755 v2/example/linux64_build.sh create mode 100644 v2/example/sonyflake_server.go create mode 100644 v2/go.mod create mode 100644 v2/mock/sonyflake_mock.go create mode 100644 v2/sonyflake.go create mode 100644 v2/sonyflake_test.go create mode 100644 v2/types/types.go diff --git a/.github/workflows/test-v1.yml b/.github/workflows/test-v1.yml new file mode 100644 index 0000000..930a359 --- /dev/null +++ b/.github/workflows/test-v1.yml @@ -0,0 +1,43 @@ +name: CI for v1 Module + +on: + push: + paths-ignore: + - 'v2/**' + pull_request: + paths-ignore: + - 'v2/**' + +jobs: + test-v1: + strategy: + matrix: + go-version: [1.23.x, 1.24.x] + os: [ubuntu-latest] + + runs-on: ${{matrix.os}} + + defaults: + run: + working-directory: ./ + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{matrix.go-version}} + + - name: gofmt + run: test -z "`gofmt -l .`" + + - name: golint + run: test -z "`golint ./...`" + + - name: go test + run: go test -v ./... + + - name: Build example + run: cd example && ./linux64_build.sh diff --git a/.github/workflows/test-v2.yml b/.github/workflows/test-v2.yml new file mode 100644 index 0000000..6ea324e --- /dev/null +++ b/.github/workflows/test-v2.yml @@ -0,0 +1,46 @@ +name: CI for v2 Module + +on: + push: + paths: + - 'v2/**' + pull_request: + paths: + - 'v2/**' + +jobs: + test-v2: + strategy: + matrix: + go-version: [1.23.x, 1.24.x] + os: [ubuntu-latest] + + runs-on: ${{matrix.os}} + + defaults: + run: + working-directory: ./v2 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: ./v2/go.mod + + - name: Tidy modules + run: go mod tidy + + - name: Run tests + run: go test ./... -v + + - name: gofmt + run: test -z "`gofmt -l .`" + + - name: golint + run: test -z "`golint ./...`" + + - name: Build example + run: cd example && ./linux64_build.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index ced6f74..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,24 +0,0 @@ -on: [push, pull_request] -name: Test -jobs: - test: - strategy: - matrix: - go-version: [1.23.x, 1.24.x] - os: [ubuntu-latest] - runs-on: ${{matrix.os}} - steps: - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: ${{matrix.go-version}} - - name: Checkout - uses: actions/checkout@v2 - - name: gofmt - run: test -z "`gofmt -l .`" - - name: golint - run: test -z "`golint ./...`" - - name: go test - run: go test -v ./... - - name: Build example - run: cd example && ./linux64_build.sh diff --git a/v2/awsutil/awsutil.go b/v2/awsutil/awsutil.go new file mode 100644 index 0000000..4ad57b9 --- /dev/null +++ b/v2/awsutil/awsutil.go @@ -0,0 +1,64 @@ +// Package awsutil provides utility functions for using Sonyflake on AWS. +package awsutil + +import ( + "errors" + "io/ioutil" + "net" + "net/http" + "os/exec" + "regexp" + "strconv" + "time" +) + +func amazonEC2PrivateIPv4() (net.IP, error) { + res, err := http.Get("http://169.254.169.254/latest/meta-data/local-ipv4") + if err != nil { + return nil, err + } + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + ip := net.ParseIP(string(body)) + if ip == nil { + return nil, errors.New("invalid ip address") + } + return ip.To4(), nil +} + +// AmazonEC2MachineID retrieves the private IP address of the Amazon EC2 instance +// and returns its lower 16 bits. +// It works correctly on Docker as well. +func AmazonEC2MachineID() (uint16, error) { + ip, err := amazonEC2PrivateIPv4() + if err != nil { + return 0, err + } + + return uint16(ip[2])<<8 + uint16(ip[3]), nil +} + +// TimeDifference returns the time difference between the localhost and the given NTP server. +func TimeDifference(server string) (time.Duration, error) { + output, err := exec.Command("/usr/sbin/ntpdate", "-q", server).CombinedOutput() + if err != nil { + return time.Duration(0), err + } + + re, _ := regexp.Compile("offset (.*) sec") + submatched := re.FindSubmatch(output) + if len(submatched) != 2 { + return time.Duration(0), errors.New("invalid ntpdate output") + } + + f, err := strconv.ParseFloat(string(submatched[1]), 64) + if err != nil { + return time.Duration(0), err + } + return time.Duration(f*1000) * time.Millisecond, nil +} diff --git a/v2/example/Dockerfile b/v2/example/Dockerfile new file mode 100644 index 0000000..6344389 --- /dev/null +++ b/v2/example/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:14.04 + +ADD ./sonyflake_server / +ENTRYPOINT ["/sonyflake_server"] + +EXPOSE 8080 diff --git a/v2/example/Dockerrun.aws.json b/v2/example/Dockerrun.aws.json new file mode 100644 index 0000000..eb8d0a2 --- /dev/null +++ b/v2/example/Dockerrun.aws.json @@ -0,0 +1,8 @@ +{ + "AWSEBDockerrunVersion": "1", + "Ports": [ + { + "ContainerPort": "8080" + } + ] +} diff --git a/v2/example/README.md b/v2/example/README.md new file mode 100644 index 0000000..17fbcab --- /dev/null +++ b/v2/example/README.md @@ -0,0 +1,21 @@ +Example +======= + +This example runs Sonyflake on AWS Elastic Beanstalk. + +Setup +----- + +1. Build the cross compiler for linux/amd64 if using other platforms. + + ``` + cd $GOROOT/src && GOOS=linux GOARCH=amd64 ./make.bash + ``` + +2. Build sonyflake_server in the example directory. + + ``` + ./linux64_build.sh + ``` + +3. Upload the example directory to AWS Elastic Beanstalk. diff --git a/v2/example/linux64_build.sh b/v2/example/linux64_build.sh new file mode 100755 index 0000000..a164783 --- /dev/null +++ b/v2/example/linux64_build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +GOOS=linux GOARCH=amd64 go build sonyflake_server.go diff --git a/v2/example/sonyflake_server.go b/v2/example/sonyflake_server.go new file mode 100644 index 0000000..f85cf6e --- /dev/null +++ b/v2/example/sonyflake_server.go @@ -0,0 +1,42 @@ +package main + +import ( + "encoding/json" + "net/http" + + "github.com/sony/sonyflake/v2" + "github.com/sony/sonyflake/v2/awsutil" +) + +var sf *sonyflake.Sonyflake + +func init() { + var st sonyflake.Settings + st.MachineID = awsutil.AmazonEC2MachineID + sf = sonyflake.NewSonyflake(st) + if sf == nil { + panic("sonyflake not created") + } +} + +func handler(w http.ResponseWriter, r *http.Request) { + id, err := sf.NextID() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + body, err := json.Marshal(sonyflake.Decompose(id)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header()["Content-Type"] = []string{"application/json; charset=utf-8"} + w.Write(body) +} + +func main() { + http.HandleFunc("/", handler) + http.ListenAndServe(":8080", nil) +} diff --git a/v2/go.mod b/v2/go.mod new file mode 100644 index 0000000..ee45ee3 --- /dev/null +++ b/v2/go.mod @@ -0,0 +1,3 @@ +module github.com/sony/sonyflake/v2 + +go 1.22 diff --git a/v2/mock/sonyflake_mock.go b/v2/mock/sonyflake_mock.go new file mode 100644 index 0000000..8de6107 --- /dev/null +++ b/v2/mock/sonyflake_mock.go @@ -0,0 +1,35 @@ +// Package mock offers implementations of interfaces defined in types.go +// This allows complete control over input / output for any given method that consumes +// a given type +package mock + +import ( + "fmt" + "net" + + "github.com/sony/sonyflake/v2/types" +) + +// NewSuccessfulInterfaceAddrs returns a single private IP address +func NewSuccessfulInterfaceAddrs() types.InterfaceAddrs { + ifat := make([]net.Addr, 0, 1) + ifat = append(ifat, &net.IPNet{IP: []byte{192, 168, 0, 1}, Mask: []byte{255, 0, 0, 0}}) + + return func() ([]net.Addr, error) { + return ifat, nil + } +} + +// NewFailingInterfaceAddrs returns an error +func NewFailingInterfaceAddrs() types.InterfaceAddrs { + return func() ([]net.Addr, error) { + return nil, fmt.Errorf("test error") + } +} + +// NewFailingInterfaceAddrs returns an empty slice of addresses +func NewNilInterfaceAddrs() types.InterfaceAddrs { + return func() ([]net.Addr, error) { + return []net.Addr{}, nil + } +} diff --git a/v2/sonyflake.go b/v2/sonyflake.go new file mode 100644 index 0000000..96d45d7 --- /dev/null +++ b/v2/sonyflake.go @@ -0,0 +1,229 @@ +// Package sonyflake implements Sonyflake, a distributed unique ID generator inspired by Twitter's Snowflake. +// +// A Sonyflake ID is composed of +// +// 39 bits for time in units of 10 msec +// 8 bits for a sequence number +// 16 bits for a machine id +package sonyflake + +import ( + "errors" + "net" + "sync" + "time" + + "github.com/sony/sonyflake/v2/types" +) + +// These constants are the bit lengths of Sonyflake ID parts. +const ( + BitLenTime = 39 // bit length of time + BitLenSequence = 8 // bit length of sequence number + BitLenMachineID = 63 - BitLenTime - BitLenSequence // bit length of machine id +) + +// Settings configures Sonyflake: +// +// StartTime is the time since which the Sonyflake time is defined as the elapsed time. +// If StartTime is 0, the start time of the Sonyflake is set to "2014-09-01 00:00:00 +0000 UTC". +// If StartTime is ahead of the current time, Sonyflake is not created. +// +// MachineID returns the unique ID of the Sonyflake instance. +// If MachineID returns an error, Sonyflake is not created. +// If MachineID is nil, default MachineID is used. +// Default MachineID returns the lower 16 bits of the private IP address. +// +// CheckMachineID validates the uniqueness of the machine ID. +// If CheckMachineID returns false, Sonyflake is not created. +// If CheckMachineID is nil, no validation is done. +type Settings struct { + StartTime time.Time + MachineID func() (uint16, error) + CheckMachineID func(uint16) bool +} + +// Sonyflake is a distributed unique ID generator. +type Sonyflake struct { + mutex *sync.Mutex + startTime int64 + elapsedTime int64 + sequence uint16 + machineID uint16 +} + +var ( + ErrStartTimeAhead = errors.New("start time is ahead of now") + ErrNoPrivateAddress = errors.New("no private ip address") + ErrOverTimeLimit = errors.New("over the time limit") + ErrInvalidMachineID = errors.New("invalid machine id") +) + +var defaultInterfaceAddrs = net.InterfaceAddrs + +// New returns a new Sonyflake configured with the given Settings. +// New returns an error in the following cases: +// - Settings.StartTime is ahead of the current time. +// - Settings.MachineID returns an error. +// - Settings.CheckMachineID returns false. +func New(st Settings) (*Sonyflake, error) { + if st.StartTime.After(time.Now()) { + return nil, ErrStartTimeAhead + } + + sf := new(Sonyflake) + sf.mutex = new(sync.Mutex) + sf.sequence = uint16(1<= current + sf.sequence = (sf.sequence + 1) & maskSequence + if sf.sequence == 0 { + sf.elapsedTime++ + overtime := sf.elapsedTime - current + time.Sleep(sleepTime((overtime))) + } + } + + return sf.toID() +} + +const sonyflakeTimeUnit = 1e7 // nsec, i.e. 10 msec + +func toSonyflakeTime(t time.Time) int64 { + return t.UTC().UnixNano() / sonyflakeTimeUnit +} + +func currentElapsedTime(startTime int64) int64 { + return toSonyflakeTime(time.Now()) - startTime +} + +func sleepTime(overtime int64) time.Duration { + return time.Duration(overtime*sonyflakeTimeUnit) - + time.Duration(time.Now().UTC().UnixNano()%sonyflakeTimeUnit) +} + +func (sf *Sonyflake) toID() (uint64, error) { + if sf.elapsedTime >= 1<= 16 && ip[1] < 32) || ip[0] == 192 && ip[1] == 168 || ip[0] == 169 && ip[1] == 254) +} + +func lower16BitPrivateIP(interfaceAddrs types.InterfaceAddrs) (uint16, error) { + ip, err := privateIPv4(interfaceAddrs) + if err != nil { + return 0, err + } + + return uint16(ip[2])<<8 + uint16(ip[3]), nil +} + +// ElapsedTime returns the elapsed time when the given Sonyflake ID was generated. +func ElapsedTime(id uint64) time.Duration { + return time.Duration(elapsedTime(id) * sonyflakeTimeUnit) +} + +func elapsedTime(id uint64) uint64 { + return id >> (BitLenSequence + BitLenMachineID) +} + +// SequenceNumber returns the sequence number of a Sonyflake ID. +func SequenceNumber(id uint64) uint64 { + const maskSequence = uint64((1<> BitLenMachineID +} + +// MachineID returns the machine ID of a Sonyflake ID. +func MachineID(id uint64) uint64 { + const maskMachineID = uint64(1<> 63 + time := elapsedTime(id) + sequence := SequenceNumber(id) + machineID := MachineID(id) + return map[string]uint64{ + "id": id, + "msb": msb, + "time": time, + "sequence": sequence, + "machine-id": machineID, + } +} diff --git a/v2/sonyflake_test.go b/v2/sonyflake_test.go new file mode 100644 index 0000000..c5830f2 --- /dev/null +++ b/v2/sonyflake_test.go @@ -0,0 +1,314 @@ +package sonyflake + +import ( + "errors" + "fmt" + "net" + "runtime" + "testing" + "time" + + "github.com/sony/sonyflake/v2/mock" + "github.com/sony/sonyflake/v2/types" +) + +var sf *Sonyflake + +var startTime int64 +var machineID uint64 + +func init() { + var st Settings + st.StartTime = time.Now() + + sf = NewSonyflake(st) + if sf == nil { + panic("sonyflake not created") + } + + startTime = toSonyflakeTime(st.StartTime) + + ip, _ := lower16BitPrivateIP(defaultInterfaceAddrs) + machineID = uint64(ip) +} + +func nextID(t *testing.T) uint64 { + id, err := sf.NextID() + if err != nil { + t.Fatal("id not generated") + } + return id +} + +func TestNew(t *testing.T) { + genError := fmt.Errorf("an error occurred while generating ID") + + tests := []struct { + name string + settings Settings + err error + }{ + { + name: "failure: time ahead", + settings: Settings{ + StartTime: time.Now().Add(time.Minute), + }, + err: ErrStartTimeAhead, + }, + { + name: "failure: machine ID", + settings: Settings{ + MachineID: func() (uint16, error) { + return 0, genError + }, + }, + err: genError, + }, + { + name: "failure: invalid machine ID", + settings: Settings{ + CheckMachineID: func(uint16) bool { + return false + }, + }, + err: ErrInvalidMachineID, + }, + { + name: "success", + settings: Settings{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + sonyflake, err := New(test.settings) + + if !errors.Is(err, test.err) { + t.Fatalf("unexpected value, want %#v, got %#v", test.err, err) + } + + if sonyflake == nil && err == nil { + t.Fatal("unexpected value, sonyflake should not be nil") + } + }) + } +} + +func TestSonyflakeOnce(t *testing.T) { + sleepTime := time.Duration(50 * sonyflakeTimeUnit) + time.Sleep(sleepTime) + + id := nextID(t) + + actualTime := ElapsedTime(id) + if actualTime < sleepTime || actualTime > sleepTime+sonyflakeTimeUnit { + t.Errorf("unexpected time: %d", actualTime) + } + + actualSequence := SequenceNumber(id) + if actualSequence != 0 { + t.Errorf("unexpected sequence: %d", actualSequence) + } + + actualMachineID := MachineID(id) + if actualMachineID != machineID { + t.Errorf("unexpected machine id: %d", actualMachineID) + } + + fmt.Println("sonyflake id:", id) + fmt.Println("decompose:", Decompose(id)) +} + +func currentTime() int64 { + return toSonyflakeTime(time.Now()) +} + +func TestSonyflakeFor10Sec(t *testing.T) { + var numID uint32 + var lastID uint64 + var maxSequence uint64 + + initial := currentTime() + current := initial + for current-initial < 1000 { + id := nextID(t) + parts := Decompose(id) + numID++ + + if id == lastID { + t.Fatal("duplicated id") + } + if id < lastID { + t.Fatal("must increase with time") + } + lastID = id + + current = currentTime() + + actualMSB := parts["msb"] + if actualMSB != 0 { + t.Errorf("unexpected msb: %d", actualMSB) + } + + actualTime := int64(parts["time"]) + overtime := startTime + actualTime - current + if overtime > 0 { + t.Errorf("unexpected overtime: %d", overtime) + } + + actualSequence := parts["sequence"] + if maxSequence < actualSequence { + maxSequence = actualSequence + } + + actualMachineID := parts["machine-id"] + if actualMachineID != machineID { + t.Errorf("unexpected machine id: %d", actualMachineID) + } + } + + if maxSequence != 1<