From 0f43462de3720b917e859a8d3bad9de368d41e82 Mon Sep 17 00:00:00 2001 From: Godruoyi Date: Thu, 15 Apr 2021 11:04:49 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=9B=20first=20commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 5 + atomic_resolver.go | 31 ++++++ atomic_resolver_test.go | 29 ++++++ example/example.go | 21 +++++ go.mod | 3 + package.go | 8 ++ privateip.go | 65 +++++++++++++ privateip_test.go | 14 +++ readme.md | 187 +++++++++++++++++++++++++++++++++++++ snowflake.go | 172 ++++++++++++++++++++++++++++++++++ snowflake_test.go | 202 ++++++++++++++++++++++++++++++++++++++++ 11 files changed, 737 insertions(+) create mode 100644 .gitignore create mode 100644 atomic_resolver.go create mode 100644 atomic_resolver_test.go create mode 100644 example/example.go create mode 100644 go.mod create mode 100644 package.go create mode 100644 privateip.go create mode 100644 privateip_test.go create mode 100644 readme.md create mode 100644 snowflake.go create mode 100644 snowflake_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b4b8202 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.code +.git +.idea +*.DS_Store +/vendor diff --git a/atomic_resolver.go b/atomic_resolver.go new file mode 100644 index 0000000..ff337c2 --- /dev/null +++ b/atomic_resolver.go @@ -0,0 +1,31 @@ +package snowflake + +import "sync/atomic" + +var lastTime int64 +var lastSeq uint32 + +// AtomicResolver define as atomic sequence resolver, base on standard sync/atomic. +func AtomicResolver(ms int64) (uint16, error) { + var last int64 + var seq, localSeq uint32 + + for { + last = atomic.LoadInt64(&lastTime) + localSeq = atomic.LoadUint32(&lastSeq) + if last > ms { + return MaxSequence, nil + } + + if last == ms { + seq = MaxSequence & (localSeq + 1) + if seq == 0 { + return MaxSequence, nil + } + } + + if atomic.CompareAndSwapInt64(&lastTime, last, ms) && atomic.CompareAndSwapUint32(&lastSeq, localSeq, seq) { + return uint16(seq), nil + } + } +} diff --git a/atomic_resolver_test.go b/atomic_resolver_test.go new file mode 100644 index 0000000..1ffc17c --- /dev/null +++ b/atomic_resolver_test.go @@ -0,0 +1,29 @@ +package snowflake_test + +import ( + "testing" + + "github.com/godruoyi/go-snowflake" +) + +func TestAtomicResolver(t *testing.T) { + id, _ := snowflake.AtomicResolver(1) + + if id != 0 { + t.Error("Sequence should be equal 0") + } +} + +func BenchmarkCombinationParallel(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + snowflake.AtomicResolver(1) + } + }) +} + +func BenchmarkAtomicResolver(b *testing.B) { + for i := 0; i < b.N; i++ { + snowflake.AtomicResolver(1) + } +} diff --git a/example/example.go b/example/example.go new file mode 100644 index 0000000..16d4eb9 --- /dev/null +++ b/example/example.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + + "github.com/godruoyi/go-snowflake" +) + +func main() { + id := snowflake.ID() + fmt.Println(id) + + sid := snowflake.ParseID(id) + // SID { + // Sequence: 0 + // MachineID: 0 + // Timestamp: x + // ID: x + // } + fmt.Println(sid) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..0f536f5 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/godruoyi/go-snowflake + +go 1.15 diff --git a/package.go b/package.go new file mode 100644 index 0000000..5972201 --- /dev/null +++ b/package.go @@ -0,0 +1,8 @@ +// Package snowflake is a network service for generating unique ID numbers at high scale with some simple guarantees. +// The first bit is unused sign bit. +// The second part consists of a 41-bit timestamp (milliseconds) whose value is the offset of the current time relative to a certain time. +// The 5 bits of the third and fourth parts represent data center and worker, and max value is 2^5 -1 = 31. +// The last part consists of 12 bits, its means the length of the serial number generated per millisecond per working node, a maximum of 2^12 -1 = 4095 IDs can be generated in the same millisecond. +// In a distributed environment, five-bit datacenter and worker mean that can deploy 31 datacenters, and each datacenter can deploy up to 31 nodes. +// The binary length of 41 bits is at most 2^41 -1 millisecond = 69 years. So the snowflake algorithm can be used for up to 69 years, In order to maximize the use of the algorithm, you should specify a start time for it. +package snowflake diff --git a/privateip.go b/privateip.go new file mode 100644 index 0000000..7f0d276 --- /dev/null +++ b/privateip.go @@ -0,0 +1,65 @@ +package snowflake + +import ( + "errors" + "net" +) + +// PrivateIPToMachineID convert private ip to machine id. +// From https://github.com/sony/sonyflake/blob/master/sonyflake.go +func PrivateIPToMachineID() uint16 { + ip, err := lower16BitPrivateIP() + if err != nil { + return 0 + } + + return ip +} + +//-------------------------------------------------------------------- +// private function defined. +//-------------------------------------------------------------------- + +func privateIPv4() (net.IP, error) { + as, err := net.InterfaceAddrs() + if err != nil { + return nil, err + } + + for _, a := range as { + ipnet, ok := a.(*net.IPNet) + if !ok || ipnet.IP.IsLoopback() { + continue + } + + ip := ipnet.IP.To4() + if isPrivateIPv4(ip) { + return ip, nil + } + } + + return nil, errors.New("no private ip address") +} + +func isPrivateIPv4(ip net.IP) bool { + return ip != nil && + (ip[0] == 10 || ip[0] == 172 && (ip[1] >= 16 && ip[1] < 32) || ip[0] == 192 && ip[1] == 168) +} + +func lower16BitPrivateIP() (uint16, error) { + ip, err := privateIPv4() + if err != nil { + return 0, err + } + + // Snowflake macnineID max length is 10, max value is 1023 + // If ip[2] > 3, return ip[2] + ip[3], but: + // 10.21.5.211 => return 5 + 211 = 216 + // 10.21.4.212 => return 4 + 212 = 216 + // need help if you can do this. + if ip[2] > 3 { + return uint16(ip[2] + ip[3]), nil + } + + return uint16(ip[2])<<8 + uint16(ip[3]), nil +} diff --git a/privateip_test.go b/privateip_test.go new file mode 100644 index 0000000..76e167b --- /dev/null +++ b/privateip_test.go @@ -0,0 +1,14 @@ +package snowflake_test + +import ( + "testing" + + "github.com/godruoyi/go-snowflake" +) + +func TestPrivateIPToMachineID(t *testing.T) { + mid := snowflake.PrivateIPToMachineID() + if mid <= 0 { + t.Error("MachineID should be > 0") + } +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..6668d25 --- /dev/null +++ b/readme.md @@ -0,0 +1,187 @@ +
+

+ +

+

An Lock Free ID Generator for Golang based on Snowflake Algorithm (Twitter announced).

+ +
+ +## Description + +An Lock Free ID Generator for Golang implementation. + +![file](https://images.godruoyi.com/logos/201908/13/_1565672621_LPW65Pi8cG.png) + +Snowflake is a network service for generating unique ID numbers at high scale with some simple guarantees. + +* The first bit is unused sign bit. +* The second part consists of a 41-bit timestamp (milliseconds) whose value is the offset of the current time relative to a certain time. +* The 10 bits machineID, and max value is 2^10 -1 = 1023. +* The last part consists of 12 bits, its means the length of the serial number generated per millisecond per working node, a maximum of 2^12 -1 = 4095 IDs can be generated in the same millisecond. +* The binary length of 41 bits is at most 2^41 -1 millisecond = 69 years. So the snowflake algorithm can be used for up to 69 years, In order to maximize the use of the algorithm, you should specify a start time for it. + +> You must know, The ID generated by the snowflake algorithm is not guaranteed to be unique. +> For example, when two different requests enter the same machine at the same time, and the sequence generated by the node is the same, the generated ID will be duplicated. + +So if you want use the snowflake algorithm to generate unique ID, You must ensure: The sequence-number generated in the same millisecond of the same node is unique. +Based on this, we created this package and integrated multiple sequence-number providers into it. + +* AtomicResolver (base sync/atmoic) +* Custom (you can custom) + +> Each provider only needs to ensure that the serial number generated in the same millisecond is different. You can get a unique ID. + +## Installation + +```shell +$ go get github.com/godruoyi/go-snowflake +``` + +## Useage + +1. simple to use. + +```go +import ( + "fmt" + + "github.com/godruoyi/go-snowflake" +) + +func main() { + id := snowflake.ID() + fmt.Println(id) + // 1537200202186752 +} +``` + +2. Specify the MachineID. + +```go +import ( + "fmt" + + "github.com/godruoyi/go-snowflake" +) + +func main() { + snowflake.SetMachineID(1) + id := snowflake.ID() + fmt.Println(id) +} +``` + +3. Specify start time. + +```go +import ( + "fmt" + "time" + + "github.com/godruoyi/go-snowflake" +) + +func main() { + snowflake.SetStartTime(time.Date(2014, 9, 1, 0, 0, 0, 0, time.UTC)) + id := snowflake.ID() + fmt.Println(id) +} +``` + +4. Parse ID. + +```go +import ( + "fmt" + "time" + + "github.com/godruoyi/go-snowflake" +) + +func main() { + id := snowflake.ID() + sid := snowflake.ParseID(id) + + fmt.Println(sid.ID) // + fmt.Println(sid.MachineID) // + fmt.Println(sid.Sequence) // + fmt.Println(sid.Timestamp) // + fmt.Println(sid.GenerateTime()) // +} +``` + +## Best practices + +> ⚠️⚠️ All SetXXX method is thread-unsafe, recommended you call him in the main function. + +```go +import ( + "fmt" + "time" + "net/http" + + "github.com/godruoyi/go-snowflake" +) + +func main() { + snowflake.SetMachineID(1) // change to your machineID + snowflake.SetStartTime(time.Date(2014, 9, 1, 0, 0, 0, 0, time.UTC)) + + http.HandleFunc("/order", submitOrder) + http.ListenAndServe(":8090", nil) +} + +func submitOrder(w http.ResponseWriter, req *http.Request) { + orderId := snowflake.ID() + // save order +} +``` + +## Advanced + +1. Custom sequence resolver. + +You can customize the sequence-number resolver by following way: + +```go +import ( + "fmt" + "time" + + "github.com/godruoyi/go-snowflake" +) + +func yourSequenceNumber(ms int64) (uint16, error) { + +} + +// usage + +snowflake.SetSequenceResolver(yourSequenceNumber) +snowflake.ID() +``` + +And you can use closure: + +## License + +MIT diff --git a/snowflake.go b/snowflake.go new file mode 100644 index 0000000..cf793b6 --- /dev/null +++ b/snowflake.go @@ -0,0 +1,172 @@ +package snowflake + +import ( + "errors" + "time" +) + +// These constants are the bit lengths of snowflake ID parts. +const ( + TimestampLength = 41 + MachineIDLength = 10 + SequenceLength = 12 + MaxSequence = 1<= MaxSequence { + c = waitForNextMillis(c) + seq, err = seqResolver(c) + if err != nil { + return 0, err + } + } + + df := int(elapsedTime(startTime)) + if df < 0 || df > MaxTimestamp { + return 0, errors.New("The maximum life cycle of the snowflake algorithm is 2^41-1(millis), please check starttime") + } + + id := uint64(df< current millisecond +// current millisecond - s > 2^41(69 years). +// This function is thread-unsafe, recommended you call him in the main function. +func SetStartTime(s time.Time) { + s = s.UTC() + + if s.IsZero() { + panic("The start time cannot be a zero value") + } + + if s.After(time.Now()) { + panic("The s cannot be greater than the current millisecond") + } + + // Because s must after now, so the `df` not < 0. + df := elapsedTime(s) + if df > MaxTimestamp { + panic("The maximum life cycle of the snowflake algorithm is 69 years") + } + + startTime = s +} + +// SetMachineID specify the machine ID. It will panic when machineid > max limit for 2^10-1. +// This function is thread-unsafe, recommended you call him in the main function. +func SetMachineID(m uint16) { + if m > MaxMachineID { + panic("The machineid cannot be greater than 1023") + } + machineID = int(m) +} + +// SetSequenceResolver set an custom sequence resolver. +// This function is thread-unsafe, recommended you call him in the main function. +func SetSequenceResolver(seq SequenceResolver) { + if seq != nil { + resolver = seq + } +} + +// SID snowflake id +type SID struct { + Sequence uint64 + MachineID uint64 + Timestamp uint64 + ID uint64 +} + +// GenerateTime snowflake generate at, return a UTC time. +func (id *SID) GenerateTime() time.Time { + ms := startTime.UTC().UnixNano()/1e6 + int64(id.Timestamp) + + return time.Unix(0, (ms * int64(time.Millisecond))).UTC() +} + +// ParseID parse snowflake it to SID struct. +func ParseID(id uint64) SID { + time := id >> (SequenceLength + MachineIDLength) + sequence := id & MaxSequence + machineID := (id & (MaxMachineID << SequenceLength)) >> SequenceLength + + return SID{ + ID: id, + Sequence: sequence, + MachineID: machineID, + Timestamp: time, + } +} + +//-------------------------------------------------------------------- +// private function defined. +//-------------------------------------------------------------------- + +func waitForNextMillis(last int64) int64 { + now := currentMillis() + for now == last { + now = currentMillis() + } + return now +} + +func callSequenceResolver() SequenceResolver { + if resolver == nil { + return AtomicResolver + } + + return resolver +} + +func elapsedTime(s time.Time) int64 { + return currentMillis() - s.UTC().UnixNano()/1e6 +} + +// currentMillis get current millisecond. +func currentMillis() int64 { + return time.Now().UTC().UnixNano() / 1e6 +} diff --git a/snowflake_test.go b/snowflake_test.go new file mode 100644 index 0000000..ebb9e8b --- /dev/null +++ b/snowflake_test.go @@ -0,0 +1,202 @@ +package snowflake_test + +import ( + "errors" + "testing" + "time" + + "github.com/godruoyi/go-snowflake" +) + +func TestID(t *testing.T) { + id := snowflake.ID() + + if id <= 0 { + t.Error("The snowflake should't < 0.") + } +} + +func TestSetStartTime(t *testing.T) { + t.Run("A nil time", func(tt *testing.T) { + defer func() { + if e := recover(); e == nil { + tt.Error("Should throw a error when start time is zero") + } else if e.(string) != "The start time cannot be a zero value" { + tt.Error("The error message should equal [The start time cannot be a zero value]") + } + }() + var time time.Time + snowflake.SetStartTime(time) + }) + + t.Run("Start time too big", func(tt *testing.T) { + defer func() { + if e := recover(); e == nil { + tt.Error("Should throw a error when start time is too big") + } else if e.(string) != "The s cannot be greater than the current millisecond" { + tt.Error("The error message should equal [The s cannot be greater than the current millisecond]") + } + }() + time := time.Date(2035, 1, 1, 1, 0, 0, 0, time.UTC) + snowflake.SetStartTime(time) + }) + + t.Run("Start time too small", func(tt *testing.T) { + defer func() { + if e := recover(); e == nil { + tt.Error("Should throw a error when starttime is too small") + } else if e.(string) != "The maximum life cycle of the snowflake algorithm is 69 years" { + tt.Error("The error message should equal [The maximum life cycle of the snowflake algorithm is 69 years]") + } + }() + // because 2021-69 = 1952, set df time > 69 years to test. + time := time.Date(1951, 1, 1, 1, 0, 0, 0, time.UTC) + snowflake.SetStartTime(time) + }) + + t.Run("Default start time", func(tt *testing.T) { + defaultTime := time.Date(2014, 9, 1, 0, 0, 0, 0, time.UTC) + defaultNano := defaultTime.UTC().UnixNano() / 1e6 + + sid := snowflake.ParseID(snowflake.ID()) + currentTime := sid.Timestamp + uint64(defaultNano) + + nowNano := time.Now().UTC().UnixNano() / 1e6 + + // approximate equality, Assuming that the program is completed in one second. + if currentTime/1000 != uint64(nowNano)/1000 { + t.Error("The timestamp should be equal") + } + }) + + t.Run("Basic", func(tt *testing.T) { + date := time.Date(2002, 1, 1, 1, 0, 0, 0, time.UTC) + snowflake.SetStartTime(date) + + nowNano := time.Now().UTC().UnixNano() / 1e6 + startNano := date.UTC().UnixNano() / 1e6 + df := nowNano - startNano + + sid := snowflake.ParseID(snowflake.ID()) + + // approximate equality, Assuming that the program is completed in one second. + if sid.Timestamp/1000 != uint64(df)/1000 { + t.Error("The timestamp should be equal") + } + }) +} + +func TestSetMachineID(t *testing.T) { + // first test, + sid := snowflake.ParseID(snowflake.ID()) + if sid.MachineID != 0 { + t.Error("MachineID should be equal 0") + } + + t.Run("No Panic", func(tt *testing.T) { + defer func() { + if err := recover(); err != nil { + t.Error("An error should not be returned") + } + }() + + snowflake.SetMachineID(1) + id := snowflake.ID() + sid := snowflake.ParseID(id) + + if sid.MachineID != 1 { + tt.Error("The machineID should be equal 1") + } + }) + + t.Run("Panic", func(tt *testing.T) { + defer func() { + if err := recover(); err == nil { + tt.Error("Should throw a error") + } else if err.(string) != "The machineid cannot be greater than 1023" { + tt.Error("The error message should be eq 「The machineid cannot be greater than 1023」") + } + }() + + snowflake.SetMachineID(1024) + }) + + snowflake.SetMachineID(100) + sid = snowflake.ParseID(snowflake.ID()) + if sid.MachineID != 100 { + t.Error("MachineID should be equal 100") + } +} + +func TestSetSequenceResolver(t *testing.T) { + snowflake.SetSequenceResolver(func(c int64) (uint16, error) { + return 100, nil + }) + + id := snowflake.ID() + sid := snowflake.ParseID(id) + + if sid.Sequence != 100 { + t.Error("The snowflake number part of sequence should be equal 100") + } + + time.Sleep(time.Millisecond) + + id = snowflake.ID() + sid2 := snowflake.ParseID(id) + + if sid2.Sequence != 100 { + t.Error("The snowflake number part of sequence should be equal 100") + } + + if sid2.Timestamp <= sid.Timestamp { + t.Error("It should be bigger than the previous time") + } +} + +func TestNextID(t *testing.T) { + _, err := snowflake.NextID() + if err != nil { + t.Error(err) + return + } + + snowflake.SetSequenceResolver(func(ms int64) (uint16, error) { + return 0, errors.New("test error") + }) + _, e := snowflake.NextID() + if e == nil { + t.Error("Should be throw error") + } else if e.Error() != "test error" { + t.Error("NextID error message should be equal [test error]") + } +} + +func TestParseID(t *testing.T) { + time := 101 << (snowflake.MachineIDLength + snowflake.SequenceLength) + machineid := 1023 << snowflake.SequenceLength + seq := 999 + + id := uint64(time | machineid | seq) + + d := snowflake.ParseID(id) + if d.Sequence != 999 { + t.Error("Sequence should be equal 999") + } + + if d.MachineID != 1023 { + t.Error("MachineID should be equal 1023") + } + + if d.Timestamp != 101 { + t.Error("Timestamp should be equal 101") + } +} + +func TestSID_GenerateTime(t *testing.T) { + sid := snowflake.ParseID(snowflake.ID()) + + if sid.GenerateTime().UTC().Second() != time.Now().UTC().Second() { + t.Error("The id generate time should be equal") + } +}