2.55

trigger migrated travis build

README.md

@@ -3,7 +3,7 @@
 
 [![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
 [![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs)
-[![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
+[![Build Status](https://travis-ci.com/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.com/chrislusf/seaweedfs)
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
 [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)

go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6
 	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e
-	github.com/golang/protobuf v1.4.2
+	github.com/golang/protobuf v1.4.3
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.4
@@ -42,7 +42,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
 	github.com/jinzhu/copier v0.2.8
-	github.com/json-iterator/go v1.1.10
+	github.com/json-iterator/go v1.1.11
 	github.com/karlseguin/ccache v2.0.3+incompatible // indirect
 	github.com/karlseguin/ccache/v2 v2.0.7
 	github.com/klauspost/compress v1.10.9 // indirect
@@ -59,7 +59,7 @@ require (
 	github.com/peterh/liner v1.1.0
 	github.com/pierrec/lz4 v2.2.7+incompatible // indirect
 	github.com/pquerna/cachecontrol v0.1.0
-	github.com/prometheus/client_golang v1.3.0
+	github.com/prometheus/client_golang v1.11.0
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
 	github.com/seaweedfs/fuse v1.1.7
 	github.com/seaweedfs/goexif v1.0.2
@@ -88,11 +88,11 @@ require (
 	gocloud.dev/pubsub/rabbitpubsub v0.20.0
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
 	golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
-	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
+	golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40
 	golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78
 	google.golang.org/api v0.26.0
 	google.golang.org/grpc v1.29.1
-	google.golang.org/protobuf v1.24.0
+	google.golang.org/protobuf v1.26.0-rc.1
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect

go.sum

@@ -109,6 +109,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
 github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@@ -251,6 +252,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo=
 github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
@@ -333,6 +335,8 @@ github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
@@ -353,6 +357,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-replayers/grpcreplay v0.1.0 h1:eNb1y9rZFmY4ax45uEEECSa8fsxGRU+8Bil52ASAwic=
 github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
 github.com/google/go-replayers/httpreplay v0.1.0 h1:AX7FUb4BjrrzNvblr/OlgwrmFiep6soj5K2QSDW7BGk=
@@ -456,17 +462,21 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU=
 github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w=
 github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM=
@@ -497,6 +507,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGi
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -564,6 +576,7 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt v0.2.6/go.mod h1:mQxQ0uHQ9FhEVPIcTSKwx2lqZEpXWWcCgA7R6NrWvvY=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
 github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI=
@@ -654,6 +667,9 @@ github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc=
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
@@ -662,6 +678,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCb
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE=
 github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
@@ -669,6 +687,9 @@ github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -676,6 +697,9 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
 github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
@@ -724,6 +748,8 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q=
 github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -964,6 +990,7 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U=
@@ -989,6 +1016,8 @@ golang.org/x/sync v0.0.0-20200930132711-30421366ff76 h1:JnxiSYT3Nm0BT2a8CyvYyM6c
 golang.org/x/sync v0.0.0-20200930132711-30421366ff76/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1020,6 +1049,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1038,12 +1068,16 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd h1:WgqgiQvkiZWz7XLhphjt2GI2GcGCTIZs9jqXMWmH+oc=
 golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1217,6 +1251,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1256,6 +1292,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=

k8s/seaweedfs/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.51"
-version: "2.51"
+appVersion: "2.55"
+version: "2.55"

k8s/seaweedfs/values.yaml

@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  # imageTag: "2.51" - started using {.Chart.appVersion}
+  # imageTag: "2.55" - started using {.Chart.appVersion}
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always

other/metrics/grafana_seaweedfs.json

@@ -1805,6 +1805,336 @@
               "show": true
             }
           ]
+        },
+        {
+          "type": "graph",
+          "title": "Process memory",
+          "gridPos": {
+            "x": 0,
+            "y": 40,
+            "w": 24,
+            "h": 7
+          },
+          "aliasColors": {},
+          "dashLength": 10,
+          "datasource": "${DS_PROMETHEUS-DEV}",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "id": 66,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "pluginVersion": "7.4.5",
+          "pointradius": 5,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "$$hashKey": "object:258",
+              "alias": "max limit",
+              "color": "#890f02"
+            }
+          ],
+          "spaceLength": 10,
+          "targets": [
+            {
+              "expr": "process_resident_memory_bytes{exported_job=\"filer\"}",
+              "legendFormat": "resident {{exported_instance}}",
+              "interval": "",
+              "format": "time_series",
+              "groupBy": [
+                {
+                  "params": [
+                    "$__interval"
+                  ],
+                  "type": "time"
+                },
+                {
+                  "params": [
+                    "null"
+                  ],
+                  "type": "fill"
+                }
+              ],
+              "intervalFactor": 2,
+              "orderByTime": "ASC",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "mean"
+                  }
+                ]
+              ],
+              "step": 120,
+              "tags": []
+            },
+            {
+              "expr": "process_virtual_memory_bytes{exported_job=\"filer\"}",
+              "legendFormat": "virtual {{exported_instance}}",
+              "interval": "",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "refId": "B",
+              "step": 120
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "$$hashKey": "object:265",
+              "format": "bytes",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "$$hashKey": "object:266",
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          },
+          "bars": false,
+          "dashes": false,
+          "fillGradient": 0,
+          "hiddenSeries": false,
+          "percentage": false,
+          "points": false,
+          "stack": false,
+          "steppedLine": false,
+          "timeFrom": null,
+          "timeShift": null
+        },
+        {
+          "type": "graph",
+          "title": "Open File Descriptor",
+          "gridPos": {
+            "x": 0,
+            "y": 47,
+            "w": 24,
+            "h": 7
+          },
+          "aliasColors": {},
+          "dashLength": 10,
+          "datasource": "${DS_PROMETHEUS-DEV}",
+          "editable": true,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "id": 68,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "pluginVersion": "7.4.5",
+          "pointradius": 5,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "targets": [
+            {
+              "expr": "process_open_fds{exported_job=\"filer\"}",
+              "legendFormat": "{{exported_instance}}",
+              "interval": "",
+              "format": "time_series",
+              "groupBy": [
+                {
+                  "params": [
+                    "$interval"
+                  ],
+                  "type": "time"
+                },
+                {
+                  "params": [
+                    "null"
+                  ],
+                  "type": "fill"
+                }
+              ],
+              "intervalFactor": 2,
+              "metric": "",
+              "policy": "default",
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "mean"
+                  }
+                ]
+              ],
+              "step": 240,
+              "tags": []
+            },
+            {
+              "expr": "process_max_fds{exported_job=\"filer\"}",
+              "legendFormat": "{{exported_instance}}",
+              "interval": "",
+              "format": "time_series",
+              "groupBy": [
+                {
+                  "params": [
+                    "$interval"
+                  ],
+                  "type": "time"
+                },
+                {
+                  "params": [
+                    "null"
+                  ],
+                  "type": "fill"
+                }
+              ],
+              "intervalFactor": 2,
+              "metric": "",
+              "policy": "default",
+              "refId": "B",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "mean"
+                  }
+                ]
+              ],
+              "step": 240,
+              "tags": [],
+              "hide": false
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "$$hashKey": "object:1087",
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "$$hashKey": "object:1088",
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          },
+          "bars": false,
+          "dashes": false,
+          "error": false,
+          "fillGradient": 0,
+          "hiddenSeries": false,
+          "percentage": false,
+          "points": false,
+          "stack": false,
+          "steppedLine": false,
+          "timeFrom": null,
+          "timeShift": null
         }
       ],
       "repeat": null,

weed/command/filer_copy.go

@@ -213,11 +213,15 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 
 	mode := fi.Mode()
 	uid, gid := util.GetFileUidGid(fi)
+	fileSize := fi.Size()
+	if mode.IsDir() {
+		fileSize = 0
+	}
 
 	fileCopyTaskChan <- FileCopyTask{
 		sourceLocation:     fileOrDir,
 		destinationUrlPath: destPath,
-		fileSize:           fi.Size(),
+		fileSize:           fileSize,
 		fileMode:           fi.Mode(),
 		uid:                uid,
 		gid:                gid,

weed/command/fuse.go

@@ -22,6 +22,7 @@ func runFuse(cmd *Command, args []string) bool {
 	rawArgsLen := len(rawArgs)
 	option := strings.Builder{}
 	options := []parameter{}
+	masterProcess := true
 
 	// first parameter
 	i := 0
@@ -98,6 +99,8 @@ func runFuse(cmd *Command, args []string) bool {
 		parameter := options[i]
 
 		switch parameter.name  {
+		case "child":
+			masterProcess = false
 		case "arg0":
 			mountOptions.dir = &parameter.value
 		case "filer":
@@ -187,6 +190,27 @@ func runFuse(cmd *Command, args []string) bool {
 		}
 	}
 
+	// the master start the child, release it then finish himself
+	if masterProcess {
+		arg0 := os.Args[0]
+		argv := append(os.Args, "-o", "child")
+
+		attr := os.ProcAttr{}
+		child, err := os.StartProcess(arg0, argv, &attr)
+
+		if err != nil {
+			panic(fmt.Errorf("master process can not start child process: %s", err))
+		}
+
+		err = child.Release()
+
+		if err != nil {
+			panic(fmt.Errorf("master process can not release child process: %s", err))
+		}
+
+		return true
+	}
+
 	// I don't know why PATH environment variable is lost
 	if err := os.Setenv("PATH", "/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"); err != nil {
 		panic(fmt.Errorf("setenv: %s", err))

weed/command/upload.go

@@ -84,7 +84,7 @@ func runUpload(cmd *Command, args []string) bool {
 		if *upload.dir == "" {
 			return false
 		}
-		filepath.Walk(util.ResolvePath(*upload.dir), func(path string, info os.FileInfo, err error) error {
+		err = filepath.Walk(util.ResolvePath(*upload.dir), func(path string, info os.FileInfo, err error) error {
 			if err == nil {
 				if !info.IsDir() {
 					if *upload.include != "" {
@@ -108,12 +108,21 @@ func runUpload(cmd *Command, args []string) bool {
 			}
 			return err
 		})
+		if err != nil {
+			fmt.Println(err.Error())
+			return false;
+		}
 	} else {
 		parts, e := operation.NewFileParts(args)
 		if e != nil {
 			fmt.Println(e.Error())
+			return false
+		}
+		results, err := operation.SubmitFiles(func() string { return *upload.master }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl)
+		if err != nil {
+			fmt.Println(err.Error())
+			return false
 		}
-		results, _ := operation.SubmitFiles(func() string { return *upload.master }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl)
 		bytes, _ := json.Marshal(results)
 		fmt.Println(string(bytes))
 	}

weed/filer/abstract_sql/abstract_sql_store.go

@@ -10,6 +10,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"strings"
 	"sync"
+	"time"
 )
 
 type SqlGenerator interface {
@@ -32,6 +33,9 @@ type AbstractSqlStore struct {
 	dbsLock            sync.Mutex
 }
 
+func (store *AbstractSqlStore) CanDropWholeBucket() bool {
+	return store.SupportBucketTable
+}
 func (store *AbstractSqlStore) OnBucketCreation(bucket string) {
 	store.dbsLock.Lock()
 	defer store.dbsLock.Unlock()
@@ -258,7 +262,7 @@ func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.Fu
 	return nil
 }
 
-func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
+func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) error {
 
 	db, bucket, shortPath, err := store.getTxOrDB(ctx, fullpath, true)
 	if err != nil {
@@ -276,18 +280,23 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
 		}
 	}
 
-	glog.V(4).Infof("delete %s SQL %s %d", string(shortPath), store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)))
+	for {
+		glog.V(4).Infof("delete %s SQL %s %d", string(shortPath), store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)))
+		res, err := db.ExecContext(ctx, store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)), string(shortPath), limit)
+		if err != nil {
+			return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err)
+		}
 
-	res, err := db.ExecContext(ctx, store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)), string(shortPath))
-	if err != nil {
-		return fmt.Errorf("deleteFolderChildren %s: %s", fullpath, err)
+		rowCount, err := res.RowsAffected()
+		if err != nil {
+			return fmt.Errorf("deleteFolderChildren %s but no rows affected: %s", fullpath, err)
+		}
+		if rowCount < limit {
+			break
+		}
+		// to give the Galera Cluster a chance to breath
+		time.Sleep(time.Second)
 	}
-
-	_, err = res.RowsAffected()
-	if err != nil {
-		return fmt.Errorf("deleteFolderChildren %s but no rows affected: %s", fullpath, err)
-	}
-
 	return nil
 }
 

weed/filer/cassandra/cassandra_store.go

@@ -154,7 +154,7 @@ func (store *CassandraStore) DeleteEntry(ctx context.Context, fullpath util.Full
 	return nil
 }
 
-func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
+func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) error {
 	if _, ok := store.isSuperLargeDirectory(string(fullpath)); ok {
 		return nil // filer.ErrUnsupportedSuperLargeDirectoryListing
 	}

weed/filer/elastic/v7/elastic_store.go

@@ -186,7 +186,7 @@ func (store *ElasticStore) deleteEntry(ctx context.Context, index, id string) (e
 	return fmt.Errorf("delete entry %v.", err)
 }
 
-func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
 	_, err = store.ListDirectoryEntries(ctx, fullpath, "", false, math.MaxInt32, func(entry *filer.Entry) bool {
 		if err := store.DeleteEntry(ctx, entry.FullPath); err != nil {
 			glog.Errorf("elastic delete %s: %v.", entry.FullPath, err)

weed/filer/etcd/etcd_store.go

@@ -130,7 +130,7 @@ func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.Full
 	return nil
 }
 
-func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
 	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
 
 	if _, err := store.client.Delete(ctx, string(directoryPrefix), clientv3.WithPrefix()); err != nil {

weed/filer/filer.go

@@ -19,6 +19,7 @@ import (
 const (
 	LogFlushInterval = time.Minute
 	PaginationSize   = 1024
+	DeleteMaxRows    = 10000
 	FilerStoreId     = "filer.store.id"
 )
 

weed/filer/filer_delete_entry.go

@@ -71,7 +71,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 
 	lastFileName := ""
 	includeLastFile := false
-	if !isDeletingBucket {
+	if !isDeletingBucket || !f.Store.CanDropWholeBucket() {
 		for {
 			entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "")
 			if err != nil {
@@ -115,7 +115,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
 
 	glog.V(3).Infof("deleting directory %v delete %d chunks: %v", entry.FullPath, len(chunks), shouldDeleteChunks)
 
-	if storeDeletionErr := f.Store.DeleteFolderChildren(ctx, entry.FullPath); storeDeletionErr != nil {
+	if storeDeletionErr := f.Store.DeleteFolderChildren(ctx, entry.FullPath, DeleteMaxRows); storeDeletionErr != nil {
 		return nil, nil, fmt.Errorf("filer store delete: %v", storeDeletionErr)
 	}
 

weed/filer/filerstore.go

@@ -25,7 +25,7 @@ type FilerStore interface {
 	// err == filer_pb.ErrNotFound if not found
 	FindEntry(context.Context, util.FullPath) (entry *Entry, err error)
 	DeleteEntry(context.Context, util.FullPath) (err error)
-	DeleteFolderChildren(context.Context, util.FullPath) (err error)
+	DeleteFolderChildren(context.Context, util.FullPath, int64) (err error)
 	ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
 	ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
 
@@ -43,4 +43,5 @@ type FilerStore interface {
 type BucketAware interface {
 	OnBucketCreation(bucket string)
 	OnBucketDeletion(bucket string)
+	CanDropWholeBucket() bool
 }

weed/filer/filerstore_translate_path.go

@@ -100,10 +100,10 @@ func (t *FilerStorePathTranlator) DeleteOneEntry(ctx context.Context, existingEn
 	return t.actualStore.DeleteEntry(ctx, existingEntry.FullPath)
 }
 
-func (t *FilerStorePathTranlator) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
+func (t *FilerStorePathTranlator) DeleteFolderChildren(ctx context.Context, fp util.FullPath, limit int64) (err error) {
 	newFullPath := t.translatePath(fp)
 
-	return t.actualStore.DeleteFolderChildren(ctx, newFullPath)
+	return t.actualStore.DeleteFolderChildren(ctx, newFullPath, limit)
 }
 
 func (t *FilerStorePathTranlator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {

weed/filer/filerstore_wrapper.go

@@ -23,6 +23,7 @@ type VirtualFilerStore interface {
 	AddPathSpecificStore(path string, storeId string, store FilerStore)
 	OnBucketCreation(bucket string)
 	OnBucketDeletion(bucket string)
+	CanDropWholeBucket() bool
 }
 
 type FilerStoreWrapper struct {
@@ -42,6 +43,13 @@ func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
 	}
 }
 
+func (fsw *FilerStoreWrapper) CanDropWholeBucket() bool {
+	if ba, ok := fsw.defaultStore.(BucketAware); ok {
+		return ba.CanDropWholeBucket()
+	}
+	return false
+}
+
 func (fsw *FilerStoreWrapper) OnBucketCreation(bucket string) {
 	for _, store := range fsw.storeIdToStore {
 		if ba, ok := store.(BucketAware); ok {
@@ -205,7 +213,7 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
 	return actualStore.DeleteEntry(ctx, existingEntry.FullPath)
 }
 
-func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
+func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath, limit int64) (err error) {
 	actualStore := fsw.getActualStore(fp + "/")
 	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Inc()
 	start := time.Now()
@@ -214,7 +222,7 @@ func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.
 	}()
 
 	glog.V(4).Infof("DeleteFolderChildren %s", fp)
-	return actualStore.DeleteFolderChildren(ctx, fp)
+	return actualStore.DeleteFolderChildren(ctx, fp, limit)
 }
 
 func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {

weed/filer/hbase/hbase_store.go

@@ -109,7 +109,7 @@ func (store *HbaseStore) DeleteEntry(ctx context.Context, path util.FullPath) (e
 	return store.doDelete(ctx, store.cfMetaDir, []byte(path))
 }
 
-func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) (err error) {
+func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath, limit int64) (err error) {
 
 	family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}}
 	expectedPrefix := []byte(path.Child(""))

weed/filer/leveldb/leveldb_store.go

@@ -136,7 +136,7 @@ func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.F
 	return nil
 }
 
-func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
 
 	batch := new(leveldb.Batch)
 

weed/filer/leveldb2/leveldb2_store.go

@@ -144,7 +144,7 @@ func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath weed_util.
 	return nil
 }
 
-func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+func (store *LevelDB2Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
 	directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, "", store.dbCount)
 
 	batch := new(leveldb.Batch)

weed/filer/leveldb3/leveldb3_store.go

@@ -245,7 +245,7 @@ func (store *LevelDB3Store) DeleteEntry(ctx context.Context, fullpath weed_util.
 	return nil
 }
 
-func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+func (store *LevelDB3Store) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
 
 	db, bucket, shortPath, err := store.findDB(fullpath, true)
 	if err != nil {

weed/filer/mongodb/mongodb_store.go

@@ -167,7 +167,7 @@ func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPa
 	return nil
 }
 
-func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
+func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) error {
 
 	where := bson.M{"directory": fullpath}
 	_, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteMany(ctx, where)

weed/filer/mysql/mysql_sql_gen.go

@@ -38,7 +38,7 @@ func (gen *SqlGenMysql) GetSqlDelete(tableName string) string {
 }
 
 func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(tableName string) string {
-	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=?", tableName)
+	return fmt.Sprintf("DELETE FROM `%s` WHERE dirhash=? AND directory=? LIMIT ?", tableName)
 }
 
 func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {

weed/filer/redis/universal_redis_store.go

@@ -107,7 +107,7 @@ func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath util
 	return nil
 }
 
-func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) (err error) {
 
 	members, err := store.Client.SMembers(ctx, genDirectoryListKey(string(fullpath))).Result()
 	if err != nil {

weed/filer/redis2/universal_redis_store.go

@@ -127,7 +127,7 @@ func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath uti
 	return nil
 }
 
-func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath, limit int64) (err error) {
 
 	if store.isSuperLargeDirectory(string(fullpath)) {
 		return nil

weed/filer/rocksdb/rocksdb_store.go

@@ -148,7 +148,7 @@ func (store *RocksDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.F
 	return nil
 }
 
-func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+func (store *RocksDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath, limit int64) (err error) {
 	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
 
 	batch := gorocksdb.NewWriteBatch()

weed/filesys/dir.go

@@ -443,7 +443,10 @@ func (dir *Dir) removeOneFile(req *fuse.RemoveRequest) error {
 	dir.wfs.handlesLock.Lock()
 	defer dir.wfs.handlesLock.Unlock()
 	inodeId := filePath.AsInode()
-	delete(dir.wfs.handles, inodeId)
+	if fh, ok := dir.wfs.handles[inodeId]; ok {
+		delete(dir.wfs.handles, inodeId)
+		fh.isDeleted = true
+	}
 
 	return nil
 

weed/filesys/filehandle.go

@@ -33,11 +33,12 @@ type FileHandle struct {
 	Uid       uint32         // user ID of process making request
 	Gid       uint32         // group ID of process making request
 	writeOnly bool
+	isDeleted bool
 }
 
 func newFileHandle(file *File, uid, gid uint32, writeOnly bool) *FileHandle {
 	fh := &FileHandle{
-		f:          file,
+		f: file,
 		// dirtyPages: newContinuousDirtyPages(file, writeOnly),
 		dirtyPages: newTempFileDirtyPages(file, writeOnly),
 		Uid:        uid,
@@ -222,6 +223,11 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 
 	glog.V(4).Infof("Flush %v fh %d", fh.f.fullpath(), fh.handle)
 
+	if fh.isDeleted {
+		glog.V(4).Infof("Flush %v fh %d skip deleted", fh.f.fullpath(), fh.handle)
+		return nil
+	}
+
 	fh.Lock()
 	defer fh.Unlock()
 

weed/iamapi/iamapi_handlers.go

@@ -1,55 +1,13 @@
 package iamapi
 
 import (
-	"bytes"
-	"encoding/xml"
 	"fmt"
-	"strconv"
-
-	"net/http"
-	"net/url"
-	"time"
-
+	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
-
-	"github.com/aws/aws-sdk-go/service/iam"
+	"net/http"
 )
 
-type mimeType string
-
-const (
-	mimeNone mimeType = ""
-	mimeXML  mimeType = "application/xml"
-)
-
-func setCommonHeaders(w http.ResponseWriter) {
-	w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano()))
-	w.Header().Set("Accept-Ranges", "bytes")
-}
-
-// Encodes the response headers into XML format.
-func encodeResponse(response interface{}) []byte {
-	var bytesBuffer bytes.Buffer
-	bytesBuffer.WriteString(xml.Header)
-	e := xml.NewEncoder(&bytesBuffer)
-	e.Encode(response)
-	return bytesBuffer.Bytes()
-}
-
-// If none of the http routes match respond with MethodNotAllowed
-func notFoundHandler(w http.ResponseWriter, r *http.Request) {
-	glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI)
-	writeErrorResponse(w, s3err.ErrMethodNotAllowed, r.URL)
-}
-
-func writeErrorResponse(w http.ResponseWriter, errorCode s3err.ErrorCode, reqURL *url.URL) {
-	apiError := s3err.GetAPIError(errorCode)
-	errorResponse := getRESTErrorResponse(apiError, reqURL.Path)
-	encodedErrorResponse := encodeResponse(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
-}
-
 func writeIamErrorResponse(w http.ResponseWriter, err error, object string, value string, msg error) {
 	errCode := err.Error()
 	errorResp := ErrorResponse{}
@@ -64,42 +22,10 @@ func writeIamErrorResponse(w http.ResponseWriter, err error, object string, valu
 	case iam.ErrCodeNoSuchEntityException:
 		msg := fmt.Sprintf("The %s with name %s cannot be found.", object, value)
 		errorResp.Error.Message = &msg
-		writeResponse(w, http.StatusNotFound, encodeResponse(errorResp), mimeXML)
+		s3err.WriteXMLResponse(w, http.StatusNotFound, errorResp)
 	case iam.ErrCodeServiceFailureException:
-		writeResponse(w, http.StatusInternalServerError, encodeResponse(errorResp), mimeXML)
+		s3err.WriteXMLResponse(w, http.StatusInternalServerError, errorResp)
 	default:
-		writeResponse(w, http.StatusInternalServerError, encodeResponse(errorResp), mimeXML)
+		s3err.WriteXMLResponse(w, http.StatusInternalServerError, errorResp)
 	}
 }
-
-func getRESTErrorResponse(err s3err.APIError, resource string) s3err.RESTErrorResponse {
-	return s3err.RESTErrorResponse{
-		Code:      err.Code,
-		Message:   err.Description,
-		Resource:  resource,
-		RequestID: fmt.Sprintf("%d", time.Now().UnixNano()),
-	}
-}
-
-func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
-	setCommonHeaders(w)
-	if response != nil {
-		w.Header().Set("Content-Length", strconv.Itoa(len(response)))
-	}
-	if mType != mimeNone {
-		w.Header().Set("Content-Type", string(mType))
-	}
-	w.WriteHeader(statusCode)
-	if response != nil {
-		glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response))
-		_, err := w.Write(response)
-		if err != nil {
-			glog.V(0).Infof("write err: %v", err)
-		}
-		w.(http.Flusher).Flush()
-	}
-}
-
-func writeSuccessResponseXML(w http.ResponseWriter, response []byte) {
-	writeResponse(w, http.StatusOK, response, mimeXML)
-}

weed/iamapi/iamapi_management_handlers.go

@@ -362,7 +362,7 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu
 
 func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	if err := r.ParseForm(); err != nil {
-		writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r)
 		return
 	}
 	values := r.PostForm
@@ -370,7 +370,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 	s3cfgLock.RLock()
 	s3cfg := &iam_pb.S3ApiConfiguration{}
 	if err := iama.s3ApiConfig.GetS3ApiConfiguration(s3cfg); err != nil {
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 	s3cfgLock.RUnlock()
@@ -411,14 +411,14 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		response, err = iama.CreatePolicy(s3cfg, values)
 		if err != nil {
 			glog.Errorf("CreatePolicy:  %+v", err)
-			writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r)
 			return
 		}
 	case "PutUserPolicy":
 		response, err = iama.PutUserPolicy(s3cfg, values)
 		if err != nil {
 			glog.Errorf("PutUserPolicy:  %+v", err)
-			writeErrorResponse(w, s3err.ErrInvalidRequest, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInvalidRequest, r)
 			return
 		}
 	case "GetUserPolicy":
@@ -437,7 +437,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 		errorResponse := ErrorResponse{}
 		errorResponse.Error.Code = &errNotImplemented.Code
 		errorResponse.Error.Message = &errNotImplemented.Description
-		writeResponse(w, errNotImplemented.HTTPStatusCode, encodeResponse(errorResponse), mimeXML)
+		s3err.WriteXMLResponse(w, errNotImplemented.HTTPStatusCode, errorResponse)
 		return
 	}
 	if changed {
@@ -449,5 +449,5 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	writeSuccessResponseXML(w, encodeResponse(response))
+	s3err.WriteXMLResponse(w, http.StatusOK, response)
 }

weed/iamapi/iamapi_server.go

@@ -12,6 +12,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
 	"github.com/chrislusf/seaweedfs/weed/s3api"
 	. "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"github.com/gorilla/mux"
 	"google.golang.org/grpc"
@@ -71,7 +72,7 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) {
 	apiRouter.Methods("POST").Path("/").HandlerFunc(iama.iam.Auth(iama.DoActions, ACTION_ADMIN))
 	//
 	// NotFound
-	apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler)
+	apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler)
 }
 
 func (iam IamS3ApiConfigure) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) (err error) {

weed/operation/delete_content.go

@@ -100,7 +100,7 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str
 		go func(server string, fidList []string) {
 			defer wg.Done()
 
-			if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, grpcDialOption, fidList, true); deleteErr != nil {
+			if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, grpcDialOption, fidList, false); deleteErr != nil {
 				err = deleteErr
 			} else if deleteResults != nil {
 				resultChan <- deleteResults

weed/s3api/auth_credentials.go

@@ -150,7 +150,7 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
 			f(w, r)
 			return
 		}
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 	}
 }
 

weed/s3api/filer_multipart_test.go

@@ -3,6 +3,7 @@ package s3api
 import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"testing"
 	"time"
 )
@@ -19,7 +20,7 @@ func TestInitiateMultipartUploadResult(t *testing.T) {
 		},
 	}
 
-	encoded := string(encodeResponse(response))
+	encoded := string(s3err.EncodeXMLResponse(response))
 	if encoded != expected {
 		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
 	}
@@ -41,7 +42,7 @@ func TestListPartsResult(t *testing.T) {
 		},
 	}
 
-	encoded := string(encodeResponse(response))
+	encoded := string(s3err.EncodeXMLResponse(response))
 	if encoded != expected {
 		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
 	}

weed/s3api/s3api_bucket_handlers.go

@@ -32,7 +32,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 	if s3a.iam.isEnabled() {
 		identity, s3Err = s3a.iam.authUser(r)
 		if s3Err != s3err.ErrNone {
-			writeErrorResponse(w, s3Err, r.URL)
+			s3err.WriteErrorResponse(w, s3Err, r)
 			return
 		}
 	}
@@ -42,7 +42,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 	entries, _, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)
 
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 
@@ -69,7 +69,7 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques
 		Buckets: buckets,
 	}
 
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, response)
 }
 
 func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -95,14 +95,14 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 		}
 		return nil
 	}); err != nil {
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 	if exist, err := s3a.exists(s3a.option.BucketsPath, bucket, true); err == nil && exist {
 		errCode = s3err.ErrBucketAlreadyExists
 	}
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 
@@ -118,7 +118,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
 	// create the folder for bucket, but lazily create actual collection
 	if err := s3a.mkdir(s3a.option.BucketsPath, bucket, fn); err != nil {
 		glog.Errorf("PutBucketHandler mkdir: %v", err)
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 
@@ -130,7 +130,7 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
 	bucket, _ := getBucketAndObject(r)
 
 	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
-		writeErrorResponse(w, err, r.URL)
+		s3err.WriteErrorResponse(w, err, r)
 		return
 	}
 
@@ -152,11 +152,11 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
 	err = s3a.rm(s3a.option.BucketsPath, bucket, false, true)
 
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 
-	writeResponse(w, http.StatusNoContent, nil, mimeNone)
+	s3err.WriteEmptyResponse(w, http.StatusNoContent)
 }
 
 func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
@@ -164,7 +164,7 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
 	bucket, _ := getBucketAndObject(r)
 
 	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
-		writeErrorResponse(w, err, r.URL)
+		s3err.WriteErrorResponse(w, err, r)
 		return
 	}
 

weed/s3api/s3api_bucket_handlers_test.go

@@ -1,6 +1,7 @@
 package s3api
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"testing"
 	"time"
 
@@ -32,7 +33,7 @@ func TestListBucketsHandler(t *testing.T) {
 		Buckets: buckets,
 	}
 
-	encoded := string(encodeResponse(response))
+	encoded := string(s3err.EncodeXMLResponse(response))
 	if encoded != expected {
 		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
 	}

weed/s3api/s3api_handlers.go

@@ -1,45 +1,16 @@
 package s3api
 
 import (
-	"bytes"
 	"encoding/base64"
-	"encoding/xml"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
-	"net/http"
-	"net/url"
-	"strconv"
-	"time"
-
 	"google.golang.org/grpc"
+	"net/http"
 
-	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
-type mimeType string
-
-const (
-	mimeNone mimeType = ""
-	mimeJSON mimeType = "application/json"
-	mimeXML  mimeType = "application/xml"
-)
-
-func setCommonHeaders(w http.ResponseWriter) {
-	w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano()))
-	w.Header().Set("Accept-Ranges", "bytes")
-}
-
-// Encodes the response headers into XML format.
-func encodeResponse(response interface{}) []byte {
-	var bytesBuffer bytes.Buffer
-	bytesBuffer.WriteString(xml.Header)
-	e := xml.NewEncoder(&bytesBuffer)
-	e.Encode(response)
-	return bytesBuffer.Bytes()
-}
-
 var _ = filer_pb.FilerClient(&S3ApiServer{})
 
 func (s3a *S3ApiServer) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
@@ -54,53 +25,12 @@ func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string {
 	return location.Url
 }
 
-// If none of the http routes match respond with MethodNotAllowed
-func notFoundHandler(w http.ResponseWriter, r *http.Request) {
-	glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI)
-	writeErrorResponse(w, s3err.ErrMethodNotAllowed, r.URL)
-}
-
-func writeErrorResponse(w http.ResponseWriter, errorCode s3err.ErrorCode, reqURL *url.URL) {
-	apiError := s3err.GetAPIError(errorCode)
-	errorResponse := getRESTErrorResponse(apiError, reqURL.Path)
-	encodedErrorResponse := encodeResponse(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
-}
-
-func getRESTErrorResponse(err s3err.APIError, resource string) s3err.RESTErrorResponse {
-	return s3err.RESTErrorResponse{
-		Code:      err.Code,
-		Message:   err.Description,
-		Resource:  resource,
-		RequestID: fmt.Sprintf("%d", time.Now().UnixNano()),
-	}
-}
-
-func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
-	setCommonHeaders(w)
-	if response != nil {
-		w.Header().Set("Content-Length", strconv.Itoa(len(response)))
-	}
-	if mType != mimeNone {
-		w.Header().Set("Content-Type", string(mType))
-	}
-	w.WriteHeader(statusCode)
-	if response != nil {
-		glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response))
-		_, err := w.Write(response)
-		if err != nil {
-			glog.V(0).Infof("write err: %v", err)
-		}
-		w.(http.Flusher).Flush()
-	}
-}
-
-func writeSuccessResponseXML(w http.ResponseWriter, response []byte) {
-	writeResponse(w, http.StatusOK, response, mimeXML)
+func writeSuccessResponseXML(w http.ResponseWriter, response interface{}) {
+	s3err.WriteXMLResponse(w, http.StatusOK, response)
 }
 
 func writeSuccessResponseEmpty(w http.ResponseWriter) {
-	writeResponse(w, http.StatusOK, nil, mimeNone)
+	s3err.WriteEmptyResponse(w, http.StatusOK)
 }
 
 func validateContentMd5(h http.Header) ([]byte, error) {

weed/s3api/s3api_object_copy_handlers.go

@@ -32,28 +32,28 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 		dir, name := fullPath.DirAndName()
 		entry, err := s3a.getEntry(dir, name)
 		if err != nil {
-			writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
 		}
 		entry.Extended = weed_server.SaveAmzMetaData(r, entry.Extended, isReplace(r))
 		err = s3a.touch(dir, name, entry)
 		if err != nil {
-			writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
 		}
-		writeSuccessResponseXML(w, encodeResponse(CopyObjectResult{
+		writeSuccessResponseXML(w, CopyObjectResult{
 			ETag:         fmt.Sprintf("%x", entry.Attributes.Md5),
 			LastModified: time.Now().UTC(),
-		}))
+		})
 		return
 	}
 
 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
-		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
 		return
 	}
 
 	if srcBucket == dstBucket && srcObject == dstObject {
-		writeErrorResponse(w, s3err.ErrInvalidCopyDest, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopyDest, r)
 		return
 	}
 
@@ -64,7 +64,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 
 	_, _, resp, err := util.DownloadFile(srcUrl)
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
 		return
 	}
 	defer util.CloseResponse(resp)
@@ -73,7 +73,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 	etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body)
 
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 
@@ -84,7 +84,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 		LastModified: time.Now().UTC(),
 	}
 
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, response)
 
 }
 
@@ -117,7 +117,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 	srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
-		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
 		return
 	}
 
@@ -126,13 +126,13 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 
 	partID, err := strconv.Atoi(partIDString)
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrInvalidPart, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r)
 		return
 	}
 
 	// check partID with maximum part ID for multipart objects
 	if partID > globalMaxPartID {
-		writeErrorResponse(w, s3err.ErrInvalidMaxParts, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
 		return
 	}
 
@@ -145,7 +145,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 
 	dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, rangeHeader)
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrInvalidCopySource, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidCopySource, r)
 		return
 	}
 	defer dataReader.Close()
@@ -154,7 +154,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 	etag, errCode := s3a.putToFiler(r, dstUrl, dataReader)
 
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 
@@ -165,7 +165,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
 		LastModified: time.Now().UTC(),
 	}
 
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, response)
 
 }
 

weed/s3api/s3api_object_handlers.go

@@ -44,20 +44,20 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 
 	_, err := validateContentMd5(r.Header)
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrInvalidDigest, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r)
 		return
 	}
 
 	if r.Header.Get("Cache-Control") != "" {
 		if _, err = cacheobject.ParseRequestCacheControl(r.Header.Get("Cache-Control")); err != nil {
-			writeErrorResponse(w, s3err.ErrInvalidDigest, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r)
 			return
 		}
 	}
 
 	if r.Header.Get("Expires") != "" {
 		if _, err = time.Parse(http.TimeFormat, r.Header.Get("Expires")); err != nil {
-			writeErrorResponse(w, s3err.ErrInvalidDigest, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInvalidDigest, r)
 			return
 		}
 	}
@@ -75,12 +75,12 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
 		}
 		if s3ErrCode != s3err.ErrNone {
-			writeErrorResponse(w, s3ErrCode, r.URL)
+			s3err.WriteErrorResponse(w, s3ErrCode, r)
 			return
 		}
 	} else {
 		if authTypeStreamingSigned == rAuthType {
-			writeErrorResponse(w, s3err.ErrAuthNotSetup, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrAuthNotSetup, r)
 			return
 		}
 	}
@@ -88,7 +88,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 
 	if strings.HasSuffix(object, "/") {
 		if err := s3a.mkdir(s3a.option.BucketsPath, bucket+object, nil); err != nil {
-			writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 			return
 		}
 	} else {
@@ -97,7 +97,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 		etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
 
 		if errCode != s3err.ErrNone {
-			writeErrorResponse(w, errCode, r.URL)
+			s3err.WriteErrorResponse(w, errCode, r)
 			return
 		}
 
@@ -120,7 +120,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 	bucket, object := getBucketAndObject(r)
 
 	if strings.HasSuffix(r.URL.Path, "/") {
-		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
 		return
 	}
 
@@ -195,13 +195,13 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 
 	deleteXMLBytes, err := ioutil.ReadAll(r.Body)
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 
 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
-		writeErrorResponse(w, s3err.ErrMalformedXML, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrMalformedXML, r)
 		return
 	}
 
@@ -253,7 +253,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 	}
 	deleteResp.Errors = deleteErrors
 
-	writeSuccessResponseXML(w, encodeResponse(deleteResp))
+	writeSuccessResponseXML(w, deleteResp)
 
 }
 
@@ -297,7 +297,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 
 	if err != nil {
 		glog.Errorf("NewRequest %s: %v", destUrl, err)
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 
@@ -327,19 +327,19 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 
 	if postErr != nil {
 		glog.Errorf("post to filer: %v", postErr)
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 	defer util.CloseResponse(resp)
 
 	if resp.StatusCode == http.StatusPreconditionFailed {
-		writeErrorResponse(w, s3err.ErrPreconditionFailed, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrPreconditionFailed, r)
 		return
 	}
 
 	if (resp.ContentLength == -1 || resp.StatusCode == 404) && resp.StatusCode != 304 {
 		if r.Method != "DELETE" {
-			writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
 			return
 		}
 	}

weed/s3api/s3api_object_handlers_postpolicy.go

@@ -26,23 +26,23 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 
 	reader, err := r.MultipartReader()
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
 		return
 	}
 	form, err := reader.ReadForm(int64(5 * humanize.MiByte))
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
 		return
 	}
 	defer form.RemoveAll()
 
 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(form)
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
 		return
 	}
 	if fileBody == nil {
-		writeErrorResponse(w, s3err.ErrPOSTFileRequired, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrPOSTFileRequired, r)
 		return
 	}
 	defer fileBody.Close()
@@ -60,7 +60,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 	if successRedirect != "" {
 		redirectURL, err = url.Parse(successRedirect)
 		if err != nil {
-			writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
 			return
 		}
 	}
@@ -68,13 +68,13 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 	// Verify policy signature.
 	errCode := s3a.iam.doesPolicySignatureMatch(formValues)
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 
 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrMalformedPOSTRequest, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrMalformedPOSTRequest, r)
 		return
 	}
 
@@ -83,7 +83,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 
 		postPolicyForm, err := policy.ParsePostPolicyForm(string(policyBytes))
 		if err != nil {
-			writeErrorResponse(w, s3err.ErrPostPolicyConditionInvalidFormat, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrPostPolicyConditionInvalidFormat, r)
 			return
 		}
 
@@ -99,12 +99,12 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 		lengthRange := postPolicyForm.Conditions.ContentLengthRange
 		if lengthRange.Valid {
 			if fileSize < lengthRange.Min {
-				writeErrorResponse(w, s3err.ErrEntityTooSmall, r.URL)
+				s3err.WriteErrorResponse(w, s3err.ErrEntityTooSmall, r)
 				return
 			}
 
 			if fileSize > lengthRange.Max {
-				writeErrorResponse(w, s3err.ErrEntityTooLarge, r.URL)
+				s3err.WriteErrorResponse(w, s3err.ErrEntityTooLarge, r)
 				return
 			}
 		}
@@ -115,7 +115,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 	etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody)
 
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 
@@ -123,7 +123,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 		// Replace raw query params..
 		redirectURL.RawQuery = getRedirectPostRawQuery(bucket, object, etag)
 		w.Header().Set("Location", redirectURL.String())
-		writeResponse(w, http.StatusSeeOther, nil, mimeNone)
+		s3err.WriteEmptyResponse(w, http.StatusSeeOther)
 		return
 	}
 
@@ -132,15 +132,15 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 	// Decide what http response to send depending on success_action_status parameter
 	switch successStatus {
 	case "201":
-		resp := encodeResponse(PostResponse{
+		resp := PostResponse{
 			Bucket:   bucket,
 			Key:      object,
 			ETag:     `"` + etag + `"`,
 			Location: w.Header().Get("Location"),
-		})
-		writeResponse(w, http.StatusCreated, resp, mimeXML)
+		}
+		s3err.WriteXMLResponse(w, http.StatusCreated, resp)
 	case "200":
-		writeResponse(w, http.StatusOK, nil, mimeNone)
+		s3err.WriteEmptyResponse(w, http.StatusOK)
 	default:
 		writeSuccessResponseEmpty(w)
 	}

weed/s3api/s3api_object_multipart_handlers.go

@@ -29,14 +29,14 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
 		Key:    objectKey(aws.String(object)),
 	})
 
-	glog.V(2).Info("NewMultipartUploadHandler", string(encodeResponse(response)), errCode)
+	glog.V(2).Info("NewMultipartUploadHandler", s3err.EncodeXMLResponse(response), errCode)
 
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, response)
 
 }
 
@@ -53,14 +53,14 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
 		UploadId: aws.String(uploadID),
 	})
 
-	glog.V(2).Info("CompleteMultipartUploadHandler", string(encodeResponse(response)), errCode)
+	glog.V(2).Info("CompleteMultipartUploadHandler", s3err.EncodeXMLResponse(response), errCode)
 
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, response)
 
 }
 
@@ -78,13 +78,13 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht
 	})
 
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 
-	glog.V(2).Info("AbortMultipartUploadHandler", string(encodeResponse(response)))
+	glog.V(2).Info("AbortMultipartUploadHandler", s3err.EncodeXMLResponse(response))
 
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, response)
 
 }
 
@@ -94,13 +94,13 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht
 
 	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query())
 	if maxUploads < 0 {
-		writeErrorResponse(w, s3err.ErrInvalidMaxUploads, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxUploads, r)
 		return
 	}
 	if keyMarker != "" {
 		// Marker not common with prefix is not implemented.
 		if !strings.HasPrefix(keyMarker, prefix) {
-			writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
 			return
 		}
 	}
@@ -115,16 +115,16 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht
 		UploadIdMarker: aws.String(uploadIDMarker),
 	})
 
-	glog.V(2).Info("ListMultipartUploadsHandler", string(encodeResponse(response)), errCode)
+	glog.V(2).Info("ListMultipartUploadsHandler", s3err.EncodeXMLResponse(response), errCode)
 
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 
 	// TODO handle encodingType
 
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, response)
 }
 
 // ListObjectPartsHandler - Lists object parts in a multipart upload.
@@ -133,11 +133,11 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
 
 	uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query())
 	if partNumberMarker < 0 {
-		writeErrorResponse(w, s3err.ErrInvalidPartNumberMarker, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidPartNumberMarker, r)
 		return
 	}
 	if maxParts < 0 {
-		writeErrorResponse(w, s3err.ErrInvalidMaxParts, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
 		return
 	}
 
@@ -149,14 +149,14 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
 		UploadId:         aws.String(uploadID),
 	})
 
-	glog.V(2).Info("ListObjectPartsHandler", string(encodeResponse(response)), errCode)
+	glog.V(2).Info("ListObjectPartsHandler", s3err.EncodeXMLResponse(response), errCode)
 
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, response)
 
 }
 
@@ -167,18 +167,18 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
 	uploadID := r.URL.Query().Get("uploadId")
 	exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
 	if !exists {
-		writeErrorResponse(w, s3err.ErrNoSuchUpload, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrNoSuchUpload, r)
 		return
 	}
 
 	partIDString := r.URL.Query().Get("partNumber")
 	partID, err := strconv.Atoi(partIDString)
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrInvalidPart, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidPart, r)
 		return
 	}
 	if partID > globalMaxPartID {
-		writeErrorResponse(w, s3err.ErrInvalidMaxParts, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxParts, r)
 		return
 	}
 
@@ -195,7 +195,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
 			_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
 		}
 		if s3ErrCode != s3err.ErrNone {
-			writeErrorResponse(w, s3ErrCode, r.URL)
+			s3err.WriteErrorResponse(w, s3ErrCode, r)
 			return
 		}
 	}
@@ -207,7 +207,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
 	etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader)
 
 	if errCode != s3err.ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		s3err.WriteErrorResponse(w, errCode, r)
 		return
 	}
 

weed/s3api/s3api_object_tagging_handlers.go

@@ -25,15 +25,15 @@ func (s3a *S3ApiServer) GetObjectTaggingHandler(w http.ResponseWriter, r *http.R
 	if err != nil {
 		if err == filer_pb.ErrNotFound {
 			glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err)
-			writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
 		} else {
 			glog.Errorf("GetObjectTaggingHandler %s: %v", r.URL, err)
-			writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		}
 		return
 	}
 
-	writeSuccessResponseXML(w, encodeResponse(FromTags(tags)))
+	writeSuccessResponseXML(w, FromTags(tags))
 
 }
 
@@ -50,29 +50,29 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R
 	input, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		glog.Errorf("PutObjectTaggingHandler read input %s: %v", r.URL, err)
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 	if err = xml.Unmarshal(input, tagging); err != nil {
 		glog.Errorf("PutObjectTaggingHandler Unmarshal %s: %v", r.URL, err)
-		writeErrorResponse(w, s3err.ErrMalformedXML, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrMalformedXML, r)
 		return
 	}
 	tags := tagging.ToTags()
 	if len(tags) > 10 {
 		glog.Errorf("PutObjectTaggingHandler tags %s: %d tags more than 10", r.URL, len(tags))
-		writeErrorResponse(w, s3err.ErrInvalidTag, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r)
 		return
 	}
 	for k, v := range tags {
 		if len(k) > 128 {
 			glog.Errorf("PutObjectTaggingHandler tags %s: tag key %s longer than 128", r.URL, k)
-			writeErrorResponse(w, s3err.ErrInvalidTag, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r)
 			return
 		}
 		if len(v) > 256 {
 			glog.Errorf("PutObjectTaggingHandler tags %s: tag value %s longer than 256", r.URL, v)
-			writeErrorResponse(w, s3err.ErrInvalidTag, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInvalidTag, r)
 			return
 		}
 	}
@@ -80,10 +80,10 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R
 	if err = s3a.setTags(dir, name, tagging.ToTags()); err != nil {
 		if err == filer_pb.ErrNotFound {
 			glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err)
-			writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
 		} else {
 			glog.Errorf("PutObjectTaggingHandler setTags %s: %v", r.URL, err)
-			writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		}
 		return
 	}
@@ -105,10 +105,10 @@ func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *htt
 	if err != nil {
 		if err == filer_pb.ErrNotFound {
 			glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err)
-			writeErrorResponse(w, s3err.ErrNoSuchKey, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrNoSuchKey, r)
 		} else {
 			glog.Errorf("DeleteObjectTaggingHandler %s: %v", r.URL, err)
-			writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		}
 		return
 	}

weed/s3api/s3api_objects_list_handlers.go

@@ -44,11 +44,11 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 	originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())
 
 	if maxKeys < 0 {
-		writeErrorResponse(w, s3err.ErrInvalidMaxKeys, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxKeys, r)
 		return
 	}
 	if delimiter != "" && delimiter != "/" {
-		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
 		return
 	}
 
@@ -60,13 +60,13 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
 
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 
 	if len(response.Contents) == 0 {
 		if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
-			writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrNoSuchBucket, r)
 			return
 		}
 	}
@@ -86,7 +86,7 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
 		StartAfter:            startAfter,
 	}
 
-	writeSuccessResponseXML(w, encodeResponse(responseV2))
+	writeSuccessResponseXML(w, responseV2)
 }
 
 func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
@@ -99,29 +99,29 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 	originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())
 
 	if maxKeys < 0 {
-		writeErrorResponse(w, s3err.ErrInvalidMaxKeys, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInvalidMaxKeys, r)
 		return
 	}
 	if delimiter != "" && delimiter != "/" {
-		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrNotImplemented, r)
 		return
 	}
 
 	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
 
 	if err != nil {
-		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
+		s3err.WriteErrorResponse(w, s3err.ErrInternalError, r)
 		return
 	}
 
 	if len(response.Contents) == 0 {
 		if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
-			writeErrorResponse(w, s3err.ErrNoSuchBucket, r.URL)
+			s3err.WriteErrorResponse(w, s3err.ErrNoSuchBucket, r)
 			return
 		}
 	}
 
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, response)
 }
 
 func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) {

weed/s3api/s3api_objects_list_handlers_test.go

@@ -1,6 +1,7 @@
 package s3api
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"testing"
 	"time"
 )
@@ -31,7 +32,7 @@ func TestListObjectsHandler(t *testing.T) {
 		}},
 	}
 
-	encoded := string(encodeResponse(response))
+	encoded := string(s3err.EncodeXMLResponse(response))
 	if encoded != expected {
 		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
 	}

weed/s3api/s3api_server.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	. "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"net/http"
 	"strings"
 	"time"
@@ -132,6 +133,6 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 	apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.ListBucketsHandler, "LIST"))
 
 	// NotFound
-	apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler)
+	apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler)
 
 }

weed/s3api/s3api_test.go

@@ -1,6 +1,7 @@
 package s3api
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"testing"
 	"time"
 )
@@ -14,7 +15,7 @@ func TestCopyObjectResponse(t *testing.T) {
 		LastModified: time.Now(),
 	}
 
-	println(string(encodeResponse(response)))
+	println(string(s3err.EncodeXMLResponse(response)))
 
 }
 
@@ -27,6 +28,6 @@ func TestCopyPartResponse(t *testing.T) {
 		LastModified: time.Now(),
 	}
 
-	println(string(encodeResponse(response)))
+	println(string(s3err.EncodeXMLResponse(response)))
 
 }

weed/s3api/s3err/error_handler.go

@@ -0,0 +1,92 @@
+package s3err
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/gorilla/mux"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type mimeType string
+
+const (
+	mimeNone mimeType = ""
+	MimeXML  mimeType = "application/xml"
+)
+
+func WriteXMLResponse(w http.ResponseWriter, statusCode int, response interface{}) {
+	WriteResponse(w, statusCode, EncodeXMLResponse(response), MimeXML)
+}
+
+func WriteEmptyResponse(w http.ResponseWriter, statusCode int) {
+	WriteResponse(w, statusCode, []byte{}, mimeNone)
+}
+
+func WriteErrorResponse(w http.ResponseWriter, errorCode ErrorCode, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+	if strings.HasPrefix(object, "/") {
+		object = object[1:]
+	}
+
+	apiError := GetAPIError(errorCode)
+	errorResponse := getRESTErrorResponse(apiError, r.URL.Path, bucket, object)
+	encodedErrorResponse := EncodeXMLResponse(errorResponse)
+	WriteResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, MimeXML)
+}
+
+func getRESTErrorResponse(err APIError, resource string, bucket, object string) RESTErrorResponse {
+	return RESTErrorResponse{
+		Code:       err.Code,
+		BucketName: bucket,
+		Key:        object,
+		Message:    err.Description,
+		Resource:   resource,
+		RequestID:  fmt.Sprintf("%d", time.Now().UnixNano()),
+	}
+}
+
+// Encodes the response headers into XML format.
+func EncodeXMLResponse(response interface{}) []byte {
+	var bytesBuffer bytes.Buffer
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	e.Encode(response)
+	return bytesBuffer.Bytes()
+}
+
+func setCommonHeaders(w http.ResponseWriter) {
+	w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano()))
+	w.Header().Set("Accept-Ranges", "bytes")
+}
+
+func WriteResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
+	setCommonHeaders(w)
+	if response != nil {
+		w.Header().Set("Content-Length", strconv.Itoa(len(response)))
+	}
+	if mType != mimeNone {
+		w.Header().Set("Content-Type", string(mType))
+	}
+	w.WriteHeader(statusCode)
+	if response != nil {
+		glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response))
+		_, err := w.Write(response)
+		if err != nil {
+			glog.V(0).Infof("write err: %v", err)
+		}
+		w.(http.Flusher).Flush()
+	}
+}
+
+// If none of the http routes match respond with MethodNotAllowed
+func NotFoundHandler(w http.ResponseWriter, r *http.Request) {
+	glog.V(0).Infof("unsupported %s %s", r.Method, r.RequestURI)
+	WriteErrorResponse(w, ErrMethodNotAllowed, r)
+}

weed/s3api/s3err/s3api_errors.go

@@ -15,11 +15,13 @@ type APIError struct {
 
 // RESTErrorResponse - error response format
 type RESTErrorResponse struct {
-	XMLName   xml.Name `xml:"Error" json:"-"`
-	Code      string   `xml:"Code" json:"Code"`
-	Message   string   `xml:"Message" json:"Message"`
-	Resource  string   `xml:"Resource" json:"Resource"`
-	RequestID string   `xml:"RequestId" json:"RequestId"`
+	XMLName    xml.Name `xml:"Error" json:"-"`
+	Code       string   `xml:"Code" json:"Code"`
+	Message    string   `xml:"Message" json:"Message"`
+	Resource   string   `xml:"Resource" json:"Resource"`
+	RequestID  string   `xml:"RequestId" json:"RequestId"`
+	Key        string   `xml:"Key,omitempty" json:"Key,omitempty"`
+	BucketName string   `xml:"BucketName,omitempty" json:"BucketName,omitempty"`
 
 	// Underlying HTTP status code for the returned error
 	StatusCode int `xml:"-" json:"-"`

weed/s3api/tags_test.go

@@ -2,6 +2,7 @@ package s3api
 
 import (
 	"encoding/xml"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"github.com/stretchr/testify/assert"
 	"testing"
 )
@@ -41,7 +42,7 @@ func TestXMLMarshall(t *testing.T) {
 		},
 	}
 
-	actual := string(encodeResponse(tags))
+	actual := string(s3err.EncodeXMLResponse(tags))
 
 	expected := `<?xml version="1.0" encoding="UTF-8"?>
 <Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><TagSet><Tag><Key>key1</Key><Value>value1</Value></Tag></TagSet></Tagging>`

weed/server/filer_grpc_server_rename.go

@@ -115,8 +115,7 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat
 		Extended: entry.Extended,
 		Content:  entry.Content,
 	}
-	createErr := fs.filer.CreateEntry(ctx, newEntry, false, false, nil)
-	if createErr != nil {
+	if createErr := fs.filer.CreateEntry(ctx, newEntry, false, false, nil); createErr != nil {
 		return createErr
 	}
 

weed/server/filer_server_handlers_write.go

@@ -28,8 +28,6 @@ type FilerPostResult struct {
 	Name  string `json:"name,omitempty"`
 	Size  int64  `json:"size,omitempty"`
 	Error string `json:"error,omitempty"`
-	Fid   string `json:"fid,omitempty"`
-	Url   string `json:"url,omitempty"`
 }
 
 func (fs *FilerServer) assignNewFileInfo(so *operation.StorageOption) (fileId, urlLocation string, auth security.EncodedJwt, err error) {

weed/server/filer_server_handlers_write_upload.go

@@ -29,7 +29,6 @@ var bufPool = sync.Pool{
 }
 
 func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, contentLength int64, so *operation.StorageOption) ([]*filer_pb.FileChunk, hash.Hash, int64, error, []byte) {
-	var fileChunks []*filer_pb.FileChunk
 
 	md5Hash := md5.New()
 	var partReader = ioutil.NopCloser(io.TeeReader(reader, md5Hash))
@@ -41,6 +40,8 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
 	var wg sync.WaitGroup
 	var bytesBufferCounter int64
 	bytesBufferLimitCond := sync.NewCond(new(sync.Mutex))
+	var fileChunks []*filer_pb.FileChunk
+	var fileChunksLock sync.Mutex
 	for {
 
 		// need to throttle used byte buffer
@@ -94,7 +95,9 @@ func (fs *FilerServer) uploadReaderToChunks(w http.ResponseWriter, r *http.Reque
 				uploadErr = toChunkErr
 			}
 			if chunk != nil {
+				fileChunksLock.Lock()
 				fileChunks = append(fileChunks, chunk)
+				fileChunksLock.Unlock()
 				glog.V(4).Infof("uploaded %s chunk %d to %s [%d,%d)", fileName, len(fileChunks), chunk.FileId, offset, offset+int64(chunk.Size))
 			}
 		}(chunkOffset)

weed/server/master_grpc_server.go

@@ -205,8 +205,8 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 			_, err := stream.Recv()
 			if err != nil {
 				glog.V(2).Infof("- client %v: %v", clientName, err)
-				stopChan <- true
-				break
+				close(stopChan)
+				return
 			}
 		}
 	}()

weed/server/volume_grpc_batch_delete.go

@@ -8,7 +8,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
-	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
 func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.BatchDeleteRequest) (*volume_server_pb.BatchDeleteResponse, error) {
@@ -30,7 +29,7 @@ func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.B
 		n := new(needle.Needle)
 		volumeId, _ := needle.NewVolumeId(vid)
 		if req.SkipCookieCheck {
-			n.Id, err = types.ParseNeedleId(id_cookie)
+			n.Id, _, err = needle.ParseNeedleIdCookie(id_cookie)
 			if err != nil {
 				resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
 					FileId: fid,

weed/shell/shell_liner.go

@@ -148,9 +148,11 @@ func loadHistory() {
 
 func saveHistory() {
 	if f, err := os.Create(historyPath); err != nil {
-		fmt.Printf("Error writing history file: %v\n", err)
+		fmt.Printf("Error creating history file: %v\n", err)
 	} else {
-		line.WriteHistory(f)
+		if _, err = line.WriteHistory(f); err != nil {
+			fmt.Printf("Error writing history file: %v\n", err)
+		}
 		f.Close()
 	}
 }

weed/stats/metrics.go

@@ -8,11 +8,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/client_golang/prometheus/push"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
 )
 
 var (
@@ -127,12 +127,12 @@ var (
 )
 
 func init() {
-
 	Gather.MustRegister(FilerRequestCounter)
 	Gather.MustRegister(FilerRequestHistogram)
 	Gather.MustRegister(FilerStoreCounter)
 	Gather.MustRegister(FilerStoreHistogram)
-	Gather.MustRegister(prometheus.NewGoCollector())
+	Gather.MustRegister(collectors.NewGoCollector())
+	Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
 
 	Gather.MustRegister(VolumeServerRequestCounter)
 	Gather.MustRegister(VolumeServerRequestHistogram)
@@ -147,7 +147,6 @@ func init() {
 }
 
 func LoopPushingMetric(name, instance, addr string, intervalSeconds int) {
-
 	if addr == "" || intervalSeconds == 0 {
 		return
 	}
@@ -165,7 +164,6 @@ func LoopPushingMetric(name, instance, addr string, intervalSeconds int) {
 			intervalSeconds = 15
 		}
 		time.Sleep(time.Duration(intervalSeconds) * time.Second)
-
 	}
 }
 

weed/storage/store.go

@@ -250,6 +250,11 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
 			}
 			if !deleteVolume {
 				collectionVolumeSize[v.Collection] += volumeMessage.Size
+			} else {
+				collectionVolumeSize[v.Collection] -= volumeMessage.Size
+				if collectionVolumeSize[v.Collection] <= 0 {
+					delete(collectionVolumeSize, v.Collection)
+				}
 			}
 
 			if _, exist := collectionVolumeReadOnlyCount[v.Collection]; !exist {

weed/topology/topology.go

@@ -37,7 +37,7 @@ type Topology struct {
 	chanFullVolumes    chan storage.VolumeInfo
 	chanCrowdedVolumes chan storage.VolumeInfo
 
-	Configuration      *Configuration
+	Configuration *Configuration
 
 	RaftServer raft.Server
 }
@@ -70,6 +70,11 @@ func (t *Topology) IsLeader() bool {
 		if t.RaftServer.State() == raft.Leader {
 			return true
 		}
+		if leader, err := t.Leader(); err == nil {
+			if t.RaftServer.Name() == leader {
+				return true
+			}
+		}
 	}
 	return false
 }

weed/util/constants.go

@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 51)
+	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 55)
 	COMMIT  = ""
 )
 

weed/util/grace/pprof.go

@@ -14,9 +14,30 @@ func SetupProfiling(cpuProfile, memProfile string) {
 		if err != nil {
 			glog.Fatal(err)
 		}
+		runtime.SetBlockProfileRate(1)
+		runtime.SetMutexProfileFraction(1)
 		pprof.StartCPUProfile(f)
 		OnInterrupt(func() {
 			pprof.StopCPUProfile()
+
+			// write block pprof
+			blockF, err := os.Create(cpuProfile+".block")
+			if err != nil {
+				return
+			}
+			p := pprof.Lookup("block")
+			p.WriteTo(blockF,0)
+			blockF.Close()
+
+			// write mutex pprof
+			mutexF, err := os.Create(cpuProfile+".mutex")
+			if err != nil {
+				return
+			}
+			p = pprof.Lookup("mutex")
+			p.WriteTo(mutexF,0)
+			mutexF.Close()
+
 		})
 	}
 	if memProfile != "" {