Compare commits

...
4 Commits
Author SHA1 Message Date
Chris Lu 71f0c19515 2.26 2021-02-15 13:38:38 -08:00
Chris Lu 8b169aa47c s3: Critical - S3 Delete Multiple Objects, delete the bucket
fix https://github.com/chrislusf/seaweedfs/issues/1806
2021-02-15 13:38:29 -08:00
Chris LuandGitHub 50202d1391 Merge pull request #1805 from LazyDBA247-Anyvision/master
better postgresql connection pool management
2021-02-14 22:26:29 -08:00
LazyDBA247-Anyvision 7f458d5e78 better postgres connection pool management
adding SetConnMaxLifetime configuration (https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime)
to enable refresh of stale connections.
2021-02-15 07:45:09 +02:00
8 changed files with 31 additions and 9 deletions
+2 -2
View File
@@ -1,5 +1,5 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "2.25"
version: 2.25
appVersion: "2.26"
version: 2.26
+1 -1
View File
@@ -4,7 +4,7 @@ global:
registry: ""
repository: ""
imageName: chrislusf/seaweedfs
# imageTag: "2.25" - started using {.Chart.appVersion}
# imageTag: "2.26" - started using {.Chart.appVersion}
imagePullPolicy: IfNotPresent
imagePullSecrets: imagepullsecret
restartPolicy: Always
+2
View File
@@ -160,6 +160,7 @@ schema = ""
sslmode = "disable"
connection_max_idle = 100
connection_max_open = 100
connection_max_lifetime_seconds = 0
[postgres2]
enabled = false
@@ -181,6 +182,7 @@ schema = ""
sslmode = "disable"
connection_max_idle = 100
connection_max_open = 100
connection_max_lifetime_seconds = 0
[cassandra]
# CREATE TABLE filemeta (
+11
View File
@@ -0,0 +1,11 @@
[elastic7]
enabled = true
servers = [
"http://localhost:9200",
]
username = ""
password = ""
sniff_enabled = false
healthcheck_enabled = false
# increase the value is recommend, be sure the value in Elastic is greater or equal here
index.max_result_window = 10000
+4 -1
View File
@@ -3,6 +3,7 @@ package postgres
import (
"database/sql"
"fmt"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
@@ -37,10 +38,11 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
configuration.GetString(prefix+"sslmode"),
configuration.GetInt(prefix+"connection_max_idle"),
configuration.GetInt(prefix+"connection_max_open"),
configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
)
}
func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen int) (err error) {
func (store *PostgresStore) initialize(user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = false
store.SqlGenerator = &SqlGenPostgres{
@@ -71,6 +73,7 @@ func (store *PostgresStore) initialize(user, password, hostname string, port int
store.DB.SetMaxIdleConns(maxIdle)
store.DB.SetMaxOpenConns(maxOpen)
store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
if err = store.DB.Ping(); err != nil {
return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
+4 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
@@ -40,10 +41,11 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
configuration.GetString(prefix+"sslmode"),
configuration.GetInt(prefix+"connection_max_idle"),
configuration.GetInt(prefix+"connection_max_open"),
configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
)
}
func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen int) (err error) {
func (store *PostgresStore2) initialize(createTable, user, password, hostname string, port int, database, schema, sslmode string, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = true
store.SqlGenerator = &postgres.SqlGenPostgres{
@@ -74,6 +76,7 @@ func (store *PostgresStore2) initialize(createTable, user, password, hostname st
store.DB.SetMaxIdleConns(maxIdle)
store.DB.SetMaxOpenConns(maxOpen)
store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)
if err = store.DB.Ping(); err != nil {
return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
+6 -3
View File
@@ -185,7 +185,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
for _, object := range deleteObjects.Objects {
lastSeparator := strings.LastIndex(object.ObjectName, "/")
parentDirectoryPath, entryName, isDeleteData, isRecursive := "/", object.ObjectName, true, false
parentDirectoryPath, entryName, isDeleteData, isRecursive := "", object.ObjectName, true, false
if lastSeparator > 0 && lastSeparator+1 < len(object.ObjectName) {
entryName = object.ObjectName[lastSeparator+1:]
parentDirectoryPath = "/" + object.ObjectName[:lastSeparator]
@@ -208,7 +208,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
// purge empty folders, only checking folders with deletions
for len(directoriesWithDeletion) > 0 {
directoriesWithDeletion = doDeleteEmptyDirectories(client, directoriesWithDeletion)
directoriesWithDeletion = s3a.doDeleteEmptyDirectories(client, directoriesWithDeletion)
}
return nil
@@ -224,7 +224,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
}
func doDeleteEmptyDirectories(client filer_pb.SeaweedFilerClient, directoriesWithDeletion map[string]int) (newDirectoriesWithDeletion map[string]int) {
func (s3a *S3ApiServer) doDeleteEmptyDirectories(client filer_pb.SeaweedFilerClient, directoriesWithDeletion map[string]int) (newDirectoriesWithDeletion map[string]int) {
var allDirs []string
for dir, _ := range directoriesWithDeletion {
allDirs = append(allDirs, dir)
@@ -235,6 +235,9 @@ func doDeleteEmptyDirectories(client filer_pb.SeaweedFilerClient, directoriesWit
newDirectoriesWithDeletion = make(map[string]int)
for _, dir := range allDirs {
parentDir, dirName := util.FullPath(dir).DirAndName()
if parentDir == s3a.option.BucketsPath {
continue
}
if err := doDeleteEntry(client, parentDir, dirName, false, false); err != nil {
glog.V(4).Infof("directory %s has %d deletion but still not empty: %v", dir, directoriesWithDeletion[dir], err)
} else {
+1 -1
View File
@@ -5,7 +5,7 @@ import (
)
var (
VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 25)
VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 26)
COMMIT = ""
)