Files
seaweedfs/weed/filer/postgres/postgres_store.go
Chris Lu a24f4844d3 filer: keep S3 list order byte-lexicographic regardless of SQL name column collation (#9824)
* mysql: keep S3 list order byte-lexicographic regardless of name column collation

ORDER BY name and the name > ? pagination predicate follow the column
collation, so a case-insensitive filemeta.name (e.g. utf8mb3_general_ci)
returns S3 keys out of byte order and breaks clients that merge two sorted
listings.

Detect the live name collation at startup; only when it isn't binary, wrap
the list comparison, prefix, and ORDER BY in BINARY name so order and
pagination stay consistent. Correctly configured utf8mb4_bin tables keep
their indexed range scan unchanged, and the operator gets a warning to
convert the column.

* postgres: keep S3 list order byte-lexicographic regardless of name column collation

ORDER BY name and the name > $n pagination predicate follow the column or
database collation, so a locale-aware filemeta.name (e.g. the en_US.UTF-8
database default) returns S3 keys out of byte order and breaks clients that
merge two sorted listings.

Detect the live name collation at startup; only when it isn't byte-ordered,
wrap the list comparison, prefix, and ORDER BY in COLLATE "C" so order and
pagination stay consistent. A byte-ordered (C/POSIX/C.UTF-8) column keeps its
indexed range scan unchanged, and the operator gets a warning to declare the
column COLLATE "C".
2026-06-04 14:33:41 -07:00

124 lines
3.7 KiB
Go

// Package postgres provides PostgreSQL filer store implementation
// Migrated from github.com/lib/pq to github.com/jackc/pgx for:
// - Active development and support
// - Better performance and PostgreSQL-specific features
// - Improved error handling (no more panics)
// - Built-in logging capabilities
// - Superior SSL certificate support
package postgres
import (
"strconv"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func init() {
filer.Stores = append(filer.Stores, &PostgresStore{})
}
type PostgresStore struct {
abstract_sql.AbstractSqlStore
}
func (store *PostgresStore) GetName() string {
return "postgres"
}
func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
// Absent key keeps a pooled default; an explicit 0 disables the idle pool.
configuration.SetDefault(prefix+"connection_max_idle", 2)
// Default on so minimal configs are not exposed to duplicate-key tx
// poisoning on Postgres; an explicit false still disables it.
configuration.SetDefault(prefix+"enableUpsert", true)
return store.initialize(
configuration.GetString(prefix+"upsertQuery"),
configuration.GetBool(prefix+"enableUpsert"),
configuration.GetString(prefix+"username"),
configuration.GetString(prefix+"password"),
configuration.GetString(prefix+"hostname"),
configuration.GetInt(prefix+"port"),
configuration.GetString(prefix+"database"),
configuration.GetString(prefix+"schema"),
configuration.GetString(prefix+"sslmode"),
configuration.GetString(prefix+"sslcert"),
configuration.GetString(prefix+"sslkey"),
configuration.GetString(prefix+"sslrootcert"),
configuration.GetString(prefix+"sslcrl"),
configuration.GetBool(prefix+"pgbouncer_compatible"),
configuration.GetInt(prefix+"connection_max_idle"),
configuration.GetInt(prefix+"connection_max_open"),
configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
)
}
func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = false
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
upsertQuery = DefaultUpsertQuery
}
gen := &SqlGenPostgres{
CreateTableSqlTemplate: "",
DropTableSqlTemplate: `drop table "%s"`,
UpsertQueryTemplate: upsertQuery,
}
store.SqlGenerator = gen
// pgx-optimized connection string with better timeouts and connection handling
sqlUrl := "connect_timeout=30"
if hostname != "" {
sqlUrl += " host=" + hostname
}
if port != 0 {
sqlUrl += " port=" + strconv.Itoa(port)
}
// SSL configuration - pgx provides better SSL support than lib/pq
if sslmode != "" {
sqlUrl += " sslmode=" + sslmode
}
if sslcert != "" {
sqlUrl += " sslcert=" + sslcert
}
if sslkey != "" {
sqlUrl += " sslkey=" + sslkey
}
if sslrootcert != "" {
sqlUrl += " sslrootcert=" + sslrootcert
}
if sslcrl != "" {
sqlUrl += " sslcrl=" + sslcrl
}
if user != "" {
sqlUrl += " user=" + user
}
adaptedSqlUrl := sqlUrl
if password != "" {
sqlUrl += " password=" + password
adaptedSqlUrl += " password=ADAPTED"
}
if database != "" {
sqlUrl += " dbname=" + database
adaptedSqlUrl += " dbname=" + database
}
if schema != "" && !pgbouncerCompatible {
sqlUrl += " search_path=" + schema
adaptedSqlUrl += " search_path=" + schema
}
db, openErr := OpenPGXDB(sqlUrl, adaptedSqlUrl, pgbouncerCompatible, maxIdle, maxOpen, maxLifetimeSeconds)
if openErr != nil {
return openErr
}
store.DB = db
ConfigureListOrdering(store.DB, gen)
return nil
}