versitygw/cmd/internal/gwcli/utils.go

// Copyright 2023 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package gwcli

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/urfave/cli/v2"
	"github.com/versity/versitygw/backend/meta"
	"github.com/versity/versitygw/s3event"
)

// UtilsCommand returns the "utils" subcommand, common to all versitygw
// binaries.
func UtilsCommand() *cli.Command {
	return &cli.Command{
		Name:  "utils",
		Usage: "utility helper CLI tool",
		Subcommands: []*cli.Command{
			{
				Name:    "gen-event-filter-config",
				Aliases: []string{"gefc"},
				Usage:   "Create a new configuration file for bucket event notifications filter.",
				Action:  generateEventFiltersConfig,
				Flags: []cli.Flag{
					&cli.StringFlag{
						Name:    "path",
						Usage:   "the path where the config file has to be created",
						Aliases: []string{"p"},
					},
				},
			},
			{
				Name:    "convert-xattr-metadata",
				Aliases: []string{"cxm"},
				Usage:   "Convert legacy X-Amz-Meta.* xattrs into user.metadata JSON and remove legacy keys.",
				Action:  convertXattrMetadata,
			},
		},
	}
}

func generateEventFiltersConfig(ctx *cli.Context) error {
	pathFlag := ctx.String("path")
	path, err := filepath.Abs(filepath.Join(pathFlag, "event_config.json"))
	if err != nil {
		return err
	}

	config := s3event.EventFilter{
		s3event.EventObjectCreated:              true,
		s3event.EventObjectCreatedPut:           true,
		s3event.EventObjectCreatedPost:          true,
		s3event.EventObjectCreatedCopy:          true,
		s3event.EventCompleteMultipartUpload:    true,
		s3event.EventObjectRemoved:              true,
		s3event.EventObjectRemovedDelete:        true,
		s3event.EventObjectRemovedDeleteObjects: true,
		s3event.EventObjectTagging:              true,
		s3event.EventObjectTaggingPut:           true,
		s3event.EventObjectTaggingDelete:        true,
		s3event.EventObjectAclPut:               true,
		s3event.EventObjectRestore:              true,
		s3event.EventObjectRestorePost:          true,
		s3event.EventObjectRestoreCompleted:     true,
	}

	configBytes, err := json.MarshalIndent(config, "", "  ")
	if err != nil {
		return fmt.Errorf("parse event config: %w", err)
	}

	file, err := os.Create(path)
	if err != nil {
		return fmt.Errorf("create config file: %w", err)
	}
	defer file.Close()

	_, err = file.Write(configBytes)
	if err != nil {
		return fmt.Errorf("write config file: %w", err)
	}

	return nil
}

const (
	newMetadataAttr = "metadata"   // stored as user.metadata
	oldMetadataHdr  = "X-Amz-Meta" // legacy prefix
)

func convertXattrMetadata(ctx *cli.Context) error {
	root := strings.TrimSpace(ctx.Args().First())
	if root == "" {
		return cli.Exit("missing directory: should be provided as command argument", 2)
	}

	absRoot, err := filepath.Abs(root)
	if err != nil {
		return fmt.Errorf("resolve directory: %w", err)
	}

	info, err := os.Stat(absRoot)
	if err != nil {
		return fmt.Errorf("stat directory: %w", err)
	}
	if !info.IsDir() {
		return cli.Exit(fmt.Sprintf("not a directory: %s", absRoot), 2)
	}

	xm := meta.XattrMeta{}
	err = xm.Test(absRoot)
	if err != nil {
		return err
	}

	var (
		scanned   int
		converted int
		skipped   int
		errCount  int
	)

	walkErr := filepath.WalkDir(absRoot, func(path string, d os.DirEntry, walkErr error) error {
		if walkErr != nil {
			errCount++
			// keep going
			return nil
		}

		rel, err := filepath.Rel(absRoot, path)
		if err != nil {
			errCount++
			return nil
		}
		if rel == "." {
			// skip root itself
			return nil
		}

		attrs, err := xm.ListAttributes(absRoot, rel)
		if err != nil {
			errCount++
			return nil
		}
		if len(attrs) == 0 {
			// not an s3 object, do not track as skipped
			return nil
		}

		scanned++

		// Collect legacy metadata attributes.
		oldAttrs := make([]string, 0)
		for _, a := range attrs {
			if strings.HasPrefix(a, oldMetadataHdr+".") {
				oldAttrs = append(oldAttrs, a)
			}
		}
		if len(oldAttrs) == 0 {
			skipped++
			return nil
		}

		// Build key/value map from legacy attrs.
		md := make(map[string]string, len(oldAttrs))
		for _, a := range oldAttrs {
			b, err := xm.RetrieveAttribute(nil, absRoot, rel, a)
			if err != nil {
				// If we can't read one key, don't convert this entry.
				errCount++
				return nil
			}
			key := strings.TrimPrefix(a, oldMetadataHdr+".")
			md[key] = string(b)
		}

		// Marshal to JSON and store as user.metadata.
		j, err := json.Marshal(md)
		if err != nil {
			errCount++
			return nil
		}

		if err := xm.StoreAttribute(nil, absRoot, rel, newMetadataAttr, j); err != nil {
			errCount++
			return nil
		}

		// Cleanup old metadata only after successful write of user.metadata.
		for _, a := range oldAttrs {
			if err := xm.DeleteAttribute(absRoot, rel, a); err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
				// Count, but continue cleanup attempts.
				errCount++
			}
		}

		converted++
		return nil
	})
	if walkErr != nil {
		return fmt.Errorf("walk directory: %w", walkErr)
	}

	fmt.Printf(
		"xattr metadata conversion is finished:\n  directory: %s\n  scanned: %d\n  converted: %d\n  skipped: %d\n  errors: %d\n",
		absRoot, scanned, converted, skipped, errCount,
	)

	return nil
}