seaweedfs/weed/storage/blockvol/csi/node.go

package csi

import (
	"context"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"

	"github.com/container-storage-interface/spec/lib/go/csi"
	"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

const (
	transportISCSI = "iscsi"
	transportNVMe  = "nvme"
)

// stagedVolumeInfo tracks info needed for NodeUnstageVolume and NodeExpandVolume.
type stagedVolumeInfo struct {
	iqn         string
	iscsiAddr   string
	nqn         string // NVMe subsystem NQN
	nvmeAddr    string // NVMe/TCP target address
	transport   string // "iscsi" or "nvme"
	isLocal     bool   // true if volume is served by local VolumeManager
	fsType      string // filesystem type (ext4, xfs, etc.)
	stagingPath string // staging mount path
}

type nodeServer struct {
	csi.UnimplementedNodeServer
	mgr       *VolumeManager // may be nil in controller-only mode
	nodeID    string
	iqnPrefix string // for IQN derivation fallback on restart
	nqnPrefix string // for NQN derivation fallback on restart
	iscsiUtil ISCSIUtil
	nvmeUtil  NVMeUtil // may be nil if NVMe not available
	mountUtil MountUtil
	logger    *log.Logger

	stagedMu sync.Mutex
	staged   map[string]*stagedVolumeInfo // volumeID -> staged info
}

// transportFile is the filename written inside the staging directory to persist
// the transport type across CSI plugin restarts.
const transportFile = ".transport"

func (s *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
	volumeID := req.VolumeId
	stagingPath := req.StagingTargetPath

	if volumeID == "" {
		return nil, status.Error(codes.InvalidArgument, "volume ID is required")
	}
	if stagingPath == "" {
		return nil, status.Error(codes.InvalidArgument, "staging target path is required")
	}
	if req.VolumeCapability == nil {
		return nil, status.Error(codes.InvalidArgument, "volume capability is required")
	}

	// Idempotency: if already mounted at staging path, return OK.
	mounted, err := s.mountUtil.IsMounted(ctx, stagingPath)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "check mount: %v", err)
	}
	if mounted {
		s.logger.Printf("NodeStageVolume: %s already mounted at %s", volumeID, stagingPath)
		return &csi.NodeStageVolumeResponse{}, nil
	}

	// Resolve iSCSI target info.
	// Priority: publish_context > volume_context > local volume manager fallback.
	var iqn, portal string
	isLocal := false

	if req.PublishContext != nil && req.PublishContext["iscsiAddr"] != "" && req.PublishContext["iqn"] != "" {
		portal = req.PublishContext["iscsiAddr"]
		iqn = req.PublishContext["iqn"]
	} else if req.VolumeContext != nil && req.VolumeContext["iscsiAddr"] != "" && req.VolumeContext["iqn"] != "" {
		portal = req.VolumeContext["iscsiAddr"]
		iqn = req.VolumeContext["iqn"]
	} else if s.mgr != nil {
		isLocal = true
		if err := s.mgr.OpenVolume(volumeID); err != nil {
			return nil, status.Errorf(codes.Internal, "open volume: %v", err)
		}
		iqn = s.mgr.VolumeIQN(volumeID)
		portal = s.mgr.ListenAddr()
	}

	// Resolve NVMe target info (same priority chain).
	// PublishContext > VolumeContext > local VolumeManager.
	var nqn, nvmeAddr string
	if req.PublishContext != nil && req.PublishContext["nvmeAddr"] != "" && req.PublishContext["nqn"] != "" {
		nvmeAddr = req.PublishContext["nvmeAddr"]
		nqn = req.PublishContext["nqn"]
	} else if req.VolumeContext != nil && req.VolumeContext["nvmeAddr"] != "" && req.VolumeContext["nqn"] != "" {
		nvmeAddr = req.VolumeContext["nvmeAddr"]
		nqn = req.VolumeContext["nqn"]
	} else if s.mgr != nil && s.mgr.NvmeAddr() != "" {
		nvmeAddr = s.mgr.NvmeAddr()
		nqn = s.mgr.VolumeNQN(volumeID)
	}

	// No transport info at all (neither iSCSI nor NVMe resolved, no local mgr).
	if iqn == "" && nqn == "" {
		return nil, status.Error(codes.FailedPrecondition, "no volume_context and no local volume manager")
	}

	// Cleanup on error.
	success := false
	defer func() {
		if !success {
			s.logger.Printf("NodeStageVolume: cleaning up %s after error", volumeID)
			if isLocal && s.mgr != nil {
				s.mgr.CloseVolume(volumeID)
			}
		}
	}()

	// Transport selection: prefer NVMe if supported, fall back to iSCSI.
	var device, transport string

	nvmeAvailable := nvmeAddr != "" && nqn != "" && s.nvmeUtil != nil && s.nvmeUtil.IsNVMeTCPAvailable()
	if nvmeAvailable {
		// NVMe path — fail fast on error, no fallback to iSCSI.
		transport = transportNVMe

		connected, cerr := s.nvmeUtil.IsConnected(ctx, nqn)
		if cerr != nil {
			return nil, status.Errorf(codes.Internal, "check nvme connection: %v", cerr)
		}
		if !connected {
			if cerr := s.nvmeUtil.Connect(ctx, nqn, nvmeAddr); cerr != nil {
				return nil, status.Errorf(codes.Internal, "nvme connect: %v", cerr)
			}
		}

		// Cleanup NVMe on subsequent failures.
		defer func() {
			if !success {
				s.nvmeUtil.Disconnect(ctx, nqn)
			}
		}()

		device, err = s.nvmeUtil.GetDeviceByNQN(ctx, nqn)
		if err != nil {
			return nil, status.Errorf(codes.Internal, "nvme get device: %v", err)
		}
	} else if iqn != "" && portal != "" {
		// iSCSI path (existing code).
		transport = transportISCSI

		loggedIn, lerr := s.iscsiUtil.IsLoggedIn(ctx, iqn)
		if lerr != nil {
			return nil, status.Errorf(codes.Internal, "check iscsi login: %v", lerr)
		}
		if !loggedIn {
			if err := s.iscsiUtil.Discovery(ctx, portal); err != nil {
				return nil, status.Errorf(codes.Internal, "iscsi discovery: %v", err)
			}
			if err := s.iscsiUtil.Login(ctx, iqn, portal); err != nil {
				return nil, status.Errorf(codes.Internal, "iscsi login: %v", err)
			}
		}

		device, err = s.iscsiUtil.GetDeviceByIQN(ctx, iqn)
		if err != nil {
			return nil, status.Errorf(codes.Internal, "get device: %v", err)
		}
	} else {
		return nil, status.Error(codes.FailedPrecondition, "no transport available")
	}

	// Ensure staging directory exists.
	if err := os.MkdirAll(stagingPath, 0750); err != nil {
		return nil, status.Errorf(codes.Internal, "create staging dir: %v", err)
	}

	// Format (if needed) and mount.
	fsType := "ext4"
	if req.VolumeCapability != nil {
		if mnt := req.VolumeCapability.GetMount(); mnt != nil && mnt.FsType != "" {
			fsType = mnt.FsType
		}
	}

	if err := s.mountUtil.FormatAndMount(ctx, device, stagingPath, fsType); err != nil {
		return nil, status.Errorf(codes.Internal, "format and mount: %v", err)
	}

	// Write transport marker for restart recovery.
	if werr := writeTransportFile(stagingPath, transport); werr != nil {
		s.logger.Printf("NodeStageVolume: %s: %v (non-fatal)", volumeID, werr)
	}

	// Track staged volume for unstage.
	s.stagedMu.Lock()
	if s.staged == nil {
		s.staged = make(map[string]*stagedVolumeInfo)
	}
	s.staged[volumeID] = &stagedVolumeInfo{
		iqn:         iqn,
		iscsiAddr:   portal,
		nqn:         nqn,
		nvmeAddr:    nvmeAddr,
		transport:   transport,
		isLocal:     isLocal,
		fsType:      fsType,
		stagingPath: stagingPath,
	}
	s.stagedMu.Unlock()

	success = true
	s.logger.Printf("NodeStageVolume: %s staged at %s (device=%s, transport=%s, local=%v)", volumeID, stagingPath, device, transport, isLocal)
	return &csi.NodeStageVolumeResponse{}, nil
}

func (s *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
	volumeID := req.VolumeId
	stagingPath := req.StagingTargetPath

	if volumeID == "" {
		return nil, status.Error(codes.InvalidArgument, "volume ID is required")
	}
	if stagingPath == "" {
		return nil, status.Error(codes.InvalidArgument, "staging target path is required")
	}

	// Look up staged info.
	s.stagedMu.Lock()
	info := s.staged[volumeID]
	s.stagedMu.Unlock()

	// Determine transport and identifiers.
	var iqn, nqn, transport string
	isLocal := false

	if info != nil {
		iqn = info.iqn
		nqn = info.nqn
		transport = info.transport
		isLocal = info.isLocal
	} else {
		// Restart fallback: read .transport file from staging path.
		transport = readTransportFile(stagingPath)

		// Derive identifiers.
		if s.mgr != nil {
			iqn = s.mgr.VolumeIQN(volumeID)
			nqn = s.mgr.VolumeNQN(volumeID)
			isLocal = true
		} else {
			if s.iqnPrefix != "" {
				iqn = s.iqnPrefix + ":" + blockvol.SanitizeIQN(volumeID)
			}
			if s.nqnPrefix != "" {
				nqn = blockvol.BuildNQN(s.nqnPrefix, volumeID)
			}
		}

		// If no .transport file, probe NVMe connection to determine transport.
		if transport == "" && nqn != "" && s.nvmeUtil != nil {
			if connected, _ := s.nvmeUtil.IsConnected(ctx, nqn); connected {
				transport = transportNVMe
			}
		}
		if transport == "" {
			transport = transportISCSI // default fallback
		}

		s.logger.Printf("NodeUnstageVolume: %s not in staged map, derived transport=%s", volumeID, transport)
	}

	// Best-effort cleanup: always attempt all steps even if one fails.
	var firstErr error

	// Unmount.
	if err := s.mountUtil.Unmount(ctx, stagingPath); err != nil {
		s.logger.Printf("NodeUnstageVolume: unmount error: %v", err)
		firstErr = err
	}

	// Disconnect transport.
	switch transport {
	case transportNVMe:
		if nqn != "" && s.nvmeUtil != nil {
			if err := s.nvmeUtil.Disconnect(ctx, nqn); err != nil {
				s.logger.Printf("NodeUnstageVolume: nvme disconnect error: %v", err)
				if firstErr == nil {
					firstErr = err
				}
			}
		}
	default: // iSCSI
		if iqn != "" {
			if err := s.iscsiUtil.Logout(ctx, iqn); err != nil {
				s.logger.Printf("NodeUnstageVolume: logout error: %v", err)
				if firstErr == nil {
					firstErr = err
				}
			}
		}
	}

	// Close the local volume if applicable.
	if isLocal && s.mgr != nil {
		if err := s.mgr.CloseVolume(volumeID); err != nil {
			s.logger.Printf("NodeUnstageVolume: close volume error: %v", err)
			if firstErr == nil {
				firstErr = err
			}
		}
	}

	if firstErr != nil {
		// Keep staged entry so retry has correct info.
		return nil, status.Errorf(codes.Internal, "unstage: %v", firstErr)
	}

	// Clean up transport file.
	os.Remove(filepath.Join(stagingPath, transportFile))

	// Remove from staged map only after successful cleanup.
	s.stagedMu.Lock()
	delete(s.staged, volumeID)
	s.stagedMu.Unlock()

	s.logger.Printf("NodeUnstageVolume: %s unstaged from %s", volumeID, stagingPath)
	return &csi.NodeUnstageVolumeResponse{}, nil
}

func (s *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
	volumeID := req.VolumeId
	targetPath := req.TargetPath
	stagingPath := req.StagingTargetPath

	if volumeID == "" {
		return nil, status.Error(codes.InvalidArgument, "volume ID is required")
	}
	if targetPath == "" {
		return nil, status.Error(codes.InvalidArgument, "target path is required")
	}
	if stagingPath == "" {
		return nil, status.Error(codes.InvalidArgument, "staging target path is required")
	}

	// Idempotency: if already bind-mounted, return OK.
	mounted, err := s.mountUtil.IsMounted(ctx, targetPath)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "check mount: %v", err)
	}
	if mounted {
		s.logger.Printf("NodePublishVolume: %s already mounted at %s", volumeID, targetPath)
		return &csi.NodePublishVolumeResponse{}, nil
	}

	// Ensure target directory exists.
	if err := os.MkdirAll(targetPath, 0750); err != nil {
		return nil, status.Errorf(codes.Internal, "create target dir: %v", err)
	}

	// Bind mount staging path to target path.
	readOnly := req.Readonly
	if err := s.mountUtil.BindMount(ctx, stagingPath, targetPath, readOnly); err != nil {
		return nil, status.Errorf(codes.Internal, "bind mount: %v", err)
	}

	s.logger.Printf("NodePublishVolume: %s published at %s", volumeID, targetPath)
	return &csi.NodePublishVolumeResponse{}, nil
}

func (s *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
	if req.VolumeId == "" {
		return nil, status.Error(codes.InvalidArgument, "volume ID is required")
	}
	if req.TargetPath == "" {
		return nil, status.Error(codes.InvalidArgument, "target path is required")
	}

	// Idempotent: only unmount if still mounted.
	mounted, err := s.mountUtil.IsMounted(ctx, req.TargetPath)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "check mount: %v", err)
	}
	if mounted {
		if err := s.mountUtil.Unmount(ctx, req.TargetPath); err != nil {
			return nil, status.Errorf(codes.Internal, "unmount: %v", err)
		}
	}

	// CSI spec: remove mount point at target path.
	os.RemoveAll(req.TargetPath)

	s.logger.Printf("NodeUnpublishVolume: %s unpublished from %s", req.VolumeId, req.TargetPath)
	return &csi.NodeUnpublishVolumeResponse{}, nil
}

func (s *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) {
	if req.VolumeId == "" {
		return nil, status.Error(codes.InvalidArgument, "volume ID is required")
	}

	// Look up staged volume info.
	s.stagedMu.Lock()
	info, ok := s.staged[req.VolumeId]
	s.stagedMu.Unlock()
	if !ok {
		return nil, status.Errorf(codes.FailedPrecondition, "volume %q not staged", req.VolumeId)
	}

	var device string
	var err error

	switch info.transport {
	case transportNVMe:
		// NVMe: rescan namespace, then find device.
		if s.nvmeUtil == nil {
			return nil, status.Errorf(codes.Internal, "nvme util not available")
		}
		if err := s.nvmeUtil.Rescan(ctx, info.nqn); err != nil {
			return nil, status.Errorf(codes.Internal, "nvme rescan: %v", err)
		}
		device, err = s.nvmeUtil.GetDeviceByNQN(ctx, info.nqn)
		if err != nil {
			return nil, status.Errorf(codes.Internal, "nvme find device: %v", err)
		}
	default: // iSCSI
		// Check that iSCSI session is active.
		loggedIn, lerr := s.iscsiUtil.IsLoggedIn(ctx, info.iqn)
		if lerr != nil {
			return nil, status.Errorf(codes.Internal, "check iSCSI session: %v", lerr)
		}
		if !loggedIn {
			return nil, status.Errorf(codes.FailedPrecondition, "volume %q not staged: iSCSI session not active", req.VolumeId)
		}
		if err := s.iscsiUtil.RescanDevice(ctx, info.iqn); err != nil {
			return nil, status.Errorf(codes.Internal, "rescan iSCSI device: %v", err)
		}
		device, err = s.iscsiUtil.GetDeviceByIQN(ctx, info.iqn)
		if err != nil {
			return nil, status.Errorf(codes.Internal, "find device: %v", err)
		}
	}

	// Determine mount path and fsType.
	mountPath := info.stagingPath
	if req.VolumePath != "" {
		mountPath = req.VolumePath
	}
	fsType := info.fsType
	if fsType == "" {
		fsType = "ext4"
	}

	// Resize the filesystem.
	if err := s.mountUtil.ResizeFS(ctx, device, mountPath, fsType); err != nil {
		return nil, status.Errorf(codes.Internal, "resize filesystem: %v", err)
	}

	s.logger.Printf("NodeExpandVolume: %s expanded (device=%s, transport=%s, fs=%s)", req.VolumeId, device, info.transport, fsType)

	capacity := int64(0)
	if req.CapacityRange != nil {
		capacity = req.CapacityRange.RequiredBytes
	}
	return &csi.NodeExpandVolumeResponse{
		CapacityBytes: capacity,
	}, nil
}

func (s *nodeServer) NodeGetCapabilities(_ context.Context, _ *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
	caps := []csi.NodeServiceCapability_RPC_Type{
		csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
		csi.NodeServiceCapability_RPC_EXPAND_VOLUME,
	}

	var result []*csi.NodeServiceCapability
	for _, c := range caps {
		result = append(result, &csi.NodeServiceCapability{
			Type: &csi.NodeServiceCapability_Rpc{
				Rpc: &csi.NodeServiceCapability_RPC{Type: c},
			},
		})
	}
	return &csi.NodeGetCapabilitiesResponse{Capabilities: result}, nil
}

func (s *nodeServer) NodeGetInfo(_ context.Context, _ *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) {
	return &csi.NodeGetInfoResponse{
		NodeId:            s.nodeID,
		MaxVolumesPerNode: 256,
		AccessibleTopology: &csi.Topology{
			Segments: map[string]string{
				fmt.Sprintf("topology.%s/node", DriverName): s.nodeID,
			},
		},
	}, nil
}

// writeTransportFile writes the transport type to a file inside the staging path
// so it can be recovered after CSI plugin restart.
func writeTransportFile(stagingPath, transport string) error {
	path := filepath.Join(stagingPath, transportFile)
	if err := os.WriteFile(path, []byte(transport), 0600); err != nil {
		return fmt.Errorf("write transport file: %w", err)
	}
	return nil
}

// readTransportFile reads the transport type from the staging path.
// Returns empty string if the file doesn't exist.
func readTransportFile(stagingPath string) string {
	path := filepath.Join(stagingPath, transportFile)
	data, err := os.ReadFile(path)
	if err != nil {
		return ""
	}
	t := string(data)
	if t == transportISCSI || t == transportNVMe {
		return t
	}
	return ""
}