diff --git a/doc/scst_user_spec.sgml b/doc/scst_user_spec.sgml index 2c1bfa173..19a6b9d65 100644 --- a/doc/scst_user_spec.sgml +++ b/doc/scst_user_spec.sgml @@ -25,8 +25,9 @@ feature virtual SCSI devices in the SCST environment. This document assumes that the reader is familiar with the SCST architecture and the states through which SCSI commands go during processing in SCST. Module scst_user basically only provides hooks to -them. Their description could be found on the SCST web page on -http://scst.sf.net. +them. See doc/scst_pg.sgml for an architecture overview and verify current +lifecycle details against scst/include/scst.h and +scst/src/dev_handlers/scst_user.c. User space API @@ -50,21 +51,23 @@ following system calls available: Device /dev/scst_user could be opened in blocking or non-blocking mode using O_NONBLOCK flag. In the blocking mode ioctl() -SCST_USER_REPLY_GET_CMD function blocks until there is a new subcommand +SCST_USER_REPLY_AND_GET_CMD function blocks until there is a new subcommand to process. In the non-blocking mode if there are no pending subcommands -SCST_USER_REPLY_GET_CMD function returns immediately with EAGAIN error +SCST_USER_REPLY_AND_GET_CMD function returns immediately with EAGAIN error code, and the user space device handler can use poll() call to get notification about new subcommands arrival. The blocking mode is the default. -The module scst_user API is defined in scst_user.h file. +The module scst_user ABI is defined by scst/include/scst_user.h. Structure +definitions in this document are explanatory copies; use the installed header +when building a handler and update this document whenever that header changes. IOCTL() functions

-There are following IOCTL functions available. All of them has one -argument. They all, except SCST_USER_REGISTER_DEVICE return 0 for -success or -1 in case of error, and errno is set appropriately. +The following IOCTL functions are available. Their arguments, where +applicable, are described below. They return 0 for success or -1 in case +of error, and errno is set appropriately. SCST_USER_REGISTER_DEVICE @@ -82,12 +85,12 @@ struct scst_user_dev_desc uint8_t sgv_disable_clustered_pool; int32_t sgv_single_alloc_pages; int32_t sgv_purge_interval; - uint8_t has_own_order_mgmt; struct scst_user_opt opt; uint32_t block_size; + uint8_t enable_pr_cmds_notifications; char name[SCST_MAX_NAME]; char sgv_name[SCST_MAX_NAME]; -}, +}; where: @@ -96,11 +99,9 @@ where: 0, then the SGV cache will work in the fixed size buffers mode. In this case it sets - the size of each buffer in pages. See the SGV cache documentation - (http://scst.sourceforge.net/sgv_cache.txt) for more details. + the size of each buffer in pages. See the SGV cache section in + doc/scst_pg.sgml for more details. -SCST_USER_REGISTER_DEVICE returns registered device's handler or -1 -in case of error, and errno is set appropriately. +SCST_USER_REGISTER_DEVICE returns 0 on success or -1 in case of error, +and errno is set appropriately. The registration is associated with the +open file descriptor. -In order to unregister the device, either call SCST_USER_UNREGISTER_DEVICE -function, or close on its file descriptor. +Close the device's file descriptor to unregister it. SCST_USER_UNREGISTER_DEVICE @@ -171,16 +171,20 @@ struct scst_user_opt uint8_t on_free_cmd_type; uint8_t memory_reuse_type; uint8_t partial_transfers_type; - uint32_t partial_len; + int32_t partial_len; uint8_t tst; + uint8_t tmf_only; uint8_t queue_alg; + uint8_t qerr; uint8_t tas; uint8_t swp; uint8_t d_sense; uint8_t has_own_order_mgmt; -}, + + uint8_t ext_copy_remap_supported; +}; where: @@ -233,18 +237,12 @@ where: - - Flags where: @@ -350,8 +352,9 @@ struct scst_user_get_multi { int16_t replies_cnt; int16_t replies_done; int16_t cmds_cnt; - struct scst_user_get_cmd cmds[0]; -}, + int16_t pad; + struct scst_user_get_cmd cmds[]; +}; where: @@ -396,8 +399,9 @@ struct scst_user_reply_cmd struct scst_user_scsi_cmd_reply_parse parse_reply; struct scst_user_scsi_cmd_reply_alloc_mem alloc_reply; struct scst_user_scsi_cmd_reply_exec exec_reply; + struct scst_user_ext_copy_reply_remap remap_reply; }; -}, +}; where: @@ -417,20 +421,33 @@ where: struct scst_user_scsi_cmd_reply_parse { - uint8_t queue_type; - uint8_t data_direction; - uint16_t cdb_len; - aligned_i64 lba; - uint32_t op_flags; - aligned_i64 data_len; - int32_t bufflen; - int32_t out_bufflen; -}, + uint8_t status; + union { + struct { + uint8_t queue_type; + uint8_t data_direction; + uint16_t cdb_len; + aligned_i64 lba; + aligned_i64 data_len; + int32_t bufflen; + uint32_t op_flags; + int32_t out_bufflen; + }; + struct { + uint8_t sense_len; + aligned_u64 psense_buffer; + }; + }; +}; where: + struct scst_user_scsi_cmd_reply_alloc_mem { - uint64_t pbuf; -}, + aligned_u64 pbuf; +}; where: @@ -477,22 +499,41 @@ where: +struct scst_user_data_descriptor +{ + aligned_u64 usdd_lba; + aligned_u64 usdd_blocks; +}; + struct scst_user_scsi_cmd_reply_exec { int32_t resp_data_len; - uint64_t pbuf; + aligned_u64 pbuf; uint8_t reply_type; uint8_t status; - uint8_t sense_len; - aligned_u64 psense_buffer; -}, + union { + struct { + uint8_t sense_len; + aligned_u64 psense_buffer; + }; + struct { + uint16_t ws_descriptors_len; + aligned_u64 ws_descriptors; + }; + }; +}; where: + @@ -574,7 +625,7 @@ SCST_USER_GET_EXTENDED_CDB has the following arguments: struct scst_user_get_ext_cdb { uint32_t cmd_h; aligned_u64 ext_cdb_buffer; -}, +}; where: @@ -600,7 +651,7 @@ It has the following arguments: union scst_user_prealloc_buffer { struct scst_user_prealloc_buffer_in in; struct scst_user_prealloc_buffer_out out; -}, +}; where: @@ -618,7 +669,7 @@ struct scst_user_prealloc_buffer_in { aligned_u64 pbuf; uint32_t bufflen; uint8_t for_clust_pool; -}, +}; where: @@ -638,7 +689,7 @@ Structure struct scst_user_prealloc_buffer_out { uint32_t cmd_h; -} +}; where: @@ -663,15 +714,15 @@ contains struct scst_user_sess, which is defined as the following: struct scst_user_sess { - uint64_t sess_h; - uint64_t lun; + aligned_u64 sess_h; + aligned_u64 lun; uint16_t threads_num; uint8_t rd_only; uint16_t scsi_transport_version; uint16_t phys_transport_version; - char initiator_name[SCST_MAX_NAME]; - char target_name[SCST_MAX_NAME]; -}, + char initiator_name[SCST_MAX_EXTERNAL_NAME]; + char target_name[SCST_MAX_EXTERNAL_NAME]; +}; where: @@ -739,17 +790,17 @@ the following: struct scst_user_scsi_cmd_parse { - uint64_t sess_h; + aligned_u64 sess_h; uint8_t cdb[SCST_MAX_CDB_SIZE]; uint16_t cdb_len; aligned_i64 lba; - uint32_t timeout; - int32_t bufflen; aligned_i64 data_len; + int32_t bufflen; int32_t out_bufflen; + int32_t timeout; uint32_t op_flags; @@ -762,7 +813,7 @@ struct scst_user_scsi_cmd_parse int32_t expected_out_transfer_len; uint32_t sn; -}, +}; where: @@ -808,11 +859,15 @@ where: -Bits of scst_cdb_flags can be: +The bit values are defined by enum scst_cdb_flags in +scst/include/scst_const.h. Treat that header as authoritative; the common +parsing flags below are not an exhaustive copy of the enum: In the PARSE state of SCSI commands processing the user space device @@ -863,7 +923,7 @@ as the following: struct scst_user_scsi_cmd_alloc_mem { - uint64_t sess_h; + aligned_u64 sess_h; uint8_t cdb[SCST_MAX_CDB_SIZE]; uint16_t cdb_len; @@ -874,7 +934,7 @@ struct scst_user_scsi_cmd_alloc_mem uint8_t data_direction; uint32_t sn; -}, +}; where: @@ -912,8 +972,7 @@ caches unused SG vectors for future allocations to improve performance. Then, after some time of inactivity or when the system is under memory pressure, the cache entries will be freed and the user space handler will be notified using SCST_USER_ON_CACHED_MEM_FREE. See the SGV cache -documentation for more -details. +section in doc/scst_pg.sgml for more details. Since the SGV cache caches SG vectors, which can be bigger, than actual data sizes of SCSI commands, alloc_len field could also be bigger, than @@ -969,7 +1028,7 @@ following: struct scst_user_scsi_cmd_exec { - uint64_t sess_h; + aligned_u64 sess_h; uint8_t cdb[SCST_MAX_CDB_SIZE]; uint16_t cdb_len; @@ -979,11 +1038,11 @@ struct scst_user_scsi_cmd_exec aligned_i64 data_len; int32_t bufflen; int32_t alloc_len; - uint64_t pbuf; + aligned_u64 pbuf; uint8_t queue_type; uint8_t data_direction; uint8_t partial; - uint32_t timeout; + int32_t timeout; aligned_u64 p_out_buf; int32_t out_bufflen; @@ -993,7 +1052,7 @@ struct scst_user_scsi_cmd_exec uint32_t parent_cmd_h; int32_t parent_cmd_data_len; uint32_t partial_offset; -}, +}; where: @@ -1024,14 +1083,8 @@ where: - + It is guaranteed that only commands of the same queue_type per session can be returned simultaneously. In case of any error it should be reported via appropriate SAM status -and sense. If it happens for a subcommand of a partial data transfers -command, all other subcommands of this command, which already passed the -the user space handler or will be passed in the future, will be aborted -by scst_user, the user space handler should ignore them. +and sense. + + + SCST_USER_EXT_COPY_REMAP + +

+SCST_USER_EXT_COPY_REMAP asks a handler that advertised +ext_copy_remap_supported to remap an EXTENDED COPY segment. The payload +contains the current device session, source and destination sessions, and the +range to remap: + + +struct scst_user_ext_copy_data_descr +{ + aligned_u64 src_lba; + aligned_u64 dst_lba; + int32_t data_len; +}; + +struct scst_user_ext_copy_remap +{ + aligned_u64 sess_h; + aligned_u64 src_sess_h; + aligned_u64 dst_sess_h; + struct scst_user_ext_copy_data_descr data_descr; +}; + + +where: + + + + +The corresponding reply payload is: + + +struct scst_user_ext_copy_reply_remap +{ + aligned_u64 remap_descriptors; + uint16_t remap_descriptors_len; + uint8_t status; + uint8_t sense_len; + aligned_u64 psense_buffer; +}; + + + + SCST_USER_ON_FREE_CMD @@ -1081,13 +1196,13 @@ as the following: struct scst_user_scsi_on_free_cmd { - uint64_t pbuf; + aligned_u64 pbuf; int32_t resp_data_len; uint8_t buffer_cached; uint8_t aborted; uint8_t status; uint8_t delivery_status; -}, +}; where: @@ -1111,6 +1226,8 @@ where: @@ -1129,10 +1246,10 @@ Payload contains struct scst_user_on_cached_mem_free, which is defined as the following: -struct scst_user_scsi_cmd_alloc_mem +struct scst_user_on_cached_mem_free { - uint64_t pbuf; -}, + aligned_u64 pbuf; +}; where: @@ -1152,12 +1269,12 @@ is defined as the following: struct scst_user_tm { - uint64_t sess_h; + aligned_u64 sess_h; uint32_t fn; uint32_t cmd_h_to_abort; uint32_t cmd_sn; uint8_t cmd_sn_set; -}, +}; where: diff --git a/usr/events/README b/usr/events/README index e354bb4dd..e9cdfad56 100644 --- a/usr/events/README +++ b/usr/events/README @@ -1,2 +1,43 @@ -This is test/demo application for SCST events subsystem. For more -information about this subsystem, see scst_event.h file. +SCST events example +=================== + +events is a test and demonstration consumer for the SCST event ABI. The +authoritative ioctl definitions, wildcard rules, event codes, and payload +layouts are in scst/include/scst_event.h and scst/src/scst_event.c. The +example's current behavior is implemented in usr/events/events.c. + +Build +----- + +The example is intentionally omitted from the default "make usr" build and +install targets. Build it explicitly with "make -C usr/events". Installing or +running it is not required for normal SCST operation. + +Usage +----- + +events opens /dev/scst_event, registers one or more filters, and prints the +events it receives. + +events [OPTIONS] + + - -e, --allowed_event=CODE: allow CODE; zero matches any event code + - -i, --allowed_issuer=ISSUER: allow ISSUER; "*" matches any issuer + - -n, --non_blocking: use nonblocking event retrieval + - -v, --version: print the program version + - -h, --help: print usage information + +The -d/--debug option is present only when tracing or debugging is compiled +in. Repeat an event-code/issuer pair to add filters, up to the implementation +limit. With no filter options, the program subscribes to every event code from +every issuer. + +Safety +------ + +Do not run this example on a live target or alongside a production event +consumer such as stpgd. Its default wildcard subscription can receive an +event that requires SCST_EVENT_NOTIFY_DONE, while the example only sends a +completion for its synthetic event code 0x12345. A real completion-bearing +event can therefore be diverted from its intended consumer and remain pending +until the kernel timeout. diff --git a/usr/fileio/README b/usr/fileio/README index 0e9926469..6a35de21b 100644 --- a/usr/fileio/README +++ b/usr/fileio/README @@ -2,82 +2,69 @@ User space FILEIO handler ========================= Version 3.11.0-pre, 29 December 2025 ----------------------------- +------------------------------------ -User space program fileio_tgt uses interface of SCST's scst_user dev -handler and allows to see how it works in various modes. Fileio_tgt -provides mostly the same functionality as the kernel space SCST's -scst_vdisk handler with the only exceptions that it supports O_DIRECT -mode and doesn't support BLOCKIO one. O_DIRECT mode is basically the -same as BLOCKIO, but also supports files, so for some loads it could be -significantly faster, than the regular FILEIO access. All the words -about BLOCKIO mode from SCST's README file apply to O_DIRECT mode as -well. +fileio_tgt is a user-space disk handler that demonstrates and exercises the +SCST scst_user interface. The interface is defined by +scst/include/scst_user.h and documented in doc/scst_user_spec.sgml. -Installation ------------- +Build +----- -Make sure you have installed SCST core, if not, see its README file for -details how to do it. - -Then run "make all install" - -For the best performance make sure you don't have any debug options -(i.e. lines like "CFLAGS += -DDEBUG" containing "-DDEBUG*" string) -enabled in the Makefile. +From the repository root, "make usr" builds fileio_tgt and stpgd. To build +only this program, run "make -C usr/fileio". Installation is a separate, +privileged operation; use the corresponding install target only when system +installation is intended. Usage ----- -Load scst_user module using modprobe or insmod, then start fileio_tgt -program. It can be used as the following: +The scst_user kernel module must be loaded before fileio_tgt is started. +Running fileio_tgt registers live SCST devices and opens every supplied +backing path. Unless read-only mode is selected, initiators can cause writes +to those paths. Do not use a real block device or valuable backing file +without an explicitly prepared test configuration. -fileio_tgt [OPTION] name path +fileio_tgt [OPTIONS] name path [name path] ... -Where: - - - OPTION - one or several not required options, see below. - - - name - name of the virtual device as it will be seen by SCST and used - in the subsequent access management using the corresponding SCST interface. - - - path - path to the device file. +Each name is the SCST device name used by later access-management operations. +Each path is the backing file or device from which the handler determines the +virtual device size. Up to the implementation limit, multiple name/path pairs +can be registered by one process. The following options are supported: - -b or --block=size: block size, must be power of 2 and >=512 + - -b, --block=size: block size; it must be a power of two and at least 512 + - -e, --threads=count: number of worker threads + - -t, --write_through: write-through mode + - -r, --read_only: read-only mode + - -o, --direct: open the backing path with O_DIRECT for data I/O + - -n, --nullio: NULLIO mode; see scst/README for its safety warning + - -c, --nv_cache: NV_CACHE mode; see scst/README for its semantics + - -p, --parse=type: "std" (default), "call", or "excpt" + - -f, --on_free=type: "ignore" (default) or "call" + - -m, --mem_reuse=type: "all" (default), "read", "write", or "none" + - -l, --non_blocking: use nonblocking scst_user operations + - -I, --vdisk_id=ID: virtual disk ID for multi-target setups + - -F, --flush=n: flush the SGV cache every n seconds + - -s, --sgv_shared: share the first device's SGV cache + - -S, --sgv_single_cache=n: use a single-entry SGV cache with n pages + - -P, --sgv_purge_interval=n: set the SGV purge interval in seconds + - -u, --unreg_before_close: exercise the obsolete unregister ioctl + - -D, --sgv_disable_clustered_pool: disable the clustered SGV pool + - -R, --prealloc_buffers=n: preallocate n buffers + - -Z, --prealloc_buffer_size=n: preallocated buffer size in KiB + - -M, --multi_cmd=v: enable or disable multi-command processing (default 1) + - -v, --version: print the program version + - -h, --help: print usage information - -e or --threads=count: number of threads +The following options are present only when their corresponding tracing or +debug feature is compiled in: - -t or --write_through: write through mode + - -d, --debug=level: tracing level + - -g, --debug_tm_ignore: enable the DEBUG_TM_IGNORE test feature - -r or --read_only: read only - - -o or --direct: O_DIRECT mode, see above for details - - -n or --nullio: NULLIO mode, see SCST's README file for details - - -c or --nv_cache: NV_CACHE mode, see SCST's README file for details - - -p or --parse=type: parse type, one of "std" (default), "call" or "excpt" - - -f or --on_free=type: on free call type, one of "ignore" (default) or "call" - - -m or --mem_reuse=type: Memory reuse type, one of "all" (default), "read", - "write" or "none" - - -s or --prio_thread: Use separate thread for mgmt (prio) commands - - -l or --non_blocking: Use non-blocking operations - -Also in the debug builds the following options are supported: - - -d or --debug=level: debug tracing level - - -g or --debug_tm_ignore: turn on DEBUG_TM_IGNORE, one of the task management - debugging features - -If you don't understand some these options, don't use them, default -values provide the best performance. +The current option parser and usage() output in usr/fileio/fileio.c are the +authoritative command-line reference. Vladislav Bolkhovitin , http://scst.sourceforge.net diff --git a/usr/stpgd/README b/usr/stpgd/README index 64b5ad98b..18e5bc05c 100644 --- a/usr/stpgd/README +++ b/usr/stpgd/README @@ -1,9 +1,61 @@ -This is stpgd service called by the SCST core upon receiving SET TARGET -PORT GROUPS command via SCST events subsystem. In turn, stpgd calls -external script to actually modify ALUA state as requested by the SET -TARGET PORT GROUPS command via cluster manager and SCST sysfs. You can -find example of such script in scst_on_stpg file. +stpgd explicit ALUA helper +========================== -Reason why such dual stage approach is used is, because there is no way -from inside the kernel to control execution of external programs and -there is no way to write a service calling IOCTLs on shell. +stpgd is the user-space consumer for explicit ALUA state-transition events. +It opens the SCST event device, subscribes to events issued by the SCST core, +and handles SCST_EVENT_STPG_USER_INVOKE by executing an external helper once +for each target-port-group descriptor. The helper's exit status is reported +back to the kernel through SCST_EVENT_NOTIFY_DONE. + +The event payload and ioctl ABI are defined in scst/include/scst_event.h. +The current event filtering, helper invocation, timeout handling, and +command-line parsing are implemented in usr/stpgd/stpgd_main.c. + +Build and invocation +-------------------- + +From the repository root, "make usr" builds stpgd and fileio_tgt. To build +only this daemon, run "make -C usr/stpgd". Installation is a separate, +privileged operation. + +stpgd [OPTIONS] + + - -f, --foreground: do not daemonize + - -p, --path=PATH: absolute path to the executable transition helper + - -t, --timeout=SECONDS: helper timeout; the default is 17 seconds + - -v, --version: print the program version + - -h, --help: print usage information + +The -d/--debug option is present only when tracing or debugging is compiled +in. Without -p, the daemon uses /usr/local/bin/scst/scst_on_stpg. If a +different installation prefix is used, pass the installed helper path +explicitly. + +Helper interface +---------------- + +The helper receives these five positional arguments, in order: + + 1. SCST device name + 2. previous ALUA state + 3. requested ALUA state + 4. device-group name + 5. target-group name + +The same values are supplied as SCST_DEVICE_NAME, SCST_PREV_ALUA_STATE, +SCST_ALUA_STATE, SCST_DEVICE_GROUP, and SCST_TARGET_GROUP in the helper's +environment. The daemon supplies a restricted PATH containing /bin, +/usr/bin, /sbin, and /usr/sbin. + +Safety +------ + +usr/stpgd/scst_on_stpg is only a minimal example. It writes the requested +state directly to the target group's live SCST sysfs attribute; it does not +coordinate replication, fencing, failover, or cluster-manager state. Do not +run stpgd or deploy that helper until the transition workflow has been adapted +and validated for the exact hosts, storage, target groups, and HA stack. + +SCST blocks the affected devices around the explicit transition event, as +described in the explicit ALUA section of scst/README. The external helper is +still responsible for performing every site-specific transition safely.