diff --git a/AskingQuestions b/AskingQuestions index 1d9441c45..99fedfcc3 100644 --- a/AskingQuestions +++ b/AskingQuestions @@ -107,7 +107,7 @@ For example, this will cause problems in the following situation: 7. scst_send_to_midlev returns with SCST_CMD_STATE_RES_NEED_THREAD 8. Now, the scst_process_active_cmd will try to reschedule command 0x8000100 which is already destroyed at this point ! - + Can anyone on the list confirm my guess? Or, this situation should never happen because of some other condition which I may have missed? Right now I can't think of any of simple methods to work around the issue, @@ -124,104 +124,104 @@ I have two machines (SCST targets) with the following parameters: - Linux 2.6.21.3, running in 64 bit mode with 16G RAM - SCST trunk version -On the client side there is a Solaris 10 U3 machine, with the same (chip +On the client side there is a Solaris 10 U3 machine, with the same (chip wise) Qlogic controller. -There is an FC switch between the three machines, and each of the -targets are zoned to the client's port in a one-by-one manner, so HBA +There is an FC switch between the three machines, and each of the +targets are zoned to the client's port in a one-by-one manner, so HBA port 1 sees only target 1 and port 2 sees only target 2. -The targets are configured with two large sparse files on XFS (8 TB +The targets are configured with two large sparse files on XFS (8 TB each, with dd if=/dev/zero of=file bs=1M count=0 seek=8388608). -In Solaris I do various tests with SVM (Sun's built in volume manager) +In Solaris I do various tests with SVM (Sun's built in volume manager) and multiterabyte UFS. Occasionally, there are some strange write errors, where the volume manager drops its volumes and without a VM, a simple UFS fs write can fail too. -I see various errors logged by the kernel (Solaris'), these are some +I see various errors logged by the kernel (Solaris'), these are some examples, both with and without SVM: -Jun 21 10:42:14 solaris fctl: [ID 517869 kern.warning] WARNING: +Jun 21 10:42:14 solaris fctl: [ID 517869 kern.warning] WARNING: fp(1)::GPN_ID for D_ID=621200 failed -Jun 21 10:42:14 solaris fctl: [ID 517869 kern.warning] WARNING: -fp(1)::N_x Port with D_ID=621200, PWWN=210000e08b944419 disappeared from +Jun 21 10:42:14 solaris fctl: [ID 517869 kern.warning] WARNING: +fp(1)::N_x Port with D_ID=621200, PWWN=210000e08b944419 disappeared from fabric -Jun 21 10:42:53 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 10:42:53 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): -Jun 21 10:42:53 solaris SCSI transport failed: reason +Jun 21 10:42:53 solaris SCSI transport failed: reason 'tran_err': retrying command -Jun 21 10:43:06 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 10:43:06 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): -Jun 21 10:43:06 solaris SCSI transport failed: reason 'timeout': +Jun 21 10:43:06 solaris SCSI transport failed: reason 'timeout': retrying command Jun 21 10:43:13 solaris scsi: [ID 107833 kern.notice] Device is gone -Jun 21 10:43:13 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 10:43:13 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 10:43:13 solaris transport rejected fatal error -Jun 21 10:43:13 solaris md_stripe: [ID 641072 kern.warning] WARNING: md: +Jun 21 10:43:13 solaris md_stripe: [ID 641072 kern.warning] WARNING: md: d10: write error on /dev/dsk/c2t210000E08B944419d0s6 Jun 21 10:43:13 solaris last message repeated 9 times -Jun 21 10:43:13 solaris scsi: [ID 243001 kern.info] +Jun 21 10:43:13 solaris scsi: [ID 243001 kern.info] /pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0 (fcp1): -Jun 21 10:43:13 solaris offlining lun=0 (trace=0), target=621200 +Jun 21 10:43:13 solaris offlining lun=0 (trace=0), target=621200 (trace=2800004) -Jun 21 10:43:13 solaris ufs: [ID 702911 kern.warning] WARNING: Error +Jun 21 10:43:13 solaris ufs: [ID 702911 kern.warning] WARNING: Error writing master during ufs log roll -Jun 21 10:43:13 solaris ufs: [ID 127457 kern.warning] WARNING: ufs log +Jun 21 10:43:13 solaris ufs: [ID 127457 kern.warning] WARNING: ufs log for /mnt changed state to Error -Jun 21 10:43:13 solaris ufs: [ID 616219 kern.warning] WARNING: Please +Jun 21 10:43:13 solaris ufs: [ID 616219 kern.warning] WARNING: Please umount(1M) /mnt and run fsck(1M) -Jun 21 11:08:55 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:08:55 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:08:55 solaris offline or reservation conflict -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris offline or reservation conflict -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris offline or reservation conflict -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris i/o to invalid geometry -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris offline or reservation conflict -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris i/o to invalid geometry -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris offline or reservation conflict -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris i/o to invalid geometry -Jun 21 11:09:43 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:43 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:43 solaris offline or reservation conflict -Jun 21 11:09:43 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:43 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:43 solaris SYNCHRONIZE CACHE command failed (5) I don't see anything in the dmesg on the target side. -After these errors SCST seems to be dead. I can't unload its modules and +After these errors SCST seems to be dead. I can't unload its modules and can't communicate it via /proc. A simple cat vdisk just waits and waits. -Could you please help? What should I set/collect/send in this case to +Could you please help? What should I set/collect/send in this case to help resolving this issue? ====================================================================== diff --git a/Makefile b/Makefile index 0652efe09..f34290161 100644 --- a/Makefile +++ b/Makefile @@ -308,7 +308,7 @@ emulex_install: emulex_uninstall: cd $(EMULEX_DIR) && $(MAKE) uninstall -emulex_clean: +emulex_clean: cd $(EMULEX_DIR) && $(MAKE) clean emulex_extraclean: @@ -323,7 +323,7 @@ lsi_install: lsi_uninstall: cd $(LSI_DIR) && $(MAKE) uninstall -lsi_clean: +lsi_clean: cd $(LSI_DIR) && $(MAKE) clean lsi_extraclean: @@ -368,7 +368,7 @@ usr_install: usr_uninstall: cd $(USR_DIR) && $(MAKE) uninstall -usr_clean: +usr_clean: cd $(USR_DIR) && $(MAKE) clean usr_extraclean: diff --git a/fcst/Makefile b/fcst/Makefile index ca25c5dd4..357a6c897 100644 --- a/fcst/Makefile +++ b/fcst/Makefile @@ -4,12 +4,12 @@ # # Copyright (C) 2006 - 2008 Jacky Feng # Copyright (C) 2011 - 2018 Bart Van Assche -# +# # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation, version 2 # of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the @@ -20,8 +20,8 @@ # all (the default) : make all # clean : clean files # extraclean : clean + clean dependencies -# install : install -# uninstall : uninstall +# install : install +# uninstall : uninstall # # Notes : # - install and uninstall must be made as root diff --git a/iscsi-scst/AskingQuestions b/iscsi-scst/AskingQuestions index 1d9441c45..99fedfcc3 100644 --- a/iscsi-scst/AskingQuestions +++ b/iscsi-scst/AskingQuestions @@ -107,7 +107,7 @@ For example, this will cause problems in the following situation: 7. scst_send_to_midlev returns with SCST_CMD_STATE_RES_NEED_THREAD 8. Now, the scst_process_active_cmd will try to reschedule command 0x8000100 which is already destroyed at this point ! - + Can anyone on the list confirm my guess? Or, this situation should never happen because of some other condition which I may have missed? Right now I can't think of any of simple methods to work around the issue, @@ -124,104 +124,104 @@ I have two machines (SCST targets) with the following parameters: - Linux 2.6.21.3, running in 64 bit mode with 16G RAM - SCST trunk version -On the client side there is a Solaris 10 U3 machine, with the same (chip +On the client side there is a Solaris 10 U3 machine, with the same (chip wise) Qlogic controller. -There is an FC switch between the three machines, and each of the -targets are zoned to the client's port in a one-by-one manner, so HBA +There is an FC switch between the three machines, and each of the +targets are zoned to the client's port in a one-by-one manner, so HBA port 1 sees only target 1 and port 2 sees only target 2. -The targets are configured with two large sparse files on XFS (8 TB +The targets are configured with two large sparse files on XFS (8 TB each, with dd if=/dev/zero of=file bs=1M count=0 seek=8388608). -In Solaris I do various tests with SVM (Sun's built in volume manager) +In Solaris I do various tests with SVM (Sun's built in volume manager) and multiterabyte UFS. Occasionally, there are some strange write errors, where the volume manager drops its volumes and without a VM, a simple UFS fs write can fail too. -I see various errors logged by the kernel (Solaris'), these are some +I see various errors logged by the kernel (Solaris'), these are some examples, both with and without SVM: -Jun 21 10:42:14 solaris fctl: [ID 517869 kern.warning] WARNING: +Jun 21 10:42:14 solaris fctl: [ID 517869 kern.warning] WARNING: fp(1)::GPN_ID for D_ID=621200 failed -Jun 21 10:42:14 solaris fctl: [ID 517869 kern.warning] WARNING: -fp(1)::N_x Port with D_ID=621200, PWWN=210000e08b944419 disappeared from +Jun 21 10:42:14 solaris fctl: [ID 517869 kern.warning] WARNING: +fp(1)::N_x Port with D_ID=621200, PWWN=210000e08b944419 disappeared from fabric -Jun 21 10:42:53 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 10:42:53 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): -Jun 21 10:42:53 solaris SCSI transport failed: reason +Jun 21 10:42:53 solaris SCSI transport failed: reason 'tran_err': retrying command -Jun 21 10:43:06 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 10:43:06 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): -Jun 21 10:43:06 solaris SCSI transport failed: reason 'timeout': +Jun 21 10:43:06 solaris SCSI transport failed: reason 'timeout': retrying command Jun 21 10:43:13 solaris scsi: [ID 107833 kern.notice] Device is gone -Jun 21 10:43:13 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 10:43:13 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 10:43:13 solaris transport rejected fatal error -Jun 21 10:43:13 solaris md_stripe: [ID 641072 kern.warning] WARNING: md: +Jun 21 10:43:13 solaris md_stripe: [ID 641072 kern.warning] WARNING: md: d10: write error on /dev/dsk/c2t210000E08B944419d0s6 Jun 21 10:43:13 solaris last message repeated 9 times -Jun 21 10:43:13 solaris scsi: [ID 243001 kern.info] +Jun 21 10:43:13 solaris scsi: [ID 243001 kern.info] /pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0 (fcp1): -Jun 21 10:43:13 solaris offlining lun=0 (trace=0), target=621200 +Jun 21 10:43:13 solaris offlining lun=0 (trace=0), target=621200 (trace=2800004) -Jun 21 10:43:13 solaris ufs: [ID 702911 kern.warning] WARNING: Error +Jun 21 10:43:13 solaris ufs: [ID 702911 kern.warning] WARNING: Error writing master during ufs log roll -Jun 21 10:43:13 solaris ufs: [ID 127457 kern.warning] WARNING: ufs log +Jun 21 10:43:13 solaris ufs: [ID 127457 kern.warning] WARNING: ufs log for /mnt changed state to Error -Jun 21 10:43:13 solaris ufs: [ID 616219 kern.warning] WARNING: Please +Jun 21 10:43:13 solaris ufs: [ID 616219 kern.warning] WARNING: Please umount(1M) /mnt and run fsck(1M) -Jun 21 11:08:55 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:08:55 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:08:55 solaris offline or reservation conflict -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris offline or reservation conflict -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris offline or reservation conflict -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris i/o to invalid geometry -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris offline or reservation conflict -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris i/o to invalid geometry -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris offline or reservation conflict -Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:41 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:41 solaris i/o to invalid geometry -Jun 21 11:09:43 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:43 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:43 solaris offline or reservation conflict -Jun 21 11:09:43 solaris scsi: [ID 107833 kern.warning] WARNING: -/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 +Jun 21 11:09:43 solaris scsi: [ID 107833 kern.warning] WARNING: +/pci@1,0/pci1022,7450@a/pcie11,105@1,1/fp@0,0/disk@w210000e08b944419,0 (sd1): Jun 21 11:09:43 solaris SYNCHRONIZE CACHE command failed (5) I don't see anything in the dmesg on the target side. -After these errors SCST seems to be dead. I can't unload its modules and +After these errors SCST seems to be dead. I can't unload its modules and can't communicate it via /proc. A simple cat vdisk just waits and waits. -Could you please help? What should I set/collect/send in this case to +Could you please help? What should I set/collect/send in this case to help resolving this issue? ====================================================================== diff --git a/iscsi-scst/README b/iscsi-scst/README index a8dbc6f77..37bd611dd 100644 --- a/iscsi-scst/README +++ b/iscsi-scst/README @@ -981,7 +981,7 @@ Background information about zero-copy data sending As explained above the most efficient operation of the iSCSI-SCST target driver is achieved when the following two conditions are met: * Data is sent from target to initiator in a zero-copy fashion. -* Data buffers are cached for reuse (by the so-called sgv pool). +* Data buffers are cached for reuse (by the so-called sgv pool). Unfortunately the zero-copy API in the Linux kernel (proto.sendpage() / tcp_sendpage()) does not yet support completion notifications. Hence the put_page_callback patch which adds completion notification support to @@ -999,7 +999,7 @@ Notes: pool to allocate a data buffer. * Zero-copy sending is only possible with network interface drivers that support scatter/gather and checksumming (NETIF_F_SG and NETIF_F_ALL_CSUM - respectively). + respectively). References: [1] James Bottomley, Linux Kernel Mailing List, December 2008, diff --git a/iscsi-scst/ToDo b/iscsi-scst/ToDo index 0d942d7cd..36f4eee62 100644 --- a/iscsi-scst/ToDo +++ b/iscsi-scst/ToDo @@ -7,7 +7,7 @@ same structure. This architecture decision derived from IET and makes the code a lot less readable, maintainable and effective as well as more errors prone, so it must be corrected. - + - Code beautifying, i.e. make it be written in the same nice looking style. Particularly, all functions and variables names should share the same style. diff --git a/iscsi-scst/doc/SCST_Gentoo_HOWTO.txt b/iscsi-scst/doc/SCST_Gentoo_HOWTO.txt index e79ed917b..b83c80c1b 100644 --- a/iscsi-scst/doc/SCST_Gentoo_HOWTO.txt +++ b/iscsi-scst/doc/SCST_Gentoo_HOWTO.txt @@ -36,11 +36,11 @@ a) Latest development version cd /root svn checkout svn://svn.code.sf.net/p/scst/svn/trunk scst -b) Specific branch (3.0 used in example) +b) Specific branch (3.0 used in example) cd /root svn checkout svn://svn.code.sf.net/p/scst/svn/branches/3.0.x scst - + 2. Build SCST ! Important: SCST 3.1.x and trunk: Gentoo Hardened kernels uses diff --git a/iscsi-scst/doc/iscsi-scst-howto.txt b/iscsi-scst/doc/iscsi-scst-howto.txt index 73d5fd50d..68f91b073 100644 --- a/iscsi-scst/doc/iscsi-scst-howto.txt +++ b/iscsi-scst/doc/iscsi-scst-howto.txt @@ -398,7 +398,7 @@ Device (host:ch:id:lun or name) Device handler LUN masking ----------- -SCST defines security groups. For each group, you can add LUNs. +SCST defines security groups. For each group, you can add LUNs. If you want to have all LUNs available for all targets, just add them to the "Default" group: diff --git a/iscsi-scst/doc/manpages/iscsi-scst-adm.8 b/iscsi-scst/doc/manpages/iscsi-scst-adm.8 index 39c597673..bb73242e5 100644 --- a/iscsi-scst/doc/manpages/iscsi-scst-adm.8 +++ b/iscsi-scst/doc/manpages/iscsi-scst-adm.8 @@ -137,7 +137,7 @@ iscsi-scst-adm \- iSCSI SCST Target Administration Utility. \&\fBiscsi-scst-adm \-\-op [operation] \-\-tid=[id] [\-\-sid [id]] [\-\-params [key=value,...]]\fR .SH "DESCRIPTION" .IX Header "DESCRIPTION" -iscsi-scst-adm is used to monitor and modify in real-time the iSCSI SCST Target +iscsi-scst-adm is used to monitor and modify in real-time the iSCSI SCST Target targets. .SH "USAGE" .IX Header "USAGE" diff --git a/iscsi-scst/doc/manpages/iscsi-scstd.8 b/iscsi-scst/doc/manpages/iscsi-scstd.8 index 2ddee95b6..6167a65fe 100644 --- a/iscsi-scst/doc/manpages/iscsi-scstd.8 +++ b/iscsi-scst/doc/manpages/iscsi-scstd.8 @@ -56,7 +56,7 @@ Specify running user id, default is current uid. .SH FILES .I /etc/iscsi-scstd.conf .RS -The system wide configuration file. See +The system wide configuration file. See .BR iscsi-scstd.conf (5) for further details. .RE diff --git a/iscsi-scst/doc/manpages/iscsi-scstd.conf.5 b/iscsi-scst/doc/manpages/iscsi-scstd.conf.5 index 6958b15ec..052d4c3f8 100644 --- a/iscsi-scst/doc/manpages/iscsi-scstd.conf.5 +++ b/iscsi-scst/doc/manpages/iscsi-scstd.conf.5 @@ -1,6 +1,6 @@ .\" Process this file with .\" groff -man -Tascii iscsi-scstd.conf.5 -.\" +.\" .TH "ISCSI_SCSTD.CONF" "5" "May 2007" "A. Lehmann, M. Zhang and A. Redlich" "File formats" .SH "NAME" /etc/iscsi-scstd.conf \- configuration for iSCSI SCST Target Daemon @@ -15,12 +15,12 @@ Only lines starting with `#' are ignored. Putting '#' in the middle of a line is .P The "Yes" and "No" for parameter values are case sensitive. The parameter names are case insensitive. .P -The file consists of a global part and zero or more "Target" stanzas. Everything until the first target definition belongs to the global configuration. +The file consists of a global part and zero or more "Target" stanzas. Everything until the first target definition belongs to the global configuration. Here is an example: IncomingUser joe secret -.br +.br OutgoingUser jack secret2 Target iqn.2007\-05.com.example:storage.disk2.sys1.xyz @@ -46,11 +46,11 @@ Target iqn.2007\-05.com.example:storage.disk2.sys1.xyz Stanzas start with the word "Target" and the target name. This name must be a globally unique name, as defined by the iSCSI standard : the "iSCSI Qualified Name". The daemon brings the targets up in the order listed. .SH "GLOBAL OPTIONS" Global Options are case sensitive. -.TP +.TP .B [IncomingUser ] -The +The .I -and +and .I used during discovery sessions to authenticate iSCSI initiators. Several of those can be specified for discovery. If no .B IncomingUser @@ -60,13 +60,13 @@ HINT: RFC 3720 requires .I to be 12 characters long. This is enforced e.g. by MS Initiator. .RE -.TP +.TP .B [OutgoingUser ] -The +The .I -and +and .I -used during discovery sessions to authenticate the target to initiators. Only one outgoing +used during discovery sessions to authenticate the target to initiators. Only one outgoing .I / combination may be specified. .RS @@ -74,16 +74,16 @@ HINT: RFC 3720 requires .I to be 12 characters long. This is enforced e.g. by MS Initiator. .RE -.TP +.TP .B Target iqn..[:] A target definition and the target name. The targets name (the .B iSCSI Qualified Name ) must be a globally unique name (as defined by the iSCSI standard) and has to start with .I iqn -followed by a single dot. The EUI\-64 form is not supported. +followed by a single dot. The EUI\-64 form is not supported. .I is the date (year and month) at which the domain is valid. This has to be followed by a single dot and the reversed domain name. -The optional +The optional .I \- which is freely selectable \- has to be separated by a single colon. For further details please check the iSCSI spec. @@ -92,7 +92,7 @@ Here is an example: Target iqn.2007\-05.com.example.host:storage.disk2.sys1.xyz .SH "TARGET OPTIONS" Target options are also case sensitive. -.TP +.TP .B [IncomingUser ] The .I @@ -102,55 +102,55 @@ used to authenticate the iSCSI initiators to this target. It may be different fr .B IncomingUser Option, connections are allowed without authentication. A .I -has to be provided, if there is a +has to be provided, if there is a .I given. Specifying several different .B IncomingUser accounts is supported. -.TP +.TP .B [OutgoingUser ] The .I and .I used to authenticate this iSCSI target to initiators. Only one -.B +.B OutgoingUser per target is supported. It may be different from the username and password in section GLOBAL OPTIONS, which is used for discovery. A .I -has to be provided, if there is a +has to be provided, if there is a .I given. -.TP +.TP .B [Alias ] -This assigns an optional +This assigns an optional .I to the target. -.TP +.TP .B [HeaderDigest ] Optional. If set to "CRC32C" and the initiator is configured accordingly, the integrity of an iSCSI PDU's header segments will be protected by a CRC32C checksum. The default is "None". Note that header digests are not supported during discovery sessions. -.TP +.TP .B [DataDigest ] Optional. If set to "CRC32C" and the initiator is configured accordingly, the integrity of an iSCSI PDU's data segment will be protected by a CRC32C checksum. The default is "None". Note that data digests are not supported during discovery sessions. -.TP +.TP .B [MaxConnections ] Optional. The number of connections within a session. Has to be set to "1" (in words: one), which is also the default since MC/S is not supported. -.TP +.TP .B [InitialR2T ] Optional. If set to "Yes", the initiator has to wait for the target to solicit SCSI data before sending it. Setting it to "No" (default) allows the initiator to send a burst of .B FirstBurstLength bytes unsolicited right after and/or (depending on the setting of .B ImmediateData ) together with the command. Thus setting it to "No" may improve performance. -.TP +.TP .B [ImmediateData ] Optional. This allows the initiator to append unsolicited data to a command. To achieve better performance, this should be set to "Yes". Which is the default. -.TP +.TP .B [MaxRecvDataSegmentLength ] -Optional. Sets the maximum data segment length that can be received. The +Optional. Sets the maximum data segment length that can be received. The .I should be set to multiples of PAGE_SIZE. Configuring too large values may lead to problems allocating sufficient memory, which in turn may lead to SCSI commands timing out at the initiator host. The default value is the highes possible for current platform (1 or 2 MB). -.TP +.TP .B [MaxXmitDataSegmentLength ] Optional. Sets the maximum data segment length that can be sent. The .I @@ -158,15 +158,15 @@ actually used is the minimum of .B MaxXmitDataSegmentLength and the .B MaxRecvDataSegmentLength -announced by the initiator. The +announced by the initiator. The .I should be set to multiples of PAGE_SIZE. Configuring too large values may lead to problems allocating sufficient memory, which in turn may lead to SCSI commands timing out at the initiator host. The default value is the highes possible for current platform (1 or 2 MB). -.TP +.TP .B [MaxBurstLength ] -Optional. Sets the maximum amount of either unsolicited or solicited data the initiator may send in a single burst. Any amount of data exceeding this value must be explicitly solicited by the target. The +Optional. Sets the maximum amount of either unsolicited or solicited data the initiator may send in a single burst. Any amount of data exceeding this value must be explicitly solicited by the target. The .I should be set to multiples of PAGE_SIZE. Configuring too large values may lead to problems allocating sufficient memory, which in turn may lead to SCSI commands timing out at the initiator host. The default value is the highes possible for current platform (1 or 2 MB). -.TP +.TP .B [FirstBurstLength ] Optional. Sets the amount of unsolicited data the initiator may transmit in the first burst of a transfer either with and/or right after the command, depending on the settings of .B InitialR2T @@ -175,27 +175,27 @@ and . .I should be set to multiples of PAGE_SIZE. Configuring too large values may lead to problems allocating sufficient memory, which in turn may lead to SCSI commands timing out at the initiator host. The default value is the highes possible for current platform (1 or 2 MB). -.TP +.TP .B [DefaultTime2Wait ] Currently not supported. -.TP +.TP .B [DefaultTime2Retain ] Currently not supported. -.TP +.TP .B [MaxOutstandingR2T ] Optional. Controls the maximum number of data transfers the target may request at once, each of up to .B MaxBurstLength bytes. The default is 20. -.TP +.TP .B [DataPDUInOrder ] Optional. Has to be set to "Yes" \- which is also the default. -.TP +.TP .B [DataSequenceInOrder ] Optional. Has to be set to "Yes" \- which is also the default. -.TP +.TP .B [ErrorRecoveryLevel ] Optional. Has to be set to "0" (in words: zero), which is also the default. -.TP +.TP .B [QueuedCommands ] Optional. This parameter defines a window of commands an initiator may send and that will be buffered by the target. Depending on your hardware and your (expected) workload, the .I value @@ -204,7 +204,7 @@ may be carefully adjusted. The default value of 32 should be sufficient for most Currently (as of 0.4.11) not all iSCSI target parameters are used. Header and data digests are not supported during discovery sessions. .SH "SEE ALSO" .B iscsi-scstd (8) -.TP +.TP You should have a look at .B RFC 3720 for all the glory details. diff --git a/iscsi-scst/etc/obsolete/initiators.deny b/iscsi-scst/etc/obsolete/initiators.deny index 72fdf0bf3..5e0d1113c 100644 --- a/iscsi-scst/etc/obsolete/initiators.deny +++ b/iscsi-scst/etc/obsolete/initiators.deny @@ -1,9 +1,9 @@ # The semantics are: # * By default, every initiator can see and connect to all targets. -# +# # * Deny for some or every initiator access to one or all targets, # by adding a line to this file. -# +# # * Then allow some named initiators access to selected targets, # by adding lines to initiators.allow # diff --git a/iscsi-scst/kernel/config.c b/iscsi-scst/kernel/config.c index aac5749b9..f09c58915 100644 --- a/iscsi-scst/kernel/config.c +++ b/iscsi-scst/kernel/config.c @@ -340,7 +340,7 @@ static int add_conn(void __user *ptr) session = session_lookup(target, info.sid); if (!session) { PRINT_ERROR("Session %lld not found", - (unsigned long long int)info.tid); + (unsigned long long)info.tid); err = -ENOENT; goto out_unlock; } @@ -384,7 +384,7 @@ static int del_conn(void __user *ptr) session = session_lookup(target, info.sid); if (!session) { PRINT_ERROR("Session %llx not found", - (unsigned long long int)info.sid); + (unsigned long long)info.sid); err = -ENOENT; goto out_unlock; } diff --git a/iscsi-scst/kernel/conn.c b/iscsi-scst/kernel/conn.c index 71c119352..4aff2a69d 100644 --- a/iscsi-scst/kernel/conn.c +++ b/iscsi-scst/kernel/conn.c @@ -606,7 +606,7 @@ static void conn_rsp_timer_fn(struct timer_list *timer) "%s (SID %llx), closing connection %p", iscsi_get_timeout(cmnd)/HZ, conn->session->initiator_name, - (unsigned long long int) conn->session->sid, + (unsigned long long)conn->session->sid, conn); /* * We must call mark_conn_closed() outside of @@ -837,13 +837,13 @@ static int conn_setup_sock(struct iscsi_conn *conn) mm_segment_t oldfs; struct iscsi_session *session = conn->session; - TRACE_DBG("%llx", (unsigned long long int)session->sid); + TRACE_DBG("%llx", (unsigned long long)session->sid); conn->sock = SOCKET_I(file_inode(conn->file)); if (conn->sock->ops->sendpage == NULL) { PRINT_ERROR("Socket for sid %llx doesn't support sendpage()", - (unsigned long long int)session->sid); + (unsigned long long)session->sid); res = -EINVAL; goto out; } @@ -882,7 +882,7 @@ void conn_free(struct iscsi_conn *conn) TRACE_ENTRY(); TRACE(TRACE_MGMT, "Freeing conn %p (sess=%p, %#Lx %u, initiator %s)", - conn, session, (unsigned long long int)session->sid, conn->cid, + conn, session, (unsigned long long)session->sid, conn->cid, session->scst_sess->initiator_name); lockdep_assert_held(&conn->target->target_mutex); @@ -1000,7 +1000,7 @@ int iscsi_conn_alloc(struct iscsi_session *session, } TRACE(TRACE_MGMT, "Creating connection %p for sid %#Lx, cid %u " - "(initiator %s)", conn, (unsigned long long int)session->sid, + "(initiator %s)", conn, (unsigned long long)session->sid, info->cid, session->scst_sess->initiator_name); conn->transport = t; diff --git a/iscsi-scst/kernel/nthread.c b/iscsi-scst/kernel/nthread.c index 68dc0d1bd..fe2dc74ba 100644 --- a/iscsi-scst/kernel/nthread.c +++ b/iscsi-scst/kernel/nthread.c @@ -1375,7 +1375,7 @@ retry: res = scst_writev(file, iop, count, &off); set_fs(oldfs); TRACE_WRITE("sid %#Lx, cid %u, res %d, iov_len %zd", - (unsigned long long int)conn->session->sid, + (unsigned long long)conn->session->sid, conn->cid, res, iop->iov_len); if (unlikely(res <= 0)) { if (res == -EAGAIN) { @@ -1507,7 +1507,7 @@ retry2: "index %lu, offset %u, size %u, cmd %p, " "page %p)", (sendpage != sock_no_sendpage) ? "sendpage" : "sock_no_sendpage", - (unsigned long long int)conn->session->sid, + (unsigned long long)conn->session->sid, conn->cid, res, page->index, offset, size, write_cmnd, page); if (unlikely(res <= 0)) { @@ -1593,7 +1593,7 @@ out_err: { #endif PRINT_ERROR("error %d at sid:cid %#Lx:%u, cmnd %p", res, - (unsigned long long int)conn->session->sid, + (unsigned long long)conn->session->sid, conn->cid, conn->write_cmnd); } if (ref_cmd_to_parent && diff --git a/iscsi-scst/kernel/session.c b/iscsi-scst/kernel/session.c index 12a802fac..051e26f0c 100644 --- a/iscsi-scst/kernel/session.c +++ b/iscsi-scst/kernel/session.c @@ -382,7 +382,7 @@ int __del_session(struct iscsi_target *target, u64 sid) if (!list_empty(&session->conn_list)) { PRINT_ERROR("%llx still have connections", - (unsigned long long int)session->sid); + (unsigned long long)session->sid); return -EBUSY; } @@ -399,7 +399,7 @@ void iscsi_sess_force_close(struct iscsi_session *sess) lockdep_assert_held(&sess->target->target_mutex); PRINT_INFO("Force closing session %llx with initiator %s (%p)", - (unsigned long long int)sess->sid, sess->initiator_name, sess); + (unsigned long long)sess->sid, sess->initiator_name, sess); list_for_each_entry(conn, &sess->conn_list, conn_list_entry) { TRACE(TRACE_MGMT, "Force closing connection %p", conn); @@ -423,7 +423,7 @@ static void iscsi_session_info_show(struct seq_file *seq, list_for_each_entry(session, &target->session_list, session_list_entry) { seq_printf(seq, "\tsid:%llx initiator:%s (reinstating %s)\n", - (unsigned long long int)session->sid, + (unsigned long long)session->sid, session->initiator_name, session->sess_reinstating ? "yes" : "no"); conn_info_show(seq, session); diff --git a/qla2x00t/qla2x00-target/qla2x00t.c b/qla2x00t/qla2x00-target/qla2x00t.c index 6eacddb14..9720c1499 100644 --- a/qla2x00t/qla2x00-target/qla2x00t.c +++ b/qla2x00t/qla2x00-target/qla2x00t.c @@ -2631,8 +2631,8 @@ static void q2t_load_cont_data_segments(struct q2t_prm *prm) *dword_ptr++ = cpu_to_le32(sg_dma_len(prm->sg)); TRACE_SG("S/G Segment Cont. phys_addr=%llx:%llx, len=%d", - (unsigned long long int)pci_dma_hi32(dma_addr), - (unsigned long long int)pci_dma_lo32(dma_addr), + (unsigned long long)pci_dma_hi32(dma_addr), + (unsigned long long)pci_dma_lo32(dma_addr), (int)sg_dma_len(prm->sg)); /* prm->sg might be last here */ @@ -2693,8 +2693,8 @@ static void q2x_load_data_segments(struct q2t_prm *prm) *dword_ptr++ = cpu_to_le32(sg_dma_len(prm->sg)); TRACE_SG("S/G Segment phys_addr=%llx:%llx, len=%d", - (unsigned long long int)pci_dma_hi32(dma_addr), - (unsigned long long int)pci_dma_lo32(dma_addr), + (unsigned long long)pci_dma_hi32(dma_addr), + (unsigned long long)pci_dma_lo32(dma_addr), (int)sg_dma_len(prm->sg)); /* prm->sg might be last here */ @@ -2755,8 +2755,8 @@ static void q24_load_data_segments(struct q2t_prm *prm) *dword_ptr++ = cpu_to_le32(sg_dma_len(prm->sg)); TRACE_SG("S/G Segment phys_addr=%llx:%llx, len=%d", - (unsigned long long int)pci_dma_hi32(dma_addr), - (unsigned long long int)pci_dma_lo32(dma_addr), + (unsigned long long)pci_dma_hi32(dma_addr), + (unsigned long long)pci_dma_lo32(dma_addr), (int)sg_dma_len(prm->sg)); /* prm->sg might be last here */ diff --git a/qla2x00t/qla_dfs.c b/qla2x00t/qla_dfs.c index 499c74e39..02b32bfca 100644 --- a/qla2x00t/qla_dfs.c +++ b/qla2x00t/qla_dfs.c @@ -25,14 +25,14 @@ qla2x00_dfs_fce_show(struct seq_file *s, void *unused) seq_printf(s, "FCE Trace Buffer\n"); seq_printf(s, "In Pointer = %llx\n\n", (unsigned long long)ha->fce_wr); - seq_printf(s, "Base = %llx\n\n", (unsigned long long) ha->fce_dma); + seq_printf(s, "Base = %llx\n\n", (unsigned long long)ha->fce_dma); seq_printf(s, "FCE Enable Registers\n"); seq_printf(s, "%08x %08x %08x %08x %08x %08x\n", ha->fce_mb[0], ha->fce_mb[2], ha->fce_mb[3], ha->fce_mb[4], ha->fce_mb[5], ha->fce_mb[6]); fce = (uint32_t *) ha->fce; - fce_start = (unsigned long long) ha->fce_dma; + fce_start = (unsigned long long)ha->fce_dma; for (cnt = 0; cnt < fce_calc_size(ha->fce_bufs) / 4; cnt++) { if (cnt % 8 == 0) seq_printf(s, "\n%llx: ", diff --git a/scripts/checkpatch b/scripts/checkpatch new file mode 100755 index 000000000..21797c67f --- /dev/null +++ b/scripts/checkpatch @@ -0,0 +1,3 @@ +#!/bin/bash + +../linux-kernel/scripts/checkpatch.pl -f --show-types --ignore SPDX_LICENSE_TAG,LINUX_VERSION_CODE $(list-source-files | grep -vE 'fcst/linux-patches|patch$|pdf$|png$|ibmvstgt/') | sed 's/^#[0-9]*: FILE: \(.*\):/\1:1:/' diff --git a/scst/README b/scst/README index 1c3dcaa4a..fed367fe1 100644 --- a/scst/README +++ b/scst/README @@ -114,7 +114,7 @@ log message: "tgt_dev for LUN 0 not found, command to unexisting LU?" It is highly recommended to use scstadmin utility for configuring devices and security groups. -The flow of SCST initialization should be as the following: +The flow of SCST initialization should be as follows: 1. Load of SCST modules with necessary module parameters, if needed. @@ -1102,9 +1102,13 @@ cache. The following parameters possible for vdisk_fileio: - read_only - read only. Default is 0. - - o_direct - disables both read and write caching. This mode isn't - currently fully implemented, you should use user space fileio_tgt - program in O_DIRECT mode instead (see below). + - async - submit I/O asynchronously to the device handler. This mode + allows concurrent processing of SCSI commands even when using only + a single SCST command thread. + + - o_direct - disables both read and write caching if asynchronous + I/O is used. This mode bypasses the page cache and hence improves + performance. - nv_cache - enables "non-volatile cache" mode. In this mode it is assumed that the target has a GOOD UPS with ability to cleanly @@ -1139,8 +1143,8 @@ cache. The following parameters possible for vdisk_fileio: - rotational - if set, this device reported as rotational. Otherwise, it is reported as non-rotational (SSD, etc.) - - zero_copy - if set, then this device uses zero copy access to the - page cache. At the moment, only read side zero copy is implemented. + - zero_copy - ignored. For zero-copy I/O, set the async flag and + possibly also the o_direct flag and use Linux kernel v4.10 or later. - dif_mode - specifies which T10-PI, or DIF, mode this device will use. See SCSI standards from more info about T10-PI. Available DIF modes diff --git a/scst/README_in-tree b/scst/README_in-tree deleted file mode 100644 index 837df1428..000000000 --- a/scst/README_in-tree +++ /dev/null @@ -1,2703 +0,0 @@ -Generic SCSI target mid-level for Linux (SCST) -============================================== - -SCST is designed to provide unified, consistent interface between SCSI -target drivers and Linux kernel and simplify target drivers development -as much as possible. Detail description of SCST's features and internals -could be found on its Internet page http://scst.sourceforge.net. - -SCST supports the following I/O modes: - - * Pass-through mode with one to many relationship, i.e. when multiple - initiators can connect to the exported pass-through devices, for - the following SCSI devices types: disks (type 0), tapes (type 1), - processors (type 3), CDROMs (type 5), MO disks (type 7), medium - changers (type 8) and RAID controllers (type 0xC). - - * FILEIO mode, which allows to use files on file systems or block - devices as virtual remotely available SCSI disks or CDROMs with - benefits of the Linux page cache. - - * BLOCKIO mode, which performs direct block IO with a block device, - bypassing page-cache for all operations. This mode works ideally with - high-end storage HBAs and for applications that either do not need - caching between application and disk or need the large block - throughput. - - * User space mode using scst_user device handler, which allows to - implement in the user space high performance virtual SCSI - devices. Comparing with fully in-kernel dev handlers this mode has - very low overhead (few %%). - - * "Performance" device handlers, which provide in pseudo pass-through - mode a way for direct performance measurements without overhead of - actual data transferring from/to underlying SCSI device. - -In addition, SCST supports advanced per-initiator access and devices -visibility management, so different initiators could see different set -of devices with different access permissions. See below for details. - -Full list of SCST features and comparison with other Linux targets you -can find on http://scst.sourceforge.net/comparison.html. - - -Installation ------------- - -To see your devices remotely, you need to add a corresponding LUN for -them (see below how). By default, no local devices are seen remotely. -There must be LUN 0 in each LUNs set (security group), i.e. LUs -numeration must not start from, e.g., 1. Otherwise you will see no -devices on remote initiators and SCST core will write into the kernel -log message: "tgt_dev for LUN 0 not found, command to unexisting LU?" - -It is highly recommended to use scstadmin utility for configuring -devices and security groups. - -The flow of SCST inialization should be as the following: - -1. Load of SCST modules with necessary module parameters, if needed. - -2. Configure targets, devices, LUNs, etc. using either scstadmin -(recommended), or the sysfs interface directly as described below. - -If you experience problems during modules load or running, check your -kernel logs (or run dmesg command for the few most recent messages). - -IMPORTANT: Without loading appropriate device handler, corresponding devices -========= will be invisible for remote initiators, which could lead to holes - in the LUN addressing, so automatic device scanning by remote SCSI - mid-level could not notice the devices. Therefore you will have - to add them manually via - 'echo "- - -" >/sys/class/scsi_host/hostX/scan', - where X - is the host number. - -IMPORTANT: Working of target and initiator on the same host is -========= supported, except the following 2 cases: swap over target exported - device and using a writable mmap over a file from target - exported device. The latter means you can't mount a file - system over target exported device. In other words, you can - freely use any sg, sd, st, etc. devices imported from target - on the same host, but you can't mount file systems or put - swap on them. This is a limitation of Linux memory/cache - manager, because in this case a memory allocation deadlock is - possible like: system needs some memory -> it decides to - clear some cache -> the cache is needed to be written on a - target exported device -> initiator sends request to the - target located on the same system -> the target needs memory - -> the system needs even more memory -> deadlock. - -IMPORTANT: In the current version simultaneous access to local SCSI devices -========= via standard high-level SCSI drivers (sd, st, sg, etc.) and - SCST's target drivers is unsupported. Especially it is - important for execution via sg and st commands that change - the state of devices and their parameters, because that could - lead to data corruption. If any such command is done, at - least related device handler(s) must be restarted. For block - devices READ/WRITE commands using direct disk handler are - generally safe. - - -Usage in failover mode ----------------------- - -It is recommended to use TEST UNIT READY ("tur") command to check if -SCST target is alive in MPIO configurations. - - -Device handlers ---------------- - -Device specific drivers (device handlers) are plugins for SCST, which -help SCST to analyze incoming requests and determine parameters, -specific to various types of devices. If an appropriate device handler -for a SCSI device type isn't loaded, SCST doesn't know how to handle -devices of this type, so they will be invisible for remote initiators -(more precisely, "LUN not supported" sense code will be returned). - -In addition to device handlers for real devices, there are VDISK, user -space and "performance" device handlers. - -VDISK device handler works over files on file systems and makes from -them virtual remotely available SCSI disks or CDROM's. In addition, it -allows to work directly over a block device, e.g. local IDE or SCSI disk -or ever disk partition, where there is no file systems overhead. Using -block devices comparing to sending SCSI commands directly to SCSI -mid-level via scsi_do_req()/scsi_execute_async() has advantage that data -are transferred via system cache, so it is possible to fully benefit -from caching and read ahead performed by Linux's VM subsystem. The only -disadvantage here that in the FILEIO mode there is superfluous data -copying between the cache and SCST's buffers. This issue is going to be -addressed in one of the future releases. Virtual CDROM's are useful for -remote installation. See below for details how to setup and use VDISK -device handler. - -"Performance" device handlers for disks, MO disks and tapes in their -exec() method skip (pretend to execute) all READ and WRITE operations -and thus provide a way for direct link performance measurements without -overhead of actual data transferring from/to underlying SCSI device. - -NOTE: Since "perf" device handlers on READ operations don't touch the -==== commands' data buffer, it is returned to remote initiators as it - was allocated, without even being zeroed. Thus, "perf" device - handlers impose some security risk, so use them with caution. - - -Compilation options -------------------- - -There are the following compilation options, that could be change using -your favorite kernel configuration Makefile target, e.g. "make xconfig": - - - CONFIG_SCST_DEBUG - if defined, turns on some debugging code, - including some logging. Makes the driver considerably bigger and slower, - producing large amount of log data. - - - CONFIG_SCST_TRACING - if defined, turns on ability to log events. Makes the - driver considerably bigger and leads to some performance loss. - - - CONFIG_SCST_EXTRACHECKS - if defined, adds extra validity checks in - the various places. - - - CONFIG_SCST_USE_EXPECTED_VALUES - if not defined (default), initiator - supplied expected data transfer length and direction will be used - only for verification purposes to return error or warn in case if one - of them is invalid. Instead, locally decoded from SCSI command values - will be used. This is necessary for security reasons, because - otherwise a faulty initiator can crash target by supplying invalid - value in one of those parameters. This is especially important in - case of pass-through mode. If CONFIG_SCST_USE_EXPECTED_VALUES is - defined, initiator supplied expected data transfer length and - direction will override the locally decoded values. This might be - necessary if internal SCST commands translation table doesn't contain - SCSI command, which is used in your environment. You can know that if - you enable "minor" trace level and have messages like "Unknown - opcode XX for YY. Should you update scst_scsi_op_table?" in your - kernel log and your initiator returns an error. Also report those - messages in the SCST mailing list scst-devel@lists.sourceforge.net. - Note, that not all SCSI transports support supplying expected values. - You should try to enable this option if you have a not working with - SCST pass-through device, for instance, an SATA CDROM. - - - CONFIG_SCST_DEBUG_TM - if defined, turns on task management functions - debugging, when on LUN 6 some of the commands will be delayed for - about 60 sec., so making the remote initiator send TM functions, eg - ABORT TASK and TARGET RESET. Also define - CONFIG_SCST_TM_DBG_GO_OFFLINE symbol in the Makefile if you want that - the device eventually become completely unresponsive, or otherwise to - circle around ABORTs and RESETs code. Needs CONFIG_SCST_DEBUG turned - on. - - - CONFIG_SCST_STRICT_SERIALIZING - if defined, makes SCST send all commands to - underlying SCSI device synchronously, one after one. This makes task - management more reliable, with cost of some performance penalty. This - is mostly actual for stateful SCSI devices like tapes, where the - result of command's execution depends from device's settings defined - by previous commands. Disk and RAID devices are stateless in the most - cases. The current SCSI core in Linux doesn't allow to abort all - commands reliably if they sent asynchronously to a stateful device. - Turned off by default, turn it on if you use stateful device(s) and - need as much error recovery reliability as possible. As a side effect - of CONFIG_SCST_STRICT_SERIALIZING, on kernels below 2.6.30 no kernel - patching is necessary for pass-through device handlers (scst_disk, - etc.). - - - CONFIG_SCST_TEST_IO_IN_SIRQ - if defined, allows SCST to submit selected - SCSI commands (TUR and READ/WRITE) from soft-IRQ context (tasklets). - Enabling it will decrease amount of context switches and slightly - improve performance. The goal of this option is to be able to measure - overhead of the context switches. If after enabling this option you - don't see under load in vmstat output on the target significant - decrease of amount of context switches, then your target driver - doesn't submit commands to SCST in IRQ context. For instance, - iSCSI-SCST doesn't do that, but qla2x00t with - CONFIG_QLA_TGT_DEBUG_WORK_IN_THREAD disabled - does. This option is - designed to be used with vdisk NULLIO backend. - - WARNING! Using this option enabled with other backend than vdisk - NULLIO is unsafe and can lead you to a kernel crash! - - - CONFIG_SCST_STRICT_SECURITY - if defined, makes SCST zero allocated data - buffers. Undefining it (default) considerably improves performance - and eases CPU load, but could create a security hole (information - leakage), so enable it, if you have strict security requirements. - - - CONFIG_SCST_ABORT_CONSIDER_FINISHED_TASKS_AS_NOT_EXISTING - if defined, - in case when TASK MANAGEMENT function ABORT TASK is trying to abort a - command, which has already finished, remote initiator, which sent the - ABORT TASK request, will receive TASK NOT EXIST (or ABORT FAILED) - response for the ABORT TASK request. This is more logical response, - since, because the command finished, attempt to abort it failed, but - some initiators, particularly VMware iSCSI initiator, consider TASK - NOT EXIST response as if the target got crazy and try to RESET it. - Then sometimes get crazy itself. So, this option is disabled by - default. - - - CONFIG_SCST_DIF_INJECT_CORRUPTED_TAGS - if defined, allows injection - of corrupted DIF tags according to the Oracle specification. This - functionality is working only if dif_mode doesn't contain dev_store - and dif_type is 1. - - - CONFIG_SCST_FORWARD_MODE_PASS_THROUGH - if defined, the pass-through - subsystem starts working in the forwarding mode, where reservation - commands processed locally and not passed to the backend SCSI device, - while COMPARE AND WRITE, EXTENDED COPY and RECEIVE COPY RESULTS - commands, which normally processed locally by the SCST core, not - processed locally, but passed to the backend device. Intended to be - used to implement NON-OPTIMIZED ALUA state together with "forwarding" - target attribute on the remote node. See below for more details. - Disabled by default for safety. - - - CONFIG_SCST_NO_TOTAL_MEM_CHECKS - disables checks of allocated - memory, see scst_max_cmd_mem below. Allows to avoid 2 global - variables on the fast path, hence get better multi-queue performance. - -HIGHMEM kernel configurations are fully supported, but not recommended -for performance reasons. - - -Module parameters ------------------ - -Module scst supports the following parameters: - - - scst_threads - allows to set count of SCST's threads. By default it - is CPU count. - - - scst_max_cmd_mem - sets maximum amount of memory in MB allowed to be - consumed by the SCST commands for data buffers at any given time. By - default it is approximately TotalMem/4. - - - auto_cm_assignment - enables the copy managers auto registration. - If a device is not registered in the copy manager, it can not be - source or target of EXTENDED COPY commands. Enabled by default. - Disable, if you want to manually control the copy manager - registration or need to change a device, e.g. a DM cache device, with - SCST LUN on top of it to avoid extra reference the copy manager holds - on this device. In the later case you can also remove this reference - by manually deleting the corresponding copy manager LUN via sysfs interface - (/sys/kernel/scst_tgt/targets/copy_manager/copy_manager_tgt/luns/mgmt). - - -SCST sysfs interface --------------------- - -SCST sysfs interface designed to be self descriptive and self -containing. This means that a high level management tool for it can be -written once and automatically support any future sysfs interface -changes (attributes additions or removals, new target drivers and dev -handlers, etc.) without any modifications. Scstadmin is an example of -such management tool. - -To implement that an management tool should not be implemented around -drivers and their attributes, but around common rules those drivers and -attributes follow. You can find those rules in SysfsRules file. For -instance, each SCST sysfs file (attribute) can contain in the last line -mark "[key]". It is automatically added to allow scstadmin and other -management tools to see which attributes it should save in the config -file. If you are doing manual attributes manipulations, you can ignore -this mark. - -Root of SCST sysfs interface is /sys/kernel/scst_tgt. It has the -following entries: - - - devices - this is a root subdirectory for all SCST devices - - - handlers - this is a root subdirectory for all SCST dev handlers - - - max_tasklet_cmd - specifies how many commands at max can be queued in - the SCST core simultaneously on a single CPU from all connected - initiators to allow processing commands on this CPU in soft-IRQ - context in tasklets. If the count of the commands exceeds this value, - then all of them will be processed only in SCST threads. This is to - to prevent possible under heavy load starvation of processes on the - CPUs serving soft IRQs and in some cases to improve performance by - more evenly spreading load over available CPUs. - - - sgv - this is a root subdirectory for all SCST SGV caches - - - targets - this is a root subdirectory for all SCST targets - - - setup_id - allows to read and write SCST setup ID. This ID can be - used in cases, when the same SCST configuration should be installed - on several targets, but exported from those targets devices should - have different IDs and SNs. For instance, VDISK dev handler uses this - ID to generate T10 vendor specific identifier and SN of the devices. - - - poll_us - if polling is desired, sets how many us each SCST thread - is polling its queue after it became empty in a hope that a new - command can come. In some cases, polling can significantly increase - IOPS, especially if low power states on CPU not disabled, because on - high IOPS polling could be cheaper comparing to spending significant - time on entering, then exiting CPU low power states + corresponding - context switches. Disabled, i.e. set to 0, by default. - - - suspend - globally suspends or releases all SCSI activities on all - devices. Useful for mass management, like adding or deleting LUNs. - Writing to it value v: - - * v > 0 - suspends activities, but waits no more, than v seconds - - * v = 0 - suspends activities, waits indefinitely - - * V < 0 - releases activities. - - Reading from this attribute returns number of previous suspend - requests. - - - threads - allows to read and set number of global SCST I/O threads. - Those threads used with async. dev handlers, for instance, vdisk - BLOCKIO or NULLIO. - - - trace_cmds - shows current SCST commands up to size of the sysfs - buffer (4KB) - - - trace_mcmds - shows current SCST management commands up to size of - the sysfs buffer (4KB) - - - trace_level - allows to enable and disable various tracing - facilities. See content of this file for help how to use it. See also - section "Dealing with massive logs" for more info how to make correct - logs when you enabled trace levels producing a lot of logs data. - - - version - read-only attribute, which allows to see version of - SCST and enabled optional features. - - - last_sysfs_mgmt_res - read-only attribute returning completion status - of the last management command. In the sysfs implementation there are - some problems between internal sysfs and internal SCST locking. To - avoid them in some cases sysfs calls can return error with errno - EAGAIN. This doesn't mean the operation failed. It only means that - the operation queued and not yet completed. To wait for it to - complete, an management tool should poll this file. If the operation - hasn't yet completed, it will also return EAGAIN. But after it's - completed, it will return the result of this operation (0 for success - or -errno for error). The following two shell functions show how to do - this: - - - force_global_sgv_pool - if not set, buffers for SCSI commands are - allocated from per-CPU SGV pool. Otherwise, global SGV pool is used. - -# Read the SCST sysfs attribute $1. See also scst/README for more information. -scst_sysfs_read() { - local EAGAIN val - - EAGAIN="Resource temporarily unavailable" - while true; do - if val="$(LC_ALL=C cat "$1" 2>&1)"; then - echo -n "${val%\[key\]}" - return 0 - elif [ "${val/*: }" != "$EAGAIN" ]; then - return 1 - fi - sleep 1 - done -} - -# Write $1 into the SCST sysfs attribute $2. See also scst/README for more -# information. -scst_sysfs_write() { - local EAGAIN status - - EAGAIN="Resource temporarily unavailable" - if status="$(LC_ALL=C; (echo -n "$1" > "$2") 2>&1)"; then - return 0 - elif [ "${status/*: }" != "$EAGAIN" ]; then - return 1 - fi - scst_sysfs_read /sys/kernel/scst_tgt/last_sysfs_mgmt_res >/dev/null -} - -"Devices" subdirectory contains subdirectories for each SCST devices. - -Content of each device's subdirectory is dev handler specific. See -documentation for your dev handlers for more info about it as well as -SysfsRules file for more info about common to all dev handlers rules. -SCST dev handlers can have the following common entries: - - - block - allows to temporary block and unblock this device. See below. - - - exported - subdirectory containing links to all LUNs where this - device was exported. - - - handler - if dev handler determined for this device, this link points - to it. The handler can be not set for pass-through devices. - - - threads_num - shows and allows to set number of threads in this device's - threads pool. If 0 - no threads will be created, and global SCST - threads pool will be used. If <0 - creation of the threads pool is - prohibited. - - - threads_pool_type - shows and allows to sets threads pool type. - Possible values: "per_initiator" and "shared". When the value is - "per_initiator" (default), each session from each initiator will use - separate dedicated pool of threads. When the value is "shared", all - sessions from all initiators will share the same per-device pool of - threads. Valid only if threads_num attribute >0. - - - dump_prs - allows to dump persistent reservations information in the - kernel log. - - - type - SCSI type of this device - - - max_tgt_dev_commands - maximum number of SCSI commands any session to - this device can have in flight. - - - numa_node_id - NUMA node id this device physically belongs to. SCST - NUMA handling assumes that being used in the system NUMA memory - allocation policy is to always allocate from the current node. - -Attribute "block" allows to temporary block and unblock this device. -"Blocking" means that no new commands for this device will go into the -execution stage, but instead will be suspended just before it. The -blocked state is not reached until queue of the corresponding device is -completely drained. You can also call this state "frozen". It is useful -in many cases, like consistent snapshots and graceful shutdown. - -On write "block" entry allows the following 3 types of parameters: - - - 1 - block device synchronously, i.e. don't return until this device - becomes blocked, i.e. until queue of it is not completely drained. Can - be called as many times as needed. - - - 11 params - block device asynchronously, i.e. return immediately. - Notification about completing is delivered using SCST_EVENT_EXT_BLOCKING_DONE - event. "Params" delivered to it as is in "data" payload. Can be - called as many times as needed. Alternatively, status of blocking could be - polled by reading this attributes until the second number reaches 0 - (see below). - - - 0 - unblock this device. - -Reading from "block" entry returns two numbers separated by space: - -1. How many times this device was blocked, i.e. how many times writing -"0" to it is needed to unblock this device. - -2. Boolean (0 or 1) if blocking, if any, is done (0) or still pending (1). - -See below for more information about other entries of this subdirectory -of the standard SCST dev handlers. - -"Handlers" subdirectory contains subdirectories for each SCST dev -handler. - -Content of each handler's subdirectory is dev handler specific. See -documentation for your dev handlers for more info about it as well as -SysfsRules file for more info about common to all dev handlers rules. -SCST dev handlers can have the following common entries: - - - mgmt - this entry allows to create virtual devices and their - attributes (for virtual devices dev handlers) or assign/unassign real - SCSI devices to/from this dev handler (for pass-through dev - handlers). - - - trace_level - allows to enable and disable various tracing - facilities. See content of this file for help how to use it. See also - section "Dealing with massive logs" for more info how to make correct - logs when you enabled trace levels producing a lot of logs data. - - - type - SCSI type of devices served by this dev handler. - -See below for more information about other entries of this subdirectory -of the standard SCST dev handlers. - -"Sgv" subdirectory contains statistic information of SCST SGV caches. It -has the following entries: - - - None, one or more subdirectories for each existing SGV cache. - - - global_stats - file containing global SGV caches statistics. - -Each SGV cache's subdirectory has the following item: - - - stats - file containing statistics for this SGV caches. - -"Targets" subdirectory contains subdirectories for each SCST target. - -Content of each target's subdirectory is target specific. See -documentation for your target for more info about it as well as -SysfsRules file for more info about common to all targets rules. -Every target should have at least the following entries: - - - ini_groups - subdirectory, which contains and allows to define - initiator-oriented access control information, see below. - - - luns - subdirectory, which contains list of available LUNs in the - target-oriented access control and allows to define it, see below. - - - sessions - subdirectory containing connected to this target sessions. - - - comment - this attribute can be used to store any human readable info - to help identify target. For instance, to help identify the target's - mapping to the corresponding hardware port. It isn't anyhow used by - SCST. - - - enabled - using this attribute you can enable or disable this target. - It allows to finish configuring it before it starts accepting new - connections. 0 by default. - - - addr_method - used LUNs addressing method. Possible values: - "Peripheral", "Flat" or "LUN". Most initiators work well with - Peripheral addressing method (default), but some (HP-UX, for instance) - may require the Flat method or the LUN method (e.g. IBM systems). This - attribute is also available in the initiators security groups, so you - can assign the addressing method on per-initiator basis. See also the - "Logical unit addressing (LUN)" section in SAM-5 for more information. - - - black_hole - if set, all LUNs in the corresponding initiator group, - default target group in this case, start "swallowing" requests from - initiators. Possible values are: - - * 0 - disable black hole mode - - * 1 - immediately abort all coming SCSI commands, i.e. all SCSI commands - are dropped and TM requests return that they completed. It is - supposed to simulate lost front end responses. - - * 2 - immediately abort all coming SCSI commands and drop all coming TM - commands. It is supposed to simulate logical target hang, when the - target stops responding, but on the HW/TCP connection level still - appears to be online. - - * 3 - immediately abort all coming data transfer SCSI commands, i.e. - only data transfer SCSI commands are dropped, while commands like - INQUIRY and TEST UNIT READY pass well. It is supposed to simulate - flaky front end connectivity, when responses for small commands - pass well, but big data transfers fail. - - * 4 - immediately abort all coming data transfer SCSI commands and - drop all coming TM commands. It is supposed to simulate really - flaky front end connectivity, when TM requests or responses are - also lost. - - Modes 3 and 4 are the most evil ones, because they are not too well - handled by many initiator OS'es, including Linux, so they may never - recover from it. - - Note, dropping TM commands, i.e. not sending response on them, - implemented not for all target drivers. If it's implemented for your - particular target driver or not, you can find out by checking traces - or the target driver's source code. - - - dif_capabilities - if this target supports T10-PI, returns which - exact DIF capabilities this target supports. - - - dif_checks_failed - if this target supports T10-PI, returns - statistics how many DIF errors have been detected on the - corresponding processing stages on this target. It returns 3 rows of - numbers with 3 numbers in each row: for target driver stage, for SCST - stage and for dev handler stage. Numbers in each row: how many errors - detected checking application, reference and guard tags - correspondingly. Writing to this attribute resets the numbers. - - - cpu_mask - defines CPU affinity mask for threads serving this target. - For threads serving LUNs it is used only for devices with - threads_pool_type "per_initiator". - - - io_grouping_type - defines how I/O from sessions to this target are - grouped together. This I/O grouping is very important for - performance. By setting this attribute in a right value, you can - considerably increase performance of your setup. This grouping is - performed only if you use CFQ I/O scheduler on the target and for - devices with threads_num >= 0 and, if threads_num > 0, with - threads_pool_type "per_initiator". Possible values: - "this_group_only", "never", "auto", or I/O group number >0. When the - value is "this_group_only" all I/O from all sessions in this target - will be grouped together. When the value is "never", I/O from - different sessions will not be grouped together, i.e. all sessions in - this target will have separate dedicated I/O groups. When the value - is "auto" (default), all I/O from initiators with the same name - (iSCSI initiator name, for instance) in all targets will be grouped - together with a separate dedicated I/O group for each initiator name. - For iSCSI this mode works well, but other transports usually use - different initiator names for different sessions, so using such - transports in MPIO configurations you should either use value - "this_group_only", or an explicit I/O group number. This attribute is - also available in the initiators security groups, so you can assign - the I/O grouping on per-initiator basis. See below for more info how - to use this attribute. - - - rel_tgt_id - allows to read or write SCSI Relative Target Port - Identifier attribute. This identifier is used to identify SCSI Target - Ports by some SCSI commands, mainly by Persistent Reservations - commands. This identifier must be unique among all SCST targets, but - for convenience SCST allows disabled targets to have not unique - rel_tgt_id. In this case SCST will not allow to enable this target - until rel_tgt_id becomes unique. This attribute initialized unique by - SCST by default. - - - forwarding - if set this target is forwarding target, i.e. does not check - any local SCSI events (reservations, etc.). Those event supposed to - be checked on the another, requester's side. - - - *count*, e.g. read_io_count_kb, - statistics about executed - commands and transferred data. Those attributes have speaking names - built from parts: - - 1. Data transfer direction - - 2. Alignment type: not specified or unaligned (on 4K boundaries) - - 3. Type: IO (commands) count or amount of transferred data - - 4. For transferred data: measurement units - - For instance, read_unaligned_cmd_count means number of 4K unaligned IOs. - -A target driver may have also the following entries: - - - "hw_target" - if the target driver supports both hardware and virtual - targets (for instance, an FC adapter supporting NPIV, which has - hardware targets for its physical ports as well as virtual NPIV - targets), this read only attribute for all hardware targets will - exist and contain value 1. - -Subdirectory "sessions" contains one subdirectory for each connected -session with name equal to name of the connected initiator with the -following entries: - - - initiator_name - contains initiator name - - - force_close - optional write-only attribute, which allows to force - close this session. - - - active_commands - contains number of active, i.e. not yet or being - executed, SCSI commands in this session. - - - commands - contains overall number of SCSI commands in this session. - - - dif_checks_failed - if target of this session supports T10-PI, returns - statistics how many DIF errors have been detected on the - corresponding processing stages on all DIF-enabled LUNs in this - session. It returns 3 rows of numbers with 3 numbers in each row: for - target driver stage, for SCST stage and for dev handler stage. - Numbers in each row: how many errors detected checking application, - reference and guard tags correspondingly. Writing to this attribute - resets the numbers. Similar statistics returned in attribute with the - same name for each LUN in this session in this LUN's subdirectory, if - its device configured with dif_type > 0. - - - read_cmd_count - number of READ SCSI commands received since beginning - or last reset (writing 0 in this attribute) - - - read_io_count_kb - amount of data in KB read by the initiator since - beginning or last reset (writing 0 in this attribute) - - - write_cmd_count - number of WRITE SCSI commands received since - beginning or last reset (writing 0 in this attribute) - - - write_io_count_kb - amount of data in KB written by the initiator - since beginning or last reset (writing 0 in this attribute) - - - bidi_cmd_count - number of BIDI SCSI commands received since - beginning or last reset (writing 0 in this attribute) - - - bidi_io_count_kb - amount of data in KB transferred by the - initiator since beginning or last reset (writing 0 in this attribute) - - - none_cmd_count - number of not transferring data SCSI commands - (e.g. INQUIRY or TEST UNIT READY) received since beginning or last - reset (writing 0 in this attribute) - - - unknown_cmd_count - number of unknown SCSI commands received since - beginning or last reset (writing 0 in this attribute) - - - *count*, e.g. read_io_count_kb, - statistics about executed - commands and transferred data. See above for more details. - - - luns - a link pointing out to the corresponding LUNs set (security - group) where this session was attached to. - - - One or more "lunX" subdirectories, where 'X' is a number, for each LUN - this session has (see below). - - - other target driver specific attributes and subdirectories. - -See below description of the VDISK's sysfs interface for samples. - - -Each sessions//lun subdirectory contains the following entries: - - - active_commands - contains number of active, i.e. not yet or being - executed, SCSI commands for lun in session . - - - thread_pid - contains a single line with all the process identifiers - (PIDs) of the kernel threads that process SCSI commands intended for - lun in session . - - - thread_index - thread index assigned by scst_add_threads(). - Can be used to look up which export thread is serving which target - since this index also appears in the export thread name. This - information then could be used to set CPU affinity for those threads - to improve performance. Has a value in the range 0..n-1 for - threads_pool_type per_initiator or -1 when using a shared thread pool - per LUN or the global thread pool. - - -Access and devices visibility management (LUN masking) ------------------------------------------------------- - -Access and devices visibility management allows for an initiator or -group of initiators to see different devices with different LUNs -with necessary access permissions. - -SCST supports two modes of access control: - -1. Target-oriented. In this mode you define for each target a default -set of LUNs, which are accessible to all initiators, connected to that -target. This is a regular access control mode, which people usually mean -thinking about access control in general. For instance, in IET this is -the only supported mode. - -2. Initiator-oriented. In this mode you define which LUNs are accessible -for each initiator. In this mode you should create for each set of one -or more initiators, which should access to the same set of devices with -the same LUNs, a separate security group, then add to it devices and -names of allowed initiator(s). - -Both modes can be used simultaneously. In this case the -initiator-oriented mode has higher priority, than the target-oriented, -i.e. initiators are at first searched in all defined security groups for -this target and, if none matches, the default target's set of LUNs is -used. This set of LUNs might be empty, then the initiator will not see -any LUNs from the target. - -You can at any time find out which set of LUNs each session is assigned -to by looking where link -/sys/kernel/scst_tgt/targets/target_driver/target_name/sessions/initiator_name/luns -points to. - -To configure the target-oriented access control SCST provides the -following interface. Each target's sysfs subdirectory -(/sys/kernel/scst_tgt/targets/target_driver/target_name) has "luns" -subdirectory. This subdirectory contains the list of already defined -target-oriented access control LUNs for this target as well as file -"mgmt". This file has the following commands, which you can send to it, -for instance, using "echo" shell command. You can always get a small -help about supported commands by looking inside this file. "Parameters" -are one or more param_name=value pairs separated by ';'. - - - "add H:C:I:L lun [parameters]" - adds a pass-through device with - host:channel:id:lun with LUN "lun". Optionally, the device could be - marked as read only by using parameter "read_only". The recommended - way to find out H:C:I:L numbers is use of lsscsi utility. - - - "replace H:C:I:L lun [parameters]" - replaces by pass-through device - with host:channel:id:lun existing with LUN "lun" device with - generation of INQUIRY DATA HAS CHANGED Unit Attention. If the old - device doesn't exist, this command acts as the "add" command. - Optionally, the device could be marked as read only by using - parameter "read_only". The recommended way to find out H:C:I:L - numbers is use of lsscsi utility. - - - "add VNAME lun [parameters]" - adds a virtual device with name VNAME - with LUN "lun". Optionally, the device could be marked as read only - by using parameter "read_only". - - - "replace VNAME lun [parameters]" - replaces by virtual device - with name VNAME existing with LUN "lun" device with generation of - INQUIRY DATA HAS CHANGED Unit Attention. If the old device doesn't - exist, this command acts as the "add" command. Optionally, the device - could be marked as read only by using parameter "read_only". - - - "del lun" - deletes LUN lun - - - "clear" - clears the list of devices - -To configure the initiator-oriented access control SCST provides the -following interface. Each target's sysfs subdirectory -(/sys/kernel/scst_tgt/targets/target_driver/target_name) has "ini_groups" -subdirectory. This subdirectory contains the list of already defined -security groups for this target as well as file "mgmt". This file has -the following commands, which you can send to it, for instance, using -"echo" shell command. You can always get a small help about supported -commands by looking inside this file. - - - "create GROUP_NAME" - creates a new security group. - - - "del GROUP_NAME" - deletes a new security group. - -Each security group's subdirectory contains 2 subdirectories: initiators -and luns as well as the following attributes: addr_method, cpu_mask and -io_grouping_type, black_hole. See above description of them. - -Each "initiators" subdirectory contains list of added to this groups -initiator as well as as well as file "mgmt". This file has the following -commands, which you can send to it, for instance, using "echo" shell -command. You can always get a small help about supported commands by -looking inside this file. - - - "add INITIATOR_NAME" - adds initiator with name INITIATOR_NAME to the - group. - - - "del INITIATOR_NAME" - deletes initiator with name INITIATOR_NAME - from the group. - - - "move INITIATOR_NAME DEST_GROUP_NAME" moves initiator with name - INITIATOR_NAME from the current group to group with name - DEST_GROUP_NAME. - - - "clear" - deletes all initiators from this group. - -For "add" and "del" commands INITIATOR_NAME can be a simple DOS-type -patterns, containing '*' and '?' symbols. '*' means match all any -symbols, '?' means match only any single symbol. For instance, -"blah.xxx" will match "bl?h.*". Additionally, you can use negative sign -'!' to revert the value of the pattern. For instance, "ah.xxx" will -match "!bl?h.*". - -Each "luns" subdirectory contains the list of already defined LUNs for -this group as well as file "mgmt". Content of this file as well as list -of available in it commands is fully identical to the "luns" -subdirectory of the target-oriented access control. - -Examples: - - - echo "create INI" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.vlnb:tgt1/ini_groups/mgmt - - creates security group INI for target iqn.2006-10.net.vlnb:tgt1. - - - echo "add 2:0:1:0 11" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.vlnb:tgt1/ini_groups/INI/luns/mgmt - - adds a pass-through device sitting on host 2, channel 0, ID 1, LUN 0 - to group with name INI as LUN 11. - - - echo "add disk1 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.vlnb:tgt1/ini_groups/INI/luns/mgmt - - adds a virtual disk with name disk1 to group with name INI as LUN 0. - - - echo "add 21:*:e0:?b:83:*" >/sys/kernel/scst_tgt/targets/21:00:00:a0:8c:54:52:12/ini_groups/INI/initiators/mgmt - - adds a pattern to group with name INI to Fibre Channel target with - WWN 21:00:00:a0:8c:54:52:12, which matches WWNs of Fibre Channel - initiator ports. - -Consider you need to have an iSCSI target with name -"iqn.2007-05.com.example:storage.disk1.sys1.xyz", which should export -virtual device "dev1" with LUN 0 and virtual device "dev2" with LUN 1, -but initiator with name -"iqn.2007-05.com.example:storage.disk1.spec_ini.xyz" should see only -virtual device "dev2" read only with LUN 0. To achieve that you should -do the following commands: - -# echo "iqn.2007-05.com.example:storage.disk1.sys1.xyz" >/sys/kernel/scst_tgt/targets/iscsi/mgmt -# echo "add dev1 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2007-05.com.example:storage.disk1.sys1.xyz/luns/mgmt -# echo "add dev2 1" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2007-05.com.example:storage.disk1.sys1.xyz/luns/mgmt -# echo "create SPEC_INI" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2007-05.com.example:storage.disk1.sys1.xyz/ini_groups/mgmt -# echo "add dev2 0 read_only=1" \ - >/sys/kernel/scst_tgt/targets/iscsi/iqn.2007-05.com.example:storage.disk1.sys1.xyz/ini_groups/SPEC_INI/luns/mgmt -# echo "iqn.2007-05.com.example:storage.disk1.spec_ini.xyz" \ - >/sys/kernel/scst_tgt/targets/iscsi/iqn.2007-05.com.example:storage.disk1.sys1.xyz/ini_groups/SPEC_INI/initiators/mgmt - -For Fibre Channel or SAS in the above example you should use target's -and initiator ports WWNs instead of iSCSI names. - -It is highly recommended to use scstadmin utility instead of described -in this section low level interface. - -IMPORTANT -========= - -There must be LUN 0 in each set of LUNs, i.e. LUs numeration must not -start from, e.g., 1. Otherwise you will see no devices on remote -initiators and SCST core will write into the kernel log message: "tgt_dev -for LUN 0 not found, command to unexisting LU?" - -IMPORTANT -========= - -All the access control must be fully configured BEFORE the corresponding -target is enabled. When you enable a target, it will immediately start -accepting new connections, hence creating new sessions, and those new -sessions will be assigned to security groups according to the -*currently* configured access control settings. For instance, to -the default target's set of LUNs, instead of "HOST004" group as you may -need, because "HOST004" doesn't exist yet. So, you must configure all -the security groups before new connections from the initiators are -created, i.e. before the target enabled. - - -VDISK device handler --------------------- - -VDISK has 4 built-in dev handlers: vdisk_fileio, vdisk_blockio, -vdisk_nullio and vcdrom. Roots of their sysfs interface are -/sys/kernel/scst_tgt/handlers/handler_name, e.g. for vdisk_fileio: -/sys/kernel/scst_tgt/handlers/vdisk_fileio. Each root has the following -entries: - - - None, one or more links to devices with name equal to names - of the corresponding devices. - - - trace_level - allows to enable and disable various tracing - facilities. See content of this file for help how to use it. See also - section "Dealing with massive logs" for more info how to make correct - logs when you enabled trace levels producing a lot of logs data. - - - mgmt - main management entry, which allows to add/delete VDISK - devices with the corresponding type. - -The "mgmt" file has the following commands, which you can send to it, -for instance, using "echo" shell command. You can always get a small -help about supported commands by looking inside this file. "Parameters" -are one or more param_name=value pairs separated by ';'. - - - echo "add_device device_name [parameters]" - adds a virtual device - with name device_name and specified parameters (see below) - - - echo "del_device device_name" - deletes a virtual device with name - device_name. - -Handler vdisk_fileio provides FILEIO mode to create virtual devices. -This mode uses as backend files and accesses to them using regular -read()/write() file calls. This allows to use full power of Linux page -cache. The following parameters possible for vdisk_fileio: - - - filename - specifies path and file name of the backend file. The path - must be absolute. - - - blocksize - specifies block size used by this virtual device. The - block size must be power of 2 and >= 512 bytes. Default is 512. - - - write_through - disables write back caching. Note, this option - has sense only if you also *manually* disable write-back cache in - *all* your backstorage devices and make sure it's actually disabled, - since many devices are known to lie about this mode to get better - benchmark results. Default is 0. - - - read_only - read only. Default is 0. - - - o_direct - disables both read and write caching. This mode isn't - currently fully implemented, you should use user space fileio_tgt - program in O_DIRECT mode instead (see below). - - - nv_cache - enables "non-volatile cache" mode. In this mode it is - assumed that the target has a GOOD UPS with ability to cleanly - shutdown target in case of power failure and it is software/hardware - bugs free, i.e. all data from the target's cache are guaranteed - sooner or later to go to the media. Hence all data synchronization - with media operations, like SYNCHRONIZE_CACHE, are ignored in order - to bring more performance. Also in this mode target reports to - initiators that the corresponding device has write-through cache to - disable all write-back cache workarounds used by initiators. Use with - extreme caution, since in this mode after a crash of the target - journaled file systems don't guarantee the consistency after journal - recovery, therefore manual fsck MUST be ran. Note, that since usually - the journal barrier protection (see "IMPORTANT" note below) turned - off, enabling NV_CACHE could change nothing from data protection - point of view, since no data synchronization with media operations - will go from the initiator. This option overrides "write_through" - option. Disabled by default. - - - thin_provisioned - enables thin provisioning facility, when remote - initiators can unmap blocks of storage, if they don't need them - anymore. Backend storage also must support this facility. - - - tst - allows to specify TST control mode page field. It specifies - the type of task set in the device. Possible values are: 0 - the - device maintains one task set for all I_T nexuses and 1 - the device - maintains separate task sets for each I_T nexus. Default - 1. - - - removable - with this flag set the device is reported to remote - initiators as removable. - - - rotational - if set, this device reported as rotational. Otherwise, - it is reported as non-rotational (SSD, etc.) - - - zero_copy - if set, then this device uses zero copy access to the - page cache. At the moment, only read side zero copy is implemented. - - - dif_mode - specifies which T10-PI, or DIF, mode this device will use. - See SCSI standards from more info about T10-PI. Available DIF modes - (can be combined using '|'): - - * tgt - DIF tags are checked on the target hardware, if supported - - * scst - DIF tags are checked inside SCST core - - * dev_check - DIF tags are checked inside backend device. No DIF - tags storing is required, but optionally possible. - - * dev_store - DIF tags are stored inside backend device on the WRITE - path and read from it on the READ path. No DIF tags checking is - required, but optionally possible. - - For instance, if only tgt DIF mode specified, then target driver, - serving this device, will inside hardware check, then STRIP DIF tags - from SCSI commands on the WRITE path and generate, then INSERT DIF - tags into SCSI commands on the READ path, so neither SCST core, nor - dev handler will see them. - - Similarly, if only scst DIF mode specified, then target driver will - PASS DIF tags into SCST core, which then check/STRIP/generate/INSERT - them, so dev handler will not see them. - - If only dev_check DIF mode specified, then both target driver and - SCST core will PASS DIF tags into the dev handler, which is then - responsible to check them in the backend hardware. If only dev_store - specified, then DIF tags will only be stored by the dev handler in - the backend hardware without checking at any level. - - If all "tgt|scst|dev_check|dev_store" DIF mode specified, then all - target driver, SCST core and dev handler will check DIF tags, then - dev handler will store them in the backend hardware. - - - dif_type - specifies which DIF SCSI type this device will use. - - - dif_static_app_tag - specifies fixed (static) DIF application tag for - this device. - - - dif_filename - specifies full path to filename, where DIF tags will - be stored. - -Handler vdisk_blockio provides BLOCKIO mode to create virtual devices. -This mode performs direct block I/O with a block device, bypassing the -page cache for all operations. This mode works ideally with high-end -storage HBAs and for applications that either do not need caching -between application and disk or need the large block throughput. See -below for more info. - -The following parameters possible for vdisk_blockio: filename, -blocksize, nv_cache, read_only, removable, rotational, thin_provisioned, -tst, dif_mode, dif_type, dif_static_app_tag, dif_filename. See -vdisk_fileio above for description of those parameters. - -vdisk_blockio devices have the following two additional attributes: - -- active - if this flag is set (the default), the backing block device - will be opened when the SCST device is added/opened. If a SCST device - is opened with active=0 then the backing block device will not be - opened, allowing for an active/passive SCST configuration. In addition, - this attribute is writable via sysfs allowing the user to open/close the - backing block device on the fly, or via a script. - -- bind_alua_state - if this flag is set (the default), when the device is - associated with an ALUA device group, and a target group ALUA state - changes to the active/nonoptimized state, the active attribute will be - set to 1 which attempts to open the backing block device. If the target - group ALUA state changes to a value other than active/nonoptimized, the - backing device will be closed (active=0). If bind_alua_state=0 for a - device the ALUA state changes have NO effect on the active attribute, - it is left up to the user to use a script, or manually set the active - attribute to open/close the backing block device. - -Handler vdisk_nullio provides NULLIO mode to create virtual devices. In -this mode no real I/O is done, but success returned to initiators. -Intended to be used for performance measurements at the same way as -"*_perf" handlers. The following parameters possible for vdisk_nullio: -blocksize, read_only, removable, tst. See vdisk_fileio above for -description of those parameters. - -vdisk_nullio devices have the following two additional attributes: - - - dummy - if this flag is set, LUNs corresponding to this device will - not appear at the initiator side. This is because SCST will set the - PERIPHERAL QUALIFIER qualifier field to 1 (not connected) and the - PERIPHERAL DEVICE TYPE to 0x1f (no device) in the INQUIRY response. - See also SPC-4 for more information. It is designed to be used as a - "dummy" placeholder on LUN 0, if LUN 0 is not desired. - - - read_zero - if this flag is set, reading from a vdisk_nullio device - returns a buffer filled with byte 0x00. If this flag is cleared - (which is the default behavior), the buffer returned to the - initiator is not cleared. Although this results in slightly faster - operation this is a security hole since any data that is present in - kernel memory can be returned to the initiator. - -Handler vcdrom allows emulation of a virtual CDROM device using an ISO -file as backend. It has only single parameter: tst. - -For example: - -echo "add_device disk1 filename=/disk1; blocksize=4096; nv_cache=1" >/sys/kernel/scst_tgt/handlers/vdisk_fileio/mgmt - -will create a FILEIO virtual device disk1 with backend file /disk1 -with block size 4K and NV_CACHE enabled. - -Each vdisk_fileio's device has the following attributes in -/sys/kernel/scst_tgt/devices/device_name: - - - filename - contains path and file name of the backend file. - - - blocksize - contains block size used by this virtual device. - - - write_through - contains status of write back caching of this virtual - device. - - - sync - writing into this attribute causes the page cache contents to - be flushed to disk. - - - read_only - contains read only status of this virtual device. - - - o_direct - contains O_DIRECT status of this virtual device. - - - inq_vend_specific - Vendor specific data that will be reported via - either bytes 36..55 or bytes 96..256 of the INQUIRY response, depending - on whether this field is <= 20 or > 20 bytes long. - - - nv_cache - contains NV_CACHE status of this virtual device. - - - prod_id - PRODUCT IDENTIFICATION as reported via the INQUIRY response. - The default value for this field is the SCST device name. - - - prod_rev_lvl - PRODUCT REVISION LEVEL as reported via the INQUIRY - response. The default value for this field is " 300". - - - scsi_device_name - optional SCSI target device name to which this - SCST device belongs to (in SCSI terminology all SCST devices called - Logical Units). See SPC for more info. - - - tst - contains TST field of SCSI Control mode page. See SPC-4 for - more details about this field. - - - thin_provisioned - contains thin provisioning status of this virtual - device. - - - gen_tp_soft_threshold_reached_UA - for thin provisioned devices - writing of anything into this write-only attribute will generate THIN - PROVISIONING SOFT THRESHOLD REACHED Unit Attention to all connected - to this device initiators. - - - removable - contains removable status of this virtual device. - - - rotational - contains rotational status of this virtual device. - - - size_mb - contains size of this virtual device in MB. - - - pr_file_name - Full path of the file or block device in which to store - persistent reservation information. The default value for this attribute is - /var/lib/scst/pr/${device_name}. Writing a new value into this sysfs - attribute is only allowed if the device is not exported. Modifying this - sysfs attribute causes the persistent reservation state to be reloaded. - - - t10_dev_id - contains and allows to set T10 vendor specific - identifier for Device Identification VPD page (0x83) of INQUIRY data. - By default VDISK handler always generates t10_dev_id for every new - created device at creation time based on the device name and - scst_vdisk_ID scst_vdisk.ko module parameter (see below). - Note: some initiators, e.g. VMware's ESXi or MS Hyper-V, only looks - at the first eight characters of t10_dev_id. You have to make sure - that these first eight characters are unique or VMware will consider - these devices as identical. - - - eui64_id - allows to set the EUI-64 based device identifier in the - SCSI device identification VPD page (83h). This identifier must be 8, - 12 or 16 bytes long and must be specified in hexadecimal format (EUI = - Extended Unique Identifier). A leading "0x" is allowed but is not - required. Writing a newline into this attribute discards the EUI-64 - identifier. If neither eui64_id nor naa_id have been set the first - eight bytes of the t10_dev_id are used as the EUI-64 ID. If naa_id has - been set but eui64_id has not been set no EUI-64 identifier is - reported in the SCSI device identification VPD page. If eui64_id has - been set the value of this attribute is reported as the EUI-64 ID. The - first three bytes of an EUI-64 ID are a so-called organizationally - unique identifier (OUI). The remaining bytes may be chosen by the - organization that owns the OUI. For more information about OUIs, see - also http://standards.ieee.org/develop/regauth/oui/public.html. - - - naa_id - allows to set the NAA ID in the SCSI INQUIRY response (NAA = - Network Address Authority). This identifier must be 8 or 16 bytes long - and must be specified in hex format. A leading "0x" is allowed but is - not required. Writing a newline into this attribute discards the NAA - ID. If this ID is set it is reported in the SCSI VPD device - identification page (83h). More information about NAA identifiers can - be found in the following documents: - * ANSI T11 committee, Fibre Channel Framing and Signaling Interface - 4 - (FC-FS-4) rev 0.50, May 2014 (http://www.t11.org/). - * IETF, RFC 3980 - T11 Network Address Authority (NAA) Naming Format for - iSCSI Node Names, February 2005 (https://tools.ietf.org/html/rfc3980). - - - t10_vend_id - Contents of the T10 VENDOR IDENTIFICATION field of the - INQUIRY response. The default value for this field is "SCST_BIO" for - vdisk_block devices and "SCST_FIO" for vdisk_fileio devices. - - - usn - contains the virtual device's serial number of INQUIRY data. It - is created at the device creation time based on the device name and - scst_vdisk_ID scst_vdisk.ko module parameter (see below). - - - type - contains SCSI type of this virtual device. - - - resync_size - write only attribute, which makes vdisk_fileio to - rescan size of the backend file. It is useful if you changed it, for - instance, if you resized it. - - - vend_specific_id - Vendor specific ID as reported via the Device - Identification VPD page (83h). The default value for this attribute - is the value of the t10_dev_id attribute. - -For example: - -/sys/kernel/scst_tgt/devices/disk1 -|-- block -|-- blocksize -|-- exported -| |-- export0 -> ../../../targets/iscsi/iqn.2006-10.net.vlnb:tgt/luns/0 -| |-- export1 -> ../../../targets/iscsi/iqn.2006-10.net.vlnb:tgt/ini_groups/INI/luns/0 -| |-- export2 -> ../../../targets/iscsi/iqn.2006-10.net.vlnb:tgt1/luns/0 -| |-- export3 -> ../../../targets/iscsi/iqn.2006-10.net.vlnb:tgt1/ini_groups/INI1/luns/0 -| |-- export4 -> ../../../targets/iscsi/iqn.2006-10.net.vlnb:tgt1/ini_groups/INI2/luns/0 -|-- filename -|-- handler -> ../../handlers/vdisk_fileio -|-- nv_cache -|-- o_direct -|-- read_only -|-- removable -|-- resync_size -|-- rotational -|-- size_mb -|-- t10_dev_id -|-- thin_provisioned -|-- threads_num -|-- threads_pool_type -|-- tst -|-- type -|-- usn -`-- write_through - -Each vdisk_blockio's device has the following attributes in -/sys/kernel/scst_tgt/devices/device_name: blocksize, filename, nv_cache, -read_only, removable, resync_size, rotational, size_mb, t10_dev_id, -thin_provisioned, gen_tp_soft_threshold_reached_UA, threads_num, -threads_pool_type, tst, type, usn. See above description of those -parameters. - -Each vdisk_nullio's device has the following attributes in -/sys/kernel/scst_tgt/devices/device_name: blocksize, read_only, -removable, size_mb, t10_dev_id, threads_num, threads_pool_type, type, -tst, usn, dummy. See above description of those parameters. - -Each vcdrom's device has the following attributes in -/sys/kernel/scst_tgt/devices/device_name: filename, size_mb, -t10_dev_id, threads_num, threads_pool_type, type, usn, tst. See above -description of those parameters. Exception is filename attribute. For -vcdrom it is writable. Writing to it allows to virtually insert or -change virtual CD media in the virtual CDROM device. For example: - - - echo "/image.iso" >/sys/kernel/scst_tgt/devices/cdrom/filename - will - insert file /image.iso as virtual media to the virtual CDROM cdrom. - - - echo "" >/sys/kernel/scst_tgt/devices/cdrom/filename - will remove - "media" from the virtual CDROM cdrom. - -Additionally VDISK handler has module parameter "num_threads", which -specifies count of I/O threads for each FILEIO VDISK's or VCDROM device. -If you have a workload, which tends to produce rather random accesses -(e.g. DB-like), you should increase this count to a bigger value, like -32. If you have a rather sequential workload, you should decrease it to -a lower value, like number of CPUs on the target or even 1. Due to some -limitations of Linux I/O subsystem, increasing number of I/O threads too -much leads to sequential performance drop, especially with deadline -scheduler, so decreasing it can improve sequential performance. The -default provides a good compromise between random and sequential -accesses. - -You shouldn't be afraid to have too many VDISK I/O threads if you have -many VDISK devices. Kernel threads consume very little amount of -resources (several KBs) and only necessary threads will be used by SCST, -so the threads will not trash your system. - -CAUTION: If you partitioned/formatted your device with block size X, *NEVER* -======== ever try to export and then mount it (even accidentally) with another - block size. Otherwise you can *instantly* damage it pretty - badly as well as all your data on it. Messages on initiator - like: "attempt to access beyond end of device" is the sign of - such damage. - - Moreover, if you want to compare how well different block sizes - work for you, you **MUST** EVERY TIME AFTER CHANGING BLOCK SIZE - **COMPLETELY** **WIPE OFF** ALL THE DATA FROM THE DEVICE. In - other words, THE **WHOLE** DEVICE **MUST** HAVE ONLY **ZEROS** - AS THE DATA AFTER YOU SWITCH TO NEW BLOCK SIZE. Switching block - sizes isn't like switching between FILEIO and BLOCKIO, after - changing block size all previously written with another block - size data MUST BE ERASED. Otherwise you will have a full set of - very weird behaviors, because blocks addressing will be - changed, but initiators in most cases will not have a - possibility to detect that old addresses written on the device - in, e.g., partition table, don't refer anymore to what they are - intended to refer. - -IMPORTANT: Some disk and partition table management utilities don't support -========= block sizes >512 bytes, therefore make sure that your favorite one - supports it. Currently only cfdisk is known to work only with - 512 bytes blocks, other utilities like fdisk on Linux or - standard disk manager on Windows are proved to work well with - non-512 bytes blocks. Note, if you export a disk file or - device with some block size, different from one, with which - it was already partitioned, you could get various weird - things like utilities hang up or other unexpected behavior. - Hence, to be sure, zero the exported file or device before - the first access to it from the remote initiator with another - block size. On Window initiator make sure you "Set Signature" - in the disk manager on the imported from the target drive - before doing any other partitioning on it. After you - successfully mounted a file system over non-512 bytes block - size device, the block size stops matter, any program will - work with files on such file system. - - -Dealing with massive logs -------------------------- - -If you want to enable using "trace_level" file logging levels, which -produce a lot of events, like "debug", to not loose logged events you -should also: - - * Increase in .config of your kernel CONFIG_LOG_BUF_SHIFT variable - to much bigger value, then recompile it. For example, value 25 will - provide good protection from logging overflow even under high volume - of logging events. To use it you will need to modify the maximum - allowed value for CONFIG_LOG_BUF_SHIFT in the corresponding Kconfig - file to 25 as well. - - * Change in your /etc/syslog.conf or other config file of your favorite - logging program to store kernel logs in async manner. For example, - you can add in rsyslog.conf line "kern.info -/var/log/kernel" and - add "kern.none" in line for /var/log/messages, so the resulting line - would looks like: - - "*.info;kern.none;mail.none;authpriv.none;cron.none /var/log/messages" - - -Persistent Reservations ------------------------ - -SCST implements Persistent Reservations with full set of capabilities, -including "Persistence Through Power Loss". - -The "Persistence Through Power Loss" data are saved in /var/lib/scst/pr -with files with names the same as the names of the corresponding -devices. Also this directory contains backup versions of those files -with suffix ".1". Those backup files are used in case of power or other -failure to prevent Persistent Reservation information from corruption -during update. It is safe to assume that each of those files can be up -to 1KB big. - -The Persistent Reservations available on all transports implementing -get_initiator_port_transport_id() callback. Transports not implementing -this callback will act in one of 2 possible scenarios ("all or -nothing"): - -1. If a device has such transport connected and doesn't have persistent -reservations, it will refuse Persistent Reservations commands as if it -doesn't support them. - -2. If a device has persistent reservations, all initiators newly -connecting via such transports will not see this device. After all -persistent reservations from this device are released, upon reconnect -the initiators will see it. - - -ALUA Support ------------- - -SCST supports both implicit and explicit asymmetric logical unit access -(ALUA). ALUA is a feature defined by the ANSI T10 SCSI committee. It -allows a target to tell the initiator which path to use in a multipath -setup plus, in the explicit case, control state of each path via SET -TARGET PORT GROUPS SCSI command. The redundant paths between initiator -and target can be used either for redundancy or for load sharing -purposes. The target can either be a single target system running SCST -with multiple communication interfaces or two target systems each -running SCST and configured in a high availability setup. - -In the SPC-4 standard the following concepts are defined related to ALUA: -* Relative target port ID. A number between 1 and 65535 that uniquely - identifies a target port. These numbers must be unique over the target as - a whole, even if that target consists of multiple systems each running SCST. -* Target port group asymmetric access state. One of active/optimized, - active/non-optimized, standby, unavailable, logical block dependent or - offline. The access state of a port defines which (if any) SCSI commands - will be processed by the target port. -* Target port preference indicator. This indicator is additional information - next to the asymmetric access state that is provided by the target to an - initiator and that may impact the decision taken by the initiator about - which path that will be chosen. - -More detailed information about ALUA can be found in section 5.11.2 of the -ANSI T10 standard called SPC-4. - -ALUA support in SCST -.................... - -SCST allows to define ALUA settings for each unique combination of SCST -device and SCST target. An initiator however queries ALUA settings by -sending an appropriate SCSI command to a specific LUN of an SCST target. -Each such LUN maps uniquely to an SCST device. For hardware SCST target -drivers, e.g. ib_srpt, there is a one-to-one correspondence between SCST -target and SCSI target port. With other SCST targets, e.g. iSCSI-SCST, -by default the only relationship between SCST targets and SCSI target -ports is that all SCST targets defined on a system are visible via all -SCSI target ports. See also the iSCSI-SCST documentation about the -allowed_portal attribute for information about how to associate iSCSI -targets with a single physical interface. - -Notes: -- In a H.A. setup it is the responsibility of the user to synchronize ALUA - information between the individual systems running SCST. There are no - provisions in SCST to exchange ALUA information automatically between - individual systems. -- In order to support H.A. setups it is possible to let one SCST system - report information about target ports present in other SCST systems. -- With SCST, and certainly in a H.A. setup, it is possible to configure ALUA - such that an initiator receives information that is not standard compliant, - e.g. setting all target ports in the offline state. It is the responsibility - of the user to make sure that the information queried by an initiator is - consistent independent of the LUN and the target port used by the initiator - to query this information. -- Before building a H.A. setup consisting of two or more SCST systems one - should evaluate whether it's acceptable that persistent reservation commands, - SCSI task management commands and MODE SELECT commands will only be processed - by a single node instead of being processed by all nodes. - -Configuring ALUA in SCST -........................ - -SCST allows to configure the following settings related to ALUA -for each unique combination of SCST target and virtual SCST device -(vdisk_fileio, vdisk_blockio, vcdrom, ...): -* The target port group asymmetric access state. SCST supports all ALUA port - states except logical block dependent. -* The preference indicator for a target port group. -* The relative target port ID associated with the SCST target. - -It is possible to configure the following ALUA-related information via the -sysfs interface of SCST: -* Device groups, where each device group has a name and contains zero or more - SCST devices. If a device group contains only a single SCST device, the name - of the group may be identical to the device name. See also - /sys/kernel/scst_tgt/device_groups/mgmt. -* Which devices are inside a device group. See also - /sys/kernel/scst_tgt/device_groups//devices/mgmt. -* Target groups, where each target group has a name and contains zero or more - SCST target names. See also - /sys/kernel/scst_tgt/device_groups//target_groups/mgmt. -* Target port group identifier. This is a number in the range 0..65535 and is - called the TARGET PORT GROUP in SPC-4. See also - /sys/kernel/scst_tgt/device_groups//target_groups//group_id. -* Target port group preference indicator. This is a boolean value called the - PREF bit in SPC-4. See also /sys/kernel/scst_tgt/device_groups//target_groups//preferred. -* Target port group state name. One of active, nonoptimized, standby, - unavailable, offline or transitioning. See also - /sys/kernel/scst_tgt/device_groups//target_groups//state. -* Target group contents - zero or more target names. The target names either - exist on the local system or on a remote system in a H.A. setup. For target - names that refer to SCST targets on another system only the relative target - port identifier matters, not the assigned name. See also - /sys/kernel/scst_tgt/device_groups//target_groups//mgmt. -* Relative target identifier. See also - /sys/kernel/scst_tgt/device_groups//target_groups///rel_tgt_id. - -The steps involved in configuring ALUA are: -* Identify the SCST devices that will always share the same ALUA settings and - state. Assign a name to each such group of SCST devices. If a device group - only contains a single device, the group name may be identical to the device - name. -* Configure that device group in SCST via sysfs. -* Identify the SCSI target ports that will always share the same ALUA settings - and state. Assign a name, a group ID and preference indicator to each such - SCSI target port group. -* Configure the target port group information in SCST via sysfs. -* Identify all SCST targets that can be accessed via a target port group. -* Assign all these SCST target names to the target group via sysfs. -* Assign a relative target port identifier to each target. - -As an example, in a H.A. setup with two systems each having one InfiniBand -HCA controlled by the ib_srpt driver and where each system exports two LUNs -the following configuration can be used in scst.conf on both systems: - -DEVICE_GROUP dgroup1 { - DEVICE disk01 - - TARGET_GROUP tgroup1 { - group_id 256 - preferred 1 - state active - TARGET fe80:0000:0000:0000:0002:c903:00fa:b7e1 { - rel_tgt_id 1 - } - } - TARGET_GROUP tgroup2 { - group_id 257 - state standby - TARGET fe80:0000:0000:0000:0002:c903:00fa:b7f2 { - rel_tgt_id 2 - } - } -} - -DEVICE_GROUP dgroup2 { - DEVICE disk02 - - TARGET_GROUP tgroup1 { - group_id 258 - state standby - TARGET fe80:0000:0000:0000:0002:c903:00fa:b7e1 { - rel_tgt_id 1 - } - } - TARGET_GROUP tgroup2 { - group_id 259 - preferred 1 - state active - TARGET fe80:0000:0000:0000:0002:c903:00fa:b7f2 { - rel_tgt_id 2 - } - } -} - -Note, if you are using "active" BLOCKIO device attribute to prevent open -of the backend block device on the passive node, it is not recommended -to set both active ("active", "nonoptimized") and passive ("standby", -etc.) ALUA states for the same device if "bind_alua_state=1" is used, as -shown above to keep internal "active" state of the BLOCKIO device consistent. - -If using the "active" BLOCKIO device attribute and multiple target groups -exist per device on a SCST instance then "bind_alua_state=0" should be used -and it is left up to the user to modify the "active" attribute value. - -Explicit ALUA -............. - -To enable explicit ALUA you need in addition to the above settings set -expl_alua device attribute to 1 (by default it is 0). Also you need to -run stpgd and supply to it path to a script or program, which will -perform actual path state switching on SET TARGET PORT GROUPS command, -for instance, by calling drbdadm. For more information see stpgd README -as well as sample script scst_on_stpg. - -DRBD and other replication/failover SW compatibility -.................................................... - -DRBD as well as other replication/failover SW does not allow to open its -device on the secondary as well as does not allow to perform primary to -secondary transition, if this device is open. - -SCST BLOCKIO handler has necessary support for such behavior: - -1. If you need to prevent an SCST BLOCKIO device from opening its block -device, you need to create it with parameter "active=0". In case of DRBD -it would be done automatically, you don't have to use the "active" -attribute. - -2. By default, if you write new ALUA state in the "state" attribute and -"bind_alua_state=1" for the device, SCST BLOCKIO handler before transition -closes open handles on all affected SCST devices and after transition -reopens them, if the new state is active or nonoptimized. Alternatively, -set "bind_alua_state=0" for SCST BLOCKIO devices and ALUA state changes -will not open/close the backing block device, the user will neeed to handle -this manually or via a cluster RA in an HA setup. - -Thus, the recommended implicit ALUA state change procedure for primary -to secondary transition is: - -1. Block all involved SCST devices using "block" sysfs attribute (see -above). Wait until the blocking finished. - -2. Change the ALUA state to "transitioning". At this moment all open -file handles will be closed. - -3. Perform the DRBD or other replication/failover SW state transition - -4. Change the ALUA state to your desired secondary state. - -5. Unblock the blocked on step 1 devices. - -Optionally, if your initiators support Transitioning ALUA state, for -more responsive behavior the blocked devices can be unblocked -immediately after step (2). However, not all initiators correctly -behave, if they receive ASYMMETRIC STATE TRANSITION sense. - -For the secondary to primary transition procedure is similar. - -In case of explicit ALUA, SCST automatically performs the necessary -devices blocking around sending SCST_EVENT_STPG_USER_INVOKE event. - -Checking the Target Configuration -................................. - -One way to verify the ALUA configuration from a Linux initiator is via -the commands provided in the sg3_utils package. The first step is to -verify whether for a certain LUN ALUA has been configured on the target. -This is possible by checking whether the TPGS=1 text appears in the -sg_inq output, where /dev/sdb is a device node created by the ib_srp -initiator: - -# sg_inq /dev/sdb -standard INQUIRY: - PQual=0 Device_type=0 RMB=0 version=0x05 [SPC-3] - [AERC=0] [TrmTsk=0] NormACA=0 HiSUP=1 Resp_data_format=2 - SCCS=0 ACC=0 TPGS=1 3PC=0 Protect=0 BQue=0 - EncServ=0 MultiP=0 [MChngr=0] [ACKREQQ=0] Addr16=1 - [RelAdr=0] WBus16=0 Sync=0 Linked=0 [TranDis=0] CmdQue=1 - [SPI: Clocking=0x0 QAS=0 IUS=0] - length=66 (0x42) Peripheral device type: disk - Vendor identification: SCST_FIO - Product identification: disk01 - Product revision level: 300 - Unit serial number: 27cddc71 - -The next step is to verify the target group configuration. That is possible -by verifying whether the output of the sg_rtpg command matches the values -configured on the target: - -# sg_rtpg /dev/sdb -Report target port groups: - target port group id : 0x100 , Pref=1 - target port group asymmetric access state : 0x00 - T_SUP : 0, O_SUP : 0, LBD_SUP : 0, U_SUP : 1, S_SUP : 1, AN_SUP : 1, AO_SUP : 1 - status code : 0x02 - vendor unique status : 0x00 - target port count : 01 - Relative target port ids: - 0x01 - target port group id : 0x101 , Pref=0 - target port group asymmetric access state : 0x02 - T_SUP : 0, O_SUP : 0, LBD_SUP : 0, U_SUP : 1, S_SUP : 1, AN_SUP : 1, AO_SUP : 1 - status code : 0x02 - vendor unique status : 0x00 - target port count : 01 - Relative target port ids: - 0x02 - -The relative target port ID and the target port group ID for a certain path -can be queried e.g. as follows: - -# sg_vpd -p di /dev/sdb -Device Identification VPD page: - Addressed logical unit: - designator type: T10 vendor identification, code set: ASCII - vendor id: SCST_FIO - vendor specific: 27cddc71-disk01 - designator type: EUI-64 based, code set: Binary - 0x3237636464633731 - Target port: - designator type: Relative target port, code set: Binary - Relative target port: 0x1 - designator type: Target port group, code set: Binary - Target port group: 0x100 - - -Initiator Support -................. - -On Linux systems ALUA support is provided by the scsi_dh_alua kernel -driver in combination with the user space multipathd daemon. You will -have to modify at least the following in /etc/multipath.conf to enable -ALUA: - -* hardware_handler "1 alua" -* prio alua -* path_grouping_policy group_by_prio -* path_checker tur - -Notes: -- Newer versions of multipathd support a parameter called - "detect_prio". It can be more convenient to enable this parameter instead of - setting the parameter "prio" to "alua" for only those LUNs that support ALUA. -- Older versions of multipathd (e.g. RHEL 5 and SLES 10 SP1) need - 'prio_callout "/sbin/mpath_prio_alua /dev/%n"' instead of 'prio alua'. - -# multipath -ll -23237636464633731 dm-3 SCST_FIO,disk01 -size=1.0G features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 alua' wp=rw -|-+- policy='service-time 0' prio=1 status=active -| `- 10:0:0:0 sdd 8:48 active ready running -`-+- policy='service-time 0' prio=130 status=enabled - `- 11:0:0:0 sde 8:64 active ready running -23133326137346538 dm-4 SCST_FIO,disk02 -size=1.0G features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 alua' wp=rw -|-+- policy='service-time 0' prio=130 status=active -| `- 10:0:0:2 sdn 8:208 active ready running -`-+- policy='service-time 0' prio=1 status=enabled - `- 11:0:0:2 sdp 8:240 active ready running - -The following information can be derived from the above output: -* That the hardware handler (hw_handler) has been set to "1 alua". -* That multipathd created two priority groups - one with priority 1 and one - with priority 130. -* That the SRP path with SCSI host number 10 will be used for communication - with LUN "disk01" and that the SRP path with SCSI host number 11 will be used - for communication with LUN "disk02". - -More information about how to configure the device mapper and the scsi_dh_alua -driver can be found in the manual of your Linux distribution ("man -multipath.conf", "man multipath" and "man multipathd"). - -Windows initiator systems support ALUA from Windows Server 2008 on. For more -information about ALUA support in Windows Server, see also: -* Microsoft, Windows Server 2008 R2 Multipath I/O Overview, MSDN - (http://technet.microsoft.com/en-us/library/cc725907.aspx). -* Microsoft, Multipathing Support in Windows Server 2008, July 2008, MSDN - (http://blogs.msdn.com/b/san/archive/2008/07/27/multipathing-support-in-windows-server-2008.aspx). -* Microsoft, ALUA MPIO Logo Test, MSDN - (http://msdn.microsoft.com/en-us/library/gg607458%28v=vs.85%29.aspx). - -Active/Non-Optimized via internal redirection -............................................. - -The Active-Standby configuration is simple to understand and setup, -however, it might have serious interoperability issues, because not all -initiators handle Standby state correctly. For instance, some versions -of VMware reported to have such issues. Same for Windows. - -Hence, it is better to use Non-Optimized state on the passive node -instead of Standby with internal commands redirection to the active -node. This is what the vast majority of storage vendors are doing, which -is, actually, the reason why Standby and Unavailable states have all -those initiator issues. Simply, they have had too few testing, because -only marginally used. - -SCST has necessary support for such redirection, it just needs to be -configured correctly. It's a little bit of effort, especially to -understand how it's going to function, but then it would work MUCH more -reliable for full range of initiators. Ever poor initiators, who have no -idea about ALUA (boot from SAN, e.g.) would work now. - -1. Build SCST with CONFIG_SCST_FORWARD_MODE_PASS_THROUGH enabled in scst.h - -2. Setup on active node internal redirect target, which is going to -accept redirected commands from the passive node. It must be visible -only to the passive node. - -3. Set "forwarding" attribute for this target to 1. This is necessary to -correctly handle PRs. - -4. Export through this target the SAME backend SCST device as being -served to initiator(s) (consider for simplicity that there is only one -served device) - -5. Connect to this SCST device through this internal target from the -passive node, for instance, using iSCSI. Now you have a local SCSI -device on the passive side pointing to the active node. - -6. Export this local device to the initiator(s) using SCST -*pass-through* handler (scst_disk). Pass-though is needed to redirect -non-block commands as well: ATS, XCOPY, etc. - -7. Set ALUA state to this target as "nonoptimized". - -That's it on the normal path. Now the initiator(s) would see 2 paths: -OPTIMIZED going to the active node and NON-OPTIMIZED going to the -passive node, then redirected to the active node. - -On failover (i.e. switching active and passive states): - -1. Setup similar redirect target on the new active node. - -2. Setup connectivity to that new redirect target from the new passive -node - -3. Start ALUA change (see above) on both nodes - -4. !! Exchange in the sysfs security group(s) for the initiator(s) *LUN* -from old SCST device to the new one (blockio -> pass-through on the new -passive and pass-through -> blockio on the new active) using "replace_no_ua" -SCST command. You need to do it directly in the sysfs interface, -scstadmin can't do it. - -5. Set ALUA states to "active" on the new active node and "nonoptimized" -on the new passive node. - -6. Finish ALUA states change. - -Example using direct sysfs interface could look like: - -Active-Optimized node: - -modprobe scst -modprobe scst_disk -modprobe scst_vdisk - -# Main device, DRBD primary here -echo "add_device aa filename=/dev/drbd1" >/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt - -# Redirect device, not used here. Coming from connecting via iSCSI to the -# corresponding redirect target on the other side. -DEVICE=10:0:0:0 -echo add_device $DEVICE >/sys/kernel/scst_tgt/handlers/dev_disk/mgmt - -service iscsi-scst start - -# This is a regular, user-visible target -echo "add_target iqn.2006-10.net.v:tgt " >/sys/kernel/scst_tgt/targets/iscsi/mgmt -echo 1 >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgt/rel_tgt_id -echo "add aa 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgt/luns/mgmt - -# This is redirect target, 192.168.9.x is the redirect network -echo "add_target iqn.2006-10.net.v:tgtR" >/sys/kernel/scst_tgt/targets/iscsi/mgmt -echo 2 >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgtR/rel_tgt_id -echo "add_target_attribute iqn.2006-10.net.v:tgtR allowed_portal 192.168.9.1" >/sys/kernel/scst_tgt/targets/iscsi/mgmt -echo "1" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgtR/forwarding -echo "add aa 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgtR/luns/mgmt - -echo 1 >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgt/enabled -echo 1 >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgtR/enabled - -echo 1 >/sys/kernel/scst_tgt/targets/iscsi/enabled - -# ALUA config - -echo create aa >/sys/kernel/scst_tgt/device_groups/mgmt -echo add aa >/sys/kernel/scst_tgt/device_groups/aa/devices/mgmt - -echo add tgt_a >/sys/kernel/scst_tgt/device_groups/aa/target_groups/mgmt -echo add iqn.2006-10.net.v:tgt >/sys/kernel/scst_tgt/device_groups/aa/target_groups/tgt_a/mgmt -echo 1 >/sys/kernel/scst_tgt/device_groups/aa/target_groups/tgt_a/group_id -echo active >/sys/kernel/scst_tgt/device_groups/aa/target_groups/tgt_a/state - -echo add tgt_n >/sys/kernel/scst_tgt/device_groups/aa/target_groups/mgmt -echo add iqn.2006-10.net.v:tgt1 >/sys/kernel/scst_tgt/device_groups/aa/target_groups/tgt_n/mgmt -echo 2 >/sys/kernel/scst_tgt/device_groups/aa/target_groups/tgt_n/iqn.2006-10.net.v:tgt1/rel_tgt_id -echo 2 >/sys/kernel/scst_tgt/device_groups/aa/target_groups/tgt_n/group_id -echo nonoptimized >/sys/kernel/scst_tgt/device_groups/aa/target_groups/tgt_n/state - -Active-Non-Optimized node: - -modprobe scst -modprobe scst_disk -modprobe scst_vdisk - -# Main device, DRBD secondary, not used here -echo "add_device aa filename=/dev/drbd1" >/sys/kernel/scst_tgt/handlers/vdisk_blockio/mgmt - -# Redirect device. Coming from connecting via iSCSI to the -# corresponding redirect target on the other side. -DEVICE=10:0:0:0 -echo add_device $DEVICE >/sys/kernel/scst_tgt/handlers/dev_disk/mgmt - -service iscsi-scst start - -echo "add_target iqn.2006-10.net.v:tgt1" >/sys/kernel/scst_tgt/targets/iscsi/mgmt -echo 2 >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgt1/rel_tgt_id -echo "add $DEVICE 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgt1/luns/mgmt - -# Redirect target, 192.168.9.x is the redirect network -echo "add_target iqn.2006-10.net.v:tgtR" >/sys/kernel/scst_tgt/targets/iscsi/mgmt -echo 2 >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgtR/rel_tgt_id -echo "add_target_attribute iqn.2006-10.net.v:tgtR allowed_portal 192.168.9.2" >/sys/kernel/scst_tgt/targets/iscsi/mgmt -echo "1" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgtR/forwarding -echo "add aa 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgtR/luns/mgmt - -echo 1 >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgt1/enabled - -echo 1 >/sys/kernel/scst_tgt/targets/iscsi/enabled -echo 1 >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgtR/enabled - -# ALUA config - -echo create $DEVICE >/sys/kernel/scst_tgt/device_groups/mgmt -echo add $DEVICE >/sys/kernel/scst_tgt/device_groups/$DEVICE/devices/mgmt - -echo add tgt_a >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/mgmt -echo add iqn.2006-10.net.v:tgt >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_a/mgmt -echo 1 >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_a/iqn.2006-10.net.v:tgt/rel_tgt_id -echo 1 >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_a/group_id -echo active >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_a/state - -echo add tgt_n >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/mgmt -echo add iqn.2006-10.net.v:tgt1 >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_n/mgmt -echo 1 >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_n/group_id -echo nonoptimized >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_n/state - -ALUA state switch after DRBD primary-secondary transition: - -Ex-Optimized: - -echo "replace_no_ua $DEVICE 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgt1/luns/mgmt -echo nonoptimized >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_a/state -echo active >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_n/state - -Ex-Non-Optimized: - -echo "replace_no_ua aa 0" >/sys/kernel/scst_tgt/targets/iscsi/iqn.2006-10.net.v:tgt1/luns/mgmt -echo nonoptimized >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_a/state -echo active >/sys/kernel/scst_tgt/device_groups/$DEVICE/target_groups/tgt_n/state - -If you have any questions, please read this above text at least 3 times -before asking. It might be tricky to understand :-) - - -VAAI ----- - -SCST supports all 3 VAAI SCSI commands: WRITE SAME, COMPARE AND WRITE -(ATS) and EXTENDED COPY. Additionally, it supports not directly related -to VAAI Thin Provisioning capabilities, particularly, UNMAP SCSI -commands, WRITE SAME with UNMAP bit as well as thin provisioning related -devices' sysfs attributes (see above). - -In some cases dev handlers should perform some manual actions to fully -benefit from SCST VAAI implementation. Those actions described in the -implementation notes below. For vdisk and fileio_tgt handlers they have -already been implemented. - -IMPORTANT: To use EXTENDED COPY command between LUNs (datastores) they all -========= MUST have the same PRODUCT IDENTIFICATION INQUIRY field. By - default, to simplify remote devices identification, SCST uses - vdisk names as PRODUCT IDENTIFICATION, so SCST devices look - differently from the initiators. However, for some reasons, - VMware does not use EXTENDED COPY between LUNs with different - PRODUCT IDENTIFICATION. Thus, to be able to use full VAAI in - your VMware setups you must manually set PRODUCT - IDENTIFICATION for all your VMware LUNs to the same value, - for instance, "SCST", via using "prod_id" attribute. It could - be done either by adding "prod_id" attribute to scstadmin - scst.conf, or by directly writing to SCST sysfs attribute. - For example: - - HANDLER vdisk_blockio { - DEVICE blockio1 { - filename /dev/sda5 - prod_id SCST - } - - or - - echo SCST >/sys/kernel/scst_tgt/devices/blockio1/prod_id - correspondingly. - - Note, this prod_id modification must be done on all - datastores BEFORE VMware connects to them. - - -Implementation notes -.................... - -WRITE SAME -~~~~~~~~~~ - -WRITE SAME command supports 2 modes: - -1. Manual writing mode. In this mode WRITE SAME generates a set of -internal WRITE(16) SCSI commands to perform requested writing. - -2. Remap mode. In this mode a dev handler, if supported, can remap being -written blocks to a single block and then tell SCST to manually write -parts of the requested area, which for some reason can not be remapped. - -In both cases dev handlers should call from WRITE SAME command handler -scst_write_same() function. This function as the second argument gets -array of descriptors where to write the requested block of data. Last -element in this array must have len 0. If this argument is NULL, then -the whole area will be manually written by SCST. This value should be -used by dev handlers not supporting remapping blocks. - -User space dev handlers should use SCST_EXEC_REPLY_DO_WRITE_SAME -reply_type of SCST_USER_EXEC subcommand. See scst_user doc for more -info. - - -COMPARE AND WRITE -~~~~~~~~~~~~~~~~~ - -COMPARE AND WRITE implemented by SCST a set of read, compare and write -actions done in atomic manner against affected blocks as well as regular -RESERVE SCSI commands. Particularly, COMPARE AND WRITE doesn't need any -queue flushing and unlimited number of COMPARE AND WRITE commands on -different blocks can be executed simultaneously. - -The read and write actions implemented as generation of internal -READ(16) and WRITE(16) SCSI commands. - -COMPARE AND WRITE command is completely transparent to dev handlers -(they only see the corresponding READ(16) and WRITE(16) commands), so -doesn't require any manual actions from them. - - -EXTENDED COPY -~~~~~~~~~~~~~ - -SCST implements EXTENDED COPY via internal Copy Manager target. This -target has the following specific attribute in its sysfs: - - - allow_not_connected_copy - if not set (default), an initiator can -perform copy only between devices it has direct access to via any -target/session. If set, any initiator can copy between any devices in -the system. - -The Copy Manager has access only to those devices, for which it has LUNs -in /sys/kernel/scst_tgt/targets/copy_manager/copy_manager_tgt/luns/. -Devices from scst_vdisk dev handler added to it automatically upon -registration, but for other devices you need to manually add LUNs there -the same way as for any target driver. You can also delete any device at -any time from the Copy Manager visibility by deleting the corresponding -LUN from the sysfs. It might be useful during ALUA state switching. - -Internally SCST implements EXTENDED COPY as generation of sets of -internal READ(16) and WRITE(16) SCSI commands. Dev handlers don't need -any manual actions to use it. - -Also SCST provides for dev handlers possibility to remap blocks instead -of copy them, if they support this feature. It allows them to perform -EXTENDED COPY command much faster by just metadata update of their -backend storage, which supposed to be nearly instantaneous. - -To use this feature, a dev handler should setup ext_copy_remap() -callback in its struct scst_dev_type. This callback is called by SCST -during EXTENDED COPY command processing to let the dev handler try to -remap affected blocks at first. - -Upon finish, the dev handler should call scst_ext_copy_remap_done(). In -case of error, the dev handler should set the corresponding sense to cmd -and then also call scst_ext_copy_remap_done(cmd, NULL, 0). - -If dev handler is not able to remap any part of the segment, if should -kmalloc(), then fill all leftover subsegments and supply them to -scst_ext_copy_remap_done(). SCST then will copy the subsegments using -internal copy machine, then kfree() the supplied array. If the dev -handler is not able to remap the whole segment, it can simply directly -supply the original segment to scst_ext_copy_remap_done(). - -It is highly recommended that in normal circumstances dev handlers call -scst_ext_copy_remap_done() from another thread context than one where -ext_copy_remap() callback was originally called, because otherwise there -could be recursion in the segments processing. Hopefully, this thread -context switch is natural for such potentially long operation as -EXTENDED COPY. - - -VMware and Ceph RBD space reclaim ---------------------------------- - -VMware with VMFS5 filesystem ignores UNMAP alignment, so if you use 4MB -Ceph RBD objects and VMFS5, only some discards will reclaim RBD space -due to 1MB discard not often hitting the tail of objects. - -Thus, to have efficient ESXi space reclamation with RBD and VMFS5, you are -recommended to use 1 MB object size in Ceph. - -See https://sourceforge.net/p/scst/mailman/message/35287598 thread for -details. - - -Caching -------- - -By default for performance reasons VDISK FILEIO devices use write back -caching policy. - -Generally, write back caching is safe for use and danger of it is -greatly overestimated, because most modern (especially, Enterprise -level) applications are well prepared to work with write back cached -storage. Particularly, such are all transactions-based applications. -Those applications flush cache to completely avoid ANY data loss on a -crash or power failure. For instance, journaled file systems flush cache -on each meta data update, so they survive power/hardware/software -failures pretty well. - -Since locally on initiators write back caching is always on, if an -application cares about its data consistency, it does flush the cache -when necessary or on any write, if open files with O_SYNC. If it doesn't -care, it doesn't flush the cache. As soon as the cache flushes -propagated to the storage, write back caching on it doesn't make any -difference. If application doesn't flush the cache, it's doomed to loose -data in case of a crash or power failure doesn't matter where this cache -located, locally or on the storage. - -To illustrate that consider, for example, a user who wants to copy /src -directory to /dst directory reliably, i.e. after the copy finished no -power failure or software/hardware crash could lead to a loss of the -data in /dst. There are 2 ways to achieve this. Let's suppose for -simplicity cp opens files for writing with O_SYNC flag, hence bypassing -the local cache. - -1. Slow. Make the device behind /dst working in write through caching -mode and then run "cp -a /src /dst". - -2. Fast. Let the device behind /dst working in write back caching mode -and then run "cp -a /src /dst; sync". The reliability of the result is -the same, but it's much faster than (1). Nobody would care if a crash -happens during the copy, because after recovery simply leftovers from -the not completed attempt would be deleted and the operation would be -restarted from the very beginning. - -So, you can see in (2) there is no danger of ANY data loss from the -write back caching. Moreover, since on practice cp doesn't open files -for writing with O_SYNC flag, to get the copy done reliably, sync -command must be called after cp anyway, so enabling write back caching -wouldn't make any difference for reliability. - -Also you can consider it from another side. Modern HDDs have at least -16MB of cache working in write back mode by default, so for a 10 drives -RAID it is 160MB of a write back cache. How many people are happy with -it and how many disabled write back cache of their HDDs? Almost all and -almost nobody correspondingly? Moreover, many HDDs lie about state of -their cache and report write through while working in write back mode. -They are also successfully used. - -Note, Linux I/O subsystem guarantees to propagated cache flushes to the -storage only using data protection barriers, which usually turned off by -default (see http://lwn.net/Articles/283161). Without barriers enabled -Linux doesn't provide a guarantee that after sync()/fsync() all written -data really hit permanent storage. They can be stored in the cache of -your backstorage devices and, hence, lost on a power failure event. -Thus, ever with write-through cache mode, you still either need to -enable barriers on your backend file system on the target (for direct -/dev/sdX devices this is, indeed, impossible), or need a good UPS to -protect yourself from not committed data loss. Some info about barriers -from the XFS point of view could be found at -http://xfs.org/index.php/XFS_FAQ#Write_barrier_support. On Linux -initiators for Ext3 and ReiserFS file systems the barrier protection -could be turned on using "barrier=1" and "barrier=flush" mount options -correspondingly. You can check if the barriers turn on or off by looking -in /proc/mounts. Windows and, AFAIK, other UNIX'es don't need any -special explicit options and do necessary barrier actions on write-back -caching devices by default. - -To limit this data loss with write back caching you can use files in -/proc/sys/vm to limit amount of unflushed data in the system cache. - -If you for some reason have to use VDISK FILEIO devices in write through -caching mode, don't forget to disable internal caching on their backend -devices or make sure they have additional battery or supercapacitors -power supply on board. Otherwise, you still on a power failure would -loose all the unsaved yet data in the devices internal cache. - -Note, on some real-life workloads write through caching might perform -better, than write back one with the barrier protection turned on. - - -Errors caching -.............. - -When using virtual device in FILEIO mode, the Linux page cache comes -into picture. The negative side of it is that it's sometimes also -caching errored pages. That is, if the underlying file experiences IO -errors, those errors might be cached by the Linux page cache. As a -result, even when the underlying file recovers and stops failing IOs, -the initiator may still hit IO errors returned by the Linux page cache, -until the cache re-reads the errored pages (usually it happens pretty -soon, but not immediately). To make sure that cached pages are dropped, -one of the following can be done: - -- Detach the SCSI virtual device (del_device) and re-attach it - (add_device). This should evict all the cached pages, unless somebody - else holds the same "filename" opened. - -- Issue a BLKFLSBUF ioctl to the same "filename" you provided for "add_device". - -For the second option, a rudimentary C code is required: - -fd = open(filename, O_RDWR); -if (fd < 0) { - err = errno; - ... -} else { - err = ioctl(fd, BLKFLSBUF); - if (err < 0) { - err = errno; - ... - } - close(fd); -} - - -BLOCKIO VDISK mode ------------------- - -This module works best for these types of scenarios: - -1) Data that are not aligned to 4K sector boundaries and <4K block sizes -are used, which is normally found in virtualization environments where -operating systems start partitions on odd sectors (Windows and it's -sector 63). - -2) Large block data transfers normally found in database loads/dumps and -streaming media. - -3) Advanced relational database systems that perform their own caching -which prefer or demand direct IO access and, because of the nature of -their data access, can actually see worse performance with -non-discriminate caching. - -4) Multiple layers of targets were the secondary and above layers need -to have a consistent view of the primary targets in order to preserve -data integrity which a page cache backed IO type might not provide -reliably. - -Also it has an advantage over FILEIO that it doesn't copy data between -the system cache and the commands data buffers, so it saves a -considerable amount of CPU power and memory bandwidth. - -IMPORTANT: Since data in BLOCKIO and FILEIO modes are not consistent between -========= each other, if you try to use a device in both those modes - simultaneously, you will almost instantly corrupt your data - on that device. - -IMPORTANT: If SCST 1.x BLOCKIO worked by default in NV_CACHE mode, when -========= each device reported to remote initiators as having write through - caching. But if your backend block device has internal write - back caching it might create a possibility for data loss of - the cached in the internal cache data in case of a power - failure. Starting from SCST 2.0 BLOCKIO works by default in - non-NV_CACHE mode, when each device reported to remote - initiators as having write back caching, and synchronizes the - internal device's cache on each SYNCHRONIZE_CACHE command - from the initiators. It might lead to some *PERFORMANCE LOSS*, - so if you are are sure in your power supply and want to - restore the 1.x behavior, your should recreate your BLOCKIO - devices in NV_CACHE mode. - - -Pass-through mode ------------------ - -In the pass-through mode (i.e. using the pass-through device handlers -scst_disk, scst_tape, etc) SCSI commands, coming from remote initiators, -are passed to local SCSI devices on target as is, without any -modifications. - -SCST supports 1 to many pass-through, when several initiators can safely -connect a single pass-through device (a tape, for instance). For such -cases SCST emulates all the necessary functionality. - -In the sysfs interface all real SCSI devices are listed in -/sys/kernel/scst_tgt/devices in form host:channel:id:lun numbers, for -instance 1:0:0:0. The recommended way to match those numbers to your -devices is use of lsscsi utility. - -Each pass-through dev handler has in its root subdirectory -/sys/kernel/scst_tgt/handlers/handler_name, e.g. -/sys/kernel/scst_tgt/handlers/dev_disk, "mgmt" file. It allows the -following commands. They can be sent to it using, e.g., echo command. - - - "add_device" - this command assigns SCSI device with -host:channel:id:lun numbers to this dev handler. - -echo "add_device 1:0:0:0" >/sys/kernel/scst_tgt/handlers/dev_disk/mgmt - -will assign SCSI device 1:0:0:0 to this dev handler. - - - "del_device" - this command unassigns SCSI device with -host:channel:id:lun numbers from this dev handler. - -As usually, on read the "mgmt" file returns small help about available -commands. - -You need to manually assign each your real SCSI device to the -corresponding pass-through dev handler using the "add_device" command, -otherwise the real SCSI devices will not be visible remotely. The -assignment isn't done automatically, because it could lead to the -pass-through dev handlers load and initialization problems if any of the -local real SCSI devices are malfunctioning. - -As any other hardware, the local SCSI hardware can not handle commands -with amount of data and/or segments count in scatter-gather array bigger -some values. Therefore, when using the pass-through mode you should note -that values for maximum number of segments and maximum amount of -transferred data (max_sectors) for each SCSI command on devices on -initiators can not be bigger, than corresponding values of the -corresponding SCSI devices on the target. Otherwise you will see -symptoms like small transfers work well, but large ones stall and -messages like: "Unable to complete command due to SG IO count -limitation" are printed in the kernel logs. - -You can't control from the user space limit of the scatter-gather -segments, but for block devices usually it is sufficient if you set on -the initiators /sys/block/DEVICE_NAME/queue/max_sectors_kb in the same -or lower value as in /sys/block/DEVICE_NAME/queue/max_hw_sectors_kb for -the corresponding devices on the target. - -For not-block devices SCSI commands are usually generated directly by -applications, so, if you experience large transfers stalls, you should -check documentation for your application how to limit the transfer -sizes. - -Another way to solve this issue is to build SG entries with more than 1 -page each. See the following patch as an example: -http://scst.sourceforge.net/sgv_big_order_alloc.diff - - -Performance ------------ - -SCST from the very beginning has been designed and implemented to -provide the best possible performance. Since there is no "one fit all" -the best performance configuration for different setups and loads, SCST -provides extensive set of settings to allow to tune it for the best -performance in each particular case. You don't have to necessary use -those settings. If you don't, SCST will do very good job to autotune for -you, so the resulting performance will, in average, be better -(sometimes, much better) than with other SCSI targets. But in some cases -you can by manual tuning improve it even more. - -Before doing any performance measurements note that performance results -are very much dependent from your type of load, so it is crucial that -you choose access mode (FILEIO, BLOCKIO, O_DIRECT, pass-through), which -suits your needs the best. - -In order to get the maximum performance you should: - -1. For SCST: - - - Disable CONFIG_SCST_STRICT_SERIALIZING, CONFIG_SCST_EXTRACHECKS, - CONFIG_SCST_TRACING, CONFIG_SCST_DEBUG*, CONFIG_SCST_STRICT_SECURITY. - -2. For target drivers: - - - Disable in Makefiles CONFIG_SCST_EXTRACHECKS, CONFIG_SCST_TRACING, - CONFIG_SCST_DEBUG* - -3. For device handlers, including VDISK: - - - Disable in Makefile CONFIG_SCST_TRACING and CONFIG_SCST_DEBUG. - -Note, by disabling CONFIG_SCST_TRACING and CONFIG_SCST_DEBUG you are -disabling many useful SCST diagnostic messages, which can significantly -help in many troubleshooting cases. So, if you may consider to keep -CONFIG_SCST_TRACING, its performance impact is very limited. - -4. Make sure you have io_grouping_type option set correctly, especially -in the following cases: - - - Several initiators share your target's backstorage. It can be a - shared LU using some cluster FS, like VMFS, as well as can be - different LUs located on the same backstorage (RAID array). For - instance, if you have 3 initiators and each of them using its own - dedicated FILEIO device file from the same RAID-6 array on the - target. - - In this case for the best performance you should have - io_grouping_type option set in value "never" in all the LUNs' targets - and security groups. - - - Your initiator connected to your target in MPIO mode. In this case for - the best performance you should: - - * Either connect all the sessions from the initiator to a single - target or security group and have io_grouping_type option set in - value "this_group_only" in the target or security group, - - * Or, if it isn't possible to connect all the sessions from the - initiator to a single target or security group, assign the same - numeric io_grouping_type value for each target/security group this - initiator connected to. The exact value itself doesn't matter, - important only that all the targets/security groups use the same - value. - -Don't forget, io_grouping_type makes sense only if you use CFQ I/O -scheduler on the target and for devices with threads_num >= 0 and, if -threads_num > 0, with threads_pool_type "per_initiator". - -You can check if in your setup io_grouping_type set correctly as well as -if the "auto" io_grouping_type value works for you by tests like the -following: - - - For not MPIO case you can run single thread sequential reading, e.g. - using buffered dd, from one initiator, then run the same single - thread sequential reading from the second initiator in parallel. If - io_grouping_type is set correctly the aggregate throughput measured - on the target should only slightly decrease as well as all initiators - should have nearly equal share of it. If io_grouping_type is not set - correctly, the aggregate throughput and/or throughput on any - initiator will decrease significantly, in 2 times or even more. For - instance, you have 80MB/s single thread sequential reading from the - target on any initiator. When then both initiators are reading in - parallel you should see on the target aggregate throughput something - like 70-75MB/s with correct io_grouping_type and something like - 35-40MB/s or 8-10MB/s on any initiator with incorrect. - - - For the MPIO case it's quite easier. With incorrect io_grouping_type - you simply won't see performance increase from adding the second - session (assuming your hardware is capable to transfer data through - both sessions in parallel), or can even see a performance decrease. - -5. If you are going to use your target in an VM environment, for -instance as a shared storage with VMware, make sure all your VMs -connected to the target via *separate* sessions. For instance, for iSCSI -it means that each VM has own connection to the target, not all VMs -connected using a single connection. You can check it using SCST sysfs -interface. For other transports you should use available facilities, -like NPIV for Fibre Channel, to make separate sessions for each VM. If -you miss it, you can greatly loose performance of parallel access to -your target from different VMs. This isn't related to the case if your -VMs are using the same shared storage, like with VMFS, for instance. In -this case all your VM hosts will be connected to the target via separate -sessions, which is enough. - -6. For other target and initiator software parts: - - - Make sure you applied on your kernel all available SCST patches. - If for your kernel version this patch doesn't exist, it is strongly - recommended to upgrade your kernel to version, for which this patch - exists. - - - Don't enable debug/hacking features in the kernel, i.e. use them as - they are by default. - - - The default kernel read-ahead and queuing settings are optimized - for locally attached disks, therefore they are not optimal if they - attached remotely (SCSI target case), which sometimes could lead to - unexpectedly low throughput. You should increase read-ahead size to at - least 512KB or even more on all initiators and the target. - - You should also limit on all initiators maximum amount of sectors per - SCSI command. This tuning is also recommended on targets with large - read-ahead values. To do it on Linux, run: - - echo “64” > /sys/block/sdX/queue/max_sectors_kb - - where specify instead of X your imported from target device letter, - like 'b', i.e. sdb. - - To increase read-ahead size on Linux, run: - - blockdev --setra N /dev/sdX - - where N is a read-ahead number in 512-byte sectors and X is a device - letter like above. - - Note: you need to set read-ahead setting for device sdX again after - you changed the maximum amount of sectors per SCSI command for that - device. - - Note2: you need to restart SCST after you changed read-ahead settings - on the target. It is a limitation of the Linux read ahead - implementation. It reads RA values for each file only when the file - is open and not updates them when the global RA parameters changed. - Hence, the need for vdisk to reopen all its files/devices. - - - You may need to increase amount of requests that OS on initiator - sends to the target device. To do it on Linux initiators, run - - echo “64” > /sys/block/sdX/queue/nr_requests - - where X is a device letter like above. - - You may also experiment with other parameters in /sys/block/sdX - directory, they also affect performance. If you find the best values, - please share them with us. - - - On the target use CFQ IO scheduler. In most cases it has performance - advantage over other IO schedulers, sometimes huge (2+ times - aggregate throughput increase). - - - It is recommended to turn the kernel preemption off, i.e. set - the kernel preemption model to "No Forced Preemption (Server)". - - - Looks like XFS is the best filesystem on the target to store device - files, because it allows considerably better linear write throughput, - than ext3. - -7. For hardware on target. - - - Make sure that your target hardware (e.g. target FC or network card) - and underlying IO hardware (e.g. IO card, like SATA, SCSI or RAID to - which your disks connected) don't share the same PCI bus. You can - check it using lspci utility. They have to work in parallel, so it - will be better if they don't compete for the bus. The problem is not - only in the bandwidth, which they have to share, but also in the - interaction between cards during that competition. This is very - important, because in some cases if target and backend storage - controllers share the same PCI bus, it could lead up to 5-10 times - less performance, than expected. Moreover, some motherboard (by - Supermicro, particularly) have serious stability issues if there are - several high speed devices on the same bus working in parallel. If - you have no choice, but PCI bus sharing, set in the BIOS PCI latency - as low as possible. - -8. If you use VDISK IO module in FILEIO mode, NV_CACHE option will -provide you the best performance. But using it make sure you use a good -UPS with ability to shutdown the target on the power failure. - -Baseline performance numbers you can find in those measurements: -http://lkml.org/lkml/2009/3/30/283. - -IMPORTANT: If you use on initiator some versions of Windows (at least W2K) -========= you can't get good write performance for VDISK FILEIO devices with - default 512 bytes block sizes. You could get about 10% of the - expected one. This is because of the partition alignment, which - is (simplifying) incompatible with how Linux page cache - works, so for each write the corresponding block must be read - first. Use 4096 bytes block sizes for VDISK devices and you - will have the expected write performance. Actually, any OS on - initiators, not only Windows, will benefit from block size - max(PAGE_SIZE, BLOCK_SIZE_ON_UNDERLYING_FS), where PAGE_SIZE - is the page size, BLOCK_SIZE_ON_UNDERLYING_FS is block size - on the underlying FS, on which the device file located, or 0, - if a device node is used. Both values are from the target. - See also important notes about setting block sizes >512 bytes - for VDISK FILEIO devices above. - - -9. In some cases, for instance working with SSD devices, which consume -100% of a single CPU load for data transfers in their internal threads, -to maximize IOPS it can be needed to assign for those threads dedicated -CPUs. Consider using cpu_mask attribute for devices with -threads_pool_type "per_initiator" or Linux CPU affinity facilities for -other threads_pool_types. No IRQ processing should be done on those -CPUs. Check that using /proc/interrupts. See taskset command and -Documentation/IRQ-affinity.txt in your kernel's source tree for how to -assign IRQ affinity to tasks and IRQs. - -The reason for that is that processing of coming commands in SIRQ -context might be done on the same CPUs as SSD devices' threads doing data -transfers. As the result, those threads won't receive all the processing -power of those CPUs and perform worse. - -10. If your storage is capable of operation on hundreds of thousands -IOPS level, you can use poll_us sysfs attribute to set how many us each -SCST thread is polling its queue after it became empty in a hope that a -new command can come. In some cases, polling can significantly increase -IOPS, especially if low power states on CPU not disabled, because on -high IOPS polling could be cheaper comparing to spending significant -time on entering, then exiting CPU low power states + corresponding -context switches. Polling is disabled by default. The recommended value -to start from is 5-10 us. Then you can increase or decrease it to see if -your IOPS are increasing or decreasing. - - -Commands suspending takes too long ----------------------------------- - -SCST is suspending commands during some management activities like -adding/deleting LUNs or devices. It is done to have lockless LUNs -translation on the hot commands processing path. This brings significant -performance advantage. You will see a message like "Waiting for X active -commands to complete" when this wait started. - -But downside of it is that no new commands start executing until older -ones, which had started before the suspending begun, finished. This -wait can not be any longer, than the worst command latency any your -initiator is seeing at this particular time. - -So, if this wait takes too long, in majority of cases it means that you -are overloading your storage. A proper storage should have worst case -latency below few hundreds of milliseconds. In this case the SCST -suspending will finish in few hundreds of milliseconds at worse. - -Another case, when it can take too long to suspend is a hung user space -device (i.e. scst_user device) not responding to any command. In this -case you should kill the corresponding user space program to finish -suspending. - - -Work if target's backstorage or link is too slow ------------------------------------------------- - -Under high I/O load, when your target's backstorage gets overloaded, or -working over a slow link between initiator and target, when the link -can't serve all the queued commands on time, you can experience I/O -stalls or see in the kernel log abort or reset messages. - -At first, consider the case of too slow target's backstorage. On some -seek intensive workloads even fast disks or RAIDs, which able to serve -continuous data stream on 500+ MB/s speed, can be as slow as 0.3 MB/s. -Another possible cause for that can be MD/LVM/RAID on your target as in -http://lkml.org/lkml/2008/2/27/96 (check the whole thread as well). - -Thus, in such situations simply processing of one or more commands takes -too long time, hence initiator decides that they are stuck on the target -and tries to recover. Particularly, it is known that the default amount -of simultaneously queued commands (48) is sometimes too high if you do -intensive writes from VMware on a target disk, which uses LVM in the -snapshot mode. In this case value like 16 or even 8-10 depending of your -backstorage speed could be more appropriate. - -There are 6 possible actions, which you can do to workaround or fix such -issues: - -1. Ignore incoming task management (TM) commands. It's fine if there are -not too many of them, so average performance isn't hurt and the -corresponding device isn't getting put offline, i.e. if the backstorage -isn't a way too slow. - -2. Decrease /sys/block/sdX/device/queue_depth on the initiator in case -if it's Linux (see below how) or/and SCST_MAX_TGT_DEV_COMMANDS constant -in scst_priv.h file until you stop seeing incoming TM commands. -ISCSI-SCST driver also has its own iSCSI specific parameter for that, -see its README file. - -To decrease device queue depth on Linux initiators you can run command: - -# echo Y >/sys/block/sdX/device/queue_depth - -where Y is the new number of simultaneously queued commands, X - your -imported device letter, like 'a' for sda device. There are no special -limitations for Y value, it can be any value from 1 to possible maximum -(usually, 32), so start from dividing the current value on 2, i.e. set -16, if /sys/block/sdX/device/queue_depth contains 32. - -3. Increase the corresponding timeout on the initiator. For Linux it is -located in -/sys/devices/platform/host*/session*/target*:0:0/*:0:0:1/timeout. It can -be done automatically by an udev rule. For instance, the following -rule will increase it to 300 seconds: - -SUBSYSTEM=="scsi", KERNEL=="[0-9]*:[0-9]*", ACTION=="add", ATTR{type}=="0|7|14", ATTR{timeout}="300" - -By default, this timeout is 30 or 60 seconds, depending on your distribution. - -4. Try to avoid such seek intensive workloads. - -5. Increase speed of the target's backstorage. - -6. Implement in SCST QoS, so queue depth size on the target is -dynamically adjusted, hence worst case initiator seen latencies are -controlled. - -Next, consider the case of too slow link between initiator and target, -when the initiator tries to simultaneously push N commands to the target -over it. In this case time to serve those commands, i.e. send or receive -data for them over the link, can be more, than timeout for any single -command, hence one or more commands in the tail of the queue can not be -served on time less than the timeout, so the initiator will decide that -they are stuck on the target and will try to recover. - -To workaround/fix this issue in this case you can use ways 1, 2, 3 above -or (7): increase speed of the link between target and initiator. - -Note, that logged messages about QUEUE_FULL status are quite different -by nature. This is a normal work, just SCSI flow control in action. -Simply don't enable "mgmt_minor" logging level, or, alternatively, if -you are confident in the worst case performance of your back-end storage -or initiator-target link, you can increase SCST_MAX_TGT_DEV_COMMANDS in -scst_priv.h to 64. Usually initiators don't try to push more commands on -the target. - - -Credits -------- - -Thanks to: - - * Mark Buechler for a lot of useful - suggestions, bug reports and help in debugging. - - * Ming Zhang for fixes and comments. - - * Nathaniel Clark for fixes and comments. - - * Calvin Morrow for testing and useful - suggestions. - - * Hu Gang for the original version of the - LSI target driver. - - * Erik Habbinga for fixes and support - of the LSI target driver. - - * Ross S. W. Walker for BLOCKIO inspiration - and Vu Pham who implemented it for VDISK dev handler. - - * Alessandro Premoli for fixes - - * Nathan Bullock for fixes. - - * Terry Greeniaus for fixes. - - * Krzysztof Blaszkowski for many fixes and bug reports. - - * Jianxi Chen for fixing problem with - devices >2TB in size - - * Bart Van Assche for a lot of help - - * Daniel Debonzi for a big part of the - initial SCST sysfs tree implementation - - -Vladislav Bolkhovitin , http://scst.sourceforge.net diff --git a/scst/include/backport.h b/scst/include/backport.h index 64ad38e13..63659596b 100644 --- a/scst/include/backport.h +++ b/scst/include/backport.h @@ -327,6 +327,25 @@ static inline int vfs_fsync_backport(struct file *file, int datasync) #define vfs_fsync vfs_fsync_backport #endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +/* + * See also commit bb7462b6fd64 ("vfs: use helpers for calling + * f_op->{read,write}_iter()"). + */ +static inline ssize_t call_read_iter(struct file *file, struct kiocb *kio, + struct iov_iter *iter) +{ + return file->f_op->read_iter(kio, iter); +} + +static inline ssize_t call_write_iter(struct file *file, struct kiocb *kio, + struct iov_iter *iter) +{ + return file->f_op->write_iter(kio, iter); +} +#endif + /* */ #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19) diff --git a/scst/src/Makefile b/scst/src/Makefile index 0a7b3f6c4..159aa769a 100644 --- a/scst/src/Makefile +++ b/scst/src/Makefile @@ -155,7 +155,7 @@ INSTALL_DIR_H := $(DESTDIR)$(PREFIX)/include/scst enable-Wextra = $(shell uname_r="$$(uname -r)"; if [ "$${uname_r%.el5}" = "$${uname_r}" ]; then echo "$(1)"; fi) EXTRA_CFLAGS += -I$(SCST_INC_DIR) $(call enable-Wextra,-Wextra \ - -Wno-unused-parameter -Wno-missing-field-initializers) \ + -Wno-unused-parameter -Wno-missing-field-initializers -Wno-sign-compare) \ $(shell [ -n "${CONFIG_SCST_NO_DLM}" ] && echo -DCONFIG_SCST_NO_DLM) #EXTRA_CFLAGS += -DCONFIG_SCST_STRICT_SERIALIZING diff --git a/scst/src/dev_handlers/Makefile b/scst/src/dev_handlers/Makefile index be8ca4332..8d998197f 100644 --- a/scst/src/dev_handlers/Makefile +++ b/scst/src/dev_handlers/Makefile @@ -75,7 +75,7 @@ endif enable-Wextra = $(shell uname_r="$$(uname -r)"; if [ "$${uname_r%.el5}" = "$${uname_r}" ]; then echo "$(1)"; fi) EXTRA_CFLAGS += -I$(SUBDIRS) -I$(SCST_INC_DIR) $(call enable-Wextra,-Wextra \ - -Wno-unused-parameter -Wno-missing-field-initializers) + -Wno-unused-parameter -Wno-missing-field-initializers -Wno-sign-compare) EXTRA_CFLAGS += -DCONFIG_SCST_EXTRACHECKS diff --git a/scst/src/dev_handlers/scst_user.c b/scst/src/dev_handlers/scst_user.c index 0b31824ea..d213c691a 100644 --- a/scst/src/dev_handlers/scst_user.c +++ b/scst/src/dev_handlers/scst_user.c @@ -1427,7 +1427,7 @@ out_process: out_inval: PRINT_ERROR("Invalid parse_reply parameters (LUN %lld, op %s, cmd %p)", - (unsigned long long int)cmd->lun, scst_get_opcode_name(cmd), cmd); + (unsigned long long)cmd->lun, scst_get_opcode_name(cmd), cmd); PRINT_BUFFER("Invalid parse_reply", reply, sizeof(*reply)); scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error)); res = -EINVAL; @@ -1875,7 +1875,7 @@ out: out_inval: PRINT_ERROR("Invalid exec_reply parameters (LUN %lld, op %s, cmd %p)", - (unsigned long long int)cmd->lun, scst_get_opcode_name(cmd), cmd); + (unsigned long long)cmd->lun, scst_get_opcode_name(cmd), cmd); PRINT_BUFFER("Invalid exec_reply", reply, sizeof(*reply)); out_intern_fail: @@ -2353,7 +2353,7 @@ static int dev_user_reply_get_cmd(struct file *file, void __user *arg) goto out; } - TRACE_DBG("ureply %lld (dev %s)", (unsigned long long int)ureply, + TRACE_DBG("ureply %lld (dev %s)", (unsigned long long)ureply, dev->name); if (ureply != 0) { diff --git a/scst/src/dev_handlers/scst_vdisk.c b/scst/src/dev_handlers/scst_vdisk.c index abf5f1bb2..8d24696a2 100644 --- a/scst/src/dev_handlers/scst_vdisk.c +++ b/scst/src/dev_handlers/scst_vdisk.c @@ -25,6 +25,7 @@ #ifndef INSIDE_KERNEL_TREE #include #endif +#include #include #include #include @@ -51,7 +52,6 @@ #include #include #include -#include #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38) #include #endif @@ -177,6 +177,7 @@ struct scst_vdisk_dev { unsigned int nv_cache:1; unsigned int o_direct_flag:1; unsigned int zero_copy:1; + unsigned int async:1; unsigned int media_changed:1; unsigned int prevent_allow_medium_removal:1; unsigned int nullio:1; @@ -265,14 +266,21 @@ struct scst_vdisk_dev { }; struct vdisk_cmd_params { - struct scatterlist small_sg[4]; - struct iovec *iv; - int iv_count; - struct iovec small_iv[4]; + union { + struct { + struct iovec *iv; + int iv_count; + struct iovec small_iv[4]; + } sync; + struct { + struct kiocb iocb; + struct kvec *kvec; + struct kvec small_kvec[4]; + } async; + }; struct scst_cmd *cmd; loff_t loff; int fua; - bool use_zero_copy; }; static bool vdev_saved_mode_pages_enabled = true; @@ -316,7 +324,6 @@ static int vdisk_get_supported_opcodes(struct scst_cmd *cmd, static int vcdrom_get_supported_opcodes(struct scst_cmd *cmd, const struct scst_opcode_descriptor ***out_supp_opcodes, int *out_supp_opcodes_cnt); -static int fileio_alloc_data_buf(struct scst_cmd *cmd); static int vdisk_parse(struct scst_cmd *); static int vcdrom_parse(struct scst_cmd *); static int non_fileio_parse(struct scst_cmd *); @@ -495,6 +502,8 @@ static ssize_t vdev_sysfs_inq_vend_specific_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf); static ssize_t vdev_zero_copy_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf); +static ssize_t vdev_async_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); static ssize_t vdev_dif_filename_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf); @@ -590,6 +599,8 @@ static struct kobj_attribute vdev_inq_vend_specific_attr = vdev_sysfs_inq_vend_specific_store); static struct kobj_attribute vdev_zero_copy_attr = __ATTR(zero_copy, S_IRUGO, vdev_zero_copy_show, NULL); +static struct kobj_attribute vdev_async_attr = + __ATTR(async, S_IRUGO, vdev_async_show, NULL); static struct kobj_attribute vdev_dif_filename_attr = __ATTR(dif_filename, S_IRUGO, vdev_dif_filename_show, NULL); @@ -625,6 +636,7 @@ static const struct attribute *vdisk_fileio_attrs[] = { &vdev_usn_attr.attr, &vdev_inq_vend_specific_attr.attr, &vdev_zero_copy_attr.attr, + &vdev_async_attr.attr, NULL, }; @@ -743,7 +755,6 @@ static struct scst_dev_type vdisk_file_devtype = { .attach_tgt = vdisk_attach_tgt, .detach_tgt = vdisk_detach_tgt, .parse = vdisk_parse, - .dev_alloc_data_buf = fileio_alloc_data_buf, .exec = fileio_exec, .on_free_cmd = fileio_on_free_cmd, .task_mgmt_fn_done = vdisk_task_mgmt_fn_done, @@ -760,12 +771,13 @@ static struct scst_dev_type vdisk_file_devtype = { .del_device = vdisk_del_device, .dev_attrs = vdisk_fileio_attrs, .add_device_parameters = + "async, " "blocksize, " + "cluster_mode, " "filename, " "numa_node_id, " "nv_cache, " "o_direct, " - "cluster_mode, " "read_only, " "removable, " "rotational, " @@ -905,7 +917,6 @@ static struct scst_dev_type vcdrom_devtype = { .attach_tgt = vdisk_attach_tgt, .detach_tgt = vdisk_detach_tgt, .parse = vcdrom_parse, - .dev_alloc_data_buf = fileio_alloc_data_buf, .exec = vcdrom_exec, .on_free_cmd = fileio_on_free_cmd, .task_mgmt_fn_done = vdisk_task_mgmt_fn_done, @@ -979,8 +990,6 @@ static struct file *vdev_open_fd(const struct scst_vdisk_dev *virt_dev, open_flags |= O_RDONLY; else open_flags |= O_RDWR; - if (virt_dev->o_direct_flag) - open_flags |= O_DIRECT; if (virt_dev->wt_flag && !virt_dev->nv_cache) open_flags |= O_DSYNC; @@ -1744,6 +1753,12 @@ next: res = -EINVAL; goto out; } + if (!virt_dev->async && virt_dev->o_direct_flag) { + PRINT_ERROR("%s: using o_direct without setting async is not" + " supported", virt_dev->filename); + res = -EINVAL; + goto out; + } dev->dev_rd_only = virt_dev->rd_only; @@ -1763,8 +1778,8 @@ next: (dev->type == TYPE_DISK) ? "disk" : "cdrom", virt_dev->name, vdev_get_filename(virt_dev), virt_dev->file_size >> 20, dev->block_size, - (unsigned long long int)virt_dev->nblocks, - (unsigned long long int)virt_dev->nblocks/64/32, + (unsigned long long)virt_dev->nblocks, + (unsigned long long)virt_dev->nblocks/64/32, virt_dev->nblocks < 64*32 ? " !WARNING! cyln less than 1" : ""); } else { @@ -1958,8 +1973,8 @@ static enum compl_status_e vdisk_synchronize_cache(struct vdisk_cmd_params *p) TRACE(TRACE_ORDER, "SYNCHRONIZE_CACHE: " "loff=%lld, data_len=%lld, immed=%d", - (unsigned long long int)loff, - (unsigned long long int)data_len, immed); + (unsigned long long)loff, + (unsigned long long)data_len, immed); if (data_len == 0) { struct scst_vdisk_dev *virt_dev = dev->dh_priv; @@ -2941,31 +2956,9 @@ static int vcdrom_get_supported_opcodes(struct scst_cmd *cmd, return 0; } -static bool vdisk_use_zero_copy(const struct scst_cmd *cmd) -{ - struct scst_vdisk_dev *virt_dev = cmd->dev->dh_priv; - - if (!virt_dev->zero_copy) - return false; - - switch (cmd->cdb[0]) { - case VARIABLE_LENGTH_CMD: - if (cmd->cdb[9] != SUBCODE_READ_32) - break; - /* fall through */ - case READ_6: - case READ_10: - case READ_12: - case READ_16: - return true; - } - - return false; -} - /* * Compute p->loff and p->fua. - * Returns true for success or false otherwise and set error in the commeand. + * Returns true for success or false otherwise and set error in the command. */ static bool vdisk_parse_offset(struct vdisk_cmd_params *p, struct scst_cmd *cmd) { @@ -3016,9 +3009,9 @@ static bool vdisk_parse_offset(struct vdisk_cmd_params *p, struct scst_cmd *cmd) loff = (loff_t)lba_start << dev->block_shift; TRACE_DBG("cmd %p, lba_start %lld, loff %lld, data_len %lld", cmd, - (unsigned long long int)lba_start, - (unsigned long long int)loff, - (unsigned long long int)data_len); + (unsigned long long)lba_start, + (unsigned long long)loff, + (unsigned long long)data_len); EXTRACHECKS_BUG_ON((loff < 0) || unlikely(data_len < 0)); @@ -3030,9 +3023,9 @@ static bool vdisk_parse_offset(struct vdisk_cmd_params *p, struct scst_cmd *cmd) } else { PRINT_INFO("Access beyond the end of device %s " "(%lld of %lld, data len %lld)", virt_dev->name, - (unsigned long long int)loff, - (unsigned long long int)virt_dev->file_size, - (unsigned long long int)data_len); + (unsigned long long)loff, + (unsigned long long)virt_dev->file_size, + (unsigned long long)data_len); scst_set_cmd_error(cmd, SCST_LOAD_SENSE( scst_sense_block_out_range_error)); } @@ -3055,15 +3048,14 @@ static bool vdisk_parse_offset(struct vdisk_cmd_params *p, struct scst_cmd *cmd) fua = (cdb[1] & 0x8); if (fua) { TRACE(TRACE_ORDER, "FUA: loff=%lld, " - "data_len=%lld", (unsigned long long int)loff, - (unsigned long long int)data_len); + "data_len=%lld", (unsigned long long)loff, + (unsigned long long)data_len); } break; } p->loff = loff; p->fua = fua; - p->use_zero_copy = vdisk_use_zero_copy(cmd); res = true; @@ -3146,397 +3138,6 @@ out: return res; } -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 30) -/* - * finish_read - Release the pages referenced by prepare_read(). - */ -static void finish_read(struct scatterlist *sg, int sg_cnt) -{ - struct page *page; - int i; - - TRACE_ENTRY(); - - for (i = 0; i < sg_cnt; ++i) { - page = sg_page(&sg[i]); - EXTRACHECKS_BUG_ON(!page); - put_page(page); - } - - TRACE_EXIT(); - return; -} - -/** - * prepare_read_page - Bring a single page into the page cache. - * - * @filp: file pointer - * @len: number of bytes to read from the file - * @offset: offset of first byte to read (from start of file) - * @last: offset of first byte that will not be read - used for readahead - * @pageptr: page pointer output variable. - * - * Returns a negative number if an error occurred, zero upon EOF or a positive - * number - the number of bytes that can be read from the file via the returned - * page. If a positive number is returned, it is the responsibility of the - * caller to release the returned page. - * - * Based on do_generic_file_read(). - */ -static int prepare_read_page(struct file *filp, int len, - loff_t offset, loff_t last, struct page **pageptr) -{ - struct address_space *mapping = filp->f_mapping; - struct inode *inode = mapping->host; - struct file_ra_state *ra = &filp->f_ra; - struct page *page; - unsigned long index, last_index; - long end_index, nr; - loff_t isize; -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) - read_descriptor_t desc = { .count = len }; -#endif - int error; - - TRACE_ENTRY(); - - WARN((offset & ~PAGE_MASK) + len > PAGE_SIZE, - "offset = %lld + %lld, len = %d\n", offset & PAGE_MASK, - offset & ~PAGE_MASK, len); - sBUG_ON(!mapping->a_ops); - - index = offset >> PAGE_SHIFT; - last_index = (last + PAGE_SIZE - 1) >> PAGE_SHIFT; - -find_page: - page = find_get_page(mapping, index); - if (!page) { - page_cache_sync_readahead(mapping, ra, filp, index, - last_index - index); - page = find_get_page(mapping, index); - if (unlikely(!page)) { - /* - * Not cached so create a new page. - */ - page = page_cache_alloc(mapping); - if (!page) { - error = -ENOMEM; - goto err; - } - error = add_to_page_cache_lru(page, mapping, index, - GFP_KERNEL); - if (error) { - put_page(page); - if (error == -EEXIST) - goto find_page; - else - goto err; - } else { - goto readpage; - } - } - } - if (PageReadahead(page)) - page_cache_async_readahead(mapping, ra, filp, page, - index, last_index - index); - if (!PageUptodate(page)) { - if (inode->i_blkbits == PAGE_SHIFT || - !mapping->a_ops->is_partially_uptodate) - goto page_not_up_to_date; - if (!trylock_page(page)) - goto page_not_up_to_date; - /* Did it get truncated before we got the lock? */ - if (!page->mapping) - goto page_not_up_to_date_locked; -#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0) - if (!mapping->a_ops->is_partially_uptodate(page, - offset & ~PAGE_MASK, len)) -#else - if (!mapping->a_ops->is_partially_uptodate(page, &desc, - offset & ~PAGE_MASK)) -#endif - goto page_not_up_to_date_locked; - unlock_page(page); - } -page_ok: - /* - * i_size must be checked after we know the page is Uptodate. - * - * Checking i_size after the check allows us to calculate the correct - * value for "nr", which means the zero-filled part of the page is not - * accessed (unless another truncate extends the file - this is - * desired though). - */ - - isize = i_size_read(inode); - end_index = (isize - 1) >> PAGE_SHIFT; - if (unlikely(isize == 0 || index > end_index)) { - put_page(page); - goto eof; - } - - /* nr is the maximum number of bytes to copy from this page */ - if (index < end_index) { - nr = PAGE_SIZE - (offset & ~PAGE_MASK); - } else { - nr = ((isize - 1) & ~PAGE_MASK) + 1 - (offset & ~PAGE_MASK); - if (nr <= 0) { - put_page(page); - goto eof; - } - } - - /* - * If users can be writing to this page using arbitrary virtual - * addresses, take care about potential aliasing before reading the - * page on the kernel side. - */ - if (mapping_writably_mapped(mapping)) - flush_dcache_page(page); - - mark_page_accessed(page); - - /* Ok, we have the page and it's up to date. */ - *pageptr = page; - TRACE_EXIT_RES(nr); - return nr; -eof: - TRACE_EXIT(); - return 0; -err: - TRACE_EXIT_RES(error); - return error; - -page_not_up_to_date: - /* Try to get exclusive access to the page. */ - error = lock_page_killable(page); - if (unlikely(error != 0)) { - put_page(page); - goto err; - } - -page_not_up_to_date_locked: - /* Did it get truncated before we got the lock? */ - if (!page->mapping) { - unlock_page(page); - put_page(page); - goto find_page; - } - - /* Did somebody else fill it already? */ - if (PageUptodate(page)) { - unlock_page(page); - goto page_ok; - } - -readpage: - /* - * A previous I/O error may have been due to temporary - * failures, eg. multipath errors. - * PG_error will be set again if readpage fails. - */ - ClearPageError(page); - /* Start the actual read. The read will unlock the page. */ - error = mapping->a_ops->readpage(filp, page); - if (unlikely(error)) { - if (error == AOP_TRUNCATED_PAGE) { - put_page(page); - goto find_page; - } - WARN(error >= 0, "error = %d\n", error); - put_page(page); - goto err; - } - - if (!PageUptodate(page)) { - error = lock_page_killable(page); - if (unlikely(error != 0)) { - put_page(page); - goto err; - } - if (!PageUptodate(page)) { - if (page->mapping == NULL) { - /* - * invalidate_mapping_pages got it - */ - unlock_page(page); - put_page(page); - goto find_page; - } - unlock_page(page); - put_page(page); - error = -EIO; - goto err; - } - unlock_page(page); - } - - goto page_ok; -} - -/** - * prepare_read - Lock page cache pages corresponding to an sg vector - * @filp: file the sg vector applies to - * @sg: sg vector - * @sg_cnt: sg vector size - * @offset: file offset the first byte of the first sg element corresponds to - */ -static int prepare_read(struct file *filp, struct scatterlist *sg, int sg_cnt, - pgoff_t offset) -{ - struct page *page = NULL; - int i, res; - loff_t off, last = ((offset + sg_cnt - 1) << PAGE_SHIFT) + - sg[sg_cnt - 1].offset + sg[sg_cnt - 1].length; - - TRACE_ENTRY(); - - for (i = 0; i < sg_cnt; ++i) { - off = (offset + i) << PAGE_SHIFT | sg[i].offset; - res = prepare_read_page(filp, sg[i].length, off, last, &page); - if (res <= 0) - goto err; - if (res < sg[i].length) { - put_page(page); - goto err; - } - sg_assign_page(&sg[i], page); - } - - file_accessed(filp); - -out: - TRACE_EXIT_RES(i); - return i; - -err: - finish_read(sg, i); - i = -EIO; - goto out; -} - -/** - * alloc_sg - Allocate an SG vector. - * @size: number of bytes that will be stored in the pages of the sg vector - * @off: first page data offset - * @gfp_mask: allocation flags for dynamic sg vector allocation - * @small_sg: pointer to a candidate sg vector - * @small_sg_size: size of @small_sg - * @p_sg_cnt: pointer to an int where the sg vector size will be written - */ -static struct scatterlist *alloc_sg(size_t size, unsigned int off, - gfp_t gfp_mask, - struct scatterlist *small_sg, - int small_sg_size, int *p_sg_cnt) -{ - struct scatterlist *sg; - int i, sg_cnt, remaining_sz, sg_sz, sg_off; - - TRACE_ENTRY(); - - sg_cnt = PAGE_ALIGN(size + off) >> PAGE_SHIFT; - sg = sg_cnt <= small_sg_size ? small_sg : - kmalloc_array(sg_cnt, sizeof(*sg), gfp_mask); - if (!sg) - goto out; - - sg_init_table(sg, sg_cnt); - remaining_sz = size; - sg_off = off; - for (i = 0; i < sg_cnt; ++i) { - sg_sz = min_t(int, PAGE_SIZE - sg_off, remaining_sz); - sg_set_page(&sg[i], NULL, sg_sz, sg_off); - remaining_sz -= sg_sz; - sg_off = 0; - } - *p_sg_cnt = sg_cnt; - -out: - TRACE_EXIT(); - return sg; -} - -static int fileio_alloc_data_buf(struct scst_cmd *cmd) -{ - struct vdisk_cmd_params *p; - struct scst_vdisk_dev *virt_dev; - int sg_cnt, nr; - const gfp_t gfp_mask = GFP_KERNEL; - struct scatterlist *sg; - - TRACE_ENTRY(); - - p = cmd->dh_priv; - EXTRACHECKS_BUG_ON(!p); - virt_dev = cmd->dev->dh_priv; - /* - * If the target driver (e.g. scst_local) allocates the sg vector - * itself or the command is a write or bidi command, don't use zero - * copy. - */ - if (cmd->tgt_i_data_buf_alloced || - (cmd->data_direction & SCST_DATA_READ) == 0 || - (virt_dev->fd && !virt_dev->fd->f_mapping->a_ops->readpage)) { - p->use_zero_copy = false; - } - if (!p->use_zero_copy) - goto out; - - EXTRACHECKS_BUG_ON(!(cmd->data_direction & SCST_DATA_READ)); - - scst_cmd_set_dh_data_buff_alloced(cmd); - - cmd->sg = alloc_sg(cmd->bufflen, p->loff & ~PAGE_MASK, gfp_mask, - p->small_sg, ARRAY_SIZE(p->small_sg), &cmd->sg_cnt); - if (!cmd->sg) { - PRINT_ERROR("sg allocation failed (bufflen = %d, off = %lld)\n", - cmd->bufflen, p->loff & ~PAGE_MASK); - goto enomem; - } - sg_cnt = scst_cmd_get_sg_cnt(cmd); - sg = cmd->sg; - nr = prepare_read(virt_dev->fd, sg, sg_cnt, p->loff >> PAGE_SHIFT); - if (nr < 0) { - PRINT_ERROR("prepare_read() failed: %d", nr); - goto out_free_sg; - } -out: - TRACE_EXIT(); - return SCST_CMD_STATE_DEFAULT; - -out_free_sg: - kfree(cmd->sg); - cmd->sg = NULL; - cmd->sg_cnt = 0; - -enomem: - scst_set_busy(cmd); - TRACE_EXIT_RES(-ENOMEM); - return scst_get_cmd_abnormal_done_state(cmd); -} - -#else - -static int fileio_alloc_data_buf(struct scst_cmd *cmd) -{ - struct vdisk_cmd_params *p; - - TRACE_ENTRY(); - - p = cmd->dh_priv; - EXTRACHECKS_BUG_ON(!p); - p->use_zero_copy = false; - - TRACE_EXIT(); - return SCST_CMD_STATE_DEFAULT; -} - -static void finish_read(struct scatterlist *sg, int sg_cnt) -{ -} - -#endif - static int vdev_do_job(struct scst_cmd *cmd, const vdisk_op_fn *ops) { int res; @@ -3610,36 +3211,150 @@ static int fileio_exec(struct scst_cmd *cmd) return vdev_do_job(cmd, ops); } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0) +static bool do_fileio_async(const struct vdisk_cmd_params *p) +{ + struct scst_cmd *cmd = p->cmd; + struct scst_device *dev = cmd->dev; + struct scst_vdisk_dev *virt_dev = dev->dh_priv; + + return virt_dev->async && dev->dev_dif_mode == SCST_DIF_MODE_NONE; +} + +static bool vdisk_alloc_kvec(struct scst_cmd *cmd, struct vdisk_cmd_params *p) +{ + int n; + + n = scst_get_buf_count(cmd); + if (n <= ARRAY_SIZE(p->async.small_kvec)) { + p->async.kvec = &p->async.small_kvec[0]; + return true; + } + + p->async.kvec = kmalloc_array(n, sizeof(*p->async.kvec), + cmd->cmd_gfp_mask); + if (p->async.kvec == NULL) { + PRINT_ERROR("Unable to allocate kvecv (%d)", n); + return false; + } + + return true; +} + +static void fileio_async_complete(struct kiocb *iocb, long ret, long ret2) +{ + struct vdisk_cmd_params *p = container_of(iocb, typeof(*p), async.iocb); + struct scst_cmd *cmd = p->cmd; + + if (ret < 0 && + scst_cmd_get_data_direction(cmd) & SCST_DATA_WRITE) + scst_set_cmd_error(cmd, + SCST_LOAD_SENSE(scst_sense_write_error)); + else if (ret < 0) + scst_set_cmd_error(cmd, + SCST_LOAD_SENSE(scst_sense_hardw_error)); + else + scst_set_resp_data_len(cmd, ret); + cmd->completed = 1; + cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME); +} + +static enum compl_status_e fileio_exec_async(struct vdisk_cmd_params *p) +{ + struct scst_cmd *cmd = p->cmd; + struct scst_device *dev = cmd->dev; + struct scst_vdisk_dev *virt_dev = dev->dh_priv; + struct file *fd = virt_dev->fd; + struct iov_iter iter = { }; + ssize_t length, total = 0; + struct kvec *kvec; + uint8_t *address; + int dir, ret; + + switch (cmd->data_direction) { + case SCST_DATA_READ: + dir = READ; + break; + case SCST_DATA_WRITE: + dir = WRITE; + break; + default: + WARN_ON_ONCE(true); + return CMD_FAILED; + } + + if (!vdisk_alloc_kvec(cmd, p)) { + scst_set_busy(cmd); + return CMD_SUCCEEDED; + } + + kvec = p->async.kvec; + length = scst_get_buf_first(cmd, &address); + while (length) { + *kvec++ = (struct kvec){ + .iov_base = address, + .iov_len = length, + }; + total += length; + length = scst_get_buf_next(cmd, &address); + } + + iov_iter_kvec(&iter, ITER_KVEC | dir, p->async.kvec, + kvec - p->async.kvec, total); + p->async.iocb = (struct kiocb) { + .ki_pos = p->loff, + .ki_filp = fd, + .ki_complete = fileio_async_complete, + }; + if (virt_dev->o_direct_flag) + p->async.iocb.ki_flags |= IOCB_DIRECT; + if (dir == WRITE) { + if (virt_dev->wt_flag && !virt_dev->nv_cache) + p->async.iocb.ki_flags |= IOCB_DSYNC; + ret = call_write_iter(fd, &p->async.iocb, &iter); + } else { + ret = call_read_iter(fd, &p->async.iocb, &iter); + } + if (p->async.kvec != p->async.small_kvec) + kfree(p->async.kvec); + if (ret != -EIOCBQUEUED) + fileio_async_complete(&p->async.iocb, ret, 0); + /* + * Return RUNNING_ASYNC even if fileio_async_complete() has been + * called because that function calls cmd->scst_cmd_done(). + */ + return RUNNING_ASYNC; +} +#else +static bool do_fileio_async(const struct vdisk_cmd_params *p) +{ + return false; +} + +static enum compl_status_e fileio_exec_async(struct vdisk_cmd_params *p) +{ + WARN_ON_ONCE(true); + return CMD_FAILED; +} +#endif + static void vdisk_on_free_cmd_params(const struct vdisk_cmd_params *p) { - if (p->iv != p->small_iv) - kfree(p->iv); + if (!do_fileio_async(p)) { + if (p->sync.iv != p->sync.small_iv) + kfree(p->sync.iv); + } } static void fileio_on_free_cmd(struct scst_cmd *cmd) { struct vdisk_cmd_params *p = cmd->dh_priv; - struct scst_vdisk_dev *virt_dev; TRACE_ENTRY(); if (!p) goto out; - virt_dev = cmd->dev->dh_priv; - - if (p->use_zero_copy) { - if ((cmd->data_direction & SCST_DATA_READ) && - virt_dev->zero_copy) - finish_read(cmd->sg, cmd->sg_cnt); - if (cmd->sg != p->small_sg) - kfree(cmd->sg); - cmd->sg_cnt = 0; - cmd->sg = NULL; - cmd->bufflen = 0; - cmd->data_len = 0; - } - vdisk_on_free_cmd_params(p); kmem_cache_free(vdisk_cmd_param_cachep, p); @@ -5844,23 +5559,24 @@ static struct iovec *vdisk_alloc_iv(struct scst_cmd *cmd, int iv_count; iv_count = min_t(int, scst_get_buf_count(cmd), UIO_MAXIOV); - if (iv_count > p->iv_count) { - if (p->iv != p->small_iv) - kfree(p->iv); - p->iv_count = 0; + if (iv_count > p->sync.iv_count) { + if (p->sync.iv != p->sync.small_iv) + kfree(p->sync.iv); + p->sync.iv_count = 0; /* It can't be called in atomic context */ - p->iv = (iv_count <= ARRAY_SIZE(p->small_iv)) ? p->small_iv : - kmalloc_array(iv_count, sizeof(*p->iv), + p->sync.iv = iv_count <= ARRAY_SIZE(p->sync.small_iv) ? + p->sync.small_iv : + kmalloc_array(iv_count, sizeof(*p->sync.iv), cmd->cmd_gfp_mask); - if (p->iv == NULL) { + if (p->sync.iv == NULL) { PRINT_ERROR("Unable to allocate iv (%d)", iv_count); goto out; } - p->iv_count = iv_count; + p->sync.iv_count = iv_count; } out: - return p->iv; + return p->sync.iv; } static enum compl_status_e nullio_exec_read(struct vdisk_cmd_params *p) @@ -5919,11 +5635,6 @@ static int vdev_read_dif_tags(struct vdisk_cmd_params *p) EXTRACHECKS_BUG_ON(virt_dev->nullio); -#if 0 /* no zero-copy (yet) */ - if (p->use_zero_copy) - goto out; -#endif - EXTRACHECKS_BUG_ON(!(cmd->dev->dev_dif_mode & SCST_DIF_MODE_DEV_STORE) || (scst_get_dif_action(scst_get_dev_dif_actions(cmd->cmd_dif_actions)) == SCST_DIF_ACTION_NONE)); @@ -5931,7 +5642,7 @@ static int vdev_read_dif_tags(struct vdisk_cmd_params *p) if (unlikely(tags_num == 0)) goto out; - iv = p->iv; + iv = p->sync.iv; if (iv == NULL) { iv = vdisk_alloc_iv(cmd, p); if (iv == NULL) { @@ -5945,7 +5656,7 @@ static int vdev_read_dif_tags(struct vdisk_cmd_params *p) goto out; } } - max_iv_count = p->iv_count; + max_iv_count = p->sync.iv_count; old_fs = get_fs(); set_fs(get_ds()); @@ -6045,11 +5756,6 @@ static int vdev_write_dif_tags(struct vdisk_cmd_params *p) EXTRACHECKS_BUG_ON(virt_dev->nullio); -#if 0 /* no zero-copy (yet) */ - if (p->use_zero_copy) - goto out; -#endif - EXTRACHECKS_BUG_ON(!(cmd->dev->dev_dif_mode & SCST_DIF_MODE_DEV_STORE) || (scst_get_dif_action(scst_get_dev_dif_actions(cmd->cmd_dif_actions)) == SCST_DIF_ACTION_NONE)); @@ -6057,7 +5763,7 @@ static int vdev_write_dif_tags(struct vdisk_cmd_params *p) if (unlikely(tags_num == 0)) goto out; - iv = p->iv; + iv = p->sync.iv; if (iv == NULL) { iv = vdisk_alloc_iv(cmd, p); if (iv == NULL) { @@ -6071,7 +5777,7 @@ static int vdev_write_dif_tags(struct vdisk_cmd_params *p) goto out; } } - max_iv_count = p->iv_count; + max_iv_count = p->sync.iv_count; old_fs = get_fs(); set_fs(get_ds()); @@ -6200,14 +5906,14 @@ static enum compl_status_e fileio_exec_read(struct vdisk_cmd_params *p) EXTRACHECKS_BUG_ON(virt_dev->nullio); - if (p->use_zero_copy) - goto out_dif; + if (do_fileio_async(p)) + return fileio_exec_async(p); iv = vdisk_alloc_iv(cmd, p); if (iv == NULL) goto out_nomem; - max_iv_count = p->iv_count; + max_iv_count = p->sync.iv_count; length = scst_get_buf_first(cmd, (uint8_t __force **)&address); if (unlikely(length < 0)) { @@ -6252,7 +5958,7 @@ static enum compl_status_e fileio_exec_read(struct vdisk_cmd_params *p) err = scst_readv(fd, iv, iv_count, &loff); if ((err < 0) || (err < full_len)) { PRINT_ERROR("readv() returned %lld from %zd", - (unsigned long long int)err, + (unsigned long long)err, full_len); if (err == -EAGAIN) scst_set_busy(cmd); @@ -6281,7 +5987,6 @@ static enum compl_status_e fileio_exec_read(struct vdisk_cmd_params *p) goto out; } -out_dif: scst_dif_process_read(cmd); out: @@ -6387,18 +6092,18 @@ static enum compl_status_e fileio_exec_write(struct vdisk_cmd_params *p) EXTRACHECKS_BUG_ON(virt_dev->nullio); + if (do_fileio_async(p)) + return fileio_exec_async(p); + rc = scst_dif_process_write(cmd); if (unlikely(rc != 0)) goto out; - if (p->use_zero_copy) - goto out_sync; - iv = vdisk_alloc_iv(cmd, p); if (iv == NULL) goto out_nomem; - max_iv_count = p->iv_count; + max_iv_count = p->sync.iv_count; length = scst_get_buf_first(cmd, &address); if (unlikely(length < 0)) { @@ -6445,7 +6150,7 @@ restart: err = scst_writev(fd, eiv, eiv_count, &loff); if (err < 0) { PRINT_ERROR("write() returned %lld from %zd", - (unsigned long long int)err, + (unsigned long long)err, full_len); if (err == -EAGAIN) scst_set_busy(cmd); @@ -7419,7 +7124,7 @@ static enum compl_status_e vdev_exec_verify(struct vdisk_cmd_params *p) err = vdev_read_sync(virt_dev, mem_verify, len_mem, &loff); if ((err < 0) || (err < len_mem)) { PRINT_ERROR("verify() returned %lld from %zd", - (unsigned long long int)err, len_mem); + (unsigned long long)err, len_mem); if (err == -EAGAIN) scst_set_busy(cmd); else { @@ -7766,6 +7471,10 @@ static void vdisk_report_registering(const struct scst_vdisk_dev *virt_dev) i += snprintf(&buf[i], buf_size - i, "%sZERO_COPY", (j == i) ? "(" : ", "); + if (virt_dev->async) + i += snprintf(&buf[i], buf_size - i, "%sASYNC", + (j == i) ? "(" : ", "); + if (virt_dev->dummy) i += snprintf(&buf[i], buf_size - i, "%sDUMMY", (j == i) ? "(" : ", "); @@ -7813,8 +7522,8 @@ static int vdisk_resync_size(struct scst_vdisk_dev *virt_dev) "(fs=%lldMB, bs=%d, nblocks=%lld, cyln=%lld%s)", virt_dev->name, virt_dev->file_size >> 20, virt_dev->dev->block_size, - (unsigned long long int)virt_dev->nblocks, - (unsigned long long int)virt_dev->nblocks/64/32, + (unsigned long long)virt_dev->nblocks, + (unsigned long long)virt_dev->nblocks/64/32, virt_dev->nblocks < 64*32 ? " !WARNING! cyln less " "than 1" : ""); @@ -8217,14 +7926,8 @@ static int vdev_parse_add_dev_params(struct scst_vdisk_dev *virt_dev, virt_dev->nv_cache = ull_val; TRACE_DBG("NON-VOLATILE CACHE %d", virt_dev->nv_cache); } else if (!strcasecmp("o_direct", p)) { -#if 0 virt_dev->o_direct_flag = ull_val; TRACE_DBG("O_DIRECT %d", virt_dev->o_direct_flag); -#else - PRINT_INFO("O_DIRECT flag doesn't currently" - " work, ignoring it, use fileio_tgt " - "in O_DIRECT mode instead (device %s)", virt_dev->name); -#endif } else if (!strcasecmp("read_only", p)) { virt_dev->rd_only = ull_val; TRACE_DBG("READ ONLY %d", virt_dev->rd_only); @@ -8264,6 +7967,8 @@ static int vdev_parse_add_dev_params(struct scst_vdisk_dev *virt_dev, virt_dev->thin_provisioned); } else if (!strcasecmp("zero_copy", p)) { virt_dev->zero_copy = !!ull_val; + } else if (!strcasecmp("async", p)) { + virt_dev->async = !!ull_val; } else if (!strcasecmp("size", p)) { virt_dev->file_size = ull_val; } else if (!strcasecmp("size_mb", p)) { @@ -8833,8 +8538,8 @@ static int vcdrom_change(struct scst_vdisk_dev *virt_dev, " cyln=%lld%s)", virt_dev->name, vdev_get_filename(virt_dev), virt_dev->file_size >> 20, virt_dev->dev->block_size, - (unsigned long long int)virt_dev->nblocks, - (unsigned long long int)virt_dev->nblocks/64/32, + (unsigned long long)virt_dev->nblocks, + (unsigned long long)virt_dev->nblocks/64/32, virt_dev->nblocks < 64*32 ? " !WARNING! cyln less " "than 1" : ""); } else { @@ -10603,6 +10308,17 @@ static ssize_t vdev_zero_copy_show(struct kobject *kobj, return pos; } +static ssize_t vdev_async_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct scst_device *dev = + container_of(kobj, struct scst_device, dev_kobj); + struct scst_vdisk_dev *virt_dev = dev->dh_priv; + + return sprintf(buf, "%d\n%s", virt_dev->async, + virt_dev->async ? SCST_SYSFS_KEY_MARK "\n" : ""); +} + static ssize_t vdev_dif_filename_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { diff --git a/scst/src/scst_event.c b/scst/src/scst_event.c index 16d1efd5f..310ec1073 100644 --- a/scst/src/scst_event.c +++ b/scst/src/scst_event.c @@ -321,7 +321,7 @@ int scst_event_queue_lun_not_found(const struct scst_cmd *cmd) if (event_entry == NULL) { PRINT_ERROR("Unable to allocate event (size %d). LUN not found " "event is lost (LUN %lld, initiator %s, target %s)!", - event_entry_len, (unsigned long long int)cmd->lun, + event_entry_len, (unsigned long long)cmd->lun, cmd->sess->initiator_name, cmd->tgt->tgt_name); res = -ENOMEM; goto out; diff --git a/scst/src/scst_lib.c b/scst/src/scst_lib.c index bef37bdb5..e6f014463 100644 --- a/scst/src/scst_lib.c +++ b/scst/src/scst_lib.c @@ -3317,7 +3317,7 @@ static bool __scst_adjust_sg(struct scst_cmd *cmd, struct scatterlist *sg, TRACE_DBG_FLAG(TRACE_SG_OP|TRACE_MEMORY|TRACE_DEBUG, "cmd %p (tag %llu), sg %p, sg_cnt %d, " "adjust_len %d, i %d, sg[j].length %d, left %d", - cmd, (unsigned long long int)cmd->tag, + cmd, (unsigned long long)cmd->tag, sg, *sg_cnt, adjust_len, i, sgi->length, left); @@ -5389,7 +5389,7 @@ static int scst_alloc_add_tgt_dev(struct scst_session *sess, scst_sgv_pool_use_dma(tgt_dev); TRACE_MGMT_DBG("Device %s on SCST lun=%lld", - dev->virt_name, (unsigned long long int)tgt_dev->lun); + dev->virt_name, (unsigned long long)tgt_dev->lun); spin_lock_init(&tgt_dev->tgt_dev_lock); INIT_LIST_HEAD(&tgt_dev->UA_list); @@ -7462,7 +7462,7 @@ void scst_free_cmd(struct scst_cmd *cmd) TRACE_ENTRY(); TRACE_DBG("Freeing cmd %p (tag %llu)", - cmd, (unsigned long long int)cmd->tag); + cmd, (unsigned long long)cmd->tag); if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) TRACE_MGMT_DBG("Freeing aborted cmd %p", cmd); @@ -7505,7 +7505,7 @@ void scst_free_cmd(struct scst_cmd *cmd) &cmd->cmd_flags); TRACE_SN("Out of SN cmd %p (tag %llu, sn %d), " "destroy=%d", cmd, - (unsigned long long int)cmd->tag, + (unsigned long long)cmd->tag, cmd->sn, destroy); } } @@ -13193,7 +13193,7 @@ static void scst_free_all_UA(struct scst_tgt_dev *tgt_dev) list_for_each_entry_safe(UA_entry, t, &tgt_dev->UA_list, UA_list_entry) { TRACE_MGMT_DBG("Clearing UA for tgt_dev LUN %lld", - (unsigned long long int)tgt_dev->lun); + (unsigned long long)tgt_dev->lun); list_del(&UA_entry->UA_list_entry); mempool_free(UA_entry, scst_ua_mempool); } @@ -13294,7 +13294,7 @@ restart: * !! destroyed! !! */ TRACE_SN("cmd %p (tag %llu) with skipped sn %d found", - cmd, (unsigned long long int)cmd->tag, cmd->sn); + cmd, (unsigned long long)cmd->tag, cmd->sn); order_data->def_cmd_count--; list_del(&cmd->deferred_cmd_list_entry); spin_unlock_irq(&order_data->sn_lock); @@ -13394,12 +13394,12 @@ bool __scst_check_blocked_dev(struct scst_cmd *cmd) if (dev->block_count > 0) { TRACE_BLOCK("Delaying cmd %p due to blocking " "(tag %llu, op %s, dev %s)", cmd, - (unsigned long long int)cmd->tag, + (unsigned long long)cmd->tag, scst_get_opcode_name(cmd), dev->virt_name); goto out_block; } else if ((cmd->op_flags & SCST_STRICTLY_SERIALIZED) == SCST_STRICTLY_SERIALIZED) { TRACE_BLOCK("Strictly serialized cmd %p (tag %llu, op %s, dev %s)", - cmd, (unsigned long long int)cmd->tag, + cmd, (unsigned long long)cmd->tag, scst_get_opcode_name(cmd), dev->virt_name); if ((cmd->cdb[0] == MAINTENANCE_OUT) && @@ -13432,7 +13432,7 @@ bool __scst_check_blocked_dev(struct scst_cmd *cmd) } else if ((dev->dev_double_ua_possible) || ((cmd->op_flags & SCST_SERIALIZED) != 0)) { TRACE_BLOCK("cmd %p (tag %llu, op %s): blocking further cmds " - "on dev %s due to %s", cmd, (unsigned long long int)cmd->tag, + "on dev %s due to %s", cmd, (unsigned long long)cmd->tag, scst_get_opcode_name(cmd), dev->virt_name, dev->dev_double_ua_possible ? "possible double reset UA" : "serialized cmd"); @@ -13963,13 +13963,13 @@ void scst_xmit_process_aborted_cmd(struct scst_cmd *cmd) if (test_bit(SCST_CMD_DEVICE_TAS, &cmd->cmd_flags)) { TRACE_MGMT_DBG("Flag ABORTED OTHER set for cmd %p " "(tag %llu), returning TASK ABORTED ", cmd, - (unsigned long long int)cmd->tag); + (unsigned long long)cmd->tag); scst_set_cmd_error_status(cmd, SAM_STAT_TASK_ABORTED); } else { TRACE_MGMT_DBG("Flag ABORTED OTHER set for cmd %p " "(tag %llu), aborting without delivery or " "notification", - cmd, (unsigned long long int)cmd->tag); + cmd, (unsigned long long)cmd->tag); /* * There is no need to check/requeue possible UA, * because, if it exists, it will be delivered diff --git a/scst/src/scst_proc.c b/scst/src/scst_proc.c index 48b311d41..0da66b59a 100644 --- a/scst/src/scst_proc.c +++ b/scst/src/scst_proc.c @@ -2199,7 +2199,7 @@ static int scst_groups_devices_show(struct seq_file *seq, void *v) list_for_each_entry(acg_dev, &acg->acg_dev_list, acg_dev_list_entry) { seq_printf(seq, "%-60s%-13lld%s\n", acg_dev->dev->virt_name, - (unsigned long long int)acg_dev->lun, + (unsigned long long)acg_dev->lun, acg_dev->acg_dev_rd_only ? "RO" : ""); } mutex_unlock(&scst_mutex); diff --git a/scst/src/scst_targ.c b/scst/src/scst_targ.c index bb1295ff0..95cb926cc 100644 --- a/scst/src/scst_targ.c +++ b/scst/src/scst_targ.c @@ -514,7 +514,7 @@ void __scst_check_unblock_dev(struct scst_cmd *cmd) if (unlikely(cmd->unblock_dev)) { TRACE_BLOCK("cmd %p (tag %llu): unblocking dev %s", cmd, - (unsigned long long int)cmd->tag, dev->virt_name); + (unsigned long long)cmd->tag, dev->virt_name); cmd->unblock_dev = 0; scst_unblock_dev(dev); } else if (unlikely(dev->strictly_serialized_cmd_waiting)) { @@ -796,9 +796,9 @@ void scst_cmd_init_done(struct scst_cmd *cmd, TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd); TRACE(TRACE_SCSI, "NEW CDB: len %d, lun %lld, initiator %s, " "target %s, queue_type %x, tag %llu (cmd %p, sess %p)", - cmd->cdb_len, (unsigned long long int)cmd->lun, + cmd->cdb_len, (unsigned long long)cmd->lun, cmd->sess->initiator_name, cmd->tgt->tgt_name, cmd->queue_type, - (unsigned long long int)cmd->tag, cmd, sess); + (unsigned long long)cmd->tag, cmd, sess); PRINT_BUFF_FLAG(TRACE_SCSI, "CDB", cmd->cdb, cmd->cdb_len); #ifdef CONFIG_SCST_EXTRACHECKS @@ -1653,7 +1653,7 @@ void scst_restart_cmd(struct scst_cmd *cmd, int status, TRACE_DBG("Preferred context: %d", pref_context); TRACE_DBG("tag=%llu, status=%#x", - (unsigned long long int)scst_cmd_get_tag(cmd), + (unsigned long long)scst_cmd_get_tag(cmd), status); #ifdef CONFIG_SCST_EXTRACHECKS @@ -2760,7 +2760,7 @@ static int scst_reserve_local(struct scst_cmd *cmd) if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) { PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented " - "(lun=%lld)", (unsigned long long int)cmd->lun); + "(lun=%lld)", (unsigned long long)cmd->lun); scst_set_invalid_field_in_cdb(cmd, 2, SCST_INVAL_FIELD_BIT_OFFS_VALID | 4); goto out_done; @@ -3435,7 +3435,7 @@ static int scst_do_real_exec(struct scst_cmd *cmd) if (unlikely(scsi_dev == NULL)) { PRINT_ERROR("Command for virtual device must be " "processed by device handler (LUN %lld)!", - (unsigned long long int)cmd->lun); + (unsigned long long)cmd->lun); goto out_error; } @@ -4150,7 +4150,7 @@ next: (cmd->cdb[0] == MODE_SELECT_10) || (cmd->cdb[0] == LOG_SELECT))) { TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded (LUN %lld)", - (unsigned long long int)cmd->lun); + (unsigned long long)cmd->lun); scst_set_cmd_state(cmd, SCST_CMD_STATE_MODE_SELECT_CHECKS); goto out; } @@ -4163,7 +4163,7 @@ next: SCST_SENSE_ASCx_VALID, 0, 0x2a, 0x01)) { TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun " - "%lld)", (unsigned long long int)cmd->lun); + "%lld)", (unsigned long long)cmd->lun); scst_set_cmd_state(cmd, SCST_CMD_STATE_MODE_SELECT_CHECKS); goto out; } @@ -4201,7 +4201,7 @@ static int scst_mode_select_checks(struct scst_cmd *cmd) TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, " "setting the SELECT UA (lun=%lld)", - (unsigned long long int)cmd->lun); + (unsigned long long)cmd->lun); spin_lock_bh(&dev->dev_lock); if (cmd->cdb[0] == LOG_SELECT) { @@ -4248,7 +4248,7 @@ static int scst_mode_select_checks(struct scst_cmd *cmd) TRACE(TRACE_SCSI, "Possible parameters changed UA %x " "(LUN %lld): getting new parameters", cmd->sense[12], - (unsigned long long int)cmd->lun); + (unsigned long long)cmd->lun); scst_obtain_device_parameters(cmd->dev, NULL); } else @@ -4453,7 +4453,7 @@ again: if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) { EXTRACHECKS_BUG_ON(!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)); TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu), " - "skipping", cmd, (unsigned long long int)cmd->tag); + "skipping", cmd, (unsigned long long)cmd->tag); scst_set_cmd_state(cmd, SCST_CMD_STATE_FINISHED); goto out_same; } @@ -4988,7 +4988,7 @@ static int scst_translate_lun(struct scst_cmd *cmd) if (likely(!test_bit(SCST_FLAG_SUSPENDED, &scst_flags))) { TRACE_DBG("Finding tgt_dev for cmd %p (lun %lld)", cmd, - (unsigned long long int)cmd->lun); + (unsigned long long)cmd->lun); res = -1; rcu_read_lock(); @@ -5009,7 +5009,7 @@ static int scst_translate_lun(struct scst_cmd *cmd) } else { PRINT_INFO("Dev handler for device %lld is NULL, " "the device will not be visible remotely", - (unsigned long long int)cmd->lun); + (unsigned long long)cmd->lun); nul_dev = true; } } @@ -5018,7 +5018,7 @@ static int scst_translate_lun(struct scst_cmd *cmd) TRACE(TRACE_MINOR, "tgt_dev for LUN %lld not found, command to " "unexisting LU (initiator %s, target %s)?", - (unsigned long long int)cmd->lun, + (unsigned long long)cmd->lun, cmd->sess->initiator_name, cmd->tgt->tgt_name); scst_event_queue_lun_not_found(cmd); } @@ -5241,7 +5241,7 @@ restart: } } else { TRACE_MGMT_DBG("Aborting not inited cmd %p (tag %llu)", - cmd, (unsigned long long int)cmd->tag); + cmd, (unsigned long long)cmd->tag); scst_set_cmd_abnormal_done_state(cmd); } @@ -5465,7 +5465,7 @@ void scst_process_active_cmd(struct scst_cmd *cmd, bool atomic) res = SCST_CMD_STATE_RES_CONT_NEXT; TRACE_MGMT_DBG("Skipping cmd %p (tag %llu), " "because of TM DBG delay", cmd, - (unsigned long long int)cmd->tag); + (unsigned long long)cmd->tag); break; } res = scst_exec_check_sn(&cmd); @@ -5851,7 +5851,7 @@ static int scst_mgmt_translate_lun(struct scst_mgmt_cmd *mcmd) TRACE_ENTRY(); TRACE_DBG("Finding tgt_dev for mgmt cmd %p (lun %lld)", mcmd, - (unsigned long long int)mcmd->lun); + (unsigned long long)mcmd->lun); res = scst_get_mgmt(mcmd); if (unlikely(res != 0)) @@ -5885,7 +5885,7 @@ void scst_done_cmd_mgmt(struct scst_cmd *cmd) TRACE_ENTRY(); TRACE_MGMT_DBG("cmd %p done (tag %llu)", - cmd, (unsigned long long int)cmd->tag); + cmd, (unsigned long long)cmd->tag); spin_lock_irqsave(&scst_mcmd_lock, flags); @@ -6041,7 +6041,7 @@ void scst_finish_cmd_mgmt(struct scst_cmd *cmd) TRACE_ENTRY(); TRACE(TRACE_MGMT, "Aborted cmd %p finished (tag %llu, ref %d)", cmd, - (unsigned long long int)cmd->tag, atomic_read(&cmd->cmd_ref)); + (unsigned long long)cmd->tag, atomic_read(&cmd->cmd_ref)); spin_lock_irqsave(&scst_mcmd_lock, flags); @@ -6145,7 +6145,7 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd, EXTRACHECKS_BUG_ON(!mcmd); TRACE(TRACE_SCSI|TRACE_MGMT_DEBUG, "Aborting cmd %p (tag %llu, op %s)", - cmd, (unsigned long long int)cmd->tag, scst_get_opcode_name(cmd)); + cmd, (unsigned long long)cmd->tag, scst_get_opcode_name(cmd)); /* To protect from concurrent aborts */ spin_lock_irqsave(&other_ini_lock, flags); @@ -6230,7 +6230,7 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd, if (cmd->sent_for_exec && !cmd->done) { TRACE_MGMT_DBG("cmd %p (tag %llu) is being executed", - cmd, (unsigned long long int)cmd->tag); + cmd, (unsigned long long)cmd->tag); mstb->done_counted = 1; mcmd->cmd_done_wait_count++; } @@ -6258,7 +6258,7 @@ void scst_abort_cmd(struct scst_cmd *cmd, struct scst_mgmt_cmd *mcmd, "deferring ABORT (cmd_done_wait_count %d, " "cmd_finish_wait_count %d, internal %d, mcmd " "fn %d (mcmd %p), initiator %s, target %s)", - cmd, (unsigned long long int)cmd->tag, + cmd, (unsigned long long)cmd->tag, cmd->sn, scst_get_cmd_state_name(state_name, sizeof(state_name), cmd->state), scst_get_opcode_name(cmd), @@ -6497,7 +6497,7 @@ static int scst_abort_task_set(struct scst_mgmt_cmd *mcmd) struct scst_tgt_dev *tgt_dev = mcmd->mcmd_tgt_dev; TRACE(TRACE_MGMT, "Aborting task set (lun=%lld, mcmd=%p)", - (unsigned long long int)tgt_dev->lun, mcmd); + (unsigned long long)tgt_dev->lun, mcmd); __scst_abort_task_set(mcmd, tgt_dev); @@ -6547,7 +6547,7 @@ static bool scst_is_cmd_belongs_to_dev(struct scst_cmd *cmd, TRACE_ENTRY(); TRACE_DBG("Finding match for dev %s and cmd %p (lun %lld)", - dev->virt_name, cmd, (unsigned long long int)cmd->lun); + dev->virt_name, cmd, (unsigned long long)cmd->lun); rcu_read_lock(); tgt_dev = scst_lookup_tgt_dev(cmd->sess, cmd->lun); @@ -6569,7 +6569,7 @@ static int scst_clear_task_set(struct scst_mgmt_cmd *mcmd) TRACE_ENTRY(); TRACE(TRACE_MGMT, "Clearing task set (lun=%lld, mcmd=%p)", - (unsigned long long int)mcmd->lun, mcmd); + (unsigned long long)mcmd->lun, mcmd); #if 0 /* we are SAM-3 */ /* @@ -6686,7 +6686,7 @@ static int scst_mgmt_cmd_init(struct scst_mgmt_cmd *mcmd) if (cmd == NULL) { TRACE_MGMT_DBG("ABORT TASK: command " "for tag %llu not found", - (unsigned long long int)mcmd->tag); + (unsigned long long)mcmd->tag); scst_mgmt_cmd_set_status(mcmd, SCST_MGMT_STATUS_TASK_NOT_EXIST); spin_unlock_irq(&sess->sess_list_lock); res = scst_set_mcmd_next_state(mcmd); @@ -6698,7 +6698,7 @@ static int scst_mgmt_cmd_init(struct scst_mgmt_cmd *mcmd) mcmd->cpu_cmd_counter = scst_get(); spin_unlock_irq(&sess->sess_list_lock); TRACE_DBG("Cmd to abort %p for tag %llu found (tgt_dev %p)", - cmd, (unsigned long long int)mcmd->tag, tgt_dev); + cmd, (unsigned long long)mcmd->tag, tgt_dev); mcmd->cmd_to_abort = cmd; sBUG_ON(mcmd->mcmd_tgt_dev != NULL); mcmd->mcmd_tgt_dev = tgt_dev; @@ -6740,7 +6740,7 @@ static int scst_mgmt_cmd_init(struct scst_mgmt_cmd *mcmd) mcmd->state = SCST_MCMD_STATE_EXEC; else if (rc < 0) { PRINT_ERROR("Corresponding device for LUN %lld not " - "found", (unsigned long long int)mcmd->lun); + "found", (unsigned long long)mcmd->lun); scst_mgmt_cmd_set_status(mcmd, SCST_MGMT_STATUS_LUN_NOT_EXIST); res = scst_set_mcmd_next_state(mcmd); } else @@ -6881,7 +6881,7 @@ static int scst_lun_reset(struct scst_mgmt_cmd *mcmd) TRACE_ENTRY(); TRACE(TRACE_MGMT, "Resetting LUN %lld (mcmd %p)", - (unsigned long long int)tgt_dev->lun, mcmd); + (unsigned long long)tgt_dev->lun, mcmd); mcmd->needs_unblocking = 1; @@ -7103,21 +7103,21 @@ static int scst_abort_task(struct scst_mgmt_cmd *mcmd) TRACE_MGMT_DBG("Aborting task (cmd %p, sn %d, set %d, tag %llu, " "queue_type %x)", cmd, cmd->sn, cmd->sn_set, - (unsigned long long int)mcmd->tag, cmd->queue_type); + (unsigned long long)mcmd->tag, cmd->queue_type); if (mcmd->lun_set && (mcmd->lun != cmd->lun)) { PRINT_ERROR("ABORT TASK: LUN mismatch: mcmd LUN %llx, " "cmd LUN %llx, cmd tag %llu", - (unsigned long long int)mcmd->lun, - (unsigned long long int)cmd->lun, - (unsigned long long int)mcmd->tag); + (unsigned long long)mcmd->lun, + (unsigned long long)cmd->lun, + (unsigned long long)mcmd->tag); scst_mgmt_cmd_set_status(mcmd, SCST_MGMT_STATUS_REJECTED); } else if (mcmd->cmd_sn_set && (scst_sn_before(mcmd->cmd_sn, cmd->tgt_sn) || (mcmd->cmd_sn == cmd->tgt_sn))) { PRINT_ERROR("ABORT TASK: SN mismatch: mcmd SN %x, " "cmd SN %x, cmd tag %llu", mcmd->cmd_sn, - cmd->tgt_sn, (unsigned long long int)mcmd->tag); + cmd->tgt_sn, (unsigned long long)mcmd->tag); scst_mgmt_cmd_set_status(mcmd, SCST_MGMT_STATUS_REJECTED); } else { spin_lock_irq(&cmd->sess->sess_list_lock); @@ -7208,7 +7208,7 @@ static int scst_clear_aca_mcmd(struct scst_mgmt_cmd *mcmd) TRACE(TRACE_MGMT, "CLEAR ACA (dev %s, lun=%lld, mcmd %p, tgt_dev %p)", mcmd_tgt_dev->dev->virt_name, - (unsigned long long int)mcmd_tgt_dev->lun, mcmd, mcmd_tgt_dev); + (unsigned long long)mcmd_tgt_dev->lun, mcmd, mcmd_tgt_dev); spin_lock_irq(&order_data->sn_lock); @@ -7812,9 +7812,9 @@ int scst_rx_mgmt_fn(struct scst_session *sess, TRACE_MGMT_DBG("sess=%p, tag_set %d, tag %lld, lun_set %d, " "lun=%lld, cmd_sn_set %d, cmd_sn %d, priv %p", sess, params->tag_set, - (unsigned long long int)params->tag, + (unsigned long long)params->tag, params->lun_set, - (unsigned long long int)mcmd->lun, + (unsigned long long)mcmd->lun, params->cmd_sn_set, params->cmd_sn, params->tgt_priv); @@ -8555,7 +8555,7 @@ static struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess, /* ToDo: hash list */ TRACE_DBG("%s (sess=%p, tag=%llu)", "Searching in sess cmd list", - sess, (unsigned long long int)tag); + sess, (unsigned long long)tag); list_for_each_entry(cmd, &sess->sess_cmd_list, sess_cmd_list_entry) { diff --git a/scst_local/scst_local.c b/scst_local/scst_local.c index df41160e5..0528af537 100644 --- a/scst_local/scst_local.c +++ b/scst_local/scst_local.c @@ -799,7 +799,7 @@ out_up: #endif /* CONFIG_SCST_PROC */ -static int scst_local_abort(struct scsi_cmnd *SCpnt) +static int scst_local_abort(struct scsi_cmnd *scmd) { struct scst_local_sess *sess; int ret; @@ -807,10 +807,11 @@ static int scst_local_abort(struct scsi_cmnd *SCpnt) TRACE_ENTRY(); - sess = to_scst_lcl_sess(scsi_get_device(SCpnt->device->host)); + sess = to_scst_lcl_sess(scsi_get_device(scmd->device->host)); - ret = scst_rx_mgmt_fn_tag(sess->scst_sess, SCST_ABORT_TASK, SCpnt->tag, - false, &dev_reset_completion); + ret = scst_rx_mgmt_fn_tag(sess->scst_sess, SCST_ABORT_TASK, + blk_mq_unique_tag(scmd->request), + false, &dev_reset_completion); /* Now wait for the completion ... */ wait_for_completion_interruptible(&dev_reset_completion); @@ -824,7 +825,7 @@ static int scst_local_abort(struct scsi_cmnd *SCpnt) return ret; } -static int scst_local_device_reset(struct scsi_cmnd *SCpnt) +static int scst_local_device_reset(struct scsi_cmnd *scmd) { struct scst_local_sess *sess; struct scsi_lun lun; @@ -833,9 +834,9 @@ static int scst_local_device_reset(struct scsi_cmnd *SCpnt) TRACE_ENTRY(); - sess = to_scst_lcl_sess(scsi_get_device(SCpnt->device->host)); + sess = to_scst_lcl_sess(scsi_get_device(scmd->device->host)); - int_to_scsilun(SCpnt->device->lun, &lun); + int_to_scsilun(scmd->device->lun, &lun); ret = scst_rx_mgmt_fn_lun(sess->scst_sess, SCST_LUN_RESET, lun.scsi_lun, sizeof(lun), false, @@ -854,7 +855,7 @@ static int scst_local_device_reset(struct scsi_cmnd *SCpnt) } #if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 25)) -static int scst_local_target_reset(struct scsi_cmnd *SCpnt) +static int scst_local_target_reset(struct scsi_cmnd *scmd) { struct scst_local_sess *sess; struct scsi_lun lun; @@ -863,9 +864,9 @@ static int scst_local_target_reset(struct scsi_cmnd *SCpnt) TRACE_ENTRY(); - sess = to_scst_lcl_sess(scsi_get_device(SCpnt->device->host)); + sess = to_scst_lcl_sess(scsi_get_device(scmd->device->host)); - int_to_scsilun(SCpnt->device->lun, &lun); + int_to_scsilun(scmd->device->lun, &lun); ret = scst_rx_mgmt_fn_lun(sess->scst_sess, SCST_TARGET_RESET, lun.scsi_lun, sizeof(lun), false, @@ -936,9 +937,9 @@ static int scst_local_send_resp(struct scsi_cmnd *cmnd, */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37) static int scst_local_queuecommand(struct Scsi_Host *host, - struct scsi_cmnd *SCpnt) + struct scsi_cmnd *scmd) #else -static int scst_local_queuecommand_lck(struct scsi_cmnd *SCpnt, +static int scst_local_queuecommand_lck(struct scsi_cmnd *scmd, void (*done)(struct scsi_cmnd *)) __acquires(&h->host_lock) __releases(&h->host_lock) @@ -956,26 +957,26 @@ static int scst_local_queuecommand_lck(struct scsi_cmnd *SCpnt, TRACE_ENTRY(); - TRACE_DBG("lun %lld, cmd: 0x%02X", (u64)SCpnt->device->lun, - SCpnt->cmnd[0]); + TRACE_DBG("lun %lld, cmd: 0x%02X", (u64)scmd->device->lun, + scmd->cmnd[0]); #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37) /* - * We save a pointer to the done routine in SCpnt->scsi_done and + * We save a pointer to the done routine in scmd->scsi_done and * we save that as tgt specific stuff below. */ - SCpnt->scsi_done = done; + scmd->scsi_done = done; #endif - sess = to_scst_lcl_sess(scsi_get_device(SCpnt->device->host)); + sess = to_scst_lcl_sess(scsi_get_device(scmd->device->host)); if (sess->unregistering) { - SCpnt->result = DID_BAD_TARGET << 16; - SCpnt->scsi_done(SCpnt); + scmd->result = DID_BAD_TARGET << 16; + scmd->scsi_done(scmd); return 0; } - scsi_set_resid(SCpnt, 0); + scsi_set_resid(scmd, 0); #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)) /* @@ -988,7 +989,7 @@ static int scst_local_queuecommand_lck(struct scsi_cmnd *SCpnt, sizeof(*tgt_specific)); return SCSI_MLQUEUE_HOST_BUSY; } - tgt_specific->cmnd = SCpnt; + tgt_specific->cmnd = scmd; tgt_specific->done = done; #endif @@ -1000,22 +1001,22 @@ static int scst_local_queuecommand_lck(struct scsi_cmnd *SCpnt, * get into mem alloc deadlock when mounting file systems over * our devices. */ - int_to_scsilun(SCpnt->device->lun, &lun); + int_to_scsilun(scmd->device->lun, &lun); scst_cmd = scst_rx_cmd(sess->scst_sess, lun.scsi_lun, sizeof(lun), - SCpnt->cmnd, SCpnt->cmd_len, true); + scmd->cmnd, scmd->cmd_len, true); if (!scst_cmd) { PRINT_ERROR("%s", "scst_rx_cmd() failed"); return SCSI_MLQUEUE_HOST_BUSY; } - scst_cmd_set_tag(scst_cmd, SCpnt->tag); + scst_cmd_set_tag(scst_cmd, blk_mq_unique_tag(scmd->request)); #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) - if (SCpnt->device->tagged_supported && SCpnt->device->simple_tags) + if (scmd->device->tagged_supported && scmd->device->simple_tags) scst_cmd_set_queue_type(scst_cmd, SCST_CMD_QUEUE_SIMPLE); else scst_cmd_set_queue_type(scst_cmd, SCST_CMD_QUEUE_UNTAGGED); #else - switch (scsi_get_tag_type(SCpnt->device)) { + switch (scsi_get_tag_type(scmd->device)) { case MSG_SIMPLE_TAG: scst_cmd_set_queue_type(scst_cmd, SCST_CMD_QUEUE_SIMPLE); break; @@ -1038,9 +1039,9 @@ static int scst_local_queuecommand_lck(struct scsi_cmnd *SCpnt, * to one. We use scsi_sg_count to isolate us from the changes from * version to version */ - if (scsi_sg_count(SCpnt)) { - sgl = scsi_sglist(SCpnt); - sgl_count = scsi_sg_count(SCpnt); + if (scsi_sg_count(scmd)) { + sgl = scsi_sglist(scmd); + sgl_count = scsi_sg_count(scmd); } else { /* * Build a one-element scatter list out of the buffer @@ -1050,11 +1051,11 @@ static int scst_local_queuecommand_lck(struct scsi_cmnd *SCpnt, * We use the sglist and bufflen function/macros to isolate * us from kernel version differences. */ - if (scsi_sglist(SCpnt)) { - sg_init_one(&(tgt_specific->sgl), - scsi_sglist(SCpnt), - scsi_bufflen(SCpnt)); - sgl = &(tgt_specific->sgl); + if (scsi_sglist(scmd)) { + sg_init_one(&tgt_specific->sgl, + scsi_sglist(scmd), + scsi_bufflen(scmd)); + sgl = &tgt_specific->sgl; sgl_count = 1; } else { sgl = NULL; @@ -1062,30 +1063,30 @@ static int scst_local_queuecommand_lck(struct scsi_cmnd *SCpnt, } } #else - sgl = scsi_sglist(SCpnt); - sgl_count = scsi_sg_count(SCpnt); + sgl = scsi_sglist(scmd); + sgl_count = scsi_sg_count(scmd); #endif - if (scsi_bidi_cmnd(SCpnt)) { + if (scsi_bidi_cmnd(scmd)) { #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 24) /* Some of these symbols are only defined after 2.6.24 */ dir = SCST_DATA_BIDI; - scst_cmd_set_expected(scst_cmd, dir, scsi_bufflen(SCpnt)); + scst_cmd_set_expected(scst_cmd, dir, scsi_bufflen(scmd)); scst_cmd_set_expected_out_transfer_len(scst_cmd, - scsi_in(SCpnt)->length); + scsi_in(scmd)->length); scst_cmd_set_noio_mem_alloc(scst_cmd); - scst_cmd_set_tgt_sg(scst_cmd, scsi_in(SCpnt)->table.sgl, - scsi_in(SCpnt)->table.nents); + scst_cmd_set_tgt_sg(scst_cmd, scsi_in(scmd)->table.sgl, + scsi_in(scmd)->table.nents); scst_cmd_set_tgt_out_sg(scst_cmd, sgl, sgl_count); #endif - } else if (SCpnt->sc_data_direction == DMA_TO_DEVICE) { + } else if (scmd->sc_data_direction == DMA_TO_DEVICE) { dir = SCST_DATA_WRITE; - scst_cmd_set_expected(scst_cmd, dir, scsi_bufflen(SCpnt)); + scst_cmd_set_expected(scst_cmd, dir, scsi_bufflen(scmd)); scst_cmd_set_noio_mem_alloc(scst_cmd); scst_cmd_set_tgt_sg(scst_cmd, sgl, sgl_count); - } else if (SCpnt->sc_data_direction == DMA_FROM_DEVICE) { + } else if (scmd->sc_data_direction == DMA_FROM_DEVICE) { dir = SCST_DATA_READ; - scst_cmd_set_expected(scst_cmd, dir, scsi_bufflen(SCpnt)); + scst_cmd_set_expected(scst_cmd, dir, scsi_bufflen(scmd)); scst_cmd_set_noio_mem_alloc(scst_cmd); scst_cmd_set_tgt_sg(scst_cmd, sgl, sgl_count); } else { @@ -1097,7 +1098,7 @@ static int scst_local_queuecommand_lck(struct scsi_cmnd *SCpnt, #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)) scst_cmd_set_tgt_priv(scst_cmd, tgt_specific); #else - scst_cmd_set_tgt_priv(scst_cmd, SCpnt); + scst_cmd_set_tgt_priv(scst_cmd, scmd); #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37) @@ -1108,7 +1109,7 @@ static int scst_local_queuecommand_lck(struct scsi_cmnd *SCpnt, * real effect. */ #ifdef CONFIG_SMP - if (spin_is_locked(SCpnt->device->host->host_lock)) + if (spin_is_locked(scmd->device->host->host_lock)) scst_cmd_init_done(scst_cmd, SCST_CONTEXT_THREAD); else #endif @@ -1452,7 +1453,7 @@ static int scst_local_targ_xmit_response(struct scst_cmd *scst_cmd) #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)) struct scst_local_tgt_specific *tgt_specific; #endif - struct scsi_cmnd *SCpnt = NULL; + struct scsi_cmnd *scmd = NULL; void (*done)(struct scsi_cmnd *); TRACE_ENTRY(); @@ -1469,11 +1470,11 @@ static int scst_local_targ_xmit_response(struct scst_cmd *scst_cmd) #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)) tgt_specific = scst_cmd_get_tgt_priv(scst_cmd); - SCpnt = tgt_specific->cmnd; + scmd = tgt_specific->cmnd; done = tgt_specific->done; #else - SCpnt = scst_cmd_get_tgt_priv(scst_cmd); - done = SCpnt->scsi_done; + scmd = scst_cmd_get_tgt_priv(scst_cmd); + done = scmd->scsi_done; #endif /* @@ -1484,20 +1485,20 @@ static int scst_local_targ_xmit_response(struct scst_cmd *scst_cmd) /* Calculate the residual ... */ if (likely(!scst_get_resid(scst_cmd, &resid, &out_resid))) { - TRACE_DBG("No residuals for request %p", SCpnt); + TRACE_DBG("No residuals for request %p", scmd); } else { if (out_resid != 0) PRINT_ERROR("Unable to return OUT residual %d " - "(op %02x)", out_resid, SCpnt->cmnd[0]); + "(op %02x)", out_resid, scmd->cmnd[0]); } - scsi_set_resid(SCpnt, resid); + scsi_set_resid(scmd, resid); /* * It seems like there is no way to set out_resid ... */ - (void)scst_local_send_resp(SCpnt, scst_cmd, done, + (void)scst_local_send_resp(scmd, scst_cmd, done, scst_cmd_get_status(scst_cmd)); } @@ -1682,6 +1683,10 @@ static int scst_local_driver_probe(struct device *dev) goto out; } +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0) + hpnt->nr_hw_queues = num_possible_cpus(); +#endif + sess->shost = hpnt; hpnt->max_id = 1; /* Don't want more than one id */