2 missed functions added to scst_user interface:

- SCST_USER_UNREGISTER_DEVICE - to unregister device, flush mem reuse (SGV) cache and send UCMD_STATE_ON_CACHE_FREEING notifications for all freed buffers to the user space handler. Simple device close doesn't allow that, so all the cached buffers might look as "leaked" from the user space handler POV.

 - SCST_USER_FLUSH_CACHE - to flush mem reuse (SGV) cache and send UCMD_STATE_ON_CACHE_FREEING notifications for all freed buffers to the user space handler

+ some cosmetics

Docs update is coming



git-svn-id: http://svn.code.sf.net/p/scst/svn/trunk@665 d57e44dd-8a1f-0410-8b47-8ef2f437770f

scst/include/scst.h

@@ -444,6 +444,7 @@ enum scst_cdb_flags {
  * have to provide in order to work with the target mid-level.
  * MUST HAVEs define functions that are expected to be in order to work.
  * OPTIONAL says that there is a choice.
+ *
  * Also, pay attention to the fact that a command is BLOCKING or NON-BLOCKING.
  * NON-BLOCKING means that a function returns immediately and will not wait
  * for actual data transfer to finish. Blocking in such command could have
@@ -451,6 +452,7 @@ enum scst_cdb_flags {
  * it is worth to consider creating dedicated thread(s) in target driver, to
  * which the commands would be passed and which would perform blocking
  * operations instead of SCST.
+ *
  * If the function allowed to sleep or not is determined by its last
  * argument, which is true, if sleeping is not allowed. In this case,
  * if the function requires sleeping, it  can return
@@ -623,6 +625,7 @@ struct scst_tgt_template {
 	 * as the mid-level is concerned. Any information that must be
 	 * stored about the command is the responsibility of the low-
 	 * level driver. No return value expected.
+	 *
 	 * This function is expected to be NON-BLOCKING
 	 *
 	 * Called without any locks held from a thread context.

scst/include/scst_sgv.h

@@ -53,6 +53,7 @@ enum sgv_clustering_types {
 struct sgv_pool *sgv_pool_create(const char *name,
 	enum sgv_clustering_types clustered);
 void sgv_pool_destroy(struct sgv_pool *pool);
+void sgv_pool_flush(struct sgv_pool *pool);
 
 void sgv_pool_set_allocator(struct sgv_pool *pool,
 	struct page *(*alloc_pages_fn)(struct scatterlist *, gfp_t, void *),

scst/include/scst_user.h

@@ -236,10 +236,12 @@ struct scst_user_reply_cmd {
 };
 
 #define SCST_USER_REGISTER_DEVICE	_IOW('u', 1, struct scst_user_dev_desc)
+#define SCST_USER_UNREGISTER_DEVICE	_IO('u', 2)
 #define SCST_USER_SET_OPTIONS		_IOW('u', 3, struct scst_user_opt)
 #define SCST_USER_GET_OPTIONS		_IOR('u', 4, struct scst_user_opt)
 #define SCST_USER_REPLY_AND_GET_CMD	_IOWR('u', 5, struct scst_user_get_cmd)
 #define SCST_USER_REPLY_CMD		_IOW('u', 6, struct scst_user_reply_cmd)
+#define SCST_USER_FLUSH_CACHE		_IO('u', 7)
 
 /* Values for scst_user_get_cmd.subcode */
 #define SCST_USER_ATTACH_SESS		\

scst/src/dev_handlers/scst_user.c

@@ -184,6 +184,8 @@ static int dev_user_process_reply_tm_exec(struct scst_user_cmd *ucmd,
 static int dev_user_process_reply_sess(struct scst_user_cmd *ucmd, int status);
 static int dev_user_register_dev(struct file *file,
 	const struct scst_user_dev_desc *dev_desc);
+static int dev_user_unregister_dev(struct file *file);
+static int dev_user_flush_cache(struct file *file);
 static int __dev_user_set_opt(struct scst_user_dev *dev,
 	const struct scst_user_opt *opt);
 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt);
@@ -511,7 +513,6 @@ static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
 	int flags = 0;
 	int bufflen = cmd->bufflen;
 	int last_len = 0;
-	struct sgv_pool *pool;
 
 	TRACE_ENTRY();
 
@@ -541,11 +542,6 @@ static int dev_user_alloc_sg(struct scst_user_cmd *ucmd, int cached_buff)
 	}
 	ucmd->buff_cached = cached_buff;
 
-	if (test_bit(SCST_TGT_DEV_CLUST_POOL, &cmd->tgt_dev->tgt_dev_flags))
-		pool = dev->pool_clust;
-	else
-		pool = dev->pool;
-
 	cmd->sg = sgv_pool_alloc((struct sgv_pool *)cmd->tgt_dev->dh_priv,
 			bufflen, gfp_mask, flags, &cmd->sg_cnt, &ucmd->sgv,
 			&dev->udev_mem_lim, ucmd);
@@ -1831,6 +1827,16 @@ static long dev_user_ioctl(struct file *file, unsigned int cmd,
 		break;
 	}
 
+	case SCST_USER_UNREGISTER_DEVICE:
+		TRACE_DBG("%s", "UNREGISTER_DEVICE");
+		res = dev_user_unregister_dev(file);
+		break;
+
+	case SCST_USER_FLUSH_CACHE:
+		TRACE_DBG("%s", "FLUSH_CACHE");
+		res = dev_user_flush_cache(file);
+		break;
+
 	case SCST_USER_SET_OPTIONS:
 	{
 		struct scst_user_opt opt;
@@ -2681,6 +2687,76 @@ out_put:
 	goto out;
 }
 
+static int dev_user_unregister_dev(struct file *file)
+{
+	int res;
+	struct scst_user_dev *dev;
+
+	TRACE_ENTRY();
+
+	mutex_lock(&dev_priv_mutex);
+	dev = (struct scst_user_dev *)file->private_data;
+	res = dev_user_check_reg(dev);
+	if (res != 0) {
+		mutex_unlock(&dev_priv_mutex);
+		goto out;
+	}
+	down_read(&dev->dev_rwsem);
+	mutex_unlock(&dev_priv_mutex);
+
+	res = scst_suspend_activity(true);
+	if (res != 0)
+		goto out_up;
+
+	up_read(&dev->dev_rwsem);
+
+	dev_user_release(NULL, file);
+
+	scst_resume_activity();
+
+out:
+	TRACE_EXIT_RES(res);
+	return res;
+
+out_up:
+	up_read(&dev->dev_rwsem);
+	goto out;
+}
+
+static int dev_user_flush_cache(struct file *file)
+{
+	int res;
+	struct scst_user_dev *dev;
+
+	TRACE_ENTRY();
+
+	mutex_lock(&dev_priv_mutex);
+	dev = (struct scst_user_dev *)file->private_data;
+	res = dev_user_check_reg(dev);
+	if (res != 0) {
+		mutex_unlock(&dev_priv_mutex);
+		goto out;
+	}
+	down_read(&dev->dev_rwsem);
+	mutex_unlock(&dev_priv_mutex);
+
+	res = scst_suspend_activity(true);
+	if (res != 0)
+		goto out_up;
+
+	sgv_pool_flush(dev->pool);
+	sgv_pool_flush(dev->pool_clust);
+
+	scst_resume_activity();
+
+out_up:
+	up_read(&dev->dev_rwsem);
+
+out:
+	TRACE_EXIT_RES(res);
+	return res;
+}
+
 static int __dev_user_set_opt(struct scst_user_dev *dev,
 	const struct scst_user_opt *opt)
 {
@@ -2744,7 +2820,7 @@ out:
 
 static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
 {
-	int res = 0;
+	int res;
 	struct scst_user_dev *dev;
 
 	TRACE_ENTRY();
@@ -2756,7 +2832,7 @@ static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
 		mutex_unlock(&dev_priv_mutex);
 		goto out;
 	}
-	down_write(&dev->dev_rwsem);
+	down_read(&dev->dev_rwsem);
 	mutex_unlock(&dev_priv_mutex);
 
 	res = scst_suspend_activity(true);
@@ -2767,7 +2843,7 @@ static int dev_user_set_opt(struct file *file, const struct scst_user_opt *opt)
 
 	scst_resume_activity();
 
-	up_write(&dev->dev_rwsem);
+	up_read(&dev->dev_rwsem);
 
 out:
 	TRACE_EXIT_RES(res);
@@ -2776,7 +2852,7 @@ out:
 
 static int dev_user_get_opt(struct file *file, void __user *arg)
 {
-	int res = 0;
+	int res;
 	struct scst_user_dev *dev;
 	struct scst_user_opt opt;
 
@@ -2853,16 +2929,17 @@ static int dev_user_release(struct inode *inode, struct file *file)
 	list_del(&dev->dev_list_entry);
 	spin_unlock(&dev_list_lock);
 
-	mutex_unlock(&dev_priv_mutex);
+	dev->blocking = 0;
+	wake_up_all(&dev->cmd_lists.cmd_list_waitQ);
 
 	down_write(&dev->dev_rwsem);
+	mutex_unlock(&dev_priv_mutex);
 
 	spin_lock(&cleanup_lock);
 	list_add_tail(&dev->cleanup_list_entry, &cleanup_list);
 	spin_unlock(&cleanup_lock);
 
 	wake_up(&cleanup_list_waitQ);
-	wake_up(&dev->cmd_lists.cmd_list_waitQ);
 
 	scst_unregister_virtual_device(dev->virt_id);
 	scst_unregister_virtual_dev_driver(&dev->devtype);
@@ -2899,7 +2976,8 @@ static int dev_user_process_cleanup(struct scst_user_dev *dev)
 
 	TRACE_ENTRY();
 
-	dev->blocking = 0;
+	sBUG_ON(dev->blocking);
+	wake_up_all(&dev->cmd_lists.cmd_list_waitQ); /* just in case */
 
 	while (1) {
 		TRACE_DBG("Cleanuping dev %p", dev);

scst/src/scst_mem.c

@@ -1081,23 +1081,19 @@ static void sgv_pool_evaluate_local_order(struct scst_sgv_pools_manager *pmgr)
 		+ sizeof(struct sgv_pool_obj));
 }
 
-void sgv_pool_deinit(struct sgv_pool *pool)
+void sgv_pool_flush(struct sgv_pool *pool)
 {
 	int i;
 
 	TRACE_ENTRY();
 
-	mutex_lock(&sgv_pools_mgr.scst_sgv_pool_mutex);
-	list_del(&pool->sgv_pool_list_entry);
-	mutex_unlock(&sgv_pools_mgr.scst_sgv_pool_mutex);
-
 	for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
 		struct sgv_pool_obj *e;
 
 		spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
 		while (!list_empty(&pool->recycling_lists[i])) {
 			e = list_entry(pool->recycling_lists[i].next,
-				 struct sgv_pool_obj,
+				struct sgv_pool_obj,
 				recycle_entry.recycling_list_entry);
 
 			__sgv_pool_cached_purge(e);
@@ -1109,7 +1105,26 @@ void sgv_pool_deinit(struct sgv_pool *pool)
 			spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
 		}
 		spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
+	}
 
+	TRACE_EXIT();
+	return;
+}
+EXPORT_SYMBOL(sgv_pool_flush);
+
+void sgv_pool_deinit(struct sgv_pool *pool)
+{
+	int i;
+
+	TRACE_ENTRY();
+
+	mutex_lock(&sgv_pools_mgr.scst_sgv_pool_mutex);
+	list_del(&pool->sgv_pool_list_entry);
+	mutex_unlock(&sgv_pools_mgr.scst_sgv_pool_mutex);
+
+	sgv_pool_flush(pool);
+
+	for (i = 0; i < SGV_POOL_ELEMENTS; i++) {
 		if (pool->caches[i])
 			kmem_cache_destroy(pool->caches[i]);
 		pool->caches[i] = NULL;

usr/fileio/common.c

@@ -781,10 +781,10 @@ void *main_loop(void *arg)
 			switch(res) {
 			case ESRCH:
 			case EBUSY:
-			case EINTR:
 				TRACE_MGMT_DBG("SCST_USER_REPLY_AND_GET_CMD returned "
 					"%d (%s)", res, strerror(res));
 				cmd.preply = 0;
+			case EINTR:
 				continue;
 			case EAGAIN:
 				TRACE_DBG("SCST_USER_REPLY_AND_GET_CMD returned "
@@ -815,9 +815,9 @@ again_poll:
 				case ESRCH:
 				case EBUSY:
 				case EAGAIN:
-				case EINTR:
 					TRACE_MGMT_DBG("poll() returned %d "
 						"(%s)", res, strerror(res));
+				case EINTR:
 					goto again_poll;
 				default:
 					PRINT_ERROR("poll() failed: %s", strerror(res));

usr/fileio/debug.h

@@ -129,6 +129,8 @@ do {								\
   }								\
 } while(0)
 
+#define TRACE_DBG_SPECIAL(args...)	TRACE(TRACE_DEBUG|TRACE_SPECIAL, args)
+
 #define TRACE_MGMT_DBG(format, args...)				\
 do {								\
   if (trace_flag & TRACE_MGMT_DEBUG)				\
@@ -246,6 +248,7 @@ do {								\
 
 #define TRACE_MEM(format, args...) {}
 #define TRACE_DBG(format, args...) {}
+#define TRACE_DBG_SPECIAL(args...) {}
 #define TRACE_MGMT_DBG(format, args...) {}
 #define TRACE_BUFFER(message, buff, len) {}
 #define TRACE_BUFF_FLAG(flag, message, buff, len) {}

usr/fileio/fileio.c

@@ -26,7 +26,7 @@
 #include <getopt.h>
 #include <malloc.h>
 #include <inttypes.h>
-
+#include <signal.h>
 #include <sys/types.h>
 #include <sys/user.h>
 #include <sys/poll.h>
@@ -69,7 +69,9 @@ unsigned long trace_flag = DEFAULT_LOG_FLAGS;
 #define VERSION_STR		"1.0.1"
 #define THREADS			7
 
+struct vdisk_dev dev;
 int vdisk_ID;
+int flush_interval;
 
 static struct option const long_options[] =
 {
@@ -85,6 +87,8 @@ static struct option const long_options[] =
 	{"mem_reuse", required_argument, 0, 'm'},
 	{"non_blocking", no_argument, 0, 'l'},
 	{"vdisk_id", required_argument, 0, 'I'},
+	{"flush", required_argument, 0, 'F'},
+	{"unreg_before_close", no_argument, 0, 'u'},
 #if defined(DEBUG) || defined(TRACING)
 	{"debug", required_argument, 0, 'd'},
 #endif
@@ -115,6 +119,8 @@ static void usage(void)
 		"(default), \"read\", \"write\" or \"none\"\n");
 	printf("  -l, --non_blocking	Use non-blocking operations\n");
 	printf("  -I, --vdisk_id=ID	Vdisk ID (used in multi-targets setups)\n");
+	printf("  -F, --flush=n		Flush SGV cache each n seconds\n");
+	printf("  -u, --unreg_before_close Unregister before close\n");
 #if defined(DEBUG) || defined(TRACING)
 	printf("  -d, --debug=level	Debug tracing level\n");
 #endif
@@ -154,6 +160,36 @@ static void *align_alloc(size_t size)
 	return memalign(PAGE_SIZE, size);
 }
 
+void sigalrm_handler(int signo)
+{
+	int res;
+
+	TRACE_ENTRY();
+
+	TRACE_DBG("%s", "Flushing cache...");
+
+	res = ioctl(dev.scst_usr_fd, SCST_USER_FLUSH_CACHE, NULL);
+	if (res != 0) {
+		res = errno;
+		PRINT_ERROR("Unable to flush cache: %s",
+			strerror(res));
+		goto out;
+	}
+
+	TRACE_DBG("%s", "Flushing cache done.");
+
+	res = alarm(flush_interval);
+	if (res != 0) {
+		res = errno;
+		PRINT_ERROR("alarm() failed: %s", strerror(res));
+		goto out;
+	}
+
+out:
+	TRACE_EXIT();
+	return;
+}
+
 int main(int argc, char **argv)
 {
 	int res = 0;
@@ -165,7 +201,7 @@ int main(int argc, char **argv)
 	int memory_reuse_type = SCST_USER_MEM_REUSE_ALL;
 	int threads = THREADS;
 	struct scst_user_dev_desc desc;
-	struct vdisk_dev dev;
+	int unreg_before_close = 0;
 
 	setlinebuf(stdout);
 
@@ -181,7 +217,7 @@ int main(int argc, char **argv)
 	dev.type = TYPE_DISK;
 	dev.alloc_fn = align_alloc;
 
-	while ((ch = getopt_long(argc, argv, "+b:e:tronglI:cp:f:m:d:vh", long_options,
+	while ((ch = getopt_long(argc, argv, "+b:e:trongluF:I:cp:f:m:d:vh", long_options,
 				&longindex)) >= 0) {
 		switch (ch) {
 		case 'b':
@@ -254,6 +290,17 @@ int main(int argc, char **argv)
 		case 'I':
 			vdisk_ID = strtol(optarg, (char **)NULL, 0);
 			break;
+		case 'F':
+			flush_interval = strtol(optarg, (char **)NULL, 0);
+			if (flush_interval < 0) {
+				PRINT_ERROR("Wrong flush interval %d",
+					flush_interval);
+				flush_interval = 0;
+			}
+			break;
+		case 'u':
+			unreg_before_close = 1;
+			break;
 #if defined(DEBUG_TM_IGNORE) || defined(DEBUG_TM_IGNORE_ALL)
 		case 'g':
 			dev.debug_tm_ignore = 1;
@@ -407,7 +454,7 @@ int main(int argc, char **argv)
 		if (res != 0) {
 			res = errno;
 			PRINT_ERROR("Unable to get options: %s", strerror(res));
-			goto out_close;
+			goto out_unreg;
 		}
 
 		opt.parse_type = parse_type;
@@ -418,7 +465,7 @@ int main(int argc, char **argv)
 		if (res != 0) {
 			res = errno;
 			PRINT_ERROR("Unable to set options: %s", strerror(res));
-			goto out_close;
+			goto out_unreg;
 		}
 	}
 #endif
@@ -427,7 +474,7 @@ int main(int argc, char **argv)
 	if (res != 0) {
 		res = errno;
 		PRINT_ERROR("pthread_mutex_init() failed: %s", strerror(res));
-		goto out_close;
+		goto out_unreg;
 	}
 
 	{
@@ -445,6 +492,31 @@ int main(int argc, char **argv)
 			}
 		}
 
+		if (flush_interval != 0) {
+			struct sigaction act;
+
+			memset(&act, 0, sizeof(act));
+			act.sa_handler = sigalrm_handler;
+			act.sa_flags = SA_RESTART;
+			sigemptyset(&act.sa_mask);
+			res = sigaction(SIGALRM, &act, NULL);
+			if (res != 0) {
+				res = errno;
+				PRINT_ERROR("sigaction() failed: %s",
+					strerror(res));
+				goto join;
+			}       
+
+			res = alarm(flush_interval);
+			if (res != 0) {
+				res = errno;
+				PRINT_ERROR("alarm() failed: %s",
+					strerror(res));
+				goto join;
+			}
+		}
+
+join:
 		j = i;
 		for(i = 0; i < j; i++) {
 			rc = pthread_join(thread[i], &rc1);
@@ -463,6 +535,19 @@ int main(int argc, char **argv)
 
 	pthread_mutex_destroy(&dev.dev_mutex);
 
+	alarm(0);
+
+out_unreg:
+	if (unreg_before_close) {
+		res = ioctl(dev.scst_usr_fd, SCST_USER_UNREGISTER_DEVICE, NULL);
+		if (res != 0) {
+			res = errno;
+			PRINT_ERROR("Unable to unregister device: %s",
+				strerror(res));
+			/* go through */
+		}
+	}
+
 out_close:
 	close(dev.scst_usr_fd);