Add forced unmount support

Add super_ops->umount_begin so that we can implement a forced unmount
which tries to avoid issuing any more network or storage ops.  It can
return errors and lose unsynchronized data.

Signed-off-by: Zach Brown <zab@versity.com>

kmod/src/block.c

@@ -200,7 +200,9 @@ static void block_free(struct super_block *sb, struct block_private *bp)
 	else
 		BUG();
 
-	WARN_ON_ONCE(!list_empty(&bp->dirty_entry));
+	/* ok to tear down dirty blocks when forcing unmount */
+	WARN_ON_ONCE(!scoutfs_forcing_unmount(sb) && !list_empty(&bp->dirty_entry));
+
 	WARN_ON_ONCE(atomic_read(&bp->refcount));
 	WARN_ON_ONCE(atomic_read(&bp->io_count));
 	kfree(bp);
@@ -485,6 +487,9 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp,
 	sector_t sector;
 	int ret = 0;
 
+	if (scoutfs_forcing_unmount(sb))
+		return -EIO;
+
 	sector = bp->bl.blkno << (SCOUTFS_BLOCK_LG_SHIFT - 9);
 
 	WARN_ON_ONCE(bp->bl.blkno == U64_MAX);
@@ -1148,7 +1153,7 @@ static void sm_block_bio_end_io(struct bio *bio, int err)
  * only layer that sees the full block buffer so we pass the calculated
  * crc to the caller for them to check in their context.
  */
-static int sm_block_io(struct block_device *bdev, int rw, u64 blkno,
+static int sm_block_io(struct super_block *sb, struct block_device *bdev, int rw, u64 blkno,
 		       struct scoutfs_block_header *hdr, size_t len,
 		       __le32 *blk_crc)
 {
@@ -1160,6 +1165,9 @@ static int sm_block_io(struct block_device *bdev, int rw, u64 blkno,
 
 	BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SM_SIZE);
 
+	if (scoutfs_forcing_unmount(sb))
+		return -EIO;
+
 	if (WARN_ON_ONCE(len > SCOUTFS_BLOCK_SM_SIZE) ||
 	    WARN_ON_ONCE(!(rw & WRITE) && !blk_crc))
 		return -EINVAL;
@@ -1212,14 +1220,14 @@ int scoutfs_block_read_sm(struct super_block *sb,
 			  struct scoutfs_block_header *hdr, size_t len,
 			  __le32 *blk_crc)
 {
-	return sm_block_io(bdev, READ, blkno, hdr, len, blk_crc);
+	return sm_block_io(sb, bdev, READ, blkno, hdr, len, blk_crc);
 }
 
 int scoutfs_block_write_sm(struct super_block *sb,
 			   struct block_device *bdev, u64 blkno,
 			   struct scoutfs_block_header *hdr, size_t len)
 {
-	return sm_block_io(bdev, WRITE, blkno, hdr, len, NULL);
+	return sm_block_io(sb, bdev, WRITE, blkno, hdr, len, NULL);
 }
 
 int scoutfs_block_setup(struct super_block *sb)

kmod/src/btree.c

@@ -1101,6 +1101,10 @@ static int btree_walk(struct super_block *sb,
 	if (WARN_ON_ONCE((flags & BTW_DIRTY) && (!alloc || !wri)))
 		return -EINVAL;
 
+	/* all ops come through walk and walk calls all reads */
+	if (scoutfs_forcing_unmount(sb))
+		return -EIO;
+
 	scoutfs_inc_counter(sb, btree_walk);
 
 restart:

kmod/src/client.c

@@ -466,9 +466,11 @@ static void scoutfs_client_connect_worker(struct work_struct *work)
 out:
 
 	/* always have a small delay before retrying to avoid storms */
-	if (ret && !atomic_read(&client->shutting_down))
+	if (ret && !atomic_read(&client->shutting_down) &&
+	    !scoutfs_forcing_unmount(sb)) {
 		queue_delayed_work(client->workq, &client->connect_dwork,
 				   msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS));
+	}
 }
 
 static scoutfs_net_request_t client_req_funcs[] = {
@@ -580,7 +582,7 @@ void scoutfs_client_destroy(struct super_block *sb)
 	if (client == NULL)
 		return;
 
-	if (client->server_term != 0) {
+	if (client->server_term != 0 && !scoutfs_forcing_unmount(sb)) {
 		client->sending_farewell = true;
 		ret = scoutfs_net_submit_request(sb, client->conn,
 						 SCOUTFS_NET_CMD_FAREWELL,

kmod/src/net.c

@@ -330,6 +330,9 @@ static int submit_send(struct super_block *sb,
 	    WARN_ON_ONCE(id == 0 && (flags & SCOUTFS_NET_FLAG_RESPONSE)))
 		return -EINVAL;
 
+	if (scoutfs_forcing_unmount(sb))
+		return -EIO;
+
 	msend = kmalloc(offsetof(struct message_send,
 				 nh.data[data_len]), GFP_NOFS);
 	if (!msend)
@@ -420,6 +423,16 @@ static int process_request(struct scoutfs_net_connection *conn,
 			mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len));
 }
 
+static int call_resp_func(struct super_block *sb, struct scoutfs_net_connection *conn,
+			  scoutfs_net_response_t resp_func, void *resp_data,
+			  void *resp, unsigned int resp_len, int error)
+{
+	if (resp_func)
+		return resp_func(sb, conn, resp, resp_len, error, resp_data);
+	else
+		return 0;
+}
+
 /*
  * An incoming response finds the queued request and calls its response
  * function.  The response function for a given request will only be
@@ -434,7 +447,6 @@ static int process_response(struct scoutfs_net_connection *conn,
 	struct message_send *msend;
 	scoutfs_net_response_t resp_func = NULL;
 	void *resp_data;
-	int ret = 0;
 
 	spin_lock(&conn->lock);
 
@@ -449,11 +461,8 @@ static int process_response(struct scoutfs_net_connection *conn,
 
 	spin_unlock(&conn->lock);
 
-	if (resp_func)
-		ret = resp_func(sb, conn, mrecv->nh.data,
-				le16_to_cpu(mrecv->nh.data_len),
-				net_err_to_host(mrecv->nh.error), resp_data);
-	return ret;
+	return call_resp_func(sb, conn, resp_func, resp_data, mrecv->nh.data,
+			      le16_to_cpu(mrecv->nh.data_len), net_err_to_host(mrecv->nh.error));
 }
 
 /*
@@ -823,9 +832,15 @@ static void scoutfs_net_destroy_worker(struct work_struct *work)
 	if (conn->listening_conn && conn->notify_down)
 		conn->notify_down(sb, conn, conn->info, conn->rid);
 
-	/* free all messages, refactor and complete for forced unmount? */
+	/*
+	 * Usually networking is idle and we destroy pending sends, but when forcing unmount
+	 * we can have to wake up waiters by failing pending sends.
+	 */
 	list_splice_init(&conn->resend_queue, &conn->send_queue);
 	list_for_each_entry_safe(msend, tmp, &conn->send_queue, head) {
+		if (scoutfs_forcing_unmount(sb))
+			call_resp_func(sb, conn, msend->resp_func, msend->resp_data,
+				       NULL, 0, -ECONNABORTED);
 		free_msend(ninf, msend);
 	}
 

kmod/src/server.c

@@ -284,6 +284,11 @@ static void scoutfs_server_commit_func(struct work_struct *work)
 
 	down_write(&server->commit_rwsem);
 
+	if (scoutfs_forcing_unmount(sb)) {
+		ret = -EIO;
+		goto out;
+	}
+
 	/* make sure next avail has sufficient blocks */
 	ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri,
 				      server->other_avail,

kmod/src/super.c

@@ -283,6 +283,21 @@ static void scoutfs_put_super(struct super_block *sb)
 	sb->s_fs_info = NULL;
 }
 
+/*
+ * Record that we're performing a forced unmount.  As put_super drives
+ * destruction of the filesystem we won't issue more network or storage
+ * operations because we assume that they'll hang.  Pending operations
+ * can return errors when it's possible to do so.  We may be racing with
+ * pending operations which can't be canceled.
+ */
+static void scoutfs_umount_begin(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+
+	scoutfs_warn(sb, "forcing unmount, can return errors and lose unsynced data");
+	sbi->forced_unmount = true;
+}
+
 static const struct super_operations scoutfs_super_ops = {
 	.alloc_inode = scoutfs_alloc_inode,
 	.drop_inode = scoutfs_drop_inode,
@@ -292,6 +307,7 @@ static const struct super_operations scoutfs_super_ops = {
 	.statfs = scoutfs_statfs,
 	.show_options = scoutfs_show_options,
 	.put_super = scoutfs_put_super,
+	.umount_begin = scoutfs_umount_begin,
 };
 
 /*

kmod/src/super.h

@@ -87,6 +87,8 @@ struct scoutfs_sb_info {
 
 	struct dentry *debug_root;
 
+	bool forced_unmount;
+
 	unsigned long corruption_messages_once[SC_NR_LONGS];
 };
 
@@ -107,6 +109,13 @@ static inline bool SCOUTFS_IS_META_BDEV(struct scoutfs_super_block *super_block)
 
 #define SCOUTFS_META_BDEV_MODE (FMODE_READ | FMODE_WRITE | FMODE_EXCL)
 
+static inline bool scoutfs_forcing_unmount(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+
+	return sbi->forced_unmount;
+}
+
 /*
  * A small string embedded in messages that's used to identify a
  * specific mount.  It's the three most significant bytes of the fsid

kmod/src/trans.c

@@ -185,6 +185,11 @@ void scoutfs_trans_write_func(struct work_struct *work)
 
 	wait_event(sbi->trans_hold_wq, drained_holders(tri));
 
+	if (scoutfs_forcing_unmount(sb)) {
+		ret = -EIO;
+		goto out;
+	}
+
 	trace_scoutfs_trans_write_func(sb,
 			scoutfs_block_writer_dirty_bytes(sb, &tri->wri));
 
@@ -202,7 +207,7 @@ void scoutfs_trans_write_func(struct work_struct *work)
 			if (ret < 0)
 			      s = "clean advance seq";
 		}
-		goto out;
+		goto err;
 	}
 
 	if (sbi->trans_deadline_expired)
@@ -222,11 +227,12 @@ void scoutfs_trans_write_func(struct work_struct *work)
 	      scoutfs_item_write_done(sb) ?:
 	      (s = "advance seq", scoutfs_client_advance_seq(sb, &trans_seq)) ?:
 	      (s = "get log trees", scoutfs_trans_get_log_trees(sb));
-out:
+err:
 	if (ret < 0)
 		scoutfs_err(sb, "critical transaction commit failure: %s, %d",
 			    s, ret);
 
+out:
 	spin_lock(&sbi->trans_write_lock);
 	sbi->trans_write_count++;
 	sbi->trans_write_ret = ret;