diff --git a/kmod/src/block.c b/kmod/src/block.c index 53578725..58c12d5b 100644 --- a/kmod/src/block.c +++ b/kmod/src/block.c @@ -200,7 +200,9 @@ static void block_free(struct super_block *sb, struct block_private *bp) else BUG(); - WARN_ON_ONCE(!list_empty(&bp->dirty_entry)); + /* ok to tear down dirty blocks when forcing unmount */ + WARN_ON_ONCE(!scoutfs_forcing_unmount(sb) && !list_empty(&bp->dirty_entry)); + WARN_ON_ONCE(atomic_read(&bp->refcount)); WARN_ON_ONCE(atomic_read(&bp->io_count)); kfree(bp); @@ -485,6 +487,9 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp, sector_t sector; int ret = 0; + if (scoutfs_forcing_unmount(sb)) + return -EIO; + sector = bp->bl.blkno << (SCOUTFS_BLOCK_LG_SHIFT - 9); WARN_ON_ONCE(bp->bl.blkno == U64_MAX); @@ -1148,7 +1153,7 @@ static void sm_block_bio_end_io(struct bio *bio, int err) * only layer that sees the full block buffer so we pass the calculated * crc to the caller for them to check in their context. */ -static int sm_block_io(struct block_device *bdev, int rw, u64 blkno, +static int sm_block_io(struct super_block *sb, struct block_device *bdev, int rw, u64 blkno, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc) { @@ -1160,6 +1165,9 @@ static int sm_block_io(struct block_device *bdev, int rw, u64 blkno, BUILD_BUG_ON(PAGE_SIZE < SCOUTFS_BLOCK_SM_SIZE); + if (scoutfs_forcing_unmount(sb)) + return -EIO; + if (WARN_ON_ONCE(len > SCOUTFS_BLOCK_SM_SIZE) || WARN_ON_ONCE(!(rw & WRITE) && !blk_crc)) return -EINVAL; @@ -1212,14 +1220,14 @@ int scoutfs_block_read_sm(struct super_block *sb, struct scoutfs_block_header *hdr, size_t len, __le32 *blk_crc) { - return sm_block_io(bdev, READ, blkno, hdr, len, blk_crc); + return sm_block_io(sb, bdev, READ, blkno, hdr, len, blk_crc); } int scoutfs_block_write_sm(struct super_block *sb, struct block_device *bdev, u64 blkno, struct scoutfs_block_header *hdr, size_t len) { - return sm_block_io(bdev, WRITE, blkno, hdr, len, NULL); + return sm_block_io(sb, bdev, WRITE, blkno, hdr, len, NULL); } int scoutfs_block_setup(struct super_block *sb) diff --git a/kmod/src/btree.c b/kmod/src/btree.c index 97778352..b9b02696 100644 --- a/kmod/src/btree.c +++ b/kmod/src/btree.c @@ -1101,6 +1101,10 @@ static int btree_walk(struct super_block *sb, if (WARN_ON_ONCE((flags & BTW_DIRTY) && (!alloc || !wri))) return -EINVAL; + /* all ops come through walk and walk calls all reads */ + if (scoutfs_forcing_unmount(sb)) + return -EIO; + scoutfs_inc_counter(sb, btree_walk); restart: diff --git a/kmod/src/client.c b/kmod/src/client.c index fe50b45f..f36b72ab 100644 --- a/kmod/src/client.c +++ b/kmod/src/client.c @@ -466,9 +466,11 @@ static void scoutfs_client_connect_worker(struct work_struct *work) out: /* always have a small delay before retrying to avoid storms */ - if (ret && !atomic_read(&client->shutting_down)) + if (ret && !atomic_read(&client->shutting_down) && + !scoutfs_forcing_unmount(sb)) { queue_delayed_work(client->workq, &client->connect_dwork, msecs_to_jiffies(CLIENT_CONNECT_DELAY_MS)); + } } static scoutfs_net_request_t client_req_funcs[] = { @@ -580,7 +582,7 @@ void scoutfs_client_destroy(struct super_block *sb) if (client == NULL) return; - if (client->server_term != 0) { + if (client->server_term != 0 && !scoutfs_forcing_unmount(sb)) { client->sending_farewell = true; ret = scoutfs_net_submit_request(sb, client->conn, SCOUTFS_NET_CMD_FAREWELL, diff --git a/kmod/src/net.c b/kmod/src/net.c index 9d1fedef..2837307c 100644 --- a/kmod/src/net.c +++ b/kmod/src/net.c @@ -330,6 +330,9 @@ static int submit_send(struct super_block *sb, WARN_ON_ONCE(id == 0 && (flags & SCOUTFS_NET_FLAG_RESPONSE))) return -EINVAL; + if (scoutfs_forcing_unmount(sb)) + return -EIO; + msend = kmalloc(offsetof(struct message_send, nh.data[data_len]), GFP_NOFS); if (!msend) @@ -420,6 +423,16 @@ static int process_request(struct scoutfs_net_connection *conn, mrecv->nh.data, le16_to_cpu(mrecv->nh.data_len)); } +static int call_resp_func(struct super_block *sb, struct scoutfs_net_connection *conn, + scoutfs_net_response_t resp_func, void *resp_data, + void *resp, unsigned int resp_len, int error) +{ + if (resp_func) + return resp_func(sb, conn, resp, resp_len, error, resp_data); + else + return 0; +} + /* * An incoming response finds the queued request and calls its response * function. The response function for a given request will only be @@ -434,7 +447,6 @@ static int process_response(struct scoutfs_net_connection *conn, struct message_send *msend; scoutfs_net_response_t resp_func = NULL; void *resp_data; - int ret = 0; spin_lock(&conn->lock); @@ -449,11 +461,8 @@ static int process_response(struct scoutfs_net_connection *conn, spin_unlock(&conn->lock); - if (resp_func) - ret = resp_func(sb, conn, mrecv->nh.data, - le16_to_cpu(mrecv->nh.data_len), - net_err_to_host(mrecv->nh.error), resp_data); - return ret; + return call_resp_func(sb, conn, resp_func, resp_data, mrecv->nh.data, + le16_to_cpu(mrecv->nh.data_len), net_err_to_host(mrecv->nh.error)); } /* @@ -823,9 +832,15 @@ static void scoutfs_net_destroy_worker(struct work_struct *work) if (conn->listening_conn && conn->notify_down) conn->notify_down(sb, conn, conn->info, conn->rid); - /* free all messages, refactor and complete for forced unmount? */ + /* + * Usually networking is idle and we destroy pending sends, but when forcing unmount + * we can have to wake up waiters by failing pending sends. + */ list_splice_init(&conn->resend_queue, &conn->send_queue); list_for_each_entry_safe(msend, tmp, &conn->send_queue, head) { + if (scoutfs_forcing_unmount(sb)) + call_resp_func(sb, conn, msend->resp_func, msend->resp_data, + NULL, 0, -ECONNABORTED); free_msend(ninf, msend); } diff --git a/kmod/src/server.c b/kmod/src/server.c index 884ec50d..fb353b25 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -284,6 +284,11 @@ static void scoutfs_server_commit_func(struct work_struct *work) down_write(&server->commit_rwsem); + if (scoutfs_forcing_unmount(sb)) { + ret = -EIO; + goto out; + } + /* make sure next avail has sufficient blocks */ ret = scoutfs_alloc_fill_list(sb, &server->alloc, &server->wri, server->other_avail, diff --git a/kmod/src/super.c b/kmod/src/super.c index d5d1063f..dbeb3015 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -283,6 +283,21 @@ static void scoutfs_put_super(struct super_block *sb) sb->s_fs_info = NULL; } +/* + * Record that we're performing a forced unmount. As put_super drives + * destruction of the filesystem we won't issue more network or storage + * operations because we assume that they'll hang. Pending operations + * can return errors when it's possible to do so. We may be racing with + * pending operations which can't be canceled. + */ +static void scoutfs_umount_begin(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + scoutfs_warn(sb, "forcing unmount, can return errors and lose unsynced data"); + sbi->forced_unmount = true; +} + static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .drop_inode = scoutfs_drop_inode, @@ -292,6 +307,7 @@ static const struct super_operations scoutfs_super_ops = { .statfs = scoutfs_statfs, .show_options = scoutfs_show_options, .put_super = scoutfs_put_super, + .umount_begin = scoutfs_umount_begin, }; /* diff --git a/kmod/src/super.h b/kmod/src/super.h index 820ee6b8..cbbbc3a0 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -87,6 +87,8 @@ struct scoutfs_sb_info { struct dentry *debug_root; + bool forced_unmount; + unsigned long corruption_messages_once[SC_NR_LONGS]; }; @@ -107,6 +109,13 @@ static inline bool SCOUTFS_IS_META_BDEV(struct scoutfs_super_block *super_block) #define SCOUTFS_META_BDEV_MODE (FMODE_READ | FMODE_WRITE | FMODE_EXCL) +static inline bool scoutfs_forcing_unmount(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + + return sbi->forced_unmount; +} + /* * A small string embedded in messages that's used to identify a * specific mount. It's the three most significant bytes of the fsid diff --git a/kmod/src/trans.c b/kmod/src/trans.c index 186239d4..07eea0fa 100644 --- a/kmod/src/trans.c +++ b/kmod/src/trans.c @@ -185,6 +185,11 @@ void scoutfs_trans_write_func(struct work_struct *work) wait_event(sbi->trans_hold_wq, drained_holders(tri)); + if (scoutfs_forcing_unmount(sb)) { + ret = -EIO; + goto out; + } + trace_scoutfs_trans_write_func(sb, scoutfs_block_writer_dirty_bytes(sb, &tri->wri)); @@ -202,7 +207,7 @@ void scoutfs_trans_write_func(struct work_struct *work) if (ret < 0) s = "clean advance seq"; } - goto out; + goto err; } if (sbi->trans_deadline_expired) @@ -222,11 +227,12 @@ void scoutfs_trans_write_func(struct work_struct *work) scoutfs_item_write_done(sb) ?: (s = "advance seq", scoutfs_client_advance_seq(sb, &trans_seq)) ?: (s = "get log trees", scoutfs_trans_get_log_trees(sb)); -out: +err: if (ret < 0) scoutfs_err(sb, "critical transaction commit failure: %s, %d", s, ret); +out: spin_lock(&sbi->trans_write_lock); sbi->trans_write_count++; sbi->trans_write_ret = ret;