diff --git a/utils/src/format.h b/utils/src/format.h
index 909c9894..3c95f421 100644
--- a/utils/src/format.h
+++ b/utils/src/format.h
@@ -241,6 +241,7 @@ struct scoutfs_segment_block {
 #define SCOUTFS_INODE_INDEX_ZONE		1
 #define SCOUTFS_NODE_ZONE			2
 #define SCOUTFS_FS_ZONE				3
+#define SCOUTFS_MAX_ZONE			4 /* power of 2 is efficient */
 
 /* inode index zone */
 #define SCOUTFS_INODE_INDEX_SIZE_TYPE		3
@@ -264,6 +265,8 @@ struct scoutfs_segment_block {
 #define SCOUTFS_FILE_EXTENT_TYPE		7
 #define SCOUTFS_ORPHAN_TYPE			8
 
+#define SCOUTFS_MAX_TYPE			16 /* power of 2 is efficient */
+
 /* XXX don't need these now that we have dlm lock spaces and resources */
 #define SCOUTFS_NET_ADDR_TYPE			254
 #define SCOUTFS_NET_LISTEN_TYPE			255
diff --git a/utils/src/key.c b/utils/src/key.c
index 99730c75..beff97b2 100644
--- a/utils/src/key.c
+++ b/utils/src/key.c
@@ -2,6 +2,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <stdarg.h>
 
 #include "sparse.h"
 #include "util.h"
@@ -9,170 +10,366 @@
 #include "key.h"
 
 /*
- * This is mechanically derived from scoutfs_key_str() in the kernel:
- *  - :.,$s/key->data/key_data/g
- *  - :.,$s/key->key_len/key_len/g
- *  - :.,$s/return snprintf_null(buf, size, /return printf(/g
+ * To print keys we wrap the key snprintf code from the kernel with a
+ * few support functions.  We need a few functions that the kernel has
+ * that we don't provide, then we implement our printing function by
+ * allocating a buffer for the formatted output then just printing it.
+ *
+ * To update the key printing code from the kernel we just need to make
+ * scoutfs_key_str_size() static and replace the snprintf call with the
+ * kernel's "%phN" format with the call to our replacement.
+ *
+ * This is not efficient but this isn't a performant path.
  */
-int print_key(void *key_data, unsigned key_len)
+
+#define min_t(t, a, b) min(a, b)
+
+struct scoutfs_key_buf {
+	void *data;
+	unsigned key_len;
+};
+
+/*
+ * like snprintf(buf, size, "%*phN", nr, bytes) in the kernel, but this
+ * is only called when there's room for the formatted output because
+ * we've already been through once with a 0 buffer to allocate a buffer
+ * for the output.
+ */
+static int snprintf_phN(char *buf, size_t size, unsigned nr, char *bytes)
 {
-	struct scoutfs_inode_key *ikey;
-	u8 zone = 0;
-	u8 type = 0;
-	int len;
+	int ret = 0;
+	int i;
 
-	if (key_data == NULL)
-		return printf("[NULL]");
-
-	if (key_len == 0)
-		return printf("[0 len]");
-
-	zone = *(u8 *)key_data;
-
-	/* handle smaller and unknown zones, fall through to fs types */
-	switch(zone) {
-	case SCOUTFS_INODE_INDEX_ZONE: {
-		struct scoutfs_inode_index_key *ikey = key_data;
-		static char *type_strings[] = {
-			[SCOUTFS_INODE_INDEX_SIZE_TYPE]		= "siz",
-			[SCOUTFS_INODE_INDEX_META_SEQ_TYPE]	= "msq",
-			[SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE]	= "dsq",
-		};
-
-		if (key_len < sizeof(struct scoutfs_inode_index_key))
-			break;
-
-		if (type_strings[ikey->type])
-			return printf("iin.%s.%llu.%u.%llu",
-					     type_strings[ikey->type],
-					     be64_to_cpu(ikey->major),
-					     be32_to_cpu(ikey->minor),
-					     be64_to_cpu(ikey->ino));
-		else
-			return printf("[iin type %u?]",
-					     ikey->type);
-	}
-
-	/* node zone keys start with zone, node, type */
-	case SCOUTFS_NODE_ZONE: {
-		struct scoutfs_free_extent_blkno_key *fkey = key_data;
-
-		static char *type_strings[] = {
-			[SCOUTFS_FREE_EXTENT_BLKNO_TYPE]	= "fno",
-			[SCOUTFS_FREE_EXTENT_BLOCKS_TYPE]	= "fks",
-		};
-
-		switch(fkey->type) {
-		case SCOUTFS_ORPHAN_TYPE: {
-			struct scoutfs_orphan_key *okey = key_data;
-
-			if (key_len < sizeof(struct scoutfs_orphan_key))
-				break;
-			return printf("nod.%llu.orp.%llu",
-					     be64_to_cpu(okey->node_id),
-					     be64_to_cpu(okey->ino));
-		}
-
-		case SCOUTFS_FREE_EXTENT_BLKNO_TYPE:
-		case SCOUTFS_FREE_EXTENT_BLOCKS_TYPE:
-			return printf("nod.%llu.%s.%llu.%llu",
-					     be64_to_cpu(fkey->node_id),
-					     type_strings[fkey->type],
-					     be64_to_cpu(fkey->last_blkno),
-					     be64_to_cpu(fkey->blocks));
-		default:
-			return printf("[nod type %u?]",
-					     fkey->type);
-		}
-	}
-
-	case SCOUTFS_FS_ZONE:
-		break;
-
-	default:
-		return printf("[zone %u?]", zone);
-	}
-
-	/* everything in the fs tree starts with zone, ino, type */
-	ikey = key_data;
-	switch(ikey->type) {
-	case SCOUTFS_INODE_TYPE: {
-		struct scoutfs_inode_key *ikey = key_data;
-
-		if (key_len < sizeof(struct scoutfs_inode_key))
-			break;
-
-		return printf("fs.%llu.ino",
-				     be64_to_cpu(ikey->ino));
-	}
-
-	case SCOUTFS_XATTR_TYPE: {
-		struct scoutfs_xattr_key *xkey = key_data;
-
-		len = (int)key_len - offsetof(struct scoutfs_xattr_key,
-						   name[1]);
-		if (len <= 0)
-			break;
-
-		return printf("fs.%llu.xat.%.*s",
-				     be64_to_cpu(xkey->ino), len, xkey->name);
-	}
-
-	case SCOUTFS_DIRENT_TYPE: {
-		struct scoutfs_dirent_key *dkey = key_data;
-
-		len = (int)key_len - sizeof(struct scoutfs_dirent_key);
-		if (len <= 0)
-			break;
-
-		return printf("fs.%llu.dnt.%.*s",
-				     be64_to_cpu(dkey->ino), len, dkey->name);
-	}
-
-	case SCOUTFS_READDIR_TYPE: {
-		struct scoutfs_readdir_key *rkey = key_data;
-
-		return printf("fs.%llu.rdr.%llu",
-				     be64_to_cpu(rkey->ino),
-				     be64_to_cpu(rkey->pos));
-	}
-
-	case SCOUTFS_LINK_BACKREF_TYPE: {
-		struct scoutfs_link_backref_key *lkey = key_data;
-
-		len = (int)key_len - sizeof(*lkey);
-		if (len <= 0)
-			break;
-
-		return printf("fs.%llu.lbr.%llu.%.*s",
-				     be64_to_cpu(lkey->ino),
-				     be64_to_cpu(lkey->dir_ino), len,
-				     lkey->name);
-	}
-
-	case SCOUTFS_SYMLINK_TYPE: {
-		struct scoutfs_symlink_key *skey = key_data;
-
-		return printf("fs.%llu.sym",
-				     be64_to_cpu(skey->ino));
-	}
-
-	case SCOUTFS_FILE_EXTENT_TYPE: {
-		struct scoutfs_file_extent_key *ekey = key_data;
-
-		return printf("fs.%llu.ext.%llu.%llu.%llu.%x",
-				     be64_to_cpu(ekey->ino),
-				     be64_to_cpu(ekey->last_blk_off),
-				     be64_to_cpu(ekey->last_blkno),
-				     be64_to_cpu(ekey->blocks),
-				     ekey->flags);
-	}
-
-	default:
-		return printf("[fs type %u?]", type);
-	}
-
-	return printf("[fs type %u trunc len %u]",
-			     type, key_len);
+	for (i = 0; i < nr; i++)
+		ret += sprintf(buf + ret, "%02x", bytes[i]);
 
+	return ret;
+}
+
+static char *memchr_inv(char *str, int c, size_t len)
+{
+	while (len--) {
+		if (*(str++) != c)
+			return str - 1;
+	}
+
+	return NULL;
+}
+
+static int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key,
+				size_t size);
+
+void print_key(void *key_data, unsigned key_len)
+{
+	struct scoutfs_key_buf key = {.data = key_data, .key_len = key_len};
+	char *buf;
+	int size;
+
+	size = scoutfs_key_str_size(NULL, &key, 0);
+	if (size > 0) {
+		buf = malloc(size);
+		if (buf) {
+			size = scoutfs_key_str_size(buf, &key, size);
+			if (size > 0)
+				printf("%s", buf);
+			free(buf);
+		}
+	}
+}
+
+/* ------ copied code follows --------- */
+
+#define snprintf_null(buf, size, fmt, args...) \
+	(snprintf((buf), (size), fmt, ##args) + 1)
+
+/*
+ * Store a formatted string representing the key in the buffer.  The key
+ * must be at least min_len to store the data needed by the format at
+ * all.  fmt_len is the length of data that's used by the format.  These
+ * are different because we have badly designed keys with variable
+ * length data that isn't described by the key.  It's assumed from the
+ * length of the key.  Take dirents -- they need to at least have a
+ * dirent struct, but the name length is the rest of the key.
+ *
+ * (XXX And this goes horribly wrong when we pad out dirent keys to max
+ * len to increment at high precision.  We'll never see these items used
+ * by real fs code, but temporary keys and range endpoints can be full
+ * precision and we can try and print them and get very confused.  We
+ * need to rev the format to include explicit lengths.)
+ *
+ * If the format doesn't cover the entire key then we append more
+ * formatting to represent the trailing bytes: runs of zeros compresesd
+ * to _ and then hex output of non-zero bytes.
+ */
+static int snprintf_key(char *buf, size_t size, struct scoutfs_key_buf *key,
+			unsigned min_len, unsigned fmt_len,
+			const char *fmt, ...)
+
+{
+	va_list args;
+	char *data;
+	char *end;
+	int left;
+	int part;
+	int ret;
+	int nr;
+
+	if (key->key_len < min_len)
+		return snprintf_null(buf, size, "[trunc len %u < min %u]",
+				     key->key_len, min_len);
+
+	if (fmt_len == 0)
+		fmt_len = min_len;
+
+	va_start(args, fmt);
+	ret = vsnprintf(buf, size, fmt, args);
+	va_end(args);
+	/* next formatting overwrites null */
+	if (buf) {
+		buf += ret;
+		size -= min_t(int, size, ret);
+	}
+
+	data = key->data + fmt_len;
+	left = key->key_len - fmt_len;
+
+	while (left && (!buf || size > 1)) {
+		/* compress runs of zero bytes to _ */
+		end = memchr_inv(data, 0, left);
+		nr = end ? end - data : left;
+		if (nr) {
+			if (buf) {
+				*(buf++) = '_';
+				size--;
+			}
+			ret++;
+			data += nr;
+			left -= nr;
+			continue;
+		}
+
+		/*
+		 * hex print non-zero bytes.  %ph is limited to 64 bytes
+		 * and is buggy in that it still tries to print to buf
+		 * past size.  (so buf = null, size = 0 crashes instead
+		 * of printing the length of the formatted string.)
+		 */
+		end = memchr(data, 0, left);
+		nr = end ? end - data : left;
+		nr = min(nr, 64);
+
+		if (buf)
+			part = snprintf_phN(buf, size, nr, data);
+		else
+			part = nr * 2;
+		if (buf) {
+			buf += part;
+			size -= min_t(int, size, part);
+		}
+		ret += part;
+
+		data += nr;
+		left -= nr;
+	}
+
+	/* always store and include null */
+	if (buf)
+		*buf = '\0';
+	return ret + 1;
+}
+
+typedef int (*key_printer_t)(char *buf, struct scoutfs_key_buf *key,
+			     size_t size);
+
+static int pr_ino_idx(char *buf, struct scoutfs_key_buf *key, size_t size)
+{
+	static char *type_strings[] = {
+		[SCOUTFS_INODE_INDEX_SIZE_TYPE]		= "siz",
+		[SCOUTFS_INODE_INDEX_META_SEQ_TYPE]	= "msq",
+		[SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE]	= "dsq",
+	};
+	struct scoutfs_inode_index_key *ikey = key->data;
+
+	return snprintf_key(buf, size, key,
+			    sizeof(struct scoutfs_inode_index_key), 0,
+			    "iin.%s.%llu.%u.%llu",
+			    type_strings[ikey->type], be64_to_cpu(ikey->major),
+			    be32_to_cpu(ikey->minor), be64_to_cpu(ikey->ino));
+}
+
+static int pr_free_ext(char *buf, struct scoutfs_key_buf *key, size_t size)
+{
+	struct scoutfs_free_extent_blkno_key *fkey = key->data;
+
+	static char *type_strings[] = {
+		[SCOUTFS_FREE_EXTENT_BLKNO_TYPE]	= "fno",
+		[SCOUTFS_FREE_EXTENT_BLOCKS_TYPE]	= "fks",
+	};
+
+	return snprintf_key(buf, size, key,
+			    sizeof(struct scoutfs_free_extent_blkno_key), 0,
+			    "nod.%llu.%s.%llu.%llu",
+			    be64_to_cpu(fkey->node_id),
+			    type_strings[fkey->type],
+			    be64_to_cpu(fkey->last_blkno),
+			    be64_to_cpu(fkey->blocks));
+}
+
+static int pr_orphan(char *buf, struct scoutfs_key_buf *key, size_t size)
+{
+	struct scoutfs_orphan_key *okey = key->data;
+
+	return snprintf_key(buf, size, key,
+			    sizeof(struct scoutfs_orphan_key), 0,
+			    "nod.%llu.orp.%llu",
+			    be64_to_cpu(okey->node_id),
+			    be64_to_cpu(okey->ino));
+}
+
+static int pr_inode(char *buf, struct scoutfs_key_buf *key, size_t size)
+{
+	struct scoutfs_inode_key *ikey = key->data;
+
+	return snprintf_key(buf, size, key,
+			    sizeof(struct scoutfs_inode_key), 0,
+			    "fs.%llu.ino",
+			    be64_to_cpu(ikey->ino));
+}
+
+static int pr_xattr(char *buf, struct scoutfs_key_buf *key, size_t size)
+{
+	struct scoutfs_xattr_key *xkey = key->data;
+	int len = (int)key->key_len -
+		  offsetof(struct scoutfs_xattr_key, name[1]);
+
+	return snprintf_key(buf, size, key,
+			    sizeof(struct scoutfs_xattr_key), key->key_len,
+			    "fs.%llu.xat.%.*s",
+			    be64_to_cpu(xkey->ino), len, xkey->name);
+}
+
+static int pr_dirent(char *buf, struct scoutfs_key_buf *key, size_t size)
+{
+	struct scoutfs_dirent_key *dkey = key->data;
+	int len = (int)key->key_len - sizeof(struct scoutfs_dirent_key);
+
+	return snprintf_key(buf, size, key,
+			    sizeof(struct scoutfs_dirent_key), key->key_len,
+			    "fs.%llu.dnt.%.*s",
+			    be64_to_cpu(dkey->ino), len, dkey->name);
+}
+
+static int pr_readdir(char *buf, struct scoutfs_key_buf *key, size_t size)
+{
+	struct scoutfs_readdir_key *rkey = key->data;
+
+	return snprintf_key(buf, size, key,
+			    sizeof(struct scoutfs_readdir_key), 0,
+			    "fs.%llu.rdr.%llu",
+			    be64_to_cpu(rkey->ino), be64_to_cpu(rkey->pos));
+}
+
+static int pr_link_backref(char *buf, struct scoutfs_key_buf *key, size_t size)
+{
+	struct scoutfs_link_backref_key *lkey = key->data;
+	int len = (int)key->key_len - sizeof(*lkey);
+
+	return snprintf_key(buf, size, key,
+			    sizeof(struct scoutfs_link_backref_key),
+			    key->key_len,
+			    "fs.%llu.lbr.%llu.%.*s",
+			    be64_to_cpu(lkey->ino), be64_to_cpu(lkey->dir_ino),
+			    len, lkey->name);
+}
+
+static int pr_symlink(char *buf, struct scoutfs_key_buf *key, size_t size)
+{
+	struct scoutfs_symlink_key *skey = key->data;
+
+	return snprintf_key(buf, size, key,
+			    sizeof(struct scoutfs_symlink_key), 0,
+			    "fs.%llu.sym",
+			    be64_to_cpu(skey->ino));
+}
+
+static int pr_file_ext(char *buf, struct scoutfs_key_buf *key, size_t size)
+{
+	struct scoutfs_file_extent_key *ekey = key->data;
+
+	return snprintf_key(buf, size, key,
+			    sizeof(struct scoutfs_file_extent_key), 0,
+			    "fs.%llu.ext.%llu.%llu.%llu.%x",
+			    be64_to_cpu(ekey->ino),
+			    be64_to_cpu(ekey->last_blk_off),
+			    be64_to_cpu(ekey->last_blkno),
+			    be64_to_cpu(ekey->blocks),
+			    ekey->flags);
+}
+
+const static key_printer_t key_printers[SCOUTFS_MAX_ZONE][SCOUTFS_MAX_TYPE] = {
+	[SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_SIZE_TYPE] =
+		pr_ino_idx,
+	[SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_META_SEQ_TYPE] =
+		pr_ino_idx,
+	[SCOUTFS_INODE_INDEX_ZONE][SCOUTFS_INODE_INDEX_DATA_SEQ_TYPE] =
+		pr_ino_idx,
+	[SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLKNO_TYPE] = pr_free_ext,
+	[SCOUTFS_NODE_ZONE][SCOUTFS_FREE_EXTENT_BLOCKS_TYPE] = pr_free_ext,
+	[SCOUTFS_NODE_ZONE][SCOUTFS_ORPHAN_TYPE] = pr_orphan,
+	[SCOUTFS_FS_ZONE][SCOUTFS_INODE_TYPE] = pr_inode,
+	[SCOUTFS_FS_ZONE][SCOUTFS_XATTR_TYPE] = pr_xattr,
+	[SCOUTFS_FS_ZONE][SCOUTFS_DIRENT_TYPE] = pr_dirent,
+	[SCOUTFS_FS_ZONE][SCOUTFS_READDIR_TYPE] = pr_readdir,
+	[SCOUTFS_FS_ZONE][SCOUTFS_LINK_BACKREF_TYPE] = pr_link_backref,
+	[SCOUTFS_FS_ZONE][SCOUTFS_SYMLINK_TYPE] = pr_symlink,
+	[SCOUTFS_FS_ZONE][SCOUTFS_FILE_EXTENT_TYPE] = pr_file_ext,
+};
+
+/*
+ * Write the null-terminated string that describes the key to the
+ * buffer.  The bytes copied (including the null) is returned.  A null
+ * buffer can be used to find the string size without writing anything.
+ *
+ * XXX nonprintable characters in the trace?
+ */
+static int scoutfs_key_str_size(char *buf, struct scoutfs_key_buf *key,
+				size_t size)
+{
+	u8 zone;
+	u8 type;
+
+	if (key == NULL || key->data == NULL)
+		return snprintf_null(buf, size, "[NULL]");
+
+	/* always at least zone, some id, and type */
+	if (key->key_len < (1 + 8 + 1))
+		return snprintf_null(buf, size, "[trunc len %u]", key->key_len);
+
+	zone = *(u8 *)key->data;
+
+	/*
+	 * each zone's keys always start with the same fields that let
+	 * us deref any key to get the type.  We chose a few representative
+	 * keys from each zone to get the type.
+	 */
+	if (zone == SCOUTFS_INODE_INDEX_ZONE) {
+		struct scoutfs_inode_index_key *ikey = key->data;
+		type = ikey->type;
+	} else if (zone == SCOUTFS_NODE_ZONE) {
+		struct scoutfs_free_extent_blkno_key *fkey = key->data;
+		type = fkey->type;
+	} else if (zone == SCOUTFS_FS_ZONE) {
+		struct scoutfs_inode_key *ikey = key->data;
+		type = ikey->type;
+	} else {
+		type = 255;
+	}
+
+	if (zone > SCOUTFS_MAX_ZONE || type > SCOUTFS_MAX_TYPE ||
+	    key_printers[zone][type] == NULL) {
+		return snprintf_null(buf, size, "[unk zone %u type %u]",
+				     zone, type);
+	}
+
+	return key_printers[zone][type](buf, key, size);
 }
diff --git a/utils/src/key.h b/utils/src/key.h
index ebefbf9c..bb51b80f 100644
--- a/utils/src/key.h
+++ b/utils/src/key.h
@@ -1,6 +1,6 @@
 #ifndef _KEY_H_
 #define _KEY_H_
 
-int print_key(void *key_data, unsigned key_len);
+void print_key(void *key_data, unsigned key_len);
 
 #endif