Store stack trace in a separate filed during logging (#2444)

For reasons unclear to me, if the stack trace is appended directly to
the message, the log entry will be lumped together with following logs
on GKE.

Also updated the GKE service account for Nomulus in the manifest so we
can use workload identity just for Nomulus, not other pods on the same
cluster.

jetty/kubernetes/nomulus-deployment.yaml

@@ -13,6 +13,7 @@ spec:
       labels:
         app: nomulus
     spec:
+      serviceAccountName: nomulus
       containers:
       - name: nomulus
         image: gcr.io/GCP_PROJECT/nomulus

util/src/main/java/google/registry/util/GcpJsonFormatter.java

@@ -57,6 +57,16 @@ public class GcpJsonFormatter extends Formatter {
   /** JSON field that contains the content, this will show up as the main entry in a log. */
   private static final String MESSAGE = "message";
 
+  /**
+   * JSON field that contains the stack trace, if any.
+   *
+   * <p>Note that this field is not part of the structured logging that stackdriver understands.
+   * Normally we'd just append the stack trace to the message itself. However, for unclear reasons,
+   * if we do that on GKE, the log entry containing a stack trace will be lumped together with
+   * several following log entries and makes it hard to read.
+   */
+  private static final String STACKTRACE = "stacktrace";
+
   private static final String FILE = "file";
 
   private static final String FUNCTION = "function";
@@ -99,16 +109,17 @@ public class GcpJsonFormatter extends Formatter {
     String severity = severityFor(record.getLevel());
 
     // The rest is mostly lifted from java.util.logging.SimpleFormatter.
-    String stacktrace = "";
+    String throwable = "";
     if (record.getThrown() != null) {
       StringWriter sw = new StringWriter();
-      try (PrintWriter pw = new PrintWriter(sw)) {
-        pw.println();
-        record.getThrown().printStackTrace(pw);
-      }
-      stacktrace = sw.toString();
+      PrintWriter pw = new PrintWriter(sw);
+      pw.println();
+      record.getThrown().printStackTrace(pw);
+      pw.close();
+      throwable = sw.toString();
     }
-    String message = '\n' + record.getMessage() + stacktrace;
+
+    String message = '\n' + formatMessage(record);
 
     String function = "";
     if (record.getSourceClassName() != null) {
@@ -144,6 +155,9 @@ public class GcpJsonFormatter extends Formatter {
     json.put(SEVERITY, severity);
     json.put(SOURCE_LOCATION, sourceLocation);
     json.put(MESSAGE, message);
+    if (!throwable.isEmpty()) {
+      json.put(STACKTRACE, throwable);
+    }
     if (traceId.get() != null) {
       json.put(TRACE, traceId.get());
     }

util/src/test/java/google/registry/util/GcpJsonFormatterTest.java

@@ -109,14 +109,11 @@ class GcpJsonFormatterTest {
     handler.close();
     String output = ostream.toString(StandardCharsets.US_ASCII);
     String prefix =
-        makeJson(
-            "ERROR",
-            108,
-            "testSuccess_withCause",
-            "Something went terribly wrong\\njava.lang.RuntimeException: boom!");
-    // Remove the last three characters (", }, \n) from the template as the actual output contains
+        makeJson("ERROR", 108, "testSuccess_withCause", "Something went terribly wrong");
+    // Remove the last two characters (}, \n) from the template as the actual output contains
     // the full stack trace.
-    prefix = prefix.substring(0, prefix.length() - 3);
+    prefix = prefix.substring(0, prefix.length() - 2);
+    prefix += ",\"stacktrace\":\"\\njava.lang.RuntimeException: boom!\\n";
     assertThat(output).startsWith(prefix);
   }
 
@@ -126,14 +123,11 @@ class GcpJsonFormatterTest {
     handler.close();
     String output = ostream.toString(StandardCharsets.US_ASCII);
     String prefix =
-        makeJson(
-            "ERROR",
-            125,
-            "testSuccess_withStackTrace",
-            "Something is worth checking\\ncom.google.common.flogger.LogSiteStackTrace: FULL");
-    // Remove the last three characters (", }, \n) from the template as the actual output contains
+        makeJson("ERROR", 122, "testSuccess_withStackTrace", "Something is worth checking");
+    // Remove the last three characters (}, \n) from the template as the actual output contains
     // the full stack trace.
-    prefix = prefix.substring(0, prefix.length() - 3);
+    prefix = prefix.substring(0, prefix.length() - 2);
+    prefix += ",\"stacktrace\":\"\\ncom.google.common.flogger.LogSiteStackTrace: FULL\\n";
     assertThat(output).startsWith(prefix);
   }