Add check for build_environment flag in updateReservedListCommand and updatePremiumListCommand (#2317)

* Add check for build_environment flag in updateReservedListCommand

* Do the same for premium list

.github/workflows/codeql.yml

@@ -33,7 +33,7 @@ jobs:
       uses: actions/setup-java@v3
       with:
         distribution: 'temurin'
-        java-version: '11'
+        java-version: '17'
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

build.gradle

@@ -211,6 +211,30 @@ allprojects {
         options.fork = true
         options.forkOptions.executable =
           file("${System.env.JAVA_HOME}/bin/javac")
+        options.compilerArgs = ["--add-exports",
+                                "jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
+                                "--add-exports",
+                                "jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED",
+                                "--add-exports",
+                                "jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
+                                "--add-exports",
+                                "jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
+                                "--add-exports",
+                                "jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
+                                "--add-exports",
+                                "jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
+                                "--add-exports",
+                                "jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
+                                "--add-exports",
+                                "jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED"]
+        options.forkOptions.jvmArgs = ["-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
+                                       "-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
+                                       "-J--add-exports=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED",
+                                       "-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
+                                       "-J--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
+                                       "-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
+                                       "-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
+                                       "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED"]
       }
     }
   }
@@ -358,16 +382,11 @@ subprojects {
 
   apply from: "${rootDir.path}/java_common.gradle"
 
-  if (!rootProject.enableCrossReferencing.toBoolean()) {
-    compileJava {
-      // TODO: Remove this once we migrate off of AppEngine.
-      options.release = 8
-    }
-    compileTestJava {
-      // TODO: Remove this once we migrate off of AppEngine.
-      options.release = 8
-    }
-  }
+  // When changing Java version here, be sure to update BEAM Java runtime:
+  // in core/build.gradle, search for `flex-template-base-image` and update
+  // the parameter value.
+  sourceCompatibility = JavaVersion.VERSION_17
+  targetCompatibility = JavaVersion.VERSION_17
 
   project.tasks.test.dependsOn runPresubmits
 
@@ -479,10 +498,14 @@ def createGetBuildSrcDirectDepsTask(outputFileName) {
 
 rootProject.ext {
   invokeJavaDiffFormatScript = { action ->
-    println("JAVA_HOME=${System.env.JAVA_HOME}")
-    println("PATH=${System.env.PATH}")
-    println(ext.execInBash("type java", "${rootDir}"))
-    println(ext.execInBash("java -version", "${rootDir}"))
+    def javaHome = project.findProperty('org.gradle.java.home')
+    def javaBin
+    if (javaHome != null) {
+      javaBin = "$javaHome/bin/java"
+    } else {
+      javaBin = ext.execInBash("which java", rootDir)
+    }
+    println("Running the formatting tool with $javaBin")
     def scriptDir = rootDir.path.endsWith('buildSrc')
             ? "${rootDir}/../java-format"
             : "${rootDir}/java-format"
@@ -493,7 +516,7 @@ rootProject.ext {
     def pythonExe = getPythonExecutable()
 
     return ext.execInBash(
-        "PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
+        "JAVA=${javaBin} PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
   }
 }
 

config/presubmits.py

@@ -87,7 +87,7 @@ PRESUBMITS = {
     PresubmitCheck(
         r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
         ("java", "js", "soy", "sql", "py", "sh", "gradle", "ts"), {
-            ".git", "/build/", "/generated/", "/generated_tests/",
+            ".git", "/build/", "/bin/generated-sources/", "/bin/generated-test-sources/",
             "node_modules/", "LoggerConfig.java", "registrar_bin.",
             "registrar_dbg.", "google-java-format-diff.py",
             "nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"

console-webapp/src/app/app.component.ts

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { AfterViewInit, Component, ViewChild } from '@angular/core';
+import { AfterViewInit, Component, ViewChild, effect } from '@angular/core';
 import { RegistrarService } from './registrar/registrar.service';
 import { UserDataService } from './shared/services/userData.service';
 import { GlobalLoaderService } from './shared/services/globalLoader.service';
@@ -36,11 +36,13 @@ export class AppComponent implements AfterViewInit {
     protected globalLoader: GlobalLoaderService,
     protected router: Router
   ) {
-    registrarService.activeRegistrarIdChange.subscribe(() => {
-      this.renderRouter = false;
-      setTimeout(() => {
-        this.renderRouter = true;
-      }, 400);
+    effect(() => {
+      if (registrarService.registrarId()) {
+        this.renderRouter = false;
+        setTimeout(() => {
+          this.renderRouter = true;
+        }, 400);
+      }
     });
   }
 

console-webapp/src/app/domains/domainList.service.ts

@@ -52,7 +52,7 @@ export class DomainListService {
   ) {
     return this.backendService
       .getDomains(
-        this.registrarService.activeRegistrarId,
+        this.registrarService.registrarId(),
         this.checkpointTime,
         pageNumber,
         resultsPerPage,

console-webapp/src/app/home/widgets/billingWidget.component.ts

@@ -23,8 +23,10 @@ export class BillingWidgetComponent {
   constructor(public registrarService: RegistrarService) {}
 
   public get driveFolderUrl(): string {
-    if (this.registrarService?.registrar.driveFolderId) {
-      return `https://drive.google.com/drive/folders/${this.registrarService?.registrar.driveFolderId}`;
+    if (this.registrarService.registrar()?.driveFolderId) {
+      return `https://drive.google.com/drive/folders/${
+        this.registrarService.registrar()?.driveFolderId
+      }`;
     }
     return '';
   }

console-webapp/src/app/registrar/emptyRegistrar.component.ts

@@ -12,9 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Component } from '@angular/core';
+import { Component, effect } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
-import { Subscription } from 'rxjs';
 
 import { RegistrarService } from './registrar.service';
 
@@ -24,22 +23,15 @@ import { RegistrarService } from './registrar.service';
   styleUrls: ['./emptyRegistrar.component.scss'],
 })
 export class EmptyRegistrar {
-  private registrarIdChangeSubscription?: Subscription;
-
   constructor(
     private route: ActivatedRoute,
     protected registrarService: RegistrarService,
     private router: Router
   ) {
-    this.registrarIdChangeSubscription =
-      registrarService.activeRegistrarIdChange.subscribe((newRegistrarId) => {
-        if (newRegistrarId) {
-          this.router.navigate([this.route.snapshot.paramMap.get('nextUrl')]);
-        }
-      });
-  }
-
-  ngOnDestroy() {
-    this.registrarIdChangeSubscription?.unsubscribe();
+    effect(() => {
+      if (registrarService.registrarId()) {
+        this.router.navigate([this.route.snapshot.paramMap.get('nextUrl')]);
+      }
+    });
   }
 }

console-webapp/src/app/registrar/registrar.guard.ts

@@ -34,7 +34,7 @@ export class RegistrarGuard {
     _: ActivatedRouteSnapshot,
     state: RouterStateSnapshot
   ): Promise<boolean> | boolean {
-    if (this.registrarService.activeRegistrarId) {
+    if (this.registrarService.registrarId()) {
       return true;
     }
     return this.router.navigate([

console-webapp/src/app/registrar/registrar.service.ts

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Injectable } from '@angular/core';
-import { Observable, Subject, tap } from 'rxjs';
+import { Injectable, computed, signal } from '@angular/core';
+import { Observable, tap } from 'rxjs';
 
 import { BackendService } from '../shared/services/backend.service';
 import {
@@ -52,9 +52,11 @@ export interface Registrar {
   providedIn: 'root',
 })
 export class RegistrarService implements GlobalLoader {
-  activeRegistrarId: string = '';
-  registrars: Registrar[] = [];
-  activeRegistrarIdChange: Subject<string> = new Subject<string>();
+  registrarId = signal<string>('');
+  registrars = signal<Registrar[]>([]);
+  registrar = computed<Registrar | undefined>(() =>
+    this.registrars().find((r) => r.registrarId === this.registrarId())
+  );
 
   constructor(
     private backend: BackendService,
@@ -67,22 +69,15 @@ export class RegistrarService implements GlobalLoader {
     this.globalLoader.startGlobalLoader(this);
   }
 
-  public get registrar(): Registrar {
-    return this.registrars.filter(
-      (r) => r.registrarId === this.activeRegistrarId
-    )[0];
-  }
-
   public updateSelectedRegistrar(registrarId: string) {
-    this.activeRegistrarId = registrarId;
-    this.activeRegistrarIdChange.next(registrarId);
+    this.registrarId.set(registrarId);
   }
 
   public loadRegistrars(): Observable<Registrar[]> {
     return this.backend.getRegistrars().pipe(
       tap((registrars) => {
         if (registrars) {
-          this.registrars = registrars;
+          this.registrars.set(registrars);
         }
       })
     );

console-webapp/src/app/registrar/registrarSelector.component.html

@@ -5,20 +5,20 @@
     routerLinkActive="active"
     *ngIf="isMobile; else desktop"
   >
-    {{ registrarService.activeRegistrarId || "Select registrar" }}
+    {{ registrarService.registrarId() || "Select registrar" }}
     <mat-icon>open_in_new</mat-icon>
   </button>
   <ng-template #desktop>
     <mat-form-field class="mat-form-field-density-5" appearance="fill">
       <mat-label>Registrar</mat-label>
       <mat-select
-        [ngModel]="registrarService.activeRegistrarId"
+        [ngModel]="registrarService.registrarId()"
         (selectionChange)="
           registrarService.updateSelectedRegistrar($event.value)
         "
       >
         <mat-option
-          *ngFor="let registrar of registrarService.registrars"
+          *ngFor="let registrar of registrarService.registrars()"
           [value]="registrar.registrarId"
         >
           {{ registrar.registrarId }}

console-webapp/src/app/registrar/registrarsTable.component.ts

@@ -87,7 +87,7 @@ export class RegistrarComponent {
 
   constructor(protected registrarService: RegistrarService) {
     this.dataSource = new MatTableDataSource<Registrar>(
-      registrarService.registrars
+      registrarService.registrars()
     );
   }
 

console-webapp/src/app/settings/contact/contact.component.html

@@ -1,41 +1,41 @@
-<div *ngIf="loading" class="contact__loading">
+@if (loading) {
+<div class="contact__loading">
   <mat-progress-bar mode="indeterminate"></mat-progress-bar>
 </div>
-<div *ngIf="!loading">
-  <div
-    class="contact__empty-contacts"
-    *ngIf="contactService.contacts.length === 0"
-  >
+} @else {
+<div>
+  @if (contactService.contacts().length === 0) {
+  <div class="contact__empty-contacts">
     <mat-icon class="contact__empty-contacts-icon secondary-text"
       >apps_outage</mat-icon
     >
     <h1>No contacts found</h1>
   </div>
-  <div *ngFor="let group of groupedData">
-    <div class="contact__cards-wrapper" *ngIf="group.contacts.length">
-      <h3>{{ group.label }}s</h3>
-      <mat-divider></mat-divider>
-      <div class="contact__cards">
-        <mat-card class="contact__card" *ngFor="let contact of group.contacts">
-          <mat-card-title>{{ contact.name }}</mat-card-title>
-          <p *ngIf="contact.phoneNumber">{{ contact.phoneNumber }}</p>
-          <p *ngIf="contact.emailAddress">{{ contact.emailAddress }}</p>
-          <mat-card-actions class="contact__card-actions">
-            <button
-              mat-button
-              color="primary"
-              (click)="openDetails($event, contact)"
-            >
-              <mat-icon>edit</mat-icon>Edit
-            </button>
-            <button mat-button color="accent" (click)="deleteContact(contact)">
-              <mat-icon>delete</mat-icon>Delete
-            </button>
-          </mat-card-actions>
-        </mat-card>
-      </div>
+  } @else { @for (group of groupedContacts(); track group.emailAddress) {
+  <div class="contact__cards-wrapper">
+    <h3>{{ group.label }}s</h3>
+    <mat-divider></mat-divider>
+    <div class="contact__cards">
+      <mat-card class="contact__card" *ngFor="let contact of group.contacts">
+        <mat-card-title>{{ contact.name }}</mat-card-title>
+        <p *ngIf="contact.phoneNumber">{{ contact.phoneNumber }}</p>
+        <p *ngIf="contact.emailAddress">{{ contact.emailAddress }}</p>
+        <mat-card-actions class="contact__card-actions">
+          <button
+            mat-button
+            color="primary"
+            (click)="openDetails($event, contact)"
+          >
+            <mat-icon>edit</mat-icon>Edit
+          </button>
+          <button mat-button color="accent" (click)="deleteContact(contact)">
+            <mat-icon>delete</mat-icon>Delete
+          </button>
+        </mat-card-actions>
+      </mat-card>
     </div>
   </div>
+  } }
   <div class="contact__actions">
     <button mat-raised-button color="primary" (click)="openCreateNew($event)">
       <mat-icon>add</mat-icon>Create a Contact
@@ -45,3 +45,4 @@
     #contactDetailsWrapper
   ></app-dialog-bottom-sheet-wrapper>
 </div>
+}

console-webapp/src/app/settings/contact/contact.component.ts

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Component, ViewChild } from '@angular/core';
+import { Component, ViewChild, computed } from '@angular/core';
 import { Contact, ContactService } from './contact.service';
 import { HttpErrorResponse } from '@angular/common/http';
 import { MatSnackBar } from '@angular/material/snack-bar';
@@ -70,9 +70,9 @@ export class ContactDetailsDialogComponent implements DialogBottomSheetContent {
 
   init(params: ContactDetailsParams) {
     this.params = params;
-    this.contactIndex = this.contactService.contacts.findIndex(
-      (c) => c === params.data.contact
-    );
+    this.contactIndex = this.contactService
+      .contacts()
+      .findIndex((c) => c === params.data.contact);
     this.contact = JSON.parse(JSON.stringify(params.data.contact));
   }
 
@@ -115,6 +115,16 @@ export class ContactDetailsDialogComponent implements DialogBottomSheetContent {
 })
 export default class ContactComponent {
   public static PATH = 'contact';
+  public groupedContacts = computed(() => {
+    return this.contactService.contacts().reduce((acc, contact) => {
+      contact.types.forEach((contactType) => {
+        acc
+          .find((group: GroupedContacts) => group.value === contactType)
+          ?.contacts.push(contact);
+      });
+      return acc;
+    }, JSON.parse(JSON.stringify(contactTypes)));
+  });
 
   @ViewChild('contactDetailsWrapper')
   detailsComponentWrapper!: DialogBottomSheetWrapper;
@@ -131,17 +141,6 @@ export default class ContactComponent {
     });
   }
 
-  public get groupedData() {
-    return this.contactService.contacts?.reduce((acc, contact) => {
-      contact.types.forEach((type) => {
-        acc
-          .find((group: GroupedContacts) => group.value === type)
-          ?.contacts.push(contact);
-      });
-      return acc;
-    }, JSON.parse(JSON.stringify(contactTypes)));
-  }
-
   deleteContact(contact: Contact) {
     if (confirm(`Please confirm contact ${contact.name} delete`)) {
       this.contactService.deleteContact(contact).subscribe({

console-webapp/src/app/settings/contact/contact.service.ts

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Injectable } from '@angular/core';
+import { Injectable, signal } from '@angular/core';
 import { Observable, tap } from 'rxjs';
 import { RegistrarService } from 'src/app/registrar/registrar.service';
 import { BackendService } from 'src/app/shared/services/backend.service';
@@ -33,7 +33,7 @@ export interface Contact {
   providedIn: 'root',
 })
 export class ContactService {
-  contacts: Contact[] = [];
+  contacts = signal<Contact[]>([]);
 
   constructor(
     private backend: BackendService,
@@ -42,39 +42,37 @@ export class ContactService {
 
   // TODO: Come up with a better handling for registrarId
   fetchContacts(): Observable<Contact[]> {
-    return this.backend
-      .getContacts(this.registrarService.activeRegistrarId)
-      .pipe(
-        tap((contacts = []) => {
-          this.contacts = contacts;
-        })
-      );
+    return this.backend.getContacts(this.registrarService.registrarId()).pipe(
+      tap((contacts = []) => {
+        this.contacts.set(contacts);
+      })
+    );
   }
 
   saveContacts(contacts: Contact[]): Observable<Contact[]> {
     return this.backend
-      .postContacts(this.registrarService.activeRegistrarId, contacts)
+      .postContacts(this.registrarService.registrarId(), contacts)
       .pipe(
         tap((_) => {
-          this.contacts = contacts;
+          this.contacts.set(contacts);
         })
       );
   }
 
   updateContact(index: number, contact: Contact) {
-    const newContacts = this.contacts.map((c, i) =>
+    const newContacts = this.contacts().map((c, i) =>
       i === index ? contact : c
     );
     return this.saveContacts(newContacts);
   }
 
   addContact(contact: Contact) {
-    const newContacts = this.contacts.concat([contact]);
+    const newContacts = this.contacts().concat([contact]);
     return this.saveContacts(newContacts);
   }
 
   deleteContact(contact: Contact) {
-    const newContacts = this.contacts.filter((c) => c !== contact);
+    const newContacts = this.contacts().filter((c) => c !== contact);
     return this.saveContacts(newContacts);
   }
 }

console-webapp/src/app/settings/security/security.component.ts

@@ -40,7 +40,7 @@ export default class SecurityComponent {
     private _snackBar: MatSnackBar,
     public registrarService: RegistrarService
   ) {
-    this.dataSource = apiToUiConverter(this.registrarService.registrar);
+    this.dataSource = apiToUiConverter(this.registrarService.registrar());
   }
 
   enableEdit() {
@@ -76,6 +76,6 @@ export default class SecurityComponent {
   }
 
   resetDataSource() {
-    this.dataSource = apiToUiConverter(this.registrarService.registrar);
+    this.dataSource = apiToUiConverter(this.registrarService.registrar());
   }
 }

console-webapp/src/app/settings/security/security.service.ts

@@ -64,7 +64,7 @@ export class SecurityService {
   saveChanges(newSecuritySettings: SecuritySettings) {
     return this.backend
       .postSecuritySettings(
-        this.registrarService.activeRegistrarId,
+        this.registrarService.registrarId(),
         uiToApiConverter(newSecuritySettings)
       )
       .pipe(

core/Dockerfile

@@ -1,3 +1,3 @@
-FROM gcr.io/distroless/java:debug
+FROM gcr.io/distroless/java17-debian11:debug
 ADD build/libs/nomulus.jar /nomulus.jar
 ENTRYPOINT ["/usr/bin/java", "-jar", "/nomulus.jar"]

core/build.gradle

@@ -776,7 +776,7 @@ if (environment == 'alpha') {
             gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \
             --image-gcr-path ${imageName}:live \
             --sdk-language JAVA \
-            --flex-template-base-image JAVA11 \
+            --flex-template-base-image JAVA17 \
             --metadata-file ${projectDir}/src/main/resources/${metaData} \
             --jar ${uberJarName} \
             --env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \

core/src/main/java/google/registry/beam/billing/InvoicingPipeline.java

@@ -163,7 +163,12 @@ public class InvoicingPipeline implements Serializable {
     }
   }
 
-  /** Saves the billing events to a single overall invoice CSV file. */
+  /**
+   * Saves the billing events to a single overall invoice CSV file. TextIO always produces the file
+   * of type text/plain, which we then update to desired text/csv before sending an email to billing
+   * team {@link google.registry.reporting.billing.BillingEmailUtils#emailOverallInvoice()
+   * emailOverallInvoice}
+   */
   static void saveInvoiceCsv(
       PCollection<google.registry.beam.billing.BillingEvent> billingEvents,
       InvoicingPipelineOptions options) {

core/src/main/java/google/registry/bsa/BsaRefreshAction.java

@@ -54,7 +54,7 @@ public class BsaRefreshAction implements Runnable {
   private final GcsClient gcsClient;
   private final BsaReportSender bsaReportSender;
   private final int transactionBatchSize;
-  private final Duration domainTxnMaxDuration;
+  private final Duration domainCreateTxnCommitTimeLag;
   private final BsaLock bsaLock;
   private final Clock clock;
   private final Response response;
@@ -65,7 +65,7 @@ public class BsaRefreshAction implements Runnable {
       GcsClient gcsClient,
       BsaReportSender bsaReportSender,
       @Config("bsaTxnBatchSize") int transactionBatchSize,
-      @Config("domainCreateTxnCommitTimeLag") Duration domainTxnMaxDuration,
+      @Config("domainCreateTxnCommitTimeLag") Duration domainCreateTxnCommitTimeLag,
       BsaLock bsaLock,
       Clock clock,
       Response response) {
@@ -73,7 +73,7 @@ public class BsaRefreshAction implements Runnable {
     this.gcsClient = gcsClient;
     this.bsaReportSender = bsaReportSender;
     this.transactionBatchSize = transactionBatchSize;
-    this.domainTxnMaxDuration = domainTxnMaxDuration;
+    this.domainCreateTxnCommitTimeLag = domainCreateTxnCommitTimeLag;
     this.bsaLock = bsaLock;
     this.clock = clock;
     this.response = response;
@@ -109,7 +109,10 @@ public class BsaRefreshAction implements Runnable {
     RefreshSchedule schedule = maybeSchedule.get();
     DomainsRefresher refresher =
         new DomainsRefresher(
-            schedule.prevRefreshTime(), clock.nowUtc(), domainTxnMaxDuration, transactionBatchSize);
+            schedule.prevRefreshTime(),
+            clock.nowUtc(),
+            domainCreateTxnCommitTimeLag,
+            transactionBatchSize);
     switch (schedule.stage()) {
       case CHECK_FOR_CHANGES:
         ImmutableList<UnblockableDomainChange> blockabilityChanges =
@@ -132,10 +135,12 @@ public class BsaRefreshAction implements Runnable {
       case UPLOAD_REMOVALS:
         try (Stream<UnblockableDomainChange> changes =
             gcsClient.readRefreshChanges(schedule.jobName())) {
+          // Unblockables with changes in REASON are removed then added back. That is why they are
+          // included in this stage.
           Optional<String> report =
               JsonSerializations.toUnblockableDomainsRemovalReport(
                   changes
-                      .filter(UnblockableDomainChange::isDelete)
+                      .filter(UnblockableDomainChange::isChangeOrDelete)
                       .map(UnblockableDomainChange::domainName));
           if (report.isPresent()) {
             gcsClient.logRemovedUnblockableDomainsReport(
@@ -153,12 +158,12 @@ public class BsaRefreshAction implements Runnable {
           Optional<String> report =
               JsonSerializations.toUnblockableDomainsReport(
                   changes
-                      .filter(UnblockableDomainChange::isAddOrChange)
+                      .filter(UnblockableDomainChange::isNewOrChange)
                       .map(UnblockableDomainChange::newValue));
           if (report.isPresent()) {
-            gcsClient.logRemovedUnblockableDomainsReport(
+            gcsClient.logAddedUnblockableDomainsReport(
                 schedule.jobName(), LINE_SPLITTER.splitToStream(report.get()));
-            bsaReportSender.removeUnblockableDomainsUpdates(report.get());
+            bsaReportSender.addUnblockableDomainsUpdates(report.get());
           } else {
             logger.atInfo().log("No new Unblockable domains to add.");
           }

core/src/main/java/google/registry/bsa/BsaStringUtils.java

@@ -30,7 +30,7 @@ public class BsaStringUtils {
   public static final Splitter LINE_SPLITTER = Splitter.on('\n');
 
   public static String getLabelInDomain(String domainName) {
-    List<String> parts = DOMAIN_SPLITTER.limit(1).splitToList(domainName);
+    List<String> parts = DOMAIN_SPLITTER.splitToList(domainName);
     checkArgument(!parts.isEmpty(), "Not a valid domain: [%s]", domainName);
     return parts.get(0);
   }

core/src/main/java/google/registry/bsa/BsaTransactions.java

@@ -21,6 +21,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import google.registry.persistence.transaction.TransactionManager.ThrowingRunnable;
 import java.util.concurrent.Callable;
 
 /**
@@ -39,10 +40,16 @@ public final class BsaTransactions {
     return tm().transact(work, TRANSACTION_REPEATABLE_READ);
   }
 
+  public static void bsaTransact(ThrowingRunnable work) {
+    verify(!isInTransaction(), "May only be used for top-level transactions.");
+    tm().transact(work, TRANSACTION_REPEATABLE_READ);
+  }
+
   @CanIgnoreReturnValue
   public static <T> T bsaQuery(Callable<T> work) {
     verify(!isInTransaction(), "May only be used for top-level transactions.");
-    return replicaTm().transact(work, TRANSACTION_REPEATABLE_READ);
+    // TRANSACTION_REPEATABLE_READ is default on replica.
+    return replicaTm().transact(work);
   }
 
   private BsaTransactions() {}

core/src/main/java/google/registry/bsa/GcsClient.java

@@ -161,7 +161,7 @@ public class GcsClient {
   }
 
   Stream<UnblockableDomainChange> readRefreshChanges(String jobName) {
-    BlobId blobId = getBlobId(jobName, UNBLOCKABLE_DOMAINS_FILE);
+    BlobId blobId = getBlobId(jobName, REFRESHED_UNBLOCKABLE_DOMAINS_FILE);
     return readStream(blobId).map(UnblockableDomainChange::deserialize);
   }
 

core/src/main/java/google/registry/bsa/UploadBsaUnavailableDomainsAction.java

@@ -80,7 +80,7 @@ public class UploadBsaUnavailableDomainsAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  private static final int BATCH_SIZE = 50000;
+  private static final int BATCH_SIZE = 40000;
 
   Clock clock;
 

core/src/main/java/google/registry/bsa/api/UnblockableDomain.java

@@ -28,9 +28,9 @@ import java.util.List;
 // TODO(1/15/2024): rename to UnblockableDomain.
 @AutoValue
 public abstract class UnblockableDomain {
-  abstract String domainName();
+  public abstract String domainName();
 
-  abstract Reason reason();
+  public abstract Reason reason();
 
   /** Reasons why a valid domain name cannot be blocked. */
   public enum Reason {

core/src/main/java/google/registry/bsa/api/UnblockableDomainChange.java

@@ -52,12 +52,16 @@ public abstract class UnblockableDomainChange {
     return UnblockableDomain.of(unblockable().domainName(), newReason().get());
   }
 
-  public boolean isAddOrChange() {
+  public boolean isNewOrChange() {
     return newReason().isPresent();
   }
 
+  public boolean isChangeOrDelete() {
+    return !isNew();
+  }
+
   public boolean isDelete() {
-    return !this.isAddOrChange();
+    return !this.isNewOrChange();
   }
 
   public boolean isNew() {

core/src/main/java/google/registry/bsa/persistence/BsaUnblockableDomain.java

@@ -105,6 +105,10 @@ class BsaUnblockableDomain {
     return new BsaUnblockableDomain(parts.get(0), parts.get(1), reason);
   }
 
+  static BsaUnblockableDomain of(UnblockableDomain unblockable) {
+    return of(unblockable.domainName(), Reason.valueOf(unblockable.reason().name()));
+  }
+
   static VKey<BsaUnblockableDomain> vKey(String domainName) {
     ImmutableList<String> parts = ImmutableList.copyOf(DOMAIN_SPLITTER.splitToList(domainName));
     verify(parts.size() == 2, "Invalid domain name: %s", domainName);

core/src/main/java/google/registry/bsa/persistence/DomainsRefresher.java

@@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.bsa.BsaTransactions.bsaQuery;
+import static google.registry.bsa.BsaTransactions.bsaTransact;
 import static google.registry.bsa.ReservedDomainsUtils.getAllReservedNames;
 import static google.registry.bsa.ReservedDomainsUtils.isReservedDomain;
 import static google.registry.bsa.persistence.Queries.batchReadUnblockables;
@@ -44,6 +45,7 @@ import google.registry.model.domain.Domain;
 import google.registry.model.tld.Tld;
 import google.registry.model.tld.Tld.TldType;
 import google.registry.util.BatchedStreams;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -107,6 +109,8 @@ public final class DomainsRefresher {
 
     ImmutableSet<String> upgradedDomains =
         upgrades.stream().map(UnblockableDomainChange::domainName).collect(toImmutableSet());
+    // Upgrades are collected in its own txn after downgrades so need reconciliation. Conflicts are
+    // unlikely but possible: domains may be recreated or reserved names added during the interval.
     ImmutableList<UnblockableDomainChange> trueDowngrades =
         downgrades.stream()
             .filter(c -> !upgradedDomains.contains(c.domainName()))
@@ -198,22 +202,50 @@ public final class DomainsRefresher {
   }
 
   public ImmutableList<UnblockableDomainChange> getNewUnblockables() {
-    return bsaQuery(
-        () -> {
-          // TODO(weiminyu): both methods below use `queryBsaLabelByLabels`. Should combine.
-          ImmutableSet<String> newCreated = getNewlyCreatedUnblockables(prevRefreshStartTime, now);
-          ImmutableSet<String> newReserved =
-              getNewlyReservedUnblockables(now, transactionBatchSize);
-          SetView<String> reservedNotCreated = Sets.difference(newReserved, newCreated);
-          return Streams.concat(
-                  newCreated.stream()
-                      .map(name -> UnblockableDomain.of(name, Reason.REGISTERED))
-                      .map(UnblockableDomainChange::ofNew),
-                  reservedNotCreated.stream()
-                      .map(name -> UnblockableDomain.of(name, Reason.RESERVED))
-                      .map(UnblockableDomainChange::ofNew))
-              .collect(toImmutableList());
-        });
+    // TODO(weiminyu): both methods below use `queryBsaLabelByLabels`. Should combine in a single
+    // query.
+    HashSet<String> newCreated =
+        Sets.newHashSet(bsaQuery(() -> getNewlyCreatedUnblockables(prevRefreshStartTime, now)));
+    // We cannot identify new reserved unblockables so must look at all of them. There are not many
+    // of these.
+    HashSet<String> allReserved =
+        Sets.newHashSet(bsaQuery(() -> getAllReservedUnblockables(now, transactionBatchSize)));
+    HashSet<String> reservedNotCreated = Sets.newHashSet(Sets.difference(allReserved, newCreated));
+
+    ImmutableList.Builder<UnblockableDomainChange> changes = new ImmutableList.Builder<>();
+    ImmutableList<BsaUnblockableDomain> batch;
+    Optional<BsaUnblockableDomain> lastRead = Optional.empty();
+    do {
+      batch = batchReadUnblockables(lastRead, transactionBatchSize);
+      if (batch.isEmpty()) {
+        break;
+      }
+      lastRead = Optional.of(batch.get(batch.size() - 1));
+      for (BsaUnblockableDomain unblockable : batch) {
+        String domainName = unblockable.domainName();
+        if (unblockable.reason.equals(BsaUnblockableDomain.Reason.REGISTERED)) {
+          newCreated.remove(domainName);
+          reservedNotCreated.remove(domainName);
+        } else {
+          reservedNotCreated.remove(domainName);
+          if (newCreated.remove(domainName)) {
+            changes.add(
+                UnblockableDomainChange.ofChanged(
+                    unblockable.toUnblockableDomain(), Reason.REGISTERED));
+          }
+        }
+      }
+    } while (batch.size() == transactionBatchSize);
+
+    Streams.concat(
+            newCreated.stream()
+                .map(name -> UnblockableDomain.of(name, Reason.REGISTERED))
+                .map(UnblockableDomainChange::ofNew),
+            reservedNotCreated.stream()
+                .map(name -> UnblockableDomain.of(name, Reason.RESERVED))
+                .map(UnblockableDomainChange::ofNew))
+        .forEach(changes::add);
+    return changes.build();
   }
 
   static ImmutableSet<String> getNewlyCreatedUnblockables(
@@ -225,18 +257,18 @@ public final class DomainsRefresher {
             .collect(toImmutableSet());
     ImmutableSet<String> liveDomains =
         queryNewlyCreatedDomains(bsaEnabledTlds, prevRefreshStartTime, now);
-    return getUnblockedDomainNames(liveDomains);
+    return getBlockedDomainNames(liveDomains);
   }
 
-  static ImmutableSet<String> getNewlyReservedUnblockables(DateTime now, int batchSize) {
+  static ImmutableSet<String> getAllReservedUnblockables(DateTime now, int batchSize) {
     Stream<String> allReserved = getAllReservedNames(now);
     return BatchedStreams.toBatches(allReserved, batchSize)
-        .map(DomainsRefresher::getUnblockedDomainNames)
+        .map(DomainsRefresher::getBlockedDomainNames)
         .flatMap(ImmutableSet::stream)
         .collect(toImmutableSet());
   }
 
-  static ImmutableSet<String> getUnblockedDomainNames(ImmutableCollection<String> domainNames) {
+  static ImmutableSet<String> getBlockedDomainNames(ImmutableCollection<String> domainNames) {
     Map<String, List<String>> labelToNames =
         domainNames.stream().collect(groupingBy(BsaStringUtils::getLabelInDomain));
     ImmutableSet<String> bsaLabels =
@@ -244,7 +276,7 @@ public final class DomainsRefresher {
             .map(BsaLabel::getLabel)
             .collect(toImmutableSet());
     return labelToNames.entrySet().stream()
-        .filter(entry -> !bsaLabels.contains(entry.getKey()))
+        .filter(entry -> bsaLabels.contains(entry.getKey()))
         .map(Entry::getValue)
         .flatMap(List::stream)
         .collect(toImmutableSet());
@@ -257,20 +289,21 @@ public final class DomainsRefresher {
                 .collect(
                     groupingBy(
                         change -> change.isDelete() ? "remove" : "change", toImmutableSet())));
-    tm().transact(
-            () -> {
-              if (changesByType.containsKey("remove")) {
-                tm().delete(
-                        changesByType.get("remove").stream()
-                            .map(c -> BsaUnblockableDomain.vKey(c.domainName()))
-                            .collect(toImmutableSet()));
-              }
-              if (changesByType.containsKey("change")) {
-                tm().putAll(
-                        changesByType.get("change").stream()
-                            .map(UnblockableDomainChange::newValue)
-                            .collect(toImmutableSet()));
-              }
-            });
+    bsaTransact(
+        () -> {
+          if (changesByType.containsKey("remove")) {
+            tm().delete(
+                    changesByType.get("remove").stream()
+                        .map(c -> BsaUnblockableDomain.vKey(c.domainName()))
+                        .collect(toImmutableSet()));
+          }
+          if (changesByType.containsKey("change")) {
+            tm().putAll(
+                    changesByType.get("change").stream()
+                        .map(UnblockableDomainChange::newValue)
+                        .map(BsaUnblockableDomain::of)
+                        .collect(toImmutableSet()));
+          }
+        });
   }
 }

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -721,6 +721,17 @@ public final class RegistryConfig {
       return "gs://" + billingBucket;
     }
 
+    /**
+     * Returns origin part of the URL of the billing invoice file
+     *
+     * @see google.registry.beam.billing.InvoicingPipeline
+     */
+    @Provides
+    @Config("billingInvoiceOriginUrl")
+    public static String provideBillingInvoiceOriginUrl(RegistryConfigSettings config) {
+      return config.billing.billingInvoiceOriginUrl;
+    }
+
     /**
      * Returns whether or not we should publish invoices to partners automatically by default.
      *

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -173,6 +173,7 @@ public class RegistryConfigSettings {
     public List<String> invoiceEmailRecipients;
     public String invoiceReplyToEmailAddress;
     public String invoiceFilePrefix;
+    public String billingInvoiceOriginUrl;
   }
 
   /** Configuration for Registry Data Escrow (RDE). */

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -381,6 +381,7 @@ billing:
   # Optional return address that overrides the default.
   invoiceReplyToEmailAddress: null
   invoiceFilePrefix: REG-INV
+  billingInvoiceOriginUrl: https://billing-origin-url/
 
 rde:
   # URL prefix of ICANN's server to upload RDE reports to. Nomulus adds /TLD/ID

core/src/main/java/google/registry/config/files/tld/example.yaml

@@ -0,0 +1,60 @@
+# Example of a TLD configuration file. TLD configuration files are stored in
+# the internal repository and synced regularly to the database using the
+# release/cloudbuild-tld-sync.yaml job. The file can be created using the exact
+# output from the getTldCommand of an existing TLD.
+
+addGracePeriodLength: "PT432000S"
+allowedFullyQualifiedHostNames: []
+allowedRegistrantContactIds: []
+anchorTenantAddGracePeriodLength: "PT2592000S"
+autoRenewGracePeriodLength: "PT3888000S"
+automaticTransferLength: "PT432000S"
+claimsPeriodEnd: "294247-01-10T04:00:54.775Z"
+createBillingCost:
+  currency: "USD"
+  amount: 8.00
+creationTime: "2017-01-03T19:52:47.770Z"
+currency: "USD"
+defaultPromoTokens: []
+dnsAPlusAaaaTtl: null
+dnsDsTtl: null
+dnsNsTtl: null
+dnsPaused: true
+dnsWriters:
+- "VoidDnsWriter"
+driveFolderId: null
+eapFeeSchedule:
+  "1970-01-01T00:00:00.000Z":
+    currency: "USD"
+    amount: 0.00
+escrowEnabled: false
+idnTables: []
+invoicingEnabled: false
+lordnUsername: null
+numDnsPublishLocks: 1
+pendingDeleteLength: "PT432000S"
+premiumListName: null
+pricingEngineClassName: "google.registry.model.pricing.StaticPremiumListPricingEngine"
+redemptionGracePeriodLength: "PT2592000S"
+registryLockOrUnlockBillingCost:
+  currency: "USD"
+  amount: 0.00
+renewBillingCostTransitions:
+  "1970-01-01T00:00:00.000Z":
+    currency: "USD"
+    amount: 8.00
+renewGracePeriodLength: "PT432000S"
+reservedListNames: []
+restoreBillingCost:
+  currency: "USD"
+  amount: 50.00
+roidSuffix: "EXAMPLE"
+serverStatusChangeBillingCost:
+  currency: "USD"
+  amount: 0.00
+tldStateTransitions:
+  "1970-01-01T00:00:00.000Z": "GENERAL_AVAILABILITY"
+tldStr: "example"
+tldType: "TEST"
+tldUnicode: "example"
+transferGracePeriodLength: "PT432000S"

core/src/main/java/google/registry/env/production/backend/WEB-INF/appengine-web.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 
-  <runtime>java8</runtime>
+  <runtime>java17</runtime>
   <service>backend</service>
-  <!--app-engine-apis>true</app-engine-apis-->
-  <threadsafe>true</threadsafe>
+  <app-engine-apis>true</app-engine-apis>
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4_1G</instance-class>
   <basic-scaling>

core/src/main/java/google/registry/env/production/bsa/WEB-INF/appengine-web.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 
-  <runtime>java8</runtime>
+  <runtime>java17</runtime>
   <service>bsa</service>
-  <!--app-engine-apis>true</app-engine-apis-->
-  <threadsafe>true</threadsafe>
+  <app-engine-apis>true</app-engine-apis>
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4_1G</instance-class>
   <basic-scaling>

core/src/main/java/google/registry/env/production/default/WEB-INF/appengine-web.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 
-  <runtime>java8</runtime>
+  <runtime>java17</runtime>
   <service>default</service>
-  <!--app-engine-apis>true</app-engine-apis-->
-  <threadsafe>true</threadsafe>
+  <app-engine-apis>true</app-engine-apis>
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4_1G</instance-class>
   <manual-scaling>

core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml

@@ -285,6 +285,18 @@
     <schedule>0 * * * *</schedule>
   </task>
 
+  <task>
+    <url><![CDATA[/_dr/task/bsaRefresh]]></url>
+    <name>bsaRefresh</name>
+    <service>bsa</service>
+    <description>
+      Checks for changes in registered domains and reserved labels, and updates
+      the unblockable domains list.
+    </description>
+    <!-- Runs every 30 minutes. -->
+    <schedule>15,45 * * * *</schedule>
+  </task>
+
   <task>
     <url><![CDATA[/_dr/task/uploadBsaUnavailableNames]]></url>
     <name>uploadBsaUnavailableNames</name>

core/src/main/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 
-  <runtime>java8</runtime>
+  <runtime>java17</runtime>
   <service>pubapi</service>
-  <!--app-engine-apis>true</app-engine-apis-->
-  <threadsafe>true</threadsafe>
+  <app-engine-apis>true</app-engine-apis>
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4_1G</instance-class>
   <manual-scaling>

core/src/main/java/google/registry/env/production/tools/WEB-INF/appengine-web.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 
-  <runtime>java8</runtime>
+  <runtime>java17</runtime>
   <service>tools</service>
-  <!--app-engine-apis>true</app-engine-apis-->
-  <threadsafe>true</threadsafe>
+  <app-engine-apis>true</app-engine-apis>
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4_1G</instance-class>
   <basic-scaling>

core/src/main/java/google/registry/env/sandbox/backend/WEB-INF/appengine-web.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 
-  <runtime>java8</runtime>
+  <runtime>java17</runtime>
   <service>backend</service>
-  <!--app-engine-apis>true</app-engine-apis-->
-  <threadsafe>true</threadsafe>
+  <app-engine-apis>true</app-engine-apis>
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4</instance-class>
   <basic-scaling>

core/src/main/java/google/registry/env/sandbox/bsa/WEB-INF/appengine-web.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 
-  <runtime>java8</runtime>
+  <runtime>java17</runtime>
   <service>bsa</service>
-  <!--app-engine-apis>true</app-engine-apis-->
-  <threadsafe>true</threadsafe>
+  <app-engine-apis>true</app-engine-apis>
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4</instance-class>
   <basic-scaling>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/appengine-web.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 
-  <runtime>java8</runtime>
+  <runtime>java17</runtime>
   <service>default</service>
-  <!--app-engine-apis>true</app-engine-apis-->
-  <threadsafe>true</threadsafe>
+  <app-engine-apis>true</app-engine-apis>
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4_1G</instance-class>
   <manual-scaling>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cloud-scheduler-tasks.xml

@@ -174,6 +174,18 @@
     <schedule>0 * * * *</schedule>
   </task>
 
+  <task>
+    <url><![CDATA[/_dr/task/bsaRefresh]]></url>
+    <name>bsaRefresh</name>
+    <service>bsa</service>
+    <description>
+      Checks for changes in registered domains and reserved labels, and updates
+      the unblockable domains list.
+    </description>
+    <!-- Runs every 30 minutes. -->
+    <schedule>15,45 * * * *</schedule>
+  </task>
+
   <task>
     <url><![CDATA[/_dr/task/uploadBsaUnavailableNames]]></url>
     <name>uploadBsaUnavailableNames</name>

core/src/main/java/google/registry/env/sandbox/pubapi/WEB-INF/appengine-web.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 
-  <runtime>java8</runtime>
+  <runtime>java17</runtime>
   <service>pubapi</service>
-  <!--app-engine-apis>true</app-engine-apis-->
-  <threadsafe>true</threadsafe>
+  <app-engine-apis>true</app-engine-apis>
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4_1G</instance-class>
   <manual-scaling>

core/src/main/java/google/registry/env/sandbox/tools/WEB-INF/appengine-web.xml

@@ -1,10 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
 
-  <runtime>java8</runtime>
+  <runtime>java17</runtime>
   <service>tools</service>
-  <!--app-engine-apis>true</app-engine-apis-->
-  <threadsafe>true</threadsafe>
+  <app-engine-apis>true</app-engine-apis>
   <sessions-enabled>true</sessions-enabled>
   <instance-class>B4</instance-class>
   <basic-scaling>

core/src/main/java/google/registry/gcs/GcsUtils.java

@@ -118,6 +118,14 @@ public class GcsUtils implements Serializable {
     storage().delete(blobId);
   }
 
+  /** Update file content type on existing GCS file */
+  public void updateContentType(BlobId blobId, String contentType) throws StorageException {
+    if (existsAndNotEmpty(blobId)) {
+      Blob blob = storage().get(blobId);
+      blob.toBuilder().setContentType(contentType).build().update();
+    }
+  }
+
   /**
    * Returns a list of all object names within a bucket for a given prefix.
    *

core/src/main/java/google/registry/reporting/billing/BillingEmailUtils.java

@@ -15,20 +15,17 @@
 package google.registry.reporting.billing;
 
 import static com.google.common.base.Throwables.getRootCause;
-import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.StorageException;
 import com.google.common.collect.ImmutableList;
-import com.google.common.io.CharStreams;
+import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
 import google.registry.groups.GmailClient;
 import google.registry.reporting.billing.BillingModule.InvoiceDirectoryPrefix;
 import google.registry.util.EmailMessage;
-import google.registry.util.EmailMessage.Attachment;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.util.Optional;
 import javax.inject.Inject;
 import javax.mail.internet.InternetAddress;
@@ -37,6 +34,7 @@ import org.joda.time.YearMonth;
 /** Utility functions for sending emails involving monthly invoices. */
 public class BillingEmailUtils {
 
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private final GmailClient gmailClient;
   private final YearMonth yearMonth;
   private final InternetAddress outgoingEmailAddress;
@@ -46,6 +44,7 @@ public class BillingEmailUtils {
   private final String billingBucket;
   private final String invoiceFilePrefix;
   private final String invoiceDirectoryPrefix;
+  private final String billingInvoiceOriginUrl;
   private final GcsUtils gcsUtils;
 
   @Inject
@@ -58,6 +57,7 @@ public class BillingEmailUtils {
       @Config("invoiceReplyToEmailAddress") Optional<InternetAddress> replyToEmailAddress,
       @Config("billingBucket") String billingBucket,
       @Config("invoiceFilePrefix") String invoiceFilePrefix,
+      @Config("billingInvoiceOriginUrl") String billingInvoiceOriginUrl,
       @InvoiceDirectoryPrefix String invoiceDirectoryPrefix,
       GcsUtils gcsUtils) {
     this.gmailClient = gmailClient;
@@ -69,31 +69,36 @@ public class BillingEmailUtils {
     this.billingBucket = billingBucket;
     this.invoiceFilePrefix = invoiceFilePrefix;
     this.invoiceDirectoryPrefix = invoiceDirectoryPrefix;
+    this.billingInvoiceOriginUrl = billingInvoiceOriginUrl;
     this.gcsUtils = gcsUtils;
   }
 
   /** Sends an e-mail to all expected recipients with an attached overall invoice from GCS. */
-  void emailOverallInvoice() {
+  public void emailOverallInvoice() {
     try {
       String invoiceFile = String.format("%s-%s.csv", invoiceFilePrefix, yearMonth);
+      String fileUrl = billingInvoiceOriginUrl + invoiceDirectoryPrefix + invoiceFile;
       BlobId invoiceFilename = BlobId.of(billingBucket, invoiceDirectoryPrefix + invoiceFile);
-      try (InputStream in = gcsUtils.openInputStream(invoiceFilename)) {
-        gmailClient.sendEmail(
-            EmailMessage.newBuilder()
-                .setSubject(String.format("Domain Registry invoice data %s", yearMonth))
-                .setBody(
-                    String.format("Attached is the %s invoice for the domain registry.", yearMonth))
-                .setFrom(outgoingEmailAddress)
-                .setRecipients(invoiceEmailRecipients)
-                .setReplyToEmailAddress(replyToEmailAddress)
-                .setAttachment(
-                    Attachment.newBuilder()
-                        .setContent(CharStreams.toString(new InputStreamReader(in, UTF_8)))
-                        .setContentType(MediaType.CSV_UTF_8)
-                        .setFilename(invoiceFile)
-                        .build())
-                .build());
+      try {
+        gcsUtils.updateContentType(invoiceFilename, "text/csv");
+      } catch (StorageException e) {
+        // We want to continue with email anyway, it just will be less convenient for billing team
+        // to process the file.
+        logger.atWarning().withCause(e).log("Failed to update invoice file type");
       }
+      gmailClient.sendEmail(
+          EmailMessage.newBuilder()
+              .setSubject(String.format("Domain Registry invoice data %s", yearMonth))
+              .setBody(
+                  String.format(
+                      "<p>Use the following link to download %s invoice for the domain registry -"
+                          + " <a href=\"%s\">invoice</a>.</p>",
+                      yearMonth, fileUrl))
+              .setFrom(outgoingEmailAddress)
+              .setRecipients(invoiceEmailRecipients)
+              .setReplyToEmailAddress(replyToEmailAddress)
+              .setContentType(MediaType.HTML_UTF_8)
+              .build());
     } catch (Throwable e) {
       // Strip one layer, because callWithRetry wraps in a RuntimeException
       sendAlertEmail(

core/src/main/java/google/registry/tools/ConfigureTldCommand.java

@@ -105,6 +105,12 @@ public class ConfigureTldCommand extends MutatingCommand {
 
   @Override
   protected void init() throws Exception {
+    if (RegistryToolEnvironment.get().equals(RegistryToolEnvironment.PRODUCTION)) {
+      checkArgument(
+          buildEnv || breakGlass != null,
+          "Either the --break_glass or --build_environment flag must be used when"
+              + " running the configure_tld command on Production");
+    }
     String name = convertFilePathToName(inputFile);
     Map<String, Object> tldData = new Yaml().load(Files.newBufferedReader(inputFile));
     checkName(name, tldData);

core/src/main/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java

@@ -292,8 +292,8 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
       }
       if (!isNullOrEmpty(email)) {
         builder.setEmailAddress(email);
-      } else if (!isNullOrEmpty(
-          icannReferralEmail)) { // fall back to ICANN referral email if present
+      } else if (!isNullOrEmpty(icannReferralEmail) && oldRegistrar == null) {
+        // On creates, fall back to ICANN referral email (if present).
         builder.setEmailAddress(icannReferralEmail);
       }
       if (url != null) {

core/src/main/java/google/registry/tools/RegistryCli.java

@@ -23,6 +23,7 @@ import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
 import com.beust.jcommander.ParametersDelegate;
+import com.google.common.base.Ascii;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
@@ -218,12 +219,21 @@ final class RegistryCli implements CommandRunner {
 
     // Reset the JPA transaction manager after every command to avoid a situation where a test can
     // interfere with other tests
-    JpaTransactionManager cachedJpaTm = tm();
-    TransactionManagerFactory.setJpaTm(() -> component.nomulusToolJpaTransactionManager().get());
-    TransactionManagerFactory.setReplicaJpaTm(
-        () -> component.nomulusToolReplicaJpaTransactionManager().get());
-    command.run();
-    TransactionManagerFactory.setJpaTm(() -> cachedJpaTm);
+    try {
+      JpaTransactionManager cachedJpaTm = tm();
+      TransactionManagerFactory.setJpaTm(() -> component.nomulusToolJpaTransactionManager().get());
+      TransactionManagerFactory.setReplicaJpaTm(
+          () -> component.nomulusToolReplicaJpaTransactionManager().get());
+      command.run();
+      TransactionManagerFactory.setJpaTm(() -> cachedJpaTm);
+    } catch (Exception e) {
+      String env = Ascii.toLowerCase(environment.name());
+      System.err.printf(
+          "Could not get tool transaction manager; try running nomulus -e %s logout "
+              + "and then nomulus -e %s login.\n",
+          env, env);
+      throw e;
+    }
   }
 
   void setEnvironment(RegistryToolEnvironment environment) {

core/src/main/java/google/registry/tools/UpdatePremiumListCommand.java

@@ -35,11 +35,22 @@ class UpdatePremiumListCommand extends CreateOrUpdatePremiumListCommand {
       description = "Does not execute the entity mutation")
   boolean dryRun;
 
-  // indicates if there is a new change made by this command
+  @Parameter(
+      names = {"--build_environment"},
+      description =
+          "DO NOT USE THIS FLAG ON THE COMMAND LINE! This flag indicates the command is being run"
+              + " by the build environment tools. This flag should never be used by a human user"
+              + " from the command line.")
+  boolean buildEnv;
+
+  // Indicates if there is a new change made by this command
   private boolean newChange = false;
 
   @Override
   protected String prompt() throws Exception {
+    checkArgument(
+        !RegistryToolEnvironment.get().equals(RegistryToolEnvironment.PRODUCTION) || buildEnv,
+        "The --build_environment flag must be used when running update_premium_list in production");
     name = Strings.isNullOrEmpty(name) ? convertFilePathToName(inputFile) : name;
     PremiumList existingList =
         PremiumListDao.getLatestRevision(name)

core/src/main/java/google/registry/tools/UpdateReservedListCommand.java

@@ -14,10 +14,12 @@
 
 package google.registry.tools;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.util.DiffUtils.prettyPrintEntityDeepDiff;
 import static google.registry.util.ListNamingUtils.convertFilePathToName;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
+import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.common.base.Strings;
 import google.registry.model.tld.label.ReservedList;
@@ -28,8 +30,28 @@ import java.util.List;
 @Parameters(separators = " =", commandDescription = "Update a ReservedList.")
 final class UpdateReservedListCommand extends CreateOrUpdateReservedListCommand {
 
+  @Parameter(
+      names = {"-d", "--dry_run"},
+      description = "Does not execute the entity mutation")
+  boolean dryRun;
+
+  @Parameter(
+      names = {"--build_environment"},
+      description =
+          "DO NOT USE THIS FLAG ON THE COMMAND LINE! This flag indicates the command is being run"
+              + " by the build environment tools. This flag should never be used by a human user"
+              + " from the command line.")
+  boolean buildEnv;
+
+  // indicates if there is a new change made by this command
+  private boolean newChange = true;
+
   @Override
   protected String prompt() throws Exception {
+    checkArgument(
+        !RegistryToolEnvironment.get().equals(RegistryToolEnvironment.PRODUCTION) || buildEnv,
+        "The --build_environment flag must be used when running update_reserved_list in"
+            + " production");
     name = Strings.isNullOrEmpty(name) ? convertFilePathToName(input) : name;
     ReservedList existingReservedList =
         ReservedList.get(name)
@@ -54,6 +76,7 @@ final class UpdateReservedListCommand extends CreateOrUpdateReservedListCommand
             .getReservedListEntries()
             .equals(reservedList.getReservedListEntries());
     if (!shouldPublishChanged && !reservedListEntriesChanged) {
+      newChange = false;
       return "No entity changes to apply.";
     }
     String result = String.format("Update reserved list for %s?\n", name);
@@ -70,4 +93,9 @@ final class UpdateReservedListCommand extends CreateOrUpdateReservedListCommand
     }
     return result;
   }
+
+  @Override
+  protected boolean dontRunCommand() {
+    return dryRun || !newChange;
+  }
 }

core/src/test/java/google/registry/bsa/BsaDownloadFunctionalTest.java

@@ -118,15 +118,8 @@ class BsaDownloadFunctionalTest {
         gcsClient.readBlockList(downloadJob, BlockListType.BLOCK_PLUS)) {
       assertThat(blockListFile).containsExactly(BSA_CSV_HEADER, "abc,2", "def,3");
     }
-    ImmutableList<String> persistedLabels =
-        ImmutableList.copyOf(
-            tm().transact(
-                    () ->
-                        tm().getEntityManager()
-                            .createNativeQuery("SELECT label from \"BsaLabel\"")
-                            .getResultList()));
     // TODO(weiminyu): check intermediate files
-    assertThat(persistedLabels).containsExactly("abc", "def");
+    assertThat(getPersistedLabels()).containsExactly("abc", "def");
   }
 
   @Test

core/src/test/java/google/registry/bsa/BsaRefreshFunctionalTest.java

@@ -0,0 +1,331 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.bsa;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.bsa.ReservedDomainsTestingUtils.addReservedDomainToList;
+import static google.registry.bsa.ReservedDomainsTestingUtils.addReservedListsToTld;
+import static google.registry.bsa.ReservedDomainsTestingUtils.createReservedList;
+import static google.registry.bsa.ReservedDomainsTestingUtils.removeReservedDomainFromList;
+import static google.registry.bsa.persistence.BsaTestingUtils.createDownloadScheduler;
+import static google.registry.bsa.persistence.BsaTestingUtils.persistBsaLabel;
+import static google.registry.bsa.persistence.BsaTestingUtils.queryUnblockableDomains;
+import static google.registry.model.tld.Tlds.getTldEntitiesOfType;
+import static google.registry.model.tld.label.ReservationType.RESERVED_FOR_SPECIFIC_USE;
+import static google.registry.testing.DatabaseHelper.createTlds;
+import static google.registry.testing.DatabaseHelper.deleteTestDomain;
+import static google.registry.testing.DatabaseHelper.persistActiveDomain;
+import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+
+import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.bsa.api.BsaReportSender;
+import google.registry.bsa.api.UnblockableDomain;
+import google.registry.bsa.api.UnblockableDomain.Reason;
+import google.registry.bsa.api.UnblockableDomainChange;
+import google.registry.bsa.persistence.BsaTestingUtils;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.domain.Domain;
+import google.registry.model.tld.Tld.TldType;
+import google.registry.persistence.transaction.JpaTestExtensions;
+import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension;
+import google.registry.request.Response;
+import google.registry.testing.FakeClock;
+import google.registry.testing.FakeLockHandler;
+import google.registry.testing.FakeResponse;
+import java.io.UncheckedIOException;
+import java.util.Optional;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.mockito.InOrder;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+/**
+ * Functional tests for refreshing the unblockable domains with recent registration and reservation
+ * changes.
+ */
+@ExtendWith(MockitoExtension.class)
+class BsaRefreshFunctionalTest {
+
+  static final DateTime TEST_START_TIME = DateTime.parse("2024-01-01T00:00:00Z");
+
+  static final String RESERVED_LIST_NAME = "reserved";
+
+  private final FakeClock fakeClock = new FakeClock(TEST_START_TIME);
+
+  @RegisterExtension
+  JpaIntegrationWithCoverageExtension jpa =
+      new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension();
+
+  @Mock BsaReportSender bsaReportSender;
+
+  private GcsClient gcsClient;
+  private Response response;
+  private BsaRefreshAction action;
+
+  @BeforeEach
+  void setup() throws Exception {
+    gcsClient =
+        new GcsClient(new GcsUtils(LocalStorageHelper.getOptions()), "my-bucket", "SHA-256");
+    response = new FakeResponse();
+    action =
+        new BsaRefreshAction(
+            BsaTestingUtils.createRefreshScheduler(),
+            gcsClient,
+            bsaReportSender,
+            /* transactionBatchSize= */ 5,
+            /* domainCreateTxnCommitTimeLag= */ Duration.millis(1),
+            new BsaLock(
+                new FakeLockHandler(/* lockSucceeds= */ true), Duration.standardSeconds(30)),
+            fakeClock,
+            response);
+
+    initDb();
+  }
+
+  private String getRefreshJobName(DateTime jobStartTime) {
+    return jobStartTime.toString() + "-refresh";
+  }
+
+  private void initDb() {
+    createTlds("app", "dev");
+    getTldEntitiesOfType(TldType.REAL)
+        .forEach(
+            tld ->
+                persistResource(
+                    tld.asBuilder().setBsaEnrollStartTime(Optional.of(START_OF_TIME)).build()));
+
+    createReservedList(RESERVED_LIST_NAME, "dummy", RESERVED_FOR_SPECIFIC_USE);
+    addReservedListsToTld("app", ImmutableList.of(RESERVED_LIST_NAME));
+
+    persistBsaLabel("blocked1");
+    persistBsaLabel("blocked2");
+    // Creates a download record so that refresher will not quit immediately.
+    createDownloadScheduler(fakeClock).schedule().get().updateJobStage(DownloadStage.DONE);
+    fakeClock.advanceOneMilli();
+  }
+
+  @Test
+  void newReservedDomain_addedAsUnblockable() throws Exception {
+    addReservedDomainToList(
+        RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
+    String jobName = getRefreshJobName(fakeClock.nowUtc());
+    action.run();
+    UnblockableDomain newUnblockable = UnblockableDomain.of("blocked1.app", Reason.RESERVED);
+    assertThat(queryUnblockableDomains()).containsExactly(newUnblockable);
+    assertThat(gcsClient.readRefreshChanges(jobName))
+        .containsExactly(UnblockableDomainChange.ofNew(newUnblockable));
+    verify(bsaReportSender, never()).removeUnblockableDomainsUpdates(anyString());
+    verify(bsaReportSender, times(1))
+        .addUnblockableDomainsUpdates("{\n  \"reserved\": [\n    \"blocked1.app\"\n  ]\n}");
+  }
+
+  @Test
+  void newRegisteredDomain_addedAsUnblockable() throws Exception {
+    persistActiveDomain("blocked1.dev", fakeClock.nowUtc());
+    persistActiveDomain("dummy.dev", fakeClock.nowUtc());
+    String jobName = getRefreshJobName(fakeClock.nowUtc());
+    action.run();
+    UnblockableDomain newUnblockable = UnblockableDomain.of("blocked1.dev", Reason.REGISTERED);
+    assertThat(queryUnblockableDomains()).containsExactly(newUnblockable);
+    assertThat(gcsClient.readRefreshChanges(jobName))
+        .containsExactly(UnblockableDomainChange.ofNew(newUnblockable));
+
+    verify(bsaReportSender, never()).removeUnblockableDomainsUpdates(anyString());
+    verify(bsaReportSender, times(1))
+        .addUnblockableDomainsUpdates("{\n  \"registered\": [\n    \"blocked1.dev\"\n  ]\n}");
+  }
+
+  @Test
+  void registeredUnblockable_unregistered() {
+    Domain domain = persistActiveDomain("blocked1.dev", fakeClock.nowUtc());
+    action.run();
+    assertThat(queryUnblockableDomains())
+        .containsExactly(UnblockableDomain.of("blocked1.dev", Reason.REGISTERED));
+    fakeClock.advanceOneMilli();
+    deleteTestDomain(domain, fakeClock.nowUtc());
+    fakeClock.advanceOneMilli();
+
+    String jobName = getRefreshJobName(fakeClock.nowUtc());
+    Mockito.reset(bsaReportSender);
+    action.run();
+    assertThat(queryUnblockableDomains()).isEmpty();
+    assertThat(gcsClient.readRefreshChanges(jobName))
+        .containsExactly(
+            UnblockableDomainChange.ofDeleted(
+                UnblockableDomain.of("blocked1.dev", Reason.REGISTERED)));
+
+    verify(bsaReportSender, never()).addUnblockableDomainsUpdates(anyString());
+    verify(bsaReportSender, times(1)).removeUnblockableDomainsUpdates("[\n  \"blocked1.dev\"\n]");
+  }
+
+  @Test
+  void reservedUnblockable_noLongerReserved() {
+    addReservedDomainToList(
+        RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
+    action.run();
+    assertThat(queryUnblockableDomains())
+        .containsExactly(UnblockableDomain.of("blocked1.app", Reason.RESERVED));
+    fakeClock.advanceOneMilli();
+    removeReservedDomainFromList(RESERVED_LIST_NAME, ImmutableSet.of("blocked1"));
+
+    String jobName = getRefreshJobName(fakeClock.nowUtc());
+    Mockito.reset(bsaReportSender);
+    action.run();
+    assertThat(queryUnblockableDomains()).isEmpty();
+    assertThat(gcsClient.readRefreshChanges(jobName))
+        .containsExactly(
+            UnblockableDomainChange.ofDeleted(
+                UnblockableDomain.of("blocked1.app", Reason.RESERVED)));
+
+    verify(bsaReportSender, never()).addUnblockableDomainsUpdates(anyString());
+    verify(bsaReportSender, times(1)).removeUnblockableDomainsUpdates("[\n  \"blocked1.app\"\n]");
+  }
+
+  @Test
+  void registeredAndReservedUnblockable_noLongerRegistered_stillUnblockable() throws Exception {
+    addReservedDomainToList(
+        RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
+    Domain domain = persistActiveDomain("blocked1.app", fakeClock.nowUtc());
+    action.run();
+    assertThat(queryUnblockableDomains())
+        .containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
+    fakeClock.advanceOneMilli();
+    deleteTestDomain(domain, fakeClock.nowUtc());
+    fakeClock.advanceOneMilli();
+
+    String jobName = getRefreshJobName(fakeClock.nowUtc());
+    Mockito.reset(bsaReportSender);
+    action.run();
+    assertThat(queryUnblockableDomains())
+        .containsExactly(UnblockableDomain.of("blocked1.app", Reason.RESERVED));
+    assertThat(gcsClient.readRefreshChanges(jobName))
+        .containsExactly(
+            UnblockableDomainChange.ofChanged(
+                UnblockableDomain.of("blocked1.app", Reason.REGISTERED), Reason.RESERVED));
+    InOrder inOrder = Mockito.inOrder(bsaReportSender);
+    inOrder.verify(bsaReportSender).removeUnblockableDomainsUpdates("[\n  \"blocked1.app\"\n]");
+    inOrder
+        .verify(bsaReportSender)
+        .addUnblockableDomainsUpdates("{\n  \"reserved\": [\n    \"blocked1.app\"\n  ]\n}");
+  }
+
+  @Test
+  void reservedUblockable_becomesRegistered_changeToRegisterd() throws Exception {
+    addReservedDomainToList(
+        RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
+    action.run();
+    assertThat(queryUnblockableDomains())
+        .containsExactly(UnblockableDomain.of("blocked1.app", Reason.RESERVED));
+    fakeClock.advanceOneMilli();
+    persistActiveDomain("blocked1.app", fakeClock.nowUtc());
+    fakeClock.advanceOneMilli();
+
+    Mockito.reset(bsaReportSender);
+    String jobName = getRefreshJobName(fakeClock.nowUtc());
+    action.run();
+    UnblockableDomain changed = UnblockableDomain.of("blocked1.app", Reason.REGISTERED);
+    assertThat(queryUnblockableDomains()).containsExactly(changed);
+    assertThat(gcsClient.readRefreshChanges(jobName))
+        .containsExactly(
+            UnblockableDomainChange.ofChanged(
+                UnblockableDomain.of("blocked1.app", Reason.RESERVED), Reason.REGISTERED));
+    InOrder inOrder = Mockito.inOrder(bsaReportSender);
+    inOrder.verify(bsaReportSender).removeUnblockableDomainsUpdates("[\n  \"blocked1.app\"\n]");
+    inOrder
+        .verify(bsaReportSender)
+        .addUnblockableDomainsUpdates("{\n  \"registered\": [\n    \"blocked1.app\"\n  ]\n}");
+  }
+
+  @Test
+  void newRegisteredAndReservedDomain_addedAsRegisteredUnblockable() throws Exception {
+    addReservedDomainToList(
+        RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
+    persistActiveDomain("blocked1.app", fakeClock.nowUtc());
+    String jobName = getRefreshJobName(fakeClock.nowUtc());
+    action.run();
+    UnblockableDomain newUnblockable = UnblockableDomain.of("blocked1.app", Reason.REGISTERED);
+    assertThat(queryUnblockableDomains()).containsExactly(newUnblockable);
+    assertThat(gcsClient.readRefreshChanges(jobName))
+        .containsExactly(UnblockableDomainChange.ofNew(newUnblockable));
+  }
+
+  @Test
+  void registeredAndReservedUnblockable_noLongerReserved_noChange() throws Exception {
+    addReservedDomainToList(
+        RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
+    persistActiveDomain("blocked1.app", fakeClock.nowUtc());
+    action.run();
+    assertThat(queryUnblockableDomains())
+        .containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
+    fakeClock.advanceOneMilli();
+    removeReservedDomainFromList(RESERVED_LIST_NAME, ImmutableSet.of("blocked1"));
+    fakeClock.advanceOneMilli();
+
+    Mockito.reset(bsaReportSender);
+    String jobName = getRefreshJobName(fakeClock.nowUtc());
+    action.run();
+    assertThat(queryUnblockableDomains())
+        .containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
+    // Verify that refresh change file does not exist (404 error) since there is no change.
+    assertThat(
+            assertThrows(
+                UncheckedIOException.class, () -> gcsClient.readRefreshChanges(jobName).findAny()))
+        .hasMessageThat()
+        .contains("404");
+    verifyNoInteractions(bsaReportSender);
+  }
+
+  @Test
+  void registeredUblockable_becomesReserved_noChange() throws Exception {
+    persistActiveDomain("blocked1.app", fakeClock.nowUtc());
+    action.run();
+    assertThat(queryUnblockableDomains())
+        .containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
+    fakeClock.advanceOneMilli();
+    addReservedDomainToList(
+        RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
+    fakeClock.advanceOneMilli();
+
+    Mockito.reset(bsaReportSender);
+    String jobName = getRefreshJobName(fakeClock.nowUtc());
+    action.run();
+    assertThat(queryUnblockableDomains())
+        .containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
+    // Verify that refresh change file does not exist (404 error) since there is no change.
+    assertThat(
+            assertThrows(
+                UncheckedIOException.class, () -> gcsClient.readRefreshChanges(jobName).findAny()))
+        .hasMessageThat()
+        .contains("404");
+    verifyNoInteractions(bsaReportSender);
+  }
+}

core/src/test/java/google/registry/bsa/ReservedDomainsTestingUtils.java

@@ -62,4 +62,43 @@ public final class ReservedDomainsTestingUtils {
             .build();
     persistResource(tld.asBuilder().setReservedListsByName(reservedLists).build());
   }
+
+  public static void addReservedDomainToList(
+      String listName, ImmutableMap<String, ReservationType> reservedLabels) {
+    ImmutableMap<String, ReservedListEntry> existingEntries =
+        ReservedList.get(listName).get().getReservedListEntries();
+    ImmutableMap<String, ReservedListEntry> newEntries =
+        ImmutableMap.copyOf(
+            Maps.transformEntries(
+                reservedLabels, (key, value) -> ReservedListEntry.create(key, value, "")));
+
+    ReservedListDao.save(
+        new ReservedList.Builder()
+            .setName(listName)
+            .setCreationTimestamp(START_OF_TIME)
+            .setShouldPublish(true)
+            .setReservedListMap(
+                new ImmutableMap.Builder<String, ReservedListEntry>()
+                    .putAll(existingEntries)
+                    .putAll(newEntries)
+                    .buildKeepingLast())
+            .build());
+  }
+
+  public static void removeReservedDomainFromList(
+      String listName, ImmutableSet<String> removedLabels) {
+    ImmutableMap<String, ReservedListEntry> existingEntries =
+        ReservedList.get(listName).get().getReservedListEntries();
+    ImmutableMap<String, ReservedListEntry> newEntries =
+        ImmutableMap.copyOf(
+            Maps.filterEntries(existingEntries, entry -> !removedLabels.contains(entry.getKey())));
+
+    ReservedListDao.save(
+        new ReservedList.Builder()
+            .setName(listName)
+            .setCreationTimestamp(START_OF_TIME)
+            .setShouldPublish(true)
+            .setReservedListMap(newEntries)
+            .build());
+  }
 }

core/src/test/java/google/registry/bsa/UploadBsaUnavailableDomainsActionTest.java

@@ -48,6 +48,10 @@ import org.mockito.junit.jupiter.MockitoExtension;
 @ExtendWith(MockitoExtension.class)
 public class UploadBsaUnavailableDomainsActionTest {
 
+  private static final String BUCKET = "domain-registry-bsa";
+
+  private static final String API_URL = "https://upload.test/bsa";
+
   private final FakeClock clock = new FakeClock(DateTime.parse("2024-02-02T02:02:02Z"));
 
   @RegisterExtension
@@ -62,9 +66,7 @@ public class UploadBsaUnavailableDomainsActionTest {
 
   private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
 
-  private final String BUCKET = "domain-registry-bsa";
 
-  private final String API_URL = "https://upload.test/bsa";
 
   private final FakeResponse response = new FakeResponse();
 

core/src/test/java/google/registry/bsa/persistence/BsaTestingUtils.java

@@ -14,8 +14,11 @@
 
 package google.registry.bsa.persistence;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
+import com.google.common.collect.ImmutableList;
+import google.registry.bsa.api.UnblockableDomain;
 import google.registry.util.Clock;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
@@ -35,7 +38,21 @@ public final class BsaTestingUtils {
     tm().transact(() -> tm().put(new BsaLabel(domainLabel, BSA_LABEL_CREATION_TIME)));
   }
 
+  public static void persistUnblockableDomain(UnblockableDomain unblockableDomain) {
+    tm().transact(() -> tm().put(BsaUnblockableDomain.of(unblockableDomain)));
+  }
+
   public static DownloadScheduler createDownloadScheduler(Clock clock) {
     return new DownloadScheduler(DEFAULT_DOWNLOAD_INTERVAL, DEFAULT_NOP_INTERVAL, clock);
   }
+
+  public static RefreshScheduler createRefreshScheduler() {
+    return new RefreshScheduler();
+  }
+
+  public static ImmutableList<UnblockableDomain> queryUnblockableDomains() {
+    return tm().transact(() -> tm().loadAllOf(BsaUnblockableDomain.class)).stream()
+        .map(BsaUnblockableDomain::toUnblockableDomain)
+        .collect(toImmutableList());
+  }
 }

core/src/test/java/google/registry/bsa/persistence/LabelDiffUpdatesTest.java

@@ -16,6 +16,8 @@ package google.registry.bsa.persistence;
 
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.bsa.ReservedDomainsTestingUtils.addReservedListsToTld;
+import static google.registry.bsa.ReservedDomainsTestingUtils.createReservedList;
 import static google.registry.bsa.persistence.LabelDiffUpdates.applyLabelDiff;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
@@ -26,7 +28,6 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import google.registry.bsa.IdnChecker;
@@ -36,9 +37,6 @@ import google.registry.bsa.api.UnblockableDomain;
 import google.registry.bsa.persistence.BsaUnblockableDomain.Reason;
 import google.registry.model.tld.Tld;
 import google.registry.model.tld.label.ReservationType;
-import google.registry.model.tld.label.ReservedList;
-import google.registry.model.tld.label.ReservedList.ReservedListEntry;
-import google.registry.model.tld.label.ReservedListDao;
 import google.registry.persistence.transaction.JpaTestExtensions;
 import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension;
 import google.registry.testing.FakeClock;
@@ -139,18 +137,8 @@ class LabelDiffUpdatesTest {
   @Test
   void applyLabelDiffs_newLabel() {
     persistActiveDomain("label.app");
-    ReservedListDao.save(
-        new ReservedList.Builder()
-            .setReservedListMap(
-                ImmutableMap.of(
-                    "label",
-                    ReservedListEntry.create(
-                        "label", ReservationType.RESERVED_FOR_SPECIFIC_USE, null)))
-            .setName("page_reserved")
-            .setCreationTimestamp(fakeClock.nowUtc())
-            .build());
-    ReservedList reservedList = ReservedList.get("page_reserved").get();
-    tm().transact(() -> tm().put(page.asBuilder().setReservedLists(reservedList).build()));
+    createReservedList("page_reserved", "label", ReservationType.RESERVED_FOR_SPECIFIC_USE);
+    addReservedListsToTld("page", ImmutableList.of("page_reserved"));
 
     when(idnChecker.getForbiddingTlds(any()))
         .thenReturn(Sets.difference(ImmutableSet.of(dev), ImmutableSet.of()).immutableCopy());

core/src/test/java/google/registry/bsa/persistence/QueriesTest.java

@@ -208,6 +208,32 @@ class QueriesTest {
         .containsExactly("d1.tld", "d2.tld");
   }
 
+  @Test
+  void queryNewlyCreatedDomains_onlyDomainsAfterMinCreationTimeReturned() {
+    DateTime testStartTime = fakeClock.nowUtc();
+    createTlds("tld");
+    persistNewRegistrar("TheRegistrar");
+    // time 0:
+    persistResource(
+        newDomain("d1.tld").asBuilder().setCreationTimeForTest(fakeClock.nowUtc()).build());
+    // time 0, deletion time 1
+    persistDomainAsDeleted(
+        newDomain("will-delete.tld").asBuilder().setCreationTimeForTest(fakeClock.nowUtc()).build(),
+        fakeClock.nowUtc().plusMillis(1));
+    fakeClock.advanceOneMilli();
+    // time 1
+    persistResource(
+        newDomain("d2.tld").asBuilder().setCreationTimeForTest(fakeClock.nowUtc()).build());
+    fakeClock.advanceOneMilli();
+    // Now is time 2, ask for domains created since time 1
+    assertThat(
+            bsaQuery(
+                () ->
+                    queryNewlyCreatedDomains(
+                        ImmutableList.of("tld"), testStartTime.plusMillis(1), fakeClock.nowUtc())))
+        .containsExactly("d2.tld");
+  }
+
   @Test
   void queryNewlyCreatedDomains_onlyDomainsInRequestedTldsReturned() {
     DateTime testStartTime = fakeClock.nowUtc();

core/src/test/java/google/registry/reporting/billing/BillingEmailUtilsTest.java

@@ -21,17 +21,12 @@ import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
-import com.google.cloud.storage.BlobId;
 import com.google.common.collect.ImmutableList;
 import com.google.common.net.MediaType;
 import google.registry.gcs.GcsUtils;
 import google.registry.groups.GmailClient;
 import google.registry.util.EmailMessage;
-import google.registry.util.EmailMessage.Attachment;
-import java.io.ByteArrayInputStream;
-import java.nio.charset.StandardCharsets;
 import java.util.Optional;
 import javax.mail.MessagingException;
 import javax.mail.internet.InternetAddress;
@@ -45,18 +40,14 @@ class BillingEmailUtilsTest {
 
   private GmailClient gmailClient;
   private BillingEmailUtils emailUtils;
-  private GcsUtils gcsUtils;
   private ArgumentCaptor<EmailMessage> contentCaptor;
+  private GcsUtils gcsUtils;
 
   @BeforeEach
   void beforeEach() throws Exception {
     gmailClient = mock(GmailClient.class);
-    gcsUtils = mock(GcsUtils.class);
-    when(gcsUtils.openInputStream(BlobId.of("test-bucket", "results/REG-INV-2017-10.csv")))
-        .thenReturn(
-            new ByteArrayInputStream("test,data\nhello,world".getBytes(StandardCharsets.UTF_8)));
     contentCaptor = ArgumentCaptor.forClass(EmailMessage.class);
-
+    gcsUtils = mock(GcsUtils.class);
     emailUtils = getEmailUtils(Optional.of(new InternetAddress("reply-to@test.com")));
   }
 
@@ -72,6 +63,7 @@ class BillingEmailUtilsTest {
         replyToAddress,
         "test-bucket",
         "REG-INV",
+        "www.google.com/",
         "results/",
         gcsUtils);
   }
@@ -89,14 +81,11 @@ class BillingEmailUtilsTest {
                 ImmutableList.of(
                     new InternetAddress("hello@world.com"), new InternetAddress("hola@mundo.com")))
             .setSubject("Domain Registry invoice data 2017-10")
-            .setBody("Attached is the 2017-10 invoice for the domain registry.")
+            .setBody(
+                "<p>Use the following link to download 2017-10 invoice for the domain registry -"
+                    + " <a href=\"www.google.com/results/REG-INV-2017-10.csv\">invoice</a>.</p>")
             .setReplyToEmailAddress(new InternetAddress("reply-to@test.com"))
-            .setAttachment(
-                Attachment.newBuilder()
-                    .setContent("test,data\nhello,world")
-                    .setContentType(MediaType.CSV_UTF_8)
-                    .setFilename("REG-INV-2017-10.csv")
-                    .build())
+            .setContentType(MediaType.HTML_UTF_8)
             .build();
     assertThat(emailMessage).isEqualTo(expectedContent);
   }

core/src/test/java/google/registry/tools/ConfigureTldCommandTest.java

@@ -671,4 +671,46 @@ public class ConfigureTldCommandTest extends CommandTestCase<ConfigureTldCommand
     Tld notUpdatedTld = Tld.get("tld");
     assertThat(notUpdatedTld.getCreateBillingCost()).isEqualTo(Money.of(USD, 13));
   }
+
+  @Test
+  void testFailure_runCommandOnProduction_noFlag() throws Exception {
+    createTld("tld");
+    File tldFile = tmpDir.resolve("tld.yaml").toFile();
+    Files.asCharSink(tldFile, UTF_8).write(loadFile(getClass(), "tld.yaml"));
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandInEnvironment(RegistryToolEnvironment.PRODUCTION, "--input=" + tldFile));
+    assertThat(thrown.getMessage())
+        .isEqualTo(
+            "Either the --break_glass or --build_environment flag must be used when running the"
+                + " configure_tld command on Production");
+  }
+
+  @Test
+  void testSuccess_runCommandOnProduction_breakGlassFlag() throws Exception {
+    createTld("tld");
+    File tldFile = tmpDir.resolve("tld.yaml").toFile();
+    Files.asCharSink(tldFile, UTF_8).write(loadFile(getClass(), "tld.yaml"));
+    runCommandInEnvironment(
+        RegistryToolEnvironment.PRODUCTION, "--input=" + tldFile, "--break_glass=true", "-f");
+    Tld updatedTld = Tld.get("tld");
+    assertThat(updatedTld.getCreateBillingCost()).isEqualTo(Money.of(USD, 25));
+    testTldConfiguredSuccessfully(updatedTld, "tld.yaml");
+    assertThat(updatedTld.getBreakglassMode()).isTrue();
+  }
+
+  @Test
+  void testSuccess_runCommandOnProduction_buildEnvFlag() throws Exception {
+    createTld("tld");
+    File tldFile = tmpDir.resolve("tld.yaml").toFile();
+    Files.asCharSink(tldFile, UTF_8).write(loadFile(getClass(), "tld.yaml"));
+    runCommandInEnvironment(
+        RegistryToolEnvironment.PRODUCTION, "--input=" + tldFile, "--build_environment", "-f");
+    Tld updatedTld = Tld.get("tld");
+    assertThat(updatedTld.getCreateBillingCost()).isEqualTo(Money.of(USD, 25));
+    testTldConfiguredSuccessfully(updatedTld, "tld.yaml");
+    assertThat(updatedTld.getBreakglassMode()).isFalse();
+  }
 }

core/src/test/java/google/registry/tools/UpdatePremiumListCommandTest.java

@@ -205,4 +205,39 @@ class UpdatePremiumListCommandTest<C extends UpdatePremiumListCommand>
         .comparingElementsUsing(immutableObjectCorrespondence("revisionId"))
         .containsExactly(PremiumEntry.create(0L, new BigDecimal("9090.00"), "doge"));
   }
+
+  @Test
+  void testFailure_runCommandOnProduction_noFlag() throws Exception {
+    File tmpFile = tmpDir.resolve(String.format("%s.txt", TLD_TEST)).toFile();
+    String newPremiumListData = "eth,USD 9999";
+    Files.asCharSink(tmpFile, UTF_8).write(newPremiumListData);
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandInEnvironment(
+                    RegistryToolEnvironment.PRODUCTION,
+                    "--name=" + TLD_TEST,
+                    "--input=" + Paths.get(tmpFile.getPath())));
+    assertThat(thrown.getMessage())
+        .isEqualTo(
+            "The --build_environment flag must be used when running update_premium_list in"
+                + " production");
+  }
+
+  @Test
+  void testSuccess_runCommandOnProduction_buildEnvFlag() throws Exception {
+    File tmpFile = tmpDir.resolve(String.format("%s.txt", TLD_TEST)).toFile();
+    String newPremiumListData = "eth,USD 9999";
+    Files.asCharSink(tmpFile, UTF_8).write(newPremiumListData);
+    runCommandInEnvironment(
+        RegistryToolEnvironment.PRODUCTION,
+        "--name=" + TLD_TEST,
+        "--input=" + Paths.get(tmpFile.getPath()),
+        "--build_environment",
+        "-f");
+    assertThat(PremiumListDao.loadAllPremiumEntries(TLD_TEST))
+        .comparingElementsUsing(immutableObjectCorrespondence("revisionId"))
+        .containsExactly(PremiumEntry.create(0L, new BigDecimal("9999.00"), "eth"));
+  }
 }

core/src/test/java/google/registry/tools/UpdateRegistrarCommandTest.java

@@ -637,10 +637,12 @@ class UpdateRegistrarCommandTest extends CommandTestCase<UpdateRegistrarCommand>
 
   @Test
   void testSuccess_setIcannEmail() throws Exception {
-    runCommand("--icann_referral_email=foo@bar.test", "--force", "TheRegistrar");
     Registrar registrar = loadRegistrar("TheRegistrar");
-    assertThat(registrar.getIcannReferralEmail()).isEqualTo("foo@bar.test");
-    assertThat(registrar.getEmailAddress()).isEqualTo("foo@bar.test");
+    assertThat(registrar.getEmailAddress()).isEqualTo("the.registrar@example.com");
+    runCommand("--icann_referral_email=foo@bar.test", "--force", "TheRegistrar");
+    Registrar updatedRegistrar = loadRegistrar("TheRegistrar");
+    assertThat(updatedRegistrar.getIcannReferralEmail()).isEqualTo("foo@bar.test");
+    assertThat(updatedRegistrar.getEmailAddress()).isEqualTo("the.registrar@example.com");
   }
 
   @Test

core/src/test/java/google/registry/tools/UpdateReservedListCommandTest.java

@@ -142,4 +142,50 @@ class UpdateReservedListCommandTest
     assertThat(command.prompt()).contains("baddies: null -> baddies,FULLY_BLOCKED");
     assertThat(command.prompt()).contains("ford: null -> ford,FULLY_BLOCKED # random comment");
   }
+
+  @Test
+  void testSuccess_dryRun() throws Exception {
+    runCommandForced("--input=" + reservedTermsPath, "--dry_run");
+    assertThat(command.prompt()).contains("Update reserved list for xn--q9jyb4c_common-reserved?");
+    assertThat(ReservedList.get("xn--q9jyb4c_common-reserved")).isPresent();
+    ReservedList reservedList = ReservedList.get("xn--q9jyb4c_common-reserved").get();
+    assertThat(reservedList.getReservedListEntries()).hasSize(1);
+    assertThat(reservedList.getReservationInList("helicopter")).hasValue(FULLY_BLOCKED);
+  }
+
+  @Test
+  void testFailure_runCommandOnProduction_noFlag() throws Exception {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandInEnvironment(
+                    RegistryToolEnvironment.PRODUCTION,
+                    "--name=xn--q9jyb4c_common-reserved",
+                    "--input=" + reservedTermsPath));
+    assertThat(thrown.getMessage())
+        .isEqualTo(
+            "The --build_environment flag must be used when running update_reserved_list in"
+                + " production");
+  }
+
+  @Test
+  void testSuccess_runCommandOnProduction_buildEnvFlag() throws Exception {
+    runCommandInEnvironment(
+        RegistryToolEnvironment.PRODUCTION,
+        "--name=xn--q9jyb4c_common-reserved",
+        "--input=" + reservedTermsPath,
+        "--build_environment",
+        "-f");
+    assertThat(ReservedList.get("xn--q9jyb4c_common-reserved")).isPresent();
+    ReservedList reservedList = ReservedList.get("xn--q9jyb4c_common-reserved").get();
+    assertThat(reservedList.getReservedListEntries()).hasSize(2);
+    assertThat(reservedList.getReservationInList("baddies")).hasValue(FULLY_BLOCKED);
+    assertThat(reservedList.getReservationInList("ford")).hasValue(FULLY_BLOCKED);
+    assertThat(reservedList.getReservationInList("helicopter")).isEmpty();
+    assertInStdout("Update reserved list for xn--q9jyb4c_common-reserved?");
+    assertInStdout("helicopter: helicopter,FULLY_BLOCKED -> null");
+    assertInStdout("baddies: null -> baddies,FULLY_BLOCKED");
+    assertInStdout("ford: null -> ford,FULLY_BLOCKED # random comment");
+  }
 }

db/src/main/resources/sql/flyway.txt

@@ -150,3 +150,5 @@ V149__add_bsa_domain_in_use_table.sql
 V150__add_tld_bsa_enroll_date.sql
 V151__add_bsa_unblockable_domain_table.sql
 V152__add_bsa_domain_refresh_table.sql
+V153__drop_bsa_domain_in_use_table.sql
+V154__add_create_billing_cost_transitions_to_tld.sql

db/src/main/resources/sql/flyway/V153__drop_bsa_domain_in_use_table.sql

@@ -0,0 +1,15 @@
+-- Copyright 2023 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+DROP TABLE IF EXISTS "BsaDomainInUse";

db/src/main/resources/sql/flyway/V154__add_create_billing_cost_transitions_to_tld.sql

@@ -0,0 +1,15 @@
+-- Copyright 2024 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+ALTER TABLE "Tld" ADD COLUMN create_billing_cost_transitions hstore;

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -123,18 +123,6 @@ CREATE TABLE public."BillingRecurrence" (
 );
 
 
---
--- Name: BsaDomainInUse; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public."BsaDomainInUse" (
-    label text NOT NULL,
-    tld text NOT NULL,
-    creation_time timestamp with time zone NOT NULL,
-    reason text NOT NULL
-);
-
-
 --
 -- Name: BsaDomainRefresh; Type: TABLE; Schema: public; Owner: -
 --
@@ -1190,7 +1178,8 @@ CREATE TABLE public."Tld" (
     dns_ns_ttl interval,
     idn_tables text[],
     breakglass_mode boolean DEFAULT false NOT NULL,
-    bsa_enroll_start_time timestamp with time zone
+    bsa_enroll_start_time timestamp with time zone,
+    create_billing_cost_transitions public.hstore
 );
 
 
@@ -1369,14 +1358,6 @@ ALTER TABLE ONLY public."BillingRecurrence"
     ADD CONSTRAINT "BillingRecurrence_pkey" PRIMARY KEY (billing_recurrence_id);
 
 
---
--- Name: BsaDomainInUse BsaDomainInUse_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public."BsaDomainInUse"
-    ADD CONSTRAINT "BsaDomainInUse_pkey" PRIMARY KEY (label, tld);
-
-
 --
 -- Name: BsaDomainRefresh BsaDomainRefresh_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -2781,14 +2762,6 @@ ALTER TABLE ONLY public."DomainHistoryHost"
     ADD CONSTRAINT fka9woh3hu8gx5x0vly6bai327n FOREIGN KEY (domain_history_domain_repo_id, domain_history_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;
 
 
---
--- Name: BsaDomainInUse fkbsadomaininuse2label; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public."BsaDomainInUse"
-    ADD CONSTRAINT fkbsadomaininuse2label FOREIGN KEY (label) REFERENCES public."BsaLabel"(label) ON DELETE CASCADE;
-
-
 --
 -- Name: BsaUnblockableDomain fkbsaunblockabledomainlabel; Type: FK CONSTRAINT; Schema: public; Owner: -
 --

java-format/google-java-format-git-diff.sh

@@ -40,37 +40,22 @@ where:
     show   show the effect of the formatting as unified diff"
 
 SCRIPT_DIR="$(realpath $(dirname $0))"
-JAR_NAME="google-java-format-1.8-all-deps.jar"
+JAR_NAME="google-java-format-1.19.2-all-deps.jar"
 
 # Make sure we have a valid python interpreter.
 if [ -z "$PYTHON" ]; then
-  echo "You must specify the name of a python3 interpreter in the PYTHON" \
+  echo "You must specify the name of a Python interpreter in the PYTHON" \
        "environment variable."
   exit 1
 elif ! "$PYTHON" -c ''; then
-  echo "Invalid python interpreter: $PYTHON"
+  echo "Invalid Python interpreter: $PYTHON"
   exit 1
 fi
 
-# Locate the java binary.
-if [ -n "$JAVA_HOME" ]; then
-  JAVA_BIN="$JAVA_HOME/bin/java"
-  if [ ! -x "$JAVA_BIN" ]; then
-    echo "No java binary found in JAVA_HOME (JAVA_HOME is $JAVA_HOME)"
-    exit 1
-  fi
-else
-  # Use java from the path.
-  JAVA_BIN="$(which java)" || JAVA_BIN=""
-  if [ -z "$JAVA_BIN" ]; then
-    echo "JAVA_HOME is not defined and java was not found on the path"
-    exit 1
-  fi
-fi
-
-if ! "$JAVA_BIN" -version 2>&1 | grep 'version "11\.' >/dev/null; then
-  echo "Bad java version.  Requires java 11, got:"
-  "$JAVA_BIN" -version
+# Make sure we have a valid JRE binary
+if [ -z "$JAVA" ]; then
+  echo "You must specify the name of a JRE binary in the JAVA" \
+       "environment variable."
   exit 1
 fi
 
@@ -80,7 +65,7 @@ function runGoogleJavaFormatAgainstDiffs() {
 
   git diff -U0 "$forkPoint" | \
       "${PYTHON}" "${SCRIPT_DIR}/google-java-format-diff.py" \
-          --java-binary "$JAVA_BIN" \
+          --java-binary "$JAVA" \
           --google-java-format-jar "${SCRIPT_DIR}/${JAR_NAME}" \
           -p1 "$@" | \
       tee gjf.out

processor/src/main/java/google/registry/processors/VKeyProcessor.java

@@ -50,7 +50,7 @@ import javax.persistence.Converter;
 
 /** Processor to generate {@link AttributeConverter} for {@code VKey} type. */
 @SupportedAnnotationTypes("google.registry.persistence.WithVKey")
-@SupportedSourceVersion(SourceVersion.RELEASE_8)
+@SupportedSourceVersion(SourceVersion.RELEASE_17)
 public class VKeyProcessor extends AbstractProcessor {
 
   private static final String CONVERTER_CLASS_NAME_TEMP = "VKeyConverter_%s";

release/cloudbuild-kythe.yaml

@@ -29,6 +29,12 @@ steps:
     mv $${JAVA_HOME}/bin/javac $${JAVA_HOME}/bin/javac.real
     cp ./kythe/extractors/javac-wrapper.sh $${JAVA_HOME}/bin/javac
     export JAVAC_EXTRACTOR_JAR="$${PWD}/kythe/extractors/javac_extractor.jar"
+    jvmopts="--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED"
+    jvmopts="$${jvmopts} --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"
+    jvmopts="$${jvmopts} --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED"
+    jvmopts="$${jvmopts} --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED"
+    jvmopts="$${jvmopts} --add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED"
+    export KYTHE_JAVA_RUNTIME_OPTIONS=$${jvmopts}
     export KYTHE_VNAMES="$${PWD}/vnames.json"
     export KYTHE_ROOT_DIRECTORY="$${PWD}"
     export KYTHE_OUTPUT_DIRECTORY="$${PWD}/kythe_output"

release/cloudbuild-nomulus.yaml

@@ -91,7 +91,7 @@ steps:
       --format="get(digest)" --filter="tags = ${TAG_NAME}")
     sed -i s/'prober_cert_updater:latest'/prober_cert_updater@$digest/g \
       release/cloudbuild-renew-prober-certs-*.yaml
-# Build the tld_updater image and upload it to GCR. This image extends
+# Build the db_object_updater image and upload it to GCR. This image extends
 # from the `builder` and the nomulus.jar built earlier.
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: /bin/bash
@@ -101,14 +101,14 @@ steps:
     set -e
     # The nomulus jar is not under the working dir. Must be copied over. 
     cp ../../output/nomulus.jar .
-    docker build -t gcr.io/${PROJECT_ID}/tld_updater:${TAG_NAME} \
+    docker build -t gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME} \
       --build-arg TAG_NAME=${TAG_NAME} --build-arg PROJECT_ID=${PROJECT_ID} .
-    docker tag gcr.io/${PROJECT_ID}/tld_updater:${TAG_NAME} \
-      gcr.io/${PROJECT_ID}/tld_updater:latest
-    docker push gcr.io/${PROJECT_ID}/tld_updater:latest
-    docker push gcr.io/${PROJECT_ID}/tld_updater:${TAG_NAME}
-  dir: 'release/tld-updater/'
-# Update the tld_updater image digest in relevant GCB files.
+    docker tag gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME} \
+      gcr.io/${PROJECT_ID}/db_object_updater:latest
+    docker push gcr.io/${PROJECT_ID}/db_object_updater:latest
+    docker push gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME}
+  dir: 'release/db-object-updater/'
+# Update the db_object_updater image digest in relevant GCB files.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
   args:
@@ -116,10 +116,10 @@ steps:
   - |
     set -e
     digest=$(gcloud container images list-tags \
-      gcr.io/${PROJECT_ID}/tld_updater \
+      gcr.io/${PROJECT_ID}/db_object_updater \
       --format="get(digest)" --filter="tags = ${TAG_NAME}")
-    sed -i s/'tld_updater:latest'/tld_updater@$digest/g \
-      release/cloudbuild-tld-sync-*.yaml
+    sed -i s/'db_object_updater:latest'/db_object_updater@$digest/g \
+      release/cloudbuild-sync-db-objects-*.yaml
 # Build and stage Dataflow Flex templates.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
@@ -190,7 +190,7 @@ artifacts:
     - 'release/cloudbuild-renew-prober-certs-*.yaml'
     - 'release/cloudbuild-schema-deploy-*.yaml'
     - 'release/cloudbuild-schema-verify-*.yaml'
-    - 'release/cloudbuild-tld-sync-*.yaml'
+    - 'release/cloudbuild-sync-db-objects-*.yaml'
 
 timeout: 7200s
 options:

release/cloudbuild-release.yaml

@@ -139,9 +139,9 @@ steps:
       gcloud container images list-tags \
       gcr.io/${PROJECT_ID}/prober_cert_updater \
       --format='get(digest)' --filter='tags = ${TAG_NAME}')
-     tld_updater_digest=$( \
+     db_object_updater_digest=$( \
       gcloud container images list-tags \
-      gcr.io/${PROJECT_ID}/tld_updater \
+      gcr.io/${PROJECT_ID}/db_object_updater \
       --format='get(digest)' --filter='tags = ${TAG_NAME}')
     sed -i s/builder:latest/builder@$builder_digest/g \
       release/cloudbuild-schema-deploy.yaml
@@ -150,7 +150,7 @@ steps:
     sed -i s/builder:latest/builder@$builder_digest/g \
       release/cloudbuild-renew-prober-certs.yaml
     sed -i s/builder:latest/builder@$builder_digest/g \
-      release/cloudbuild-tld-sync.yaml
+      release/cloudbuild-sync-db-objects.yaml
     sed -i s/schema_deployer:latest/schema_deployer@$schema_deployer_digest/g \
       release/cloudbuild-schema-deploy.yaml
     sed -i s/schema_verifier:latest/schema_verifier@$schema_verifier_digest/g \
@@ -163,8 +163,8 @@ steps:
         > release/cloudbuild-schema-verify-${environment}.yaml
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-renew-prober-certs.yaml \
         > release/cloudbuild-renew-prober-certs-${environment}.yaml
-      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-tld-sync.yaml \
-        > release/cloudbuild-tld-sync-${environment}.yaml
+      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-sync-db-objects.yaml \
+        > release/cloudbuild-sync-db-objects-${environment}.yaml
     done
 # Upload the gradle binary to GCS if it does not exist and point URL in gradle wrapper to it.
 - name: 'gcr.io/cloud-builders/gsutil'

release/cloudbuild-sync-db-objects.yaml

@@ -1,7 +1,8 @@
-# This will sync the Tld configurations in the internal repo with the Tld objects in the database.
+# This will sync the configuration files in the internal repo with their
+# corresponding objects in the database.
 #
 # To manually trigger a build on GCB, run:
-# gcloud builds submit --config cloudbuild-tld-sync.yaml --substitutions \
+# gcloud builds submit --config cloudbuild-sync-db-objects.yaml --substitutions \
 #   _INTERNAL_REPO_URL=[URL] ..
 #
 # To trigger a build automatically, follow the instructions below and add a trigger:
@@ -34,11 +35,26 @@ steps:
       --secret nomulus-tool-cloudbuild-credential \
       > nomulus_tool_credential.json
 # Configure the TLDs using the stored configuration files in the internal repo
-- name: 'gcr.io/$PROJECT_ID/tld_updater:latest'
+- name: 'gcr.io/$PROJECT_ID/db_object_updater:latest'
   args:
   - ${_ENV}
   - ./nomulus_tool_credential.json
+  - configure_tld
   - nomulus-internal/core/src/main/java/google/registry/config/files/tld/
+# Configure the premium lists using the stored configuration files in the internal repo
+- name: 'gcr.io/$PROJECT_ID/db_object_updater:latest'
+  args:
+  - ${_ENV}
+  - ./nomulus_tool_credential.json
+  - update_premium_list
+  - nomulus-internal/core/src/main/java/google/registry/config/files/premium/
+# Configure the reserved lists using the stored configuration files in the internal repo
+- name: 'gcr.io/$PROJECT_ID/db_object_updater:latest'
+  args:
+  - ${_ENV}
+  - ./nomulus_tool_credential.json
+  - update_reserved_list
+  - nomulus-internal/core/src/main/java/google/registry/config/files/reserved/
 
 timeout: 7200s
 options:

release/db-object-updater/Dockerfile

@@ -17,6 +17,6 @@ ARG TAG_NAME
 FROM gcr.io/${PROJECT_ID}/builder:${TAG_NAME}
 
 COPY nomulus.jar /
-COPY sync_tlds.sh /usr/local/bin
+COPY sync_db_objects.sh /usr/local/bin
 
-ENTRYPOINT [ "bash", "sync_tlds.sh" ]
+ENTRYPOINT [ "bash", "sync_db_objects.sh" ]

release/db-object-updater/sync_db_objects.sh

@@ -13,23 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Sync the TLD configuration files from the internal repo with the Tld object
-# in the database. Loops through the Tld configuration files and runs the configure_tld command
-# with the file.
+# Sync the configuration files in the internal repo with the objects in the
+# database. Loops through the configuration files in the inputted directory and
+# runs the passed in nomulus update command with the file.
 
 # - env: The Nomulus environment, production, sandbox, etc.
 # - tools_credential: The credential (.json) needed to run the nomulus command.
+# - nomulus_command: The nomulus command to run.
 # - config_file_directory: The internal directory storing the TLD config files.
 
 set -e
-if [ "$#" -ne 3 ]; then
-  echo "Expecting three parameters in order: env tools_credential config_file_directory"
+if [ "$#" -ne 4 ]; then
+  echo "Expecting four parameters in order: env tools_credential nomulus_command config_file_directory"
   exit 1
 fi
 
 nomulus_env="${1}"
 tools_credential="${2}"
-config_file_directory="${3}"
+nomulus_command="${3}"
+config_file_directory="${4}"
 
 echo ${config_file_directory}
 
@@ -37,5 +39,5 @@ for FILE in ${config_file_directory}/${nomulus_env}/*; do
   echo $FILE
   java -jar /nomulus.jar -e "${nomulus_env}" \
   --credential "${tools_credential}" \
-  configure_tld -i $FILE --force
+  "${nomulus_command}" -i $FILE --force --build_environment
 done