Add Java changes for new breakglass_mode column on Tld (#2053)

* Add Java changes for new breakglass_mode column on Tld

* Add generated sql schema

console-webapp/dev-proxy.config.json

@@ -1,7 +1,7 @@
 {
-  "/registrar":
+  "/console-api":
   {
-    "target": "http://localhost:8080/registrar",
-    "secure": false
+    "target": "http://localhost:8080",
+    "secure": true
   }
 }

console-webapp/src/app/app-routing.module.ts

@@ -22,14 +22,18 @@ import SettingsRegistrarsComponent from './settings/registrars/registrars.compon
 import SettingsWhoisComponent from './settings/whois/whois.component';
 import SettingsUsersComponent from './settings/users/users.component';
 import SettingsSecurityComponent from './settings/security/security.component';
+import { RegistrarGuard } from './registrar/registrar.guard';
+import { RegistrarComponent } from './registrar/registrar.component';
 
 const routes: Routes = [
-  { path: '', redirectTo: '/settings/contact', pathMatch: 'full' },
-  { path: 'home', component: HomeComponent },
-  { path: 'tlds', component: TldsComponent },
+  { path: '', redirectTo: '/home', pathMatch: 'full' },
+  { path: 'registrars', component: RegistrarComponent },
+  { path: 'home', component: HomeComponent, canActivate: [RegistrarGuard] },
+  { path: 'tlds', component: TldsComponent, canActivate: [RegistrarGuard] },
   {
     path: 'settings',
     component: SettingsComponent,
+    canActivate: [RegistrarGuard],
     children: [
       {
         path: '',

console-webapp/src/app/app.component.html

@@ -12,6 +12,9 @@
         <a mat-list-item [routerLink]="'/settings'" routerLinkActive="active">
           Settings
         </a>
+        <a mat-list-item [routerLink]="'/registrars'" routerLinkActive="active">
+          Select Registrar
+        </a>
       </mat-nav-list>
     </mat-sidenav>
     <mat-sidenav-content class="console-app__content">

console-webapp/src/app/app.module.ts

@@ -31,6 +31,8 @@ import SettingsContactComponent, {
   ContactDetailsDialogComponent,
 } from './settings/contact/contact.component';
 import { HttpClientModule } from '@angular/common/http';
+import { RegistrarComponent } from './registrar/registrar.component';
+import { RegistrarGuard } from './registrar/registrar.guard';
 
 @NgModule({
   declarations: [
@@ -41,6 +43,7 @@ import { HttpClientModule } from '@angular/common/http';
     SettingsComponent,
     SettingsContactComponent,
     ContactDetailsDialogComponent,
+    RegistrarComponent,
   ],
   imports: [
     HttpClientModule,
@@ -50,7 +53,7 @@ import { HttpClientModule } from '@angular/common/http';
     AppRoutingModule,
     BrowserAnimationsModule,
   ],
-  providers: [BackendService],
+  providers: [BackendService, RegistrarGuard],
   bootstrap: [AppComponent],
 })
 export class AppModule {}

console-webapp/src/app/registrar/registrar.component.html

@@ -0,0 +1,17 @@
+<div class="console-app__registrar">
+  <h4 class="console-app__title">
+    {{ registrarService.activeRegistrarId === "" ? "Select" : "Switch" }}
+    registrar:
+  </h4>
+  <mat-form-field>
+    <mat-label>Registrar</mat-label>
+    <mat-select [(ngModel)]="registrarService.activeRegistrarId">
+      <mat-option
+        *ngFor="let registrar of registrarService.registrars"
+        [value]="registrar"
+      >
+        {{ registrar }}
+      </mat-option>
+    </mat-select>
+  </mat-form-field>
+</div>

console-webapp/src/app/registrar/registrar.component.less

@@ -0,0 +1,11 @@
+.console-app {
+  &__registrar {
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    margin-top: 1rem;
+  }
+  &__title {
+    margin-right: 1rem;
+  }
+}

console-webapp/src/app/registrar/registrar.component.spec.ts

@@ -0,0 +1,44 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { RegistrarComponent } from './registrar.component';
+import { BackendService } from '../shared/services/backend.service';
+import { ActivatedRoute } from '@angular/router';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+
+describe('RegistrarComponent', () => {
+  let component: RegistrarComponent;
+  let fixture: ComponentFixture<RegistrarComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [RegistrarComponent],
+      imports: [HttpClientTestingModule],
+      providers: [
+        BackendService,
+        { provide: ActivatedRoute, useValue: {} as ActivatedRoute },
+      ],
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(RegistrarComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

console-webapp/src/app/registrar/registrar.component.ts

@@ -0,0 +1,45 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Component } from '@angular/core';
+import { RegistrarService } from './registrar.service';
+import { ActivatedRoute, Router } from '@angular/router';
+
+@Component({
+  selector: 'app-registrar',
+  templateUrl: './registrar.component.html',
+  styleUrls: ['./registrar.component.less'],
+})
+export class RegistrarComponent {
+  private lastActiveRegistrarId: string;
+
+  constructor(
+    private route: ActivatedRoute,
+    protected registrarService: RegistrarService,
+    private router: Router
+  ) {
+    this.lastActiveRegistrarId = registrarService.activeRegistrarId;
+  }
+
+  ngDoCheck() {
+    if (
+      this.registrarService.activeRegistrarId &&
+      this.registrarService.activeRegistrarId !== this.lastActiveRegistrarId &&
+      this.route.snapshot.paramMap.get('nextUrl')
+    ) {
+      this.lastActiveRegistrarId = this.registrarService.activeRegistrarId;
+      this.router.navigate([this.route.snapshot.paramMap.get('nextUrl')]);
+    }
+  }
+}

console-webapp/src/app/registrar/registrar.guard.spec.ts

@@ -0,0 +1,64 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { TestBed } from '@angular/core/testing';
+
+import { RegistrarGuard } from './registrar.guard';
+import { Router, RouterStateSnapshot } from '@angular/router';
+import { RegistrarService } from './registrar.service';
+
+describe('RegistrarGuard', () => {
+  let guard: RegistrarGuard;
+  let dummyRegistrarService: RegistrarService;
+  let routeSpy: Router;
+  let dummyRoute: RouterStateSnapshot = {} as RouterStateSnapshot;
+
+  beforeEach(() => {
+    routeSpy = jasmine.createSpyObj<Router>('Router', ['navigate']);
+    dummyRegistrarService = { activeRegistrarId: '' } as RegistrarService;
+
+    TestBed.configureTestingModule({
+      providers: [
+        RegistrarGuard,
+        { provide: Router, useValue: routeSpy },
+        { provide: RegistrarService, useValue: dummyRegistrarService },
+      ],
+    });
+  });
+
+  it('should not be able to activate when activeRegistrarId is empty', () => {
+    guard = TestBed.inject(RegistrarGuard);
+    const res = guard.canActivate(dummyRoute);
+    expect(res).toBeFalsy();
+  });
+
+  it('should be able to activate when activeRegistrarId is not empty', () => {
+    TestBed.overrideProvider(RegistrarService, {
+      useValue: { activeRegistrarId: 'value' },
+    });
+    guard = TestBed.inject(RegistrarGuard);
+    const res = guard.canActivate(dummyRoute);
+    expect(res).toBeTrue();
+  });
+
+  it('should navigate to registrars when activeRegistrarId is empty', () => {
+    const dummyRoute = { url: '/value' } as RouterStateSnapshot;
+    guard = TestBed.inject(RegistrarGuard);
+    guard.canActivate(dummyRoute);
+    expect(routeSpy.navigate).toHaveBeenCalledOnceWith([
+      '/registrars',
+      { nextUrl: '/value' },
+    ]);
+  });
+});

console-webapp/src/app/registrar/registrar.guard.ts

@@ -0,0 +1,34 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Injectable } from '@angular/core';
+import { Router, RouterStateSnapshot } from '@angular/router';
+import { RegistrarService } from './registrar.service';
+
+@Injectable({
+  providedIn: 'root',
+})
+export class RegistrarGuard {
+  constructor(
+    private router: Router,
+    private registrarService: RegistrarService
+  ) {}
+
+  canActivate(state: RouterStateSnapshot): Promise<boolean> | boolean {
+    if (this.registrarService.activeRegistrarId) {
+      return true;
+    }
+    return this.router.navigate([`/registrars`, { nextUrl: state.url }]);
+  }
+}

console-webapp/src/app/registrar/registrar.service.spec.ts

@@ -0,0 +1,35 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { TestBed } from '@angular/core/testing';
+
+import { RegistrarService } from './registrar.service';
+import { BackendService } from '../shared/services/backend.service';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
+
+describe('RegistrarService', () => {
+  let service: RegistrarService;
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({
+      imports: [HttpClientTestingModule],
+      providers: [BackendService],
+    });
+    service = TestBed.inject(RegistrarService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});

console-webapp/src/app/registrar/registrar.service.ts

@@ -0,0 +1,29 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Injectable } from '@angular/core';
+import { BackendService } from '../shared/services/backend.service';
+
+@Injectable({
+  providedIn: 'root',
+})
+export class RegistrarService {
+  activeRegistrarId: string = '';
+  registrars: string[] = [];
+  constructor(private backend: BackendService) {
+    this.backend.getRegistrars().subscribe((r) => {
+      this.registrars = r;
+    });
+  }
+}

console-webapp/src/app/settings/contact/contact-details.component.html

@@ -1,11 +1,12 @@
 <h3 mat-dialog-title>Contact details</h3>
 <div mat-dialog-content>
-  <form>
+  <form (ngSubmit)="saveAndClose($event)">
     <div>
       <mat-form-field class="contact-details__input">
         <mat-label>Name: </mat-label>
         <input
           matInput
+          [required]="true"
           [(ngModel)]="contact.name"
           [ngModelOptions]="{ standalone: true }"
         />
@@ -19,6 +20,7 @@
           type="email"
           matInput
           [email]="true"
+          [required]="true"
           [(ngModel)]="contact.emailAddress"
           [ngModelOptions]="{ standalone: true }"
         />
@@ -94,12 +96,9 @@
         (per CL&D requirements)</mat-checkbox
       >
     </section>
+    <mat-dialog-actions>
+      <button mat-button (click)="onClose()">Cancel</button>
+      <button type="submit" mat-button>Save</button>
+    </mat-dialog-actions>
   </form>
-
-  <p></p>
-
-  <mat-dialog-actions>
-    <button mat-button (click)="onClose()">Cancel</button>
-    <button mat-button (click)="saveAndClose()">Save</button>
-  </mat-dialog-actions>
 </div>

console-webapp/src/app/settings/contact/contact.component.spec.ts

@@ -14,7 +14,10 @@
 
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
-import { ContactComponent } from './contact.component';
+import ContactComponent from './contact.component';
+import { MaterialModule } from 'src/app/material.module';
+import { BackendService } from 'src/app/shared/services/backend.service';
+import { HttpClientTestingModule } from '@angular/common/http/testing';
 
 describe('ContactComponent', () => {
   let component: ContactComponent;
@@ -23,8 +26,9 @@ describe('ContactComponent', () => {
   beforeEach(async () => {
     await TestBed.configureTestingModule({
       declarations: [ContactComponent],
+      imports: [HttpClientTestingModule, MaterialModule],
+      providers: [BackendService],
     }).compileComponents();
-
     fixture = TestBed.createComponent(ContactComponent);
     component = fixture.componentInstance;
     fixture.detectChanges();

console-webapp/src/app/settings/contact/contact.component.ts

@@ -104,7 +104,11 @@ export class ContactDetailsDialogComponent {
     this.operation = data.operation;
   }
 
-  saveAndClose() {
+  saveAndClose(e: any) {
+    e.preventDefault();
+    if (!e.target.checkValidity()) {
+      return;
+    }
     let operationObservable;
     if (this.operation === Operations.ADD) {
       operationObservable = this.contactService.addContact(this.contact);
@@ -143,7 +147,7 @@ export default class ContactComponent {
   ) {
     // TODO: Refactor to registrarId service
     this.loading = true;
-    this.contactService.fetchContacts('zoomco').subscribe(() => {
+    this.contactService.fetchContacts().subscribe(() => {
       this.loading = false;
     });
   }
@@ -160,7 +164,9 @@ export default class ContactComponent {
   }
 
   deleteContact(contact: Contact) {
-    this.contactService.deleteContact(contact);
+    if (confirm(`Please confirm contact ${contact.name} delete`)) {
+      this.contactService.deleteContact(contact).subscribe();
+    }
   }
 
   openCreateNew(e: Event) {

console-webapp/src/app/settings/contact/contact.service.ts

@@ -14,6 +14,7 @@
 
 import { Injectable } from '@angular/core';
 import { Observable, tap } from 'rxjs';
+import { RegistrarService } from 'src/app/registrar/registrar.service';
 import { BackendService } from 'src/app/shared/services/backend.service';
 
 export interface Contact {
@@ -34,26 +35,30 @@ export interface Contact {
 export class ContactService {
   contacts: Contact[] = [];
 
-  constructor(private backend: BackendService) {}
+  constructor(
+    private backend: BackendService,
+    private registrarService: RegistrarService
+  ) {}
 
   // TODO: Come up with a better handling for registrarId
-  fetchContacts(registrarId: string): Observable<Contact[]> {
-    return this.backend.getContacts(registrarId).pipe(
-      tap((contacts) => {
-        this.contacts = contacts;
-      })
-    );
+  fetchContacts(): Observable<Contact[]> {
+    return this.backend
+      .getContacts(this.registrarService.activeRegistrarId)
+      .pipe(
+        tap((contacts) => {
+          this.contacts = contacts;
+        })
+      );
   }
 
-  saveContacts(
-    contacts: Contact[],
-    registrarId?: string
-  ): Observable<Contact[]> {
-    return this.backend.postContacts(registrarId || 'default', contacts).pipe(
-      tap((_) => {
-        this.contacts = contacts;
-      })
-    );
+  saveContacts(contacts: Contact[]): Observable<Contact[]> {
+    return this.backend
+      .postContacts(this.registrarService.activeRegistrarId, contacts)
+      .pipe(
+        tap((_) => {
+          this.contacts = contacts;
+        })
+      );
   }
 
   updateContact(index: number, contact: Contact) {

console-webapp/src/app/settings/registrars/registrars.component.spec.ts

@@ -14,7 +14,7 @@
 
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
-import { RegistrarsComponent } from './registrars.component';
+import RegistrarsComponent from './registrars.component';
 
 describe('RegistrarsComponent', () => {
   let component: RegistrarsComponent;

console-webapp/src/app/settings/security/security.component.spec.ts

@@ -14,7 +14,7 @@
 
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
-import { SecurityComponent } from './security.component';
+import SecurityComponent from './security.component';
 
 describe('SecurityComponent', () => {
   let component: SecurityComponent;

console-webapp/src/app/settings/users/users.component.spec.ts

@@ -14,7 +14,7 @@
 
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
-import { UsersComponent } from './users.component';
+import UsersComponent from './users.component';
 
 describe('UsersComponent', () => {
   let component: UsersComponent;

console-webapp/src/app/settings/whois/whois.component.spec.ts

@@ -14,7 +14,7 @@
 
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
-import { WhoisComponent } from './whois.component';
+import WhoisComponent from './whois.component';
 
 describe('WhoisComponent', () => {
   let component: WhoisComponent;

console-webapp/src/app/shared/services/backend.service.ts

@@ -41,35 +41,11 @@ export class BackendService {
   }
 
   getContacts(registrarId: string): Observable<Contact[]> {
-    const mockData = [
-      {
-        name: 'Name Lastname',
-        emailAddress: 'test@google.com',
-        registrarId: 'zoomco',
-        types: ['ADMIN'],
-        visibleInWhoisAsAdmin: false,
-        visibleInWhoisAsTech: false,
-        visibleInDomainWhoisAsAbuse: false,
-      },
-      {
-        name: 'Testname testlastname',
-        emailAddress: 'testasd@google.com',
-        registrarId: 'zoomco',
-        visibleInWhoisAsAdmin: false,
-        visibleInWhoisAsTech: false,
-        visibleInDomainWhoisAsAbuse: false,
-        types: ['BILLING'],
-      },
-    ];
     return this.http
       .get<Contact[]>(
         `/console-api/settings/contacts?registrarId=${registrarId}`
       )
-      .pipe(
-        catchError((err) =>
-          this.errorCatcher<Contact[]>(err, <Contact[]>mockData)
-        )
-      );
+      .pipe(catchError((err) => this.errorCatcher<Contact[]>(err)));
   }
 
   postContacts(
@@ -81,4 +57,10 @@ export class BackendService {
       { contacts }
     );
   }
+
+  getRegistrars(): Observable<string[]> {
+    return this.http
+      .get<string[]>('/console-api/registrars')
+      .pipe(catchError((err) => this.errorCatcher<string[]>(err)));
+  }
 }

core/src/main/java/google/registry/dns/RefreshDnsAction.java

@@ -60,21 +60,18 @@ public final class RefreshDnsAction implements Runnable {
     if (!domainOrHostName.contains(".")) {
       throw new BadRequestException("URL parameter 'name' must be fully qualified");
     }
-    tm().transact(
-            () -> {
-              switch (type) {
-                case DOMAIN:
-                  loadAndVerifyExistence(Domain.class, domainOrHostName);
-                  requestDomainDnsRefresh(domainOrHostName);
-                  break;
-                case HOST:
-                  verifyHostIsSubordinate(loadAndVerifyExistence(Host.class, domainOrHostName));
-                  requestHostDnsRefresh(domainOrHostName);
-                  break;
-                default:
-                  throw new BadRequestException("Unsupported type: " + type);
-              }
-            });
+    switch (type) {
+      case DOMAIN:
+        loadAndVerifyExistence(Domain.class, domainOrHostName);
+        tm().transact(() -> requestDomainDnsRefresh(domainOrHostName));
+        break;
+      case HOST:
+        verifyHostIsSubordinate(loadAndVerifyExistence(Host.class, domainOrHostName));
+        tm().transact(() -> requestHostDnsRefresh(domainOrHostName));
+        break;
+      default:
+        throw new BadRequestException("Unsupported type: " + type);
+    }
   }
 
   private <T extends EppResource & ForeignKeyedEppResource>

core/src/main/java/google/registry/model/tld/Tld.java

@@ -475,6 +475,9 @@ public class Tld extends ImmutableObject implements Buildable, UnsafeSerializabl
   /** An allowlist of hosts allowed to be used on domains on this TLD (ignored if empty). */
   @Nullable Set<String> allowedFullyQualifiedHostNames;
 
+  @Column(nullable = false)
+  boolean breakglassMode = false;
+
   /**
    * References to allocation tokens that can be used on the TLD if no other token is passed in on a
    * domain create.
@@ -701,6 +704,10 @@ public class Tld extends ImmutableObject implements Buildable, UnsafeSerializabl
     return nullToEmptyImmutableCopy(idnTables);
   }
 
+  public boolean getBreakglassMode() {
+    return breakglassMode;
+  }
+
   @Override
   public Builder asBuilder() {
     return new Builder(clone(this));
@@ -1004,6 +1011,11 @@ public class Tld extends ImmutableObject implements Buildable, UnsafeSerializabl
       return this;
     }
 
+    public Builder setBreakglassMode(boolean breakglassMode) {
+      getInstance().breakglassMode = breakglassMode;
+      return this;
+    }
+
     @Override
     public Tld build() {
       final Tld instance = getInstance();

core/src/main/java/google/registry/tools/CurlCommand.java

@@ -75,6 +75,11 @@ class CurlCommand implements CommandWithConnection {
       required = true)
   private Service service;
 
+  @Parameter(
+      names = {"--canary"},
+      description = "If set, use the canary end-point; otherwise use the regular end-point.")
+  private Boolean canary = Boolean.FALSE;
+
   @Override
   public void setConnection(ServiceConnection connection) {
     this.connection = connection;
@@ -90,7 +95,7 @@ class CurlCommand implements CommandWithConnection {
       throw new IllegalArgumentException("You may not specify a body for a get method.");
     }
 
-    ServiceConnection connectionToService = connection.withService(service);
+    ServiceConnection connectionToService = connection.withService(service, canary);
     String response =
         (method == Method.GET)
             ? connectionToService.sendGetRequest(path, ImmutableMap.<String, String>of())

core/src/main/java/google/registry/tools/ServiceConnection.java

@@ -15,6 +15,8 @@
 package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static com.google.common.base.Verify.verify;
 import static com.google.common.net.HttpHeaders.X_REQUESTED_WITH;
 import static com.google.common.net.MediaType.JSON_UTF_8;
 import static google.registry.security.JsonHttp.JSON_SAFETY_PREFIX;
@@ -26,6 +28,7 @@ import com.google.api.client.http.HttpHeaders;
 import com.google.api.client.http.HttpRequest;
 import com.google.api.client.http.HttpRequestFactory;
 import com.google.api.client.http.HttpResponse;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.io.CharStreams;
@@ -36,6 +39,7 @@ import google.registry.config.RegistryConfig;
 import google.registry.request.Action.Service;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Map;
 import javax.annotation.Nullable;
@@ -55,20 +59,23 @@ public class ServiceConnection {
 
   @Inject HttpRequestFactory requestFactory;
   private final Service service;
+  private final boolean useCanary;
 
   @Inject
   ServiceConnection() {
     service = Service.TOOLS;
+    useCanary = false;
   }
 
-  private ServiceConnection(Service service, HttpRequestFactory requestFactory) {
+  private ServiceConnection(Service service, HttpRequestFactory requestFactory, boolean useCanary) {
     this.service = service;
     this.requestFactory = requestFactory;
+    this.useCanary = useCanary;
   }
 
-  /** Returns a copy of this connection that talks to a different service. */
-  public ServiceConnection withService(Service service) {
-    return new ServiceConnection(service, requestFactory);
+  /** Returns a copy of this connection that talks to a different service endpoint. */
+  public ServiceConnection withService(Service service, boolean isCanary) {
+    return new ServiceConnection(service, requestFactory, isCanary);
   }
 
   /** Returns the contents of the title tag in the given HTML, or null if not found. */
@@ -85,7 +92,7 @@ public class ServiceConnection {
   private String internalSend(
       String endpoint, Map<String, ?> params, MediaType contentType, @Nullable byte[] payload)
       throws IOException {
-    GenericUrl url = new GenericUrl(String.format("%s%s", getServer(service), endpoint));
+    GenericUrl url = new GenericUrl(String.format("%s%s", getServer(), endpoint));
     url.putAll(params);
     HttpRequest request =
         (payload != null)
@@ -120,6 +127,20 @@ public class ServiceConnection {
     }
   }
 
+  @VisibleForTesting
+  URL getServer() {
+    URL url = getServer(service);
+    if (useCanary) {
+      verify(!isNullOrEmpty(url.getHost()), "Null host in url");
+      try {
+        return new URL(url.getProtocol(), "nomulus-dot-" + url.getHost(), url.getFile());
+      } catch (MalformedURLException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    return url;
+  }
+
   public String sendPostRequest(
       String endpoint, Map<String, ?> params, MediaType contentType, byte[] payload)
       throws IOException {

core/src/main/java/google/registry/tools/server/RefreshDnsForAllDomainsAction.java

@@ -15,20 +15,25 @@
 package google.registry.tools.server;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.Iterables.getLast;
 import static google.registry.dns.DnsUtils.requestDomainDnsRefresh;
 import static google.registry.model.tld.Tlds.assertTldsExist;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.RequestParameters.PARAM_TLDS;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.util.Clock;
+import java.util.Optional;
 import java.util.Random;
 import javax.inject.Inject;
+import org.apache.arrow.util.VisibleForTesting;
 import org.apache.http.HttpStatus;
 import org.joda.time.Duration;
 
@@ -43,10 +48,8 @@ import org.joda.time.Duration;
  * run internally, or by pretending to be internal by setting the X-AppEngine-QueueName header,
  * which only admin users can do.
  *
- * <p>You must pass in a number of {@code smearMinutes} as a URL parameter so that the DNS queue
- * doesn't get overloaded. A rough rule of thumb for Cloud DNS is 1 minute per every 1,000 domains.
- * This smears the updates out over the next N minutes. For small TLDs consisting of fewer than
- * 1,000 domains, passing in 1 is fine (which will execute all the updates immediately).
+ * <p>You may pass in a {@code batchSize} for the batched read of domains from the database. This is
+ * recommended to be somewhere between 200 and 500. The default value is 250.
  */
 @Action(
     service = Action.Service.TOOLS,
@@ -56,47 +59,78 @@ public class RefreshDnsForAllDomainsAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  @Inject Response response;
+  private static final int DEFAULT_BATCH_SIZE = 250;
+
+  private final Response response;
+  private final ImmutableSet<String> tlds;
+
+  // Recommended value for batch size is between 200 and 500
+  private final int batchSize;
+  private final Random random;
 
   @Inject
-  @Parameter(PARAM_TLDS)
-  ImmutableSet<String> tlds;
-
-  @Inject
-  @Parameter("smearMinutes")
-  int smearMinutes;
-
-  @Inject Clock clock;
-  @Inject Random random;
-
-  @Inject
-  RefreshDnsForAllDomainsAction() {}
+  RefreshDnsForAllDomainsAction(
+      Response response,
+      @Parameter(PARAM_TLDS) ImmutableSet<String> tlds,
+      @Parameter("batchSize") Optional<Integer> batchSize,
+      Random random) {
+    this.response = response;
+    this.tlds = tlds;
+    this.batchSize = batchSize.orElse(DEFAULT_BATCH_SIZE);
+    this.random = random;
+  }
 
   @Override
   public void run() {
     assertTldsExist(tlds);
-    checkArgument(smearMinutes > 0, "Must specify a positive number of smear minutes");
-    tm().transact(
-            () ->
-                tm().query(
-                        "SELECT domainName FROM Domain "
-                            + "WHERE tld IN (:tlds) "
-                            + "AND deletionTime > :now",
-                        String.class)
-                    .setParameter("tlds", tlds)
-                    .setParameter("now", clock.nowUtc())
-                    .getResultStream()
-                    .forEach(
-                        domainName -> {
-                          try {
-                            // Smear the task execution time over the next N minutes.
-                            requestDomainDnsRefresh(
-                                domainName, Duration.standardMinutes(random.nextInt(smearMinutes)));
-                          } catch (Throwable t) {
-                            logger.atSevere().withCause(t).log(
-                                "Error while enqueuing DNS refresh for domain '%s'.", domainName);
-                            response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR);
-                          }
-                        }));
+    checkArgument(batchSize > 0, "Must specify a positive number for batch size");
+    int smearMinutes = tm().transact(this::calculateSmearMinutes);
+    ImmutableList<String> previousBatch = ImmutableList.of("");
+    do {
+      String lastInPreviousBatch = getLast(previousBatch);
+      previousBatch = tm().transact(() -> refreshBatch(lastInPreviousBatch, smearMinutes));
+    } while (previousBatch.size() == batchSize);
+  }
+
+  /**
+   * Calculates the number of smear minutes to enqueue refreshes so that the DNS queue does not get
+   * overloaded.
+   */
+  private int calculateSmearMinutes() {
+    Long activeDomains =
+        tm().query(
+                "SELECT COUNT(*) FROM Domain WHERE tld IN (:tlds) AND deletionTime = :endOfTime",
+                Long.class)
+            .setParameter("tlds", tlds)
+            .setParameter("endOfTime", END_OF_TIME)
+            .getSingleResult();
+    return Math.max(activeDomains.intValue() / 1000, 1);
+  }
+
+  private ImmutableList<String> getBatch(String lastInPreviousBatch) {
+    return tm().query(
+            "SELECT domainName FROM Domain WHERE tld IN (:tlds) AND"
+                + " deletionTime = :endOfTime  AND domainName >"
+                + " :lastInPreviousBatch ORDER BY domainName ASC",
+            String.class)
+        .setParameter("tlds", tlds)
+        .setParameter("endOfTime", END_OF_TIME)
+        .setParameter("lastInPreviousBatch", lastInPreviousBatch)
+        .setMaxResults(batchSize)
+        .getResultStream()
+        .collect(toImmutableList());
+  }
+
+  @VisibleForTesting
+  ImmutableList<String> refreshBatch(String lastInPreviousBatch, int smearMinutes) {
+    ImmutableList<String> domainBatch = getBatch(lastInPreviousBatch);
+    try {
+      // Smear the task execution time over the next N minutes.
+      requestDomainDnsRefresh(domainBatch, Duration.standardMinutes(random.nextInt(smearMinutes)));
+    } catch (Throwable t) {
+      logger.atSevere().withCause(t).log("Error while enqueuing DNS refresh batch");
+      response.setStatus(HttpStatus.SC_OK);
+    }
+    return domainBatch;
   }
 }

core/src/main/java/google/registry/tools/server/ToolsServerModule.java

@@ -16,6 +16,7 @@ package google.registry.tools.server;
 
 import static com.google.common.base.Strings.emptyToNull;
 import static google.registry.request.RequestParameters.extractIntParameter;
+import static google.registry.request.RequestParameters.extractOptionalIntParameter;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
 
@@ -76,8 +77,8 @@ public class ToolsServerModule {
   }
 
   @Provides
-  @Parameter("smearMinutes")
-  static int provideSmearMinutes(HttpServletRequest req) {
-    return extractIntParameter(req, "smearMinutes");
+  @Parameter("batchSize")
+  static Optional<Integer> provideBatchSize(HttpServletRequest req) {
+    return extractOptionalIntParameter(req, "batchSize");
   }
 }

core/src/test/java/google/registry/testing/DatabaseHelper.java

@@ -1357,5 +1357,16 @@ public final class DatabaseHelper {
         .isEqualTo(1);
   }
 
+  public static void assertDnsRequestsWithRequestTime(DateTime requestTime, int numOfDomains) {
+    assertThat(
+            tm().transact(
+                    () ->
+                        tm().createQueryComposer(DnsRefreshRequest.class)
+                            .where("type", EQ, DnsUtils.TargetType.DOMAIN)
+                            .where("requestTime", EQ, requestTime)
+                            .count()))
+        .isEqualTo(numOfDomains);
+  }
+
   private DatabaseHelper() {}
 }

core/src/test/java/google/registry/tools/CurlCommandTest.java

@@ -22,6 +22,7 @@ import static google.registry.request.Action.Service.TOOLS;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -29,6 +30,7 @@ import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.net.MediaType;
+import google.registry.request.Action.Service;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentCaptor;
@@ -46,7 +48,7 @@ class CurlCommandTest extends CommandTestCase<CurlCommand> {
   @BeforeEach
   void beforeEach() {
     command.setConnection(connection);
-    when(connection.withService(any())).thenReturn(connectionForService);
+    when(connection.withService(any(Service.class), anyBoolean())).thenReturn(connectionForService);
   }
 
   @Captor ArgumentCaptor<ImmutableMap<String, String>> urlParamCaptor;
@@ -54,7 +56,7 @@ class CurlCommandTest extends CommandTestCase<CurlCommand> {
   @Test
   void testGetInvocation() throws Exception {
     runCommand("--path=/foo/bar?a=1&b=2", "--service=TOOLS");
-    verify(connection).withService(TOOLS);
+    verify(connection).withService(eq(TOOLS), eq(false));
     verifyNoMoreInteractions(connection);
     verify(connectionForService)
         .sendGetRequest(eq("/foo/bar?a=1&b=2"), eq(ImmutableMap.<String, String>of()));
@@ -63,7 +65,7 @@ class CurlCommandTest extends CommandTestCase<CurlCommand> {
   @Test
   void testExplicitGetInvocation() throws Exception {
     runCommand("--path=/foo/bar?a=1&b=2", "--request=GET", "--service=BACKEND");
-    verify(connection).withService(BACKEND);
+    verify(connection).withService(eq(BACKEND), eq(false));
     verifyNoMoreInteractions(connection);
     verify(connectionForService)
         .sendGetRequest(eq("/foo/bar?a=1&b=2"), eq(ImmutableMap.<String, String>of()));
@@ -72,7 +74,7 @@ class CurlCommandTest extends CommandTestCase<CurlCommand> {
   @Test
   void testPostInvocation() throws Exception {
     runCommand("--path=/foo/bar?a=1&b=2", "--data=some data", "--service=DEFAULT");
-    verify(connection).withService(DEFAULT);
+    verify(connection).withService(eq(DEFAULT), eq(false));
     verifyNoMoreInteractions(connection);
     verify(connectionForService)
         .sendPostRequest(
@@ -89,7 +91,7 @@ class CurlCommandTest extends CommandTestCase<CurlCommand> {
         "--data=some data",
         "--service=DEFAULT",
         "--content-type=application/json");
-    verify(connection).withService(DEFAULT);
+    verify(connection).withService(eq(DEFAULT), eq(false));
     verifyNoMoreInteractions(connection);
     verify(connectionForService)
         .sendPostRequest(
@@ -118,7 +120,7 @@ class CurlCommandTest extends CommandTestCase<CurlCommand> {
   void testMultiDataPost() throws Exception {
     runCommand(
         "--path=/foo/bar?a=1&b=2", "--data=first=100", "-d", "second=200", "--service=PUBAPI");
-    verify(connection).withService(PUBAPI);
+    verify(connection).withService(eq(PUBAPI), eq(false));
     verifyNoMoreInteractions(connection);
     verify(connectionForService)
         .sendPostRequest(
@@ -132,7 +134,7 @@ class CurlCommandTest extends CommandTestCase<CurlCommand> {
   void testDataDoesntSplit() throws Exception {
     runCommand(
         "--path=/foo/bar?a=1&b=2", "--data=one,two", "--service=PUBAPI");
-    verify(connection).withService(PUBAPI);
+    verify(connection).withService(eq(PUBAPI), eq(false));
     verifyNoMoreInteractions(connection);
     verify(connectionForService)
         .sendPostRequest(
@@ -145,7 +147,20 @@ class CurlCommandTest extends CommandTestCase<CurlCommand> {
   @Test
   void testExplicitPostInvocation() throws Exception {
     runCommand("--path=/foo/bar?a=1&b=2", "--request=POST", "--service=TOOLS");
-    verify(connection).withService(TOOLS);
+    verify(connection).withService(eq(TOOLS), eq(false));
+    verifyNoMoreInteractions(connection);
+    verify(connectionForService)
+        .sendPostRequest(
+            eq("/foo/bar?a=1&b=2"),
+            eq(ImmutableMap.<String, String>of()),
+            eq(MediaType.PLAIN_TEXT_UTF_8),
+            eq("".getBytes(UTF_8)));
+  }
+
+  @Test
+  void testCanaryInvocation() throws Exception {
+    runCommand("--path=/foo/bar?a=1&b=2", "--request=POST", "--service=TOOLS", "--canary");
+    verify(connection).withService(eq(TOOLS), eq(true));
     verifyNoMoreInteractions(connection);
     verify(connectionForService)
         .sendPostRequest(

core/src/test/java/google/registry/tools/ServiceConnectionTest.java

@@ -0,0 +1,38 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.request.Action.Service.DEFAULT;
+
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link google.registry.tools.ServiceConnection}. */
+public class ServiceConnectionTest {
+
+  @Test
+  void testServerUrl_notCanary() {
+    ServiceConnection connection = new ServiceConnection().withService(DEFAULT, false);
+    String serverUrl = connection.getServer().toString();
+    assertThat(serverUrl).isEqualTo("https://localhost"); // See default-config.yaml
+  }
+
+  @Test
+  void testServerUrl_canary() {
+    ServiceConnection connection = new ServiceConnection().withService(DEFAULT, true);
+    String serverUrl = connection.getServer().toString();
+    assertThat(serverUrl).isEqualTo("https://nomulus-dot-localhost");
+  }
+}

core/src/test/java/google/registry/tools/server/RefreshDnsForAllDomainsActionTest.java

@@ -15,18 +15,24 @@
 package google.registry.tools.server;
 
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.QueryComposer.Comparator.EQ;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatabaseHelper.assertDnsRequestsWithRequestTime;
 import static google.registry.testing.DatabaseHelper.assertDomainDnsRequestWithRequestTime;
 import static google.registry.testing.DatabaseHelper.assertNoDnsRequestsExcept;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.persistActiveDomain;
 import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
-import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import google.registry.dns.DnsUtils;
+import google.registry.model.common.DnsRefreshRequest;
 import google.registry.persistence.transaction.JpaTestExtensions;
 import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
+import java.util.Optional;
 import java.util.Random;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
@@ -46,20 +52,16 @@ public class RefreshDnsForAllDomainsActionTest {
 
   @BeforeEach
   void beforeEach() {
-    action = new RefreshDnsForAllDomainsAction();
-    action.smearMinutes = 1;
-    action.random = new Random();
-    action.random.setSeed(123L);
-    action.clock = clock;
-    action.response = response;
     createTld("bar");
+    action =
+        new RefreshDnsForAllDomainsAction(
+            response, ImmutableSet.of("bar"), Optional.of(10), new Random());
   }
 
   @Test
   void test_runAction_successfullyEnqueuesDnsRefreshes() throws Exception {
     persistActiveDomain("foo.bar");
     persistActiveDomain("low.bar");
-    action.tlds = ImmutableSet.of("bar");
     action.run();
     assertDomainDnsRequestWithRequestTime("foo.bar", clock.nowUtc());
     assertDomainDnsRequestWithRequestTime("low.bar", clock.nowUtc());
@@ -69,18 +71,27 @@ public class RefreshDnsForAllDomainsActionTest {
   void test_runAction_smearsOutDnsRefreshes() throws Exception {
     persistActiveDomain("foo.bar");
     persistActiveDomain("low.bar");
-    action.tlds = ImmutableSet.of("bar");
-    action.smearMinutes = 1000;
-    action.run();
-    assertDomainDnsRequestWithRequestTime("foo.bar", clock.nowUtc().plusMinutes(450));
-    assertDomainDnsRequestWithRequestTime("low.bar", clock.nowUtc().plusMinutes(782));
+    // Set batch size to 1 since each batch will be enqueud at the same time
+    action =
+        new RefreshDnsForAllDomainsAction(
+            response, ImmutableSet.of("bar"), Optional.of(1), new Random());
+    tm().transact(() -> action.refreshBatch("", 1000));
+    tm().transact(() -> action.refreshBatch("", 1000));
+    ImmutableList<DnsRefreshRequest> refreshRequests =
+        tm().transact(
+                () ->
+                    tm().createQueryComposer(DnsRefreshRequest.class)
+                        .where("type", EQ, DnsUtils.TargetType.DOMAIN)
+                        .list());
+    assertThat(refreshRequests.size()).isEqualTo(2);
+    assertThat(refreshRequests.get(0).getRequestTime())
+        .isNotEqualTo(refreshRequests.get(1).getRequestTime());
   }
 
   @Test
   void test_runAction_doesntRefreshDeletedDomain() throws Exception {
     persistActiveDomain("foo.bar");
     persistDeletedDomain("deleted.bar", clock.nowUtc().minusYears(1));
-    action.tlds = ImmutableSet.of("bar");
     action.run();
     assertDomainDnsRequestWithRequestTime("foo.bar", clock.nowUtc());
     assertNoDnsRequestsExcept("foo.bar");
@@ -92,7 +103,6 @@ public class RefreshDnsForAllDomainsActionTest {
     persistActiveDomain("foo.bar");
     persistActiveDomain("low.bar");
     persistActiveDomain("ignore.baz");
-    action.tlds = ImmutableSet.of("bar");
     action.run();
     assertDomainDnsRequestWithRequestTime("foo.bar", clock.nowUtc());
     assertDomainDnsRequestWithRequestTime("low.bar", clock.nowUtc());
@@ -100,13 +110,11 @@ public class RefreshDnsForAllDomainsActionTest {
   }
 
   @Test
-  void test_smearMinutesMustBeSpecified() {
-    action.tlds = ImmutableSet.of("bar");
-    action.smearMinutes = 0;
-    IllegalArgumentException thrown =
-        assertThrows(IllegalArgumentException.class, () -> action.run());
-    assertThat(thrown)
-        .hasMessageThat()
-        .isEqualTo("Must specify a positive number of smear minutes");
+  void test_successfullyBatchesNames() {
+    for (int i = 0; i <= 10; i++) {
+      persistActiveDomain(String.format("test%s.bar", i));
+    }
+    action.run();
+    assertDnsRequestsWithRequestTime(clock.nowUtc(), 11);
   }
 }

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -704,6 +704,7 @@
         anchor_tenant_add_grace_period_length interval not null,
         auto_renew_grace_period_length interval not null,
         automatic_transfer_length interval not null,
+        breakglass_mode boolean not null,
         claims_period_end timestamptz not null,
         create_billing_cost_amount numeric(19, 2),
         create_billing_cost_currency text,