Compare commits

..
Author SHA1 Message Date
Lai JiangandGitHub c0ac9bdba4 Compile to Java 17 bytecode (#2304)
Also fix a linter warning.
2024-01-25 18:29:07 -05:00
Weimin YuandGitHub 58ec0f826d Stop saving BSA empty refresh changes (#2307)
* Stop saving BSA empty refresh changes

We thought that as a way to verify the refresh job to be running, browsing
the GCS bucket with empty files is easier than quering the DB or go to GCP
logging dashboard, but there are too many of them to be useful.
2024-01-25 16:02:04 -05:00
Pavlo TkachandGitHub f9e0908022 Replace invoice email attachement with bucket link (#2299) 2024-01-25 14:08:08 -05:00
sarahcaseybotandGitHub b21e1a1935 Add required --build_environment flag to tld-sync Cloud Build job (#2306) 2024-01-25 12:27:05 -05:00
Lai JiangandGitHub 0112b3ae06 Make the formatting tasks work with Java 17 (take 2) (#2305)
We should not assume org.gradle.java.home to exist on kokoro or GCB.
2024-01-25 12:08:30 -05:00
Lai JiangandGitHub a4903c27b9 Make the formatting tasks work with Java 17 (#2301)
TESTED=ran gradle jIFA locally after intentionally mis-formatting a Java
file.
2024-01-24 17:15:13 -05:00
sarahcaseybotandGitHub 2166c28d6d Update to only include changes to check for production required tags (#2273) 2024-01-24 17:12:46 -05:00
Lai JiangandGitHub 891e7c0174 Make Kythe work with Java 17 (#2293)
TESTED=submitted a GCB job locally and it ran successfully.
2024-01-24 13:26:45 -05:00
Ben McIlwainandGitHub 64f5971275 Include a better error message to debug nomulus tool not working (#2275)
Failures to initialize the tool transaction manager seem to often be caused by
stale local credentials.
2024-01-24 13:08:33 -05:00
sarahcaseybotandGitHub 818944317f Add some updates to UpdateReservedListCommand to facilitate internal config presubmits and syncing (#2292)
* Add some updates to UpdateReservedListCommand to facilitate internal config presubmits and syncing

Added a dry-run tag for presubmit tests

Added early exit behavior when there are no new changes to the list

Added a new --build_environment tag to be used to indicate command runs from build tools. This tag was also added to UpdatePremiumListCommand. Once this new tag is deployed, and break glass behavior is added, these commands will be modified to prevent runs on the command line in the production environment unless the --build_environment or --break_glass flag is used.

* Fix capitalization

* Added in commented out production environment check for buildEnv flag
2024-01-23 17:32:33 -05:00
Weimin YuandGitHub ea96ed300f Drop the BsaDomainInUse table (#2298)
Already renamed to BsaUnlockableDomain table.
2024-01-23 17:07:35 -05:00
Weimin YuandGitHub 8415c8bbe4 Fix typo in BsaRefreshAction (#2297) 2024-01-22 16:03:35 -05:00
Lai JiangandGitHub dc48c257b5 Use Java 17 runtime on sandbox and production (#2296)
The blocking issue is fixed in
https://github.com/google/nomulus/pull/2224.

Java 8 support is being deprecated on 2024-01-31 and no further deployment is
possible afterwards without exception:

https://cloud.google.com/appengine/docs/legacy/standard/java/deprecations

We have been using Java 17 on alpha/crash/qa for several months and have
not oberved any other blocking issue other than possible missing email
attachements, which is being mitigated by including a link to the
attachments saved in GCS.
2024-01-22 15:21:17 -05:00
sarahcaseybotandGitHub 2bf3867532 Add an example tld YAML config file (#2295) 2024-01-22 13:32:36 -05:00
44 changed files with 5134 additions and 9403 deletions
+1 -1
View File
@@ -33,7 +33,7 @@ jobs:
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '11'
java-version: '17'
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
+35 -15
View File
@@ -211,6 +211,30 @@ allprojects {
options.fork = true
options.forkOptions.executable =
file("${System.env.JAVA_HOME}/bin/javac")
options.compilerArgs = ["--add-exports",
"jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
"--add-exports",
"jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED",
"--add-exports",
"jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
"--add-exports",
"jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
"--add-exports",
"jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
"--add-exports",
"jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
"--add-exports",
"jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
"--add-exports",
"jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED"]
options.forkOptions.jvmArgs = ["-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
"-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED"]
}
}
}
@@ -358,16 +382,8 @@ subprojects {
apply from: "${rootDir.path}/java_common.gradle"
if (!rootProject.enableCrossReferencing.toBoolean()) {
compileJava {
// TODO: Remove this once we migrate off of AppEngine.
options.release = 8
}
compileTestJava {
// TODO: Remove this once we migrate off of AppEngine.
options.release = 8
}
}
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
project.tasks.test.dependsOn runPresubmits
@@ -479,10 +495,14 @@ def createGetBuildSrcDirectDepsTask(outputFileName) {
rootProject.ext {
invokeJavaDiffFormatScript = { action ->
println("JAVA_HOME=${System.env.JAVA_HOME}")
println("PATH=${System.env.PATH}")
println(ext.execInBash("type java", "${rootDir}"))
println(ext.execInBash("java -version", "${rootDir}"))
def javaHome = project.findProperty('org.gradle.java.home')
def javaBin
if (javaHome != null) {
javaBin = "$javaHome/bin/java"
} else {
javaBin = ext.execInBash("which java", rootDir)
}
println("Running the formatting tool with $javaBin")
def scriptDir = rootDir.path.endsWith('buildSrc')
? "${rootDir}/../java-format"
: "${rootDir}/java-format"
@@ -493,7 +513,7 @@ rootProject.ext {
def pythonExe = getPythonExecutable()
return ext.execInBash(
"PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
"JAVA=${javaBin} PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
}
}
+1 -1
View File
@@ -87,7 +87,7 @@ PRESUBMITS = {
PresubmitCheck(
r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
("java", "js", "soy", "sql", "py", "sh", "gradle", "ts"), {
".git", "/build/", "/generated/", "/generated_tests/",
".git", "/build/", "/bin/generated-sources/", "/bin/generated-test-sources/",
"node_modules/", "LoggerConfig.java", "registrar_bin.",
"registrar_dbg.", "google-java-format-diff.py",
"nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
+1 -1
View File
@@ -1,3 +1,3 @@
FROM gcr.io/distroless/java:debug
FROM gcr.io/distroless/java17-debian11:debug
ADD build/libs/nomulus.jar /nomulus.jar
ENTRYPOINT ["/usr/bin/java", "-jar", "/nomulus.jar"]
@@ -163,7 +163,12 @@ public class InvoicingPipeline implements Serializable {
}
}
/** Saves the billing events to a single overall invoice CSV file. */
/**
* Saves the billing events to a single overall invoice CSV file. TextIO always produces the file
* of type text/plain, which we then update to desired text/csv before sending an email to billing
* team {@link google.registry.reporting.billing.BillingEmailUtils#emailOverallInvoice()
* emailOverallInvoice}
*/
static void saveInvoiceCsv(
PCollection<google.registry.beam.billing.BillingEvent> billingEvents,
InvoicingPipelineOptions options) {
@@ -117,13 +117,12 @@ public class BsaRefreshAction implements Runnable {
case CHECK_FOR_CHANGES:
ImmutableList<UnblockableDomainChange> blockabilityChanges =
refresher.checkForBlockabilityChanges();
// Always write even if no change. Easier for manual inspection of GCS bucket.
gcsClient.writeRefreshChanges(schedule.jobName(), blockabilityChanges.stream());
if (blockabilityChanges.isEmpty()) {
logger.atInfo().log("No change to Unblockable domains found.");
schedule.updateJobStage(RefreshStage.DONE);
return null;
}
gcsClient.writeRefreshChanges(schedule.jobName(), blockabilityChanges.stream());
schedule.updateJobStage(RefreshStage.APPLY_CHANGES);
// Fall through
case APPLY_CHANGES:
@@ -162,7 +161,7 @@ public class BsaRefreshAction implements Runnable {
.filter(UnblockableDomainChange::isNewOrChange)
.map(UnblockableDomainChange::newValue));
if (report.isPresent()) {
gcsClient.logRemovedUnblockableDomainsReport(
gcsClient.logAddedUnblockableDomainsReport(
schedule.jobName(), LINE_SPLITTER.splitToStream(report.get()));
bsaReportSender.addUnblockableDomainsUpdates(report.get());
} else {
@@ -48,7 +48,8 @@ public final class BsaTransactions {
@CanIgnoreReturnValue
public static <T> T bsaQuery(Callable<T> work) {
verify(!isInTransaction(), "May only be used for top-level transactions.");
return replicaTm().transact(work, TRANSACTION_REPEATABLE_READ);
// TRANSACTION_REPEATABLE_READ is default on replica.
return replicaTm().transact(work);
}
private BsaTransactions() {}
@@ -721,6 +721,17 @@ public final class RegistryConfig {
return "gs://" + billingBucket;
}
/**
* Returns origin part of the URL of the billing invoice file
*
* @see google.registry.beam.billing.InvoicingPipeline
*/
@Provides
@Config("billingInvoiceOriginUrl")
public static String provideBillingInvoiceOriginUrl(RegistryConfigSettings config) {
return config.billing.billingInvoiceOriginUrl;
}
/**
* Returns whether or not we should publish invoices to partners automatically by default.
*
@@ -173,6 +173,7 @@ public class RegistryConfigSettings {
public List<String> invoiceEmailRecipients;
public String invoiceReplyToEmailAddress;
public String invoiceFilePrefix;
public String billingInvoiceOriginUrl;
}
/** Configuration for Registry Data Escrow (RDE). */
@@ -381,6 +381,7 @@ billing:
# Optional return address that overrides the default.
invoiceReplyToEmailAddress: null
invoiceFilePrefix: REG-INV
billingInvoiceOriginUrl: https://billing-origin-url/
rde:
# URL prefix of ICANN's server to upload RDE reports to. Nomulus adds /TLD/ID
@@ -0,0 +1,60 @@
# Example of a TLD configuration file. TLD configuration files are stored in
# the internal repository and synced regularly to the database using the
# release/cloudbuild-tld-sync.yaml job. The file can be created using the exact
# output from the getTldCommand of an existing TLD.
addGracePeriodLength: "PT432000S"
allowedFullyQualifiedHostNames: []
allowedRegistrantContactIds: []
anchorTenantAddGracePeriodLength: "PT2592000S"
autoRenewGracePeriodLength: "PT3888000S"
automaticTransferLength: "PT432000S"
claimsPeriodEnd: "294247-01-10T04:00:54.775Z"
createBillingCost:
currency: "USD"
amount: 8.00
creationTime: "2017-01-03T19:52:47.770Z"
currency: "USD"
defaultPromoTokens: []
dnsAPlusAaaaTtl: null
dnsDsTtl: null
dnsNsTtl: null
dnsPaused: true
dnsWriters:
- "VoidDnsWriter"
driveFolderId: null
eapFeeSchedule:
"1970-01-01T00:00:00.000Z":
currency: "USD"
amount: 0.00
escrowEnabled: false
idnTables: []
invoicingEnabled: false
lordnUsername: null
numDnsPublishLocks: 1
pendingDeleteLength: "PT432000S"
premiumListName: null
pricingEngineClassName: "google.registry.model.pricing.StaticPremiumListPricingEngine"
redemptionGracePeriodLength: "PT2592000S"
registryLockOrUnlockBillingCost:
currency: "USD"
amount: 0.00
renewBillingCostTransitions:
"1970-01-01T00:00:00.000Z":
currency: "USD"
amount: 8.00
renewGracePeriodLength: "PT432000S"
reservedListNames: []
restoreBillingCost:
currency: "USD"
amount: 50.00
roidSuffix: "EXAMPLE"
serverStatusChangeBillingCost:
currency: "USD"
amount: 0.00
tldStateTransitions:
"1970-01-01T00:00:00.000Z": "GENERAL_AVAILABILITY"
tldStr: "example"
tldType: "TEST"
tldUnicode: "example"
transferGracePeriodLength: "PT432000S"
@@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<runtime>java17</runtime>
<service>backend</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4_1G</instance-class>
<basic-scaling>
@@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<runtime>java17</runtime>
<service>bsa</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4_1G</instance-class>
<basic-scaling>
@@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<runtime>java17</runtime>
<service>default</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4_1G</instance-class>
<manual-scaling>
@@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<runtime>java17</runtime>
<service>pubapi</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4_1G</instance-class>
<manual-scaling>
@@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<runtime>java17</runtime>
<service>tools</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4_1G</instance-class>
<basic-scaling>
@@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<runtime>java17</runtime>
<service>backend</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4</instance-class>
<basic-scaling>
@@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<runtime>java17</runtime>
<service>bsa</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4</instance-class>
<basic-scaling>
@@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<runtime>java17</runtime>
<service>default</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4_1G</instance-class>
<manual-scaling>
@@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<runtime>java17</runtime>
<service>pubapi</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4_1G</instance-class>
<manual-scaling>
@@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
<runtime>java8</runtime>
<runtime>java17</runtime>
<service>tools</service>
<!--app-engine-apis>true</app-engine-apis-->
<threadsafe>true</threadsafe>
<app-engine-apis>true</app-engine-apis>
<sessions-enabled>true</sessions-enabled>
<instance-class>B4</instance-class>
<basic-scaling>
@@ -118,6 +118,14 @@ public class GcsUtils implements Serializable {
storage().delete(blobId);
}
/** Update file content type on existing GCS file */
public void updateContentType(BlobId blobId, String contentType) throws StorageException {
if (existsAndNotEmpty(blobId)) {
Blob blob = storage().get(blobId);
blob.toBuilder().setContentType(contentType).build().update();
}
}
/**
* Returns a list of all object names within a bucket for a given prefix.
*
@@ -15,20 +15,17 @@
package google.registry.reporting.billing;
import static com.google.common.base.Throwables.getRootCause;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.cloud.storage.BlobId;
import com.google.cloud.storage.StorageException;
import com.google.common.collect.ImmutableList;
import com.google.common.io.CharStreams;
import com.google.common.flogger.FluentLogger;
import com.google.common.net.MediaType;
import google.registry.config.RegistryConfig.Config;
import google.registry.gcs.GcsUtils;
import google.registry.groups.GmailClient;
import google.registry.reporting.billing.BillingModule.InvoiceDirectoryPrefix;
import google.registry.util.EmailMessage;
import google.registry.util.EmailMessage.Attachment;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Optional;
import javax.inject.Inject;
import javax.mail.internet.InternetAddress;
@@ -37,6 +34,7 @@ import org.joda.time.YearMonth;
/** Utility functions for sending emails involving monthly invoices. */
public class BillingEmailUtils {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final GmailClient gmailClient;
private final YearMonth yearMonth;
private final InternetAddress outgoingEmailAddress;
@@ -46,6 +44,7 @@ public class BillingEmailUtils {
private final String billingBucket;
private final String invoiceFilePrefix;
private final String invoiceDirectoryPrefix;
private final String billingInvoiceOriginUrl;
private final GcsUtils gcsUtils;
@Inject
@@ -58,6 +57,7 @@ public class BillingEmailUtils {
@Config("invoiceReplyToEmailAddress") Optional<InternetAddress> replyToEmailAddress,
@Config("billingBucket") String billingBucket,
@Config("invoiceFilePrefix") String invoiceFilePrefix,
@Config("billingInvoiceOriginUrl") String billingInvoiceOriginUrl,
@InvoiceDirectoryPrefix String invoiceDirectoryPrefix,
GcsUtils gcsUtils) {
this.gmailClient = gmailClient;
@@ -69,31 +69,36 @@ public class BillingEmailUtils {
this.billingBucket = billingBucket;
this.invoiceFilePrefix = invoiceFilePrefix;
this.invoiceDirectoryPrefix = invoiceDirectoryPrefix;
this.billingInvoiceOriginUrl = billingInvoiceOriginUrl;
this.gcsUtils = gcsUtils;
}
/** Sends an e-mail to all expected recipients with an attached overall invoice from GCS. */
void emailOverallInvoice() {
public void emailOverallInvoice() {
try {
String invoiceFile = String.format("%s-%s.csv", invoiceFilePrefix, yearMonth);
String fileUrl = billingInvoiceOriginUrl + invoiceDirectoryPrefix + invoiceFile;
BlobId invoiceFilename = BlobId.of(billingBucket, invoiceDirectoryPrefix + invoiceFile);
try (InputStream in = gcsUtils.openInputStream(invoiceFilename)) {
gmailClient.sendEmail(
EmailMessage.newBuilder()
.setSubject(String.format("Domain Registry invoice data %s", yearMonth))
.setBody(
String.format("Attached is the %s invoice for the domain registry.", yearMonth))
.setFrom(outgoingEmailAddress)
.setRecipients(invoiceEmailRecipients)
.setReplyToEmailAddress(replyToEmailAddress)
.setAttachment(
Attachment.newBuilder()
.setContent(CharStreams.toString(new InputStreamReader(in, UTF_8)))
.setContentType(MediaType.CSV_UTF_8)
.setFilename(invoiceFile)
.build())
.build());
try {
gcsUtils.updateContentType(invoiceFilename, "text/csv");
} catch (StorageException e) {
// We want to continue with email anyway, it just will be less convenient for billing team
// to process the file.
logger.atWarning().withCause(e).log("Failed to update invoice file type");
}
gmailClient.sendEmail(
EmailMessage.newBuilder()
.setSubject(String.format("Domain Registry invoice data %s", yearMonth))
.setBody(
String.format(
"<p>Use the following link to download %s invoice for the domain registry -"
+ " <a href=\"%s\">invoice</a>.</p>",
yearMonth, fileUrl))
.setFrom(outgoingEmailAddress)
.setRecipients(invoiceEmailRecipients)
.setReplyToEmailAddress(replyToEmailAddress)
.setContentType(MediaType.HTML_UTF_8)
.build());
} catch (Throwable e) {
// Strip one layer, because callWithRetry wraps in a RuntimeException
sendAlertEmail(
@@ -105,6 +105,12 @@ public class ConfigureTldCommand extends MutatingCommand {
@Override
protected void init() throws Exception {
if (RegistryToolEnvironment.get().equals(RegistryToolEnvironment.PRODUCTION)) {
checkArgument(
buildEnv || breakGlass != null,
"Either the --break_glass or --build_environment flag must be used when"
+ " running the configure_tld command on Production");
}
String name = convertFilePathToName(inputFile);
Map<String, Object> tldData = new Yaml().load(Files.newBufferedReader(inputFile));
checkName(name, tldData);
@@ -23,6 +23,7 @@ import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import com.google.common.base.Ascii;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
@@ -218,12 +219,21 @@ final class RegistryCli implements CommandRunner {
// Reset the JPA transaction manager after every command to avoid a situation where a test can
// interfere with other tests
JpaTransactionManager cachedJpaTm = tm();
TransactionManagerFactory.setJpaTm(() -> component.nomulusToolJpaTransactionManager().get());
TransactionManagerFactory.setReplicaJpaTm(
() -> component.nomulusToolReplicaJpaTransactionManager().get());
command.run();
TransactionManagerFactory.setJpaTm(() -> cachedJpaTm);
try {
JpaTransactionManager cachedJpaTm = tm();
TransactionManagerFactory.setJpaTm(() -> component.nomulusToolJpaTransactionManager().get());
TransactionManagerFactory.setReplicaJpaTm(
() -> component.nomulusToolReplicaJpaTransactionManager().get());
command.run();
TransactionManagerFactory.setJpaTm(() -> cachedJpaTm);
} catch (Exception e) {
String env = Ascii.toLowerCase(environment.name());
System.err.printf(
"Could not get tool transaction manager; try running nomulus -e %s logout "
+ "and then nomulus -e %s login.\n",
env, env);
throw e;
}
}
void setEnvironment(RegistryToolEnvironment environment) {
@@ -35,11 +35,24 @@ class UpdatePremiumListCommand extends CreateOrUpdatePremiumListCommand {
description = "Does not execute the entity mutation")
boolean dryRun;
// indicates if there is a new change made by this command
@Parameter(
names = {"--build_environment"},
description =
"DO NOT USE THIS FLAG ON THE COMMAND LINE! This flag indicates the command is being run"
+ " by the build environment tools. This flag should never be used by a human user"
+ " from the command line.")
boolean buildEnv;
// Indicates if there is a new change made by this command
private boolean newChange = false;
@Override
protected String prompt() throws Exception {
// TODO(sarahbot): uncomment once go/r3pr/2292 is deployed
// checkArgument(
// !RegistryToolEnvironment.get().equals(RegistryToolEnvironment.PRODUCTION) || buildEnv,
// "The --build_environment flag must be used when running update_premium_list in
// production");
name = Strings.isNullOrEmpty(name) ? convertFilePathToName(inputFile) : name;
PremiumList existingList =
PremiumListDao.getLatestRevision(name)
@@ -18,6 +18,7 @@ import static google.registry.util.DiffUtils.prettyPrintEntityDeepDiff;
import static google.registry.util.ListNamingUtils.convertFilePathToName;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Strings;
import google.registry.model.tld.label.ReservedList;
@@ -28,8 +29,29 @@ import java.util.List;
@Parameters(separators = " =", commandDescription = "Update a ReservedList.")
final class UpdateReservedListCommand extends CreateOrUpdateReservedListCommand {
@Parameter(
names = {"-d", "--dry_run"},
description = "Does not execute the entity mutation")
boolean dryRun;
@Parameter(
names = {"--build_environment"},
description =
"DO NOT USE THIS FLAG ON THE COMMAND LINE! This flag indicates the command is being run"
+ " by the build environment tools. This flag should never be used by a human user"
+ " from the command line.")
boolean buildEnv;
// indicates if there is a new change made by this command
private boolean newChange = true;
@Override
protected String prompt() throws Exception {
// TODO(sarahbot): uncomment once go/r3pr/2292 is deployed
// checkArgument(
// !RegistryToolEnvironment.get().equals(RegistryToolEnvironment.PRODUCTION) || buildEnv,
// "The --build_environment flag must be used when running update_reserved_list in"
// + " production");
name = Strings.isNullOrEmpty(name) ? convertFilePathToName(input) : name;
ReservedList existingReservedList =
ReservedList.get(name)
@@ -54,6 +76,7 @@ final class UpdateReservedListCommand extends CreateOrUpdateReservedListCommand
.getReservedListEntries()
.equals(reservedList.getReservedListEntries());
if (!shouldPublishChanged && !reservedListEntriesChanged) {
newChange = false;
return "No entity changes to apply.";
}
String result = String.format("Update reserved list for %s?\n", name);
@@ -70,4 +93,9 @@ final class UpdateReservedListCommand extends CreateOrUpdateReservedListCommand
}
return result;
}
@Override
protected boolean dontRunCommand() {
return dryRun || !newChange;
}
}
@@ -30,6 +30,7 @@ import static google.registry.testing.DatabaseHelper.deleteTestDomain;
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
import static google.registry.testing.DatabaseHelper.persistResource;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -54,6 +55,7 @@ import google.registry.request.Response;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeLockHandler;
import google.registry.testing.FakeResponse;
import java.io.UncheckedIOException;
import java.util.Optional;
import org.joda.time.DateTime;
import org.joda.time.Duration;
@@ -293,7 +295,12 @@ class BsaRefreshFunctionalTest {
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
assertThat(gcsClient.readRefreshChanges(jobName)).isEmpty();
// Verify that refresh change file does not exist (404 error) since there is no change.
assertThat(
assertThrows(
UncheckedIOException.class, () -> gcsClient.readRefreshChanges(jobName).findAny()))
.hasMessageThat()
.contains("404");
verifyNoInteractions(bsaReportSender);
}
@@ -313,7 +320,12 @@ class BsaRefreshFunctionalTest {
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
assertThat(gcsClient.readRefreshChanges(jobName)).isEmpty();
// Verify that refresh change file does not exist (404 error) since there is no change.
assertThat(
assertThrows(
UncheckedIOException.class, () -> gcsClient.readRefreshChanges(jobName).findAny()))
.hasMessageThat()
.contains("404");
verifyNoInteractions(bsaReportSender);
}
}
@@ -48,6 +48,10 @@ import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
public class UploadBsaUnavailableDomainsActionTest {
private static final String BUCKET = "domain-registry-bsa";
private static final String API_URL = "https://upload.test/bsa";
private final FakeClock clock = new FakeClock(DateTime.parse("2024-02-02T02:02:02Z"));
@RegisterExtension
@@ -62,9 +66,7 @@ public class UploadBsaUnavailableDomainsActionTest {
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
private final String BUCKET = "domain-registry-bsa";
private final String API_URL = "https://upload.test/bsa";
private final FakeResponse response = new FakeResponse();
@@ -21,17 +21,12 @@ import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.google.cloud.storage.BlobId;
import com.google.common.collect.ImmutableList;
import com.google.common.net.MediaType;
import google.registry.gcs.GcsUtils;
import google.registry.groups.GmailClient;
import google.registry.util.EmailMessage;
import google.registry.util.EmailMessage.Attachment;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import javax.mail.MessagingException;
import javax.mail.internet.InternetAddress;
@@ -45,18 +40,14 @@ class BillingEmailUtilsTest {
private GmailClient gmailClient;
private BillingEmailUtils emailUtils;
private GcsUtils gcsUtils;
private ArgumentCaptor<EmailMessage> contentCaptor;
private GcsUtils gcsUtils;
@BeforeEach
void beforeEach() throws Exception {
gmailClient = mock(GmailClient.class);
gcsUtils = mock(GcsUtils.class);
when(gcsUtils.openInputStream(BlobId.of("test-bucket", "results/REG-INV-2017-10.csv")))
.thenReturn(
new ByteArrayInputStream("test,data\nhello,world".getBytes(StandardCharsets.UTF_8)));
contentCaptor = ArgumentCaptor.forClass(EmailMessage.class);
gcsUtils = mock(GcsUtils.class);
emailUtils = getEmailUtils(Optional.of(new InternetAddress("reply-to@test.com")));
}
@@ -72,6 +63,7 @@ class BillingEmailUtilsTest {
replyToAddress,
"test-bucket",
"REG-INV",
"www.google.com/",
"results/",
gcsUtils);
}
@@ -89,14 +81,11 @@ class BillingEmailUtilsTest {
ImmutableList.of(
new InternetAddress("hello@world.com"), new InternetAddress("hola@mundo.com")))
.setSubject("Domain Registry invoice data 2017-10")
.setBody("Attached is the 2017-10 invoice for the domain registry.")
.setBody(
"<p>Use the following link to download 2017-10 invoice for the domain registry -"
+ " <a href=\"www.google.com/results/REG-INV-2017-10.csv\">invoice</a>.</p>")
.setReplyToEmailAddress(new InternetAddress("reply-to@test.com"))
.setAttachment(
Attachment.newBuilder()
.setContent("test,data\nhello,world")
.setContentType(MediaType.CSV_UTF_8)
.setFilename("REG-INV-2017-10.csv")
.build())
.setContentType(MediaType.HTML_UTF_8)
.build();
assertThat(emailMessage).isEqualTo(expectedContent);
}
@@ -671,4 +671,46 @@ public class ConfigureTldCommandTest extends CommandTestCase<ConfigureTldCommand
Tld notUpdatedTld = Tld.get("tld");
assertThat(notUpdatedTld.getCreateBillingCost()).isEqualTo(Money.of(USD, 13));
}
@Test
void testFailure_runCommandOnProduction_noFlag() throws Exception {
createTld("tld");
File tldFile = tmpDir.resolve("tld.yaml").toFile();
Files.asCharSink(tldFile, UTF_8).write(loadFile(getClass(), "tld.yaml"));
IllegalArgumentException thrown =
assertThrows(
IllegalArgumentException.class,
() ->
runCommandInEnvironment(RegistryToolEnvironment.PRODUCTION, "--input=" + tldFile));
assertThat(thrown.getMessage())
.isEqualTo(
"Either the --break_glass or --build_environment flag must be used when running the"
+ " configure_tld command on Production");
}
@Test
void testSuccess_runCommandOnProduction_breakGlassFlag() throws Exception {
createTld("tld");
File tldFile = tmpDir.resolve("tld.yaml").toFile();
Files.asCharSink(tldFile, UTF_8).write(loadFile(getClass(), "tld.yaml"));
runCommandInEnvironment(
RegistryToolEnvironment.PRODUCTION, "--input=" + tldFile, "--break_glass=true", "-f");
Tld updatedTld = Tld.get("tld");
assertThat(updatedTld.getCreateBillingCost()).isEqualTo(Money.of(USD, 25));
testTldConfiguredSuccessfully(updatedTld, "tld.yaml");
assertThat(updatedTld.getBreakglassMode()).isTrue();
}
@Test
void testSuccess_runCommandOnProduction_buildEnvFlag() throws Exception {
createTld("tld");
File tldFile = tmpDir.resolve("tld.yaml").toFile();
Files.asCharSink(tldFile, UTF_8).write(loadFile(getClass(), "tld.yaml"));
runCommandInEnvironment(
RegistryToolEnvironment.PRODUCTION, "--input=" + tldFile, "--build_environment", "-f");
Tld updatedTld = Tld.get("tld");
assertThat(updatedTld.getCreateBillingCost()).isEqualTo(Money.of(USD, 25));
testTldConfiguredSuccessfully(updatedTld, "tld.yaml");
assertThat(updatedTld.getBreakglassMode()).isFalse();
}
}
@@ -205,4 +205,40 @@ class UpdatePremiumListCommandTest<C extends UpdatePremiumListCommand>
.comparingElementsUsing(immutableObjectCorrespondence("revisionId"))
.containsExactly(PremiumEntry.create(0L, new BigDecimal("9090.00"), "doge"));
}
// TODO(sarahbot): uncomment once go/r3pr/2292 is deployed
// @Test
// void testFailure_runCommandOnProduction_noFlag() throws Exception {
// File tmpFile = tmpDir.resolve(String.format("%s.txt", TLD_TEST)).toFile();
// String newPremiumListData = "eth,USD 9999";
// Files.asCharSink(tmpFile, UTF_8).write(newPremiumListData);
// IllegalArgumentException thrown =
// assertThrows(
// IllegalArgumentException.class,
// () ->
// runCommandInEnvironment(
// RegistryToolEnvironment.PRODUCTION,
// "--name=" + TLD_TEST,
// "--input=" + Paths.get(tmpFile.getPath())));
// assertThat(thrown.getMessage())
// .isEqualTo(
// "The --build_environment flag must be used when running update_premium_list in"
// + " production");
// }
//
// @Test
// void testSuccess_runCommandOnProduction_buildEnvFlag() throws Exception {
// File tmpFile = tmpDir.resolve(String.format("%s.txt", TLD_TEST)).toFile();
// String newPremiumListData = "eth,USD 9999";
// Files.asCharSink(tmpFile, UTF_8).write(newPremiumListData);
// runCommandInEnvironment(
// RegistryToolEnvironment.PRODUCTION,
// "--name=" + TLD_TEST,
// "--input=" + Paths.get(tmpFile.getPath()),
// "--build_environment",
// "-f");
// assertThat(PremiumListDao.loadAllPremiumEntries(TLD_TEST))
// .comparingElementsUsing(immutableObjectCorrespondence("revisionId"))
// .containsExactly(PremiumEntry.create(0L, new BigDecimal("9999.00"), "eth"));
// }
}
@@ -142,4 +142,51 @@ class UpdateReservedListCommandTest
assertThat(command.prompt()).contains("baddies: null -> baddies,FULLY_BLOCKED");
assertThat(command.prompt()).contains("ford: null -> ford,FULLY_BLOCKED # random comment");
}
@Test
void testSuccess_dryRun() throws Exception {
runCommandForced("--input=" + reservedTermsPath, "--dry_run");
assertThat(command.prompt()).contains("Update reserved list for xn--q9jyb4c_common-reserved?");
assertThat(ReservedList.get("xn--q9jyb4c_common-reserved")).isPresent();
ReservedList reservedList = ReservedList.get("xn--q9jyb4c_common-reserved").get();
assertThat(reservedList.getReservedListEntries()).hasSize(1);
assertThat(reservedList.getReservationInList("helicopter")).hasValue(FULLY_BLOCKED);
}
// TODO(sarahbot): uncomment once go/r3pr/2292 is deployed
// @Test
// void testFailure_runCommandOnProduction_noFlag() throws Exception {
// IllegalArgumentException thrown =
// assertThrows(
// IllegalArgumentException.class,
// () ->
// runCommandInEnvironment(
// RegistryToolEnvironment.PRODUCTION,
// "--name=xn--q9jyb4c_common-reserved",
// "--input=" + reservedTermsPath));
// assertThat(thrown.getMessage())
// .isEqualTo(
// "The --build_environment flag must be used when running update_reserved_list in"
// + " production");
// }
//
// @Test
// void testSuccess_runCommandOnProduction_buildEnvFlag() throws Exception {
// runCommandInEnvironment(
// RegistryToolEnvironment.PRODUCTION,
// "--name=xn--q9jyb4c_common-reserved",
// "--input=" + reservedTermsPath,
// "--build_environment",
// "-f");
// assertThat(ReservedList.get("xn--q9jyb4c_common-reserved")).isPresent();
// ReservedList reservedList = ReservedList.get("xn--q9jyb4c_common-reserved").get();
// assertThat(reservedList.getReservedListEntries()).hasSize(2);
// assertThat(reservedList.getReservationInList("baddies")).hasValue(FULLY_BLOCKED);
// assertThat(reservedList.getReservationInList("ford")).hasValue(FULLY_BLOCKED);
// assertThat(reservedList.getReservationInList("helicopter")).isEmpty();
// assertInStdout("Update reserved list for xn--q9jyb4c_common-reserved?");
// assertInStdout("helicopter: helicopter,FULLY_BLOCKED -> null");
// assertInStdout("baddies: null -> baddies,FULLY_BLOCKED");
// assertInStdout("ford: null -> ford,FULLY_BLOCKED # random comment");
// }
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+1
View File
@@ -150,3 +150,4 @@ V149__add_bsa_domain_in_use_table.sql
V150__add_tld_bsa_enroll_date.sql
V151__add_bsa_unblockable_domain_table.sql
V152__add_bsa_domain_refresh_table.sql
V153__drop_bsa_domain_in_use_table.sql
@@ -0,0 +1,15 @@
-- Copyright 2023 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
DROP TABLE IF EXISTS "BsaDomainInUse";
@@ -123,18 +123,6 @@ CREATE TABLE public."BillingRecurrence" (
);
--
-- Name: BsaDomainInUse; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public."BsaDomainInUse" (
label text NOT NULL,
tld text NOT NULL,
creation_time timestamp with time zone NOT NULL,
reason text NOT NULL
);
--
-- Name: BsaDomainRefresh; Type: TABLE; Schema: public; Owner: -
--
@@ -1369,14 +1357,6 @@ ALTER TABLE ONLY public."BillingRecurrence"
ADD CONSTRAINT "BillingRecurrence_pkey" PRIMARY KEY (billing_recurrence_id);
--
-- Name: BsaDomainInUse BsaDomainInUse_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public."BsaDomainInUse"
ADD CONSTRAINT "BsaDomainInUse_pkey" PRIMARY KEY (label, tld);
--
-- Name: BsaDomainRefresh BsaDomainRefresh_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -2781,14 +2761,6 @@ ALTER TABLE ONLY public."DomainHistoryHost"
ADD CONSTRAINT fka9woh3hu8gx5x0vly6bai327n FOREIGN KEY (domain_history_domain_repo_id, domain_history_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;
--
-- Name: BsaDomainInUse fkbsadomaininuse2label; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public."BsaDomainInUse"
ADD CONSTRAINT fkbsadomaininuse2label FOREIGN KEY (label) REFERENCES public."BsaLabel"(label) ON DELETE CASCADE;
--
-- Name: BsaUnblockableDomain fkbsaunblockabledomainlabel; Type: FK CONSTRAINT; Schema: public; Owner: -
--
Binary file not shown.
Binary file not shown.
+8 -23
View File
@@ -40,37 +40,22 @@ where:
show show the effect of the formatting as unified diff"
SCRIPT_DIR="$(realpath $(dirname $0))"
JAR_NAME="google-java-format-1.8-all-deps.jar"
JAR_NAME="google-java-format-1.19.2-all-deps.jar"
# Make sure we have a valid python interpreter.
if [ -z "$PYTHON" ]; then
echo "You must specify the name of a python3 interpreter in the PYTHON" \
echo "You must specify the name of a Python interpreter in the PYTHON" \
"environment variable."
exit 1
elif ! "$PYTHON" -c ''; then
echo "Invalid python interpreter: $PYTHON"
echo "Invalid Python interpreter: $PYTHON"
exit 1
fi
# Locate the java binary.
if [ -n "$JAVA_HOME" ]; then
JAVA_BIN="$JAVA_HOME/bin/java"
if [ ! -x "$JAVA_BIN" ]; then
echo "No java binary found in JAVA_HOME (JAVA_HOME is $JAVA_HOME)"
exit 1
fi
else
# Use java from the path.
JAVA_BIN="$(which java)" || JAVA_BIN=""
if [ -z "$JAVA_BIN" ]; then
echo "JAVA_HOME is not defined and java was not found on the path"
exit 1
fi
fi
if ! "$JAVA_BIN" -version 2>&1 | grep 'version "11\.' >/dev/null; then
echo "Bad java version. Requires java 11, got:"
"$JAVA_BIN" -version
# Make sure we have a valid JRE binary
if [ -z "$JAVA" ]; then
echo "You must specify the name of a JRE binary in the JAVA" \
"environment variable."
exit 1
fi
@@ -80,7 +65,7 @@ function runGoogleJavaFormatAgainstDiffs() {
git diff -U0 "$forkPoint" | \
"${PYTHON}" "${SCRIPT_DIR}/google-java-format-diff.py" \
--java-binary "$JAVA_BIN" \
--java-binary "$JAVA" \
--google-java-format-jar "${SCRIPT_DIR}/${JAR_NAME}" \
-p1 "$@" | \
tee gjf.out
@@ -50,7 +50,7 @@ import javax.persistence.Converter;
/** Processor to generate {@link AttributeConverter} for {@code VKey} type. */
@SupportedAnnotationTypes("google.registry.persistence.WithVKey")
@SupportedSourceVersion(SourceVersion.RELEASE_8)
@SupportedSourceVersion(SourceVersion.RELEASE_17)
public class VKeyProcessor extends AbstractProcessor {
private static final String CONVERTER_CLASS_NAME_TEMP = "VKeyConverter_%s";
+6
View File
@@ -29,6 +29,12 @@ steps:
mv $${JAVA_HOME}/bin/javac $${JAVA_HOME}/bin/javac.real
cp ./kythe/extractors/javac-wrapper.sh $${JAVA_HOME}/bin/javac
export JAVAC_EXTRACTOR_JAR="$${PWD}/kythe/extractors/javac_extractor.jar"
jvmopts="--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED"
jvmopts="$${jvmopts} --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"
jvmopts="$${jvmopts} --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED"
jvmopts="$${jvmopts} --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED"
jvmopts="$${jvmopts} --add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED"
export KYTHE_JAVA_RUNTIME_OPTIONS=$${jvmopts}
export KYTHE_VNAMES="$${PWD}/vnames.json"
export KYTHE_ROOT_DIRECTORY="$${PWD}"
export KYTHE_OUTPUT_DIRECTORY="$${PWD}/kythe_output"
+1 -1
View File
@@ -37,5 +37,5 @@ for FILE in ${config_file_directory}/${nomulus_env}/*; do
echo $FILE
java -jar /nomulus.jar -e "${nomulus_env}" \
--credential "${tools_credential}" \
configure_tld -i $FILE --force
configure_tld -i $FILE --force --build_environment
done